diff --git a/Dockerfile b/Dockerfile index b2231cbe..1f34420f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,8 +5,8 @@ FROM docker.io/alpine:${ALPINE_VERSION} ARG ALPINE_VERSION ARG KUBE_VERSION=1.31 -ARG SOPS_VERSION="3.9.1" -ARG VALS_VERSION="0.37.6" +ARG SOPS_VERSION="3.9.4" +ARG VALS_VERSION="0.39.1" ARG HELM_SECRETS_VERSION="4.6.2" RUN cd /etc/apk/keys && \ diff --git a/admin/dev_apply.sh b/admin/dev_apply.sh index 4085ebe9..249d2b6b 100755 --- a/admin/dev_apply.sh +++ b/admin/dev_apply.sh @@ -49,7 +49,6 @@ function cert-manager-post() { wait_for "kubectl get deployment -n $namespace cert-manager-webhook" kubectl rollout status deployment -n $namespace cert-manager-webhook wait_for 'kubectl get validatingwebhookconfigurations -o yaml | grep "caBundle: LS0"' - apply fi wait_for "kubectl get ClusterIssuer -n $namespace kubezero-local-ca-issuer" @@ -82,11 +81,11 @@ function metrics-pre() { get_kubezero_values $ARGOCD # Always use embedded kubezero chart -helm template $CHARTS/kubezero -f $WORKDIR/kubezero-values.yaml --kube-version $KUBE_VERSION --version ~$KUBE_VERSION --devel --output-dir $WORKDIR +helm template $CHARTS/kubezero -f $WORKDIR/kubezero-values.yaml --kube-version $KUBE_VERSION --name-template kubezero --version ~$KUBE_VERSION --devel --output-dir $WORKDIR # Root KubeZero apply directly and exit if [ ${ARTIFACTS[0]} == "kubezero" ]; then - kubectl apply --server-side --force-conflicts -f $WORKDIR/kubezero/templates + kubectl replace -f $WORKDIR/kubezero/templates exit $? # "catch all" apply all enabled modules diff --git a/admin/libhelm.sh b/admin/libhelm.sh index 3075b081..42bc4fc3 100644 --- a/admin/libhelm.sh +++ b/admin/libhelm.sh @@ -50,7 +50,7 @@ function get_kubezero_values() { function update_kubezero_cm() { kubectl get cm -n kubezero kubezero-values -o=yaml | \ yq e ".data.\"values.yaml\" |= load_str(\"$WORKDIR/kubezero-values.yaml\")" | \ - kubectl apply --server-side --force-conflicts -f - + kubectl replace -f - } # sync kubezero-values CM from ArgoCD app diff --git a/charts/kubezero-argo/README.md b/charts/kubezero-argo/README.md index 708b90e2..fbb76f45 100644 --- a/charts/kubezero-argo/README.md +++ b/charts/kubezero-argo/README.md @@ -1,6 +1,6 @@ # kubezero-argo -![Version: 0.2.7](https://img.shields.io/badge/Version-0.2.7-informational?style=flat-square) +![Version: 0.2.8](https://img.shields.io/badge/Version-0.2.8-informational?style=flat-square) KubeZero Argo - Events, Workflow, CD @@ -18,16 +18,17 @@ Kubernetes: `>= 1.26.0-0` | Repository | Name | Version | |------------|------|---------| -| https://argoproj.github.io/argo-helm | argo-cd | 7.7.7 | -| https://argoproj.github.io/argo-helm | argo-events | 2.4.9 | +| https://argoproj.github.io/argo-helm | argo-cd | 7.8.2 | +| https://argoproj.github.io/argo-helm | argo-events | 2.4.13 | | https://argoproj.github.io/argo-helm | argocd-apps | 2.0.2 | -| https://argoproj.github.io/argo-helm | argocd-image-updater | 0.11.2 | +| https://argoproj.github.io/argo-helm | argocd-image-updater | 0.12.0 | | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 | ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| +| argo-cd.configs.cm."application.instanceLabelKey" | string | `nil` | | | argo-cd.configs.cm."application.resourceTrackingMethod" | string | `"annotation"` | | | argo-cd.configs.cm."resource.customizations" | string | `"cert-manager.io/Certificate:\n # Lua script for customizing the health status assessment\n health.lua: |\n hs = {}\n if obj.status ~= nil then\n if obj.status.conditions ~= nil then\n for i, condition in ipairs(obj.status.conditions) do\n if condition.type == \"Ready\" and condition.status == \"False\" then\n hs.status = \"Degraded\"\n hs.message = condition.message\n return hs\n end\n if condition.type == \"Ready\" and condition.status == \"True\" then\n hs.status = \"Healthy\"\n hs.message = condition.message\n return hs\n end\n end\n end\n end\n hs.status = \"Progressing\"\n hs.message = \"Waiting for certificate\"\n return hs\n"` | | | argo-cd.configs.cm."timeout.reconciliation" | string | `"300s"` | | @@ -35,6 +36,7 @@ Kubernetes: `>= 1.26.0-0` | argo-cd.configs.cm."ui.bannerpermanent" | string | `"true"` | | | argo-cd.configs.cm."ui.bannerposition" | string | `"bottom"` | | | argo-cd.configs.cm."ui.bannerurl" | string | `"https://kubezero.com/releases/v1.31"` | | +| argo-cd.configs.cm.installationID | string | `"KubeZero-ArgoCD"` | | | argo-cd.configs.cm.url | string | `"https://argocd.example.com"` | | | argo-cd.configs.params."controller.diff.server.side" | string | `"true"` | | | argo-cd.configs.params."controller.operation.processors" | string | `"5"` | | @@ -94,7 +96,7 @@ Kubernetes: `>= 1.26.0-0` | argo-events.configs.jetstream.streamConfig.maxMsgs | int | `1000000` | Maximum number of messages before expiring oldest message | | argo-events.configs.jetstream.streamConfig.replicas | int | `1` | Number of replicas, defaults to 3 and requires minimal 3 | | argo-events.configs.jetstream.versions[0].configReloaderImage | string | `"natsio/nats-server-config-reloader:0.14.1"` | | -| argo-events.configs.jetstream.versions[0].metricsExporterImage | string | `"natsio/prometheus-nats-exporter:0.14.0"` | | +| argo-events.configs.jetstream.versions[0].metricsExporterImage | string | `"natsio/prometheus-nats-exporter:0.16.0"` | | | argo-events.configs.jetstream.versions[0].natsImage | string | `"nats:2.10.11-scratch"` | | | argo-events.configs.jetstream.versions[0].startCommand | string | `"/nats-server"` | | | argo-events.configs.jetstream.versions[0].version | string | `"2.10.11"` | | diff --git a/charts/kubezero-argo/values.yaml b/charts/kubezero-argo/values.yaml index 731dc6b6..3902ac97 100644 --- a/charts/kubezero-argo/values.yaml +++ b/charts/kubezero-argo/values.yaml @@ -45,7 +45,7 @@ argo-cd: format: json image: repository: public.ecr.aws/zero-downtime/zdt-argocd - tag: v2.13.1 + tag: v2.14.2 networkPolicy: create: true diff --git a/charts/kubezero-cert-manager/README.md b/charts/kubezero-cert-manager/README.md index 5feab8ba..7f1b08f6 100644 --- a/charts/kubezero-cert-manager/README.md +++ b/charts/kubezero-cert-manager/README.md @@ -1,6 +1,6 @@ # kubezero-cert-manager -![Version: 0.9.11](https://img.shields.io/badge/Version-0.9.11-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) +![Version: 0.9.12](https://img.shields.io/badge/Version-0.9.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) KubeZero Umbrella Chart for cert-manager @@ -14,12 +14,12 @@ KubeZero Umbrella Chart for cert-manager ## Requirements -Kubernetes: `>= 1.26.0-0` +Kubernetes: `>= 1.30.0-0` | Repository | Name | Version | |------------|------|---------| | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 | -| https://charts.jetstack.io | cert-manager | v1.17.0 | +| https://charts.jetstack.io | cert-manager | v1.17.1 | ## AWS - OIDC IAM roles diff --git a/charts/kubezero-cert-manager/jsonnetfile.lock.json b/charts/kubezero-cert-manager/jsonnetfile.lock.json index d2f033eb..bc978950 100644 --- a/charts/kubezero-cert-manager/jsonnetfile.lock.json +++ b/charts/kubezero-cert-manager/jsonnetfile.lock.json @@ -18,7 +18,7 @@ "subdir": "contrib/mixin" } }, - "version": "c0e7e8c873a6067f9ae9076c3c243a20fa713a58", + "version": "8c52b414f324d6369b77096af98d8f0416fe20cb", "sum": "XmXkOCriQIZmXwlIIFhqlJMa0e6qGWdxZD+ZDYaN0Po=" }, { @@ -78,7 +78,7 @@ "subdir": "grafana-builder" } }, - "version": "d6c38bb26f576b128cadca4137d73a037afdd872", + "version": "393630ca7ba9b25258c098f1fd4c81962e3ca046", "sum": "yxqWcq/N3E/a/XreeU6EuE6X7kYPnG0AspAQFKOjASo=" }, { @@ -118,8 +118,8 @@ "subdir": "" } }, - "version": "e27267571be06c2bdc3d2fd8dbd70161cd709cb4", - "sum": "je1RPCp2aFNefYs5Q57Q5wDm93p8pL4pdBtA5rC7jLA=" + "version": "4ff562d5e8145940cf355f62cf2308895c4dca81", + "sum": "kiL19fTbXOtNglsmT62kOzIf/Xpu+YwoiMPAApDXhkE=" }, { "source": { @@ -158,7 +158,7 @@ "subdir": "jsonnet/mixin" } }, - "version": "4da36fdd2377362c285aee3a96f7b0516f6e41bf", + "version": "7deab71d6d5921eeaf8c79e3ae8e31efe63783a9", "sum": "gi+knjdxs2T715iIQIntrimbHRgHnpM8IFBJDD1gYfs=", "name": "prometheus-operator-mixin" }, @@ -169,8 +169,8 @@ "subdir": "jsonnet/prometheus-operator" } }, - "version": "4da36fdd2377362c285aee3a96f7b0516f6e41bf", - "sum": "tb5PzIT75Hv4m3kbOHXvmrlcplg+EbS4++NfTttDNOk=" + "version": "7deab71d6d5921eeaf8c79e3ae8e31efe63783a9", + "sum": "LctDdofQostvviE5y8vpRKWGGO1ZKO3dgJe7P9xifW0=" }, { "source": { @@ -200,7 +200,7 @@ "subdir": "documentation/prometheus-mixin" } }, - "version": "906f6a33b60cec2596018ac8cc97ac41b16b06b7", + "version": "a5ffa83be83be22e2ec9fd1d4765299d8d16119e", "sum": "2c+wttfee9TwuQJZIkNV7Tekem74Qgc7iZ842P28rNw=", "name": "prometheus" }, diff --git a/charts/kubezero-istio/Chart.yaml b/charts/kubezero-istio/Chart.yaml index 86388f3c..a226e621 100644 --- a/charts/kubezero-istio/Chart.yaml +++ b/charts/kubezero-istio/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-istio description: KubeZero Umbrella Chart for Istio type: application -version: 0.24.3 +version: 0.24.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: diff --git a/charts/kubezero-istio/README.md b/charts/kubezero-istio/README.md index d60094ec..8194df8f 100644 --- a/charts/kubezero-istio/README.md +++ b/charts/kubezero-istio/README.md @@ -23,7 +23,7 @@ Kubernetes: `>= 1.30.0-0` | https://cdn.zero-downtime.net/charts/ | kubezero-lib | >= 0.1.6 | | https://istio-release.storage.googleapis.com/charts | base | 1.24.2 | | https://istio-release.storage.googleapis.com/charts | istiod | 1.24.2 | -| https://kiali.org/helm-charts | kiali-server | 1.89.7 | +| https://kiali.org/helm-charts | kiali-server | 2.5.0 | ## Values diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md index b87ed65f..96dc090e 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md @@ -1,4 +1,77 @@ # Helm chart + +## v2.39.3 + +### Urgent Upgrade Notes + +Please upgrade from v2.39.2 directly to v2.39.3 to avoid upgrade failures if you are using this chart as a subchart. + +### Bug or Regression +- Fix sub-charting by removing values schema ([#2322](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2322), [@ElijahQuinones]((https://github.com/ElijahQuinones) + +## v2.39.2 + +### Urgent Upgrade Notes + +Please upgrade from v2.38.1 directly to v2.39.2 to avoid upgrade failures if you are relying on `a1CompatibilityDaemonSet`. + +### Bug or Regression +- Fix helm regression when `a1CompatibilityDaemonSet=true` ([#2316](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2316), [@AndrewSirenko](https://github.com/AndrewSirenko)) + +## v2.39.1 + +### Bug or Regression +- Fix `node.selinux` to properly set SELinux-specific mounts as ReadOnly ([#2311](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2311), [@AndrewSirenko](https://github.com/AndrewSirenko)) + +## v2.39.0 + +### Feature + +- Add Helm parameter `node.selinux` to enable SELinux-specific mounts on the node DaemonSet ([#2253](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2253), [@ConnorJC3](https://github.com/ConnorJC3)) +- Add Helm FIPS parameter ([#2244](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2244), [@ConnorJC3](https://github.com/ConnorJC3)) + +## v2.38.1 + +### Feature + +- Render templated controller service account parameters ([#2243](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2243), [@ElijahQuinones](https://github.com/ElijahQuinones)) + +### Bug or Regression + +- Fix rendering failrue when `node.enableMetrics` is set to `true` ([#2250](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2250), [@mindw](https://github.com/mindw)) +- Remove duplicate 'enableMetrics' key ([#2256](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2256), [@sule26](https://github.com/sule26)) + +## v2.37.0 +* Bump driver version to `v1.37.0` +* Add init containers to node daemonset ([#2215](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2215), [@clbx](https://github.com/clbx)) +* Fix fetching test package version for kubetest in helm-tester ([#2203](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2203), [@torredil](https://github.com/torredil)) + +## v2.36.0 +* Bump driver version to `v1.36.0` +* Add recommended autoscalar Tolerations to driver DaemonSet ([#2165](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2165), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Add support for unhealthyPodEvictionPolicy on PodDisruptionBudget ([#2159](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2159), [@peterabarr](https://github.com/peterabarr)) + +## v2.35.1 +* Fix an issue causing the `csi-attacher` container to get stuck in `CrashLoopBackoff` on clusters with VAC enabled. Users with a VAC-enabled cluster are strongly encouraged to skip `v2.35.0` and/or upgrade directly to `v2.35.1` or later. + +## v2.35.0 +* Bump driver version to `v1.35.0` +* Add reservedVolumeAttachments to windows nodes ([#2134](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2134),[@AndrewSirenko](https://github.com/AndrewSirenko)) +* Add legacy-xfs driver option for clusters that mount XFS volumes to nodes with Linux kernel <= 5.4. Warning: This is a temporary workaround for customers unable to immediately upgrade their nodes. It will be removed in a future release. See [the options documentation](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/release-1.35/docs/options.md) for more details.([#2121](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2121),[@AndrewSirenko](https://github.com/AndrewSirenko)) +* Add back "Auto-enable VAC on clusters with beta API version" ([#2141](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2141), [@ConnorJC3](https://github.com/ConnorJC3)) + +## v2.34.0 +* Bump driver version to `v1.34.0` +* Add toggle for PodDisruptionBudget in chart ([#2109](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2109), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* Add nodeComponentOnly parameter to helm chart ([#2106](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2106), [@AndrewSirenko](https://github.com/AndrewSirenko)) +* fix: sidecars.snapshotter.logLevel not being respect ([#2102](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2102), [@zyue110026](https://github.com/zyue110026)) + +## v2.33.0 +* Bump driver version to `v1.33.0` +* Bump CSI sidecar container versions +* Add fix for enableLinux node parameter ([#2078](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2078), [@ElijahQuinones](https://github.com/ElijahQuinones)) +* Fix dnsConfig indentation in controller template file ([#2084](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/2084), [@cHiv0rz](https://github.com/cHiv0rz)) + ## v2.32.0 * Bump driver version to `v1.32.0` * Bump CSI sidecar container versions diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml index f22a10f7..d017be89 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.32.0 +appVersion: 1.39.0 description: A Helm chart for AWS EBS CSI Driver home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver keywords: @@ -13,4 +13,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 2.32.0 +version: 2.39.3 diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt index cb3e6cec..fc4cab42 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/NOTES.txt @@ -2,4 +2,6 @@ To verify that aws-ebs-csi-driver has started, run: kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-ebs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -NOTE: The [CSI Snapshotter](https://github.com/kubernetes-csi/external-snapshotter) controller and CRDs will no longer be installed as part of this chart and moving forward will be a prerequisite of using the snap shotting functionality. +[ACTION REQUIRED] Update to the EBS CSI Driver IAM Policy + +Due to an upcoming change in handling of IAM polices for the CreateVolume API when creating a volume from an EBS snapshot, a change to your EBS CSI Driver policy may be needed. For more information and remediation steps, see GitHub issue #2190 (https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/2190). This change affects all versions of the EBS CSI Driver and action may be required even on clusters where the driver is not upgraded. diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_helpers.tpl b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_helpers.tpl index 42fc1382..62941efc 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_helpers.tpl +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_helpers.tpl @@ -31,6 +31,13 @@ Create chart name and version as used by the chart label. {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Determine image +*/}} +{{- define "aws-ebs-csi-driver.fullImagePath" -}} +{{ printf "%s%s:%s%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (.Values.image.tag | toString)) (.Values.fips | ternary "-fips" "") }} +{{- end -}} + {{/* Common labels */}} @@ -83,3 +90,21 @@ Handle http proxy env vars - name: NO_PROXY value: {{ .Values.proxy.no_proxy | quote }} {{- end -}} + +{{/* +Recommended daemonset tolerations +*/}} +{{- define "aws-ebs-csi-driver.daemonset-tolerations" -}} +# Prevents stateful workloads from being scheduled to node before CSI Driver reports volume attachment limit +- key: "ebs.csi.aws.com/agent-not-ready" + operator: "Exists" +# Prevents undesired eviction by Cluster Autoscalar +- key: "ToBeDeletedByClusterAutoscaler" + operator: Exists +# Prevents undesired eviction by v1 Karpenter +- key: "karpenter.sh/disrupted" + operator: Exists +# Prevents undesired eviction by v1beta1 Karpenter +- key: "karpenter.sh/disruption" + operator: Exists +{{- end -}} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node-windows.tpl b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node-windows.tpl index 76ab90fd..f53bffd0 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node-windows.tpl +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node-windows.tpl @@ -49,6 +49,7 @@ spec: {{- with .Values.node.tolerations }} {{- toYaml . | nindent 8 }} {{- end }} + {{- include "aws-ebs-csi-driver.daemonset-tolerations" . | nindent 8 }} {{- end }} {{- if .Values.node.windowsHostProcess }} securityContext: @@ -56,10 +57,14 @@ spec: hostProcess: true runAsUserName: "NT AUTHORITY\\SYSTEM" hostNetwork: true + {{- with .Values.node.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} containers: - name: ebs-plugin - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + image: {{ include "aws-ebs-csi-driver.fullImagePath" $ }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if .Values.node.windowsHostProcess }} command: @@ -68,9 +73,15 @@ spec: args: - node - --endpoint=$(CSI_ENDPOINT) + {{- with .Values.node.reservedVolumeAttachments }} + - --reserved-volume-attachments={{ . }} + {{- end }} {{- with .Values.node.volumeAttachLimit }} - --volume-attach-limit={{ . }} {{- end }} + {{- if .Values.node.legacyXFS }} + - --legacy-xfs=true + {{- end}} {{- with .Values.node.loggingFormat }} - --logging-format={{ . }} {{- end }} @@ -100,6 +111,10 @@ spec: value: {{ .otelServiceName }} - name: OTEL_EXPORTER_OTLP_ENDPOINT value: {{ .otelExporterEndpoint }} + {{- if .Values.fips }} + - name: AWS_USE_FIPS_ENDPOINT + value: "true" + {{- end }} {{- end }} {{- with .Values.node.env }} {{- . | toYaml | nindent 12 }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node.tpl b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node.tpl index 383e2a21..0a3b2c66 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node.tpl +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/_node.tpl @@ -1,5 +1,5 @@ {{- define "node" }} -{{- if or (eq (default true .Values.node.enableLinux) true) }} +{{- if .Values.node.enableLinux }} --- kind: DaemonSet apiVersion: apps/v1 @@ -53,17 +53,20 @@ spec: {{- with .Values.node.tolerations }} {{- toYaml . | nindent 8 }} {{- end }} - - key: "ebs.csi.aws.com/agent-not-ready" - operator: "Exists" + {{- include "aws-ebs-csi-driver.daemonset-tolerations" . | nindent 8 }} {{- end }} hostNetwork: {{ .Values.node.hostNetwork }} {{- with .Values.node.securityContext }} securityContext: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.node.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: ebs-plugin - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} + image: {{ include "aws-ebs-csi-driver.fullImagePath" $ }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: - node @@ -71,9 +74,18 @@ spec: {{- with .Values.node.reservedVolumeAttachments }} - --reserved-volume-attachments={{ . }} {{- end }} + {{- if .Values.node.enableMetrics }} + - --http-endpoint=0.0.0.0:3302 + {{- end}} + {{- with .Values.node.kubeletPath }} + - --csi-mount-point-prefix={{ . }}/plugins/kubernetes.io/csi/ebs.csi.aws.com/ + {{- end}} {{- with .Values.node.volumeAttachLimit }} - --volume-attach-limit={{ . }} {{- end }} + {{- if .Values.node.legacyXFS }} + - --legacy-xfs=true + {{- end}} {{- with .Values.node.loggingFormat }} - --logging-format={{ . }} {{- end }} @@ -100,6 +112,10 @@ spec: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: {{ .otelExporterEndpoint }} {{- end }} + {{- if .Values.fips }} + - name: AWS_USE_FIPS_ENDPOINT + value: "true" + {{- end }} {{- with .Values.node.env }} {{- . | toYaml | nindent 12 }} {{- end }} @@ -115,6 +131,14 @@ spec: mountPath: /csi - name: device-dir mountPath: /dev + {{- if .Values.node.selinux }} + - name: selinux-sysfs + mountPath: /sys/fs/selinux + readOnly: true + - name: selinux-config + mountPath: /etc/selinux/config + readOnly: true + {{- end }} {{- with .Values.node.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -232,6 +256,16 @@ spec: hostPath: path: /dev type: Directory + {{- if .Values.node.selinux }} + - name: selinux-sysfs + hostPath: + path: /sys/fs/selinux + type: Directory + - name: selinux-config + hostPath: + path: /etc/selinux/config + type: File + {{- end }} - name: probe-dir {{- if .Values.node.probeDirVolume }} {{- toYaml .Values.node.probeDirVolume | nindent 10 }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml index bff6577b..e3eaf001 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-attacher.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -5,22 +6,23 @@ metadata: name: ebs-external-attacher-role labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +# Do not modify the rules below manually, see `make update-sidecar-dependencies` +# BEGIN AUTOGENERATED RULES rules: - - apiGroups: [ "" ] - resources: [ "persistentvolumes" ] - verbs: [ "get", "list", "watch", "update", "patch" ] - - apiGroups: [ "" ] - resources: [ "nodes" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "csi.storage.k8s.io" ] - resources: [ "csinodeinfos" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "storage.k8s.io" ] - resources: [ "volumeattachments" ] - verbs: [ "get", "list", "watch", "update", "patch" ] - - apiGroups: [ "storage.k8s.io" ] - resources: [ "volumeattachments/status" ] - verbs: [ "patch" ] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] +# END AUTOGENERATED RULES {{- with .Values.sidecars.attacher.additionalClusterRoleRules }} {{- . | toYaml | nindent 2 }} {{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml index cb69cc8d..9c4673dd 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-provisioner.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -5,37 +6,51 @@ metadata: name: ebs-external-provisioner-role labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +# Do not modify the rules below manually, see `make update-sidecar-dependencies` +# BEGIN AUTOGENERATED RULES rules: - - apiGroups: [ "" ] - resources: [ "persistentvolumes" ] - verbs: [ "get", "list", "watch", "create", "patch", "delete" ] - - apiGroups: [ "" ] - resources: [ "persistentvolumeclaims" ] - verbs: [ "get", "list", "watch", "update" ] - - apiGroups: [ "storage.k8s.io" ] - resources: [ "storageclasses" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "" ] - resources: [ "events" ] - verbs: [ "list", "watch", "create", "update", "patch" ] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshots" ] - verbs: [ "get", "list" ] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshotcontents" ] - verbs: [ "get", "list" ] - - apiGroups: [ "storage.k8s.io" ] - resources: [ "csinodes" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "" ] - resources: [ "nodes" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "storage.k8s.io" ] - resources: [ "volumeattachments" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "storage.k8s.io" ] - resources: [ "volumeattributesclasses" ] - verbs: [ "get" ] + # The following rule should be uncommented for plugins that require secrets + # for provisioning. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "patch", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + # Access to volumeattachments is only needed when the CSI driver + # has the PUBLISH_UNPUBLISH_VOLUME controller capability. + # In that case, external-provisioner will watch volumeattachments + # to determine when it is safe to delete a volume. + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] +# END AUTOGENERATED RULES + # Extra rule: VAC rules not present in upstream example + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattributesclasses"] + verbs: ["get"] {{- with .Values.sidecars.provisioner.additionalClusterRoleRules }} {{- . | toYaml | nindent 2 }} {{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml index 81858af3..f175a61d 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -5,33 +6,35 @@ metadata: name: ebs-external-resizer-role labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +# Do not modify the rules below manually, see `make update-sidecar-dependencies` +# BEGIN AUTOGENERATED RULES rules: # The following rule should be uncommented for plugins that require secrets # for provisioning. # - apiGroups: [""] # resources: ["secrets"] # verbs: ["get", "list", "watch"] - - apiGroups: [ "" ] - resources: [ "persistentvolumes" ] - verbs: [ "get", "list", "watch", "update", "patch" ] - - apiGroups: [ "" ] - resources: [ "persistentvolumeclaims" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "" ] - resources: [ "persistentvolumeclaims/status" ] - verbs: [ "update", "patch" ] - - apiGroups: [ "storage.k8s.io" ] - resources: [ "storageclasses" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "" ] - resources: [ "events" ] - verbs: [ "list", "watch", "create", "update", "patch" ] - - apiGroups: [ "" ] - resources: [ "pods" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "storage.k8s.io" ] - resources: [ "volumeattributesclasses" ] - verbs: [ "get", "list", "watch" ] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + # only required if enabling the alpha volume modify feature + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattributesclasses"] + verbs: ["get", "list", "watch"] +# END AUTOGENERATED RULES {{- with .Values.sidecars.resizer.additionalClusterRoleRules }} {{- . | toYaml | nindent 2 }} {{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml index 697e818d..8a3fadfa 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrole-snapshotter.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 @@ -5,26 +6,39 @@ metadata: name: ebs-external-snapshotter-role labels: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} +# Do not modify the rules below manually, see `make update-sidecar-dependencies` +# BEGIN AUTOGENERATED RULES rules: - - apiGroups: [ "" ] - resources: [ "events" ] - verbs: [ "list", "watch", "create", "update", "patch" ] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] # Secret permission is optional. # Enable it if your driver needs secret. # For example, `csi.storage.k8s.io/snapshotter-secret-name` is set in VolumeSnapshotClass. # See https://kubernetes-csi.github.io/docs/secrets-and-credentials.html for more details. - # - apiGroups: [ "" ] - # resources: [ "secrets" ] - # verbs: [ "get", "list" ] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshotclasses" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshotcontents" ] - verbs: [ "create", "get", "list", "watch", "update", "delete", "patch" ] - - apiGroups: [ "snapshot.storage.k8s.io" ] - resources: [ "volumesnapshotcontents/status" ] - verbs: [ "update", "patch" ] + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["groupsnapshot.storage.k8s.io"] + resources: ["volumegroupsnapshotcontents/status"] + verbs: ["update", "patch"] +# END AUTOGENERATED RULES {{- with .Values.sidecars.snapshotter.additionalClusterRoleRules }} {{- . | toYaml | nindent 2 }} {{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml index bb23044b..32c41961 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-attacher.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -13,3 +14,4 @@ roleRef: kind: ClusterRole name: ebs-external-attacher-role apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml index 9d2749ac..32008486 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-provisioner.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -13,3 +14,4 @@ roleRef: kind: ClusterRole name: ebs-external-provisioner-role apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml index 88cb47de..f0c694fe 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-resizer.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -13,3 +14,4 @@ roleRef: kind: ClusterRole name: ebs-external-resizer-role apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml index 2d429054..4c349efb 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/clusterrolebinding-snapshotter.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -13,3 +14,4 @@ roleRef: kind: ClusterRole name: ebs-external-snapshotter-role apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml index 6b393f40..598984ed 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} # Controller Service kind: Deployment apiVersion: apps/v1 @@ -70,14 +71,10 @@ spec: {{- end }} containers: - name: ebs-plugin - image: {{ printf "%s%s:%s" (default "" .Values.image.containerRegistry) .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (.Values.image.tag | toString)) }} + image: {{ include "aws-ebs-csi-driver.fullImagePath" $ }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: - {{- if ne .Release.Name "kustomize" }} - controller - {{- else }} - # - {all,controller,node} # specify the driver mode - {{- end }} - --endpoint=$(CSI_ENDPOINT) {{- if .Values.controller.extraVolumeTags }} {{- include "aws-ebs-csi-driver.extra-volume-tags" . | nindent 12 }} @@ -153,6 +150,10 @@ spec: - name: OTEL_EXPORTER_OTLP_ENDPOINT value: {{ .otelExporterEndpoint }} {{- end }} + {{- if .Values.fips }} + - name: AWS_USE_FIPS_ENDPOINT + value: "true" + {{- end }} {{- with .Values.controller.envFrom }} envFrom: {{- . | toYaml | nindent 12 }} @@ -230,6 +231,9 @@ spec: {{- if not (regexMatch "(-retry-interval-max)" (join " " .Values.sidecars.provisioner.additionalArgs)) }} - --retry-interval-max=30m {{- end }} + {{- if .Capabilities.APIVersions.Has "storage.k8s.io/v1beta1/VolumeAttributesClass" }} + - --feature-gates=VolumeAttributesClass=true + {{- end }} {{- range .Values.sidecars.provisioner.additionalArgs }} - {{ . }} {{- end }} @@ -320,6 +324,7 @@ spec: args: - --csi-address=$(ADDRESS) - --leader-election=true + - --v={{ .Values.sidecars.snapshotter.logLevel }} {{- if .Values.controller.extraCreateMetadata }} - --extra-create-metadata {{- end}} @@ -447,6 +452,9 @@ spec: {{- if not (regexMatch "(-retry-interval-max)" (join " " .Values.sidecars.resizer.additionalArgs)) }} - --retry-interval-max=30m {{- end }} + {{- if .Capabilities.APIVersions.Has "storage.k8s.io/v1beta1/VolumeAttributesClass" }} + - --feature-gates=VolumeAttributesClass=true + {{- end }} {{- range .Values.sidecars.resizer.additionalArgs }} - {{ . }} {{- end }} @@ -513,7 +521,8 @@ spec: {{- with .Values.controller.volumes }} {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.controller.dnsConfig }} - dnsConfig: - {{- toYaml .Values.controller.dnsConfig | nindent 4 }} - {{- end }} + {{- if .Values.controller.dnsConfig }} + dnsConfig: + {{- toYaml .Values.controller.dnsConfig | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml index a46d4b50..a78eb58d 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} apiVersion: {{ ternary "storage.k8s.io/v1" "storage.k8s.io/v1beta1" (semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.Version) }} kind: CSIDriver metadata: @@ -10,3 +11,4 @@ spec: {{- if not .Values.useOldCSIDriver }} fsGroupPolicy: File {{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/ebs-csi-default-sc.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/ebs-csi-default-sc.yaml index a5859572..95d74387 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/ebs-csi-default-sc.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/ebs-csi-default-sc.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} {{- if .Values.defaultStorageClass.enabled }} apiVersion: storage.k8s.io/v1 kind: StorageClass @@ -9,3 +10,4 @@ provisioner: ebs.csi.aws.com volumeBindingMode: WaitForFirstConsumer allowVolumeExpansion: true {{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml index d68bd7ab..5fc2c254 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml @@ -1,4 +1,4 @@ -{{- if .Values.controller.enableMetrics -}} +{{- if and .Values.controller.enableMetrics (not .Values.nodeComponentOnly) -}} --- apiVersion: v1 kind: Service @@ -40,3 +40,21 @@ spec: interval: {{ .Values.controller.serviceMonitor.interval | default "15s"}} {{- end }} {{- end }} +--- +{{- if .Values.node.enableMetrics }} +apiVersion: v1 +kind: Service +metadata: + name: ebs-csi-node + namespace: {{ .Release.Namespace }} + labels: + app: ebs-csi-node +spec: + selector: + app: ebs-csi-node + ports: + - name: metrics + port: 3302 + targetPort: 3302 + type: ClusterIP +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml index a891513b..0035686d 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml @@ -12,6 +12,9 @@ {{- include "node" (deepCopy $ | mustMerge $args) -}} {{- end }} {{- if .Values.a1CompatibilityDaemonSet }} +{{- if .Values.fips -}} +{{- fail "FIPS mode not supported for A1 instance family compatibility image" -}} +{{- end -}} {{$args := dict "NodeName" "ebs-csi-node-a1compat" "Values" (dict @@ -27,7 +30,7 @@ (dict "key" "eks.amazonaws.com/compute-type" "operator" "NotIn" - "values" (list "fargate") + "values" (list "fargate" "auto" "hybrid") ) (dict "key" "node.kubernetes.io/instance-type" diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml index 0a1e97cc..8f1659db 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/poddisruptionbudget-controller.yaml @@ -1,3 +1,4 @@ +{{- if and .Values.controller.podDisruptionBudget.enabled (not .Values.nodeComponentOnly) -}} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: @@ -10,8 +11,12 @@ spec: matchLabels: app: ebs-csi-controller {{- include "aws-ebs-csi-driver.selectorLabels" . | nindent 6 }} + {{- if .Values.controller.podDisruptionBudget.unhealthyPodEvictionPolicy }} + unhealthyPodEvictionPolicy: {{ .Values.controller.podDisruptionBudget.unhealthyPodEvictionPolicy }} + {{- end }} {{- if le (.Values.controller.replicaCount | int) 2 }} maxUnavailable: 1 {{- else }} minAvailable: 2 {{- end }} +{{- end -}} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/role-leases.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/role-leases.yaml index 1ec62bb4..f1260c0a 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/role-leases.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/role-leases.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -9,3 +10,4 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/rolebinding-leases.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/rolebinding-leases.yaml index 88fded8a..f2826cbc 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/rolebinding-leases.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/rolebinding-leases.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.nodeComponentOnly -}} kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -13,3 +14,4 @@ roleRef: kind: Role name: ebs-csi-leases-role apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml index d819f549..9f1d480e 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/serviceaccount-csi-controller.yaml @@ -1,4 +1,4 @@ -{{- if .Values.controller.serviceAccount.create -}} +{{- if and .Values.controller.serviceAccount.create (not .Values.nodeComponentOnly) -}} apiVersion: v1 kind: ServiceAccount metadata: @@ -8,12 +8,7 @@ metadata: {{- include "aws-ebs-csi-driver.labels" . | nindent 4 }} {{- with .Values.controller.serviceAccount.annotations }} annotations: - {{- toYaml . | nindent 4 }} - {{- end }} - {{- if eq .Release.Name "kustomize" }} - #Enable if EKS IAM roles for service accounts (IRSA) is used. See https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html for details. - #annotations: - # eks.amazonaws.com/role-arn: arn::iam:::role/ebs-csi-role + {{- tpl (toYaml .) $ | nindent 4 }} {{- end }} {{- if .Values.controller.serviceAccount.automountServiceAccountToken }} automountServiceAccountToken: {{ .Values.controller.serviceAccount.automountServiceAccountToken }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/tests/helm-tester.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/tests/helm-tester.yaml index 08113b7c..0960b530 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/tests/helm-tester.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/tests/helm-tester.yaml @@ -1,4 +1,4 @@ -{{- if .Values.helmTester.enabled -}} +{{- if and .Values.helmTester.enabled (not .Values.nodeComponentOnly) -}} --- apiVersion: v1 kind: ServiceAccount @@ -220,15 +220,28 @@ spec: else FOCUS_REGEX="${FOCUS_REGEX})" fi - export KUBE_VERSION=$(kubectl version --output json | jq -r '.serverVersion.major + "." + .serverVersion.minor') - kubetest2 noop --run-id='e2e-kubernetes' --test=ginkgo -- --test-package-version="$(curl -L https://dl.k8s.io/release/stable-${KUBE_VERSION}.txt)" --skip-regex='[Disruptive]|[Serial]' --focus-regex="$FOCUS_REGEX" --parallel=25 --test-args='-storage.testdriver=/etc/config/manifests.yaml' + + echo "Detecting Kubernetes server version" + export KUBE_VERSION=$(kubectl version --output json | jq -r '.serverVersion.major + "." + .serverVersion.minor' | sed 's/[^0-9.]*$//') + echo "Detected KUBE_VERSION=${KUBE_VERSION}" + + echo "Fetching the stable test package version for KUBE_VERSION=${KUBE_VERSION}" + test_package_version=$(curl -L https://dl.k8s.io/release/stable-${KUBE_VERSION}.txt 2>/dev/null) + + if echo "$test_package_version" | grep -q "Error"; then + echo "Error: Failed to fetch test package version for KUBE_VERSION=${KUBE_VERSION}. Exiting." + exit 1 + fi + echo "Fetched test package version ${test_package_version}" + + echo "Starting kubetest2 with ginkgo tests..." + kubetest2 noop --run-id='e2e-kubernetes' --test=ginkgo -- --test-package-version="$test_package_version" --skip-regex='[Disruptive]|[Serial]' --focus-regex="$FOCUS_REGEX" --parallel=25 --test-args='-storage.testdriver=/etc/config/manifests.yaml' + echo "kubetest2 test run completed." volumeMounts: - name: config-vol mountPath: /etc/config - # kubekins-e2e v1 image is linux amd64 only. nodeSelector: kubernetes.io/os: linux - kubernetes.io/arch: amd64 serviceAccountName: ebs-csi-driver-test volumes: - name: config-vol diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml index 6b7216ff..3ef73a82 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml @@ -11,13 +11,18 @@ image: customLabels: {} # k8s-app: aws-ebs-csi-driver +# Instruct the AWS SDK to use AWS FIPS endpoints, and deploy container built with BoringCrypto (a FIPS-validated cryptographic library) instead of the Go default +# +# The EBS CSI Driver FIPS images have not undergone FIPS certification, and no official guarnatee is made about the compliance of these images under the FIPS standard +# Users relying on these images for FIPS compliance should perform their own independent evaluation +fips: false sidecars: provisioner: env: [] image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner - tag: "v5.0.1-eks-1-30-8" + tag: "v5.1.0-eks-1-31-12" logLevel: 2 # Additional parameters provided by external-provisioner. additionalArgs: [] @@ -44,7 +49,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-attacher - tag: "v4.6.1-eks-1-30-8" + tag: "v4.8.0-eks-1-31-12" # Tune leader lease election for csi-attacher. # Leader election is on by default. leaderElection: @@ -73,7 +78,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-snapshotter/csi-snapshotter - tag: "v8.0.1-eks-1-30-8" + tag: "v8.2.0-eks-1-31-12" logLevel: 2 # Additional parameters provided by csi-snapshotter. additionalArgs: [] @@ -89,7 +94,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe - tag: "v2.13.0-eks-1-30-8" + tag: "v2.14.0-eks-1-31-12" # Additional parameters provided by livenessprobe. additionalArgs: [] resources: {} @@ -101,7 +106,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/external-resizer - tag: "v1.11.1-eks-1-30-8" + tag: "v1.12.0-eks-1-31-11" # Tune leader lease election for csi-resizer. # Leader election is on by default. leaderElection: @@ -128,7 +133,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar - tag: "v2.11.0-eks-1-30-8" + tag: "v2.13.0-eks-1-31-12" logLevel: 2 # Additional parameters provided by node-driver-registrar. additionalArgs: [] @@ -150,7 +155,7 @@ sidecars: image: pullPolicy: IfNotPresent repository: public.ecr.aws/ebs-csi-driver/volume-modifier-for-k8s - tag: "v0.3.0" + tag: "v0.5.1" leaderElection: enabled: true # Optional values to tune lease behavior. @@ -196,6 +201,8 @@ controller: operator: NotIn values: - fargate + - auto + - hybrid podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - podAffinityTerm: @@ -245,6 +252,11 @@ controller: deploymentAnnotations: {} podAnnotations: {} podLabels: {} + podDisruptionBudget: + # Warning: Disabling PodDisruptionBudget may lead to delays in stateful workloads starting due to controller + # pod restarts or evictions. + enabled: true + # unhealthyPodEvictionPolicy: priorityClassName: system-cluster-critical # AWS region to use. If not specified then the region will be looked up via the AWS EC2 metadata # service. @@ -335,11 +347,15 @@ controller: # Enable dnsConfig for the controller and node pods dnsConfig: {} node: + # Enable SELinux-only optimizations on the EBS CSI Driver node pods + # Must only be set true if all linux nodes in the DaemonSet have SELinux enabled + selinux: false env: [] envFrom: [] kubeletPath: /var/lib/kubelet loggingFormat: text logLevel: 2 + enableMetrics: false priorityClassName: additionalArgs: [] affinity: @@ -351,6 +367,8 @@ node: operator: NotIn values: - fargate + - auto + - hybrid - key: node.kubernetes.io/instance-type operator: NotIn values: @@ -388,6 +406,10 @@ node: # Enable the linux daemonset creation enableLinux: true enableWindows: false + # Warning: This option will be removed in a future release. It is a temporary workaround for users unable to immediately migrate off of older kernel versions. + # Formats XFS volumes with bigtime=0,inobtcount=0,reflink=0, for mounting onto nodes with linux kernel version <= 5.4. + # Note that XFS volumes formatted with this option will only have timestamp records until 2038. + legacyXFS: false # The number of attachment slots to reserve for system use (and not to be used for CSI volumes) # When this parameter is not specified (or set to -1), the EBS CSI Driver will attempt to determine the number of reserved slots via heuristic # Cannot be specified at the same time as `node.volumeAttachLimit` @@ -426,6 +448,14 @@ node: containerSecurityContext: readOnlyRootFilesystem: true privileged: true + initContainers: [] + # containers to be run before the csi-node's container starts. + # + # Example: + # + # - name: wait + # image: busybox + # command: [ 'sh', '-c', "sleep 20" ] # Enable opentelemetry tracing for the plugin running on the daemonset otelTracing: {} # otelServiceName: ebs-csi-node @@ -476,7 +506,9 @@ volumeSnapshotClasses: [] # Intended for use with older clusters that cannot easily replace the CSIDriver object # This parameter should always be false for new installations useOldCSIDriver: false +# Deploy EBS CSI Driver without controller and associated resources +nodeComponentOnly: false helmTester: enabled: true # Supply a custom image to the ebs-csi-driver-test pod in helm-tester.yaml - image: "gcr.io/k8s-staging-test-infra/kubekins-e2e:v20240611-597c402033-master" + image: "us-central1-docker.pkg.dev/k8s-staging-test-infra/images/kubekins-e2e:v20241230-3006692a6f-master" diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md b/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md index a0a7190d..707d9944 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md @@ -1,4 +1,24 @@ # Helm chart +# v3.1.6 +* Bump app/driver version to `v2.1.5` +# v3.1.5 +* Bump app/driver version to `v2.1.4` +# v3.1.4 +* Bump app/driver version to `v2.1.3` +# v3.1.3 +* Bump app/driver version to `v2.1.2` +# v3.1.2 +* Bump app/driver version to `v2.1.1` +# v3.1.1 +* Bump app/driver version to `v2.1.0` +# v3.1.0 +* Bump app/driver version to `v2.0.9` +# v3.0.9 +* Bump app/driver version to `v2.0.8` +# v3.0.8 +* Bump app/driver version to `v2.0.7` +# v3.0.7 +* Bump app/driver version to `v2.0.6` # v3.0.6 * Bump app/driver version to `v2.0.5` # v3.0.5 @@ -224,4 +244,4 @@ for Controller deployment and Node daemonset * Fixing Controller deployment using `podAnnotations` and `tolerations` values from Node daemonset * Let the user define the whole `tolerations` array, default to `- operator: Exists` * Default `logLevel` lowered from `5` to `2` -* Default `imagePullPolicy` everywhere set to `IfNotPresent` +* Default `imagePullPolicy` everywhere set to `IfNotPresent` \ No newline at end of file diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml index d6a13911..3c2b059d 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 2.0.5 +appVersion: 2.1.5 description: A Helm chart for AWS EFS CSI Driver home: https://github.com/kubernetes-sigs/aws-efs-csi-driver keywords: @@ -15,4 +15,4 @@ maintainers: name: aws-efs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-efs-csi-driver -version: 3.0.6 +version: 3.1.6 diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml index 94e321ca..4753b16f 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml @@ -3,17 +3,18 @@ kind: Deployment apiVersion: apps/v1 metadata: - name: efs-csi-controller + name: {{ .Values.controller.name }} + namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} {{- with .Values.controller.additionalLabels }} {{ toYaml . | nindent 4 }} {{- end }} spec: - replicas: {{ .Values.replicaCount }} + replicas: {{ .Values.controller.replicaCount }} selector: matchLabels: - app: efs-csi-controller + app: {{ .Values.controller.name }} app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- with .Values.controller.updateStrategy }} @@ -23,7 +24,7 @@ spec: template: metadata: labels: - app: efs-csi-controller + app: {{ .Values.controller.name }} app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- with .Values.controller.podLabels }} @@ -93,6 +94,8 @@ spec: - name: AWS_USE_FIPS_ENDPOINT value: "true" {{- end }} + - name: PORT_RANGE_UPPER_BOUND + value: "{{ .Values.portRangeUpperBound }}" {{- with .Values.controller.env }} {{- toYaml . | nindent 12 }} {{- end }} @@ -134,13 +137,16 @@ spec: {{- if hasKey .Values.controller "leaderElectionLeaseDuration" }} - --leader-election-lease-duration={{ .Values.controller.leaderElectionLeaseDuration }} {{- end }} + {{- range .Values.sidecars.csiProvisioner.additionalArgs }} + - {{ . }} + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - {{- with .Values.sidecars.csiProvisioner.resources }} + {{- with default .Values.controller.resources .Values.sidecars.csiProvisioner.resources }} resources: {{ toYaml . | nindent 12 }} {{- end }} {{- with .Values.sidecars.csiProvisioner.securityContext }} @@ -159,7 +165,7 @@ spec: {{- with .Values.controller.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} - {{- with .Values.sidecars.livenessProbe.resources }} + {{- with default .Values.controller.resources .Values.sidecars.livenessProbe.resources }} resources: {{ toYaml . | nindent 12 }} {{- end }} {{- with .Values.sidecars.livenessProbe.securityContext }} @@ -175,4 +181,13 @@ spec: {{- with .Values.controller.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} + {{- if .Values.controller.topologySpreadConstraints }} + {{- $tscLabelSelector := dict "labelSelector" ( dict "matchLabels" ( dict "app" "efs-csi-controller" ) ) }} + {{- $constraints := list }} + {{- range .Values.controller.topologySpreadConstraints }} + {{- $constraints = mustAppend $constraints (mergeOverwrite . $tscLabelSelector) }} + {{- end }} + topologySpreadConstraints: + {{- $constraints | toYaml | nindent 8 }} + {{- end }} {{- end }} diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-pdb.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-pdb.yaml new file mode 100644 index 00000000..43d5e697 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-pdb.yaml @@ -0,0 +1,24 @@ +{{- if .Values.controller.podDisruptionBudget.enabled -}} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ .Values.controller.name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "aws-efs-csi-driver.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ .Values.controller.name }} + app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.controller.podDisruptionBudget.unhealthyPodEvictionPolicy }} + unhealthyPodEvictionPolicy: {{ .Values.controller.podDisruptionBudget.unhealthyPodEvictionPolicy }} + {{- end }} + {{- if .Values.controller.podDisruptionBudget.maxUnavailable }} + maxUnavailable: {{ .Values.controller.podDisruptionBudget.maxUnavailable }} + {{- end }} + {{- if .Values.controller.podDisruptionBudget.minAvailable }} + minAvailable: {{ .Values.controller.podDisruptionBudget.minAvailable }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml index 44326cd5..1a113307 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-serviceaccount.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ .Values.controller.serviceAccount.name }} + namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} {{- with .Values.controller.serviceAccount.annotations }} @@ -74,6 +75,7 @@ kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: efs-csi-provisioner-binding-describe-secrets + namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} subjects: diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/csidriver.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/csidriver.yaml index e6b4d419..6b68cc14 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/csidriver.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/csidriver.yaml @@ -3,8 +3,10 @@ kind: CSIDriver metadata: name: efs.csi.aws.com annotations: + {{- if .Values.useHelmHooksForCSIDriver }} "helm.sh/hook": pre-install, pre-upgrade "helm.sh/hook-delete-policy": before-hook-creation + {{- end }} "helm.sh/resource-policy": keep spec: attachRequired: false diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml index c472b4a9..324350ce 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml @@ -3,8 +3,12 @@ kind: DaemonSet apiVersion: apps/v1 metadata: name: efs-csi-node + namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} + {{- with .Values.node.additionalLabels }} + {{ toYaml . | nindent 4 }} + {{- end }} spec: selector: matchLabels: @@ -88,12 +92,14 @@ spec: - name: AWS_USE_FIPS_ENDPOINT value: "true" {{- end }} + - name: PORT_RANGE_UPPER_BOUND + value: "{{ .Values.portRangeUpperBound }}" {{- with .Values.node.env }} {{- toYaml . | nindent 12 }} {{- end }} volumeMounts: - name: kubelet-dir - mountPath: /var/lib/kubelet + mountPath: {{ .Values.node.kubeletPath }} mountPropagation: "Bidirectional" - name: plugin-dir mountPath: /csi @@ -132,7 +138,7 @@ spec: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/efs.csi.aws.com/csi.sock + value: {{ printf "%s/plugins/efs.csi.aws.com/csi.sock" (trimSuffix "/" .Values.node.kubeletPath) }} - name: KUBE_NODE_NAME valueFrom: fieldRef: @@ -169,15 +175,15 @@ spec: volumes: - name: kubelet-dir hostPath: - path: /var/lib/kubelet + path: {{ .Values.node.kubeletPath }} type: Directory - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/efs.csi.aws.com/ + path: {{ printf "%s/plugins/efs.csi.aws.com/" (trimSuffix "/" .Values.node.kubeletPath) }} type: DirectoryOrCreate - name: registration-dir hostPath: - path: /var/lib/kubelet/plugins_registry/ + path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} type: Directory - name: efs-state-dir hostPath: @@ -193,4 +199,4 @@ spec: type: DirectoryOrCreate {{- with .Values.node.volumes }} {{- toYaml . | nindent 8 }} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml index c90d62be..797f1b34 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-serviceaccount.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: ServiceAccount metadata: name: {{ .Values.node.serviceAccount.name }} + namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} {{- with .Values.node.serviceAccount.annotations }} diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml index 84b48288..4db8025a 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml @@ -5,20 +5,20 @@ nameOverride: "" fullnameOverride: "" -replicaCount: 2 - useFIPS: false +portRangeUpperBound: "21049" + image: repository: public.ecr.aws/efs-csi-driver/amazon/aws-efs-csi-driver - tag: "v2.0.5" + tag: "v2.1.5" pullPolicy: IfNotPresent sidecars: livenessProbe: image: repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe - tag: v2.13.0-eks-1-30-8 + tag: v2.14.0-eks-1-31-5 pullPolicy: IfNotPresent resources: {} securityContext: @@ -27,7 +27,7 @@ sidecars: nodeDriverRegistrar: image: repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar - tag: v2.11.0-eks-1-30-8 + tag: v2.12.0-eks-1-31-5 pullPolicy: IfNotPresent resources: {} securityContext: @@ -36,12 +36,13 @@ sidecars: csiProvisioner: image: repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner - tag: v5.0.1-eks-1-30-8 + tag: v5.1.0-eks-1-31-5 pullPolicy: IfNotPresent resources: {} securityContext: readOnlyRootFilesystem: true allowPrivilegeEscalation: false + additionalArgs: [] imagePullSecrets: [] @@ -50,6 +51,10 @@ imagePullSecrets: [] controller: # Specifies whether a deployment should be created create: true + # Name of the CSI controller service + name: efs-csi-controller + # Number of replicas for the CSI controller service deployment + replicaCount: 2 # Number for the log level verbosity logLevel: 2 # If set, add pv/pvc metadata to plugin create requests as parameters. @@ -63,7 +68,7 @@ controller: # path on efs when deleteing an access point deleteAccessPointRootDir: false podAnnotations: {} - podLabel: {} + podLabels: {} hostNetwork: false priorityClassName: system-cluster-critical dnsPolicy: ClusterFirst @@ -101,6 +106,12 @@ controller: # eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/efs-csi-role healthPort: 9909 regionalStsEndpoints: false + # Pod Disruption Budget + podDisruptionBudget: + enabled: false + # maxUnavailable: 1 + minAvailable: 1 + unhealthyPodEvictionPolicy: IfHealthyBudget # securityContext on the controller pod securityContext: runAsNonRoot: false @@ -113,7 +124,18 @@ controller: privileged: true leaderElectionRenewDeadline: 10s leaderElectionLeaseDuration: 15s - + # TSCs without the label selector stanza + # + # Example: + # + # topologySpreadConstraints: + # - maxSkew: 1 + # topologyKey: topology.kubernetes.io/zone + # whenUnsatisfiable: ScheduleAnyway + # - maxSkew: 1 + # topologyKey: kubernetes.io/hostname + # whenUnsatisfiable: ScheduleAnyway + topologySpreadConstraints: [] ## Node daemonset variables @@ -144,6 +166,7 @@ node: # - 169.254.169.253 podLabels: {} podAnnotations: {} + additionalLabels: {} resources: {} # limits: @@ -153,7 +176,8 @@ node: # cpu: 100m # memory: 128Mi nodeSelector: {} - updateStrategy: {} + updateStrategy: + {} # Override default strategy (RollingUpdate) to speed up deployment. # This can be useful if helm timeouts are observed. # type: OnDelete @@ -168,6 +192,7 @@ node: operator: NotIn values: - fargate + - hybrid # Specifies whether a service account should be created serviceAccount: create: true @@ -186,6 +211,7 @@ node: env: [] volumes: [] volumeMounts: [] + kubeletPath: /var/lib/kubelet storageClasses: [] # Add StorageClass resources like: @@ -206,3 +232,6 @@ storageClasses: [] # ensureUniqueDirectory: true # reclaimPolicy: Delete # volumeBindingMode: Immediate + +# Specifies wether to use helm hooks to apply the CSI driver +useHelmHooksForCSIDriver: true diff --git a/charts/kubezero-storage/charts/lvm-localpv/Chart.yaml b/charts/kubezero-storage/charts/lvm-localpv/Chart.yaml index 50d04e8c..ebef807d 100644 --- a/charts/kubezero-storage/charts/lvm-localpv/Chart.yaml +++ b/charts/kubezero-storage/charts/lvm-localpv/Chart.yaml @@ -1,10 +1,10 @@ apiVersion: v2 -appVersion: 1.6.0 +appVersion: 1.6.1 dependencies: - condition: crds.enabled name: crds repository: "" - version: 1.6.0 + version: 1.6.2 description: CSI Driver for dynamic provisioning of LVM Persistent Local Volumes. home: https://openebs.io/ icon: https://raw.githubusercontent.com/cncf/artwork/master/projects/openebs/icon/color/openebs-icon-color.png @@ -18,4 +18,4 @@ keywords: name: lvm-localpv sources: - https://github.com/openebs/lvm-localpv -version: 1.6.0 +version: 1.6.2 diff --git a/charts/kubezero-storage/charts/lvm-localpv/README.md b/charts/kubezero-storage/charts/lvm-localpv/README.md index e9385ec4..4796e35c 100644 --- a/charts/kubezero-storage/charts/lvm-localpv/README.md +++ b/charts/kubezero-storage/charts/lvm-localpv/README.md @@ -38,7 +38,7 @@ $ helm install [RELEASE_NAME] openebs-lvmlocalpv/lvm-localpv --namespace [NAMESP **Note:** If moving from the operator to helm -- Make sure the namespace provided in the helm install command is same as `LVM_NAMESPACE` (by default it is `openebs`) env in the controller deployment. +- Make sure the namespace provided in the helm install command is same as `OPENEBS_NAMESPACE` (by default it is `openebs`) env in the controller deployment. - Before installing, clean up the stale deployment and daemonset from `kube-system` namespace using the below commands ```sh kubectl delete deployment openebs-lvm-controller -n kube-system @@ -92,7 +92,7 @@ helm install openebs-lvmlocalpv openebs-lvmlocalpv/lvm-localpv --namespace opene | `lvmPlugin.image.registry` | Registry for openebs-lvm-plugin image | `""` | | `lvmPlugin.image.repository` | Image repository for openebs-lvm-plugin | `openebs/lvm-driver` | | `lvmPlugin.image.pullPolicy` | Image pull policy for openebs-lvm-plugin | `IfNotPresent` | -| `lvmPlugin.image.tag` | Image tag for openebs-lvm-plugin | `1.6.0` | +| `lvmPlugin.image.tag` | Image tag for openebs-lvm-plugin | `1.6.1` | | `lvmPlugin.metricsPort` | The TCP port number used for exposing lvm-metrics | `9500` | | `lvmPlugin.allowedTopologies` | The comma seperated list of allowed node topologies | `kubernetes.io/hostname,` | | `lvmNode.driverRegistrar.image.registry` | Registry for csi-node-driver-registrar image | `registry.k8s.io/` | diff --git a/charts/kubezero-storage/charts/lvm-localpv/charts/crds/Chart.yaml b/charts/kubezero-storage/charts/lvm-localpv/charts/crds/Chart.yaml index bf85f488..604a2359 100644 --- a/charts/kubezero-storage/charts/lvm-localpv/charts/crds/Chart.yaml +++ b/charts/kubezero-storage/charts/lvm-localpv/charts/crds/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v2 description: A Helm chart that collects CustomResourceDefinitions (CRDs) from lvm-localpv. name: crds -version: 1.6.0 +version: 1.6.2 diff --git a/charts/kubezero-storage/charts/lvm-localpv/templates/_helpers.tpl b/charts/kubezero-storage/charts/lvm-localpv/templates/_helpers.tpl index b43ec257..34580e9e 100644 --- a/charts/kubezero-storage/charts/lvm-localpv/templates/_helpers.tpl +++ b/charts/kubezero-storage/charts/lvm-localpv/templates/_helpers.tpl @@ -60,7 +60,7 @@ Define meta labels for openebs lvm-localpv components {{- define "lvmlocalpv.common.metaLabels" -}} chart: {{ template "lvmlocalpv.chart" . }} heritage: {{ .Release.Service }} -openebs.io/version: {{ .Values.release.version | quote }} +openebs.io/version: {{ .Chart.AppVersion | quote }} role: {{ .Values.role | quote }} {{- end -}} diff --git a/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-controller.yaml b/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-controller.yaml index b1729d4f..2f97edb6 100644 --- a/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-controller.yaml +++ b/charts/kubezero-storage/charts/lvm-localpv/templates/lvm-controller.yaml @@ -107,6 +107,14 @@ spec: value: "lvm-localpv-helm" - name: OPENEBS_IO_ENABLE_ANALYTICS value: "{{ .Values.analytics.enabled }}" + {{- if .Values.analytics.gaId }} + - name: GA_ID + value: {{ .Values.analytics.gaId | quote }} + {{- end }} + {{- if .Values.analytics.gaKey }} + - name: GA_KEY + value: {{ .Values.analytics.gaKey | quote }} + {{- end }} args : - "--endpoint=$(OPENEBS_CSI_ENDPOINT)" - "--plugin=$(OPENEBS_CONTROLLER_DRIVER)" diff --git a/charts/kubezero-storage/charts/lvm-localpv/values.yaml b/charts/kubezero-storage/charts/lvm-localpv/values.yaml index c40f7405..255ca067 100644 --- a/charts/kubezero-storage/charts/lvm-localpv/values.yaml +++ b/charts/kubezero-storage/charts/lvm-localpv/values.yaml @@ -1,8 +1,6 @@ # Default values for openebs-lvmlocalpv. # This is a YAML-formatted file. # Declare variables to be passed into your templates. -release: - version: "1.6.0" imagePullSecrets: # - name: "image-pull-secret" @@ -151,7 +149,7 @@ lvmPlugin: repository: openebs/lvm-driver pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: 1.6.0 + tag: 1.6.1 ioLimits: enabled: false containerRuntime: containerd diff --git a/charts/kubezero-storage/jsonnet/dashboards/openebs-diskpool.json b/charts/kubezero-storage/jsonnet/dashboards/openebs-diskpool.json deleted file mode 100644 index 4c679eff..00000000 --- a/charts/kubezero-storage/jsonnet/dashboards/openebs-diskpool.json +++ /dev/null @@ -1,836 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 41, - "links": [ ], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [ - { - "options": { - "1": { - "index": 0, - "text": "Online" - }, - "2": { - "index": 1, - "text": "Degraded" - }, - "3": { - "index": 2, - "text": "Faulted" - }, - "4": { - "index": 3, - "text": "Removed" - }, - "5": { - "index": 4, - "text": "Unavail" - }, - "6": { - "index": 5, - "text": "NoPoolsAvailable" - } - }, - "type": "value" - } - ], - "max": 6, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "maxDataPoints": 827, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "timezone": [ - "browser" - ], - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "diskpool_status{name=~\"$mayastor_diskpool\"}", - "instant": false, - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "DiskPool Status", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "gbytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "maxDataPoints": 827, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "diskpool_total_size_bytes{name=~\"$mayastor_diskpool\"}/(1024*1024*1024)", - "instant": false, - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "DiskPool Total Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "gbytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 3, - "maxDataPoints": 827, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "diskpool_used_size_bytes{name=~\"$mayastor_diskpool\"}/(1024*1024*1024)", - "instant": false, - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "DiskPool Used Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "gbytes" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 4, - "maxDataPoints": 827, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "(diskpool_total_size_bytes{name=~\"$mayastor_diskpool\"}-diskpool_used_size_bytes{name=~\"$mayastor_diskpool\"})/(1024*1024*1024)", - "instant": false, - "legendFormat": "{{name}}", - "range": true, - "refId": "A" - } - ], - "title": "DiskPool Available Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "iops" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": false, - "expr": "irate(diskpool_num_read_ops{name=~\"$mayastor_diskpool\"}[1m])", - "interval": "", - "legendFormat": "read iops", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "irate(diskpool_num_write_ops{name=~\"$mayastor_diskpool\"}[1m])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "write iops", - "range": true, - "refId": "B" - } - ], - "title": "IOPS", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "binBps" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": true, - "expr": "irate(diskpool_bytes_read{name=~\"$mayastor_diskpool\"}[1m])", - "interval": "", - "legendFormat": "read", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "irate(diskpool_bytes_written{name=~\"$mayastor_diskpool\"}[1m])", - "hide": false, - "instant": false, - "legendFormat": "write", - "range": true, - "refId": "B" - } - ], - "title": "Throughput", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "s" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 6, - "y": 24 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": true, - "expr": "((irate(diskpool_read_latency_us{name=~\"$mayastor_diskpool\"}[1m]))/(irate(diskpool_num_read_ops{name=~\"$mayastor_diskpool\"}[1m])))/1000000", - "interval": "", - "legendFormat": "read", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "((irate(diskpool_write_latency_us{name=~\"$mayastor_diskpool\"}[1m]))/(irate(diskpool_num_write_ops{name=~\"$mayastor_diskpool\"}[1m])))/1000000", - "hide": false, - "instant": false, - "legendFormat": "write", - "range": true, - "refId": "B" - } - ], - "title": "Latency", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 39, - "tags": [ - "OpenEBS", - "Mayastor" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "Prometheus", - "value": "prometheus" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "datasource", - "options": [ ], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": true, - "text": [ - "pool-on-node-1" - ], - "value": [ - "pool-on-node-1" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "definition": "label_values(diskpool_status,name)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "mayastor_diskpool", - "options": [ ], - "query": { - "qryType": 1, - "query": "label_values(diskpool_status,name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { }, - "timezone": "browser", - "title": "OpenEBS / Replicated PV / Mayastor / DiskPool", - "uid": "edl0359q6u1a8f", - "version": 7, - "weekStart": "" -} diff --git a/charts/kubezero-storage/jsonnet/dashboards/openebs-volume-replica.json b/charts/kubezero-storage/jsonnet/dashboards/openebs-volume-replica.json deleted file mode 100644 index 1081c480..00000000 --- a/charts/kubezero-storage/jsonnet/dashboards/openebs-volume-replica.json +++ /dev/null @@ -1,422 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 43, - "links": [ ], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "iops" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": false, - "expr": "irate(replica_num_read_ops{name=~\"$replica_name\"}[1m])", - "interval": "", - "legendFormat": "read", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "irate(replica_num_write_ops{name=~\"$replica_name\"}[1m])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "write", - "range": true, - "refId": "B" - } - ], - "title": "IOPS", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "binBps" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": true, - "expr": "irate(replica_bytes_read{name=~\"$replica_name\"}[1m])", - "interval": "", - "legendFormat": "read", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "irate(replica_bytes_written{name=~\"$replica_name\"}[1m])", - "hide": false, - "instant": false, - "legendFormat": "write", - "range": true, - "refId": "B" - } - ], - "title": "Throughput", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 6, - "y": 8 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": true, - "expr": "((irate(replica_read_latency_us{name=~\"$replica_name\"}[1m]))/(irate(replica_num_read_ops{name=~\"$replica_name\"}[1m])))/1000000", - "interval": "", - "legendFormat": "read", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "((irate(replica_write_latency_us{name=~\"$replica_name\"}[1m]))/(irate(replica_num_write_ops{name=~\"$replica_name\"}[1m])))/1000000", - "hide": false, - "instant": false, - "legendFormat": "write", - "range": true, - "refId": "B" - } - ], - "title": "Latency", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 39, - "tags": [ - "OpenEBS", - "Mayastor" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "Prometheus", - "value": "prometheus" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(replica_num_read_ops,name)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "replica_name", - "options": [ ], - "query": { - "qryType": 1, - "query": "label_values(replica_num_read_ops,name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { }, - "timezone": "browser", - "title": "OpenEBS / Replicated PV / Mayastor / Volume Replica", - "uid": "fdl05xto1hn28e", - "version": 6, - "weekStart": "" -} diff --git a/charts/kubezero-storage/jsonnet/dashboards/openebs-volume.json b/charts/kubezero-storage/jsonnet/dashboards/openebs-volume.json deleted file mode 100644 index 44537e86..00000000 --- a/charts/kubezero-storage/jsonnet/dashboards/openebs-volume.json +++ /dev/null @@ -1,422 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 42, - "links": [ ], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "iops" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": false, - "expr": "irate(volume_num_read_ops{pv_name=~\"$pv_name\"}[1m])", - "interval": "", - "legendFormat": "read", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "irate(volume_num_write_ops{pv_name=~\"$pv_name\"}[1m])", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "write", - "range": true, - "refId": "B" - } - ], - "title": "IOPS", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "binBps" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": true, - "expr": "irate(volume_bytes_read{pv_name=~\"$pv_name\"}[1m])", - "interval": "", - "legendFormat": "read", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "irate(volume_bytes_written{pv_name=~\"$pv_name\"}[1m])", - "hide": false, - "instant": false, - "legendFormat": "write", - "range": true, - "refId": "B" - } - ], - "title": "Throughput", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [ ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 6, - "y": 8 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "exemplar": true, - "expr": "((irate(volume_read_latency_us{pv_name=~\"$pv_name\"}[1m]))/(irate(volume_num_read_ops{pv_name=~\"$pv_name\"}[1m])))/1000000", - "interval": "", - "legendFormat": "read", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "((irate(volume_write_latency_us{pv_name=~\"$pv_name\"}[1m]))/(irate(volume_num_write_ops{pv_name=~\"$pv_name\"}[1m])))/1000000", - "hide": false, - "instant": false, - "legendFormat": "write", - "range": true, - "refId": "B" - } - ], - "title": "Latency", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 39, - "tags": [ - "OpenEBS", - "Mayastor" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "Prometheus", - "value": "prometheus" - }, - "hide": 0, - "includeAll": false, - "multi": false, - "name": "datasource", - "options": [ ], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "current": { - "selected": true, - "text": [ - "pvc-d070ac1c-6bb7-4ece-9893-5471c0c636a8" - ], - "value": [ - "pvc-d070ac1c-6bb7-4ece-9893-5471c0c636a8" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "definition": "label_values(volume_num_read_ops,pv_name)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "pv_name", - "options": [ ], - "query": { - "qryType": 1, - "query": "label_values(volume_num_read_ops,pv_name)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { }, - "timezone": "browser", - "title": "OpenEBS / Replicated PV / Mayastor / Volume", - "uid": "fdl05gxyisqo0d", - "version": 6, - "weekStart": "" -} diff --git a/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json b/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json index 468ee948..2b7c226d 100644 --- a/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json +++ b/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json @@ -18,8 +18,8 @@ "subdir": "contrib/mixin" } }, - "version": "9f59ef8ead097f836271f125d0e3774ddae4e71d", - "sum": "IXI3LQIT9NmTPJAk8WLUJd5+qZfcGpeNCyWIK7oEpws=" + "version": "8c52b414f324d6369b77096af98d8f0416fe20cb", + "sum": "XmXkOCriQIZmXwlIIFhqlJMa0e6qGWdxZD+ZDYaN0Po=" }, { "source": { @@ -41,16 +41,6 @@ "version": "a1d61cce1da59c71409b99b5c7568511fec661ea", "sum": "342u++/7rViR/zj2jeJOjshzglkZ1SY+hFNuyCBFMdc=" }, - { - "source": { - "git": { - "remote": "https://github.com/grafana/grafonnet-lib.git", - "subdir": "grafonnet-7.0" - } - }, - "version": "a1d61cce1da59c71409b99b5c7568511fec661ea", - "sum": "gCtR9s/4D5fxU9aKXg0Bru+/njZhA0YjLjPiASc61FM=" - }, { "source": { "git": { @@ -58,8 +48,8 @@ "subdir": "gen/grafonnet-latest" } }, - "version": "5a66b0f6a0f4f7caec754dd39a0e263b56a0f90a", - "sum": "eyuJ0jOXeA4MrobbNgU4/v5a7ASDHslHZ0eS6hDdWoI=" + "version": "d20e609202733790caf5b554c9945d049f243ae3", + "sum": "V9vAj21qJOc2DlMPDgB1eEjSQU4A+sAA4AXuJ6bd4xc=" }, { "source": { @@ -68,18 +58,18 @@ "subdir": "gen/grafonnet-v10.0.0" } }, - "version": "5a66b0f6a0f4f7caec754dd39a0e263b56a0f90a", + "version": "d20e609202733790caf5b554c9945d049f243ae3", "sum": "xdcrJPJlpkq4+5LpGwN4tPAuheNNLXZjE6tDcyvFjr0=" }, { "source": { "git": { "remote": "https://github.com/grafana/grafonnet.git", - "subdir": "gen/grafonnet-v11.0.0" + "subdir": "gen/grafonnet-v11.4.0" } }, - "version": "5a66b0f6a0f4f7caec754dd39a0e263b56a0f90a", - "sum": "Fuo+qTZZzF+sHDBWX/8fkPsUmwW6qhH8hRVz45HznfI=" + "version": "d20e609202733790caf5b554c9945d049f243ae3", + "sum": "aVAX09paQYNOoCSKVpuk1exVIyBoMt/C50QJI+Q/3nA=" }, { "source": { @@ -88,7 +78,7 @@ "subdir": "grafana-builder" } }, - "version": "3f0a5b0eeb2f5dc381a420b35d27198bd9b72e8c", + "version": "393630ca7ba9b25258c098f1fd4c81962e3ca046", "sum": "yxqWcq/N3E/a/XreeU6EuE6X7kYPnG0AspAQFKOjASo=" }, { @@ -108,8 +98,8 @@ "subdir": "" } }, - "version": "63d430b69a95741061c2f7fc9d84b1a778511d9c", - "sum": "qiZi3axUSXCVzKUF83zSAxklwrnitMmrDK4XAfjPMdE=" + "version": "1199b50e9d2ff53d4bb5fb2304ad1fb69d38e609", + "sum": "LfbgcJbilu4uBdKYZSvmkoOTPwEAzg10L3/VqKAIWtA=" }, { "source": { @@ -118,8 +108,8 @@ "subdir": "" } }, - "version": "dd5c59ab4491159593ed370a344a553b57146a7d", - "sum": "2tFZyRtLw9nasUQdFn5LGGqJplJyAeJxd59u6mHU+mw=" + "version": "4ff562d5e8145940cf355f62cf2308895c4dca81", + "sum": "kiL19fTbXOtNglsmT62kOzIf/Xpu+YwoiMPAApDXhkE=" }, { "source": { @@ -128,8 +118,8 @@ "subdir": "jsonnet/kube-state-metrics" } }, - "version": "e96dfc0a39d8e2ae759a954a98d8bc9b29bf1a3e", - "sum": "h6H5AsU7JsCAWttnPgevTNituobj2eIr2ebxdkaABQo=" + "version": "2a95d4649b2fea55799032fb9c0b571c4ba7f776", + "sum": "3bioG7CfTfY9zeu5xU4yon6Zt3kYvNkyl492nOhQxnM=" }, { "source": { @@ -138,7 +128,7 @@ "subdir": "jsonnet/kube-state-metrics-mixin" } }, - "version": "e96dfc0a39d8e2ae759a954a98d8bc9b29bf1a3e", + "version": "2a95d4649b2fea55799032fb9c0b571c4ba7f776", "sum": "qclI7LwucTjBef3PkGBkKxF0mfZPbHnn4rlNWKGtR4c=" }, { @@ -148,8 +138,8 @@ "subdir": "" } }, - "version": "ac1fc45d5b5012c1c0cd97e49d8168b783f6ec74", - "sum": "x4ichkWQBusHv8ph2UddBrbe4oDbKlOtwXqzQ4X+8mI=" + "version": "d2dc72021d0247a5199007ed6e425d4615f9fa5c", + "sum": "rHh5ItS3fs1kwz8GKNEPiBBn58m4Bn5v9KAdBU+tf1U=" }, { "source": { @@ -158,8 +148,8 @@ "subdir": "jsonnet/kube-prometheus" } }, - "version": "defa2bd1e242519c62a5c2b3b786b1caa6d906d4", - "sum": "INKeZ+QIIPImq+TrfHT8CpYdoRzzxRk0txG07XlOo/Q=" + "version": "1eea946a1532f1e8cccfceea98d907bf3a10b1d9", + "sum": "17LhiwefVfoNDsF3DcFZw/UL4PMU7YpNNUaOdaYd1gE=" }, { "source": { @@ -168,7 +158,7 @@ "subdir": "jsonnet/mixin" } }, - "version": "105b88afada91ecd4dab14b6d091b0933c749972", + "version": "7deab71d6d5921eeaf8c79e3ae8e31efe63783a9", "sum": "gi+knjdxs2T715iIQIntrimbHRgHnpM8IFBJDD1gYfs=", "name": "prometheus-operator-mixin" }, @@ -179,8 +169,8 @@ "subdir": "jsonnet/prometheus-operator" } }, - "version": "105b88afada91ecd4dab14b6d091b0933c749972", - "sum": "lTyttpFADJ40Zd7FuwgXcXswU+7grlQBeXms7gyabYc=" + "version": "7deab71d6d5921eeaf8c79e3ae8e31efe63783a9", + "sum": "LctDdofQostvviE5y8vpRKWGGO1ZKO3dgJe7P9xifW0=" }, { "source": { @@ -189,8 +179,8 @@ "subdir": "doc/alertmanager-mixin" } }, - "version": "cad5fa580108431e6ed209f2a23a373aa50c098f", - "sum": "IpF46ZXsm+0wJJAPtAre8+yxTNZA57mBqGpBP/r7/kw=", + "version": "b5d1a64ad5bb0ff879705714d1e40cea82efbd5c", + "sum": "Mf4h1BYLle2nrgjf/HXrBbl0Zk8N+xaoEM017o0BC+k=", "name": "alertmanager" }, { @@ -200,8 +190,8 @@ "subdir": "docs/node-mixin" } }, - "version": "b9d0932179a0c5b3a8863f3d6cdafe8584cedc8e", - "sum": "rhUvbqviGjQ2mwsRhHKMN0TiS3YvnYpUXHew3XlQ+Wg=" + "version": "11365f97bef6cb0e6259d536a7e21c49e3f5c065", + "sum": "xYj6VYFT/eafsbleNlC+Z2VfLy1CndyYrJs9BcTmnX8=" }, { "source": { @@ -210,19 +200,19 @@ "subdir": "documentation/prometheus-mixin" } }, - "version": "d4f098ae80fb276153efc757e373c813163da0e8", - "sum": "dYLcLzGH4yF3qB7OGC/7z4nqeTNjv42L7Q3BENU8XJI=", + "version": "a5ffa83be83be22e2ec9fd1d4765299d8d16119e", + "sum": "2c+wttfee9TwuQJZIkNV7Tekem74Qgc7iZ842P28rNw=", "name": "prometheus" }, { "source": { "git": { "remote": "https://github.com/pyrra-dev/pyrra.git", - "subdir": "config/crd/bases" + "subdir": "jsonnet/controller-gen" } }, - "version": "551856d42dff02ec38c5b0ea6a2d99c4cb127e82", - "sum": "bY/Pcrrbynguq8/HaI88cQ3B2hLv/xc+76QILY7IL+g=", + "version": "d723f4d1a066dd657e9d09c46a158519dda0faa8", + "sum": "cxAPQovFkM16zNB5/94O+sk/n3SETk6ao6Oas2Sa6RE=", "name": "pyrra" }, { @@ -232,8 +222,8 @@ "subdir": "mixin" } }, - "version": "639bf8f216494ad9c375ebaac45f5d15715065ba", - "sum": "HhSSbGGCNHCMy1ee5jElYDm0yS9Vesa7QB2/SHKdjsY=", + "version": "346d18bb0f8011c63d7106de494cf3b9253161a1", + "sum": "ieCD4eMgGbOlrI8GmckGPHBGQDcLasE1rULYq56W/bs=", "name": "thanos-mixin" } ], diff --git a/charts/kubezero-storage/jsonnet/rules/openebs-mixin-prometheusRules b/charts/kubezero-storage/jsonnet/rules/openebs-mixin-prometheusRules index febe0936..0ac7eccf 100644 --- a/charts/kubezero-storage/jsonnet/rules/openebs-mixin-prometheusRules +++ b/charts/kubezero-storage/jsonnet/rules/openebs-mixin-prometheusRules @@ -24,6 +24,8 @@ "expr": "lvm_vg_missing_pv_count > 0", "for": "5m", "labels": { + "engine": "localpv-lvm", + "product": "openebs", "severity": "critical" } }, @@ -38,6 +40,8 @@ "expr": "((lvm_vg_total_size_bytes - lvm_vg_free_size_bytes)/lvm_vg_total_size_bytes)*100 > 90", "for": "5m", "labels": { + "engine": "localpv-lvm", + "product": "openebs", "severity": "critical" } }, @@ -52,6 +56,8 @@ "expr": "lvm_lv_used_percent{segtype=\"thin-pool\"} > 90", "for": "5m", "labels": { + "engine": "localpv-lvm", + "product": "openebs", "severity": "critical" } } diff --git a/charts/kubezero-storage/templates/lvm/prometheus-rules.yaml b/charts/kubezero-storage/templates/lvm/prometheus-rules.yaml index 65c7e348..96fbb57c 100644 --- a/charts/kubezero-storage/templates/lvm/prometheus-rules.yaml +++ b/charts/kubezero-storage/templates/lvm/prometheus-rules.yaml @@ -19,6 +19,8 @@ spec: expr: lvm_vg_missing_pv_count > 0 for: 5m labels: + engine: localpv-lvm + product: openebs severity: critical - alert: LVMVolumeGroupCapacityLow annotations: @@ -29,6 +31,8 @@ spec: expr: ((lvm_vg_total_size_bytes - lvm_vg_free_size_bytes)/lvm_vg_total_size_bytes)*100 > 90 for: 5m labels: + engine: localpv-lvm + product: openebs severity: critical - alert: LVMThinPoolCapacityLow annotations: @@ -39,6 +43,8 @@ spec: expr: lvm_lv_used_percent{segtype="thin-pool"} > 90 for: 5m labels: + engine: localpv-lvm + product: openebs severity: critical - name: persistent-volume-claim rules: diff --git a/charts/kubezero-storage/templates/snapshot-controller/rbac.yaml b/charts/kubezero-storage/templates/snapshot-controller/rbac.yaml index 4305413c..e458ec23 100644 --- a/charts/kubezero-storage/templates/snapshot-controller/rbac.yaml +++ b/charts/kubezero-storage/templates/snapshot-controller/rbac.yaml @@ -39,7 +39,7 @@ rules: verbs: ["patch"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] - verbs: ["get", "list", "watch", "update", "patch", "delete"] + verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots/status"] verbs: ["update", "patch"] diff --git a/charts/kubezero-storage/values.yaml b/charts/kubezero-storage/values.yaml index bb4be298..d19e648e 100644 --- a/charts/kubezero-storage/values.yaml +++ b/charts/kubezero-storage/values.yaml @@ -104,6 +104,7 @@ aws-ebs-csi-driver: replicaCount: 1 loggingFormat: json + revisionHistoryLimit: 3 defaultFsType: xfs @@ -116,10 +117,8 @@ aws-ebs-csi-driver: # k8sTagClusterId: # region: - # Enable once we have a customer needing it - # so far the default storageclass does just fine and we rather have different classes volumeModificationFeature: - enabled: false + enabled: true #env: # ebsPlugin: @@ -153,6 +152,8 @@ aws-ebs-csi-driver: node: loggingFormat: json + revisionHistoryLimit: 3 + tolerateAllTaints: false priorityClassName: system-node-critical @@ -170,6 +171,8 @@ aws-ebs-csi-driver: effect: NoSchedule operator: Exists + enableMetrics: false + resources: requests: cpu: 10m @@ -202,15 +205,17 @@ aws-ebs-csi-driver: aws-efs-csi-driver: enabled: false - replicaCount: 1 + useHelmHooksForCSIDriver: false - image: - tag: "v2.0.9" +# image: +# tag: "v2.0.9" controller: create: true logLevel: 2 + replicaCount: 1 + tolerations: - key: node-role.kubernetes.io/control-plane effect: NoSchedule diff --git a/charts/kubezero/templates/istio.yaml b/charts/kubezero/templates/istio.yaml index 6df9429d..90bc4230 100644 --- a/charts/kubezero/templates/istio.yaml +++ b/charts/kubezero/templates/istio.yaml @@ -40,7 +40,7 @@ rateLimiting: - group: apiextensions.k8s.io kind: CustomResourceDefinition jsonPointers: - - /metadata/labels + - /metadata/annotations - /spec/additionalPrinterColumns - group: admissionregistration.k8s.io kind: ValidatingWebhookConfiguration diff --git a/charts/kubezero/templates/storage.yaml b/charts/kubezero/templates/storage.yaml index 2eb5467e..f957efc4 100644 --- a/charts/kubezero/templates/storage.yaml +++ b/charts/kubezero/templates/storage.yaml @@ -26,6 +26,10 @@ aws-ebs-csi-driver: enabled: {{ default false (index .Values "storage" "aws-ebs-csi-driver" "enabled")}} controller: replicaCount: {{ ternary 2 1 .Values.global.highAvailable }} + {{- if not .Values.global.highAvailable }} + podDisruptionBudget: + enabled: false + {{- end }} k8sTagClusterId: {{ .Values.global.clusterName }} enableMetrics: {{ .Values.metrics.enabled }} region: {{ .Values.global.aws.region }} @@ -37,6 +41,9 @@ aws-ebs-csi-driver: - name: AWS_STS_REGIONAL_ENDPOINTS value: regional + node: + enableMetrics: {{ .Values.metrics.enabled }} + {{- if not .Values.global.highAvailable }} sidecars: provisioner: @@ -55,8 +62,13 @@ aws-ebs-csi-driver: aws-efs-csi-driver: enabled: {{ default false (index .Values "storage" "aws-efs-csi-driver" "enabled")}} - replicaCount: {{ ternary 2 1 .Values.global.highAvailable }} controller: + replicaCount: {{ ternary 2 1 .Values.global.highAvailable }} + {{- if .Values.global.highAvailable }} + podDisruptionBudget: + enabled: true + {{- end }} + {{- with index .Values "storage" "aws-efs-csi-driver" "IamArn" }} extraEnv: - name: AWS_ROLE_ARN diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index 7344e055..bdfb87ec 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -43,11 +43,11 @@ network: cert-manager: enabled: false namespace: cert-manager - targetRevision: 0.9.11 + targetRevision: 0.9.12 storage: enabled: false - targetRevision: 0.8.9 + targetRevision: 0.8.10 lvm-localpv: enabled: false aws-ebs-csi-driver: @@ -119,7 +119,7 @@ logging: argo: enabled: false namespace: argocd - targetRevision: 0.2.7 + targetRevision: 0.2.8 argo-cd: enabled: false istio: