diff --git a/charts/kubezero-storage/Chart.yaml b/charts/kubezero-storage/Chart.yaml index f5e83336..5aa039c7 100644 --- a/charts/kubezero-storage/Chart.yaml +++ b/charts/kubezero-storage/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-storage description: KubeZero umbrella chart for all things storage incl. AWS EBS/EFS, openEBS-lvm, gemini type: application -version: 0.7.1 +version: 0.7.2 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png keywords: @@ -28,11 +28,11 @@ dependencies: condition: gemini.enabled # repository: https://charts.fairwinds.com/stable - name: aws-ebs-csi-driver - version: 2.11.0 + version: 2.12.0 condition: aws-ebs-csi-driver.enabled # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver - name: aws-efs-csi-driver - version: 2.2.7 + version: 2.2.9 condition: aws-efs-csi-driver.enabled # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver kubeVersion: ">= 1.20.0" diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md index 8f3281d4..6a743650 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md @@ -1,6 +1,21 @@ # Helm chart -# v2.11.0 +## v2.12.0 +* Bump app/driver to version `v1.12.0` +* Move default toleration to values.yaml so it can be overriden if desired by users ([#1400](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1400), [@cnmcavoy](https://github.com/cnmcavoy)) +* Add enableMetrics configuration ([#1380](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1380), [@torredil](https://github.com/torredil)) +* add initContainer to the controller's template ([#1379](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1379), [@InsomniaCoder](https://github.com/InsomniaCoder)) +* Add controller nodeAffinity to prefer EC2 over Fargate ([#1360](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1360), [@torredil](https://github.com/torredil)) + +## v2.11.1 +* Add `useOldCSIDriver` parameter to use old `CSIDriver` object. + +## v2.11.0 + +**Important Notice:** This version updates the `CSIDriver` object in order to fix [a bug with static volumes and the `fsGroup` parameter](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1365). This upgrade will fail on existing clusters because the associated field in `CSIDriver` is immutable. + +Users upgrading to this version should pre-delete the existing `CSIDriver` object (example: `kubectl delete csidriver ebs.csi.aws.com`). This will not affect any existing volumes, but will cause the EBS CSI Driver to be unavailable to handle future requests, and should be immediately followed by an upgrade. For users that cannot delete the `CSIDriver` object, v2.11.1 implements a new parameter `useOldCSIDriver` that will use the previous `CSIDriver`. + * Bump app/driver to version `v1.11.3` * Add support for leader election tuning for `csi-provisioner` and `csi-attacher` ([#1371](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1371), [@moogzy](https://github.com/moogzy)) * Change `fsGroupPolicy` to `File` ([#1377](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1377), [@ConnorJC3](https://github.com/ConnorJC3)) diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml index 03c789cf..d3a0d4ff 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml @@ -3,7 +3,7 @@ annotations: - kind: added description: Custom controller.updateStrategy to set controller deployment strategy. apiVersion: v2 -appVersion: 1.11.3 +appVersion: 1.12.0 description: A Helm chart for AWS EBS CSI Driver home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver keywords: @@ -19,4 +19,4 @@ maintainers: name: aws-ebs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-ebs-csi-driver -version: 2.11.0 +version: 2.12.0 diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml index 6afb4a00..9c01bb66 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml @@ -62,6 +62,10 @@ spec: securityContext: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.controller.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: ebs-plugin image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (.Values.image.tag | toString)) }} @@ -79,6 +83,9 @@ spec: {{- with .Values.controller.k8sTagClusterId }} - --k8s-tag-cluster-id={{ . }} {{- end }} + {{- if and (.Values.controller.enableMetrics) (not .Values.controller.httpEndpoint) }} + - --http-endpoint=0.0.0.0:3301 + {{- end}} {{- with .Values.controller.httpEndpoint }} - --http-endpoint={{ . }} {{- end }} @@ -136,6 +143,11 @@ spec: - name: healthz containerPort: 9808 protocol: TCP + {{- if .Values.controller.enableMetrics }} + - name: metrics + containerPort: 3301 + protocol: TCP + {{- end}} livenessProbe: httpGet: path: /healthz diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml index baf05409..a46d4b50 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml @@ -7,4 +7,6 @@ metadata: spec: attachRequired: true podInfoOnMount: false + {{- if not .Values.useOldCSIDriver }} fsGroupPolicy: File + {{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml new file mode 100644 index 00000000..db4ac0d3 --- /dev/null +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml @@ -0,0 +1,40 @@ +{{- if .Values.controller.enableMetrics -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: ebs-csi-controller + namespace: kube-system + labels: + app: ebs-csi-controller +spec: + selector: + app: ebs-csi-controller + ports: + - name: metrics + port: 3301 + targetPort: 3301 + type: ClusterIP +--- +{{- if (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: ebs-csi-controller + namespace: kube-system + labels: + app: ebs-csi-controller + release: prometheus +spec: + selector: + matchLabels: + app: ebs-csi-controller + namespaceSelector: + matchNames: + - kube-system + endpoints: + - targetPort: 3301 + path: /metrics + interval: 15s +{{- end }} +{{- end }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml index f57b949d..5c98fab2 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml @@ -40,13 +40,10 @@ spec: {{- if .Values.node.tolerateAllTaints }} - operator: Exists {{- else }} - - operator: Exists - effect: NoExecute - tolerationSeconds: 300 - {{- end }} {{- with .Values.node.tolerations }} {{- toYaml . | nindent 8 }} {{- end }} + {{- end }} containers: - name: ebs-plugin image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml index ac68406a..ab1db918 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml @@ -40,13 +40,10 @@ spec: {{- if .Values.node.tolerateAllTaints }} - operator: Exists {{- else }} - - operator: Exists - effect: NoExecute - tolerationSeconds: 300 - {{- end }} {{- with .Values.node.tolerations }} {{- toYaml . | nindent 8 }} {{- end }} + {{- end }} {{- with .Values.node.securityContext }} securityContext: {{- toYaml . | nindent 8 }} diff --git a/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml index dc31ef10..047f3fc8 100644 --- a/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml +++ b/charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml @@ -110,7 +110,16 @@ fullnameOverride: controller: # If arbitrary args like "--aws-sdk-debug-log=true" need to be passed, use this value additionalArgs: [] - affinity: {} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate # The default filesystem type of the volume to provision when fstype is unspecified in the StorageClass. # If the default is not set and fstype is unset in the StorageClass, then no fstype will be set defaultFsType: ext4 @@ -126,6 +135,14 @@ controller: # key2: value2 extraVolumeTags: {} httpEndpoint: + # (deprecated) The TCP network address where the prometheus metrics endpoint + # will run (example: `:8080` which corresponds to port 8080 on local host). + # The default is empty string, which means metrics endpoint is disabled. + # --- + enableMetrics: false + # If set to true, AWS API call metrics will be exported to the following + # TCP endpoint: "0.0.0.0:3301" + # --- # ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional). k8sTagClusterId: logLevel: 2 @@ -184,6 +201,14 @@ controller: containerSecurityContext: readOnlyRootFilesystem: true allowPrivilegeEscalation: false + initContainers: [] + # containers to be run before the controller's container starts. + # + # Example: + # + # - name: wait + # image: busybox + # command: [ 'sh', '-c', "sleep 20" ] node: env: [] @@ -204,7 +229,10 @@ node: podAnnotations: {} podLabels: {} tolerateAllTaints: true - tolerations: [] + tolerations: + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 resources: {} serviceAccount: create: true @@ -244,3 +272,8 @@ storageClasses: [] # reclaimPolicy: Retain # parameters: # encrypted: "true" + +# Use old CSIDriver without an fsGroupPolicy set +# Intended for use with older clusters that cannot easily replace the CSIDriver object +# This parameter should always be false for new installations +useOldCSIDriver: false diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md b/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md index 4be84986..23d892c2 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md @@ -1,5 +1,11 @@ # Helm chart +# v2.2.9 +* Bump app/driver version to `v1.4.2` + +# v2.2.8 +* Bump app/driver version to `v1.4.1` + # v2.2.7 * Bump app/driver version to `v1.4.0` # v2.2.6 diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml index 63f86261..0ca18ea1 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.4.0 +appVersion: 1.4.2 description: A Helm chart for AWS EFS CSI Driver home: https://github.com/kubernetes-sigs/aws-efs-csi-driver keywords: @@ -15,4 +15,4 @@ maintainers: name: aws-efs-csi-driver sources: - https://github.com/kubernetes-sigs/aws-efs-csi-driver -version: 2.2.7 +version: 2.2.9 diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml index 06b76e75..aa201082 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml @@ -69,14 +69,18 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - {{- if .Values.controller.extraEnv }} + {{- if .Values.useFIPS }} + - name: AWS_USE_FIPS_ENDPOINT + value: "true" + {{- end }} + {{- if .Values.controller.extraEnv }} {{- toYaml .Values.controller.extraEnv | nindent 12 }} - {{- end }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: aws-token - mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" + mountPath: /var/run/secrets/sts.amazonaws.com/serviceaccount/ ports: - name: healthz containerPort: {{ .Values.controller.healthPort }} diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml index 75ade6dc..d7cd908f 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml @@ -67,6 +67,10 @@ spec: env: - name: CSI_ENDPOINT value: unix:/csi/csi.sock + {{- if .Values.useFIPS }} + - name: AWS_USE_FIPS_ENDPOINT + value: "true" + {{- end }} volumeMounts: - name: kubelet-dir mountPath: /var/lib/kubelet diff --git a/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml b/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml index 511d70e3..f49beb18 100644 --- a/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml +++ b/charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml @@ -7,9 +7,11 @@ fullnameOverride: "" replicaCount: 2 +useFIPS: false + image: repository: amazon/aws-efs-csi-driver - tag: "v1.4.0" + tag: "v1.4.2" pullPolicy: IfNotPresent sidecars: diff --git a/charts/kubezero-storage/efs.patch b/charts/kubezero-storage/efs.patch index e7c5168b..28d0ea72 100644 --- a/charts/kubezero-storage/efs.patch +++ b/charts/kubezero-storage/efs.patch @@ -1,6 +1,6 @@ diff -tuNr charts/aws-efs-csi-driver.orig/templates/controller-deployment.yaml charts/aws-efs-csi-driver/templates/controller-deployment.yaml ---- charts/aws-efs-csi-driver.orig/templates/controller-deployment.yaml 2022-08-24 13:58:54.208936918 +0200 -+++ charts/aws-efs-csi-driver/templates/controller-deployment.yaml 2022-08-24 14:02:53.031437096 +0200 +--- charts/aws-efs-csi-driver.orig/templates/controller-deployment.yaml 2022-10-11 20:57:20.000000000 +0200 ++++ charts/aws-efs-csi-driver/templates/controller-deployment.yaml 2022-10-12 22:40:48.100658738 +0200 @@ -13,6 +13,10 @@ app: efs-csi-controller app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }} @@ -12,22 +12,22 @@ diff -tuNr charts/aws-efs-csi-driver.orig/templates/controller-deployment.yaml c template: metadata: labels: -@@ -65,9 +69,14 @@ - valueFrom: - fieldRef: - fieldPath: spec.nodeName -+ {{- if .Values.controller.extraEnv }} +@@ -69,9 +73,14 @@ + - name: AWS_USE_FIPS_ENDPOINT + value: "true" + {{- end }} ++ {{- if .Values.controller.extraEnv }} + {{- toYaml .Values.controller.extraEnv | nindent 12 }} -+ {{- end }} ++ {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: aws-token -+ mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/" ++ mountPath: /var/run/secrets/sts.amazonaws.com/serviceaccount/ ports: - name: healthz containerPort: {{ .Values.controller.healthPort }} -@@ -118,6 +127,13 @@ +@@ -122,6 +131,13 @@ volumes: - name: socket-dir emptyDir: {} @@ -42,8 +42,8 @@ diff -tuNr charts/aws-efs-csi-driver.orig/templates/controller-deployment.yaml c affinity: {{- toYaml . | nindent 8 }} {{- end }} diff -tuNr charts/aws-efs-csi-driver.orig/templates/node-daemonset.yaml charts/aws-efs-csi-driver/templates/node-daemonset.yaml ---- charts/aws-efs-csi-driver.orig/templates/node-daemonset.yaml 2022-08-24 13:58:54.208936918 +0200 -+++ charts/aws-efs-csi-driver/templates/node-daemonset.yaml 2022-08-24 14:03:42.095284086 +0200 +--- charts/aws-efs-csi-driver.orig/templates/node-daemonset.yaml 2022-10-11 20:57:20.000000000 +0200 ++++ charts/aws-efs-csi-driver/templates/node-daemonset.yaml 2022-10-12 22:39:19.796397838 +0200 @@ -40,15 +40,10 @@ {{- with .Values.node.nodeSelector }} {{- toYaml . | nindent 8 }} diff --git a/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json b/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json index 560a4e29..926e4f07 100644 --- a/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json +++ b/charts/kubezero-storage/jsonnet/jsonnetfile.lock.json @@ -18,7 +18,7 @@ "subdir": "contrib/mixin" } }, - "version": "19002cfc689fba2b8f56605e5797bf79f8b61fdd", + "version": "acc7463fb2dc7037f571a02e65afd6e573a6344c", "sum": "W/Azptf1PoqjyMwJON96UY69MFugDA4IAYiKURscryc=" }, { @@ -38,7 +38,7 @@ "subdir": "grafana-builder" } }, - "version": "dbf6fc14105c28b6fd0253005f7ca2da37d3d4e1", + "version": "d73aff453c9784cd6922119f3ce33d8d355a79e1", "sum": "tDR6yT2GVfw0wTU12iZH+m01HrbIr6g/xN+/8nzNkU0=" }, { @@ -48,8 +48,8 @@ "subdir": "" } }, - "version": "fb9d8ed4bc4a3d6efac525f72e8a0d2c583a0fe2", - "sum": "xjKkdp+5fkekCNBUIgZCHTRmVdUEmQNFKslrL2Ho8gs=" + "version": "ba16d78ef81ca3d183472c2e3d1d7d3549e18ecb", + "sum": "ntQv86Jd9HR4YQlb2yzjYVCAQHMUnAaQVIdhR2QaNNg=" }, { "source": { @@ -58,7 +58,7 @@ "subdir": "lib/promgrafonnet" } }, - "version": "5e44626d70c2bf2d35c37f3fee5a6261a5335cc6", + "version": "7b559e800a32a2a80caf4c968f37c4999ec44689", "sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps=" }, { @@ -98,7 +98,7 @@ "subdir": "jsonnet/kube-prometheus" } }, - "version": "452aaed72e36acb31cae93cfa85a5d9c3d3d2ec7", + "version": "fcd73c717f54a49cce69e78b9bcb709d0b4571ff", "sum": "pupXEvlRbhLdEO9b8LfFZB66+Z7fEqvRZ9m3MyEvsv4=" }, { diff --git a/charts/kubezero-storage/values.yaml b/charts/kubezero-storage/values.yaml index 459a35fc..852b1760 100644 --- a/charts/kubezero-storage/values.yaml +++ b/charts/kubezero-storage/values.yaml @@ -95,10 +95,17 @@ aws-ebs-csi-driver: memory: 40Mi node: + tolerateAllTaints: false tolerations: - key: kubezero-workergroup effect: NoSchedule operator: Exists + - key: nvidia.com/gpu + effect: NoSchedule + operator: Exists + - key: aws.amazon.com/neuron + effect: NoSchedule + operator: Exists resources: requests: @@ -171,6 +178,12 @@ aws-efs-csi-driver: - key: kubezero-workergroup effect: NoSchedule operator: Exists + - key: nvidia.com/gpu + effect: NoSchedule + operator: Exists + - key: aws.amazon.com/neuron + effect: NoSchedule + operator: Exists #storageClasses: #- name: efs-sc diff --git a/charts/kubezero/values.yaml b/charts/kubezero/values.yaml index df63ec15..3d43478b 100644 --- a/charts/kubezero/values.yaml +++ b/charts/kubezero/values.yaml @@ -36,7 +36,7 @@ cert-manager: storage: enabled: false - targetRevision: 0.7.1 + targetRevision: 0.7.2 aws-ebs-csi-driver: enabled: false aws-efs-csi-driver: