fix: update EFS/EBS drivers, adjust tolerations to allow EFS on Nvidia, remove EBS nodes from controllers

charts/kubezero-storage/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-storage
 description: KubeZero umbrella chart for all things storage incl. AWS EBS/EFS, openEBS-lvm, gemini
 type: application
-version: 0.7.1
+version: 0.7.2
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
 keywords:
@@ -28,11 +28,11 @@ dependencies:
     condition: gemini.enabled
     # repository: https://charts.fairwinds.com/stable
   - name: aws-ebs-csi-driver
-    version: 2.11.0
+    version: 2.12.0
     condition: aws-ebs-csi-driver.enabled
     # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver
   - name: aws-efs-csi-driver
-    version: 2.2.7
+    version: 2.2.9
     condition: aws-efs-csi-driver.enabled
     # repository: https://kubernetes-sigs.github.io/aws-ebs-csi-driver
 kubeVersion: ">= 1.20.0"

charts/kubezero-storage/charts/aws-ebs-csi-driver/CHANGELOG.md

@@ -1,6 +1,21 @@
 # Helm chart
 
-# v2.11.0
+## v2.12.0
+* Bump app/driver to version `v1.12.0`
+* Move default toleration to values.yaml so it can be overriden if desired by users ([#1400](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1400), [@cnmcavoy](https://github.com/cnmcavoy))
+* Add enableMetrics configuration ([#1380](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1380), [@torredil](https://github.com/torredil))
+* add initContainer to the controller's template ([#1379](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1379), [@InsomniaCoder](https://github.com/InsomniaCoder))
+* Add controller nodeAffinity to prefer EC2 over Fargate ([#1360](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1360), [@torredil](https://github.com/torredil))
+
+## v2.11.1
+* Add `useOldCSIDriver` parameter to use old `CSIDriver` object.
+
+## v2.11.0
+
+**Important Notice:** This version updates the `CSIDriver` object in order to fix [a bug with static volumes and the `fsGroup` parameter](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/issues/1365). This upgrade will fail on existing clusters because the associated field in `CSIDriver` is immutable.
+
+Users upgrading to this version should pre-delete the existing `CSIDriver` object (example: `kubectl delete csidriver ebs.csi.aws.com`). This will not affect any existing volumes, but will cause the EBS CSI Driver to be unavailable to handle future requests, and should be immediately followed by an upgrade. For users that cannot delete the `CSIDriver` object, v2.11.1 implements a new parameter `useOldCSIDriver` that will use the previous `CSIDriver`.
+
 * Bump app/driver to version `v1.11.3`
 * Add support for leader election tuning for `csi-provisioner` and `csi-attacher` ([#1371](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1371), [@moogzy](https://github.com/moogzy))
 * Change `fsGroupPolicy` to `File` ([#1377](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1377), [@ConnorJC3](https://github.com/ConnorJC3))

charts/kubezero-storage/charts/aws-ebs-csi-driver/Chart.yaml

@@ -3,7 +3,7 @@ annotations:
     - kind: added
       description: Custom controller.updateStrategy to set controller deployment strategy.
 apiVersion: v2
-appVersion: 1.11.3
+appVersion: 1.12.0
 description: A Helm chart for AWS EBS CSI Driver
 home: https://github.com/kubernetes-sigs/aws-ebs-csi-driver
 keywords:
@@ -19,4 +19,4 @@ maintainers:
 name: aws-ebs-csi-driver
 sources:
 - https://github.com/kubernetes-sigs/aws-ebs-csi-driver
-version: 2.11.0
+version: 2.12.0

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/controller.yaml

@@ -62,6 +62,10 @@ spec:
       securityContext:  
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.controller.initContainers }}
+      initContainers:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       containers:
         - name: ebs-plugin
           image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (.Values.image.tag | toString)) }}
@@ -79,6 +83,9 @@ spec:
             {{- with .Values.controller.k8sTagClusterId }}
             - --k8s-tag-cluster-id={{ . }}
             {{- end }}
+            {{- if and (.Values.controller.enableMetrics) (not .Values.controller.httpEndpoint) }}
+            - --http-endpoint=0.0.0.0:3301
+            {{- end}}
             {{- with .Values.controller.httpEndpoint }}
             - --http-endpoint={{ . }}
             {{- end }}
@@ -136,6 +143,11 @@ spec:
             - name: healthz
               containerPort: 9808
               protocol: TCP
+            {{- if .Values.controller.enableMetrics }}
+            - name: metrics
+              containerPort: 3301
+              protocol: TCP
+            {{- end}}
           livenessProbe:
             httpGet:
               path: /healthz

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/csidriver.yaml

@@ -7,4 +7,6 @@ metadata:
 spec:
   attachRequired: true
   podInfoOnMount: false
+  {{- if not .Values.useOldCSIDriver }}
   fsGroupPolicy: File
+  {{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/metrics.yaml

@@ -0,0 +1,40 @@
+{{- if .Values.controller.enableMetrics -}}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ebs-csi-controller
+  namespace: kube-system
+  labels:
+    app: ebs-csi-controller
+spec:
+  selector:
+    app: ebs-csi-controller
+  ports:
+    - name: metrics
+      port: 3301
+      targetPort: 3301
+  type: ClusterIP
+---
+{{- if (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") -}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: ebs-csi-controller
+  namespace: kube-system
+  labels:
+    app: ebs-csi-controller
+    release: prometheus
+spec:
+  selector:
+    matchLabels:
+      app: ebs-csi-controller
+  namespaceSelector:
+    matchNames:
+      - kube-system
+  endpoints:
+    - targetPort: 3301
+      path: /metrics
+      interval: 15s
+{{- end }}
+{{- end }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node-windows.yaml

@@ -40,13 +40,10 @@ spec:
         {{- if .Values.node.tolerateAllTaints }}
         - operator: Exists
         {{- else }}
-        - operator: Exists
-          effect: NoExecute
-          tolerationSeconds: 300
-        {{- end }}
         {{- with .Values.node.tolerations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
+        {{- end }}
       containers:
         - name: ebs-plugin
           image: {{ printf "%s:%s" .Values.image.repository (default (printf "v%s" .Chart.AppVersion) (toString .Values.image.tag)) }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/templates/node.yaml

@@ -40,13 +40,10 @@ spec:
         {{- if .Values.node.tolerateAllTaints }}
         - operator: Exists
         {{- else }}
-        - operator: Exists
-          effect: NoExecute
-          tolerationSeconds: 300
-        {{- end }}
         {{- with .Values.node.tolerations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
+        {{- end }}
       {{- with .Values.node.securityContext }}
       securityContext:  
         {{- toYaml . | nindent 8 }}

charts/kubezero-storage/charts/aws-ebs-csi-driver/values.yaml

@@ -110,7 +110,16 @@ fullnameOverride:
 controller:
   # If arbitrary args like "--aws-sdk-debug-log=true" need to be passed, use this value
   additionalArgs: []
-  affinity: {}
+  affinity:
+    nodeAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+      - weight: 1
+        preference:
+          matchExpressions:
+          - key: eks.amazonaws.com/compute-type
+            operator: NotIn
+            values:
+            - fargate
   # The default filesystem type of the volume to provision when fstype is unspecified in the StorageClass.
   # If the default is not set and fstype is unset in the StorageClass, then no fstype will be set
   defaultFsType: ext4
@@ -126,6 +135,14 @@ controller:
   #   key2: value2
   extraVolumeTags: {}
   httpEndpoint:
+  # (deprecated) The TCP network address where the prometheus metrics endpoint
+  # will run (example: `:8080` which corresponds to port 8080 on local host). 
+  # The default is empty string, which means metrics endpoint is disabled.
+  # ---
+  enableMetrics: false
+  # If set to true, AWS API call metrics will be exported to the following 
+  # TCP endpoint: "0.0.0.0:3301"
+  # ---
   # ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional).
   k8sTagClusterId:
   logLevel: 2
@@ -184,6 +201,14 @@ controller:
   containerSecurityContext:
     readOnlyRootFilesystem: true
     allowPrivilegeEscalation: false
+  initContainers: []
+  # containers to be run before the controller's container starts.
+  #
+  # Example:
+  #
+  # - name: wait
+  #   image: busybox
+  #   command: [ 'sh', '-c', "sleep 20" ]
 
 node:
   env: []
@@ -204,7 +229,10 @@ node:
   podAnnotations: {}
   podLabels: {}
   tolerateAllTaints: true
-  tolerations: []
+  tolerations:
+  - operator: Exists
+    effect: NoExecute
+    tolerationSeconds: 300
   resources: {}
   serviceAccount:
     create: true
@@ -244,3 +272,8 @@ storageClasses: []
 #   reclaimPolicy: Retain
 #   parameters:
 #     encrypted: "true"
+
+# Use old CSIDriver without an fsGroupPolicy set
+# Intended for use with older clusters that cannot easily replace the CSIDriver object
+# This parameter should always be false for new installations
+useOldCSIDriver: false

charts/kubezero-storage/charts/aws-efs-csi-driver/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Helm chart
 
+# v2.2.9
+* Bump app/driver version to `v1.4.2`
+
+# v2.2.8
+* Bump app/driver version to `v1.4.1`
+
 # v2.2.7
 * Bump app/driver version to `v1.4.0`
 # v2.2.6

charts/kubezero-storage/charts/aws-efs-csi-driver/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v2
-appVersion: 1.4.0
+appVersion: 1.4.2
 description: A Helm chart for AWS EFS CSI Driver
 home: https://github.com/kubernetes-sigs/aws-efs-csi-driver
 keywords:
@@ -15,4 +15,4 @@ maintainers:
 name: aws-efs-csi-driver
 sources:
 - https://github.com/kubernetes-sigs/aws-efs-csi-driver
-version: 2.2.7
+version: 2.2.9

charts/kubezero-storage/charts/aws-efs-csi-driver/templates/controller-deployment.yaml

@@ -69,14 +69,18 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
-          {{- if .Values.controller.extraEnv }}
+            {{- if .Values.useFIPS }}
+            - name: AWS_USE_FIPS_ENDPOINT
+              value: "true"
+            {{- end }}
+            {{- if .Values.controller.extraEnv }}
             {{- toYaml .Values.controller.extraEnv | nindent 12 }}
-          {{- end }}
+            {{- end }}
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
             - name: aws-token
-              mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/"
+              mountPath: /var/run/secrets/sts.amazonaws.com/serviceaccount/
           ports:
             - name: healthz
               containerPort: {{ .Values.controller.healthPort }}

charts/kubezero-storage/charts/aws-efs-csi-driver/templates/node-daemonset.yaml

@@ -67,6 +67,10 @@ spec:
           env:
             - name: CSI_ENDPOINT
               value: unix:/csi/csi.sock
+            {{- if .Values.useFIPS }}
+            - name: AWS_USE_FIPS_ENDPOINT
+              value: "true"
+            {{- end }}
           volumeMounts:
             - name: kubelet-dir
               mountPath: /var/lib/kubelet

charts/kubezero-storage/charts/aws-efs-csi-driver/values.yaml

@@ -7,9 +7,11 @@ fullnameOverride: ""
 
 replicaCount: 2
 
+useFIPS: false
+
 image:
   repository: amazon/aws-efs-csi-driver
-  tag: "v1.4.0"
+  tag: "v1.4.2"
   pullPolicy: IfNotPresent
 
 sidecars:

charts/kubezero-storage/efs.patch

@@ -1,6 +1,6 @@
 diff -tuNr charts/aws-efs-csi-driver.orig/templates/controller-deployment.yaml charts/aws-efs-csi-driver/templates/controller-deployment.yaml
---- charts/aws-efs-csi-driver.orig/templates/controller-deployment.yaml	2022-08-24 13:58:54.208936918 +0200
+--- charts/aws-efs-csi-driver.orig/templates/controller-deployment.yaml	2022-10-11 20:57:20.000000000 +0200
-+++ charts/aws-efs-csi-driver/templates/controller-deployment.yaml	2022-08-24 14:02:53.031437096 +0200
++++ charts/aws-efs-csi-driver/templates/controller-deployment.yaml	2022-10-12 22:40:48.100658738 +0200
 @@ -13,6 +13,10 @@
        app: efs-csi-controller
        app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
@@ -12,22 +12,22 @@ diff -tuNr charts/aws-efs-csi-driver.orig/templates/controller-deployment.yaml c
    template:
      metadata:
        labels:
-@@ -65,9 +69,14 @@
+@@ -69,9 +73,14 @@
-               valueFrom:
+             - name: AWS_USE_FIPS_ENDPOINT
-                 fieldRef:
+               value: "true"
-                   fieldPath: spec.nodeName
+             {{- end }}
-+          {{- if .Values.controller.extraEnv }}
++            {{- if .Values.controller.extraEnv }}
 +            {{- toYaml .Values.controller.extraEnv | nindent 12 }}
-+          {{- end }}
++            {{- end }}
            volumeMounts:
              - name: socket-dir
                mountPath: /var/lib/csi/sockets/pluginproxy/
 +            - name: aws-token
-+              mountPath: "/var/run/secrets/sts.amazonaws.com/serviceaccount/"
++              mountPath: /var/run/secrets/sts.amazonaws.com/serviceaccount/
            ports:
              - name: healthz
                containerPort: {{ .Values.controller.healthPort }}
-@@ -118,6 +127,13 @@
+@@ -122,6 +131,13 @@
        volumes:
          - name: socket-dir
            emptyDir: {}
@@ -42,8 +42,8 @@ diff -tuNr charts/aws-efs-csi-driver.orig/templates/controller-deployment.yaml c
        affinity: {{- toYaml . | nindent 8 }}
        {{- end }}
 diff -tuNr charts/aws-efs-csi-driver.orig/templates/node-daemonset.yaml charts/aws-efs-csi-driver/templates/node-daemonset.yaml
---- charts/aws-efs-csi-driver.orig/templates/node-daemonset.yaml	2022-08-24 13:58:54.208936918 +0200
+--- charts/aws-efs-csi-driver.orig/templates/node-daemonset.yaml	2022-10-11 20:57:20.000000000 +0200
-+++ charts/aws-efs-csi-driver/templates/node-daemonset.yaml	2022-08-24 14:03:42.095284086 +0200
++++ charts/aws-efs-csi-driver/templates/node-daemonset.yaml	2022-10-12 22:39:19.796397838 +0200
 @@ -40,15 +40,10 @@
          {{- with .Values.node.nodeSelector }}
          {{- toYaml . | nindent 8 }}

charts/kubezero-storage/jsonnet/jsonnetfile.lock.json

@@ -18,7 +18,7 @@
           "subdir": "contrib/mixin"
         }
       },
-      "version": "19002cfc689fba2b8f56605e5797bf79f8b61fdd",
+      "version": "acc7463fb2dc7037f571a02e65afd6e573a6344c",
       "sum": "W/Azptf1PoqjyMwJON96UY69MFugDA4IAYiKURscryc="
     },
     {
@@ -38,7 +38,7 @@
           "subdir": "grafana-builder"
         }
       },
-      "version": "dbf6fc14105c28b6fd0253005f7ca2da37d3d4e1",
+      "version": "d73aff453c9784cd6922119f3ce33d8d355a79e1",
       "sum": "tDR6yT2GVfw0wTU12iZH+m01HrbIr6g/xN+/8nzNkU0="
     },
     {
@@ -48,8 +48,8 @@
           "subdir": ""
         }
       },
-      "version": "fb9d8ed4bc4a3d6efac525f72e8a0d2c583a0fe2",
+      "version": "ba16d78ef81ca3d183472c2e3d1d7d3549e18ecb",
-      "sum": "xjKkdp+5fkekCNBUIgZCHTRmVdUEmQNFKslrL2Ho8gs="
+      "sum": "ntQv86Jd9HR4YQlb2yzjYVCAQHMUnAaQVIdhR2QaNNg="
     },
     {
       "source": {
@@ -58,7 +58,7 @@
           "subdir": "lib/promgrafonnet"
         }
       },
-      "version": "5e44626d70c2bf2d35c37f3fee5a6261a5335cc6",
+      "version": "7b559e800a32a2a80caf4c968f37c4999ec44689",
       "sum": "zv7hXGui6BfHzE9wPatHI/AGZa4A2WKo6pq7ZdqBsps="
     },
     {
@@ -98,7 +98,7 @@
           "subdir": "jsonnet/kube-prometheus"
         }
       },
-      "version": "452aaed72e36acb31cae93cfa85a5d9c3d3d2ec7",
+      "version": "fcd73c717f54a49cce69e78b9bcb709d0b4571ff",
       "sum": "pupXEvlRbhLdEO9b8LfFZB66+Z7fEqvRZ9m3MyEvsv4="
     },
     {

charts/kubezero-storage/values.yaml

@@ -95,10 +95,17 @@ aws-ebs-csi-driver:
         memory: 40Mi
 
   node:
+    tolerateAllTaints: false
     tolerations:
     - key: kubezero-workergroup
       effect: NoSchedule
       operator: Exists
+    - key: nvidia.com/gpu
+      effect: NoSchedule
+      operator: Exists
+    - key: aws.amazon.com/neuron
+      effect: NoSchedule
+      operator: Exists
 
     resources:
       requests:
@@ -171,6 +178,12 @@ aws-efs-csi-driver:
     - key: kubezero-workergroup
       effect: NoSchedule
       operator: Exists
+    - key: nvidia.com/gpu
+      effect: NoSchedule
+      operator: Exists
+    - key: aws.amazon.com/neuron
+      effect: NoSchedule
+      operator: Exists
 
   #storageClasses:
   #- name: efs-sc

charts/kubezero/values.yaml

@@ -36,7 +36,7 @@ cert-manager:
 
 storage:
   enabled: false
-  targetRevision: 0.7.1
+  targetRevision: 0.7.2
   aws-ebs-csi-driver:
     enabled: false
   aws-efs-csi-driver: