Add custom support for nodeaffinity and resources to aws-efs-csi-driver

This commit is contained in:
Stefan Reimer 2021-03-24 12:11:47 +01:00
parent 6ea7500e41
commit 48d3d269e7
14 changed files with 611 additions and 8 deletions

View File

@ -1,7 +1,7 @@
apiVersion: v2
name: kubezero-aws-efs-csi-driver
description: KubeZero Umbrella Chart for aws-efs-csi-driver
version: 0.3.0
version: 0.3.1
appVersion: 1.2.0
kubeVersion: ">=1.18.0-0"
home: https://kubezero.com

View File

@ -0,0 +1,46 @@
diff -tubNr charts/aws-efs-csi-driver/templates/node-daemonset.yaml charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml
--- charts/aws-efs-csi-driver/templates/node-daemonset.yaml 2021-03-19 00:22:54.000000000 +0100
+++ charts/aws-efs-csi-driver.zdt/templates/node-daemonset.yaml 2021-03-24 12:02:07.152569061 +0100
@@ -40,15 +40,14 @@
{{- with .Values.nodeSelector }}
{{- . | toYaml | nindent 8 }}
{{- end }}
+ {{- with .Values.affinity }}
affinity:
- nodeAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- nodeSelectorTerms:
- - matchExpressions:
- - key: eks.amazonaws.com/compute-type
- operator: NotIn
- values:
- - fargate
+ {{- . | toYaml | nindent 8 }}
+ {{- end }}
+ {{- with .Values.resources }}
+ resources:
+ {{- . | toYaml | nindent 8 }}
+ {{- end }}
hostNetwork: true
{{- if .Values.dnsPolicy }}
dnsPolicy: "{{ .Values.dnsPolicy }}"
diff -tubNr charts/aws-efs-csi-driver/values.yaml charts/aws-efs-csi-driver.zdt/values.yaml
--- charts/aws-efs-csi-driver/values.yaml 2021-03-19 00:22:54.000000000 +0100
+++ charts/aws-efs-csi-driver.zdt/values.yaml 2021-03-24 11:54:19.252585893 +0100
@@ -43,7 +43,15 @@
tolerations: []
-affinity: {}
+affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: eks.amazonaws.com/compute-type
+ operator: NotIn
+ values:
+ - fargate
node:
podAnnotations: {}

View File

@ -0,0 +1,22 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

View File

@ -0,0 +1,18 @@
apiVersion: v1
appVersion: 1.2.0
description: A Helm chart for AWS EFS CSI Driver
home: https://github.com/kubernetes-sigs/aws-efs-csi-driver
keywords:
- aws
- efs
- csi
kubeVersion: '>=1.17.0-0'
maintainers:
- name: leakingtapan
url: https://github.com/leakingtapan
- name: krmichel
url: https://github.com/krmichel
name: aws-efs-csi-driver
sources:
- https://github.com/kubernetes-sigs/aws-efs-csi-driver
version: 1.2.1

View File

@ -0,0 +1,3 @@
To verify that aws-efs-csi-driver has started, run:
kubectl get pod -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aws-efs-csi-driver.name" . }},app.kubernetes.io/instance={{ .Release.Name }}"

View File

@ -0,0 +1,56 @@
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "aws-efs-csi-driver.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "aws-efs-csi-driver.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "aws-efs-csi-driver.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Common labels
*/}}
{{- define "aws-efs-csi-driver.labels" -}}
app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
helm.sh/chart: {{ include "aws-efs-csi-driver.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end -}}
{{/*
Create the name of the service account to use
*/}}
{{- define "aws-efs-csi-driver.serviceAccountName" -}}
{{- if .Values.serviceAccount.controller.create -}}
{{ default (include "aws-efs-csi-driver.fullname" .) .Values.serviceAccount.controller.name }}
{{- else -}}
{{ default "default" .Values.serviceAccount.controller.name }}
{{- end -}}
{{- end -}}

View File

@ -0,0 +1,100 @@
{{- if .Values.controller.create }}
# Controller Service
kind: Deployment
apiVersion: apps/v1
metadata:
name: efs-csi-controller
labels:
app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
app: efs-csi-controller
app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
template:
metadata:
labels:
app: efs-csi-controller
app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- if .Values.node.podAnnotations }}
annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.imagePullSecrets }}
imagePullSecrets:
{{- range .Values.imagePullSecrets }}
- name: {{ . }}
{{- end }}
{{- end }}
nodeSelector:
kubernetes.io/os: linux
{{- with .Values.nodeSelector }}
{{- . | toYaml | nindent 8 }}
{{- end }}
{{- if .Values.serviceAccount.controller.create }}
serviceAccountName: {{ include "aws-efs-csi-driver.serviceAccountName" . }}
{{- end }}
priorityClassName: system-cluster-critical
tolerations:
- operator: Exists
{{- with .Values.node.tolerations }}
{{- . | toYaml | nindent 8 }}
{{- end }}
containers:
- name: efs-plugin
securityContext:
privileged: true
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: IfNotPresent
args:
- --endpoint=$(CSI_ENDPOINT)
- --logtostderr
- --v=5
# Uncomment below line to allow access point root directory to be deleted by controller.
#- --delete-access-point-root-dir
env:
- name: CSI_ENDPOINT
value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
volumeMounts:
- name: socket-dir
mountPath: /var/lib/csi/sockets/pluginproxy/
ports:
- name: healthz
containerPort: 9808
protocol: TCP
livenessProbe:
httpGet:
path: /healthz
port: healthz
initialDelaySeconds: 10
timeoutSeconds: 3
periodSeconds: 10
failureThreshold: 5
- name: csi-provisioner
image: {{ printf "%s:%s" .Values.sidecars.csiProvisionerImage.repository .Values.sidecars.csiProvisionerImage.tag }}
args:
- --csi-address=$(ADDRESS)
- --v=5
- --feature-gates=Topology=true
- --leader-election
env:
- name: ADDRESS
value: /var/lib/csi/sockets/pluginproxy/csi.sock
volumeMounts:
- name: socket-dir
mountPath: /var/lib/csi/sockets/pluginproxy/
- name: liveness-probe
image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }}
args:
- --csi-address=/csi/csi.sock
- --health-port=9808
volumeMounts:
- name: socket-dir
mountPath: /csi
volumes:
- name: socket-dir
emptyDir: {}
{{- end }}

View File

@ -0,0 +1,59 @@
{{- if .Values.serviceAccount.controller.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "aws-efs-csi-driver.serviceAccountName" . }}
labels:
app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
{{- with .Values.serviceAccount.controller.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: efs-csi-external-provisioner-role
labels:
app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create"]
- apiGroups: ["storage.k8s.io"]
resources: ["csinodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["coordination.k8s.io"]
resources: ["leases"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: efs-csi-provisioner-binding
labels:
app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
subjects:
- kind: ServiceAccount
name: {{ include "aws-efs-csi-driver.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: efs-csi-external-provisioner-role
apiGroup: rbac.authorization.k8s.io
{{- end }}

View File

@ -0,0 +1,10 @@
apiVersion: storage.k8s.io/v1beta1
kind: CSIDriver
metadata:
name: efs.csi.aws.com
annotations:
"helm.sh/hook": pre-install, pre-upgrade
"helm.sh/hook-delete-policy": before-hook-creation
"helm.sh/resource-policy": keep
spec:
attachRequired: false

View File

@ -0,0 +1,154 @@
# Node Service
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: efs-csi-node
labels:
app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
spec:
selector:
matchLabels:
app: efs-csi-node
app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
template:
metadata:
labels:
app: efs-csi-node
app.kubernetes.io/name: {{ include "aws-efs-csi-driver.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- if .Values.node.podAnnotations }}
annotations: {{ toYaml .Values.node.podAnnotations | nindent 8 }}
{{- end }}
spec:
{{- if .Values.hostAliases }}
hostAliases:
{{- range $k, $v := .Values.hostAliases }}
- ip: {{ $v.ip }}
hostnames:
- {{ $k }}.efs.{{ $v.region }}.amazonaws.com
{{- end }}
{{- end }}
{{- if .Values.imagePullSecrets }}
imagePullSecrets:
{{- range .Values.imagePullSecrets }}
- name: {{ . }}
{{- end }}
{{- end }}
nodeSelector:
beta.kubernetes.io/os: linux
{{- with .Values.nodeSelector }}
{{- . | toYaml | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- . | toYaml | nindent 8 }}
{{- end }}
{{- with .Values.resources }}
resources:
{{- . | toYaml | nindent 8 }}
{{- end }}
hostNetwork: true
{{- if .Values.dnsPolicy }}
dnsPolicy: "{{ .Values.dnsPolicy }}"
{{- end }}
{{- with .Values.dnsConfig }}
dnsConfig:
{{- toYaml . | nindent 8 }}
{{- end }}
priorityClassName: system-node-critical
tolerations:
- operator: Exists
{{- with .Values.node.tolerations }}
{{- . | toYaml | nindent 8 }}
{{- end }}
containers:
- name: efs-plugin
securityContext:
privileged: true
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
args:
- --endpoint=$(CSI_ENDPOINT)
- --logtostderr
- --v={{ .Values.logLevel }}
env:
- name: CSI_ENDPOINT
value: unix:/csi/csi.sock
volumeMounts:
- name: kubelet-dir
mountPath: /var/lib/kubelet
mountPropagation: "Bidirectional"
- name: plugin-dir
mountPath: /csi
- name: efs-state-dir
mountPath: /var/run/efs
- name: efs-utils-config
mountPath: /var/amazon/efs
- name: efs-utils-config-legacy
mountPath: /etc/amazon/efs-legacy
ports:
- name: healthz
containerPort: 9809
protocol: TCP
livenessProbe:
httpGet:
path: /healthz
port: healthz
initialDelaySeconds: 10
timeoutSeconds: 3
periodSeconds: 2
failureThreshold: 5
- name: csi-driver-registrar
image: {{ printf "%s:%s" .Values.sidecars.nodeDriverRegistrarImage.repository .Values.sidecars.nodeDriverRegistrarImage.tag }}
args:
- --csi-address=$(ADDRESS)
- --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
- --v={{ .Values.logLevel }}
env:
- name: ADDRESS
value: /csi/csi.sock
- name: DRIVER_REG_SOCK_PATH
value: /var/lib/kubelet/plugins/efs.csi.aws.com/csi.sock
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: plugin-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
- name: liveness-probe
image: {{ printf "%s:%s" .Values.sidecars.livenessProbeImage.repository .Values.sidecars.livenessProbeImage.tag }}
args:
- --csi-address=/csi/csi.sock
- --health-port=9809
- --v={{ .Values.logLevel }}
volumeMounts:
- name: plugin-dir
mountPath: /csi
volumes:
- name: kubelet-dir
hostPath:
path: /var/lib/kubelet
type: Directory
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins/efs.csi.aws.com/
type: DirectoryOrCreate
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins_registry/
type: Directory
- name: efs-state-dir
hostPath:
path: /var/run/efs
type: DirectoryOrCreate
- name: efs-utils-config
hostPath:
path: /var/amazon/efs
type: DirectoryOrCreate
- name: efs-utils-config-legacy
hostPath:
path: /etc/amazon/efs
type: DirectoryOrCreate

View File

@ -0,0 +1,15 @@
{{- range .Values.storageClasses }}
kind: StorageClass
apiVersion: storage.k8s.io/v1
metadata:
name: {{ .name }}
provisioner: efs.csi.aws.com
{{- with .mountOptions }}
mountOptions:
{{ toYaml . }}
{{- end }}
{{- with .parameters }}
parameters:
{{ toYaml . | indent 2 }}
{{- end }}
{{- end }}

View File

@ -0,0 +1,105 @@
# Default values for aws-efs-csi-driver.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 2
image:
repository: amazon/aws-efs-csi-driver
tag: "v1.2.0"
pullPolicy: IfNotPresent
sidecars:
livenessProbeImage:
repository: public.ecr.aws/eks-distro/kubernetes-csi/livenessprobe
tag: v2.1.0-eks-1-18-1
nodeDriverRegistrarImage:
repository: public.ecr.aws/eks-distro/kubernetes-csi/node-driver-registrar
tag: v2.0.1-eks-1-18-1
csiProvisionerImage:
repository: public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner
tag: v2.0.3-eks-1-18-1
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
podAnnotations: {}
resources:
{}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
nodeSelector: {}
tolerations: []
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: eks.amazonaws.com/compute-type
operator: NotIn
values:
- fargate
node:
podAnnotations: {}
tolerations: []
logLevel: 5
hostAliases:
{}
# for cross VPC EFS, you need to poison or overwrite the DNS for the efs volume as per
# https://docs.aws.amazon.com/efs/latest/ug/efs-different-vpc.html#wt6-efs-utils-step3
# implementing the suggested solution found here:
# https://github.com/kubernetes-sigs/aws-efs-csi-driver/issues/240#issuecomment-676849346
# EFS Vol ID, IP, Region
# "fs-01234567":
# ip: 10.10.2.2
# region: us-east-2
dnsPolicy: ""
dnsConfig:
{}
# Example config which uses the AWS nameservers
# dnsPolicy: "None"
# dnsConfig:
# nameservers:
# - 169.254.169.253
serviceAccount:
controller:
# Specifies whether a service account should be created
create: true
annotations: {}
## Enable if EKS IAM for SA is used
# eks.amazonaws.com/role-arn: arn:aws:iam::111122223333:role/efs-csi-role
name: efs-csi-controller-sa
controller:
create: true
storageClasses: []
# Add StorageClass resources like:
# - name: efs-sc
# mountOptions:
# - tls
# parameters:
# provisioningMode: efs-ap
# fileSystemId: fs-92107410
# directoryPerms: "700"
# gidRangeStart: "1000"
# gidRangeEnd: "2000"
# basePath: "/dynamic_provisioning"

View File

@ -0,0 +1,8 @@
#!/bin/bash
VERSION=1.2.1
rm -rf charts/aws-efs-csi-driver
curl -L -s -o - https://github.com/kubernetes-sigs/aws-efs-csi-driver/releases/download/helm-chart-aws-efs-csi-driver-${VERSION}/aws-efs-csi-driver-${VERSION}.tgz | tar xfz - -C charts
patch -i affinity_resources.patch -p0 --no-backup-if-mismatch

View File

@ -3,13 +3,20 @@ aws-efs-csi-driver:
# node.kubernetes.io/csi.efs.fs: EFS-FS_ID
logLevel: 1
# Resources are currently not supported in the helm templates :-(
#resources:
# requests:
# cpu: 20m
# memory: 64Mi
# limits:
# memory: 128Mi
resources:
requests:
cpu: 20m
memory: 64Mi
limits:
memory: 128Mi
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node.kubernetes.io/csi.efs.fs
operator: Exists
StorageClass:
create: true