From d51e3cb0480c9dba3b9f6e826c9830f48b7361a3 Mon Sep 17 00:00:00 2001
From: Stefan Reimer <stefan@zero-downtime.net>
Date: Thu, 18 Mar 2021 14:31:10 +0100
Subject: [PATCH] Kubelet tuning

---
 .../templates/KubeletConfiguration.yaml       | 15 +++++++++---
 .../kubeadm/templates/patches/coredns0.yaml   | 24 +++++++++++++++++++
 charts/kubeadm/templates/patches/etcd0.yaml   |  7 ++++++
 .../templates/patches/kube-apiserver0.yaml    |  7 ++++++
 .../patches/kube-controller-manager0.yaml     |  7 ++++++
 .../templates/patches/kube-scheduler0.yaml    |  7 ++++++
 6 files changed, 64 insertions(+), 3 deletions(-)
 create mode 100644 charts/kubeadm/templates/patches/coredns0.yaml
 create mode 100644 charts/kubeadm/templates/patches/etcd0.yaml
 create mode 100644 charts/kubeadm/templates/patches/kube-apiserver0.yaml
 create mode 100644 charts/kubeadm/templates/patches/kube-controller-manager0.yaml
 create mode 100644 charts/kubeadm/templates/patches/kube-scheduler0.yaml

diff --git a/charts/kubeadm/templates/KubeletConfiguration.yaml b/charts/kubeadm/templates/KubeletConfiguration.yaml
index eaa960f..04617ad 100644
--- a/charts/kubeadm/templates/KubeletConfiguration.yaml
+++ b/charts/kubeadm/templates/KubeletConfiguration.yaml
@@ -5,7 +5,8 @@ metadata:
 failSwapOn: false
 cgroupDriver: cgroupfs
 logging:
-  format: json
+  # We already have syslog-ng logging json
+  # format: json
 hairpinMode: hairpin-veth
 {{- if .Values.systemd }}
 resolvConf: /run/systemd/resolve/resolv.conf
@@ -22,7 +23,15 @@ featureGates:
   CSIMigrationAWS: true
   CSIMigrationAWSComplete: true
 {{- end }}
+# Minimal unit is 50m per pod
+podsPerCore: 20
+# Basic OS on Ubuntu 20.04 incl. crio
+systemReserved:
+  memory: 256Mi
+# This should be dynamic based on number of maxpods and available cores
+# https://cloud.google.com/kubernetes-engine/docs/concepts/cluster-architecture#memory_cpu
+# Below are default minimal for 2 cores and minimum kubelet 
 kubeReserved:
-  cpu: 50m
-  memory: 128m
+  cpu: 70m
+  memory: 128Mi
 # cpuCFSQuotaPeriod: 10ms
diff --git a/charts/kubeadm/templates/patches/coredns0.yaml b/charts/kubeadm/templates/patches/coredns0.yaml
new file mode 100644
index 0000000..ef42b4e
--- /dev/null
+++ b/charts/kubeadm/templates/patches/coredns0.yaml
@@ -0,0 +1,24 @@
+spec:
+  replicas: {{ ternary 3 1 .Values.clusterHighAvailable }}
+  template:
+    spec:
+      containers:
+      - name: coredns
+        resources:
+          requests:
+            cpu: 100m
+            memory: 32Mi
+          limits:
+            memory: 128Mi
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: k8s-app
+                operator: In
+                values:
+                - kube-dns
+            topologyKey: "kubernetes.io/hostname"
diff --git a/charts/kubeadm/templates/patches/etcd0.yaml b/charts/kubeadm/templates/patches/etcd0.yaml
new file mode 100644
index 0000000..b17a0e5
--- /dev/null
+++ b/charts/kubeadm/templates/patches/etcd0.yaml
@@ -0,0 +1,7 @@
+spec:
+  containers:
+  - name: etcd
+    resources:
+      requests:
+        cpu: 250m
+        memory: 192Mi
diff --git a/charts/kubeadm/templates/patches/kube-apiserver0.yaml b/charts/kubeadm/templates/patches/kube-apiserver0.yaml
new file mode 100644
index 0000000..05c2a18
--- /dev/null
+++ b/charts/kubeadm/templates/patches/kube-apiserver0.yaml
@@ -0,0 +1,7 @@
+spec:
+  containers:
+  - name: kube-apiserver
+    resources:
+      requests:
+        cpu: 250m
+        memory: 1Gi
diff --git a/charts/kubeadm/templates/patches/kube-controller-manager0.yaml b/charts/kubeadm/templates/patches/kube-controller-manager0.yaml
new file mode 100644
index 0000000..cb08409
--- /dev/null
+++ b/charts/kubeadm/templates/patches/kube-controller-manager0.yaml
@@ -0,0 +1,7 @@
+spec:
+  containers:
+  - name: kube-controller-manager
+    resources:
+      requests:
+        cpu: 200m
+        memory: 128Mi
diff --git a/charts/kubeadm/templates/patches/kube-scheduler0.yaml b/charts/kubeadm/templates/patches/kube-scheduler0.yaml
new file mode 100644
index 0000000..837fd63
--- /dev/null
+++ b/charts/kubeadm/templates/patches/kube-scheduler0.yaml
@@ -0,0 +1,7 @@
+spec:
+  containers:
+  - name: kube-scheduler
+    resources:
+      requests:
+        cpu: 100m
+        memory: 64Mi