From e56d0661d6c1b9712e483f6f6d9ed97ca4e01dd2 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 11 Sep 2020 11:00:51 +0100 Subject: [PATCH] Make ES heap configurable, set resources accordingly --- charts/kubezero-logging/Chart.yaml | 2 +- charts/kubezero-logging/README.md | 4 ++-- .../kubezero-logging/templates/eck/elasticsearch.yaml | 11 +++++++---- charts/kubezero-logging/values-all.yaml | 5 +++++ 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/charts/kubezero-logging/Chart.yaml b/charts/kubezero-logging/Chart.yaml index 9d82968..af2c956 100644 --- a/charts/kubezero-logging/Chart.yaml +++ b/charts/kubezero-logging/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubezero-logging description: KubeZero Umbrella Chart for complete EFK stack type: application -version: 0.3.2 +version: 0.3.3 appVersion: 1.2.1 home: https://kubezero.com icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md index 3e1a2a1..336afd8 100644 --- a/charts/kubezero-logging/README.md +++ b/charts/kubezero-logging/README.md @@ -2,7 +2,7 @@ kubezero-logging ================ KubeZero Umbrella Chart for complete EFK stack -Current chart version is `0.3.2` +Current chart version is `0.3.3` Source code can be found [here](https://kubezero.com) @@ -56,7 +56,7 @@ Source code can be found [here](https://kubezero.com) | es.s3Snapshot.enabled | bool | `false` | | | es.s3Snapshot.iamrole | string | `""` | | | fluent-bit.config.filters | string | `"[FILTER]\n Name lua\n Match kube.*\n script /fluent-bit/etc/functions.lua\n call reassemble_cri_logs\n\n[FILTER]\n Name kubernetes\n Match kube.*\n Merge_Log On\n Keep_Log Off\n K8S-Logging.Parser On\n K8S-Logging.Exclude On\n\n[FILTER]\n Name lua\n Match kube.*\n script /fluent-bit/etc/functions.lua\n call dedot\n"` | | -| fluent-bit.config.inputs | string | `"[INPUT]\n Name tail\n Path /var/log/containers/*.log\n Parser cri\n Tag kube.*\n Mem_Buf_Limit 5MB\n Skip_Long_Lines On\n Refresh_Interval 10\n DB /var/log/flb_kube.db\n DB.Sync Normal\n[INPUT]\n Name tail\n Path /var/log/kubernetes/audit.log\n Parser json\n Tag kube.api.audit\n Mem_Buf_Limit 5MB\n Skip_Long_Lines On\n Refresh_Interval 60\n DB /var/log/flb_kube_audit.db\n DB.Sync Normal\n"` | | +| fluent-bit.config.inputs | string | `"[INPUT]\n Name tail\n Path /var/log/containers/*.log\n Parser cri\n Tag kube.*\n Mem_Buf_Limit 5MB\n Skip_Long_Lines On\n Refresh_Interval 10\n DB /var/log/flb_kube.db\n DB.Sync Normal\n[INPUT]\n Name tail\n Path /var/log/kubernetes/audit.log\n Parser json\n Tag audit.api-server\n Mem_Buf_Limit 5MB\n Skip_Long_Lines On\n Refresh_Interval 60\n DB /var/log/flb_kube_audit.db\n DB.Sync Normal\n"` | | | fluent-bit.config.lua | string | `"function dedot(tag, timestamp, record)\n if record[\"kubernetes\"] == nil then\n return 0, 0, 0\n end\n dedot_keys(record[\"kubernetes\"][\"annotations\"])\n dedot_keys(record[\"kubernetes\"][\"labels\"])\n return 1, timestamp, record\nend\n\nfunction dedot_keys(map)\n if map == nil then\n return\n end\n local new_map = {}\n local changed_keys = {}\n for k, v in pairs(map) do\n local dedotted = string.gsub(k, \"%.\", \"_\")\n if dedotted ~= k then\n new_map[dedotted] = v\n changed_keys[k] = true\n end\n end\n for k in pairs(changed_keys) do\n map[k] = nil\n end\n for k, v in pairs(new_map) do\n map[k] = v\n end\nend\n\nlocal reassemble_state = {}\n\nfunction reassemble_cri_logs(tag, timestamp, record)\n -- IMPORTANT: reassemble_key must be unique for each parser stream\n -- otherwise entries from different sources will get mixed up.\n -- Either make sure that your parser tags satisfy this or construct\n -- reassemble_key some other way\n local reassemble_key = tag\n -- if partial line, accumulate\n if record.logtag == 'P' then\n reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or \"\" .. record.message\n return -1, 0, 0\n end\n -- otherwise it's a full line, concatenate with accumulated partial lines if any\n record.message = reassemble_state[reassemble_key] or \"\" .. (record.message or \"\")\n reassemble_state[reassemble_key] = nil\n return 1, timestamp, record\nend\n"` | | | fluent-bit.config.outputs | string | `"[OUTPUT]\n Match *\n Name forward\n Host logging-fluentd\n Port 24224\n tls on\n tls.verify off\n Shared_Key cloudbender\n"` | | | fluent-bit.config.service | string | `"[SERVICE]\n Flush 5\n Daemon Off\n Log_Level warn\n Parsers_File parsers.conf\n Parsers_File custom_parsers.conf\n HTTP_Server On\n HTTP_Listen 0.0.0.0\n HTTP_Port 2020\n"` | | diff --git a/charts/kubezero-logging/templates/eck/elasticsearch.yaml b/charts/kubezero-logging/templates/eck/elasticsearch.yaml index a9dfd27..854affc 100644 --- a/charts/kubezero-logging/templates/eck/elasticsearch.yaml +++ b/charts/kubezero-logging/templates/eck/elasticsearch.yaml @@ -48,13 +48,16 @@ spec: - name: elasticsearch resources: requests: - cpu: 100m - memory: 2500Mi + cpu: {{ default "200m" .cpu_request }} + memory: {{ mul 2 ( default "4" .jvm_heap ) }}Gi limits: - memory: 4Gi + {{- if .cpu_limit }} + cpu: {{ .cpu_limit }} + {{- end }} + memory: {{ mul 2 ( default "4" .jvm_heap ) }}Gi env: - name: ES_JAVA_OPTS - value: "-Xms2g -Xmx2g" + value: -Xms{{ default "2" .jvm_heap }}g -Xmx{{ default "2" .jvm_heap }}g affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: diff --git a/charts/kubezero-logging/values-all.yaml b/charts/kubezero-logging/values-all.yaml index c87d0ad..e5997b7 100644 --- a/charts/kubezero-logging/values-all.yaml +++ b/charts/kubezero-logging/values-all.yaml @@ -18,6 +18,8 @@ es: size: 512Gi class: ebs-sc-gp2-xfs zone: us-west-2a + jvm_heap: 4 + cpu_limit: 2 s3Snapshot: enabled: true iamrole: "dfsf" # INSERT_CLOUDFORMATION_OUTPUT_ElasticSearchSnapshots @@ -32,3 +34,6 @@ kibana: fluentd: enabled: true + +fluent-bit: + enabled: true