Make ES heap configurable, set resources accordingly

charts/kubezero-logging/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.3.2
+version: 0.3.3
 appVersion: 1.2.1
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png

charts/kubezero-logging/README.md

@@ -2,7 +2,7 @@ kubezero-logging
 ================
 KubeZero Umbrella Chart for complete EFK stack
 
-Current chart version is `0.3.2`
+Current chart version is `0.3.3`
 
 Source code can be found [here](https://kubezero.com)
 
@@ -56,7 +56,7 @@ Source code can be found [here](https://kubezero.com)
 | es.s3Snapshot.enabled | bool | `false` |  |
 | es.s3Snapshot.iamrole | string | `""` |  |
 | fluent-bit.config.filters | string | `"[FILTER]\n    Name    lua\n    Match   kube.*\n    script  /fluent-bit/etc/functions.lua\n    call    reassemble_cri_logs\n\n[FILTER]\n    Name kubernetes\n    Match kube.*\n    Merge_Log On\n    Keep_Log Off\n    K8S-Logging.Parser On\n    K8S-Logging.Exclude On\n\n[FILTER]\n    Name    lua\n    Match   kube.*\n    script  /fluent-bit/etc/functions.lua\n    call    dedot\n"` |  |
-| fluent-bit.config.inputs | string | `"[INPUT]\n    Name tail\n    Path /var/log/containers/*.log\n    Parser cri\n    Tag kube.*\n    Mem_Buf_Limit 5MB\n    Skip_Long_Lines On\n    Refresh_Interval 10\n    DB /var/log/flb_kube.db\n    DB.Sync Normal\n[INPUT]\n    Name tail\n    Path /var/log/kubernetes/audit.log\n    Parser json\n    Tag kube.api.audit\n    Mem_Buf_Limit 5MB\n    Skip_Long_Lines On\n    Refresh_Interval 60\n    DB /var/log/flb_kube_audit.db\n    DB.Sync Normal\n"` |  |
+| fluent-bit.config.inputs | string | `"[INPUT]\n    Name tail\n    Path /var/log/containers/*.log\n    Parser cri\n    Tag kube.*\n    Mem_Buf_Limit 5MB\n    Skip_Long_Lines On\n    Refresh_Interval 10\n    DB /var/log/flb_kube.db\n    DB.Sync Normal\n[INPUT]\n    Name tail\n    Path /var/log/kubernetes/audit.log\n    Parser json\n    Tag audit.api-server\n    Mem_Buf_Limit 5MB\n    Skip_Long_Lines On\n    Refresh_Interval 60\n    DB /var/log/flb_kube_audit.db\n    DB.Sync Normal\n"` |  |
 | fluent-bit.config.lua | string | `"function dedot(tag, timestamp, record)\n    if record[\"kubernetes\"] == nil then\n        return 0, 0, 0\n    end\n    dedot_keys(record[\"kubernetes\"][\"annotations\"])\n    dedot_keys(record[\"kubernetes\"][\"labels\"])\n    return 1, timestamp, record\nend\n\nfunction dedot_keys(map)\n    if map == nil then\n        return\n    end\n    local new_map = {}\n    local changed_keys = {}\n    for k, v in pairs(map) do\n        local dedotted = string.gsub(k, \"%.\", \"_\")\n        if dedotted ~= k then\n            new_map[dedotted] = v\n            changed_keys[k] = true\n        end\n    end\n    for k in pairs(changed_keys) do\n        map[k] = nil\n    end\n    for k, v in pairs(new_map) do\n        map[k] = v\n    end\nend\n\nlocal reassemble_state = {}\n\nfunction reassemble_cri_logs(tag, timestamp, record)\n   -- IMPORTANT: reassemble_key must be unique for each parser stream\n   -- otherwise entries from different sources will get mixed up.\n   -- Either make sure that your parser tags satisfy this or construct\n   -- reassemble_key some other way\n   local reassemble_key = tag\n   -- if partial line, accumulate\n   if record.logtag == 'P' then\n      reassemble_state[reassemble_key] = reassemble_state[reassemble_key] or \"\" .. record.message\n      return -1, 0, 0\n   end\n   -- otherwise it's a full line, concatenate with accumulated partial lines if any\n   record.message = reassemble_state[reassemble_key] or \"\" .. (record.message or \"\")\n   reassemble_state[reassemble_key] = nil\n   return 1, timestamp, record\nend\n"` |  |
 | fluent-bit.config.outputs | string | `"[OUTPUT]\n    Match *\n    Name forward\n    Host logging-fluentd\n    Port 24224\n    tls on\n    tls.verify off\n    Shared_Key cloudbender\n"` |  |
 | fluent-bit.config.service | string | `"[SERVICE]\n    Flush 5\n    Daemon Off\n    Log_Level warn\n    Parsers_File parsers.conf\n    Parsers_File custom_parsers.conf\n    HTTP_Server On\n    HTTP_Listen 0.0.0.0\n    HTTP_Port 2020\n"` |  |

charts/kubezero-logging/templates/eck/elasticsearch.yaml

@@ -48,13 +48,16 @@ spec:
         - name: elasticsearch
           resources:
             requests:
-              cpu: 100m
+              cpu: {{ default "200m" .cpu_request }}
-              memory: 2500Mi
+              memory: {{ mul 2 ( default "4" .jvm_heap ) }}Gi
             limits:
-              memory: 4Gi
+              {{- if .cpu_limit }}
+              cpu: {{ .cpu_limit }}
+              {{- end }}
+              memory: {{ mul 2 ( default "4" .jvm_heap ) }}Gi
           env:
           - name: ES_JAVA_OPTS
-            value: "-Xms2g -Xmx2g"
+            value: -Xms{{ default "2" .jvm_heap }}g -Xmx{{ default "2" .jvm_heap }}g
         affinity:
           podAntiAffinity:
             requiredDuringSchedulingIgnoredDuringExecution:

charts/kubezero-logging/values-all.yaml

@@ -18,6 +18,8 @@ es:
       size: 512Gi
       class: ebs-sc-gp2-xfs
     zone: us-west-2a
+    jvm_heap: 4
+    cpu_limit: 2
   s3Snapshot:
     enabled: true
     iamrole: "dfsf" # INSERT_CLOUDFORMATION_OUTPUT_ElasticSearchSnapshots
@@ -32,3 +34,6 @@ kibana:
 
 fluentd:
   enabled: true
+
+fluent-bit:
+  enabled: true