Add fluentd to logging

charts/kubezero-logging/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kubezero-logging
 description: KubeZero Umbrella Chart for complete EFK stack
 type: application
-version: 0.1.0
+version: 0.2.0
 appVersion: 1.2.1
 home: https://kubezero.com
 icon: https://cdn.zero-downtime.net/assets/kubezero/logo-small-64.png
@@ -18,4 +18,8 @@ dependencies:
   - name: kubezero-lib
     version: ">= 0.1.3"
     repository: https://zero-down-time.github.io/kubezero/
+  - name: fluentd
+    version: 2.5.1
+    repository: https://kubernetes-charts.storage.googleapis.com/
+    condition: fluentd.enabled
 kubeVersion: ">= 1.16.0"

charts/kubezero-logging/templates/fluentd/fluentd-certificate.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.fluentd.enabled }}
+apiVersion: cert-manager.io/v1alpha2
+kind: Certificate
+metadata:
+  name: fluentd-ingress-cert
+  namespace: {{ .Release.Namespace }}
+  labels:
+{{ include "kubezero-lib.labels" . | indent 4 }}
+spec:
+  secretName: fluentd-certificate
+  issuerRef:
+    name: letsencrypt-dns-prod
+    kind: ClusterIssuer
+  dnsNames:
+  - "{{ .Values.fluentd.url }}"
+{{- end }}

charts/kubezero-logging/templates/fluentd/istio-service.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.fluentd.istio.enabled }}
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: fluentd
+  namespace: {{ .Release.Namespace }}
+  labels:
+{{ include "kubezero-lib.labels" . | indent 4 }}
+spec:
+  gateways:
+  - {{ .Values.fluentd.istio.gateway }}
+  hosts:
+  - {{ .Values.fluentd.url }}
+  tcp:
+  - match:
+    - port: 24224
+    route:
+    - destination:
+        host: fluentd
+        port:
+          number: 24224
+  http:
+  - route:
+    - destination:
+        host: fluentd
+        port:
+          number: 9880
+{{- end }}

charts/kubezero-logging/values-all.yaml

@@ -29,3 +29,6 @@ kibana:
     enabled: true
     url: kibana.example.com
     gateway: istio-system/private-ingressgateway
+
+fluentd:
+  enabled: true

charts/kubezero-logging/values-fluentd.yaml

@@ -0,0 +1,11 @@
+fluentd:
+  enabled: true
+
+  metrics:
+    enabled: true
+
+  url: fluentd.example.com
+  istio:
+    enabled: true
+    gateway: istio-system/private-ingressgateway
+

charts/kubezero-logging/values-no-eck.yaml

@@ -1,6 +0,0 @@
-# Default values for zdt-logging.
-# This is a YAML-formatted file.
-# Declare variables to be passed into your templates.
-
-kibana: 
-  count: 0

charts/kubezero-logging/values.yaml

@@ -26,3 +26,168 @@ kibana:
     enabled: false
     gateway: "istio-system/ingressgateway"
     url: "" # kibana.example.com
+
+fluentd:
+  enabled: false
+  #image:
+    #repository: quay.io/fluentd_elasticsearch/fluentd
+    #tag: v2.9.0
+  istio:
+    enabled: false
+
+  # we wont persistent buffering
+  useStatefulSet: true
+  replicaCount: 2
+
+  plugins:
+    enabled: true
+    pluginsList:
+    - fluent-plugin-detect-exceptions
+  # - fluent-plugin-s3
+  # - fluent-plugin-grok-parser
+
+  persistence:
+    enabled: true
+    storageClass: "ebs-sc-gp2-xfs"
+    accessMode: ReadWriteOnce
+    size: 4Gi
+
+  service:
+    ports:
+    - name: tcp-forward
+      protocol: TCP
+      containerPort: 24224
+    - name: http-fluentd
+      protocol: TCP
+      containerPort: 9880
+
+  metrics:
+    enabled: false
+    serviceMonitor:
+      enabled: true
+      additionalLabels:
+        release: metrics
+      namespace: monitoring
+
+  output:
+    host: logging-es-http
+
+  env:
+    OUTPUT_USER: elastic
+    OUTPUT_SSL_VERIFY: "false"
+
+  extraEnvVars:
+  - name: OUTPUT_PASSWORD
+    valueFrom:
+      secretKeyRef:
+        name: logging-es-elastic-user
+        key: elastic
+  - name: FLUENTD_SHARED_KEY
+    valueFrom:
+      secretKeyRef:
+        name: fluentd-config
+        key: shared_key
+
+  extraVolumes:
+    - name: fluentd-certs
+      secret:
+        secretName: fluentd-certificate
+  extraVolumeMounts:
+    - name: fluentd-certs
+      mountPath: /mnt/fluentd-certs
+      readOnly: true
+
+  configMaps:
+    forward-input.conf: |
+      <source>
+        @type forward
+        port 24224
+        bind 0.0.0.0
+        skip_invalid_event true
+        <transport tls>
+          cert_path /mnt/fluentd-certs/tls.crt
+          private_key_path /mnt/fluentd-certs/tls.key
+        </transport>
+        <security>
+          self_hostname "#{ENV['HOSTNAME']}"
+          shared_key "#{ENV['FLUENTD_SHARED_KEY']}"
+        </security>
+      </source>
+
+    output.conf: |
+      <match **>
+        @id elasticsearch
+        @type elasticsearch
+        @log_level info
+        include_tag_key true
+        id_key id
+        remove_keys id
+
+        # This pipeline incl. eg. GeoIP
+        pipeline fluentd
+
+        host "#{ENV['OUTPUT_HOST']}"
+        port "#{ENV['OUTPUT_PORT']}"
+        scheme "#{ENV['OUTPUT_SCHEME']}"
+        ssl_version "#{ENV['OUTPUT_SSL_VERSION']}"
+        ssl_verify "#{ENV['OUTPUT_SSL_VERIFY']}"
+        user "#{ENV['OUTPUT_USER']}"
+        password "#{ENV['OUTPUT_PASSWORD']}"
+
+        logstash_format true
+        reload_connections false
+        reconnect_on_error true
+        reload_on_failure true
+        request_timeout 15s
+
+        <buffer>
+          @type file
+          path /var/log/fluentd-buffers/kubernetes.system.buffer
+          flush_mode interval
+          flush_thread_count 2
+          flush_interval 5s
+          flush_at_shutdown true
+          retry_type exponential_backoff
+          retry_timeout 60m
+          retry_max_interval 30
+          chunk_limit_size "#{ENV['OUTPUT_BUFFER_CHUNK_LIMIT']}"
+          queue_limit_length "#{ENV['OUTPUT_BUFFER_QUEUE_LIMIT']}"
+          overflow_action drop_oldest_chunk
+        </buffer>
+      </match>
+
+#  filter.conf: |
+#    <filter auth system.auth>
+#      @type parser
+#      key_name message
+#      reserve_data true
+#      reserve_time true
+#      <parse>
+#        @type grok
+#
+#        # SSH
+#        <grok>
+#          pattern %{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:system.auth.user} from %{IPORHOST:system.auth.ip} port %{NUMBER:system.auth.port} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?
+#        </grok>
+#        <grok>
+#          pattern %{DATA:system.auth.ssh.event} user %{DATA:system.auth.user} from %{IPORHOST:system.auth.ip}
+#        </grok>
+#
+#        # sudo
+#        <grok>
+#          pattern \s*%{DATA:system.auth.user} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}
+#        </grok>
+#
+#        # Users
+#        <grok>
+#          pattern new group: name=%{DATA:system.auth.groupadd.name}, GID=%{NUMBER:system.auth.groupadd.gid}
+#        </grok>
+#        <grok>
+#          pattern new user: name=%{DATA:system.auth.useradd.name}, UID=%{NUMBER:system.auth.useradd.uid}, GID=%{NUMBER:system.auth.useradd.gid}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$
+#        </grok>
+#
+#        <grok>
+#          pattern %{GREEDYDATA:message}
+#        </grok>
+#      </parse>
+#    </filter>

deploy/templates/values.yaml

@@ -139,6 +139,7 @@ kubezero:
       fullnameOverride: {{ .Values.logging.fullnameOverride }}
       {{- end }}
 
+      {{- if .Values.logging.es }}
       es:
         {{- if .Values.logging.es.nodeSets }}
         nodeSets:
@@ -154,6 +155,7 @@ kubezero:
           {{- toYaml . | nindent 10 }}
           {{- end }}
         {{- end }}
+      {{- end }}
 
       {{- if .Values.logging.kibana }}
       kibana:
@@ -162,6 +164,19 @@ kubezero:
         {{- end }}
       {{- end }}
 
+      fluentd:
+        enabled: {{ .Values.logging.fluentd.enabled }}
+        metrics:
+          enabled: {{ .Values.metrics.enabled }}
+        url: {{ .Values.logging.fluentd.url }}
+        {{- if and .Values.logging.fluentd.istio .Values.istio.enabled }}
+        istio:
+          {{- with .Values.logging.fluentd.istio }}
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
+        {{- end }}
+
+        
 argo-cd:
   controller:
     metrics:

deploy/values.yaml

@@ -35,6 +35,8 @@ metrics:
 
 logging:
   enabled: false
+  fluentd:
+   enabled: false
 
 argo-cd:
   server: {}