From 88725c33beb1e0a8c10affff7b972230f25f368f Mon Sep 17 00:00:00 2001
From: Stefan Reimer <stefan@zero-downtime.net>
Date: Fri, 4 Sep 2020 01:13:39 +0100
Subject: [PATCH] Move scrape username to its own secret as eck operator cleans
 up otherwise

---
 charts/kubezero-istio/README.md                |  5 +++--
 charts/kubezero-logging/README.md              |  9 +++++----
 .../templates/eck/secrets.yaml                 | 13 +++++++++++++
 .../templates/eck/servicemonitor.yaml          |  2 +-
 charts/kubezero-metrics/README.md              | 18 +++++++++++++++---
 charts/kubezero-metrics/README.md.gotmpl       | 10 ++++++++++
 docs/misc.md                                   |  3 +++
 7 files changed, 50 insertions(+), 10 deletions(-)

diff --git a/charts/kubezero-istio/README.md b/charts/kubezero-istio/README.md
index fa4e2989..23cbeacf 100644
--- a/charts/kubezero-istio/README.md
+++ b/charts/kubezero-istio/README.md
@@ -5,7 +5,7 @@ KubeZero Umbrella Chart for Istio
 Installs Istio Operator and KubeZero Istio profile
 
 
-Current chart version is `0.2.3`
+Current chart version is `0.2.4`
 
 Source code can be found [here](https://kubezero.com)
 
@@ -24,8 +24,9 @@ Source code can be found [here](https://kubezero.com)
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | ingress.autoscaleEnabled | bool | `false` |  |
+| ingress.dnsNames[0] | string | `"*"` |  |
 | ingress.private.enabled | bool | `true` |  |
-| ingress.private.nodeSelector | string | `"31080_31443_30671_30672_31224"` |  |
+| ingress.private.nodeSelector | string | `"31080_31443_31671_31672_31224"` |  |
 | ingress.replicaCount | int | `2` |  |
 | ingress.type | string | `"NodePort"` |  |
 | istio-operator.hub | string | `"docker.io/istio"` |  |
diff --git a/charts/kubezero-logging/README.md b/charts/kubezero-logging/README.md
index 7a4de686..5d33343d 100644
--- a/charts/kubezero-logging/README.md
+++ b/charts/kubezero-logging/README.md
@@ -49,13 +49,12 @@ Source code can be found [here](https://kubezero.com)
 | es.prometheus | bool | `false` |  |
 | es.s3Snapshot.enabled | bool | `false` |  |
 | es.s3Snapshot.iamrole | string | `""` |  |
-| fluent-bit.config.customParsers | string | `"[PARSER]\n    # http://rubular.com/r/tjUt3Awgg4\n    Name cri\n    Format regex\n    Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$\n    Time_Key    time\n    Time_Format %Y-%m-%dT%H:%M:%S.%L%z\n    # Decode_Field_As json log\n"` |  |
 | fluent-bit.config.filters | string | `"[FILTER]\n    Name kubernetes\n    Match kube.*\n    Merge_Log On\n    Keep_Log Off\n    K8S-Logging.Parser On\n    K8S-Logging.Exclude On\n\n[FILTER]\n    Name    lua\n    Match   kube.*\n    script  /fluent-bit/etc/functions.lua\n    call    dedot\n"` |  |
 | fluent-bit.config.inputs | string | `"[INPUT]\n    Name tail\n    Path /var/log/containers/*.log\n    Parser cri\n    Tag kube.*\n    Mem_Buf_Limit 5MB\n    Skip_Long_Lines On\n    Refresh_Interval 10\n    DB /var/log/flb_kube.db\n    DB.Sync Normal\n"` |  |
 | fluent-bit.config.lua | string | `"function dedot(tag, timestamp, record)\n    if record[\"kubernetes\"] == nil then\n        return 0, 0, 0\n    end\n    dedot_keys(record[\"kubernetes\"][\"annotations\"])\n    dedot_keys(record[\"kubernetes\"][\"labels\"])\n    return 1, timestamp, record\nend\n\nfunction dedot_keys(map)\n    if map == nil then\n        return\n    end\n    local new_map = {}\n    local changed_keys = {}\n    for k, v in pairs(map) do\n        local dedotted = string.gsub(k, \"%.\", \"_\")\n        if dedotted ~= k then\n            new_map[dedotted] = v\n            changed_keys[k] = true\n        end\n    end\n    for k in pairs(changed_keys) do\n        map[k] = nil\n    end\n    for k, v in pairs(new_map) do\n        map[k] = v\n    end\nend\n"` |  |
-| fluent-bit.config.outputs | string | `"[OUTPUT]\n    Match *\n    Name forward\n    Host fluentd\n    Port 24224\n    tls on\n    tls.verify off\n    Shared_Key cloudbender\n"` |  |
+| fluent-bit.config.outputs | string | `"[OUTPUT]\n    Match *\n    Name forward\n    Host logging-fluentd\n    Port 24224\n    tls on\n    tls.verify off\n    Shared_Key cloudbender\n"` |  |
 | fluent-bit.config.service | string | `"[SERVICE]\n    Flush 5\n    Daemon Off\n    Log_Level warn\n    Parsers_File parsers.conf\n    Parsers_File custom_parsers.conf\n    HTTP_Server On\n    HTTP_Listen 0.0.0.0\n    HTTP_Port 2020\n"` |  |
-| fluent-bit.enabled | bool | `true` |  |
+| fluent-bit.enabled | bool | `false` |  |
 | fluent-bit.serviceMonitor.enabled | bool | `true` |  |
 | fluent-bit.serviceMonitor.namespace | string | `"monitoring"` |  |
 | fluent-bit.serviceMonitor.selector.release | string | `"metrics"` |  |
@@ -63,7 +62,7 @@ Source code can be found [here](https://kubezero.com)
 | fluent-bit.tolerations[0].effect | string | `"NoSchedule"` |  |
 | fluent-bit.tolerations[0].key | string | `"node-role.kubernetes.io/master"` |  |
 | fluentd.configMaps."forward-input.conf" | string | `"<source>\n  @type forward\n  port 24224\n  bind 0.0.0.0\n  skip_invalid_event true\n  <transport tls>\n    cert_path /mnt/fluentd-certs/tls.crt\n    private_key_path /mnt/fluentd-certs/tls.key\n  </transport>\n  <security>\n    self_hostname \"#{ENV['HOSTNAME']}\"\n    shared_key \"#{ENV['FLUENTD_SHARED_KEY']}\"\n  </security>\n</source>\n"` |  |
-| fluentd.configMaps."output.conf" | string | `"<match **>\n  @id elasticsearch\n  @type elasticsearch\n  @log_level info\n  include_tag_key true\n  id_key id\n  remove_keys id\n\n  # KubeZero pipeline incl. GeoIP etc.\n  pipeline fluentd\n\n  host \"#{ENV['OUTPUT_HOST']}\"\n  port \"#{ENV['OUTPUT_PORT']}\"\n  scheme \"#{ENV['OUTPUT_SCHEME']}\"\n  ssl_version \"#{ENV['OUTPUT_SSL_VERSION']}\"\n  ssl_verify \"#{ENV['OUTPUT_SSL_VERIFY']}\"\n  user \"#{ENV['OUTPUT_USER']}\"\n  password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n  logstash_format true\n  reload_connections false\n  reconnect_on_error true\n  reload_on_failure true\n  request_timeout 15s\n\n  <buffer>\n    @type file\n    path /var/log/fluentd-buffers/kubernetes.system.buffer\n    flush_mode interval\n    flush_thread_count 2\n    flush_interval 5s\n    flush_at_shutdown true\n    retry_type exponential_backoff\n    retry_timeout 60m\n    retry_max_interval 30\n    chunk_limit_size \"#{ENV['OUTPUT_BUFFER_CHUNK_LIMIT']}\"\n    queue_limit_length \"#{ENV['OUTPUT_BUFFER_QUEUE_LIMIT']}\"\n    overflow_action drop_oldest_chunk\n  </buffer>\n</match>\n"` |  |
+| fluentd.configMaps."output.conf" | string | `"<match **>\n  @id elasticsearch\n  @type elasticsearch\n  @log_level info\n  include_tag_key true\n  id_key id\n  remove_keys id\n\n  # KubeZero pipeline incl. GeoIP etc.\n  # Freaking ES jams under load and all is lost ...\n  # pipeline fluentd\n\n  host \"#{ENV['OUTPUT_HOST']}\"\n  port \"#{ENV['OUTPUT_PORT']}\"\n  scheme \"#{ENV['OUTPUT_SCHEME']}\"\n  ssl_version \"#{ENV['OUTPUT_SSL_VERSION']}\"\n  ssl_verify \"#{ENV['OUTPUT_SSL_VERIFY']}\"\n  user \"#{ENV['OUTPUT_USER']}\"\n  password \"#{ENV['OUTPUT_PASSWORD']}\"\n\n  log_es_400_reason\n  logstash_format true\n  reconnect_on_error true\n  # reload_on_failure true\n  request_timeout 15s\n  suppress_type_name true\n\n  <buffer>\n    @type file\n    path /var/log/fluentd-buffers/kubernetes.system.buffer\n    flush_mode interval\n    flush_thread_count 2\n    flush_interval 30s\n    flush_at_shutdown true\n    retry_type exponential_backoff\n    retry_timeout 60m\n    chunk_limit_size 16M\n    overflow_action drop_oldest_chunk\n  </buffer>\n</match>\n"` |  |
 | fluentd.enabled | bool | `false` |  |
 | fluentd.env.OUTPUT_SSL_VERIFY | string | `"false"` |  |
 | fluentd.env.OUTPUT_USER | string | `"elastic"` |  |
@@ -78,6 +77,8 @@ Source code can be found [here](https://kubezero.com)
 | fluentd.extraVolumeMounts[0].readOnly | bool | `true` |  |
 | fluentd.extraVolumes[0].name | string | `"fluentd-certs"` |  |
 | fluentd.extraVolumes[0].secret.secretName | string | `"fluentd-certificate"` |  |
+| fluentd.image.repository | string | `"quay.io/fluentd_elasticsearch/fluentd"` |  |
+| fluentd.image.tag | string | `"v2.9.0"` |  |
 | fluentd.istio.enabled | bool | `false` |  |
 | fluentd.metrics.enabled | bool | `false` |  |
 | fluentd.metrics.serviceMonitor.additionalLabels.release | string | `"metrics"` |  |
diff --git a/charts/kubezero-logging/templates/eck/secrets.yaml b/charts/kubezero-logging/templates/eck/secrets.yaml
index 897b9689..04b9e604 100644
--- a/charts/kubezero-logging/templates/eck/secrets.yaml
+++ b/charts/kubezero-logging/templates/eck/secrets.yaml
@@ -12,5 +12,18 @@ metadata:
 {{ include "kubezero-lib.labels" . | indent 4 }}
 data:
   elastic: {{ .Values.elastic_password | b64enc | quote }}
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  labels:
+    common.k8s.elastic.co/type: elasticsearch
+    elasticsearch.k8s.elastic.co/cluster-name: {{ template "kubezero-lib.fullname" $ }}
+  name: {{ template "kubezero-lib.fullname" $ }}-es-elastic-username
+  namespace: {{ .Release.Namespace }}
+  labels:
+{{ include "kubezero-lib.labels" . | indent 4 }}
+data:
   username: {{ "elastic" | b64enc | quote }}
 {{- end }}
diff --git a/charts/kubezero-logging/templates/eck/servicemonitor.yaml b/charts/kubezero-logging/templates/eck/servicemonitor.yaml
index 8c809f58..85712632 100644
--- a/charts/kubezero-logging/templates/eck/servicemonitor.yaml
+++ b/charts/kubezero-logging/templates/eck/servicemonitor.yaml
@@ -11,7 +11,7 @@ spec:
   endpoints:
   - basicAuth:
       username:
-        name: {{ template "kubezero-lib.fullname" $ }}-es-elastic-user
+        name: {{ template "kubezero-lib.fullname" $ }}-es-elastic-username
         key: username
       password:
         name: {{ template "kubezero-lib.fullname" $ }}-es-elastic-user
diff --git a/charts/kubezero-metrics/README.md b/charts/kubezero-metrics/README.md
index 4275afb9..b4c8a4c3 100644
--- a/charts/kubezero-metrics/README.md
+++ b/charts/kubezero-metrics/README.md
@@ -2,7 +2,7 @@ kubezero-metrics
 ================
 KubeZero Umbrella Chart for prometheus-operator
 
-Current chart version is `0.1.3`
+Current chart version is `0.1.4`
 
 Source code can be found [here](https://kubezero.com)
 
@@ -11,7 +11,7 @@ Source code can be found [here](https://kubezero.com)
 | Repository | Name | Version |
 |------------|------|---------|
 | https://kubernetes-charts.storage.googleapis.com/ | prometheus-adapter | 2.5.0 |
-| https://kubernetes-charts.storage.googleapis.com/ | prometheus-operator | 9.3.0 |
+| https://kubernetes-charts.storage.googleapis.com/ | prometheus-operator | 9.3.1 |
 | https://zero-down-time.github.io/kubezero/ | kubezero-lib | >= 0.1.3 |
 
 ## Chart Values
@@ -58,7 +58,9 @@ Source code can be found [here](https://kubezero.com)
 | prometheus-operator.kubeControllerManager.serviceMonitor.https | bool | `true` |  |
 | prometheus-operator.kubeControllerManager.serviceMonitor.insecureSkipVerify | bool | `true` |  |
 | prometheus-operator.kubeDns.enabled | bool | `false` |  |
-| prometheus-operator.kubeEtcd.enabled | bool | `false` |  |
+| prometheus-operator.kubeEtcd.enabled | bool | `true` |  |
+| prometheus-operator.kubeEtcd.service.port | int | `2381` |  |
+| prometheus-operator.kubeEtcd.service.targetPort | int | `2381` |  |
 | prometheus-operator.kubeProxy.enabled | bool | `true` |  |
 | prometheus-operator.kubeScheduler.enabled | bool | `true` |  |
 | prometheus-operator.kubeScheduler.service.port | int | `10259` |  |
@@ -96,3 +98,13 @@ Source code can be found [here](https://kubezero.com)
 | prometheus.istio.enabled | bool | `false` |  |
 | prometheus.istio.gateway | string | `"istio-system/ingressgateway"` |  |
 | prometheus.istio.url | string | `""` |  |
+
+
+# Dashboards
+
+## Etcs
+- https://grafana.com/grafana/dashboards/3070
+
+## ElasticSearch
+- https://grafana.com/grafana/dashboards/266
+
diff --git a/charts/kubezero-metrics/README.md.gotmpl b/charts/kubezero-metrics/README.md.gotmpl
index 1e4d03ba..d186c403 100644
--- a/charts/kubezero-metrics/README.md.gotmpl
+++ b/charts/kubezero-metrics/README.md.gotmpl
@@ -8,3 +8,13 @@
 {{ template "chart.requirementsSection" . }}
 
 {{ template "chart.valuesSection" . }}
+
+
+# Dashboards
+
+## Etcs
+- https://grafana.com/grafana/dashboards/3070
+
+## ElasticSearch
+- https://grafana.com/grafana/dashboards/266
+
diff --git a/docs/misc.md b/docs/misc.md
index 8eecdf24..a48b318c 100644
--- a/docs/misc.md
+++ b/docs/misc.md
@@ -4,6 +4,9 @@
 - https://kubernetes.io/docs/tasks/debug-application-cluster/audit/
 - https://kubernetes.io/docs/tasks/debug-application-cluster/falco/
 
+## DNS - Todo
+- https://github.com/kubernetes-sigs/external-dns/blob/0ef226f77ef80158257e1fe9705c095452a51545/docs/tutorials/istio.md
+
 ## Performance - Todo
 - https://kubernetes.io/docs/tasks/administer-cluster/limit-storage-consumption/