From 49e17c008f2e8259c2c4037710cfb65e3ac10bf5 Mon Sep 17 00:00:00 2001 From: Stefan Reimer Date: Fri, 9 Feb 2024 16:24:37 +0000 Subject: [PATCH] Feat: KubeZero-Telemetry module incl. Jaeger Collector/UI and OpenSearch --- charts/kubezero-cert-manager/values.yaml | 3 + charts/kubezero-operators/README.md | 2 +- charts/kubezero-operators/values.yaml | 7 +- charts/kubezero-telemetry/dashboards.yaml | 14 ++++ .../templates/grafana-dashboards.yaml | 15 ++++ .../jaeger/istio-authorization-policy.yaml | 28 +++++++ .../templates/jaeger/istio-service.yaml | 2 +- .../templates/opensearch/certificates.yaml | 70 +++++++++++++++++ .../templates/opensearch/cluster.yaml | 76 +++++++++++++++---- .../istio-authorization-policy.yaml | 28 +++++++ .../opensearch/istio-virtualservice.yaml | 20 +++++ charts/kubezero-telemetry/update.sh | 2 + charts/kubezero-telemetry/values-nodes.yaml | 75 ++++++++++++++++++ charts/kubezero-telemetry/values.yaml | 61 ++++++++++++--- charts/kubezero/templates/telemetry.yaml | 30 ++++---- 15 files changed, 391 insertions(+), 42 deletions(-) create mode 100644 charts/kubezero-telemetry/dashboards.yaml create mode 100644 charts/kubezero-telemetry/templates/grafana-dashboards.yaml create mode 100644 charts/kubezero-telemetry/templates/jaeger/istio-authorization-policy.yaml create mode 100644 charts/kubezero-telemetry/templates/opensearch/certificates.yaml create mode 100644 charts/kubezero-telemetry/templates/opensearch/istio-authorization-policy.yaml create mode 100644 charts/kubezero-telemetry/templates/opensearch/istio-virtualservice.yaml create mode 100644 charts/kubezero-telemetry/values-nodes.yaml diff --git a/charts/kubezero-cert-manager/values.yaml b/charts/kubezero-cert-manager/values.yaml index 2a5118d6..e35137ee 100644 --- a/charts/kubezero-cert-manager/values.yaml +++ b/charts/kubezero-cert-manager/values.yaml @@ -23,6 +23,9 @@ cert-manager: leaderElection: namespace: "cert-manager" + # remove secrets if the cert is deleted + enableCertificateOwnerRef: true + extraArgs: - "--logging-format=json" - "--leader-elect=false" diff --git a/charts/kubezero-operators/README.md b/charts/kubezero-operators/README.md index 81a0a515..8cdb1de9 100644 --- a/charts/kubezero-operators/README.md +++ b/charts/kubezero-operators/README.md @@ -32,7 +32,7 @@ Kubernetes: `>= 1.26.0` | eck-operator.tolerations[0].effect | string | `"NoSchedule"` | | | eck-operator.tolerations[0].key | string | `"node-role.kubernetes.io/control-plane"` | | | opensearch-operator.enabled | bool | `false` | | -| opensearch-operator.fullnameOverride | string | `"telemetry"` | | +| opensearch-operator.fullnameOverride | string | `"opensearch-operator"` | | | opensearch-operator.kubeRbacProxy.enable | bool | `false` | | | opensearch-operator.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | | | opensearch-operator.tolerations[0].effect | string | `"NoSchedule"` | | diff --git a/charts/kubezero-operators/values.yaml b/charts/kubezero-operators/values.yaml index 6142ab9a..34be69ad 100644 --- a/charts/kubezero-operators/values.yaml +++ b/charts/kubezero-operators/values.yaml @@ -2,12 +2,17 @@ opensearch-operator: enabled: false # otherwise service names will be >63 chars - fullnameOverride: telemetry + fullnameOverride: opensearch-operator # not needed for now kubeRbacProxy: enable: false + manager: + extraEnv: + - name: SKIP_INIT_CONTAINER + value: "true" + tolerations: - key: node-role.kubernetes.io/control-plane effect: NoSchedule diff --git a/charts/kubezero-telemetry/dashboards.yaml b/charts/kubezero-telemetry/dashboards.yaml new file mode 100644 index 00000000..52b8cb71 --- /dev/null +++ b/charts/kubezero-telemetry/dashboards.yaml @@ -0,0 +1,14 @@ +configmap: grafana-dashboards +gzip: true +folder: Telemetry +dashboards: +- name: jaeger + url: https://grafana.com/api/dashboards/10001/revisions/2/download + tags: + - Jaeger + - Telemetry +- name: opensearch + url: https://grafana.com/api/dashboards/15178/revisions/2/download + tags: + - OpenSearch + - Telemetry diff --git a/charts/kubezero-telemetry/templates/grafana-dashboards.yaml b/charts/kubezero-telemetry/templates/grafana-dashboards.yaml new file mode 100644 index 00000000..1b0f760f --- /dev/null +++ b/charts/kubezero-telemetry/templates/grafana-dashboards.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" (include "kubezero-lib.fullname" $) "grafana-dashboards" | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace }} + labels: + grafana_dashboard: "1" + {{- include "kubezero-lib.labels" . | nindent 4 }} + annotations: + k8s-sidecar-target-directory: Telemetry +binaryData: + jaeger.json.gz: + H4sIAAAAAAAC/+2dW3PbuBmG7/0rWF507K1sS7LlOJ3Zab3JppNOmmSTbHuRzXAgEpZQUwADgo4Vj/rbC/AgHiX6JFly3ptEBEACePHhw2McyOsdy7Idh/EgUqH9V+uzvras6/hfHcPJhOpQ++VH5/2Hdy9/f/Hp9bu3+2e/250shU+G1DdJAim8yFVM8H0S5fEeDV3JAhNuUuURahrEj/aIIqGIpEvzuMCPRoy/9tLnTqga0yisxr9NC/c+TxEnmOl/v3SSikn6NWKSNlQty38kyTnhJH848xqDMyn+UY24pDJMa3dy0D3opoXoNGcXEK71qmcWjBuzKgYXMhq0Z9SkK1uoKK9r2ZbvXGXCuVDEtLCROSmI7bNQzUXPi6djhhHz1WvzuF4nDy0Ut7kYOg3lZOibeCUjWggfM68hlLmCvxC+kOaBcjQku92O1e/19D+DQcfq7RUfndX/LK+L9WfrzKdSlYqQqxuOh4JIz07jZvH/X3bSBqna/cssvXUupDURnCkhGR9Z/yV0RKUlI87NJeMWsS5OQ4vySyYFn1CuDqz/CHkRWt+YGlu9g+d/ObBeUeoNiXsR/s1671MSUiuk3LOUsPxI8NHBpfg7nQS+mJr7x1SKA1dM4nrY1GOqIqM94lTFva3X7XZ7SZCxvU9C+IoFOqIbB8b2wyPfj698xi/ijpWYQWzZDR3NFb5PgpB6lSbSWTDvvchtJmnLil1809f940LAVVaa9HpqrrNGKFv6UT/3GOXClU3S2LDPSBhbS1yeWacYOSRx6DnxQ1qKMFbwhvKRikvdrcXRRbe12rpOdc58vyzGAs3muj3rlMOMdqeVsIp8cwmP+oWwcvUTLY9LYb62Wu7Vi0EuR01VNmYQSamNcUHshFwtimF8QUw4Ft9q3T7ppboP+wvuuiR+lLfLwkpr445TVZ8eR3xjnhrX2qbcH3LXovvLe8G4+peI3VQcUG7qgLh0kR0FVNsJV2REG20pME+WxGORyXpQj2s2Qd0UHpU09o3nvlDl8oRUMhq+045fmxttqFO4rMSh0p6pMddQ0SCg3hstYWO8InJEVbmbVrtqMhJcBXHRw2iyK4miu4kbdYyvoa52rY4uIQ/18O9Sdkk9J7aIz4PJl709u1N9mnbJE2Is01ZsQp2k9vVk6TDTYFSmk3BFpbatV8RkX7WNQp95Nc+snoOk5wn0nNmlqFnhqtwQaqzhZix8r6GRTGVeaedS8NmluA90lI7ZDTd+HLNz1XynigcP+6NR2PqQKnwYUrdsRGo+eFxX+y2RtdEgixNSNXiouNM62eDLuMcumRfpPr6w/xbwrsBQiQskV6zBgw4j9yIxvmqdjRdK+66RptJuGTs03LXYQ8290OcvC6swJVf0Bn0ht16dnVR1q8oQvV5AEylGv2h+aDTYxCk33pZ45caoQqXLRtx5IvVY2BmnzYal2WK0aAyL497Qy3nFdpoyKkj341LL6V2o5QTUAmrZNGqpwQrjzteI6hHO1yzD3amjucY6tNrTuSLiCkDzMEBzpi1HG6z+G3w/VtlKVbZ2NdvsAW4AN4AbwM1q4KZ3cge6Oe6DbkA3G083Q6LcsROy7/TOpPLjwcgvRjTroxZN44eZaQGAAEAAIACQda4JPW+ZXTkFf4A/Np4/0imTpGSYK3kQPPktniBJWtva/QhAAaAAUAAo617+aQGU4y4ABYCyaYCybMOKJ4XJrLBfBXMmt9yR8jKREBtSQCQgEhDJutds2pAE+2iBJJs/ZxKSy9bdKKU099uJ8mOyimUUxC4TEAuIZfXEslPJLu+K/0yOYL3IHFv9jKLU1Wg8bLeGs029BWebuj/W2aaTe6xj9ZYjGVaxQGQbR2TVk01aJa4cDSwaLJzICwxK6Vo4urhmyHUCKVwahuWjTtZQY4WOUUI7qpXOJG0Ve91WUkwlAcwAZphKWh21nN6eWvpY2wK2bBy2aHO7GbHE+4EBKM2AcksVwSZgE7AJ2GQ1bNK4zNUGJ1jlApxsI5wke4XBJneePMkFBJWASkAloJL1rfM8a4ESvAwGULK1Cz2SEs/RtdG9G0s896WUgphY3AGqAFWAKutd3GlDFbzZBaiy9XtS6oeZgCv32pHi4WgTkAXIAmR5lDWfNmbBPlowy/Yxi8r2eQrpjAn3fJ0I0yz35JbFooJdwC5gF7DLmk8AHS1nF6wMAV22BF0kDbTXo+lrdWnohNFwwpRqmGdJPCU+eLQS2HE113DqL2kPoA5QB6gD1FnzsaEW1OmBdcA628Y6ycvxVkc6T3tepoYqFTkBKgAVgApAZd1niNpIBZtgQCobRyq1Q0TlCYDC2SFASRlK6sevmudQcKgZQAIgAZCsfZGo7dW9wBHgyHZMnIyVKpx1+aqbW9W3s5hRDZ8XuOOkSpPCmEcBtgBbgC1rXvBpwZYeuAXcsq2bW84J8yM94mPF52E2p2R6AlWAKkAVoMq6l3zaWAVniMAq27k5BaTykHtTwCngFHAKOOVxVoL6g5ajzqAUUMr2rQQtONac9L6OpcuqonCtK0LX10nms9mB/hnnP5s91WUiHIAG0YBoQDQPSTS1WZZFq0RtSNPD54TANBvJNKXxNP/6dCDF1TRhmQRjdI1jQTrZa1o6GgEiHzMvczKJxVqqJ3bdgkxAJuv5/rSkASXVDlv7KvWZ6aab9UXqfvMXqY9PfqwvUj9rwLHyd4+WzDC1HH06Xu06mHmxx6+TQE2bPRtgDbB2B1jLyEIGbrYl1vGJotydOhrkrENrSQJXRFzt/dTrdrvYknxjojvTVqGN0UrVtFI1rd2JdmcspK7gXgioA9QB6jDd9AB8c+OjVN3lfDPAbBMAZuNX0AynxAs4TUeofjrpbsyL9c58P0OA0No91Ea+dwuw6TyoOukyngY6j/78vz/s/tXVH/bs/prdQRadtfXhxrL8sjZZ/vTzo8ryVvD920nzYgtQOKuNqYu1W+wR4F/wL/gX/Lsi/j29C/9iBxn4d1v59zpbaTV4d0gCdqglc2n4WDyzUfS7UJsOuPjugoGYV07M83aoNAPoGfQMegY9r4iem4+JtuEzXmkBfH5q+Hy4H///+uU+QPpmKgGpH0Q6wPUjwXWxQYDZwGxgNjD7lph92nQmqH/TXRotX3Qe4FNBwOyng9nmGC6Lh9301/6h0EoTZcZ5MPcdJAOAP7yOoPG103hL6wDNgeZAc6D5qtC8FngjNsebEsHmT4/NgeHL1QFx30sywPWjwTU4GhwNjl7XyyWqr5H4LaJyuuw1Ejvp3cb1GE9iYo+6iduzQ3dMJ+TfVIbaT8xf063xYZpk4hF5kaTUXJSbZJp5mq39ifpUs7UuxzwzRSeBr//O5qO5LBrbQpUZ/ixJxeIOlMafJ57M5uLbfi/rutqbpGF26baA6U4r85vT2jmZ8y72H3uQe3m71y1cHBUvepP896Dwu1e8OOoWY3IHY/cLv3teIv6XrA5muBGBypxxay7FB58UH1zMpX9cvPDy38+8YnmzspTk+y5iMrTTxi0bVRIYxX+O2N/fvGPPB1eT1A4u57Zy1N2Z/R92O4kbZvgAAA== + opensearch.json.gz: + H4sIAAAAAAAC/+1dWXPbOLZ+71/B4WSm45TaEWVJlrsqD7YT9+TejuO2nZ5UJR4VREISYi4KSNpW+/r+9jkASRHcJFKWJSrGgxcCIJaDs3w4AA7vf1IUtd8n9sT3XPVX5Qs8K8o9/w05NrIwpKpvL/pn5x8/vLv817tPF2ojyjbRAJssf0IdC3tj7LtxpoFdnZKJRxybFTmbFVEM5CHX8amO49LedMJbysubmP6I2O+NopaC/NOwr3FDKi/wAL+vGsE4Kf7uE4pzRhq1P6JoiGwUV06M3OSIMr+lM24wdcMh7+92drVm2ItGfnsTZAMJs61NxrlticlCS/PbyKMpKaSmnaVjXpPabnN3mbG5xB6Z2PWQl23yIiev9CjzW8tvZ0ELM45Btu1AWchlLBM0qZrE9WYMFHcEcgY+Mb33rCatEacK5M8nK5TBNhqYLN+jPhbSx8TISSW6Yx87pkNZhXQ0QC+bDaWlafCr02ko2o5YdTTow3gsyj+VQxNTL9GFmFvc8cBB1FDDvAf+9+qnkPRpwf44wfYFRlQfK7rpux6miuXYxHMozKZyS7yxIhQR1MC7u4lDWXFkG8qpY+BZCu+Wig3ipaiijmzscUWgdbT9XpDEpOLScUyPTCCjyRP57Nu+aQZPUCcK+6t19zr7+51Wt9fu7vFck9jXXCMEc875KEdD6I5poomLWc1DZLo41nTiBEeN8pwRJcaZE/NOMKcp/riF51ZbSLiLhhE+T9lzNBlJBu/FSnDW7atYqRKPk089DiYmq26pc5svVTrSx/iSWNjxvfSoFjJ0mkXijCHBpnHs2EMyShLFwEPkm16SVAHVOZuLiZBsOVwuVG8M2nzsmIarCgUeGmIVFppMgBVdQWbTkhv1DoaS6G8scUZy0iIy4jsvUNgY29m3PCe/tpD62fpukOnzcTXVRNZD41EdbxV2fIpNE1hghT3XVtrzvcKeUxDEFXa7leq28HSV4CaB4wq5Eg1cx/Q9nOoIWCM8ybJhlioC3xewltB1Jpqp3IdGhQZcbJFfDESvf1nYVHM17RQwndCQtpqG8nhEaKWVbiXxfPVTXo7QE9UB5AAKPoCTVxkFXaj891LKX2stUP5avvJvxUA8YcKCzk2SsCWm0YeQSwdIvx5Rx7dFIgXmNCqCKEZi5jcwI2Q4nWX7IHAiQSjBthdZ2kw2TIev4485HeMGx9SzsgFLDNc7dbxTYHK1UCa5VXEzoh9MtBvZ69w5DNXJfTotMcQM+YMVx58ZqB/bV0RH2EuOJzFcfIetiYloBtzxvEnAyL710gHo5HLo1A/RVZ9hWt+9Dx/f/P9X9UX4/1f1YUd5regwp17emzYMyu3bvjXAtOj9JAWJDclAxixto5wTpHtc7hJGRjXxCNvGiUMt5GVf/u5jOr2MMAjgP8f6NzKvk4UoHgZrvsN45iNBFDGOhU8CI5IAKSz9YkyGXjYjCYqUC07PLDbia4dccLTFAGizxixWzQOmgitatQ2p5zQ0T6jrOeq5XUEXB8OWajjz2gX5i42x0y18rcaauoy+XZe+3gKFfO7bNvMZMF/AivWxVLvbr3a75dRuV6rdZ652mTp4rOotW0cN1W+oTQ91j9xg5S0MRGrUrfPKLHBd9Jo11dKtkr4LrSnV9DNX05BigDT3QdNcP0ZVV6mnvur6LBiFcslGUUVVb3Zfql0g3q1yG1MXY0QNt8K+lHS9LGmx6Gjwku8OR792tnQdkObATgEHSvfLczcwLlcucy1CgymcN19VxMHyIkuzVIW1XyEU6WC5Onh2uraT0rXtkrpW+lykrq2oGvsTSixEpyvXuWLFtde9Z0Ffn0QHSyD8RMp5ucMoDnDOCFc69VITK3BQ0gr0pBWQVqCssiY28QgyyV+gFVZoA9LV1tcCvBd6KjG4VPMbV/NaSbTfkq57qedLK2SKTUeHKVuplk9WWl8dfz7rp0T4UvX/qjR3O2s52V5bE1NyKdFqSRMjTUxZa+DbyHXJyMbGCk1MstL6mphPs34uYWI2u3fcKxD+brm9Y3aoqcLOMSy1kMtvzbqJCx/qAFE3Z/zu+Hdsj7xx8rAKS8fufHIV2GqdWIgPplVkwI+QS3Tl+OyTQuyh8whzHt8HWka9D4lpJu6WsoTfKDKYahTnsZgXelWvOB1kejEmhsGuDYNCztI7YJR4izdXIkNpTPYO3YzStTFB8CkNBpfOsdBdXiqxc1IB5Nxmb2t7jsc7li6dq9ofxDtdPFesjSfeEsNLyJvK5PPMARpE+pwlqPNNKGIXwC8jZBa2kzUBGFga7OEofQt9sXGYsA4xpvGTPM/TszMK5DcwxdwkDE1HuJ/vchb4KPJsvrXhEs7ef/2fI9+dKu+dW0S810n1OgNTfz9ua92WqGQbC6t9b5i4sL5O97DbPqpUH+/mxRQsj1VY7bvmUbvZrF7tJ6BbYaXaSbd53K5e6UfQasW19k72jo5688yWO0E6ztOrYKP06zSPMfA/wcbvwPQZfimBPWb4QhlMlZfEhjZsHe+8JBR5+CU73dvXJ37fxbpjAyrgcnrPFiRg/l0+KQAHotcYSth99YK9tfsK0r85Ayj2Av4APvjSsa52dpRXMKLm0hfo2nORhMAqyv191KuHhxLgobF6Av3sA3f9XC/iMIYvQ5qjJyYN4WqnZsQJdGEZ8hw/LXk4MQj9XjfRen/+h1uGPG+rkWcxRf4GHANm5edG+BBIVvgQqKHZY8Ra0SP9Hpd0hh5/rhVVub1QeCcK6PmuOruxEZeTRE7XuhCEgYe5pDiZu+ATnWhfyl23O8ejEPilXlhwMRoWIHwpIqyuZvF17kW4iyg2soDXdaiXgvgc6/ajNRqxDXJDDB+ZWaeDEIhLDHZ1h+5ICsEOfP06QADiMESvJYw0JxJSqnQ+aJ+B85z10RTdzYOhL16MYZH4v5gta1Rn8A3r3q+atneQnPXhjC+gA9RLuRHCwGrJrrIMZwQzk4llEq5SMsWDZUomWRjyQuHLH027WbvRpBxkGQGaZjkI0O0obx3H03/HN7NeJyJh1c6xkONDsLF369Br7kdQYBGncF01RGIIuM14FbSn9CpkruYv5VboSQfC0zgQksTeqAehwpI02cpK1qQCdgkltU+xjtnpxcHUwxGKKY9cXvWeCLRAt24AFhv4hjBQvOKlZw4dYMJt1yJe3QjB+1WREkc1AXSnoTW4pGg4BOsgwd3S4G5bYFvt+rk+PLbZTS1tP39Xa69ZblfrraP7FrY9d3u2tkoByfUixYPKIfZ6SyDFvdZ8pMjY9tC9zA/tG+DIVGIMI1MZgQSmE7n8pRX0chgyF70thJAtCSHXDiErH6lgFlkHLGVhqDuEVH3D0d0+Pyix5kiAWQW38rMR60NWs8Eo74Fz7oRTWRJVPdZl1uz2NuZkUlP+rUd7zJrduvv/no3DLCeufJBOKVdfyfpq7SVLkm95NJQNabkMHOp0JRyaE5E5g5B0x7ZBOYhGoxYwqfO8YZILKAb3XfJX6Iwrh49ifc5saD8gxBNBKI41FH68twgeJfU9UJKvXZv1QE05/c+HS5Z7jvm9gIBvM6K9JJgCzOabiF3Afi5gqrvfKmJXzuPq41FGc6WAqbu/VwUvSVi0Hi9OrbYDKzt5WvvLoJrec0c1WiVUU0O/z3MBNHw3LQfREGZvWYhE/s9cr88XbXy19G2X+3uFbc0ppbYGa+W2Uc6Rh6XvZlVwY7+zv4S3I56W165aw2NP+52e9OFIsPJEYCXjg1kKrXSbEq1ItLLVaCV8HGKP3dx99ljlD7YxJvHJSvFJr91aBp+wqSA1BSe99p4EJ3KDaTMbTL2qp6T2ullsg8lonAyaUAbvtJ7JYW25tbTFW0sMLiBjAla7xFcqGgqzjH978xW40QaasxUxSw0Dk3z3sR9EKmd34VjRpzqpcwp1A7fc8/7MA0q13mz6g9FLOWY4Uu42rQdetbS9Tj12bxqVOt19bltOG/52TMHHY/baJT8ewy9HVzgnrSN9jC9BThw/oy94FIuj+JPYaYLw/D/D+Go5WUkRYTHo0MtWu9NQOu3gp7l7sJMIJcZL7O03FK110FDaTVakl1OmAznafguKdHgt+ztqhkM2AkqroM6C7WZ1hHyOKu4TGCyis9ZsJmFYlNHMqrS0JMxMzO9srePOK/EB0WtMC11OpcPddRcxfKcU6g2EoJsX3CjBsjlAMAzCGJ6j1fLS5+jy0GQE5kDxHIWHlsvGo0tEGCzSsVFlPCji/Mpac3QfcAP7hOFZBBcFligFe1mZyyBCXoJ4E8f1huQuSfkw8cSxvTB4ntpp/kPIB9CTeYenFb7Ch/8BTebdKQoAT9IXGXBo0HH19HUKZHnO7IX50XbotRmAXIF12YrvWAiWiV7uaaBitB786h0wHaP1Empo6Jt5SxpWs1hPUE2LqauDvUQFWTMnQCCmlaAa30oFMV0ZEOf3Cy1sOXTa/4CtS37+PTjaJV4snF0rnHf0XesVxYcoCaof7W0UKRRZQj4k5QMfYk78P5B8E/MogLM8Lnoi0/ZEpuW585nWYeBZfVOSXyNJX8iyvOBpqDqidbo05tKYb9yYdx9jzHvSmEtjLo15uXB9aYN9QjH+ce21woYnjbY02tJor95oZw7VVLHa7aa02tJqS6u9nNU+vEHEZBX/wKZ7NkZpv6X9lvb7Cex37zH2uyXtt7Tf0n5Xsd8wMBwEne4PFyy7G4rFDlXwgzxvvqqvt9eavyXu9fOz5fKU5IZPSS68r9peJgpHuy1PRMoTkdt0IpL/+6gzkWBByM3cQ5HrDM/xY5ybvOTTopzBFLny4OSaYp5VC3lWj48BVItsJu+XSOS0euSU9fMvFcCsKaGThE61vyibg53mXJNdAJ0o/sbnNgqg/0SYKL5u+6MAI0Z/JaAer0VCpLVApLZ2sHUQqd1qSoi0Voj0DIKZdbrLOIe6z+7bRjIKyMbATWBXDkejZJl0uZg91WRYhnA0nkfsUVpicwAK8j1HTRV5yNQXmUVQIrg/Jq7ngIG0ku89CE9XCbVquFGMemwi1yN6gMnURlVQB6I2/xOW+eclwu9UDqbBVzDzPli5WldXexGss/KiqIBCpkTPkKk/AOjENor7kM34Ku+t0syiqYWTy/Fw2TldDDyTezQMUjL7wiUPEdOnAMhgsqxJTULGHN6MdhX2Fc9PLtNEMmzMqu4199rLRIBT1fp9kACGIj9IIP15W/NBguXQbh1D91J2WumCGHh1GFl+rODHdwICO/aZqPaZPRS9gd9uLHbqtj/GaNL3XWzM/WrBF40BSOW1UlQDMPLcCja3rVoiYl+t/YaAypR/AYkVYiswC6AbpMtwPbuq7XZhOJpQEfk2WQV40578uwgwlq789LoEbttyhK3bWWYfVn5XU2KzLTvbltmkZd8Ln4BC7tO7PvSXWcv50Yy5Z29z8Or88/KwqvFUtPOq0E55pfyibY6Al58LPwNfa1x6/vn15WceYFrpWBKTrgeTdprtrdvG7jQ7chtbAsyanfRbDmHKSxISSNYPSJI0EmKuuZHeZ3Fag3Nm/RDH8J3ieWhoLef4RvrWeuN+O1ZCKkrIs6bd04NOa/uiQh90fvwPkf4UVqu6+hhbKFbLrf0g2Zua4ZkZes0HDOpOONSjfpwwcyschFEvsYnZcY6pOqvc4/qPneWZdRqMkusJXHSfZ2wTc+RiMzBQmUEKt2cnWSiVvEQrFMhFPmAodEomocinhCUCX6nkcbClJ0o8sUEpG/gw/+p3rmtWBa3gkZziUSwAATSKUG1mpb8IZ1vYtwLotJAiQXYUEEJNhM3AQ6aJU1G9KRiDu3RR95pMPlHzYmrreREi4tNWUbfTuiTBv2bUnRRZBHZITNQiCB2gX2KTcC4DsvcD1SZa2tB89tktb9+FRpQwZUfdDsYIu1uBKxKCNWOVRxFIBAAXHrJBYRh/IkrYIoZ/yCpf3lbGbjlmFzQV5yn3j2iEajI3Qx+Wll84ZOaAVkKG7+LLoKJcmL5BDueHC33GI31iD537b86AocXdV9f+AFMbe9jdffV/u694OYChQD/Q/vyoYUOBwhvjfvV/nIFagf2/JYs/nvVXSbmaiEVHisWcyYU1FPzh08ey05fI18v9p2yRxXk7RwZSOD0SgajXTy8HC0gl2X2L2D2ewjfhQXP2P5/b6Aj65sTgfdiDKpaADWF9IlCRfFI0RNHgf5lz4SFYIRIrjrMVx/Rybn/RZgvLMGxX8Imf+LUJ0a+5++8+TvvL4X46dUCdWzdExrEDKF6zKil5Uv3gfsFn7d8Xt9cn++GS9yZeFnPyCZOi7jVhJfnwXxIW2TBO7wAA diff --git a/charts/kubezero-telemetry/templates/jaeger/istio-authorization-policy.yaml b/charts/kubezero-telemetry/templates/jaeger/istio-authorization-policy.yaml new file mode 100644 index 00000000..cda4d219 --- /dev/null +++ b/charts/kubezero-telemetry/templates/jaeger/istio-authorization-policy.yaml @@ -0,0 +1,28 @@ +{{- if .Values.jaeger.istio.enabled }} +{{- if .Values.jaeger.istio.ipBlocks }} +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: jaeger-deny-not-in-ipblocks + namespace: istio-system + labels: + {{- include "kubezero-lib.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: istio-ingressgateway + action: DENY + rules: + - from: + - source: + notIpBlocks: + {{- toYaml .Values.jaeger.istio.ipBlocks | nindent 8 }} + to: + - operation: + hosts: [{{ .Values.jaeger.istio.url }}] + when: + - key: connection.sni + values: + - '*' +{{- end }} +{{- end }} diff --git a/charts/kubezero-telemetry/templates/jaeger/istio-service.yaml b/charts/kubezero-telemetry/templates/jaeger/istio-service.yaml index 127343a5..c3b01a79 100644 --- a/charts/kubezero-telemetry/templates/jaeger/istio-service.yaml +++ b/charts/kubezero-telemetry/templates/jaeger/istio-service.yaml @@ -16,5 +16,5 @@ spec: - destination: host: {{ .Release.Name }}-jaeger-query port: - number: 16686 + number: 80 {{- end }} diff --git a/charts/kubezero-telemetry/templates/opensearch/certificates.yaml b/charts/kubezero-telemetry/templates/opensearch/certificates.yaml new file mode 100644 index 00000000..0072e560 --- /dev/null +++ b/charts/kubezero-telemetry/templates/opensearch/certificates.yaml @@ -0,0 +1,70 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kubezero-lib.fullname" . }}-nodes-transport + namespace: {{ .Release.Namespace }} + labels: + {{ include "kubezero-lib.labels" . | nindent 4 }} +spec: + secretName: {{ template "kubezero-lib.fullname" . }}-nodes-transport-tls + issuerRef: + name: kubezero-local-ca-issuer + kind: ClusterIssuer + duration: 8760h0m0s + privateKey: + encoding: PKCS8 + usages: + - "client auth" + - "server auth" + commonName: {{ template "kubezero-lib.fullname" . }}-nodes + dnsNames: + # -- + - '{{ template "kubezero-lib.fullname" . }}-nodes' + - '{{ template "kubezero-lib.fullname" . }}-nodes-*' + - '{{ template "kubezero-lib.fullname" . }}-bootstrap-0' +--- + +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kubezero-lib.fullname" . }}-nodes-http + namespace: {{ .Release.Namespace }} + labels: + {{ include "kubezero-lib.labels" . | nindent 4 }} +spec: + secretName: {{ template "kubezero-lib.fullname" . }}-nodes-http-tls + issuerRef: + name: kubezero-local-ca-issuer + kind: ClusterIssuer + duration: 8760h0m0s + privateKey: + encoding: PKCS8 + usages: + - "client auth" + - "server auth" + commonName: {{ template "kubezero-lib.fullname" . }} + dnsNames: + # , ., ..svc,..svc.cluster.local + - '{{ template "kubezero-lib.fullname" . }}' + - '{{ template "kubezero-lib.fullname" . }}.{{ .Release.Namespace }}.svc' + - '{{ template "kubezero-lib.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local' +--- + +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "kubezero-lib.fullname" . }}-admin + namespace: {{ .Release.Namespace }} + labels: + {{ include "kubezero-lib.labels" . | nindent 4 }} +spec: + secretName: {{ template "kubezero-lib.fullname" . }}-admin-tls + issuerRef: + name: kubezero-local-ca-issuer + kind: ClusterIssuer + duration: 8760h0m0s + usages: + - "client auth" + commonName: {{ template "kubezero-lib.fullname" . }}-admin + privateKey: + encoding: PKCS8 diff --git a/charts/kubezero-telemetry/templates/opensearch/cluster.yaml b/charts/kubezero-telemetry/templates/opensearch/cluster.yaml index edb81be9..4a906a23 100644 --- a/charts/kubezero-telemetry/templates/opensearch/cluster.yaml +++ b/charts/kubezero-telemetry/templates/opensearch/cluster.yaml @@ -1,3 +1,4 @@ +#pluginsList: ["repository-s3","https://github.com/aiven/prometheus-exporter-plugin-for-opensearch/releases/download/2.11.1.0/prometheus-exporter-2.11.1.0.zip"] {{- if .Values.opensearch.nodeSets }} apiVersion: opensearch.opster.io/v1 kind: OpenSearchCluster @@ -9,31 +10,76 @@ metadata: spec: general: serviceName: {{ template "kubezero-lib.fullname" . }} - version: 2.11.0 + version: {{ .Values.opensearch.version }} + setVMMaxMapCount: false + pluginsList: ["repository-s3"] + monitoring: + enable: {{ .Values.opensearch.prometheus }} + tlsConfig: + insecureSkipVerify: true + {{- if .Values.opensearch.dashboard.enabled }} + # https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/config/opensearch_dashboards.yml dashboards: enable: true - version: 2.11.0 + version: {{ .Values.opensearch.version }} replicas: 1 resources: requests: memory: "512Mi" cpu: "200m" limits: - memory: "512Mi" - cpu: "200m" + memory: "1Gi" + #cpu: "200m" + {{- end }} nodePools: - - component: nodes - replicas: 2 - diskSize: "16Gi" - nodeSelector: - resources: - requests: - memory: "2Gi" - cpu: "500m" - limits: - memory: "2Gi" - cpu: "500m" + {{- range .Values.opensearch.nodeSets }} + - component: nodes-{{ .name }} + replicas: {{ .replicas }} + diskSize: {{ .storage.size }} + {{- with .storage.class }} + persistence: + pvc: + storageClass: {{ . }} + {{- end }} + {{- with .resources }} + resources: {{ toYaml . | nindent 8 }} + {{- end }} roles: - "cluster_manager" - "data" + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + opster.io/opensearch-cluster: {{ template "kubezero-lib.fullname" $ }} + additionalConfig: + index.codec: zstd_no_dict + indices.time_series_index.default_index_merge_policy: log_byte_size + {{- with .zone }} + cluster.routing.allocation.awareness.attributes: k8s_node_name,zone + node.attr.zone: {{ . }} + {{- end }} + {{- end }} + security: + config: + adminSecret: + name: {{ template "kubezero-lib.fullname" . }}-admin-tls + tls: + transport: + generate: false + perNode: false + secret: + name: {{ template "kubezero-lib.fullname" . }}-nodes-transport-tls + nodesDn: + - 'CN={{ template "kubezero-lib.fullname" . }}-nodes' + - 'CN={{ template "kubezero-lib.fullname" . }}-nodes-*' + - 'CN={{ template "kubezero-lib.fullname" . }}-bootstrap-0' + adminDn: + - 'CN={{ template "kubezero-lib.fullname" . }}-admin' + http: + generate: false + secret: + name: {{ template "kubezero-lib.fullname" . }}-nodes-http-tls {{- end }} diff --git a/charts/kubezero-telemetry/templates/opensearch/istio-authorization-policy.yaml b/charts/kubezero-telemetry/templates/opensearch/istio-authorization-policy.yaml new file mode 100644 index 00000000..7e2ce03b --- /dev/null +++ b/charts/kubezero-telemetry/templates/opensearch/istio-authorization-policy.yaml @@ -0,0 +1,28 @@ +{{- if .Values.opensearch.dashboard.istio.enabled }} +{{- if .Values.opensearch.dashboard.istio.ipBlocks }} +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: telemetry-dashboard-deny-not-in-ipblocks + namespace: istio-system + labels: + {{- include "kubezero-lib.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: istio-ingressgateway + action: DENY + rules: + - from: + - source: + notIpBlocks: + {{- toYaml .Values.opensearch.dashboard.istio.ipBlocks | nindent 8 }} + to: + - operation: + hosts: [{{ .Values.opensearch.dashboard.istio.url }}] + when: + - key: connection.sni + values: + - '*' +{{- end }} +{{- end }} diff --git a/charts/kubezero-telemetry/templates/opensearch/istio-virtualservice.yaml b/charts/kubezero-telemetry/templates/opensearch/istio-virtualservice.yaml new file mode 100644 index 00000000..e48b2ac9 --- /dev/null +++ b/charts/kubezero-telemetry/templates/opensearch/istio-virtualservice.yaml @@ -0,0 +1,20 @@ +{{- if .Values.opensearch.dashboard.istio.enabled }} +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: {{ template "kubezero-lib.fullname" . }}-kibana + namespace: {{ .Release.Namespace }} + labels: +{{ include "kubezero-lib.labels" . | indent 4 }} +spec: + hosts: + - {{ .Values.opensearch.dashboard.istio.url }} + gateways: + - {{ default "istio-system/ingressgateway" .Values.opensearch.dashboard.istio.gateway }} + http: + - route: + - destination: + host: telemetry-dashboards + port: + number: 5601 +{{- end }} diff --git a/charts/kubezero-telemetry/update.sh b/charts/kubezero-telemetry/update.sh index a8d36bee..f2f1aa1c 100755 --- a/charts/kubezero-telemetry/update.sh +++ b/charts/kubezero-telemetry/update.sh @@ -3,5 +3,7 @@ set -ex . ../../scripts/lib-update.sh +../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/grafana-dashboards.yaml + #login_ecr_public update_helm diff --git a/charts/kubezero-telemetry/values-nodes.yaml b/charts/kubezero-telemetry/values-nodes.yaml new file mode 100644 index 00000000..60c6e197 --- /dev/null +++ b/charts/kubezero-telemetry/values-nodes.yaml @@ -0,0 +1,75 @@ +opentelemetry-collector: + enabled: false + + mode: deployment + +jaeger: + enabled: false + + agent: + enabled: false + + collector: + service: + otlp: + grpc: + name: otlp-grpc + port: 4317 + http: + name: otlp-http + port: 4318 + serviceMonitor: + enabled: false + + # https://www.jaegertracing.io/docs/1.53/deployment/#collector + storage: + type: elasticsearch + elasticsearch: + scheme: https + host: telemetry + user: admin + password: admin + cmdlineParams: + es.tls.enabled: "" + es.tls.skip-host-verify: "" + + provisionDataStore: + cassandra: false + elasticsearch: false + + query: + agentSidecar: + enabled: false + serviceMonitor: + enabled: false + + istio: + enabled: false + gateway: istio-ingress/private-ingressgateway + url: jaeger.example.com + +opensearch: + version: 2.11.1 + prometheus: false + + nodeSets: + - name: default + replicas: 2 + storage: + size: 16Gi + class: my-fancy-SSDs + zone: us-west-2a + resources: + limits: + #cpu: 1 + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi + + dashboard: + enabled: false + istio: + enabled: false + gateway: istio-ingress/private-ingressgateway + url: telemetry-dashboard.example.com diff --git a/charts/kubezero-telemetry/values.yaml b/charts/kubezero-telemetry/values.yaml index 61695936..d124a372 100644 --- a/charts/kubezero-telemetry/values.yaml +++ b/charts/kubezero-telemetry/values.yaml @@ -6,29 +6,70 @@ opentelemetry-collector: jaeger: enabled: false -# allInOne: -# enabled: true -# storage: -# type: none -# collector: -# enabled: false -# query: -# enabled: false - agent: enabled: false + collector: + service: + otlp: + grpc: + name: otlp-grpc + port: 4317 + http: + name: otlp-http + port: 4318 + serviceMonitor: + enabled: false + + # https://www.jaegertracing.io/docs/1.53/deployment/#collector storage: type: elasticsearch + elasticsearch: + scheme: https + host: telemetry + user: admin + password: admin + cmdlineParams: + es.tls.enabled: "" + es.tls.skip-host-verify: "" provisionDataStore: cassandra: false elasticsearch: false + query: + agentSidecar: + enabled: false + serviceMonitor: + enabled: false + istio: enabled: false gateway: istio-ingress/private-ingressgateway url: jaeger.example.com opensearch: - nodeSets: {} + version: 2.11.1 + prometheus: false + + nodeSets: [] + #- name: default-nodes + # replicas: 2 + # storage: + # size: 16Gi + # class: my-fancy-SSDs + # zone: us-west-2a + # resources: + # limits: + # #cpu: 1 + # memory: 2Gi + # requests: + # cpu: 500m + # memory: 2Gi + + dashboard: + enabled: false + istio: + enabled: false + gateway: istio-ingress/private-ingressgateway + url: telemetry-dashboard.example.com diff --git a/charts/kubezero/templates/telemetry.yaml b/charts/kubezero/templates/telemetry.yaml index 511ff8fe..8d751711 100644 --- a/charts/kubezero/templates/telemetry.yaml +++ b/charts/kubezero/templates/telemetry.yaml @@ -5,31 +5,33 @@ jaeger: {{- with .Values.telemetry.jaeger }} {{- toYaml . | nindent 2 }} {{- end }} + + collector: + serviceMonitor: + enabled: {{ .Values.metrics.enabled }} + query: + serviceMonitor: + enabled: {{ .Values.metrics.enabled }} {{- end }} {{- if .Values.telemetry.opensearch }} opensearch: {{- if .Values.telemetry.opensearch.nodeSets }} nodeSets: - {{- with .Values.telemetry.opensearch.nodeSets }} - {{- toYaml . | nindent 2 }} - {{- end }} - {{- end }} - prometheus: {{ .Values.metrics.enabled }} - - {{- if .Values.telemetry.opensearch.s3Snapshot }} - s3Snapshot: - {{- with .Values.telemetry.opensearch.s3Snapshot }} + {{- with .Values.telemetry.opensearch.nodeSets }} {{- toYaml . | nindent 4 }} {{- end }} {{- end }} -{{- end }} -{{- if .Values.telemetry.dashboard }} -dashboard: - {{- with .Values.telemetry.dashboard }} - {{- toYaml . | nindent 2 }} + {{- if .Values.telemetry.opensearch.dashboard }} + dashboard: + {{- with .Values.telemetry.opensearch.dashboard }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} + + prometheus: {{ .Values.metrics.enabled }} + {{- end }} {{- end }}