Feat: KubeZero-Telemetry module incl. Jaeger Collector/UI and OpenSearch

charts/kubezero-cert-manager/values.yaml

@@ -23,6 +23,9 @@ cert-manager:
     leaderElection:
       namespace: "cert-manager"
 
+  # remove secrets if the cert is deleted
+  enableCertificateOwnerRef: true
+
   extraArgs:
     - "--logging-format=json"
     - "--leader-elect=false"

charts/kubezero-operators/README.md

@@ -32,7 +32,7 @@ Kubernetes: `>= 1.26.0`
 | eck-operator.tolerations[0].effect | string | `"NoSchedule"` |  |
 | eck-operator.tolerations[0].key | string | `"node-role.kubernetes.io/control-plane"` |  |
 | opensearch-operator.enabled | bool | `false` |  |
-| opensearch-operator.fullnameOverride | string | `"telemetry"` |  |
+| opensearch-operator.fullnameOverride | string | `"opensearch-operator"` |  |
 | opensearch-operator.kubeRbacProxy.enable | bool | `false` |  |
 | opensearch-operator.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` |  |
 | opensearch-operator.tolerations[0].effect | string | `"NoSchedule"` |  |

charts/kubezero-operators/values.yaml

@@ -2,12 +2,17 @@ opensearch-operator:
   enabled: false
 
   # otherwise service names will be >63 chars
-  fullnameOverride: telemetry
+  fullnameOverride: opensearch-operator
 
   # not needed for now
   kubeRbacProxy:
     enable: false
 
+  manager:
+    extraEnv:
+      - name: SKIP_INIT_CONTAINER
+        value: "true"
+
   tolerations:
   - key: node-role.kubernetes.io/control-plane
     effect: NoSchedule

charts/kubezero-telemetry/dashboards.yaml

@@ -0,0 +1,14 @@
+configmap: grafana-dashboards
+gzip: true
+folder: Telemetry
+dashboards:
+- name: jaeger
+  url: https://grafana.com/api/dashboards/10001/revisions/2/download
+  tags:
+  - Jaeger
+  - Telemetry
+- name: opensearch
+  url: https://grafana.com/api/dashboards/15178/revisions/2/download
+  tags:
+  - OpenSearch
+  - Telemetry

charts/kubezero-telemetry/templates/jaeger/istio-authorization-policy.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.jaeger.istio.enabled }}
+{{- if .Values.jaeger.istio.ipBlocks }}
+apiVersion: security.istio.io/v1beta1
+kind: AuthorizationPolicy
+metadata:
+  name: jaeger-deny-not-in-ipblocks
+  namespace: istio-system
+  labels:
+    {{- include "kubezero-lib.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app: istio-ingressgateway
+  action: DENY
+  rules:
+  - from:
+    - source:
+        notIpBlocks:
+        {{- toYaml .Values.jaeger.istio.ipBlocks | nindent 8 }}
+    to:
+    - operation:
+        hosts: [{{ .Values.jaeger.istio.url }}]
+    when:
+    - key: connection.sni
+      values:
+      - '*'
+{{- end }}
+{{- end }}

charts/kubezero-telemetry/templates/jaeger/istio-service.yaml

@@ -16,5 +16,5 @@ spec:
     - destination:
         host: {{ .Release.Name }}-jaeger-query
         port:
-          number: 16686
+          number: 80
 {{- end }}

charts/kubezero-telemetry/templates/opensearch/certificates.yaml

@@ -0,0 +1,70 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ template "kubezero-lib.fullname" . }}-nodes-transport
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{ include "kubezero-lib.labels" . | nindent 4 }}
+spec:
+  secretName: {{ template "kubezero-lib.fullname" . }}-nodes-transport-tls
+  issuerRef:
+    name: kubezero-local-ca-issuer
+    kind: ClusterIssuer
+  duration: 8760h0m0s
+  privateKey:
+    encoding: PKCS8
+  usages:
+  - "client auth"
+  - "server auth"
+  commonName: {{ template "kubezero-lib.fullname" . }}-nodes
+  dnsNames:
+  # <cluster-name>-<nodepool-component>-<index>
+  - '{{ template "kubezero-lib.fullname" . }}-nodes'
+  - '{{ template "kubezero-lib.fullname" . }}-nodes-*'
+  - '{{ template "kubezero-lib.fullname" . }}-bootstrap-0'
+---
+
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ template "kubezero-lib.fullname" . }}-nodes-http
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{ include "kubezero-lib.labels" . | nindent 4 }}
+spec:
+  secretName: {{ template "kubezero-lib.fullname" . }}-nodes-http-tls
+  issuerRef:
+    name: kubezero-local-ca-issuer
+    kind: ClusterIssuer
+  duration: 8760h0m0s
+  privateKey:
+    encoding: PKCS8
+  usages:
+  - "client auth"
+  - "server auth"
+  commonName: {{ template "kubezero-lib.fullname" . }}
+  dnsNames:
+  # <cluster-name>, <cluster-name>.<namespace>, <cluster-name>.<namespace>.svc,<cluster-name>.<namespace>.svc.cluster.local
+  - '{{ template "kubezero-lib.fullname" . }}'
+  - '{{ template "kubezero-lib.fullname" . }}.{{ .Release.Namespace }}.svc'
+  - '{{ template "kubezero-lib.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local'
+---
+
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ template "kubezero-lib.fullname" . }}-admin
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{ include "kubezero-lib.labels" . | nindent 4 }}
+spec:
+  secretName: {{ template "kubezero-lib.fullname" . }}-admin-tls
+  issuerRef:
+    name: kubezero-local-ca-issuer
+    kind: ClusterIssuer
+  duration: 8760h0m0s
+  usages:
+  - "client auth"
+  commonName: {{ template "kubezero-lib.fullname" . }}-admin
+  privateKey:
+    encoding: PKCS8

charts/kubezero-telemetry/templates/opensearch/cluster.yaml

@@ -1,3 +1,4 @@
+#pluginsList: ["repository-s3","https://github.com/aiven/prometheus-exporter-plugin-for-opensearch/releases/download/2.11.1.0/prometheus-exporter-2.11.1.0.zip"]
 {{- if .Values.opensearch.nodeSets }}
 apiVersion: opensearch.opster.io/v1
 kind: OpenSearchCluster
@@ -9,31 +10,76 @@ metadata:
 spec:
   general:
     serviceName: {{ template "kubezero-lib.fullname" . }}
-    version: 2.11.0
+    version: {{ .Values.opensearch.version }}
+    setVMMaxMapCount: false
+    pluginsList: ["repository-s3"]
+    monitoring:
+      enable: {{ .Values.opensearch.prometheus }}
+      tlsConfig:
+        insecureSkipVerify: true
+  {{- if .Values.opensearch.dashboard.enabled }}
+  # https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/config/opensearch_dashboards.yml
   dashboards:
     enable: true
-    version: 2.11.0
+    version: {{ .Values.opensearch.version }}
     replicas: 1
     resources:
       requests:
          memory: "512Mi"
          cpu: "200m"
       limits:
-         memory: "512Mi"
+         memory: "1Gi"
-         cpu: "200m"
+         #cpu: "200m"
+  {{- end }}
   nodePools:
-    - component: nodes
+    {{- range .Values.opensearch.nodeSets }}
-      replicas: 2
+    - component: nodes-{{ .name }}
-      diskSize: "16Gi"
+      replicas: {{ .replicas }}
-      nodeSelector:
+      diskSize:  {{ .storage.size }}
-      resources:
+      {{- with .storage.class }}
-         requests:
+      persistence:
-            memory: "2Gi"
+        pvc:
-            cpu: "500m"
+          storageClass: {{ . }}
-         limits:
+      {{- end }}
-            memory: "2Gi"
+      {{- with .resources }}
-            cpu: "500m"
+      resources: {{ toYaml . | nindent 8 }}
+      {{- end }}
       roles:
         - "cluster_manager"
         - "data"
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              opster.io/opensearch-cluster: {{ template "kubezero-lib.fullname" $ }}
+      additionalConfig:
+        index.codec: zstd_no_dict
+        indices.time_series_index.default_index_merge_policy: log_byte_size 
+        {{- with .zone }}
+        cluster.routing.allocation.awareness.attributes: k8s_node_name,zone
+        node.attr.zone: {{ . }}
+        {{- end }}
+    {{- end }}
+  security:
+    config:
+      adminSecret: 
+        name: {{ template "kubezero-lib.fullname" . }}-admin-tls
+    tls:
+      transport:
+        generate: false
+        perNode: false
+        secret:
+          name: {{ template "kubezero-lib.fullname" . }}-nodes-transport-tls
+        nodesDn:
+          - 'CN={{ template "kubezero-lib.fullname" . }}-nodes'
+          - 'CN={{ template "kubezero-lib.fullname" . }}-nodes-*'
+          - 'CN={{ template "kubezero-lib.fullname" . }}-bootstrap-0'
+        adminDn:
+          - 'CN={{ template "kubezero-lib.fullname" . }}-admin'
+      http:
+        generate: false
+        secret:
+          name: {{ template "kubezero-lib.fullname" . }}-nodes-http-tls
 {{- end }}

charts/kubezero-telemetry/templates/opensearch/istio-authorization-policy.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.opensearch.dashboard.istio.enabled }}
+{{- if .Values.opensearch.dashboard.istio.ipBlocks }}
+apiVersion: security.istio.io/v1beta1
+kind: AuthorizationPolicy
+metadata:
+  name: telemetry-dashboard-deny-not-in-ipblocks
+  namespace: istio-system
+  labels:
+    {{- include "kubezero-lib.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app: istio-ingressgateway
+  action: DENY
+  rules:
+  - from:
+    - source:
+        notIpBlocks:
+        {{- toYaml .Values.opensearch.dashboard.istio.ipBlocks | nindent 8 }}
+    to:
+    - operation:
+        hosts: [{{ .Values.opensearch.dashboard.istio.url }}]
+    when:
+    - key: connection.sni
+      values:
+      - '*'
+{{- end }}
+{{- end }}

charts/kubezero-telemetry/templates/opensearch/istio-virtualservice.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.opensearch.dashboard.istio.enabled }}
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: {{ template "kubezero-lib.fullname" . }}-kibana
+  namespace: {{ .Release.Namespace }}
+  labels:
+{{ include "kubezero-lib.labels" . | indent 4 }} 
+spec:
+  hosts:
+  - {{ .Values.opensearch.dashboard.istio.url }}
+  gateways:
+  - {{ default "istio-system/ingressgateway" .Values.opensearch.dashboard.istio.gateway }}
+  http:
+  - route:
+    - destination:
+        host: telemetry-dashboards
+        port:
+          number: 5601
+{{- end }}

charts/kubezero-telemetry/update.sh

@@ -3,5 +3,7 @@ set -ex
 
 . ../../scripts/lib-update.sh
 
+../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/grafana-dashboards.yaml
+
 #login_ecr_public
 update_helm

charts/kubezero-telemetry/values-nodes.yaml

@@ -0,0 +1,75 @@
+opentelemetry-collector:
+  enabled: false
+
+  mode: deployment
+
+jaeger:
+  enabled: false
+
+  agent:
+    enabled: false
+
+  collector:
+    service:
+      otlp:
+        grpc:
+          name: otlp-grpc
+          port: 4317
+        http:
+          name: otlp-http
+          port: 4318
+    serviceMonitor:
+      enabled: false
+
+  # https://www.jaegertracing.io/docs/1.53/deployment/#collector
+  storage:
+    type: elasticsearch
+    elasticsearch:
+      scheme: https
+      host: telemetry
+      user: admin
+      password: admin
+      cmdlineParams:
+        es.tls.enabled: ""
+        es.tls.skip-host-verify: ""
+
+  provisionDataStore:
+    cassandra: false
+    elasticsearch: false
+
+  query:
+    agentSidecar:
+      enabled: false
+    serviceMonitor:
+      enabled: false
+
+  istio:
+    enabled: false
+    gateway: istio-ingress/private-ingressgateway
+    url: jaeger.example.com
+
+opensearch:
+  version: 2.11.1
+  prometheus: false
+
+  nodeSets:
+  - name: default
+    replicas: 2
+    storage:
+      size: 16Gi
+      class: my-fancy-SSDs
+    zone: us-west-2a
+    resources:
+      limits:
+        #cpu: 1
+        memory: 2Gi
+      requests:
+        cpu: 500m
+        memory: 2Gi
+
+  dashboard:
+    enabled: false
+    istio:
+      enabled: false
+      gateway: istio-ingress/private-ingressgateway
+      url: telemetry-dashboard.example.com

charts/kubezero-telemetry/values.yaml

@@ -6,29 +6,70 @@ opentelemetry-collector:
 jaeger:
   enabled: false
 
-# allInOne:
-#   enabled: true
-# storage:
-#   type: none
-# collector:
-#   enabled: false
-# query:
-#   enabled: false
-
   agent:
     enabled: false
 
+  collector:
+    service:
+      otlp:
+        grpc:
+          name: otlp-grpc
+          port: 4317
+        http:
+          name: otlp-http
+          port: 4318
+    serviceMonitor:
+      enabled: false
+
+  # https://www.jaegertracing.io/docs/1.53/deployment/#collector
   storage:
     type: elasticsearch
+    elasticsearch:
+      scheme: https
+      host: telemetry
+      user: admin
+      password: admin
+      cmdlineParams:
+        es.tls.enabled: ""
+        es.tls.skip-host-verify: ""
 
   provisionDataStore:
     cassandra: false
     elasticsearch: false
 
+  query:
+    agentSidecar:
+      enabled: false
+    serviceMonitor:
+      enabled: false
+
   istio:
     enabled: false
     gateway: istio-ingress/private-ingressgateway
     url: jaeger.example.com
 
 opensearch:
-  nodeSets: {}
+  version: 2.11.1
+  prometheus: false
+
+  nodeSets: []
+  #- name: default-nodes 
+  #  replicas: 2
+  #  storage:
+  #    size: 16Gi
+  #    class: my-fancy-SSDs
+  #  zone: us-west-2a
+  #  resources:
+  #    limits:
+  #      #cpu: 1
+  #      memory: 2Gi
+  #    requests:
+  #      cpu: 500m
+  #      memory: 2Gi
+
+  dashboard:
+    enabled: false
+    istio:
+      enabled: false
+      gateway: istio-ingress/private-ingressgateway
+      url: telemetry-dashboard.example.com

charts/kubezero/templates/telemetry.yaml

@@ -5,31 +5,33 @@ jaeger:
   {{- with .Values.telemetry.jaeger }}
   {{- toYaml . | nindent 2 }}
   {{- end }}
+
+  collector:
+    serviceMonitor:
+      enabled: {{ .Values.metrics.enabled }}
+  query:
+    serviceMonitor:
+      enabled: {{ .Values.metrics.enabled }}
 {{- end }}
 
 {{- if .Values.telemetry.opensearch }}
 opensearch:
   {{- if .Values.telemetry.opensearch.nodeSets }}
   nodeSets:
-  {{- with .Values.telemetry.opensearch.nodeSets }}
+    {{- with .Values.telemetry.opensearch.nodeSets }}
-  {{- toYaml . | nindent 2 }}
-  {{- end }}
-  {{- end }}
-  prometheus: {{ .Values.metrics.enabled }}
-
-  {{- if .Values.telemetry.opensearch.s3Snapshot }}
-  s3Snapshot:
-    {{- with .Values.telemetry.opensearch.s3Snapshot }}
     {{- toYaml . | nindent 4 }}
     {{- end }}
   {{- end }}
-{{- end }}
 
-{{- if .Values.telemetry.dashboard }}
+  {{- if .Values.telemetry.opensearch.dashboard }}
-dashboard:
+  dashboard:
-  {{- with .Values.telemetry.dashboard }}
+    {{- with .Values.telemetry.opensearch.dashboard }}
-  {{- toYaml . | nindent 2 }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   {{- end }}
+
+  prometheus: {{ .Values.metrics.enabled }}
+
 {{- end }}
 
 {{- end }}