ZeroDownTime/KubeZero
3
0
Fork 0
Code Issues 1 Pull Requests 25 Packages Projects Releases Wiki Activity

Feat: KubeZero-Telemetry module incl. Jaeger Collector/UI and OpenSearch

Browse Source
This commit is contained in:
Stefan Reimer 2024-02-09 16:24:37 +00:00
parent d2621ca2e6
commit 49e17c008f
15 changed files with 391 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
charts/kubezero-cert-manager/values.yaml
View File

@ -23,6 +23,9 @@ cert-manager:
leaderElection:
namespace: "cert-manager"
# remove secrets if the cert is deleted
enableCertificateOwnerRef: true
extraArgs:
- "--logging-format=json"
- "--leader-elect=false"

2
charts/kubezero-operators/README.md
View File

@ -32,7 +32,7 @@ Kubernetes: `>= 1.26.0`
| eck-operator.tolerations[0].effect | string | `"NoSchedule"` | |
| eck-operator.tolerations[0].key | string | `"node-role.kubernetes.io/control-plane"` | |
| opensearch-operator.enabled | bool | `false` | |
| opensearch-operator.fullnameOverride | string | `"telemetry"` | |
| opensearch-operator.fullnameOverride | string | `"opensearch-operator"` | |
| opensearch-operator.kubeRbacProxy.enable | bool | `false` | |
| opensearch-operator.nodeSelector."node-role.kubernetes.io/control-plane" | string | `""` | |
| opensearch-operator.tolerations[0].effect | string | `"NoSchedule"` | |

7
charts/kubezero-operators/values.yaml
View File

@ -2,12 +2,17 @@ opensearch-operator:
enabled: false
# otherwise service names will be >63 chars
fullnameOverride: telemetry
fullnameOverride: opensearch-operator
# not needed for now
kubeRbacProxy:
enable: false
manager:
extraEnv:
- name: SKIP_INIT_CONTAINER
value: "true"
tolerations:
- key: node-role.kubernetes.io/control-plane
effect: NoSchedule

14
charts/kubezero-telemetry/dashboards.yaml Normal file
View File

@ -0,0 +1,14 @@
configmap: grafana-dashboards
gzip: true
folder: Telemetry
dashboards:
- name: jaeger
url: https://grafana.com/api/dashboards/10001/revisions/2/download
tags:
- Jaeger
- Telemetry
- name: opensearch
url: https://grafana.com/api/dashboards/15178/revisions/2/download
tags:
- OpenSearch
- Telemetry

15
charts/kubezero-telemetry/templates/grafana-dashboards.yaml Normal file
View File

File diff suppressed because one or more lines are too long

28
charts/kubezero-telemetry/templates/jaeger/istio-authorization-policy.yaml Normal file
View File

@ -0,0 +1,28 @@
{{- if .Values.jaeger.istio.enabled }}
{{- if .Values.jaeger.istio.ipBlocks }}
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: jaeger-deny-not-in-ipblocks
namespace: istio-system
labels:
{{- include "kubezero-lib.labels" . | nindent 4 }}
spec:
selector:
matchLabels:
app: istio-ingressgateway
action: DENY
rules:
- from:
- source:
notIpBlocks:
{{- toYaml .Values.jaeger.istio.ipBlocks | nindent 8 }}
to:
- operation:
hosts: [{{ .Values.jaeger.istio.url }}]
when:
- key: connection.sni
values:
- '*'
{{- end }}
{{- end }}

2
charts/kubezero-telemetry/templates/jaeger/istio-service.yaml
View File

@ -16,5 +16,5 @@ spec:
- destination:
host: {{ .Release.Name }}-jaeger-query
port:
number: 16686
number: 80
{{- end }}

70
charts/kubezero-telemetry/templates/opensearch/certificates.yaml Normal file
View File

@ -0,0 +1,70 @@
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: {{ template "kubezero-lib.fullname" . }}-nodes-transport
namespace: {{ .Release.Namespace }}
labels:
{{ include "kubezero-lib.labels" . | nindent 4 }}
spec:
secretName: {{ template "kubezero-lib.fullname" . }}-nodes-transport-tls
issuerRef:
name: kubezero-local-ca-issuer
kind: ClusterIssuer
duration: 8760h0m0s
privateKey:
encoding: PKCS8
usages:
- "client auth"
- "server auth"
commonName: {{ template "kubezero-lib.fullname" . }}-nodes
dnsNames:
# <cluster-name>-<nodepool-component>-<index>
- '{{ template "kubezero-lib.fullname" . }}-nodes'
- '{{ template "kubezero-lib.fullname" . }}-nodes-*'
- '{{ template "kubezero-lib.fullname" . }}-bootstrap-0'
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: {{ template "kubezero-lib.fullname" . }}-nodes-http
namespace: {{ .Release.Namespace }}
labels:
{{ include "kubezero-lib.labels" . | nindent 4 }}
spec:
secretName: {{ template "kubezero-lib.fullname" . }}-nodes-http-tls
issuerRef:
name: kubezero-local-ca-issuer
kind: ClusterIssuer
duration: 8760h0m0s
privateKey:
encoding: PKCS8
usages:
- "client auth"
- "server auth"
commonName: {{ template "kubezero-lib.fullname" . }}
dnsNames:
# <cluster-name>, <cluster-name>.<namespace>, <cluster-name>.<namespace>.svc,<cluster-name>.<namespace>.svc.cluster.local
- '{{ template "kubezero-lib.fullname" . }}'
- '{{ template "kubezero-lib.fullname" . }}.{{ .Release.Namespace }}.svc'
- '{{ template "kubezero-lib.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local'
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: {{ template "kubezero-lib.fullname" . }}-admin
namespace: {{ .Release.Namespace }}
labels:
{{ include "kubezero-lib.labels" . | nindent 4 }}
spec:
secretName: {{ template "kubezero-lib.fullname" . }}-admin-tls
issuerRef:
name: kubezero-local-ca-issuer
kind: ClusterIssuer
duration: 8760h0m0s
usages:
- "client auth"
commonName: {{ template "kubezero-lib.fullname" . }}-admin
privateKey:
encoding: PKCS8

76
charts/kubezero-telemetry/templates/opensearch/cluster.yaml
View File

@ -1,3 +1,4 @@
#pluginsList: ["repository-s3","https://github.com/aiven/prometheus-exporter-plugin-for-opensearch/releases/download/2.11.1.0/prometheus-exporter-2.11.1.0.zip"]
{{- if .Values.opensearch.nodeSets }}
apiVersion: opensearch.opster.io/v1
kind: OpenSearchCluster
@ -9,31 +10,76 @@ metadata:
spec:
general:
serviceName: {{ template "kubezero-lib.fullname" . }}
version: 2.11.0
version: {{ .Values.opensearch.version }}
setVMMaxMapCount: false
pluginsList: ["repository-s3"]
monitoring:
enable: {{ .Values.opensearch.prometheus }}
tlsConfig:
insecureSkipVerify: true
{{- if .Values.opensearch.dashboard.enabled }}
# https://github.com/opensearch-project/OpenSearch-Dashboards/blob/main/config/opensearch_dashboards.yml
dashboards:
enable: true
version: 2.11.0
version: {{ .Values.opensearch.version }}
replicas: 1
resources:
requests:
memory: "512Mi"
cpu: "200m"
limits:
memory: "512Mi"
cpu: "200m"
memory: "1Gi"
#cpu: "200m"
{{- end }}
nodePools:
- component: nodes
replicas: 2
diskSize: "16Gi"
nodeSelector:
resources:
requests:
memory: "2Gi"
cpu: "500m"
limits:
memory: "2Gi"
cpu: "500m"
{{- range .Values.opensearch.nodeSets }}
- component: nodes-{{ .name }}
replicas: {{ .replicas }}
diskSize: {{ .storage.size }}
{{- with .storage.class }}
persistence:
pvc:
storageClass: {{ . }}
{{- end }}
{{- with .resources }}
resources: {{ toYaml . | nindent 8 }}
{{- end }}
roles:
- "cluster_manager"
- "data"
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
opster.io/opensearch-cluster: {{ template "kubezero-lib.fullname" $ }}
additionalConfig:
index.codec: zstd_no_dict
indices.time_series_index.default_index_merge_policy: log_byte_size
{{- with .zone }}
cluster.routing.allocation.awareness.attributes: k8s_node_name,zone
node.attr.zone: {{ . }}
{{- end }}
{{- end }}
security:
config:
adminSecret:
name: {{ template "kubezero-lib.fullname" . }}-admin-tls
tls:
transport:
generate: false
perNode: false
secret:
name: {{ template "kubezero-lib.fullname" . }}-nodes-transport-tls
nodesDn:
- 'CN={{ template "kubezero-lib.fullname" . }}-nodes'
- 'CN={{ template "kubezero-lib.fullname" . }}-nodes-*'
- 'CN={{ template "kubezero-lib.fullname" . }}-bootstrap-0'
adminDn:
- 'CN={{ template "kubezero-lib.fullname" . }}-admin'
http:
generate: false
secret:
name: {{ template "kubezero-lib.fullname" . }}-nodes-http-tls
{{- end }}

28
charts/kubezero-telemetry/templates/opensearch/istio-authorization-policy.yaml Normal file
View File

@ -0,0 +1,28 @@
{{- if .Values.opensearch.dashboard.istio.enabled }}
{{- if .Values.opensearch.dashboard.istio.ipBlocks }}
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
name: telemetry-dashboard-deny-not-in-ipblocks
namespace: istio-system
labels:
{{- include "kubezero-lib.labels" . | nindent 4 }}
spec:
selector:
matchLabels:
app: istio-ingressgateway
action: DENY
rules:
- from:
- source:
notIpBlocks:
{{- toYaml .Values.opensearch.dashboard.istio.ipBlocks | nindent 8 }}
to:
- operation:
hosts: [{{ .Values.opensearch.dashboard.istio.url }}]
when:
- key: connection.sni
values:
- '*'
{{- end }}
{{- end }}

20
charts/kubezero-telemetry/templates/opensearch/istio-virtualservice.yaml Normal file
View File

@ -0,0 +1,20 @@
{{- if .Values.opensearch.dashboard.istio.enabled }}
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
name: {{ template "kubezero-lib.fullname" . }}-kibana
namespace: {{ .Release.Namespace }}
labels:
{{ include "kubezero-lib.labels" . | indent 4 }}
spec:
hosts:
- {{ .Values.opensearch.dashboard.istio.url }}
gateways:
- {{ default "istio-system/ingressgateway" .Values.opensearch.dashboard.istio.gateway }}
http:
- route:
- destination:
host: telemetry-dashboards
port:
number: 5601
{{- end }}

2
charts/kubezero-telemetry/update.sh
View File

@ -3,5 +3,7 @@ set -ex
. ../../scripts/lib-update.sh
../kubezero-metrics/sync_grafana_dashboards.py dashboards.yaml templates/grafana-dashboards.yaml
#login_ecr_public
update_helm

75
charts/kubezero-telemetry/values-nodes.yaml Normal file
View File

@ -0,0 +1,75 @@
opentelemetry-collector:
enabled: false
mode: deployment
jaeger:
enabled: false
agent:
enabled: false
collector:
service:
otlp:
grpc:
name: otlp-grpc
port: 4317
http:
name: otlp-http
port: 4318
serviceMonitor:
enabled: false
# https://www.jaegertracing.io/docs/1.53/deployment/#collector
storage:
type: elasticsearch
elasticsearch:
scheme: https
host: telemetry
user: admin
password: admin
cmdlineParams:
es.tls.enabled: ""
es.tls.skip-host-verify: ""
provisionDataStore:
cassandra: false
elasticsearch: false
query:
agentSidecar:
enabled: false
serviceMonitor:
enabled: false
istio:
enabled: false
gateway: istio-ingress/private-ingressgateway
url: jaeger.example.com
opensearch:
version: 2.11.1
prometheus: false
nodeSets:
- name: default
replicas: 2
storage:
size: 16Gi
class: my-fancy-SSDs
zone: us-west-2a
resources:
limits:
#cpu: 1
memory: 2Gi
requests:
cpu: 500m
memory: 2Gi
dashboard:
enabled: false
istio:
enabled: false
gateway: istio-ingress/private-ingressgateway
url: telemetry-dashboard.example.com

61
charts/kubezero-telemetry/values.yaml
View File

@ -6,29 +6,70 @@ opentelemetry-collector:
jaeger:
enabled: false
# allInOne:
# enabled: true
# storage:
# type: none
# collector:
# enabled: false
# query:
# enabled: false
agent:
enabled: false
collector:
service:
otlp:
grpc:
name: otlp-grpc
port: 4317
http:
name: otlp-http
port: 4318
serviceMonitor:
enabled: false
# https://www.jaegertracing.io/docs/1.53/deployment/#collector
storage:
type: elasticsearch
elasticsearch:
scheme: https
host: telemetry
user: admin
password: admin
cmdlineParams:
es.tls.enabled: ""
es.tls.skip-host-verify: ""
provisionDataStore:
cassandra: false
elasticsearch: false
query:
agentSidecar:
enabled: false
serviceMonitor:
enabled: false
istio:
enabled: false
gateway: istio-ingress/private-ingressgateway
url: jaeger.example.com
opensearch:
nodeSets: {}
version: 2.11.1
prometheus: false
nodeSets: []
#- name: default-nodes
# replicas: 2
# storage:
# size: 16Gi
# class: my-fancy-SSDs
# zone: us-west-2a
# resources:
# limits:
# #cpu: 1
# memory: 2Gi
# requests:
# cpu: 500m
# memory: 2Gi
dashboard:
enabled: false
istio:
enabled: false
gateway: istio-ingress/private-ingressgateway
url: telemetry-dashboard.example.com

28
charts/kubezero/templates/telemetry.yaml
View File

@ -5,6 +5,13 @@ jaeger:
{{- with .Values.telemetry.jaeger }}
{{- toYaml . | nindent 2 }}
{{- end }}
collector:
serviceMonitor:
enabled: {{ .Values.metrics.enabled }}
query:
serviceMonitor:
enabled: {{ .Values.metrics.enabled }}
{{- end }}
{{- if .Values.telemetry.opensearch }}
@ -12,24 +19,19 @@ opensearch:
{{- if .Values.telemetry.opensearch.nodeSets }}
nodeSets:
{{- with .Values.telemetry.opensearch.nodeSets }}
{{- toYaml . | nindent 2 }}
{{- end }}
{{- end }}
prometheus: {{ .Values.metrics.enabled }}
{{- if .Values.telemetry.opensearch.s3Snapshot }}
s3Snapshot:
{{- with .Values.telemetry.opensearch.s3Snapshot }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.telemetry.dashboard }}
dashboard:
{{- with .Values.telemetry.dashboard }}
{{- toYaml . | nindent 2 }}
{{- if .Values.telemetry.opensearch.dashboard }}
dashboard:
{{- with .Values.telemetry.opensearch.dashboard }}
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
prometheus: {{ .Values.metrics.enabled }}
{{- end }}
{{- end }}