151 lines
7.0 KiB
YAML
151 lines
7.0 KiB
YAML
{{- /*
|
|
Generated from 'etcd' group from https://raw.githubusercontent.com/etcd-io/website/master/content/docs/v3.4.0/op-guide/etcd3_alert.rules.yml
|
|
Do not change in-place! In order to change this file first read following link:
|
|
https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
|
|
*/ -}}
|
|
{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }}
|
|
{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.kubeEtcd.enabled .Values.defaultRules.rules.etcd }}
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "etcd" | trunc 63 | trimSuffix "-" }}
|
|
namespace: {{ template "kube-prometheus-stack.namespace" . }}
|
|
labels:
|
|
app: {{ template "kube-prometheus-stack.name" . }}
|
|
{{ include "kube-prometheus-stack.labels" . | indent 4 }}
|
|
{{- if .Values.defaultRules.labels }}
|
|
{{ toYaml .Values.defaultRules.labels | indent 4 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.annotations }}
|
|
annotations:
|
|
{{ toYaml .Values.defaultRules.annotations | indent 4 }}
|
|
{{- end }}
|
|
spec:
|
|
groups:
|
|
- name: etcd
|
|
rules:
|
|
- alert: etcdInsufficientMembers
|
|
annotations:
|
|
message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": insufficient members ({{`{{`}} $value {{`}}`}}).'
|
|
expr: sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"}) by (job) + 1) / 2)
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: etcdNoLeader
|
|
annotations:
|
|
message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member {{`{{`}} $labels.instance {{`}}`}} has no leader.'
|
|
expr: etcd_server_has_leader{job=~".*etcd.*"} == 0
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: etcdHighNumberOfLeaderChanges
|
|
annotations:
|
|
message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": instance {{`{{`}} $labels.instance {{`}}`}} has seen {{`{{`}} $value {{`}}`}} leader changes within the last hour.'
|
|
expr: rate(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}[15m]) > 3
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: etcdGRPCRequestsSlow
|
|
annotations:
|
|
message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": gRPC requests to {{`{{`}} $labels.grpc_method {{`}}`}} are taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
|
|
expr: |-
|
|
histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=~".*etcd.*", grpc_type="unary"}[5m])) by (job, instance, grpc_service, grpc_method, le))
|
|
> 0.15
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: etcdMemberCommunicationSlow
|
|
annotations:
|
|
message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": member communication with {{`{{`}} $labels.To {{`}}`}} is taking {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
|
|
expr: |-
|
|
histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket{job=~".*etcd.*"}[5m]))
|
|
> 0.15
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: etcdHighNumberOfFailedProposals
|
|
annotations:
|
|
message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": {{`{{`}} $value {{`}}`}} proposal failures within the last hour on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
|
|
expr: rate(etcd_server_proposals_failed_total{job=~".*etcd.*"}[15m]) > 5
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: etcdHighFsyncDurations
|
|
annotations:
|
|
message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile fync durations are {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
|
|
expr: |-
|
|
histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
|
|
> 0.5
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: etcdHighCommitDurations
|
|
annotations:
|
|
message: 'etcd cluster "{{`{{`}} $labels.job {{`}}`}}": 99th percentile commit durations {{`{{`}} $value {{`}}`}}s on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
|
|
expr: |-
|
|
histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
|
|
> 0.25
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: etcdHighNumberOfFailedHTTPRequests
|
|
annotations:
|
|
message: '{{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}'
|
|
expr: |-
|
|
sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m]))
|
|
BY (method) > 0.01
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: etcdHighNumberOfFailedHTTPRequests
|
|
annotations:
|
|
message: '{{`{{`}} $value {{`}}`}}% of requests for {{`{{`}} $labels.method {{`}}`}} failed on etcd instance {{`{{`}} $labels.instance {{`}}`}}.'
|
|
expr: |-
|
|
sum(rate(etcd_http_failed_total{job=~".*etcd.*", code!="404"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job=~".*etcd.*"}[5m]))
|
|
BY (method) > 0.05
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: etcdHTTPRequestsSlow
|
|
annotations:
|
|
message: etcd instance {{`{{`}} $labels.instance {{`}}`}} HTTP requests to {{`{{`}} $labels.method {{`}}`}} are slow.
|
|
expr: |-
|
|
histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m]))
|
|
> 0.15
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
{{- end }} |