2020-12-18 00:46:15 +00:00
{{- /*
2021-02-26 21:25:43 +00:00
Generated from 'kubernetes-system-kubelet' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/master/manifests/kubernetes-prometheusRule.yaml
2020-12-18 00:46:15 +00:00
Do not change in-place! In order to change this file first read following link :
https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
*/ -}}
{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }}
{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesSystem }}
apiVersion : monitoring.coreos.com/v1
kind : PrometheusRule
metadata :
name : {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-system-kubelet" | trunc 63 | trimSuffix "-" }}
namespace : {{ template "kube-prometheus-stack.namespace" . }}
labels :
app : {{ template "kube-prometheus-stack.name" . }}
{{ include "kube-prometheus-stack.labels" . | indent 4 }}
{{- if .Values.defaultRules.labels }}
{{ toYaml .Values.defaultRules.labels | indent 4 }}
{{- end }}
{{- if .Values.defaultRules.annotations }}
annotations :
{{ toYaml .Values.defaultRules.annotations | indent 4 }}
{{- end }}
spec :
groups :
- name : kubernetes-system-kubelet
rules :
- alert : KubeNodeNotReady
annotations :
description : '{{`{{`}} $labels.node {{`}}`}} has been unready for more than 15 minutes.'
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodenotready
summary : Node is not ready.
expr : kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
for : 15m
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : KubeNodeUnreachable
annotations :
description : '{{`{{`}} $labels.node {{`}}`}} is unreachable and some workloads may be rescheduled.'
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodeunreachable
summary : Node is unreachable.
expr : (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1
for : 15m
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : KubeletTooManyPods
annotations :
description : Kubelet '{{`{{`}} $labels.node {{`}}`}}' is running at {{`{{`}} $value | humanizePercentage {{`}}`}} of its Pod capacity.
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubelettoomanypods
summary : Kubelet is running at capacity.
expr : |-
count by(node) (
(kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
)
/
max by(node) (
2021-05-11 08:07:38 +00:00
kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1
2020-12-18 00:46:15 +00:00
) > 0.95
for : 15m
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : KubeNodeReadinessFlapping
annotations :
description : The readiness status of node {{`{{`}} $labels.node {{`}}`}} has changed {{`{{`}} $value {{`}}`}} times in the last 15 minutes.
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubenodereadinessflapping
summary : Node readiness status is flapping.
expr : sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2
for : 15m
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : KubeletPlegDurationHigh
annotations :
description : The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}.
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletplegdurationhigh
summary : Kubelet Pod Lifecycle Event Generator is taking too long to relist.
expr : node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
for : 5m
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : KubeletPodStartUpLatencyHigh
annotations :
description : Kubelet Pod startup 99th percentile latency is {{`{{`}} $value {{`}}`}} seconds on node {{`{{`}} $labels.node {{`}}`}}.
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletpodstartuplatencyhigh
summary : Kubelet Pod startup latency is too high.
expr : histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60
for : 15m
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : KubeletClientCertificateExpiration
annotations :
description : Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}.
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletclientcertificateexpiration
summary : Kubelet client certificate is about to expire.
expr : kubelet_certificate_manager_client_ttl_seconds < 604800
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : KubeletClientCertificateExpiration
annotations :
description : Client certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}.
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletclientcertificateexpiration
summary : Kubelet client certificate is about to expire.
expr : kubelet_certificate_manager_client_ttl_seconds < 86400
labels :
severity : critical
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : KubeletServerCertificateExpiration
annotations :
description : Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}.
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletservercertificateexpiration
summary : Kubelet server certificate is about to expire.
expr : kubelet_certificate_manager_server_ttl_seconds < 604800
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : KubeletServerCertificateExpiration
annotations :
description : Server certificate for Kubelet on node {{`{{`}} $labels.node {{`}}`}} expires in {{`{{`}} $value | humanizeDuration {{`}}`}}.
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletservercertificateexpiration
summary : Kubelet server certificate is about to expire.
expr : kubelet_certificate_manager_server_ttl_seconds < 86400
labels :
severity : critical
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : KubeletClientCertificateRenewalErrors
annotations :
description : Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its client certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes).
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletclientcertificaterenewalerrors
summary : Kubelet has failed to renew its client certificate.
expr : increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0
for : 15m
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : KubeletServerCertificateRenewalErrors
annotations :
description : Kubelet on node {{`{{`}} $labels.node {{`}}`}} has failed to renew its server certificate ({{`{{`}} $value | humanize {{`}}`}} errors in the last 5 minutes).
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletservercertificaterenewalerrors
summary : Kubelet has failed to renew its server certificate.
expr : increase(kubelet_server_expiration_renew_errors[5m]) > 0
for : 15m
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
{{- if .Values.prometheusOperator.kubeletService.enabled }}
- alert : KubeletDown
annotations :
description : Kubelet has disappeared from Prometheus target discovery.
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-kubeletdown
summary : Target disappeared from Prometheus target discovery.
expr : absent(up{job="kubelet", metrics_path="/metrics"} == 1)
for : 15m
labels :
severity : critical
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
{{- end }}
{{- end }}