2020-12-18 00:46:15 +00:00
{{- /*
2021-06-30 16:26:06 +00:00
Generated from 'node-exporter' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/node-exporter-prometheusRule.yaml
2020-12-18 00:46:15 +00:00
Do not change in-place! In order to change this file first read following link :
https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
*/ -}}
{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }}
{{- if and (semverCompare ">=1.14.0-0" $kubeTargetVersion) (semverCompare "<9.9.9-9" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.node }}
apiVersion : monitoring.coreos.com/v1
kind : PrometheusRule
metadata :
name : {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "node-exporter" | trunc 63 | trimSuffix "-" }}
namespace : {{ template "kube-prometheus-stack.namespace" . }}
labels :
app : {{ template "kube-prometheus-stack.name" . }}
{{ include "kube-prometheus-stack.labels" . | indent 4 }}
{{- if .Values.defaultRules.labels }}
{{ toYaml .Values.defaultRules.labels | indent 4 }}
{{- end }}
{{- if .Values.defaultRules.annotations }}
annotations :
{{ toYaml .Values.defaultRules.annotations | indent 4 }}
{{- end }}
spec :
groups :
- name : node-exporter
rules :
- alert : NodeFilesystemSpaceFillingUp
annotations :
description : Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up.
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemspacefillingup
2020-12-18 00:46:15 +00:00
summary : Filesystem is predicted to run out of space within the next 24 hours.
expr : |-
(
2021-04-15 13:03:52 +00:00
node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 25
2020-12-18 00:46:15 +00:00
and
predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0
and
node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
)
for : 1h
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeFilesystemSpaceFillingUp
annotations :
description : Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast.
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemspacefillingup
2020-12-18 00:46:15 +00:00
summary : Filesystem is predicted to run out of space within the next 4 hours.
expr : |-
(
2021-04-15 13:03:52 +00:00
node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 10
2020-12-18 00:46:15 +00:00
and
predict_linear(node_filesystem_avail_bytes{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0
and
node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
)
for : 1h
labels :
severity : critical
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeFilesystemAlmostOutOfSpace
annotations :
description : Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left.
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutofspace
2020-12-18 00:46:15 +00:00
summary : Filesystem has less than 5% space left.
expr : |-
(
node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 5
and
node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
)
for : 1h
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeFilesystemAlmostOutOfSpace
annotations :
description : Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left.
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutofspace
2020-12-18 00:46:15 +00:00
summary : Filesystem has less than 3% space left.
expr : |-
(
node_filesystem_avail_bytes{job="node-exporter",fstype!=""} / node_filesystem_size_bytes{job="node-exporter",fstype!=""} * 100 < 3
and
node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
)
for : 1h
labels :
severity : critical
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeFilesystemFilesFillingUp
annotations :
description : Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up.
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemfilesfillingup
2020-12-18 00:46:15 +00:00
summary : Filesystem is predicted to run out of inodes within the next 24 hours.
expr : |-
(
node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 40
and
predict_linear(node_filesystem_files_free{job="node-exporter",fstype!=""}[6h], 24*60*60) < 0
and
node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
)
for : 1h
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeFilesystemFilesFillingUp
annotations :
description : Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast.
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemfilesfillingup
2020-12-18 00:46:15 +00:00
summary : Filesystem is predicted to run out of inodes within the next 4 hours.
expr : |-
(
node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 20
and
predict_linear(node_filesystem_files_free{job="node-exporter",fstype!=""}[6h], 4*60*60) < 0
and
node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
)
for : 1h
labels :
severity : critical
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeFilesystemAlmostOutOfFiles
annotations :
description : Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left.
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutoffiles
2020-12-18 00:46:15 +00:00
summary : Filesystem has less than 5% inodes left.
expr : |-
(
node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 5
and
node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
)
for : 1h
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeFilesystemAlmostOutOfFiles
annotations :
description : Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left.
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodefilesystemalmostoutoffiles
2020-12-18 00:46:15 +00:00
summary : Filesystem has less than 3% inodes left.
expr : |-
(
node_filesystem_files_free{job="node-exporter",fstype!=""} / node_filesystem_files{job="node-exporter",fstype!=""} * 100 < 3
and
node_filesystem_readonly{job="node-exporter",fstype!=""} == 0
)
for : 1h
labels :
severity : critical
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeNetworkReceiveErrs
annotations :
description : '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.'
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworkreceiveerrs
2020-12-18 00:46:15 +00:00
summary : Network interface is reporting many receive errors.
expr : rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01
for : 1h
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeNetworkTransmitErrs
annotations :
description : '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.'
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodenetworktransmiterrs
2020-12-18 00:46:15 +00:00
summary : Network interface is reporting many transmit errors.
expr : rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01
for : 1h
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeHighNumberConntrackEntriesUsed
annotations :
description : '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.'
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodehighnumberconntrackentriesused
2020-12-18 00:46:15 +00:00
summary : Number of conntrack are getting close to the limit.
expr : (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeTextFileCollectorScrapeError
annotations :
description : Node Exporter text file collector failed to scrape.
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodetextfilecollectorscrapeerror
2020-12-18 00:46:15 +00:00
summary : Node Exporter text file collector failed to scrape.
expr : node_textfile_scrape_error{job="node-exporter"} == 1
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeClockSkewDetected
annotations :
2021-06-30 16:26:06 +00:00
description : Clock on {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 300s. Ensure NTP is configured correctly on this host.
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodeclockskewdetected
2020-12-18 00:46:15 +00:00
summary : Clock skew detected.
expr : |-
(
node_timex_offset_seconds > 0.05
and
deriv(node_timex_offset_seconds[5m]) >= 0
)
or
(
node_timex_offset_seconds < -0.05
and
deriv(node_timex_offset_seconds[5m]) <= 0
)
for : 10m
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeClockNotSynchronising
annotations :
2021-06-30 16:26:06 +00:00
description : Clock on {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host.
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-nodeclocknotsynchronising
2020-12-18 00:46:15 +00:00
summary : Clock not synchronising.
expr : |-
min_over_time(node_timex_sync_status[5m]) == 0
and
node_timex_maxerror_seconds >= 16
for : 10m
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeRAIDDegraded
annotations :
description : RAID array '{{`{{`}} $labels.device {{`}}`}}' on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-noderaiddegraded
2020-12-18 00:46:15 +00:00
summary : RAID Array is degraded
expr : node_md_disks_required - ignoring (state) (node_md_disks{state="active"}) > 0
for : 15m
labels :
severity : critical
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
- alert : NodeRAIDDiskFailure
annotations :
description : At least one device in RAID array on {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap.
2021-06-30 16:26:06 +00:00
runbook_url : {{ .Values.defaultRules.runbookUrl }}alert-name-noderaiddiskfailure
2020-12-18 00:46:15 +00:00
summary : Failed device in RAID array
2021-06-30 16:26:06 +00:00
expr : node_md_disks{state="failed"} > 0
2020-12-18 00:46:15 +00:00
labels :
severity : warning
{{- if .Values.defaultRules.additionalRuleLabels }}
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
{{- end }}
2021-07-01 11:36:35 +00:00
{{- end }}