274 lines
16 KiB
YAML
274 lines
16 KiB
YAML
|
apiVersion: monitoring.coreos.com/v1
|
||
|
|
kind: PrometheusRule
|
||
|
|
metadata:
|
||
|
|
name: {{ printf "%s-%s" (include "kubezero-lib.fullname" $) "node-exporter" | trunc 63 | trimSuffix "-" }}
|
||
|
|
namespace: {{ .Release.Namespace }}
|
||
|
|
labels:
|
||
|
|
{{- include "kubezero-lib.labels" . | nindent 4 }}
|
||
|
|
spec:
|
||
|
|
groups:
|
||
|
|
- name: node-exporter
|
||
|
|
rules:
|
||
|
|
- alert: NodeFilesystemSpaceFillingUp
|
||
|
|
annotations:
|
||
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
|
||
|
|
summary: Filesystem is predicted to run out of space within the next 24 hours.
|
||
|
|
expr: "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 20\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n"
|
||
|
|
for: 1h
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: warning
|
||
|
|
- alert: NodeFilesystemSpaceFillingUp
|
||
|
|
annotations:
|
||
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup
|
||
|
|
summary: Filesystem is predicted to run out of space within the next 4 hours.
|
||
|
|
expr: "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 15\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 4*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n"
|
||
|
|
for: 1h
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: critical
|
||
|
|
- alert: NodeFilesystemAlmostOutOfSpace
|
||
|
|
annotations:
|
||
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
|
||
|
|
summary: Filesystem has less than 5% space left.
|
||
|
|
expr: "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 5\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n"
|
||
|
|
for: 30m
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: warning
|
||
|
|
- alert: NodeFilesystemAlmostOutOfSpace
|
||
|
|
annotations:
|
||
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace
|
||
|
|
summary: Filesystem has less than 3% space left.
|
||
|
|
expr: "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 3\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n"
|
||
|
|
for: 30m
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: critical
|
||
|
|
- alert: NodeFilesystemFilesFillingUp
|
||
|
|
annotations:
|
||
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
|
||
|
|
summary: Filesystem is predicted to run out of inodes within the next 24 hours.
|
||
|
|
expr: "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\"} * 100 < 40\nand\n predict_linear(node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n"
|
||
|
|
for: 1h
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: warning
|
||
|
|
- alert: NodeFilesystemFilesFillingUp
|
||
|
|
annotations:
|
||
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup
|
||
|
|
summary: Filesystem is predicted to run out of inodes within the next 4 hours.
|
||
|
|
expr: "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\"} * 100 < 20\nand\n predict_linear(node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"}[6h], 4*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n"
|
||
|
|
for: 1h
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: critical
|
||
|
|
- alert: NodeFilesystemAlmostOutOfFiles
|
||
|
|
annotations:
|
||
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
|
||
|
|
summary: Filesystem has less than 5% inodes left.
|
||
|
|
expr: "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\"} * 100 < 5\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n"
|
||
|
|
for: 1h
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: warning
|
||
|
|
- alert: NodeFilesystemAlmostOutOfFiles
|
||
|
|
annotations:
|
||
|
|
description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles
|
||
|
|
summary: Filesystem has less than 3% inodes left.
|
||
|
|
expr: "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\"} * 100 < 3\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n"
|
||
|
|
for: 1h
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: critical
|
||
|
|
- alert: NodeNetworkReceiveErrs
|
||
|
|
annotations:
|
||
|
|
description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.'
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworkreceiveerrs
|
||
|
|
summary: Network interface is reporting many receive errors.
|
||
|
|
expr: 'rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01
|
||
|
|
|
||
|
|
'
|
||
|
|
for: 1h
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: warning
|
||
|
|
- alert: NodeNetworkTransmitErrs
|
||
|
|
annotations:
|
||
|
|
description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.'
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworktransmiterrs
|
||
|
|
summary: Network interface is reporting many transmit errors.
|
||
|
|
expr: 'rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01
|
||
|
|
|
||
|
|
'
|
||
|
|
for: 1h
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: warning
|
||
|
|
- alert: NodeHighNumberConntrackEntriesUsed
|
||
|
|
annotations:
|
||
|
|
description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.'
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodehighnumberconntrackentriesused
|
||
|
|
summary: Number of conntrack are getting close to the limit.
|
||
|
|
expr: '(node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75
|
||
|
|
|
||
|
|
'
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: warning
|
||
|
|
- alert: NodeTextFileCollectorScrapeError
|
||
|
|
annotations:
|
||
|
|
description: Node Exporter text file collector failed to scrape.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodetextfilecollectorscrapeerror
|
||
|
|
summary: Node Exporter text file collector failed to scrape.
|
||
|
|
expr: 'node_textfile_scrape_error{job="node-exporter"} == 1
|
||
|
|
|
||
|
|
'
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: warning
|
||
|
|
- alert: NodeClockSkewDetected
|
||
|
|
annotations:
|
||
|
|
description: Clock on {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 300s. Ensure NTP is configured correctly on this host.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclockskewdetected
|
||
|
|
summary: Clock skew detected.
|
||
|
|
expr: "(\n node_timex_offset_seconds > 0.05\nand\n deriv(node_timex_offset_seconds[5m]) >= 0\n)\nor\n(\n node_timex_offset_seconds < -0.05\nand\n deriv(node_timex_offset_seconds[5m]) <= 0\n)\n"
|
||
|
|
for: 10m
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: warning
|
||
|
|
- alert: NodeClockNotSynchronising
|
||
|
|
annotations:
|
||
|
|
description: Clock on {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodeclocknotsynchronising
|
||
|
|
summary: Clock not synchronising.
|
||
|
|
expr: 'min_over_time(node_timex_sync_status[5m]) == 0
|
||
|
|
|
||
|
|
and
|
||
|
|
|
||
|
|
node_timex_maxerror_seconds >= 16
|
||
|
|
|
||
|
|
'
|
||
|
|
for: 10m
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: warning
|
||
|
|
- alert: NodeRAIDDegraded
|
||
|
|
annotations:
|
||
|
|
description: RAID array '{{`{{`}} $labels.device {{`}}`}}' on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddegraded
|
||
|
|
summary: RAID Array is degraded
|
||
|
|
expr: 'node_md_disks_required - ignoring (state) (node_md_disks{state="active"}) > 0
|
||
|
|
|
||
|
|
'
|
||
|
|
for: 15m
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: critical
|
||
|
|
- alert: NodeRAIDDiskFailure
|
||
|
|
annotations:
|
||
|
|
description: At least one device in RAID array on {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddiskfailure
|
||
|
|
summary: Failed device in RAID array
|
||
|
|
expr: 'node_md_disks{state="failed"} > 0
|
||
|
|
|
||
|
|
'
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: warning
|
||
|
|
- alert: NodeFileDescriptorLimit
|
||
|
|
annotations:
|
||
|
|
description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
|
||
|
|
summary: Kernel is predicted to exhaust file descriptors limit soon.
|
||
|
|
expr: "(\n node_filefd_allocated{job=\"node-exporter\"} * 100 / node_filefd_maximum{job=\"node-exporter\"} > 70\n)\n"
|
||
|
|
for: 15m
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: warning
|
||
|
|
- alert: NodeFileDescriptorLimit
|
||
|
|
annotations:
|
||
|
|
description: File descriptors limit at {{`{{`}} $labels.instance {{`}}`}} is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%.
|
||
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit
|
||
|
|
summary: Kernel is predicted to exhaust file descriptors limit soon.
|
||
|
|
expr: "(\n node_filefd_allocated{job=\"node-exporter\"} * 100 / node_filefd_maximum{job=\"node-exporter\"} > 90\n)\n"
|
||
|
|
for: 15m
|
||
|
|
labels:
|
||
|
|
{{- if .Values.additionalRuleLabels }}
|
||
|
|
{{- toYaml .Values.additionalRuleLabels | nindent 8 }}
|
||
|
|
{{- end }}
|
||
|
|
severity: critical
|
||
|
|
- name: node-exporter.rules
|
||
|
|
rules:
|
||
|
|
- expr: "count without (cpu, mode) (\n node_cpu_seconds_total{job=\"node-exporter\",mode=\"idle\"}\n)\n"
|
||
|
|
record: instance:node_num_cpu:sum
|
||
|
|
- expr: "1 - avg without (cpu, mode) (\n rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\"}[5m])\n)\n"
|
||
|
|
record: instance:node_cpu_utilisation:rate5m
|
||
|
|
- expr: "(\n node_load1{job=\"node-exporter\"}\n/\n instance:node_num_cpu:sum{job=\"node-exporter\"}\n)\n"
|
||
|
|
record: instance:node_load1_per_cpu:ratio
|
||
|
|
- expr: "1 - (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n or\n (\n node_memory_Buffers_bytes{job=\"node-exporter\"}\n +\n node_memory_Cached_bytes{job=\"node-exporter\"}\n +\n node_memory_MemFree_bytes{job=\"node-exporter\"}\n +\n node_memory_Slab_bytes{job=\"node-exporter\"}\n )\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n"
|
||
|
|
record: instance:node_memory_utilisation:ratio
|
||
|
|
- expr: 'rate(node_vmstat_pgmajfault{job="node-exporter"}[5m])
|
||
|
|
|
||
|
|
'
|
||
|
|
record: instance:node_vmstat_pgmajfault:rate5m
|
||
|
|
- expr: 'rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[5m])
|
||
|
|
|
||
|
|
'
|
||
|
|
record: instance_device:node_disk_io_time_seconds:rate5m
|
||
|
|
- expr: 'rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[5m])
|
||
|
|
|
||
|
|
'
|
||
|
|
record: instance_device:node_disk_io_time_weighted_seconds:rate5m
|
||
|
|
- expr: "sum without (device) (\n rate(node_network_receive_bytes_total{job=\"node-exporter\", device!=\"lo\"}[5m])\n)\n"
|
||
|
|
record: instance:node_network_receive_bytes_excluding_lo:rate5m
|
||
|
|
- expr: "sum without (device) (\n rate(node_network_transmit_bytes_total{job=\"node-exporter\", device!=\"lo\"}[5m])\n)\n"
|
||
|
|
record: instance:node_network_transmit_bytes_excluding_lo:rate5m
|
||
|
|
- expr: "sum without (device) (\n rate(node_network_receive_drop_total{job=\"node-exporter\", device!=\"lo\"}[5m])\n)\n"
|
||
|
|
record: instance:node_network_receive_drop_excluding_lo:rate5m
|
||
|
|
- expr: "sum without (device) (\n rate(node_network_transmit_drop_total{job=\"node-exporter\", device!=\"lo\"}[5m])\n)\n"
|
||
|
|
record: instance:node_network_transmit_drop_excluding_lo:rate5m
|
||
|
|
|