306 lines
18 KiB
Plaintext
306 lines
18 KiB
Plaintext
{
|
|
"apiVersion": "monitoring.coreos.com/v1",
|
|
"kind": "PrometheusRule",
|
|
"metadata": {
|
|
"labels": {
|
|
"app.kubernetes.io/component": "exporter",
|
|
"app.kubernetes.io/name": "node-exporter",
|
|
"app.kubernetes.io/part-of": "kube-prometheus",
|
|
"app.kubernetes.io/version": "1.2.2",
|
|
"prometheus": "k8s",
|
|
"role": "alert-rules"
|
|
},
|
|
"name": "node-exporter-rules",
|
|
"namespace": "monitoring"
|
|
},
|
|
"spec": {
|
|
"groups": [
|
|
{
|
|
"name": "node-exporter",
|
|
"rules": [
|
|
{
|
|
"alert": "NodeFilesystemSpaceFillingUp",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left and is filling up.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup",
|
|
"summary": "Filesystem is predicted to run out of space within the next 24 hours."
|
|
},
|
|
"expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 40\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemSpaceFillingUp",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left and is filling up fast.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemspacefillingup",
|
|
"summary": "Filesystem is predicted to run out of space within the next 4 hours."
|
|
},
|
|
"expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 15\nand\n predict_linear(node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"}[6h], 4*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemAlmostOutOfSpace",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace",
|
|
"summary": "Filesystem has less than 5% space left."
|
|
},
|
|
"expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 5\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
|
|
"for": "30m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemAlmostOutOfSpace",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available space left.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutofspace",
|
|
"summary": "Filesystem has less than 3% space left."
|
|
},
|
|
"expr": "(\n node_filesystem_avail_bytes{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_size_bytes{job=\"node-exporter\",fstype!=\"\"} * 100 < 3\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
|
|
"for": "30m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemFilesFillingUp",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left and is filling up.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup",
|
|
"summary": "Filesystem is predicted to run out of inodes within the next 24 hours."
|
|
},
|
|
"expr": "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\"} * 100 < 40\nand\n predict_linear(node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"}[6h], 24*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemFilesFillingUp",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left and is filling up fast.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemfilesfillingup",
|
|
"summary": "Filesystem is predicted to run out of inodes within the next 4 hours."
|
|
},
|
|
"expr": "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\"} * 100 < 20\nand\n predict_linear(node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"}[6h], 4*60*60) < 0\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemAlmostOutOfFiles",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles",
|
|
"summary": "Filesystem has less than 5% inodes left."
|
|
},
|
|
"expr": "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\"} * 100 < 5\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFilesystemAlmostOutOfFiles",
|
|
"annotations": {
|
|
"description": "Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf \"%.2f\" $value }}% available inodes left.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefilesystemalmostoutoffiles",
|
|
"summary": "Filesystem has less than 3% inodes left."
|
|
},
|
|
"expr": "(\n node_filesystem_files_free{job=\"node-exporter\",fstype!=\"\"} / node_filesystem_files{job=\"node-exporter\",fstype!=\"\"} * 100 < 3\nand\n node_filesystem_readonly{job=\"node-exporter\",fstype!=\"\"} == 0\n)\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeNetworkReceiveErrs",
|
|
"annotations": {
|
|
"description": "{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworkreceiveerrs",
|
|
"summary": "Network interface is reporting many receive errors."
|
|
},
|
|
"expr": "rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeNetworkTransmitErrs",
|
|
"annotations": {
|
|
"description": "{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodenetworktransmiterrs",
|
|
"summary": "Network interface is reporting many transmit errors."
|
|
},
|
|
"expr": "rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01\n",
|
|
"for": "1h",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeHighNumberConntrackEntriesUsed",
|
|
"annotations": {
|
|
"description": "{{ $value | humanizePercentage }} of conntrack entries are used.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodehighnumberconntrackentriesused",
|
|
"summary": "Number of conntrack are getting close to the limit."
|
|
},
|
|
"expr": "(node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75\n",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeTextFileCollectorScrapeError",
|
|
"annotations": {
|
|
"description": "Node Exporter text file collector failed to scrape.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodetextfilecollectorscrapeerror",
|
|
"summary": "Node Exporter text file collector failed to scrape."
|
|
},
|
|
"expr": "node_textfile_scrape_error{job=\"node-exporter\"} == 1\n",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeClockSkewDetected",
|
|
"annotations": {
|
|
"description": "Clock on {{ $labels.instance }} is out of sync by more than 300s. Ensure NTP is configured correctly on this host.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodeclockskewdetected",
|
|
"summary": "Clock skew detected."
|
|
},
|
|
"expr": "(\n node_timex_offset_seconds > 0.05\nand\n deriv(node_timex_offset_seconds[5m]) >= 0\n)\nor\n(\n node_timex_offset_seconds < -0.05\nand\n deriv(node_timex_offset_seconds[5m]) <= 0\n)\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeClockNotSynchronising",
|
|
"annotations": {
|
|
"description": "Clock on {{ $labels.instance }} is not synchronising. Ensure NTP is configured on this host.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodeclocknotsynchronising",
|
|
"summary": "Clock not synchronising."
|
|
},
|
|
"expr": "min_over_time(node_timex_sync_status[5m]) == 0\nand\nnode_timex_maxerror_seconds >= 16\n",
|
|
"for": "10m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeRAIDDegraded",
|
|
"annotations": {
|
|
"description": "RAID array '{{ $labels.device }}' on {{ $labels.instance }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddegraded",
|
|
"summary": "RAID Array is degraded"
|
|
},
|
|
"expr": "node_md_disks_required - ignoring (state) (node_md_disks{state=\"active\"}) > 0\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeRAIDDiskFailure",
|
|
"annotations": {
|
|
"description": "At least one device in RAID array on {{ $labels.instance }} failed. Array '{{ $labels.device }}' needs attention and possibly a disk swap.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/noderaiddiskfailure",
|
|
"summary": "Failed device in RAID array"
|
|
},
|
|
"expr": "node_md_disks{state=\"failed\"} > 0\n",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFileDescriptorLimit",
|
|
"annotations": {
|
|
"description": "File descriptors limit at {{ $labels.instance }} is currently at {{ printf \"%.2f\" $value }}%.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit",
|
|
"summary": "Kernel is predicted to exhaust file descriptors limit soon."
|
|
},
|
|
"expr": "(\n node_filefd_allocated{job=\"node-exporter\"} * 100 / node_filefd_maximum{job=\"node-exporter\"} > 70\n)\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "warning"
|
|
}
|
|
},
|
|
{
|
|
"alert": "NodeFileDescriptorLimit",
|
|
"annotations": {
|
|
"description": "File descriptors limit at {{ $labels.instance }} is currently at {{ printf \"%.2f\" $value }}%.",
|
|
"runbook_url": "https://runbooks.prometheus-operator.dev/runbooks/node/nodefiledescriptorlimit",
|
|
"summary": "Kernel is predicted to exhaust file descriptors limit soon."
|
|
},
|
|
"expr": "(\n node_filefd_allocated{job=\"node-exporter\"} * 100 / node_filefd_maximum{job=\"node-exporter\"} > 90\n)\n",
|
|
"for": "15m",
|
|
"labels": {
|
|
"severity": "critical"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "node-exporter.rules",
|
|
"rules": [
|
|
{
|
|
"expr": "count without (cpu) (\n count without (mode) (\n node_cpu_seconds_total{job=\"node-exporter\"}\n )\n)\n",
|
|
"record": "instance:node_num_cpu:sum"
|
|
},
|
|
{
|
|
"expr": "1 - avg without (cpu, mode) (\n rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\"}[5m])\n)\n",
|
|
"record": "instance:node_cpu_utilisation:rate5m"
|
|
},
|
|
{
|
|
"expr": "(\n node_load1{job=\"node-exporter\"}\n/\n instance:node_num_cpu:sum{job=\"node-exporter\"}\n)\n",
|
|
"record": "instance:node_load1_per_cpu:ratio"
|
|
},
|
|
{
|
|
"expr": "1 - (\n node_memory_MemAvailable_bytes{job=\"node-exporter\"}\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\"}\n)\n",
|
|
"record": "instance:node_memory_utilisation:ratio"
|
|
},
|
|
{
|
|
"expr": "rate(node_vmstat_pgmajfault{job=\"node-exporter\"}[5m])\n",
|
|
"record": "instance:node_vmstat_pgmajfault:rate5m"
|
|
},
|
|
{
|
|
"expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[5m])\n",
|
|
"record": "instance_device:node_disk_io_time_seconds:rate5m"
|
|
},
|
|
{
|
|
"expr": "rate(node_disk_io_time_weighted_seconds_total{job=\"node-exporter\", device=~\"mmcblk.p.+|nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+\"}[5m])\n",
|
|
"record": "instance_device:node_disk_io_time_weighted_seconds:rate5m"
|
|
},
|
|
{
|
|
"expr": "sum without (device) (\n rate(node_network_receive_bytes_total{job=\"node-exporter\", device!=\"lo\"}[5m])\n)\n",
|
|
"record": "instance:node_network_receive_bytes_excluding_lo:rate5m"
|
|
},
|
|
{
|
|
"expr": "sum without (device) (\n rate(node_network_transmit_bytes_total{job=\"node-exporter\", device!=\"lo\"}[5m])\n)\n",
|
|
"record": "instance:node_network_transmit_bytes_excluding_lo:rate5m"
|
|
},
|
|
{
|
|
"expr": "sum without (device) (\n rate(node_network_receive_drop_total{job=\"node-exporter\", device!=\"lo\"}[5m])\n)\n",
|
|
"record": "instance:node_network_receive_drop_excluding_lo:rate5m"
|
|
},
|
|
{
|
|
"expr": "sum without (device) (\n rate(node_network_transmit_drop_total{job=\"node-exporter\", device!=\"lo\"}[5m])\n)\n",
|
|
"record": "instance:node_network_transmit_drop_excluding_lo:rate5m"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|