686 lines
34 KiB
YAML
686 lines
34 KiB
YAML
|
"groups":
|
||
|
|
- "name": "kubernetes-apps"
|
||
|
|
"rules":
|
||
|
|
- "alert": "KubePodCrashLooping"
|
||
|
|
"annotations":
|
||
|
|
"description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf \"%.2f\" $value }} times / 10 minutes."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping"
|
||
|
|
"summary": "Pod is crash looping."
|
||
|
|
"expr": |
|
||
|
|
increase(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[10m]) > 0
|
||
|
|
and
|
||
|
|
kube_pod_container_status_waiting{job="kube-state-metrics"} == 1
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubePodNotReady"
|
||
|
|
"annotations":
|
||
|
|
"description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready"
|
||
|
|
"summary": "Pod has been in a non-ready state for more than 15 minutes."
|
||
|
|
"expr": |
|
||
|
|
sum by (namespace, pod) (
|
||
|
|
max by(namespace, pod) (
|
||
|
|
kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown"}
|
||
|
|
) * on(namespace, pod) group_left(owner_kind) topk by(namespace, pod) (
|
||
|
|
1, max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})
|
||
|
|
)
|
||
|
|
) > 0
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeDeploymentGenerationMismatch"
|
||
|
|
"annotations":
|
||
|
|
"description": "Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch"
|
||
|
|
"summary": "Deployment generation mismatch due to possible roll-back"
|
||
|
|
"expr": |
|
||
|
|
kube_deployment_status_observed_generation{job="kube-state-metrics"}
|
||
|
|
!=
|
||
|
|
kube_deployment_metadata_generation{job="kube-state-metrics"}
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeDeploymentReplicasMismatch"
|
||
|
|
"annotations":
|
||
|
|
"description": "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch"
|
||
|
|
"summary": "Deployment has not matched the expected number of replicas."
|
||
|
|
"expr": |
|
||
|
|
(
|
||
|
|
kube_deployment_spec_replicas{job="kube-state-metrics"}
|
||
|
|
>
|
||
|
|
kube_deployment_status_replicas_available{job="kube-state-metrics"}
|
||
|
|
) and (
|
||
|
|
changes(kube_deployment_status_replicas_updated{job="kube-state-metrics"}[10m])
|
||
|
|
==
|
||
|
|
0
|
||
|
|
)
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeStatefulSetReplicasMismatch"
|
||
|
|
"annotations":
|
||
|
|
"description": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch"
|
||
|
|
"summary": "Deployment has not matched the expected number of replicas."
|
||
|
|
"expr": |
|
||
|
|
(
|
||
|
|
kube_statefulset_status_replicas_ready{job="kube-state-metrics"}
|
||
|
|
!=
|
||
|
|
kube_statefulset_status_replicas{job="kube-state-metrics"}
|
||
|
|
) and (
|
||
|
|
changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[10m])
|
||
|
|
==
|
||
|
|
0
|
||
|
|
)
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeStatefulSetGenerationMismatch"
|
||
|
|
"annotations":
|
||
|
|
"description": "StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch"
|
||
|
|
"summary": "StatefulSet generation mismatch due to possible roll-back"
|
||
|
|
"expr": |
|
||
|
|
kube_statefulset_status_observed_generation{job="kube-state-metrics"}
|
||
|
|
!=
|
||
|
|
kube_statefulset_metadata_generation{job="kube-state-metrics"}
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeStatefulSetUpdateNotRolledOut"
|
||
|
|
"annotations":
|
||
|
|
"description": "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout"
|
||
|
|
"summary": "StatefulSet update has not been rolled out."
|
||
|
|
"expr": |
|
||
|
|
(
|
||
|
|
max without (revision) (
|
||
|
|
kube_statefulset_status_current_revision{job="kube-state-metrics"}
|
||
|
|
unless
|
||
|
|
kube_statefulset_status_update_revision{job="kube-state-metrics"}
|
||
|
|
)
|
||
|
|
*
|
||
|
|
(
|
||
|
|
kube_statefulset_replicas{job="kube-state-metrics"}
|
||
|
|
!=
|
||
|
|
kube_statefulset_status_replicas_updated{job="kube-state-metrics"}
|
||
|
|
)
|
||
|
|
) and (
|
||
|
|
changes(kube_statefulset_status_replicas_updated{job="kube-state-metrics"}[5m])
|
||
|
|
==
|
||
|
|
0
|
||
|
|
)
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeDaemonSetRolloutStuck"
|
||
|
|
"annotations":
|
||
|
|
"description": "DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} has not finished or progressed for at least 15 minutes."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck"
|
||
|
|
"summary": "DaemonSet rollout is stuck."
|
||
|
|
"expr": |
|
||
|
|
(
|
||
|
|
(
|
||
|
|
kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"}
|
||
|
|
!=
|
||
|
|
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
|
||
|
|
) or (
|
||
|
|
kube_daemonset_status_number_misscheduled{job="kube-state-metrics"}
|
||
|
|
!=
|
||
|
|
0
|
||
|
|
) or (
|
||
|
|
kube_daemonset_updated_number_scheduled{job="kube-state-metrics"}
|
||
|
|
!=
|
||
|
|
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
|
||
|
|
) or (
|
||
|
|
kube_daemonset_status_number_available{job="kube-state-metrics"}
|
||
|
|
!=
|
||
|
|
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
|
||
|
|
)
|
||
|
|
) and (
|
||
|
|
changes(kube_daemonset_updated_number_scheduled{job="kube-state-metrics"}[5m])
|
||
|
|
==
|
||
|
|
0
|
||
|
|
)
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeContainerWaiting"
|
||
|
|
"annotations":
|
||
|
|
"description": "Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} has been in waiting state for longer than 1 hour."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting"
|
||
|
|
"summary": "Pod container waiting longer than 1 hour"
|
||
|
|
"expr": |
|
||
|
|
sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0
|
||
|
|
"for": "1h"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeDaemonSetNotScheduled"
|
||
|
|
"annotations":
|
||
|
|
"description": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled"
|
||
|
|
"summary": "DaemonSet pods are not scheduled."
|
||
|
|
"expr": |
|
||
|
|
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
|
||
|
|
-
|
||
|
|
kube_daemonset_status_current_number_scheduled{job="kube-state-metrics"} > 0
|
||
|
|
"for": "10m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeDaemonSetMisScheduled"
|
||
|
|
"annotations":
|
||
|
|
"description": "{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled"
|
||
|
|
"summary": "DaemonSet pods are misscheduled."
|
||
|
|
"expr": |
|
||
|
|
kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeJobCompletion"
|
||
|
|
"annotations":
|
||
|
|
"description": "Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than 12 hours to complete."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion"
|
||
|
|
"summary": "Job did not complete in time"
|
||
|
|
"expr": |
|
||
|
|
kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"} > 0
|
||
|
|
"for": "12h"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeJobFailed"
|
||
|
|
"annotations":
|
||
|
|
"description": "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete. Removing failed job after investigation should clear this alert."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed"
|
||
|
|
"summary": "Job failed to complete."
|
||
|
|
"expr": |
|
||
|
|
kube_job_failed{job="kube-state-metrics"} > 0
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeHpaReplicasMismatch"
|
||
|
|
"annotations":
|
||
|
|
"description": "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has not matched the desired number of replicas for longer than 15 minutes."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch"
|
||
|
|
"summary": "HPA has not matched descired number of replicas."
|
||
|
|
"expr": |
|
||
|
|
(kube_horizontalpodautoscaler_status_desired_replicas{job="kube-state-metrics"}
|
||
|
|
!=
|
||
|
|
kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"})
|
||
|
|
and
|
||
|
|
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
|
||
|
|
>
|
||
|
|
kube_horizontalpodautoscaler_spec_min_replicas{job="kube-state-metrics"})
|
||
|
|
and
|
||
|
|
(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
|
||
|
|
<
|
||
|
|
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"})
|
||
|
|
and
|
||
|
|
changes(kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}[15m]) == 0
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeHpaMaxedOut"
|
||
|
|
"annotations":
|
||
|
|
"description": "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has been running at max replicas for longer than 15 minutes."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout"
|
||
|
|
"summary": "HPA is running at max replicas"
|
||
|
|
"expr": |
|
||
|
|
kube_horizontalpodautoscaler_status_current_replicas{job="kube-state-metrics"}
|
||
|
|
==
|
||
|
|
kube_horizontalpodautoscaler_spec_max_replicas{job="kube-state-metrics"}
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "name": "kubernetes-resources"
|
||
|
|
"rules":
|
||
|
|
- "alert": "KubeCPUOvercommit"
|
||
|
|
"annotations":
|
||
|
|
"description": "Cluster has overcommitted CPU resource requests for Pods by {{ $value }} CPU shares and cannot tolerate node failure."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit"
|
||
|
|
"summary": "Cluster has overcommitted CPU resource requests."
|
||
|
|
"expr": |
|
||
|
|
sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="cpu"}) - max(kube_node_status_allocatable{resource="cpu"})) > 0
|
||
|
|
and
|
||
|
|
(sum(kube_node_status_allocatable{resource="cpu"}) - max(kube_node_status_allocatable{resource="cpu"})) > 0
|
||
|
|
"for": "10m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeMemoryOvercommit"
|
||
|
|
"annotations":
|
||
|
|
"description": "Cluster has overcommitted memory resource requests for Pods by {{ $value }} bytes and cannot tolerate node failure."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryovercommit"
|
||
|
|
"summary": "Cluster has overcommitted memory resource requests."
|
||
|
|
"expr": |
|
||
|
|
sum(namespace_memory:kube_pod_container_resource_requests:sum{}) - (sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0
|
||
|
|
and
|
||
|
|
(sum(kube_node_status_allocatable{resource="memory"}) - max(kube_node_status_allocatable{resource="memory"})) > 0
|
||
|
|
"for": "10m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeCPUQuotaOvercommit"
|
||
|
|
"annotations":
|
||
|
|
"description": "Cluster has overcommitted CPU resource requests for Namespaces."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuquotaovercommit"
|
||
|
|
"summary": "Cluster has overcommitted CPU resource requests."
|
||
|
|
"expr": |
|
||
|
|
sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="cpu"})
|
||
|
|
/
|
||
|
|
sum(kube_node_status_allocatable{resource="cpu"})
|
||
|
|
> 1.5
|
||
|
|
"for": "5m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeMemoryQuotaOvercommit"
|
||
|
|
"annotations":
|
||
|
|
"description": "Cluster has overcommitted memory resource requests for Namespaces."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryquotaovercommit"
|
||
|
|
"summary": "Cluster has overcommitted memory resource requests."
|
||
|
|
"expr": |
|
||
|
|
sum(kube_resourcequota{job="kube-state-metrics", type="hard", resource="memory"})
|
||
|
|
/
|
||
|
|
sum(kube_node_status_allocatable{resource="memory",job="kube-state-metrics"})
|
||
|
|
> 1.5
|
||
|
|
"for": "5m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeQuotaAlmostFull"
|
||
|
|
"annotations":
|
||
|
|
"description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaalmostfull"
|
||
|
|
"summary": "Namespace quota is going to be full."
|
||
|
|
"expr": |
|
||
|
|
kube_resourcequota{job="kube-state-metrics", type="used"}
|
||
|
|
/ ignoring(instance, job, type)
|
||
|
|
(kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
|
||
|
|
> 0.9 < 1
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "info"
|
||
|
|
- "alert": "KubeQuotaFullyUsed"
|
||
|
|
"annotations":
|
||
|
|
"description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotafullyused"
|
||
|
|
"summary": "Namespace quota is fully used."
|
||
|
|
"expr": |
|
||
|
|
kube_resourcequota{job="kube-state-metrics", type="used"}
|
||
|
|
/ ignoring(instance, job, type)
|
||
|
|
(kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
|
||
|
|
== 1
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "info"
|
||
|
|
- "alert": "KubeQuotaExceeded"
|
||
|
|
"annotations":
|
||
|
|
"description": "Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded"
|
||
|
|
"summary": "Namespace quota has exceeded the limits."
|
||
|
|
"expr": |
|
||
|
|
kube_resourcequota{job="kube-state-metrics", type="used"}
|
||
|
|
/ ignoring(instance, job, type)
|
||
|
|
(kube_resourcequota{job="kube-state-metrics", type="hard"} > 0)
|
||
|
|
> 1
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "CPUThrottlingHigh"
|
||
|
|
"annotations":
|
||
|
|
"description": "{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh"
|
||
|
|
"summary": "Processes experience elevated CPU throttling."
|
||
|
|
"expr": |
|
||
|
|
sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace)
|
||
|
|
/
|
||
|
|
sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)
|
||
|
|
> ( 25 / 100 )
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "info"
|
||
|
|
- "name": "kubernetes-storage"
|
||
|
|
"rules":
|
||
|
|
- "alert": "KubePersistentVolumeFillingUp"
|
||
|
|
"annotations":
|
||
|
|
"description": "The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup"
|
||
|
|
"summary": "PersistentVolume is filling up."
|
||
|
|
"expr": |
|
||
|
|
(
|
||
|
|
kubelet_volume_stats_available_bytes{job="kubelet"}
|
||
|
|
/
|
||
|
|
kubelet_volume_stats_capacity_bytes{job="kubelet"}
|
||
|
|
) < 0.03
|
||
|
|
and
|
||
|
|
kubelet_volume_stats_used_bytes{job="kubelet"} > 0
|
||
|
|
"for": "1m"
|
||
|
|
"labels":
|
||
|
|
"severity": "critical"
|
||
|
|
- "alert": "KubePersistentVolumeFillingUp"
|
||
|
|
"annotations":
|
||
|
|
"description": "Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup"
|
||
|
|
"summary": "PersistentVolume is filling up."
|
||
|
|
"expr": |
|
||
|
|
(
|
||
|
|
kubelet_volume_stats_available_bytes{job="kubelet"}
|
||
|
|
/
|
||
|
|
kubelet_volume_stats_capacity_bytes{job="kubelet"}
|
||
|
|
) < 0.15
|
||
|
|
and
|
||
|
|
kubelet_volume_stats_used_bytes{job="kubelet"} > 0
|
||
|
|
and
|
||
|
|
predict_linear(kubelet_volume_stats_available_bytes{job="kubelet"}[6h], 4 * 24 * 3600) < 0
|
||
|
|
"for": "1h"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubePersistentVolumeErrors"
|
||
|
|
"annotations":
|
||
|
|
"description": "The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors"
|
||
|
|
"summary": "PersistentVolume is having issues with provisioning."
|
||
|
|
"expr": |
|
||
|
|
kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
|
||
|
|
"for": "5m"
|
||
|
|
"labels":
|
||
|
|
"severity": "critical"
|
||
|
|
- "name": "kubernetes-system"
|
||
|
|
"rules":
|
||
|
|
- "alert": "KubeVersionMismatch"
|
||
|
|
"annotations":
|
||
|
|
"description": "There are {{ $value }} different semantic versions of Kubernetes components running."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch"
|
||
|
|
"summary": "Different semantic versions of Kubernetes components running."
|
||
|
|
"expr": |
|
||
|
|
count(count by (git_version) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"git_version","$1","git_version","(v[0-9]*.[0-9]*).*"))) > 1
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeClientErrors"
|
||
|
|
"annotations":
|
||
|
|
"description": "Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'"
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors"
|
||
|
|
"summary": "Kubernetes API server client is experiencing errors."
|
||
|
|
"expr": |
|
||
|
|
(sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job, namespace)
|
||
|
|
/
|
||
|
|
sum(rate(rest_client_requests_total[5m])) by (instance, job, namespace))
|
||
|
|
> 0.01
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "name": "kube-apiserver-slos"
|
||
|
|
"rules":
|
||
|
|
- "alert": "KubeAPIErrorBudgetBurn"
|
||
|
|
"annotations":
|
||
|
|
"description": "The API server is burning too much error budget."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn"
|
||
|
|
"summary": "The API server is burning too much error budget."
|
||
|
|
"expr": |
|
||
|
|
sum(apiserver_request:burnrate1h) > (14.40 * 0.01000)
|
||
|
|
and
|
||
|
|
sum(apiserver_request:burnrate5m) > (14.40 * 0.01000)
|
||
|
|
"for": "2m"
|
||
|
|
"labels":
|
||
|
|
"long": "1h"
|
||
|
|
"severity": "critical"
|
||
|
|
"short": "5m"
|
||
|
|
- "alert": "KubeAPIErrorBudgetBurn"
|
||
|
|
"annotations":
|
||
|
|
"description": "The API server is burning too much error budget."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn"
|
||
|
|
"summary": "The API server is burning too much error budget."
|
||
|
|
"expr": |
|
||
|
|
sum(apiserver_request:burnrate6h) > (6.00 * 0.01000)
|
||
|
|
and
|
||
|
|
sum(apiserver_request:burnrate30m) > (6.00 * 0.01000)
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"long": "6h"
|
||
|
|
"severity": "critical"
|
||
|
|
"short": "30m"
|
||
|
|
- "alert": "KubeAPIErrorBudgetBurn"
|
||
|
|
"annotations":
|
||
|
|
"description": "The API server is burning too much error budget."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn"
|
||
|
|
"summary": "The API server is burning too much error budget."
|
||
|
|
"expr": |
|
||
|
|
sum(apiserver_request:burnrate1d) > (3.00 * 0.01000)
|
||
|
|
and
|
||
|
|
sum(apiserver_request:burnrate2h) > (3.00 * 0.01000)
|
||
|
|
"for": "1h"
|
||
|
|
"labels":
|
||
|
|
"long": "1d"
|
||
|
|
"severity": "warning"
|
||
|
|
"short": "2h"
|
||
|
|
- "alert": "KubeAPIErrorBudgetBurn"
|
||
|
|
"annotations":
|
||
|
|
"description": "The API server is burning too much error budget."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorbudgetburn"
|
||
|
|
"summary": "The API server is burning too much error budget."
|
||
|
|
"expr": |
|
||
|
|
sum(apiserver_request:burnrate3d) > (1.00 * 0.01000)
|
||
|
|
and
|
||
|
|
sum(apiserver_request:burnrate6h) > (1.00 * 0.01000)
|
||
|
|
"for": "3h"
|
||
|
|
"labels":
|
||
|
|
"long": "3d"
|
||
|
|
"severity": "warning"
|
||
|
|
"short": "6h"
|
||
|
|
- "name": "kubernetes-system-apiserver"
|
||
|
|
"rules":
|
||
|
|
- "alert": "KubeClientCertificateExpiration"
|
||
|
|
"annotations":
|
||
|
|
"description": "A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration"
|
||
|
|
"summary": "Client certificate is about to expire."
|
||
|
|
"expr": |
|
||
|
|
apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 604800
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeClientCertificateExpiration"
|
||
|
|
"annotations":
|
||
|
|
"description": "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration"
|
||
|
|
"summary": "Client certificate is about to expire."
|
||
|
|
"expr": |
|
||
|
|
apiserver_client_certificate_expiration_seconds_count{job="kube-apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="kube-apiserver"}[5m]))) < 86400
|
||
|
|
"labels":
|
||
|
|
"severity": "critical"
|
||
|
|
- "alert": "AggregatedAPIErrors"
|
||
|
|
"annotations":
|
||
|
|
"description": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. It has appeared unavailable {{ $value | humanize }} times averaged over the past 10m."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapierrors"
|
||
|
|
"summary": "An aggregated API has reported errors."
|
||
|
|
"expr": |
|
||
|
|
sum by(name, namespace)(increase(aggregator_unavailable_apiservice_total[10m])) > 4
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "AggregatedAPIDown"
|
||
|
|
"annotations":
|
||
|
|
"description": "An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has been only {{ $value | humanize }}% available over the last 10m."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapidown"
|
||
|
|
"summary": "An aggregated API is down."
|
||
|
|
"expr": |
|
||
|
|
(1 - max by(name, namespace)(avg_over_time(aggregator_unavailable_apiservice[10m]))) * 100 < 85
|
||
|
|
"for": "5m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeAPIDown"
|
||
|
|
"annotations":
|
||
|
|
"description": "KubeAPI has disappeared from Prometheus target discovery."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapidown"
|
||
|
|
"summary": "Target disappeared from Prometheus target discovery."
|
||
|
|
"expr": |
|
||
|
|
absent(up{job="kube-apiserver"} == 1)
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "critical"
|
||
|
|
- "alert": "KubeAPITerminatedRequests"
|
||
|
|
"annotations":
|
||
|
|
"description": "The apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapiterminatedrequests"
|
||
|
|
"summary": "The apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests."
|
||
|
|
"expr": |
|
||
|
|
sum(rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) / ( sum(rate(apiserver_request_total{job="kube-apiserver"}[10m])) + sum(rate(apiserver_request_terminations_total{job="kube-apiserver"}[10m])) ) > 0.20
|
||
|
|
"for": "5m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "name": "kubernetes-system-kubelet"
|
||
|
|
"rules":
|
||
|
|
- "alert": "KubeNodeNotReady"
|
||
|
|
"annotations":
|
||
|
|
"description": "{{ $labels.node }} has been unready for more than 15 minutes."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready"
|
||
|
|
"summary": "Node is not ready."
|
||
|
|
"expr": |
|
||
|
|
kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeNodeUnreachable"
|
||
|
|
"annotations":
|
||
|
|
"description": "{{ $labels.node }} is unreachable and some workloads may be rescheduled."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodeunreachable"
|
||
|
|
"summary": "Node is unreachable."
|
||
|
|
"expr": |
|
||
|
|
(kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} unless ignoring(key,value) kube_node_spec_taint{job="kube-state-metrics",key=~"ToBeDeletedByClusterAutoscaler|cloud.google.com/impending-node-termination|aws-node-termination-handler/spot-itn"}) == 1
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeletTooManyPods"
|
||
|
|
"annotations":
|
||
|
|
"description": "Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods"
|
||
|
|
"summary": "Kubelet is running at capacity."
|
||
|
|
"expr": |
|
||
|
|
count by(node) (
|
||
|
|
(kube_pod_status_phase{job="kube-state-metrics",phase="Running"} == 1) * on(instance,pod,namespace,cluster) group_left(node) topk by(instance,pod,namespace,cluster) (1, kube_pod_info{job="kube-state-metrics"})
|
||
|
|
)
|
||
|
|
/
|
||
|
|
max by(node) (
|
||
|
|
kube_node_status_capacity{job="kube-state-metrics",resource="pods"} != 1
|
||
|
|
) > 0.95
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "info"
|
||
|
|
- "alert": "KubeNodeReadinessFlapping"
|
||
|
|
"annotations":
|
||
|
|
"description": "The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodereadinessflapping"
|
||
|
|
"summary": "Node readiness status is flapping."
|
||
|
|
"expr": |
|
||
|
|
sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeletPlegDurationHigh"
|
||
|
|
"annotations":
|
||
|
|
"description": "The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletplegdurationhigh"
|
||
|
|
"summary": "Kubelet Pod Lifecycle Event Generator is taking too long to relist."
|
||
|
|
"expr": |
|
||
|
|
node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
|
||
|
|
"for": "5m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeletPodStartUpLatencyHigh"
|
||
|
|
"annotations":
|
||
|
|
"description": "Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletpodstartuplatencyhigh"
|
||
|
|
"summary": "Kubelet Pod startup latency is too high."
|
||
|
|
"expr": |
|
||
|
|
histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet"} > 60
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeletClientCertificateExpiration"
|
||
|
|
"annotations":
|
||
|
|
"description": "Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletclientcertificateexpiration"
|
||
|
|
"summary": "Kubelet client certificate is about to expire."
|
||
|
|
"expr": |
|
||
|
|
kubelet_certificate_manager_client_ttl_seconds < 604800
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeletClientCertificateExpiration"
|
||
|
|
"annotations":
|
||
|
|
"description": "Client certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletclientcertificateexpiration"
|
||
|
|
"summary": "Kubelet client certificate is about to expire."
|
||
|
|
"expr": |
|
||
|
|
kubelet_certificate_manager_client_ttl_seconds < 86400
|
||
|
|
"labels":
|
||
|
|
"severity": "critical"
|
||
|
|
- "alert": "KubeletServerCertificateExpiration"
|
||
|
|
"annotations":
|
||
|
|
"description": "Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletservercertificateexpiration"
|
||
|
|
"summary": "Kubelet server certificate is about to expire."
|
||
|
|
"expr": |
|
||
|
|
kubelet_certificate_manager_server_ttl_seconds < 604800
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeletServerCertificateExpiration"
|
||
|
|
"annotations":
|
||
|
|
"description": "Server certificate for Kubelet on node {{ $labels.node }} expires in {{ $value | humanizeDuration }}."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletservercertificateexpiration"
|
||
|
|
"summary": "Kubelet server certificate is about to expire."
|
||
|
|
"expr": |
|
||
|
|
kubelet_certificate_manager_server_ttl_seconds < 86400
|
||
|
|
"labels":
|
||
|
|
"severity": "critical"
|
||
|
|
- "alert": "KubeletClientCertificateRenewalErrors"
|
||
|
|
"annotations":
|
||
|
|
"description": "Kubelet on node {{ $labels.node }} has failed to renew its client certificate ({{ $value | humanize }} errors in the last 5 minutes)."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletclientcertificaterenewalerrors"
|
||
|
|
"summary": "Kubelet has failed to renew its client certificate."
|
||
|
|
"expr": |
|
||
|
|
increase(kubelet_certificate_manager_client_expiration_renew_errors[5m]) > 0
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeletServerCertificateRenewalErrors"
|
||
|
|
"annotations":
|
||
|
|
"description": "Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes)."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletservercertificaterenewalerrors"
|
||
|
|
"summary": "Kubelet has failed to renew its server certificate."
|
||
|
|
"expr": |
|
||
|
|
increase(kubelet_server_expiration_renew_errors[5m]) > 0
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "warning"
|
||
|
|
- "alert": "KubeletDown"
|
||
|
|
"annotations":
|
||
|
|
"description": "Kubelet has disappeared from Prometheus target discovery."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletdown"
|
||
|
|
"summary": "Target disappeared from Prometheus target discovery."
|
||
|
|
"expr": |
|
||
|
|
absent(up{job="kubelet"} == 1)
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "critical"
|
||
|
|
- "name": "kubernetes-system-scheduler"
|
||
|
|
"rules":
|
||
|
|
- "alert": "KubeSchedulerDown"
|
||
|
|
"annotations":
|
||
|
|
"description": "KubeScheduler has disappeared from Prometheus target discovery."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeschedulerdown"
|
||
|
|
"summary": "Target disappeared from Prometheus target discovery."
|
||
|
|
"expr": |
|
||
|
|
absent(up{job="kube-scheduler"} == 1)
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "critical"
|
||
|
|
- "name": "kubernetes-system-controller-manager"
|
||
|
|
"rules":
|
||
|
|
- "alert": "KubeControllerManagerDown"
|
||
|
|
"annotations":
|
||
|
|
"description": "KubeControllerManager has disappeared from Prometheus target discovery."
|
||
|
|
"runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontrollermanagerdown"
|
||
|
|
"summary": "Target disappeared from Prometheus target discovery."
|
||
|
|
"expr": |
|
||
|
|
absent(up{job="kube-controller-manager"} == 1)
|
||
|
|
"for": "15m"
|
||
|
|
"labels":
|
||
|
|
"severity": "critical"
|