200 lines
10 KiB
YAML
200 lines
10 KiB
YAML
{{- /*
|
|
Generated from 'kubernetes-apps' group from https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/release-0.1/manifests/prometheus-rules.yaml
|
|
Do not change in-place! In order to change this file first read following link:
|
|
https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
|
|
*/ -}}
|
|
{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }}
|
|
{{- if and (semverCompare ">=1.10.0-0" $kubeTargetVersion) (semverCompare "<1.14.0-0" $kubeTargetVersion) .Values.defaultRules.create .Values.defaultRules.rules.kubernetesApps }}
|
|
{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }}
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: {{ printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "kubernetes-apps" | trunc 63 | trimSuffix "-" }}
|
|
namespace: {{ template "kube-prometheus-stack.namespace" . }}
|
|
labels:
|
|
app: {{ template "kube-prometheus-stack.name" . }}
|
|
{{ include "kube-prometheus-stack.labels" . | indent 4 }}
|
|
{{- if .Values.defaultRules.labels }}
|
|
{{ toYaml .Values.defaultRules.labels | indent 4 }}
|
|
{{- end }}
|
|
{{- if .Values.defaultRules.annotations }}
|
|
annotations:
|
|
{{ toYaml .Values.defaultRules.annotations | indent 4 }}
|
|
{{- end }}
|
|
spec:
|
|
groups:
|
|
- name: kubernetes-apps
|
|
rules:
|
|
- alert: KubePodCrashLooping
|
|
annotations:
|
|
message: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} ({{`{{`}} $labels.container {{`}}`}}) is restarting {{`{{`}} printf "%.2f" $value {{`}}`}} times / 5 minutes.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodcrashlooping
|
|
expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}[15m]) * 60 * 5 > 0
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: KubePodNotReady
|
|
annotations:
|
|
message: Pod {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.pod {{`}}`}} has been in a non-ready state for longer than an hour.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubepodnotready
|
|
expr: sum by (namespace, pod) (kube_pod_status_phase{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}", phase=~"Pending|Unknown"}) > 0
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: KubeDeploymentGenerationMismatch
|
|
annotations:
|
|
message: Deployment generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} does not match, this indicates that the Deployment has failed but has not been rolled back.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentgenerationmismatch
|
|
expr: |-
|
|
kube_deployment_status_observed_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
!=
|
|
kube_deployment_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: KubeDeploymentReplicasMismatch
|
|
annotations:
|
|
message: Deployment {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.deployment {{`}}`}} has not matched the expected number of replicas for longer than an hour.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedeploymentreplicasmismatch
|
|
expr: |-
|
|
kube_deployment_spec_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
!=
|
|
kube_deployment_status_replicas_available{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
for: 1h
|
|
labels:
|
|
severity: critical
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: KubeStatefulSetReplicasMismatch
|
|
annotations:
|
|
message: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} has not matched the expected number of replicas for longer than 15 minutes.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetreplicasmismatch
|
|
expr: |-
|
|
kube_statefulset_status_replicas_ready{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
!=
|
|
kube_statefulset_status_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: KubeStatefulSetGenerationMismatch
|
|
annotations:
|
|
message: StatefulSet generation for {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} does not match, this indicates that the StatefulSet has failed but has not been rolled back.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetgenerationmismatch
|
|
expr: |-
|
|
kube_statefulset_status_observed_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
!=
|
|
kube_statefulset_metadata_generation{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: KubeStatefulSetUpdateNotRolledOut
|
|
annotations:
|
|
message: StatefulSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.statefulset {{`}}`}} update has not been rolled out.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubestatefulsetupdatenotrolledout
|
|
expr: |-
|
|
max without (revision) (
|
|
kube_statefulset_status_current_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
unless
|
|
kube_statefulset_status_update_revision{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
)
|
|
*
|
|
(
|
|
kube_statefulset_replicas{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
!=
|
|
kube_statefulset_status_replicas_updated{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
)
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: KubeDaemonSetRolloutStuck
|
|
annotations:
|
|
message: Only {{`{{`}} $value {{`}}`}}% of the desired Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are scheduled and ready.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetrolloutstuck
|
|
expr: |-
|
|
kube_daemonset_status_number_ready{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
/
|
|
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} * 100 < 100
|
|
for: 15m
|
|
labels:
|
|
severity: critical
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: KubeDaemonSetNotScheduled
|
|
annotations:
|
|
message: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are not scheduled.'
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetnotscheduled
|
|
expr: |-
|
|
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"}
|
|
-
|
|
kube_daemonset_status_current_number_scheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: KubeDaemonSetMisScheduled
|
|
annotations:
|
|
message: '{{`{{`}} $value {{`}}`}} Pods of DaemonSet {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.daemonset {{`}}`}} are running where they are not supposed to run.'
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubedaemonsetmisscheduled
|
|
expr: kube_daemonset_status_number_misscheduled{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: KubeCronJobRunning
|
|
annotations:
|
|
message: CronJob {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.cronjob {{`}}`}} is taking more than 1h to complete.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubecronjobrunning
|
|
expr: time() - kube_cronjob_next_schedule_time{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 3600
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: KubeJobCompletion
|
|
annotations:
|
|
message: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} is taking more than one hour to complete.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobcompletion
|
|
expr: kube_job_spec_completions{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} - kube_job_status_succeeded{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
- alert: KubeJobFailed
|
|
annotations:
|
|
message: Job {{`{{`}} $labels.namespace {{`}}`}}/{{`{{`}} $labels.job_name {{`}}`}} failed to complete.
|
|
runbook_url: {{ .Values.defaultRules.runbookUrl }}alert-name-kubejobfailed
|
|
expr: kube_job_status_failed{job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"} > 0
|
|
for: 1h
|
|
labels:
|
|
severity: warning
|
|
{{- if .Values.defaultRules.additionalRuleLabels }}
|
|
{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}
|
|
{{- end }}
|
|
{{- end }} |