add alert for unhealthy pod

prometheus/config-map.yaml

@@ -44,6 +44,16 @@ data:
         annotations:
           summary: "Free disk space below 2 GB"
           description: "Disk space on server {{ $labels.node }} is under 2 GB. Plox fix. Free Space: {{ $value }} GB on partition {{ $labels.device }}"
+    - name: KubernetesUnhealthyPod
+      rules:
+      - alert: KubernetesUnhealthyPod
+        expr: kube_pod_container_status_waiting_reason = 1
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "The Pod {{ $labels.pod }} is {{ $labels.reason }}""
+          description: "The Pod {{ $labels.pod }} is in the state {{ $labels.reason }} for more than 5m. The Pod is in namespace {{ $labels.namespace }} and on node {{ $labels.node }}."
     - name: PrometheusTargetMissing
       rules:
       - alert: PrometheusTargetMissing
@@ -53,7 +63,7 @@ data:
           severity: critical
         annotations:
           summary: Prometheus target missing (instance {{ $labels.instance }})
-          description: "A Prometheus target has disappeared. \n  Job: {{ $labels.job }}\n  App: {{ $labels.app }}\n  Pod: {{ $labels.pod }}\n  Node: {{ $labels.node }}\n  Namespace: {{ $labels.namespace }}"
+          description: "A Prometheus target has disappeared. {{if ne $labels.job ""}}\n  Job: {{ $labels.job }}{{end}}{{if ne $labels.app ""}}\n  App: {{ $labels.app }}{{end}}{{if ne $labels.pod ""}}\n  Pod: {{ $labels.pod }}{{end}}{{if ne $labels.node ""}}\n  Node: {{ $labels.node }}{{end}}{{if ne $labels.namespace ""}}\n  Namespace: {{ $labels.namespace }}{{end}}"
     - name: PrometheusConfigurationReloadFailure
       rules:
       - alert: PrometheusConfigurationReloadFailure