core-deployments/prometheus/alerts.yaml

---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: prometheus-core-deployment-rules
  namespace: prometheus
  labels:
    monitor: core-deployment
spec:
  groups:
    - name: hardware
      rules:
      - alert: MemoryHigh
        expr: round((((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * 100), 0.1) > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Memory over 80%"
          description: "Memory on node {{ $labels.node }} is over 80% for more than 5 minutes. Plox fix. Memory usage: {{ $value }}%"
      - alert: DiskspaceLow
        expr: round(node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100, 1) < 5
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "Free disk space at {{ $value }}%"
          description: "Disk space on node {{ $labels.node }} is only {{ $value }}%. Plox fix. Partition: {{ $labels.device }}"
      - alert: HostMemoryUnderMemoryPressure
        expr: rate(node_vmstat_pgmajfault[1m]) > 1000
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: Host memory under memory pressure {{ $labels.node }}
          description: "The node is under heavy memory pressure. High rate of major page faults\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostUnusualDiskReadRate
        expr: sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 200
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Host unusual disk read rate {{ $labels.node }}
          description: "Disk is probably reading too much data (> 200 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostUnusualDiskWriteRate
        expr: sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 200
        for: 3m
        labels:
          severity: warning
        annotations:
          summary: Host unusual disk write rate {{ $labels.node }}
          description: "Disk is probably writing too much data (> 200 MB/s)\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostCpuStealNoisyNeighbor
        expr: avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: Host CPU steal noisy neighbor {{ $labels.node }}
          description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: HostPhysicalComponentTooHot
        expr: node_hwmon_temp_celsius > 90
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: Host physical component too hot {{ $labels.node }}
          description: "Physical hardware component too hot\n  Sensor = {{ $labels.sensor }}\n Temp = {{ $value }}"
      - alert: SMARTbad
        expr: smartmon_device_smart_healthy < 1
        for: 0m
        labels:
          severity: critical
        annotations:
          summary: SMART check bad of drive {{ $labels.exported_disk }} in node {{ $labels.node }}
          description: "SMART check returned bad health of {{ $labels.exported_disk }} in node {{ $labels.node }}. VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: "SMARTcheck too old"
        expr: (time() - smartmon_smartctl_run) > 10800
        labels:
          severity: warning
        annotations:
          summary: "SMARTcheck not running"
          description: 'The last SMARTcheck on node {{ $labels.node }} was more than 3h ago. Plox fix.'
      - alert: "ECC Memory errors"
        expr: (node_edac_correctable_errors_total) > 100
        labels:
          severity: warning
        annotations:
          summary: "ECC errors on {{ $labels.node }}"
          description: 'The node {{ $labels.node }} accumulated {{ $value }} correctable errors.'
      - alert: "ECC Memory uncorrectable errors"
        expr: (node_edac_uncorrectable_errors_total) > 0
        labels:
          severity: critical
        annotations:
          summary: "ECC errors on {{ $labels.node }}"
          description: 'The node {{ $labels.node }} accumulated {{ $value }} uncorrectable errors.'
    - name: etcdbackup
      rules:
      - alert: "etcdbackup too old"
        expr: (time() - etcdbackup_time) > 10800
        labels:
          severity: warning
        annotations:
          summary: "etcd backup not running"
          description: 'The last etcd backup on node {{ $labels.node }} was more than 3h ago. Plox fix.'
      - alert: "etcdbackup failed"
        expr: etcdbackup_result > 0
        labels:
          severity: warning
        annotations:
          summary: "etcdbackup failed"
          description: "The backup script for etcd failed on node {{ $labels.node }}. Plox fix."
    - name: kubernetes
      rules:
      - alert: KubernetesUnhealthyPod
        expr: kube_pod_container_status_waiting_reason == 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "The Pod {{ $labels.pod }} is {{ $labels.reason }}"
          description: "The Pod {{ $labels.pod }} is in the state {{ $labels.reason }} for more than 5m. The Pod is in namespace {{ $labels.namespace }} and on node {{ $labels.node }}."