From ffaf6a079e64f210a804c9712eacb76fd8cff7df Mon Sep 17 00:00:00 2001 From: Aaron Riedel Date: Wed, 30 Aug 2023 21:27:13 +0200 Subject: [PATCH 1/4] put alertmanager config back into helm values --- prometheus/values.yaml | 46 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/prometheus/values.yaml b/prometheus/values.yaml index 650e76f..6cf2e01 100644 --- a/prometheus/values.yaml +++ b/prometheus/values.yaml @@ -1,6 +1,51 @@ alertmanager: alertmanagerSpec: replicas: 2 + secrets: + - telegram-api + config: + global: + resolve_timeout: 5m + templates: + - '/etc/alertmanager/config/*.tmpl' + route: + group_by: ['alertname'] + group_wait: 30s + group_interval: 30s + repeat_interval: 24h + receiver: 'tg1' + routes: + - matchers: + - severity=warning + receiver: 'tg1' + - matchers: + - severity=critical + receiver: 'tg1' + receivers: + - name: tg1 + telegram_configs: + - bot_token_file: '/etc/alertmanager/secrets/api_key' + chat_id: -995270884 + api_url: "https://api.telegram.org" + send_resolved: true + parse_mode: "HTML" + message: '{{ template "telegram.aaron" .}}' + inhibit_rules: + - source_matchers: + - severity = critical + target_matchers: + - severity = warning + equal: ['alertname', 'server', 'instance'] + templateFiles: + telegram.tmpl: |- + {{ define "telegram.aaron" }} + {{ range .Alerts }} + {{ if eq .Status "firing"}}🔥 {{ .Labels.alertname }} 🔥{{ else }}✅ {{ .Labels.alertname }} ✅{{ end }} + {{ .Annotations.summary }} + + {{ .Annotations.description }} + {{ end }} + {{ end }} storage: volumeClaimTemplate: spec: @@ -8,7 +53,6 @@ alertmanager: resources: requests: storage: 5Gi - useExistingSecret: true ingress: paths: - / From 778306127fb86162ddae58433579e30ae09a048d Mon Sep 17 00:00:00 2001 From: Aaron Riedel Date: Wed, 30 Aug 2023 21:32:18 +0200 Subject: [PATCH 2/4] try to fix prometheus try to fix prometheus 2 try to fix prometheus 3 --- prometheus/templates.yaml | 15 +++++++++++++++ prometheus/values.yaml | 17 +++++------------ 2 files changed, 20 insertions(+), 12 deletions(-) create mode 100644 prometheus/templates.yaml diff --git a/prometheus/templates.yaml b/prometheus/templates.yaml new file mode 100644 index 0000000..6c8c408 --- /dev/null +++ b/prometheus/templates.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: templates + namespace: prometheus +data: + telegram.tmpl: |- + {{ define "telegram.aaron" }} + {{ range .Alerts }} + {{ if eq .Status "firing"}}🔥 {{ .Labels.alertname }} 🔥{{ else }}✅ {{ .Labels.alertname }} ✅{{ end }} + {{ .Annotations.summary }} + + {{ .Annotations.description }} + {{ end }} + {{ end }} \ No newline at end of file diff --git a/prometheus/values.yaml b/prometheus/values.yaml index 6cf2e01..9c5a5da 100644 --- a/prometheus/values.yaml +++ b/prometheus/values.yaml @@ -2,12 +2,14 @@ alertmanager: alertmanagerSpec: replicas: 2 secrets: - - telegram-api + - "telegram-api" + configMaps: + - "templates" config: global: resolve_timeout: 5m templates: - - '/etc/alertmanager/config/*.tmpl' + - '/etc/alertmanager/configmaps/*.tmpl' route: group_by: ['alertname'] group_wait: 30s @@ -36,16 +38,6 @@ alertmanager: target_matchers: - severity = warning equal: ['alertname', 'server', 'instance'] - templateFiles: - telegram.tmpl: |- - {{ define "telegram.aaron" }} - {{ range .Alerts }} - {{ if eq .Status "firing"}}🔥 {{ .Labels.alertname }} 🔥{{ else }}✅ {{ .Labels.alertname }} ✅{{ end }} - {{ .Annotations.summary }} - - {{ .Annotations.description }} - {{ end }} - {{ end }} storage: volumeClaimTemplate: spec: @@ -53,6 +45,7 @@ alertmanager: resources: requests: storage: 5Gi + useExistingSecret: false ingress: paths: - / From 94c2a34aac7c478a7e4dd2850ba3c12bd7d6f222 Mon Sep 17 00:00:00 2001 From: Aaron Riedel Date: Thu, 31 Aug 2023 00:29:12 +0200 Subject: [PATCH 3/4] try to fix prometheus 2 --- prometheus/values.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/prometheus/values.yaml b/prometheus/values.yaml index 9c5a5da..0906755 100644 --- a/prometheus/values.yaml +++ b/prometheus/values.yaml @@ -5,6 +5,14 @@ alertmanager: - "telegram-api" configMaps: - "templates" + storage: + volumeClaimTemplate: + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi + useExistingSecret: false config: global: resolve_timeout: 5m @@ -38,14 +46,6 @@ alertmanager: target_matchers: - severity = warning equal: ['alertname', 'server', 'instance'] - storage: - volumeClaimTemplate: - spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 5Gi - useExistingSecret: false ingress: paths: - / From c6b97ac1e8312b88a02017399152188104a9aad3 Mon Sep 17 00:00:00 2001 From: Aaron Riedel Date: Fri, 15 Sep 2023 01:01:12 +0200 Subject: [PATCH 4/4] update kube-prometheus-stack to 51.0.3 --- core-deployments.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-deployments.yaml b/core-deployments.yaml index 6e84d64..dc35495 100644 --- a/core-deployments.yaml +++ b/core-deployments.yaml @@ -98,7 +98,7 @@ spec: sources: - chart: kube-prometheus-stack repoURL: https://prometheus-community.github.io/helm-charts - targetRevision: 49.2.0 + targetRevision: 51.0.3 helm: releaseName: prometheus valueFiles: