Prometheus使用企业微信Webhook推送告警通知

构建微信Webhook镜像

代码依赖文件：requirements.txt

certifi==2018.10.15
chardet==3.0.4
Click==7.0
Flask==1.0.2
idna==2.7
itsdangerous==1.1.0
Jinja2==2.10
MarkupSafe==1.1.0
requests==2.20.1
urllib3==1.24.1
Werkzeug==0.15.3

微信Webhook代码示例文件：app.py

import os
import sys
import json
import logging
import requests

from urllib.parse import urlparse
from flask import Flask
from flask import request

import urllib3
urllib3.disable_warnings()

app = Flask(__name__)

logging.basicConfig(
    level=logging.DEBUG if os.getenv('LOG_LEVEL') == "debug" else logging.INFO,
    format="%(asctime)s %(levelname)s %(message)s")


@app.route('/', methods=['POST', 'GET'])
def send():
    if request.method == "POST":
        post_data = request.get_data()
        app.logger.debug(post_data)
        alert_data = json.loads(post_data)
        send_message(alert_data)
        return "Success!"
    else:
        return "Weclome to use weixin webhook server!"


def send_message(alert_data):
    corp_id = os.getenv('CORP_ID')
    secret = os.getenv('SECRET')

    token = get_token(corp_id, secret)
    url = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=%s" % token

    user_id = os.getenv('USER_ID')
    party_id = os.getenv('PARTY_ID')
    agent_id = os.getenv('AGENT_ID')

    alerts = alert_data['alerts']
    alert_name = alerts[0]['labels']['alertname']
    subject = "[容器告警通知] %s 有 %d 条新的报警" % (alert_name, len(alerts))

    external_url = alerts[0]['generatorURL']
    prometheus_url = os.getenv('PROME_URL')
    if prometheus_url:
        res = urlparse(external_url)
        external_url = external_url.replace(res.netloc, prometheus_url)

    data = {
        "touser": user_id,
        #"totag": tag_id,
        "toparty": party_id,
        "msgtype": "markdown",
        "agentid": agent_id,
        "markdown": {
            "content": subject + "\n" + _mark_item(alerts[0]) + "\n" + "[>>点击查看完整信息](" + external_url + ")\n"
        },
        "safe": "0"
    }
    r = requests.post(url=url,data=json.dumps(data),verify=False)
    n = 0
    while r.json()['errcode'] != 0 and n < 4:
        n+=1
        token = get_token(corp_id, secret)
        if token:
            url = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=%s" % token
            r = requests.post(url=url,data=json.dumps(data),verify=False)
    return r.json()


def get_token(corp_id,secret):
    url = "https://qyapi.weixin.qq.com/cgi-bin/gettoken"
    data = {
        "corpid":corp_id,
        "corpsecret":secret
    }
    r = requests.get(url=url,params=data,verify=False)
    print(r.json())
    if r.json()['errcode'] != 0:
        return False
    else:
        token = r.json()['access_token']
        return token


def _mark_item(alert):
    labels = alert['labels']
    annotations = ""
    for k, v in alert['annotations'].items():
        annotations += "{0}: {1}\n".format(k, v)
    if "job" in labels:
        mark_item = "\n> job: " + labels['job'] + "\n\n" + annotations + "\n"
    else:
        mark_item = "\n> " + annotations + "\n"
    return mark_item


if __name__ == '__main__':
    app.debug=False
    app.run(host='0.0.0.0', port=5000)

镜像构建模板文件：Dockerfile

FROM python:3.6.4-alpine3.4

MAINTAINER varden

RUN echo "https://mirrors.aliyun.com/alpine/v3.4/main/" > /etc/apk/repositories
RUN echo "https://mirrors.aliyun.com/alpine/v3.4/community/" >> /etc/apk/repositories
RUN apk update
RUN apk upgrade

RUN apk add --no-cache ca-certificates tzdata curl bash && rm -rf /var/cache/apk/*
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN echo "Asia/Shanghai" > /etc/timezone

WORKDIR /app

COPY requirements.txt /app/requirements.txt
RUN pip install -r requirements.txt -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com

COPY app.py /app/app.py

CMD python app.py

构建命令

docker build -t weixin-hook:1.0.0 .

docker部署

#!/bin/bash
#
set -u

docker run -detach \
  --publish 5000:5000 \
  --name weixin-hook \
  --restart always \
  --env LOG_LEVEL=debug \
  --env PROME_URL=10.10.10.200:32101 \
  --env CORP_ID=xxxxxxxxxxxxxxx \
  --env SECRET=xxxxxxxxx-xxxxxxxxxxxxxxxxxxxxx \
  --env USER_ID="" \
  --env PARTY_ID=2 \
  --env AGENT_ID=1000002 \
  weixin-hook:1.0.0

docker-compose部署

version: "3.3"

networks:
  bridge_net:
    external:
      name: bridge

services:
  weixin_hook:
    image: weixin-hook:1.0.0
    ports:
      - "15000:5000"
    networks:
      - bridge_net
    environment:
      LOG_LEVEL: debug
      PROME_URL: 10.10.10.200:32101
    env_file: robot.env
    deploy:
      mode: replicated
      replicas: 1
      resources:
        limits:
          cpus: '0.2'
          memory: 128M
        reservations:
          cpus: '0.1'
          memory: 64M
    healthcheck:
      test: curl -f http://localhost:5000 || exit 1
      interval: 30s
      timeout: 30s
      retries: 5

环境变量文件：robot.env

CORP_ID=xxxxxxxxxxxxxxx 
SECRET=xxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxx
USER_ID=""
PARTY_ID=2
AGENT_ID=1000002

K8s部署

创建Secret资源对象

kubectl create secret generic weixin-secret \
--from-literal=corp_id=xxxxxxxxxxxxxxxx \
--from-literal=secret=xxxxxxxx-xxxxxxxxxxxxxx \
--from-literal=user_id="" \
--from-literal=party_id=2 \
--from-literal=agent_id=1000002 \
-n monitoring

squid代理访问外网：

kubectl create secret generic proxy-secret \
--from-literal=http_proxy=http://<username>:<password>@10.10.10.15:3128 \
--from-literal=https_proxy=http://<username>:<password>@10.10.10.15:3128 \
-n monitoring

K8s部署清单

apiVersion: apps/v1
kind: Deployment
metadata:
  name: weixin-hook
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: weixin-hook
  template:
    metadata:
      labels:
        app: weixin-hook
    spec:
      containers:
      - name: weixin-hook
        image: weixin-hook:1.0.0
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 5000
          name: http
        env:
        - name: PROME_URL
          value: 10.10.10.200:32101
        - name: LOG_LEVEL
          value: debug
        - name: CORP_ID
          valueFrom:
            secretKeyRef:
              name: weixin-secret
              key: corp_id
        - name: SECRET
          valueFrom:
            secretKeyRef:
              name: weixin-secret
              key: secret
        - name: USER_ID
          valueFrom:
            secretKeyRef:
              name: weixin-secret
              key: user_id
        - name: PARTY_ID
          valueFrom:
            secretKeyRef:
              name: weixin-secret
              key: party_id
        - name: AGENT_ID
          valueFrom:
            secretKeyRef:
              name: weixin-secret
              key: agent_id
        - name: http_proxy
          valueFrom:
            secretKeyRef:
              name: proxy-secret
              key: http_proxy
        - name: https_proxy
          valueFrom:
            secretKeyRef:
              name: proxy-secret
              key: https_proxy
        resources:
          requests:
            cpu: 100m
            memory: 200Mi
          limits:
            cpu: 100m
            memory: 200Mi
        livenessProbe:
          httpGet:
            scheme: HTTP
            path: /
            port: 5000
          initialDelaySeconds: 30
          timeoutSeconds: 30
        readinessProbe:
          httpGet:
            scheme: HTTP
            path: /
            port: 5000
          initialDelaySeconds: 30
          timeoutSeconds: 30
---
apiVersion: v1
kind: Service
metadata:
  name: weixin-hook
  namespace: monitoring
spec:
  selector:
    app: weixin-hook
  ports:
  - name: hook
    port: 5000
    targetPort: http

在AlertManager中webhook地址直接通过DNS形式访问即可：

receivers:
- name: 'webhook'
  webhook_configs:
  - url: 'http://weixin-hook:5000'
    send_resolved: true

作者：Varden