install

wget https://github.com/prometheus/prometheus/releases/download/v2.16.0/prometheus-2.16.0.linux-amd64.tar.gz
tar -xvf prometheus-2.16.0.linux-amd64.tar.gz
cd prometheus-2.16.0.linux-amd64
./prometheus
# http://127.0.0.1:9090
# http://127.0.0.1:9090/metrics

# 监控物理设备
# wget https://github.com/prometheus/node_exporter/releases/download/v0.18.1/node_exporter-0.18.1.linux-amd64.tar.gz
# ./node_exporter
# 监控网络
# wget https://github.com/prometheus/blackbox_exporter/releases/download/v0.16.0/blackbox_exporter-0.16.0.linux-amd64.tar.gz
# ./blackbox_exporter --web.listen-address=:9115 --config.file=blackbox.yml
# docker run prom/node-exporter:v1.0.0-rc.0

# for test
docker run --rm --name node-exporter \
-v "/proc:/host/proc" -v "/sys:/host/sys" -v "/:/rootfs" --net="host" prom/node-exporter:v1.0.0-rc.0 --path.procfs /host/proc --path.sysfs /host/proc --collector.filesystem.ignored-mount-points "^/(sys|proc|dev|host|etc)($|/)"

# for daemon
docker run -d --name node-exporter \
-v "/proc:/host/proc" -v "/sys:/host/sys" -v "/:/rootfs" --net="host" prom/node-exporter:v1.0.0-rc.0 --path.procfs /host/proc --path.sysfs /host/proc --collector.filesystem.ignored-mount-points "^/(sys|proc|dev|host|etc)($|/)"

mkdir -p $(pwd)/prometheus/alert.rules.d

# docker run -it --rm \
docker run -d --name prometheus -p 19090:9090 \
-v $(pwd)/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml \
-v $(pwd)/prometheus/alert.rules.d:/etc/prometheus/alert.rules.d \
prom/prometheus:v2.16.0  --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.retention.time=10d --web.enable-lifecycle


# cadvisor

docker run \
  --volume=/:/rootfs:ro \
  --volume=/var/run:/var/run:ro \
  --volume=/sys:/sys:ro \
  --volume=/var/lib/docker/:/var/lib/docker:ro \
  --volume=/dev/disk/:/dev/disk:ro \
  --publish=8080:8080 \
  --detach=true \
  --name=cadvisor \
  gcr.azk8s.cn/google_containers/cadvisor:v0.36.0


# 默认用户名密码为admin/admin
docker run -d --name=grafana -p 23000:3000 grafana/grafana:6.6.2  

# https://grafana.com/grafana/dashboards/8919
# https://grafana.com/grafana/dashboards/179



# a1
alertmanager  --web.listen-address=":9193" --cluster.listen-address="0.0.0.0:9194" --config.file=/etc/prometheus/alertmanager-ha.yml  --storage.path=/data/alertmanager/1/ --log.level=debug

# a2
alertmanager  --web.listen-address=":9293" --cluster.listen-address="0.0.0.0:9294" --config.file=/etc/prometheus/alertmanager-ha.yml  --storage.path=/data/alertmanager/2/ --log.level=debug --cluster.peer=127.0.0.1:9194

# a3
alertmanager  --web.listen-address=":9393" --cluster.listen-address="0.0.0.0:9394" --config.file=/etc/prometheus/alertmanager-ha.yml  --storage.path=/data/alertmanager/3/ --log.level=debug --cluster.peer=127.0.0.1:9194


mkdir -p $(pwd)/alertmanager/template

cat >$(pwd)/alertmanager/alertmanager.yml <<EOL
global:
  resolve_timeout: 5m

route:
  group_by: ['alertname']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 1h
  receiver: 'web.hook'
receivers:
- name: 'web.hook'
  webhook_configs:
  - url: 'http://192.168.122.37:19000/hooks/redeploy-webhook'
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']
EOL


docker run -d --name alertmanager-1 \
-v $(pwd)/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml \
-v $(pwd)/alertmanager/template:/etc/alertmanager/template \
-p 19193:9193 \
-p 19194:9194 \
prom/alertmanager:v0.20.0 \
--config.file=/etc/alertmanager/alertmanager.yml \
--storage.path=/alertmanager \
--web.listen-address=":9193" \
--cluster.listen-address="0.0.0.0:9194"


docker run -d --name alertmanager-2 \
-v $(pwd)/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml \
-v $(pwd)/alertmanager/template:/etc/alertmanager/template \
-p 19293:9193 \
-p 19294:9194 \
prom/alertmanager:v0.20.0 \
--config.file=/etc/alertmanager/alertmanager.yml \
--storage.path=/alertmanager \
--web.listen-address=":9193" \
--cluster.listen-address="0.0.0.0:9194" \
--cluster.peer="192.168.122.37:19194"

docker run -d --name alertmanager-3 \
-v $(pwd)/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml \
-v $(pwd)/alertmanager/template:/etc/alertmanager/template \
-p 19393:9193 \
-p 19394:9194 \
prom/alertmanager:v0.20.0 \
--config.file=/etc/alertmanager/alertmanager.yml \
--storage.path=/alertmanager \
--web.listen-address=":9193" \
--cluster.listen-address="0.0.0.0:9194" \
--cluster.peer="192.168.122.37:19194"



cat >$(pwd)/data.json <<EOL
[
  {
    "labels": {
       "alertname": "DiskRunningFull",
       "dev": "sda1",
       "instance": "example1"
     },
     "annotations": {
        "info": "The disk sda1 is running full",
        "summary": "please check the instance example1"
      }
  },
  {
    "labels": {
       "alertname": "DiskRunningFull",
       "dev": "sdb2",
       "instance": "example2"
     },
     "annotations": {
        "info": "The disk sdb2 is running full",
        "summary": "please check the instance example2"
      }
  },
  {
    "labels": {
       "alertname": "DiskRunningFull",
       "dev": "sda1",
       "instance": "example3",
       "severity": "critical"
     }
  },
  {
    "labels": {
       "alertname": "DiskRunningFull",
       "dev": "sda1",
       "instance": "example3",
       "severity": "warning"
     }
  }
]
EOL
curl -XPOST --data @data.json http://192.168.122.37:19193/api/v1/alerts




docker run --name ding -d -p18060:8060 \
-v $(pwd)/prometheus-webhook-dingtalk/config.yml:/etc/prometheus-webhook-dingtalk/config.yml \
timonwong/prometheus-webhook-dingtalk:v1.4.0 \
--config.file=/etc/prometheus-webhook-dingtalk/config.yml \
--web.enable-ui \
--web.enable-lifecycle \
--log.level=debug



docker run -d -p 19000:9000 --name=webhook -v $(pwd)/webhook:/etc/webhook  almir/webhook:2.6.11 -verbose -hooks=/etc/webhook/hooks.json -hotreload

cat >$(pwd)/webhook/hooks.json <<EOL
[
  {
    "id": "redeploy-webhook",
    "execute-command": "/etc/webhook/redeploy.sh",
    "command-working-directory": "/tmp",
    "response-message": "I got the payload!"
  }
]
EOL
# https://github.com/adnanh/webhook
# https://github.com/adnanh/webhook/blob/master/docs/Hook-Examples.md

ref