说明

1
2
3
192.168.33.55   consul-server     nomad-server
192.168.33.56   consul-server     nomad-server
192.168.33.57   consul-server     nomad-server
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
  [55,56,57].each do |i|
    config.vm.define "n#{i}" do |node|
      node.vm.network "private_network", ip: "192.168.33.#{i}"
      node.vm.synced_folder "/data/vagrant/shell", "/shell"
      node.vm.network :forwarded_port, guest: 22, host: "2#{i}22", host_ip: "0.0.0.0"
      node.vm.provider "virtualbox" do |vb|
        vb.memory = "2048"
        vb.cpus = 2
      end
      node.vm.provision "shell", inline: <<-SHELL
        echo "vagrant:vagrant" | sudo chpasswd
        mkdir -p /data
        chown -R vagrant:vagrant /data
        hostnamectl set-hostname n#{i}
      SHELL
    end
  end

下载

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
# download nomad
curl -Ls https://releases.hashicorp.com/nomad/0.9.1/nomad_0.9.1_linux_amd64.zip -o nomad.zip
unzip nomad.zip
sudo chmod +x nomad
sudo mv nomad /usr/local/bin
# download consul
curl -Ls https://releases.hashicorp.com/consul/1.4.4/consul_1.4.4_linux_amd64.zip -o consul.zip
unzip consul.zip
sudo chmod +x consul
sudo mv consul /usr/local/bin

# sudo mkdir -p /etc/consul.d
# sudo mkdir -p /etc/nomad.d
# sudo mkdir -p /data/consul
# sudo mkdir -p /data/nomad

consul集群

server

1
2
3
4
5
6
7
# 单机模式
# /usr/local/bin/consul agent -dev
# /usr/local/bin/consul agent -server -bootstrap-expect 1 -data-dir /data/consul -bind=10.10.49.193 -ui -client 0.0.0.0 
/usr/local/bin/consul keygen
# 替换"encrypt": "goplCZgdmOFMZ2Q43To0jw=="
# sudo vi /etc/consul.d/config.json
sudo /usr/local/bin/consul agent -config-dir /etc/consul.d/
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
{
  "bootstrap_expect": 3,
  "server": true,
  "client_addr": "0.0.0.0",
  "datacenter": "east-aws",
  "data_dir": "/data/consul",
  "dns_config": {
    "enable_truncate": true,
    "only_passing": true
  },
  "encrypt": "goplCZgdmOFMZ2Q43To0jw==",
  "leave_on_terminate": true,
  "log_level": "INFO",
  "log_file": "/var/log/consul/",
  "rejoin_after_leave": true,
  "ui": true,
  "enable_debug": false,
  "retry_join": [
    "192.168.33.55",
    "192.168.33.56",
    "192.168.33.57"
  ],
  "retry_interval": "30s",
  "start_join": [
    "192.168.33.55",
    "192.168.33.56",
    "192.168.33.57"
  ],
  "disable_update_check": true
}
1
==> Multiple private IPv4 addresses found. Please configure one with 'bind' and/or 'advertise'.

client

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
./consul agent -data-dir="/data/consul" -bind=192.168.33.8 \
-retry-join=192.168.33.55 \
-retry-join=192.168.33.56 \
-retry-join=192.168.33.57 \
-retry-interval=10s \
-datacenter="east-aws" \
-server=false \
-log-level="INFO" \
-client=0.0.0.0
####
####
####
sudo netstat -tunlp |grep consul
tcp        0      0 192.168.33.8:8301       0.0.0.0:*               LISTEN      28326/./consul      
tcp6       0      0 :::8500                 :::*                    LISTEN      28326/./consul      
tcp6       0      0 :::8600                 :::*                    LISTEN      28326/./consul      
udp        0      0 192.168.33.8:8301       0.0.0.0:*                           28326/./consul      
udp6       0      0 :::8600                 :::*                                28326/./consul   

systemd

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
[Unit]
Description=Consul Startup process
After=network.target

[Service]
Type=simple
ExecStart=/bin/bash -c '/usr/local/bin/consul agent -bind=192.168.33.55 -config-dir /etc/consul.d/'
ExecStop=/usr/local/bin/consul leave
ExecReload=/bin/kill -HUP $MAINPID
TimeoutStartSec=0

[Install]
WantedBy=default.target
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
sudo vi /lib/systemd/system/consul.service;
sudo systemctl daemon-reload
sudo systemctl enable consul.service
sudo systemctl start consul.service
sudo systemctl status consul.service
## 查看端口情况
sudo netstat -tunlp |grep consul
tcp        0      0 192.168.33.55:8300      0.0.0.0:*               LISTEN      5687/consul         
tcp        0      0 192.168.33.55:8301      0.0.0.0:*               LISTEN      5687/consul         
tcp        0      0 192.168.33.55:8302      0.0.0.0:*               LISTEN      5687/consul         
tcp6       0      0 :::8500                 :::*                    LISTEN      5687/consul         
tcp6       0      0 :::8600                 :::*                    LISTEN      5687/consul         
udp        0      0 192.168.33.55:8301      0.0.0.0:*                           5687/consul         
udp        0      0 192.168.33.55:8302      0.0.0.0:*                           5687/consul         
udp6       0      0 :::8600                 :::*                                5687/consul 

端口

  • 8600 dns
  • 8500 http
  • 8501 https
  • 8502 grpc
  • 8300 server
  • 8301 serf_lan
  • 8302 serf_wan

HTTP API

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
#############################
# kv - Key/Value存储
# agent - Agent控制
# catalog - 管理nodes和services
# health - 管理健康监测
# session - Session操作
# acl - ACL创建和管理event - 用户Events
# status - Consul系统状态
# /v1/agent/checks : 返回本地agent注册的所有检查(包括配置文件和HTTP接口)
# /v1/agent/services : 返回本地agent注册的所有 服务
# /v1/agent/members : 返回agent在集群的gossip pool中看到的成员
# /v1/agent/self : 返回本地agent的配置和成员信息/v1/agent/join/<address> : 触发本地agent加入node/v1/agent/force-leave/<node>>: 强制删除node
# /v1/agent/check/register : 在本地agent增加一个检查项,使用PUT方法传输一个json格式的数据/v1/agent/check/deregister/<checkID> : 注销一个本地agent的检查项/v1/agent/# check/pass/<checkID> : 设置一个本地检查项的状态为passing/v1/agent/check/warn/<checkID> : 设置一个本地检查项的状态为warning/v1/agent/check/fail/<checkID> : 设置一个本地检查项的状态为critical
# /v1/agent/service/register : 在本地agent增加一个新的服务项,使用PUT方法传输一个json格式的数据/v1/agent/service/deregister/<serviceID> : 注销一个本地agent的服务项
# /v1/catalog/register : Registers a new node, service, or check/v1/catalog/deregister : Deregisters a node, service, or check/v1/catalog/datacenters : Lists known datacenters
# /v1/catalog/nodes : Lists nodes in a given DC
# /v1/catalog/services : Lists services in a given DC
# /v1/catalog/service/<service> : Lists the nodes in a given service
# /v1/catalog/node/<node> : Lists the services provided by a node
# /v1/healt/node/<node>: 返回node所定义的检查,可用参数?dc=
# /v1/health/checks/<service>: 返回和服务相关联的检查,可用参数?dc=
# /v1/health/service/<service>: 返回给定datacenter中给定node中service
# /v1/health/state/<state>: 返回给定datacenter中指定状态的服务,state可以是"any", "unknown", "passing", "warning", or "critical",可用参数?dc=
# /v1/session/create: Creates a new session
# /v1/session/destroy/<session>: Destroys a given session
# /v1/session/info/<session>: Queries a given session
# /v1/session/node/<node>: Lists sessions belonging to a node
# /v1/session/list: Lists all the active sessions
# /v1/acl/create: Creates a new token with policy
# /v1/acl/update: Update the policy of a token
# /v1/acl/destroy/<id>: Destroys a given token
# /v1/acl/info/<id>: Queries the policy of a given token
# /v1/acl/clone/<id>: Creates a new token by cloning an existing token
# /v1/acl/list: Lists all the active tokens
# /v1/status/leader : 返回当前集群的Raft leader
# /v1/status/peers : 返回当前集群中同事
#############################
# 获取服务
curl -s http://127.0.0.1:8500/v1/catalog/services |jq
# 集群成员
curl -s http://127.0.0.1:8500/v1/agent/members |jq
# kv
# kv 新增
curl --request PUT --data "hehe" http://127.0.0.1:8500/v1/kv/my-key
# kv 查询
curl -s http://127.0.0.1:8500/v1/kv/my-key |jq -r '.[].Value' |base64 -d
# kv 删除
curl --request DELETE http://127.0.0.1:8500/v1/kv/my-key
# 服务注册
curl -X PUT -d '{"id":"tomcat1","name":"tomcat","address":"192.168.33.101","port":9999,"tags":["dev"],"checks":[{"http":"http://192.168.33.101:9999","interval":"1s"}]}'     http://127.0.0.1:8500/v1/agent/service/register
curl -X PUT -d '{"id":"tomcat2","name":"tomcat","address":"192.168.33.102","port":9999,"tags":["dev"],"checks":[{"http":"http://192.168.33.102:9999","interval":"1s"}]}'     http://127.0.0.1:8500/v1/agent/service/register
curl -X PUT -d '{"id":"tomcat3","name":"tomcat","address":"192.168.33.109","port":9999,"tags":["dev"],"checks":[{"http":"http://192.168.33.109:9999","interval":"1s"}]}'     http://127.0.0.1:8500/v1/agent/service/register
# 服务查询
curl -s http://127.0.0.1:8500/v1/catalog/service/tomcat\?pretty
# 检查服务
curl -s http://127.0.0.1:8500/v1/health/service/tomcat\?pretty | grep -A 2 Status
curl -GLs http://127.0.0.1:8500/v1/health/checks/tomcat --data-urlencode 'pretty'  | grep -A 3 Status
# 取消注册服务
curl -X PUT http://127.0.0.1:8500/v1/agent/service/deregister/my-service-id
# dns发现
dig @127.0.0.1 -p 8600 tomcat.service.consul SRV
#################################################
# bootstrap server
sudo docker run -d -p 18300:8300 -p 18330:8301 -p 18360:8302 -p 18500:8500 -p 18600:8600  --name docker-consul-0 consul agent -server -bootstrap
# 获取 docker-consul-0 的 IP 地址
# docker inspect -f '{{.NetworkSettings.IPAddress}}' docker-consul-0`
sudo docker run -d -p 18301:8300 -p 18331:8301 -p 18361:8302 -p 18501:8500 -p 18601:8600  --name docker-consul-1 consul agent -server -join=`sudo docker inspect -f '{{.NetworkSettings.IPAddress}}' docker-consul-0`
sudo docker run -d -p 18302:8300 -p 18332:8301 -p 18362:8302 -p 18502:8500 -p 18602:8600  --name docker-consul-2 consul agent -server -join=`sudo docker inspect -f '{{.NetworkSettings.IPAddress}}' docker-consul-0`
sudo docker run -d -p 18303:8300 -p 18333:8301 -p 18363:8302 -p 18503:8500 -p 18603:8600  --name docker-consul-3 consul agent -server -join=`sudo docker inspect -f '{{.NetworkSettings.IPAddress}}' docker-consul-0`
sudo docker run -d -p 18304:8300 -p 18334:8301 -p 18364:8302 -p 18504:8500 -p 18604:8600  --name docker-consul-4 consul agent -client -join=`sudo docker inspect -f '{{.NetworkSettings.IPAddress}}' docker-consul-0`

nomad集群

server

1
sudo vi /etc/nomad.d/server.conf
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
log_level = "DEBUG"
data_dir = "/data/nomad"
name = "client1"
bind_addr = "0.0.0.0"
advertise {
  http = "192.168.33.51"
  rpc = "192.168.33.51"
  serf = "192.168.33.51"
}
ports {
  http = 4646
  rpc  = 4647
  serf = 4648
}
client {
  enabled = true
  servers = ["192.168.33.51:4647"]
  options {
    "driver.raw_exec.enable" = "1"
  }
}
server {
  enabled = true
  bootstrap_expect = 1
}
1
/usr/local/bin/nomad agent -config=/etc/nomad.d

systemd

参考