abstract

192.168.33.2  主机名为n2,  安装coredns
192.168.33.11 主机名为n11, 安装etcd
192.168.33.12 主机名为n12, 安装etcd
192.168.33.13 主机名为n13, 安装etcd
192.168.33.14 主机名为n14, 安装consul(server=true)
192.168.33.15 主机名为n15, 安装consul(server=true)
192.168.33.16 主机名为n16, 安装consul(server=true)
192.168.33.21 主机名为n21, 安装consul(server=false)、安装flanneld、docker
192.168.33.22 主机名为n22, 安装consul(server=false)、安装flanneld、docker
192.168.33.23 主机名为n23, 安装consul(server=false)、安装flanneld、docker
192.168.33.24 主机名为n24, 安装consul(server=false)、安装flanneld、docker、traefik
uname -r
4.4.233-1.el7.elrepo.x86_64
cat /etc/redhat-release
CentOS Linux release 7.8.2003 (Core)
# 网卡为 eth1
[vagrant@n13 ~]$ ip a show eth1
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 08:00:27:15:3d:14 brd ff:ff:ff:ff:ff:ff
    inet 192.168.33.13/24 brd 192.168.33.255 scope global noprefixroute eth1
       valid_lft forever preferred_lft forever
    inet6 fe80::a00:27ff:fe15:3d14/64 scope link
       valid_lft forever preferred_lft forever

requestments

os

set -eux;
cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup \
&& sed -i "s@#baseurl@baseurl@g" /etc/yum.repos.d/CentOS-Base.repo \
&& sed -i "s@mirrorlist=http@#mirrorlist=http@g" /etc/yum.repos.d/CentOS-Base.repo \
&& sed -i "s@baseurl=.*/centos@baseurl=https://mirrors.huaweicloud.com/centos@g" /etc/yum.repos.d/CentOS-Base.repo \
&& yum clean all \
&& yum makecache \
&& yum --exclude=kernel* update -y


rpm -qa|grep epel-release |xargs --no-run-if-empty sudo yum remove -y
sudo yum install -y epel-release
sudo yum makecache fast

rpm -qa|grep elrepo-release |xargs --no-run-if-empty sudo yum remove -y
sudo rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
sudo yum install -y https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm

sudo yum makecache fast

# yum --enablerepo=elrepo-kernel -y install kernel-ml kernel-ml-devel
sudo yum --enablerepo=elrepo-kernel -y install kernel-lt kernel-lt-devel

sudo grub2-set-default 0;
sudo grub2-mkconfig -o /etc/grub2.cfg
sudo grubby --default-kernel
sudo reboot

# rpm -qa|grep kernel-headers|grep 3.10 |xargs --no-run-if-empty sudo yum remove -y
# sudo yum --enablerepo=elrepo-kernel -y install kernel-ml-headers
sudo yum --enablerepo=elrepo-kernel -y install kernel-lt-headers

# rpm -e --nodeps nginx
 
sudo yum install -y ca-certificates traceroute htop whois psmisc bc aria2 nmap openssh-server lsof net-tools curl wget git vim jq socat conntrack ipvsadm ipset sysstat libseccomp gcc gcc-c++ cmake make bzip2 automake autoconf libtool flex bison pcre-devel zlib-devel openssl openssl-devel bridge-utils bind-utils libnfnetlink-devel libnl3 libnl3-devel systemd-devel libuuid-devel device-mapper-persistent-data lvm2 libmnl-devel libnftnl-devel libnetfilter_conntrack-devel libnetfilter_queue-devel libpcap-devel


# 安装VBoxLinuxAdditions 非必须
sudo mkdir -p /mnt/cd
sudo mount -o ro /dev/cdrom /mnt/cd/ && cd /mnt/cd && sudo ./VBoxLinuxAdditions.run




grep -e "^docker" /etc/group >& /dev/null
if [ $? -ne 0 ]
then
    groupadd docker
fi

sudo usermod --append -G docker vagrant


sudo sh -c "cat > /etc/modules-load.d/10-virtio-net.conf"<<EOF
overlay
br_netfilter
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
EOF

sudo systemctl daemon-reload && \
sudo systemctl enable systemd-modules-load.service && \
sudo systemctl start systemd-modules-load.service && \
sudo systemctl status systemd-modules-load.service -l 


sudo timedatectl set-timezone Asia/Shanghai
sudo systemctl disable firewalld
sudo systemctl stop firewalld
sudo sed -ri '/^[^#]*SELINUX=/s#=.+$#=disabled#' /etc/selinux/config




sudo sh -c "cat > /etc/hosts"<<EOF
192.168.33.11   n11
192.168.33.12   n12
192.168.33.13   n13
192.168.33.21   n21
192.168.33.22   n22
192.168.33.23   n23
192.168.33.24   n24
192.168.33.25   n25
192.168.33.26   n26
192.168.33.27   n27
192.168.33.28   n28
192.168.33.29   n29
192.168.33.30   n30
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
EOF

sudo sh -c "cat > /etc/sysctl.d/mysysctl.conf"<<EOF
fs.file-max = 1024000
fs.inotify.max_user_instances = 8192
fs.inotify.max_user_watches = 89100
#net.core.default_qdisc = fq
net.core.somaxconn = 65535
net.core.netdev_max_backlog = 4096
net.ipv4.ip_forward = 1
#net.ipv4.tcp_congestion_control = bbr
net.ipv4.tcp_max_orphans = 65535
net.ipv4.tcp_max_syn_backlog = 4096
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_tw_reuse = 0
#net.ipv4.tcp_tw_recycle = 0
#/proc/sys/net/ipv4/tcp_tw_recycle: No such file or directory
net.ipv4.tcp_notsent_lowat = 16384
net.ipv4.tcp_slow_start_after_idle = 0
net.ipv4.tcp_fastopen = 3
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
user.max_user_namespaces = 15000
vm.max_map_count = 262144
EOF
sysctl --system

sudo sh -c 'echo "* soft nofile 65535" >> /etc/security/limits.conf'
sudo sh -c 'echo "* hard nofile 65535" >> /etc/security/limits.conf'

bin

mkdir -p /shell/install2/
cd /shell/install2/

# 把需要的二进制手工下载解压缩
curl -fksSL -O https://github.com/etcd-io/etcd/releases/download/v3.4.12/etcd-v3.4.12-linux-amd64.tar.gz
curl -fksSL -O https://github.com/coreos/flannel/releases/download/v0.12.0/flannel-v0.12.0-linux-amd64.tar.gz
curl -fksSL -O https://download.docker.com/linux/static/stable/x86_64/docker-19.03.12.tgz
curl -fksSL -O https://github.com/coredns/coredns/releases/download/v1.7.0/coredns_1.7.0_linux_amd64.tgz
curl -fksSL -O https://github.com/containous/traefik/releases/download/v2.2.11/traefik_v2.2.11_linux_amd64.tar.gz

mkdir -p coredns-1.7.0
tar -xvf coredns_1.7.0_linux_amd64.tgz -C coredns-1.7.0

tar -xvf etcd-v3.4.12-linux-amd64.tar.gz

mkdir -p docker-19.03.12
tar -xvf docker-19.03.12.tgz -C docker-19.03.12

mkdir -p flanneld-v0.12.0
tar -xvf flannel-v0.12.0-linux-amd64.tar.gz -C flanneld-v0.12.0

mkdir -p traefik-v2.2.11
tar -xvf traefik_v2.2.11_linux_amd64.tar.gz -C traefik-v2.2.11

mkdir cfssl-v1.4.1
cd cfssl-v1.4.1
ver='1.4.1'
for i in "cfssl" "cfssl-bundle" "cfssl-certinfo" "cfssl-newkey" "cfssl-scan" "cfssljson" "mkbundle" "multirootca"; do
    curl -sSL -o ${i} https://github.com/cloudflare/cfssl/releases/download/v${ver}/${i}_${ver}_linux_amd64
done

gentcert

cd /shell/install2/
git clone https://github.com/dyrnq/centos7-docker-flanneld-consul-example.git cert
# 准备证书
cd /shell/install2/cert

./gentcert.sh

install-etcd-v3.4.12

在 192.168.33.11、192.168.33.12、192.168.33.13 安装

sudo cp /shell/install2/etcd-v3.4.12-linux-amd64/etcd /usr/local/bin/
sudo cp /shell/install2/etcd-v3.4.12-linux-amd64/etcdctl /usr/local/bin/
sudo mkdir -p /opt/etcd-data
sudo chmod 700 /opt/etcd-data
sudo mkdir -p /etc/etcd
sudo cp -r /shell/install2/cert/tmp2/*.pem /etc/etcd/
ip4=$(/sbin/ip -o -4 addr list eth1 | awk '{print $4}' |cut -d/ -f1);
tmpn=$(echo -n ${ip4} |awk -F "." '{print $NF}')
sudo cp /shell/install2/cert/etcd.conf.yml /etc/etcd && \
sudo cp /shell/install2/cert/etcd.service /lib/systemd/system/ && \
sudo sed -i "s@^name:.*@name: 'n${tmpn}'@g" /etc/etcd/etcd.conf.yml && \
sudo sed -i "s@^data-dir:.*@data-dir: /opt/etcd-data@g" /etc/etcd/etcd.conf.yml && \
sudo sed -i "s@^listen-peer-urls:.*@listen-peer-urls: https://${ip4}:2380@g" /etc/etcd/etcd.conf.yml && \
sudo sed -i "s@^listen-client-urls:.*@listen-client-urls: https://${ip4}:2379@g" /etc/etcd/etcd.conf.yml && \
sudo sed -i "s@^initial-advertise-peer-urls:.*@initial-advertise-peer-urls: https://${ip4}:2380@g" /etc/etcd/etcd.conf.yml && \
sudo sed -i "s@^advertise-client-urls:.*@advertise-client-urls: https://${ip4}:2379@g" /etc/etcd/etcd.conf.yml && \
sudo sed -i "s@^initial-cluster:.*@initial-cluster: etcd-11=https://192.168.33.11:2380,etcd-12=https://192.168.33.12:2380,etcd-13=https://192.168.33.13:2380@g" /etc/etcd/etcd.conf.yml && \
cat /etc/etcd/etcd.conf.yml && \
cat /lib/systemd/system/etcd.service && \
sudo systemctl daemon-reload && \
sudo systemctl enable etcd.service && \
sudo systemctl restart etcd.service;
sudo systemctl status etcd.service -l
## 查看etcd集群状态
[vagrant@n11 ~]$ sudo /usr/local/bin/etcdctl endpoint health \
--endpoints "https://192.168.33.11:2379,https://192.168.33.12:2379,https://192.168.33.13:2379" \
--cacert=/etc/etcd/etcd-ca.pem \
--cert=/etc/etcd/etcd-healthcheck-client.pem \
--key=/etc/etcd/etcd-healthcheck-client-key.pem \
--cluster=true

https://192.168.33.11:2379 is healthy: successfully committed proposal: took = 48.748634ms
https://192.168.33.12:2379 is healthy: successfully committed proposal: took = 49.391402ms
https://192.168.33.13:2379 is healthy: successfully committed proposal: took = 54.411539ms

install-flanneld-v0.12.0

在 192.168.33.21、192.168.33.22、192.168.33.23 安装flanneld和docker

sudo mkdir -p /etc/etcd
sudo cp -r /shell/install2/cert/tmp2/*.pem /etc/etcd/
sudo cp /shell/install2/etcd-v3.4.12-linux-amd64/etcd /usr/local/bin/
sudo cp /shell/install2/etcd-v3.4.12-linux-amd64/etcdctl /usr/local/bin/
sudo cp /shell/install2/flanneld-v0.12.0/flanneld /usr/local/bin/
sudo cp /shell/install2/flanneld-v0.12.0/mk-docker-opts.sh /usr/local/bin/
sudo cp /shell/install2/cfssl-v1.4.1/* /usr/local/bin
sudo cp /shell/install2/docker-19.03.12/docker/* /usr/local/bin
# 只操作一次
sudo ETCDCTL_API=2 /usr/local/bin/etcdctl \
--endpoints "https://192.168.33.11:2379" \
--debug \
--ca-file=/etc/etcd/etcd-ca.pem \
--cert-file=/etc/etcd/etcd-healthcheck-client.pem \
--key-file=/etc/etcd/etcd-healthcheck-client-key.pem \
mk /coreos.com/network/config '{ "Network": "10.5.0.0/16", "Backend": {"Type": "vxlan"} }'

# '{ "Network": "10.0.0.0/8", "Backend": {"Type": "vxlan"} }'
# 如果节点很多,修改为如上值
## 查看各节点分配情况
[vagrant@n22 ~]$ sudo ETCDCTL_API=2 /usr/local/bin/etcdctl \
--endpoints "https://192.168.33.11:2379" \
--ca-file=/etc/etcd/etcd-ca.pem \
--cert-file=/etc/etcd/etcd-healthcheck-client.pem \
--key-file=/etc/etcd/etcd-healthcheck-client-key.pem \
ls /coreos.com/network/subnets/
/coreos.com/network/subnets/10.5.17.0-24
/coreos.com/network/subnets/10.5.7.0-24
/coreos.com/network/subnets/10.5.36.0-24
/coreos.com/network/subnets/10.5.90.0-24
sudo mkdir -p /etc/docker/
sudo mkdir -p /etc/containerd/
sudo mkdir -p /var/lib/docker/
sudo mkdir -p /var/lib/containerd/
sudo mkdir -p /etc/cni/net.d
sudo mkdir -p /opt/cni/bin

sudo cp /shell/install2/cert/flanneld.service /lib/systemd/system/ && \
sudo systemctl daemon-reload && \
cat /lib/systemd/system/flanneld.service && \
sudo systemctl enable flanneld.service && \
sudo systemctl start flanneld.service && \
sudo systemctl status flanneld.service -l

sudo cp /shell/install2/cert/docker.service /lib/systemd/system/ && \
sudo cp /shell/install2/cert/docker.socket /lib/systemd/system/ && \
sudo cp /shell/install2/cert/containerd.service /lib/systemd/system/ && \
sudo cp /shell/install2/cert/daemon.json /etc/docker/ && \
sudo systemctl daemon-reload && \
cat /lib/systemd/system/docker.service && \
sudo systemctl enable docker.service && \
sudo systemctl start docker.service && \
sudo systemctl status docker.service -l

install-coredns-1.7.0

在 192.168.33.2 安装

sudo mkdir -p /etc/coredns/
sudo mkdir -p /etc/etcd
sudo cp -r /shell/install2/cert/tmp2/*.pem /etc/etcd/
sudo cp /shell/install2/etcd-v3.4.12-linux-amd64/etcd /usr/local/bin/
sudo cp /shell/install2/etcd-v3.4.12-linux-amd64/etcdctl /usr/local/bin/
sudo cp /shell/install2/coredns-1.7.0/coredns /usr/local/bin/
sudo cp /shell/install2/cert/Corefile /etc/coredns/Corefile

sudo cp /shell/install2/cert/coredns.service /lib/systemd/system/ && \
sudo systemctl daemon-reload && \
cat /lib/systemd/system/coredns.service && \
sudo systemctl enable coredns.service && \
sudo systemctl start coredns.service && \
sudo systemctl status coredns.service -l

install-consul-1.8.3

192.168.33.14、192.168.33.15、192.168.33.16 机器上安装consul

cd /shell/install2/
mkdir -p /shell/install2/consul-1.8.3
curl -Ls https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip -o consul.zip
unzip consul.zip


consul keygen
GroHLpFsmeLrTaTxGjHa7zhoi5zhqMbu+C3y0yuTUjY=


ip4=$(/sbin/ip -o -4 addr list eth1 | awk '{print $4}' |cut -d/ -f1);
tmpn=$(echo -n ${ip4} |awk -F "." '{print $NF}')
sudo mkdir -p /etc/consul.d && \
sudo mkdir -p /data/consul && \
sudo cp /shell/install2/consul-1.8.3/consul /usr/local/bin && \
sudo cp /shell/install2/cert/consul-server.json /etc/consul.d && \
sudo sed -i "s@_encrypt_@GroHLpFsmeLrTaTxGjHa7zhoi5zhqMbu+C3y0yuTUjY=@g" /etc/consul.d/consul-server.json && \
sudo cp /shell/install2/cert/consul.service /lib/systemd/system/ && \
sudo sed -i "s@_BINDIP_@${ip4}@g" /lib/systemd/system/consul.service && \
sudo sed -i "s@_NODENAME_@n${tmpn}@g" /lib/systemd/system/consul.service && \
sudo systemctl daemon-reload && \
cat /lib/systemd/system/consul.service && \
sudo systemctl enable consul.service && \
sudo systemctl start consul.service && \
sudo systemctl status consul.service -l


[vagrant@n15 ~]$ consul operator raft list-peers
Node   ID                                    Address             State     Voter  RaftProtocol
"n15"  77be84ed-d803-2a2a-5d1a-2bbc57409ff8  192.168.33.15:8300  follower  true   3
"n16"  d9fe1ba8-094d-1e34-e430-7c98adbb9fd9  192.168.33.16:8300  leader    true   3
"n14"  d5cadf6b-a881-cd7b-96bc-8c5b87903c2d  192.168.33.14:8300  follower  true   3

192.168.33.21、192.168.33.22、192.168.33.23、192.168.33.24 机器上安装consul

ip4=$(/sbin/ip -o -4 addr list eth1 | awk '{print $4}' |cut -d/ -f1);
tmpn=$(echo -n ${ip4} |awk -F "." '{print $NF}')
sudo mkdir -p /etc/consul.d && \
sudo mkdir -p /data/consul && \
sudo cp /shell/install2/consul-1.8.3/consul /usr/local/bin && \
sudo cp /shell/install2/cert/consul-node.json /etc/consul.d && \
sudo sed -i "s@_encrypt_@GroHLpFsmeLrTaTxGjHa7zhoi5zhqMbu+C3y0yuTUjY=@g" /etc/consul.d/consul-node.json && \
sudo cp /shell/install2/cert/consul.service /lib/systemd/system/ && \
sudo sed -i "s@_BINDIP_@${ip4}@g" /lib/systemd/system/consul.service && \
sudo sed -i "s@_NODENAME_@n${tmpn}@g" /lib/systemd/system/consul.service && \
sudo systemctl daemon-reload && \
cat /lib/systemd/system/consul.service && \
sudo systemctl enable consul.service && \
sudo systemctl start consul.service && \
sudo systemctl status consul.service -l



[vagrant@n21 ~]$ consul members
Node   Address             Status  Type    Build  Protocol  DC   Segment
"n14"  192.168.33.14:8301  alive   server  1.8.3  2         dc1  <all>
"n15"  192.168.33.15:8301  alive   server  1.8.3  2         dc1  <all>
"n16"  192.168.33.16:8301  alive   server  1.8.3  2         dc1  <all>
"n21"  192.168.33.21:8301  alive   client  1.8.3  2         dc1  <default>
"n22"  192.168.33.22:8301  alive   client  1.8.3  2         dc1  <default>
"n23"  192.168.33.23:8301  alive   client  1.8.3  2         dc1  <default>

install-traefik-2.2.11

192.168.33.24 机器上安装traefik2

sudo mkdir -p /etc/traefik && \
sudo mkdir -p /etc/traefik/file && \
sudo cp /shell/install2/traefik-v2.2.11/traefik /usr/local/bin && \
sudo cp /shell/install2/cert/traefik.toml /etc/traefik && \
sudo cp /shell/install2/cert/traefik.service /lib/systemd/system/ && \
sudo systemctl daemon-reload && \
cat /lib/systemd/system/traefik.service && \
sudo systemctl enable traefik.service && \
sudo systemctl start traefik.service && \
sudo systemctl status traefik.service -l

test

container-to-container

[vagrant@n21 ~]$ docker run -d --name 21-nginx nginx:1.18.0
[vagrant@n21 ~]$ docker inspect -f='{{ .NetworkSettings.IPAddress }}' 21-nginx
10.5.7.2


[vagrant@n22 ~]$ docker run -d --name 22-nginx nginx:1.18.0
[vagrant@n22 ~]$ docker inspect -f='{{ .NetworkSettings.IPAddress }}' 22-nginx
10.5.36.2

[vagrant@n22 ~]$ docker exec -it 22-nginx bash -c 'sed -i "s|deb.debian.org|mirrors.huaweicloud.com|g" /etc/apt/sources.list && \
sed -i "s|security.debian.org|mirrors.huaweicloud.com|g" /etc/apt/sources.list && \
apt-get clean && \
apt-get update && \
apt-get -y upgrade && \
apt-get install -yq curl iproute2 iputils-ping psmisc procps nmap'


[vagrant@n22 ~]$ docker exec -it 22-nginx bash -c "ping -c4 10.5.7.2"
PING 10.5.7.2 (10.5.7.2) 56(84) bytes of data.
64 bytes from 10.5.7.2: icmp_seq=1 ttl=62 time=0.518 ms
64 bytes from 10.5.7.2: icmp_seq=2 ttl=62 time=1.18 ms
64 bytes from 10.5.7.2: icmp_seq=3 ttl=62 time=1.09 ms
64 bytes from 10.5.7.2: icmp_seq=4 ttl=62 time=1.22 ms

--- 10.5.7.2 ping statistics ---
4 packets transmitted, 4 received, 0% packet loss, time 9ms
rtt min/avg/max/mdev = 0.518/1.002/1.218/0.283 ms

[vagrant@n22 ~]$ docker exec -it 22-nginx bash -c "echo `curl -o /dev/null -s -w %{http_code} --connect-timeout 1 --max-time 1  http://10.5.7.2`"
200

[vagrant@n22 ~]$ docker exec -it 22-nginx bash -c "nc -nvz 10.5.7.2 80"
(UNKNOWN) [10.5.7.2] 80 (?) open

host-to-container

[vagrant@n22 ~]$ ping -c2 10.5.7.2
PING 10.5.7.2 (10.5.7.2) 56(84) bytes of data.
64 bytes from 10.5.7.2: icmp_seq=1 ttl=63 time=1.21 ms
64 bytes from 10.5.7.2: icmp_seq=2 ttl=63 time=1.26 ms

--- 10.5.7.2 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1174ms
rtt min/avg/max/mdev = 1.219/1.242/1.266/0.042 ms
[vagrant@n22 ~]$ curl http://10.5.7.2
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>

container-dns

## 手工注册容器域名和ip
sudo /usr/local/bin/etcdctl put \
--endpoints "https://192.168.33.11:2379,https://192.168.33.12:2379,https://192.168.33.13:2379" \
--cacert=/etc/etcd/etcd-ca.pem \
--cert=/etc/etcd/etcd-healthcheck-client.pem \
--key=/etc/etcd/etcd-healthcheck-client-key.pem \
/skydns/local/dev/nginx21 '{"host":"10.5.7.2","ttl":60}'
## 手工注册容器域名和ip
sudo /usr/local/bin/etcdctl put \
--endpoints "https://192.168.33.11:2379,https://192.168.33.12:2379,https://192.168.33.13:2379" \
--cacert=/etc/etcd/etcd-ca.pem \
--cert=/etc/etcd/etcd-healthcheck-client.pem \
--key=/etc/etcd/etcd-healthcheck-client-key.pem \
/skydns/local/dev/nginx22 '{"host":"10.5.36.2","ttl":60}'

## 宿主机nslookup
[vagrant@n22 ~]$ nslookup nginx21.dev.local 192.168.33.2
Server:		192.168.33.2
Address:	192.168.33.2#53

Name:	nginx21.dev.local
Address: 10.5.7.2


## 在容器上ping域名
[vagrant@n22 ~]$ docker run -it --rm --dns=192.168.33.2 praqma/network-multitool sh -c "ping -c2 nginx21.dev.local"
The directory /usr/share/nginx/html is not mounted.
Over-writing the default index.html file with some useful information.
PING nginx21.dev.local (10.5.7.2) 56(84) bytes of data.
64 bytes from 10.5.7.2 (10.5.7.2): icmp_seq=1 ttl=62 time=0.537 ms
64 bytes from 10.5.7.2 (10.5.7.2): icmp_seq=2 ttl=62 time=1.22 ms

--- nginx21.dev.local ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1059ms
rtt min/avg/max/mdev = 0.537/0.880/1.223/0.343 ms

## 在容器上nslookup域名
[vagrant@n22 ~]$ docker run -it --rm --dns=192.168.33.2 praqma/network-multitool sh -c "nslookup nginx21.dev.local"
The directory /usr/share/nginx/html is not mounted.
Over-writing the default index.html file with some useful information.
Server:		192.168.33.2
Address:	192.168.33.2#53

Name:	nginx21.dev.local
Address: 10.5.7.2

service-discovery

## 在33.21上操作
docker rm -f reg;docker run --restart=always -d --name=reg --net=host --volume=/var/run/docker.sock:/tmp/docker.sock gliderlabs/registrator:master -internal=true consul://127.0.0.1:8500
docker rm -f foo1 foo2;
docker run -d --restart always \
-l "SERVICE_NAME=foo" \
-l "SERVICE_80_CHECK_HTTP=/health" \
-l "SERVICE_TAGS=traefik.enable=true,traefik.http.routers.foo.entrypoints=http,traefik.http.routers.foo.rule=Host(\`foo.com\`)" \
--name foo1 containous/whoami:v1.5.0
docker run -d --restart always \
-l "SERVICE_NAME=foo" \
-l "SERVICE_80_CHECK_HTTP=/health" \
-l "SERVICE_TAGS=traefik.enable=true,traefik.http.routers.foo.entrypoints=http,traefik.http.routers.foo.rule=Host(\`foo.com\`)" \
--name foo2 containous/whoami:v1.5.0

## 在33.22上操作
docker rm -f reg;docker run --restart=always -d --name=reg --net=host --volume=/var/run/docker.sock:/tmp/docker.sock gliderlabs/registrator:master -internal=true consul://127.0.0.1:8500
docker rm -f foo1 foo2;
docker run -d --restart always \
-l "SERVICE_NAME=foo" \
-l "SERVICE_80_CHECK_HTTP=/health" \
-l "SERVICE_TAGS=traefik.enable=true,traefik.http.routers.foo.entrypoints=http,traefik.http.routers.foo.rule=Host(\`foo.com\`)" \
--name foo1 containous/whoami:v1.5.0
docker run -d --restart always \
-l "SERVICE_NAME=foo" \
-l "SERVICE_80_CHECK_HTTP=/health" \
-l "SERVICE_TAGS=traefik.enable=true,traefik.http.routers.foo.entrypoints=http,traefik.http.routers.foo.rule=Host(\`foo.com\`)" \
--name foo2 containous/whoami:v1.5.0

## 在22.23上操作
docker rm -f reg;docker run --restart=always -d --name=reg --net=host --volume=/var/run/docker.sock:/tmp/docker.sock gliderlabs/registrator:master -internal=true consul://127.0.0.1:8500
docker rm -f foo1 foo2;
docker run -d --restart always \
-l "SERVICE_NAME=foo" \
-l "SERVICE_80_CHECK_HTTP=/health" \
-l "SERVICE_TAGS=traefik.enable=true,traefik.http.routers.foo.entrypoints=http,traefik.http.routers.foo.rule=Host(\`foo.com\`)" \
--name foo1 containous/whoami:v1.5.0
docker run -d --restart always \
-l "SERVICE_NAME=foo" \
-l "SERVICE_80_CHECK_HTTP=/health" \
-l "SERVICE_TAGS=traefik.enable=true,traefik.http.routers.foo.entrypoints=http,traefik.http.routers.foo.rule=Host(\`foo.com\`)" \
--name foo2 containous/whoami:v1.5.0

# PathPrefix:/
# PathPrefixStrip:
# traefik.http.routers.{name-of-your-choice}.rule
# defaultRule = "Host(`{{ .Name }}.{{ index .Labels \"customLabel\"}}`)"
# traefik.http.routers.myrouter.entrypoints=web,websecure
# https://docs.traefik.io/routing/providers/consul-catalog/
# https://docs.traefik.io/providers/consul/#routing-configuration
# 查看注册的服务
[vagrant@n21 ~]$ curl -s -XGET http://127.0.0.1:8500/v1/health/service/foo?passing=true |jq -r '.[] | [.Service.Address, .Service.Port] |join(":") '
10.5.7.2:80
10.5.7.3:80
10.5.36.2:80
10.5.36.3:80
10.5.90.4:80
10.5.90.5:80

# 删除服务
curl -X PUT http://127.0.0.1:8500/v1/agent/service/deregister/n21:zen_tu:80


[vagrant@n24 ~]$ curl -H "host:foo.com" http://192.168.33.24:80
Hostname: 92a274472418
IP: 127.0.0.1
IP: 10.5.7.3
RemoteAddr: 10.5.17.0:49242
GET / HTTP/1.1
Host: foo.com
User-Agent: curl/7.29.0
Accept: */*
Accept-Encoding: gzip
X-Forwarded-For: 192.168.33.24
X-Forwarded-Host: foo
X-Forwarded-Port: 80
X-Forwarded-Proto: http
X-Forwarded-Server: n24
X-Real-Ip: 192.168.33.24

至此实现了在3台宿主机上分别启动了2个whoami的应用实例,就是总共6个应用实例,并通过traefik从consul上发现了这六个实例,然后实现了负载。

nomad

nomad实现了多机的任务调度,有点类似于ansible,可自动化实现多机docker run

other

flannel vxlan 8472

flannel udp 8285

cross-subnet

ref