说明

1
2
3
4
5
6
7
8
192.168.33.11 etcd集群节点1,安装etcd。
192.168.33.12 etcd集群节点2,安装etcd。
192.168.33.13 etcd集群节点3,安装etcd。
192.168.33.21 apiserver集群节点1,安装kube-apiserver、kube-controller-manager、kube-scheduler并同时安装kubelet、kube-proxy和dockerd并label为master。安装keepalived+haproxy,设置KUBE_MASTER_VIP="https://192.168.33.100:8443"192.168.33.22 apiserver集群节点2,安装kube-apiserver、kube-controller-manager、kube-scheduler并同时安装kubelet、kube-proxy和dockerd并label为master。安装keepalived+haproxy,设置KUBE_MASTER_VIP="https://192.168.33.100:8443"192.168.33.23 apiserver集群节点3,安装kube-apiserver、kube-controller-manager、kube-scheduler并同时安装kubelet、kube-proxy和dockerd并label为master。安装keepalived+haproxy,设置KUBE_MASTER_VIP="https://192.168.33.100:8443"192.168.33.26 仅安装kubelet、kube-proxy和dockerd并label为worker。
192.168.33.27 仅安装kubelet、kube-proxy和dockerd并label为worker。

系统

Vagrantfile

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
  config.ssh.username = "vagrant"
  config.ssh.password = "vagrant"
  #config.ssh.insert_key = false

  [11,12,13,21,22,23,26,27].each do |i|
    config.vm.define "n#{i}" do |node|
      node.vm.network "private_network", ip: "192.168.33.#{i}"
      node.vm.synced_folder "/data/vagrant/shell", "/shell"
      node.vm.network :forwarded_port, guest: 22, host: "2#{i}22", host_ip: "0.0.0.0"

      node.vm.provider "virtualbox" do |vb|
        vb.memory = "2048"
        vb.cpus = 2
      end

      node.vm.provision "shell", inline: <<-SHELL
        echo "vagrant:vagrant" | sudo chpasswd
        mkdir -p /data
        echo IP=\"192.168.33.#{i}\" >>/data/env.sh
        echo ETCD_IP=\"192.168.33.#{i}\" >>/data/env.sh
        echo ETCD_NAME=\"etcd-#{i}\" >>/data/env.sh
        echo KUBE_NODENAME=\"n#{i}\" >>/data/env.sh
        echo KUBE_NODE_HOSTNAME=\"n#{i}.dev\" >>/data/env.sh
        echo HOSTNAME_=\"n#{i}\" >>/data/env.sh
        chown -R vagrant:vagrant /data
        hostnamectl set-hostname n#{i}
        timedatectl set-timezone Asia/Shanghai
      SHELL
    end
  end

centos7

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
# 所有的机器安装执行
#!/bin/bash
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.bak
wget -O /etc/yum.repos.d/CentOS-Base.repo https://mirrors.163.com/.help/CentOS7-Base-163.repo
rm -rf /etc/yum.repos.d/epel.repo
yum clean all
yum makecache
yum install -y epel-release
yum clean all
yum makecache
#yum repolist enabled
#yum repolist all
yum update
yum upgrade
yum install -y net-tools curl wget git vim jq socat conntrack ipvsadm ipset sysstat libseccomp gcc gcc-c++ cmake make bzip2 automake autoconf libtool flex bison pcre-devel zlib-devel openssl openssl-devel lsof htop patch lsof
yum install -y libnfnetlink-devel libnl3 libnl3-devel systemd-devel
yum install -y bridge-utils bind-utils psmisc
yum install -y device-mapper-persistent-data lvm2
#yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r)
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
sudo timedatectl set-timezone Asia/Shanghai
cat > /tmp/mysysctl.conf <<EOF
net.ipv4.ip_forward=1
net.bridge.bridge-nf-call-ip6tables=1
net.bridge.bridge-nf-call-iptables=1
fs.inotify.max_user_watches=89100
user.max_user_namespaces=15000
vm.max_map_count=262144
EOF
sudo cp /tmp/mysysctl.conf  /etc/sysctl.d/mysysctl.conf
sudo modprobe br_netfilter
sudo sysctl -p /etc/sysctl.d/mysysctl.conf
sudo groupadd docker
sudo usermod -a -G docker,vagrant vagrant
#sudo usermod --append --groups docker,vagrant vagrant
sudo gpasswd -a vagrant docker
sudo gpasswd --add $USER docker

#sudo gpasswd --add vagrant docker
id vagrant
newgrp docker

准备证书

etcd集群

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
cat > /tmp/etcd.service <<EOF
[Unit]
Description=etcd
Documentation=https://github.com/coreos/etcd
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
#User=etcd
Type=notify
ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) $ETCD_BIN_DIR/etcd \\
--name=$ETCD_NAME \\
--data-dir=$ETCD_DATA_DIR \\
--advertise-client-urls=https://$ETCD_IP:2379 \\
--listen-peer-urls=https://$ETCD_IP:2380 \\
--listen-client-urls=https://$ETCD_IP:2379 \\
--initial-advertise-peer-urls=https://$ETCD_IP:2380 \\
--initial-cluster=$ETCD_CLUS \\
--initial-cluster-state=new \\
--cert-file=$ETCD_PKI_DIR/etcd-server.pem \\
--key-file=$ETCD_PKI_DIR/etcd-server-key.pem \\
--trusted-ca-file=$ETCD_PKI_DIR/etcd-ca.pem \\
--client-cert-auth \\
--peer-cert-file=$ETCD_PKI_DIR/etcd-peer.pem \\
--peer-key-file=$ETCD_PKI_DIR/etcd-peer-key.pem \\
--peer-trusted-ca-file=$ETCD_PKI_DIR/etcd-ca.pem \\
--peer-client-cert-auth"
Restart=on-failure
RestartSec=10s
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF

k8s集群

master

1
2
3
4
5
6
7
8
keepalived
haproxy
kube-apiserver
kube-controller-manager
kube-scheduler
dockerd
kubelet
kube-proxy

worker

1
2
3
dockerd
kubelet
kube-proxy

部署cni之bridge

1
vi /etc/cni/net.d/19-bridge.conf;
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
{
	"name": "mynet",
	"type": "bridge",
	"bridge": "mynet0",
	"isDefaultGateway": true,
	"forceAddress": false,
	"ipMasq": true,
	"hairpinMode": true,
	"ipam": {
		"type": "host-local",
		"subnet": "172.19.0.0/16"
	}
}

部署cni之flanneld

部署cni之calico

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
# https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/calico#installing-with-the-kubernetes-api-datastore50-nodes-or-less
# https://docs.projectcalico.org/v3.6/maintenance/kubernetes-upgrade#upgrading-an-installation-that-uses-the-kubernetes-api-datastore
wget https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml
diff calico_deploy.yaml calico.yaml
298,299d297
<             - name: IP_AUTODETECTION_METHOD
<               value: "interface=eth1"  
313c311
<               value: "172.20.0.0/16"
---
>               value: "192.168.0.0/16"
1
2
3
4
5
kubectl apply -f calico_deploy.yaml
# 部署calico后cni之bridge自动失效
# calicoctl
vi calicoctl.yaml
kubectl apply -f calicoctl.yaml
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
apiVersion: v1
kind: Pod
metadata:
  name: calicoctl
  namespace: kube-system
spec:
  hostNetwork: true
  serviceAccountName: calico-node
  containers:
  - name: calicoctl
    image: quay.io/calico/ctl
    command: ["/bin/sh", "-c", "while true; do sleep 3600; done"]
    env:
    - name: DATASTORE_TYPE
      value: kubernetes
1
2
3
4
5
6
kubectl exec -ti -n kube-system calicoctl -- /calicoctl get workloadEndpoint -o wide
kubectl exec -ti -n kube-system calicoctl -- /calicoctl get node
kubectl exec -ti -n kube-system calicoctl -- /calicoctl get ipPool
kubectl exec -ti -n kube-system calicoctl -- /calicoctl get ipPool -o json
kubectl exec -ti -n kube-system calicoctl -- /calicoctl ipam show --ip=172.20.1.1
kubectl exec -ti -n kube-system calicoctl -- /calicoctl get bgpConfiguration

部署coredns

//部署

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
#!/bin/bash
#export http_proxy=
#export https_proxy=
#10.254.0.2 为cluster-ip,根据情况部署写死指定
#cluster.local. 为域名
wget https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/coredns.yaml.sed
wget https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/deploy.sh
chmod +x deploy.sh
./deploy.sh -i 10.254.0.2 -d cluster.local. > dns.yaml
kubectl apply -f dns.yaml;
## 升级coredns
curl -Ls -o coredns_1.5.0.yaml.sed https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/coredns.yaml.sed

//测试

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
# --service-cluster-ip-range (kube-apiserver)
# --cluster-cidr (kube-controller-manager、kube-proxy)
# --cluster-dns (kubelet)
kubectl config view -o jsonpath="{.clusters[?(@.name == \"kubernetes\")].cluster.server}";
docker run -d -p 9000:9000 --name portainer --restart always -v /var/run/docker.sock:/var/run/docker.sock -v /data/portainer_data:/data portainer/portainer
#####
kubectl get no,svc,ep,pod,deployment -o wide -n kube-system
kubectl get no,svc,ep,pod,deployment -o wide -n default 
kubectl get svc,ep,po -n kube-system -l k8s-app=kube-dns -o wide --show-labels
kubectl edit configmap coredns -n kube-system
kubectl get ds,rs,sts,cm --all-namespaces
kubectl get ev --all-namespaces
kubectl get pod -n kube-system -o wide --show-labels
#####
kubectl get svc,ep,po -n kube-system -l k8s-app=kubernetes-dashboard -o wide --show-labels
kubectl logs $(kubectl get po -o jsonpath={.items[0].metadata.name} -l k8s-app=kubernetes-dashboard -n kube-system) -n kube-system
kubectl describe pod $(kubectl get po -o jsonpath={.items[0].metadata.name} -l k8s-app=kubernetes-dashboard -n kube-system) -n kube-system
kubectl describe svc kubernetes-dashboard -n kube-system
#####
kubectl logs $(kubectl get pod -o json -l k8s-app=kube-dns -n kube-system | jq -r '.items | .[0] | .metadata | .name') -n kube-system
kubectl logs $(kubectl get pod -o jsonpath={.items[0].metadata.name} -l k8s-app=kube-dns -n kube-system) -n kube-system
kubectl describe pod $(kubectl get pod -o jsonpath={.items[0].metadata.name} -l k8s-app=kube-dns -n kube-system) -n kube-system
#########
kubectl get svc,ep,pod,cm -n ingress-nginx -l app.kubernetes.io/name=ingress-nginx -o wide --show-labels
kubectl logs $(kubectl get pod -n ingress-nginx -l app.kubernetes.io/name=ingress-nginx -o jsonpath={.items[0].metadata.name}) -n ingress-nginx
kubectl describe pod $(kubectl get pod -n ingress-nginx -l app.kubernetes.io/name=ingress-nginx -o jsonpath={.items[0].metadata.name}) -n ingress-nginx
## 查看coredns
kubectl run -it --rm --generator=run-pod/v1 --image=tutum/dnsutils dns -- /bin/sh
kubectl run -it --rm --generator=run-pod/v1 --image=nginx:1.15.11-alpine nginx1 -- /bin/sh
kubectl run -it --rm --generator=run-pod/v1 --image=nginx:1.15.11-perl nginx2 -- /bin/sh
kubectl run -it --rm --generator=run-pod/v1 --image=infoblox/dnstools dns-client
############
sudo ipvsadm -Ln -t 10.254.0.2:53 --stats
sudo ipvsadm --list -n --tcp-service 10.254.0.2:53
sudo ipvsadm --list -n --tcp-service 10.254.0.1:443
ping 10.254.0.1
ping 10.254.0.2
## apt-get install dnsutils inetutils-ping
## 查看53端口
nc -nvz 10.254.0.2 53
dig -t A -p 53 @10.254.0.2 kubernetes.default.svc.cluster.local. +short
dig kubernetes.default.svc.cluster.local. -t A +short
## 执行命令
nslookup kubernetes
## 返回如下信息
Server:		10.254.0.2
Address:	10.254.0.2#53

Name:	kubernetes.default.svc.cluster.local
Address: 10.254.0.1

部署metrics-server

//开启apiserver聚合层

1
2
3
4
5
6
7
8
9
## 开启聚合层
## https://kubernetes.io/docs/tasks/access-kubernetes-api/configure-aggregation-layer/#enable-kubernetes-apiserver-flags
--requestheader-client-ca-file=<path to aggregator CA cert>
--requestheader-allowed-names=front-proxy-client
--requestheader-extra-headers-prefix=X-Remote-Extra-
--requestheader-group-headers=X-Remote-Group
--requestheader-username-headers=X-Remote-User
--proxy-client-cert-file=<path to aggregator proxy cert>
--proxy-client-key-file=<path to aggregator proxy key>

//部署

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
mkdir metrics-server
cd metrics-server
# https://github.com/kubernetes-incubator/metrics-server/tree/master/deploy
url=https://raw.githubusercontent.com/kubernetes-incubator/metrics-server/master/deploy/1.8%2B
wget --no-check-certificate $url/aggregated-metrics-reader.yaml
wget --no-check-certificate $url/auth-delegator.yaml
wget --no-check-certificate $url/auth-reader.yaml
wget --no-check-certificate $url/metrics-apiservice.yaml
wget --no-check-certificate $url/metrics-server-deployment.yaml
wget --no-check-certificate $url/metrics-server-service.yaml
wget --no-check-certificate $url/resource-reader.yaml
# k8s.gcr.io/metrics-server-amd64:v0.3.1
# 修改metrics-server-deployment.yaml,添加证书
#        command:
#        - /metrics-server
#        - --metric-resolution=20s
#        - --requestheader-client-ca-file=/etc/kubernetes/pki/kubernetes-front-proxy-ca.pem
#        - --requestheader-allowed-names=aggregator
#        - --kubelet-insecure-tls
#        - --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP

# --metric-resolution=30s:从 kubelet 采集数据的周期;
# --requestheader-allowed-names=aggregator:允许请求 metrics-server API 的用户名,该名称与 kube-apiserver 的 --proxy-client-cert-file 指定的证书 CN 一致;
# --kubelet-preferred-address-types:优先使用 InternalIP 来访问 kubelet,这样可以避免节点名称没有 DNS 解析记录时,通过节点名称调用节点 kubelet API 失败的情况(未配置时默认的情况);

kubectl create -f .
# https://github.com/kubernetes-incubator/metrics-server/issues/45
# W0312 01:45:37.102313       1 x509.go:172] x509: subject with cn=front-proxy-client is not in the allowed list: [aggregator]
# E0312 01:45:37.102354       1 authentication.go:62] Unable to authenticate the request due to an error: [x509: subject with cn=front-proxy-client is not allowed, x509: certificate signed by unknown author
# https://github.com/kubernetes-incubator/metrics-server/issues/58
# error: Metrics not available for pod default
# https://github.com/kubernetes-incubator/metrics-server/issues/150
# unable to fetch metrics from Kubelet
# https://github.com/kubernetes-incubator/metrics-server/issues/146


# E0311 16:43:38.416358       1 manager.go:102] unable to fully collect metrics: [unable to fully scrape metrics from source kubelet_summary:n22.dev: unable to fetch metrics from Kubelet n22.dev (192.168.33.22): Get https://192.168.33.22:10250/stats/summary/: x509: cannot validate certificate for 192.168.33.22 because it doesn't contain any IP SANs, unable to fully scrape metrics from source kubelet_summary:n26.dev: unable to fetch metrics from Kubelet n26.dev (192.168.33.26): Get https://192.168.33.26:10250/stats/summary/: x509: cannot validate certificate for 192.168.33.26 because it doesn't contain any IP SANs, unable to fully scrape metrics from source kubelet_summary:n23.dev: unable to fetch metrics from Kubelet n23.dev (192.168.33.23): Get https://192.168.33.23:10250/stats/summary/: x509: cannot validate certificate for 192.168.33.23 because it doesn't contain any IP SANs, unable to fully scrape metrics from source kubelet_summary:n21.dev: unable to fetch metrics from Kubelet n21.dev (192.168.33.21): Get https://192.168.33.21:10250/stats/summary/: x509: cannot validate certificate for 192.168.33.21 because it doesn't contain any IP SANs, unable to fully scrape metrics from source kubelet_summary:n27.dev: unable to fetch metrics from Kubelet n27.dev (192.168.33.27): Get https://192.168.33.27:10250/stats/summary/: x509: cannot validate certificate for 192.168.33.27 because it doesn't contain any IP SANs]

# Error from server (Forbidden): nodes.metrics.k8s.io is forbidden: User “system:kube-proxy” cannot list nodes.metrics.k8s.io at the cluster scope
# Forbidden (user=system:anonymous, verb=get, resource=nodes, subresource=proxy)
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: view-metrics
rules:
- apiGroups:
    - metrics.k8s.io
  resources:
    - pods
    - nodes
  verbs:
    - get
    - list
    - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: view-metrics
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: view-metrics
subjects:
  - apiGroup: rbac.authorization.k8s.io
    kind: User
    name: system:kube-proxy
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: anonymous-logs-role
rules:
- apiGroups: [""]
  resources: ["nodes/proxy"]
  verbs: ["create", "get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: anonymous-logs-binding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: anonymous-logs-role
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: User
  name: system:anonymous
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
kubectl get apiservice v1beta1.metrics.k8s.io -o json
kubectl get pods -n kube-system -l k8s-app=metrics-server -o wide
kubectl get --raw "/apis/metrics.k8s.io"
# 查看node指标
kubectl get --raw "/apis/metrics.k8s.io/v1beta1/nodes"
# 查看pod指标
kubectl get --raw "/apis/metrics.k8s.io/v1beta1/pods"
kubectl get --raw "/apis/metrics.k8s.io/v1beta1" | jq .
kubectl logs $(kubectl get po -o json -n kube-system -l k8s-app=metrics-server | jq -r '.items | .[] | .metadata | .name') -n kube-system

# kubectl get --raw "/apis/metrics.k8s.io/v1beta1" | jq .
# error: You must be logged in to the server (Unauthorized)
# front-proxy-ca.pem证书问题,--requestheader-client-ca-file参数
# error: metrics not available yet
# 刚搭建完会没有数据,过几分钟就能收集到数据了
# '/sys/fs/cgroup/cpuset': operation not permitted
# Could not configure a source for OOM detection, disabling OOM events: open /dev/kmsg: no such file or directory

//测试

1
2
kubectl top node
kubectl top pod

部署dashboard

//部署

1
2
3
# [Release](https://github.com/kubernetes/dashboard)
# [Dashboard arguments](https://github.com/kubernetes/dashboard/wiki/Dashboard-arguments)
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml

//创建账户

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kube-system 
---
kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard2
  namespace: kube-system
spec:
  type: NodePort
  ports:
  - port: 443
    targetPort: 8443
    nodePort: 30014
  selector:
    k8s-app: kubernetes-dashboard
1
2
3
4
## 获取登录token
#kubectl create sa dashboard-admin -n kube-system
#kubectl create clusterrolebinding dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin
kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') | grep -E '^token' | awk '{print $2}'

部署ingress-nginx

//部署

1
2
3
4
#kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/mandatory.yaml
## service-nodeport.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/baremetal/service-nodeport.yaml
1
2
3
## pod固定
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
apiVersion: v1
kind: Service
metadata:
  name: ingress-nginx
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
  type: NodePort
  ports:
    - name: http
      port: 80
      targetPort: 80
      protocol: TCP
    - name: https
      port: 443
      targetPort: 443
      protocol: TCP
    - name: proxied-tcp-3306
      port: 3306
      targetPort: 3306
      protocol: TCP
      nodePort: 30016
  selector:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

//L4转发

1
2
3
4
5
6
7
kind: ConfigMap
apiVersion: v1
metadata:
  name: tcp-services
  namespace: ingress-nginx
data:
  3306: "default/mysql-svc:3306"

//L7转发

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: my-ingress
  annotations:
    nginx.ingress.kubernetes.io/use-regex: "true"
spec:
  rules:
  - host: demo.mydomain.com
    http:
      paths:
      - backend:
          serviceName: api
          servicePort: 80

//测试

1
curl -H "host:demo.mydomain.com" http://192.168.33.27:31025

部署traefik-ingress

//部署

1
2
3
4
5
6
7
8
9
# https://github.com/containous/traefik/tree/v1.7/examples/k8s
kubectl apply -f https://raw.githubusercontent.com/containous/traefik/v1.7/examples/k8s/traefik-rbac.yaml
# examples/k8s/traefik-deployment.yaml
kubectl apply -f https://raw.githubusercontent.com/containous/traefik/v1.7/examples/k8s/traefik-deployment.yaml
# examples/k8s/traefik-ds.yaml
#kubectl apply -f https://raw.githubusercontent.com/containous/traefik/v1.7/examples/k8s/traefik-ds.yaml
kubectl --namespace=kube-system get pod,svc -o wide
# examples/k8s/ui.yaml
kubectl apply -f https://raw.githubusercontent.com/containous/traefik/v1.7/examples/k8s/ui.yaml

//Ingress

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
---
apiVersion: v1
kind: Service
metadata:
  name: traefik-web-ui
  namespace: kube-system
spec:
  selector:
    k8s-app: traefik-ingress-lb
  ports:
  - name: web
    port: 80
    targetPort: 8080
---    
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: traefik-web-ui
  namespace: kube-system
  annotations:
    kubernetes.io/ingress.class: traefik
spec:
  rules:
  - host: traefik-ui.minikube
    http:
      paths:
      - path: /
        backend:
          serviceName: traefik-web-ui
          servicePort: 80

//测试

1
curl -fsSL http://$(kubectl get svc traefik-web-ui -n kube-system -o jsonpath='{.spec.clusterIP}'):80/api/providers |jq

部署helm

1
2
3
4
5
6
7
# 下载
wget https://get.helm.sh/helm-v2.14.3-linux-amd64.tar.gz
# 初始化SA
kubectl create serviceaccount --namespace kube-system tiller
kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
# 部署tiller
helm init --service-account tiller
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
Creating /home/vagrant/.helm
Creating /home/vagrant/.helm/repository
Creating /home/vagrant/.helm/repository/cache
Creating /home/vagrant/.helm/repository/local
Creating /home/vagrant/.helm/plugins
Creating /home/vagrant/.helm/starters
Creating /home/vagrant/.helm/cache/archive
Creating /home/vagrant/.helm/repository/repositories.yaml
Adding stable repo with URL: https://kubernetes-charts.storage.googleapis.com
Adding local repo with URL: http://127.0.0.1:8879/charts
$HELM_HOME has been configured at /home/vagrant/.helm.

Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.

Please note: by default, Tiller is deployed with an insecure 'allow unauthenticated users' policy.
To prevent this, run `helm init` with the --tiller-tls-verify flag.
For more information on securing your installation see: https://docs.helm.sh/using_helm/#securing-your-helm-installation
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
helm init --service-account tiller --upgrade --tiller-image sapcc/tiller:v2.14.3 --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts
# --tiller-image gcr.io/kubernetes-helm/tiller:v2.14.3 \
# --tiller-tls-cert /etc/kubernetes/ssl/tiller001.pem \
# --tiller-tls-key /etc/kubernetes/ssl/tiller001-key.pem \
# --tls-ca-cert /etc/kubernetes/ssl/ca.pem \
# --tiller-namespace kube-system 
# 验证安装状态
kubectl get deploy -n kube-system  tiller-deploy  --output yaml|grep  serviceAccount
kubectl describe deploy -n kube-system  tiller-deploy
kubectl get pod -n kube-system -o wide -l app=helm
kubectl -n kube-system get pods|grep tiller


## helm error installing the server could not find the requested resource

helm init --debug --dry-run \
--service-account tiller \
--tiller-namespace kube-system \
--upgrade \
--tiller-image sapcc/tiller:v2.14.3 \
--stable-repo-url https://mirror.azure.cn/kubernetes/charts/ \
|sed -e "s/apiVersion: extensions\/v1beta1/apiVersion: apps\/v1/g" \
|sed '/^\./d' \
|sed -e "/replicas: 1/a \  selector:\n\    matchLabels:\n\      app: helm\n\      name: tiller" \
|kubectl apply -f -

## 查看tiller
kubectl get deployments,pod -n kube-system -o wide -l app=helm -l name=tiller

# 添加fabric8库
helm repo add fabric8 https://fabric8.io/helm
helm search prometheus
helm fetch stable/prometheus
helm search jenkins
helm install --name my-release --set Persistence.StorageClass=slow stable/jenkins
helm get my-release
helm install stable/nginx-ingress --set controller.hostNetwork=true,rbac.create=true
helm repo add incubator http://storage.googleapis.com/kubernetes-charts-incubator 
helm install incubator/sparkoperator --namespace spark-operator

# 更换仓库

helm repo remove stable
helm repo add stable https://mirror.azure.cn/kubernetes/charts/
helm repo update
helm repo list
helm search fluentd

# 产生yaml
helm install stable/fluentd --version 2.0.0 --dry-run --debug |sed -n '/^---/,$p' > fluentd.yaml

部署operators

部署glusterfs集群

//准备机器

1
2
3
4
5
6
7
8
9
192.168.33.50 安装heketi服务器
192.168.33.55 glusterfs集群server
192.168.33.56 glusterfs集群server
192.168.33.57 glusterfs集群server
# 配置 /etc/hosts
192.168.33.50   n50.dev
192.168.33.55   n55.dev
192.168.33.56   n56.dev
192.168.33.57   n57.dev

//格式化磁盘

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# 格式化磁盘
# 查看CentOS 7支持的文件系统格式
cat /etc/filesystems
# 查看mount查看分区的文件系统格式
mount
# 查看磁盘
fdisk -l
# 挂载磁盘
# /dev/sda 
# /dev/sdb
# 格式化磁盘并挂载到/data/brick1
mkfs.xfs -i size=512 /dev/sdb
mkdir -p /data/brick1
echo '/dev/sdb /data/brick1 xfs defaults 1 2' >> /etc/fstab
mount -a && mount
###################
# 查看挂载情况
df |grep /data/brick1
# 参考
# https://docs.gluster.org/en/latest/Quick-Start-Guide/Quickstart/
# https://kubernetes.feisky.xyz/cha-jian-kuo-zhan/volume/glusterfs
# https://www.linuxidc.com/Linux/2017-02/140517.htm
# https://github.com/gluster/gluster-kubernetes/blob/master/docs/setup-guide.md

//编译安装

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
# 编译源代码
# https://docs.gluster.org/en/latest/Developer-guide/Building-GlusterFS/#building-from-source
sudo yum remove  epel-release-7-11.noarch && sudo yum install -y epel-release
sudo yum install centos-release-gluster -y
sudo yum clean all && sudo yum makecache
sudo yum install autoconf automake bison cmockery2-devel dos2unix flex fuse-devel glib2-devel libacl-devel libaio-devel libattr-devel libcurl-devel libibverbs-devel librdmacm-devel libtirpc-devel libtool libxml2-devel lvm2-devel make openssl-devel pkgconfig pyliblzma python-devel python-eventlet python-netifaces python-paste-deploy python-simplejson python-sphinx python-webob pyxattr readline-devel rpm-build sqlite-devel systemtap-sdt-devel tar userspace-rcu-devel
## 下载源代码
wget https://github.com/gluster/glusterfs/archive/v6.5.tar.gz
tar -xvf v6.5.tar.gz
cd glusterfs-6.5
./autogen.sh
./configure --sysconfdir=/etc --without-libtirpc --enable-gnfs
################################
GlusterFS configure summary
===========================
FUSE client          : yes
Infiniband verbs     : yes
epoll IO multiplex   : yes
fusermount           : yes
readline             : yes
georeplication       : yes
Linux-AIO            : yes
Enable Debug         : no
Enable ASAN          : no
Enable TSAN          : no
Use syslog           : yes
XML output           : yes
Unit Tests           : no
Track priv ports     : yes
POSIX ACLs           : yes
SELinux features     : yes
firewalld-config     : no
Events               : yes
EC dynamic support   : x64 sse avx
Use memory pools     : yes
Nanosecond m/atimes  : yes
Server components    : yes
Legacy gNFS server   : yes
IPV6 default         : no
Use TIRPC            : no
With Python          : 2.7
Cloudsync            : yes
################################
make
sudo make install
#############
ls /usr/local/sbin/ |grep gluster && ls /usr/local/bin/ |grep gluster
ls -l /etc/glusterfs/
cat /usr/local/lib/systemd/system/glusterd.service
# 修改 glusterd 目录
sed -i 's/var\/lib/opt/g' /etc/glusterfs/glusterd.vol
sudo yum install -y rpcbind attr psmisc
# 设置 glusterd.service 开机启动
sudo systemctl enable glusterd.service
# 启动 glusterd.service
sudo systemctl start glusterd.service
# 查看 glusterd.service
systemctl status glusterd.service
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
● glusterd.service - GlusterFS, a clustered file-system server
   Loaded: loaded (/usr/local/lib/systemd/system/glusterd.service; enabled; vendor preset: disabled)
   Active: active (running) since 五 2019-08-09 21:58:51 CST; 7s ago
     Docs: man:glusterd(8)
  Process: 3717 ExecStart=/usr/local/sbin/glusterd -p /var/run/glusterd.pid --log-level $LOG_LEVEL $GLUSTERD_OPTIONS (code=exited, status=0/SUCCESS)
 Main PID: 3718 (glusterd)
    Tasks: 9
   Memory: 10.2M
   CGroup: /system.slice/glusterd.service
           └─3718 /usr/local/sbin/glusterd -p /var/run/glusterd.pid --log-level INFO

8月 09 21:58:50 n55 systemd[1]: Starting GlusterFS, a clustered file-system server...
8月 09 21:58:51 n55 systemd[1]: Started GlusterFS, a clustered file-system server.

//创建测试卷

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
# 加载内核模块
sudo modprobe dm_snapshot
sudo modprobe dm_thin_pool
sudo modprobe dm_mirror
# 确认模块已加载
lsmod |egrep "dm_snapshot|dm_mirror|dm_thin_pool"
# 分区、格式化
vgcreate v_gluster /dev/sdb
lvcreate -n lv_gluster -L 2G vg_gluster
mkfs.ext4 /dev/v_gluster/lv_gluster
# 添加节点到集群
sudo /usr/local/sbin/gluster peer probe n56.dev
sudo /usr/local/sbin/gluster peer probe n57.dev
# 查看集群状态
sudo /usr/local/sbin/gluster peer status
# 创建test-volume 
sudo /usr/local/sbin/gluster volume create test-volume transport rdma \
n55.dev:/data/brick1/gv0 \
n56.dev:/data/brick1/gv0 \
n57.dev:/data/brick1/gv0

# volume create: test-volume: failed: The brick n55.dev:/data/brick1 is a mount point. Please create a sub-directory under the mount point and use that as the brick directory. Or use 'force' at the end of the command if you want to override this behavior.

# volume create: test-volume: failed: The brick n55.dev:/opt/gfs_data is being created in the root partition. It is recommended that you don't use the system's root partition for storage backend. Or use 'force' at the end of the command if you want to override this behavior.

sudo /usr/local/sbin/gluster volume start test-volume

# volume start: test-volume: failed: Commit failed on localhost. Please check log file for details.
# 好像不支持 rdma 协议

sudo /usr/local/sbin/gluster volume set test-volume config.transport tcp
sudo /usr/local/sbin/gluster volume info test-volume

Volume Name: test-volume
Type: Distribute
Volume ID: 5cf6bcd6-d2b3-495e-aa6c-601d54c23707
Status: Created
Snapshot Count: 0
Number of Bricks: 3
Transport-type: rdma
Bricks:
Brick1: n55.dev:/data/brick1/gv0
Brick2: n56.dev:/data/brick1/gv0
Brick3: n57.dev:/data/brick1/gv0
Options Reconfigured:
nfs.disable: on

sudo /usr/local/sbin/gluster volume status test-volume
Volume test-volume is not started


sudo /usr/local/sbin/gluster volume status test-volume
Status of volume: test-volume
Gluster process                             TCP Port  RDMA Port  Online  Pid
------------------------------------------------------------------------------
Brick n55.dev:/data/brick1/gv0              49153     0          Y       4086
Brick n56.dev:/data/brick1/gv0              49152     0          Y       7782
Brick n57.dev:/data/brick1/gv0              49152     0          Y       6744

Task Status of Volume test-volume
------------------------------------------------------------------------------
There are no active volume tasks

# 查询端口
sudo netstat -tunlp |grep gluster
tcp        0      0 0.0.0.0:49153           0.0.0.0:*               LISTEN      4086/glusterfsd
tcp        0      0 0.0.0.0:24007           0.0.0.0:*               LISTEN      3718/glusterd



# Unmount the volume on all the clients using the following command:
umount mount-point
# Stop the volumes using the following command:
gluster volume stop test-volume
# Mount the volume on all the clients.
mount -t glusterfs -o transport=rdma server1:/test-volume /mnt/glusterfs
# Change the transport type.
gluster volume set test-volume config.transport tcp,rdma
#########################
# 默认模式,既DHT, 也叫 分布卷: 将文件已hash算法随机分布到 一台服务器节点中存储。
gluster volume create test-volume server1:/exp1 server2:/exp2
# 复制模式,既AFR, 创建volume 时带 replica x 数量: 将文件复制到 replica x 个节点中。
gluster volume create test-volume replica 2 transport tcp server1:/exp1 server2:/exp2
# 条带模式,既Striped, 创建volume 时带 stripe x 数量: 将文件切割成数据块,分别存储到 stripe x 个节点中 ( 类似raid 0 )。
gluster volume create test-volume stripe 2 transport tcp server1:/exp1 server2:/exp2
# 分布式条带模式(组合型),最少需要4台服务器才能创建。 创建volume 时 stripe 2 server = 4 个节点: 是DHT 与 Striped 的组合型。
gluster volume create test-volume stripe 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4
# 分布式复制模式(组合型), 最少需要4台服务器才能创建。 创建volume 时 replica 2 server = 4 个节点:是DHT 与 AFR 的组合型。
gluster volume create test-volume replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4
# 条带复制卷模式(组合型), 最少需要4台服务器才能创建。 创建volume 时 stripe 2 replica 2 server = 4 个节点: 是 Striped 与 AFR 的组合型。
gluster volume create test-volume stripe 2 replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4
# 三种模式混合, 至少需要8台 服务器才能创建。 stripe 2 replica 2 , 每4个节点 组成一个 组。
gluster volume create test-volume stripe 2 replica 2 transport tcp server1:/exp1 server2:/exp2 server3:/exp3 server4:/exp4 server5:/exp5 server6:/exp6 server7:/exp7 server8:/exp8
# 验证可信存储池的状态
gluster peer status
gluster pool list

//部署heketi(物理器二进制安装)

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
wget https://github.com/heketi/heketi/releases/download/v9.0.0/heketi-v9.0.0.linux.amd64.tar.gz
tar -xvf heketi-v9.0.0.linux.amd64.tar.gz
cd heketi;
sudo mv heketi-cli /usr/local/bin
sudo mv heketi /usr/local/bin
heketi-cli --version
heketi-cli v9.0.0
sudo mkdir -p /etc/heketi/
sudo mkdir -p /var/lib/heketi

ssh-keygen -f /etc/heketi/heketi_key
# ssh-keygen -t rsa -q -f /etc/heketi/heketi_key -N ''
chmod 600 /etc/heketi/heketi_key.pub

ssh-copy-id -i /etc/heketi/heketi_key.pub vagrant@n55.dev
ssh-copy-id -i /etc/heketi/heketi_key.pub vagrant@n56.dev
ssh-copy-id -i /etc/heketi/heketi_key.pub vagrant@n57.dev
ssh vagrant@57.dev -i /etc/heketi/heketi_key

wget https://raw.githubusercontent.com/heketi/heketi/master/extras/systemd/heketi.env
# https://github.com/heketi/heketi/blob/master/extras/systemd/heketi.service

cd /lib/systemd/system/
sudo wget https://raw.githubusercontent.com/heketi/heketi/master/extras/systemd/heketi.service

sudo ln -s /usr/local/bin/heketi /usr/bin/heketi
sudo ln -s /usr/local/bin/heketi-cli /usr/bin/heketi-cli


heketi-cli -s http://localhost:7070 --user admin --secret '' topology load --json=/shell/deploy/glusterfs/topology.json
Creating cluster ... ID: 606bbee818c0e23c8d09d793824c23ba
	Allowing file volumes on cluster.
	Allowing block volumes on cluster.
	Creating node n55.dev ... ID: 9121797acba88cdff782253669041503
		Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?): /bin/bash: pvcreate: command not found
	Creating node n56.dev ... Unable to create node: /bin/bash: gluster: command not found
	Creating node n57.dev ... Unable to create node: /bin/bash: gluster: command not found


heketi-cli -s http://localhost:7070 --user admin --secret '' topology load --json=/shell/deploy/glusterfs/topology.json
	Found node n55.dev on cluster 606bbee818c0e23c8d09d793824c23ba
		Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?):   Can't open /dev/sdb exclusively.  Mounted filesystem?
  Can't open /dev/sdb exclusively.  Mounted filesystem?
	Creating node n56.dev ... Unable to create node: sudo: gluster: command not found
	Creating node n57.dev ... Unable to create node: sudo: gluster: command not found

sudo vi /etc/sudoers
#Defaults    secure_path = /sbin:/bin:/usr/sbin:/usr/bin
Defaults    secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin


[vagrant@n55 ~]$ sudo /usr/local/sbin/gluster volume start test-volume
volume start: test-volume: failed: Volume test-volume already started
[vagrant@n55 ~]$ sudo /usr/local/sbin/gluster volume stop test-volume
Stopping volume will make its data inaccessible. Do you want to continue? (y/n) y
volume stop: test-volume: success
[vagrant@n55 ~]$ sudo /usr/local/sbin/gluster volume delete test-volume
Deleting volume will erase all information about the volume. Do you want to continue? (y/n) y
volume delete: test-volume: success
sudo umount /data/brick1/


	Found node n55.dev on cluster 606bbee818c0e23c8d09d793824c23ba
		Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?):   Can't open /dev/sdb exclusively.  Mounted filesystem?
  Can't open /dev/sdb exclusively.  Mounted filesystem?
	Found node n56.dev on cluster 606bbee818c0e23c8d09d793824c23ba
		Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?):   Can't open /dev/sdb exclusively.  Mounted filesystem?
  Can't open /dev/sdb exclusively.  Mounted filesystem?
	Found node n57.dev on cluster 606bbee818c0e23c8d09d793824c23ba
		Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?):   Can't open /dev/sdb exclusively.  Mounted filesystem?
  Can't open /dev/sdb exclusively.  Mounted filesystem?
[vagrant@n50 heketi]$ heketi-cli -s http://localhost:7070 --user admin --secret '' topology load --json=/shell/deploy/glusterfs/topology.json
	Found node n55.dev on cluster 606bbee818c0e23c8d09d793824c23ba
		Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?): WARNING: xfs signature detected on /dev/sdb at offset 0. Wipe it? [y/n]: [n]
  Aborted wiping of xfs.
  1 existing signature left on the device.
	Found node n56.dev on cluster 606bbee818c0e23c8d09d793824c23ba
		Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?): WARNING: xfs signature detected on /dev/sdb at offset 0. Wipe it? [y/n]: [n]
  Aborted wiping of xfs.
  1 existing signature left on the device.
	Found node n57.dev on cluster 606bbee818c0e23c8d09d793824c23ba
		Adding device /dev/sdb ... Unable to add device: Setup of device /dev/sdb failed (already initialized or contains data?): WARNING: xfs signature detected on /dev/sdb at offset 0. Wipe it? [y/n]: [n]
  Aborted wiping of xfs.
  1 existing signature left on the device.

# Heketi需要空块设备,如果您已经在这些块设备上创建了lvm pvs或文件系统或分区等,则无法使用。如果您知道这些设备未使用,您可以在尝试初始化heketi之前使用像wipefs -a这样的命令来清除它们。

sudo wipefs -a /dev/sdb

heketi-cli -s http://localhost:7070 --user admin --secret '' topology load --json=/shell/deploy/glusterfs/topology.json
	Found node n55.dev on cluster 606bbee818c0e23c8d09d793824c23ba
		Adding device /dev/sdb ... OK
	Found node n56.dev on cluster 606bbee818c0e23c8d09d793824c23ba
		Adding device /dev/sdb ... OK
	Found node n57.dev on cluster 606bbee818c0e23c8d09d793824c23ba
		Adding device /dev/sdb ... OK


heketi-cli -s http://localhost:7070 --user admin --secret '' topology info

Cluster Id: 606bbee818c0e23c8d09d793824c23ba

    File:  true
    Block: true

    Volumes:


    Nodes:

	Node Id: 2315d10e5bf974a1c9069a7181e3b8a3
	State: online
	Cluster Id: 606bbee818c0e23c8d09d793824c23ba
	Zone: 1
	Management Hostnames: n56.dev
	Storage Hostnames: 192.168.33.56
	Devices:
		Id:ff84b184c51e3ef7f86b38c387bb9ed5   Name:/dev/sdb            State:online    Size (GiB):249     Used (GiB):0       Free (GiB):249
			Bricks:

	Node Id: 9121797acba88cdff782253669041503
	State: online
	Cluster Id: 606bbee818c0e23c8d09d793824c23ba
	Zone: 1
	Management Hostnames: n55.dev
	Storage Hostnames: 192.168.33.55
	Devices:
		Id:67c05c37402ba2eaf85223966401cf05   Name:/dev/sdb            State:online    Size (GiB):249     Used (GiB):0       Free (GiB):249
			Bricks:

	Node Id: ece7a447bd1987ad6fd862db418dfcef
	State: online
	Cluster Id: 606bbee818c0e23c8d09d793824c23ba
	Zone: 1
	Management Hostnames: n57.dev
	Storage Hostnames: 192.168.33.57
	Devices:
		Id:5d312fe9a5ab47c0b2384d5be5053eaf   Name:/dev/sdb            State:online    Size (GiB):249     Used (GiB):0       Free (GiB):249
			Bricks:


# 创建一个大小为3G,副本为2的volume
heketi-cli -s http://localhost:7070 --user admin --secret '' volume create --size 3 --replica 2

Name: vol_58ebc1e318fdabbf3c905cc9ffd2f6f8
Size: 3
Volume Id: 58ebc1e318fdabbf3c905cc9ffd2f6f8
Cluster Id: 606bbee818c0e23c8d09d793824c23ba
Mount: 192.168.33.56:vol_58ebc1e318fdabbf3c905cc9ffd2f6f8
Mount Options: backup-volfile-servers=192.168.33.55,192.168.33.57
Block: false
Free Size: 0
Reserved Size: 0
Block Hosting Restriction: (none)
Block Volumes: []
Durability Type: replicate
Distribute Count: 1
Replica Count: 2

//部署heketi(k8s方式)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
wget https://raw.githubusercontent.com/gluster/gluster-kubernetes/master/deploy/kube-templates/heketi-service-account.yaml
wget https://raw.githubusercontent.com/gluster/gluster-kubernetes/master/deploy/kube-templates/deploy-heketi-deployment.yaml
wget https://raw.githubusercontent.com/gluster/gluster-kubernetes/master/docs/examples/hello_world/gluster-storage-class.yaml

diff deploy-heketi-deployment.yaml deploy-heketi-deployment1.yaml
11a12
>   clusterIP: 10.254.8.2
36a38
>       nodeName: n26.dev
77a80,81
>         hostPath:
>           path: /data/heketi-data


diff gluster-storage-class.yaml gluster-storage-class1.yaml
1c1
< apiVersion: storage.k8s.io/v1beta1
---
> apiVersion: storage.k8s.io/v1
8,10c8,11
<   resturl: "http://10.42.0.0:8080"
<   restuser: "joe"
<   restuserkey: "My Secret Life"
---
>   resturl: "http://10.254.8.2:8080"
>   restauthenabled: "false"
>   #restuser: "joe"
>   #restuserkey: "My Secret Life"

## 创建资源
curl -L -o heketi.json https://raw.githubusercontent.com/gluster/gluster-kubernetes/master/deploy/heketi.json.template
# curl -L -o heketi.json https://raw.githubusercontent.com/heketi/heketi/master/etc/heketi.json
curl -L -o topology.json https://raw.githubusercontent.com/gluster/gluster-kubernetes/master/deploy/topology.json.sample

kubectl -n default create secret generic heketi-config-secret --from-file=private_key=/dev/null --from-file=./heketi.json 
--from-file=topology.json=topology.json

sed \
-e 's/\${HEKETI_EXECUTOR}/kubernetes/' \
-e 's/\${SSH_PORT}/22/' \
-e 's/\${SSH_USER}/vagrant/' \
-e 's/\${SSH_SUDO}/true/' \
./heketi.json

sed \
-e 's/\${HEKETI_EXECUTOR}/kubernetes/' \
-e 's#\${HEKETI_FSTAB}#/var/lib/heketi/fstab#' \
-e 's/\${HEKETI_ADMIN_KEY}//' \
-e 's/\${HEKETI_USER_KEY}//' ./deploy-heketi-deployment1.yaml | kubectl -n default create -f -


kubectl apply -f heketi-service-account.yaml
kubectl apply -f deploy-heketi-deployment1.yaml
kubectl apply -f gluster-storage-class1.yaml

# ERROR: Unable to parse configuration: json: cannot unmarshal string into Go struct field SshConfig.xfs_sw of type int

kubectl logs $(kubectl get pod -l glusterfs=heketi-pod -o jsonpath={.items[0].metadata.name})

kubectl -n default exec -i $(kubectl get pod -l glusterfs=heketi-pod -o jsonpath={.items[0].metadata.name}) -- heketi-cli -s http://localhost:8080 --user admin --secret '' topology load --json=/etc/heketi/topology.json
kubectl -n default exec -i $(kubectl get pod -l glusterfs=heketi-pod -o jsonpath={.items[0].metadata.name}) cat /etc/heketi/topology.json
kubectl -n default exec -i $(kubectl get pod -l glusterfs=heketi-pod -o jsonpath={.items[0].metadata.name}) -- heketi-cli --server http://localhost:8080 --user admin --secret "" topology load --json=/etc/heketi/topology.json
kubectl -n default delete secret heketi-config-secret && \
kubectl -n default create secret generic heketi-config-secret --from-file=private_key=/dev/null --from-file=heketi.json=heketi2.json --from-file=topology.json=topology.json && \
kubectl delete -f deploy-heketi-deployment1.yaml && \
sed \
-e 's/\${HEKETI_EXECUTOR}/kubernetes/' \
-e 's#\${HEKETI_FSTAB}#/var/lib/heketi/fstab#' \
-e 's/\${HEKETI_ADMIN_KEY}//' \
-e 's/\${HEKETI_USER_KEY}//' ./deploy-heketi-deployment1.yaml | kubectl -n default create -f -

# https://blog.fleeto.us/post/glusterfs-and-heketi-on-physical-server/
# https://www.jianshu.com/p/f70d370582ef
# https://fkpwolf.net/2018/09/11/glusterfs.html
# https://lichi6174.github.io/glusterfs-heketi

//pvc测试

cat gluster-storage-class1.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: gluster-heketi
provisioner: kubernetes.io/glusterfs
parameters:
  #endpoint: "heketi-storage-endpoints"
  resturl: "http://192.168.33.50:7070"
  restauthenabled: "false"
  volumetype: "replicate:2"
  #restuser: "joe"
  #restuserkey: "My Secret Life"

cat gluster-pvc.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
 name: gluster1
 annotations:
   volume.beta.kubernetes.io/storage-class: gluster-heketi
spec:
 accessModes:
  - ReadWriteMany
 resources:
   requests:
     storage: 2Gi

cat nginx-pod.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
apiVersion: v1
kind: Pod
metadata:
  name: gluster-pod1
  labels:
    name: gluster-pod1
spec:
  containers:
  - name: gluster-pod1
    image: nginx:1.17.2-alpine
    ports:
    - name: web
      containerPort: 80
    securityContext:
      privileged: true
    volumeMounts:
    - name: gluster-vol1
      mountPath: /usr/share/nginx/html
  volumes:
  - name: gluster-vol1
    persistentVolumeClaim:
      claimName: gluster1
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
## 查看
kubectl get pv,pvc,ep
NAME                                                        CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM              STORAGECLASS     REASON   AGE
persistentvolume/pvc-17535729-c745-11e9-837f-08002763a99f   2Gi        RWX            Delete           Bound    default/gluster1   gluster-heketi            2m

NAME                             STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS     AGE
persistentvolumeclaim/gluster1   Bound    pvc-17535729-c745-11e9-837f-08002763a99f   2Gi        RWX            gluster-heketi   2m20s

NAME                                                               ENDPOINTS                                         AGE
endpoints/glusterfs-dynamic-17535729-c745-11e9-837f-08002763a99f   192.168.33.55:1,192.168.33.56:1,192.168.33.57:1   2m1s


## 一些错误

Mounting command: systemd-run
Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/064e4421-bc43-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-0649fb80-bc43-11e9-9644-08002763a99f --scope -- mount -t glusterfs -o log-file=/var/lib/kubelet/plugins/kubernetes.io/glusterfs/pvc-0649fb80-bc43-11e9-9644-08002763a99f/gluster-pod1-glusterfs.log,log-level=ERROR,backup-volfile-servers=192.168.33.55:192.168.33.56:192.168.33.57,auto_unmount 192.168.33.55:vol_fe3d76a59b5805d057eda0f2f24d4a66 /var/lib/kubelet/pods/064e4421-bc43-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-0649fb80-bc43-11e9-9644-08002763a99f
Output: Running scope as unit run-15406.scope.
mount: unknown filesystem type 'glusterfs'


Warning  FailedMount  2m27s (x25 over 2m35s)  kubelet, n27.dev   MountVolume.NewMounter initialization failed for volume "pvc-10c304b0-bc48-11e9-9644-08002763a99f" : endpoints "glusterfs-dynamic-gluster1" not found

# 安装glusterfs-fuse包



 the following error information was pulled from the glusterfs log to help diagnose this issue: could not open log file for pod gluster-pod1
  Warning  FailedMount  2s (x7 over 15m)  kubelet, n27.dev  Unable to mount volumes for pod "gluster-pod1_default(064e4421-bc43-11e9-9644-08002763a99f)": timeout expired waiting for volumes to attach or mount for pod "default"/"gluster-pod1". list of unmounted volumes=[gluster-vol1]. list of unattached volumes=[gluster-vol1 default-token-4vjc4]



 the following error information was pulled from the glusterfs log to help diagnose this issue:
[2019-08-11 14:41:42.236008] E [MSGID: 100026] [glusterfsd.c:2351:glusterfs_process_volfp] 0-: failed to construct the graph
[2019-08-11 14:41:42.236212] E [graph.c:1142:glusterfs_graph_destroy] (-->/usr/sbin/glusterfs(mgmt_getspec_cbk+0x532) [0x559227d11332] -->/usr/sbin/glusterfs(glusterfs_process_volfp+0x150) [0x559227d0ae60] -->/lib64/libglusterfs.so.0(glusterfs_graph_destroy+0x84) [0x7f2dedacb1e4] ) 0-graph: invalid argument: graph [Invalid argument]
  Warning  FailedMount  49s  kubelet, n27.dev  MountVolume.SetUp failed for volume "pvc-111c0388-bc46-11e9-9644-08002763a99f" : mount failed: mount failed: exit status 1
Mounting command: systemd-run
Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/1123119f-bc46-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f --scope -- mount -t glusterfs -o log-file=/var/lib/kubelet/plugins/kubernetes.io/glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f/gluster-pod1-glusterfs.log,log-level=ERROR,backup-volfile-servers=192.168.33.55:192.168.33.56:192.168.33.57,auto_unmount 192.168.33.55:vol_050b5c59f74b1ca53e838b9544866dd2 /var/lib/kubelet/pods/1123119f-bc46-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f
Output: Running scope as unit run-24918.scope.
Mount failed. Please check the log file for more details.

 the following error information was pulled from the glusterfs log to help diagnose this issue:
[2019-08-11 14:41:50.612078] E [MSGID: 100026] [glusterfsd.c:2351:glusterfs_process_volfp] 0-: failed to construct the graph
[2019-08-11 14:41:50.612277] E [graph.c:1142:glusterfs_graph_destroy] (-->/usr/sbin/glusterfs(mgmt_getspec_cbk+0x532) [0x558767795332] -->/usr/sbin/glusterfs(glusterfs_process_volfp+0x150) [0x55876778ee60] -->/lib64/libglusterfs.so.0(glusterfs_graph_destroy+0x84) [0x7fe047c8f1e4] ) 0-graph: invalid argument: graph [Invalid argument]
  Warning  FailedMount  33s  kubelet, n27.dev  MountVolume.SetUp failed for volume "pvc-111c0388-bc46-11e9-9644-08002763a99f" : mount failed: mount failed: exit status 1
Mounting command: systemd-run
Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/1123119f-bc46-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f --scope -- mount -t glusterfs -o log-level=ERROR,backup-volfile-servers=192.168.33.55:192.168.33.56:192.168.33.57,auto_unmount,log-file=/var/lib/kubelet/plugins/kubernetes.io/glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f/gluster-pod1-glusterfs.log 192.168.33.55:vol_050b5c59f74b1ca53e838b9544866dd2 /var/lib/kubelet/pods/1123119f-bc46-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f
Output: Running scope as unit run-25208.scope.
Mount failed. Please check the log file for more details.

 the following error information was pulled from the glusterfs log to help diagnose this issue:
[2019-08-11 14:42:06.929857] E [MSGID: 100026] [glusterfsd.c:2351:glusterfs_process_volfp] 0-: failed to construct the graph
[2019-08-11 14:42:06.930063] E [graph.c:1142:glusterfs_graph_destroy] (-->/usr/sbin/glusterfs(mgmt_getspec_cbk+0x532) [0x55a84024d332] -->/usr/sbin/glusterfs(glusterfs_process_volfp+0x150) [0x55a840246e60] -->/lib64/libglusterfs.so.0(glusterfs_graph_destroy+0x84) [0x7f8a74e401e4] ) 0-graph: invalid argument: graph [Invalid argument]
  Warning  FailedMount  0s  kubelet, n27.dev  MountVolume.SetUp failed for volume "pvc-111c0388-bc46-11e9-9644-08002763a99f" : mount failed: mount failed: exit status 1
Mounting command: systemd-run
Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/1123119f-bc46-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f --scope -- mount -t glusterfs -o auto_unmount,log-file=/var/lib/kubelet/plugins/kubernetes.io/glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f/gluster-pod1-glusterfs.log,log-level=ERROR,backup-volfile-servers=192.168.33.55:192.168.33.56:192.168.33.57 192.168.33.55:vol_050b5c59f74b1ca53e838b9544866dd2 /var/lib/kubelet/pods/1123119f-bc46-11e9-9644-08002763a99f/volumes/kubernetes.io~glusterfs/pvc-111c0388-bc46-11e9-9644-08002763a99f
Output: Running scope as unit run-25563.scope.
Mount failed. Please check the log file for more details.

 the following error information was pulled from the glusterfs log to help diagnose this issue:
[2019-08-11 14:42:39.344185] E [MSGID: 100026] [glusterfsd.c:2351:glusterfs_process_volfp] 0-: failed to construct the graph
[2019-08-11 14:42:39.344398] E [graph.c:1142:glusterfs_graph_destroy] (-->/usr/sbin/glusterfs(mgmt_getspec_cbk+0x532) [0x5639cfa58332] -->/usr/sbin/glusterfs(glusterfs_process_volfp+0x150) [0x5639cfa51e60] -->/lib64/libglusterfs.so.0(glusterfs_graph_destroy+0x84) [0x7f93ffa2c1e4] ) 0-graph: invalid argument: graph [Invalid argument]

# 服务端和客户端版本不一致
sudo yum list installed |grep gluster
sudo rpm -qa |grep gluster
sudo yum install centos-release-gluster -y && sudo yum clean all && sudo yum makecache
sudo yum remove glusterfs-fuse -y && sudo yum autoremove -y && sudo yum install glusterfs-fuse

部署istio

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
curl -L https://git.io/getLatestIstio | ISTIO_VERSION=1.2.4 sh -
kubectl create namespace istio-system
helm template install/kubernetes/helm/istio-init --name istio-init --namespace istio-system | kubectl apply -f -
kubectl get crds | grep 'istio.io\|certmanager.k8s.io' | wc -l
# 23 个crd If cert-manager is enabled, then the CRD count will be 28 instead.
helm template install/kubernetes/helm/istio --name istio --namespace istio-system | kubectl apply -f -
kubectl -n istio-system get svc,ep,pod,deployments,job -o wide
kubectl patch svc istio-ingressgateway -n istio-system -p '{"spec":{"externalIPs":["your_external_ip"]}}'
####
kubectl run -it --rm --restart=Never --generator=run-pod/v1 --image=praqma/network-multitool pod-$RANDOM -- /bin/sh -c 'nslookup istio-sidecar-injector.istio-system.svc'
kubectl run -it --rm --restart=Never --generator=run-pod/v1 --image=praqma/network-multitool pod-$RANDOM -- /bin/sh -c 'nslookup istio-sidecar-injector.istio-system.svc && curl https://istio-sidecar-injector.istio-system.svc:443/inject?timeout=30s -k -i -v && curl -ivk --connect-timeout 3 -m 5 -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" https://10.254.0.1:443/apis/admissionregistration.k8s.io/v1beta1/mutatingwebhookconfigurations'
# /apis/admissionregistration.k8s.io/v1beta1/mutatingwebhookconfigurations
kubectl label namespace MYNAMESPACE istio-injection=enabled
kubectl get mutatingwebhookconfiguration istio-sidecar-injector -o yaml -o jsonpath='{.webhooks[0].clientConfig.caBundle}' | md5sum
kubectl -n istio-system get secret istio.istio-sidecar-injector-service-account -o jsonpath='{.data.root-cert\.pem}' | md5sum
kubectl -n istio-system logs $(kubectl get pod -n istio-system -l istio=sidecar-injector -o jsonpath={.items[0].metadata.name})
kubectl get validatingwebhookconfiguration istio-galley -o yaml
kubectl -n istio-system get configmap istio-galley-configuration -o jsonpath='{.data}'
kubectl get ValidatingWebhookConfiguration istio-galley -oyaml
kubectl get ValidatingWebhookConfiguration istio-galley -oyaml |grep -C3 'admitpilot\|admitmixer'
kubectl -n istio-system get pod -listio=galley -o wide
kubectl -n istio-system get ep istio-galley
kubectl get clusterrole istio-galley-istio-system -o yaml
for pod in $(kubectl -n istio-system get pod -listio=galley -o jsonpath='{.items[*].metadata.name}'); do
    kubectl -n istio-system logs ${pod}
done
# https://istio.io/docs/ops/setup/validation/
# https://istio.io/docs/ops/setup/webhook/

# 验证webhook服务的连通性
kubectl get pod,svc -o wide -n istio-system -l app=sidecarInjectorWebhook
kubectl get pod,svc -o wide -n istio-system -l istio=galley
sudo ipvsadm --list -n |grep -A2  10.254.98.23
nc -nvz 10.254.98.23 443
kubectl get svc -o wide -n istio-system -l istio=galley -o jsonpath='{.items[0].spec.clusterIP}'
kubectl get svc -o wide -n istio-system -l app=sidecarInjectorWebhook -o jsonpath='{.items[0].spec.clusterIP}'
curl -ikv https://`kubectl get svc -o wide -n istio-system -l app=sidecarInjectorWebhook -o jsonpath='{.items[0].spec.clusterIP}'`:443
curl -ikv https://`kubectl get svc -o wide -n istio-system -l istio=galley -o jsonpath='{.items[0].spec.clusterIP}'`:443

## 部署demo
kubectl label namespace default istio-injection=enabled

kubectl apply -f samples/bookinfo/platform/kube/bookinfo.yaml
kubectl apply -f samples/bookinfo/networking/bookinfo-gateway.yaml


kubectl apply -f samples/helloworld/helloworld-gateway.yaml
kubectl apply -f samples/helloworld/helloworld.yaml

curl http://192.168.33.180/hello
curl http://192.168.33.180/productpage


## 检测istio

kubectl run -it --rm --restart=Never --generator=run-pod/v1 \
--overrides='{ "metadata": {"annotations":  {"sidecar.istio.io/inject" : "false" }}}' \
--image=7error/kkkbox:2.1.6 pod-$RANDOM \
-- /bin/sh -c 'sleep 3s; \
echo "################################################################" \
&& cat /etc/resolv.conf \
&& echo "################################################################" \
&& cat /etc/resolv.conf |grep -i '^nameserver' |head -n1 | awk "{print \$2}" | xargs --no-run-if-empty -I% nc -nzv -w1 % 53 \
&& dig +nocmd kubernetes.default.svc.cluster.local -t A +noall +answer \
&& nping --tcp-connect -c1 -H -p 8060,15014 istio-citadel.istio-system.svc \
&& nping --tcp-connect -c1 -H -p 443,15014,9901 istio-galley.istio-system.svc \
&& nping --tcp-connect -c1 -H -p 15020,80,443,31400,15029,15030,15031,15032,15443,15090 istio-ingressgateway.istio-system.svc \
&& nping --tcp-connect -c1 -H -p 15010,15011,8080,15014 istio-pilot.istio-system.svc \
&& nping --tcp-connect -c1 -H -p 9091,15004,15014 istio-policy.istio-system.svc \
&& nping --tcp-connect -c1 -H -p 443,15014 istio-sidecar-injector.istio-system.svc \
&& nping --tcp-connect -c1 -H -p 9091,15004,15014,42422 istio-telemetry.istio-system.svc \
&& nping --tcp-connect -c1 -H -p 443 kubernetes \
&& nslookup istio-sidecar-injector.istio-system.svc \
&& curl --connect-timeout 2 --max-time 2 -iv http://istio-ingressgateway.istio-system.svc:15020/healthz/ready \
&& env';
#########

kubectl run -it --rm --restart=Never --generator=run-pod/v1 \
--image=raisiqueira/mtr pod-$RANDOM \
-- /bin/sh -c 'sleep 3s; \
echo "################################################################" \
&& ip a && mtr -c2 -r 192.168.33.21';

###

kubectl exec -n istio-system $(kubectl get pod --namespace istio-system --selector istio=ingressgateway --output jsonpath='{.items[0].metadata.name}') -- netstat -tunlp

Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:80              0.0.0.0:*               LISTEN      27/envoy
tcp        0      0 0.0.0.0:15090           0.0.0.0:*               LISTEN      27/envoy
tcp        0      0 127.0.0.1:15000         0.0.0.0:*               LISTEN      27/envoy
tcp6       0      0 :::15020                :::*                    LISTEN      1/pilot-agent


kubectl exec -it helloworld-v1-b6c7975c7-77xvl -c helloworld -- curl http://helloworld.default.svc:5000/hello

kubectl -n istio-system describe deployments istio-policy
kubectl -n istio-system logs $(kubectl -n istio-system get pods -lapp=policy -o jsonpath='{.items[0].metadata.name}') -cmixer
kubectl -n istio-system logs $(kubectl -n istio-system get pods -lapp=policy -o jsonpath='{.items[0].metadata.name}') -cistio-proxy
kubectl -n istio-system logs istio-pilot-78998f5bc4-s8kdj discovery -f

nnection error: desc = "transport: Error while dialing dial tcp 10.254.248.13:9901: connect: connection refused"
2019-09-27T02:35:21.354124Z	info	mcp	(re)trying to establish new MCP sink stream
2019-09-27T02:35:21.354202Z	error	mcp	Failed to create a new MCP sink stream: rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.254.248.13:9901: connect: connection refused"
2019-09-27T02:35:22.354985Z	info	mcp	(re)trying to establish new MCP sink stream
2019-09-27T02:35:22.355068Z	error	mcp	Failed to create a new MCP sink stream: rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.254.248.13:9901: connect: connection refused"
2019-09-27T02:35:23.360141Z	info	mcp	(re)trying to establish new MCP sink stream
2019-09-27T02:35:23.360231Z	error	mcp	Failed to create a new MCP sink stream: rpc error: code = Unavailable desc = all SubConns are in TransientFailure, latest connection error: connection error: desc = "transport: Error while dialing dial tcp 10.254.248.13:9901: connect: connection refused"


kubectl -n istio-system logs  istio-galley-677bf9cff7-22sr7

2019-09-27T02:40:37.791750Z	error	istio.io/istio/galley/pkg/crd/validation/endpoint.go:103: Failed to list *v1.Endpoints: Get https://10.254.0.1:443/api/v1/namespaces/istio-system/endpoints?fieldSelector=metadata.name%3Distio-galley&limit=500&resourceVersion=0: dial tcp 10.254.0.1:443: connect: no route to host
2019-09-27T02:40:40.803000Z	error	istio.io/istio/galley/pkg/crd/validation/endpoint.go:103: Failed to list *v1.Endpoints: Get https://10.254.0.1:443/api/v1/namespaces/istio-system/endpoints?fieldSelector=metadata.name%3Distio-galley&limit=500&resourceVersion=0: dial tcp 10.254.0.1:443: connect: no route to host
2019-09-27T02:40:43.808687Z	error	istio.io/istio/galley/pkg/crd/validation/endpoint.go:103: Failed to list *v1.Endpoints: Get https://10.254.0.1:443/api/v1/namespaces/istio-system/endpoints?fieldSelector=metadata.name%3Distio-galley&limit=500&resourceVersion=0: dial tcp 10.254.0.1:443: connect: no route to host
2019-09-27T02:40:46.819131Z	error	istio.io/istio/galley/pkg/crd/validation/endpoint.go:103: Failed to list *v1.Endpoints: Get https://10.254.0.1:443/api/v1/namespaces/istio-system/endpoints?fieldSelector=metadata.name%3Distio-galley&limit=500&resourceVersion=0: dial tcp 10.254.0.1:443: connect: no route to host
2019-09-27T02:40:49.830907Z	error	istio.io/istio/galley/pkg/crd/validation/endpoint.go:103: Failed to list *v1.Endpoints: Get https://10.254.0.1:443/api/v1/namespaces/istio-system/endpoints?fieldSelector=metadata.name%3Distio-galley&limit=500&resourceVersion=0: dial tcp 10.254.0.1:443: connect: no route to host



2019-09-27T02:41:46.992999Z	error	kube	Expected resources (CRDs) not found: [template QuotaSpec rule VirtualService ServiceEntry ClusterRbacConfig QuotaSpecBinding ServiceRoleBinding attributemanifest HTTPAPISpecBinding Sidecar adapter Ingress Gateway Policy instance DestinationRule EnvoyFilter MeshPolicy RbacConfig handler ServiceRole HTTPAPISpec]
2019-09-27T02:41:46.993023Z	error	kube	To stop Galley from waiting for these resources (CRDs), consider using the --excludedResourceKinds flag
2019-09-27T02:41:46.993379Z	fatal	Error creating server: 54 errors occurred:
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find networking.istio.io/v1alpha3: Get https://10.254.0.1:443/apis/networking.istio.io/v1alpha3?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find networking.istio.io/v1alpha3: Get https://10.254.0.1:443/apis/networking.istio.io/v1alpha3?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find networking.istio.io/v1alpha3: Get https://10.254.0.1:443/apis/networking.istio.io/v1alpha3?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find networking.istio.io/v1alpha3: Get https://10.254.0.1:443/apis/networking.istio.io/v1alpha3?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find networking.istio.io/v1alpha3: Get https://10.254.0.1:443/apis/networking.istio.io/v1alpha3?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find authentication.istio.io/v1alpha1: Get https://10.254.0.1:443/apis/authentication.istio.io/v1alpha1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find rbac.istio.io/v1alpha1: Get https://10.254.0.1:443/apis/rbac.istio.io/v1alpha1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find extensions/v1beta1: Get https://10.254.0.1:443/apis/extensions/v1beta1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find rbac.istio.io/v1alpha1: Get https://10.254.0.1:443/apis/rbac.istio.io/v1alpha1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find rbac.istio.io/v1alpha1: Get https://10.254.0.1:443/apis/rbac.istio.io/v1alpha1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find rbac.istio.io/v1alpha1: Get https://10.254.0.1:443/apis/rbac.istio.io/v1alpha1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find authentication.istio.io/v1alpha1: Get https://10.254.0.1:443/apis/authentication.istio.io/v1alpha1?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find networking.istio.io/v1alpha3: Get https://10.254.0.1:443/apis/networking.istio.io/v1alpha3?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host
	* could not find config.istio.io/v1alpha2: Get https://10.254.0.1:443/apis/config.istio.io/v1alpha2?timeout=32s: dial tcp 10.254.0.1:443: connect: no route to host

: the following resource type(s) were not found: [template QuotaSpec rule VirtualService ServiceEntry ClusterRbacConfig QuotaSpecBinding ServiceRoleBinding attributemanifest HTTPAPISpecBinding Sidecar adapter Ingress Gateway Policy instance DestinationRule EnvoyFilter MeshPolicy RbacConfig handler ServiceRole HTTPAPISpec]





## 升级
curl -L https://git.io/getLatestIstio | ISTIO_VERSION=1.3.0 sh -
sudo cp istio-1.3.0/bin/istioctl /usr/bin/istioctl
helm template install/kubernetes/helm/istio-init --name istio-init --namespace istio-system > istio-init.yaml
helm template install/kubernetes/helm/istio --name istio --namespace istio-system > istio.yaml


kubectl patch service istio-ingressgateway -n istio-system -p '{"spec":{"type":"NodePort"}}'
kubectl patch service istio-ingressgateway -n istio-system -p '{"spec":{"type":"LoadBalancer"}}'
kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.spec.ports[?(@.name=="http2")].nodePort}'
kubectl -n istio-system get pod -l istio=ingressgateway  -o jsonpath={.items[0].status.hostIP}

## 部署metallb
kubectl apply -f https://raw.githubusercontent.com/danderson/metallb/master/manifests/metallb.yaml

####
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: metallb-system
  name: config
data:
  config: |
    address-pools:
    - name: default
      protocol: layer2
      addresses:
      # 此IP地址在集群外无法ping通,但可以访问80端口
      # 根据情况自定义,范围与node节点ip的同网络
      - 192.168.33.200-192.168.33.210
EOF



####

kubectl get pod,svc,cm -n metallb-system -owide

####
curl -v http://127.0.0.1:15020/healthz/ready
curl -v http://127.0.0.1:15000/stats


#######
level=warn ts=2019-09-03T13:42:00.81256519Z caller=main.go:274 deprecation_notice="'storage.tsdb.retention' flag is deprecated use 'storage.tsdb.retention.time' instead."
level=info ts=2019-09-03T13:42:00.812643225Z caller=main.go:321 msg="Starting Prometheus" version="(version=2.8.0, branch=HEAD, revision=59369491cfdfe8dcb325723d6d28a837887a07b9)"
level=info ts=2019-09-03T13:42:00.812671456Z caller=main.go:322 build_context="(go=go1.11.5, user=root@4c4d5c29b71f, date=20190312-07:46:58)"
level=info ts=2019-09-03T13:42:00.812691213Z caller=main.go:323 host_details="(Linux 4.4.190-1.el7.elrepo.x86_64 #1 SMP Sun Aug 25 07:32:44 EDT 2019 x86_64 prometheus-7b65765ddd-4s42g (none))"
level=info ts=2019-09-03T13:42:00.812714715Z caller=main.go:324 fd_limits="(soft=1048576, hard=1048576)"
level=info ts=2019-09-03T13:42:00.812731803Z caller=main.go:325 vm_limits="(soft=unlimited, hard=unlimited)"
level=info ts=2019-09-03T13:42:00.814091896Z caller=main.go:640 msg="Starting TSDB ..."
level=info ts=2019-09-03T13:42:00.814102568Z caller=web.go:418 component=web msg="Start listening for connections" address=0.0.0.0:9090
level=info ts=2019-09-03T13:42:00.81416798Z caller=main.go:509 msg="Stopping scrape discovery manager..."
level=info ts=2019-09-03T13:42:00.814180948Z caller=main.go:523 msg="Stopping notify discovery manager..."
level=info ts=2019-09-03T13:42:00.814186902Z caller=main.go:545 msg="Stopping scrape manager..."
level=info ts=2019-09-03T13:42:00.814193295Z caller=main.go:519 msg="Notify discovery manager stopped"
level=info ts=2019-09-03T13:42:00.81420815Z caller=main.go:505 msg="Scrape discovery manager stopped"
level=info ts=2019-09-03T13:42:00.814226179Z caller=main.go:539 msg="Scrape manager stopped"
level=info ts=2019-09-03T13:42:00.814237714Z caller=manager.go:736 component="rule manager" msg="Stopping rule manager..."
level=info ts=2019-09-03T13:42:00.814250851Z caller=manager.go:742 component="rule manager" msg="Rule manager stopped"
level=info ts=2019-09-03T13:42:00.814258689Z caller=notifier.go:521 component=notifier msg="Stopping notification manager..."
level=info ts=2019-09-03T13:42:00.814270521Z caller=main.go:708 msg="Notifier manager stopped"
level=error ts=2019-09-03T13:42:00.814675543Z caller=main.go:717 err="opening storage failed: mkdir data/: permission denied"
########
# 增加权限
      securityContext:
        runAsUser: 0

常用命令

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
#!/bin/bash
# create bootstrap token
pwdoo=$(dirname "$(readlink -fn "$0")")
source $pwdoo/__global_env.sh
echo $1
TOKEN=$(kubeadm token create --groups system:bootstrappers:$1:default-node-token)
echo $TOKEN
kubectl --kubeconfig=$K8S_ETC_DIR/bootstrap.conf config set-cluster kubernetes --certificate-authority=$K8S_PKI_DIR/kubernetes-ca.pem --embed-certs=true --server=https://192.168.33.100:8443
kubectl --kubeconfig=$K8S_ETC_DIR/bootstrap.conf config set-credentials kubelet-bootstrap --token=$TOKEN
kubectl --kubeconfig=$K8S_ETC_DIR/bootstrap.conf config set-context default --cluster=kubernetes --user=kubelet-bootstrap
kubectl --kubeconfig=$K8S_ETC_DIR/bootstrap.conf config use-context default
kubeadm token list;
cat $K8S_ETC_DIR/bootstrap.conf
1
2
3
4
5
6
7
#!/bin/bash
nodes=$(kubectl get csr -o json | jq -r '[.items | .[].metadata | .name ] | join(" ")')
for node in $nodes; do
	kubectl certificate approve $node
done
kubectl get csr;
kubectl get no;
1
2
3
4
5
6
7
### 批量删除被驱逐的pod
### ImageInspectError、Evicted、Terminating
### --force --grace-period=0
for ns in $(kubectl get ns -o json | jq -r '[.items[] | .metadata.name ] | join(" ")'); do
  #echo $ns;
  kubectl -n $ns get po | grep -E "Evicted|Terminating" | awk '{print $1}'| xargs --no-run-if-empty -I% kubectl -n $ns delete pod % --force --grace-period=0
done
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
###
# Unregistered Authentication Agent for unix-process:18639:13186955 (system bus name :1.629, object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale

#
# 如果还没有 docker group 就添加一个:
sudo groupadd docker
# 将用户加入该 group 内。然后退出并重新登录就生效啦。
sudo gpasswd -a ${USER} docker
# 重启 docker 服务
sudo service docker restart
# 切换当前会话到新 group 
newgrp - docker
###
cat /var/run/secrets/kubernetes.io/serviceaccount/namespace
cat /var/run/secrets/kubernetes.io/serviceaccount/token
cat /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
###
journalctl -xe
###########
sudo systemctl daemon-reload
sudo systemctl restart docker.service
sudo systemctl status network.service
sudo journalctl -ex --unit docker --no-pager
systemctl cat docker --no-pager |grep -v ^#
sudo netstat -tlunp
sudo ss -tlunp
nmap 127.0.0.1  -p1-65535
kubectl label node n21.dev node-role.kubernetes.io/master=master
kubectl label node n21.dev node-role.kubernetes.io/worker=worker
kubectl label node n22.dev node-role.kubernetes.io/master=master
kubectl label node n22.dev node-role.kubernetes.io/worker=worker
kubectl label node n23.dev node-role.kubernetes.io/master=master
kubectl label node n23.dev node-role.kubernetes.io/worker=worker
kubectl label node n26.dev node-role.kubernetes.io/worker=worker
kubectl label node n27.dev node-role.kubernetes.io/worker=worker
#########
kubectl patch deploy --namespace default adminer -p '{"spec":{"replicas":2}}'
kubectl patch deployment adminer -p '{"spec":{"replicas":4}}'
kubectl patch deployment web -p "{\"spec\":{\"template\":{\"metadata\":{\"labels\":{\"date\":\"`date +'%s'`\"}}}}}"
#####
kubectl patch deployment nginx -p '{"spec":{"template":{"spec":{"containers":[{"name":"nginx","image":"nginx:1.17.1-alpine"}]}}}}'
kubectl patch deployment nginx -p '{"spec":{"replicas":4}}'
curl --cacert ./ca.pem --cert ./admin.pem --key ./admin-key.pem -k \
-X PATCH --header 'Content-Type: application/strategic-merge-patch+json' \
--data '{"spec":{"template":{"spec":{"containers":[{"name":"nginx","image":"nginx:1.15.2-alpine"}]}}}}' \
https://192.168.33.100:8443/apis/apps/v1/namespaces/default/deployments/nginx
#####
time curl -vo /dev/null http://baidu.com
# 部分更新节点
$ kubectl patch node k8s-node-1 -p '{"spec":{"unschedulable":true}}'
# 更新容器镜像; spec.containers[*].name 是必须的,因为这是合并的关键字
$ kubectl patch pod valid-pod -p '{"spec":{"containers":[{"name":"kubernetes-serve-hostname","image":"new image"}]}}'
# 使用具有位置数组的 json 补丁更新容器镜像
$ kubectl patch pod valid-pod --type='json' -p='[{"op": "replace", "path": "/spec/containers/0/image", "value":"new image"}]'
# 使用具有位置数组的 json 补丁禁用 deployment 的 livenessProbe
$ kubectl patch deployment valid-deployment  --type json   -p='[{"op": "remove", "path": "/spec/template/spec/containers/0/livenessProbe"}]'
#########
docker system info | grep Driver
docker system prune
docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
docker run -d --name uuu ubuntu:18.04 /bin/sh -c "while true; do sleep 30; done;"
docker exec -it uuu /bin/bash
docker run -it --rm radial/busyboxplus:curl
docker run -it --rm mcr.microsoft.com/java/jre-headless:8u192-zulu-alpine /bin/sh
docker run -it --rm --env AAA=value -v /shell:/hehe mcr.microsoft.com/java/jdk:8u212-zulu-alpine /bin/sh -c "javac /hehe/Env.java &&cd /hehe && java Env |grep AAA"
docker run --rm -p 18081:8080 --name tomcat tomcat:7.0.94-jre8-alpine
docker run -d -P --name iamfoo containous/whoami
docker run -d -p 18080:80 --name iamfoo containous/whoami
docker run -it --rm --name web python:3.7.4-alpine3.10 /bin/sh -c "python -V"
############
docker network create my-net
docker run -itd --name box3 --rm --network my-net busybox /bin/sh -c "sleep 60s"
docker run -itd --name box4 --rm --network my-net busybox /bin/sh -c "sleep 60s"
docker exec -it box3 /bin/sh -c "ping -c3 box4"
docker exec -it box4 /bin/sh -c "ping -c3 box3"
docker run -d --name pause -p 8880:80 k8s.gcr.io/pause-amd64:3.1
docker run -d --name nginx -v `pwd`/nginx.conf:/etc/nginx/nginx.conf --net=container:pause --ipc=container:pause --pid=container:pause nginx
docker run -d --name ghost --net=container:pause --ipc=container:pause --pid=container:pause ghost
############
docker inspect --format '{{ .NetworkSettings.Ports }}'  iamfoo
docker inspect iamfoo --format '{{.HostConfig.LogConfig.Type}}'
docker container ps -a -q
docker container ps |awk '{print $1}'
docker container ls -a -f name=^k8s -q | wc -l
docker container inspect -f "{{.Id}} {{.State.Pid}} {{.Config.Hostname}}" <dockerid>
docker container inspect -f "{{.State.Pid}}" <dockerid>
docker container inspect -f "{{.Id}}" <dockerid>
docker container ps --format='table {{.Ports}}\t{{.Names}}'
#####
docker container run -d --restart always --name centos77 centos:centos7.6.1810 /bin/sh -c "sleep 365d;"
docker container exec -it centos77 /bin/bash
docker container stop centos77
docker container rm centos77
#####
docker container run -d --restart always --name ddd debian:stretch /bin/sh -c "while true; do sleep 1s;echo \$HOSTNAME; done;"
docker container exec -it ddd /bin/bash
docker container logs -f ddd
docker container stop ddd && docker container rm ddd
#########
# 导入导出镜像
docker image save registry.access.redhat.com/rhel7/pod-infrastructure > /shell/pod-infrastructure.tar
docker image load < pod-infrastructure.tar
#####
kubectl api-resources
kubectl api-versions
kubectl cluster-info
kubectl get all --all-namespaces
kubectl get apiservices
kubectl get replicaset
kubectl get --raw "/apis" | jq
kubectl get svc,ep,deploy,po,no,rs,ns,sa --all-namespaces -o wide
kubectl get ds
kubectl get cs
kubectl get csr
kubectl certificate approve <CSR>
kubectl certificate approve `kubectl get csr --no-headers |awk '{print $1}'`
kubectl logs calico-node-2nct7 --namespace kube-system
# --field-selector status.phase=Running
kubectl logs --namespace kube-system $(kubectl get pod -n kube-system -l k8s-app=calico-node  -o jsonpath={.items[0].metadata.name}) 
## 强制删除 pod
kubectl delete -n istio-system pod istio-policy-5dd5cb9645-l5wfd --force --grace-period=0
## 强制删除 namespace
kubectl delete namespace NAMESPACENAME --force --grace-period=0
#########
kubectl get pod -n kube-system -l k8s-app=calico-node  -o wide
kubectl get pod -n kube-system -l k8s-app=calico-kube-controllers  -o wide
kubectl describe cm calico-config -n kube-system
cat /etc/cni/net.d/10-calico.conflist
ls -l  /etc/cni/net.d/
# 滚动跟新
kubectl set image deployment/my-nginx nginx=nginx:1.9.1
kubectl rollout status deployment/my-nginx
kubectl rollout history deployment/my-nginx
kubectl apply -f <deploy.yaml> --record
# spec.revisionHistoryLimit
# Mark node as unschedulable or schedulable
kubectl cordon <node>
kubectl uncordon <node>
# 污点
kubectl taint
# 驱赶node上的pod
kubectl drain
kubectl run -it --rm --generator=run-pod/v1 --image=busybox:1.30.1-musl busy11 -- /bin/sh
kubectl run -it --rm --generator=run-pod/v1 --image=radial/busyboxplus:curl busyboxplus -- /bin/sh
kubectl run -it --rm --generator=run-pod/v1 --image=alpine:3.9.4 alpine -- /bin/sh
kubectl run -it --rm --generator=run-pod/v1 --image=infoblox/dnstools dns-client
kubectl run -it --rm --generator=run-pod/v1 --image=tutum/dnsutils dns -- /bin/sh
kubectl run -it --rm --generator=run-pod/v1 --image=sjourdan/toolbox t111 -- /bin/sh
kubectl run -it --rm --generator=run-pod/v1 --image=praqma/network-multitool t222 -- /bin/sh
kubectl run -it --rm --generator=run-pod/v1 --image=buildpack-deps:18.04-curl ubuntu -- /bin/sh
kubectl run -it --rm --generator=run-pod/v1 --image=buildpack-deps:18.04-curl ubuntu1 -- /bin/bash
kubectl run -it --rm --restart=Never --generator=run-pod/v1 --image=praqma/network-multitool pod-$RANDOM -- /bin/sh -c 'nc -nvz 10.254.98.23 443'
##############
kubectl run --generator=deployment/apps.v1beta1 --image=nginx:1.16.0-alpine  --replicas=2 nginx11
kubectl exec -it $(kubectl get pod -l run=nginx11 -o jsonpath={.items[0].metadata.name}) -- /bin/sh -c "ping -c 200 10.254.0.1"
##############
kubectl run -it --rm --restart=Never --generator=run-pod/v1 --image=busybox:1.31.1-glibc busybox-$RANDOM -- /bin/sh -c 'nslookup kubernetes.default.svc.cluster.local';
kubectl run --generator=run-pod/v1 --image=busybox:1.30.1-musl busybox11 -- /bin/sh -c 'sleep 3699d';
kubectl exec -it busybox11 -- /bin/sh -c "nslookup kubernetes.default.svc.cluster.local."
kubectl exec -it busybox11 -- nslookup kubernetes.default.svc.cluster.local.
kubectl run --generator=run-pod/v1 --image=sjourdan/toolbox t111 -- /bin/sh -c 'sleep 3699d'
kubectl run --generator=run-pod/v1 --image=praqma/network-multitool t222 -- /bin/sh -c 'sleep 3699d'
#########
kubectl run --generator=run-pod/v1 --image=tutum/dnsutils dns11 -- /bin/sh -c 'sleep 3699d'
kubectl exec -it dns11 -- nslookup kubernetes.default
############
kubectl run --generator=run-pod/v1 --image=ubuntu:18.04 ubu -- /bin/sh -c "sleep 36500d;"
kubectl run --generator=run-pod/v1 --image=centos:centos7.6.1810 centos7 -- /bin/sh -c "sleep 36500d;"
###########
kubectl run --generator=run-pod/v1 --image=containous/whoami who333
kubectl expose pod who333 --port=80 --target-port=80 --type=NodePort
curl http://$(kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'):$(kubectl get svc who333 -o jsonpath='{.spec.ports..nodePort}')
########
kubectl run --generator=deployment/apps.v1beta1 --image=nginx  --replicas=5 nginx009 
kubectl expose deployments nginx009 --port=80 --target-port=80 --type=NodePort
curl http://$(kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'):$(kubectl get svc nginx009 -o jsonpath='{.spec.ports..nodePort}')
kubectl scale --replicas=1 deployments/nginx009
#################
kubectl run --restart=Never --generator=run-pod/v1 --image=gcr.io/kuar-demo/kuard-amd64:blue kuard
kubectl expose pod kuard --port=8080 --target-port=8080 --type=NodePort
###
mtr -r -c 1 10.254.0.1
traceroute -n -q1 10.254.0.1
nc -nvz 10.254.0.2 53
nslookup -qt=A kubernetes
nslookup -qt=A www.baidu.com
###
kubectl get service kubernetes --output jsonpath='{.spec.ports[?(@.name=="https")].targetPort}'
kubectl get node -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'
kubectl get po -l k8s-app=nginx-demo -o jsonpath={.items..metadata.name}
kubectl exec `kubectl get po -l run=ubuntu  -o=name|cut -d "/" -f2` env
kubectl get po -o=custom-columns=NAME:.metadata.name,RSRC:.metadata.resourceVersion
kubectl get po --all-namespaces -o=custom-columns=NAME:.metadata.name,USER:.metadata.user,VERSION:.metadata.version
kubectl get po -o go-template --template '{{range .items}}{{.metadata.name}}{{"\t"}}{{range .spec.containers}}{{.image}}{{","}}{{end}}{{"\n"}}{{end}}'
##################
for ns in $(kubectl get ns --no-headers | cut -d ' ' -f1); do for po in $(kubectl -n $ns get po --no-headers --ignore-not-found | grep Terminating | cut -d ' ' -f1); do kubectl -n $ns delete po $po --force --grace-period 0; done; done;
##################
kubectl -n kube-system edit cm kubeadm-config
##################
# alpine更换阿里云repo
sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
# alpine更换科大repo
sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
kubectl exec -it <pod> /bin/sh
kubectl exec -it $(kubectl get pods -o jsonpath='{.items[0].metadata.name}') /bin/sh
kubectl exec -it $(kubectl get po -o json | jq -r '.items | sort_by(.spec.nodeName)[] | [.metadata.name] | @tsv' |grep test) /bin/sh
# pod 中通过访问api
curl -ik -H "Authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" https://kubernetes/api/v1/namespaces/default/pods
kubectl get secret --namespace default dokuwiki-dokuwiki -o jsonpath="{.data.dokuwiki-password}" | base64 --decode
kubectl get pods -o go-template --template '{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}'
kubectl get secret -o jsonpath='{range .items[?(@.metadata.annotations.kubernetes\.io/service-account\.name=="default")].data}{"token: "}{.token}{"\n\n"}{"ca.crt: "}{.ca\.crt}{"\n"}{end}'
###
kubectl get no -o json | jq -r '.items[].status.addresses[] | select(.type=="InternalIP") | .address'
kubectl delete pod $(kubectl get pods -o go-template='{{range .items}}{{if eq .status.phase "Failed"}}{{.metadata.name}}{{"\n"}}{{end}}{{end}}')
###
export jsonpath='{range .items[*]}{"\n"}{@.metadata.name}{range @.status.conditions[*]}{"\t"}{@.type}={@.status}{end}{end}' 
kubectl get po -ao jsonpath="$jsonpath" && echo
###
kubectl get --all-namespaces svc -o json | jq -r '.items[] | [.metadata.name,([.spec.ports[]?.nodePort | tostring ] | join("|"))] | @csv'
kubectl get svc --all-namespaces -o go-template='{{range .items}}{{.metadata.name}}{{"\t"}}{{range.spec.ports}}{{if .nodePort}}{{.nodePort}}{{","}}{{end}}{{end}}{{"\n"}}{{end}}'
kubectl get no -o jsonpath="{.items[?(@.spec.unschedulable)].metadata.name}"
kubectl get pv -o jsonpath='{range.items[?(@.status.phase=="Released")]}{.metadata.name}{"\n"}{end}'
kubectl get pv -o json | jq -r '.items | sort_by(.spec.capacity.storage)[]|[.metadata.name,.spec.capacity.storage]| @tsv'
kubectl get po -o json | jq -r '.items | sort_by(.spec.nodeName)[]|select(.spec.nodeName=="node3")|[.metadata.name,.spec.nodeName]| @tsv'
kubectl get no -o json | jq -r '.items | sort_by(.status.capacity.memory)[]|[.metadata.name,.status.capacity.memory]| @tsv'
# 查看nodelabel
kubectl get nodes --show-labels
# You can remove incorrect label with <label>-
kubectl label node 192.168.1.1 networkSpeed-
# 给node设置label
kubectl label node n11.dev node-role.kubernetes.io/master=master
kubectl label node n12.dev node-role.kubernetes.io/worker=worker
# 创建token
kubeadm token create  --groups system:bootstrappers:kube-n11 --kubeconfig ~/.kube/config
# 删除node节点
kubectl delete no n11.dev;
curl --cacert ca.pem --cert etcd-client.pem --key etcd-client-key.pem https://192.168.33.11:2379/health
curl --cacert ca.pem --cert kubectl.pem --key kubectl-key.pem https://192.168.33.12:6443
curl --cacert ca.pem --cert kubectl.pem --key kubectl-key.pem https://192.168.33.100:8443/api/v1/nodes | jq '.items | .[] | .metadata | .name';
#############
export ETCDCTL_API=3;
etcdctl endpoint health --endpoints "https://192.168.33.11:2379,https://192.168.33.12:2379,https://192.168.33.13:2379" --cacert=/data/k8s/certs/ca.pem --cert=/data/k8s/certs/etcd-client.pem --key=/data/k8s/certs/etcd-client-key.pem --cluster=true
##############
kubectl get role,rolebindings,serviceaccount,clusterrole,clusterrolebindings --all-namespaces -o wide
yaml2json_linux_amd64 < ~/.kube/config |jq '.clusters | .[] | .cluster.server'
kubectl get pods/{NAME} -n {NAMESPACE} --export=true -o yaml
kubectl get configmap extension-apiserver-authentication  -n kube-system
kubectl delete configmap extension-apiserver-authentication  -n kube-system
sudo iptables -L -n -v -t filter|grep -C 2 policy
sudo iptables -L -n -v -t nat|grep -C 2 policy
ss --listening --tcp --numeric --processes '( sport = 10250 )'
# 根据端口号获取pid
ss --listening --tcp --processes "( sport = 10250 )" | grep -oP 'pid=\K\d+'
# [ -n "$pid" ] && kill "$pid"
cat /lib/modules/`uname -r`/build/.config |grep CONFIG_CGROUP_PIDS
cat /boot/config-`uname -r` |grep CONFIG_CGROUP_PIDS
cat /proc/cgroups |grep pids
cat /etc/redhat-release
# https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/20190129-pid-limiting.md#version-114
# https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
# SupportPodPidsLimit
# SupportNodePidsLimit
# kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}'
printf "$(uname -srm)\n$(cat /etc/os-release)\n"
#####
#####
Error: failed to start container "sh": Error response from daemon: OCI runtime create failed: container_linux.go:348: starting container process caused "exec: \"sleep 999999;\": executable file not found in $PATH": unknown
## 命令传错了

Error killing pod: failed to "KillPodSandbox" for "bdbc8477-3a93-11e9-9c65-08002763a99f" with KillPodSandboxError: "rpc error: code = Unknown desc = NetworkPlugin cni failed to teardown pod \"_\" network: operation Delete is not supported on WorkloadEndpoint(default/n26.dev-cni-4af9815b898043486d12acf1b8d0668ca7f2a05f225784003a6704d5453c1388-eth0)"
####################
ls -l /etc/cni/net.d/
cat /run/flannel/subnet.env

sudo iptables -I INPUT -p tcp -m tcp --dport 8472 -j ACCEPT
sudo iptables -I INPUT -p tcp -m tcp --dport 6443 -j ACCEPT
sudo iptables -I INPUT -p tcp -m tcp --dport 9898 -j ACCEPT
sudo iptables -I INPUT -p tcp -m tcp --dport 10250 -j ACCEPT
# 10250是kubectl exec使用的,不加会报“Error from server: error dialing backend: dial tcp 192.168.128.164:10250: getsockopt: no route to host”。
############
ip r
ip r list type blackhole
ip a show type ipip
ip a show type veth
ip -d a list type veth
ip -d link show flannel.1
ip n
ip -d r |grep bird
ip link set dev tunl0 down
ip link delete dev tunl0
nc -nvz 192.168.33.26 179
nc -nvz 192.168.33.26 179
sudo iptables -L -n |grep 179
sudo netstat -natp|grep ESTABLISHED|grep 179
sudo netstat -natp  |grep bird
sudo ss -tnp|grep 179
# 查看当前的模式
calicoctl node status
calicoctl get bgpconfig
calicoctl get bgppeer
###
sudo iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT
sudo iptables -A INPUT -p tcp -m state --state NEW -m tcp --dport 179 -j ACCEPT
sudo iptables -I INPUT -p udp -m udp --dport 8472 -j ACCEPT
sudo iptables -A PREROUTING -t nat -p tcp --dport 179 -j REDIRECT --to-port 1790
-A INPUT -p tcp -m state --state NEW -m tcp --dport 179 -j ACCEPT
-A INPUT -p tcp -m state --state NEW -m tcp --dport 2375 -j ACCEPT
-A INPUT -p tcp -m state --state NEW -m tcp --dport 2379 -j ACCEPT
-A INPUT -p tcp -m state --state NEW -m tcp --dport 2380 -j ACCEPT
-A PREROUTING -p tcp -m tcp --sport 179 -m ttl --ttl-eq 1 -j TTL --ttl-set 2
-A POSTROUTING -p udp -m udp --dport 3784 -j TTL --ttl-set 255
###
kubectl exec nginx11-696b76565d-4jdgp -- /bin/sh -c "ip route"
kubectl exec nginx11-696b76565d-4jdgp -- /bin/sh -c "ip link show"
kubectl exec nginx11-696b76565d-4jdgp -- /bin/sh -c "ip address show"
kubectl exec nginx11-696b76565d-4jdgp -- /bin/sh -c "ping 0.0.0.0"
###
sudo tcpdump -n -vvv -i tunl0
sudo tcpdump -n -vvv -i any dst 192.168.33.26
sudo tcpdump -n -vvv -i any src 192.168.33.26
## nfs 默认端口
nfs       tcp   2049
sunrpc    tcp   111 
sunrpc    udp   111
acp-proto udp   4046
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
# Docker
kubelet-> Kubernetes CRI-> Docker-> containerd-> OCI-> runC->container image
# cri-o 
kubelet-> Kubernetes CRI-> cri-o-> OCI-> runC-> container image
# containerd
kubelet-> Kubernetes CRI-> cri-containerd-> gRPC-> containerd-> OCI-> runC-> container image
# Kubernetes 1.10和containerd 1.1及更高版本
kubelet-> Kubernetes CRI-> containerd-> OCI-> runC-> container image
##########
# level=info msg="devmapper: Creating filesystem xfs on device docker-253:1-34265854-base, mkfs args: [-m crc=0,finobt=0 /dev/mapper/docker-253:1-34265854-base]"
# level=info msg="devmapper: Error while creating filesystem xfs on device docker-253:1-34265854-base: exit status 1"
# level=error msg="[graphdriver] prior storage driver devicemapper failed: exit status 1"
yum update xfsprogs
sudo apt-get install linux-image-extra-`uname -r`
#
kube-proxy 的 --cluster-cidr
kube-controller-manager 的 --cluster-cidr
kubelet 的 podCIDR // 
// The CIDR to use for pod IP addresses, only used in standalone mode.
// In cluster mode, this is obtained from the master.
#
kubeapi-server 的 --service-cluster-ip-range
kube-controller-manager 的 --service-cluster-ip-range
##
kubectl describe ippools default-ipv4-ippool
######
sudo ipvsadm --list --numeric --connection
sudo ipvsadm --list -n |grep 443

TCP  10.96.0.1:443 rr
  -> 192.168.33.21:6443           Masq    1      1          0
  -> 192.168.33.22:6443           Masq    1      1          0
  -> 192.168.33.23:6443           Masq    1      0          0
######
sudo iptables -L -n -t nat |grep -E 'KUBE-SVC.*kubernetes.*443'

KUBE-SVC-NPX46M4PTMTKRN6Y  tcp  --  0.0.0.0/0            10.96.0.1            /* default/kubernetes:https cluster IP */ tcp dpt:443

####
kubectl -n kube-system get cm kube-proxy -o yaml |grep mode
kubectl -n kube-system logs $(kubectl get po -o jsonpath={.items[0].metadata.name} -l k8s-app=kube-proxy -n kube-system)
dpkg-reconfigure popularity-contest
cat /proc/filesystems

###

Error from server (Forbidden): Error: configmaps is forbidden: User "system:node:sjr-k8s-master" cannot list resource "configmaps" in API group "" in the namespace "kube-system": No Object name found

配置模板

kube-proxy-config.json

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
{
    "kind": "KubeProxyConfiguration", 
    "apiVersion": "kubeproxy.config.k8s.io/v1alpha1", 
    "clientConnection": {
        "kubeconfig": "/etc/kubernetes/kube-proxy.kubeconfig"
    }, 
    "bindAddress": "0.0.0.0", 
    "clusterCIDR": "172.20.0.0/16", 
    "healthzBindAddress": "0.0.0.0:10256", 
    "hostnameOverride": "n21.dev", 
    "metricsBindAddress": "127.0.0.1:10249", 
    "mode": "ipvs"
}

kubelet-config.json

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
{
  "kind": "KubeletConfiguration", 
  "apiVersion": "kubelet.config.k8s.io/v1beta1", 
  "authentication": {
    "x509": {
      "clientCAFile": "/etc/kubernetes/cert/kubernetes-ca.pem"
    }, 
    "webhook": {
      "enabled": true, 
      "cacheTTL": "2m0s"
    }, 
    "anonymous": {
      "enabled": false
    }
  }, 
  "authorization": {
    "mode": "Webhook", 
    "webhook": {
      "cacheAuthorizedTTL": "5m0s", 
      "cacheUnauthorizedTTL": "30s"
    }
  }, 
  "address": "0.0.0.0", 
  "port": 10250, 
  "readOnlyPort": 0, 
  "cgroupDriver": "cgroupfs", 
  "hairpinMode": "promiscuous-bridge", 
  "serializeImagePulls": false, 
  "featureGates": {
    "RotateKubeletClientCertificate": true, 
    "RotateKubeletServerCertificate": true
  }, 
  "clusterDomain": "cluster.local.", 
  "clusterDNS": [
    "10.254.0.2"
  ]
}

kube-proxy-cm.yml

1
https://github.com/QingCloudAppcenter/kubernetes/blob/master/k8s/addons/kube-proxy/kube-proxy-cm.yaml
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
apiVersion: v1
data:
  config.conf: |-
    apiVersion: kubeproxy.config.k8s.io/v1alpha1
    bindAddress: 0.0.0.0
    clientConnection:
      acceptContentTypes: ""
      burst: 10
      contentType: application/vnd.kubernetes.protobuf
      kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
      qps: 5
    clusterCIDR: 192.244.0.0/16
    configSyncPeriod: 15m0s
    conntrack:
      max: null
      maxPerCore: 32768
      min: 131072
      tcpCloseWaitTimeout: 1h0m0s
      tcpEstablishedTimeout: 24h0m0s
    enableProfiling: false
    healthzBindAddress: 0.0.0.0:10256
    hostnameOverride: ""
    iptables:
      masqueradeAll: false
      masqueradeBit: 14
      minSyncPeriod: 0s
      syncPeriod: 30s
    ipvs:
      excludeCIDRs: null
      minSyncPeriod: 0s
      scheduler: ""
      syncPeriod: 30s
    kind: KubeProxyConfiguration
    metricsBindAddress: 127.0.0.1:10249
    mode: ""
    nodePortAddresses: null
    oomScoreAdj: -999
    portRange: ""
    resourceContainer: /kube-proxy
    udpIdleTimeout: 250ms
  kubeconfig.conf: |-
    apiVersion: v1
    kind: Config
    clusters:
    - cluster:
        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        server: https://10.42.87.182:6443
      name: default
    contexts:
    - context:
        cluster: default
        namespace: default
        user: default
      name: default
    current-context: default
    users:
    - name: default
      user:
        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
kind: ConfigMap
metadata:
  labels:
    app: kube-proxy
  name: kube-proxy
  namespace: kube-system

参考资料