1.3、部署flannel网络
所有节点都需要flannel
1.3.0、下载flannel二进制文件
k8s-01:~ # cd /opt/k8s/packages/ k8s-01:/opt/k8s/packages # mkdir flannel k8s-01:/opt/k8s/packages # wget https://github.com/coreos/flannel/releases/download/v0.12.0/flannel-v0.12.0-linux-amd64.tar.gz k8s-01:/opt/k8s/packages # tar xf flannel-v0.12.0-linux-amd64.tar.gz -C /opt/k8s/packages/flannel/
1.3.1、创建flannel证书和私钥
k8s-01:~ # cd /opt/k8s/ssl/ k8s-01:/opt/k8s/ssl # cat > flanneld-csr.json <<EOF { "CN": "flanneld", "hosts": [ ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "ShangHai", "L": "ShangHai", "O": "k8s", "OU": "bandian" } ] } EOF
1.3.2、生成flannel证书和私钥
k8s-01:/opt/k8s/ssl # cfssl gencert -ca=/opt/k8s/ssl/ca.pem \ -ca-key=/opt/k8s/ssl/ca-key.pem \ -config=/opt/k8s/ssl/ca-config.json \ -profile=kubernetes flanneld-csr.json | cfssljson -bare flanneld
1.3.3、将pod网段写入etcd
k8s-01:~ # cd /opt/k8s/ssl/ k8s-01:/opt/k8s/ssl # source /opt/k8s/bin/k8s-env.sh k8s-01:/opt/k8s/ssl # ETCDCTL_API=2 etcdctl \ --endpoints=${ETCD_ENDPOINTS} \ --ca-file=/opt/k8s/ssl/ca.pem \ --cert-file=/opt/k8s/ssl/flanneld.pem \ --key-file=/opt/k8s/ssl/flanneld-key.pem \ mk ${FLANNEL_ETCD_PREFIX}/config '{"Network":"'${CLUSTER_CIDR}'", "SubnetLen": 21, "Backend": {"Type": "vxlan"}}'
- 因为flannel当前版本
0.12.0不支持etcd v3，因此需要使用etcd v2API写入配置，否则后面启动flanneld会找不到写入的key
1.3.4、配置flannel为systemctl管理
k8s-01:~ # cd /opt/k8s/conf/ k8s-01:/opt/k8s/conf # source /opt/k8s/bin/k8s-env.sh k8s-01:/opt/k8s/conf # cat > flanneld.service << EOF [Unit] Description=Flanneld overlay address etcd agent After=network.target After=network-online.target Wants=network-online.target After=etcd.service Before=docker.service [Service] Type=notify ExecStart=/opt/k8s/bin/flanneld \\ -etcd-cafile=/etc/kubernetes/cert/ca.pem \\ -etcd-certfile=/etc/flanneld/cert/flanneld.pem \\ -etcd-keyfile=/etc/flanneld/cert/flanneld-key.pem \\ -etcd-endpoints=${ETCD_ENDPOINTS} \\ -etcd-prefix=${FLANNEL_ETCD_PREFIX} ExecStartPost=/opt/k8s/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker Restart=always RestartSec=5 StartLimitInterval=0 [Install] WantedBy=multi-user.target RequiredBy=docker.service EOF
mk-docker-opts.sh脚本将分配给 flanneld 的 Pod 子网段信息写入 /run/flannel/docker 文件，后续 docker 启动时使用这个文件中的环境变量配置 docker0 网桥- flanneld 使用系统缺省路由所在的接口与其它节点通信，对于有多个网络接口（如内网和公网）的节点，可以用
-iface参数指定通信接口 - -ip-masq flanneld 为访问 Pod 网络外的流量设置 SNAT 规则，同时将传递给 Docker 的变量 -ip-masq（/run/flannel/docker 文件中）设置为 false，这样 Docker 将不再创建 SNAT 规则；
Docker 的 -ip-masq 为 true 时，创建的 SNAT 规则比较“暴力”：将所有本节点 Pod 发起的、访问非 docker0 接口的请求做 SNAT，这样访问其他节点 Pod 的请求来源 IP 会被设置为 flannel.1 接口的 IP，导致目的 Pod 看不到真实的来源 Pod IP。
flanneld 创建的 SNAT 规则比较温和，只对访问非 Pod 网段的请求做 SNAT。
1.3.5、分发flannel证书和启动文件到所有节点
#!/usr/bin/env bash source /opt/k8s/bin/k8s-env.sh for host in ${NODE_IPS[@]} do printf "\e[1;34m${host}\e[0m\n" ssh root@${host} "mkdir -p /etc/flanneld/cert" scp /opt/k8s/ssl/flanneld*.pem ${host}:/etc/flanneld/cert/ scp /opt/k8s/packages/flannel/{flanneld,mk-docker-opts.sh} ${host}:/opt/k8s/bin/ scp /opt/k8s/conf/flanneld.service ${host}:/etc/systemd/system/ done
1.3.6、配置并启动flannel服务
#!/usr/bin/env bash source /opt/k8s/bin/k8s-env.sh for host in ${NODE_IPS[@]} do printf "\e[1;34m${host}\e[0m\n" ssh root@${host} "systemctl daemon-reload && \ systemctl enable flanneld && \ systemctl restart flanneld && \ systemctl status flanneld | grep Active" done
1.3.7、查看已分配的pod网段列表
k8s-01:~ # ETCDCTL_API=2 etcdctl \ --endpoints=${ETCD_ENDPOINTS} \ --ca-file=/etc/kubernetes/cert/ca.pem \ --cert-file=/etc/flanneld/cert/flanneld.pem \ --key-file=/etc/flanneld/cert/flanneld-key.pem \ ls ${FLANNEL_ETCD_PREFIX}/subnets
1.3.8、查看各节点是否都存在flannel网卡
#!/usr/bin/env bash source /opt/k8s/bin/k8s-env.sh for host in ${NODE_IPS[@]} do printf "\e[1;34m${host}\e[0m\n" ssh root@${host} "ip a | grep flannel | grep -w inet" done
