环境准备
IP | HOSTNAME | SYSTEM |
192.168.131.145 | master | CentOS 7.6 |
192.168.131.146 | node1 | CentOS 7.6 |
192.168.131.147 | node2 | CentOS 7.6 |
[root@localhost ~]# cat /etc/redhat-release CentOS Linux release 7.6.1810 (Core) [root@localhost ~]# sestatus SELinux status: disabled [root@localhost ~]# systemctl status firewalld.service ● firewalld.service - firewalld - dynamic firewall daemon Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled) Active: inactive (dead) Docs: man:firewalld(1)
配置模板机
建议内核是4.X以上的,为了方便后面直接克隆虚拟机,所以创建一个模板机器,不能克隆的环境,需要自己手动升级 ---------------------------------------------------------------------------------------------- 更新:2021年2月6日 此脚本已经不适合升级内核了,现在默认的repo里面的内核都是"5.x了,4.x需要自己编译升级了,千万不要直接使用"
# 以下脚本会升级内核,以及安装ansible,使用的DNS是阿里云的 #!/bin/bash # k8s集群模板机 # from 半癫 uname=$(uname -r | awk -F '.' '{printf $1}') check_network() { printf "\e[1;32m###########正在检查网络######################################\e[0m\n" ping -c1 www.baidu.com > /dev/null 2>&1 if [ $? -ne 0 ] then printf "\e[1;32m###########网络链接失败,请检查网络###########################\e[0m\n" exit 1 else printf "\e[1;32m###########网络链接成功,继续执行,请稍后#####################\e[0m\n" fi } check_network set_network() { printf "\e[1;32m###########正在配置网络######################################\e[0m\n" echo "请输入主机名:" read HOST_NAME if [ -z $HOST_NAME ];then echo "Hostname will not be changed." fi echo "请输入ip地址:" read NETWORK_IP if [ -z $NETWORK_IP ];then echo "You need input IP here. Please try again." exit fi echo "请输入网关: " read GATEWAY_CUSTOM if [ -z $GATEWAY_CUSTOM ];then echo "You need input GATEWAY here. Please try again." exit else NETWORK_GATEWAY=$GATEWAY_CUSTOM fi echo "请输入子网掩码:" read NETMASK_CUSTOM if [ -z $NETMASK_CUSTOM ];then NETWORK_NETMASK=$NETMASK_DEFAULT else NETWORK_NETMASK=$NETMASK_CUSTOM fi echo "$NETWORK_IP $HOST_NAME" >> /etc/hosts sed -i '/^ONBOOT/s/=.*/=yes/' /etc/sysconfig/network-scripts/ifcfg-eth0 sed -i '/^BOOTPROTO/s/=.*/=none/' /etc/sysconfig/network-scripts/ifcfg-eth0 sed -i '/^IPADDR/d' /etc/sysconfig/network-scripts/ifcfg-eth0 sed -i '/^NETMASK/d' /etc/sysconfig/network-scripts/ifcfg-eth0 sed -i '/^GATEWAY/d' /etc/sysconfig/network-scripts/ifcfg-eth0 echo "IPADDR=$NETWORK_IP NETMASK=$NETWORK_NETMASK GATEWAY=$NETWORK_GATEWAY">> /etc/sysconfig/network-scripts/ifcfg-eth0 echo "DNS1=223.5.5.5" >> /etc/sysconfig/network-scripts/ifcfg-eth0 echo "DNS2=223.6.6.6" >> /etc/sysconfig/network-scripts/ifcfg-eth0 echo $HOST_NAME > /etc/hostname systemctl restart network check_network } elrepo() { printf "\e[1;32m###########正在升级内核######################################\e[0m\n" rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm yum --disablerepo=\* --enablerepo=elrepo-kernel repolist yum --disablerepo=\* --enablerepo=elrepo-kernel install kernel-lt.x86_64 -y yum remove kernel-tools-libs.x86_64 kernel-tools.x86_64 -y yum --disablerepo=\* --enablerepo=elrepo-kernel install kernel-lt-tools* -y grub2-set-default 0 printf "\e[1;32m###########内核升级完成######################################\e[0m\n" } init_yum() { printf "\e[1;32m###########正在配置yum源#####################################\e[0m\n" mkdir /etc/yum.repos.d/bak mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/bak wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo yum clean all yum makecache if [ $? -eq 0 ] then printf "\e[1;32m###########yum源配置成功#####################################\e[0m\n" else printf "\e[1;32m###########yum源配置失败#####################################\e[0m\n" exit 3 fi } install_ansible() { printf "\e[1;32m###########正在安装Ansible###################################\e[0m\n" yum install git python-pip -y pip install pip --upgrade -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com pip install ansible==2.6.12 -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com pip install netaddr -i http://pypi.douban.com/simple --trusted-host pypi.douban.com if [ $? -eq 0 ] then printf "\e[1;32m###########Ansible 部署成功##################################\e[0m\n" else printf "\e[1;32m###########Ansible 部署失败##################################\e[0m\n" exit 4 fi } swappoff() { swapoff -a && sysctl -w vm.swappiness=0 sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab } selinuxstatus() { systemctl disable --now firewalld setenforce 0 sed -i "s/SELINUX=enforcing/SELINUX=disabled/g" /etc/selinux/config } if [ $uname -eq 4 ] then printf "\e[1;32m###########内核版本为4,无须修改##############################\e[0m\n" selinuxstatus set_network init_yum install_ansible swappoff /usr/sbin/init 0 else printf "\e[1;32m###########内核版本为3,准备升级内核##########################\e[0m\n" selinuxstatus set_network init_yum install_ansible swappoff elrepo sleep 3 /usr/sbin/init 0 fi
# 克隆完机器后,记得修改ip和hostname
配置hosts解析
[root@master ~]# sed -i 's/192.168.131.144.*//g' /etc/hosts # 模板机里面的要删掉 [root@master ~]# cat >> /etc/hosts <<EOF 192.168.131.145 master 192.168.131.146 node1 192.168.131.147 node2 EOF
配置ssh 免密钥登陆
[root@master ~]# ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa [root@master ~]# yum -y install sshpass [root@master ~]# cat ssh-key.sh for node in master node1 node2 do sshpass -p '123.com' ssh-copy-id ${node} -o StrictHostKeyChecking=no scp /etc/hosts ${node}:/etc/hosts if [ $? -eq 0 ];then echo "${node} 秘钥copy完成" else echo "${node} 秘钥copy失败" fi done [root@master ~]# sh ssh-key.sh
kubeasz 部署服务准备
- github上的kubeasz已经更新了,导致和之前写的博客不一样,之前保留了一份kubeasz在本地,现已经上传到百度云了,新版的kubeasz暂时没有好好的把玩过
- 链接:https://pan.baidu.com/s/1rFscCCLHhD4O3os_9yKqEQ
提取码:o1bs
[root@master ~]# git clone https://github.com/easzlab/kubeasz.git [root@master ~]# cd kubeasz/ [root@master kubeasz]# ll total 88 -rw-r--r-- 1 root root 395 Sep 7 16:56 01.prepare.yml -rw-r--r-- 1 root root 58 Sep 7 16:56 02.etcd.yml -rw-r--r-- 1 root root 149 Sep 7 16:56 03.containerd.yml -rw-r--r-- 1 root root 137 Sep 7 16:56 03.docker.yml -rw-r--r-- 1 root root 470 Sep 7 16:56 04.kube-master.yml -rw-r--r-- 1 root root 140 Sep 7 16:56 05.kube-node.yml -rw-r--r-- 1 root root 408 Sep 7 16:56 06.network.yml -rw-r--r-- 1 root root 77 Sep 7 16:56 07.cluster-addon.yml -rw-r--r-- 1 root root 3686 Sep 7 16:56 11.harbor.yml -rw-r--r-- 1 root root 431 Sep 7 16:56 22.upgrade.yml -rw-r--r-- 1 root root 1975 Sep 7 16:56 23.backup.yml -rw-r--r-- 1 root root 113 Sep 7 16:56 24.restore.yml -rw-r--r-- 1 root root 1752 Sep 7 16:56 90.setup.yml -rw-r--r-- 1 root root 1127 Sep 7 16:56 91.start.yml -rw-r--r-- 1 root root 1120 Sep 7 16:56 92.stop.yml -rw-r--r-- 1 root root 337 Sep 7 16:56 99.clean.yml -rw-r--r-- 1 root root 10283 Sep 7 16:56 ansible.cfg drwxr-xr-x 2 root root 23 Sep 7 16:56 bin drwxr-xr-x 2 root root 23 Sep 7 16:56 dockerfiles drwxr-xr-x 8 root root 92 Sep 7 16:56 docs drwxr-xr-x 2 root root 25 Sep 7 16:56 down drwxr-xr-x 2 root root 52 Sep 7 16:56 example drwxr-xr-x 14 root root 218 Sep 7 16:56 manifests drwxr-xr-x 2 root root 322 Sep 7 16:56 pics -rw-r--r-- 1 root root 5653 Sep 7 16:56 README.md drwxr-xr-x 23 root root 4096 Sep 7 16:56 roles drwxr-xr-x 2 root root 294 Sep 7 16:56 tools [root@master kubeasz]# tools/easzup -D
配置主机清单
[root@master kubeasz]# cd /etc/ansible/ # 完成后,所需的文件都会下载到ansible目录下 [root@master ansible]# ll total 92 -rw-rw-r-- 1 root root 395 May 28 21:11 01.prepare.yml -rw-rw-r-- 1 root root 58 May 28 21:11 02.etcd.yml -rw-rw-r-- 1 root root 149 May 28 21:11 03.containerd.yml -rw-rw-r-- 1 root root 137 May 28 21:11 03.docker.yml -rw-rw-r-- 1 root root 470 May 28 21:11 04.kube-master.yml -rw-rw-r-- 1 root root 140 May 28 21:11 05.kube-node.yml -rw-rw-r-- 1 root root 408 May 28 21:11 06.network.yml -rw-rw-r-- 1 root root 77 May 28 21:11 07.cluster-addon.yml -rw-rw-r-- 1 root root 3686 May 28 21:11 11.harbor.yml -rw-rw-r-- 1 root root 431 May 28 21:11 22.upgrade.yml -rw-rw-r-- 1 root root 1975 May 28 21:11 23.backup.yml -rw-rw-r-- 1 root root 113 May 28 21:11 24.restore.yml -rw-rw-r-- 1 root root 1752 May 28 21:11 90.setup.yml -rw-rw-r-- 1 root root 1127 May 28 21:11 91.start.yml -rw-rw-r-- 1 root root 1120 May 28 21:11 92.stop.yml -rw-rw-r-- 1 root root 337 May 28 21:11 99.clean.yml -rw-rw-r-- 1 root root 10283 May 28 21:11 ansible.cfg drwxrwxr-x 2 root root 4096 Sep 7 16:59 bin drwxrwxr-x 2 root root 23 May 29 09:15 dockerfiles drwxrwxr-x 8 root root 92 May 29 09:15 docs drwxrwxr-x 2 root root 292 Sep 7 17:01 down drwxrwxr-x 2 root root 52 May 29 09:15 example drwxrwxr-x 14 root root 218 May 29 09:15 manifests drwxrwxr-x 2 root root 322 May 29 09:15 pics -rw-rw-r-- 1 root root 5653 May 28 21:11 README.md drwxrwxr-x 23 root root 4096 May 29 09:15 roles drwxrwxr-x 2 root root 294 May 29 09:15 tools [root@master ansible]# cp example/hosts.multi-node ./hosts [root@master ansible]# vim hosts # 'etcd' cluster should have odd member(s) (1,3,5,...) # variable 'NODE_NAME' is the distinct name of a member in 'etcd' cluster [etcd] 192.168.131.145 NODE_NAME=etcd1 # master node(s) [kube-master] 192.168.131.145 # work node(s) [kube-node] 192.168.131.146 192.168.131.147 # [optional] harbor server, a private docker registry # 'NEW_INSTALL': 'yes' to install a harbor server; 'no' to integrate with existed one # 'SELF_SIGNED_CERT': 'no' you need put files of certificates named harbor.pem and harbor-key.pem in directory 'down' [harbor] #192.168.1.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no SELF_SIGNED_CERT=yes # [optional] loadbalance for accessing k8s from outside [ex-lb] #192.168.1.6 LB_ROLE=backup EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443 #192.168.1.7 LB_ROLE=master EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443 # [optional] ntp server for the cluster [chrony] 192.168.131.145 [all:vars] # --------- Main Variables --------------- # Cluster container-runtime supported: docker, containerd CONTAINER_RUNTIME="docker" # Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn CLUSTER_NETWORK="calico" # Service proxy mode of kube-proxy: 'iptables' or 'ipvs' PROXY_MODE="ipvs" # K8S Service CIDR, not overlap with node(host) networking SERVICE_CIDR="10.68.0.0/16" # Cluster CIDR (Pod CIDR), not overlap with node(host) networking CLUSTER_CIDR="172.20.0.0/16" # NodePort Range NODE_PORT_RANGE="20000-40000" # Cluster DNS Domain CLUSTER_DNS_DOMAIN="cluster.local." # -------- Additional Variables (don't change the default value right now) --- # Binaries Directory bin_dir="/opt/kube/bin" # CA and other components cert/key Directory ca_dir="/etc/kubernetes/ssl" # Deploy Directory (kubeasz workspace) base_dir="/etc/ansible"
配置主机清单
[root@master kubeasz]# cd /etc/ansible/ # 完成后,所需的文件都会下载到ansible目录下 [root@master ansible]# ll total 92 -rw-rw-r-- 1 root root 395 May 28 21:11 01.prepare.yml -rw-rw-r-- 1 root root 58 May 28 21:11 02.etcd.yml -rw-rw-r-- 1 root root 149 May 28 21:11 03.containerd.yml -rw-rw-r-- 1 root root 137 May 28 21:11 03.docker.yml -rw-rw-r-- 1 root root 470 May 28 21:11 04.kube-master.yml -rw-rw-r-- 1 root root 140 May 28 21:11 05.kube-node.yml -rw-rw-r-- 1 root root 408 May 28 21:11 06.network.yml -rw-rw-r-- 1 root root 77 May 28 21:11 07.cluster-addon.yml -rw-rw-r-- 1 root root 3686 May 28 21:11 11.harbor.yml -rw-rw-r-- 1 root root 431 May 28 21:11 22.upgrade.yml -rw-rw-r-- 1 root root 1975 May 28 21:11 23.backup.yml -rw-rw-r-- 1 root root 113 May 28 21:11 24.restore.yml -rw-rw-r-- 1 root root 1752 May 28 21:11 90.setup.yml -rw-rw-r-- 1 root root 1127 May 28 21:11 91.start.yml -rw-rw-r-- 1 root root 1120 May 28 21:11 92.stop.yml -rw-rw-r-- 1 root root 337 May 28 21:11 99.clean.yml -rw-rw-r-- 1 root root 10283 May 28 21:11 ansible.cfg drwxrwxr-x 2 root root 4096 Sep 7 16:59 bin drwxrwxr-x 2 root root 23 May 29 09:15 dockerfiles drwxrwxr-x 8 root root 92 May 29 09:15 docs drwxrwxr-x 2 root root 292 Sep 7 17:01 down drwxrwxr-x 2 root root 52 May 29 09:15 example drwxrwxr-x 14 root root 218 May 29 09:15 manifests drwxrwxr-x 2 root root 322 May 29 09:15 pics -rw-rw-r-- 1 root root 5653 May 28 21:11 README.md drwxrwxr-x 23 root root 4096 May 29 09:15 roles drwxrwxr-x 2 root root 294 May 29 09:15 tools [root@master ansible]# cp example/hosts.multi-node ./hosts [root@master ansible]# vim hosts # 'etcd' cluster should have odd member(s) (1,3,5,...) # variable 'NODE_NAME' is the distinct name of a member in 'etcd' cluster [etcd] 192.168.131.145 NODE_NAME=etcd1 # master node(s) [kube-master] 192.168.131.145 # work node(s) [kube-node] 192.168.131.146 192.168.131.147 # [optional] harbor server, a private docker registry # 'NEW_INSTALL': 'yes' to install a harbor server; 'no' to integrate with existed one # 'SELF_SIGNED_CERT': 'no' you need put files of certificates named harbor.pem and harbor-key.pem in directory 'down' [harbor] #192.168.1.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no SELF_SIGNED_CERT=yes # [optional] loadbalance for accessing k8s from outside [ex-lb] #192.168.1.6 LB_ROLE=backup EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443 #192.168.1.7 LB_ROLE=master EX_APISERVER_VIP=192.168.1.250 EX_APISERVER_PORT=8443 # [optional] ntp server for the cluster [chrony] 192.168.131.145 [all:vars] # --------- Main Variables --------------- # Cluster container-runtime supported: docker, containerd CONTAINER_RUNTIME="docker" # Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn CLUSTER_NETWORK="calico" # Service proxy mode of kube-proxy: 'iptables' or 'ipvs' PROXY_MODE="ipvs" # K8S Service CIDR, not overlap with node(host) networking SERVICE_CIDR="10.68.0.0/16" # Cluster CIDR (Pod CIDR), not overlap with node(host) networking CLUSTER_CIDR="172.20.0.0/16" # NodePort Range NODE_PORT_RANGE="20000-40000" # Cluster DNS Domain CLUSTER_DNS_DOMAIN="cluster.local." # -------- Additional Variables (don't change the default value right now) --- # Binaries Directory bin_dir="/opt/kube/bin" # CA and other components cert/key Directory ca_dir="/etc/kubernetes/ssl" # Deploy Directory (kubeasz workspace) base_dir="/etc/ansible"
测试是否ping通 [root@master ansible]# ansible all -m ping /usr/lib/python2.7/site-packages/ansible/parsing/vault/__init__.py:44: CryptographyDeprecationWarning: Python 2 is no longer supported by the Python core team. Support for it is now deprecated in cryptography, and will be removed in a future release. from cryptography.exceptions import InvalidSignature 192.168.131.145 | SUCCESS => { "changed": false, "ping": "pong" } 192.168.131.147 | SUCCESS => { "changed": false, "ping": "pong" } 192.168.131.146 | SUCCESS => { "changed": false, "ping": "pong" }
部署集群
[root@master ansible]# ansible-playbook 90.setup.yml PLAY RECAP ****************************************************************************************** 192.168.131.145 : ok=99 changed=84 unreachable=0 failed=0 192.168.131.146 : ok=104 changed=91 unreachable=0 failed=0 192.168.131.147 : ok=99 changed=86 unreachable=0 failed=0 localhost : ok=37 changed=33 unreachable=0 failed=0 [root@master ansible]# echo "source <(kubectl completion bash)" >> ~/.bashrc # 配置命令补全 [root@master ansible]# kubectl get nodes NAME STATUS ROLES AGE VERSION 192.168.131.145 Ready,SchedulingDisabled master 6m49s v1.18.3 192.168.131.146 Ready node 5m41s v1.18.3 192.168.131.147 Ready node 5m41s v1.18.3 [root@master ansible]# kubectl top nodes NAME CPU(cores) CPU% MEMORY(bytes) MEMORY% 192.168.131.145 202m 10% 1354Mi 105% 192.168.131.146 83m 8% 485Mi 176% 192.168.131.147 60m 6% 473Mi 172% [root@master ansible]# kubectl get pods -A NAMESPACE NAME READY STATUS RESTARTS AGE kube-system calico-kube-controllers-7fdc86d8ff-96nlk 1/1 Running 0 91s kube-system calico-node-2jc2n 1/1 Running 0 91s kube-system calico-node-dft74 1/1 Running 0 91s kube-system calico-node-fdgwc 1/1 Running 0 91s kube-system coredns-65dbdb44db-8tlpg 1/1 Running 0 59s kube-system dashboard-metrics-scraper-545bbb8767-hd7gj 1/1 Running 0 22s kube-system kubernetes-dashboard-65665f84db-qtx8b 1/1 Running 0 23s kube-system metrics-server-869ffc99cd-6w2n8 1/1 Running 0 51s [root@master ansible]# kubectl get svc -n kube-system # dashboard开放的端口是39915 NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE dashboard-metrics-scraper ClusterIP 10.68.124.158 <none> 8000/TCP 7m16s kube-dns ClusterIP 10.68.0.2 <none> 53/UDP,53/TCP,9153/TCP 7m52s kubernetes-dashboard NodePort 10.68.224.242 <none> 443:39915/TCP 7m16s metrics-server ClusterIP 10.68.152.27 <none> 443/TCP 7m44s [root@master ansible]# kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') Name: admin-user-token-cjl42 Namespace: kube-system Labels: <none> Annotations: kubernetes.io/service-account.name: admin-user kubernetes.io/service-account.uid: dd7fb3fb-a95d-4f51-9df4-efcd7bf9bbc3 Type: kubernetes.io/service-account-token Data ==== namespace: 11 bytes token: eyJhbGciOiJSUzI1NiIsImtpZCI6IjZwUjlZNUYzd0pCRzFrQVBHLTNYeFhuVEpyZi1rNU1MbnhIT2VWaG5BSXcifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLWNqbDQyIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJkZDdmYjNmYi1hOTVkLTRmNTEtOWRmNC1lZmNkN2JmOWJiYzMiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06YWRtaW4tdXNlciJ9.maXSb5FGfutG1fLQwCM6L1sNYPU9lOSr0hIE7xQOxvpDW-oDFPWWrLkVyHQgfG1bwZxqu-M3YfPy6cYSqhuGB7-UAR18TaG3rXNDmaa6QnMlLch65ZyoaUkHb_X_woa3ZL_TOd9NnckuZ4lo5e-PudDWRGmUJSmlXTG-O10kmi_RQ_txjD4wXa4XGl-GER7JXTc78Nhbacj1uyzm2SDk4xTsT2tN6C3sQt_5hfhxTxmBhM-9kw12_a9a6FVxLi8CB6GOoAqxmckPU-FMbgSOO_VOc6idN4D9OMbZtSuqXvop-SxL6PkcOUEoc9tK12U81pekBlwKPIcVjmcKnHnMQQ ca.crt: 1350 bytes # 浏览器访问:https://192.168.131.145:39915