作者:尹正杰
版权声明:原创作品,谢绝转载!否则将追究法律责任。
一.热迁移概述
1.迁移的类型
冷迁移:
将虚拟机关机,然后将配置文件和磁盘文件拷贝到另一台KVM宿主机上重启启动的过程。
热迁移:
在不关闭KVM虚拟机的情况下,迁移KVM宿主机内的相应的虚拟机到其他KVM宿主机的过程。
2.热迁移的实现方案
热迁移原理:
如上图所示,两个KVM宿主机共享挂载NFS服务器的同一个存储目录,该目录存储了虚拟机的磁盘文件。
温馨提示:
- 其实磁盘文件也可以实现共享,这样就没有必要执行迁移的命令啦;
- 基于NFS挂载磁盘,效率可能会有点低存在延迟的可能性,生产环境建议使用cephFS分布式文件系统或者专业的NAS设备;
二.KVM虚拟机热迁移方案-临时迁移
1.查看KVM虚拟机是否安装NFS环境
rpm -qa | grep nfs
2.部署NFS服务器
1.安装NFS工具包
[root@yinzhengjie-kvm ~]# yum -y install nfs-utils
2.配置共享的nfs目录
[root@yinzhengjie-kvm ~]# cat /etc/exports
/yinzhengjie/data 10.0.0.0/24(rw,sync,no_root_squash,no_all_squash)
[root@yinzhengjie-kvm ~]#
3.配置NFS开机自启动
[root@yinzhengjie-kvm ~]# systemctl enable --now nfs
Created symlink from /etc/systemd/system/multi-user.target.wants/nfs-server.service to /usr/lib/systemd/system/nfs-server.service.
[root@yinzhengjie-kvm ~]#
4.验证服务是否暴露成功
[root@yinzhengjie-kvm ~]# exportfs
/yinzhengjie/data
10.0.0.0/24
[root@yinzhengjie-kvm ~]#
[root@yinzhengjie-kvm ~]# showmount -e 10.0.0.188
Export list for 10.0.0.188:
/yinzhengjie/data 10.0.0.0/24
[root@yinzhengjie-kvm ~]#
5.创建工作目录
[root@yinzhengjie-kvm ~]# mkdir -pv /yinzhengjie/data
mkdir: created directory ‘/yinzhengjie’
mkdir: created directory ‘/yinzhengjie/data’
[root@yinzhengjie-kvm ~]#
[root@yinzhengjie-kvm ~]# ll /yinzhengjie/data
total 0
[root@yinzhengjie-kvm ~]#
3.源节点KVM宿主机环境准备
1.源虚拟机将现有的磁盘文件备份到从/opt目录备份到/mnt目录
[root@yinzhengjie-kvm189 ~]# ll /opt/
total 7824148
-rw-r--r--. 1 qemu qemu 4712300544 Apr 9 01:25 CentOS-7-x86_64-DVD-2009.iso
-rw-r--r--. 1 root root 9764864 Apr 14 05:24 k8s-master231.qcow2
-rw-r--r--. 1 root root 197120 Apr 14 05:55 k8s-worker232_add_disk01.qcow2
-rw-r--r--. 1 root root 9764864 Apr 14 05:47 k8s-worker232.qcow2
drwxr-xr-x. 2 root root 6 Oct 31 2018 rh
-rw-------. 1 root root 10739318784 Apr 14 04:08 yinzhengjie-kvm01.qcow2
-rw-------. 1 qemu qemu 1622278144 Apr 14 05:16 yinzhengjie-kvm02.qcow2
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# ll /mnt/
total 0
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# mv /opt/* /mnt/
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# ll /mnt/
total 7824148
-rw-r--r--. 1 qemu qemu 4712300544 Apr 9 01:25 CentOS-7-x86_64-DVD-2009.iso
-rw-r--r--. 1 root root 9764864 Apr 14 05:24 k8s-master231.qcow2
-rw-r--r--. 1 root root 197120 Apr 14 05:55 k8s-worker232_add_disk01.qcow2
-rw-r--r--. 1 root root 9764864 Apr 14 05:47 k8s-worker232.qcow2
drwxr-xr-x. 2 root root 6 Oct 31 2018 rh
-rw-------. 1 root root 10739318784 Apr 14 04:08 yinzhengjie-kvm01.qcow2
-rw-------. 1 qemu qemu 1622278144 Apr 14 05:16 yinzhengjie-kvm02.qcow2
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# ll /opt/
total 0
[root@yinzhengjie-kvm189 ~]#
2.源KVM虚拟机挂载NFS服务器的配置(生产环境应该采用开机自动挂载,此处我只是手动测试)
[root@yinzhengjie-kvm189 ~]# mount -t nfs 10.0.0.188:/yinzhengjie/data /opt/
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# df -h | grep opt
10.0.0.188:/yinzhengjie/data 50G 5.5G 45G 11% /opt
[root@yinzhengjie-kvm189 ~]#
3.将源KVM虚拟机磁盘文件移动到NFS的共享目录(此过程可能比较缓慢,因为设计到跨主机拷贝文件,需要等待一段时间)
[root@yinzhengjie-kvm189 ~]# mv /mnt/*.qcow2 /opt/
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# ll -h /opt/
total 3.6G
-rw-r--r--. 1 root root 9.4M Apr 14 05:24 k8s-master231.qcow2
-rw-r--r--. 1 root root 193K Apr 14 05:55 k8s-worker232_add_disk01.qcow2
-rw-r--r--. 1 root root 9.4M Apr 14 05:47 k8s-worker232.qcow2
-rw-------. 1 root root 11G Apr 14 04:08 yinzhengjie-kvm01.qcow2
-rw-------. 1 qemu qemu 1.6G Apr 14 05:16 yinzhengjie-kvm02.qcow2
[root@yinzhengjie-kvm189 ~]#
4.目标节点KVM宿主机环境准备
1.删除虚拟机配置文件及列表
[root@yinzhengjie-kvm190 ~]# virsh list --all
Id Name State
----------------------------------------------------
- k8s-master231 shut off
- k8s-worker232 shut off
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# virsh undefine k8s-master231
Domain k8s-master231 has been undefined
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# virsh undefine k8s-worker232
Domain k8s-worker232 has been undefined
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# virsh list --all
Id Name State
----------------------------------------------------
[root@yinzhengjie-kvm190 ~]#
2.删除虚拟机现有的磁盘文件
[root@yinzhengjie-kvm190 ~]# ll /opt/
total 6205584
-rw-r--r--. 1 qemu qemu 4712300544 Apr 9 01:25 CentOS-7-x86_64-DVD-2009.iso
-rw-r--r--. 1 root root 9961472 Apr 14 05:28 k8s-master231.qcow2
-rw-------. 1 root root 4258 Apr 14 05:25 k8s-master231.xml
-rw-r--r--. 1 root root 9961472 Apr 14 06:03 k8s-worker232.qcow2
-rw-------. 1 root root 4528 Apr 14 06:00 k8s-worker232.xml
drwxr-xr-x. 2 root root 6 Oct 31 2018 rh
-rw-------. 1 qemu qemu 1622278144 Apr 14 05:26 yinzhengjie-kvm02.qcow2
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# rm -rf /opt/*
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# ll /opt/
total 0
[root@yinzhengjie-kvm190 ~]#
3.挂载NFS目录,查看虚拟机的磁盘文件
[root@yinzhengjie-kvm190 ~]# ll /opt/
total 0
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# mount -t nfs 10.0.0.188:/yinzhengjie/data /opt/
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# df -h | grep opt
10.0.0.188:/yinzhengjie/data 50G 9.1G 41G 19% /opt
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# ll -h /opt/
total 3.6G
-rw-r--r--. 1 root root 9.4M Apr 14 05:24 k8s-master231.qcow2
-rw-r--r--. 1 root root 193K Apr 14 05:55 k8s-worker232_add_disk01.qcow2
-rw-r--r--. 1 root root 9.4M Apr 14 05:47 k8s-worker232.qcow2
-rw-------. 1 root root 11G Apr 14 04:08 yinzhengjie-kvm01.qcow2
-rw-------. 1 qemu qemu 1.6G Apr 14 05:16 yinzhengjie-kvm02.qcow2
[root@yinzhengjie-kvm190 ~]#
5.开始迁移虚拟机
1.准备解析主机文件
[root@yinzhengjie-kvm189 ~]# cat >> /etc/hosts <<EOF
10.0.0.189 yinzhengjie-kvm189
10.0.0.190 yinzhengjie-kvm190
EOF
2.启动虚拟机
[root@yinzhengjie-kvm189 ~]# virsh start yinzhengjie-kvm01
Domain yinzhengjie-kvm01 started
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# virsh list --all
Id Name State
----------------------------------------------------
11 yinzhengjie-kvm01 running
- k8s-master231 shut off
- k8s-worker232 shut off
- yinzhengjie-kvm02 shut off
[root@yinzhengjie-kvm189 ~]#
3.开始迁移虚拟机
[root@yinzhengjie-kvm189 ~]# virsh migrate --live --verbose yinzhengjie-kvm01 qemu+ssh://yinzhengjie-kvm190/system --unsafe
root@yinzhengjie-kvm190's password:
Migration: [100 %]
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# virsh list --all
Id Name State
----------------------------------------------------
- k8s-master231 shut off
- k8s-worker232 shut off
- yinzhengjie-kvm01 shut off
- yinzhengjie-kvm02 shut off
[root@yinzhengjie-kvm189 ~]#
4.迁移虚拟机成功后,验证查看
[root@yinzhengjie-kvm190 ~]# virsh list --all
Id Name State
----------------------------------------------------
2 yinzhengjie-kvm01 running
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# ll /etc/libvirt/qemu # 并没有迁移配置文件,说明时临时迁移的
total 0
drwx------. 3 root root 42 Apr 28 2021 networks
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]#
5.临时迁移虚拟机会存在如下的bug,当关闭虚拟机时,虚拟机莫名其妙的就消失啦~
[root@yinzhengjie-kvm190 ~]# virsh list --all
Id Name State
----------------------------------------------------
2 yinzhengjie-kvm01 running
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# virsh destroy yinzhengjie-kvm01
Domain yinzhengjie-kvm01 destroyed
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# virsh list --all
Id Name State
----------------------------------------------------
[root@yinzhengjie-kvm190 ~]#
三.KVM虚拟机热迁移方案-永久迁移
1.KVM源节点启动虚拟机并查看IP地址
[root@yinzhengjie-kvm189 ~]# virsh list --all
Id Name State
----------------------------------------------------
- k8s-master231 shut off
- k8s-worker232 shut off
- yinzhengjie-kvm01 shut off
- yinzhengjie-kvm02 shut off
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# virsh start k8s-master231
Domain k8s-master231 started
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# virsh list --all
Id Name State
----------------------------------------------------
12 k8s-master231 running
- k8s-worker232 shut off
- yinzhengjie-kvm01 shut off
- yinzhengjie-kvm02 shut off
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# virsh console k8s-master231
Connected to domain k8s-master231
Escape character is ^]
CentOS Linux 7 (Core)
Kernel 3.10.0-1160.el7.x86_64 on an x86_64
yinzhengjie-kvm01 login: root
Password:
Last login: Sun Apr 14 05:23:08 on ttyS0
[root@yinzhengjie-kvm01 ~]#
[root@yinzhengjie-kvm01 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 52:54:00:48:88:51 brd ff:ff:ff:ff:ff:ff
inet 192.168.122.92/24 brd 192.168.122.255 scope global noprefixroute dynamic eth0
valid_lft 3565sec preferred_lft 3565sec
inet6 fe80::ae66:b680:585a:e5c2/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@yinzhengjie-kvm01 ~]#
2.开始永久迁移虚拟机
[root@yinzhengjie-kvm189 ~]# virsh list --all
Id Name State
----------------------------------------------------
12 k8s-master231 running
- k8s-worker232 shut off
- yinzhengjie-kvm01 shut off
- yinzhengjie-kvm02 shut off
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# virsh migrate --live --verbose k8s-master231 qemu+ssh://yinzhengjie-kvm190/system --unsafe --persistent --undefinesource
root@yinzhengjie-kvm190's password:
Migration: [100 %]
[root@yinzhengjie-kvm189 ~]#
[root@yinzhengjie-kvm189 ~]# virsh list --all
Id Name State
----------------------------------------------------
- k8s-worker232 shut off
- yinzhengjie-kvm01 shut off
- yinzhengjie-kvm02 shut off
[root@yinzhengjie-kvm189 ~]#
温馨提示:
--persistent
表示同时虚拟机的迁移配置文件。
--undefinesource
表示迁移时删除源虚拟机。
3.迁移后查看虚拟机IP地址是否发生变化
[root@yinzhengjie-kvm190 ~]# virsh list --all
Id Name State
----------------------------------------------------
3 k8s-master231 running
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# ll /etc/libvirt/qemu
total 8
-rw-------. 1 root root 4258 Apr 15 05:19 k8s-master231.xml
drwx------. 3 root root 42 Apr 28 2021 networks
[root@yinzhengjie-kvm190 ~]#
[root@yinzhengjie-kvm190 ~]# virsh console k8s-master231
Connected to domain k8s-master231
Escape character is ^]
[root@yinzhengjie-kvm01 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 52:54:00:48:88:51 brd ff:ff:ff:ff:ff:ff
inet 192.168.122.92/24 brd 192.168.122.255 scope global noprefixroute dynamic eth0
valid_lft 2902sec preferred_lft 2902sec
inet6 fe80::ae66:b680:585a:e5c2/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@yinzhengjie-kvm01 ~]#
温馨提示:
迁移过程中,虚拟机的IP地址是不会发生变化的哟~
四.可能存在的故障
1.error: internal error: process exited while connecting to monitor: 2024-04-14T20:51:33.275513Z qemu-kvm: -drive file=/opt/yinzhengjie-kvm01.qcow2,format=qcow2,if=none,id=drive-virtio-disk0: could not open disk image /opt/yinzhengjie-kvm01.qcow2: Could not open '/opt/yinzhengjie-kvm01.qcow2': Permission denied
问题原因:
宿主机开启了selinux。
解决方案:
关闭selinux即可。
2.error: Unsafe migration: Migration may lead to data corruption if disks use cache != none or cache != directsync
问题原因:
这是由于虚拟机磁盘不是从本地读取的,无法利用磁盘缓存(或者说无法预读磁盘文件)。
解决方案:
迁移时加上"--unsafe"参数即可。