一.cephFS客户端手动挂载实战案例
1.客户端移除认证文件
[root@ceph144 ~]# rm -f /etc/ceph/ceph.client.admin.keyring
[root@ceph144 ~]#
[root@ceph144 ~]# ll /etc/ceph/
total 8
-rw-r--r-- 1 root root 353 Feb 2 11:13 ceph.conf
-rw-r--r-- 1 root root 92 Jun 30 2021 rbdmap
-rw------- 1 root root 0 Feb 1 16:50 tmpEYwKWU
[root@ceph144 ~]#
[root@ceph144 ~]# ceph -s
2024-02-02 15:01:34.969 7fd3b4f99700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory
2024-02-02 15:01:34.969 7fd3b4f99700 -1 AuthRegistry(0x7fd3b00662b8) no keyring found at /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx
2024-02-02 15:01:34.993 7fd3b4f99700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory
2024-02-02 15:01:34.993 7fd3b4f99700 -1 AuthRegistry(0x7fd3b00c7e18) no keyring found at /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx
2024-02-02 15:01:34.994 7fd3b4f99700 -1 auth: unable to find a keyring on /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,: (2) No such file or directory
2024-02-02 15:01:34.994 7fd3b4f99700 -1 AuthRegistry(0x7fd3b4f97e78) no keyring found at /etc/ceph/ceph.client.admin.keyring,/etc/ceph/ceph.keyring,/etc/ceph/keyring,/etc/ceph/keyring.bin,, disabling cephx
[errno 2] error connecting to the cluster
[root@ceph144 ~]#
2.管理节点创建用户并导出钥匙环和key文件
[root@ceph141 ~]# ceph auth add client.yinzhengjiefs mon 'allow r' mds 'allow rw' osd 'allow rwx'
added key for client.yinzhengjiefs
[root@ceph141 ~]#
[root@ceph141 ~]# ceph auth get client.yinzhengjiefs
[client.yinzhengjiefs]
key = AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
caps mds = "allow rw"
caps mon = "allow r"
caps osd = "allow rwx"
exported keyring for client.yinzhengjiefs
[root@ceph141 ~]#
[root@ceph141 ~]# ceph auth get client.yinzhengjiefs > ceph.client.yinzhengjiefs.keyring
exported keyring for client.yinzhengjiefs
[root@ceph141 ~]#
[root@ceph141 ~]# cat ceph.client.yinzhengjiefs.keyring
[client.yinzhengjiefs]
key = AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
caps mds = "allow rw"
caps mon = "allow r"
caps osd = "allow rwx"
[root@ceph141 ~]#
[root@ceph141 ~]# ceph auth print-key client.yinzhengjiefs > yinzhengjiefs.key
[root@ceph141 ~]#
[root@ceph141 ~]# more yinzhengjiefs.key
AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph141 ~]#
3.将钥匙环和秘钥key拷贝到客户端指定目录
[root@ceph141 ~]# scp ceph.client.yinzhengjiefs.keyring yinzhengjiefs.key ceph144:/etc/ceph/
root@ceph144's password:
ceph.client.yinzhengjiefs.keyring 100% 138 133.4KB/s 00:00
yinzhengjiefs.key 100% 40 3.6KB/s 00:00
[root@ceph141 ~]#
4.客户端两种挂载cephFS
1 配置ceph集群主机解析
[root@ceph144 ~]# cat >> /etc/hosts <<EOF
10.0.0.141 ceph141
10.0.0.142 ceph142
10.0.0.143 ceph143
10.0.0.144 ceph144
EOF
2 方式一： 基于key文件进行挂载
[root@ceph144 ~]# df -h | grep mnt
[root@ceph144 ~]#
[root@ceph144 ~]# mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secretfile=/etc/ceph/yinzhengjiefs.key
[root@ceph144 ~]#
[root@ceph144 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/ 630G 0 630G 0% /mnt
[root@ceph144 ~]#
[root@ceph144 ~]# ll /mnt/
total 0
[root@ceph144 ~]#
[root@ceph144 ~]# cp /etc/os-release /mnt/
[root@ceph144 ~]#
[root@ceph144 ~]# ll /mnt/
total 1
-rw-r--r-- 1 root root 393 Feb 2 15:10 os-release
[root@ceph144 ~]#
[root@ceph144 ~]# umount /mnt/
[root@ceph144 ~]#
[root@ceph144 ~]# df -h | grep mnt
[root@ceph144 ~]#
3 方式二: 基于KEY进行挂载，无需拷贝秘钥文件！
[root@ceph144 ~]# more /etc/ceph/yinzhengjiefs.key
AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph144 ~]#
[root@ceph144 ~]# mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph144 ~]#
[root@ceph144 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/ 630G 0 630G 0% /mnt
[root@ceph144 ~]#
[root@ceph144 ~]# ll /mnt/
total 1
-rw-r--r-- 1 root root 393 Feb 2 15:10 os-release
[root@ceph144 ~]#
[root@ceph144 ~]# umount /mnt
[root@ceph144 ~]#
[root@ceph144 ~]# df -h | grep mnt
[root@ceph144 ~]#
5.验证yinzhengjiefs普通用户
[root@ceph144 ~]# ll /etc/ceph/ceph.client.yinzhengjiefs.keyring
-rw-r--r-- 1 root root 138 Feb 2 15:07 /etc/ceph/ceph.client.yinzhengjiefs.keyring
[root@ceph144 ~]#
[root@ceph144 ~]# cat /etc/ceph/ceph.client.yinzhengjiefs.keyring
[client.yinzhengjiefs]
key = AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
caps mds = "allow rw"
caps mon = "allow r"
caps osd = "allow rwx"
[root@ceph144 ~]#
[root@ceph144 ~]# ceph -s --user yinzhengjiefs
cluster:
id: 5821e29c-326d-434d-a5b6-c492527eeaad
health: HEALTH_OK
services:
mon: 3 daemons, quorum ceph141,ceph142,ceph143 (age 4h)
mgr: ceph142(active, since 4h), standbys: ceph141, ceph143
mds: yinzhengjie-linux-cephfs:2 {0=ceph143=up:active,1=ceph141=up:active} 1 up:standby
osd: 7 osds: 7 up (since 4h), 7 in (since 22h)
data:
pools: 5 pools, 256 pgs
objects: 102 objects, 100 MiB
usage: 7.8 GiB used, 1.9 TiB / 2.0 TiB avail
pgs: 256 active+clean
[root@ceph144 ~]#
6.后续使用直接用就可以了，和nfs几乎一样，只不过是高可用的NFS
[root@ceph143 ~]# df -h | grep mnt
[root@ceph143 ~]#
[root@ceph143 ~]# mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph143 ~]#
[root@ceph143 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/ 630G 0 630G 0% /mnt
[root@ceph143 ~]#
[root@ceph143 ~]# ll /mnt/
total 1
-rw-r--r-- 1 root root 393 Feb 2 15:10 os-release
[root@ceph143 ~]#
[root@ceph143 ~]# cp /etc/sysconfig/network-scripts/ifcfg-ens33 /mnt/
[root@ceph143 ~]#
[root@ceph143 ~]# ll /mnt/
total 1
-rw-r--r-- 1 root root 147 Feb 2 15:17 ifcfg-ens33
-rw-r--r-- 1 root root 393 Feb 2 15:10 os-release
[root@ceph143 ~]#
7.在另一个客户端挂载测试，值得注意是，基于key进行挂载，是不需要拷贝秘钥文件的
[root@ceph142 ~]# df -h | grep mnt
[root@ceph142 ~]#
[root@ceph142 ~]# mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph142 ~]#
[root@ceph142 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/ 630G 0 630G 0% /mnt
[root@ceph142 ~]#
[root@ceph142 ~]# ll /mnt/
total 1
-rw-r--r-- 1 root root 147 Feb 2 15:17 ifcfg-ens33
-rw-r--r-- 1 root root 393 Feb 2 15:10 os-release
[root@ceph142 ~]#
二.cephFS开机自动挂载的三种方式
1.方式一: 通过"/etc/rc.d/rc.local "文件实现开机自动挂载，（推荐： 就算执行失败，不影响开机）
[root@ceph144 ~]# vim /etc/rc.d/rc.local
...
mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph144 ~]#
[root@ceph144 ~]# ll /etc/rc.d/rc.local
-rw-r--r-- 1 root root 602 Feb 2 15:19 /etc/rc.d/rc.local
[root@ceph144 ~]#
[root@ceph144 ~]# chmod +x /etc/rc.d/rc.local
[root@ceph144 ~]#
[root@ceph144 ~]# ll /etc/rc.d/rc.local
-rwxr-xr-x 1 root root 602 Feb 2 15:19 /etc/rc.d/rc.local
[root@ceph144 ~]#
[root@ceph144 ~]# df -h | grep mnt
[root@ceph144 ~]#
[root@ceph144 ~]# reboot
[root@ceph144 ~]#
[root@ceph144 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/ 630G 0 630G 0% /mnt
[root@ceph144 ~]#
2.方式二: 使用"/etc/fstab"进行开机挂载
[root@ceph144 ~]# grep yinzhengjie /etc/rc.d/rc.local
mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph144 ~]#
[root@ceph144 ~]# sed -i '/mount/s@^@#@' /etc/rc.d/rc.local
[root@ceph144 ~]#
[root@ceph144 ~]# grep yinzhengjie /etc/rc.d/rc.local
#mount -t ceph ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt -o name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph144 ~]#
[root@ceph144 ~]# reboot
[root@ceph144 ~]#
[root@ceph144 ~]# df -h | grep mnt # 发现并不会开机自动挂载，说明上面的配置生效了。
[root@ceph144 ~]#
[root@ceph144 ~]# vim /etc/fstab
[root@ceph144 ~]#
[root@ceph144 ~]# grep yinzhengjie /etc/fstab
ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt/ ceph name=yinzhengjiefs,secretfile=/etc/ceph/yinzhengjiefs.key,noatime,_netdev 0 2
[root@ceph144 ~]#
[root@ceph144 ~]# reboot
[root@ceph144 ~]#
[root@ceph144 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/ 630G 0 630G 0% /mnt
[root@ceph144 ~]#
温馨提示:
不推荐使用，一定要谨慎修改，因为一旦修改失败，会导致服务器启动失败，在开机挂载时阻塞住，需要进单用户模式救援。
3.方式三: 使用"/etc/fstab"进行开机挂载
[root@ceph144 ~]# grep yinzhengjie /etc/fstab
ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt/ ceph name=yinzhengjiefs,secretfile=/etc/ceph/yinzhengjiefs.key,noatime,_netdev 0 2
[root@ceph144 ~]#
[root@ceph144 ~]# sed -i '/ceph/s@^@#@' /etc/fstab
[root@ceph144 ~]#
[root@ceph144 ~]# grep yinzhengjie /etc/fstab
#ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt/ ceph name=yinzhengjiefs,secretfile=/etc/ceph/yinzhengjiefs.key,noatime,_netdev 0 2
[root@ceph144 ~]#
[root@ceph144 ~]# vim /etc/fstab
[root@ceph144 ~]#
[root@ceph144 ~]# more /etc/ceph/yinzhengjiefs.key
AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==
[root@ceph144 ~]#
[root@ceph144 ~]#
[root@ceph144 ~]# grep yinzhengjie /etc/fstab
#ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt/ ceph name=yinzhengjiefs,secretfile=/etc/ceph/yinzhengjiefs.key,noatime,_netdev 0 2
ceph141:6789,ceph142:6789,ceph143:6789:/ /mnt/ ceph name=yinzhengjiefs,secret=AQCqk7xl25FuIBAAgjIW2EGodfyFRLuifNoYSA==,noatime,_netdev 0 2
[root@ceph144 ~]#
[root@ceph144 ~]# reboot
[root@ceph144 ~]#
[root@ceph144 ~]# df -h | grep mnt
10.0.0.141:6789,10.0.0.142:6789,10.0.0.143:6789:/ 630G 0 630G 0% /mnt
[root@ceph144 ~]#
[root@ceph144 ~]# ll /mnt/
total 1
-rw-r--r-- 1 root root 147 Feb 2 15:17 ifcfg-ens33
-rw-r--r-- 1 root root 393 Feb 2 15:10 os-release
[root@ceph144 ~]#
