targetcli内核态操作实战
工作中一旦遇到内核态的东西感觉操作非常困难，并且各种的权限的限制导致的操作非常困难，最终往往都是重启了事。比如进程出现了僵尸D状态，活着出现内核态数据残留，例如sysfs数据残留等。targetcli是操作内核态iSCSI管理的工具，这个工具会遇到数据校验错误，从而导致管理命令无法使用，导致数据残留无法操作，也无法通过简单的重启服务解决，最终只能重启主机完成状态重置。如果这时又增加了重启加载，可能重启也不会解决问题。
targetcli工具是通过configfs的内核模块管理target的内核结构的创建和删除，也可以调用rtslib_fb的python库来管理操作。
configfs：
configfs 是一个基于内存的文件系统，它提供了与sysfs相反的功能。sysfs 是一个基于文件系统的内核对象视图，而configfs 是一个基于文件系统的内核对象管理器（或称为config_items）。configfs 的 config_item 是通过用户空间显式操作 mkdir(2) 创建， rmdir(2) 销毁的。对象的属性在 mkdir(2) 时出现，并且可以通过 read(2) 和 write(2) 进行读取或修改。
targetcli由于是内核态的操作非常不方便，如果使用targetcli创建错误的target、lun、acls后，会出现targetcli不能操作的问题。
#创建[i-a]的acls
[root@target-t1 ~]# ll /sys/kernel/config/target/iscsi/iqn.1999-06.com.target.iscsi\:ppp/tpgt_1/acls/
total 0
drwxr-xr-x 6 root root 0 Nov 10 15:56 iqn.2019-10.com.target:client
drwxr-xr-x 6 root root 0 Nov 10 15:56 'iqn.2019-10.com.target:client[i-a]'
#targetcli操作失效
[root@target-t1 ~]# targetcli ls
bad character range i-a at position 41
#直接删除target，没有操作权限
[root@target-t1 acls]# rm -rf iqn.2019-10.com.target\:client\[i-a\]/
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/fabric_statistics/iscsi_sess_stats/conn_timeout_errors': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/fabric_statistics/iscsi_sess_stats/conn_digest_errors': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/fabric_statistics/iscsi_sess_stats/rxdata_octs': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/fabric_statistics/iscsi_sess_stats/txdata_octs': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/fabric_statistics/iscsi_sess_stats/rsp_pdus': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/fabric_statistics/iscsi_sess_stats/cmd_pdus': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/fabric_statistics/iscsi_sess_stats/indx': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/fabric_statistics/iscsi_sess_stats/node': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/fabric_statistics/iscsi_sess_stats/inst': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/param/ErrorRecoveryLevel': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/param/DataSequenceInOrder': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/param/DataPDUInOrder': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/param/MaxOutstandingR2T': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/param/DefaultTime2Retain': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/param/DefaultTime2Wait': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/param/FirstBurstLength': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/param/MaxBurstLength': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/param/ImmediateData': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/param/InitialR2T': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/param/MaxConnections': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/auth/password_mutual': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/auth/userid_mutual': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/auth/authenticate_target': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/auth/password': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/auth/userid': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/attrib/random_r2t_offsets': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/attrib/random_datain_seq_offsets': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/attrib/random_datain_pdu_offsets': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/attrib/nopin_response_timeout': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/attrib/nopin_timeout': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/attrib/default_erl': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/attrib/dataout_timeout_retries': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/attrib/dataout_timeout': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/tag': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/cmdsn_depth': Operation not permitted
rm: cannot remove 'iqn.2019-10.com.target:client[i-a]/info': Operation not permitted
shell操作target
#使用mkdir创建target
[root@target-t1 ]# mkdir /sys/kernel/config/target/iscsi/123
[root@target-t1 ]# ll /sys/kernel/config/target/iscsi/123
total 0
drwxr-xr-x 7 root root 0 Nov 10 16:02 fabric_statistics
[root@target-t1 ]# mkdir /sys/kernel/config/target/iscsi/123/tpgt_1
[root@target-t1 ]# ll /sys/kernel/config/target/iscsi/123/tpgt_1
total 0
drwxr-xr-x 2 root root 0 Nov 10 16:02 acls
drwxr-xr-x 2 root root 0 Nov 10 16:02 attrib
drwxr-xr-x 2 root root 0 Nov 10 16:02 auth
-r--r--r-- 1 root root 4096 Nov 10 16:02 dynamic_sessions
-rw-r--r-- 1 root root 4096 Nov 10 16:02 enable
drwxr-xr-x 2 root root 0 Nov 10 16:02 lun
drwxr-xr-x 2 root root 0 Nov 10 16:02 np
drwxr-xr-x 2 root root 0 Nov 10 16:02 param
[root@target-t1 ]# mkdir /sys/kernel/config/target/iscsi/123/tpgt_1/acls/456
[root@target-t1 ]# ll /sys/kernel/config/target/iscsi/123/tpgt_1/acls/456
total 0
drwxr-xr-x 2 root root 0 Nov 10 16:03 attrib
drwxr-xr-x 2 root root 0 Nov 10 16:03 auth
-rw-r--r-- 1 root root 4096 Nov 10 16:03 cmdsn_depth
drwxr-xr-x 3 root root 0 Nov 10 16:03 fabric_statistics
-r--r--r-- 1 root root 4096 Nov 10 16:03 info
drwxr-xr-x 2 root root 0 Nov 10 16:03 param
-rw-r--r-- 1 root root 4096 Nov 10 16:03 tag
#使用rmdir删除target，只能一层一层的删除，否则没有执行权限
[root@target-t1 ~]# rmdir /sys/kernel/config/target/iscsi/123/tpgt_1/acls/456
[root@target-t1 ~]# ll /sys/kernel/config/target/iscsi/123/tpgt_1/acls/
total 0
[root@target-t1 ~]# rmdir /sys/kernel/config/target/iscsi/123/tpgt_1
[root@target-t1 ~]# ll /sys/kernel/config/target/iscsi/123/
total 0
drwxr-xr-x 7 root root 0 Nov 10 16:02 fabric_statistics
[root@target-t1 ~]# rmdir /sys/kernel/config/target/iscsi/123
[root@target-t1 ~]# ll /sys/kernel/config/target/iscsi/123/
ls: cannot access '/sys/kernel/config/target/iscsi/123/': No such file or directory
[root@target-t1 ~]# ll /sys/kernel/config/target/iscsi/
total 0
drwxr-xr-x 2 root root 0 Nov 10 15:20 discovery_auth
drwxr-xr-x 4 root root 0 Nov 10 15:09 iqn.1999-06.com.target.iscsi:ppp
-r--r--r-- 1 root root 4096 Nov 10 16:06 lio_version
[root@target-t1 ~]#
python操作target
# rtslib库可以创建删除targetcli，后端的删除创建操作调用的接口为os.rmdir/os.mkdir
import rtslib.root as root
rtsroot = root.RTSRoot()
for tgt in rtsroot._list_targets():
import pdb;pdb.set_trace()
tgt.delete()
print(rtsroot._list_targets())
rtslib调用接口
# node.py 创建和删除接口
def _create_in_cfs_ine(self, mode):
'''
Creates the configFS node if it does not already exist, depending on
the mode.
any -> makes sure it exists, also works if the node already does exist
lookup -> make sure it does NOT exist
create -> create the node which must not exist beforehand
'''
if mode not in ['any', 'lookup', 'create']:
raise RTSLibError("Invalid mode: %s" % mode)
if self.exists and mode == 'create':
# ensure that self.path is not stale hba-only dir
if os.path.samefile(os.path.dirname(self.path), self.configfs_dir+'/core') \
and not next(os.walk(self.path))[1]:
os.rmdir(self.path)
else:
raise RTSLibError("This %s already exists in configFS"
% self.__class__.__name__)
elif not self.exists and mode == 'lookup':
raise RTSLibNotInCFS("No such %s in configfs: %s"
% (self.__class__.__name__, self.path))
if not self.exists:
try:
os.mkdir(self.path)
except:
raise RTSLibError("Could not create %s in configFS"
% self.__class__.__name__)
def delete(self):
'''
If the underlying configFS object does not exist, this method does
nothing. If the underlying configFS object exists, this method attempts
to delete it.
'''
if self.exists:
os.rmdir(self.path)
通过targetcli内核态操作分析，很多的内核系统调用其实需要一定的操作格式，并且和用户态操作操作不太一致，操作也bi'jia比较简单，都是系统中最基础的操作命令，可以通过脚本方式，也可以使用系统命令操作实现。有的需要通过加载内核模块，或者卸载内核模块实现特定目录的创建和删除。targetcli只要子目录中不存在资源，就可以通过命令手动清理残留，一层嵌套一层，直到残留清理完成。
