一、描述：
以module的方式组件python代码，在磁盘文件清理上复用性更好
二、达到目标：
清空过期日志文件，清理掉超过自定大小日志文件
三、原码
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
|
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import
commands
import
os
import
time
import
re
import
getopt
import
sys
# commands.getstatusoutput 返回两个元素的元组tuple(status, result)，status为int类型，result为string类型
def
execute_local_shell_cmd(cmd):
status, result
=
commands.getstatusoutput(cmd)
result
=
result.split(
"\n"
)
return
status, result
def
send_alert_mail():
pass
'''
获取某一磁盘的空间使用率
'''
def
get_disk_used(disk_name):
status, result
=
execute_local_shell_cmd(
"df | grep %s | awk '{print $5}'"
%
disk_name)
return
status, result[
0
]
#print(get_disk_used('/data0'))
'''
判断文件是否在指定时间内修改过
'''
def
file_modify_in(file_path,time_interval
=
'1d'
):
current_time
=
time.time()
# os.path.getmtime 返回最后修改时间。返回从unix纪元开始的跳秒数
if
current_time
-
os.path.getmtime(file_path) < translate_time_interval_to_second(time_interval):
return
True
return
False
def
translate_file_size_to_kb(file_size):
# 将字符串所有大写字符转为小写
file_size
=
str
(file_size.lower())
# 创建匹配数字1次或多次的数字且小数点出现一次或者不出现的；小数点后数字重复0次或多次模式对象
pattern
=
re.
compile
(r
'\d+\.?\d*'
)
match
=
pattern.match(file_size)
file_size_number
=
None
if
match:
# 使用Match获得分组信息
#print(match.group())
file_size_number
=
float
(match.group())
else
:
raise
IOError(
"Input {0} can't translate to byte."
"Current support g(gb)/m(mb)/k(kb)/b(byte)"
.
format
(file_size))
# endswith() 方法用于判断字符串是否以指定后缀结尾，如果以指定后缀结尾返回True，否则返回False。
# 可选参数"start"与"end"为检索字符串的开始与结束位置。
if
file_size.endswith(
"g"
)
or
file_size.endswith(
"gb"
):
return
file_size_number
*
1024
*
1024
*
1024
elif
file_size.endswith(
"m"
)
or
file_size.endswith(
"mb"
):
return
file_size_number
*
1024
*
1024
elif
file_size.endswith(
"k"
)
or
file_size.endswith(
"kb"
):
return
file_size_number
*
1024
elif
file_size.endswith(
"b"
)
or
file_size.endswith(
"byte"
):
return
file_size_number
else
:
raise
IOError(
"Input {0} can't translate to byte."
"Current support g(gb)/m(mb)/k(kb)/b(byte)"
.
format
(file_size))
#print(translate_file_size_to_kb('10g'))
def
translate_time_interval_to_second(time_interval):
date_interval
=
str
(time_interval.lower())
pattern
=
re.
compile
(r
'\d+'
)
match
=
pattern.match(date_interval)
date_interval_number
=
None
if
match:
date_interval_number
=
int
(match.group())
else
:
raise
IOError(
"Input {0} can't translate to second."
"Current support d(day)/h(hour)/m(min)/s(sec)"
.
format
(date_interval))
if
date_interval.endswith(
'd'
)
or
date_interval.endswith(
'day'
):
return
date_interval_number
*
24
*
3600
elif
date_interval.endswith(
'h'
)
or
date_interval.endswith(
'hour'
):
return
date_interval_number
*
3600
elif
date_interval.endswith(
'm'
)
or
date_interval.endswith(
'min'
):
return
date_interval_number
*
60
elif
date_interval.endswith(
's'
)
or
date_interval.endswith(
'sec'
):
return
date_interval_number
else
:
raise
IOError(
"Input {0} cant't translate to second."
"Current support d(day)/h(hour)/m(min)/s(second)"
.
format
(date_interval))
#print(translate_time_interval_to_second('7d'))
'''
关断文件是否可能是当前log文件
1） 修改改时间1天内
2） 以pattern结尾
'''
def
probable_current_log_file(file_path,pattern
=
'log'
,modify_in
=
'1d'
):
if
file_modify_in(file_path,time_interval
=
modify_in):
return
True
return
str
(file_path).endswith(pattern)
'''
获取超过天数设置log，注意不会返回可能是当前正在修改的文件，查看probable_current_log_file
确定如何做该判断
'''
def
get_clean_log_list_by_date(target_dir,before_days_remove
=
'7d'
,pattern
=
"log"
):
before_seconds_remove
=
translate_time_interval_to_second(before_days_remove)
current_time
=
time.time()
# os.listdir 返回指定文件夹包含文件或文件夹的名字列表
for
candidate_file
in
os.listdir(target_dir):
candidate_file_fullpath
=
"%s/%s"
%
(target_dir,candidate_file)
# 是否存在一个普通文件
if
os.path.isfile(candidate_file_fullpath):
candidate_file_mtime
=
os.path.getmtime(candidate_file_fullpath)
# find\(\)根据是否包含字符串，如果包含有，返回开始的索引值，否则返回-1
if
current_time
-
candidate_file_mtime > before_seconds_remove \
and
candidate_file.find(pattern) !
=
-
1
\
and
not
probable_current_log_file(candidate_file_fullpath):
# yield 就是return一个值，并且记住这个返回值的位置，下次迭代就从这个位置后开始
yield
candidate_file_fullpath
'''
获取超过大小的日志文件（注意默认不会返回修改时间小于1天的文件）
'''
def
get_clean_log_list_by_size(target_dir,file_size_limit
=
'10g'
,pattern
=
"log"
):
file_size_limit_byte
=
translate_file_size_to_kb(file_size_limit)
for
candidate_file
in
os.listdir(target_dir):
candidate_file_fullpath
=
"%s/%s"
%
(target_dir,candidate_file)
if
os.path.isfile(candidate_file_fullpath):
# stat返回相关文件的系统状态信息
file_stat
=
os.stat(candidate_file_fullpath)
if
candidate_file.find(pattern) !
=
-
1
and
\
file_stat.st_size >
=
file_size_limit_byte:
yield
candidate_file_fullpath
# 如果文件在modify_in之内修改过，则不返回
# if not (modify_in and file_modify_in(candidate_file_fullpath, time_interval=modify_in)) and \
# not probable_current_log_file(candidate_file_fullpath):
# yield candidate_file_fullpath
'''
remove文件列表
'''
def
remove_file_list(file_list,pattern
=
'log'
,roll_back
=
False
):
for
file_item
in
file_list:
if
roll_back
or
probable_current_log_file(file_item,pattern
=
pattern,modify_in
=
'1d'
):
print
(
'roll back file %s'
%
file_item)
execute_local_shell_cmd(
"cat /dev/null > {0}"
.
format
(file_item))
else
:
print
(
'remove file %s'
%
file_item)
# os.remove 删除指定路径文件。如果指定的路径是一个目录，将抛出OSError
os.remove(file_item)
'''
清理掉超过日期的日志文件
'''
def
remove_files_by_date(target_dir,before_days_remove
=
'7d'
,pattern
=
'log'
):
file_list
=
get_clean_log_list_by_date(target_dir,before_days_remove,pattern)
remove_file_list(file_list)
'''
清理掉超过大小的日志文件
'''
def
remove_files_by_size(target_dir,file_size_limit
=
'10g'
,pattern
=
'log'
):
file_list
=
get_clean_log_list_by_size(target_dir,file_size_limit,pattern)
remove_file_list(file_list)
'''
清空当前的日志文件，使用cat /dev/null > {log_file}方式
'''
def
clean_curren_log_file(target_dir,file_size_limit
=
'10g'
,pattern
=
'log'
):
for
candidate_file
in
os.listdir(target_dir):
candidate_file_fullpath
=
'%s/%s'
%
(target_dir,candidate_file)
if
candidate_file.endswith(pattern)
and
os.path.isfile(candidate_file_fullpath):
file_stat
=
os.stat(candidate_file_fullpath)
if
file_stat.st_size >
=
translate_file_size_to_kb(file_size_limit):
remove_file_list([candidate_file_fullpath],roll_back
=
True
)
def
clean_data_release_disk(disk_name, target_dir, disk_used_limit
=
'80%'
, before_days_remove
=
'7d'
,
file_size_limit
=
'10g'
, pattern
=
'log'
):
disk_used_limit
=
disk_used_limit.replace(
'%'
, '')
# 第一步执行按时间的日志清理
print
(
'Step one remove files {0} ago.'
.
format
(before_days_remove))
remove_files_by_date(target_dir, before_days_remove
=
before_days_remove, pattern
=
pattern)
# 如果磁盘空间还是没有充分释放,则执行按大小的日志清理
current_disk_used
=
int
(get_disk_used(disk_name)[
1
].replace(
'%'
, ''))
if
current_disk_used >
int
(disk_used_limit):
print
(
"Disk {0}'s current used {1}% great than input used limit {2}%,"
"so we will remove files bigger than {3}"
.
format
(disk_name, current_disk_used, disk_used_limit, file_size_limit))
remove_files_by_size(target_dir, file_size_limit
=
file_size_limit, pattern
=
pattern)
# 如果磁盘空间开没有释放,清空当前正在写的log文件,并alert
current_disk_used
=
int
(get_disk_used(disk_name)[
1
].replace(
'%'
, ''))
if
current_disk_used >
int
(disk_used_limit):
print
(
"Disk {0}'s current used {1}% great than input used limit {2}%,"
"so we will roll back current log file"
.
format
(disk_name, current_disk_used, disk_used_limit, file_size_limit))
clean_curren_log_file(target_dir, file_size_limit
=
file_size_limit, pattern
=
pattern)
# 如果还是没有,alert mail
if
int
(get_disk_used(disk_name)[
1
].replace(
'%'
, '')) >
int
(disk_used_limit):
send_alert_mail()
def
usage():
print
(
'clean.py -d <target_disk> -r <target_dirctory -u <diskUsedLimit(default 80%)> '
'-f <fileSizeLimit(default 10gb,gb/mb/kb)> -p <filePattern(default log)> '
'-t <beforeDaysRemove(default 7d,d)> '
)
if
__name__
=
=
"__main__"
:
target_disk_input
=
'/data0'
target_dir_input
=
'/data0/hadoop2/logs'
disk_used_limit_input
=
'80%'
file_size_limit_input
=
'10g'
pattern_input
=
'log'
before_days_remove_input
=
'7d'
try
:
# getopt 命令解析，有短选项和长选项
# getopt 返回两人个参数：一个对应参数选项和value元组，另一个一般为空
opts,args
=
getopt.getopt(sys.argv[
1
:],
'hd:r:u:f:p:t:'
, [
'help'
'disk='
,
'directory='
,
'diskUsedLimit='
,
'fileSizeLimit='
,
'filePattern='
,
'beforeDaysRemove='
])
# getopt模块函数异常错误，捕获异常并打印错误
except
getopt.GetoptError as err:
print
err
usage()
sys.exit(
2
)
if
len
(opts) <
6
:
usage()
sys.exit(
2
)
for
opt,arg
in
opts:
if
opt
=
=
'-h'
:
usage()
sys.exit()
elif
opt
in
(
"-d"
,
"--disk"
):
target_disk_input
=
arg.replace(
'/'
,'')
elif
opt
in
(
"-r"
,
"--directory"
):
target_dir_input
=
arg
elif
opt
in
(
"-u"
,
"--diskUsedLimit"
):
disk_used_limit_input
=
arg
elif
opt
in
(
"-f"
,
"--fileSizeLimit"
):
file_size_limit_input
=
arg
translate_file_size_to_kb(file_size_limit_input)
elif
opt
in
(
"-p"
,
"filePattern"
):
pattern_input
=
arg
elif
opt
in
(
"-t"
,
"--beforeDaysRemove"
):
before_days_remove_input
=
arg
translate_time_interval_to_second(before_days_remove_input)
print
(
"{0} Start clean job.target_disk:{1},target_directory:{2},disk_used_limit:{3},"
"file_size_limit:{4},pattern:{5},before_days_remove:{6}"
.
format
(time.ctime(time.time()),
target_disk_input, target_dir_input,
disk_used_limit_input, file_size_limit_input,
pattern_input, before_days_remove_input))
clean_data_release_disk(target_disk_input, target_dir_input,
disk_used_limit
=
disk_used_limit_input, file_size_limit
=
file_size_limit_input,
pattern
=
pattern_input, before_days_remove
=
before_days_remove_input)
|
四、统一调用目录定时删除
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
|
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import
os
# 遍历目录
def
Lisdir(targetdir):
list_dirs
=
os.walk(targetdir)
for
root,list_dirs,files
in
list_dirs:
for
d
in
list_dirs:
yield
os.path.join(root,d)
def
log_dir(targetdir):
list_dirs
=
os.listdir(targetdir)
for
ph
in
list_dirs:
if
os.path.isdir(os.path.join(targetdir,ph)):
yield
Lisdir(os.path.join(targetdir,ph))
for
path
in
log_dir(
'/data0/backup_log-bin'
):
for
ppp
in
path:
# 以log-bin结尾 为假
if
ppp.endswith(
'log-bin'
)
is
False
:
os.system(
"db_script/clean_robo.py -d /data0 -r {0} -u 75% -f 501M -p bin -t 5d"
.
format
(ppp))
|
本文转自 zouqingyun 51CTO博客，原文链接：http://blog.51cto.com/zouqingyun/1897057，如需转载请自行联系原作者
