上次使用sftp.get同步远程目录，现在需要用python合并日志
别人很早就写出来了
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
收集Tomcat异常日志并发送邮件
http:
/
/
linux5588.blog.
51cto
.com
/
65280
/
1208951
#!/usr/bin/env python
# coding=utf-8
#---------------------------------------------------------
# Name: Tomcat错误日志发送邮件脚本
# Purpose: 收集Tomcat异常日志并发送邮件
# Version: 1.0
# Author: LEO
# BLOG: http://linux5588.blog.51cto.com
# EMAIL: chanyipiaomiao@163.com
# Created: 2013-05-22
# Copyright: (c) LEO 2013
# Python： 2.7/2.4 皆可使用
#--------------------------------------------------------
from
smtplib
import
SMTP
from
email
import
MIMEText
from
email
import
Header
from
os.path
import
getsize
from
sys
import
exit
from
re
import
compile
, IGNORECASE
#定义主机 帐号 密码 收件人 邮件主题
smtpserver
=
'smtp.163.com'
sender
=
'帐号/发件人'
password
=
'密码'
receiver
=
(
'收件人1'
,
'收件人2'
,)
subject
=
u
'Web服务器Tomcat日志错误信息'
From
=
u
'xxx Web服务器'
To
=
u
'服务器管理员'
#定义tomcat日志文件位置
tomcat_log
=
'/home/back/catalina.out'
#该文件是用于记录上次读取日志文件的位置,执行脚本的用户要有创建该文件的权限
last_position_logfile
=
'/home/back/last_position.txt'
error_list_logfile
=
'/home/back/error_list.txt'
#匹配的错误信息关键字的正则表达式
pattern
=
compile
(r
'Exception|^\t+\bat\b'
,IGNORECASE)
#发送邮件函数
def
send_mail(error):
#定义邮件的头部信息
header
=
Header.Header
msg
=
MIMEText.MIMEText(error,
'plain'
,
'utf-8'
)
msg[
'From'
]
=
header(From)
msg[
'To'
]
=
header(To)
msg[
'Subject'
]
=
header(subject
+
'\n'
)
#连接SMTP服务器，然后发送信息
smtp
=
SMTP(smtpserver)
smtp.login(sender, password)
smtp.sendmail(sender, receiver, msg.as_string())
smtp.close()
#读取上一次日志文件的读取位置
def
get_last_position(
file
):
try
:
data
=
open
(
file
,
'r'
)
last_position
=
data.readline()
if
last_position:
last_position
=
int
(last_position)
else
:
last_position
=
0
except
:
last_position
=
0
return
last_position
#写入本次日志文件的本次位置
def
write_this_position(
file
,last_positon):
try
:
data
=
open
(
file
,
'w'
)
data.write(
str
(last_positon))
data.write(
'\n'
+
"Don't Delete This File,It is Very important for Looking Tomcat Error Log !! \n"
)
data.close()
except
:
print
"Can't Create File !"
+
file
exit()
#分析文件找出异常的行
def
analysis_log(
file
):
error_list
=
[]
try
:
data
=
open
(
file
,
'r'
)
except
:
exit()
last_position
=
get_last_position(last_position_logfile)
this_postion
=
getsize(tomcat_log)
if
this_postion < last_position:
data.seek(
0
)
elif
this_postion
=
=
last_position:
exit()
elif
this_postion > last_position:
data.seek(last_position)
for
line
in
data:
if
pattern.search(line):
error_list.append(line)
write_this_position(last_position_logfile,data.tell())
data.close()
return
''.join(error_list)
def
writeToTxt(list_name,file_path):
try
:
fp
=
open
(file_path,
"w+"
)
for
item
in
list_name:
fp.write(
str
(item)
+
"\n"
)
fp.close()
except
IOError:
print
(
"fail to open file"
)
#调用发送邮件函数发送邮件
error_info
=
analysis_log(tomcat_log)
writeToTxt(erro_info,error_list_logfile)
|
自己模仿写的
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
#!/usr/bin/env python
# coding=utf-8
from
smtplib
import
SMTP
from
email
import
MIMEText
from
email
import
Header
from
os.path
import
getsize
from
sys
import
exit
from
re
import
compile
, IGNORECASE
import
os
#定义主机 帐号 密码 收件人 邮件主题
smtpserver
=
'smtp.163.com'
sender
=
'帐号/发件人'
password
=
'密码'
receiver
=
(
'收件人1'
,
'收件人2'
,)
subject
=
u
'Web服务器Tomcat日志错误信息'
From
=
u
'xxx Web服务器'
To
=
u
'服务器管理员'
#定义tomcat日志文件位置
tomcat_log
=
'/home/python/12/'
#该文件是用于记录上次读取日志文件的位置,执行脚本的用户要有创建该文件的权限
last_position_logfile
=
'/home/python/last_position.txt'
error_list_logfile
=
'/home/python/error_list.txt'
#匹配的错误信息关键字的正则表达式
pattern
=
compile
(r
'Exception|^\t+\bat\b'
,IGNORECASE)
#发送邮件函数
def
send_mail(error):
#定义邮件的头部信息
header
=
Header.Header
msg
=
MIMEText.MIMEText(error,
'plain'
,
'utf-8'
)
msg[
'From'
]
=
header(From)
msg[
'To'
]
=
header(To)
msg[
'Subject'
]
=
header(subject
+
'\n'
)
#连接SMTP服务器，然后发送信息
smtp
=
SMTP(smtpserver)
smtp.login(sender, password)
smtp.sendmail(sender, receiver, msg.as_string())
smtp.close()
#读取上一次日志文件的读取位置
def
get_last_position(
file
):
try
:
data
=
open
(
file
,
'r'
)
last_position
=
data.readline()
if
last_position:
last_position
=
int
(last_position)
else
:
last_position
=
0
except
:
last_position
=
0
return
last_position
#写入本次日志文件的本次位置
def
write_this_position(
file
,last_positon):
try
:
data
=
open
(
file
,
'a'
)
data.write(
str
(last_positon)
+
'\n'
)
#data.write('\n' + "Don't Delete This File,It is Very important for Looking Tomcat Error Log !! \n")
data.close()
except
:
print
"Can't Create File !"
+
file
exit()
#分析文件找出异常的行
def
analysis_log(
file
):
error_list
=
[]
try
:
data
=
open
(
file
,
'r'
)
except
:
exit()
#last_position = get_last_position(last_position_logfile)
last_position
=
0
this_postion
=
getsize(
file
)
if
this_postion < last_position:
data.seek(
0
)
elif
this_postion
=
=
last_position:
exit()
elif
this_postion > last_position:
data.seek(last_position)
for
line
in
data:
if
pattern.search(line):
error_list.append(line)
write_this_position(last_position_logfile,data.tell())
data.close()
return
''.join(error_list)
def
writeToTxt(list_name,file_path):
try
:
fp
=
open
(file_path,
"w+"
)
for
item
in
list_name:
fp.write(
str
(item)
+
"\n"
)
fp.close()
except
IOError:
print
(
"fail to open file"
)
def
walk(local_dir):
for
root, dirs, files
in
os.walk(local_dir):
for
filespath
in
files:
#return filespath
local_file
=
os.path.join(root,filespath)
#error_info2 = analysis_log('/home/python/12/xx.log')
#print error_info2
error_info
=
analysis_log(local_file)
fp
=
open
(error_list_logfile,
"a+"
)
fp.write(error_info)
if
__name__
=
=
'__main__'
:
walk(tomcat_log)
#调用发送邮件函数发送邮件
#error_info = analysis_log(tomcat_log)
#writeToTxt(error_info,error_list_logfile)
#fp = open(error_list_logfile,"w+")
#fp.write(str(error_info))
|
第二版
生成zabbix监控，只是监控异常数量,没有过滤exception
-
先执行第一个脚本，生成数据
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
|
# !/usr/bin/env python
#-*-coding:utf-8-*-
import
os
import
sys
import
time
import
datetime
import
string
from
smtplib
import
SMTP
from
email
import
MIMEText
from
email
import
Header
from
os.path
import
getsize
from
sys
import
exit
from
re
import
compile
, IGNORECASE
tomcat_log
=
'/home/logs/'
last_position_logfile
=
'/home/back/test/last_position.txt'
error_list_logfile
=
'/home/back/test/error_list.txt'
pattern
=
compile
(r
'Exception|^\t+\bat\b'
, IGNORECASE)
fp
=
open
(error_list_logfile,
"w+"
)
def
writeToTxt(list_name, file_path):
try
:
fp
=
open
(file_path,
"a+"
)
for
item
in
list_name:
fp.write(
str
(item)
+
"\n"
)
fp.close()
except
IOError:
print
(
"fail to open file"
)
# 分析文件找出异常的行
def
analysis_log(
file
):
#parent_file = os.path.dirname(file)
error_list
=
[]
try
:
data
=
open
(
file
,
'r'
)
except
:
exit()
for
line
in
data:
if
pattern.search(line):
error_list.append(
file
)
break
data.close()
if
len
(error_list)
=
=
1
:
#return ''.join(error_list)
writeToTxt(error_list,error_list_logfile)
#return error_list
#if errors_list == 1:
#print "%s num is %d" %(file,errors_list)
#print ''.join(errors_list)
def
walk(local_dir):
for
root, dirs, files
in
os.walk(local_dir):
for
filespath
in
files:
# return filespath
local_file
=
os.path.join(root, filespath)
# print error_info2
error_info
=
analysis_log(local_file)
def
walk_dir(home_dir):
homedir
=
[]
for
root, dirs, files
in
os.walk(home_dir):
for
dirname
in
dirs:
local_dir
=
os.path.join(root, dirname)
now
=
datetime.datetime.now()
strdatetime
=
now.strftime(
"%Y/%m/%d"
)
Date
=
now.strftime(
"%d"
)
#print "Date is %s" %Date
result
=
string.find(local_dir,strdatetime)!
=
-
1
if
result:
#print local_dir.count('/')
if
local_dir.count(
'/'
)
=
=
8
:
homedir.append(local_dir)
return
homedir
if
__name__
=
=
'__main__'
:
tomcat_last_dir
=
walk_dir(tomcat_log)
for
i
in
tomcat_last_dir:
walk(i)
|
2.zabbix脚本两个
python以后都可以这样写
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
|
# !/usr/bin/env python
#-*-coding:utf-8-*-
import
json
import
os
import
sys
import
time
import
datetime
import
string
from
smtplib
import
SMTP
from
email
import
MIMEText
from
email
import
Header
from
os.path
import
getsize
from
sys
import
exit
file
=
'/home/back/test/error_list.txt'
error_list
=
[]
service
=
[]
try
:
data
=
open
(
file
,
'r'
)
except
:
exit()
for
line
in
data:
error_list.append(line.split(
'/'
)[
8
])
#service.append(line.split('/')[3])
news_error_list
=
list
(
set
(error_list))
#news_service = list(set(service))
#print news_error_list
#print news_service
javas
=
[]
for
x
in
news_error_list:
javas.append({
'{#EXCEPTION}'
:x.strip(
'\n'
)})
print
json.dumps({
'data'
:javas},indent
=
4
,separators
=
(
','
,
':'
))
|
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
|
# !/usr/bin/env python
#-*-coding:utf-8-*-
import
json
import
os
import
sys
import
time
import
datetime
import
string
from
smtplib
import
SMTP
from
email
import
MIMEText
from
email
import
Header
from
os.path
import
getsize
from
sys
import
exit
file
=
'/home/back/test/error_list.txt'
error_list
=
[]
try
:
data
=
open
(
file
,
'r'
)
except
:
exit()
for
line
in
data:
error_list.append(line.split(
'/'
)[
8
])
news_error_list
=
list
(
set
(error_list))
if
len
(sys.argv)
=
=
2
:
argv1
=
sys.argv[
1
]
print
error_list.count(argv1)
else
:
print
-
1
|
标记一些常用写法
result = string.find(local_dir,strdatetime)!=-1
news_error_list = list(set(error_list))
print error_list.count(argv1)
python将数组写入文件
fp = open(error_list_logfile,"w+")
fp.write(str(error_info))
这些脚本，主要的作用就是针对每个文件，过滤带有exception的文件并在grafana显示。
本文转自 liqius 51CTO博客，原文链接：http://blog.51cto.com/szgb17/1861459，如需转载请自行联系原作者
