资料整理】下载和分析cdn节点汇总日志
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
|
#!/bin/bash
# filename: log_cdn.sh
#
# 2014/11/20
#
# mkdir -p /home/log/cdn/{log,stat}
action=$1
dt=$2
[ -z ${dt} ] && dt=$(
date
-d
"yesterday"
+
"%Y-%m-%d"
)
location=
"ftp.xxcdn.com"
ftpsvr=
"ftp://${location}"
d_base=
"/home/log/cdn"
d_stat=
"${d_base}/stat"
d_log=
"${d_base}/log"
f_logdetail=
"${d_base}/log/download.log"
f_logwget=
"${d_base}/log/wget.log"
usage() {
cat
<<_EOF
$0 [stat|clean]
date
_EOF
}
s_domains=
"www.xxx.com"
list_domain=(${s_domains})
lens_domain=${
#list_domain[@]}
log_download() {
local
s_URL=
"$1"
echo
"[+] START: `date +"
%Y-%m-%d %H:%M:%S
"`"
echo
"[-] ready to get: ${s_URL}"
wget -P ${d_base} --
ftp
-user=xxx --
ftp
-password=xxx -m -c -t5 ${s_URL} -a ${f_logwget} -nv
echo
"[-] END: `date +"
%Y-%m-%d %H:%M:%S
"`"
}
log_stat() {
for
((i=0;i<$lens_domain;i++))
do
local
s_URL=
"${ftpsvr}/${list_domain[$i]}/${dt}*"
log_download ${s_URL}
local
d_result=
"${d_stat}/${list_domain[$i]}/${dt}"
mkdir
-p ${d_result} &&
cd
${d_result}
local
f_pattern=
"${d_base}/${location}/${list_domain[$i]}/${dt}-*.cn.*"
ls
${f_pattern}
zcat ${f_pattern} |
awk
'{print $7}'
|
sort
|
uniq
-c |
sort
-nr >1_uri.log
done
}
log_clean() {
find
${d_base}/${location} -
type
-f -mtime +13 -delete
}
case
${action}
in
stat)
log_clean
log_stat ${dt} >${f_logdetail} 2>&1 &
;;
clean)
log_clean
;;
*)
usage
;;
esac
|
本文转自 pcnk 51CTO博客，原文链接：http://blog.51cto.com/nosmoking/1595612，如需转载请自行联系原作者
