|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
|
import MySQLdb
import MySQLdb.cursors
class
MysqlPipeline(object):
#采用同步的机制写入mysql
def __init__(self):
self.conn = MySQLdb.connect(
'192.168.0.106'
,
'root'
,
'root'
,
'article_spider'
, charset=
"utf8"
, use_unicode=True)
self.cursor = self.conn.cursor()
def process_item(self, item, spider):
insert_sql =
""
"
insert into jobbole_article(title, url, create_date, fav_nums)
VALUES (%s, %s, %s, %s)
""
"
self.cursor.execute(insert_sql, (item[
"title"
], item[
"url"
], item[
"create_date"
], item[
"fav_nums"
]))
self.conn.commit()
class
MysqlTwistedPipline(object):
def __init__(self, dbpool):
self.dbpool = dbpool
@classmethod
def from_settings(cls, settings):
dbparms = dict(
host = settings[
"MYSQL_HOST"
],
db = settings[
"MYSQL_DBNAME"
],
user = settings[
"MYSQL_USER"
],
passwd = settings[
"MYSQL_PASSWORD"
],
charset=
'utf8'
,
cursorclass=MySQLdb.cursors.DictCursor,
use_unicode=True,
)
dbpool = adbapi.ConnectionPool(
"MySQLdb"
, **dbparms)
return
cls(dbpool)
def process_item(self, item, spider):
#使用twisted将mysql插入变成异步执行
query = self.dbpool.runInteraction(self.do_insert, item)
query.addErrback(self.handle_error, item, spider) #处理异常
def handle_error(self, failure, item, spider):
# 处理异步插入的异常
print (failure)
def do_insert(self, cursor, item):
#执行具体的插入
#根据不同的item 构建不同的sql语句并插入到mysql中
insert_sql, params = item.get_insert_sql()
print (insert_sql, params)
cursor.execute(insert_sql, params)
|
本文转自ning1022 51CTO博客，原文链接：http://blog.51cto.com/ning1022/1925593
，如需转载请自行联系原作者
