python数据抓取分析（python + mongodb）

 1 def step():
 2     try:
 3         headers = {
 4            。。。。。
 5             }
 6         r = requests.get(url,headers,timeout=30)
 7         html = r.content
 8         soup = BeautifulSoup(html,"lxml")
 9         url = soup.find_all(正则表达式)
10         for i in url:
11             url2 =  i.find_all('a')
12             for j in url2:
13                  step1url =url + j['href']
14                  print step1url
15                  step2(step1url)
16     except Exception,e:
17         print e

 1 def step2(step1url):
 2     try:
 3         headers = {
 4            。。。。
 5             }
 6         r = requests.get(step1url,headers,timeout=30)
 7         html = r.content
 8         soup = BeautifulSoup(html,"lxml")
 9         a = soup.find('div',id='divTbl')
10         if a:
11             url = soup.find_all('td',class_='S-ITabs')
12             for i in url:
13                 classifyurl =  i.find_all('a')
14                 for j in classifyurl:
15                      step2url = url + j['href']
16                      #print step2url
17                      step3(step2url)
18         else:
19             postdata(step1url)

 1 def producturl(url):
 2     try:
 3         p1url = doc.xpath(正则表达式)
 4         for i in xrange(1,len(p1url) + 1):
 5             p2url = doc.xpath(正则表达式)
 6             if len(p2url) > 0:
 7                 producturl = url + p2url[0].get('href')
 8                 count = db[table].find({'url':producturl}).count()
 9                 if count <= 0:
10                         sn = getNewsn()
11                         db[table].insert({"sn":sn,"url":producturl})
12                         print str(sn) + 'inserted successfully'
13                 else:
14                         'url exist'
15 
16     except Exception,e:
17         print e

 1 def parser(sn,url):
 2     try:
 3         headers = {
 4             。。。。。。
 5             }
 6         r = requests.get(url, headers=headers,timeout=30)
 7         html = r.content
 8         soup = BeautifulSoup(html,"lxml")
 9         dt = {}
10         #partno
11         a = soup.find("meta",itemprop="mpn")
12         if a:
13             dt['partno'] = a['content']
14         #manufacturer
15         b = soup.find("meta",itemprop="manufacturer")
16         if b:
17             dt['manufacturer'] = b['content']
18         #description
19         c = soup.find("span",itemprop="description")
20         if c:
21             dt['description'] = c.get_text().strip()
22         #price
23         price = soup.find("table",class_="table table-condensed occalc_pa_table")
24         if price:
25             cost = {}
26             for i in price.find_all('tr'):
27                 if len(i) > 1:
28                     td = i.find_all('td')
29                     key=td[0].get_text().strip().replace(',','')
30                     val=td[1].get_text().replace(u'\u20ac','').strip()
31                     if key and val:
32                         cost[key] = val
33             if cost:
34                 dt['cost'] = cost
35                 dt['currency'] = 'EUR'
36         
37         #quantity
38         d = soup.find("input",id="ItemQuantity")
39         if d:
40            dt['quantity'] = d['value']
41         #specs
42         e = soup.find("div",class_="row parameter-container")
43         if e:
44             key1 = []
45             val1= []
46             for k in e.find_all('dt'):
47                 key =  k.get_text().strip().strip('.')
48                 if key:
49                     key1.append(key)
50             for i in e.find_all('dd'):
51                 val =  i.get_text().strip()
52                 if val:
53                     val1.append(val)
54             specs = dict(zip(key1,val1))
55         if specs:
56             dt['specs'] = specs
57             print dt
58 
59             
60         if dt:
61             db[table].update({'sn':sn},{'$set':dt})
62             print str(sn) +  ' insert successfully'
63             time.sleep(3)
64         else:
65             error(str(sn) + '\t' + url)
66     except Exception,e:
67         error(str(sn) + '\t' + url)
68         print "Don't data!"