昨天已经完成了 从《续蜀山剑侠传》连载网站页面上获取目录信息，包括目录名称和网址。
接下来就是读取每个目录里的连载内容了，通过对网页代码的分析， 获取连载内容的代码如下，其中涉及到用正则表达到过滤HTML标签、空格，转换换行符\n等，都是非常经典 和 实用的：
# -*- coding:UTF-8 -*- import urllib.request, sys import re #-------------------------- def openUrl(url): try: page = urllib.request.urlopen(url, data=None, timeout=5) except urllib.error.HTTPError as e: print(e.code) print(e.reason) return '' except urllib.error.URLError as e: print(e.reason) return '' else: html = page.read().decode('utf-8') return html #-------------------------- def getCon(html, tag): i = html.find(tag) if i == -1: print ('没有找到' + tag) return '' else: con = html[i+len(tag):] #print ("前30个字符：" + con[:30]) tag = 'ul' tag_pat = r'(?<=<'+ tag + '>).*?(?=</' + tag + '>)' tag_ex = re.compile(tag_pat, re.M|re.S) con = re.findall(tag_ex, con) #con = html.split('正文') #print (con[0]) return con[0] #-------------------------- def printList(list, host): #获取text res = r'(.*?)' t = re.findall(res, list, re.S|re.M) #获取href res_url = r"(?<=href=\").+?(?=\")|(?<=href=\').+?(?=\')" h = re.findall(res_url, list, re.I|re.S|re.M) for i in range(len(t)): print (str(i+1) + '\t' + t[i] + '\t' + host + h[i]) #-------------------------- def showCon(url): print (url) html = openUrl(url) if len(html) <= 0: print ('……未能打开-_-!') return #print (html) #html = html.replace('\u3000', ' ') html = html.replace('', '\n') #取回目 res = r'(.*?) ' t = re.findall(res, html, re.I|re.S|re.M) print (t[0]) #获取内容 res = r'(.*?) ' c = re.findall(res, html, re.I|re.S|re.M) #t = c[0].replace('\u3000',' ') #t = t.replace('','\n') print (c[0]) #-------------------------- def showList(list, host): #获取text res = r'(.*?)' t = re.findall(res, list, re.S|re.M) #获取href res_url = r"(?<=href=\").+?(?=\")|(?<=href=\').+?(?=\')" h = re.findall(res_url, list, re.I|re.S|re.M) showCon(host + h[0]) #for i in range(len(t)): # showCon (host + h[i]) #-------------------------- def main(): url = 'http://www.mengxi.net/book/263745/index.html' i = url.index('/', 7) host = url[0 : i] #print ('打开' + url) html = openUrl(url) if len(html) > 0: tag = '正文' list = getCon(html, tag) showList(list, host) main()
代码运行结果如下：
