一个用Python实现的多入口全网爬的多线程爬虫的实现

1
2
3
4
5
6
7
8
9
10
11
12
13
#入口url列表
urls = [
		'http://www.ailab.cn/news/ainew/',
		'http://news.chinaso.com/search?wd='+request.quote('人工智能+行业'),
		'https://www.baidu.com/s?ie=utf-8&cus_sid=18364444163281065044&tn=SE_pscse_053x7tyx&wd=%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD%20%E8%A1%8C%E4%B8%9A'
		'http://search.ifeng.com/sofeng/search.action?q=%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD+%E8%A1%8C%E4%B8%9A&c=1&chel='
		'http://search.southcn.com/web/search?channelid=216505&&searchword=%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD%20%E8%A1%8C%E4%B8%9A',
		'https://www.baidu.com/s?ie=utf-8&cus_f=0&wd=%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD%01%E8%A1%8C%E4%B8%9A&tn=71032170_cpr&rsv_lu=3_sb&si=china.com&ct=2097152&fenlei=mv6qUAdxTZPxTvb0IZRqIHDLnjmdnWc0T1YYPARYrHN-nAR4uhD1uHTL0AGo5HmLPH7hP1u9PHD4Pj9bnhm0IAYqnWm3PjfkPHRz0APh5Hn0Thcqn0K_IyVG5HD0mv4YUWYLnH01nWDLn7qWTZc0TLPs5HD0TLPsnWYk0ZwYTjYk0A4YIZ0qnfKLpHYY0Au1mv9VujYz0Zwb5HDLnjmdnWc0IgF_5y9YIZ0-nYD-nbm-nbuYuyPCFHF7mv9GUhD-nbNWUvY-nbm0',
		'http://search.sina.com.cn/?q=%C8%CB%B9%A4%D6%C7%C4%DC+%D0%D0%D2%B5&range=all&c=news&sort=rel',
		'http://sou.chinanews.com.cn/search.do?q=%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD+%E8%A1%8C%E4%B8%9A&ps=10&time_scope=0&channel=all&sort=_score',
		'http://news.sogou.com/news?query=%C8%CB%B9%A4%D6%C7%C4%DC+%D0%D0%D2%B5',
		'https://www.sogou.com/sogou?site=news.qq.com&query=%C8%CB%B9%A4%D6%C7%C4%DC+%D0%D0%D2%B5&pid=sogou-wsse-b58ac8403eb9cf17-0004&sourceid=&idx=f&idx=f'
]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
class MyQueue:
	def __init__(self):
		self.visited = []
		self.unVisitedFirst = []	#采用多级队列实现优先级分级
		self.unVisitedSecond = []

    #将最后一个元素（即队列头）移出列表
    def unVisitedDequeue(self):
        try:
            if len(self.unVisitedFirst) > 0:    #只有当一级队列为空时，才返回二级队列的内容
                return self.unVisitedFirst.pop()
            else:
                return self.unVisitedSecond.pop()
        except:
            return None

    #将新加入的url插入到列表头（即对列尾），先进先出    
    def addUnvisitedUrl(self,url,level):    
        if url!='' :
            if level == 1:
                self.unVisitedFirst.insert(0,url)
            else:
                self.unVisitedSecond.insert(0,url)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
"""
抓取页面并进行解压处理
超时三次才将页面丢弃，防止丢弃过多页面
"""
	def fetchPage(self,url):
		page = ''
		for i in range(3):
			try:
				socket.setdefaulttimeout(5)
				req = request.Request(url,headers=self.headers)
				page = request.urlopen(req).read()
				page = self.gzdecode(page)
				# print('success')
				return page
			except request.HTTPError as e:
				print(e.reason)
				return ''
			except request.URLError as e:
				print(e.reason)
				return ''
			except socket.timeout as e:
				if i < 2:
					continue
				else:
					print('timeout')
					return ''
			except:
				print('other error')
				return ''

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
"""
过滤HTML标签，合并多余空行
"""
	def replace(self,content):
		r = re.compile(r'<script.*?</script>',re.I|re.M|re.S)   #删除JavaScript
		s = r.sub ('',content)
		r = re.compile(r'<style.*?</style>',re.I|re.M|re.S)     #删除css
		s = r.sub ('', s)
		r = re.compile(r'<!--.*?-->', re.I|re.M|re.S)           #删除注释
		s = r.sub('',s)
		r = re.compile(r'<meta.*?>', re.I|re.M|re.S)            #删除meta
		s = r.sub('',s)
		r = re.compile(r'<ins.*?</ins>', re.I|re.M|re.S)        #删除ins
		s = r.sub('',s)
		r = re.compile(r'<[^>]+>',re.M|re.S)
		s = r.sub('',s).strip()                                 #删除HTML标签
		r = re.compile(r'^s+$', re.M|re.S)                     #删除空白行
		s = r.sub ('', s)
		r = re.compile(r'n+',re.M|re.S)                        #合并空行为一行
		s = r.sub('n',s)
		r = re.compile('s+',re.I|re.M|re.S)
		s = r.sub('n',s)
		return s

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
"""
	@param: values	正文段列表
			k 		每次选取的段数
	@return:left	正文起始段
			right	正文结束段
"""
def findEnd(values,k=5):
	left,right = 0,0
	# print(values)
	for i in range(len(values)):
		strCnt = sum(values[i:i+k])
	# print(strCnt)
		if strCnt > 180 and values[i] > 15:
			if left == 0:
				left = i
			right = i
	cnt = 0
	for i in range(right,len(values)):
		if cnt >= k:
			break
		if values[i] > 15:
			right += 1
		cnt += 1
	return left,right

"""
找出正文的起始位置和结束位置
	@param: content	过滤标签后的网页
			k 		每段选取的行数
	@return:left	正文起始段
			right	正文结束段
"""
def filt(content,k=1):
	if not content:
		return None,None
	lines = content.split('n')
	group_value = []
	for i in range(0,len(lines),k):
		group = 'n'.join(lines[i:i+k]).strip()
		group_value.append(len(group))
	left,right = findEnd (group_value)
	return left,right
"""