Python bs4库爬取豆瓣TOP250基础案例

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
import urllib.request
#打开url
返回HTTPResponse object
用read()解析
#获取一个get请求
#response = urllib.request.urlopen("http://www.baidu.com")
#print(response.read().decode('utf-8'))
#post请求
报错!!!!!
#解析
import urllib.parse
#模拟真实用户
#data = bytes(urllib.parse.urlencode({"1111":"222"}),encoding="utf-8")#转二进制
#response = urllib.request.urlopen("http://httpbin.org/post",data=data)
#print(response.read().decode("utf-8"))
#c超时处理
#try:
#
response = urllib.request.urlopen("http://httpbin.org/get",timeout=1)
#
print(response.read().decode("utf-8"))
#except urllib.error.URLError as e:
#
print("time out!")
#response = urllib.request.urlopen("http://httpbin.org/get",timeout=1)
#标头
#print(response.getheaders())
#print(response.status)
url="/post"
#改成键值对
headers = {
"User-Agent": "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.122
Safari / 537.36"
}
#data =bytes(urllib.parse.urlencode({'13':'5465'}),encodings="utf-8")
#伪装url
#req = urllib.request.Request(url=url,headers=headers,method="POST")
#response=urllib.request.urlopen(req)
#print(response.read().decode("utf-8"))
#即可
req = urllib.request.Request(url=url,headers=headers)
response=urllib.request.urlopen(req)
print(response.read().decode("utf-8"))

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
import re
from bs4 import BeautifulSoup
# 网页解析，获取数据
file = open("a.html","rb")
#用类型接收
html = file.read()
#解析器
bs是解析后的内容
bs = BeautifulSoup(html,"html.parser")
#bs将html的各种元素分类变成他的标签
# bs的标签及内容 bs.head
# string文字内容 拿不到注释
attras属性内容
# beatufulsoup 整个文档
# string
#文档的遍历
#一个数组可以加【0】
#print(bs.head.contents)
#文档搜索
#1 find_all()查找所有标签
#字符串过滤
会查找与字符串安全匹配的内容
#t_list = bs.find_all("a")
#2正则表达式搜索，使用search（）方法搜索！！！！！！！！！！！！强大
#re.compile("a") 包含a
#t_list=bs.find_all(re.compile("a"))
#方法：传入一个函数，根据函数要求搜索
略略
#ame__is__exists(tag) 所有所有有name的标签
#def name__is__exists(tag):
#
return tag.has_attr("name")
#t_list = bs.find_all(name__is__exists)
#2 kwargs 参数
t_list = bs.find_all(id="head")
for item in t_list:
print(item)
#3 text参数 文本的内容
t_list=bs.find_all(text="")
#limit参数
t_list=bs.find_all(text="",limit=3)
#css选择器!!!!!!!!!!!!!!!! 指定标签“”
id select("#u1")
类名 select(".mnav")
# ("head>title") "父>子"
#返回一个列表
print(bs.select("a[class='bri']"))

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
import re
#字符串模式
判断字符串是否符合一定标准
#修饰符
模式限定
#search只找第一个
findall()全部的
#模式法
#1生成一个正则对象
# 2用正则对象调用方法
#pat = re.compile("AA")#此处AA是正则表达式
，用来验证其他字符安川
#m = pat.search("AACBAA")
#print(m)
#没有模式
#m =re.search("asd","asdckoj")
#print(m)
#re.sub() 替换
print(re.sub("a","A","fkegwughikaasdasdasdarfd"))
#re.match()

1
2
3
4
5
6
7
import xlwt
workbook = xlwt.Workbook(encoding="utf-8")#创建workbook对象
work_sheef = workbook.add_sheet('sheef') #创建工作表
work_sheef.write(0,0,'wort')
#写入数据
workbook.save('student.xls')
#保存数据表

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
from bs4 import BeautifulSoup
import re
import urllib.request,urllib.error
import xlwt
import sqlite3
def main():
baseurl = "https://movie.douban.com/top250?start="
#1.爬取网页
datalist = getData(baseurl)
savepath = "豆瓣电影Top250.xls"
#3.保存数据
saveData(datalist,savepath)
#askURL("https://movie.douban.com/top250?start=")
findlink = re.compile(r'<a href="(.*?)">')#影片详情链接
findImgSrc = re.compile(r'<img.*src="(.*?)"',re.S)#让换行符包含在字符中 #影片图片链接
findtitle = re.compile(r'<span class="title">(.*)</span>')#影片片名
findRating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>')#影片评分
findJudge = re.compile(r'<span>(d*)人评价</span>')
findInq = re.compile(r'<span class="inq">(.*)</span>')
findBd = re.compile(r'<p class="">(.*?)</p>',re.S)
#爬取网页
def getData(baseurl):
datalist = []
for i in range(0,10):
url =baseurl + str(i*25)
html = askURL(url)
#2.逐一解析
soup = BeautifulSoup(html,"html.parser")
for item in soup.find_all('div',class_="item"):
#print(item)#测试查看电影item全部信息。
data=[]
item = str(item)
link = re.findall(findlink,item)[0]
data.append(link)
imgSrc = re.findall(findImgSrc,item)[0]
data.append(imgSrc)
titles = re.findall(findtitle,item)
if(len(titles)==2):
ctitle = titles[0]
data.append(ctitle)
otitle = titles[1].replace("/","")
data.append(otitle)
else:
data.append(titles[0])
data.append('')
reting =re.findall(findRating,item)[0]
data.append(reting)
judgeNum = re.findall(findJudge,item)[0]
data.append(judgeNum)
inq = re.findall(findInq,item)
if(len(inq)!=0):
inq = inq[0].replace("。","")
data.append(inq)
else:
data.append("")
bd = re.findall(findBd,item)[0]
bd = re.sub('<br(s+)?/>(s+)?'," ",bd)
bd = re.sub('/'," ",bd)
data.append(bd.strip())
datalist.append(data)
print(datalist)
return datalist
#得到指定网站url内容
def askURL(url):
head = {
"User-Agent":"Mozilla / 5.0(Windows NT 10.0;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 84.0.4147.89Safari / 537.36"
}
request = urllib.request.Request(url,headers=head)
html = ""
try:
response = urllib.request.urlopen(request)
html= response.read().decode("utf-8")
#print(html)
except urllib.error.URLError as e:
if hasattr(e,"code"):
print(e.code)
if hasattr(e, "reason"):
print(e.reason)
return html
#保存网页数据
def saveData(datalist,savepath):
book = xlwt.Workbook(encoding="utf-8",style_compression=0)
sheet = book.add_sheet('豆瓣电影TOP250',cell_overwrite_ok=True)
col = ("电影详情链接","图片链接","影片中文名","影片外国名","评分","评分数","概况","相关信息")
for i in range(0,8):
sheet.write(0,i,col[i])
for i in range(0,250):
print("第%d条"%(i+1))
data = datalist[i]
for j in range(0,8):
sheet.write(i+1,j,data[j])
book.save(savepath)
if __name__== "__main__":
main()