一键爬取资源python简直太强大了（Python爬取B）

这个夏天，《哪吒之魔童降世》碾压其他暑期档电影，成为最强黑马。我身边的朋友，不是已经N刷了这部电影，就是在赶去N刷的路上。从票房上也可窥见一斑：

数据爬取

在浏览器开发者模式CTRL F很容易就能找到所需要的信息，就在页面源码中：

因此我们用beautifulsoup库就能快速方便地获取想要的信息啦。

因为B站视频数量有限定，每次搜索只能显示20条*50页=1000个视频信息。

为了尽可能多的获取视频信息，我另外还选了“最多点击”“最新发布”“最多弹幕”和“最多收藏”4个选项。

• search.bilibili/all?keyword=哪吒之魔童降世&from_source=nav_search&order=totalrank&duration=0&tids_1=0&page={}
• search.bilibili/all?keyword=哪吒之魔童降世&from_source=nav_search&order=click&duration=0&tids_1=0&page={}
• search.bilibili/all?keyword=哪吒之魔童降世&from_source=nav_search&order=stow&duration=0&tids_1=0&page={}
• search.bilibili/all?keyword=哪吒之魔童降世&from_source=nav_search&order=dm&duration=0&tids_1=0&page={}
• search.bilibili/all?keyword=哪吒之魔童降世&from_source=nav_search&order=pubdate&duration=0&tids_1=0&page={}

5个URL，一共爬取5000条视频，去重之后还剩下2388条信息。

为了得到“转评赞”数据，我还以视频id里面的数字（去掉“av”）为索引，遍历访问了每个视频页面获取了更详细的数据，最终得到以下字段：

数据分析

电影在7月18、19日就进行了全国范围的点映，正式上映时间为7月26日，在这之后相关视频数量有明显的上升。

在这时间之前的，最早发布时间可以追溯到2018年11月份，大部分都是预告类视频：

在8月7日之后视频数量猛增，单单8月7日一天就新上传了319个相关视频。

从标题名字中我们可以大致了解视频的内容：

毫无疑问，“哪吒”和“敖丙”作为影片两大主角是视频的主要人物；因为他们同生共患难的情谊，“藕饼”（“哪吒 敖丙”组合）也是视频的关键词；除此之外，“国漫”也是一大主题词，毕竟我们这次是真正地被我们的国产动漫震撼到了。

实现代码

bilibili.py

import requests import re from datetime import datetime import pandas as pd import random import time ​ ​ ​ video_time=[] abstime=[] userid=[] comment_content=[] ​ def dateRange(beginDate, endDate): dates = [] dt = datetime.datetime.strptime(beginDate, "%Y-%m-%d") date = beginDate[:] while date <= endDate: dates.append(date) dt = dt datetime.timedelta(1) date = dt.strftime("%Y-%m-%d") return dates ​ #视频发布时间～当日 search_time=dateRange("2016-01-10", "2019-06-25") ​ headers = { 'Host': 'api.bilibili', 'Connection': 'keep-alive', 'Content-Type': 'text/xml', 'Upgrade-Insecure-Requests': '1', 'User-Agent': '', 'Origin': 'bilibili', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN,zh;q=0.9', # 'Cookie': 'finger=edc6ecda; LIVE_BUVID=AUTO1415378023816310; stardustvideo=1; CURRENT_FNVAL=8; buvid3=0D8F3D74-987D-442D-99CF-42BC9A967709149017infoc; rpdid=olwimklsiidoskmqwipww; fts=1537803390' } #cookie用火狐浏览器找，以字典形式写入 cookie={ # '_dfcaptcha':'2dd6f170a70dd9d39711013946907de0', # 'bili_jct':'9feece81d443f00759b45952bf66dfff', # 'buvid3':'DDCE08BC-0FFE-4E4E-8DCF-9C8EB7B2DD3752143infoc', # 'CURRENT_FNVAL':'16', # 'DedeUserID':'293928856', # 'DedeUserID__ckMd5':'6dc937ced82650a6', # 'LIVE_BUVID':'AUTO7815513331706031', # 'rpdid':'owolosliwxdossokkkoqw', # 'SESSDATA':'7e38d733,1564033647,804c5461', # 'sid':' 9zyorvhg', # 'stardustvideo':'1', } ​ url='api.bilibili/x/v2/dm/history?type=1&oid=5627945&date={}' ​ for search_data in search_time: print('正在爬取{}的弹幕'.format(search_data)) full_url=url.format(search_data) res=requests.get(full_url,headers=headers,timeout=10,cookies=cookie) res.encoding='utf-8' data_number=re.findall('d p="(.*?)">',res.text,re.S) data_text=re.findall('">(.*?)</d>',res.text,re.S) comment_content.extend(data_text) for each_numbers in data_number: each_numbers=each_numbers.split(',') video_time.append(each_numbers[0]) abstime.append(time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(int(each_numbers[4])))) userid.append(each_numbers[6]) time.sleep(random.random()*3) ​ ​ print(len(comment_content)) print('爬取完成') result={'用户id':userid,'评论时间':abstime,'视频位置(s)':video_time,'弹幕内容':comment_content} ​ results=pd.DataFrame(result) final= results.drop_duplicates() final() final.to_excel('B站弹幕（天鹅臂）最后.xlsx')

bilibili_danmu.py

import requests import re import datetime import pandas as pd import random import time ​ ​ ​ video_time=[] abstime=[] userid=[] comment_content=[] ​ def dateRange(beginDate, endDate): dates = [] dt = datetime.datetime.strptime(beginDate, "%Y-%m-%d") date = beginDate[:] while date <= endDate: dates.append(date) dt = dt datetime.timedelta(1) date = dt.strftime("%Y-%m-%d") return dates ​ #视频发布时间～当日 search_time=dateRange("2019-07-26", "2019-08-09") ​ headers = { 'Host': 'api.bilibili', 'Connection': 'keep-alive', 'Content-Type': 'text/xml', 'Upgrade-Insecure-Requests': '1', 'User-Agent': '', 'Origin': 'bilibili', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'zh-CN,zh;q=0.9', } ​ #cookie用火狐浏览器找，以字典形式写入 cookie={ # '_dfcaptcha':'2dd6f170a70dd9d39711013946907de0', 'bili_jct':'bili_jct5bbff2af91bd6d6c219d1fafa51ce179', 'buvid3':'4136E3A9-5B93-47FD-ACB8-6681EB0EF439155803infoc', 'CURRENT_FNVAL':'16', 'DedeUserID':'293928856', 'DedeUserID__ckMd5':'6dc937ced82650a6', 'LIVE_BUVID':'AUTO6915654009867897', # 'rpdid':'owolosliwxdossokkkoqw', 'SESSDATA':'72b81477,1567992983,bd6cb481', 'sid':'i2a1khkk', 'stardustvideo':'1', } ​ url='api.bilibili/x/v2/dm/history?type=1&oid=105743914&date={}' ​ for search_data in search_time: print('正在爬取{}的弹幕'.format(search_data)) full_url=url.format(search_data) res=requests.get(full_url,headers=headers,timeout=10,cookies=cookie) res.encoding='utf-8' data_number=re.findall('d p="(.*?)">',res.text,re.S) data_text=re.findall('">(.*?)</d>',res.text,re.S) comment_content.extend(data_text) for each_numbers in data_number: each_numbers=each_numbers.split(',') video_time.append(each_numbers[0]) abstime.append(time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(int(each_numbers[4])))) userid.append(each_numbers[6]) time.sleep(random.random()*3) ​ ​ print(len(comment_content)) print('爬取完成') result={'用户id':userid,'评论时间':abstime,'视频位置(s)':video_time,'弹幕内容':comment_content} ​ results=pd.DataFrame(result) final= results.drop_duplicates() final() final.to_excel('B站弹幕（哪吒）.xlsx')

bilibili_detailpage.py

from bs4 import BeautifulSoup import requests import warnings import re from datetime import datetime import json import pandas as pd import random import time import datetime from multiprocessing import Pool ​ ​ ​ url='api.bilibili/x/web-interface/view?aid={}' ​ headers = { 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1', 'Referer':'bilibili/', 'Connection':'keep-alive'} cookies={'cookie':'LIVE_BUVID=AUTO6415404632769145; sid=7lzefkl6; stardustvideo=1; CURRENT_FNVAL=16; rpdid=kwmqmilswxdospwpxkkpw; fts=1540466261; im_notify_type_293928856=0; CURRENT_QUALITY=64; buvid3=D1539899-8626-4E86-8D7B-B4A84FC4A29540762infoc; _uuid=79056333-ED23-6F44-690F-1296084A1AAE80543infoc; gr_user_id=32dbb555-8c7f-4e11-beb9-e3fba8a10724; grwng_uid=03b8da29-386e-40d0-b6ea-25dbc283dae5; UM_distinctid=16b8be59fb13bc-094e320148f138-37617e02-13c680-16b8be59fb282c; DedeUserID=293928856; DedeUserID__ckMd5=6dc937ced82650a6; SESSDATA=b7d13f3a,1567607524,4811bc81; bili_jct=6b3e565d30678a47c908e7a03254318f; _uuid=01B131EB-D429-CA2D-8D86-6B5CD9EA123061556infoc; bsource=seo_baidu'} ​ k=0 def get_bilibili_detail(id): global k k=k 1 print(k) full_url=url.format(id[2:]) try: res=requests.get(full_url,headers=headers,cookies=cookies,timeout=30) time.sleep(random.random() 1) print('正在爬取{}'.format(id)) content=json.loads(res.text,encoding='utf-8') test=content['data'] except: print('error') info={'视频id':id,'最新弹幕数量':'','金币数量':'','不喜欢':'','收藏':'','最高排名':'','点赞数':'','目前排名':'','回复数':'','分享数':'','观看数':''} return info else: danmu=content['data']['stat']['danmaku'] coin=content['data']['stat']['coin'] dislike=content['data']['stat']['dislike'] favorite=content['data']['stat']['favorite'] his_rank=content['data']['stat']['his_rank'] like=content['data']['stat']['like'] now_rank=content['data']['stat']['now_rank'] reply=content['data']['stat']['reply'] share=content['data']['stat']['share'] view=content['data']['stat']['view'] info={'视频id':id,'最新弹幕数量':danmu,'金币数量':coin,'不喜欢':dislike,'收藏':favorite,'最高排名':his_rank,'点赞数':like,'目前排名':now_rank,'回复数':reply,'分享数':share,'观看数':view} return info ​ if __name__=='__main__': df=pd.read_excel('哪吒.xlsx') avids=df['视频id'] detail_lists=[] for id in avids: detail_lists.append(get_bilibili_detail(id)) ​ reshape_df=pd.DataFrame(detail_lists) final_df=pdrge(df,reshape_df,how='inner',on='视频id') final_df.to_excel('藕饼cp详情new.xlsx') final_df() # final_df.duplicated(['视频id']) # reshape_df.to_excel('藕饼cp.xlsx')

bilibili_search.py

from bs4 import BeautifulSoup import requests import warnings import re from datetime import datetime import json import pandas as pd import random import time import datetime from multiprocessing import Pool ​ ​ headers = { 'User-Agent': '' 'Referer':'bilibili/', 'Connection':'keep-alive'} cookies={'cookie':''} ​ def get_bilibili_oubing(url): avid=[] video_type=[] watch_count=[] comment_count=[] up_time=[] up_name=[] title=[] duration=[] print('正在爬取{}'.format(url)) time.sleep(random.random() 2) res=requests.get(url,headers=headers,cookies=cookies,timeout=30) soup=BeautifulSoup(res.text,'html.parser') #avi号码 avids=soup.select('.avid') #视频类型 videotypes=soup.find_all('span',class_="type hide") ​ #观看数 watch_counts=soup.find_all('span',title="观看") #弹幕 comment_counts=soup.find_all('span',title="弹幕") #上传时间 up_times=soup.find_all('span',title="上传时间") #up主 up_names=soup.find_all('span',title="up主") ​ #title titles=soup.find_all('a',class_="title") #时长 durations=soup.find_all('span',class_='so-imgTag_rb') for i in range(20): avid.append(avids[i].text) video_type.append(videotypes[i].text) watch_count.append(watch_counts[i].text.strip()) comment_count.append(comment_counts[i].text.strip()) up_time.append(up_times[i].text.strip()) up_name.append(up_names[i].text) title.append(titles[i].text) duration.append(durations[i].text) result={'视频id':avid,'视频类型':video_type,'观看次数':watch_count,'弹幕数量':comment_count,'上传时间':up_time,'up主':up_name,'标题':title,'时长':duration} ​ results=pd.DataFrame(result) return results ​ if __name__=='__main__': url_original='search.bilibili/all?keyword=哪吒之魔童降世&from_source=nav_search&order=totalrank&duration=0&tids_1=0&page={}' url_click='search.bilibili/all?keyword=哪吒之魔童降世&from_source=nav_search&order=click&duration=0&tids_1=0&page={}' url_favorite='search.bilibili/all?keyword=哪吒之魔童降世&from_source=nav_search&order=stow&duration=0&tids_1=0&page={}' url_bullet='search.bilibili/all?keyword=哪吒之魔童降世&from_source=nav_search&order=dm&duration=0&tids_1=0&page={}' url_new='search.bilibili/all?keyword=哪吒之魔童降世&from_source=nav_search&order=pubdate&duration=0&tids_1=0&page={}' all_url=[url_bullet,url_click,url_favorite,url_new,url_original] ​ info_df=pd.DataFrame(columns = ['视频id','视频类型','观看次数','弹幕数量','上传时间','up主','标题','时长']) for i in range(50): for url in all_url: full_url=url.format(i 1) info_df=pd.concat([info_df,get_bilibili_oubing(full_url)],ignore_index=True) print('爬取完成！') #去重 info_df=info_df.drop_duplicates(subset=['视频id']) info_df() info_df.to_excel('哪吒.xlsx')

,