由于单线程太慢，所以刚又写了个多进程，我来为大家科普一下关于草民网多进程程爬取好莱坞大片?下面希望有你要的答案，我们一起来看看吧!
草民网多进程程爬取好莱坞大片
由于单线程太慢，所以刚又写了个多进程
from urllib import request
import urllib
from time import sleep
import socket
import random
from multiprocessing import Process
class Video():
def __init__(self):
self.headers = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3704.400 QQBrowser/10.4.3615.400"
self.url = ""
def set_url(self, i):
if i < 1000:
self.url = "https://zy.baidu-360-yyy-kubo.com/20180705/mBlJPrWZ/800kb/hls/FF9QaL5935d.ts" % i
else:
self.url = "https://zy.baidu-360-yyy-kubo.com/20180705/mBlJPrWZ/800kb/hls/FF9QaL5935d.ts" % i
# 获取并下载ts文件
def download_ts(self, i):
rq = request.Request(self.url)
rq.add_header('User-Agent', self.headers)
response = request.urlopen(rq)
res = response.read()
# 保存文件路径
with open('./HW/' str(i) ".ts", "wb") as f:
f.write(res)
response.close() # 关闭urlopen方法
def start(self, i):
self.set_url(i)
try:
self.download_ts(i)
print('第' str(i) ".ts" '已下载')
sleep(1)
except urllib.error.URLError as e:
print(e.reason)
self.download_ts(i)
except socket.timeout as e2:
print(e2.reason)
self.download_ts(i)
if __name__ == '__main__':
video = Video()
socket.setdefaulttimeout(random.randint(5, 10)) # 设置socket层超时时间20秒
I = 0
while I < 2000 1:
# 5个进程并发运行
p_l = [Process(target=video.start, args=(i,)) for i in range(I, I 5)]
for p in p_l:
p.start()
for p in p_l:
p.join()
I = I 5
,
