python抓取贴吧标题和图片代码
Python实现的爬取百度贴吧图片功能完整示例本文实例讲述了python实现的爬取百度贴吧图片功能。分享给大家供大家参考，具体如下：
|
|
#coding:utf-8import requestsimport urllib2import urllib```from lxml import etreeclass tieba: def __init__(self): self.tiebaname = raw_input("请输入需要爬取的贴吧:") self.beginpage = int(raw_input("请输入爬取的起始页：")) self.endpage = int(raw_input("请输入爬取的结束页：")) self.baseurl = "http://tieba.baidu.com" #self.headers = {"user-agent" : "mozilla/5.0 (windows nt 10.0; wow64) applewebkit/537.36 (khtml, like gecko) chrome/59.0.3071.86 safari/537.36"} self.headers = {"user-agent" : "mozilla/5.0 (compatible; msie 9.0; windows nt 6.1 trident/5.0;"} def startwork(self): """ 发送贴吧每一页的url请求 """ for page in range(self.beginpage, self.endpage + 1): pn = (page - 1) * 50 keyword = {"kw" : self.tiebaname, "pn" : pn} kw = urllib.urlencode(keyword) url = self.baseurl + "/f?" + kw #print url html = self.loadrequest(url) self.loadpage(html) def loadrequest(self, url): """ 发送请求，返回响应 url: 发送请求的url地址 """ request = urllib2.request(url, headers = self.headers) #request = urllib2.request(url) response = urllib2.urlopen(request) return response.read() def loadpage(self, html): """ 提取每个帖子的url，并发送请求，获取响应 html: 贴吧每一页的html """ content = etree.html(html) print '------' print content # xpath 返回的所有匹配成功后的结果的列表 #pagelink_list = content.xpath("//li[@class='threadlist_lz clearfix']/li/a[@class='j_th_tit']/@href") pagelink_list = content.xpath("//li[@class='col2_right j_threadlist_li_right']//li/a/@href") pagelink_list = content.xpath("//li[@class='t_con cleafix']//li/a/@href") for link in pagelink_list: print link self.loadimage(self.loadrequest(self.baseurl + link)) def loadimage(self, html): """ 提取帖子里用户发送的图片的url地址 html: 每个帖子的html """ content = etree.html(html) imagelink_list = content.xpath("//li[@class='p_content ']//img[@class='bde_image']/@src") for link in imagelink_list: self.writeimage(self.loadrequest(link), link[-10:]) def writeimage(self, data, filename): """ 将图片的响应数据，写入到本地磁盘里 data: 图片的响应数据 filename: 文件名(图片url的后10位) """ print "正在保存图片...%s" % filename with open(filename, "wb") as f: f.write(data)if __name__ == "__main__": tieba = tieba() tieba.startwork() |
希望本文所述对大家python程序设计有所帮助。
原文链接：https://blog.csdn.net/xuezhangjun0121/article/details/73719059