python爬取某网站原图作为壁纸(代理匹配)

不得不说 python真是一个神奇的东西,学三天就能爬网站真香

完整代码

# -*- coding: utf-8 -*-
"""
Created on Wed May 26 17:53:13 2021
@author: 19088
"""
import urllib.request
import os
import pickle
import re
import random
import sys

#获取http代理
class getHttpAgents:
 #初始化函数
 def __init__(self):
  self.attArray=self.__loadAgentList()
  self.myagent=""
 
 #注意 返回对象未进行解码
 def openUrl(self,url,istry=1):
  response=""
  ip=""
  if(0 != len(self.myagent.strip())):
ip=self.myagent
  i=1
  if not istry:
i=99
  while i<100:
try:
 #print(self.attArray)
 if(0 == len(self.attArray) and 0==len(ip.strip())):
  req=urllib.request.Request(url)
  #设置访问头
  req.add_header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36")
  response=urllib.request.urlopen(req)
 else:
  if(0 != len(self.attArray)):ip=random.choice(self.attArray)
  if(0 != len(self.myagent.strip())):ip=self.myagent
  print("以{}访问 {}".format(ip,url))
  #设置代理
  proxy={"http":ip}
  #print(proxy)
  #定义一个代理字段
  proxy_support=urllib.request.ProxyHandler(proxy)
  #建立一个opener
  opener=urllib.request.build_opener(proxy_support)
  opener.addheaders=[("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36")]
  #urllib.request.install_opener(opener)
  #获得网页对象
  response=opener.open(url)
except:
 if not istry:
  print("{} 无法使用".format(ip))
 else:
  print("第{}次尝试连接！".format(i))
else:
 break;
finally:
 i+=1
  if 11==i and istry:
raise ValueError
  if not response:
return 
  html=response.read()
  #print(html)
  return html
 #检查代理池 去除掉不可用代理ip
 def checkMyIpPool(self):
  agentsResult=[]
  agentList=self.attArray
  for iter in agentList:
ip=iter
self.setMyIp(ip)
b=self.__getMyIp()
if not b:
 #代理不能用
 #agentList.pop(-iter)
 pass
else:
 agentsResult.append(ip)
 #print(b)
  #记录爬取过的可以使用的代理ip
  self.__writeAgentList(agentsResult)
  self.__setAgents(agentsResult)
  self.setMyIp("")
 
 #解析读取网页中所有的代理地址
 def getAgents(self,html):
  #print(html)
  #匹配 ip地址 正则表达式
  pattern = re.compile(r'(<td>)\s*((25[0-5]|2[0-4]\d|[0-1]\d\d|\d\d|\d)\.){3}(25[0-5]|2[0-4]\d|[0-1]\d\d|\d\d|\d)\s*</td>')
  ipList=[]
  ip=pattern.finditer(html)
  for ipiter in ip:
ipText=ipiter.group()
ipGroup=re.search(r"((25[0-5]|2[0-4]\d|[0-1]\d\d|\d\d|\d)\.){3}(25[0-5]|2[0-4]\d|[0-1]\d\d|\d\d|\d)", ipText)
ipList.append(ipGroup.group())
  #匹配 端口地址 正则表达式
  portList=[]
  pattern = re.compile(r'(<td>)\s*\d+\s*</td>')
  port = pattern.finditer(html) 
  for portiter in port:
portText=portiter.group()
portGroup=re.search(r"\d+", portText)
portList.append(portGroup.group())
  if(len(ipList) is not len(portList)):
print("注意: ip和端口参数不匹配!")
return
  ipDict=dict(zip(ipList,portList))
  agentList=[]
  for key in ipDict:
agentList.append(key+":"+ipDict.get(key))  
  agentsResult=[]
  for iter in agentList:
ip=iter
self.setMyIp(ip)
b=self.__getMyIp()
if not b:
 #代理不能用
 pass
 #agentList.pop(-iter)
else :
 agentsResult.append(ip)
 self.__setAgents(agentsResult)
 print("{} 可以使用".format(ip))
  agentsResult.extend(self.attArray)  
  #记录爬取过的可以使用的代理ip
  if(0==len(agentsResult)):
return
  self.__writeAgentList(agentsResult)
  self.__setAgents(agentsResult)
  self.setMyIp("")
  return agentList
 
 def __setAgents(self,ipArray):
  self.attArray=ipArray
 def setMyIp(self,ip):
  self.myagent=ip
 #存储爬取过的ip代理
 def __writeAgentList(self, agentList): 
  if os.path.exists("agent.pkl"):
os.remove("agent.pkl") #每次重新生成 要不多次 dump需要多次 load
  with open("agent.pkl.","wb") as f:
pickle.dump(agentList, f)
  print("存储{}条代理".format(len(agentList)))
 
 #加载之前存储过的ip代理
 def __loadAgentList(self):
  agentlist=[]
  if not os.path.exists("agent.pkl"):
return agentlist
  with open("agent.pkl","rb") as f:
agentlist=pickle.load(f)
print("加载{}条代理".format(len(agentlist)))
return agentlist
 #获取当前使用的ip地址 类的内部方法 仅供内部调用
 def __getMyIp(self,ip=""):
  url="https://www.baidu.com/"
  html=""
  try:
html=self.openUrl(url,0).decode("utf-8")
  except:
return 
  #匹配ip地址
  #pattern = re.compile(r'((25[0-5]|2[0-4]\d|[0-1]\d\d|\d\d|\d)\.){3}(25[0-5]|2[0-4]\d|[0-1]\d\d|\d\d|\d)')
  #groupIp=pattern.search(html)
  #if groupIp:
#return groupIp.group()
  else:
return html
 
 #通过不同的网站去爬取代理
 def crawlingAgents(self,index):
  try:
url ="http://ip.yqie.com/ipproxy.htm"
print(url)
html=self.openUrl(url) 
html=html.decode("utf-8") 
self.setMyIp("")  #不指定ip 随机挑选一个作为代理
self.getAgents(html)
  except Exception as e:
print("{} 爬取失败".format(url))
  
  #一共搜集多少页
  page=index
  
  indexCur=1
  while indexCur<=page:
try:
 url=r"https://www.89ip.cn/index_{}.html".format(indexCur)
 print(url)
 self.setMyIp("") 
 html=self.openUrl(url) #不指定ip 随机挑选一个作为代理
 html=html.decode("utf-8")
 self.getAgents(html)
except Exception as e:
 print("{} 爬取失败".format(url))
finally:
 indexCur+=1
  
  indexCur=1
  while indexCur<=page:
try:
 url=r"http://www.66ip.cn/{}.html".format(indexCur)
 print(url)
 self.setMyIp("") 
 html=a.openUrl(url) #不指定ip 随机挑选一个作为代理
 html=html.decode("gb2312")
 self.getAgents(html)
except Exception as e:
 print("{} 爬取失败".format(url))
finally:
 indexCur+=1
  indexCur=1
  while indexCur<=page:
try:
 url=r"http://www.ip3366.net/?stype=1&page={}".format(indexCur)
 print(url)
 self.setMyIp("") 
 html=a.openUrl(url) #不指定ip 随机挑选一个作为代理
 html=html.decode("gb2312")
 self.getAgents(html)
except Exception as e:
 print("{} 爬取失败".format(url))
finally:
 indexCur+=1  
  indexCur=1
  while indexCur<=page:
try:
 url=r"http://www.kxdaili.com/dailiip/1/{}.html".format(indexCur)
 print(url)
 self.setMyIp("") 
 html=a.openUrl(url) #不指定ip 随机挑选一个作为代理
 html=html.decode("utf-8")
 self.getAgents(html)
except Exception as e:
 print("{} 爬取失败".format(url))
finally:
 indexCur+=1

#下载图片封装类
class downLoadPictures:
 #构造函数
 def __init__(self):
  self.sortKey={}  #定义一个搜索关键字的字典
  self.urlLoad=getHttpAgents()
  self.bzmenuDict={}  #分类信息 风景 美女 什么的分类
  self.sortscreenDict={}  #按照屏幕尺寸分类
  self.littleSignDict={}  #分类信息下面的小分类
  pass
 
 
 def getPictures(self,url):
  #第一步 打开网页 读取page信息 
  pagerHtml=self.urlLoad.openUrl(url)
  #第二步 获取 pageFolder 链接和各种分类信息 返回的是一堆folder链接的url
  folderPictursUrl=self.readPages(pagerHtml).values()
  if not folderPictursUrl:
print("获取图片集失败!")
return
  for floderiterUrl in folderPictursUrl:
folderUrl=str("https://www.ivsky.com/")+floderiterUrl
folderHtml=self.urlLoad.openUrl(folderUrl)
#第三步 读取图片集 获取单个图片的链接地址 返回的是图片集里面的一堆文件url
pictursUrlDict=self.readFolders(folderHtml)
for iterPictureKey in pictursUrlDict:
 fileName=iterPictureKey+".jpg"
 pictureUrl=str("https://www.ivsky.com/")+pictursUrlDict.get(iterPictureKey)
 
 #读取图片页相关信息
 pictureHtml=self.urlLoad.openUrl(pictureUrl)
 picturDownUrl=self.readPictures(pictureHtml)
 pictureDownHtml=self.urlLoad.openUrl(picturDownUrl)
 if not pictureDownHtml:
  continue
 #保存图片
 with open(fileName,"wb+") as f:
  f.write(pictureDownHtml)
  
 
 #提取匹配内容中的所有链接地址
 def getHrefMap(self,html,isPicture=0,isFolder=0):
  hrefDict={}
  pattern=re.compile(r'<a\s*.*?\s*</a>',re.I)
  if isPicture:
pattern=re.compile(r'<p>\s*?<a\s*.*?</p>',re.I)
  hrefIter=pattern.finditer(html)
  index=0
  for iter in hrefIter:
hrefText=iter.group()
#匹配分类名字
pattern=re.compile(r'"\s*?>\s*?.*?</a>',re.I)
name=""
nameGroup=pattern.search(hrefText)
if nameGroup:
 name=nameGroup.group()
 if(5==len(nameGroup.group().replace(" ", ""))):
  pattern=re.compile(r'title=".*?"',re.I)
  nameGroup=pattern.search(hrefText)
  if nameGroup:name=nameGroup.group()[7:-1]
 name=name[2:-4].replace(" ", '')
#匹配href
pattern=re.compile(r'href=".*?" rel="external nofollow" ',re.I)
url=""
urlGroup=pattern.search(hrefText)
if urlGroup:
 url=urlGroup.group()[6:-1].replace(" ", '')
if isFolder:
 index+=1
 name+="_"+str(index)
hrefDict[name]=url
  return hrefDict
  #读取首页信息 包含各种分类的链接地址 以及图片集的地址集合
 def readPages(self,html):
  html=html.decode("utf-8")
  #检索壁纸分类
  #匹配 壁纸分类信息
  pattern=re.compile(r'<ul\s*class="bzmenu".*?</ul>',re.I)
  sortClassGroup=pattern.search(html)
  if sortClassGroup:
sortMessage=sortClassGroup.group()
self.bzmenuDict=self.getHrefMap(sortMessage)
#print(self.bzmenuDict)
  else:
print("匹配壁纸分类出错!")
return
  
#匹配 按照屏幕大小分类
  pattern=re.compile(r'<ul\s*class="sall_dd".*?</ul>',re.I)
  sortClassGroup=pattern.search(html)
  if sortClassGroup:
sortMessage=sortClassGroup.group()
self.sortscreenDict=self.getHrefMap(sortMessage)
#print(self.sortscreenDict)
  else:
print("匹配屏幕尺寸分类失败!")
return 
 
#匹配 获取小分类
  pattern=re.compile(r'<div\s*class="sline".*?</div>',re.I)
  sortClassGroup=pattern.search(html)
  if sortClassGroup:
sortMessage=sortClassGroup.group()
#print(sortMessage)
self.littleSignDict=self.getHrefMap(sortMessage)
#print(self.littleSignDict)
  else:
print("匹配小分类失败")
return
  
  pictureDict={}
  #匹配 图片集地址
  pattern=re.compile(r'<ul\s*class="ali".*?</ul>',re.I)
  sortClassGroup=pattern.search(html)
  if sortClassGroup:
sortMessage=sortClassGroup.group()
pictureDict=self.getHrefMap(sortMessage,1)
#print(pictureDict)
  else:
print("匹配图片集地址失败!")
return
  #print(html)
  return pictureDict
 
 #解析每个图片集合对应的图片集内容 解析出单个图片的链接地址
 def readFolders(self,html):
  if not html:
return
  html=html.decode("utf-8")
  
  #获取图片集里面每个图片的具体地址和名称
#匹配 获取小分类
  pattern=re.compile(r'<ul\s*class="pli".*?</ul>',re.I)
  sortClassGroup=pattern.search(html)
  pictureUrlDict={}
  if sortClassGroup:
sortMessage=sortClassGroup.group()
#print(sortMessage)
pictureUrlDict=self.getHrefMap(sortMessage,1,1)
#print(pictureUrlDict)
  else:
print("匹配小分类失败")
return
  return pictureUrlDict
 
 #解析每个图片集合对应的图片集内容 解析出单个图片的链接地址
 def readPictures(self,html):
  if not html:
return  
  html=html.decode("utf-8")
  #获取图片集里面每个图片的具体地址和名称
#匹配 获取小分类
  pattern=re.compile(r'<div\s*class="pic".*?</div>',re.I)
  sortClassGroup=pattern.search(html)
  pictureUrl=""
  if sortClassGroup:
sortMessage=sortClassGroup.group()
#匹配href
pattern=re.compile(u"src='.*?'",re.I)
url=""
urlGroup=pattern.search(sortMessage)
if urlGroup:
 url=urlGroup.group()[5:-1].replace(" ", '')
url=url.replace('img-pre', 'img-picdown')
url=url.replace('pre', 'pic')
url=str("https:")+url
#print(sortMessage)
pictureUrlDict=url
#print(url)
  else:
print("匹配小分类失败")
return
  return pictureUrlDict  
  
class UrlUser:
 
 def __init__(self):
  self.agent=getHttpAgents()
  self.downPicture=downLoadPictures()
 
 #下载图片调用函数
 def downPictures(self):
  #url="https://www.ivsky.com/bizhi"
  #b.getPictures(url)
  #确定保存路径
  dirPath=input("请输入保存路径:")
  if not os.path.exists(dirPath):
os.mkdir(dirPath)
  if not os.path.isdir(dirPath):
print("savePath is wrong!")
sys.exit()
  os.chdir(dirPath)  #切换工作目录 
  #url=r"https://www.ivsky.com/bizhi/nvxing_1920x1080/index_{}.html"
  page=input("爬取前多少页的图片?\n")
  indexRe = re.search(r"\d+", page)
  if(not indexRe):
print("输入页数有误!")
  indexRe=int(indexRe.group())
  indexCur=1
  while indexCur<=indexRe:
try:
 #注意 爬取什么类型的图片可以根据不同的网址进行设计 下载类里面已经读取了所有分类对应的地址 有兴趣可以自己完善
 url=r"https://www.ivsky.com/bizhi/nvxing_1920x1080/index_{}.html".format(indexCur)
 print(url)
 self.downPicture.getPictures(url)
except:
 print("打开出错!")
 pass
finally:
 indexCur+=1
 #爬取代理
 def downAgents(self):
  page=input("爬取前多少页的代理?\n")
  indexRe = re.search(r"\d+", page)
  if(not indexRe):
print("输入页数有误!")
return
  indexRe=int(indexRe.group()) 
  self.agent.crawlingAgents(indexRe)
 
 # 检查当前代理池是否可以
 def checkPool(self):
  self.agent.checkMyIpPool() 
  
if __name__ == "__main__":
 print("*"*20)
 print("1.爬取代理\n")
 print("2.检查代理\n")
 print("3.爬取图片")
 print("*"*20)
 mode=input("请输入数字选择处理模式:\n")
 indexRe = re.search(r"\d+", mode)
 if(not indexRe):
  print("输入页数有误!")
  sys.exit()
 indexRe=int(indexRe.group())
 #实例化一个对象
 uesrObj=UrlUser()
 
 if 1 == indexRe:
  uesrObj.downAgents()
 elif 2 == indexRe:
  uesrObj.checkPool()
 elif 3 == indexRe:
  uesrObj.downPictures()
 else:
  print("模式选择错误!")
  sys.exit()
 print("爬取完毕！")