selenium在scrapy中的使用代码

import scrapy
from selenium import webdriver
from wangyiPro.items import WangyiproItem
class WangyiSpider(scrapy.Spider):
 name = 'wangyi'
 # allowed_domains = ['www.xxx.com']
 start_urls = ['https://news.163.com/']
 model_url = []

 hro = webdriver.Chrome()
 def parse(self, response):
 # 从所有版块中筛选出所需要下载的4个版块
  li_list = response.xpath('/html/body/div[1]/div[1]/div[2]/div[2]/div[2]/div[2]/div/ul/li')
  indexs = [3,4,6,7]
  for index in indexs:
model_li = li_list[index]
model_list = model_li.xpath('./a/@href').extract_first()
self.model_url.append(model_list)
  for url in self.model_url:
yield scrapy.Request(url=url,callback=self.parse_model)
 # 数据解析：新闻标题+新闻详情页的url（动态加载数据）
 def parse_model(self,response):
  # 直接对response解析无法获取该数据（动态加载数据）
  # 无法拿到动态加载数据返回的响应数据就不足
  # 使用中间件修改不满足需求的响应对象中的响应数据，将其改为包含动态加载数据的响应数据
  div_list = response.xpath('/html/body/div/div[3]/div[4]/div[1]/div[1]/div/ul/li/div/div')
  for div in div_list:
title = div.xpath('./div/div[1]/h3/a/text()').extract_first()
new_url_detail = div.xpath('./div/div[1]/h3/a/@href').extract_first()
if new_url_detail:
 item = WangyiproItem()
 item['title'] = title

 yield scrapy.Request(url=new_url_detail,callback=self.parse_detail,meta={'item':item})

 def parse_detail(self,response):
  # 返回的列表，需要将其转换为字符串
  content = response.xpath('/html/body/div[3]/div[1]/div[3]/div[2]/p/text()').extract()
  content = ''.join(content)
  item = response.meta['item']
  item['content'] = content
  yield item
 # 重写爬虫类父类方法，该方法在爬虫程序执行结束后立即执行
 def closed(self,spider):
  # 打开selenium需要关闭
  self.hro.quit()

from time import sleep
from scrapy.http import HtmlResponse#scrapy封装好的响应类
class WangyiproDownloaderMiddleware:

 def process_request(self, request, spider):
  return None
 # 拦截所有的响应对象
 # 整个工程发起的请求：1+4+n 对应相同的响应
 def process_response(self, request, response, spider):
  # 从所有拦截对象中找到4个需修改的响应对象
  if request.url in spider.model_url:
hro = spider.hro
hro.get(request.url)
sleep(2)
# 网页下拉到底，获取更多动态加载数据
hro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
sleep(1)
page_text = hro.page_source
# response.text = page_text
# 返回一个新的响应对象
return HtmlResponse(url=request.url,body=page_text,encoding='utf-8',request=request)
  # 如果不是需要修改的响应数据，不做修改
  else:
return response
 def process_exception(self, request, exception, spider):
  pass