Python scrapy爬取苏州二手房交易数据

import scrapy
class LianjiaHomeItem(scrapy.Item):
  name = scrapy.Field() # 名称
  type = scrapy.Field()  # 户型
  area = scrapy.Field()  # 面积
  direction = scrapy.Field()  #朝向
  fitment = scrapy.Field()  # 装修情况
  elevator = scrapy.Field()  # 有无电梯
  total_price = scrapy.Field()  # 总价
  unit_price = scrapy.Field()  # 单价

from scrapy import Request
from scrapy.spiders import Spider
from lianjia_home.items import LianjiaHomeItem
class HomeSpider(Spider):
 name = "home"
 current_page=1 #起始页
 def start_requests(self): #初始请求
  url="https://su.lianjia.com/ershoufang/"
  yield Request(url=url)
 def parse(self, response): #解析函数
  list_selctor=response.xpath("//li/div[@class='info clear']")
  for one_selector in list_selctor:
try:
 #房屋名称
 name=one_selector.xpath("//div[@class='flood']/div[@class='positionInfo']/a/text()").extract_first()
 #其他信息
 other=one_selector.xpath("//div[@class='address']/div[@class='houseInfo']/text()").extract_first()
 other_list=other.split("|")
 type=other_list[0].strip(" ")#户型
 area = other_list[1].strip(" ") #面积
 direction=other_list[2].strip(" ") #朝向
 fitment=other_list[3].strip(" ") #装修
 price_list=one_selector.xpath("div[@class='priceInfo']//span/text()")
 # 总价
 total_price=price_list[0].extract()
 # 单价
 unit_price=price_list[1].extract()
 item=LianjiaHomeItem()
 item["name"]=name.strip(" ")
 item["type"]=type
 item["area"] = area
 item["direction"] = direction
 item["fitment"] = fitment
 item["total_price"] = total_price
 item["unit_price"] = unit_price
#生成详细页
 url = one_selector.xpath("div[@class='title']/a/@href").extract_first()
 yield Request(url=url,
  meta={"item":item}, #把item作为数据v传递
  callback=self.property_parse) #爬取详细页
except:
 print("error")
  #获取下一页
self.current_page+=1
if self.current_page<=100:
 next_url="https://su.lianjia.com/ershoufang/pg%d"%self.current_page
 yield Request(url=next_url)

 def property_parse(self,response):#详细页
  #配备电梯
  elevator=response.xpath("//div[@class='base']/div[@class='content']/ul/li[last()]/text()").extract_first()
  item=response.meta["item"]
  item["elevator"]=elevator
  yield item

import re
from scrapy.exceptions import DropItem
class LianjiaHomePipeline:#数据的清洗
 def process_item(self, item, spider):
  #面积
  item["area"]=re.findall("\d+\.?\d*",item["area"])[0] #提取数字并存储
  #单价
  item["unit_price"] = re.findall("\d+\.?\d*", item["unit_price"])[0] #提取数字并存储
  #如果有不完全的数据，则抛弃
  if item["direction"] =="暂无数据":
raise DropItem("无数据，抛弃：%s"%item)
  return item
class CSVPipeline(object):
 file=None
 index=0 #csv文件行数判断
 def open_spider(self,spider): #爬虫开始前，打开csv文件
  self.file=open("home.csv","a",encoding="utf=8")
 def process_item(self, item, spider):#按要求存储文件。
  if self.index ==0:
column_name="name,type,area,direction,fitment,elevator,total_price,unit_price\n"
self.file.write(column_name)#插入第一行的索引信息
self.index=1
  home_str=item["name"]+","+item["type"]+","+item["area"]+","+item["direction"]+","+item["fitment"]+","+item["elevator"]+","+item["total_price"]+","+item["unit_price"]+"\n"
  self.file.write(home_str) #插入获取的信息
  return item
 def close_soider(self,spider):#爬虫结束后关闭csv
  self.file.close()

USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.72 Safari/537.36'
#为装成浏览器
ROBOTSTXT_OBEY = False #不遵循robots协议
ITEM_PIPELINES = {
 'lianjia_home.pipelines.LianjiaHomePipeline': 300,
 #先进行数字提取
 'lianjia_home.pipelines.CSVPipeline': 400
 #在进行数据的储存
 #执行顺序由后边的数字决定
}

from scrapy import cmdline
cmdline.execute("scrapy crawl home" .split())