python爬取全国水雨情信息详解

import datetime
import pandas as pd
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options #建议使用谷歌浏览器
import time
chrome_options = Options()
chrome_options.add_argument('--headless')
driver = webdriver.Chrome()
# 存储中英文对应的变量的中文名
word_dict = {"poiBsnm": "流域",
 "poiAddv": "行政区",
 "rvnm": "河名", 
 "stnm": "站名",
 "tm": "时间", 
 "zl": "水位(米)",
 "ql": "流量(立方米/秒)",
 "wrz": "警戒水位(米)"}
# 空df接收结果
rain_total = pd.DataFrame([])
url = 'http://xxfb.mwr.cn/sq_dxsk.html'
driver.get(url)
time.sleep(5)
infos = driver.find_elements_by_xpath("/html/body//tbody[@id='DataContainer']/tr")
# pd.set_option('display.max_columns', None)#所有列
# pd.set_option('display.max_rows', None)#所有行
# 列表提取
for info in infos:
 poiBsnm = info.find_element_by_xpath("./td[1]").text
 poiAddv = info.find_element_by_xpath("./td[2]").text
 rvnm = info.find_element_by_xpath("./td[3]").text
 stnm = info.find_element_by_xpath("./td[4]").text
 tm = info.find_element_by_xpath("./td[5]").text
 zl = info.find_element_by_xpath("./td[6]").text
 ql = info.find_element_by_xpath("./td[7]").text
 wrz = info.find_element_by_xpath("./td[8]").text
# 组成pandas对象
 rain_data = [[poiBsnm,poiAddv,rvnm,stnm,tm,zl,ql,wrz]]  
 rain_df = pd.DataFrame(data=rain_data,columns=list(word_dict.values()))
 rain_total = pd.concat([rain_total,rain_df])
 print(rain_total)
# 关闭浏览器
driver.close()
# 保存数据
data_str = datetime.datetime.now().strftime('%Y_%m_%d')
rain_total.to_csv("%s_全国水雨情信息.csv" % (data_str),index=None, encoding="GB18030")