73 lines
2.4 KiB
Python
73 lines
2.4 KiB
Python
from selenium import webdriver
|
||
from bs4 import BeautifulSoup
|
||
import pandas as pd
|
||
import time
|
||
|
||
def clean_text(html_content):
|
||
# 使用BeautifulSoup来解析内容，逐个元素提取文本，避免添加不必要的空格
|
||
soup = BeautifulSoup(html_content, 'html.parser')
|
||
text = "" # 初始化一个空字符串用于拼接文本
|
||
for element in soup.stripped_strings: # 遍历所有文本节点，去除首尾空白
|
||
if element == "一、" or element == "二、": # 如果是列表标记，加入换行符
|
||
text += "\n" + element
|
||
else:
|
||
text += element # 直接拼接文本，不添加额外空格
|
||
return text.strip() # 返回处理后的文本
|
||
|
||
def fetch_data(urls):
|
||
# 设置Chrome选项以在后台运行
|
||
options = webdriver.ChromeOptions()
|
||
options.add_argument('headless')
|
||
|
||
# 初始化WebDriver
|
||
driver = webdriver.Chrome(options=options)
|
||
|
||
# 初始化一个空的DataFrame，以存储最终数据
|
||
all_data = pd.DataFrame()
|
||
|
||
for url in urls:
|
||
# 访问页面
|
||
driver.get(url)
|
||
time.sleep(3) # 等待JavaScript执行
|
||
|
||
# 获取页面源码
|
||
html = driver.page_source
|
||
|
||
# 使用BeautifulSoup解析HTML
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
table = soup.find('table', class_='MsoNormalTable')
|
||
|
||
# 如果页面上有表格
|
||
if table:
|
||
rows = table.find_all('tr')
|
||
temp_dict = {}
|
||
for row in rows:
|
||
columns = row.find_all('td')
|
||
if len(columns) >= 2: # 确保每行至少有两个列（Header和Content）
|
||
header = columns[0].get_text(strip=True)
|
||
content_html = str(columns[1]) # 获取原始HTML内容
|
||
content = clean_text(content_html) # 清洗并去除不必要的空格
|
||
temp_dict[header] = content
|
||
|
||
# 将字典转换为DataFrame，并添加到总的DataFrame中
|
||
df = pd.DataFrame([temp_dict])
|
||
all_data = pd.concat([all_data, df], ignore_index=True)
|
||
|
||
# 关闭浏览器
|
||
driver.quit()
|
||
|
||
return all_data
|
||
|
||
|
||
# 定义要处理的URL列表
|
||
urls = [
|
||
"https://www.cbirc.gov.cn/cn/view/pages/ItemDetail.html?docId=1171824&itemId=4115&generaltype=9",
|
||
# 可以添加更多的URL
|
||
]
|
||
|
||
# 调用函数并获取数据
|
||
result_data = fetch_data(urls)
|
||
|
||
# 保存到Excel文件
|
||
result_data.to_excel('output_data.xlsx', index=False)
|