142 lines
5.9 KiB
Python
142 lines
5.9 KiB
Python
import json
|
||
import docx
|
||
import re
|
||
import os
|
||
from PyPDF2 import PdfReader
|
||
from flask_app.main.截取pdf import clean_page_content, extract_common_header
|
||
|
||
def extract_text_from_docx(file_path):
|
||
doc = docx.Document(file_path)
|
||
return '\n'.join([para.text for para in doc.paragraphs])
|
||
|
||
|
||
def extract_text_from_pdf(file_path):
|
||
# 从PDF文件中提取文本
|
||
common_header = extract_common_header(file_path)
|
||
pdf_document = PdfReader(file_path)
|
||
text = ""
|
||
# 遍历每一页
|
||
for page in pdf_document.pages:
|
||
# 提取当前页面的文本
|
||
page_text = page.extract_text() if page.extract_text() else ""
|
||
# 清洗页面文本
|
||
page_text = clean_page_content(page_text, common_header)
|
||
# 将清洗后的文本添加到总文本中
|
||
text += page_text + "\n"
|
||
return text
|
||
|
||
def extract_section(text, start_pattern, end_phrases):
|
||
# 查找开始模式
|
||
start_match = re.search(start_pattern, text)
|
||
if not start_match:
|
||
return "" # 如果没有找到匹配的开始模式，返回空字符串
|
||
start_index = start_match.end() # 从匹配的结束位置开始
|
||
|
||
# 初始化结束索引为文本总长度
|
||
end_index = len(text)
|
||
|
||
# 遍历所有结束短语，查找第一个出现的结束短语
|
||
for phrase in end_phrases:
|
||
match = re.search(phrase, text[start_index:], flags=re.MULTILINE)
|
||
if match:
|
||
end_index = start_index + match.start() # 更新结束索引为匹配到的开始位置
|
||
break # 找到第一个匹配后立即停止搜索
|
||
|
||
# 提取并返回从开始模式后到结束模式前的内容
|
||
return text[start_index:end_index]
|
||
|
||
def should_add_newline(content, keywords, max_length=20):
|
||
content_str = ''.join(content).strip()
|
||
return any(keyword in content_str for keyword in keywords) or len(content_str) <= max_length
|
||
|
||
def handle_content_append(current_content, line_content, append_newline, keywords):
|
||
if append_newline:
|
||
if should_add_newline(current_content, keywords):
|
||
current_content.append('\n') # 添加换行符
|
||
append_newline = False
|
||
current_content.append(line_content)
|
||
return append_newline
|
||
|
||
#对二级标题如x.x进行额外处理：如果当前处理内容包含keywords中的内容，则必须保留换行符/如果当前内容字数大于20，不保留换行。
|
||
def parse_text_by_heading(text):
|
||
keywords = ['包含', '以下']
|
||
data = {}
|
||
current_key = None
|
||
current_content = []
|
||
append_newline = False
|
||
|
||
lines = text.split('\n')
|
||
for i, line in enumerate(lines):
|
||
line_stripped = line.strip()
|
||
# 匹配形如 '1.1'、'2.2.3' 等至少包含一个点的标题，并确保其前后没有字母或括号
|
||
match = re.match(r'^(?<![a-zA-Z（(])(\d+(?:\.\d+)+)\s*(.*)', line_stripped)
|
||
if not match:
|
||
match = re.match(r'^(\d+\.)\s*(.+)$', line_stripped)
|
||
|
||
if match:
|
||
new_key, line_content = match.groups()
|
||
line_content = line_content.lstrip('.')
|
||
# 检查是否应该更新当前键和内容
|
||
if current_key is not None:
|
||
# 将之前的内容保存到data中，保留第一个换行符，后续的换行符转为空字符
|
||
content_string = ''.join(current_content).strip()
|
||
data[current_key] = content_string.replace(' ', '')
|
||
current_key = new_key
|
||
current_content = [line_content]
|
||
# 只有当标题级别为两级（如 1.1）时，才设置 append_newline 为 True
|
||
append_newline = len(new_key.split('.')) == 2
|
||
else:
|
||
if line_stripped:
|
||
append_newline = handle_content_append(current_content, line_stripped, append_newline, keywords)
|
||
|
||
if current_key is not None:
|
||
# 保存最后一部分内容
|
||
content_string = ''.join(current_content).strip()
|
||
data[current_key] = content_string.replace(' ', '')
|
||
|
||
return data
|
||
|
||
def convert_to_json(file_path, start_word, end_phrases):
|
||
if file_path.endswith('.docx'):
|
||
text = extract_text_from_docx(file_path)
|
||
elif file_path.endswith('.pdf'):
|
||
text = extract_text_from_pdf(file_path)
|
||
else:
|
||
raise ValueError("Unsupported file format")
|
||
# 提取从 start_word 开始到 end_phrases 结束的内容
|
||
text = extract_section(text, start_word, end_phrases)
|
||
# print(text)
|
||
parsed_data = parse_text_by_heading(text)
|
||
return parsed_data
|
||
|
||
def convert_clause_to_json(input_path, output_folder, type=1):
|
||
if not os.path.exists(input_path):
|
||
print(f"The specified file does not exist: {input_path}")
|
||
return ""
|
||
if type == 1:
|
||
start_word = "第四章"
|
||
end_phrases = [
|
||
"第五章"
|
||
]
|
||
else:
|
||
start_word = r'第[一二三四五六七八九十]+章\s*招标公告|第一卷|招标编号:|招标编号：'
|
||
end_phrases = [r'第[一二三四五六七八九十]+章\s*投标人须知', r'投标人须知前附表']
|
||
result = convert_to_json(input_path, start_word, end_phrases)
|
||
file_name = "clause1.json" if type == 1 else "clause2.json"
|
||
output_path = os.path.join(output_folder, file_name)
|
||
with open(output_path, 'w', encoding='utf-8') as f:
|
||
json.dump(result, f, indent=4, ensure_ascii=False)
|
||
print(f"投标人须知正文条款提取成json文件: The data has been processed and saved to '{output_path}'.")
|
||
return output_path
|
||
|
||
|
||
if __name__ == "__main__":
|
||
# file_path = 'D:\\flask_project\\flask_app\\static\\output\\cfd4959d-5ea9-4112-8b50-9e543803f029\\ztbfile_tobidders_notice.pdf'
|
||
file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output3\\2-招标文件（广水市教育局封闭管理）_qualification1.pdf'
|
||
output_folder = 'C:\\Users\\Administrator\\Desktop\\货物标\\output3\\tmp'
|
||
try:
|
||
output_path = convert_clause_to_json(file_path, output_folder)
|
||
print(f"Final JSON result saved to: {output_path}")
|
||
except ValueError as e:
|
||
print("Error:", e)
|