11.1适配货物标
This commit is contained in:
parent
58328c2f22
commit
75362c2e22
83
flask_app/general/clean_pdf.py
Normal file
83
flask_app/general/clean_pdf.py
Normal file
@ -0,0 +1,83 @@
|
||||
import re
|
||||
|
||||
|
||||
def extract_common_header(pdf_path):
|
||||
from PyPDF2 import PdfReader
|
||||
|
||||
def get_headers(pdf_document, start_page, pages_to_read):
|
||||
headers = []
|
||||
for i in range(start_page, min(start_page + pages_to_read, len(pdf_document.pages))):
|
||||
page = pdf_document.pages[i]
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
# 只取每页的前三行，去除前后的空白字符
|
||||
first_lines = [line.strip() for line in text.strip().split('\n')[:3]]
|
||||
headers.append(first_lines)
|
||||
return headers
|
||||
|
||||
def find_common_headers(headers):
|
||||
if not headers:
|
||||
return []
|
||||
|
||||
# 使用 zip 对齐所有页的对应行
|
||||
common_headers = []
|
||||
for lines in zip(*headers):
|
||||
# 检查所有行是否完全相同
|
||||
if all(line == lines[0] for line in lines[1:]):
|
||||
common_headers.append(lines[0])
|
||||
return common_headers
|
||||
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
total_pages = len(pdf_document.pages)
|
||||
|
||||
# 定义两个提取策略
|
||||
strategies = []
|
||||
if total_pages >= 3:
|
||||
# 策略1：中间的3页
|
||||
middle_page = total_pages // 2
|
||||
start_page = max(0, middle_page - 1)
|
||||
strategies.append((start_page, 3))
|
||||
elif total_pages == 2:
|
||||
# 策略1：2页
|
||||
strategies.append((0, 2))
|
||||
else:
|
||||
# 策略1：1页
|
||||
strategies.append((0, 1))
|
||||
|
||||
# 策略2：前三页
|
||||
if total_pages >= 3:
|
||||
strategies.append((0, 3))
|
||||
elif total_pages == 2:
|
||||
strategies.append((0, 2))
|
||||
elif total_pages == 1:
|
||||
strategies.append((0, 1))
|
||||
|
||||
common_headers = []
|
||||
|
||||
for idx, (start, count) in enumerate(strategies):
|
||||
headers = get_headers(pdf_document, start, count)
|
||||
if len(headers) < 2:
|
||||
continue # 需要至少2页来比较
|
||||
|
||||
current_common = find_common_headers(headers)
|
||||
if current_common:
|
||||
common_headers = current_common
|
||||
break # 找到共同部分后退出
|
||||
# 如果没有找到，继续下一个策略
|
||||
|
||||
return '\n'.join(common_headers)
|
||||
|
||||
def clean_page_content(text, common_header):
|
||||
# 首先删除抬头公共部分
|
||||
if common_header: # 确保有公共抬头才进行替换
|
||||
for header_line in common_header.split('\n'):
|
||||
if header_line.strip(): # 只处理非空行
|
||||
# 替换首次出现的完整行
|
||||
text = re.sub(r'^' + re.escape(header_line.strip()) + r'\n?', '', text, count=1)
|
||||
|
||||
# 删除页码 eg:89/129 这个代码分三步走可以把89/129完全删除
|
||||
text = re.sub(r'^\s*\d+\s*(?=\D)', '', text) # 删除开头的页码，仅当紧跟非数字字符时
|
||||
text = re.sub(r'\s+\d+\s*$', '', text) # 删除结尾的页码
|
||||
text = re.sub(r'\s*\/\s*\d+\s*', '', text) # 删除形如 /129 的页码
|
||||
text = re.sub(r'\s*[—-]\s*\d+\s*[—-]\s*', '', text) # 删除形如 '—2—' 或 '-2-' 的页码
|
||||
return text
|
||||
@ -33,12 +33,18 @@ def upload_file(file_path, url):
|
||||
|
||||
return receive_file_url
|
||||
|
||||
|
||||
def get_filename_and_folder(file_path):
|
||||
# 使用os.path.basename获取文件名
|
||||
filename = os.path.splitext(os.path.basename(file_path))[0] #ztb_tobidders_notice_table,不包括扩展名
|
||||
# 使用os.path.dirname获取文件所在的完整目录路径，再次使用basename获取文件夹名称
|
||||
original_filename = os.path.basename(file_path)
|
||||
filename_without_ext, ext = os.path.splitext(original_filename)
|
||||
|
||||
# 替换文件名中的点为下划线，避免分割问题
|
||||
safe_filename = filename_without_ext.replace('.', '_')
|
||||
|
||||
# 使用os.path.dirname获取文件所在的完整目录路径
|
||||
directory = os.path.dirname(file_path)
|
||||
return filename, directory
|
||||
return safe_filename, directory
|
||||
|
||||
#参数为要转换的文件路径，以及输出的文件名，文件类型为自动补全。
|
||||
def pdf2docx(local_path_in):
|
||||
@ -53,121 +59,121 @@ def pdf2docx(local_path_in):
|
||||
print(f"format_change p2d:have downloaded file to: {downloaded_filepath}")
|
||||
return downloaded_filepath
|
||||
|
||||
# def doc2docx(local_path_in):
|
||||
# if not local_path_in:
|
||||
# return ""
|
||||
# remote_url = 'http://47.98.59.178:8000/v3/3rd/files/transfer/d2d'
|
||||
# receive_download_url = upload_file(local_path_in, remote_url)
|
||||
# print(receive_download_url)
|
||||
# filename, folder = get_filename_and_folder(local_path_in) # 输入输出在同一个文件夹
|
||||
# local_filename = os.path.join(folder, filename) # 输出文件名
|
||||
# downloaded_filepath, file_type = download_file(receive_download_url, local_filename)
|
||||
# print(f"format_change d2d:have downloaded file to: {downloaded_filepath}")
|
||||
# return downloaded_filepath
|
||||
# def docx2pdf(local_path_in):
|
||||
# if not local_path_in:
|
||||
# return ""
|
||||
# remote_url = 'http://47.98.59.178:8000/v3/3rd/files/transfer/d2p'
|
||||
# receive_download_url = upload_file(local_path_in, remote_url)
|
||||
# filename, folder = get_filename_and_folder(local_path_in) # 输入输出在同一个文件夹
|
||||
# local_filename = os.path.join(folder, filename) # 输出文件名
|
||||
# downloaded_filepath,file_type = download_file(receive_download_url, local_filename)
|
||||
# print(f"format_change d2p:have downloaded file to: {downloaded_filepath}")
|
||||
# return downloaded_filepath
|
||||
|
||||
def docx2pdf(file_path):
|
||||
"""
|
||||
将本地的 .docx 或 .doc 文件转换为 .pdf 文件。
|
||||
|
||||
参数：
|
||||
- file_path: str, 本地文件的路径，支持 .docx 和 .doc 格式。
|
||||
"""
|
||||
# 检查文件是否存在
|
||||
if not file_path:
|
||||
def doc2docx(local_path_in):
|
||||
if not local_path_in:
|
||||
return ""
|
||||
# 获取文件名和扩展名
|
||||
base_name = os.path.basename(file_path)
|
||||
name, ext = os.path.splitext(base_name)
|
||||
ext = ext.lower().lstrip('.')
|
||||
|
||||
if ext not in ['docx', 'doc']:
|
||||
raise ValueError(f"doc2pdf 仅支持 .docx 和 .doc 文件，当前文件扩展名为: .{ext}")
|
||||
|
||||
# 定义转换接口
|
||||
endpoint = 'http://120.26.236.97:5008/convert_to_pdf'
|
||||
# endpoint = 'http://192.168.0.2:5008/convert_to_pdf'
|
||||
|
||||
# 获取文件所在目录
|
||||
output_dir = os.path.dirname(file_path)
|
||||
|
||||
# 准备上传的文件
|
||||
with open(file_path, 'rb') as f:
|
||||
files = {'file': (base_name, f)}
|
||||
try:
|
||||
print(f"正在将 {base_name} 转换为 .pdf 格式...")
|
||||
response = requests.post(endpoint, files=files)
|
||||
response.raise_for_status() # 检查请求是否成功
|
||||
except requests.RequestException as e:
|
||||
print(f"转换过程中发生错误: {e}")
|
||||
return
|
||||
|
||||
# 准备保存转换后文件的路径
|
||||
output_file_name = f"{name}.pdf"
|
||||
output_path = os.path.join(output_dir, output_file_name)
|
||||
|
||||
# 保存转换后的文件
|
||||
with open(output_path, 'wb') as out_file:
|
||||
out_file.write(response.content)
|
||||
|
||||
print(f"文件已成功转换并保存至: {output_path}")
|
||||
return output_path
|
||||
|
||||
|
||||
def doc2docx(file_path):
|
||||
"""
|
||||
将本地的 .doc 文件转换为 .docx 文件。
|
||||
|
||||
参数：
|
||||
- file_path: str, 本地文件的路径，支持 .doc 格式。
|
||||
"""
|
||||
# 检查文件是否存在
|
||||
if not file_path:
|
||||
remote_url = 'http://47.98.59.178:8000/v3/3rd/files/transfer/d2d'
|
||||
receive_download_url = upload_file(local_path_in, remote_url)
|
||||
print(receive_download_url)
|
||||
filename, folder = get_filename_and_folder(local_path_in) # 输入输出在同一个文件夹
|
||||
local_filename = os.path.join(folder, filename) # 输出文件名
|
||||
downloaded_filepath, file_type = download_file(receive_download_url, local_filename)
|
||||
print(f"format_change d2d:have downloaded file to: {downloaded_filepath}")
|
||||
return downloaded_filepath
|
||||
def docx2pdf(local_path_in):
|
||||
if not local_path_in:
|
||||
return ""
|
||||
# 获取文件名和扩展名
|
||||
base_name = os.path.basename(file_path)
|
||||
name, ext = os.path.splitext(base_name)
|
||||
ext = ext.lower().lstrip('.')
|
||||
remote_url = 'http://47.98.59.178:8000/v3/3rd/files/transfer/d2p'
|
||||
receive_download_url = upload_file(local_path_in, remote_url)
|
||||
filename, folder = get_filename_and_folder(local_path_in) # 输入输出在同一个文件夹
|
||||
local_filename = os.path.join(folder, filename) # 输出文件名
|
||||
downloaded_filepath,file_type = download_file(receive_download_url, local_filename)
|
||||
print(f"format_change d2p:have downloaded file to: {downloaded_filepath}")
|
||||
return downloaded_filepath
|
||||
|
||||
if ext != 'doc':
|
||||
raise ValueError(f"doc2docx 仅支持 .doc 文件，当前文件扩展名为: .{ext}")
|
||||
|
||||
# 定义转换接口
|
||||
endpoint = 'http://120.26.236.97:5008/convert_to_docx'
|
||||
|
||||
# 获取文件所在目录
|
||||
output_dir = os.path.dirname(file_path)
|
||||
|
||||
# 准备上传的文件
|
||||
with open(file_path, 'rb') as f:
|
||||
files = {'file': (base_name, f)}
|
||||
try:
|
||||
print(f"正在将 {base_name} 转换为 .docx 格式...")
|
||||
response = requests.post(endpoint, files=files)
|
||||
response.raise_for_status() # 检查请求是否成功
|
||||
except requests.RequestException as e:
|
||||
print(f"转换过程中发生错误: {e}")
|
||||
return
|
||||
|
||||
# 准备保存转换后文件的路径
|
||||
output_file_name = f"{name}.docx"
|
||||
output_path = os.path.join(output_dir, output_file_name)
|
||||
|
||||
# 保存转换后的文件
|
||||
with open(output_path, 'wb') as out_file:
|
||||
out_file.write(response.content)
|
||||
|
||||
print(f"文件已成功转换并保存至: {output_path}")
|
||||
return output_path
|
||||
# def docx2pdf(file_path):
|
||||
# """
|
||||
# 将本地的 .docx 或 .doc 文件转换为 .pdf 文件。
|
||||
#
|
||||
# 参数：
|
||||
# - file_path: str, 本地文件的路径，支持 .docx 和 .doc 格式。
|
||||
# """
|
||||
# # 检查文件是否存在
|
||||
# if not file_path:
|
||||
# return ""
|
||||
# # 获取文件名和扩展名
|
||||
# base_name = os.path.basename(file_path)
|
||||
# name, ext = os.path.splitext(base_name)
|
||||
# ext = ext.lower().lstrip('.')
|
||||
#
|
||||
# if ext not in ['docx', 'doc']:
|
||||
# raise ValueError(f"doc2pdf 仅支持 .docx 和 .doc 文件，当前文件扩展名为: .{ext}")
|
||||
#
|
||||
# # 定义转换接口
|
||||
# endpoint = 'http://120.26.236.97:5008/convert_to_pdf'
|
||||
# # endpoint = 'http://192.168.0.2:5008/convert_to_pdf'
|
||||
#
|
||||
# # 获取文件所在目录
|
||||
# output_dir = os.path.dirname(file_path)
|
||||
#
|
||||
# # 准备上传的文件
|
||||
# with open(file_path, 'rb') as f:
|
||||
# files = {'file': (base_name, f)}
|
||||
# try:
|
||||
# print(f"正在将 {base_name} 转换为 .pdf 格式...")
|
||||
# response = requests.post(endpoint, files=files)
|
||||
# response.raise_for_status() # 检查请求是否成功
|
||||
# except requests.RequestException as e:
|
||||
# print(f"转换过程中发生错误: {e}")
|
||||
# return
|
||||
#
|
||||
# # 准备保存转换后文件的路径
|
||||
# output_file_name = f"{name}.pdf"
|
||||
# output_path = os.path.join(output_dir, output_file_name)
|
||||
#
|
||||
# # 保存转换后的文件
|
||||
# with open(output_path, 'wb') as out_file:
|
||||
# out_file.write(response.content)
|
||||
#
|
||||
# print(f"文件已成功转换并保存至: {output_path}")
|
||||
# return output_path
|
||||
#
|
||||
#
|
||||
# def doc2docx(file_path):
|
||||
# """
|
||||
# 将本地的 .doc 文件转换为 .docx 文件。
|
||||
#
|
||||
# 参数：
|
||||
# - file_path: str, 本地文件的路径，支持 .doc 格式。
|
||||
# """
|
||||
# # 检查文件是否存在
|
||||
# if not file_path:
|
||||
# return ""
|
||||
# # 获取文件名和扩展名
|
||||
# base_name = os.path.basename(file_path)
|
||||
# name, ext = os.path.splitext(base_name)
|
||||
# ext = ext.lower().lstrip('.')
|
||||
#
|
||||
# if ext != 'doc':
|
||||
# raise ValueError(f"doc2docx 仅支持 .doc 文件，当前文件扩展名为: .{ext}")
|
||||
#
|
||||
# # 定义转换接口
|
||||
# endpoint = 'http://120.26.236.97:5008/convert_to_docx'
|
||||
#
|
||||
# # 获取文件所在目录
|
||||
# output_dir = os.path.dirname(file_path)
|
||||
#
|
||||
# # 准备上传的文件
|
||||
# with open(file_path, 'rb') as f:
|
||||
# files = {'file': (base_name, f)}
|
||||
# try:
|
||||
# print(f"正在将 {base_name} 转换为 .docx 格式...")
|
||||
# response = requests.post(endpoint, files=files)
|
||||
# response.raise_for_status() # 检查请求是否成功
|
||||
# except requests.RequestException as e:
|
||||
# print(f"转换过程中发生错误: {e}")
|
||||
# return
|
||||
#
|
||||
# # 准备保存转换后文件的路径
|
||||
# output_file_name = f"{name}.docx"
|
||||
# output_path = os.path.join(output_dir, output_file_name)
|
||||
#
|
||||
# # 保存转换后的文件
|
||||
# with open(output_path, 'wb') as out_file:
|
||||
# out_file.write(response.content)
|
||||
#
|
||||
# print(f"文件已成功转换并保存至: {output_path}")
|
||||
# return output_path
|
||||
|
||||
|
||||
|
||||
|
||||
@ -156,7 +156,7 @@ def merge_selected_pdfs_for_engineering(output_folder, truncate_files, output_pa
|
||||
if os.path.exists(before_pdf_path):
|
||||
try:
|
||||
os.remove(before_pdf_path)
|
||||
print(f"已删除文件: {before_pdf_path}")
|
||||
# print(f"已删除文件: {before_pdf_path}")
|
||||
except Exception as e:
|
||||
print(f"删除文件 {before_pdf_path} 时出错: {e}")
|
||||
else:
|
||||
@ -224,8 +224,8 @@ def merge_selected_pdfs_for_goods(output_folder, truncate_files, output_path, ba
|
||||
# 如果找到多个匹配的文件，按名称排序并添加
|
||||
matching_files_sorted = sorted(matching_files)
|
||||
all_pdfs_to_merge.extend(matching_files_sorted)
|
||||
for f in matching_files_sorted:
|
||||
print(f"选中文件: {f}")
|
||||
# for f in matching_files_sorted:
|
||||
# print(f"选中文件: {f}")
|
||||
else:
|
||||
print(f"没有找到以 '{suffix}' 结尾的文件。")
|
||||
missing_files.append(suffix) # 记录缺失的文件
|
||||
|
||||
@ -1,65 +1,10 @@
|
||||
import PyPDF2
|
||||
import re # 导入正则表达式库
|
||||
|
||||
from PyPDF2 import PdfReader
|
||||
|
||||
|
||||
def clean_page_content(text, common_header):
|
||||
# 首先删除抬头公共部分
|
||||
if common_header: # 确保有公共抬头才进行替换
|
||||
for header_line in common_header.split('\n'):
|
||||
if header_line.strip(): # 只处理非空行
|
||||
# 替换首次出现的完整行
|
||||
text = re.sub(r'^' + re.escape(header_line.strip()) + r'\n?', '', text, count=1)
|
||||
|
||||
# 删除页码 eg:89/129 这个代码分三步走可以把89/129完全删除
|
||||
text = re.sub(r'^\s*\d+\s*(?=\D)', '', text) # 删除开头的页码，仅当紧跟非数字字符时
|
||||
text = re.sub(r'\s+\d+\s*$', '', text) # 删除结尾的页码
|
||||
text = re.sub(r'\s*\/\s*\d+\s*', '', text) # 删除形如 /129 的页码
|
||||
text = re.sub(r'\s*[—-]\s*\d+\s*[—-]\s*', '', text) # 删除形如 '—2—' 或 '-2-' 的页码
|
||||
return text
|
||||
def extract_common_header(pdf_path):
|
||||
from PyPDF2 import PdfReader
|
||||
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
headers = []
|
||||
total_pages = len(pdf_document.pages)
|
||||
|
||||
# 确定要读取的页数和起始页
|
||||
if total_pages == 2:
|
||||
pages_to_read = 2
|
||||
start_page = 0
|
||||
else:
|
||||
pages_to_read = 3
|
||||
middle_page = total_pages // 2
|
||||
start_page = max(0, middle_page - 1)
|
||||
|
||||
for i in range(start_page, min(start_page + pages_to_read, total_pages)):
|
||||
page = pdf_document.pages[i]
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
# 只取每页的前三行
|
||||
first_lines = text.strip().split('\n')[:3]
|
||||
headers.append(first_lines)
|
||||
|
||||
if len(headers) < 2:
|
||||
return "" # 如果没有足够的页来比较，返回空字符串
|
||||
|
||||
# 寻找每一行中的公共部分，按顺序保留
|
||||
common_headers = []
|
||||
for lines in zip(*headers):
|
||||
# 提取第一行的词汇顺序
|
||||
first_words = lines[0].split()
|
||||
# 筛选所有页面都包含的词汇，保持顺序
|
||||
common_line = [word for word in first_words if all(word in line.split() for line in lines[1:])]
|
||||
if common_line:
|
||||
common_headers.append(' '.join(common_line))
|
||||
|
||||
return '\n'.join(common_headers)
|
||||
|
||||
from flask_app.general.clean_pdf import extract_common_header, clean_page_content
|
||||
|
||||
def extract_text_by_page(file_path):
|
||||
common_header = extract_common_header(file_path)
|
||||
print(f"公共抬头：{common_header}")
|
||||
print("--------------------正文开始-------------------")
|
||||
result = ""
|
||||
with open(file_path, 'rb') as file:
|
||||
reader = PyPDF2.PdfReader(file)
|
||||
@ -150,7 +95,7 @@ def extract_text_by_page(file_path):
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
file_path = "C:\\Users\\Administrator\\Desktop\\new招标文件\\招标文件\\HBDL-2024-0181-001-招标文件.pdf"
|
||||
input_path="C:\\Users\\Administrator\\Desktop\\招标文件\\招标test文件夹\\zbtest8.pdf"
|
||||
# file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\2-招标文件（2020年广水市中小学教师办公电脑系统及多媒体“班班通”设备采购安装项目）_tobidders_notice_part2.pdf'
|
||||
# file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\磋商文件_tobidders_notice_part2.pdf'
|
||||
# file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\截取test\\交警支队机动车查验监管系统项目采购_tobidders_notice_part1.pdf'
|
||||
@ -158,5 +103,5 @@ if __name__ == '__main__':
|
||||
# file_path="C:\\Users\\Administrator\\Desktop\\fsdownload\\45f650ce-e519-457b-9ad6-5840e2ede539\\ztbfile_procurement.pdf"
|
||||
# ress = extract_common_header(file_path)
|
||||
# print(ress)
|
||||
res=extract_text_by_page(file_path)
|
||||
res=extract_text_by_page(input_path)
|
||||
# print(res)磋商文件_tobidders_notice_part2.pdf
|
||||
@ -1,71 +1,22 @@
|
||||
import re
|
||||
import os
|
||||
import time
|
||||
|
||||
from PyPDF2 import PdfReader, PdfWriter
|
||||
from flask_app.general.merge_pdfs import merge_pdfs, merge_selected_pdfs_for_engineering
|
||||
from flask_app.general.clean_pdf import clean_page_content, extract_common_header
|
||||
from flask_app.general.merge_pdfs import merge_selected_pdfs_for_engineering
|
||||
import concurrent.futures
|
||||
import logging
|
||||
|
||||
|
||||
def get_global_logger(unique_id):
|
||||
if unique_id is None:
|
||||
return logging.getLogger() # 获取默认的日志器
|
||||
logger = logging.getLogger(unique_id)
|
||||
return logger
|
||||
|
||||
|
||||
logger = None
|
||||
|
||||
def clean_page_content(text, common_header):
|
||||
# 首先删除抬头公共部分
|
||||
if common_header: # 确保有公共抬头才进行替换
|
||||
for header_line in common_header.split('\n'):
|
||||
if header_line.strip(): # 只处理非空行
|
||||
# 替换首次出现的完整行
|
||||
text = re.sub(r'^' + re.escape(header_line.strip()) + r'\n?', '', text, count=1)
|
||||
|
||||
# 删除页码 eg:89/129 这个代码分三步走可以把89/129完全删除
|
||||
text = re.sub(r'^\s*\d+\s*(?=\D)', '', text) # 删除开头的页码，仅当紧跟非数字字符时
|
||||
text = re.sub(r'\s+\d+\s*$', '', text) # 删除结尾的页码
|
||||
text = re.sub(r'\s*\/\s*\d+\s*', '', text) # 删除形如 /129 的页码
|
||||
text = re.sub(r'\s*[—-]\s*\d+\s*[—-]\s*', '', text) # 删除形如 '—2—' 或 '-2-' 的页码
|
||||
return text
|
||||
|
||||
def extract_common_header(pdf_path):
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
headers = []
|
||||
total_pages = len(pdf_document.pages)
|
||||
|
||||
# 确定要读取的页数和起始页
|
||||
if total_pages == 2:
|
||||
pages_to_read = 2
|
||||
start_page = 0
|
||||
else:
|
||||
pages_to_read = 3
|
||||
middle_page = total_pages // 2
|
||||
start_page = max(0, middle_page - 1)
|
||||
|
||||
for i in range(start_page, min(start_page + pages_to_read, total_pages)):
|
||||
page = pdf_document.pages[i]
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
# 只取每页的前三行
|
||||
first_lines = text.strip().split('\n')[:3]
|
||||
headers.append(first_lines)
|
||||
|
||||
if len(headers) < 2:
|
||||
return "" # 如果没有足够的页来比较，返回空字符串
|
||||
|
||||
# 寻找每一行中的公共部分，按顺序保留
|
||||
common_headers = []
|
||||
for lines in zip(*headers):
|
||||
# 提取第一行的词汇顺序
|
||||
first_words = lines[0].split()
|
||||
# 筛选所有页面都包含的词汇，保持顺序
|
||||
common_line = [word for word in first_words if all(word in line.split() for line in lines[1:])]
|
||||
if common_line:
|
||||
common_headers.append(' '.join(common_line))
|
||||
|
||||
return '\n'.join(common_headers)
|
||||
|
||||
def save_pages_to_new_pdf(pdf_path, output_folder, output_suffix, start_page, end_page, common_header):
|
||||
try:
|
||||
@ -122,9 +73,12 @@ def save_pages_to_new_pdf(pdf_path, output_folder, output_suffix, start_page, en
|
||||
print(f"Error in save_pages_to_new_pdf: {e}")
|
||||
return "" # 返回空字符串
|
||||
|
||||
def extract_pages_tobidders_notice(pdf_path, output_folder,begin_pattern, begin_page,common_header,is_secondary_match):
|
||||
|
||||
def extract_pages_tobidders_notice(pdf_path, output_folder, begin_pattern, begin_page, common_header,
|
||||
is_secondary_match):
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
exclusion_pattern = re.compile(r'文件的构成|文件的组成|须对应|需对应|须按照|需按照|须根据|需根据')
|
||||
|
||||
def run_extraction():
|
||||
start_page = None
|
||||
mid_page = None
|
||||
@ -154,10 +108,10 @@ def extract_pages_tobidders_notice(pdf_path, output_folder,begin_pattern, begin_
|
||||
# 根据 chapter_type 动态生成 end_pattern
|
||||
if not is_secondary_match:
|
||||
end_pattern = re.compile(
|
||||
rf'^第[一二三四五六七八九十百千]+?(?:{chapter_type})\s*[\u4e00-\u9fff]+|'
|
||||
r'^评标办法前附表|'
|
||||
r'^附录(?:一)?[：:]|'
|
||||
r'^附件(?:一)?[：:]|'
|
||||
rf'^第[一二三四五六七八九十百千]+?(?:{chapter_type})\s*[\u4e00-\u9fff]+|'
|
||||
r'^评标办法前附表|'
|
||||
r'^附录(?:一)?[：:]|'
|
||||
r'^附件(?:一)?[：:]|'
|
||||
r'^附表(?:一)?[：:]',
|
||||
re.MULTILINE
|
||||
)
|
||||
@ -229,26 +183,29 @@ def extract_pages_tobidders_notice(pdf_path, output_folder,begin_pattern, begin_
|
||||
start_page, mid_page, end_page = run_extraction()
|
||||
if start_page is None or end_page is None or mid_page is None:
|
||||
print(f"first: tobidders_notice 未找到起始或结束页在文件 {pdf_path} 中！尝试备用提取策略。")
|
||||
return "",""
|
||||
return "", ""
|
||||
|
||||
path1 = save_pages_to_new_pdf(pdf_path, output_folder, "tobidders_notice_table", start_page,mid_page, common_header)
|
||||
path2 = save_pages_to_new_pdf(pdf_path, output_folder, "tobidders_notice", mid_page,end_page, common_header)
|
||||
return path1,path2
|
||||
path1 = save_pages_to_new_pdf(pdf_path, output_folder, "tobidders_notice_table", start_page, mid_page,
|
||||
common_header)
|
||||
path2 = save_pages_to_new_pdf(pdf_path, output_folder, "tobidders_notice", mid_page, end_page, common_header)
|
||||
return path1, path2
|
||||
|
||||
|
||||
|
||||
def extract_pages(pdf_path, output_folder, begin_pattern, begin_page, end_pattern, output_suffix,common_header,is_secondary_match=False):
|
||||
def extract_pages(pdf_path, output_folder, begin_pattern, begin_page, end_pattern, output_suffix, common_header,
|
||||
is_secondary_match=False):
|
||||
# 打开PDF文件
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
start_page = None
|
||||
end_page = None
|
||||
|
||||
exclusion_pattern = re.compile(r'文件的构成|文件的组成|须对应|需对应|须按照|需按照|须根据|需根据')
|
||||
# 遍历文档的每一页，查找开始和结束短语的位置
|
||||
for i in range(len(pdf_document.pages)):
|
||||
page = pdf_document.pages[i]
|
||||
text = page.extract_text()
|
||||
if text:
|
||||
cleaned_text = clean_page_content(text, common_header)
|
||||
if is_secondary_match and re.search(exclusion_pattern, cleaned_text): # 跳过投标人须知正文中的"投标文件的组成"
|
||||
continue
|
||||
if re.search(begin_pattern, cleaned_text) and i >= begin_page:
|
||||
if start_page and (output_suffix == "notice" or output_suffix == "invalid"):
|
||||
pass
|
||||
@ -269,28 +226,59 @@ def extract_pages(pdf_path, output_folder, begin_pattern, begin_page, end_patter
|
||||
else:
|
||||
return [save_pages_to_new_pdf(pdf_path, output_folder, output_suffix, start_page, end_page, common_header)]
|
||||
|
||||
# def process_input(input_path, output_folder, begin_pattern, begin_page, end_pattern, output_suffix,common_header,is_secondary_match):
|
||||
# # 确保输出文件夹存在
|
||||
# if not os.path.exists(output_folder):
|
||||
# os.makedirs(output_folder)
|
||||
# if os.path.isdir(input_path):
|
||||
# generated_files = []
|
||||
# # 遍历文件夹内的所有PDF文件
|
||||
# for file in os.listdir(input_path):
|
||||
# if file.endswith(".pdf"):
|
||||
# pdf_path = os.path.join(input_path, file)
|
||||
# output_pdf_path = extract_pages(pdf_path, output_folder, begin_pattern, begin_page, end_pattern, output_suffix,common_header,is_secondary_match)
|
||||
# if output_pdf_path and os.path.isfile(output_pdf_path):
|
||||
# generated_files.append(output_pdf_path)
|
||||
# return generated_files
|
||||
# elif os.path.isfile(input_path) and input_path.endswith(".pdf"):
|
||||
# # 处理单个PDF文件
|
||||
# output_pdf_path = extract_pages(input_path, output_folder, begin_pattern, begin_page, end_pattern, output_suffix,common_header,is_secondary_match)
|
||||
# if output_pdf_path and os.path.isfile(output_pdf_path):
|
||||
# return [output_pdf_path] # 以列表形式返回，以保持一致性
|
||||
# else:
|
||||
# print("提供的路径既不是文件夹也不是PDF文件。")
|
||||
# return []
|
||||
|
||||
def extract_pages_twice(pdf_path, output_folder, output_suffix, common_header, last_begin_index):
|
||||
"""
|
||||
处理 PDF 文件，作为 truncate_pdf_main 的后备方法。
|
||||
此函数将在所有模式对都失败后调用。
|
||||
"""
|
||||
try:
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
if output_suffix == "qualification":
|
||||
begin_pattern = r'^(?:附录(?:[一1])?[：:]|附件(?:[一1])?[：:]|附表(?:[一1])?[：:])'
|
||||
end_pattern = re.compile(
|
||||
r'^(?:附录[一二三四五六七八九1-9]*[：:]|附件[一二三四五六七八九1-9]*[：:]|附表[一二三四五六七八九1-9]*[：:])(?!.*(?:资质|能力|信誉)).*$|'
|
||||
r'第[一二三四五六七八九]+(?:章|部分)\s*[\u4e00-\u9fff、]\s*$|\s*评标(办法|方法)前附表\s*$|投标人须知',
|
||||
re.MULTILINE
|
||||
)
|
||||
start_page = None
|
||||
end_page = None
|
||||
# 从章节开始后的位置进行检查
|
||||
for i, page in enumerate(pdf_document.pages[last_begin_index:], start=last_begin_index):
|
||||
text = page.extract_text()
|
||||
if text:
|
||||
cleaned_text = clean_page_content(text, common_header)
|
||||
# 确定起始页，需在last_begin_index之后
|
||||
if "资格" in cleaned_text or "资质" in cleaned_text: # 当页包含这些内容可以认为是'资格审查资料部分'
|
||||
if re.search(begin_pattern, cleaned_text, re.MULTILINE):
|
||||
if start_page is None:
|
||||
start_page = i # 确保起始页不小于章节的开始页码
|
||||
# 确定结束页
|
||||
if start_page is not None and re.search(end_pattern, cleaned_text):
|
||||
if i > start_page:
|
||||
end_page = i
|
||||
break # 找到结束页后退出循环
|
||||
|
||||
if start_page is None or end_page is None:
|
||||
print(f"{output_suffix} twice: 未找到起始或结束页在文件 {pdf_path} 中！")
|
||||
return []
|
||||
else:
|
||||
return [
|
||||
save_pages_to_new_pdf(pdf_path, output_folder, output_suffix, start_page, end_page, common_header)]
|
||||
elif output_suffix == "invalid":
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
total_pages = len(pdf_document.pages)
|
||||
# 计算总页数的三分之二
|
||||
total = int(total_pages * 2 / 3)
|
||||
start_page = last_begin_index
|
||||
end_page = min(90, total)
|
||||
return [save_pages_to_new_pdf(pdf_path, output_folder, output_suffix, start_page, end_page, common_header)]
|
||||
else:
|
||||
print(f"{output_suffix} twice: 未定义的输出后缀。")
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f"Error in extract_pages_twice: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def get_start_and_common_header(input_path):
|
||||
@ -309,8 +297,9 @@ def get_start_and_common_header(input_path):
|
||||
cleaned_text = clean_page_content(text, common_header)
|
||||
if begin_pattern.search(cleaned_text):
|
||||
last_begin_index = i # 更新第一个匹配的索引，页码从0开始
|
||||
return common_header,last_begin_index
|
||||
return last_begin_index
|
||||
return common_header, last_begin_index
|
||||
return common_header, last_begin_index
|
||||
|
||||
|
||||
def truncate_pdf_main(input_path, output_folder, selection):
|
||||
if os.path.isdir(input_path):
|
||||
@ -324,7 +313,7 @@ def truncate_pdf_main(input_path, output_folder, selection):
|
||||
return generated_files
|
||||
elif os.path.isfile(input_path) and input_path.endswith('.pdf'):
|
||||
# base_file_name = os.path.splitext(os.path.basename(input_path))[0]
|
||||
common_header,last_begin_index = get_start_and_common_header(input_path)
|
||||
common_header, last_begin_index = get_start_and_common_header(input_path)
|
||||
# print(last_begin_index)
|
||||
if selection == 1:
|
||||
# Selection 1: 投标人须知前附表
|
||||
@ -338,39 +327,50 @@ def truncate_pdf_main(input_path, output_folder, selection):
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r'.*(?:投标人?|磋商|供应商|谈判供应商|磋商供应商)须知\s*$|(?:一\s*、\s*)?(?:投标人?|磋商|供应商)须知前附表', re.MULTILINE),
|
||||
r'.*(?:投标人?|磋商|供应商|谈判供应商|磋商供应商)须知\s*$|(?:一\s*、\s*)?(?:投标人?|磋商|供应商)须知前附表',
|
||||
re.MULTILINE),
|
||||
re.compile(
|
||||
r'第[一二三四五六七八九十]+(?:章|部分)\s*[\u4e00-\u9fff]+|^评标办法前附表|^附录(?:一)?[：:]|^附件(?:一)?[：:]|^附表(?:一)?[：:]',
|
||||
re.MULTILINE)
|
||||
)
|
||||
]
|
||||
output_suffix="tobidders_notice"
|
||||
output_suffix = "tobidders_notice"
|
||||
elif selection == 2:
|
||||
# Selection 2: 评标办法
|
||||
pattern_pairs = [
|
||||
(
|
||||
re.compile(r'第[一二三四五六七八九十]+(?:章|部分)\s*评标办法'), # Alternative begin pattern
|
||||
re.compile(r'评标办法正文|评标办法') # Alternative end pattern
|
||||
re.compile(r'^第[一二三四五六七八九十]+(?:章|部分)\s*(磋商|谈判|评标|评定|评审)'),
|
||||
# Alternative begin pattern
|
||||
re.compile(r'评标办法正文|评标办法|^第[一二三四五六七八九1-9]+(?:章|部分)\s*[\u4e00-\u9fff]+')
|
||||
# Alternative end pattern
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r'第[一二三四五六七八九1-9]+(?:章|部分)\s*[\u4e00-\u9fff、]*?(磋商|谈判|评标|评定|评审)[\u4e00-\u9fff、]*\s*$|\s*评标(办法|方法)前附表\s*$',
|
||||
re.MULTILINE
|
||||
),
|
||||
re.compile(r'第[一二三四五六七八九1-9]+(?:章|部分)\s*[\u4e00-\u9fff]+\s*$', re.MULTILINE)
|
||||
)
|
||||
]
|
||||
output_suffix = "evaluation_method"
|
||||
#TODO:exclusion
|
||||
# TODO:exclusion
|
||||
elif selection == 3:
|
||||
# Selection 3: 资格审查条件
|
||||
pattern_pairs = [
|
||||
(
|
||||
re.compile(r'^(?:附录(?:[一1])?[：:]|附件(?:[一1])?[：:]|附表(?:[一1])?[：:]).*(?:资质|能力|信誉).*$|^第[一二三四五六七八九十百千]+(?:章|部分).*?(资格审查).*',
|
||||
re.MULTILINE),
|
||||
re.compile(
|
||||
r'^(?:附录[一二三四五六七八九1-9]*[：:]|附件[一二三四五六七八九1-9]*[：:]|附表[一二三四五六七八九1-9]*[：:])(?!.*(?:资质|能力|信誉)).*$'
|
||||
r'^第[一二三四五六七八九1-9]+(?:章|部分)\s*[\u4e00-\u9fff]+', re.MULTILINE)
|
||||
),
|
||||
# (
|
||||
# re.compile(r'^(?:附录(?:[一1])?[：:]|附件(?:[一1])?[：:]|附表(?:[一1])?[：:]).*(?:资质|能力|信誉).*$|^第[一二三四五六七八九十百千]+(?:章|部分).*?(资格).*',
|
||||
# re.MULTILINE),
|
||||
# re.compile(
|
||||
# r'^(?:附录[一二三四五六七八九1-9]*[：:]|附件[一二三四五六七八九1-9]*[：:]|附表[一二三四五六七八九1-9]*[：:])(?!.*(?:资质|能力|信誉)).*$'
|
||||
# r'^第[一二三四五六七八九1-9]+(?:章|部分)\s*[\u4e00-\u9fff]+', re.MULTILINE)
|
||||
# ),
|
||||
(
|
||||
re.compile(
|
||||
r'第[一二三四五六七八九1-9]+(?:章|部分)\s*[\u4e00-\u9fff]*资格[\u4e00-\u9fff]*\s*$',re.MULTILINE),
|
||||
r'^(?:附录(?:[一1])?[：:]|附件(?:[一1])?[：:]|附表(?:[一1])?[：:]).*(?:资质|能力|信誉).*$|第[一二三四五六七八九1-9]+(?:章|部分)\s*[\u4e00-\u9fff]*资格[\u4e00-\u9fff]*\s*$',
|
||||
re.MULTILINE),
|
||||
re.compile(
|
||||
r'^(?:附录[一二三四五六七八九1-9]*[：:]|附件[一二三四五六七八九1-9]*[：:]|附表[一二三四五六七八九1-9]*[：:])(?!.*(?:资质|能力|信誉)).*|'
|
||||
r'第[一二三四五六七八九1-9]+(?:章|部分)\s*[\u4e00-\u9fff]+\s*$',re.MULTILINE)
|
||||
r'第[一二三四五六七八九1-9]+(?:章|部分)\s*[\u4e00-\u9fff、]+\s*$', re.MULTILINE)
|
||||
)
|
||||
]
|
||||
output_suffix = "qualification"
|
||||
@ -383,7 +383,7 @@ def truncate_pdf_main(input_path, output_folder, selection):
|
||||
re.compile(r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+', re.MULTILINE)
|
||||
),
|
||||
(
|
||||
re.compile(r'.*(?:招标公告|投标邀请书|投标邀请函|投标邀请)[\)）]?\s*$',re.MULTILINE),
|
||||
re.compile(r'.*(?:招标公告|投标邀请书|投标邀请函|投标邀请)[\)）]?\s*$', re.MULTILINE),
|
||||
re.compile(r".*(?:投标人须知|投标人须知前附表)\s*$", re.MULTILINE)
|
||||
)
|
||||
]
|
||||
@ -411,15 +411,16 @@ def truncate_pdf_main(input_path, output_folder, selection):
|
||||
for idx, (begin_pattern, end_pattern) in enumerate(pattern_pairs, start=1):
|
||||
is_secondary_match = (idx == 2)
|
||||
begin_page = last_begin_index if last_begin_index != 0 else {
|
||||
1: 3, #前附表
|
||||
2: 10, #评标
|
||||
3: 5, #资格
|
||||
4: 0, #公告
|
||||
5: 0 #无效标
|
||||
1: 3, # 前附表
|
||||
2: 10, # 评标
|
||||
3: 5, # 资格
|
||||
4: 0, # 公告
|
||||
5: 0 # 无效标
|
||||
}.get(selection, 0)
|
||||
if selection == 1: #投标人须知
|
||||
if selection == 1: # 投标人须知
|
||||
output_paths = list(
|
||||
extract_pages_tobidders_notice(input_path,output_folder, begin_pattern, begin_page, common_header,is_secondary_match))
|
||||
extract_pages_tobidders_notice(input_path, output_folder, begin_pattern, begin_page, common_header,
|
||||
is_secondary_match))
|
||||
if output_paths and any(os.path.isfile(f) for f in output_paths):
|
||||
return output_paths
|
||||
else:
|
||||
@ -457,59 +458,6 @@ def truncate_pdf_main(input_path, output_folder, selection):
|
||||
return ['']
|
||||
|
||||
|
||||
def extract_pages_twice(pdf_path, output_folder, output_suffix,common_header,last_begin_index):
|
||||
"""
|
||||
处理 PDF 文件，作为 truncate_pdf_main 的后备方法。
|
||||
此函数将在所有模式对都失败后调用。
|
||||
"""
|
||||
try:
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
if output_suffix == "qualification":
|
||||
begin_pattern = r'^(?:附录(?:[一1])?[：:]|附件(?:[一1])?[：:]|附表(?:[一1])?[：:])'
|
||||
end_pattern = re.compile(
|
||||
r'^(?:附录[一二三四五六七八九1-9]*[：:]|附件[一二三四五六七八九1-9]*[：:]|附表[一二三四五六七八九1-9]*[：:])(?!.*(?:资质|能力|信誉)).*$|'
|
||||
r'^第[一二三四五六七八九]+(?:章|部分)\s*[\u4e00-\u9fff]+|评标办法前附表|投标人须知',
|
||||
re.MULTILINE
|
||||
)
|
||||
start_page = None
|
||||
end_page = None
|
||||
# 从章节开始后的位置进行检查
|
||||
for i, page in enumerate(pdf_document.pages[last_begin_index:], start=last_begin_index):
|
||||
text = page.extract_text()
|
||||
if text:
|
||||
cleaned_text = clean_page_content(text, common_header)
|
||||
# 确定起始页，需在last_begin_index之后
|
||||
if "资格审查" in cleaned_text or "资质条件" in cleaned_text: #当页包含这些内容可以认为是'资格审查资料部分'
|
||||
if re.search(begin_pattern, cleaned_text, re.MULTILINE):
|
||||
if start_page is None:
|
||||
start_page = i # 确保起始页不小于章节的开始页码
|
||||
# 确定结束页
|
||||
if start_page is not None and re.search(end_pattern, cleaned_text):
|
||||
if i > start_page:
|
||||
end_page = i
|
||||
break # 找到结束页后退出循环
|
||||
|
||||
if start_page is None or end_page is None:
|
||||
print(f"{output_suffix} twice: 未找到起始或结束页在文件 {pdf_path} 中！")
|
||||
return []
|
||||
else:
|
||||
return [save_pages_to_new_pdf(pdf_path, output_folder, output_suffix, start_page, end_page, common_header)]
|
||||
elif output_suffix == "invalid":
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
total_pages = len(pdf_document.pages)
|
||||
# 计算总页数的三分之二
|
||||
total = int(total_pages * 2 / 3)
|
||||
start_page = last_begin_index
|
||||
end_page = min(90, total)
|
||||
return [save_pages_to_new_pdf(pdf_path, output_folder, output_suffix, start_page, end_page, common_header)]
|
||||
else:
|
||||
print(f"{output_suffix} twice: 未定义的输出后缀。")
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f"Error in extract_pages_twice: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def truncate_pdf_multiple(input_path, output_folder, unique_id="123"):
|
||||
global logger
|
||||
logger = get_global_logger(unique_id)
|
||||
@ -520,7 +468,8 @@ def truncate_pdf_multiple(input_path, output_folder, unique_id="123"):
|
||||
# 使用 ThreadPoolExecutor 进行多线程处理
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=len(selections)) as executor:
|
||||
# 提交所有任务并保持 selection 顺序
|
||||
future_to_selection = {selection: executor.submit(truncate_pdf_main, input_path, output_folder, selection) for selection in selections}
|
||||
future_to_selection = {selection: executor.submit(truncate_pdf_main, input_path, output_folder, selection) for
|
||||
selection in selections}
|
||||
|
||||
# 按 selection 顺序收集结果
|
||||
for selection in selections:
|
||||
@ -543,7 +492,8 @@ def truncate_pdf_multiple(input_path, output_folder, unique_id="123"):
|
||||
# Proceed with merging logic as before...
|
||||
if any(f for f in truncate_files if f): # 检查是否有有效的文件路径
|
||||
merged_output_path = os.path.join(output_folder, f"{base_file_name}_merged_baseinfo.pdf")
|
||||
merged_result = merge_selected_pdfs_for_engineering(output_folder, truncate_files, merged_output_path, base_file_name)
|
||||
merged_result = merge_selected_pdfs_for_engineering(output_folder, truncate_files, merged_output_path,
|
||||
base_file_name)
|
||||
if merged_result:
|
||||
truncate_files.append(merged_result)
|
||||
logger.info(f"merged_baseinfo: 已生成合并文件: {merged_output_path}")
|
||||
@ -567,7 +517,8 @@ def truncate_pdf_specific_engineering(pdf_path, output_folder, selections, uniqu
|
||||
# 使用 ThreadPoolExecutor 进行多线程处理
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=len(selections)) as executor:
|
||||
# 提交所有任务并保持 selection 顺序
|
||||
future_to_selection = {selection: executor.submit(truncate_pdf_main, pdf_path, output_folder, selection) for selection in selections}
|
||||
future_to_selection = {selection: executor.submit(truncate_pdf_main, pdf_path, output_folder, selection) for
|
||||
selection in selections}
|
||||
|
||||
# 按 selection 顺序收集结果
|
||||
for selection in selections:
|
||||
@ -590,7 +541,8 @@ def truncate_pdf_specific_engineering(pdf_path, output_folder, selections, uniqu
|
||||
# Proceed with merging logic as before...
|
||||
if any(f for f in truncate_files if f): # 检查是否有有效的文件路径
|
||||
merged_output_path = os.path.join(output_folder, f"{base_file_name}_merged_specific.pdf")
|
||||
merged_result = merge_selected_pdfs_for_engineering(output_folder, truncate_files, merged_output_path, base_file_name)
|
||||
merged_result = merge_selected_pdfs_for_engineering(output_folder, truncate_files, merged_output_path,
|
||||
base_file_name)
|
||||
if merged_result:
|
||||
truncate_files.append(merged_result)
|
||||
logger.info(f"merged_specific: 已生成合并文件: {merged_output_path}")
|
||||
@ -609,23 +561,23 @@ def truncate_pdf_specific_engineering(pdf_path, output_folder, selections, uniqu
|
||||
|
||||
|
||||
# TODO:需要完善二次请求。目前invalid一定能返回 前附表 须知正文如果为空的话要额外处理一下，比如说就不进行跳转（见xx表） 开评定标这里也要考虑 如果评分表为空，也要处理。
|
||||
#TODO:zbtest8 zbtest18有问题 后期需要完善，截取需要截两次，第一次严格第二次宽松
|
||||
#TODO:目前merged_baseinfo没有包含投标人须知正文。
|
||||
#投标人须知前附表改为货物标一样的
|
||||
# TODO:zbtest8 zbtest18有问题 后期需要完善，截取需要截两次，第一次严格第二次宽松
|
||||
# TODO:目前merged_baseinfo没有包含投标人须知正文。
|
||||
# 投标人须知前附表改为货物标一样的
|
||||
if __name__ == "__main__":
|
||||
start_time=time.time()
|
||||
input_path = "C:\\Users\\Administrator\\Desktop\\new招标文件\\招标文件\\HBDL-2024-0181-001-招标文件.pdf"
|
||||
start_time = time.time()
|
||||
input_path = "C:\\Users\\Administrator\\Desktop\\new招标文件\\工程标\\HBDL-2024-0017-001-招标文件.pdf"
|
||||
# input_path="C:\\Users\\Administrator\\Desktop\\fsdownload\\68549b0b-e892-41a9-897c-c3694535ee61\\ztbfile.pdf"
|
||||
# input_path = "C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles\\2-招标文件.pdf"
|
||||
# input_path="C:\\Users\\Administrator\\Desktop\\招标文件\\招标test文件夹\\zbtest8.pdf"
|
||||
output_folder="C:\\Users\\Administrator\\Desktop\\new招标文件\\招标文件\\tmp"
|
||||
# input_path="C:\\Users\\Administrator\\Desktop\\招标文件\\招标test文件夹\\zbtest1.pdf"
|
||||
output_folder = "C:\\Users\\Administrator\\Desktop\\new招标文件\\招标文件\\tmp"
|
||||
# files=truncate_pdf_multiple(input_path,output_folder)
|
||||
# selections = [5, 1] # 仅处理 selection 5、1
|
||||
# files=truncate_pdf_specific_engineering(input_path,output_folder,selections)
|
||||
# print(files)
|
||||
selection = 3 # 例如：1 - 投标人须知前附表+正文, 2 - 评标办法, 3 -资格审查条件 4-招标公告 5-无效标
|
||||
selection = 2 # 例如：1 - 投标人须知前附表+正文, 2 - 评标办法, 3 -资格审查条件 4-招标公告 5-无效标
|
||||
generated_files = truncate_pdf_main(input_path, output_folder, selection)
|
||||
|
||||
print(generated_files)
|
||||
# print("生成的文件:", generated_files)
|
||||
end_time=time.time()
|
||||
print("耗时："+str(end_time-start_time))
|
||||
end_time = time.time()
|
||||
print("耗时：" + str(end_time - start_time))
|
||||
|
||||
@ -106,7 +106,7 @@ def extract_from_notice(invalid_path,clause_path, type):
|
||||
elif type == 2:
|
||||
target_values = ["开标", "评标", "定标","磋商程序","中标"]
|
||||
elif type == 3:
|
||||
target_values = ["重新招标、不再招标和终止招标", "重新招标", "不再招标", "终止招标"]
|
||||
target_values = ["重新招标、不再招标和终止招标","重新招标","重新采购", "不再招标", "不再采购","终止招标","终止采购"]
|
||||
elif type == 4:
|
||||
target_values = ["评标"] # 测试
|
||||
else:
|
||||
|
||||
@ -3,6 +3,8 @@ import logging
|
||||
from PyPDF2 import PdfReader, PdfWriter
|
||||
import re # 导入正则表达式库
|
||||
import os # 用于文件和文件夹操作
|
||||
|
||||
from flask_app.general.clean_pdf import clean_page_content, extract_common_header
|
||||
from flask_app.general.format_change import docx2pdf
|
||||
from flask_app.general.merge_pdfs import merge_and_cleanup, merge_pdfs, merge_selected_pdfs_for_goods
|
||||
import concurrent.futures
|
||||
@ -13,65 +15,6 @@ def get_global_logger(unique_id):
|
||||
return logger
|
||||
logger = None
|
||||
|
||||
def clean_page_content(text, common_header):
|
||||
# 首先删除抬头公共部分
|
||||
if common_header: # 确保有公共抬头才进行替换
|
||||
for header_line in common_header.split('\n'):
|
||||
if header_line.strip(): # 只处理非空行
|
||||
# 替换首次出现的完整行
|
||||
text = re.sub(r'^' + re.escape(header_line.strip()) + r'\n?', '', text, count=1)
|
||||
|
||||
# 删除页码 eg:89/129 这个代码分三步走可以把89/129完全删除
|
||||
text = re.sub(r'^\s*\d+\s*(?=\D)', '', text) # 删除开头的页码，仅当紧跟非数字字符时
|
||||
text = re.sub(r'\s+\d+\s*$', '', text) # 删除结尾的页码
|
||||
text = re.sub(r'\s*\/\s*\d+\s*', '', text) # 删除形如 /129 的页码
|
||||
text = re.sub(r'\s*[—-]\s*\d+\s*[—-]\s*', '', text) # 删除形如 '—2—' 或 '-2-' 的页码
|
||||
return text
|
||||
|
||||
|
||||
# PYPDF2版本
|
||||
def extract_common_header(pdf_path):
|
||||
from PyPDF2 import PdfReader
|
||||
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
headers = []
|
||||
total_pages = len(pdf_document.pages)
|
||||
|
||||
# 确定要读取的页数和起始页
|
||||
if total_pages == 2:
|
||||
pages_to_read = 2
|
||||
start_page = 0
|
||||
else:
|
||||
pages_to_read = 3
|
||||
middle_page = total_pages // 2
|
||||
start_page = max(0, middle_page - 1)
|
||||
|
||||
for i in range(start_page, min(start_page + pages_to_read, total_pages)):
|
||||
page = pdf_document.pages[i]
|
||||
text = page.extract_text() or ""
|
||||
if text:
|
||||
# 只取每页的前三行
|
||||
first_lines = text.strip().split('\n')[:3]
|
||||
headers.append(first_lines)
|
||||
|
||||
if len(headers) < 2:
|
||||
return "" # 如果没有足够的页来比较，返回空字符串
|
||||
|
||||
# 寻找每一行中的公共部分，按顺序保留
|
||||
common_headers = []
|
||||
for lines in zip(*headers):
|
||||
# 提取第一行的词汇顺序
|
||||
first_words = lines[0].split()
|
||||
# 筛选所有页面都包含的词汇，保持顺序
|
||||
common_line = [word for word in first_words if all(word in line.split() for line in lines[1:])]
|
||||
if common_line:
|
||||
common_headers.append(' '.join(common_line))
|
||||
|
||||
return '\n'.join(common_headers)
|
||||
|
||||
|
||||
|
||||
|
||||
# fitz库版本
|
||||
# def extract_common_header(pdf_path):
|
||||
# doc = fitz.open(pdf_path)
|
||||
@ -216,9 +159,9 @@ def get_patterns_for_procurement():
|
||||
|
||||
def get_patterns_for_evaluation_method():
|
||||
begin_pattern = re.compile(
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(磋商|谈判|评标|评定|评审)(方法|办法).*', re.MULTILINE)
|
||||
r'第[一二三四五六七八九1-9]+(?:章|部分)\s*[\u4e00-\u9fff、]*?(磋商|谈判|评标|评定|评审)[\u4e00-\u9fff、]*\s*$', re.MULTILINE)
|
||||
end_pattern = re.compile(
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+', re.MULTILINE)
|
||||
r'第[一二三四五六七八九1-9]+(?:章|部分)\s*[\u4e00-\u9fff]+\s*$', re.MULTILINE)
|
||||
return begin_pattern, end_pattern
|
||||
|
||||
|
||||
@ -329,7 +272,7 @@ def extract_pages_tobidders_notice(pdf_document, begin_pattern, begin_page, comm
|
||||
additional_mid_pattern = ''
|
||||
|
||||
# 定义基础的 mid_pattern
|
||||
base_mid_pattern = r'^\s*(?:[（(]\s*[一1]?\s*[)）]\s*[、．.]*|[一1][、．.]+|[、．.]+)\s*(说\s*明|总\s*则)'
|
||||
base_mid_pattern = r'^\s*(?:[（(]\s*[一二12]?\s*[)）]\s*[、．.]*|[一二12][、．.]+|[、．.]+)\s*(说\s*明|总\s*则)'
|
||||
|
||||
# 合并基础模式和额外模式
|
||||
if additional_mid_pattern:
|
||||
@ -352,7 +295,7 @@ def extract_pages_tobidders_notice(pdf_document, begin_pattern, begin_page, comm
|
||||
print(f"使用默认的 end_pattern: {end_pattern.pattern}") # 打印默认的 end_pattern
|
||||
|
||||
# 定义基础的 mid_pattern
|
||||
base_mid_pattern = r'^\s*(?:[（(]\s*[一1]?\s*[)）]\s*[、．.]*|[一1][、．.]+|[、．.]+)\s*(说\s*明|总\s*则)'
|
||||
base_mid_pattern = r'^\s*(?:[（(]\s*[一二12]?\s*[)）]\s*[、．.]*|[一二12][、．.]+|[、．.]+)\s*(说\s*明|总\s*则)'
|
||||
combined_mid_pattern = re.compile(
|
||||
rf'{base_mid_pattern}',
|
||||
re.MULTILINE
|
||||
@ -518,7 +461,7 @@ def save_extracted_pages(pdf_document, start_page, end_page, pdf_path, output_fo
|
||||
before_doc.add_page(pdf_document.pages[page_num])
|
||||
with open(before_pdf_path, 'wb') as f:
|
||||
before_doc.write(f)
|
||||
print(f"已保存页面从 0 到 {start_page - 1} 为 {before_pdf_path}")
|
||||
# print(f"已保存页面从 0 到 {start_page - 1} 为 {before_pdf_path}")
|
||||
|
||||
output_doc = PdfWriter()
|
||||
for page_num in range(start_page, end_page + 1):
|
||||
@ -534,7 +477,7 @@ def save_extracted_pages(pdf_document, start_page, end_page, pdf_path, output_fo
|
||||
def get_start_and_common_header(input_path):
|
||||
common_header = extract_common_header(input_path)
|
||||
last_begin_index = 0
|
||||
begin_pattern = re.compile(r'.*(?:招标公告|投标邀请书|投标邀请函|投标邀请)[\)）]?\s*$',re.MULTILINE)
|
||||
begin_pattern = re.compile(r'.*(?:招标公告|邀请书|邀请函|投标邀请)[\)）]?\s*$',re.MULTILINE)
|
||||
pdf_document = PdfReader(input_path)
|
||||
for i, page in enumerate(pdf_document.pages):
|
||||
if i > 10:
|
||||
@ -594,12 +537,12 @@ def process_input(input_path, output_folder, selection, output_suffix):
|
||||
|
||||
# 根据选择设置对应的模式和结束模式
|
||||
if selection == 1:
|
||||
begin_pattern = re.compile(r'.*(?:招标公告|投标邀请书|投标邀请函|投标邀请)[\)）]?\s*$', re.MULTILINE)
|
||||
begin_pattern = re.compile(r'.*(?:招标公告|邀请书|邀请函|投标邀请|磋商邀请|谈判邀请)[\)）]?\s*$', re.MULTILINE)
|
||||
end_pattern = re.compile(r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+', re.MULTILINE)
|
||||
local_output_suffix = "notice"
|
||||
elif selection == 2:
|
||||
begin_pattern = re.compile(
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(磋商|谈判|评标|评定|评审)(方法|办法).*')
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff、]*?(磋商|谈判|评标|评定|评审)[\u4e00-\u9fff、]*')
|
||||
end_pattern = re.compile(r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+')
|
||||
local_output_suffix = "evaluation_method"
|
||||
elif selection == 3:
|
||||
@ -731,7 +674,7 @@ def truncate_pdf_specific_goods(pdf_path, output_folder, selections,unique_id="1
|
||||
|
||||
# TODO:交通智能系统和招标(1)(1)文件有问题 包头 绍兴 资格审查文件可能不需要默认与"evaluation"同一章 无效投标可能也要考虑 “more”的情况，类似工程标 唐山投标只有正文，没有附表
|
||||
if __name__ == "__main__":
|
||||
input_path = "C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles\\唐山市公安交通警察支队机动车查验机构视频存储回放系统竞争性谈判-招标文件正文(1).pdf"
|
||||
input_path = "C:\\Users\\Administrator\\Desktop\\new招标文件\\货物标\\HBDL-2024-0498-001-招标文件.pdf"
|
||||
# input_path = "C:\\Users\\Administrator\\Desktop\\fsdownload\\b151fcd0-4cd8-49b4-8de3-964057a9e653\\ztbfile.pdf"
|
||||
# input_path="C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles"
|
||||
# input_path = "C:\\Users\\Administrator\\Desktop\\货物标\\output1\\2-招标文件_procurement.pdf"
|
||||
@ -742,6 +685,6 @@ if __name__ == "__main__":
|
||||
# selections = [1,4]
|
||||
# files=truncate_pdf_specific_goods(input_path,output_folder,selections)
|
||||
print(files)
|
||||
# selection = 4# 例如：1 - 公告, 2 - 评标办法, 3 - 资格审查后缀有qualification1或qualification2（与评标办法一致） 4.投标人须知前附表part1 投标人须知正文part2 5-采购需求
|
||||
# selection = 2# 例如：1 - 公告, 2 - 评标办法, 3 - 资格审查后缀有qualification1或qualification2（与评标办法一致） 4.投标人须知前附表part1 投标人须知正文part2 5-采购需求
|
||||
# generated_files = truncate_pdf_main(input_path, output_folder, selection)
|
||||
# print(generated_files)
|
||||
@ -90,7 +90,7 @@ def extract_from_notice(invalid_path,clause_path, type):
|
||||
elif type == 2:
|
||||
target_values = ["开标", "评标", "定标", "磋商程序", "中标", "程序", "步骤"]
|
||||
elif type == 3:
|
||||
target_values = ["重新招标、不再招标和终止招标", "重新招标", "不再招标", "终止招标"]
|
||||
target_values = ["重新招标、不再招标和终止招标","重新招标","重新采购", "不再招标", "不再采购","终止招标","终止采购"]
|
||||
elif type == 4:
|
||||
target_values = ["评标"] # 测试
|
||||
else:
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user