1.20 解决了废标项提取1.1.1.1多级序号bug;目前仍无法有效判断是否为乱码文件,除非调用大模型
This commit is contained in:
parent
28ddae379a
commit
fb88d1e46c
@ -119,10 +119,10 @@ if __name__ == "__main__":
|
||||
# input_path = r"C:\Users\Administrator\Desktop\new招标文件\工程标"
|
||||
# pdf_path=r"C:\Users\Administrator\Desktop\货物标\zbfiles\094定稿-湖北工业大学轻武器模拟射击设备采购项目招标文件.pdf"
|
||||
# pdf_path = r"C:\Users\Administrator\Desktop\货物标\zbfiles\zbtest4_evaluation_method.pdf"
|
||||
pdf_path=r"C:\Users\Administrator\Desktop\新建文件夹 (3)\“平安城市”三期暨雪亮工程一体化 建设项目.pdf"
|
||||
pdf_path=r"C:\Users\Administrator\Desktop\新建文件夹 (3)\temp\file1736998876340 (1).doc"
|
||||
# pdf_path=r"C:\Users\Administrator\Desktop\货物标\zbfiles\zbtest4_evaluation_method.pdf"
|
||||
# input_path=r"C:\Users\Administrator\Desktop\招标文件\招标test文件夹\zbtest8.pdf"
|
||||
output_folder = r"C:\Users\Administrator\Desktop\新建文件夹 (3)"
|
||||
output_folder = r"C:\Users\Administrator\Desktop\新建文件夹 (3)\temp"
|
||||
# selections = [1, 4] # 仅处理 selection 4、1
|
||||
# selections = [1, 2, 3, 5]
|
||||
# files = truncate_pdf_multiple(pdf_path, output_folder, logger, 'goods', selections) #engineering
|
||||
|
||||
@ -406,9 +406,9 @@ def extract_from_notice(merged_baseinfo_path, clause_path, type):
|
||||
if __name__ == "__main__":
|
||||
# file_path = 'C:\\Users\\Administrator\\Desktop\\fsdownload\\3bffaa84-2434-4bd0-a8ee-5c234ccd7fa0\\clause1.json'
|
||||
merged_baseinfo_path=r"C:\Users\Administrator\Desktop\fsdownload\b29de31a-297e-42cf-b9ba-6859e530a472\ztbfile_merged_baseinfo.pdf"
|
||||
clause_path=r"D:\flask_project\flask_app\static\output\output1\2c4be864-bdab-405d-95cb-9d945d8627b3\tmp\clause1.json"
|
||||
clause_path=r"C:\Users\Administrator\Desktop\fsdownload\ec7d5328-9c57-450f-baf4-2e5a6f90ed1d\tmp\clause1.json"
|
||||
try:
|
||||
res = extract_from_notice(merged_baseinfo_path,clause_path, 2) # 可以改变此处的 type 参数测试不同的场景
|
||||
res = extract_from_notice(merged_baseinfo_path,clause_path, 1) # 可以改变此处的 type 参数测试不同的场景
|
||||
res2 = json.dumps(res, ensure_ascii=False, indent=4)
|
||||
print(res2)
|
||||
except ValueError as e:
|
||||
|
||||
@ -481,7 +481,7 @@ def convert_clause_to_json(file_path, output_folder, type=1):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
file_path = r'C:\Users\Administrator\Desktop\货物标\zbfiles\output4444\招标文件111_tobidders_notice_part2.pdf'
|
||||
file_path = r'C:\Users\Administrator\Desktop\货物标\zbfiles\output4444\唐山市公安交通警察支队机动车查验机构视频存储回放系统竞争性谈判-招标文件正文(1)_tobidders_notice_part1.pdf'
|
||||
# file_path=r'C:\Users\Administrator\Desktop\招标文件-采购类\all\2024-陕西-陕西省某单位2024年执勤化妆服采购项目_tobidders_notice_part2.pdf'
|
||||
# file_path=r'C:\Users\Administrator\Desktop\货物标\output4\磋商文件_tobidders_notice_part2.pdf'
|
||||
# file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\6.2定版视频会议磋商文件_tobidders_notice_part2.pdf'
|
||||
|
||||
@ -344,10 +344,12 @@ def extract_text_with_keywords(processed_paragraphs, keywords, follow_up_keyword
|
||||
continue # 跳过空白行，进入下一个循环
|
||||
if not found_next_number:
|
||||
# 修改后的正则，支持 '数字 、' 格式
|
||||
number_pattern=(r'^([A-Za-z0-9]+(?:[.．][A-Za-z0-9]+)*)' #A1 1.1 abc.123
|
||||
r'|([（(]\s*[一二三四五六七八九十\d]+\s*[）)])' #(1) （2）
|
||||
r'|([一二三四五六七八九十\d]+\s*、)') #1、 2、
|
||||
next_section_number = re.match(number_pattern,next_text)
|
||||
number_pattern = (
|
||||
r'^([A-Za-z0-9]+(?:[.．][A-Za-z0-9]+)*)|' # 第一部分：匹配 A1, 1.1, abc.123 等
|
||||
r'([（(]\s*(?:[一二三四五六七八九十]{1,2}|\d+)\s*[）)])|' # 第二部分：匹配 (一), （12） 等，其中中文数字1到2位或任意位数字
|
||||
r'((?:[一二三四五六七八九十]{1,2}|\d+)\s*、)' # 第三部分：匹配 一、12、等，其中中文数字1到2位或任意位数字
|
||||
)
|
||||
next_section_number = re.match(number_pattern,next_text) #re.match总是从字符串开头匹配，即使正则模式内部没有 ^ 锚点。
|
||||
if next_section_number:
|
||||
found_next_number = True
|
||||
if next_section_number.group(1):
|
||||
@ -355,9 +357,9 @@ def extract_text_with_keywords(processed_paragraphs, keywords, follow_up_keyword
|
||||
dynamic_pattern = r'^' + r'[.．]'.join(
|
||||
[r'[A-Za-z0-9]+' for _ in section_parts]) + r'\b'
|
||||
elif next_section_number.group(2):
|
||||
dynamic_pattern = r'^[\(\（]\s*[一二三四五六七八九十\d]+\s*[\)\）]'
|
||||
dynamic_pattern = r'^[\(\（]\s*(?:[一二三四五六七八九十]{1,2}|\d+)\s*[\)\）]'
|
||||
elif next_section_number.group(3):
|
||||
dynamic_pattern = r'^[一二三四五六七八九十\d]+\s*、'
|
||||
dynamic_pattern = r'^(?:[一二三四五六七八九十]{1,2}|\d+)\s*、'
|
||||
current_section_pattern = re.compile(dynamic_pattern)
|
||||
if current_section_pattern and re.match(current_section_pattern, next_text):
|
||||
extracted_paragraphs[active_key].append(next_text)
|
||||
@ -394,12 +396,12 @@ def split_cell_text(text):
|
||||
# 2. 分割句子，保证句子完整性（按标点符号和序号分割）
|
||||
split_sentences = regex.split(
|
||||
r'(?<=[。！？!?\?])|' # 在中文句号、感叹号、问号后分割
|
||||
r'(?<![A-Za-z]\s*)(?<!\d[.．]?)(?=\d+(?:[.．]\d+)+(?!\s*[号条款节章项例页段部步点年月日时分秒个元千万台份家]))|' # 匹配多级编号，限制后面不能是指定关键字
|
||||
r'(?<![A-Za-z]\s*)(?<!\d[.．]?)(?=(?>\d+(?:[.．]\d+)+)(?!\s*[号条款节章项例页段部步点年月日时分秒个元千万台份家]))|' # 匹配多级编号，(?=(?>\d+(?:[.．]\d+)+)让多级编号匹配“一口气”完成；限制后面不能是指定关键字
|
||||
r'(?<![+\-×÷*/.\．A-Za-z]\s*|\d)(?=\d+\s(?!\s*[号条款节章项例页段部步点年月日时分秒个元千万台份家]))|' # 数字后跟空格且不跟指定关键字时分割，且前面不包含 . 或 ． eg:'1.1 xx'
|
||||
r'(?<![+\-×÷*/.\．A-Za-z]\s*|\d)(?=\d+[、.．](?!\d|(?:\s*[号条款节章项例页段部步点年月日时分秒个元千万台份家])))|' # 数字后直接跟顿号、点号时分割，且点号后不跟数字 eg:'1.'
|
||||
r'(?<![A-Za-z])(?=[A-Za-z][.．]\s*(?![A-Za-z]))|' # 单个字母+点号或单个字母+数字，排除www.baidu.com 网址情况
|
||||
r'(?=[A-Za-z]+\s*\d+\s*(?:[.．]\s*\d+)*)|' # 在字母加数字或多级编号前分割
|
||||
r'(?<=^|\n)(?=[一二三四五六七八九十]+、)', # 在中文数字加顿号（如一、二、）前分割
|
||||
r'(?<=^|\n)(?=[一二三四五六七八九十]+、)', # 在中文数字加顿号（如一、二、）前分割，(?<=^|\n)增加了行首限制
|
||||
item_with_placeholders
|
||||
)
|
||||
|
||||
@ -514,6 +516,7 @@ def handle_query(file_path, user_query, output_file, result_key, keywords):
|
||||
|
||||
# 生成用户查询
|
||||
if not all_texts1_list:
|
||||
print("无需调用大模型获取选中的序号！")
|
||||
num_list=[]
|
||||
else:
|
||||
user_query = generate_full_user_query(output_file, user_query)
|
||||
@ -521,7 +524,7 @@ def handle_query(file_path, user_query, output_file, result_key, keywords):
|
||||
# file_id = upload_file(output_file)
|
||||
# model_ans = qianwen_long(file_id, user_query)
|
||||
num_list = process_string_list(model_ans) # 处理模型返回的序号
|
||||
print(result_key + "选中的序号:" + str(num_list))
|
||||
print(result_key + "选中的序号:" + str(num_list))
|
||||
|
||||
for index in num_list:
|
||||
if 1 <= index <= len(all_texts1_list):
|
||||
|
||||
@ -22,7 +22,6 @@ def extract_text_by_page(file_path):
|
||||
print(f"Page {page_num + 1} is empty or text could not be extracted.")
|
||||
return result
|
||||
|
||||
|
||||
def save_extracted_text_to_txt(pdf_path, txt_path):
|
||||
"""
|
||||
调用 extract_text_by_page 函数提取PDF文本，并将其保存到指定的TXT文件中。
|
||||
@ -118,7 +117,7 @@ def save_extracted_text_to_txt(pdf_path, txt_path):
|
||||
|
||||
if __name__ == '__main__':
|
||||
# file_path='D:\\flask_project\\flask_app\\static\\output\\output1\\648e094b-e677-47ce-9073-09e0c82af210\\ztbfile_tobidders_notice_part2.pdf'
|
||||
pdf_path=r"C:\Users\Administrator\Desktop\招标文件\output11\zb_notice.pdf"
|
||||
pdf_path=r"D:\flask_project\flask_app\static\output\output1\032c44dc-230c-4757-9160-c99f2eaac790\ztbfile_invalid.pdf"
|
||||
# file_path = r"C:\Users\Administrator\Desktop\招标文件\招标test文件夹\zbtest8.pdf"
|
||||
# file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\截取test\\交警支队机动车查验监管系统项目采购_tobidders_notice_part1.pdf'
|
||||
# file_path = "C:\\Users\\Administrator\\Desktop\\招标文件\\招标test文件夹\\zbtest8.pdf"
|
||||
@ -128,4 +127,4 @@ if __name__ == '__main__':
|
||||
print("-----------------")
|
||||
res=extract_text_by_page(pdf_path)
|
||||
# print(res)磋商文件_tobidders_notice_part2.pdf
|
||||
# save_extracted_text_to_txt(file_path,"output.txt")
|
||||
# save_extracted_text_to_txt(file_path,"output.txt")
|
||||
|
||||
@ -90,7 +90,7 @@ def read_docx_by_paragraphs(file_path):
|
||||
return []
|
||||
|
||||
if __name__ == "__main__":
|
||||
file_path = r'C:\Users\Administrator\Desktop\测试信号测试信号.docx'
|
||||
file_path=r"C:\Users\Administrator\Desktop\新建文件夹 (3)\temp\file1736998876340 (1).doc"
|
||||
read_docx(file_path) #按行读取
|
||||
|
||||
# paragraphs = read_docx_by_paragraphs(file_path) #按段落读取
|
||||
|
||||
@ -105,6 +105,17 @@ def process_and_stream(file_url, zb_type):
|
||||
logger.error(f"Failed to decode JSON: {e}")
|
||||
logger.error(f"Data received: {data}")
|
||||
continue
|
||||
if 'error' in parsed_data:
|
||||
error_message = parsed_data['error']
|
||||
logger.error(f"Processing terminated due to error: {error_message}")
|
||||
# 使用指定的格式返回错误响应
|
||||
error_response = create_response(
|
||||
message=error_message,
|
||||
status='error',
|
||||
data=''
|
||||
)
|
||||
yield sse_format(error_response)
|
||||
return # 终止进一步处理
|
||||
|
||||
if 'good_list' in parsed_data:
|
||||
good_list = parsed_data['good_list']
|
||||
|
||||
@ -6,6 +6,7 @@ from functools import wraps
|
||||
from flask import request, jsonify, current_app, g
|
||||
|
||||
from flask_app.general.清除file_id import read_file_ids, delete_file_by_ids
|
||||
from flask_app.general.通义千问long import upload_file, qianwen_long
|
||||
from flask_app.logger_setup import create_logger
|
||||
|
||||
|
||||
@ -214,4 +215,12 @@ def log_error_unique_id(unique_id, selection=6):
|
||||
|
||||
# 将 unique_id 写入错误文件
|
||||
with open(error_file_path, 'a', encoding='utf-8') as f:
|
||||
f.write(f"{unique_id}\n")
|
||||
f.write(f"{unique_id}\n")
|
||||
|
||||
def is_gibberish_file(file_path):
|
||||
user_query="该文件是否为乱码文件,你的回答仅限于'是''否',不要返回其他解释性回答"
|
||||
file_id=upload_file(file_path)
|
||||
model_ans=qianwen_long(file_id,user_query)
|
||||
if '是' in model_ans:
|
||||
return True
|
||||
return False
|
||||
@ -24,6 +24,9 @@ def judge_zbfile_exec(file_path):
|
||||
以下是常见的招标文件类型：
|
||||
公开招标文件、邀请招标文件、竞争性谈判文件、竞争性磋商文件、询价文件、问询文件、货物类招标文件、工程类招标文件、施工类招标文件、服务类招标文件、比选文件。
|
||||
若有未涵盖的类型，但其内容明确表达了项目需求、采购或招标信息，且包含指导投标人参与的关键要素，则可视为招标文件。
|
||||
排除情况:
|
||||
1. 请注意区分招标文件和投标文件,若文件仅有投标文件格式要求部分,或是投标、响应性文件，则不视为招标文件。
|
||||
2. 若文件内容为乱码,无有效信息,请直接返回'否'。
|
||||
请基于上述内容判断文件是否属于招标文件。
|
||||
"""
|
||||
file_id = upload_file(file_path)
|
||||
@ -44,7 +47,7 @@ def judge_zbfile_exec(file_path):
|
||||
|
||||
if __name__ == '__main__':
|
||||
start_time = time.time()
|
||||
pdf_path = r"C:\Users\Administrator\Downloads\商务标null1736476867734.docx"
|
||||
pdf_path = r"C:\Users\Administrator\Desktop\新建文件夹 (3)\temp\乱码文件测试.doc"
|
||||
res = judge_zbfile_exec(pdf_path)
|
||||
if res:
|
||||
print("yes")
|
||||
|
||||
@ -14,7 +14,6 @@ from flask_app.general.投标人须知正文条款提取成json文件 import con
|
||||
from flask_app.general.无效标和废标公共代码 import combine_find_invalid
|
||||
from flask_app.货物标.资格审查main import combine_qualification_review
|
||||
from flask_app.general.商务技术评分提取 import combine_evaluation_standards
|
||||
|
||||
def preprocess_files(output_folder, file_path, file_type,logger):
|
||||
logger.info("starting 文件预处理...")
|
||||
start_time = time.time()
|
||||
@ -263,21 +262,13 @@ def goods_bid_main(output_folder, file_path, file_type, unique_id):
|
||||
if collected_good_list is not None:
|
||||
yield json.dumps({'good_list': transform_json_values(collected_good_list)}, ensure_ascii=False)
|
||||
|
||||
#广水市 2022 年义务教育学校多媒体补充采购项目 资格审查有问题
|
||||
|
||||
|
||||
#TODO:把所有未知都删掉。
|
||||
|
||||
#TODO:小解析考虑提速：1：直接pdf转文本，再切分。后期考虑。
|
||||
|
||||
#TODO: ec7d5328-9c57-450f-baf4-2e5a6f90ed1d
|
||||
|
||||
#TODO:
|
||||
# 解决禅道 测试的bug
|
||||
# 货物标和工程标的资格审查整合
|
||||
##TODO:招标文件111_tobidders_notice_part2.pdf 陕西省公安厅交通警察总队高速公路交通安全智能感知巡查系统项目(1)_tobidders_notice_part2.pdf 唐山市公安交通警察支队机动车查验机构视频存储回放系统竞争性谈判-招标文件正文(1)_tobidders_notice_part1.pdf
|
||||
#TODO:2024-陕西-陕西省某单位2024年执勤化妆服采购项目.pdf
|
||||
|
||||
##TODO:陕西省公安厅交通警察总队高速公路交通安全智能感知巡查系统项目(1)_tobidders_notice_part2.pdf 唐山市公安交通警察支队机动车查验机构视频存储回放系统竞争性谈判-招标文件正文(1)_tobidders_notice_part1.pdf 不好搞
|
||||
# 无法判断用户上传的是否为乱码文件,可以考虑并行调用大模型,如果为乱码文件直接return None
|
||||
if __name__ == "__main__":
|
||||
# 配置日志器
|
||||
unique_id = "uuidzyzy11"
|
||||
|
||||
@ -229,12 +229,12 @@ if __name__ == "__main__":
|
||||
logger = get_global_logger("123")
|
||||
start_time = time.time()
|
||||
# input_path = r"C:\Users\Administrator\Desktop\new招标文件\工程标"
|
||||
pdf_path=r"C:\Users\Administrator\Desktop\招标文件\招标test文件夹"
|
||||
pdf_path=r"C:\Users\Administrator\Desktop\新建文件夹 (3)\temp\file1736998876340 (1).doc"
|
||||
|
||||
# pdf_path = r"C:\Users\Administrator\Desktop\招标文件\招标02.pdf"
|
||||
# input_path=r"C:\Users\Administrator\Desktop\招标文件\招标test文件夹\zbtest8.pdf"
|
||||
output_folder = r"C:\Users\Administrator\Desktop\招标文件\output22"
|
||||
selection = 2 # 例如：1 - 招标公告, 2 - 评标办法, 3 -资格审查条件 4-投标人须知前附表+正文 5-无效标
|
||||
output_folder = r"C:\Users\Administrator\Desktop\fsdownload\ec7d5328-9c57-450f-baf4-2e5a6f90ed1d\tmp"
|
||||
selection = 4 # 例如：1 - 招标公告, 2 - 评标办法, 3 -资格审查条件 4-投标人须知前附表+正文 5-无效标
|
||||
generated_files = truncate_pdf_main_engineering(pdf_path, output_folder, selection, logger)
|
||||
print(generated_files)
|
||||
# print("生成的文件:", generated_files)
|
||||
|
||||
@ -315,11 +315,11 @@ if __name__ == "__main__":
|
||||
logger = get_global_logger("123")
|
||||
# input_path = r"C:\Users\Administrator\Desktop\new招标文件\货物标"
|
||||
# pdf_path = r"C:\Users\Administrator\Desktop\招标文件-采购类\2024-贵州-贵州医科大学附属医院导视系统零星制作安装项目.pdf"
|
||||
pdf_path=r"C:\Users\Administrator\Desktop\新建文件夹 (3)\temp\251WA0201146未来大厦LED屏幕采购及安装-自行组织货物类招标文件V1.2.pdf"
|
||||
pdf_path=r"C:\Users\Administrator\Desktop\货物标\zbfiles\唐山市公安交通警察支队机动车查验机构视频存储回放系统竞争性谈判-招标文件正文(1).pdf"
|
||||
# input_path = r"C:\Users\Administrator\Desktop\货物标\zbfiles\2-招标文件（广水市教育局封闭管理）.pdf"
|
||||
# pdf_path=r"C:\Users\Administrator\Desktop\文件解析问题\文件解析问题\1414cb9c-7bf4-401c-8761-2acde151b9c2\ztbfile.pdf"
|
||||
output_folder = r"C:\Users\Administrator\Desktop\新建文件夹 (3)\temp"
|
||||
output_folder = r"C:\Users\Administrator\Desktop\货物标\zbfiles\output4444"
|
||||
# output_folder = r"C:\Users\Administrator\Desktop\new招标文件\output2"
|
||||
selection = 5 # 例如：1 - 公告, 2 - 评标办法, 3 - 资格审查后缀有qualification1或qualification2（与评标办法一致） 4.投标人须知前附表part1 投标人须知正文part2 5-采购需求 6-invalid_path
|
||||
selection = 4 # 例如：1 - 公告, 2 - 评标办法, 3 - 资格审查后缀有qualification1或qualification2（与评标办法一致） 4.投标人须知前附表part1 投标人须知正文part2 5-采购需求 6-invalid_path
|
||||
generated_files = truncate_pdf_main_goods(pdf_path, output_folder, selection,logger)
|
||||
print(generated_files)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user