2.20 主流程改为多线程测试
This commit is contained in:
parent
e6b46b6949
commit
3e69e9fa9c
@ -1,11 +1,10 @@
|
||||
# -*- encoding:utf-8 -*-
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from flask_app.general.json_utils import clean_json_string
|
||||
from flask_app.general.llm.多线程提问 import multi_threading
|
||||
from flask_app.general.llm.通义千问long import upload_file, qianwen_long_stream
|
||||
from flask_app.old_version.判断是否分包等_old import read_questions_from_judge
|
||||
from flask_app.general.判断是否分包 import read_questions_from_judge
|
||||
|
||||
def get_deviation_requirements(invalid_path):
|
||||
file_id=upload_file(invalid_path)
|
||||
|
||||
@ -2,7 +2,7 @@ import json
|
||||
|
||||
from flask_app.general.json_utils import clean_json_string
|
||||
from flask_app.工程标.投标人须知正文提取指定内容工程标 import extract_from_notice
|
||||
from flask_app.old_version.判断是否分包等_old import judge_whether_main, read_questions_from_judge
|
||||
from flask_app.general.判断是否分包 import judge_whether_main, read_questions_from_judge
|
||||
from flask_app.general.llm.多线程提问 import read_questions_from_file, multi_threading
|
||||
from flask_app.general.llm.通义千问long import upload_file
|
||||
from flask_app.general.通用功能函数 import judge_consortium_bidding
|
||||
|
||||
@ -7,7 +7,7 @@ from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_compl
|
||||
|
||||
from docx import Document
|
||||
|
||||
from flask_app.general.insert_del_pagemark import insert_mark,delete_mark
|
||||
from flask_app.general.insert_del_pagemark import insert_mark, delete_mark
|
||||
from flask_app.general.截取pdf_main import truncate_pdf_multiple
|
||||
from flask_app.general.merge_pdfs import merge_pdfs
|
||||
from flask_app.general.读取文件.clean_pdf import is_pure_image
|
||||
@ -23,7 +23,7 @@ from flask_app.general.商务技术评分提取 import combine_evaluation_standa
|
||||
from flask_app.general.format_change import pdf2docx, docx2pdf
|
||||
|
||||
|
||||
def preprocess_file_main(output_folder, file_path, file_type,logger):
|
||||
def preprocess_file_main(output_folder, file_path, file_type, logger):
|
||||
# 这里是你原本处理请求的地方
|
||||
with multiprocessing.Pool(processes=1) as pool:
|
||||
result = pool.apply(
|
||||
@ -31,9 +31,11 @@ def preprocess_file_main(output_folder, file_path, file_type,logger):
|
||||
args=(output_folder, file_path, file_type, logger)
|
||||
)
|
||||
return result
|
||||
def preprocess_files(output_folder, file_path, file_type,logger):
|
||||
|
||||
|
||||
def preprocess_files(output_folder, file_path, file_type, logger):
|
||||
logger.info("starting 文件预处理...")
|
||||
start_time=time.time()
|
||||
start_time = time.time()
|
||||
is_pure_image_flag = False
|
||||
pdf_path = ""
|
||||
# 根据文件类型处理文件路径
|
||||
@ -46,7 +48,7 @@ def preprocess_files(output_folder, file_path, file_type,logger):
|
||||
elif file_type == 2: # pdf
|
||||
pdf_path = file_path
|
||||
# docx_path = pdf2docx(pdf_path) # 将pdf转换为docx以供上传到知识库
|
||||
elif file_type == 3: #doc
|
||||
elif file_type == 3: # doc
|
||||
# docx_path=doc2docx(downloaded_file_path)
|
||||
pdf_path = docx2pdf(file_path) # 将docx转换为pdf以供后续处理
|
||||
else:
|
||||
@ -54,7 +56,7 @@ def preprocess_files(output_folder, file_path, file_type,logger):
|
||||
return None
|
||||
if not is_pure_image_flag: # 大多数情况 不是纯图片doc/docx
|
||||
# 调用截取PDF多次
|
||||
truncate_files = truncate_pdf_multiple(pdf_path, output_folder,logger,'engineering')
|
||||
truncate_files = truncate_pdf_multiple(pdf_path, output_folder, logger, 'engineering')
|
||||
else:
|
||||
truncate_files = ['', '', '', '', '', file_path, '']
|
||||
# print("切割出的文件："+str(truncate_files))
|
||||
@ -78,22 +80,22 @@ def preprocess_files(output_folder, file_path, file_type,logger):
|
||||
clause_path = convert_clause_to_json(tobidders_notice, output_folder) # 投标人须知正文条款pdf->json
|
||||
# invalid_docpath = copy_docx(docx_path) # docx截取无效标部分
|
||||
# invalid_docpath=pdf2docx(invalid_path)
|
||||
truncate_endtime=time.time()
|
||||
truncate_endtime = time.time()
|
||||
if not is_pure_image_flag:
|
||||
invalid_added_pdf = insert_mark(invalid_path)
|
||||
logger.info(f"文件切分CPU耗时：{truncate_endtime - start_time:.2f} 秒")
|
||||
|
||||
invalid_added_docx = pdf2docx(invalid_added_pdf) #有标记的invalid_path
|
||||
invalid_added_docx = pdf2docx(invalid_added_pdf) # 有标记的invalid_path
|
||||
try:
|
||||
# 尝试加载 .docx 文件
|
||||
doc = Document(invalid_added_docx)
|
||||
print("yes")
|
||||
except Exception as e:
|
||||
# 捕获异常并打印错误信息
|
||||
invalid_added_docx=pdf2docx(invalid_path)
|
||||
invalid_deleted_docx=delete_mark(invalid_added_docx) #无标记的invalid_path
|
||||
invalid_added_docx = pdf2docx(invalid_path)
|
||||
invalid_deleted_docx = delete_mark(invalid_added_docx) # 无标记的invalid_path
|
||||
if not invalid_deleted_docx:
|
||||
invalid_deleted_docx=pdf2docx(invalid_path)
|
||||
invalid_deleted_docx = pdf2docx(invalid_path)
|
||||
else:
|
||||
invalid_deleted_docx = file_path
|
||||
invalid_added_docx = ''
|
||||
@ -101,25 +103,27 @@ def preprocess_files(output_folder, file_path, file_type,logger):
|
||||
# 尝试加载 .docx 文件
|
||||
doc = Document(invalid_deleted_docx)
|
||||
except Exception as e:
|
||||
invalid_deleted_docx=pdf_path if pdf_path!="" else file_path #文档转换还是失败！最后的手段保证文件可用！
|
||||
end_time=time.time()
|
||||
invalid_deleted_docx = pdf_path if pdf_path != "" else file_path # 文档转换还是失败！最后的手段保证文件可用！
|
||||
end_time = time.time()
|
||||
logger.info(f"文件预处理总耗时：{end_time - start_time:.2f} 秒")
|
||||
|
||||
# 返回包含预处理后文件路径的字典
|
||||
return {
|
||||
'invalid_deleted_docx':invalid_deleted_docx,
|
||||
'invalid_deleted_docx': invalid_deleted_docx,
|
||||
'invalid_added_docx': invalid_added_docx,
|
||||
'notice_path':notice_path,
|
||||
'notice_path': notice_path,
|
||||
'tobidders_notice_table': tobidders_notice_table,
|
||||
'evaluation_method':evaluation_method,
|
||||
'evaluation_method': evaluation_method,
|
||||
'qualification': qualification,
|
||||
'merged_baseinfo_path':merged_baseinfo_path,
|
||||
'merged_baseinfo_path_more':merged_baseinfo_path_more,
|
||||
'merged_baseinfo_path': merged_baseinfo_path,
|
||||
'merged_baseinfo_path_more': merged_baseinfo_path_more,
|
||||
'clause_path': clause_path
|
||||
}
|
||||
|
||||
|
||||
# 基本信息
|
||||
def fetch_project_basic_info(invalid_deleted_docx, merged_baseinfo_path, merged_baseinfo_path_more,clause_path, logger):
|
||||
def fetch_project_basic_info(invalid_deleted_docx, merged_baseinfo_path, merged_baseinfo_path_more, clause_path,
|
||||
logger):
|
||||
logger.info("starting 基础信息...")
|
||||
start_time = time.time()
|
||||
try:
|
||||
@ -127,7 +131,8 @@ def fetch_project_basic_info(invalid_deleted_docx, merged_baseinfo_path, merged_
|
||||
merged_baseinfo_path = invalid_deleted_docx
|
||||
if not merged_baseinfo_path_more:
|
||||
merged_baseinfo_path_more = invalid_deleted_docx
|
||||
basic_res = combine_basic_info(merged_baseinfo_path, merged_baseinfo_path_more, clause_path,invalid_deleted_docx)
|
||||
basic_res = combine_basic_info(merged_baseinfo_path, merged_baseinfo_path_more, clause_path,
|
||||
invalid_deleted_docx)
|
||||
result = basic_res
|
||||
end_time = time.time()
|
||||
logger.info(f"基础信息 done，耗时：{end_time - start_time:.2f} 秒")
|
||||
@ -138,7 +143,8 @@ def fetch_project_basic_info(invalid_deleted_docx, merged_baseinfo_path, merged_
|
||||
return result
|
||||
|
||||
|
||||
def fetch_qualification_review(evaluation_method, qualification, output_folder, tobidders_notice_table, clause_path, invalid_deleted_docx, merged_baseinfo_path, notice_path, logger):
|
||||
def fetch_qualification_review(evaluation_method, qualification, output_folder, tobidders_notice_table, clause_path,
|
||||
invalid_deleted_docx, merged_baseinfo_path, notice_path, logger):
|
||||
logger.info("starting 资格审查...")
|
||||
start_time = time.time()
|
||||
try:
|
||||
@ -149,7 +155,8 @@ def fetch_qualification_review(evaluation_method, qualification, output_folder,
|
||||
if not merged_baseinfo_path:
|
||||
merged_baseinfo_path = invalid_deleted_docx
|
||||
review_standards_res = combine_review_standards(
|
||||
evaluation_method, qualification, output_folder, tobidders_notice_table, clause_path, invalid_deleted_docx, merged_baseinfo_path, notice_path)
|
||||
evaluation_method, qualification, output_folder, tobidders_notice_table, clause_path, invalid_deleted_docx,
|
||||
merged_baseinfo_path, notice_path)
|
||||
result = review_standards_res
|
||||
end_time = time.time()
|
||||
logger.info(f"资格审查 done，耗时：{end_time - start_time:.2f} 秒")
|
||||
@ -161,12 +168,12 @@ def fetch_qualification_review(evaluation_method, qualification, output_folder,
|
||||
|
||||
|
||||
# 评分细则 流式
|
||||
def fetch_evaluation_standards(invalid_deleted_docx, evaluation_method,logger):
|
||||
def fetch_evaluation_standards(invalid_deleted_docx, evaluation_method, logger):
|
||||
logger.info("starting 商务标和技术标...")
|
||||
start_time = time.time()
|
||||
if not evaluation_method:
|
||||
evaluation_method = invalid_deleted_docx
|
||||
evaluation_standards_res = combine_evaluation_standards(evaluation_method,invalid_deleted_docx,1)
|
||||
evaluation_standards_res = combine_evaluation_standards(evaluation_method, invalid_deleted_docx, 1)
|
||||
technical_standards = {"技术评分": evaluation_standards_res.get("技术评分", {})}
|
||||
commercial_standards = {"商务评分": evaluation_standards_res.get("商务评分", {})}
|
||||
end_time = time.time()
|
||||
@ -201,7 +208,8 @@ def fetch_bidding_documents_requirements(invalid_deleted_docx, merged_baseinfo_p
|
||||
if not merged_baseinfo_path_more:
|
||||
merged_baseinfo_path_more = invalid_deleted_docx
|
||||
selection = 1
|
||||
fetch_bidding_documents_requirements_json = extract_from_notice(merged_baseinfo_path_more, clause_path, selection)
|
||||
fetch_bidding_documents_requirements_json = extract_from_notice(merged_baseinfo_path_more, clause_path,
|
||||
selection)
|
||||
result = {"投标文件要求": fetch_bidding_documents_requirements_json}
|
||||
end_time = time.time()
|
||||
logger.info(f"投标文件要求 done，耗时：{end_time - start_time:.2f} 秒")
|
||||
@ -211,6 +219,7 @@ def fetch_bidding_documents_requirements(invalid_deleted_docx, merged_baseinfo_p
|
||||
result = {"投标文件要求": {}}
|
||||
return result
|
||||
|
||||
|
||||
# 开评定标流程
|
||||
def fetch_bid_opening(invalid_deleted_docx, merged_baseinfo_path_more, clause_path, logger):
|
||||
logger.info("starting 开评定标流程...")
|
||||
@ -229,30 +238,42 @@ def fetch_bid_opening(invalid_deleted_docx, merged_baseinfo_path_more, clause_pa
|
||||
result = {"开评定标流程": {}}
|
||||
return result
|
||||
|
||||
|
||||
def engineering_bid_main(output_folder, file_path, file_type, unique_id):
|
||||
logger = get_global_logger(unique_id)
|
||||
# 预处理文件，获取处理后的数据
|
||||
processed_data = preprocess_file_main(output_folder, file_path, file_type,logger)
|
||||
processed_data = preprocess_file_main(output_folder, file_path, file_type, logger)
|
||||
if not processed_data:
|
||||
error_response = {
|
||||
'error': '文件预处理失败。请检查文件类型并重试。'
|
||||
}
|
||||
yield json.dumps(error_response, ensure_ascii=False)
|
||||
return # 停止进一步处理
|
||||
with ProcessPoolExecutor() as executor:
|
||||
with ThreadPoolExecutor() as executor:
|
||||
# 立即启动不依赖 knowledge_name 和 index 的任务
|
||||
futures = {
|
||||
'base_info': executor.submit(fetch_project_basic_info,processed_data['invalid_deleted_docx'] ,processed_data['merged_baseinfo_path'],processed_data['merged_baseinfo_path_more'],
|
||||
processed_data['clause_path'],logger),
|
||||
'base_info': executor.submit(fetch_project_basic_info, processed_data['invalid_deleted_docx'],
|
||||
processed_data['merged_baseinfo_path'],
|
||||
processed_data['merged_baseinfo_path_more'],
|
||||
processed_data['clause_path'], logger),
|
||||
'qualification_review': executor.submit(fetch_qualification_review, processed_data['evaluation_method'],
|
||||
processed_data['qualification'], output_folder,
|
||||
processed_data['tobidders_notice_table'],
|
||||
processed_data['clause_path'], processed_data['invalid_deleted_docx'],
|
||||
processed_data['merged_baseinfo_path'],processed_data['notice_path'],logger),
|
||||
'evaluation_standards': executor.submit(fetch_evaluation_standards, processed_data['invalid_deleted_docx'],processed_data['evaluation_method'],logger),
|
||||
'invalid_requirements': executor.submit(fetch_invalid_requirements, processed_data['invalid_added_docx'],output_folder,logger),
|
||||
'bidding_documents_requirements': executor.submit(fetch_bidding_documents_requirements,processed_data['invalid_deleted_docx'], processed_data['merged_baseinfo_path_more'],processed_data['clause_path'],logger),
|
||||
'opening_bid': executor.submit(fetch_bid_opening,processed_data['invalid_deleted_docx'],processed_data['merged_baseinfo_path_more'], processed_data['clause_path'],logger)
|
||||
processed_data['clause_path'],
|
||||
processed_data['invalid_deleted_docx'],
|
||||
processed_data['merged_baseinfo_path'],
|
||||
processed_data['notice_path'], logger),
|
||||
'evaluation_standards': executor.submit(fetch_evaluation_standards, processed_data['invalid_deleted_docx'],
|
||||
processed_data['evaluation_method'], logger),
|
||||
'invalid_requirements': executor.submit(fetch_invalid_requirements, processed_data['invalid_added_docx'],
|
||||
output_folder, logger),
|
||||
'bidding_documents_requirements': executor.submit(fetch_bidding_documents_requirements,
|
||||
processed_data['invalid_deleted_docx'],
|
||||
processed_data['merged_baseinfo_path_more'],
|
||||
processed_data['clause_path'], logger),
|
||||
'opening_bid': executor.submit(fetch_bid_opening, processed_data['invalid_deleted_docx'],
|
||||
processed_data['merged_baseinfo_path_more'], processed_data['clause_path'],
|
||||
logger)
|
||||
}
|
||||
|
||||
# 提前处理这些不依赖的任务，按完成顺序返回
|
||||
@ -267,8 +288,10 @@ def engineering_bid_main(output_folder, file_path, file_type, unique_id):
|
||||
commercial_standards = result["commercial_standards"]
|
||||
|
||||
# 分别返回技术标和商务标
|
||||
yield json.dumps({'technical_standards': transform_json_values(technical_standards)}, ensure_ascii=False)
|
||||
yield json.dumps({'commercial_standards': transform_json_values(commercial_standards)}, ensure_ascii=False)
|
||||
yield json.dumps({'technical_standards': transform_json_values(technical_standards)},
|
||||
ensure_ascii=False)
|
||||
yield json.dumps({'commercial_standards': transform_json_values(commercial_standards)},
|
||||
ensure_ascii=False)
|
||||
|
||||
else:
|
||||
# 处理其他任务的结果
|
||||
@ -285,11 +308,12 @@ def engineering_bid_main(output_folder, file_path, file_type, unique_id):
|
||||
yield json.dumps(default_evaluation, ensure_ascii=False)
|
||||
# yield json.dumps({'error': f'Error processing {key}: {str(exc)}'}, ensure_ascii=False)
|
||||
|
||||
#TODO:基本信息，判断是否这里，打勾逻辑取消了。
|
||||
|
||||
# TODO:基本信息，判断是否这里，打勾逻辑取消了。
|
||||
if __name__ == "__main__":
|
||||
start_time = time.time()
|
||||
output_folder = r"C:\Users\Administrator\Desktop\fsdownload\bb2747ad-5578-4b3b-b60f-3a2d1b672b6f"
|
||||
file_type = 2 #1:docx 2:pdf 3:其他
|
||||
file_type = 2 # 1:docx 2:pdf 3:其他
|
||||
input_file = r"C:\Users\Administrator\Desktop\fsdownload\bb2747ad-5578-4b3b-b60f-3a2d1b672b6f\zytest.pdf"
|
||||
print("yes")
|
||||
for output in engineering_bid_main(output_folder, input_file, file_type, "zytest"):
|
||||
|
||||
@ -15,7 +15,7 @@ from flask_app.general.投标人须知正文条款提取成json文件 import con
|
||||
from flask_app.general.无效标和废标公共代码 import combine_find_invalid
|
||||
from flask_app.货物标.资格审查main import combine_qualification_review
|
||||
from flask_app.general.商务技术评分提取 import combine_evaluation_standards
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
def preprocess_file_main(output_folder, file_path, file_type,logger):
|
||||
# 这里是你原本处理请求的地方
|
||||
with multiprocessing.Pool(processes=1) as pool:
|
||||
@ -241,11 +241,11 @@ def goods_bid_main(output_folder, file_path, file_type, unique_id):
|
||||
}
|
||||
yield json.dumps(error_response, ensure_ascii=False)
|
||||
return # 停止进一步处理
|
||||
with ProcessPoolExecutor() as executor: #开启子进程，能保证运行结束后回收资源消耗
|
||||
with ThreadPoolExecutor() as executor: #开启子进程，能保证运行结束后回收资源消耗
|
||||
# 立即启动不依赖 knowledge_name 和 index 的任务
|
||||
futures = {
|
||||
'evaluation_standards': executor.submit(fetch_evaluation_standards,processed_data['invalid_deleted_docx'], #技术评分 商务评分
|
||||
processed_data['evaluation_method_path'],logger),
|
||||
# 'evaluation_standards': executor.submit(fetch_evaluation_standards,processed_data['invalid_deleted_docx'], #技术评分 商务评分
|
||||
# processed_data['evaluation_method_path'],logger),
|
||||
|
||||
'invalid_requirements': executor.submit(fetch_invalid_requirements, processed_data['invalid_added_docx'], #无效标与废标项
|
||||
output_folder,logger),
|
||||
@ -256,12 +256,12 @@ def goods_bid_main(output_folder, file_path, file_type, unique_id):
|
||||
'opening_bid': executor.submit(fetch_bid_opening, processed_data['invalid_deleted_docx'],processed_data['merged_baseinfo_path'],
|
||||
processed_data['clause_path'],logger), #开评定标流程
|
||||
|
||||
'base_info': executor.submit(fetch_project_basic_info, processed_data['invalid_deleted_docx'],processed_data['merged_baseinfo_path'], #基础信息
|
||||
processed_data['procurement_path'],processed_data['clause_path'],logger),
|
||||
|
||||
'qualification_review': executor.submit(fetch_qualification_review, processed_data['invalid_deleted_docx'], #资格审查
|
||||
processed_data['qualification_path'],
|
||||
processed_data['notice_path'],logger),
|
||||
# 'base_info': executor.submit(fetch_project_basic_info, processed_data['invalid_deleted_docx'],processed_data['merged_baseinfo_path'], #基础信息
|
||||
# processed_data['procurement_path'],processed_data['clause_path'],logger),
|
||||
#
|
||||
# 'qualification_review': executor.submit(fetch_qualification_review, processed_data['invalid_deleted_docx'], #资格审查
|
||||
# processed_data['qualification_path'],
|
||||
# processed_data['notice_path'],logger),
|
||||
}
|
||||
collected_good_list = []
|
||||
# 提前处理这些不依赖的任务，按完成顺序返回
|
||||
|
||||
@ -5,7 +5,7 @@ import concurrent.futures
|
||||
from flask_app.general.json_utils import clean_json_string, add_outer_key
|
||||
from flask_app.general.通用功能函数 import process_judge_questions, aggregate_basic_info
|
||||
from flask_app.general.投标人须知正文提取指定内容 import extract_from_notice
|
||||
from flask_app.old_version.判断是否分包等_old import merge_json_to_list
|
||||
from flask_app.general.判断是否分包 import merge_json_to_list
|
||||
from flask_app.general.llm.多线程提问 import read_questions_from_file, multi_threading
|
||||
from flask_app.general.llm.通义千问long import upload_file
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ from flask_app.general.json_utils import clean_json_string, add_outer_key
|
||||
from flask_app.general.通用功能函数 import process_judge_questions, aggregate_basic_info
|
||||
from flask_app.general.llm.多线程提问 import read_questions_from_file, multi_threading
|
||||
from flask_app.general.llm.通义千问long import upload_file
|
||||
from flask_app.old_version.判断是否分包等_old import merge_json_to_list
|
||||
from flask_app.general.判断是否分包 import merge_json_to_list
|
||||
from flask_app.general.投标人须知正文提取指定内容 import extract_from_notice
|
||||
from flask_app.货物标.提取采购需求main import fetch_procurement_reqs
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user