diff --git a/flask_app/old_version/判断是否分包等_old.py b/flask_app/general/判断是否分包.py similarity index 100% rename from flask_app/old_version/判断是否分包等_old.py rename to flask_app/general/判断是否分包.py diff --git a/flask_app/general/通用功能函数.py b/flask_app/general/通用功能函数.py index f400c63..0bb495c 100644 --- a/flask_app/general/通用功能函数.py +++ b/flask_app/general/通用功能函数.py @@ -1,11 +1,10 @@ # -*- encoding:utf-8 -*- -import json import logging import re from flask_app.general.json_utils import clean_json_string from flask_app.general.llm.多线程提问 import multi_threading from flask_app.general.llm.通义千问long import upload_file, qianwen_long_stream -from flask_app.old_version.判断是否分包等_old import read_questions_from_judge +from flask_app.general.判断是否分包 import read_questions_from_judge def get_deviation_requirements(invalid_path): file_id=upload_file(invalid_path) diff --git a/flask_app/old_version/基础信息整合_old.py b/flask_app/old_version/基础信息整合_old.py index 24d386d..2329c4c 100644 --- a/flask_app/old_version/基础信息整合_old.py +++ b/flask_app/old_version/基础信息整合_old.py @@ -2,7 +2,7 @@ import json from flask_app.general.json_utils import clean_json_string from flask_app.工程标.投标人须知正文提取指定内容工程标 import extract_from_notice -from flask_app.old_version.判断是否分包等_old import judge_whether_main, read_questions_from_judge +from flask_app.general.判断是否分包 import judge_whether_main, read_questions_from_judge from flask_app.general.llm.多线程提问 import read_questions_from_file, multi_threading from flask_app.general.llm.通义千问long import upload_file from flask_app.general.通用功能函数 import judge_consortium_bidding diff --git a/flask_app/routes/工程标解析main.py b/flask_app/routes/工程标解析main.py index 27771e7..eccae27 100644 --- a/flask_app/routes/工程标解析main.py +++ b/flask_app/routes/工程标解析main.py @@ -7,7 +7,7 @@ from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor, as_compl from docx import Document -from flask_app.general.insert_del_pagemark import insert_mark,delete_mark +from flask_app.general.insert_del_pagemark import insert_mark, delete_mark from flask_app.general.截取pdf_main import truncate_pdf_multiple from flask_app.general.merge_pdfs import merge_pdfs from flask_app.general.读取文件.clean_pdf import is_pure_image @@ -23,7 +23,7 @@ from flask_app.general.商务技术评分提取 import combine_evaluation_standa from flask_app.general.format_change import pdf2docx, docx2pdf -def preprocess_file_main(output_folder, file_path, file_type,logger): +def preprocess_file_main(output_folder, file_path, file_type, logger): # 这里是你原本处理请求的地方 with multiprocessing.Pool(processes=1) as pool: result = pool.apply( @@ -31,9 +31,11 @@ def preprocess_file_main(output_folder, file_path, file_type,logger): args=(output_folder, file_path, file_type, logger) ) return result -def preprocess_files(output_folder, file_path, file_type,logger): + + +def preprocess_files(output_folder, file_path, file_type, logger): logger.info("starting 文件预处理...") - start_time=time.time() + start_time = time.time() is_pure_image_flag = False pdf_path = "" # 根据文件类型处理文件路径 @@ -46,7 +48,7 @@ def preprocess_files(output_folder, file_path, file_type,logger): elif file_type == 2: # pdf pdf_path = file_path # docx_path = pdf2docx(pdf_path) # 将pdf转换为docx以供上传到知识库 - elif file_type == 3: #doc + elif file_type == 3: # doc # docx_path=doc2docx(downloaded_file_path) pdf_path = docx2pdf(file_path) # 将docx转换为pdf以供后续处理 else: @@ -54,7 +56,7 @@ def preprocess_files(output_folder, file_path, file_type,logger): return None if not is_pure_image_flag: # 大多数情况 不是纯图片doc/docx # 调用截取PDF多次 - truncate_files = truncate_pdf_multiple(pdf_path, output_folder,logger,'engineering') + truncate_files = truncate_pdf_multiple(pdf_path, output_folder, logger, 'engineering') else: truncate_files = ['', '', '', '', '', file_path, ''] # print("切割出的文件："+str(truncate_files)) @@ -78,22 +80,22 @@ def preprocess_files(output_folder, file_path, file_type,logger): clause_path = convert_clause_to_json(tobidders_notice, output_folder) # 投标人须知正文条款pdf->json # invalid_docpath = copy_docx(docx_path) # docx截取无效标部分 # invalid_docpath=pdf2docx(invalid_path) - truncate_endtime=time.time() + truncate_endtime = time.time() if not is_pure_image_flag: invalid_added_pdf = insert_mark(invalid_path) logger.info(f"文件切分CPU耗时：{truncate_endtime - start_time:.2f} 秒") - invalid_added_docx = pdf2docx(invalid_added_pdf) #有标记的invalid_path + invalid_added_docx = pdf2docx(invalid_added_pdf) # 有标记的invalid_path try: # 尝试加载 .docx 文件 doc = Document(invalid_added_docx) print("yes") except Exception as e: # 捕获异常并打印错误信息 - invalid_added_docx=pdf2docx(invalid_path) - invalid_deleted_docx=delete_mark(invalid_added_docx) #无标记的invalid_path + invalid_added_docx = pdf2docx(invalid_path) + invalid_deleted_docx = delete_mark(invalid_added_docx) # 无标记的invalid_path if not invalid_deleted_docx: - invalid_deleted_docx=pdf2docx(invalid_path) + invalid_deleted_docx = pdf2docx(invalid_path) else: invalid_deleted_docx = file_path invalid_added_docx = '' @@ -101,25 +103,27 @@ def preprocess_files(output_folder, file_path, file_type,logger): # 尝试加载 .docx 文件 doc = Document(invalid_deleted_docx) except Exception as e: - invalid_deleted_docx=pdf_path if pdf_path!="" else file_path #文档转换还是失败！最后的手段保证文件可用！ - end_time=time.time() + invalid_deleted_docx = pdf_path if pdf_path != "" else file_path # 文档转换还是失败！最后的手段保证文件可用！ + end_time = time.time() logger.info(f"文件预处理总耗时：{end_time - start_time:.2f} 秒") # 返回包含预处理后文件路径的字典 return { - 'invalid_deleted_docx':invalid_deleted_docx, + 'invalid_deleted_docx': invalid_deleted_docx, 'invalid_added_docx': invalid_added_docx, - 'notice_path':notice_path, + 'notice_path': notice_path, 'tobidders_notice_table': tobidders_notice_table, - 'evaluation_method':evaluation_method, + 'evaluation_method': evaluation_method, 'qualification': qualification, - 'merged_baseinfo_path':merged_baseinfo_path, - 'merged_baseinfo_path_more':merged_baseinfo_path_more, + 'merged_baseinfo_path': merged_baseinfo_path, + 'merged_baseinfo_path_more': merged_baseinfo_path_more, 'clause_path': clause_path } + # 基本信息 -def fetch_project_basic_info(invalid_deleted_docx, merged_baseinfo_path, merged_baseinfo_path_more,clause_path, logger): +def fetch_project_basic_info(invalid_deleted_docx, merged_baseinfo_path, merged_baseinfo_path_more, clause_path, + logger): logger.info("starting 基础信息...") start_time = time.time() try: @@ -127,7 +131,8 @@ def fetch_project_basic_info(invalid_deleted_docx, merged_baseinfo_path, merged_ merged_baseinfo_path = invalid_deleted_docx if not merged_baseinfo_path_more: merged_baseinfo_path_more = invalid_deleted_docx - basic_res = combine_basic_info(merged_baseinfo_path, merged_baseinfo_path_more, clause_path,invalid_deleted_docx) + basic_res = combine_basic_info(merged_baseinfo_path, merged_baseinfo_path_more, clause_path, + invalid_deleted_docx) result = basic_res end_time = time.time() logger.info(f"基础信息 done，耗时：{end_time - start_time:.2f} 秒") @@ -138,7 +143,8 @@ def fetch_project_basic_info(invalid_deleted_docx, merged_baseinfo_path, merged_ return result -def fetch_qualification_review(evaluation_method, qualification, output_folder, tobidders_notice_table, clause_path, invalid_deleted_docx, merged_baseinfo_path, notice_path, logger): +def fetch_qualification_review(evaluation_method, qualification, output_folder, tobidders_notice_table, clause_path, + invalid_deleted_docx, merged_baseinfo_path, notice_path, logger): logger.info("starting 资格审查...") start_time = time.time() try: @@ -149,7 +155,8 @@ def fetch_qualification_review(evaluation_method, qualification, output_folder, if not merged_baseinfo_path: merged_baseinfo_path = invalid_deleted_docx review_standards_res = combine_review_standards( - evaluation_method, qualification, output_folder, tobidders_notice_table, clause_path, invalid_deleted_docx, merged_baseinfo_path, notice_path) + evaluation_method, qualification, output_folder, tobidders_notice_table, clause_path, invalid_deleted_docx, + merged_baseinfo_path, notice_path) result = review_standards_res end_time = time.time() logger.info(f"资格审查 done，耗时：{end_time - start_time:.2f} 秒") @@ -161,12 +168,12 @@ def fetch_qualification_review(evaluation_method, qualification, output_folder, # 评分细则 流式 -def fetch_evaluation_standards(invalid_deleted_docx, evaluation_method,logger): +def fetch_evaluation_standards(invalid_deleted_docx, evaluation_method, logger): logger.info("starting 商务标和技术标...") start_time = time.time() if not evaluation_method: evaluation_method = invalid_deleted_docx - evaluation_standards_res = combine_evaluation_standards(evaluation_method,invalid_deleted_docx,1) + evaluation_standards_res = combine_evaluation_standards(evaluation_method, invalid_deleted_docx, 1) technical_standards = {"技术评分": evaluation_standards_res.get("技术评分", {})} commercial_standards = {"商务评分": evaluation_standards_res.get("商务评分", {})} end_time = time.time() @@ -201,7 +208,8 @@ def fetch_bidding_documents_requirements(invalid_deleted_docx, merged_baseinfo_p if not merged_baseinfo_path_more: merged_baseinfo_path_more = invalid_deleted_docx selection = 1 - fetch_bidding_documents_requirements_json = extract_from_notice(merged_baseinfo_path_more, clause_path, selection) + fetch_bidding_documents_requirements_json = extract_from_notice(merged_baseinfo_path_more, clause_path, + selection) result = {"投标文件要求": fetch_bidding_documents_requirements_json} end_time = time.time() logger.info(f"投标文件要求 done，耗时：{end_time - start_time:.2f} 秒") @@ -211,6 +219,7 @@ def fetch_bidding_documents_requirements(invalid_deleted_docx, merged_baseinfo_p result = {"投标文件要求": {}} return result + # 开评定标流程 def fetch_bid_opening(invalid_deleted_docx, merged_baseinfo_path_more, clause_path, logger): logger.info("starting 开评定标流程...") @@ -229,30 +238,42 @@ def fetch_bid_opening(invalid_deleted_docx, merged_baseinfo_path_more, clause_pa result = {"开评定标流程": {}} return result + def engineering_bid_main(output_folder, file_path, file_type, unique_id): logger = get_global_logger(unique_id) # 预处理文件，获取处理后的数据 - processed_data = preprocess_file_main(output_folder, file_path, file_type,logger) + processed_data = preprocess_file_main(output_folder, file_path, file_type, logger) if not processed_data: error_response = { 'error': '文件预处理失败。请检查文件类型并重试。' } yield json.dumps(error_response, ensure_ascii=False) return # 停止进一步处理 - with ProcessPoolExecutor() as executor: + with ThreadPoolExecutor() as executor: # 立即启动不依赖 knowledge_name 和 index 的任务 futures = { - 'base_info': executor.submit(fetch_project_basic_info,processed_data['invalid_deleted_docx'] ,processed_data['merged_baseinfo_path'],processed_data['merged_baseinfo_path_more'], - processed_data['clause_path'],logger), + 'base_info': executor.submit(fetch_project_basic_info, processed_data['invalid_deleted_docx'], + processed_data['merged_baseinfo_path'], + processed_data['merged_baseinfo_path_more'], + processed_data['clause_path'], logger), 'qualification_review': executor.submit(fetch_qualification_review, processed_data['evaluation_method'], processed_data['qualification'], output_folder, processed_data['tobidders_notice_table'], - processed_data['clause_path'], processed_data['invalid_deleted_docx'], - processed_data['merged_baseinfo_path'],processed_data['notice_path'],logger), - 'evaluation_standards': executor.submit(fetch_evaluation_standards, processed_data['invalid_deleted_docx'],processed_data['evaluation_method'],logger), - 'invalid_requirements': executor.submit(fetch_invalid_requirements, processed_data['invalid_added_docx'],output_folder,logger), - 'bidding_documents_requirements': executor.submit(fetch_bidding_documents_requirements,processed_data['invalid_deleted_docx'], processed_data['merged_baseinfo_path_more'],processed_data['clause_path'],logger), - 'opening_bid': executor.submit(fetch_bid_opening,processed_data['invalid_deleted_docx'],processed_data['merged_baseinfo_path_more'], processed_data['clause_path'],logger) + processed_data['clause_path'], + processed_data['invalid_deleted_docx'], + processed_data['merged_baseinfo_path'], + processed_data['notice_path'], logger), + 'evaluation_standards': executor.submit(fetch_evaluation_standards, processed_data['invalid_deleted_docx'], + processed_data['evaluation_method'], logger), + 'invalid_requirements': executor.submit(fetch_invalid_requirements, processed_data['invalid_added_docx'], + output_folder, logger), + 'bidding_documents_requirements': executor.submit(fetch_bidding_documents_requirements, + processed_data['invalid_deleted_docx'], + processed_data['merged_baseinfo_path_more'], + processed_data['clause_path'], logger), + 'opening_bid': executor.submit(fetch_bid_opening, processed_data['invalid_deleted_docx'], + processed_data['merged_baseinfo_path_more'], processed_data['clause_path'], + logger) } # 提前处理这些不依赖的任务，按完成顺序返回 @@ -267,8 +288,10 @@ def engineering_bid_main(output_folder, file_path, file_type, unique_id): commercial_standards = result["commercial_standards"] # 分别返回技术标和商务标 - yield json.dumps({'technical_standards': transform_json_values(technical_standards)}, ensure_ascii=False) - yield json.dumps({'commercial_standards': transform_json_values(commercial_standards)}, ensure_ascii=False) + yield json.dumps({'technical_standards': transform_json_values(technical_standards)}, + ensure_ascii=False) + yield json.dumps({'commercial_standards': transform_json_values(commercial_standards)}, + ensure_ascii=False) else: # 处理其他任务的结果 @@ -285,11 +308,12 @@ def engineering_bid_main(output_folder, file_path, file_type, unique_id): yield json.dumps(default_evaluation, ensure_ascii=False) # yield json.dumps({'error': f'Error processing {key}: {str(exc)}'}, ensure_ascii=False) -#TODO:基本信息，判断是否这里，打勾逻辑取消了。 + +# TODO:基本信息，判断是否这里，打勾逻辑取消了。 if __name__ == "__main__": start_time = time.time() output_folder = r"C:\Users\Administrator\Desktop\fsdownload\bb2747ad-5578-4b3b-b60f-3a2d1b672b6f" - file_type = 2 #1:docx 2:pdf 3:其他 + file_type = 2 # 1:docx 2:pdf 3:其他 input_file = r"C:\Users\Administrator\Desktop\fsdownload\bb2747ad-5578-4b3b-b60f-3a2d1b672b6f\zytest.pdf" print("yes") for output in engineering_bid_main(output_folder, input_file, file_type, "zytest"): diff --git a/flask_app/routes/货物标解析main.py b/flask_app/routes/货物标解析main.py index 4fa4f44..3430801 100644 --- a/flask_app/routes/货物标解析main.py +++ b/flask_app/routes/货物标解析main.py @@ -15,7 +15,7 @@ from flask_app.general.投标人须知正文条款提取成json文件 import con from flask_app.general.无效标和废标公共代码 import combine_find_invalid from flask_app.货物标.资格审查main import combine_qualification_review from flask_app.general.商务技术评分提取 import combine_evaluation_standards - +from concurrent.futures import ThreadPoolExecutor def preprocess_file_main(output_folder, file_path, file_type,logger): # 这里是你原本处理请求的地方 with multiprocessing.Pool(processes=1) as pool: @@ -241,11 +241,11 @@ def goods_bid_main(output_folder, file_path, file_type, unique_id): } yield json.dumps(error_response, ensure_ascii=False) return # 停止进一步处理 - with ProcessPoolExecutor() as executor: #开启子进程，能保证运行结束后回收资源消耗 + with ThreadPoolExecutor() as executor: #开启子进程，能保证运行结束后回收资源消耗 # 立即启动不依赖 knowledge_name 和 index 的任务 futures = { - 'evaluation_standards': executor.submit(fetch_evaluation_standards,processed_data['invalid_deleted_docx'], #技术评分 商务评分 - processed_data['evaluation_method_path'],logger), + # 'evaluation_standards': executor.submit(fetch_evaluation_standards,processed_data['invalid_deleted_docx'], #技术评分 商务评分 + # processed_data['evaluation_method_path'],logger), 'invalid_requirements': executor.submit(fetch_invalid_requirements, processed_data['invalid_added_docx'], #无效标与废标项 output_folder,logger), @@ -256,12 +256,12 @@ def goods_bid_main(output_folder, file_path, file_type, unique_id): 'opening_bid': executor.submit(fetch_bid_opening, processed_data['invalid_deleted_docx'],processed_data['merged_baseinfo_path'], processed_data['clause_path'],logger), #开评定标流程 - 'base_info': executor.submit(fetch_project_basic_info, processed_data['invalid_deleted_docx'],processed_data['merged_baseinfo_path'], #基础信息 - processed_data['procurement_path'],processed_data['clause_path'],logger), - - 'qualification_review': executor.submit(fetch_qualification_review, processed_data['invalid_deleted_docx'], #资格审查 - processed_data['qualification_path'], - processed_data['notice_path'],logger), + # 'base_info': executor.submit(fetch_project_basic_info, processed_data['invalid_deleted_docx'],processed_data['merged_baseinfo_path'], #基础信息 + # processed_data['procurement_path'],processed_data['clause_path'],logger), + # + # 'qualification_review': executor.submit(fetch_qualification_review, processed_data['invalid_deleted_docx'], #资格审查 + # processed_data['qualification_path'], + # processed_data['notice_path'],logger), } collected_good_list = [] # 提前处理这些不依赖的任务，按完成顺序返回 diff --git a/flask_app/工程标/基础信息整合工程标.py b/flask_app/工程标/基础信息整合工程标.py index aae4c79..8bf50b2 100644 --- a/flask_app/工程标/基础信息整合工程标.py +++ b/flask_app/工程标/基础信息整合工程标.py @@ -5,7 +5,7 @@ import concurrent.futures from flask_app.general.json_utils import clean_json_string, add_outer_key from flask_app.general.通用功能函数 import process_judge_questions, aggregate_basic_info from flask_app.general.投标人须知正文提取指定内容 import extract_from_notice -from flask_app.old_version.判断是否分包等_old import merge_json_to_list +from flask_app.general.判断是否分包 import merge_json_to_list from flask_app.general.llm.多线程提问 import read_questions_from_file, multi_threading from flask_app.general.llm.通义千问long import upload_file diff --git a/flask_app/货物标/基础信息解析货物标版.py b/flask_app/货物标/基础信息解析货物标版.py index 2a45132..5fb55c7 100644 --- a/flask_app/货物标/基础信息解析货物标版.py +++ b/flask_app/货物标/基础信息解析货物标版.py @@ -7,7 +7,7 @@ from flask_app.general.json_utils import clean_json_string, add_outer_key from flask_app.general.通用功能函数 import process_judge_questions, aggregate_basic_info from flask_app.general.llm.多线程提问 import read_questions_from_file, multi_threading from flask_app.general.llm.通义千问long import upload_file -from flask_app.old_version.判断是否分包等_old import merge_json_to_list +from flask_app.general.判断是否分包 import merge_json_to_list from flask_app.general.投标人须知正文提取指定内容 import extract_from_notice from flask_app.货物标.提取采购需求main import fetch_procurement_reqs