9.26 分段解析完全版

flask_app/main/start_up.py

@@ -93,7 +93,24 @@ def zbparse():
         logger.error('Exception occurred: ' + str(e))
         return jsonify({'error': str(e)}), 500
 
+def post_processing(data,includes):
+    # 初始化结果字典，预设'其他'分类为空字典
+    result = {"其他": {}}
 
+    # 遍历原始字典的每一个键值对
+    for key, value in data.items():
+        if key in includes:
+            # 如果键在includes列表中，直接保留这个键值对
+            result[key] = value
+        else:
+            # 如果键不在includes列表中，将这个键值对加入到'其他'分类中
+            result["其他"][key] = value
+
+    # 如果'其他'分类没有任何内容，可以选择删除这个键
+    if not result["其他"]:
+        del result["其他"]
+
+    return result
 # 分段返回
 def process_and_stream(file_url):
     logger = g.logger
@@ -120,50 +137,48 @@ def process_and_stream(file_url):
 
     # 从 main_processing 获取数据
     for data in main_processing(output_folder, downloaded_filepath, file_type, unique_id):
-        response = {
-            'message': 'Processing',
-            'filename': os.path.basename(downloaded_filepath),
-            'data': data
-        }
-        # 日志记录和流式响应
-        yield f"data: {json.dumps(response, ensure_ascii=False)}\n\n"
-
-        if not data:
-            logger.error(f"Empty data received: {data}")
-            continue
-
-            # 解析 data 作为 JSON 格式数据
         if not data.strip():
             logger.error("Received empty data, skipping JSON parsing.")
-        else:
+            continue  # Skip processing empty data
-            try:
-                parsed_data = json.loads(data)
-            except json.JSONDecodeError as e:
-                logger.error(f"Failed to decode JSON: {e}")
-                logger.error(f"Data received: {data}")
-                continue  # 跳过该数据处理
 
+        try:
+            parsed_data = json.loads(data)
+        except json.JSONDecodeError as e:
+            logger.error(f"Failed to decode JSON: {e}")
+            logger.error(f"Data received: {data}")
+            continue  # Skip data if JSON parsing fails
         # 遍历 parsed_data 只提取内层内容进行合并
         for outer_key, inner_dict in parsed_data.items():
             if isinstance(inner_dict, dict):
                 combined_data.update(inner_dict)
-    logger.info(json.dumps(combined_data, ensure_ascii=False,indent=4))
+                # 日志记录已合并数据
-    # 等待所有数据都处理完后，发送整合后的完整数据
+        # 每次数据更新后，流式返回当前进度
+        response = {
+            'message': 'Processing',
+            'filename': os.path.basename(downloaded_filepath),
+            'data': json.dumps(data, ensure_ascii=False)
+        }
+        yield f"data: {json.dumps(response, ensure_ascii=False)}\n\n"
+    # 日志记录已合并数据
+    logger.info(f"Updated combined data: {json.dumps(combined_data, ensure_ascii=False, indent=4)}")
+    # **保存 combined_data 到 output_folder 下的 'final_result.json'**
+    output_json_path = os.path.join(output_folder, 'final_result.json')
+    includes = ["基础信息", "资格审查", "商务标", "技术标", "无效标与废标项", "投标文件要求", "开评定标流程"]
+    result = post_processing(combined_data, includes)
+    try:
+        with open(output_json_path, 'w', encoding='utf-8') as json_file:
+            json.dump(result, json_file, ensure_ascii=False, indent=4)
+        logger.info(f"Combined data saved to '{output_json_path}'")
+    except IOError as e:
+        logger.error(f"Error saving JSON file: {e}")
+    # 最后发送合并后的完整数据
     complete_response = {
         'message': 'Combined data',
         'filename': os.path.basename(downloaded_filepath),
-        'data': json.dumps(combined_data, ensure_ascii=False)
+        'data': json.dumps(result, ensure_ascii=False)
     }
     yield f"data: {json.dumps(complete_response, ensure_ascii=False)}\n\n"
 
-    # 发送最终响应
-    final_response = {
-        'message': 'File uploaded and processed successfully',
-        'filename': os.path.basename(downloaded_filepath),
-        'data': 'END'
-    }
-    yield f"data: {json.dumps(final_response, ensure_ascii=False)}\n\n"
-
 
 def validate_request():
     if not request.is_json:

flask_app/main/商务标技术标整合.py

@@ -109,6 +109,11 @@ def combine_evaluation_standards(truncate2):
     # return evaluation_combined_res
     return update_json              #商务标技术标整合
 if __name__ == "__main__":
-    truncate2="D:\\flask_project\\flask_app\\static\\output\\c02a12c2-6f7b-49dc-b97f-c3d740c96c21\\ztbfile_evaluation_method.pdf"
+    truncate2="C:\\Users\\Administrator\\Desktop\\fsdownload\\0883895c-e61f-4a99-9308-697fca1d4b77\\ztbfile_evaluation_method.pdf"
-    res=combine_evaluation_standards(truncate2)
+    evaluation_standards_res=combine_evaluation_standards(truncate2)
-    print(json.dumps(res,ensure_ascii=False,indent=4))
+    # 从结果中提取"商务标"和"技术标"
+    technical_standards = {"技术标": evaluation_standards_res.get("技术标", {})}
+    commercial_standards = {"商务标": evaluation_standards_res.get("商务标", {})}
+    # 返回技术标和商务标
+    print(json.dumps(technical_standards,ensure_ascii=False,indent=4))
+    print(json.dumps(commercial_standards, ensure_ascii=False, indent=4))

flask_app/main/基础信息整合.py

@@ -124,8 +124,8 @@ def project_basic_info(knowledge_name,truncate0,output_folder,clause_path):    #
     update_baseinfo_list=combine_basic_info(baseinfo_list)    #整合基础信息核心代码
 
     baseinfo_combined_res = combine_json_results(update_baseinfo_list)  # 返回值是字典
-    return nest_json_under_key(baseinfo_combined_res, "基础信息")    #返回值是json字符串
+    # return nest_json_under_key(baseinfo_combined_res, "基础信息")    #返回值是json字符串
-
+    return {"基础信息":baseinfo_combined_res}
 
 if __name__ == "__main__":
     knowledge_name = "ztb"

flask_app/main/形式响应评审.py

@@ -171,7 +171,7 @@ def reformat_questions(match_keys, input_path, output_folder):
 
 def process_reviews(original_dict_data,knowledge_name, truncate0_jsonpath,clause_json_path,input_file,output_folder):
     combined_results2={}
-    matched_keys = extract_matching_keys(original_dict_data)  #[{'形式评审标准.投标文件签字盖章': '符合第一章“投标人须知”第 3.7.3（4）目 规定'}, {'形式评审标准.多标段投标': '符合第二章“投标人须知”第 10.1款规定'}] 提取值中包含"符合"的字典
+    matched_keys = extract_matching_keys(original_dict_data)  #output:[{'形式评审标准.投标文件签字盖章': '符合第一章“投标人须知”第 3.7.3（4）目 规定'}, {'形式评审标准.多标段投标': '符合第二章“投标人须知”第 10.1款规定'}] 提取值中包含"符合"的字典
     entries_with_numbers, formatted_questions1,entries_with_numbers2, clause2_path = reformat_questions(matched_keys,input_file,output_folder)
     combined_results = process_and_merge_entries(entries_with_numbers, truncate0_jsonpath,
                                                  clause_json_path)  # 调用根据条款号整合json.py

flask_app/main/招标文件解析.py

@@ -118,12 +118,17 @@ def fetch_evaluation_standards(truncate1):  # 评标办法前附表
     # 获取评标办法前附表的字典结果
     evaluation_standards_res = combine_evaluation_standards(truncate1)
 
-    # 从结果中提取"商务标"和"技术标"
+    # 获取技术标和商务标
-    technical_standards = evaluation_standards_res.get("技术标", {})
+    technical_standards = {"技术标": evaluation_standards_res.get("技术标", {})}
-    commercial_standards = evaluation_standards_res.get("商务标", {})
+    commercial_standards = {"商务标": evaluation_standards_res.get("商务标", {})}
+
     logger.info("商务标和技术标 done")
-    # 返回技术标和商务标
+
-    return {"technical_standards": technical_standards, "commercial_standards": commercial_standards}
+    # 返回将 "技术标" 和 "商务标" 包含在新的键中
+    return {
+        "technical_standards": technical_standards,
+        "commercial_standards": commercial_standards
+    }
 
 # def fetch_evaluation_standards(truncate1):  # 评标办法前附表
 #     logger.info("starting商务标技术标...")
@@ -147,8 +152,8 @@ def fetch_bidding_documents_requirements(clause_path):
     fetch_bidding_documents_requirements_json = extract_from_notice(clause_path, 1)
     qualify_nested_res = nest_json_under_key(fetch_bidding_documents_requirements_json, "投标文件要求")
     logger.info("投标文件要求done...")
-    return qualify_nested_res
+    # return qualify_nested_res
-
+    return {"投标文件要求":fetch_bidding_documents_requirements_json}
 
 # 开评定标流程
 def fetch_bid_opening(clause_path):
@@ -156,8 +161,8 @@ def fetch_bid_opening(clause_path):
     fetch_bid_opening_json = extract_from_notice(clause_path, 2)
     qualify_nested_res = nest_json_under_key(fetch_bid_opening_json, "开评定标流程")
     logger.info("开评定标流程done...")
-    return qualify_nested_res
+    # return qualify_nested_res
-
+    return {"开评定标流程":fetch_bid_opening_json}
 
 # def main_processing(output_folder, downloaded_file_path, file_type, unique_id):  # file_type=1->docx  file_type=2->pdf
 #     global logger
@@ -243,12 +248,12 @@ def main_processing(output_folder, downloaded_file_path, file_type, unique_id):
                     commercial_standards = result["commercial_standards"]
 
                     # 分别返回技术标和商务标
-                    yield json.dumps(transform_json_values({'technical_standards': technical_standards}), ensure_ascii=False)
+                    yield json.dumps({'technical_standards': transform_json_values(technical_standards)}, ensure_ascii=False)
-                    yield json.dumps(transform_json_values({'commercial_standards': commercial_standards}), ensure_ascii=False)
+                    yield json.dumps({'commercial_standards': transform_json_values(commercial_standards)}, ensure_ascii=False)
 
                 else:
                     # 处理其他任务的结果
-                    yield json.dumps(transform_json_values({key: result}), ensure_ascii=False)
+                    yield json.dumps({key: transform_json_values(result)}, ensure_ascii=False)
 
             except Exception as exc:
                 logger.error(f"Error processing {key}: {exc}")
@@ -275,7 +280,7 @@ def main_processing(output_folder, downloaded_file_path, file_type, unique_id):
                 key = next(k for k, v in future_dependencies.items() if v == future)
                 try:
                     result = future.result()
-                    yield json.dumps(transform_json_values({key: result}), ensure_ascii=False)
+                    yield json.dumps({key: transform_json_values(result)}, ensure_ascii=False)
                 except Exception as exc:
                     logger.error(f"Error processing {key}: {exc}")
                     yield json.dumps({'error': f'Error processing {key}: {str(exc)}'}, ensure_ascii=False)

flask_app/main/无效标和废标和禁止投标整合.py

@@ -338,8 +338,8 @@ def combine_find_invalid(file_path, output_dir, truncate_json_path,clause_path,t
         combined_dict.update(d)
 
     print("无效标与废标done...")
-    return nest_json_under_key(combined_dict, "无效标与废标项")
+    # return nest_json_under_key(combined_dict, "无效标与废标项")
-
+    return {"无效标与废标项":combined_dict}
 
 if __name__ == '__main__':
     start_time = time.time()

flask_app/main/资格审查模块.py

@@ -25,8 +25,8 @@ def combine_review_standards(truncate1,truncate3,knowledge_name,truncate0_jsonpa
         final_qualify_json = future_qualification.result()
         form_response_dict = future_form_response.result()
     form_response_dict.update(final_qualify_json)
-    return nest_json_under_key(form_response_dict,"资格审查")
+    # return nest_json_under_key(form_response_dict,"资格审查")
-
+    return {"资格审查":form_response_dict}
 
 if __name__ == "__main__":
     input_file="D:\\flask_project\\flask_app\\static\\output\\c02a12c2-6f7b-49dc-b97f-c3d740c96c21\\ztbfile.pdf"

flask_app/货物标/资格审查main.py

@@ -6,6 +6,7 @@ from flask_app.main.基础信息整合 import combine_basic_info
 from flask_app.main.通义千问long import qianwen_long,upload_file
 from flask_app.main.多线程提问 import multi_threading
 from flask_app.main.json_utils import combine_json_results,clean_json_string
+from flask_app.main.形式响应评审 import update_json_data,extract_matching_keys
 #这个字典可能有嵌套，你需要遍历里面的键名，对键名作判断，而不是键值，具体是这样的：如果处于同一层级的键的数量>1并且键名全由数字或点号组成。那么就将这些序号键名全部删除，重新组织成一个字典格式的数据，你可以考虑用字符串列表来保持部分平级的数据
 #对于同级的键，如果数量>1且键名都统一，那么将键名去掉，用列表保持它们的键值
 #对于同一个字典中，可能存在若干键值对，若它们的键值都是""或者"/" 你就将它们的键值删去，它们的键名用字符串列表保存
@@ -98,9 +99,30 @@ def process_dict(data):
         return result['items']
 
     return result
+
+
+def find_chapter_clause_references(data, parent_key=""):
+    result = []
+    # 正则匹配"第x章"或"第x款"
+    chapter_clause_pattern = re.compile(r'第[一二三四五六七八九十\d]+[章款]')
+
+    # 遍历字典中的键值对
+    for key, value in data.items():
+        # 生成当前的完整键名
+        full_key = f"{parent_key}.{key}" if parent_key else key
+
+        if isinstance(value, dict):
+            # 如果值是字典，递归调用函数
+            result.extend(find_chapter_clause_references(value, full_key))
+        elif isinstance(value, str):
+            # 如果值是字符串，检查是否匹配"第x章"或"第x款"
+            if chapter_clause_pattern.search(value):
+                result.append({full_key: value})
+
+    return result
 def qualification_review(truncate_file):
     file_id=upload_file(truncate_file)
-    user_query=["该招标文件中规定的资格性审查标准是怎样的？请以json格式给出，外层为'资格性审查'，你的回答要与原文一致，不可擅自总结删减，也不要回答资格性审查的内容。","该招标文件中规定的符合性审查标准是怎样的？请以json格式给出，外层为'符合性审查'，你的回答要与原文一致，不可擅自总结删减，也不要回答资格性审查的内容。"]
+    user_query=["该招标文件中规定的资格性审查标准是怎样的？请以json格式给出，外层为'资格性审查'，你的回答要与原文一致，不可擅自总结删减，也不要回答有关符合性性审查的内容。","该招标文件中规定的符合性审查标准是怎样的？请以json格式给出，外层为'符合性审查'，你的回答要与原文一致，不可擅自总结删减，也不要回答有关资格性审查的内容。"]
     results=multi_threading(user_query,"",file_id,2)
     combined_res = {}
     for question, response in results:
@@ -109,10 +131,12 @@ def qualification_review(truncate_file):
         processed1 = preprocess_dict(cleaned_data)
         processed2 = process_dict(processed1)
         combined_res.update(processed2)
+    result=find_chapter_clause_references(combined_res)
+    print(result)
  # 整合基础信息核心代码
     return combined_res
 
 if __name__ == "__main__":
-    truncate_file="C:\\Users\\Administrator\\Desktop\\货物标\\output3\\622二次视频会议磋商文件(1)_qualification2.pdf"
+    truncate_file="C:\\Users\\Administrator\\Desktop\\货物标\\output3\\6.2定版视频会议磋商文件_qualification2.pdf"
     res=qualification_review(truncate_file)
     print(json.dumps(res,ensure_ascii=False, indent=4))