diff --git a/flask_app/general/读取文件/按页读取pdf.py b/flask_app/general/读取文件/按页读取pdf.py index e44a222..c764a11 100644 --- a/flask_app/general/读取文件/按页读取pdf.py +++ b/flask_app/general/读取文件/按页读取pdf.py @@ -25,7 +25,7 @@ def extract_text_by_page_fitz(file_path): return result -def extract_text_by_page(file_path): +def extract_text_by_page_pypdf2(file_path): # common_header="" common_header = extract_common_header(file_path) # print(common_header) @@ -109,7 +109,7 @@ def save_extracted_text_to_txt(pdf_path, txt_path): """ try: # 提取文本内容 - extracted_text = extract_text_by_page(pdf_path) + extracted_text = extract_text_by_page_pypdf2(pdf_path) # 将提取的文本写入TXT文件 with open(txt_path, 'w', encoding='utf-8') as txt_file: @@ -203,7 +203,7 @@ if __name__ == '__main__': ress = extract_common_header(pdf_path) print(ress) print("-----------------") - extract_text_by_page(pdf_path) + extract_text_by_page_pypdf2(pdf_path) # res=extract_text_by_page_fitz(pdf_path) # print(res)磋商文件_tobidders_notice_part2.pdf # save_extracted_text_to_txt(file_path,"output.txt") diff --git a/flask_app/logger_setup.py b/flask_app/logger_setup.py index 4d41753..ea725e1 100644 --- a/flask_app/logger_setup.py +++ b/flask_app/logger_setup.py @@ -66,9 +66,9 @@ def create_logger(app, subfolder): file_formatter = CSTFormatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') file_handler.setFormatter(file_formatter) logger.addHandler(file_handler) - # stream_handler = logging.StreamHandler() - # stream_handler.setFormatter(logging.Formatter('%(message)s')) - # logger.addHandler(stream_handler) + stream_handler = logging.StreamHandler() + stream_handler.setFormatter(logging.Formatter('%(message)s')) + logger.addHandler(stream_handler) logger.setLevel(logging.INFO) #Logger 只会处理大于或等于 INFO 级别的日志消息（例如 INFO、WARNING、ERROR、CRITICAL），而 DEBUG 级别的消息会被忽略。 logger.propagate = False g.logger = logger diff --git a/flask_app/old_version/资格评审前判断_old.py b/flask_app/old_version/资格评审前判断_old.py index eb3dc87..384a767 100644 --- a/flask_app/old_version/资格评审前判断_old.py +++ b/flask_app/old_version/资格评审前判断_old.py @@ -1,9 +1,9 @@ -from flask_app.general.读取文件.按页读取pdf import extract_text_by_page +from flask_app.general.读取文件.按页读取pdf import extract_text_by_page_pypdf2 def check_strings_in_pdf(file_path): judge_list=['施工机械设备', '企业信息登记'] # Read text from PDF - text = extract_text_by_page(file_path) # Assuming this returns all text from the PDF + text = extract_text_by_page_pypdf2(file_path) # Assuming this returns all text from the PDF full_text = ''.join(text).replace('\n', '').replace(' ', '') # Clean up the text # Initialize the questions list diff --git a/flask_app/routes/judge_zbfile.py b/flask_app/routes/judge_zbfile.py index 8e6884a..ba43a31 100644 --- a/flask_app/routes/judge_zbfile.py +++ b/flask_app/routes/judge_zbfile.py @@ -1,9 +1,10 @@ +import multiprocessing import os import threading import time from enum import Enum from typing import Any -from flask import Blueprint, g +from flask import Blueprint, g, current_app from flask_app.general.format_change import download_file from flask_app.routes.判断是否是招标文件 import judge_zbfile_exec from flask_app.routes.utils import validate_and_setup_logger, create_response_normal, log_error_unique_id @@ -26,7 +27,18 @@ def judge_zbfile() -> Any: #判断是否是招标文件 unique_id = g.unique_id result = [None] # 用于存储结果的可变对象 done = threading.Event() # 标志判断是否完成 - + pool = current_app.process_pool # 使用全局的进程池 + def judge_zbfile_exec_sub(file_path): + # with multiprocessing.Pool(processes=1) as pool: + # result = pool.apply( + # judge_zbfile_exec, # 你的实际执行函数 + # args=(file_path,) + # ) + result = pool.apply( + judge_zbfile_exec, # 你的实际执行函数 + args=(file_path,) + ) + return result def wrapper() -> None: """ 包装整个 judge_zbfile 的函数逻辑 @@ -38,7 +50,6 @@ def judge_zbfile() -> Any: #判断是否是招标文件 downloaded_filepath, file_type = download_file(file_url, downloaded_filename,True) if not downloaded_filepath or file_type == 4: - logger.error("下载地址不存在或不支持的文件类型！") log_error_unique_id(unique_id, 4) result[0] = JudgeResult.ERROR return @@ -46,7 +57,7 @@ def judge_zbfile() -> Any: #判断是否是招标文件 logger.info(f"Local file path: {downloaded_filepath}") # 调用实际的判断函数 - judge_result = judge_zbfile_exec(downloaded_filepath) + judge_result = judge_zbfile_exec_sub(downloaded_filepath) judge = JudgeResult.YES if judge_result else JudgeResult.NO end_time = time.time() diff --git a/flask_app/routes/判断是否是招标文件.py b/flask_app/routes/判断是否是招标文件.py index 94d7b83..ef1aec6 100644 --- a/flask_app/routes/判断是否是招标文件.py +++ b/flask_app/routes/判断是否是招标文件.py @@ -1,7 +1,7 @@ import time from PyPDF2 import PdfReader # 确保已安装 PyPDF2: pip install PyPDF2 from docx import Document - +import fitz from flask_app.general.llm.通义千问long import upload_file, qianwen_long def judge_zbfile_exec(file_path): @@ -12,28 +12,40 @@ def judge_zbfile_exec(file_path): start_time = time.time() # 检查文件是否为PDF格式 if file_path.lower().endswith('.pdf'): - # 使用 with 语句确保文件关闭 - with open(file_path, 'rb') as f: - reader = PdfReader(f) - num_pages = len(reader.pages) - if num_pages <= 5: - return False - elif file_path.lower().endswith('.docx'): - doc = Document(file_path) - accumulated_text = "" - chunk_size = 10 # 每次读取10个段落 - paragraphs = doc.paragraphs + try: + with open(file_path, 'rb') as f: + reader = PdfReader(f) + num_pages = len(reader.pages) + except Exception: + try: + doc = fitz.open(file_path) + num_pages = len(doc) + except Exception: + print("PDF 文件读取失败") + return False # 两种解析方式都失败，直接返回 False - for i in range(0, len(paragraphs), chunk_size): - chunk = paragraphs[i:i + chunk_size] - for para in chunk: - accumulated_text += para.text - # 判断累计字符数是否已经达到1000字, - if len(accumulated_text) >= 1000: - break - # 若累计内容不足1000字，则直接返回False - if len(accumulated_text) < 1000: - return False + if num_pages <= 5: + return False # 小于等于 5 页的 PDF 直接判定为非招标文件 + elif file_path.lower().endswith('.docx'): + try: + doc = Document(file_path) + accumulated_text = "" + chunk_size = 10 # 每次读取10个段落 + paragraphs = doc.paragraphs + + for i in range(0, len(paragraphs), chunk_size): + chunk = paragraphs[i:i + chunk_size] + for para in chunk: + accumulated_text += para.text + if len(accumulated_text) >= 1000: + break # 读取超过1000字后即可停止 + if len(accumulated_text) < 1000: + return False # 若累计内容不足1000字，则直接返回 False + except Exception: + print("DOCX 文件读取失败，可能为乱码文件") + return False # 解析失败直接返回 False + pre_endtime=time.time() + print(f"judge_zbfile_exec预处理耗时：{pre_endtime - start_time:.2f} 秒") # 使用大模型进行判断 user_query = """该文件是否属于招标文件？如果是的话，请返回'是',如果不是的话，返回'否'。请不要返回其他解释或内容。 以下是常见的招标文件类型： @@ -48,7 +60,7 @@ def judge_zbfile_exec(file_path): model_res = qianwen_long(file_id, user_query) end_time = time.time() print(f"judge_zbfile_exec实际耗时：{end_time - start_time:.2f} 秒") - print(f"判断是否属于招标文件：{model_res}") + print(f"判断是否属于招标文件：{model_res} 实际耗时：{end_time - start_time:.2f} 秒") return '否' not in model_res @@ -58,7 +70,7 @@ def judge_zbfile_exec(file_path): if __name__ == '__main__': start_time = time.time() - pdf_path = r"C:\Users\Administrator\Downloads\file1739842556194.docx" + pdf_path = r"C:\Users\Administrator\Desktop\fsdownload\19f53a17-ad4c-43b5-a7ed-981958ec3e0fs\ztbfile.docx" res = judge_zbfile_exec(pdf_path) if res: print("yes") diff --git a/flask_app/run_serve.py b/flask_app/run_serve.py index 14a137a..233e441 100644 --- a/flask_app/run_serve.py +++ b/flask_app/run_serve.py @@ -1,8 +1,24 @@ #flask_app/run_serve.py +import threading +import time + +import requests from waitress import serve from flask_app.start_up import create_app - +def warmup_request(): + # 等待服务器完全启动，例如等待 1-2 秒 + time.sleep(5) + try: + url = "http://127.0.0.1:5000/judge_zbfile" + #url必须为永久地址，完成热启动，创建进程池 + payload = {"file_url": "https://intellbid-open.oss-cn-wuhan-lr.aliyuncs.com/test/094%E5%AE%9A%E7%A8%BF-%E6%B9%96%E5%8C%97%E5%B7%A5%E4%B8%9A%E5%A4%A7%E5%AD%A6%E8%BD%BB%E6%AD%A6%E5%99%A8%E6%A8%A1%E6%8B%9F%E5%B0%84%E5%87%BB%E8%AE%BE%E5%A4%87%E9%87%87%E8%B4%AD%E9%A1%B9%E7%9B%AE%E6%8B%9B%E6%A0%87%E6%96%87%E4%BB%B6.pdf"} # 根据实际情况设置 file_url + headers = {"Content-Type": "application/json"} + response = requests.post(url, json=payload, headers=headers) + print(f"Warm-up 请求发送成功，状态码：{response.status_code}") + except Exception as e: + print(f"Warm-up 请求出错：{e}") if __name__ == "__main__": app = create_app() - serve(app, host='0.0.0.0', port=5000) + threading.Thread(target=warmup_request, daemon=True).start() + serve(app, host='0.0.0.0', port=5000) \ No newline at end of file diff --git a/flask_app/start_up.py b/flask_app/start_up.py index 4184946..b00343d 100644 --- a/flask_app/start_up.py +++ b/flask_app/start_up.py @@ -1,6 +1,6 @@ # flask_app/start_up.py -import gc -import os +from concurrent.futures import ProcessPoolExecutor +from multiprocessing import Pool from flask import Flask, g from flask_app.ConnectionLimiter import ConnectionLimiter @@ -13,11 +13,10 @@ from flask_app.routes.test_zbparse import test_zbparse_bp from flask_app.general.llm.清除file_id import delete_file_by_ids,read_file_ids from flask_app.routes.judge_zbfile import judge_zbfile_bp from flask_app.routes.test_preprocess import test_process_bp - - def create_app(): # 创建全局日志记录器 app = Flask(__name__) + app.process_pool = Pool(processes=10, maxtasksperchild=3) app.global_logger = create_logger_main('model_log') # 全局日志记录器 # 注册蓝图 app.register_blueprint(get_deviation_bp)