zbparse/flask_app/routes/upload.py

# flask_app/routes/upload.py
from flask import Blueprint, request, jsonify,g
import json
import os
import time
from flask_app.general.format_change import download_file
from flask_app.routes.工程标解析main import engineering_bid_main
from flask_app.routes.货物标解析main import goods_bid_main
from flask_app.general.post_processing import outer_post_processing
from flask_app.routes.utils import generate_deviation_response, validate_and_setup_logger, create_response, sse_format, \
    log_error_unique_id
from flask_app.ConnectionLimiter import require_connection_limit

upload_bp = Blueprint('upload', __name__)
@upload_bp.route('/upload', methods=['POST'])
@validate_and_setup_logger
@require_connection_limit(timeout=1800)
def zbparse():                               #大解析
    logger = g.logger
    try:
        logger.info("大解析开始!!!")
        received_data = request.get_json()
        logger.info("Received JSON data: " + str(received_data))
        file_url = g.file_url
        zb_type = g.zb_type
        try:
            logger.info("starting parsing url:" + file_url)
            return process_and_stream(file_url, zb_type)     #主要执行函数
        except Exception as e:
            logger.error('Exception occurred: ' + str(e))
            if hasattr(g, 'unique_id'):
                log_error_unique_id(g.unique_id,1)
            error_response = create_response(
                message='处理文件时发生异常',
                status='error',
                data=str(e)
            )
            return jsonify(error_response)
    except Exception as e:
        logger.error('Unexpected exception: ' + str(e))
        if hasattr(g, 'unique_id'):
            log_error_unique_id(g.unique_id,1)
        error_response = create_response(
            message='内部服务器错误',
            status='error',
            data='Internal server error'
        )
        return jsonify(error_response)
def process_and_stream(file_url, zb_type):
    """
    下载文件并进行处理，支持工程标和货物标的处理。
    """
    logger = g.logger
    unique_id = g.unique_id
    output_folder = g.output_folder
    filename = "ztbfile"
    downloaded_filename = os.path.join(output_folder, filename)
    start_time = time.time()

    try:
        downloaded = download_file(file_url, downloaded_filename)
        if not downloaded:
            logger.error("下载文件失败或不支持的文件类型")
            log_error_unique_id(unique_id,1)  # 记录失败的 unique_id
            error_response = create_response(
                message='文件处理失败',
                status='error',
                data=''
            )
            yield sse_format(error_response)
            return

        downloaded_filepath, file_type = downloaded

        if file_type == 4:
            logger.error("不支持的文件类型")
            log_error_unique_id(unique_id,1)  # 记录失败的 unique_id
            error_response = create_response(
                message='不支持的文件类型',
                status='error',
                data=''
            )
            yield sse_format(error_response)
            return

        logger.info("本地文件路径: " + downloaded_filepath)

        combined_data = {}
        good_list = None

        processing_functions = {
            1: engineering_bid_main,    #工程标解析
            2: goods_bid_main           #货物标解析/服务标解析
        }
        processing_func = processing_functions.get(zb_type, goods_bid_main)

        for data in processing_func(output_folder, downloaded_filepath, file_type, unique_id):    #逐一接收货物标 工程标解析内容，为前端网页展示服务
            if not data.strip():
                logger.error("Received empty data, skipping JSON parsing.")
                continue

            try:
                parsed_data = json.loads(data)
            except json.JSONDecodeError as e:
                logger.error(f"Failed to decode JSON: {e}")
                logger.error(f"Data received: {data}")
                continue
            if 'error' in parsed_data:
                error_message = parsed_data['error']
                logger.error(f"Processing terminated due to error: {error_message}")
                # 使用指定的格式返回错误响应
                error_response = create_response(
                    message=error_message,
                    status='error',
                    data=''
                )
                yield sse_format(error_response)
                return  # 终止进一步处理

            if 'good_list' in parsed_data:    #货物列表
                good_list = parsed_data['good_list']
                logger.info("Collected good_list from the processing function: %s", good_list)
                continue

            for outer_key, inner_dict in parsed_data.items():
                if isinstance(inner_dict, dict):
                    combined_data.update(inner_dict)

            response = create_response(
                message='Processing',
                status='success',
                data=data
            )
            yield sse_format(response)     #返回给后端->前端展示

        base_end_time = time.time()
        logger.info(f"分段解析完成，耗时：{base_end_time - start_time:.2f} 秒")
        #此时前端已完整接收到解析的所有内容，后面的内容与前端展示无关，主要是后处理：1.extracted_result，关键信息存储 2.技术偏离表 3.商务偏离表 4.投标人需提交的证明材料（目前后端存储了，前端还未展示）
        #后处理开始！！！
        output_json_path = os.path.join(output_folder, 'final_result.json')
        extracted_info_path = os.path.join(output_folder, 'extracted_result.json')
        includes = ["基础信息", "资格审查", "商务评分", "技术评分", "无效标与废标项", "投标文件要求", "开评定标流程"]
        final_result, extracted_info, tech_deviation, tech_star_deviation, business_deviation, business_star_deviation, zigefuhe_deviation, proof_materials = outer_post_processing(
            combined_data, includes, good_list)    #后处理 生成 extracted_info、商务 技术偏离数据 以及证明材料返给后端

        #后处理完毕！后面都是生成响应返回，不额外修改数据
        tech_deviation_response, tech_deviation_star_response, zigefuhe_deviation_response, shangwu_deviation_response, shangwu_star_deviation_response, proof_materials_response = generate_deviation_response(
            tech_deviation, tech_star_deviation, business_deviation, business_star_deviation, zigefuhe_deviation,
            proof_materials, logger)    #生成规范的响应

        # 使用通用响应函数
        yield sse_format(tech_deviation_response)
        yield sse_format(tech_deviation_star_response)
        yield sse_format(zigefuhe_deviation_response)
        yield sse_format(shangwu_deviation_response)
        yield sse_format(shangwu_star_deviation_response)
        yield sse_format(proof_materials_response)

        try:
            with open(extracted_info_path, 'w', encoding='utf-8') as json_file:
                json.dump(extracted_info, json_file, ensure_ascii=False, indent=4)
            logger.info(f"摘取后的数据已保存到 '{extracted_info_path}'")
        except IOError as e:
            logger.error(f"保存JSON文件时出错: {e}")
            log_error_unique_id(unique_id,1)  # 记录失败的 unique_id

        try:
            with open(output_json_path, 'w', encoding='utf-8') as json_file:
                json.dump(final_result, json_file, ensure_ascii=False, indent=4)
            logger.info(f"合并后的数据已保存到 '{output_json_path}'")
        except IOError as e:
            logger.error(f"保存JSON文件时出错: {e}")
            log_error_unique_id(unique_id,1)  # 记录失败的 unique_id

        extracted_info_response = create_response(
            message='extracted_info',
            status='success',
            data=json.dumps(extracted_info, ensure_ascii=False)
        )
        yield sse_format(extracted_info_response)

        complete_response = create_response(
            message='Combined_data',
            status='success',
            data=json.dumps(final_result, ensure_ascii=False)
        )
        yield sse_format(complete_response)

        final_response = create_response(     #目前后端的逻辑是读取到'data'中有个'END',就终止连接
            message='文件上传并处理成功',
            status='success',
            data='END'
        )
        yield sse_format(final_response)

    except Exception as e:
        logger.error(f"Unexpected error in process_and_stream: {e}")
        log_error_unique_id(unique_id,1)  # 记录失败的 unique_id
        error_response = create_response(
            message='内部服务器错误',
            status='error',
            data=''
        )
        yield sse_format(error_response)

    finally:
        end_time = time.time()
        duration = end_time - start_time
        logger.info(f"Total processing time: {duration:.2f} seconds")