zbparse/flask_app/main/start_up.py

import logging
import shutil
import sys
import time
import uuid
from datetime import datetime, timedelta

from flask import Flask, request, jsonify, Response, stream_with_context
import json
import os
from flask_app.main.download import download_file
from flask_app.main.招标文件解析 import main_processing

app = Flask(__name__)


class CSTFormatter(logging.Formatter):
    """自定义的 Formatter，将日志的时间戳调整为中国标准时间（UTC+8）"""

    def formatTime(self, record, datefmt=None):
        ct = datetime.fromtimestamp(record.created) + timedelta(hours=8)
        if datefmt:
            s = ct.strftime(datefmt)
        else:
            try:
                s = ct.strftime("%Y-%m-%d %H:%M:%S")
                if self.usesTime():
                    s = f"{s},{record.msecs:03d}"
            except ValueError:
                s = ct.strftime("%Y-%m-%d %H:%M:%S")
        return s


def create_logger(unique_id):
    """为每个请求创建一个新的日志器，日志器的日志文件存放在指定的输出文件夹中"""
    output_folder = f"flask_app/static/output/{unique_id}"
    # output_folder =f"C:/Users/Administrator/Desktop/招标文件/test/{unique_id}"
    if not os.path.exists(output_folder):
        os.makedirs(output_folder, exist_ok=True)
    log_filename = "log.txt"
    log_path = os.path.join(output_folder, log_filename)
    logger = logging.getLogger(unique_id)  # 使用 unique_id 作为日志器名字
    if not logger.handlers:  # 避免重复添加处理器
        # 文件处理器
        file_handler = logging.FileHandler(log_path)
        file_formatter = CSTFormatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        file_handler.setFormatter(file_formatter)
        logger.addHandler(file_handler)

        # 流处理器（控制台输出）
        stream_handler = logging.StreamHandler(sys.stdout)
        stream_formatter = logging.Formatter('%(message)s')  # 简化的格式，只输出消息
        stream_handler.setFormatter(stream_formatter)
        logger.addHandler(stream_handler)

        logger.setLevel(logging.INFO)
    return logger, output_folder


@app.route('/upload', methods=['POST'])
def zbparse():
    file_url = validate_request()
    if isinstance(file_url, tuple):  # Check if the returned value is an error response
        return file_url
    try:
        app.logger.info("starting parsing url:" + file_url)
        final_json_path, output_folder, logger = download_and_process_file(file_url)
        if not final_json_path:
            return jsonify({'error': 'File processing failed'}), 500
        response = generate_response(final_json_path, logger)  # 先获取响应内容
        # remove_directory(output_folder)  # 然后删除文件夹
        return response  # 最后返回获取的响应
    except Exception as e:
        app.logger.error('Exception occurred: ' + str(e))  # 使用全局 logger 记录
        return jsonify({'error': str(e)}), 500


# def zbparse():
#     file_url = validate_request()
#     if isinstance(file_url, tuple):  # Check if the returned value is an error response
#         return file_url
#     try:
#         app.logger.info("starting parsing url:" + file_url)
#         return Response(stream_with_context(process_and_stream(file_url)), content_type='text/event-stream')
#     except Exception as e:
#         app.logger.error('Exception occurred: ' + str(e))
#         return jsonify({'error': str(e)}), 500


def process_and_stream(file_url):
    unique_id = str(uuid.uuid4())
    logger, output_folder = create_logger(unique_id)
    filename = "ztbfile"
    downloaded_filename = os.path.join(output_folder, filename)

    downloaded_filepath, file_type = download_file(file_url, downloaded_filename)

    if downloaded_filepath is None or file_type == 3:
        logger.error("Unsupported file type or failed to download file")
        error_response = {
            'message': 'File processing failed',
            'filename': None,
            'data': json.dumps({'error': 'File processing failed'})
        }
        yield f"data: {json.dumps(error_response)}\n\n"
        return

    logger.info("Local file path: " + downloaded_filepath)

    for data in main_processing(output_folder, downloaded_filepath, file_type, unique_id):
        response = {
            'message': 'Processing',
            'filename': os.path.basename(downloaded_filepath),
            'data': data
        }
        yield f"data: {json.dumps(response)}\n\n"

    final_response = {
        'message': 'File uploaded and processed successfully',
        'filename': os.path.basename(downloaded_filepath),
        'data': 'END'
    }
    yield f"data: {json.dumps(final_response)}\n\n"


def validate_request():
    if not request.is_json:
        return jsonify({'error': 'Missing JSON in request'}), 400
    file_url = request.json.get('file_url')[0]
    if not file_url:
        return jsonify({'error': 'No file URL provided'}), 400
    return file_url


def download_and_process_file(file_url):
    unique_id = str(uuid.uuid4())  # 生成一个唯一的 UUID
    logger, output_folder = create_logger(unique_id)
    filename = "ztbfile"
    downloaded_filename = os.path.join(output_folder, filename)

    # 下载文件，假设 download_file 函数已正确处理异常并返回文件路径
    downloaded_filepath, file_type = download_file(file_url, downloaded_filename)

    if downloaded_filepath is None or file_type == 3:
        logger.error("Unsupported file type or failed to download file")
        return None, output_folder, logger

    logger.info("Local file path: " + downloaded_filepath)
    processed_file_path = main_processing(output_folder, downloaded_filepath, file_type, unique_id)
    return processed_file_path, output_folder, logger


@app.route('/api/test_zbparse', methods=['POST'])
def test_zbparse():
    try:
        return Response(stream_with_context(test_process_and_stream()), content_type='text/event-stream')
    except Exception as e:
        app.logger.error('Exception occurred: ' + str(e))
        return jsonify({'error': str(e)}), 500


def test_process_and_stream():
    # 模拟五段数据
    data_segments = [
        {"base_info": {"project_name": "测试项目1", "project_code": "TP001"}},
        {"review_standards": ["标准1", "标准2", "标准3"]},
        {"evaluation_standards": ["评估标准A", "评估标准B"]},
        {"invalid_requirements": ["无效要求X", "无效要求Y"]},
        {"bidding_documents_requirements": ["文件要求1", "文件要求2"]}
    ]

    filename = "test_file.pdf"

    for i, data in enumerate(data_segments, 1):
        response = {
            'message': f'Processing segment {i}',
            'filename': filename,
            'data': data
        }
        yield f"data: {json.dumps(response)}\n\n"
        time.sleep(5)  # 每隔2秒发送一段数据

    # 发送结束信号
    final_response = {
        'message': 'File processed successfully',
        'filename': filename,
        'data': 'END'
    }
    yield f"data: {json.dumps(final_response)}\n\n"


def generate_response(final_json_path, logger):
    if not os.path.exists(final_json_path):
        logger.error('JSON file not found at path: ' + final_json_path)
        return jsonify({'error': 'JSON file not found'}), 404
    with open(final_json_path, 'r', encoding='utf-8') as f:
        logger.info('final_json_path:' + final_json_path)
        zbparse_data = json.load(f)
        json_str = json.dumps(zbparse_data, ensure_ascii=False)
    return jsonify({
        'message': 'File uploaded and processed successfully',
        'filename': os.path.basename(final_json_path),
        'data': json_str
    })


# @app.route('/get_json', methods=['POST'])
# def testjson():
#     final_json_path="C:\\Users\\Administrator\\Desktop\\fsdownload\\temp4\\fd55f067-2cf6-475c-b7ce-4498f6606bf6\\final_result.json"
#     with open(final_json_path, 'r', encoding='utf-8') as f:
#         print('final_json_path:'+final_json_path)
#         zbparse_data = json.load(f)
#         json_str = json.dumps(zbparse_data, ensure_ascii=False)
#         print(json_str)
#     return jsonify({
#         'message': 'File uploaded and processed successfully',
#         'filename': os.path.basename(final_json_path),
#         'data': json_str
#     })


def remove_directory(path):
    try:
        shutil.rmtree(path)
        app.logger.info(f"Successfully removed directory: {path}")  # 使用全局 logger 记录
    except Exception as e:
        app.logger.error(f"Failed to remove directory {path}: {str(e)}")  # 使用全局 logger 记录


if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0', port=5000)