208 lines
8.2 KiB
Python
208 lines
8.2 KiB
Python
# flask_app/routes/upload.py
|
|
from flask import Blueprint, request, jsonify,g
|
|
import json
|
|
import os
|
|
import time
|
|
from flask_app.general.format_change import download_file
|
|
from flask_app.routes.工程标解析main import engineering_bid_main
|
|
from flask_app.routes.货物标解析main import goods_bid_main
|
|
from flask_app.general.post_processing import outer_post_processing
|
|
from flask_app.routes.utils import generate_deviation_response, validate_and_setup_logger, create_response, sse_format, \
|
|
log_error_unique_id
|
|
from flask_app.ConnectionLimiter import require_connection_limit
|
|
|
|
upload_bp = Blueprint('upload', __name__)
|
|
@upload_bp.route('/upload', methods=['POST'])
|
|
@validate_and_setup_logger
|
|
@require_connection_limit(timeout=720)
|
|
def zbparse():
|
|
logger = g.logger
|
|
try:
|
|
logger.info("大解析开始!!!")
|
|
received_data = request.get_json()
|
|
logger.info("Received JSON data: " + str(received_data))
|
|
file_url = g.file_url
|
|
zb_type = g.zb_type
|
|
try:
|
|
logger.info("starting parsing url:" + file_url)
|
|
return process_and_stream(file_url, zb_type)
|
|
except Exception as e:
|
|
logger.error('Exception occurred: ' + str(e))
|
|
if hasattr(g, 'unique_id'):
|
|
log_error_unique_id(g.unique_id,1)
|
|
error_response = create_response(
|
|
message='处理文件时发生异常',
|
|
status='error',
|
|
data=str(e)
|
|
)
|
|
return jsonify(error_response)
|
|
except Exception as e:
|
|
logger.error('Unexpected exception: ' + str(e))
|
|
if hasattr(g, 'unique_id'):
|
|
log_error_unique_id(g.unique_id,1)
|
|
error_response = create_response(
|
|
message='内部服务器错误',
|
|
status='error',
|
|
data='Internal server error'
|
|
)
|
|
return jsonify(error_response)
|
|
def process_and_stream(file_url, zb_type):
|
|
"""
|
|
下载文件并进行处理，支持工程标和货物标的处理。
|
|
"""
|
|
logger = g.logger
|
|
unique_id = g.unique_id
|
|
output_folder = g.output_folder
|
|
filename = "ztbfile"
|
|
downloaded_filename = os.path.join(output_folder, filename)
|
|
start_time = time.time()
|
|
|
|
try:
|
|
downloaded = download_file(file_url, downloaded_filename)
|
|
if not downloaded:
|
|
logger.error("下载文件失败或不支持的文件类型")
|
|
log_error_unique_id(unique_id,1) # 记录失败的 unique_id
|
|
error_response = create_response(
|
|
message='文件处理失败',
|
|
status='error',
|
|
data=''
|
|
)
|
|
yield sse_format(error_response)
|
|
return
|
|
|
|
downloaded_filepath, file_type = downloaded
|
|
|
|
if file_type == 4:
|
|
logger.error("不支持的文件类型")
|
|
log_error_unique_id(unique_id,1) # 记录失败的 unique_id
|
|
error_response = create_response(
|
|
message='不支持的文件类型',
|
|
status='error',
|
|
data=''
|
|
)
|
|
yield sse_format(error_response)
|
|
return
|
|
|
|
logger.info("本地文件路径: " + downloaded_filepath)
|
|
|
|
combined_data = {}
|
|
good_list = None
|
|
|
|
processing_functions = {
|
|
1: engineering_bid_main,
|
|
2: goods_bid_main
|
|
}
|
|
processing_func = processing_functions.get(zb_type, goods_bid_main)
|
|
|
|
for data in processing_func(output_folder, downloaded_filepath, file_type, unique_id):
|
|
if not data.strip():
|
|
logger.error("Received empty data, skipping JSON parsing.")
|
|
continue
|
|
|
|
try:
|
|
parsed_data = json.loads(data)
|
|
except json.JSONDecodeError as e:
|
|
logger.error(f"Failed to decode JSON: {e}")
|
|
logger.error(f"Data received: {data}")
|
|
continue
|
|
if 'error' in parsed_data:
|
|
error_message = parsed_data['error']
|
|
logger.error(f"Processing terminated due to error: {error_message}")
|
|
# 使用指定的格式返回错误响应
|
|
error_response = create_response(
|
|
message=error_message,
|
|
status='error',
|
|
data=''
|
|
)
|
|
yield sse_format(error_response)
|
|
return # 终止进一步处理
|
|
|
|
if 'good_list' in parsed_data:
|
|
good_list = parsed_data['good_list']
|
|
logger.info("Collected good_list from the processing function: %s", good_list)
|
|
continue
|
|
|
|
for outer_key, inner_dict in parsed_data.items():
|
|
if isinstance(inner_dict, dict):
|
|
combined_data.update(inner_dict)
|
|
|
|
response = create_response(
|
|
message='Processing',
|
|
status='success',
|
|
data=data
|
|
)
|
|
yield sse_format(response)
|
|
|
|
base_end_time = time.time()
|
|
logger.info(f"分段解析完成，耗时：{base_end_time - start_time:.2f} 秒")
|
|
|
|
output_json_path = os.path.join(output_folder, 'final_result.json')
|
|
extracted_info_path = os.path.join(output_folder, 'extracted_result.json')
|
|
includes = ["基础信息", "资格审查", "商务评分", "技术评分", "无效标与废标项", "投标文件要求", "开评定标流程"]
|
|
final_result, extracted_info, tech_deviation, tech_star_deviation, business_deviation, business_star_deviation, zigefuhe_deviation, proof_materials = outer_post_processing(
|
|
combined_data, includes, good_list)
|
|
|
|
tech_deviation_response, tech_deviation_star_response, zigefuhe_deviation_response, shangwu_deviation_response, shangwu_star_deviation_response, proof_materials_response = generate_deviation_response(
|
|
tech_deviation, tech_star_deviation, business_deviation, business_star_deviation, zigefuhe_deviation,
|
|
proof_materials, logger)
|
|
|
|
# 使用通用响应函数
|
|
yield sse_format(tech_deviation_response)
|
|
yield sse_format(tech_deviation_star_response)
|
|
yield sse_format(zigefuhe_deviation_response)
|
|
yield sse_format(shangwu_deviation_response)
|
|
yield sse_format(shangwu_star_deviation_response)
|
|
yield sse_format(proof_materials_response)
|
|
|
|
try:
|
|
with open(extracted_info_path, 'w', encoding='utf-8') as json_file:
|
|
json.dump(extracted_info, json_file, ensure_ascii=False, indent=4)
|
|
logger.info(f"摘取后的数据已保存到 '{extracted_info_path}'")
|
|
except IOError as e:
|
|
logger.error(f"保存JSON文件时出错: {e}")
|
|
log_error_unique_id(unique_id,1) # 记录失败的 unique_id
|
|
|
|
try:
|
|
with open(output_json_path, 'w', encoding='utf-8') as json_file:
|
|
json.dump(final_result, json_file, ensure_ascii=False, indent=4)
|
|
logger.info(f"合并后的数据已保存到 '{output_json_path}'")
|
|
except IOError as e:
|
|
logger.error(f"保存JSON文件时出错: {e}")
|
|
log_error_unique_id(unique_id,1) # 记录失败的 unique_id
|
|
|
|
extracted_info_response = create_response(
|
|
message='extracted_info',
|
|
status='success',
|
|
data=json.dumps(extracted_info, ensure_ascii=False)
|
|
)
|
|
yield sse_format(extracted_info_response)
|
|
|
|
complete_response = create_response(
|
|
message='Combined_data',
|
|
status='success',
|
|
data=json.dumps(final_result, ensure_ascii=False)
|
|
)
|
|
yield sse_format(complete_response)
|
|
|
|
final_response = create_response(
|
|
message='文件上传并处理成功',
|
|
status='success',
|
|
data='END'
|
|
)
|
|
yield sse_format(final_response)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Unexpected error in process_and_stream: {e}")
|
|
log_error_unique_id(unique_id,1) # 记录失败的 unique_id
|
|
error_response = create_response(
|
|
message='内部服务器错误',
|
|
status='error',
|
|
data=''
|
|
)
|
|
yield sse_format(error_response)
|
|
|
|
finally:
|
|
end_time = time.time()
|
|
duration = end_time - start_time
|
|
logger.info(f"Total processing time: {duration:.2f} seconds")
|