220 lines
8.5 KiB
Python
220 lines
8.5 KiB
Python
|
import logging
|
|||
|
|
import shutil
|
|||
|
|
import sys
|
|||
|
|
import time
|
|||
|
|
import uuid
|
|||
|
|
from datetime import datetime, timedelta
|
|||
|
|
|
|||
|
|
from flask import Flask, request, jsonify, send_file, Response, stream_with_context
|
|||
|
|
import json
|
|||
|
|
import os
|
|||
|
|
from download import download_file
|
|||
|
|
from 招标文件解析 import main_processing
|
|||
|
|
|
|||
|
|
app = Flask(__name__)
|
|||
|
|
class CSTFormatter(logging.Formatter):
|
|||
|
|
"""自定义的 Formatter，将日志的时间戳调整为中国标准时间（UTC+8）"""
|
|||
|
|
def formatTime(self, record, datefmt=None):
|
|||
|
|
ct = datetime.fromtimestamp(record.created) + timedelta(hours=8)
|
|||
|
|
if datefmt:
|
|||
|
|
s = ct.strftime(datefmt)
|
|||
|
|
else:
|
|||
|
|
try:
|
|||
|
|
s = ct.strftime("%Y-%m-%d %H:%M:%S")
|
|||
|
|
if self.usesTime():
|
|||
|
|
s = f"{s},{record.msecs:03d}"
|
|||
|
|
except ValueError:
|
|||
|
|
s = ct.strftime("%Y-%m-%d %H:%M:%S")
|
|||
|
|
return s
|
|||
|
|
|
|||
|
|
def create_logger(unique_id):
|
|||
|
|
"""为每个请求创建一个新的日志器，日志器的日志文件存放在指定的输出文件夹中"""
|
|||
|
|
output_folder = f"/ZbparseProjects/static/output/{unique_id}"
|
|||
|
|
# output_folder =f"C:/Users/Administrator/Desktop/招标文件/test/{unique_id}"
|
|||
|
|
if not os.path.exists(output_folder):
|
|||
|
|
os.makedirs(output_folder, exist_ok=True)
|
|||
|
|
log_filename = "log.txt"
|
|||
|
|
log_path = os.path.join(output_folder, log_filename)
|
|||
|
|
logger = logging.getLogger(unique_id) # 使用 unique_id 作为日志器名字
|
|||
|
|
if not logger.handlers: # 避免重复添加处理器
|
|||
|
|
# 文件处理器
|
|||
|
|
file_handler = logging.FileHandler(log_path)
|
|||
|
|
file_formatter = CSTFormatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|||
|
|
file_handler.setFormatter(file_formatter)
|
|||
|
|
logger.addHandler(file_handler)
|
|||
|
|
|
|||
|
|
# 流处理器（控制台输出）
|
|||
|
|
stream_handler = logging.StreamHandler(sys.stdout)
|
|||
|
|
stream_formatter = logging.Formatter('%(message)s') # 简化的格式，只输出消息
|
|||
|
|
stream_handler.setFormatter(stream_formatter)
|
|||
|
|
logger.addHandler(stream_handler)
|
|||
|
|
|
|||
|
|
logger.setLevel(logging.INFO)
|
|||
|
|
return logger, output_folder
|
|||
|
|
|
|||
|
|
@app.route('/upload', methods=['POST'])
|
|||
|
|
# def zbparse():
|
|||
|
|
# file_url = validate_request()
|
|||
|
|
# if isinstance(file_url, tuple): # Check if the returned value is an error response
|
|||
|
|
# return file_url
|
|||
|
|
# try:
|
|||
|
|
# app.logger.info("starting parsing url:"+file_url)
|
|||
|
|
# final_json_path, output_folder,logger = download_and_process_file(file_url)
|
|||
|
|
# if not final_json_path:
|
|||
|
|
# return jsonify({'error': 'File processing failed'}), 500
|
|||
|
|
# response = generate_response(final_json_path,logger) # 先获取响应内容
|
|||
|
|
# # remove_directory(output_folder) # 然后删除文件夹
|
|||
|
|
# return response # 最后返回获取的响应
|
|||
|
|
# except Exception as e:
|
|||
|
|
# app.logger.error('Exception occurred: ' + str(e)) # 使用全局 logger 记录
|
|||
|
|
# return jsonify({'error': str(e)}), 500
|
|||
|
|
def zbparse():
|
|||
|
|
file_url = validate_request()
|
|||
|
|
if isinstance(file_url, tuple): # Check if the returned value is an error response
|
|||
|
|
return file_url
|
|||
|
|
try:
|
|||
|
|
app.logger.info("starting parsing url:" + file_url)
|
|||
|
|
return Response(stream_with_context(process_and_stream(file_url)), content_type='text/event-stream')
|
|||
|
|
except Exception as e:
|
|||
|
|
app.logger.error('Exception occurred: ' + str(e))
|
|||
|
|
return jsonify({'error': str(e)}), 500
|
|||
|
|
|
|||
|
|
|
|||
|
|
def process_and_stream(file_url):
|
|||
|
|
unique_id = str(uuid.uuid4())
|
|||
|
|
logger, output_folder = create_logger(unique_id)
|
|||
|
|
filename = "ztbfile"
|
|||
|
|
downloaded_filename = os.path.join(output_folder, filename)
|
|||
|
|
|
|||
|
|
downloaded_filepath, file_type = download_file(file_url, downloaded_filename)
|
|||
|
|
|
|||
|
|
if downloaded_filepath is None or file_type == 3:
|
|||
|
|
logger.error("Unsupported file type or failed to download file")
|
|||
|
|
error_response = {
|
|||
|
|
'message': 'File processing failed',
|
|||
|
|
'filename': None,
|
|||
|
|
'data': json.dumps({'error': 'File processing failed'})
|
|||
|
|
}
|
|||
|
|
yield f"data: {json.dumps(error_response)}\n\n"
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
logger.info("Local file path: " + downloaded_filepath)
|
|||
|
|
|
|||
|
|
for data in main_processing(output_folder, downloaded_filepath, file_type, unique_id):
|
|||
|
|
response = {
|
|||
|
|
'message': 'Processing',
|
|||
|
|
'filename': os.path.basename(downloaded_filepath),
|
|||
|
|
'data': data
|
|||
|
|
}
|
|||
|
|
yield f"data: {json.dumps(response)}\n\n"
|
|||
|
|
|
|||
|
|
final_response = {
|
|||
|
|
'message': 'File uploaded and processed successfully',
|
|||
|
|
'filename': os.path.basename(downloaded_filepath),
|
|||
|
|
'data': 'END'
|
|||
|
|
}
|
|||
|
|
yield f"data: {json.dumps(final_response)}\n\n"
|
|||
|
|
|
|||
|
|
def validate_request():
|
|||
|
|
if not request.is_json:
|
|||
|
|
return jsonify({'error': 'Missing JSON in request'}), 400
|
|||
|
|
file_url = request.json.get('file_url')[0]
|
|||
|
|
if not file_url:
|
|||
|
|
return jsonify({'error': 'No file URL provided'}), 400
|
|||
|
|
return file_url
|
|||
|
|
|
|||
|
|
def download_and_process_file(file_url):
|
|||
|
|
unique_id = str(uuid.uuid4()) # 生成一个唯一的 UUID
|
|||
|
|
logger, output_folder = create_logger(unique_id)
|
|||
|
|
filename = "ztbfile"
|
|||
|
|
downloaded_filename = os.path.join(output_folder, filename)
|
|||
|
|
|
|||
|
|
# 下载文件，假设 download_file 函数已正确处理异常并返回文件路径
|
|||
|
|
downloaded_filepath,file_type = download_file(file_url, downloaded_filename)
|
|||
|
|
|
|||
|
|
if downloaded_filepath is None or file_type == 3:
|
|||
|
|
logger.error("Unsupported file type or failed to download file")
|
|||
|
|
return None, output_folder, logger
|
|||
|
|
|
|||
|
|
logger.info("Local file path: " + downloaded_filepath)
|
|||
|
|
processed_file_path = main_processing(output_folder, downloaded_filepath,file_type, unique_id)
|
|||
|
|
return processed_file_path, output_folder,logger
|
|||
|
|
|
|||
|
|
@app.route('/api/test_zbparse', methods=['POST'])
|
|||
|
|
def test_zbparse():
|
|||
|
|
try:
|
|||
|
|
return Response(stream_with_context(test_process_and_stream()), content_type='text/event-stream')
|
|||
|
|
except Exception as e:
|
|||
|
|
app.logger.error('Exception occurred: ' + str(e))
|
|||
|
|
return jsonify({'error': str(e)}), 500
|
|||
|
|
|
|||
|
|
def test_process_and_stream():
|
|||
|
|
# 模拟五段数据
|
|||
|
|
data_segments = [
|
|||
|
|
{"base_info": {"project_name": "测试项目1", "project_code": "TP001"}},
|
|||
|
|
{"review_standards": ["标准1", "标准2", "标准3"]},
|
|||
|
|
{"evaluation_standards": ["评估标准A", "评估标准B"]},
|
|||
|
|
{"invalid_requirements": ["无效要求X", "无效要求Y"]},
|
|||
|
|
{"bidding_documents_requirements": ["文件要求1", "文件要求2"]}
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
filename = "test_file.pdf"
|
|||
|
|
|
|||
|
|
for i, data in enumerate(data_segments, 1):
|
|||
|
|
response = {
|
|||
|
|
'message': f'Processing segment {i}',
|
|||
|
|
'filename': filename,
|
|||
|
|
'data': data
|
|||
|
|
}
|
|||
|
|
yield f"data: {json.dumps(response)}\n\n"
|
|||
|
|
time.sleep(5) # 每隔2秒发送一段数据
|
|||
|
|
|
|||
|
|
# 发送结束信号
|
|||
|
|
final_response = {
|
|||
|
|
'message': 'File processed successfully',
|
|||
|
|
'filename': filename,
|
|||
|
|
'data': 'END'
|
|||
|
|
}
|
|||
|
|
yield f"data: {json.dumps(final_response)}\n\n"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# def generate_response(final_json_path,logger):
|
|||
|
|
# if not os.path.exists(final_json_path):
|
|||
|
|
# logger.error('JSON file not found at path: ' + final_json_path)
|
|||
|
|
# return jsonify({'error': 'JSON file not found'}), 404
|
|||
|
|
# with open(final_json_path, 'r', encoding='utf-8') as f:
|
|||
|
|
# logger.info('final_json_path:'+final_json_path)
|
|||
|
|
# zbparse_data = json.load(f)
|
|||
|
|
# json_str = json.dumps(zbparse_data, ensure_ascii=False)
|
|||
|
|
# return jsonify({
|
|||
|
|
# 'message': 'File uploaded and processed successfully',
|
|||
|
|
# 'filename': os.path.basename(final_json_path),
|
|||
|
|
# 'data': json_str
|
|||
|
|
# })
|
|||
|
|
|
|||
|
|
|
|||
|
|
# @app.route('/get_json', methods=['POST'])
|
|||
|
|
# def testjson():
|
|||
|
|
# final_json_path="C:\\Users\\Administrator\\Desktop\\fsdownload\\temp4\\fd55f067-2cf6-475c-b7ce-4498f6606bf6\\final_result.json"
|
|||
|
|
# with open(final_json_path, 'r', encoding='utf-8') as f:
|
|||
|
|
# print('final_json_path:'+final_json_path)
|
|||
|
|
# zbparse_data = json.load(f)
|
|||
|
|
# json_str = json.dumps(zbparse_data, ensure_ascii=False)
|
|||
|
|
# print(json_str)
|
|||
|
|
# return jsonify({
|
|||
|
|
# 'message': 'File uploaded and processed successfully',
|
|||
|
|
# 'filename': os.path.basename(final_json_path),
|
|||
|
|
# 'data': json_str
|
|||
|
|
# })
|
|||
|
|
|
|||
|
|
|
|||
|
|
def remove_directory(path):
|
|||
|
|
try:
|
|||
|
|
shutil.rmtree(path)
|
|||
|
|
app.logger.info(f"Successfully removed directory: {path}") # 使用全局 logger 记录
|
|||
|
|
except Exception as e:
|
|||
|
|
app.logger.error(f"Failed to remove directory {path}: {str(e)}") # 使用全局 logger 记录
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
app.run(debug=True, host='0.0.0.0', port=5000)
|