9.30 api-key修改
This commit is contained in:
parent
03a268422b
commit
99349a9968
@ -20,7 +20,7 @@ COPY . .
|
||||
# 将 flask_project 添加到 PYTHONPATH
|
||||
ENV PYTHONPATH=/flask_project:$PYTHONPATH
|
||||
# 定义环境变量
|
||||
ENV DASHSCOPE_API_KEY=sk-f7ad8ad193064cf482588f7064e75183
|
||||
ENV DASHSCOPE_API_KEY=sk-cf7a393e94bd49ddb742f73f9943a6cf
|
||||
ENV DASHSCOPE_WORKSPACE_ID=llm-mo38469hdfwtervi
|
||||
ENV ALIBABA_CLOUD_ACCESS_KEY_ID=LTAI5tRWhjktXyY5MovoiNuF
|
||||
ENV ALIBABA_CLOUD_ACCESS_KEY_SECRET=88oyw7LniqV8i0SnOuSFS5lprfrPtw
|
||||
|
||||
@ -1,56 +1,95 @@
|
||||
# -*- encoding:utf-8 -*-
|
||||
import re
|
||||
|
||||
def find_keys_by_value(target_value, json_data):
|
||||
# 找到值等于目标值或以目标值开头的键
|
||||
matched_keys = [k for k, v in json_data.items() if v == target_value]
|
||||
if not matched_keys:
|
||||
matched_keys = [k for k, v in json_data.items() if isinstance(v, str) and v.startswith(target_value)]
|
||||
return matched_keys
|
||||
|
||||
def is_numeric_key(key):
|
||||
# 这个正则表达式匹配由数字、点、括号中的数字或单个字母（小写或大写）组成的字符串，
|
||||
# 字母后跟数字，或数字后跟字母，单个字母后跟点，但不能是字母-数字-字母的组合
|
||||
pattern = r'^[\d.]+$|^\(\d+\)$|^（\d+）$|^[a-zA-Z]$|^[a-zA-Z]\d+$|^\d+[a-zA-Z]$|^[a-zA-Z]\.$'
|
||||
return re.match(pattern, key) is not None
|
||||
def process_dict(data):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
def find_keys_with_prefix(prefix, json_data):
|
||||
# 只提取直接子项，比如 prefix 为 '7.2' 时只提取 '7.2.1', '7.2.2' 但不会提取 '7.3'
|
||||
return [k for k in json_data.keys() if k.startswith(prefix) and k[len(prefix):].lstrip('.').isdigit()]
|
||||
|
||||
result = {}
|
||||
numeric_keys = []
|
||||
non_numeric_keys = {}
|
||||
def extract_json(data, target_values):
|
||||
results = {}
|
||||
|
||||
for key, value in data.items():
|
||||
if is_numeric_key(key):
|
||||
numeric_keys.append((key, value))
|
||||
else:
|
||||
non_numeric_keys[key] = value
|
||||
# 遍历所有目标值
|
||||
for target_value in target_values:
|
||||
# 找到所有与目标值匹配的键
|
||||
matched_keys = find_keys_by_value(target_value, data)
|
||||
|
||||
if numeric_keys:
|
||||
result['items'] = [process_dict(item[1]) for item in sorted(numeric_keys)]
|
||||
for key in matched_keys:
|
||||
# 查找所有以该键为前缀的子键，限制只提取直接子项
|
||||
key_and_subheadings = find_keys_with_prefix(key, data)
|
||||
|
||||
for key, value in non_numeric_keys.items():
|
||||
if isinstance(value, list):
|
||||
processed_list = []
|
||||
for item in value:
|
||||
if isinstance(item, dict):
|
||||
# 处理字典中只有一个键值对的情况
|
||||
if len(item) == 1:
|
||||
processed_item = process_dict(list(item.values())[0])
|
||||
else:
|
||||
processed_item = process_dict(item)
|
||||
else:
|
||||
processed_item = process_dict(item)
|
||||
for subkey in key_and_subheadings:
|
||||
# 如果子键有多级结构（比如 '7.2.1'），并且是直接子项
|
||||
if "." in subkey:
|
||||
parent_key = subkey.rsplit('.', 1)[0]
|
||||
top_level_key = parent_key.split('.')[0] + '.'
|
||||
|
||||
# 如果处理后的项是只包含一个元素的列表，则展平它
|
||||
if isinstance(processed_item, list) and len(processed_item) == 1:
|
||||
processed_item = processed_item[0]
|
||||
# 确保顶级键不会重复添加
|
||||
if top_level_key not in results:
|
||||
results[top_level_key] = data[top_level_key]
|
||||
|
||||
processed_list.append(processed_item)
|
||||
# 添加或更新父级键
|
||||
if parent_key not in results:
|
||||
if parent_key in data:
|
||||
results[parent_key] = data[parent_key]
|
||||
|
||||
result[key] = processed_list
|
||||
else:
|
||||
result[key] = process_dict(value)
|
||||
# 添加当前子键和它的值
|
||||
if subkey in data:
|
||||
results[subkey] = data[subkey]
|
||||
|
||||
if len(result) == 1 and 'items' in result:
|
||||
return result['items']
|
||||
return results
|
||||
|
||||
return result
|
||||
def extract_json(data, target_values):
|
||||
results = {}
|
||||
for target_value in target_values:
|
||||
matched_keys = find_keys_by_value(target_value, data)
|
||||
for key in matched_keys:
|
||||
key_and_subheadings = find_keys_with_prefix(key, data)
|
||||
for subkey in key_and_subheadings:
|
||||
if "." in subkey:
|
||||
parent_key = subkey.rsplit('.', 1)[0]
|
||||
top_level_key = parent_key.split('.')[0] + '.'
|
||||
# 特别处理定标相关的顶级键，确保不会重复添加其他键
|
||||
if top_level_key not in results:
|
||||
results[top_level_key] = target_value
|
||||
# 添加或更新父级键
|
||||
if parent_key not in results:
|
||||
if parent_key in data:
|
||||
results[parent_key] = data[parent_key]
|
||||
# 添加当前键
|
||||
results[subkey] = data[subkey]
|
||||
return results
|
||||
|
||||
# 测试数据
|
||||
data = {
|
||||
|
||||
"7.": "合同授予",
|
||||
"7.1": "定标方式",
|
||||
"7.1.1": "招标人依据评标委员会推荐的中标候选人确定中标人，国有资金占控股或者主导地位的依法必须进行招标的项目，确定排名第一的中标候选人为中标人。评标委员会推荐中标候选人的数量见投标人须知前附表。",
|
||||
"7.1.2": "排名第一的中标候选人放弃中标、因不可抗力不能签订合同、不按照招标文件要求提交履约担保，或者被查实存在影响中标结果的违法行为等情形，不符合中标条件的，招标人可以按照评标委员会提出的中标候选人名单排序依次确定其他中标候选人为中标人，也可以重新招标。",
|
||||
"7.1.3": "招标人将自收到评标报告之日起3日内在投标人须知前附表规定的媒介公示中标候选人。公示期不少于3日。投标人或者其他利害关系人对评标结果有异议的，应当在评标结果公示期间提出。招标人自收到异议之日起3日内作出答复；作出答复前，暂停招标投标活动。投标人的异议与答复应当通过“省电子交易平台”在“异议与答复”菜单以书面形式进行，其他利害关系人的异议也应以书面形式向招标人提出。中标候选人公示媒体见投标人须知前附表。",
|
||||
"7.1.4": "中标候选人经公示无异议、招标人向交通运输主管部门履行评标结果备案手续后，发出中标通知书。",
|
||||
"7.2": "中标通知",
|
||||
"7.2.1": "招标人将在本章第3.3款规定的投标有效期内，招标人通过“电子交易平台”以书面形式向中标人发出中标通知书，同时将中标结果通知未中标的投标人。中标通知书发出前，招标人将在投标人须知前附表第7.1.3款规定的媒介发布中标结果公告。",
|
||||
"7.2.2": "中标候选人的经营、财务状况发生较大变化或者存在违法行为，招标人认为可能影响其履约能力的，将在发出中标通知书前报请行政监督部门批准后，召集原评标委员会按照招标文件规定的标准和方法审查确认。",
|
||||
"7.2.3": "招标人即使已经向中标人发出了中标通知书，若发生下列情况之一者，中标无效。（1）经查实投标人以他人名义投标或以其他方式弄虚作假，骗取中标的；（2）经查实投标人有串通投标的，或向招标人或评标委员会成员以行贿手段谋取中标的；（3）法律、法规规定的其他情形。",
|
||||
"7.3": "履约担保",
|
||||
"7.3.1": "在签订合同前，中标人应按投标人须知前附表规定的金额、担保形式和招标文件第四章规定的格式向招标人提交履约担保，或事先经过招标人书面认可的履约担保格式向招标人提交履约担保。履约担保金额不得超过中标合同价的10%，详见投标人须知前附表。联合体中标的，其履约担保由牵头人提交，并应符合投标人须知前附表规定的金额、担保形式和招标文件规定的履约担保格式要求。联合体中标的，其履约担保由牵头人递交，并应符合上述要求。",
|
||||
"7.3.2": "中标人不能按本章第7.3.1项要求提交履约担保的，视为放弃中标，其投标保证金不予退还，给招标人造成的损失超过投标保证金数额的，中标人还应当对超过部分予以赔偿。",
|
||||
}
|
||||
process_dict()
|
||||
|
||||
# 提取匹配的目标
|
||||
target_values = ["中标"]
|
||||
|
||||
# 运行提取函数
|
||||
extracted_json = extract_json(data, target_values)
|
||||
|
||||
# 输出结果
|
||||
import json
|
||||
|
||||
print(json.dumps(extracted_json, ensure_ascii=False, indent=4))
|
||||
|
||||
@ -6,7 +6,8 @@ import re
|
||||
def find_keys_by_value(target_value, json_data):
|
||||
matched_keys = [k for k, v in json_data.items() if v == target_value] # 首先检查 JSON 中的每个键值对，如果值完全等于目标值，则将这些键收集起来。
|
||||
if not matched_keys:
|
||||
matched_keys = [k for k, v in json_data.items() if isinstance(v, str) and v.startswith(target_value)] #如果没有找到完全匹配的键，它会检查字符串类型的值是否以目标值开头，并收集这些键。
|
||||
matched_keys = [k for k, v in json_data.items() if
|
||||
isinstance(v, str) and v.startswith(target_value)] # 如果没有找到完全匹配的键，它会检查字符串类型的值是否以目标值开头，并收集这些键。
|
||||
return matched_keys # eg:[3.1,3.1.1,3.1.2,3.2...]
|
||||
|
||||
|
||||
@ -19,25 +20,38 @@ def find_keys_with_prefix(key_prefix, json_data):
|
||||
# 从完整的json文件中读取所需数据，eg:投标、评标
|
||||
def extract_json(data, target_values):
|
||||
results = {}
|
||||
|
||||
# 遍历所有目标值
|
||||
for target_value in target_values:
|
||||
# 找到所有与目标值匹配的键
|
||||
matched_keys = find_keys_by_value(target_value, data)
|
||||
|
||||
for key in matched_keys:
|
||||
# 查找所有以该键为前缀的子键，限制只提取直接子项
|
||||
key_and_subheadings = find_keys_with_prefix(key, data)
|
||||
|
||||
for subkey in key_and_subheadings:
|
||||
# 如果子键有多级结构（比如 '7.2.1'），并且是直接子项
|
||||
if "." in subkey:
|
||||
parent_key = subkey.rsplit('.', 1)[0]
|
||||
top_level_key = parent_key.split('.')[0] + '.'
|
||||
# 特别处理定标相关的顶级键，确保不会重复添加其他键
|
||||
|
||||
# 确保顶级键不会重复添加
|
||||
if top_level_key not in results:
|
||||
results[top_level_key] = target_value
|
||||
results[top_level_key] = data[top_level_key]
|
||||
|
||||
# 添加或更新父级键
|
||||
if parent_key not in results:
|
||||
if parent_key in data:
|
||||
results[parent_key] = data[parent_key]
|
||||
# 添加当前键
|
||||
|
||||
# 添加当前子键和它的值
|
||||
if subkey in data:
|
||||
results[subkey] = data[subkey]
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def sort_clean_data_keys(data):
|
||||
# 预处理：删除键名中的空格
|
||||
def preprocess_key(key):
|
||||
@ -58,6 +72,7 @@ def sort_clean_data_keys(data):
|
||||
|
||||
return sorted_data
|
||||
|
||||
|
||||
# 转换结构化的JSON数据
|
||||
def transform_json(data):
|
||||
result = {}
|
||||
@ -82,9 +97,20 @@ def transform_json(data):
|
||||
continue
|
||||
|
||||
if len(levels) == 1: # 一级标题
|
||||
# 优先按 '\n' 拆分
|
||||
if '\n' in value:
|
||||
new_key, *new_value = value.split('\n', 1)
|
||||
new_key = new_key.strip()
|
||||
new_value = new_value[0].strip() if new_value else ""
|
||||
# 如果没有 '\n'，再检查 ':' 或 '：'，并进行拆分
|
||||
elif ':' in value or '：' in value:
|
||||
delimiter = ':' if ':' in value else '：'
|
||||
new_key, new_value = value.split(delimiter, 1)
|
||||
new_key = new_key.strip()
|
||||
new_value = new_value.strip()
|
||||
else:
|
||||
new_key = value.strip()
|
||||
new_value = ""
|
||||
|
||||
parent[new_key] = {}
|
||||
if new_value:
|
||||
@ -123,6 +149,7 @@ def transform_json(data):
|
||||
|
||||
return remove_single_item_lists(result)
|
||||
|
||||
|
||||
# 主要是处理键值中若存在若干序号且每个序号块的内容>=50字符的时候，用列表表示。
|
||||
def post_process(value):
|
||||
# 如果传入的是非字符串值，直接返回原值
|
||||
@ -167,12 +194,12 @@ def post_process(value):
|
||||
processed_blocks.append(block.strip())
|
||||
else:
|
||||
# 如果发现有块长度小于50，返回原数据
|
||||
print(block)
|
||||
return value
|
||||
|
||||
# 如果所有的块都符合条件，返回分割后的列表
|
||||
return processed_blocks
|
||||
|
||||
|
||||
# 递归地处理嵌套结构
|
||||
def process_nested_data(data):
|
||||
# 先检查是否所有值都是 ""、"/" 或空列表
|
||||
@ -192,32 +219,36 @@ def process_nested_data(data):
|
||||
# 到达最内层，处理非字典和非列表的元素（字符串）
|
||||
return post_process(data)
|
||||
|
||||
|
||||
# 读取JSON数据，提取内容，转换结构，并打印结果
|
||||
def extract_from_notice(clause_path, type):
|
||||
if type == 1:
|
||||
target_values = ["投标","投标文件"]
|
||||
target_values = ["投标","投标文件","响应文件"]
|
||||
elif type == 2:
|
||||
target_values = ["开标", "评标", "定标"]
|
||||
target_values = ["开标", "评标", "定标","磋商程序","中标"]
|
||||
elif type == 3:
|
||||
target_values = ["重新招标、不再招标和终止招标", "重新招标", "不再招标", "终止招标"]
|
||||
elif type == 4:
|
||||
target_values = ["评标"] # 测试
|
||||
else:
|
||||
raise ValueError("Invalid type specified. Use 1 for '投标文件, 投标' or 2 for '开标, 评标, 定标'or 3 for '重新招标'")
|
||||
raise ValueError(
|
||||
"Invalid type specified. Use 1 for '投标文件, 投标' or 2 for '开标, 评标, 定标'or 3 for '重新招标'")
|
||||
with open(clause_path, 'r', encoding='utf-8') as file:
|
||||
data = json.load(file)
|
||||
extracted_data = extract_json(data, target_values) # 读取json
|
||||
# print(json.dumps(extracted_data,ensure_ascii=False,indent=4))
|
||||
sorted_data = sort_clean_data_keys(extracted_data) # 对键进行排序
|
||||
transformed_data = transform_json(sorted_data)
|
||||
final_result = process_nested_data(transformed_data)
|
||||
return final_result
|
||||
|
||||
|
||||
# TODO: 再审视一下zbtest20的处理是否合理
|
||||
if __name__ == "__main__":
|
||||
file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\tmp1\\clause1.json'
|
||||
# file_path='C:\\Users\\Administrator\\Desktop\\招标文件\\招标test文件夹\\tmp\\clause1.json'
|
||||
file_path = 'C:\\Users\\Administrator\\Desktop\\招标文件\\招标test文件夹\\tmp\\clause1.json'
|
||||
# file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\tmp1\\clause9.json'
|
||||
try:
|
||||
res = extract_from_notice(file_path, 4) # 可以改变此处的 type 参数测试不同的场景
|
||||
res = extract_from_notice(file_path, 3) # 可以改变此处的 type 参数测试不同的场景
|
||||
res2 = json.dumps(res, ensure_ascii=False, indent=4)
|
||||
print(res2)
|
||||
except ValueError as e:
|
||||
|
||||
@ -225,9 +225,9 @@ def convert_clause_to_json(input_path,output_folder,type=1):
|
||||
# # 如果没有分割需求，保留原数据
|
||||
# processed_data[key] = value
|
||||
|
||||
# 将修改后的数据重新写入到原来的 JSON 文件中
|
||||
with open(json_file_path, 'w', encoding='utf-8') as file:
|
||||
json.dump(processed_data, file, ensure_ascii=False, indent=4)
|
||||
# # 将修改后的数据重新写入到原来的 JSON 文件中
|
||||
# with open(json_file_path, 'w', encoding='utf-8') as file:
|
||||
# json.dump(processed_data, file, ensure_ascii=False, indent=4)
|
||||
|
||||
if __name__ == "__main__":
|
||||
file_path = 'C:\\Users\\Administrator\\Desktop\\招标文件\\招标test文件夹\\zbtest20_tobidders_notice.pdf'
|
||||
|
||||
@ -72,6 +72,6 @@ def extract_text_by_page(file_path):
|
||||
print(f"Page {page_num + 1} is empty or text could not be extracted.")
|
||||
return result
|
||||
if __name__ == '__main__':
|
||||
file_path = "C:\\Users\\Administrator\\Desktop\\招标文件\\招标test文件夹\\zbtest20_tobidders_notice.pdf"
|
||||
file_path = "C:\\Users\\Administrator\\Desktop\\货物标\\output4\\ztbfile_tobidders_notice_part2.pdf"
|
||||
res=extract_text_by_page(file_path)
|
||||
# print(res)
|
||||
246
flask_app/货物标/投标人须知正文提取指定内容货物标版.py
Normal file
246
flask_app/货物标/投标人须知正文提取指定内容货物标版.py
Normal file
@ -0,0 +1,246 @@
|
||||
import json
|
||||
import re
|
||||
|
||||
#提取两个大标题之间的内容
|
||||
|
||||
def extract_between_sections(data, target_values):
|
||||
target_found = False
|
||||
extracted_data = {}
|
||||
current_section_title = ""
|
||||
section_pattern = re.compile(r'^[一二三四五六七八九十]+$') # 匹配 "一", "二", "三" 等大标题
|
||||
current_block = {}
|
||||
|
||||
# 遍历所有键值对
|
||||
for key, value in data.items():
|
||||
# 只匹配形如 "一": "竞争性磋商响应文件" 的章节标题
|
||||
if section_pattern.match(key):
|
||||
if target_found:
|
||||
# 如果已经找到了符合的章节，并且遇到了另一个章节
|
||||
# 保存当前块并重置
|
||||
if current_block:
|
||||
extracted_data[current_section_title] = current_block
|
||||
current_block = {}
|
||||
target_found = False
|
||||
|
||||
# 检查当前标题是否包含 target_values 中的任意关键词
|
||||
if any(tv in value for tv in target_values):
|
||||
target_found = True # 找到了目标章节，开始捕获后续内容
|
||||
current_section_title = value # 保存章节标题内容
|
||||
|
||||
elif target_found: # 只捕获目标值之后的内容
|
||||
current_block[key] = value
|
||||
|
||||
# 保存最后一个块（如果有的话）
|
||||
if current_block:
|
||||
extracted_data[current_section_title] = current_block
|
||||
|
||||
return extracted_data
|
||||
|
||||
|
||||
def process_with_outer_key(data):
|
||||
processed_data = {}
|
||||
|
||||
# 遍历外层的键值对
|
||||
for outer_key, inner_data in data.items():
|
||||
# 调用 transform_json 函数对内层数据进行处理
|
||||
processed_inner_data = transform_json(inner_data)
|
||||
|
||||
# 将处理后的数据保留在外层键下
|
||||
processed_data[outer_key] = processed_inner_data
|
||||
|
||||
return processed_data
|
||||
def sort_clean_data_keys(data):
|
||||
# 预处理：删除键名中的空格
|
||||
def preprocess_key(key):
|
||||
return re.sub(r'\s+', '', key)
|
||||
|
||||
# 将键转换成由整数构成的元组，作为排序依据
|
||||
def key_func(key):
|
||||
return tuple(int(part) for part in re.split(r'\D+', key) if part)
|
||||
|
||||
# 创建一个新的字典，键名经过预处理
|
||||
preprocessed_data = {preprocess_key(key): value for key, value in data.items()}
|
||||
|
||||
# 对预处理后的字典键进行排序
|
||||
sorted_keys = sorted(preprocessed_data.keys(), key=key_func)
|
||||
|
||||
# 创建一个新的字典，按照排序后的键添加键值对
|
||||
sorted_data = {key: preprocessed_data[key] for key in sorted_keys}
|
||||
|
||||
return sorted_data
|
||||
|
||||
# 转换结构化的JSON数据
|
||||
def transform_json(data):
|
||||
result = {}
|
||||
temp = {0: result} # 初始化根字典
|
||||
|
||||
# 首先,创建一个临时字典用于检查是否存在三级标题
|
||||
has_subkey = {}
|
||||
for key in data.keys():
|
||||
parts = key.split('.')
|
||||
if len(parts) > 2 and parts[1]:
|
||||
parent_key = parts[0] + '.' + parts[1]
|
||||
has_subkey[parent_key] = True
|
||||
|
||||
for key, value in data.items():
|
||||
match = re.match(r'(\d+)(?:\.(\d+))?(?:\.(\d+))?', key)
|
||||
if match:
|
||||
levels = [int(l) for l in match.groups() if l is not None]
|
||||
if (len(levels) - 1) in temp:
|
||||
parent = temp[len(levels) - 1]
|
||||
else:
|
||||
print(f"No parent found at level {len(levels) - 1} for key '{key}'. Check the data structure.")
|
||||
continue
|
||||
|
||||
if len(levels) == 1: # 一级标题
|
||||
# 新增逻辑：判断值中是否有 ':' 或 '：'，并进行拆分
|
||||
# 优先按 '\n' 拆分
|
||||
if '\n' in value:
|
||||
new_key, *new_value = value.split('\n', 1)
|
||||
new_key = new_key.strip()
|
||||
new_value = new_value[0].strip() if new_value else ""
|
||||
# 如果没有 '\n'，再检查 ':' 或 '：'，并进行拆分
|
||||
elif ':' in value or '：' in value:
|
||||
delimiter = ':' if ':' in value else '：'
|
||||
new_key, new_value = value.split(delimiter, 1)
|
||||
new_key = new_key.strip()
|
||||
new_value = new_value.strip()
|
||||
else:
|
||||
new_key = value.strip()
|
||||
new_value = ""
|
||||
|
||||
parent[new_key] = {}
|
||||
if new_value:
|
||||
parent[new_key][new_key] = new_value # 使用 new_key 作为键名，而不是固定的 "content"
|
||||
temp[len(levels)] = parent[new_key]
|
||||
elif len(levels) == 2: # 二级标题
|
||||
new_key, *new_value = value.split('\n', 1)
|
||||
new_key = new_key.strip()
|
||||
new_value = new_value[0].strip() if new_value else ""
|
||||
|
||||
if f"{levels[0]}.{levels[1]}" in has_subkey:
|
||||
parent[new_key] = [new_value] if new_value else []
|
||||
else:
|
||||
parent[new_key] = new_value
|
||||
|
||||
temp[len(levels)] = parent[new_key]
|
||||
else: # 三级标题
|
||||
if isinstance(parent, dict):
|
||||
parent_key = list(parent.keys())[-1]
|
||||
if isinstance(parent[parent_key], list):
|
||||
parent[parent_key].append(value)
|
||||
elif parent[parent_key]:
|
||||
parent[parent_key] = [parent[parent_key], value]
|
||||
else:
|
||||
parent[parent_key] = [value]
|
||||
elif isinstance(parent, list):
|
||||
parent.append(value)
|
||||
|
||||
def remove_single_item_lists(node):
|
||||
if isinstance(node, dict):
|
||||
for key in list(node.keys()):
|
||||
node[key] = remove_single_item_lists(node[key])
|
||||
if isinstance(node[key], list) and len(node[key]) == 1:
|
||||
node[key] = node[key][0]
|
||||
return node
|
||||
|
||||
return remove_single_item_lists(result)
|
||||
|
||||
#主要是处理键值中若存在若干序号且每个序号块的内容>=50字符的时候，用列表表示。
|
||||
def post_process(value):
|
||||
# 如果传入的是非字符串值，直接返回原值
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
|
||||
# 定义可能的分割模式及其正则表达式
|
||||
patterns = [
|
||||
(r'\d+、', r'(?=\d+、)'), # 匹配 '1、'
|
||||
(r'[（(]\d+[）)]', r'(?=[（(]\d+[）)])'), # 匹配 '(1)' 或 '（1）'
|
||||
(r'\d+\.', r'(?=\d+\.)'), # 匹配 '1.'
|
||||
(r'[一二三四五六七八九十]、', r'(?=[一二三四五六七八九十]、)'), # 匹配 '一、'、'二、' 等
|
||||
(r'[一二三四五六七八九十]\.', r'(?=[一二三四五六七八九十]\.)') # 匹配 '一.'、'二.' 等
|
||||
]
|
||||
|
||||
# 初始化用于保存最早匹配到的模式及其位置
|
||||
first_match = None
|
||||
first_match_position = len(value) # 初始值设为文本长度，确保任何匹配都会更新它
|
||||
|
||||
# 遍历所有模式，找到第一个出现的位置
|
||||
for search_pattern, split_pattern_candidate in patterns:
|
||||
match = re.search(search_pattern, value)
|
||||
if match:
|
||||
# 如果这个匹配的位置比当前记录的更靠前，更新匹配信息
|
||||
if match.start() < first_match_position:
|
||||
first_match = split_pattern_candidate
|
||||
first_match_position = match.start()
|
||||
|
||||
# 如果找到了最早出现的匹配模式，使用它来分割文本
|
||||
if first_match:
|
||||
blocks = re.split(first_match, value)
|
||||
else:
|
||||
# 如果没有匹配的模式，保留原文本
|
||||
blocks = [value]
|
||||
|
||||
processed_blocks = []
|
||||
for block in blocks:
|
||||
if not block:
|
||||
continue
|
||||
# 计算中英文字符总数，如果大于50，则加入列表
|
||||
if block and len(re.findall(r'[\u4e00-\u9fff\w]', block)) >= 50:
|
||||
processed_blocks.append(block.strip())
|
||||
else:
|
||||
# 如果发现有块长度小于50，返回原数据
|
||||
return value
|
||||
|
||||
# 如果所有的块都符合条件，返回分割后的列表
|
||||
return processed_blocks
|
||||
|
||||
# 递归地处理嵌套结构
|
||||
def process_nested_data(data):
|
||||
# 先检查是否所有值都是 ""、"/" 或空列表
|
||||
if isinstance(data, dict) and all(v == "" or v == "/" or (isinstance(v, list) and not v) for v in data.values()):
|
||||
return list(data.keys())
|
||||
# 递归遍历字典，处理最内层的字符串
|
||||
if isinstance(data, dict):
|
||||
# 如果当前项是字典，继续递归遍历其键值对
|
||||
result = {}
|
||||
for key, value in data.items():
|
||||
result[key] = process_nested_data(value) # 递归处理子项
|
||||
return result
|
||||
elif isinstance(data, list):
|
||||
# 如果是列表，直接返回列表，保持原样
|
||||
return data
|
||||
else:
|
||||
# 到达最内层，处理非字典和非列表的元素（字符串）
|
||||
return post_process(data)
|
||||
|
||||
# 读取JSON数据，提取内容，转换结构，并打印结果
|
||||
def extract_from_notice(clause_path, type):
|
||||
if type == 1:
|
||||
target_values = ["投标文件","响应文件"]
|
||||
elif type == 2:
|
||||
target_values = ["开标", "评标", "定标","磋商程序","中标"]
|
||||
elif type == 3:
|
||||
target_values = ["重新招标、不再招标和终止招标", "重新招标", "不再招标", "终止招标"]
|
||||
elif type == 4:
|
||||
target_values = ["评标"] #测试
|
||||
else:
|
||||
raise ValueError("Invalid type specified. Use 1 for '投标文件, 投标' or 2 for '开标, 评标, 定标'or 3 for '重新招标'")
|
||||
with open(clause_path, 'r', encoding='utf-8') as file:
|
||||
data = json.load(file)
|
||||
extracted_data = extract_between_sections(data, target_values) # 读取json
|
||||
transformed_data=process_with_outer_key(extracted_data)
|
||||
# sorted_data=sort_clean_data_keys(extracted_data) #对键进行排序
|
||||
# transformed_data = transform_json(extracted_data)
|
||||
final_result=process_nested_data(transformed_data)
|
||||
return final_result
|
||||
|
||||
#TODO: 再审视一下zbtest20的处理是否合理
|
||||
if __name__ == "__main__":
|
||||
file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\tmp1\\clause9.json'
|
||||
try:
|
||||
res = extract_from_notice(file_path, 2) # 可以改变此处的 type 参数测试不同的场景
|
||||
res2=json.dumps(res,ensure_ascii=False,indent=4)
|
||||
print(res2)
|
||||
except ValueError as e:
|
||||
print(e)
|
||||
@ -90,17 +90,32 @@ def parse_text_by_heading(text):
|
||||
lines = text.split('\n') #['一、说明', '1.适用范围', '招标文件仅适用于第一章“投标邀请书”中所述项目的货物、工程及服务的采购。']
|
||||
for i, line in enumerate(lines): #由于本身就是按行读取处理，因此保存的时候不带'\n'
|
||||
line_stripped = line.strip()
|
||||
|
||||
# 匹配中文数字标题，如 "一、说明"
|
||||
chinese_match = re.match(r'^([一二三四五六七八九十]+、)\s*(.+)$', line_stripped)
|
||||
print("yes")
|
||||
print(line_stripped)
|
||||
# 匹配中文数字标题，包括带括号和不带括号的情况
|
||||
chinese_match = re.match(r'^(?:\s*[（(]?\s*([一二三四五六七八九十]+)\s*[)）]?\s*[、]*)?\s*(.+)$', line_stripped)
|
||||
if chinese_match:
|
||||
chinese_key, chinese_value = chinese_match.groups()
|
||||
chinese_key = chinese_key.rstrip('、') # 移除顿号
|
||||
if chinese_key:
|
||||
chinese_key = f"{chinese_key}" # 统一格式为"数字、"
|
||||
data[chinese_key] = chinese_value
|
||||
current_key = None
|
||||
current_content = []
|
||||
continue
|
||||
|
||||
# 新增：匹配阿拉伯数字开头后跟顿号的情况，如 "7、竞争性磋商采购文件的修改"
|
||||
arabic_match = re.match(r'^(\d+、)\s*(.+)$', line_stripped)
|
||||
if arabic_match:
|
||||
arabic_key, arabic_value = arabic_match.groups()
|
||||
arabic_key = arabic_key.replace('、', '.') # 将顿号替换为点号
|
||||
if current_key is not None:
|
||||
content_string = ''.join(current_content).strip()
|
||||
data[current_key] = content_string.replace(' ', '')
|
||||
current_key = arabic_key
|
||||
current_content = [arabic_value]
|
||||
append_newline = True
|
||||
continue
|
||||
|
||||
# 匹配形如 '1.1'、'2.2.3' 等至少包含一个点的标题，并确保其前后没有字母或括号
|
||||
match = re.match(r'^(?<![a-zA-Z（(])(\d+(?:\.\d+)+)\s*(.*)', line_stripped)
|
||||
if not match:
|
||||
@ -146,9 +161,9 @@ def parse_text_by_heading(text):
|
||||
# parsed_data = parse_text_by_heading(text)
|
||||
# return parsed_data
|
||||
|
||||
def convert_clause_to_json(input_path,output_folder,type=1):
|
||||
if not os.path.exists(input_path):
|
||||
print(f"The specified file does not exist: {input_path}")
|
||||
def convert_clause_to_json(file_path,output_folder,type=1):
|
||||
if not os.path.exists(file_path):
|
||||
print(f"The specified file does not exist: {file_path}")
|
||||
return ""
|
||||
if type == 1:
|
||||
start_word = r'^\s*[（(]?\s*[一1]\s*[)）]?\s*[、.]*\s*(说\s*明|总\s*则)'
|
||||
@ -164,6 +179,7 @@ def convert_clause_to_json(input_path,output_folder,type=1):
|
||||
os.makedirs(output_folder)
|
||||
print(f"Created output folder: {output_folder}")
|
||||
file_name = "clause1.json" if type == 1 else "clause2.json"
|
||||
# file_name = f"clause{suffix_counter}.json"
|
||||
output_path = os.path.join(output_folder, file_name)
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(result, f, indent=4, ensure_ascii=False)
|
||||
@ -171,13 +187,38 @@ def convert_clause_to_json(input_path,output_folder,type=1):
|
||||
return output_path
|
||||
|
||||
|
||||
def process_folder(input_folder, output_folder):
|
||||
# 获取输入文件夹中的所有文件，过滤掉不以 'part2' 结尾的文件
|
||||
files = [f for f in os.listdir(input_folder) if os.path.isfile(os.path.join(input_folder, f)) and f.endswith('part2.pdf')]
|
||||
|
||||
# 初始化后缀计数器
|
||||
suffix_counter = 1
|
||||
|
||||
# 遍历文件并对每个文件进行处理
|
||||
for file_name in files:
|
||||
file_path = os.path.join(input_folder, file_name)
|
||||
suffix = str(suffix_counter) # 后缀为递增的数字
|
||||
try:
|
||||
# 调用 convert_clause_to_json，传递文件路径、输出文件夹和递增的后缀
|
||||
output_path = convert_clause_to_json(file_path, output_folder, 1, suffix_counter)
|
||||
print(f"Processed file: {file_name}, JSON saved to: {output_path}")
|
||||
except ValueError as e:
|
||||
print(f"Error processing {file_name}: {e}")
|
||||
|
||||
# 后缀递增
|
||||
suffix_counter += 1
|
||||
|
||||
if __name__ == "__main__":
|
||||
# file_path = 'D:\\flask_project\\flask_app\\static\\output\\cfd4959d-5ea9-4112-8b50-9e543803f029\\ztbfile_tobidders_notice.pdf'
|
||||
file_path='C:\\Users\\Administrator\\Desktop\\货物标\\output4\\2-招标文件（2020年广水市中小学教师办公电脑系统及多媒体“班班通”设备采购安装项目）_tobidders_notice_part2.pdf'
|
||||
file_path='C:\\Users\\Administrator\\Desktop\\货物标\\output4\\ztbfile_tobidders_notice_part2.pdf'
|
||||
output_folder = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\tmp1'
|
||||
try:
|
||||
output_path = convert_clause_to_json(file_path,output_folder)
|
||||
print(f"Final JSON result saved to: {output_path}")
|
||||
except ValueError as e:
|
||||
print("Error:", e)
|
||||
# input_folder = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4'
|
||||
# output_folder = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\tmp1'
|
||||
#
|
||||
# # 调用 process_folder 来处理整个文件夹中的所有文件
|
||||
# process_folder(input_folder, output_folder)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user