10.18小解析
This commit is contained in:
parent
6b1049e3bd
commit
c50884f9ec
@ -257,7 +257,7 @@ def process_and_stream(file_url, zb_type):
|
||||
"""
|
||||
logger = g.logger
|
||||
unique_id = g.unique_id
|
||||
output_folder = f"flask_app/static/output/{unique_id}"
|
||||
output_folder = f"flask_app/static/output1/{unique_id}"
|
||||
filename = "ztbfile"
|
||||
downloaded_filename = os.path.join(output_folder, filename)
|
||||
|
||||
|
||||
@ -19,6 +19,8 @@ def clean_page_content(text, common_header):
|
||||
|
||||
#PYPDF2库
|
||||
def extract_common_header(pdf_path):
|
||||
from PyPDF2 import PdfReader
|
||||
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
headers = []
|
||||
total_pages = len(pdf_document.pages)
|
||||
@ -43,11 +45,13 @@ def extract_common_header(pdf_path):
|
||||
if len(headers) < 2:
|
||||
return "" # 如果没有足够的页来比较，返回空字符串
|
||||
|
||||
# 寻找每一行中的公共部分
|
||||
# 寻找每一行中的公共部分，按顺序保留
|
||||
common_headers = []
|
||||
for lines in zip(*headers):
|
||||
# 在每一行中寻找公共单词
|
||||
common_line = set(lines[0].split()).intersection(*[set(line.split()) for line in lines[1:]])
|
||||
# 提取第一行的词汇顺序
|
||||
first_words = lines[0].split()
|
||||
# 筛选所有页面都包含的词汇，保持顺序
|
||||
common_line = [word for word in first_words if all(word in line.split() for line in lines[1:])]
|
||||
if common_line:
|
||||
common_headers.append(' '.join(common_line))
|
||||
|
||||
|
||||
@ -19,6 +19,8 @@ def clean_page_content(text, common_header):
|
||||
|
||||
return text
|
||||
def extract_common_header(pdf_path):
|
||||
from PyPDF2 import PdfReader
|
||||
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
headers = []
|
||||
total_pages = len(pdf_document.pages)
|
||||
@ -43,16 +45,19 @@ def extract_common_header(pdf_path):
|
||||
if len(headers) < 2:
|
||||
return "" # 如果没有足够的页来比较，返回空字符串
|
||||
|
||||
# 寻找每一行中的公共部分
|
||||
# 寻找每一行中的公共部分，按顺序保留
|
||||
common_headers = []
|
||||
for lines in zip(*headers):
|
||||
# 在每一行中寻找公共单词
|
||||
common_line = set(lines[0].split()).intersection(*[set(line.split()) for line in lines[1:]])
|
||||
# 提取第一行的词汇顺序
|
||||
first_words = lines[0].split()
|
||||
# 筛选所有页面都包含的词汇，保持顺序
|
||||
common_line = [word for word in first_words if all(word in line.split() for line in lines[1:])]
|
||||
if common_line:
|
||||
common_headers.append(' '.join(common_line))
|
||||
|
||||
return '\n'.join(common_headers)
|
||||
|
||||
|
||||
def extract_text_by_page(file_path):
|
||||
common_header = extract_common_header(file_path)
|
||||
result = ""
|
||||
@ -147,6 +152,9 @@ if __name__ == '__main__':
|
||||
# file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\广水农商行门禁控制主机及基础验证设备采购项目——磋商文件（定稿）（三次）_tobidders_notice_part2.pdf'
|
||||
# file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\2-招标文件（2020年广水市中小学教师办公电脑系统及多媒体“班班通”设备采购安装项目）_tobidders_notice_part2.pdf'
|
||||
# file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\磋商文件_tobidders_notice_part2.pdf'
|
||||
file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\竞争性谈判文件(3)_tobidders_notice_part2.pdf'
|
||||
# file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\截取test\\交警支队机动车查验监管系统项目采购_tobidders_notice_part1.pdf'
|
||||
file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles\\交警支队机动车查验监管系统项目采购.pdf'
|
||||
# ress = extract_common_header(file_path)
|
||||
# print(ress)
|
||||
res=extract_text_by_page(file_path)
|
||||
# print(res)磋商文件_tobidders_notice_part2.pdf
|
||||
@ -22,6 +22,8 @@ def clean_page_content(text, common_header):
|
||||
|
||||
# PYPDF2版本
|
||||
def extract_common_header(pdf_path):
|
||||
from PyPDF2 import PdfReader
|
||||
|
||||
pdf_document = PdfReader(pdf_path)
|
||||
headers = []
|
||||
total_pages = len(pdf_document.pages)
|
||||
@ -46,17 +48,21 @@ def extract_common_header(pdf_path):
|
||||
if len(headers) < 2:
|
||||
return "" # 如果没有足够的页来比较，返回空字符串
|
||||
|
||||
# 寻找每一行中的公共部分
|
||||
# 寻找每一行中的公共部分，按顺序保留
|
||||
common_headers = []
|
||||
for lines in zip(*headers):
|
||||
# 在每一行中寻找公共单词
|
||||
common_line = set(lines[0].split()).intersection(*[set(line.split()) for line in lines[1:]])
|
||||
# 提取第一行的词汇顺序
|
||||
first_words = lines[0].split()
|
||||
# 筛选所有页面都包含的词汇，保持顺序
|
||||
common_line = [word for word in first_words if all(word in line.split() for line in lines[1:])]
|
||||
if common_line:
|
||||
common_headers.append(' '.join(common_line))
|
||||
|
||||
return '\n'.join(common_headers)
|
||||
|
||||
|
||||
|
||||
|
||||
# fitz库版本
|
||||
# def extract_common_header(pdf_path):
|
||||
# doc = fitz.open(pdf_path)
|
||||
@ -166,7 +172,6 @@ def extract_pages_generic(pdf_document, begin_pattern, end_pattern, begin_page,
|
||||
else:
|
||||
if start_page is None and re.search(begin_pattern, cleaned_text) and i > begin_page:
|
||||
start_page = i
|
||||
|
||||
if start_page is not None and re.search(end_pattern, cleaned_text) and i > start_page:
|
||||
end_page = i
|
||||
break
|
||||
@ -207,28 +212,6 @@ def extract_pages(pdf_path, output_folder, begin_pattern, begin_page, end_patter
|
||||
print(f"Error processing {pdf_path}: {e}")
|
||||
return ""
|
||||
|
||||
|
||||
def extract_pages_tobidders_notice(pdf_document, begin_pattern, end_pattern, begin_page, common_header,
|
||||
exclusion_pattern):
|
||||
start_page = None
|
||||
mid_page = None
|
||||
end_page = None
|
||||
for i, page in enumerate(pdf_document.pages):
|
||||
text = page.extract_text() or ""
|
||||
cleaned_text = clean_page_content(text, common_header)
|
||||
if exclusion_pattern and re.search(exclusion_pattern, cleaned_text) and mid_page is not None:
|
||||
continue
|
||||
if start_page is None and re.search(begin_pattern, cleaned_text) and i > begin_page:
|
||||
start_page = i
|
||||
if start_page is not None and mid_page is None and re.search(
|
||||
r'^\s*[（(]?\s*[一1]\s*[)）]?\s*[、．.]*\s*(说\s*明|总\s*则)', cleaned_text, re.MULTILINE):
|
||||
mid_page = i
|
||||
if start_page is not None and mid_page is not None and re.search(end_pattern, cleaned_text) and i > mid_page:
|
||||
end_page = i
|
||||
break
|
||||
return start_page, mid_page, end_page
|
||||
|
||||
|
||||
def get_patterns_for_procurement():
|
||||
begin_pattern = re.compile(
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:服务|项目|商务).*?要求|'
|
||||
@ -268,15 +251,68 @@ def get_patterns_for_notice():
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:公告|邀请书).*', re.MULTILINE
|
||||
)
|
||||
end_pattern = re.compile(
|
||||
r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人须知|磋商须知|供应商须知)+|(?:一\s*、\s*)?(?:投标人须知|磋商须知|供应商须知)前附表)',
|
||||
# r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人须知|磋商须知|供应商须知)+|(?:一\s*、\s*)?(?:投标人须知|磋商须知|供应商须知)前附表)',
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+',
|
||||
re.MULTILINE
|
||||
)
|
||||
return begin_pattern, end_pattern
|
||||
|
||||
# def extract_pages_tobidders_notice(pdf_document, begin_pattern, end_pattern, begin_page, common_header,
|
||||
# exclusion_pattern):
|
||||
# start_page = None
|
||||
# mid_page = None
|
||||
# end_page = None
|
||||
# for i, page in enumerate(pdf_document.pages):
|
||||
# text = page.extract_text() or ""
|
||||
# cleaned_text = clean_page_content(text, common_header)
|
||||
# if exclusion_pattern and re.search(exclusion_pattern, cleaned_text) and mid_page is not None:
|
||||
# continue
|
||||
# if start_page is None and re.search(begin_pattern, cleaned_text) and i > begin_page:
|
||||
# start_page = i
|
||||
# if start_page is not None and mid_page is None and re.search(
|
||||
# r'^\s*[（(]?\s*[一1]\s*[)）]?\s*[、．.]*\s*(说\s*明|总\s*则)', cleaned_text):
|
||||
# mid_page = i
|
||||
# if start_page is not None and mid_page is not None and re.search(end_pattern, cleaned_text) and i > mid_page:
|
||||
# end_page = i
|
||||
# break
|
||||
# return start_page, mid_page, end_page
|
||||
|
||||
def extract_pages_tobidders_notice(pdf_document, begin_pattern, end_pattern, begin_page, common_header,
|
||||
exclusion_pattern):
|
||||
def run_extraction(use_multiline=False):
|
||||
start_page = None
|
||||
mid_page = None
|
||||
end_page = None
|
||||
for i, page in enumerate(pdf_document.pages):
|
||||
text = page.extract_text() or ""
|
||||
cleaned_text = clean_page_content(text, common_header)
|
||||
# print(cleaned_text)
|
||||
if exclusion_pattern and re.search(exclusion_pattern, cleaned_text) and mid_page is not None:
|
||||
continue
|
||||
if start_page is None and re.search(begin_pattern, cleaned_text) and i > begin_page:
|
||||
start_page = i
|
||||
if start_page is not None and mid_page is None:
|
||||
mid_pattern = r'^\s*[（(]?\s*[一1]\s*[)）]?\s*[、．.]*\s*(说\s*明|总\s*则)'
|
||||
flags = re.MULTILINE if use_multiline else 0
|
||||
if re.search(mid_pattern, cleaned_text, flags):
|
||||
mid_page = i
|
||||
if start_page is not None and mid_page is not None and re.search(end_pattern, cleaned_text) and i > mid_page:
|
||||
end_page = i
|
||||
break
|
||||
return start_page, mid_page, end_page
|
||||
|
||||
# 第一次运行
|
||||
start_page, mid_page, end_page = run_extraction()
|
||||
|
||||
# 如果有任何一个值为 None，使用 re.MULTILINE 重新运行
|
||||
if start_page is None or mid_page is None or end_page is None:
|
||||
start_page, mid_page, end_page = run_extraction(use_multiline=True)
|
||||
|
||||
return start_page, mid_page, end_page
|
||||
|
||||
def extract_pages_twice_tobidders_notice(pdf_path, output_folder, output_suffix, common_header): # 投标人须知前附表/正文二次提取
|
||||
begin_pattern = re.compile(
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:(?:投标人|磋商|供应商|谈判供应商|磋商供应商)须知前附表)+'
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:(?:投标人?|磋商|供应商|谈判供应商|磋商供应商)须知前附表)+'
|
||||
)
|
||||
end_pattern = re.compile(
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+'
|
||||
@ -460,9 +496,10 @@ def truncate_pdf_main(input_path, output_folder, selection, output_suffix="defau
|
||||
# 更新的正则表达式以匹配"第x章"和"第x部分"，考虑到可能的空格和其他文字
|
||||
begin_pattern = re.compile(
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:服务|项目|商务).*?要求|'
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分).*?采购.*'
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分).*?采购.*|'
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分).*?需求.*'
|
||||
)
|
||||
begin_page = 5
|
||||
begin_page = 3
|
||||
end_pattern = re.compile(
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+'
|
||||
)
|
||||
@ -488,7 +525,7 @@ def truncate_pdf_main(input_path, output_folder, selection, output_suffix="defau
|
||||
elif selection == 4: # 投标人须知前附表和正文
|
||||
begin_page = 1
|
||||
begin_pattern = re.compile(
|
||||
r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人|磋商|供应商|谈判供应商|磋商供应商)须知+|(?:一\s*、\s*)?(?:投标人|磋商|供应商)须知前附表)',
|
||||
r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人?|磋商|供应商|谈判供应商|磋商供应商)须知+|(?:一\s*、\s*)?(?:投标人?|磋商|供应商)须知前附表)',
|
||||
re.MULTILINE
|
||||
)
|
||||
end_pattern = re.compile(
|
||||
@ -498,10 +535,11 @@ def truncate_pdf_main(input_path, output_folder, selection, output_suffix="defau
|
||||
elif selection == 5: # 招标公告
|
||||
begin_page = 0
|
||||
begin_pattern = re.compile(
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:公告|邀请书).*'
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:公告|邀请书|邀请函).*'
|
||||
)
|
||||
end_pattern = re.compile(
|
||||
r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人|磋商|供应商|谈判供应商|磋商供应商)须知+|(?:一\s*、\s*)?(?:投标人|磋商|供应商)须知前附表)'
|
||||
# r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人|磋商|供应商|谈判供应商|磋商供应商)须知+|(?:一\s*、\s*)?(?:投标人|磋商|供应商)须知前附表)'
|
||||
r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+',re.MULTILINE
|
||||
)
|
||||
local_output_suffix = "notice"
|
||||
else:
|
||||
@ -574,11 +612,12 @@ def truncate_pdf_specific_goods(pdf_path, output_folder):
|
||||
|
||||
# TODO:交通智能系统和招标(1)(1)文件有问题 sele=4的时候excludsion有问题
|
||||
if __name__ == "__main__":
|
||||
input_path = "C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles\\招标文件正文(1).pdf"
|
||||
output_folder = "C:\\Users\\Administrator\\Desktop\\货物标\\truncate_all"
|
||||
# input_path = "C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles\\交警支队机动车查验监管系统项目采购.pdf"
|
||||
input_path = "C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles"
|
||||
output_folder = "C:\\Users\\Administrator\\Desktop\\货物标\\截取test"
|
||||
# files = truncate_pdf_multiple(input_path, output_folder)
|
||||
files=truncate_pdf_specific_goods(input_path,output_folder)
|
||||
print(files[-1])
|
||||
# selection = 1 # 例如：1 - 商务技术服务要求, 2 - 评标办法, 3 - 资格审查后缀有qualification1或qualification2（与评标办法一致） 4.投标人须知前附表part1 投标人须知正文part2 5-公告
|
||||
# generated_files = truncate_pdf_main(input_path, output_folder, selection)
|
||||
# files=truncate_pdf_specific_goods(input_path,output_folder)
|
||||
# print(files)
|
||||
selection = 1# 例如：1 - 商务技术服务要求, 2 - 评标办法, 3 - 资格审查后缀有qualification1或qualification2（与评标办法一致） 4.投标人须知前附表part1 投标人须知正文part2 5-公告
|
||||
generated_files = truncate_pdf_main(input_path, output_folder, selection)
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user