From c50884f9ec3e8880c07d5cfb34d84b2de5ed7836 Mon Sep 17 00:00:00 2001 From: zy123 <646228430@qq.com> Date: Fri, 18 Oct 2024 15:44:18 +0800 Subject: [PATCH] =?UTF-8?q?10.18=E5=B0=8F=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flask_app/main/start_up.py | 2 +- flask_app/main/截取pdf.py | 10 ++- flask_app/main/读取文件/按页读取pdf.py | 16 +++- flask_app/货物标/货物标截取pdf.py | 117 ++++++++++++++++--------- 4 files changed, 98 insertions(+), 47 deletions(-) diff --git a/flask_app/main/start_up.py b/flask_app/main/start_up.py index fb0b2ab..235fa6b 100644 --- a/flask_app/main/start_up.py +++ b/flask_app/main/start_up.py @@ -257,7 +257,7 @@ def process_and_stream(file_url, zb_type): """ logger = g.logger unique_id = g.unique_id - output_folder = f"flask_app/static/output/{unique_id}" + output_folder = f"flask_app/static/output1/{unique_id}" filename = "ztbfile" downloaded_filename = os.path.join(output_folder, filename) diff --git a/flask_app/main/截取pdf.py b/flask_app/main/截取pdf.py index d764627..a61c532 100644 --- a/flask_app/main/截取pdf.py +++ b/flask_app/main/截取pdf.py @@ -19,6 +19,8 @@ def clean_page_content(text, common_header): #PYPDF2库 def extract_common_header(pdf_path): + from PyPDF2 import PdfReader + pdf_document = PdfReader(pdf_path) headers = [] total_pages = len(pdf_document.pages) @@ -43,11 +45,13 @@ def extract_common_header(pdf_path): if len(headers) < 2: return "" # 如果没有足够的页来比较，返回空字符串 - # 寻找每一行中的公共部分 + # 寻找每一行中的公共部分，按顺序保留 common_headers = [] for lines in zip(*headers): - # 在每一行中寻找公共单词 - common_line = set(lines[0].split()).intersection(*[set(line.split()) for line in lines[1:]]) + # 提取第一行的词汇顺序 + first_words = lines[0].split() + # 筛选所有页面都包含的词汇，保持顺序 + common_line = [word for word in first_words if all(word in line.split() for line in lines[1:])] if common_line: common_headers.append(' '.join(common_line)) diff --git a/flask_app/main/读取文件/按页读取pdf.py b/flask_app/main/读取文件/按页读取pdf.py index 35ff38c..e49c0d6 100644 --- a/flask_app/main/读取文件/按页读取pdf.py +++ b/flask_app/main/读取文件/按页读取pdf.py @@ -19,6 +19,8 @@ def clean_page_content(text, common_header): return text def extract_common_header(pdf_path): + from PyPDF2 import PdfReader + pdf_document = PdfReader(pdf_path) headers = [] total_pages = len(pdf_document.pages) @@ -43,16 +45,19 @@ def extract_common_header(pdf_path): if len(headers) < 2: return "" # 如果没有足够的页来比较，返回空字符串 - # 寻找每一行中的公共部分 + # 寻找每一行中的公共部分，按顺序保留 common_headers = [] for lines in zip(*headers): - # 在每一行中寻找公共单词 - common_line = set(lines[0].split()).intersection(*[set(line.split()) for line in lines[1:]]) + # 提取第一行的词汇顺序 + first_words = lines[0].split() + # 筛选所有页面都包含的词汇，保持顺序 + common_line = [word for word in first_words if all(word in line.split() for line in lines[1:])] if common_line: common_headers.append(' '.join(common_line)) return '\n'.join(common_headers) + def extract_text_by_page(file_path): common_header = extract_common_header(file_path) result = "" @@ -147,6 +152,9 @@ if __name__ == '__main__': # file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\广水农商行门禁控制主机及基础验证设备采购项目——磋商文件（定稿）（三次）_tobidders_notice_part2.pdf' # file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\2-招标文件（2020年广水市中小学教师办公电脑系统及多媒体“班班通”设备采购安装项目）_tobidders_notice_part2.pdf' # file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\磋商文件_tobidders_notice_part2.pdf' - file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\竞争性谈判文件(3)_tobidders_notice_part2.pdf' + # file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\截取test\\交警支队机动车查验监管系统项目采购_tobidders_notice_part1.pdf' + file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles\\交警支队机动车查验监管系统项目采购.pdf' + # ress = extract_common_header(file_path) + # print(ress) res=extract_text_by_page(file_path) # print(res)磋商文件_tobidders_notice_part2.pdf \ No newline at end of file diff --git a/flask_app/货物标/货物标截取pdf.py b/flask_app/货物标/货物标截取pdf.py index 4d95ea9..703e198 100644 --- a/flask_app/货物标/货物标截取pdf.py +++ b/flask_app/货物标/货物标截取pdf.py @@ -22,6 +22,8 @@ def clean_page_content(text, common_header): # PYPDF2版本 def extract_common_header(pdf_path): + from PyPDF2 import PdfReader + pdf_document = PdfReader(pdf_path) headers = [] total_pages = len(pdf_document.pages) @@ -46,17 +48,21 @@ def extract_common_header(pdf_path): if len(headers) < 2: return "" # 如果没有足够的页来比较，返回空字符串 - # 寻找每一行中的公共部分 + # 寻找每一行中的公共部分，按顺序保留 common_headers = [] for lines in zip(*headers): - # 在每一行中寻找公共单词 - common_line = set(lines[0].split()).intersection(*[set(line.split()) for line in lines[1:]]) + # 提取第一行的词汇顺序 + first_words = lines[0].split() + # 筛选所有页面都包含的词汇，保持顺序 + common_line = [word for word in first_words if all(word in line.split() for line in lines[1:])] if common_line: common_headers.append(' '.join(common_line)) return '\n'.join(common_headers) + + # fitz库版本 # def extract_common_header(pdf_path): # doc = fitz.open(pdf_path) @@ -166,7 +172,6 @@ def extract_pages_generic(pdf_document, begin_pattern, end_pattern, begin_page, else: if start_page is None and re.search(begin_pattern, cleaned_text) and i > begin_page: start_page = i - if start_page is not None and re.search(end_pattern, cleaned_text) and i > start_page: end_page = i break @@ -207,28 +212,6 @@ def extract_pages(pdf_path, output_folder, begin_pattern, begin_page, end_patter print(f"Error processing {pdf_path}: {e}") return "" - -def extract_pages_tobidders_notice(pdf_document, begin_pattern, end_pattern, begin_page, common_header, - exclusion_pattern): - start_page = None - mid_page = None - end_page = None - for i, page in enumerate(pdf_document.pages): - text = page.extract_text() or "" - cleaned_text = clean_page_content(text, common_header) - if exclusion_pattern and re.search(exclusion_pattern, cleaned_text) and mid_page is not None: - continue - if start_page is None and re.search(begin_pattern, cleaned_text) and i > begin_page: - start_page = i - if start_page is not None and mid_page is None and re.search( - r'^\s*[（(]?\s*[一1]\s*[)）]?\s*[、．.]*\s*(说\s*明|总\s*则)', cleaned_text, re.MULTILINE): - mid_page = i - if start_page is not None and mid_page is not None and re.search(end_pattern, cleaned_text) and i > mid_page: - end_page = i - break - return start_page, mid_page, end_page - - def get_patterns_for_procurement(): begin_pattern = re.compile( r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:服务|项目|商务).*?要求|' @@ -268,15 +251,68 @@ def get_patterns_for_notice(): r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:公告|邀请书).*', re.MULTILINE ) end_pattern = re.compile( - r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人须知|磋商须知|供应商须知)+|(?:一\s*、\s*)?(?:投标人须知|磋商须知|供应商须知)前附表)', + # r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人须知|磋商须知|供应商须知)+|(?:一\s*、\s*)?(?:投标人须知|磋商须知|供应商须知)前附表)', + r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+', re.MULTILINE ) return begin_pattern, end_pattern +# def extract_pages_tobidders_notice(pdf_document, begin_pattern, end_pattern, begin_page, common_header, +# exclusion_pattern): +# start_page = None +# mid_page = None +# end_page = None +# for i, page in enumerate(pdf_document.pages): +# text = page.extract_text() or "" +# cleaned_text = clean_page_content(text, common_header) +# if exclusion_pattern and re.search(exclusion_pattern, cleaned_text) and mid_page is not None: +# continue +# if start_page is None and re.search(begin_pattern, cleaned_text) and i > begin_page: +# start_page = i +# if start_page is not None and mid_page is None and re.search( +# r'^\s*[（(]?\s*[一1]\s*[)）]?\s*[、．.]*\s*(说\s*明|总\s*则)', cleaned_text): +# mid_page = i +# if start_page is not None and mid_page is not None and re.search(end_pattern, cleaned_text) and i > mid_page: +# end_page = i +# break +# return start_page, mid_page, end_page + +def extract_pages_tobidders_notice(pdf_document, begin_pattern, end_pattern, begin_page, common_header, + exclusion_pattern): + def run_extraction(use_multiline=False): + start_page = None + mid_page = None + end_page = None + for i, page in enumerate(pdf_document.pages): + text = page.extract_text() or "" + cleaned_text = clean_page_content(text, common_header) + # print(cleaned_text) + if exclusion_pattern and re.search(exclusion_pattern, cleaned_text) and mid_page is not None: + continue + if start_page is None and re.search(begin_pattern, cleaned_text) and i > begin_page: + start_page = i + if start_page is not None and mid_page is None: + mid_pattern = r'^\s*[（(]?\s*[一1]\s*[)）]?\s*[、．.]*\s*(说\s*明|总\s*则)' + flags = re.MULTILINE if use_multiline else 0 + if re.search(mid_pattern, cleaned_text, flags): + mid_page = i + if start_page is not None and mid_page is not None and re.search(end_pattern, cleaned_text) and i > mid_page: + end_page = i + break + return start_page, mid_page, end_page + + # 第一次运行 + start_page, mid_page, end_page = run_extraction() + + # 如果有任何一个值为 None，使用 re.MULTILINE 重新运行 + if start_page is None or mid_page is None or end_page is None: + start_page, mid_page, end_page = run_extraction(use_multiline=True) + + return start_page, mid_page, end_page def extract_pages_twice_tobidders_notice(pdf_path, output_folder, output_suffix, common_header): # 投标人须知前附表/正文二次提取 begin_pattern = re.compile( - r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:(?:投标人|磋商|供应商|谈判供应商|磋商供应商)须知前附表)+' + r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:(?:投标人?|磋商|供应商|谈判供应商|磋商供应商)须知前附表)+' ) end_pattern = re.compile( r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+' @@ -460,9 +496,10 @@ def truncate_pdf_main(input_path, output_folder, selection, output_suffix="defau # 更新的正则表达式以匹配"第x章"和"第x部分"，考虑到可能的空格和其他文字 begin_pattern = re.compile( r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:服务|项目|商务).*?要求|' - r'^第[一二三四五六七八九十百千]+(?:章|部分).*?采购.*' + r'^第[一二三四五六七八九十百千]+(?:章|部分).*?采购.*|' + r'^第[一二三四五六七八九十百千]+(?:章|部分).*?需求.*' ) - begin_page = 5 + begin_page = 3 end_pattern = re.compile( r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+' ) @@ -488,7 +525,7 @@ def truncate_pdf_main(input_path, output_folder, selection, output_suffix="defau elif selection == 4: # 投标人须知前附表和正文 begin_page = 1 begin_pattern = re.compile( - r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人|磋商|供应商|谈判供应商|磋商供应商)须知+|(?:一\s*、\s*)?(?:投标人|磋商|供应商)须知前附表)', + r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人?|磋商|供应商|谈判供应商|磋商供应商)须知+|(?:一\s*、\s*)?(?:投标人?|磋商|供应商)须知前附表)', re.MULTILINE ) end_pattern = re.compile( @@ -498,10 +535,11 @@ def truncate_pdf_main(input_path, output_folder, selection, output_suffix="defau elif selection == 5: # 招标公告 begin_page = 0 begin_pattern = re.compile( - r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:公告|邀请书).*' + r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:公告|邀请书|邀请函).*' ) end_pattern = re.compile( - r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人|磋商|供应商|谈判供应商|磋商供应商)须知+|(?:一\s*、\s*)?(?:投标人|磋商|供应商)须知前附表)' + # r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人|磋商|供应商|谈判供应商|磋商供应商)须知+|(?:一\s*、\s*)?(?:投标人|磋商|供应商)须知前附表)' + r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+',re.MULTILINE ) local_output_suffix = "notice" else: @@ -574,11 +612,12 @@ def truncate_pdf_specific_goods(pdf_path, output_folder): # TODO:交通智能系统和招标(1)(1)文件有问题 sele=4的时候excludsion有问题 if __name__ == "__main__": - input_path = "C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles\\招标文件正文(1).pdf" - output_folder = "C:\\Users\\Administrator\\Desktop\\货物标\\truncate_all" + # input_path = "C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles\\交警支队机动车查验监管系统项目采购.pdf" + input_path = "C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles" + output_folder = "C:\\Users\\Administrator\\Desktop\\货物标\\截取test" # files = truncate_pdf_multiple(input_path, output_folder) - files=truncate_pdf_specific_goods(input_path,output_folder) - print(files[-1]) - # selection = 1 # 例如：1 - 商务技术服务要求, 2 - 评标办法, 3 - 资格审查后缀有qualification1或qualification2（与评标办法一致） 4.投标人须知前附表part1 投标人须知正文part2 5-公告 - # generated_files = truncate_pdf_main(input_path, output_folder, selection) + # files=truncate_pdf_specific_goods(input_path,output_folder) + # print(files) + selection = 1# 例如：1 - 商务技术服务要求, 2 - 评标办法, 3 - 资格审查后缀有qualification1或qualification2（与评标办法一致） 4.投标人须知前附表part1 投标人须知正文part2 5-公告 + generated_files = truncate_pdf_main(input_path, output_folder, selection)