10.18小解析

flask_app/main/start_up.py

@@ -257,7 +257,7 @@ def process_and_stream(file_url, zb_type):
     """
     logger = g.logger
     unique_id = g.unique_id
-    output_folder = f"flask_app/static/output/{unique_id}"
+    output_folder = f"flask_app/static/output1/{unique_id}"
     filename = "ztbfile"
     downloaded_filename = os.path.join(output_folder, filename)
 

flask_app/main/截取pdf.py

@@ -19,6 +19,8 @@ def clean_page_content(text, common_header):
 
 #PYPDF2库
 def extract_common_header(pdf_path):
+    from PyPDF2 import PdfReader
+
     pdf_document = PdfReader(pdf_path)
     headers = []
     total_pages = len(pdf_document.pages)
@@ -43,11 +45,13 @@ def extract_common_header(pdf_path):
     if len(headers) < 2:
         return ""  # 如果没有足够的页来比较，返回空字符串
 
-    # 寻找每一行中的公共部分
+    # 寻找每一行中的公共部分，按顺序保留
     common_headers = []
     for lines in zip(*headers):
-        # 在每一行中寻找公共单词
+        # 提取第一行的词汇顺序
-        common_line = set(lines[0].split()).intersection(*[set(line.split()) for line in lines[1:]])
+        first_words = lines[0].split()
+        # 筛选所有页面都包含的词汇，保持顺序
+        common_line = [word for word in first_words if all(word in line.split() for line in lines[1:])]
         if common_line:
             common_headers.append(' '.join(common_line))
 

flask_app/main/读取文件/按页读取pdf.py

@@ -19,6 +19,8 @@ def clean_page_content(text, common_header):
 
     return text
 def extract_common_header(pdf_path):
+    from PyPDF2 import PdfReader
+
     pdf_document = PdfReader(pdf_path)
     headers = []
     total_pages = len(pdf_document.pages)
@@ -43,16 +45,19 @@ def extract_common_header(pdf_path):
     if len(headers) < 2:
         return ""  # 如果没有足够的页来比较，返回空字符串
 
-    # 寻找每一行中的公共部分
+    # 寻找每一行中的公共部分，按顺序保留
     common_headers = []
     for lines in zip(*headers):
-        # 在每一行中寻找公共单词
+        # 提取第一行的词汇顺序
-        common_line = set(lines[0].split()).intersection(*[set(line.split()) for line in lines[1:]])
+        first_words = lines[0].split()
+        # 筛选所有页面都包含的词汇，保持顺序
+        common_line = [word for word in first_words if all(word in line.split() for line in lines[1:])]
         if common_line:
             common_headers.append(' '.join(common_line))
 
     return '\n'.join(common_headers)
 
+
 def extract_text_by_page(file_path):
     common_header = extract_common_header(file_path)
     result = ""
@@ -147,6 +152,9 @@ if __name__ == '__main__':
     # file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\广水农商行门禁控制主机及基础验证设备采购项目——磋商文件（定稿）（三次）_tobidders_notice_part2.pdf'
     # file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\2-招标文件（2020年广水市中小学教师办公电脑系统及多媒体“班班通”设备采购安装项目）_tobidders_notice_part2.pdf'
     # file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\磋商文件_tobidders_notice_part2.pdf'
-    file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\output4\\竞争性谈判文件(3)_tobidders_notice_part2.pdf'
+    # file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\截取test\\交警支队机动车查验监管系统项目采购_tobidders_notice_part1.pdf'
+    file_path = 'C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles\\交警支队机动车查验监管系统项目采购.pdf'
+    # ress = extract_common_header(file_path)
+    # print(ress)
     res=extract_text_by_page(file_path)
     # print(res)磋商文件_tobidders_notice_part2.pdf

flask_app/货物标/货物标截取pdf.py

@@ -22,6 +22,8 @@ def clean_page_content(text, common_header):
 
 # PYPDF2版本
 def extract_common_header(pdf_path):
+    from PyPDF2 import PdfReader
+
     pdf_document = PdfReader(pdf_path)
     headers = []
     total_pages = len(pdf_document.pages)
@@ -46,17 +48,21 @@ def extract_common_header(pdf_path):
     if len(headers) < 2:
         return ""  # 如果没有足够的页来比较，返回空字符串
 
-    # 寻找每一行中的公共部分
+    # 寻找每一行中的公共部分，按顺序保留
     common_headers = []
     for lines in zip(*headers):
-        # 在每一行中寻找公共单词
+        # 提取第一行的词汇顺序
-        common_line = set(lines[0].split()).intersection(*[set(line.split()) for line in lines[1:]])
+        first_words = lines[0].split()
+        # 筛选所有页面都包含的词汇，保持顺序
+        common_line = [word for word in first_words if all(word in line.split() for line in lines[1:])]
         if common_line:
             common_headers.append(' '.join(common_line))
 
     return '\n'.join(common_headers)
 
 
+
+
 # fitz库版本
 # def extract_common_header(pdf_path):
 #     doc = fitz.open(pdf_path)
@@ -166,7 +172,6 @@ def extract_pages_generic(pdf_document, begin_pattern, end_pattern, begin_page,
         else:
             if start_page is None and re.search(begin_pattern, cleaned_text) and i > begin_page:
                 start_page = i
-
         if start_page is not None and re.search(end_pattern, cleaned_text) and i > start_page:
             end_page = i
             break
@@ -207,28 +212,6 @@ def extract_pages(pdf_path, output_folder, begin_pattern, begin_page, end_patter
         print(f"Error processing {pdf_path}: {e}")
         return ""
 
-
-def extract_pages_tobidders_notice(pdf_document, begin_pattern, end_pattern, begin_page, common_header,
-                                   exclusion_pattern):
-    start_page = None
-    mid_page = None
-    end_page = None
-    for i, page in enumerate(pdf_document.pages):
-        text = page.extract_text() or ""
-        cleaned_text = clean_page_content(text, common_header)
-        if exclusion_pattern and re.search(exclusion_pattern, cleaned_text) and mid_page is not None:
-            continue
-        if start_page is None and re.search(begin_pattern, cleaned_text) and i > begin_page:
-            start_page = i
-        if start_page is not None and mid_page is None and re.search(
-                r'^\s*[（(]?\s*[一1]\s*[)）]?\s*[、．.]*\s*(说\s*明|总\s*则)', cleaned_text, re.MULTILINE):
-            mid_page = i
-        if start_page is not None and mid_page is not None and re.search(end_pattern, cleaned_text) and i > mid_page:
-            end_page = i
-            break
-    return start_page, mid_page, end_page
-
-
 def get_patterns_for_procurement():
     begin_pattern = re.compile(
         r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:服务|项目|商务).*?要求|'
@@ -268,15 +251,68 @@ def get_patterns_for_notice():
         r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:公告|邀请书).*', re.MULTILINE
     )
     end_pattern = re.compile(
-        r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人须知|磋商须知|供应商须知)+|(?:一\s*、\s*)?(?:投标人须知|磋商须知|供应商须知)前附表)',
+        # r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人须知|磋商须知|供应商须知)+|(?:一\s*、\s*)?(?:投标人须知|磋商须知|供应商须知)前附表)',
+        r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+',
         re.MULTILINE
     )
     return begin_pattern, end_pattern
 
+# def extract_pages_tobidders_notice(pdf_document, begin_pattern, end_pattern, begin_page, common_header,
+#                                    exclusion_pattern):
+#     start_page = None
+#     mid_page = None
+#     end_page = None
+#     for i, page in enumerate(pdf_document.pages):
+#         text = page.extract_text() or ""
+#         cleaned_text = clean_page_content(text, common_header)
+#         if exclusion_pattern and re.search(exclusion_pattern, cleaned_text) and mid_page is not None:
+#             continue
+#         if start_page is None and re.search(begin_pattern, cleaned_text) and i > begin_page:
+#             start_page = i
+#         if start_page is not None and mid_page is None and re.search(
+#                 r'^\s*[（(]?\s*[一1]\s*[)）]?\s*[、．.]*\s*(说\s*明|总\s*则)', cleaned_text):
+#             mid_page = i
+#         if start_page is not None and mid_page is not None and re.search(end_pattern, cleaned_text) and i > mid_page:
+#             end_page = i
+#             break
+#     return start_page, mid_page, end_page
+
+def extract_pages_tobidders_notice(pdf_document, begin_pattern, end_pattern, begin_page, common_header,
+                                   exclusion_pattern):
+    def run_extraction(use_multiline=False):
+        start_page = None
+        mid_page = None
+        end_page = None
+        for i, page in enumerate(pdf_document.pages):
+            text = page.extract_text() or ""
+            cleaned_text = clean_page_content(text, common_header)
+            # print(cleaned_text)
+            if exclusion_pattern and re.search(exclusion_pattern, cleaned_text) and mid_page is not None:
+                continue
+            if start_page is None and re.search(begin_pattern, cleaned_text) and i > begin_page:
+                start_page = i
+            if start_page is not None and mid_page is None:
+                mid_pattern = r'^\s*[（(]?\s*[一1]\s*[)）]?\s*[、．.]*\s*(说\s*明|总\s*则)'
+                flags = re.MULTILINE if use_multiline else 0
+                if re.search(mid_pattern, cleaned_text, flags):
+                    mid_page = i
+            if start_page is not None and mid_page is not None and re.search(end_pattern, cleaned_text) and i > mid_page:
+                end_page = i
+                break
+        return start_page, mid_page, end_page
+
+    # 第一次运行
+    start_page, mid_page, end_page = run_extraction()
+
+    # 如果有任何一个值为 None，使用 re.MULTILINE 重新运行
+    if start_page is None or mid_page is None or end_page is None:
+        start_page, mid_page, end_page = run_extraction(use_multiline=True)
+
+    return start_page, mid_page, end_page
 
 def extract_pages_twice_tobidders_notice(pdf_path, output_folder, output_suffix, common_header):  # 投标人须知前附表/正文二次提取
     begin_pattern = re.compile(
-        r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:(?:投标人|磋商|供应商|谈判供应商|磋商供应商)须知前附表)+'
+        r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:(?:投标人?|磋商|供应商|谈判供应商|磋商供应商)须知前附表)+'
     )
     end_pattern = re.compile(
         r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+'
@@ -460,9 +496,10 @@ def truncate_pdf_main(input_path, output_folder, selection, output_suffix="defau
             # 更新的正则表达式以匹配"第x章"和"第x部分"，考虑到可能的空格和其他文字
             begin_pattern = re.compile(
                 r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:服务|项目|商务).*?要求|'
-                r'^第[一二三四五六七八九十百千]+(?:章|部分).*?采购.*'
+                r'^第[一二三四五六七八九十百千]+(?:章|部分).*?采购.*|'
+                r'^第[一二三四五六七八九十百千]+(?:章|部分).*?需求.*'
             )
-            begin_page = 5
+            begin_page = 3
             end_pattern = re.compile(
                 r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+'
             )
@@ -488,7 +525,7 @@ def truncate_pdf_main(input_path, output_folder, selection, output_suffix="defau
         elif selection == 4:  # 投标人须知前附表和正文
             begin_page = 1
             begin_pattern = re.compile(
-                r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人|磋商|供应商|谈判供应商|磋商供应商)须知+|(?:一\s*、\s*)?(?:投标人|磋商|供应商)须知前附表)',
+                r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人?|磋商|供应商|谈判供应商|磋商供应商)须知+|(?:一\s*、\s*)?(?:投标人?|磋商|供应商)须知前附表)',
                 re.MULTILINE
             )
             end_pattern = re.compile(
@@ -498,10 +535,11 @@ def truncate_pdf_main(input_path, output_folder, selection, output_suffix="defau
         elif selection == 5:  # 招标公告
             begin_page = 0
             begin_pattern = re.compile(
-                r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:公告|邀请书).*'
+                r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:公告|邀请书|邀请函).*'
             )
             end_pattern = re.compile(
-                r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人|磋商|供应商|谈判供应商|磋商供应商)须知+|(?:一\s*、\s*)?(?:投标人|磋商|供应商)须知前附表)'
+                # r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人|磋商|供应商|谈判供应商|磋商供应商)须知+|(?:一\s*、\s*)?(?:投标人|磋商|供应商)须知前附表)'
+                r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+',re.MULTILINE
             )
             local_output_suffix = "notice"
         else:
@@ -574,11 +612,12 @@ def truncate_pdf_specific_goods(pdf_path, output_folder):
 
 # TODO:交通智能系统和招标(1)(1)文件有问题  sele=4的时候excludsion有问题
 if __name__ == "__main__":
-    input_path = "C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles\\招标文件正文(1).pdf"
+    # input_path = "C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles\\交警支队机动车查验监管系统项目采购.pdf"
-    output_folder = "C:\\Users\\Administrator\\Desktop\\货物标\\truncate_all"
+    input_path = "C:\\Users\\Administrator\\Desktop\\货物标\\zbfiles"
+    output_folder = "C:\\Users\\Administrator\\Desktop\\货物标\\截取test"
     # files = truncate_pdf_multiple(input_path, output_folder)
-    files=truncate_pdf_specific_goods(input_path,output_folder)
+    # files=truncate_pdf_specific_goods(input_path,output_folder)
-    print(files[-1])
+    # print(files)
-    # selection = 1  # 例如：1 - 商务技术服务要求, 2 - 评标办法, 3 - 资格审查后缀有qualification1或qualification2（与评标办法一致）  4.投标人须知前附表part1 投标人须知正文part2   5-公告
+    selection = 1# 例如：1 - 商务技术服务要求, 2 - 评标办法, 3 - 资格审查后缀有qualification1或qualification2（与评标办法一致）  4.投标人须知前附表part1 投标人须知正文part2   5-公告
-    # generated_files = truncate_pdf_main(input_path, output_folder, selection)
+    generated_files = truncate_pdf_main(input_path, output_folder, selection)