zbparse/flask_app/货物标/截取pdf货物标版.py

from PyPDF2 import PdfReader, PdfWriter
import regex  # 导入正则表达式库
import os  # 用于文件和文件夹操作
from flask_app.general.clean_pdf import clean_page_content, extract_common_header
from flask_app.general.merge_pdfs import merge_and_cleanup
from flask_app.general.截取pdf通用函数 import get_start_and_common_header, save_extracted_pages, is_pdf_or_doc, \
    convert_to_pdf, get_invalid_file, extract_pages_tobidders_notice, extract_pages_generic
from flask_app.general.通用功能函数 import get_global_logger

def extract_pages(pdf_path, output_folder, begin_pattern, begin_page, end_pattern, output_suffix,logger):
    try:
        common_header = extract_common_header(pdf_path)
        pdf_document = PdfReader(pdf_path)
        exclusion_pattern = None
        # 原有的处理逻辑保持不变
        if output_suffix == "qualification1" or output_suffix == "procurement" or output_suffix == "evaluation_method":
            exclusion_pattern = regex.compile(
                r'文件的构成|文件的组成|文件构成|文件组成|文件的编制|文件编制')
        start_page, end_page = extract_pages_generic(pdf_document, begin_pattern, end_pattern, begin_page,
                                                         common_header, exclusion_pattern, output_suffix)
        if start_page is None or end_page is None:
            print(f"first: {output_suffix} 未找到起始或结束页在文件 {pdf_path} 中！尝试备用提取策略。")
            return extract_pages_twice(pdf_path, output_folder, output_suffix, common_header, begin_page,logger)
        elif output_suffix == "qualification1":
            truncate_pdf_main_goods(pdf_path, output_folder, 2, logger,"qualification3")  # 合并'资格审查'章节和'评标办法'章节
        return save_extracted_pages(pdf_path, output_folder,start_page, end_page, output_suffix,common_header)
    except Exception as e:
        print(f"Error processing {pdf_path}: {e}")
        return ""

def get_patterns_for_procurement():
    begin_pattern = regex.compile(
        r'(?<!见\s*)(?<!与\s*)(?<!"\s*)(?<!“\s*)(?<!”\s*)'
        r'第[一二三四五六七八九十1-9]+(?:章|部分)\s*(?!.*说明)'  # 匹配“第X章”或“第X部分”
        r'[\u4e00-\u9fff、()（）]*?'  # 匹配允许的字符
        r'(?:(?:服务|项目|商务|技术|供货)[\u4e00-\u9fff、()（）]*?要求[\u4e00-\u9fff、()（）]*?\s*$|'  # 匹配“服务”、“项目”、“商务”或“技术”后跟“要求”
        r'(?:采购.*?(?:内容|要求|需求)|招标(?:内容|要求|需求))[\u4e00-\u9fff、()（）]*?|'  
        r'需求书[\u4e00-\u9fff、()（）]*?)\s*$',
        regex.MULTILINE
    )
    end_pattern = regex.compile(r'(?<!见\s*)(?<!与\s*)(?<!"\s*)(?<!“\s*)(?<!”\s*)第[一二三四五六七八九1-9]+(?:章|部分)\s*[\u4e00-\u9fff、()（）]+\s*$', regex.MULTILINE)
    return begin_pattern, end_pattern

 # """
 #    第四章
 #    评标
 #  """ 也能匹配上
def get_patterns_for_evaluation_method():
    begin_pattern = regex.compile(
        r'(?:(?<!见\s*)(?<!与\s*)(?<!"\s*)(?<!“\s*)(?<!“\s*)第[一二三四五六七八九十1-9]+(?:章|部分)\s*'  # 第一种模式
        r'(?:[\u4e00-\u9fff、()（）]*?)'
        r'(?=.*(?:(?:磋商|谈判)(?=.*(?:办法|方法|内容))|(?:评标|评定|评审)))'  
        r'[\u4e00-\u9fff、()（）]*\s*$|'
        r'^\s*(?<!见\s*)(?<!与\s*)(?<!"\s*)(?<!“\s*)(?<!“\s*)评标(方法|办法)前附表\s*$)',  # 第二种模式
        regex.MULTILINE
    )
    end_pattern = regex.compile(
        r'(?<!见\s*)(?<!与\s*)(?<!"\s*)(?<!“\s*)(?<!”\s*)第[一二三四五六七八九十1-9]+(?:章|部分)\s*[\u4e00-\u9fff、()（）]+\s*$', regex.MULTILINE)
    return begin_pattern, end_pattern


def get_patterns_for_notice():
    begin_pattern = regex.compile(r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:公告|邀请书|邀请函).*')
    end_pattern = regex.compile(
        # r'^(?:第[一二三四五六七八九十百千]+(?:章|部分)\s*(?:投标人须知|磋商须知|供应商须知)+|(?:一\s*、\s*)?(?:投标人须知|磋商须知|供应商须知)前附表)',
        r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+',
        regex.MULTILINE
    )
    return begin_pattern, end_pattern

def extract_pages_qualification(pdf_document, begin_page, common_header):
    # 开始匹配模式，仅匹配“附录”、“附件”或“附表”
    begin_pattern = regex.compile(
        r'^(?:附录(?:一|1)?[：:]?|附件(?:一|1)?[：:]?|附表(?:一|1)?[：:]?)',
        regex.MULTILINE
    )

    # 优先匹配模式，匹配以“资格性检查”、“资格审查”或“符合性审查”开头
    priority_pattern = regex.compile(
        r'^(资格性检查|资格审查|符合性审查)',
        regex.MULTILINE
    )

    # 结束匹配模式 - 附录、附件、附表等
    end_pattern_attachment = regex.compile(
        r'^(?:附录.*?[：:]|附件.*?[：:]|附表.*?[：:]|附件\s*\d+).*$',
        regex.MULTILINE
    )
    # 结束匹配模式 - 章节标题
    end_pattern_chapter = regex.compile(
        r'(?<!见\s*)(?<!与\s*)(?<!"\s*)(?<!“\s*)(?<!”\s*)第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff、()（）]+\s*$',
        regex.MULTILINE
    )

    print("第二次尝试 qualification:匹配附件")
    start_page = None
    end_page = None
    include_keywords = ["资格审查", "资质审查", "符合性审查", "资格性检查", "符合性检查", "资格检查"]
    exclude_keywords = ["声明函", "承诺函"]

    # 从指定的开始页开始遍历
    for i, page in enumerate(pdf_document.pages[begin_page:], start=begin_page):
        text = page.extract_text()
        if text:
            cleaned_text = clean_page_content(text, common_header)

            # 优先检查是否匹配优先模式
            priority_match = priority_pattern.search(cleaned_text)
            if priority_match and start_page is None:
                start_page = i
                # print(f"匹配到优先模式，设置起始页为: {start_page}")
                continue
            else:
                # 如果未匹配优先模式，则按照一般模式进行判断
                if (
                        any(keyword in cleaned_text for keyword in include_keywords) and
                        all(keyword not in cleaned_text for keyword in exclude_keywords) and
                        start_page is None
                ):
                    if begin_pattern.search(cleaned_text):
                        start_page = i
                        continue
                        # print(f"匹配到附录等模式，设置起始页为: {start_page}")

            # 确定结束页 - 附录、附件、附表等
            if start_page is not None and end_pattern_attachment.search(cleaned_text):
                # 额外检查当前页面是否不包含任何 include_keywords
                if not any(keyword in cleaned_text for keyword in include_keywords):
                    if i > start_page:
                        end_page = i
                        # print(f"找到结束页 (附件类): {end_page}")
                        break  # 找到结束页后退出循环
                else:
                    print(f"当前页面匹配附件结束模式但包含 include_keywords，继续查找。页面: {i}")

            # 确定结束页 - 章节标题
            elif start_page is not None and end_pattern_chapter.search(cleaned_text):
                if i > start_page:
                    end_page = i
                    print(f"找到结束页 (章节标题): {end_page}")
                    break  # 找到结束页后退出循环
    return start_page, end_page


def extract_pages_twice(pdf_path, output_folder, output_suffix, common_header, begin_page, logger):
    try:
        exclusion_pattern = regex.compile(
            r'文件的构成|文件的组成|文件组成|文件构成|文件的编制|文件编制')
        pdf_document = PdfReader(pdf_path)
        patterns = None
        start_page = None
        end_page = None

        if output_suffix == "procurement":
            patterns = [get_patterns_for_procurement()]
        elif output_suffix == "evaluation_method" or output_suffix == "qualification2" or output_suffix == "qualification3":
            patterns = [get_patterns_for_evaluation_method()]
        elif output_suffix == "notice":
            patterns = [get_patterns_for_notice()]
        elif output_suffix == "qualification1":
            start_page, end_page = extract_pages_qualification(pdf_document, begin_page, common_header)
        if patterns:
            for pattern_pair in patterns:
                start_page, end_page = extract_pages_generic(pdf_document, pattern_pair[0], pattern_pair[1], begin_page,
                                                             common_header, exclusion_pattern, output_suffix)
                if start_page is not None and end_page is not None:
                    break

        if start_page is None or end_page is None:
            if output_suffix == "qualification1":
                # print(f"second: {output_suffix} 未找到起始或结束页在文件 {pdf_path} 中！")
                print("第三次尝试资格审查:尝试提取评分办法章节...")
                base_file_name = os.path.splitext(os.path.basename(pdf_path))[0]
                evaluation_method_file = os.path.join(output_folder, f"{base_file_name}_evaluation_method.pdf")
                if os.path.isfile(evaluation_method_file):
                    print(f"找到评分办法章节文件: {evaluation_method_file}，生成新文件。")
                    # 获取文件路径和文件名
                    new_file_name = f"{base_file_name}_qualification2.pdf"
                    new_file_path = os.path.join(output_folder, new_file_name)
                    # 复制文件
                    with open(evaluation_method_file, 'rb') as original_file:
                        with open(new_file_path, 'wb') as new_file:
                            new_file.write(original_file.read())
                    return new_file_path
                else:
                    temp = truncate_pdf_main_goods(pdf_path, output_folder, 2,logger, "qualification2")
                    if len(temp) > 0:
                        return temp[0]
                    else:
                        return ""
            else:
                print(f"second: {output_suffix} 未找到起始或结束页在文件 {pdf_path} 中！")
                return ""
        return save_extracted_pages(pdf_path, output_folder,start_page, end_page, output_suffix,common_header)
    except Exception as e:
        print(f"Error in extract_pages_twice: {e}")
        return ""

def truncate_pdf_main_goods(input_path, output_folder, selection,logger, output_suffix="default"):
    try:
        # Function to handle processing of a single file
        def process_single_file(input_path, output_folder, selection, output_suffix):
            try:
                # 创建输出文件夹
                if not os.path.exists(output_folder):
                    os.makedirs(output_folder)

                # 获取起始和通用页眉
                pdf_path = convert_to_pdf(input_path)
                common_header, last_begin_index = get_start_and_common_header(input_path, 10)
                begin_page = last_begin_index if last_begin_index != 0 else {
                    4: 0,
                    2: 5,
                    3: 5,
                    1: 0,
                    5: 3,
                    6: 0  # Added default for selection 6 if needed
                }.get(selection, 0)

                # 根据选择设置对应的模式和结束模式
                if selection == 1:
                    begin_pattern = regex.compile(
                        r'.*(?<!见\s*)(?<!与\s*)(?<!"\s*)(?<!“\s*)(?<!”\s*)(?:招标公告|磋商公告|谈判公告|邀请书|邀请函|投标邀请|磋商邀请|谈判邀请|采购公告)[\)）]?\s*$',
                        regex.MULTILINE
                    )
                    end_pattern = regex.compile(
                        r'(?<!见\s*)(?<!与\s*)(?<!"\s*)(?<!“\s*)(?<!”\s*)第[一二三四五六七八九十1-9]+(?:章|部分)\s*[\u4e00-\u9fff、()（）]+\s*$|'
                        r'(?<!见\s*)(?<!与\s*)(?<!"\s*)(?<!“\s*)(?<!”\s*)(?:投标人?|磋商|供应商|谈判供应商|磋商供应商)须知前附表\s*$',
                        regex.MULTILINE
                    )
                    local_output_suffix = "notice"
                elif selection == 2:
                    begin_pattern = regex.compile(
                        r'^第[一二三四五六七八九十]+(?:章|部分)\s*'
                        r'(?=.*(?:磋商(?=.*(?:办法|方法|内容))|'
                        r'谈判(?=.*(?:办法|方法|内容))|评标|评定|评审))'
                    )
                    end_pattern = regex.compile(
                        r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+',regex.MULTILINE
                    )
                    local_output_suffix = "evaluation_method"
                elif selection == 3:
                    begin_pattern = regex.compile(
                        r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(资格审查).*', regex.MULTILINE
                    )
                    end_pattern = regex.compile(
                        r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+', regex.MULTILINE
                    )
                    local_output_suffix = "qualification1"
                elif selection == 4:
                    path1, path2 = extract_pages_tobidders_notice(pdf_path, output_folder, begin_page, common_header)
                    return [path1 or "", path2 or ""]
                elif selection == 5:
                    begin_pattern = regex.compile(
                        r'^第[一二三四五六七八九十百千]+(?:章|部分).*?(?:服务|项目|商务|技术|供货).*?要求|'
                        r'^第[一二三四五六七八九十百千]+(?:章|部分)(?!.*说明).*(?:采购.*?(?:内容|要求|需求)|招标(?:内容|要求|需求)).*|'
                        r'^第[一二三四五六七八九十百千]+(?:章|部分).*?需求书'
                    )
                    end_pattern = regex.compile(
                        r'^第[一二三四五六七八九十百千]+(?:章|部分)\s*[\u4e00-\u9fff]+',regex.MULTILINE
                    )
                    local_output_suffix = "procurement"
                elif selection == 6:
                    invalid_path = get_invalid_file(pdf_path, output_folder, common_header, begin_page)
                    return [invalid_path or ""]
                else:
                    print("无效的选择:请选择1-6")
                    return ['']

                # 如果传入的 output_suffix 是 'default'，则使用本地生成的 output_suffix
                if output_suffix == "default":
                    output_suffix = local_output_suffix
                result = extract_pages(
                        pdf_path,
                        output_folder,
                        begin_pattern,
                        begin_page,
                        end_pattern,
                        output_suffix,
                        logger
                    )
                # 根据提取结果以及不同的 output_suffix 进行处理
                if result:
                    if output_suffix == "qualification1":
                        merge_and_cleanup(result, "qualification3")
                        return [result or ""]
                    else:
                        return [result or ""]
                return [""]

            except Exception as e:
                logger.error(f"Error in processing file '{input_path}': {e}")
                return [""] * (2 if selection == 4 else 1)

        # 检查是否为文件夹
        if os.path.isdir(input_path):
            generated_files = []
            for file_name in os.listdir(input_path):
                file_path = os.path.join(input_path, file_name)
                if is_pdf_or_doc(file_path):
                    result = process_single_file(file_path, output_folder, selection, output_suffix)
                    if isinstance(result, tuple):
                        generated_files.extend([f if f else "" for f in result])
                    else:
                        generated_files.append(result)
            return generated_files

        # 单文件情况
        elif os.path.isfile(input_path) and is_pdf_or_doc(input_path):
            return process_single_file(input_path, output_folder, selection, output_suffix)

        else:
            logger.error("提供的路径既不是文件夹也不是PDF文件。")
            return [""] * (2 if selection == 4 else 1)

    except Exception as e:
        logger.error(f"Error in truncate_pdf_main_goods: {e}")
        return [""] * (2 if selection == 4 else 1)  # 返回空字符串



# TODO:交通智能系统和招标(1)(1)文件有问题  包头 绍兴     工程标中，判断是符合性审查之后，可以将它们设为同一章
#TODO: 验证所有文件的切分逻辑。
# ztbfile.pdf少资格评审  包头少符合性评审
if __name__ == "__main__":
    logger = get_global_logger("123")
    # input_path = r"C:\Users\Administrator\Desktop\new招标文件\货物标"
    # pdf_path = r"C:\Users\Administrator\Desktop\招标文件-采购类\2024-贵州-贵州医科大学附属医院导视系统零星制作安装项目.pdf"
    pdf_path=r"C:\Users\Administrator\Desktop\货物标\zbfiles\zbtest4_evaluation_method.pdf"
    # input_path = r"C:\Users\Administrator\Desktop\货物标\zbfiles\2-招标文件（广水市教育局封闭管理）.pdf"
    # pdf_path=r"C:\Users\Administrator\Desktop\文件解析问题\文件解析问题\1414cb9c-7bf4-401c-8761-2acde151b9c2\ztbfile.pdf"
    output_folder = r"C:\Users\Administrator\Desktop\货物标\output4_2"
    # output_folder = r"C:\Users\Administrator\Desktop\new招标文件\output2"
    selection = 4  # 例如：1 - 公告, 2 - 评标办法, 3 - 资格审查后缀有qualification1或qualification2（与评标办法一致）  4.投标人须知前附表part1 投标人须知正文part2   5-采购需求 6-invalid_path
    generated_files = truncate_pdf_main_goods(pdf_path, output_folder, selection,logger)
    print(generated_files)