12.30 废标项

This commit is contained in:
zy123 2024-12-30 10:26:08 +08:00
parent 8bfbfc4263
commit 7d405e28fc

View File

@ -2,16 +2,11 @@ import json
import os
import re
import time
import fitz
import PyPDF2
import tempfile
import requests
from ratelimit import sleep_and_retry, limits
from flask_app.general import table_ocr
from flask_app.general.file2markdown import convert_file_to_markdown
from flask_app.general.clean_pdf import extract_common_header, clean_page_content
from flask_app.general.table_ocr import CommonOcr
@ -205,11 +200,6 @@ def count_tokens(text):
tokens = re.findall(token_pattern, text)
return len(tokens)# 返回 tokens 数量和匹配的 token 列表
import requests
import os
def get_total_tokens(text):
"""
调用 API 计算给定文本的总 Token 数量