12.30 废标项
This commit is contained in:
parent
8bfbfc4263
commit
7d405e28fc
@ -2,16 +2,11 @@ import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
import fitz
|
||||
import PyPDF2
|
||||
import tempfile
|
||||
import requests
|
||||
from ratelimit import sleep_and_retry, limits
|
||||
|
||||
from flask_app.general import table_ocr
|
||||
from flask_app.general.file2markdown import convert_file_to_markdown
|
||||
|
||||
from flask_app.general.clean_pdf import extract_common_header, clean_page_content
|
||||
from flask_app.general.table_ocr import CommonOcr
|
||||
|
||||
@ -205,11 +200,6 @@ def count_tokens(text):
|
||||
tokens = re.findall(token_pattern, text)
|
||||
return len(tokens)# 返回 tokens 数量和匹配的 token 列表
|
||||
|
||||
|
||||
import requests
|
||||
import os
|
||||
|
||||
|
||||
def get_total_tokens(text):
|
||||
"""
|
||||
调用 API 计算给定文本的总 Token 数量。
|
||||
|
Loading…
x
Reference in New Issue
Block a user