zy123/zbparse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

12.30 废标项

Browse Source
This commit is contained in:
zy123 2024-12-30 10:26:08 +08:00
parent 8bfbfc4263
commit 7d405e28fc
1 changed files with 0 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
flask_app/general/doubao.py
View File

@ -2,16 +2,11 @@ import json
import os import os
import re import re
import time import time
import fitz import fitz
import PyPDF2 import PyPDF2
import tempfile import tempfile
import requests import requests
from ratelimit import sleep_and_retry, limits from ratelimit import sleep_and_retry, limits
from flask_app.general import table_ocr
from flask_app.general.file2markdown import convert_file_to_markdown
from flask_app.general.clean_pdf import extract_common_header, clean_page_content from flask_app.general.clean_pdf import extract_common_header, clean_page_content
from flask_app.general.table_ocr import CommonOcr from flask_app.general.table_ocr import CommonOcr
@ -205,11 +200,6 @@ def count_tokens(text):
tokens = re.findall(token_pattern, text) tokens = re.findall(token_pattern, text)
return len(tokens)# 返回 tokens 数量和匹配的 token 列表 return len(tokens)# 返回 tokens 数量和匹配的 token 列表
import requests
import os
def get_total_tokens(text): def get_total_tokens(text):
""" """
调用 API 计算给定文本的总 Token 数量。 调用 API 计算给定文本的总 Token 数量。