Skip to content

Commit

Permalink
fix: 修复openai计算token错误
Browse files Browse the repository at this point in the history
(cherry picked from commit 3d1b3ea)
  • Loading branch information
shaohuzhang1 committed Aug 29, 2024
1 parent d1e9081 commit 30b1bdf
Showing 1 changed file with 2 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,14 @@ def new_instance(model_type, model_name, model_credential: Dict[str, object], **

def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int:
try:
super().get_num_tokens_from_messages(messages)
return super().get_num_tokens_from_messages(messages)
except Exception as e:
tokenizer = TokenizerManage.get_tokenizer()
return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages])

def get_num_tokens(self, text: str) -> int:
try:
super().get_num_tokens(text)
return super().get_num_tokens(text)
except Exception as e:
tokenizer = TokenizerManage.get_tokenizer()
return len(tokenizer.encode(text))

0 comments on commit 30b1bdf

Please sign in to comment.