fix: update nomic model provider token calculation (#8705)

pull/8712/head
ice yao 2 years ago committed by GitHub
parent 4638f99aaa
commit 64baedb484
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -39,3 +39,4 @@
- zhinao
- fireworks
- mixedbread
- nomic

@ -77,15 +77,7 @@ class NomicTextEmbeddingModel(_CommonNomic, TextEmbeddingModel):
:param texts: texts to embed
:return:
"""
if len(texts) == 0:
return 0
_, prompt_tokens, _ = self.embed_text(
model=model,
credentials=credentials,
texts=texts,
)
return prompt_tokens
return sum(self._get_num_tokens_by_gpt2(text) for text in texts)
def validate_credentials(self, model: str, credentials: dict) -> None:
"""

Loading…
Cancel
Save