feat: use Gemini response metadata for token counting (#11743)

1 year ago · 7d5a385811
parent 900e93f758
commit 7d5a385811
1 changed files with 6 additions and 2 deletions
--- a/api/core/model_runtime/model_providers/google/llm/llm.py
+++ b/api/core/model_runtime/model_providers/google/llm/llm.py
@ -292,6 +292,10 @@ class GoogleLargeLanguageModel(LargeLanguageModel):
                    )
                else:
                    # calculate num tokens
+                    if hasattr(response, "usage_metadata") and response.usage_metadata:
+                        prompt_tokens = response.usage_metadata.prompt_token_count
+                        completion_tokens = response.usage_metadata.candidates_token_count
+                    else:
                        prompt_tokens = self.get_num_tokens(model, credentials, prompt_messages)
                        completion_tokens = self.get_num_tokens(model, credentials, [assistant_prompt_message])