fix: xinference-chat-stream-response (#991)

3 years ago · bd3a9b2f8d
parent 18d3877151
commit bd3a9b2f8d
1 changed files with 6 additions and 1 deletions
--- a/api/core/third_party/langchain/llms/xinference_llm.py
+++ b/api/core/third_party/langchain/llms/xinference_llm.py
@ -123,7 +123,12 @@ class XinferenceLLM(Xinference):
                if choices:
                    choice = choices[0]
                    if isinstance(choice, dict):
-                        token = choice.get("text", "")
+                        if 'text' in choice:
+                            token = choice.get("text", "")
+                        elif 'delta' in choice and 'content' in choice['delta']:
+                            token = choice.get('delta').get('content')
+                        else:
+                            continue
                        log_probs = choice.get("logprobs")
                        if run_manager:
                            run_manager.on_llm_new_token(