fix: ensure proper conversation role alternation for vLLM

- added a comment
pull/18837/head
StellaContrail 12 months ago
parent 9314b8526d
commit 37953337c6

@ -79,6 +79,10 @@ class QuestionClassifierNode(LLMNode):
memory=memory,
max_token_limit=rest_token,
)
# Some models (e.g. Gemma, Mistral) force roles alternation (user/assistant/user/assistant...).
# If both self._get_prompt_template and self._fetch_prompt_messages append a user prompt,
# two consecutive user prompts will be generated, causing model's error.
# To avoid this, set sys_query to an empty string so that only one user prompt is appended at the end.
prompt_messages, stop = self._fetch_prompt_messages(
prompt_template=prompt_template,
sys_query="",

Loading…
Cancel
Save