fix text splitter (#22596)

7 months ago · ed263aed9f
parent d37b6716cd
commit ed263aed9f
1 changed files with 1 additions and 0 deletions
--- a/api/core/rag/splitter/fixed_text_splitter.py
+++ b/api/core/rag/splitter/fixed_text_splitter.py
@ -102,6 +102,7 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter)
                splits = text.split()
            else:
                splits = text.split(separator)
+                splits = [item + separator if i < len(splits) else item for i, item in enumerate(splits)]
        else:
            splits = list(text)
        splits = [s for s in splits if (s not in {"", "\n"})]