fix text splitter

10 months ago · 324bc8cb88
parent 62586719b3
commit 324bc8cb88
1 changed files with 1 additions and 0 deletions
--- a/api/core/rag/splitter/fixed_text_splitter.py
+++ b/api/core/rag/splitter/fixed_text_splitter.py
@ -102,6 +102,7 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter)
                splits = text.split()
            else:
                splits = text.split(separator)
+                splits = [item + separator if i < len(splits) else item for i, item in splits]
        else:
            splits = list(text)
        splits = [s for s in splits if (s not in {"", "\n"})]