update text spliter

feat/support-multi-token-count
jyong 1 year ago
parent a74d428489
commit e3f5ac236c

@ -45,7 +45,7 @@ class TextSplitter(BaseDocumentTransformer, ABC):
self, self,
chunk_size: int = 4000, chunk_size: int = 4000,
chunk_overlap: int = 200, chunk_overlap: int = 200,
length_function: Callable[[str], int] = len, length_function: Callable[[str], [int]] = len,
keep_separator: bool = False, keep_separator: bool = False,
add_start_index: bool = False, add_start_index: bool = False,
) -> None: ) -> None:

Loading…
Cancel
Save