|
|
|
@ -45,13 +45,12 @@ class BaseDocumentTransformer(ABC):
|
|
|
|
.. code-block:: python
|
|
|
|
.. code-block:: python
|
|
|
|
|
|
|
|
|
|
|
|
class EmbeddingsRedundantFilter(BaseDocumentTransformer, BaseModel):
|
|
|
|
class EmbeddingsRedundantFilter(BaseDocumentTransformer, BaseModel):
|
|
|
|
|
|
|
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
|
|
|
|
|
|
|
|
|
|
|
embeddings: Embeddings
|
|
|
|
embeddings: Embeddings
|
|
|
|
similarity_fn: Callable = cosine_similarity
|
|
|
|
similarity_fn: Callable = cosine_similarity
|
|
|
|
similarity_threshold: float = 0.95
|
|
|
|
similarity_threshold: float = 0.95
|
|
|
|
|
|
|
|
|
|
|
|
class Config:
|
|
|
|
|
|
|
|
arbitrary_types_allowed = True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def transform_documents(
|
|
|
|
def transform_documents(
|
|
|
|
self, documents: Sequence[Document], **kwargs: Any
|
|
|
|
self, documents: Sequence[Document], **kwargs: Any
|
|
|
|
) -> Sequence[Document]:
|
|
|
|
) -> Sequence[Document]:
|
|
|
|
|