feat: Weaviate supports Chinese bm25 #12223

api/.env.example

@@ -142,6 +142,10 @@ WEAVIATE_ENDPOINT=http://localhost:8080
 WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
 WEAVIATE_GRPC_ENABLED=false
 WEAVIATE_BATCH_SIZE=100
+#If it is a Chinese full-text index, please set it to 'gse'
+#https://weaviate.io/developers/weaviate/config-refs/schema#gse-and-trigram-tokenization-methods
+#https://pkg.go.dev/github.com/go-ego/gse#section-readme
+WEAVIATE_TOKENIZATION=
 
 # Qdrant configuration, use `http://localhost:6333` for local mode or `https://your-qdrant-cluster-url.qdrant.io` for remote mode
 QDRANT_URL=http://localhost:6333

api/configs/middleware/vdb/weaviate_config.py

@@ -28,3 +28,8 @@ class WeaviateConfig(BaseSettings):
         description="Number of objects to be processed in a single batch operation (default is 100)",
         default=100,
     )
+
+    WEAVIATE_TOKENIZATION: Optional[str] = Field(
+        description="Tokenization for Weaviate",
+        default=None,
+    )

api/core/rag/datasource/vdb/weaviate/weaviate_vector.py

@@ -248,15 +248,27 @@ class WeaviateVector(BaseVector):
         return docs
 
     def _default_schema(self, index_name: str) -> dict:
-        return {
-            "class": index_name,
-            "properties": [
-                {
-                    "name": "text",
-                    "dataType": ["text"],
-                }
-            ],
-        }
+        if dify_config.WEAVIATE_TOKENIZATION:
+            return {
+                "class": index_name,
+                "properties": [
+                    {
+                        "name": "text",
+                        "dataType": ["text"],
+                        "tokenization": dify_config.WEAVIATE_TOKENIZATION
+                    }
+                ],
+            }
+        else:
+            return {
+                "class": index_name,
+                "properties": [
+                    {
+                        "name": "text",
+                        "dataType": ["text"],
+                    }
+                ],
+            }
 
     def _json_serializable(self, value: Any) -> Any:
         if isinstance(value, datetime.datetime):

docker/.env.example

@@ -383,6 +383,10 @@ VECTOR_STORE=weaviate
 # The Weaviate endpoint URL. Only available when VECTOR_STORE is `weaviate`.
 WEAVIATE_ENDPOINT=http://weaviate:8080
 WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
+#If it is a Chinese full-text index, please set it to 'gse'
+#https://weaviate.io/developers/weaviate/config-refs/schema#gse-and-trigram-tokenization-methods
+#https://pkg.go.dev/github.com/go-ego/gse#section-readme
+WEAVIATE_TOKENIZATION=
 
 # The Qdrant endpoint URL. Only available when VECTOR_STORE is `qdrant`.
 QDRANT_URL=http://qdrant:6333

docker/docker-compose-template.yaml

@@ -220,6 +220,7 @@ services:
   # The Weaviate vector store.
   weaviate:
     image: semitechnologies/weaviate:1.19.0
+    #image: semitechnologies/weaviate:1.28.0    If it is a Chinese full-text index, please update weaviate version
     profiles:
       - ''
       - weaviate
@@ -240,6 +241,7 @@ services:
       AUTHENTICATION_APIKEY_USERS: ${WEAVIATE_AUTHENTICATION_APIKEY_USERS:-hello@dify.ai}
       AUTHORIZATION_ADMINLIST_ENABLED: ${WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED:-true}
       AUTHORIZATION_ADMINLIST_USERS: ${WEAVIATE_AUTHORIZATION_ADMINLIST_USERS:-hello@dify.ai}
+      #ENABLE_TOKENIZER_GSE: 'true'    If it is a Chinese full-text index, please set it to 'gse'
 
   # Qdrant vector store.
   # (if used, you need to set VECTOR_STORE to qdrant in the api & worker service.)

docker/docker-compose.yaml

@@ -327,6 +327,7 @@ x-shared-env: &shared-api-worker-env
   WEAVIATE_AUTHENTICATION_APIKEY_USERS: ${WEAVIATE_AUTHENTICATION_APIKEY_USERS:-hello@dify.ai}
   WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED: ${WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED:-true}
   WEAVIATE_AUTHORIZATION_ADMINLIST_USERS: ${WEAVIATE_AUTHORIZATION_ADMINLIST_USERS:-hello@dify.ai}
+  WEAVIATE_TOKENIZATION: ${WEAVIATE_TOKENIZATION:-}
   CHROMA_SERVER_AUTHN_CREDENTIALS: ${CHROMA_SERVER_AUTHN_CREDENTIALS:-difyai123456}
   CHROMA_SERVER_AUTHN_PROVIDER: ${CHROMA_SERVER_AUTHN_PROVIDER:-chromadb.auth.token_authn.TokenAuthenticationServerProvider}
   CHROMA_IS_PERSISTENT: ${CHROMA_IS_PERSISTENT:-TRUE}
@@ -607,6 +608,7 @@ services:
   # The Weaviate vector store.
   weaviate:
     image: semitechnologies/weaviate:1.19.0
+    #image: semitechnologies/weaviate:1.28.0    If it is a Chinese full-text index, please update weaviate version
     profiles:
       - ''
       - weaviate
@@ -627,6 +629,7 @@ services:
       AUTHENTICATION_APIKEY_USERS: ${WEAVIATE_AUTHENTICATION_APIKEY_USERS:-hello@dify.ai}
       AUTHORIZATION_ADMINLIST_ENABLED: ${WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED:-true}
       AUTHORIZATION_ADMINLIST_USERS: ${WEAVIATE_AUTHORIZATION_ADMINLIST_USERS:-hello@dify.ai}
+     #ENABLE_TOKENIZER_GSE: 'true'    If it is a Chinese full-text index, please set it to 'gse'
 
   # Qdrant vector store.
   # (if used, you need to set VECTOR_STORE to qdrant in the api & worker service.)

docker/middleware.env.example

@@ -78,6 +78,10 @@ WEAVIATE_AUTHENTICATION_APIKEY_USERS=hello@dify.ai
 WEAVIATE_AUTHORIZATION_ADMINLIST_ENABLED=true
 WEAVIATE_AUTHORIZATION_ADMINLIST_USERS=hello@dify.ai
 WEAVIATE_HOST_VOLUME=./volumes/weaviate
+#If it is a Chinese full-text index, please set it to 'gse'
+#https://weaviate.io/developers/weaviate/config-refs/schema#gse-and-trigram-tokenization-methods
+#https://pkg.go.dev/github.com/go-ego/gse#section-readme
+WEAVIATE_TOKENIZATION=
 
 # ------------------------------
 # Docker Compose Service Expose Host Port Configurations