Merge remote-tracking branch 'origin/main'
JF.Hsiong
1 year ago
commit
298d0d6d0b
@ -0,0 +1,25 @@
|
|||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from pydantic import Field
|
||||||
|
from pydantic_settings import BaseSettings
|
||||||
|
|
||||||
|
|
||||||
|
class HuaweiCloudConfig(BaseSettings):
|
||||||
|
"""
|
||||||
|
Configuration settings for Huawei cloud search service
|
||||||
|
"""
|
||||||
|
|
||||||
|
HUAWEI_CLOUD_HOSTS: Optional[str] = Field(
|
||||||
|
description="Hostname or IP address of the Huawei cloud search service instance",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
HUAWEI_CLOUD_USER: Optional[str] = Field(
|
||||||
|
description="Username for authenticating with Huawei cloud search service",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
HUAWEI_CLOUD_PASSWORD: Optional[str] = Field(
|
||||||
|
description="Password for authenticating with Huawei cloud search service",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
@ -0,0 +1,52 @@
|
|||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from pydantic.fields import FieldInfo
|
||||||
|
|
||||||
|
from .http_request import NacosHttpClient
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
from configs.remote_settings_sources.base import RemoteSettingsSource
|
||||||
|
|
||||||
|
from .utils import _parse_config
|
||||||
|
|
||||||
|
|
||||||
|
class NacosSettingsSource(RemoteSettingsSource):
|
||||||
|
def __init__(self, configs: Mapping[str, Any]):
|
||||||
|
self.configs = configs
|
||||||
|
self.remote_configs: dict[str, Any] = {}
|
||||||
|
self.async_init()
|
||||||
|
|
||||||
|
def async_init(self):
|
||||||
|
data_id = os.getenv("DIFY_ENV_NACOS_DATA_ID", "dify-api-env.properties")
|
||||||
|
group = os.getenv("DIFY_ENV_NACOS_GROUP", "nacos-dify")
|
||||||
|
tenant = os.getenv("DIFY_ENV_NACOS_NAMESPACE", "")
|
||||||
|
|
||||||
|
params = {"dataId": data_id, "group": group, "tenant": tenant}
|
||||||
|
try:
|
||||||
|
content = NacosHttpClient().http_request("/nacos/v1/cs/configs", method="GET", headers={}, params=params)
|
||||||
|
self.remote_configs = self._parse_config(content)
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("[get-access-token] exception occurred")
|
||||||
|
raise
|
||||||
|
|
||||||
|
def _parse_config(self, content: str) -> dict:
|
||||||
|
if not content:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
return _parse_config(self, content)
|
||||||
|
except Exception as e:
|
||||||
|
raise RuntimeError(f"Failed to parse config: {e}")
|
||||||
|
|
||||||
|
def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
|
||||||
|
if not isinstance(self.remote_configs, dict):
|
||||||
|
raise ValueError(f"remote configs is not dict, but {type(self.remote_configs)}")
|
||||||
|
|
||||||
|
field_value = self.remote_configs.get(field_name)
|
||||||
|
if field_value is None:
|
||||||
|
return None, field_name, False
|
||||||
|
|
||||||
|
return field_value, field_name, False
|
||||||
@ -0,0 +1,83 @@
|
|||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class NacosHttpClient:
|
||||||
|
def __init__(self):
|
||||||
|
self.username = os.getenv("DIFY_ENV_NACOS_USERNAME")
|
||||||
|
self.password = os.getenv("DIFY_ENV_NACOS_PASSWORD")
|
||||||
|
self.ak = os.getenv("DIFY_ENV_NACOS_ACCESS_KEY")
|
||||||
|
self.sk = os.getenv("DIFY_ENV_NACOS_SECRET_KEY")
|
||||||
|
self.server = os.getenv("DIFY_ENV_NACOS_SERVER_ADDR", "localhost:8848")
|
||||||
|
self.token = None
|
||||||
|
self.token_ttl = 18000
|
||||||
|
self.token_expire_time: float = 0
|
||||||
|
|
||||||
|
def http_request(self, url, method="GET", headers=None, params=None):
|
||||||
|
try:
|
||||||
|
self._inject_auth_info(headers, params)
|
||||||
|
response = requests.request(method, url="http://" + self.server + url, headers=headers, params=params)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.text
|
||||||
|
except requests.exceptions.RequestException as e:
|
||||||
|
return f"Request to Nacos failed: {e}"
|
||||||
|
|
||||||
|
def _inject_auth_info(self, headers, params, module="config"):
|
||||||
|
headers.update({"User-Agent": "Nacos-Http-Client-In-Dify:v0.0.1"})
|
||||||
|
|
||||||
|
if module == "login":
|
||||||
|
return
|
||||||
|
|
||||||
|
ts = str(int(time.time() * 1000))
|
||||||
|
|
||||||
|
if self.ak and self.sk:
|
||||||
|
sign_str = self.get_sign_str(params["group"], params["tenant"], ts)
|
||||||
|
headers["Spas-AccessKey"] = self.ak
|
||||||
|
headers["Spas-Signature"] = self.__do_sign(sign_str, self.sk)
|
||||||
|
headers["timeStamp"] = ts
|
||||||
|
if self.username and self.password:
|
||||||
|
self.get_access_token(force_refresh=False)
|
||||||
|
params["accessToken"] = self.token
|
||||||
|
|
||||||
|
def __do_sign(self, sign_str, sk):
|
||||||
|
return (
|
||||||
|
base64.encodebytes(hmac.new(sk.encode(), sign_str.encode(), digestmod=hashlib.sha1).digest())
|
||||||
|
.decode()
|
||||||
|
.strip()
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_sign_str(self, group, tenant, ts):
|
||||||
|
sign_str = ""
|
||||||
|
if tenant:
|
||||||
|
sign_str = tenant + "+"
|
||||||
|
if group:
|
||||||
|
sign_str = sign_str + group + "+"
|
||||||
|
if sign_str:
|
||||||
|
sign_str += ts
|
||||||
|
return sign_str
|
||||||
|
|
||||||
|
def get_access_token(self, force_refresh=False):
|
||||||
|
current_time = time.time()
|
||||||
|
if self.token and not force_refresh and self.token_expire_time > current_time:
|
||||||
|
return self.token
|
||||||
|
|
||||||
|
params = {"username": self.username, "password": self.password}
|
||||||
|
url = "http://" + self.server + "/nacos/v1/auth/login"
|
||||||
|
try:
|
||||||
|
resp = requests.request("POST", url, headers=None, params=params)
|
||||||
|
resp.raise_for_status()
|
||||||
|
response_data = resp.json()
|
||||||
|
self.token = response_data.get("accessToken")
|
||||||
|
self.token_ttl = response_data.get("tokenTtl", 18000)
|
||||||
|
self.token_expire_time = current_time + self.token_ttl - 10
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("[get-access-token] exception occur")
|
||||||
|
raise
|
||||||
@ -0,0 +1,31 @@
|
|||||||
|
def _parse_config(self, content: str) -> dict[str, str]:
|
||||||
|
config: dict[str, str] = {}
|
||||||
|
if not content:
|
||||||
|
return config
|
||||||
|
|
||||||
|
for line in content.splitlines():
|
||||||
|
cleaned_line = line.strip()
|
||||||
|
if not cleaned_line or cleaned_line.startswith(("#", "!")):
|
||||||
|
continue
|
||||||
|
|
||||||
|
separator_index = -1
|
||||||
|
for i, c in enumerate(cleaned_line):
|
||||||
|
if c in ("=", ":") and (i == 0 or cleaned_line[i - 1] != "\\"):
|
||||||
|
separator_index = i
|
||||||
|
break
|
||||||
|
|
||||||
|
if separator_index == -1:
|
||||||
|
continue
|
||||||
|
|
||||||
|
key = cleaned_line[:separator_index].strip()
|
||||||
|
raw_value = cleaned_line[separator_index + 1 :].strip()
|
||||||
|
|
||||||
|
try:
|
||||||
|
decoded_value = bytes(raw_value, "utf-8").decode("unicode_escape")
|
||||||
|
decoded_value = decoded_value.replace(r"\=", "=").replace(r"\:", ":")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
decoded_value = raw_value
|
||||||
|
|
||||||
|
config[key] = decoded_value
|
||||||
|
|
||||||
|
return config
|
||||||
@ -0,0 +1,215 @@
|
|||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import ssl
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
from elasticsearch import Elasticsearch
|
||||||
|
from pydantic import BaseModel, model_validator
|
||||||
|
|
||||||
|
from configs import dify_config
|
||||||
|
from core.rag.datasource.vdb.field import Field
|
||||||
|
from core.rag.datasource.vdb.vector_base import BaseVector
|
||||||
|
from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory
|
||||||
|
from core.rag.datasource.vdb.vector_type import VectorType
|
||||||
|
from core.rag.embedding.embedding_base import Embeddings
|
||||||
|
from core.rag.models.document import Document
|
||||||
|
from extensions.ext_redis import redis_client
|
||||||
|
from models.dataset import Dataset
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def create_ssl_context() -> ssl.SSLContext:
|
||||||
|
ssl_context = ssl.create_default_context()
|
||||||
|
ssl_context.check_hostname = False
|
||||||
|
ssl_context.verify_mode = ssl.CERT_NONE
|
||||||
|
return ssl_context
|
||||||
|
|
||||||
|
|
||||||
|
class HuaweiCloudVectorConfig(BaseModel):
|
||||||
|
hosts: str
|
||||||
|
username: str | None
|
||||||
|
password: str | None
|
||||||
|
|
||||||
|
@model_validator(mode="before")
|
||||||
|
@classmethod
|
||||||
|
def validate_config(cls, values: dict) -> dict:
|
||||||
|
if not values["hosts"]:
|
||||||
|
raise ValueError("config HOSTS is required")
|
||||||
|
return values
|
||||||
|
|
||||||
|
def to_elasticsearch_params(self) -> dict[str, Any]:
|
||||||
|
params = {
|
||||||
|
"hosts": self.hosts.split(","),
|
||||||
|
"verify_certs": False,
|
||||||
|
"ssl_show_warn": False,
|
||||||
|
"request_timeout": 30000,
|
||||||
|
"retry_on_timeout": True,
|
||||||
|
"max_retries": 10,
|
||||||
|
}
|
||||||
|
if self.username and self.password:
|
||||||
|
params["basic_auth"] = (self.username, self.password)
|
||||||
|
return params
|
||||||
|
|
||||||
|
|
||||||
|
class HuaweiCloudVector(BaseVector):
|
||||||
|
def __init__(self, index_name: str, config: HuaweiCloudVectorConfig):
|
||||||
|
super().__init__(index_name.lower())
|
||||||
|
self._client = Elasticsearch(**config.to_elasticsearch_params())
|
||||||
|
|
||||||
|
def get_type(self) -> str:
|
||||||
|
return VectorType.HUAWEI_CLOUD
|
||||||
|
|
||||||
|
def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs):
|
||||||
|
uuids = self._get_uuids(documents)
|
||||||
|
for i in range(len(documents)):
|
||||||
|
self._client.index(
|
||||||
|
index=self._collection_name,
|
||||||
|
id=uuids[i],
|
||||||
|
document={
|
||||||
|
Field.CONTENT_KEY.value: documents[i].page_content,
|
||||||
|
Field.VECTOR.value: embeddings[i] or None,
|
||||||
|
Field.METADATA_KEY.value: documents[i].metadata or {},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self._client.indices.refresh(index=self._collection_name)
|
||||||
|
return uuids
|
||||||
|
|
||||||
|
def text_exists(self, id: str) -> bool:
|
||||||
|
return bool(self._client.exists(index=self._collection_name, id=id))
|
||||||
|
|
||||||
|
def delete_by_ids(self, ids: list[str]) -> None:
|
||||||
|
if not ids:
|
||||||
|
return
|
||||||
|
for id in ids:
|
||||||
|
self._client.delete(index=self._collection_name, id=id)
|
||||||
|
|
||||||
|
def delete_by_metadata_field(self, key: str, value: str) -> None:
|
||||||
|
query_str = {"query": {"match": {f"metadata.{key}": f"{value}"}}}
|
||||||
|
results = self._client.search(index=self._collection_name, body=query_str)
|
||||||
|
ids = [hit["_id"] for hit in results["hits"]["hits"]]
|
||||||
|
if ids:
|
||||||
|
self.delete_by_ids(ids)
|
||||||
|
|
||||||
|
def delete(self) -> None:
|
||||||
|
self._client.indices.delete(index=self._collection_name)
|
||||||
|
|
||||||
|
def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
|
||||||
|
top_k = kwargs.get("top_k", 4)
|
||||||
|
|
||||||
|
query = {
|
||||||
|
"size": top_k,
|
||||||
|
"query": {
|
||||||
|
"vector": {
|
||||||
|
Field.VECTOR.value: {
|
||||||
|
"vector": query_vector,
|
||||||
|
"topk": top_k,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
results = self._client.search(index=self._collection_name, body=query)
|
||||||
|
|
||||||
|
docs_and_scores = []
|
||||||
|
for hit in results["hits"]["hits"]:
|
||||||
|
docs_and_scores.append(
|
||||||
|
(
|
||||||
|
Document(
|
||||||
|
page_content=hit["_source"][Field.CONTENT_KEY.value],
|
||||||
|
vector=hit["_source"][Field.VECTOR.value],
|
||||||
|
metadata=hit["_source"][Field.METADATA_KEY.value],
|
||||||
|
),
|
||||||
|
hit["_score"],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
docs = []
|
||||||
|
for doc, score in docs_and_scores:
|
||||||
|
score_threshold = float(kwargs.get("score_threshold") or 0.0)
|
||||||
|
if score > score_threshold:
|
||||||
|
if doc.metadata is not None:
|
||||||
|
doc.metadata["score"] = score
|
||||||
|
docs.append(doc)
|
||||||
|
|
||||||
|
return docs
|
||||||
|
|
||||||
|
def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
|
||||||
|
query_str = {"match": {Field.CONTENT_KEY.value: query}}
|
||||||
|
results = self._client.search(index=self._collection_name, query=query_str, size=kwargs.get("top_k", 4))
|
||||||
|
docs = []
|
||||||
|
for hit in results["hits"]["hits"]:
|
||||||
|
docs.append(
|
||||||
|
Document(
|
||||||
|
page_content=hit["_source"][Field.CONTENT_KEY.value],
|
||||||
|
vector=hit["_source"][Field.VECTOR.value],
|
||||||
|
metadata=hit["_source"][Field.METADATA_KEY.value],
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return docs
|
||||||
|
|
||||||
|
def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs):
|
||||||
|
metadatas = [d.metadata if d.metadata is not None else {} for d in texts]
|
||||||
|
self.create_collection(embeddings, metadatas)
|
||||||
|
self.add_texts(texts, embeddings, **kwargs)
|
||||||
|
|
||||||
|
def create_collection(
|
||||||
|
self,
|
||||||
|
embeddings: list[list[float]],
|
||||||
|
metadatas: Optional[list[dict[Any, Any]]] = None,
|
||||||
|
index_params: Optional[dict] = None,
|
||||||
|
):
|
||||||
|
lock_name = f"vector_indexing_lock_{self._collection_name}"
|
||||||
|
with redis_client.lock(lock_name, timeout=20):
|
||||||
|
collection_exist_cache_key = f"vector_indexing_{self._collection_name}"
|
||||||
|
if redis_client.get(collection_exist_cache_key):
|
||||||
|
logger.info(f"Collection {self._collection_name} already exists.")
|
||||||
|
return
|
||||||
|
|
||||||
|
if not self._client.indices.exists(index=self._collection_name):
|
||||||
|
dim = len(embeddings[0])
|
||||||
|
mappings = {
|
||||||
|
"properties": {
|
||||||
|
Field.CONTENT_KEY.value: {"type": "text"},
|
||||||
|
Field.VECTOR.value: { # Make sure the dimension is correct here
|
||||||
|
"type": "vector",
|
||||||
|
"dimension": dim,
|
||||||
|
"indexing": True,
|
||||||
|
"algorithm": "GRAPH",
|
||||||
|
"metric": "cosine",
|
||||||
|
"neighbors": 32,
|
||||||
|
"efc": 128,
|
||||||
|
},
|
||||||
|
Field.METADATA_KEY.value: {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"doc_id": {"type": "keyword"} # Map doc_id to keyword type
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
settings = {"index.vector": True}
|
||||||
|
self._client.indices.create(index=self._collection_name, mappings=mappings, settings=settings)
|
||||||
|
|
||||||
|
redis_client.set(collection_exist_cache_key, 1, ex=3600)
|
||||||
|
|
||||||
|
|
||||||
|
class HuaweiCloudVectorFactory(AbstractVectorFactory):
|
||||||
|
def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings) -> HuaweiCloudVector:
|
||||||
|
if dataset.index_struct_dict:
|
||||||
|
class_prefix: str = dataset.index_struct_dict["vector_store"]["class_prefix"]
|
||||||
|
collection_name = class_prefix.lower()
|
||||||
|
else:
|
||||||
|
dataset_id = dataset.id
|
||||||
|
collection_name = Dataset.gen_collection_name_by_id(dataset_id).lower()
|
||||||
|
dataset.index_struct = json.dumps(self.gen_index_struct_dict(VectorType.HUAWEI_CLOUD, collection_name))
|
||||||
|
|
||||||
|
return HuaweiCloudVector(
|
||||||
|
index_name=collection_name,
|
||||||
|
config=HuaweiCloudVectorConfig(
|
||||||
|
hosts=dify_config.HUAWEI_CLOUD_HOSTS or "http://localhost:9200",
|
||||||
|
username=dify_config.HUAWEI_CLOUD_USER,
|
||||||
|
password=dify_config.HUAWEI_CLOUD_PASSWORD,
|
||||||
|
),
|
||||||
|
)
|
||||||
@ -0,0 +1,24 @@
|
|||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
|
|
||||||
|
class ResponseFormat(StrEnum):
|
||||||
|
"""Constants for model response formats"""
|
||||||
|
|
||||||
|
JSON_SCHEMA = "json_schema" # model's structured output mode. some model like gemini, gpt-4o, support this mode.
|
||||||
|
JSON = "JSON" # model's json mode. some model like claude support this mode.
|
||||||
|
JSON_OBJECT = "json_object" # json mode's another alias. some model like deepseek-chat, qwen use this alias.
|
||||||
|
|
||||||
|
|
||||||
|
class SpecialModelType(StrEnum):
|
||||||
|
"""Constants for identifying model types"""
|
||||||
|
|
||||||
|
GEMINI = "gemini"
|
||||||
|
OLLAMA = "ollama"
|
||||||
|
|
||||||
|
|
||||||
|
class SupportStructuredOutputStatus(StrEnum):
|
||||||
|
"""Constants for structured output support status"""
|
||||||
|
|
||||||
|
SUPPORTED = "supported"
|
||||||
|
UNSUPPORTED = "unsupported"
|
||||||
|
DISABLED = "disabled"
|
||||||
@ -0,0 +1,63 @@
|
|||||||
|
"""
|
||||||
|
Patch for OpenTelemetry context detach method to handle None tokens gracefully.
|
||||||
|
|
||||||
|
This patch addresses the issue where OpenTelemetry's context.detach() method raises a TypeError
|
||||||
|
when called with a None token. The error occurs in the contextvars_context.py file where it tries
|
||||||
|
to call reset() on a None token.
|
||||||
|
|
||||||
|
Related GitHub issue: https://github.com/langgenius/dify/issues/18496
|
||||||
|
|
||||||
|
Error being fixed:
|
||||||
|
```
|
||||||
|
Traceback (most recent call last):
|
||||||
|
File "opentelemetry/context/__init__.py", line 154, in detach
|
||||||
|
_RUNTIME_CONTEXT.detach(token)
|
||||||
|
File "opentelemetry/context/contextvars_context.py", line 50, in detach
|
||||||
|
self._current_context.reset(token) # type: ignore
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
TypeError: expected an instance of Token, got None
|
||||||
|
```
|
||||||
|
|
||||||
|
Instead of modifying the third-party package directly, this patch monkey-patches the
|
||||||
|
context.detach method to gracefully handle None tokens.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from functools import wraps
|
||||||
|
|
||||||
|
from opentelemetry import context
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Store the original detach method
|
||||||
|
original_detach = context.detach
|
||||||
|
|
||||||
|
|
||||||
|
# Create a patched version that handles None tokens
|
||||||
|
@wraps(original_detach)
|
||||||
|
def patched_detach(token):
|
||||||
|
"""
|
||||||
|
A patched version of context.detach that handles None tokens gracefully.
|
||||||
|
"""
|
||||||
|
if token is None:
|
||||||
|
logger.debug("Attempted to detach a None token, skipping")
|
||||||
|
return
|
||||||
|
|
||||||
|
return original_detach(token)
|
||||||
|
|
||||||
|
|
||||||
|
def is_enabled():
|
||||||
|
"""
|
||||||
|
Check if the extension is enabled.
|
||||||
|
Always enable this patch to prevent errors even when OpenTelemetry is disabled.
|
||||||
|
"""
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def init_app(app):
|
||||||
|
"""
|
||||||
|
Initialize the OpenTelemetry context patch.
|
||||||
|
"""
|
||||||
|
# Replace the original detach method with our patched version
|
||||||
|
context.detach = patched_detach
|
||||||
|
logger.info("OpenTelemetry context.detach patched to handle None tokens")
|
||||||
@ -0,0 +1,88 @@
|
|||||||
|
import os
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from _pytest.monkeypatch import MonkeyPatch
|
||||||
|
from api.core.rag.datasource.vdb.field import Field
|
||||||
|
from elasticsearch import Elasticsearch
|
||||||
|
|
||||||
|
|
||||||
|
class MockIndicesClient:
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def create(self, index, mappings, settings):
|
||||||
|
return {"acknowledge": True}
|
||||||
|
|
||||||
|
def refresh(self, index):
|
||||||
|
return {"acknowledge": True}
|
||||||
|
|
||||||
|
def delete(self, index):
|
||||||
|
return {"acknowledge": True}
|
||||||
|
|
||||||
|
def exists(self, index):
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class MockClient:
|
||||||
|
def __init__(self, **kwargs):
|
||||||
|
self.indices = MockIndicesClient()
|
||||||
|
|
||||||
|
def index(self, **kwargs):
|
||||||
|
return {"acknowledge": True}
|
||||||
|
|
||||||
|
def exists(self, **kwargs):
|
||||||
|
return True
|
||||||
|
|
||||||
|
def delete(self, **kwargs):
|
||||||
|
return {"acknowledge": True}
|
||||||
|
|
||||||
|
def search(self, **kwargs):
|
||||||
|
return {
|
||||||
|
"took": 1,
|
||||||
|
"hits": {
|
||||||
|
"hits": [
|
||||||
|
{
|
||||||
|
"_source": {
|
||||||
|
Field.CONTENT_KEY.value: "abcdef",
|
||||||
|
Field.VECTOR.value: [1, 2],
|
||||||
|
Field.METADATA_KEY.value: {},
|
||||||
|
},
|
||||||
|
"_score": 1.0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"_source": {
|
||||||
|
Field.CONTENT_KEY.value: "123456",
|
||||||
|
Field.VECTOR.value: [2, 2],
|
||||||
|
Field.METADATA_KEY.value: {},
|
||||||
|
},
|
||||||
|
"_score": 0.9,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"_source": {
|
||||||
|
Field.CONTENT_KEY.value: "a1b2c3",
|
||||||
|
Field.VECTOR.value: [3, 2],
|
||||||
|
Field.METADATA_KEY.value: {},
|
||||||
|
},
|
||||||
|
"_score": 0.8,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
MOCK = os.getenv("MOCK_SWITCH", "false").lower() == "true"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def setup_client_mock(request, monkeypatch: MonkeyPatch):
|
||||||
|
if MOCK:
|
||||||
|
monkeypatch.setattr(Elasticsearch, "__init__", MockClient.__init__)
|
||||||
|
monkeypatch.setattr(Elasticsearch, "index", MockClient.index)
|
||||||
|
monkeypatch.setattr(Elasticsearch, "exists", MockClient.exists)
|
||||||
|
monkeypatch.setattr(Elasticsearch, "delete", MockClient.delete)
|
||||||
|
monkeypatch.setattr(Elasticsearch, "search", MockClient.search)
|
||||||
|
|
||||||
|
yield
|
||||||
|
|
||||||
|
if MOCK:
|
||||||
|
monkeypatch.undo()
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
from core.rag.datasource.vdb.huawei.huawei_cloud_vector import HuaweiCloudVector, HuaweiCloudVectorConfig
|
||||||
|
from tests.integration_tests.vdb.__mock.huaweicloudvectordb import setup_client_mock
|
||||||
|
from tests.integration_tests.vdb.test_vector_store import AbstractVectorTest, get_example_text, setup_mock_redis
|
||||||
|
|
||||||
|
|
||||||
|
class HuaweiCloudVectorTest(AbstractVectorTest):
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.vector = HuaweiCloudVector(
|
||||||
|
"dify",
|
||||||
|
HuaweiCloudVectorConfig(
|
||||||
|
hosts="https://127.0.0.1:9200",
|
||||||
|
username="dify",
|
||||||
|
password="dify",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def search_by_vector(self):
|
||||||
|
hits_by_vector = self.vector.search_by_vector(query_vector=self.example_embedding)
|
||||||
|
assert len(hits_by_vector) == 3
|
||||||
|
|
||||||
|
def search_by_full_text(self):
|
||||||
|
hits_by_full_text = self.vector.search_by_full_text(query=get_example_text())
|
||||||
|
assert len(hits_by_full_text) == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_huawei_cloud_vector(setup_mock_redis, setup_client_mock):
|
||||||
|
HuaweiCloudVectorTest().run_all_tests()
|
||||||
@ -0,0 +1,27 @@
|
|||||||
|
from core.model_runtime.entities.message_entities import (
|
||||||
|
ImagePromptMessageContent,
|
||||||
|
TextPromptMessageContent,
|
||||||
|
UserPromptMessage,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_prompt_message_with_prompt_message_contents():
|
||||||
|
prompt = UserPromptMessage(content=[TextPromptMessageContent(data="Hello, World!")])
|
||||||
|
assert isinstance(prompt.content, list)
|
||||||
|
assert isinstance(prompt.content[0], TextPromptMessageContent)
|
||||||
|
assert prompt.content[0].data == "Hello, World!"
|
||||||
|
|
||||||
|
|
||||||
|
def test_dump_prompt_message():
|
||||||
|
example_url = "https://example.com/image.jpg"
|
||||||
|
prompt = UserPromptMessage(
|
||||||
|
content=[
|
||||||
|
ImagePromptMessageContent(
|
||||||
|
url=example_url,
|
||||||
|
format="jpeg",
|
||||||
|
mime_type="image/jpeg",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
data = prompt.model_dump()
|
||||||
|
assert data["content"][0].get("url") == example_url
|
||||||
@ -0,0 +1,82 @@
|
|||||||
|
import { fireEvent, render, screen } from '@testing-library/react'
|
||||||
|
import ConfigSelect from './index'
|
||||||
|
|
||||||
|
jest.mock('react-sortablejs', () => ({
|
||||||
|
ReactSortable: ({ children }: { children: React.ReactNode }) => <div>{children}</div>,
|
||||||
|
}))
|
||||||
|
|
||||||
|
jest.mock('react-i18next', () => ({
|
||||||
|
useTranslation: () => ({
|
||||||
|
t: (key: string) => key,
|
||||||
|
}),
|
||||||
|
}))
|
||||||
|
|
||||||
|
describe('ConfigSelect Component', () => {
|
||||||
|
const defaultProps = {
|
||||||
|
options: ['Option 1', 'Option 2'],
|
||||||
|
onChange: jest.fn(),
|
||||||
|
}
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
jest.clearAllMocks()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('renders all options', () => {
|
||||||
|
render(<ConfigSelect {...defaultProps} />)
|
||||||
|
|
||||||
|
defaultProps.options.forEach((option) => {
|
||||||
|
expect(screen.getByDisplayValue(option)).toBeInTheDocument()
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('renders add button', () => {
|
||||||
|
render(<ConfigSelect {...defaultProps} />)
|
||||||
|
|
||||||
|
expect(screen.getByText('appDebug.variableConfig.addOption')).toBeInTheDocument()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('handles option deletion', () => {
|
||||||
|
render(<ConfigSelect {...defaultProps} />)
|
||||||
|
const optionContainer = screen.getByDisplayValue('Option 1').closest('div')
|
||||||
|
const deleteButton = optionContainer?.querySelector('div[role="button"]')
|
||||||
|
|
||||||
|
if (!deleteButton) return
|
||||||
|
fireEvent.click(deleteButton)
|
||||||
|
expect(defaultProps.onChange).toHaveBeenCalledWith(['Option 2'])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('handles adding new option', () => {
|
||||||
|
render(<ConfigSelect {...defaultProps} />)
|
||||||
|
const addButton = screen.getByText('appDebug.variableConfig.addOption')
|
||||||
|
|
||||||
|
fireEvent.click(addButton)
|
||||||
|
|
||||||
|
expect(defaultProps.onChange).toHaveBeenCalledWith([...defaultProps.options, ''])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('applies focus styles on input focus', () => {
|
||||||
|
render(<ConfigSelect {...defaultProps} />)
|
||||||
|
const firstInput = screen.getByDisplayValue('Option 1')
|
||||||
|
|
||||||
|
fireEvent.focus(firstInput)
|
||||||
|
|
||||||
|
expect(firstInput.closest('div')).toHaveClass('border-components-input-border-active')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('applies delete hover styles', () => {
|
||||||
|
render(<ConfigSelect {...defaultProps} />)
|
||||||
|
const optionContainer = screen.getByDisplayValue('Option 1').closest('div')
|
||||||
|
const deleteButton = optionContainer?.querySelector('div[role="button"]')
|
||||||
|
|
||||||
|
if (!deleteButton) return
|
||||||
|
fireEvent.mouseEnter(deleteButton)
|
||||||
|
expect(optionContainer).toHaveClass('border-components-input-border-destructive')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('renders empty state correctly', () => {
|
||||||
|
render(<ConfigSelect options={[]} onChange={defaultProps.onChange} />)
|
||||||
|
|
||||||
|
expect(screen.queryByRole('textbox')).not.toBeInTheDocument()
|
||||||
|
expect(screen.getByText('appDebug.variableConfig.addOption')).toBeInTheDocument()
|
||||||
|
})
|
||||||
|
})
|
||||||
@ -0,0 +1,11 @@
|
|||||||
|
const IndeterminateIcon = () => {
|
||||||
|
return (
|
||||||
|
<div data-testid='indeterminate-icon'>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 12 12" fill="none">
|
||||||
|
<path d="M2.5 6H9.5" stroke="currentColor" strokeWidth="1.5" strokeLinecap="round"/>
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default IndeterminateIcon
|
||||||
@ -1,5 +0,0 @@
|
|||||||
<svg width="12" height="12" viewBox="0 0 12 12" fill="none" xmlns="http://www.w3.org/2000/svg">
|
|
||||||
<g id="check">
|
|
||||||
<path id="Vector 1" d="M2.5 6H9.5" stroke="white" stroke-width="1.5" stroke-linecap="round"/>
|
|
||||||
</g>
|
|
||||||
</svg>
|
|
||||||
|
Before Width: | Height: | Size: 217 B |
@ -1,10 +0,0 @@
|
|||||||
.mixed {
|
|
||||||
background: var(--color-components-checkbox-bg) url(./assets/mixed.svg) center center no-repeat;
|
|
||||||
background-size: 12px 12px;
|
|
||||||
border: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.checked.disabled {
|
|
||||||
background-color: #d0d5dd;
|
|
||||||
border-color: #d0d5dd;
|
|
||||||
}
|
|
||||||
@ -0,0 +1,67 @@
|
|||||||
|
import { fireEvent, render, screen } from '@testing-library/react'
|
||||||
|
import Checkbox from './index'
|
||||||
|
|
||||||
|
describe('Checkbox Component', () => {
|
||||||
|
const mockProps = {
|
||||||
|
id: 'test',
|
||||||
|
}
|
||||||
|
|
||||||
|
it('renders unchecked checkbox by default', () => {
|
||||||
|
render(<Checkbox {...mockProps} />)
|
||||||
|
const checkbox = screen.getByTestId('checkbox-test')
|
||||||
|
expect(checkbox).toBeInTheDocument()
|
||||||
|
expect(checkbox).not.toHaveClass('bg-components-checkbox-bg')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('renders checked checkbox when checked prop is true', () => {
|
||||||
|
render(<Checkbox {...mockProps} checked />)
|
||||||
|
const checkbox = screen.getByTestId('checkbox-test')
|
||||||
|
expect(checkbox).toHaveClass('bg-components-checkbox-bg')
|
||||||
|
expect(screen.getByTestId('check-icon-test')).toBeInTheDocument()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('renders indeterminate state correctly', () => {
|
||||||
|
render(<Checkbox {...mockProps} indeterminate />)
|
||||||
|
expect(screen.getByTestId('indeterminate-icon')).toBeInTheDocument()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('handles click events when not disabled', () => {
|
||||||
|
const onCheck = jest.fn()
|
||||||
|
render(<Checkbox {...mockProps} onCheck={onCheck} />)
|
||||||
|
const checkbox = screen.getByTestId('checkbox-test')
|
||||||
|
|
||||||
|
fireEvent.click(checkbox)
|
||||||
|
expect(onCheck).toHaveBeenCalledTimes(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('does not handle click events when disabled', () => {
|
||||||
|
const onCheck = jest.fn()
|
||||||
|
render(<Checkbox {...mockProps} disabled onCheck={onCheck} />)
|
||||||
|
const checkbox = screen.getByTestId('checkbox-test')
|
||||||
|
|
||||||
|
fireEvent.click(checkbox)
|
||||||
|
expect(onCheck).not.toHaveBeenCalled()
|
||||||
|
expect(checkbox).toHaveClass('cursor-not-allowed')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('applies custom className when provided', () => {
|
||||||
|
const customClass = 'custom-class'
|
||||||
|
render(<Checkbox {...mockProps} className={customClass} />)
|
||||||
|
const checkbox = screen.getByTestId('checkbox-test')
|
||||||
|
expect(checkbox).toHaveClass(customClass)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('applies correct styles for disabled checked state', () => {
|
||||||
|
render(<Checkbox {...mockProps} checked disabled />)
|
||||||
|
const checkbox = screen.getByTestId('checkbox-test')
|
||||||
|
expect(checkbox).toHaveClass('bg-components-checkbox-bg-disabled-checked')
|
||||||
|
expect(checkbox).toHaveClass('cursor-not-allowed')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('applies correct styles for disabled unchecked state', () => {
|
||||||
|
render(<Checkbox {...mockProps} disabled />)
|
||||||
|
const checkbox = screen.getByTestId('checkbox-test')
|
||||||
|
expect(checkbox).toHaveClass('bg-components-checkbox-bg-disabled')
|
||||||
|
expect(checkbox).toHaveClass('cursor-not-allowed')
|
||||||
|
})
|
||||||
|
})
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue