Merge remote-tracking branch 'upstream/main'
root
1 year ago
commit
b3cbc27e91
@ -1,3 +1,3 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
cd api && poetry install
|
cd api && uv sync
|
||||||
|
|||||||
@ -1,36 +0,0 @@
|
|||||||
name: Setup Poetry and Python
|
|
||||||
|
|
||||||
inputs:
|
|
||||||
python-version:
|
|
||||||
description: Python version to use and the Poetry installed with
|
|
||||||
required: true
|
|
||||||
default: '3.11'
|
|
||||||
poetry-version:
|
|
||||||
description: Poetry version to set up
|
|
||||||
required: true
|
|
||||||
default: '2.0.1'
|
|
||||||
poetry-lockfile:
|
|
||||||
description: Path to the Poetry lockfile to restore cache from
|
|
||||||
required: true
|
|
||||||
default: ''
|
|
||||||
|
|
||||||
runs:
|
|
||||||
using: composite
|
|
||||||
steps:
|
|
||||||
- name: Set up Python ${{ inputs.python-version }}
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ inputs.python-version }}
|
|
||||||
cache: pip
|
|
||||||
|
|
||||||
- name: Install Poetry
|
|
||||||
shell: bash
|
|
||||||
run: pip install poetry==${{ inputs.poetry-version }}
|
|
||||||
|
|
||||||
- name: Restore Poetry cache
|
|
||||||
if: ${{ inputs.poetry-lockfile != '' }}
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ inputs.python-version }}
|
|
||||||
cache: poetry
|
|
||||||
cache-dependency-path: ${{ inputs.poetry-lockfile }}
|
|
||||||
@ -0,0 +1,34 @@
|
|||||||
|
name: Setup UV and Python
|
||||||
|
|
||||||
|
inputs:
|
||||||
|
python-version:
|
||||||
|
description: Python version to use and the UV installed with
|
||||||
|
required: true
|
||||||
|
default: '3.12'
|
||||||
|
uv-version:
|
||||||
|
description: UV version to set up
|
||||||
|
required: true
|
||||||
|
default: '0.6.14'
|
||||||
|
uv-lockfile:
|
||||||
|
description: Path to the UV lockfile to restore cache from
|
||||||
|
required: true
|
||||||
|
default: ''
|
||||||
|
enable-cache:
|
||||||
|
required: true
|
||||||
|
default: true
|
||||||
|
|
||||||
|
runs:
|
||||||
|
using: composite
|
||||||
|
steps:
|
||||||
|
- name: Set up Python ${{ inputs.python-version }}
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: ${{ inputs.python-version }}
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
uses: astral-sh/setup-uv@v5
|
||||||
|
with:
|
||||||
|
version: ${{ inputs.uv-version }}
|
||||||
|
python-version: ${{ inputs.python-version }}
|
||||||
|
enable-cache: ${{ inputs.enable-cache }}
|
||||||
|
cache-dependency-glob: ${{ inputs.uv-lockfile }}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from pydantic import Field
|
||||||
|
from pydantic_settings import BaseSettings
|
||||||
|
|
||||||
|
|
||||||
|
class HuaweiCloudConfig(BaseSettings):
|
||||||
|
"""
|
||||||
|
Configuration settings for Huawei cloud search service
|
||||||
|
"""
|
||||||
|
|
||||||
|
HUAWEI_CLOUD_HOSTS: Optional[str] = Field(
|
||||||
|
description="Hostname or IP address of the Huawei cloud search service instance",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
HUAWEI_CLOUD_USER: Optional[str] = Field(
|
||||||
|
description="Username for authenticating with Huawei cloud search service",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
HUAWEI_CLOUD_PASSWORD: Optional[str] = Field(
|
||||||
|
description="Password for authenticating with Huawei cloud search service",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
@ -0,0 +1,45 @@
|
|||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
from pydantic import Field, PositiveInt
|
||||||
|
from pydantic_settings import BaseSettings
|
||||||
|
|
||||||
|
|
||||||
|
class VastbaseVectorConfig(BaseSettings):
|
||||||
|
"""
|
||||||
|
Configuration settings for Vector (Vastbase with vector extension)
|
||||||
|
"""
|
||||||
|
|
||||||
|
VASTBASE_HOST: Optional[str] = Field(
|
||||||
|
description="Hostname or IP address of the Vastbase server with Vector extension (e.g., 'localhost')",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
VASTBASE_PORT: PositiveInt = Field(
|
||||||
|
description="Port number on which the Vastbase server is listening (default is 5432)",
|
||||||
|
default=5432,
|
||||||
|
)
|
||||||
|
|
||||||
|
VASTBASE_USER: Optional[str] = Field(
|
||||||
|
description="Username for authenticating with the Vastbase database",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
VASTBASE_PASSWORD: Optional[str] = Field(
|
||||||
|
description="Password for authenticating with the Vastbase database",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
VASTBASE_DATABASE: Optional[str] = Field(
|
||||||
|
description="Name of the Vastbase database to connect to",
|
||||||
|
default=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
VASTBASE_MIN_CONNECTION: PositiveInt = Field(
|
||||||
|
description="Min connection of the Vastbase database",
|
||||||
|
default=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
VASTBASE_MAX_CONNECTION: PositiveInt = Field(
|
||||||
|
description="Max connection of the Vastbase database",
|
||||||
|
default=5,
|
||||||
|
)
|
||||||
@ -0,0 +1,9 @@
|
|||||||
|
from configs.observability.otel.otel_config import OTelConfig
|
||||||
|
|
||||||
|
|
||||||
|
class ObservabilityConfig(OTelConfig):
|
||||||
|
"""
|
||||||
|
Observability configuration settings
|
||||||
|
"""
|
||||||
|
|
||||||
|
pass
|
||||||
@ -0,0 +1,44 @@
|
|||||||
|
from pydantic import Field
|
||||||
|
from pydantic_settings import BaseSettings
|
||||||
|
|
||||||
|
|
||||||
|
class OTelConfig(BaseSettings):
|
||||||
|
"""
|
||||||
|
OpenTelemetry configuration settings
|
||||||
|
"""
|
||||||
|
|
||||||
|
ENABLE_OTEL: bool = Field(
|
||||||
|
description="Whether to enable OpenTelemetry",
|
||||||
|
default=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
OTLP_BASE_ENDPOINT: str = Field(
|
||||||
|
description="OTLP base endpoint",
|
||||||
|
default="http://localhost:4318",
|
||||||
|
)
|
||||||
|
|
||||||
|
OTLP_API_KEY: str = Field(
|
||||||
|
description="OTLP API key",
|
||||||
|
default="",
|
||||||
|
)
|
||||||
|
|
||||||
|
OTEL_EXPORTER_TYPE: str = Field(
|
||||||
|
description="OTEL exporter type",
|
||||||
|
default="otlp",
|
||||||
|
)
|
||||||
|
|
||||||
|
OTEL_SAMPLING_RATE: float = Field(default=0.1, description="Sampling rate for traces (0.0 to 1.0)")
|
||||||
|
|
||||||
|
OTEL_BATCH_EXPORT_SCHEDULE_DELAY: int = Field(
|
||||||
|
default=5000, description="Batch export schedule delay in milliseconds"
|
||||||
|
)
|
||||||
|
|
||||||
|
OTEL_MAX_QUEUE_SIZE: int = Field(default=2048, description="Maximum queue size for the batch span processor")
|
||||||
|
|
||||||
|
OTEL_MAX_EXPORT_BATCH_SIZE: int = Field(default=512, description="Maximum export batch size")
|
||||||
|
|
||||||
|
OTEL_METRIC_EXPORT_INTERVAL: int = Field(default=60000, description="Metric export interval in milliseconds")
|
||||||
|
|
||||||
|
OTEL_BATCH_EXPORT_TIMEOUT: int = Field(default=10000, description="Batch export timeout in milliseconds")
|
||||||
|
|
||||||
|
OTEL_METRIC_EXPORT_TIMEOUT: int = Field(default=30000, description="Metric export timeout in milliseconds")
|
||||||
@ -0,0 +1,52 @@
|
|||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from pydantic.fields import FieldInfo
|
||||||
|
|
||||||
|
from .http_request import NacosHttpClient
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
from configs.remote_settings_sources.base import RemoteSettingsSource
|
||||||
|
|
||||||
|
from .utils import _parse_config
|
||||||
|
|
||||||
|
|
||||||
|
class NacosSettingsSource(RemoteSettingsSource):
|
||||||
|
def __init__(self, configs: Mapping[str, Any]):
|
||||||
|
self.configs = configs
|
||||||
|
self.remote_configs: dict[str, Any] = {}
|
||||||
|
self.async_init()
|
||||||
|
|
||||||
|
def async_init(self):
|
||||||
|
data_id = os.getenv("DIFY_ENV_NACOS_DATA_ID", "dify-api-env.properties")
|
||||||
|
group = os.getenv("DIFY_ENV_NACOS_GROUP", "nacos-dify")
|
||||||
|
tenant = os.getenv("DIFY_ENV_NACOS_NAMESPACE", "")
|
||||||
|
|
||||||
|
params = {"dataId": data_id, "group": group, "tenant": tenant}
|
||||||
|
try:
|
||||||
|
content = NacosHttpClient().http_request("/nacos/v1/cs/configs", method="GET", headers={}, params=params)
|
||||||
|
self.remote_configs = self._parse_config(content)
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("[get-access-token] exception occurred")
|
||||||
|
raise
|
||||||
|
|
||||||
|
def _parse_config(self, content: str) -> dict:
|
||||||
|
if not content:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
return _parse_config(self, content)
|
||||||
|
except Exception as e:
|
||||||
|
raise RuntimeError(f"Failed to parse config: {e}")
|
||||||
|
|
||||||
|
def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
|
||||||
|
if not isinstance(self.remote_configs, dict):
|
||||||
|
raise ValueError(f"remote configs is not dict, but {type(self.remote_configs)}")
|
||||||
|
|
||||||
|
field_value = self.remote_configs.get(field_name)
|
||||||
|
if field_value is None:
|
||||||
|
return None, field_name, False
|
||||||
|
|
||||||
|
return field_value, field_name, False
|
||||||
@ -0,0 +1,83 @@
|
|||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class NacosHttpClient:
|
||||||
|
def __init__(self):
|
||||||
|
self.username = os.getenv("DIFY_ENV_NACOS_USERNAME")
|
||||||
|
self.password = os.getenv("DIFY_ENV_NACOS_PASSWORD")
|
||||||
|
self.ak = os.getenv("DIFY_ENV_NACOS_ACCESS_KEY")
|
||||||
|
self.sk = os.getenv("DIFY_ENV_NACOS_SECRET_KEY")
|
||||||
|
self.server = os.getenv("DIFY_ENV_NACOS_SERVER_ADDR", "localhost:8848")
|
||||||
|
self.token = None
|
||||||
|
self.token_ttl = 18000
|
||||||
|
self.token_expire_time: float = 0
|
||||||
|
|
||||||
|
def http_request(self, url, method="GET", headers=None, params=None):
|
||||||
|
try:
|
||||||
|
self._inject_auth_info(headers, params)
|
||||||
|
response = requests.request(method, url="http://" + self.server + url, headers=headers, params=params)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.text
|
||||||
|
except requests.exceptions.RequestException as e:
|
||||||
|
return f"Request to Nacos failed: {e}"
|
||||||
|
|
||||||
|
def _inject_auth_info(self, headers, params, module="config"):
|
||||||
|
headers.update({"User-Agent": "Nacos-Http-Client-In-Dify:v0.0.1"})
|
||||||
|
|
||||||
|
if module == "login":
|
||||||
|
return
|
||||||
|
|
||||||
|
ts = str(int(time.time() * 1000))
|
||||||
|
|
||||||
|
if self.ak and self.sk:
|
||||||
|
sign_str = self.get_sign_str(params["group"], params["tenant"], ts)
|
||||||
|
headers["Spas-AccessKey"] = self.ak
|
||||||
|
headers["Spas-Signature"] = self.__do_sign(sign_str, self.sk)
|
||||||
|
headers["timeStamp"] = ts
|
||||||
|
if self.username and self.password:
|
||||||
|
self.get_access_token(force_refresh=False)
|
||||||
|
params["accessToken"] = self.token
|
||||||
|
|
||||||
|
def __do_sign(self, sign_str, sk):
|
||||||
|
return (
|
||||||
|
base64.encodebytes(hmac.new(sk.encode(), sign_str.encode(), digestmod=hashlib.sha1).digest())
|
||||||
|
.decode()
|
||||||
|
.strip()
|
||||||
|
)
|
||||||
|
|
||||||
|
def get_sign_str(self, group, tenant, ts):
|
||||||
|
sign_str = ""
|
||||||
|
if tenant:
|
||||||
|
sign_str = tenant + "+"
|
||||||
|
if group:
|
||||||
|
sign_str = sign_str + group + "+"
|
||||||
|
if sign_str:
|
||||||
|
sign_str += ts
|
||||||
|
return sign_str
|
||||||
|
|
||||||
|
def get_access_token(self, force_refresh=False):
|
||||||
|
current_time = time.time()
|
||||||
|
if self.token and not force_refresh and self.token_expire_time > current_time:
|
||||||
|
return self.token
|
||||||
|
|
||||||
|
params = {"username": self.username, "password": self.password}
|
||||||
|
url = "http://" + self.server + "/nacos/v1/auth/login"
|
||||||
|
try:
|
||||||
|
resp = requests.request("POST", url, headers=None, params=params)
|
||||||
|
resp.raise_for_status()
|
||||||
|
response_data = resp.json()
|
||||||
|
self.token = response_data.get("accessToken")
|
||||||
|
self.token_ttl = response_data.get("tokenTtl", 18000)
|
||||||
|
self.token_expire_time = current_time + self.token_ttl - 10
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("[get-access-token] exception occur")
|
||||||
|
raise
|
||||||
@ -0,0 +1,31 @@
|
|||||||
|
def _parse_config(self, content: str) -> dict[str, str]:
|
||||||
|
config: dict[str, str] = {}
|
||||||
|
if not content:
|
||||||
|
return config
|
||||||
|
|
||||||
|
for line in content.splitlines():
|
||||||
|
cleaned_line = line.strip()
|
||||||
|
if not cleaned_line or cleaned_line.startswith(("#", "!")):
|
||||||
|
continue
|
||||||
|
|
||||||
|
separator_index = -1
|
||||||
|
for i, c in enumerate(cleaned_line):
|
||||||
|
if c in ("=", ":") and (i == 0 or cleaned_line[i - 1] != "\\"):
|
||||||
|
separator_index = i
|
||||||
|
break
|
||||||
|
|
||||||
|
if separator_index == -1:
|
||||||
|
continue
|
||||||
|
|
||||||
|
key = cleaned_line[:separator_index].strip()
|
||||||
|
raw_value = cleaned_line[separator_index + 1 :].strip()
|
||||||
|
|
||||||
|
try:
|
||||||
|
decoded_value = bytes(raw_value, "utf-8").decode("unicode_escape")
|
||||||
|
decoded_value = decoded_value.replace(r"\=", "=").replace(r"\:", ":")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
decoded_value = raw_value
|
||||||
|
|
||||||
|
config[key] = decoded_value
|
||||||
|
|
||||||
|
return config
|
||||||
@ -0,0 +1,45 @@
|
|||||||
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from configs import dify_config
|
||||||
|
from constants import DEFAULT_FILE_NUMBER_LIMITS
|
||||||
|
|
||||||
|
|
||||||
|
def get_parameters_from_feature_dict(
|
||||||
|
*, features_dict: Mapping[str, Any], user_input_form: list[dict[str, Any]]
|
||||||
|
) -> Mapping[str, Any]:
|
||||||
|
"""
|
||||||
|
Mapping from feature dict to webapp parameters
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"opening_statement": features_dict.get("opening_statement"),
|
||||||
|
"suggested_questions": features_dict.get("suggested_questions", []),
|
||||||
|
"suggested_questions_after_answer": features_dict.get("suggested_questions_after_answer", {"enabled": False}),
|
||||||
|
"speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
|
||||||
|
"text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
|
||||||
|
"retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
|
||||||
|
"annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
|
||||||
|
"more_like_this": features_dict.get("more_like_this", {"enabled": False}),
|
||||||
|
"user_input_form": user_input_form,
|
||||||
|
"sensitive_word_avoidance": features_dict.get(
|
||||||
|
"sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
|
||||||
|
),
|
||||||
|
"file_upload": features_dict.get(
|
||||||
|
"file_upload",
|
||||||
|
{
|
||||||
|
"image": {
|
||||||
|
"enabled": False,
|
||||||
|
"number_limits": DEFAULT_FILE_NUMBER_LIMITS,
|
||||||
|
"detail": "high",
|
||||||
|
"transfer_methods": ["remote_url", "local_file"],
|
||||||
|
}
|
||||||
|
},
|
||||||
|
),
|
||||||
|
"system_parameters": {
|
||||||
|
"image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
|
||||||
|
"video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
|
||||||
|
"audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
|
||||||
|
"file_size_limit": dify_config.UPLOAD_FILE_SIZE_LIMIT,
|
||||||
|
"workflow_file_upload_limit": dify_config.WORKFLOW_FILE_UPLOAD_LIMIT,
|
||||||
|
},
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue