## 安全问题
diff --git a/api/.env.example b/api/.env.example
index e7e704e135..01ddb4adfd 100644
--- a/api/.env.example
+++ b/api/.env.example
@@ -165,6 +165,7 @@ MILVUS_URI=http://127.0.0.1:19530
MILVUS_TOKEN=
MILVUS_USER=root
MILVUS_PASSWORD=Milvus
+MILVUS_ANALYZER_PARAMS=
# MyScale configuration
MYSCALE_HOST=127.0.0.1
@@ -189,6 +190,7 @@ TENCENT_VECTOR_DB_USERNAME=dify
TENCENT_VECTOR_DB_DATABASE=dify
TENCENT_VECTOR_DB_SHARD=1
TENCENT_VECTOR_DB_REPLICAS=2
+TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH=false
# ElasticSearch configuration
ELASTICSEARCH_HOST=127.0.0.1
@@ -325,6 +327,7 @@ UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
MULTIMODAL_SEND_FORMAT=base64
PROMPT_GENERATION_MAX_TOKENS=512
CODE_GENERATION_MAX_TOKENS=1024
+PLUGIN_BASED_TOKEN_COUNTING_ENABLED=false
# Mail configuration, support: resend, smtp
MAIL_TYPE=
@@ -421,6 +424,12 @@ WORKFLOW_CALL_MAX_DEPTH=5
WORKFLOW_PARALLEL_DEPTH_LIMIT=3
MAX_VARIABLE_SIZE=204800
+# Workflow storage configuration
+# Options: rdbms, hybrid
+# rdbms: Use only the relational database (default)
+# hybrid: Save new data to object storage, read from both object storage and RDBMS
+WORKFLOW_NODE_EXECUTION_STORAGE=rdbms
+
# App configuration
APP_MAX_EXECUTION_TIME=1200
APP_MAX_ACTIVE_REQUESTS=0
@@ -461,3 +470,16 @@ CREATE_TIDB_SERVICE_JOB_ENABLED=false
MAX_SUBMIT_COUNT=100
# Lockout duration in seconds
LOGIN_LOCKOUT_DURATION=86400
+
+# Enable OpenTelemetry
+ENABLE_OTEL=false
+OTLP_BASE_ENDPOINT=http://localhost:4318
+OTLP_API_KEY=
+OTEL_EXPORTER_TYPE=otlp
+OTEL_SAMPLING_RATE=0.1
+OTEL_BATCH_EXPORT_SCHEDULE_DELAY=5000
+OTEL_MAX_QUEUE_SIZE=2048
+OTEL_MAX_EXPORT_BATCH_SIZE=512
+OTEL_METRIC_EXPORT_INTERVAL=60000
+OTEL_BATCH_EXPORT_TIMEOUT=10000
+OTEL_METRIC_EXPORT_TIMEOUT=30000
\ No newline at end of file
diff --git a/api/Dockerfile b/api/Dockerfile
index fbfbd47741..cff696ff56 100644
--- a/api/Dockerfile
+++ b/api/Dockerfile
@@ -3,20 +3,11 @@ FROM python:3.12-slim-bookworm AS base
WORKDIR /app/api
-# Install Poetry
-ENV POETRY_VERSION=2.0.1
+# Install uv
+ENV UV_VERSION=0.6.14
-# if you located in China, you can use aliyun mirror to speed up
-# RUN pip install --no-cache-dir poetry==${POETRY_VERSION} -i https://mirrors.aliyun.com/pypi/simple/
-
-RUN pip install --no-cache-dir poetry==${POETRY_VERSION}
+RUN pip install --no-cache-dir uv==${UV_VERSION}
-# Configure Poetry
-ENV POETRY_CACHE_DIR=/tmp/poetry_cache
-ENV POETRY_NO_INTERACTION=1
-ENV POETRY_VIRTUALENVS_IN_PROJECT=true
-ENV POETRY_VIRTUALENVS_CREATE=true
-ENV POETRY_REQUESTS_TIMEOUT=15
FROM base AS packages
@@ -27,8 +18,8 @@ RUN apt-get update \
&& apt-get install -y --no-install-recommends gcc g++ libc-dev libffi-dev libgmp-dev libmpfr-dev libmpc-dev
# Install Python dependencies
-COPY pyproject.toml poetry.lock ./
-RUN poetry install --sync --no-cache --no-root
+COPY pyproject.toml uv.lock ./
+RUN uv sync --locked
# production stage
FROM base AS production
diff --git a/api/README.md b/api/README.md
index c3abc25be1..c542f11b16 100644
--- a/api/README.md
+++ b/api/README.md
@@ -3,7 +3,10 @@
## Usage
> [!IMPORTANT]
-> In the v0.6.12 release, we deprecated `pip` as the package management tool for Dify API Backend service and replaced it with `poetry`.
+>
+> In the v1.3.0 release, `poetry` has been replaced with
+> [`uv`](https://docs.astral.sh/uv/) as the package manager
+> for Dify API backend service.
1. Start the docker-compose stack
@@ -37,19 +40,19 @@
4. Create environment.
- Dify API service uses [Poetry](https://python-poetry.org/docs/) to manage dependencies. First, you need to add the poetry shell plugin, if you don't have it already, in order to run in a virtual environment. [Note: Poetry shell is no longer a native command so you need to install the poetry plugin beforehand]
+ Dify API service uses [UV](https://docs.astral.sh/uv/) to manage dependencies.
+ First, you need to add the uv package manager, if you don't have it already.
```bash
- poetry self add poetry-plugin-shell
+ pip install uv
+ # Or on macOS
+ brew install uv
```
-
- Then, You can execute `poetry shell` to activate the environment.
5. Install dependencies
```bash
- poetry env use 3.12
- poetry install
+ uv sync --dev
```
6. Run migrate
@@ -57,21 +60,21 @@
Before the first launch, migrate the database to the latest version.
```bash
- poetry run python -m flask db upgrade
+ uv run flask db upgrade
```
7. Start backend
```bash
- poetry run python -m flask run --host 0.0.0.0 --port=5001 --debug
+ uv run flask run --host 0.0.0.0 --port=5001 --debug
```
8. Start Dify [web](../web) service.
-9. Setup your application by visiting `http://localhost:3000`...
+9. Setup your application by visiting `http://localhost:3000`.
10. If you need to handle and debug the async tasks (e.g. dataset importing and documents indexing), please start the worker service.
```bash
- poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion
+ uv run celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion
```
## Testing
@@ -79,11 +82,11 @@
1. Install dependencies for both the backend and the test environment
```bash
- poetry install -C api --with dev
+ uv sync --dev
```
2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`
```bash
- poetry run -P api bash dev/pytest/pytest_all_tests.sh
+ uv run -P api bash dev/pytest/pytest_all_tests.sh
```
diff --git a/api/app_factory.py b/api/app_factory.py
index 52ae05583a..586f2ded9e 100644
--- a/api/app_factory.py
+++ b/api/app_factory.py
@@ -51,8 +51,10 @@ def initialize_extensions(app: DifyApp):
ext_login,
ext_mail,
ext_migrate,
+ ext_otel,
ext_proxy_fix,
ext_redis,
+ ext_repositories,
ext_sentry,
ext_set_secretkey,
ext_storage,
@@ -73,6 +75,7 @@ def initialize_extensions(app: DifyApp):
ext_migrate,
ext_redis,
ext_storage,
+ ext_repositories,
ext_celery,
ext_login,
ext_mail,
@@ -81,6 +84,7 @@ def initialize_extensions(app: DifyApp):
ext_proxy_fix,
ext_blueprints,
ext_commands,
+ ext_otel,
]
for ext in extensions:
short_name = ext.__name__.split(".")[-1]
diff --git a/api/configs/app_config.py b/api/configs/app_config.py
index ac1ce9db10..cb0adb751c 100644
--- a/api/configs/app_config.py
+++ b/api/configs/app_config.py
@@ -9,6 +9,7 @@ from .enterprise import EnterpriseFeatureConfig
from .extra import ExtraServiceConfig
from .feature import FeatureConfig
from .middleware import MiddlewareConfig
+from .observability import ObservabilityConfig
from .packaging import PackagingInfo
from .remote_settings_sources import RemoteSettingsSource, RemoteSettingsSourceConfig, RemoteSettingsSourceName
from .remote_settings_sources.apollo import ApolloSettingsSource
@@ -59,6 +60,8 @@ class DifyConfig(
MiddlewareConfig,
# Extra service configs
ExtraServiceConfig,
+ # Observability configs
+ ObservabilityConfig,
# Remote source configs
RemoteSettingsSourceConfig,
# Enterprise feature configs
diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py
index 46ded0244f..f498dccbbc 100644
--- a/api/configs/feature/__init__.py
+++ b/api/configs/feature/__init__.py
@@ -12,7 +12,7 @@ from pydantic import (
)
from pydantic_settings import BaseSettings
-from configs.feature.hosted_service import HostedServiceConfig
+from .hosted_service import HostedServiceConfig
class SecurityConfig(BaseSettings):
@@ -442,7 +442,7 @@ class LoggingConfig(BaseSettings):
class ModelLoadBalanceConfig(BaseSettings):
"""
- Configuration for model load balancing
+ Configuration for model load balancing and token counting
"""
MODEL_LB_ENABLED: bool = Field(
@@ -450,6 +450,11 @@ class ModelLoadBalanceConfig(BaseSettings):
default=False,
)
+ PLUGIN_BASED_TOKEN_COUNTING_ENABLED: bool = Field(
+ description="Enable or disable plugin based token counting. If disabled, token counting will return 0.",
+ default=False,
+ )
+
class BillingConfig(BaseSettings):
"""
@@ -514,6 +519,11 @@ class WorkflowNodeExecutionConfig(BaseSettings):
default=100,
)
+ WORKFLOW_NODE_EXECUTION_STORAGE: str = Field(
+ default="rdbms",
+ description="Storage backend for WorkflowNodeExecution. Options: 'rdbms', 'hybrid'",
+ )
+
class AuthConfig(BaseSettings):
"""
@@ -848,6 +858,11 @@ class AccountConfig(BaseSettings):
default=5,
)
+ EDUCATION_ENABLED: bool = Field(
+ description="whether to enable education identity",
+ default=False,
+ )
+
class FeatureConfig(
# place the configs in alphabet order
diff --git a/api/configs/middleware/vdb/milvus_config.py b/api/configs/middleware/vdb/milvus_config.py
index ebdf8857b9..d398ef5bd8 100644
--- a/api/configs/middleware/vdb/milvus_config.py
+++ b/api/configs/middleware/vdb/milvus_config.py
@@ -39,3 +39,8 @@ class MilvusConfig(BaseSettings):
"older versions",
default=True,
)
+
+ MILVUS_ANALYZER_PARAMS: Optional[str] = Field(
+ description='Milvus text analyzer parameters, e.g., {"type": "chinese"} for Chinese segmentation support.',
+ default=None,
+ )
diff --git a/api/configs/middleware/vdb/tencent_vector_config.py b/api/configs/middleware/vdb/tencent_vector_config.py
index 9cf4d07f6f..a51823c3f3 100644
--- a/api/configs/middleware/vdb/tencent_vector_config.py
+++ b/api/configs/middleware/vdb/tencent_vector_config.py
@@ -48,3 +48,8 @@ class TencentVectorDBConfig(BaseSettings):
description="Name of the specific Tencent Vector Database to connect to",
default=None,
)
+
+ TENCENT_VECTOR_DB_ENABLE_HYBRID_SEARCH: bool = Field(
+ description="Enable hybrid search features",
+ default=False,
+ )
diff --git a/api/configs/observability/__init__.py b/api/configs/observability/__init__.py
new file mode 100644
index 0000000000..8c6f21e28b
--- /dev/null
+++ b/api/configs/observability/__init__.py
@@ -0,0 +1,9 @@
+from configs.observability.otel.otel_config import OTelConfig
+
+
+class ObservabilityConfig(OTelConfig):
+ """
+ Observability configuration settings
+ """
+
+ pass
diff --git a/api/configs/observability/otel/otel_config.py b/api/configs/observability/otel/otel_config.py
new file mode 100644
index 0000000000..568a800d10
--- /dev/null
+++ b/api/configs/observability/otel/otel_config.py
@@ -0,0 +1,44 @@
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class OTelConfig(BaseSettings):
+ """
+ OpenTelemetry configuration settings
+ """
+
+ ENABLE_OTEL: bool = Field(
+ description="Whether to enable OpenTelemetry",
+ default=False,
+ )
+
+ OTLP_BASE_ENDPOINT: str = Field(
+ description="OTLP base endpoint",
+ default="http://localhost:4318",
+ )
+
+ OTLP_API_KEY: str = Field(
+ description="OTLP API key",
+ default="",
+ )
+
+ OTEL_EXPORTER_TYPE: str = Field(
+ description="OTEL exporter type",
+ default="otlp",
+ )
+
+ OTEL_SAMPLING_RATE: float = Field(default=0.1, description="Sampling rate for traces (0.0 to 1.0)")
+
+ OTEL_BATCH_EXPORT_SCHEDULE_DELAY: int = Field(
+ default=5000, description="Batch export schedule delay in milliseconds"
+ )
+
+ OTEL_MAX_QUEUE_SIZE: int = Field(default=2048, description="Maximum queue size for the batch span processor")
+
+ OTEL_MAX_EXPORT_BATCH_SIZE: int = Field(default=512, description="Maximum export batch size")
+
+ OTEL_METRIC_EXPORT_INTERVAL: int = Field(default=60000, description="Metric export interval in milliseconds")
+
+ OTEL_BATCH_EXPORT_TIMEOUT: int = Field(default=10000, description="Batch export timeout in milliseconds")
+
+ OTEL_METRIC_EXPORT_TIMEOUT: int = Field(default=30000, description="Metric export timeout in milliseconds")
diff --git a/api/configs/packaging/__init__.py b/api/configs/packaging/__init__.py
index 0ef5a724b3..c7aedc5b8a 100644
--- a/api/configs/packaging/__init__.py
+++ b/api/configs/packaging/__init__.py
@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
CURRENT_VERSION: str = Field(
description="Dify version",
- default="1.1.3",
+ default="1.2.0",
)
COMMIT_SHA: str = Field(
diff --git a/api/configs/remote_settings_sources/apollo/client.py b/api/configs/remote_settings_sources/apollo/client.py
index 03c64ea00f..88b30d3987 100644
--- a/api/configs/remote_settings_sources/apollo/client.py
+++ b/api/configs/remote_settings_sources/apollo/client.py
@@ -270,7 +270,7 @@ class ApolloClient:
while not self._stopping:
for namespace in self._notification_map:
self._do_heart_beat(namespace)
- time.sleep(60 * 10) # 10分钟
+ time.sleep(60 * 10) # 10 minutes
def _do_heart_beat(self, namespace):
url = "{}/configs/{}/{}/{}?ip={}".format(self.config_url, self.app_id, self.cluster, namespace, self.ip)
diff --git a/api/constants/__init__.py b/api/constants/__init__.py
index b5dfd9cb18..9162357466 100644
--- a/api/constants/__init__.py
+++ b/api/constants/__init__.py
@@ -3,6 +3,8 @@ from configs import dify_config
HIDDEN_VALUE = "[__HIDDEN__]"
UUID_NIL = "00000000-0000-0000-0000-000000000000"
+DEFAULT_FILE_NUMBER_LIMITS = 3
+
IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "webp", "gif", "svg"]
IMAGE_EXTENSIONS.extend([ext.upper() for ext in IMAGE_EXTENSIONS])
diff --git a/api/controllers/common/helpers.py b/api/controllers/common/helpers.py
index 2979375169..008f1f0f7a 100644
--- a/api/controllers/common/helpers.py
+++ b/api/controllers/common/helpers.py
@@ -4,8 +4,6 @@ import platform
import re
import urllib.parse
import warnings
-from collections.abc import Mapping
-from typing import Any
from uuid import uuid4
import httpx
@@ -29,8 +27,6 @@ except ImportError:
from pydantic import BaseModel
-from configs import dify_config
-
class FileInfo(BaseModel):
filename: str
@@ -87,38 +83,3 @@ def guess_file_info_from_response(response: httpx.Response):
mimetype=mimetype,
size=int(response.headers.get("Content-Length", -1)),
)
-
-
-def get_parameters_from_feature_dict(*, features_dict: Mapping[str, Any], user_input_form: list[dict[str, Any]]):
- return {
- "opening_statement": features_dict.get("opening_statement"),
- "suggested_questions": features_dict.get("suggested_questions", []),
- "suggested_questions_after_answer": features_dict.get("suggested_questions_after_answer", {"enabled": False}),
- "speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
- "text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
- "retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
- "annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
- "more_like_this": features_dict.get("more_like_this", {"enabled": False}),
- "user_input_form": user_input_form,
- "sensitive_word_avoidance": features_dict.get(
- "sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
- ),
- "file_upload": features_dict.get(
- "file_upload",
- {
- "image": {
- "enabled": False,
- "number_limits": 3,
- "detail": "high",
- "transfer_methods": ["remote_url", "local_file"],
- }
- },
- ),
- "system_parameters": {
- "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
- "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
- "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
- "file_size_limit": dify_config.UPLOAD_FILE_SIZE_LIMIT,
- "workflow_file_upload_limit": dify_config.WORKFLOW_FILE_UPLOAD_LIMIT,
- },
- }
diff --git a/api/controllers/console/app/annotation.py b/api/controllers/console/app/annotation.py
index 24f1020c18..fcd8ed1882 100644
--- a/api/controllers/console/app/annotation.py
+++ b/api/controllers/console/app/annotation.py
@@ -89,7 +89,7 @@ class AnnotationReplyActionStatusApi(Resource):
app_annotation_job_key = "{}_app_annotation_job_{}".format(action, str(job_id))
cache_result = redis_client.get(app_annotation_job_key)
if cache_result is None:
- raise ValueError("The job is not exist.")
+ raise ValueError("The job does not exist.")
job_status = cache_result.decode()
error_msg = ""
@@ -226,7 +226,7 @@ class AnnotationBatchImportStatusApi(Resource):
indexing_cache_key = "app_annotation_batch_import_{}".format(str(job_id))
cache_result = redis_client.get(indexing_cache_key)
if cache_result is None:
- raise ValueError("The job is not exist.")
+ raise ValueError("The job does not exist.")
job_status = cache_result.decode()
error_msg = ""
if job_status == "error":
diff --git a/api/controllers/console/app/app_import.py b/api/controllers/console/app/app_import.py
index 47acb47a2c..a159d4c5c4 100644
--- a/api/controllers/console/app/app_import.py
+++ b/api/controllers/console/app/app_import.py
@@ -8,6 +8,7 @@ from werkzeug.exceptions import Forbidden
from controllers.console.app.wraps import get_app_model
from controllers.console.wraps import (
account_initialization_required,
+ cloud_edition_billing_resource_check,
setup_required,
)
from extensions.ext_database import db
@@ -23,6 +24,7 @@ class AppImportApi(Resource):
@login_required
@account_initialization_required
@marshal_with(app_import_fields)
+ @cloud_edition_billing_resource_check("apps")
def post(self):
# Check user role first
if not current_user.is_editor:
diff --git a/api/controllers/console/app/workflow_app_log.py b/api/controllers/console/app/workflow_app_log.py
index 54640b1a19..d863747995 100644
--- a/api/controllers/console/app/workflow_app_log.py
+++ b/api/controllers/console/app/workflow_app_log.py
@@ -1,5 +1,4 @@
-from datetime import datetime
-
+from dateutil.parser import isoparse
from flask_restful import Resource, marshal_with, reqparse # type: ignore
from flask_restful.inputs import int_range # type: ignore
from sqlalchemy.orm import Session
@@ -41,10 +40,10 @@ class WorkflowAppLogApi(Resource):
args.status = WorkflowRunStatus(args.status) if args.status else None
if args.created_at__before:
- args.created_at__before = datetime.fromisoformat(args.created_at__before.replace("Z", "+00:00"))
+ args.created_at__before = isoparse(args.created_at__before)
if args.created_at__after:
- args.created_at__after = datetime.fromisoformat(args.created_at__after.replace("Z", "+00:00"))
+ args.created_at__after = isoparse(args.created_at__after)
# get paginate workflow app logs
workflow_app_service = WorkflowAppService()
diff --git a/api/controllers/console/auth/data_source_oauth.py b/api/controllers/console/auth/data_source_oauth.py
index e911c9a5e5..b4bd80fe2f 100644
--- a/api/controllers/console/auth/data_source_oauth.py
+++ b/api/controllers/console/auth/data_source_oauth.py
@@ -74,7 +74,9 @@ class OAuthDataSourceBinding(Resource):
if not oauth_provider:
return {"error": "Invalid provider"}, 400
if "code" in request.args:
- code = request.args.get("code")
+ code = request.args.get("code", "")
+ if not code:
+ return {"error": "Invalid code"}, 400
try:
oauth_provider.get_access_token(code)
except requests.exceptions.HTTPError as e:
diff --git a/api/controllers/console/auth/forgot_password.py b/api/controllers/console/auth/forgot_password.py
index 773ee65727..dc0009f36e 100644
--- a/api/controllers/console/auth/forgot_password.py
+++ b/api/controllers/console/auth/forgot_password.py
@@ -99,53 +99,64 @@ class ForgotPasswordResetApi(Resource):
parser.add_argument("password_confirm", type=valid_password, required=True, nullable=False, location="json")
args = parser.parse_args()
- new_password = args["new_password"]
- password_confirm = args["password_confirm"]
-
- if str(new_password).strip() != str(password_confirm).strip():
+ # Validate passwords match
+ if args["new_password"] != args["password_confirm"]:
raise PasswordMismatchError()
- token = args["token"]
- reset_data = AccountService.get_reset_password_data(token)
-
- if reset_data is None:
+ # Validate token and get reset data
+ reset_data = AccountService.get_reset_password_data(args["token"])
+ if not reset_data:
raise InvalidTokenError()
- AccountService.revoke_reset_password_token(token)
+ # Revoke token to prevent reuse
+ AccountService.revoke_reset_password_token(args["token"])
+ # Generate secure salt and hash password
salt = secrets.token_bytes(16)
- base64_salt = base64.b64encode(salt).decode()
+ password_hashed = hash_password(args["new_password"], salt)
- password_hashed = hash_password(new_password, salt)
- base64_password_hashed = base64.b64encode(password_hashed).decode()
+ email = reset_data.get("email", "")
with Session(db.engine) as session:
- account = session.execute(select(Account).filter_by(email=reset_data.get("email"))).scalar_one_or_none()
- if account:
- account.password = base64_password_hashed
- account.password_salt = base64_salt
- db.session.commit()
- tenant = TenantService.get_join_tenants(account)
- if not tenant and not FeatureService.get_system_features().is_allow_create_workspace:
- tenant = TenantService.create_tenant(f"{account.name}'s Workspace")
- TenantService.create_tenant_member(tenant, account, role="owner")
- account.current_tenant = tenant
- tenant_was_created.send(tenant)
- else:
- try:
- account = AccountService.create_account_and_tenant(
- email=reset_data.get("email", ""),
- name=reset_data.get("email", ""),
- password=password_confirm,
- interface_language=languages[0],
- )
- except WorkSpaceNotAllowedCreateError:
- pass
- except AccountRegisterError:
- raise AccountInFreezeError()
+ account = session.execute(select(Account).filter_by(email=email)).scalar_one_or_none()
+
+ if account:
+ self._update_existing_account(account, password_hashed, salt, session)
+ else:
+ self._create_new_account(email, args["password_confirm"])
return {"result": "success"}
+ def _update_existing_account(self, account, password_hashed, salt, session):
+ # Update existing account credentials
+ account.password = base64.b64encode(password_hashed).decode()
+ account.password_salt = base64.b64encode(salt).decode()
+ session.commit()
+
+ # Create workspace if needed
+ if (
+ not TenantService.get_join_tenants(account)
+ and FeatureService.get_system_features().is_allow_create_workspace
+ ):
+ tenant = TenantService.create_tenant(f"{account.name}'s Workspace")
+ TenantService.create_tenant_member(tenant, account, role="owner")
+ account.current_tenant = tenant
+ tenant_was_created.send(tenant)
+
+ def _create_new_account(self, email, password):
+ # Create new account if allowed
+ try:
+ AccountService.create_account_and_tenant(
+ email=email,
+ name=email,
+ password=password,
+ interface_language=languages[0],
+ )
+ except WorkSpaceNotAllowedCreateError:
+ pass
+ except AccountRegisterError:
+ raise AccountInFreezeError()
+
api.add_resource(ForgotPasswordSendEmailApi, "/forgot-password")
api.add_resource(ForgotPasswordCheckApi, "/forgot-password/validity")
diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py
index 396dae7a57..4644ac6299 100644
--- a/api/controllers/console/datasets/datasets.py
+++ b/api/controllers/console/datasets/datasets.py
@@ -641,7 +641,6 @@ class DatasetRetrievalSettingApi(Resource):
VectorType.RELYT
| VectorType.TIDB_VECTOR
| VectorType.CHROMA
- | VectorType.TENCENT
| VectorType.PGVECTO_RS
| VectorType.BAIDU
| VectorType.VIKINGDB
@@ -665,6 +664,7 @@ class DatasetRetrievalSettingApi(Resource):
| VectorType.OPENGAUSS
| VectorType.OCEANBASE
| VectorType.TABLESTORE
+ | VectorType.TENCENT
):
return {
"retrieval_method": [
@@ -688,7 +688,6 @@ class DatasetRetrievalSettingMockApi(Resource):
| VectorType.RELYT
| VectorType.TIDB_VECTOR
| VectorType.CHROMA
- | VectorType.TENCENT
| VectorType.PGVECTO_RS
| VectorType.BAIDU
| VectorType.VIKINGDB
@@ -710,6 +709,7 @@ class DatasetRetrievalSettingMockApi(Resource):
| VectorType.OPENGAUSS
| VectorType.OCEANBASE
| VectorType.TABLESTORE
+ | VectorType.TENCENT
):
return {
"retrieval_method": [
diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py
index 1b38d0776a..696aaa94db 100644
--- a/api/controllers/console/datasets/datasets_segments.py
+++ b/api/controllers/console/datasets/datasets_segments.py
@@ -398,7 +398,7 @@ class DatasetDocumentSegmentBatchImportApi(Resource):
indexing_cache_key = "segment_batch_import_{}".format(job_id)
cache_result = redis_client.get(indexing_cache_key)
if cache_result is None:
- raise ValueError("The job is not exist.")
+ raise ValueError("The job does not exist.")
return {"job_id": job_id, "job_status": cache_result.decode()}, 200
diff --git a/api/controllers/console/datasets/external.py b/api/controllers/console/datasets/external.py
index 48f360dcd1..2c031172bf 100644
--- a/api/controllers/console/datasets/external.py
+++ b/api/controllers/console/datasets/external.py
@@ -21,12 +21,6 @@ def _validate_name(name):
return name
-def _validate_description_length(description):
- if description and len(description) > 400:
- raise ValueError("Description cannot exceed 400 characters.")
- return description
-
-
class ExternalApiTemplateListApi(Resource):
@setup_required
@login_required
diff --git a/api/controllers/console/datasets/metadata.py b/api/controllers/console/datasets/metadata.py
index 14183a1e67..fc9711169f 100644
--- a/api/controllers/console/datasets/metadata.py
+++ b/api/controllers/console/datasets/metadata.py
@@ -14,18 +14,6 @@ from services.entities.knowledge_entities.knowledge_entities import (
from services.metadata_service import MetadataService
-def _validate_name(name):
- if not name or len(name) < 1 or len(name) > 40:
- raise ValueError("Name must be between 1 to 40 characters.")
- return name
-
-
-def _validate_description_length(description):
- if len(description) > 400:
- raise ValueError("Description cannot exceed 400 characters.")
- return description
-
-
class DatasetMetadataCreateApi(Resource):
@setup_required
@login_required
diff --git a/api/controllers/console/datasets/website.py b/api/controllers/console/datasets/website.py
index da995537e7..33c926b4c9 100644
--- a/api/controllers/console/datasets/website.py
+++ b/api/controllers/console/datasets/website.py
@@ -14,7 +14,12 @@ class WebsiteCrawlApi(Resource):
def post(self):
parser = reqparse.RequestParser()
parser.add_argument(
- "provider", type=str, choices=["firecrawl", "jinareader"], required=True, nullable=True, location="json"
+ "provider",
+ type=str,
+ choices=["firecrawl", "watercrawl", "jinareader"],
+ required=True,
+ nullable=True,
+ location="json",
)
parser.add_argument("url", type=str, required=True, nullable=True, location="json")
parser.add_argument("options", type=dict, required=True, nullable=True, location="json")
@@ -34,7 +39,9 @@ class WebsiteCrawlStatusApi(Resource):
@account_initialization_required
def get(self, job_id: str):
parser = reqparse.RequestParser()
- parser.add_argument("provider", type=str, choices=["firecrawl", "jinareader"], required=True, location="args")
+ parser.add_argument(
+ "provider", type=str, choices=["firecrawl", "watercrawl", "jinareader"], required=True, location="args"
+ )
args = parser.parse_args()
# get crawl status
try:
diff --git a/api/controllers/console/error.py b/api/controllers/console/error.py
index bd4ae9dc7f..b8fd1f0358 100644
--- a/api/controllers/console/error.py
+++ b/api/controllers/console/error.py
@@ -103,6 +103,18 @@ class AccountInFreezeError(BaseHTTPException):
)
+class EducationVerifyLimitError(BaseHTTPException):
+ error_code = "education_verify_limit"
+ description = "Rate limit exceeded"
+ code = 429
+
+
+class EducationActivateLimitError(BaseHTTPException):
+ error_code = "education_activate_limit"
+ description = "Rate limit exceeded"
+ code = 429
+
+
class CompilanceRateLimitError(BaseHTTPException):
error_code = "compilance_rate_limit"
description = "Rate limit exceeded for downloading compliance report."
diff --git a/api/controllers/console/explore/parameter.py b/api/controllers/console/explore/parameter.py
index 5bc74d16e7..bf9f0d6b28 100644
--- a/api/controllers/console/explore/parameter.py
+++ b/api/controllers/console/explore/parameter.py
@@ -1,10 +1,10 @@
from flask_restful import marshal_with # type: ignore
from controllers.common import fields
-from controllers.common import helpers as controller_helpers
from controllers.console import api
from controllers.console.app.error import AppUnavailableError
from controllers.console.explore.wraps import InstalledAppResource
+from core.app.app_config.common.parameters_mapping import get_parameters_from_feature_dict
from models.model import AppMode, InstalledApp
from services.app_service import AppService
@@ -36,9 +36,7 @@ class AppParameterApi(InstalledAppResource):
user_input_form = features_dict.get("user_input_form", [])
- return controller_helpers.get_parameters_from_feature_dict(
- features_dict=features_dict, user_input_form=user_input_form
- )
+ return get_parameters_from_feature_dict(features_dict=features_dict, user_input_form=user_input_form)
class ExploreAppMetaApi(InstalledAppResource):
diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py
index f1ec0f3d29..e9c25e6c5b 100644
--- a/api/controllers/console/workspace/account.py
+++ b/api/controllers/console/workspace/account.py
@@ -15,7 +15,13 @@ from controllers.console.workspace.error import (
InvalidInvitationCodeError,
RepeatPasswordNotMatchError,
)
-from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required
+from controllers.console.wraps import (
+ account_initialization_required,
+ cloud_edition_billing_enabled,
+ enterprise_license_required,
+ only_edition_cloud,
+ setup_required,
+)
from extensions.ext_database import db
from fields.member_fields import account_fields
from libs.helper import TimestampField, timezone
@@ -280,8 +286,6 @@ class AccountDeleteApi(Resource):
class AccountDeleteUpdateFeedbackApi(Resource):
@setup_required
def post(self):
- account = current_user
-
parser = reqparse.RequestParser()
parser.add_argument("email", type=str, required=True, location="json")
parser.add_argument("feedback", type=str, required=True, location="json")
@@ -292,6 +296,79 @@ class AccountDeleteUpdateFeedbackApi(Resource):
return {"result": "success"}
+class EducationVerifyApi(Resource):
+ verify_fields = {
+ "token": fields.String,
+ }
+
+ @setup_required
+ @login_required
+ @account_initialization_required
+ @only_edition_cloud
+ @cloud_edition_billing_enabled
+ @marshal_with(verify_fields)
+ def get(self):
+ account = current_user
+
+ return BillingService.EducationIdentity.verify(account.id, account.email)
+
+
+class EducationApi(Resource):
+ status_fields = {
+ "result": fields.Boolean,
+ }
+
+ @setup_required
+ @login_required
+ @account_initialization_required
+ @only_edition_cloud
+ @cloud_edition_billing_enabled
+ def post(self):
+ account = current_user
+
+ parser = reqparse.RequestParser()
+ parser.add_argument("token", type=str, required=True, location="json")
+ parser.add_argument("institution", type=str, required=True, location="json")
+ parser.add_argument("role", type=str, required=True, location="json")
+ args = parser.parse_args()
+
+ return BillingService.EducationIdentity.activate(account, args["token"], args["institution"], args["role"])
+
+ @setup_required
+ @login_required
+ @account_initialization_required
+ @only_edition_cloud
+ @cloud_edition_billing_enabled
+ @marshal_with(status_fields)
+ def get(self):
+ account = current_user
+
+ return BillingService.EducationIdentity.is_active(account.id)
+
+
+class EducationAutoCompleteApi(Resource):
+ data_fields = {
+ "data": fields.List(fields.String),
+ "curr_page": fields.Integer,
+ "has_next": fields.Boolean,
+ }
+
+ @setup_required
+ @login_required
+ @account_initialization_required
+ @only_edition_cloud
+ @cloud_edition_billing_enabled
+ @marshal_with(data_fields)
+ def get(self):
+ parser = reqparse.RequestParser()
+ parser.add_argument("keywords", type=str, required=True, location="args")
+ parser.add_argument("page", type=int, required=False, location="args", default=0)
+ parser.add_argument("limit", type=int, required=False, location="args", default=20)
+ args = parser.parse_args()
+
+ return BillingService.EducationIdentity.autocomplete(args["keywords"], args["page"], args["limit"])
+
+
# Register API resources
api.add_resource(AccountInitApi, "/account/init")
api.add_resource(AccountProfileApi, "/account/profile")
@@ -305,5 +382,8 @@ api.add_resource(AccountIntegrateApi, "/account/integrates")
api.add_resource(AccountDeleteVerifyApi, "/account/delete/verify")
api.add_resource(AccountDeleteApi, "/account/delete")
api.add_resource(AccountDeleteUpdateFeedbackApi, "/account/delete/feedback")
+api.add_resource(EducationVerifyApi, "/account/education/verify")
+api.add_resource(EducationApi, "/account/education")
+api.add_resource(EducationAutoCompleteApi, "/account/education/autocomplete")
# api.add_resource(AccountEmailApi, '/account/email')
# api.add_resource(AccountEmailVerifyApi, '/account/email-verify')
diff --git a/api/controllers/console/workspace/plugin.py b/api/controllers/console/workspace/plugin.py
index f4c32ede2b..e9c1884c60 100644
--- a/api/controllers/console/workspace/plugin.py
+++ b/api/controllers/console/workspace/plugin.py
@@ -49,6 +49,23 @@ class PluginListApi(Resource):
return jsonable_encoder({"plugins": plugins})
+class PluginListLatestVersionsApi(Resource):
+ @setup_required
+ @login_required
+ @account_initialization_required
+ def post(self):
+ req = reqparse.RequestParser()
+ req.add_argument("plugin_ids", type=list, required=True, location="json")
+ args = req.parse_args()
+
+ try:
+ versions = PluginService.list_latest_versions(args["plugin_ids"])
+ except PluginDaemonClientSideError as e:
+ raise ValueError(e)
+
+ return jsonable_encoder({"versions": versions})
+
+
class PluginListInstallationsFromIdsApi(Resource):
@setup_required
@login_required
@@ -232,11 +249,36 @@ class PluginInstallFromMarketplaceApi(Resource):
return jsonable_encoder(response)
+class PluginFetchMarketplacePkgApi(Resource):
+ @setup_required
+ @login_required
+ @account_initialization_required
+ @plugin_permission_required(install_required=True)
+ def get(self):
+ tenant_id = current_user.current_tenant_id
+
+ parser = reqparse.RequestParser()
+ parser.add_argument("plugin_unique_identifier", type=str, required=True, location="args")
+ args = parser.parse_args()
+
+ try:
+ return jsonable_encoder(
+ {
+ "manifest": PluginService.fetch_marketplace_pkg(
+ tenant_id,
+ args["plugin_unique_identifier"],
+ )
+ }
+ )
+ except PluginDaemonClientSideError as e:
+ raise ValueError(e)
+
+
class PluginFetchManifestApi(Resource):
@setup_required
@login_required
@account_initialization_required
- @plugin_permission_required(debug_required=True)
+ @plugin_permission_required(install_required=True)
def get(self):
tenant_id = current_user.current_tenant_id
@@ -260,7 +302,7 @@ class PluginFetchInstallTasksApi(Resource):
@setup_required
@login_required
@account_initialization_required
- @plugin_permission_required(debug_required=True)
+ @plugin_permission_required(install_required=True)
def get(self):
tenant_id = current_user.current_tenant_id
@@ -281,7 +323,7 @@ class PluginFetchInstallTaskApi(Resource):
@setup_required
@login_required
@account_initialization_required
- @plugin_permission_required(debug_required=True)
+ @plugin_permission_required(install_required=True)
def get(self, task_id: str):
tenant_id = current_user.current_tenant_id
@@ -295,7 +337,7 @@ class PluginDeleteInstallTaskApi(Resource):
@setup_required
@login_required
@account_initialization_required
- @plugin_permission_required(debug_required=True)
+ @plugin_permission_required(install_required=True)
def post(self, task_id: str):
tenant_id = current_user.current_tenant_id
@@ -309,7 +351,7 @@ class PluginDeleteAllInstallTaskItemsApi(Resource):
@setup_required
@login_required
@account_initialization_required
- @plugin_permission_required(debug_required=True)
+ @plugin_permission_required(install_required=True)
def post(self):
tenant_id = current_user.current_tenant_id
@@ -323,7 +365,7 @@ class PluginDeleteInstallTaskItemApi(Resource):
@setup_required
@login_required
@account_initialization_required
- @plugin_permission_required(debug_required=True)
+ @plugin_permission_required(install_required=True)
def post(self, task_id: str, identifier: str):
tenant_id = current_user.current_tenant_id
@@ -337,7 +379,7 @@ class PluginUpgradeFromMarketplaceApi(Resource):
@setup_required
@login_required
@account_initialization_required
- @plugin_permission_required(debug_required=True)
+ @plugin_permission_required(install_required=True)
def post(self):
tenant_id = current_user.current_tenant_id
@@ -360,7 +402,7 @@ class PluginUpgradeFromGithubApi(Resource):
@setup_required
@login_required
@account_initialization_required
- @plugin_permission_required(debug_required=True)
+ @plugin_permission_required(install_required=True)
def post(self):
tenant_id = current_user.current_tenant_id
@@ -391,7 +433,7 @@ class PluginUninstallApi(Resource):
@setup_required
@login_required
@account_initialization_required
- @plugin_permission_required(debug_required=True)
+ @plugin_permission_required(install_required=True)
def post(self):
req = reqparse.RequestParser()
req.add_argument("plugin_installation_id", type=str, required=True, location="json")
@@ -453,6 +495,7 @@ class PluginFetchPermissionApi(Resource):
api.add_resource(PluginDebuggingKeyApi, "/workspaces/current/plugin/debugging-key")
api.add_resource(PluginListApi, "/workspaces/current/plugin/list")
+api.add_resource(PluginListLatestVersionsApi, "/workspaces/current/plugin/list/latest-versions")
api.add_resource(PluginListInstallationsFromIdsApi, "/workspaces/current/plugin/list/installations/ids")
api.add_resource(PluginIconApi, "/workspaces/current/plugin/icon")
api.add_resource(PluginUploadFromPkgApi, "/workspaces/current/plugin/upload/pkg")
@@ -470,6 +513,7 @@ api.add_resource(PluginDeleteInstallTaskApi, "/workspaces/current/plugin/tasks/<
api.add_resource(PluginDeleteAllInstallTaskItemsApi, "/workspaces/current/plugin/tasks/delete_all")
api.add_resource(PluginDeleteInstallTaskItemApi, "/workspaces/current/plugin/tasks/high_quality High quality
- economy Economy
search_method (string) Search method
+ - hybrid_search Hybrid search
+ - semantic_search Semantic search
+ - full_text_search Full-text search
+ - reranking_enable (bool) Whether to enable reranking
+ - reranking_model (object) Rerank model configuration
+ - reranking_provider_name (string) Rerank model provider
+ - reranking_model_name (string) Rerank model name
+ - top_k (int) Number of results to return
+ - score_threshold_enabled (bool) Whether to enable score threshold
+ - score_threshold (float) Score threshold
+ high_quality High quality
+ - economy Economy
+ only_me Only me
+ - all_team_members All team members
+ - partial_members Partial members
+ search_method (text) Search method: One of the following four keywords is required
+ - keyword_search Keyword search
+ - semantic_search Semantic search
+ - full_text_search Full-text search
+ - hybrid_search Hybrid search
+ - reranking_enable (bool) Whether to enable reranking, required if the search mode is semantic_search or hybrid_search (optional)
+ - reranking_mode (object) Rerank model configuration, required if reranking is enabled
+ - reranking_provider_name (string) Rerank model provider
+ - reranking_model_name (string) Rerank model name
+ - weights (float) Semantic search weight setting in hybrid search mode
+ - top_k (integer) Number of results to return (optional)
+ - score_threshold_enabled (bool) Whether to enable score threshold
+ - score_threshold (float) Score threshold
+ search_method (文字列) 検索方法
+ - hybrid_search ハイブリッド検索
+ - semantic_search セマンティック検索
+ - full_text_search 全文検索
+ - reranking_enable (ブール値) リランキングを有効にするかどうか
+ - reranking_model (オブジェクト) リランクモデルの設定
+ - reranking_provider_name (文字列) リランクモデルのプロバイダ
+ - reranking_model_name (文字列) リランクモデル名
+ - top_k (整数) 返される結果の数
+ - score_threshold_enabled (ブール値) スコア閾値を有効にするかどうか
+ - score_threshold (浮動小数点数) スコア閾値
+ high_quality 高质量：使用 embedding 模型进行嵌入，构建为向量数据库索引
+ - high_quality 高质量：使用
+ ding 模型进行嵌入，构建为向量数据库索引
- economy 经济：使用 keyword table index 的倒排索引进行构建
semantic_search 语义检索
- full_text_search 全文检索
- reranking_enable (bool) 是否开启rerank
+ - reranking_mode (String) 混合检索
+ - weighted_score 权重设置
+ - reranking_model Rerank 模型
- reranking_model (object) Rerank 模型配置
- reranking_provider_name (string) Rerank 模型的提供商
- reranking_model_name (string) Rerank 模型的名称
@@ -329,6 +338,26 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
search_method (string) 检索方法
+ - hybrid_search 混合检索
+ - semantic_search 语义检索
+ - full_text_search 全文检索
+ - reranking_enable (bool) 是否开启rerank
+ - reranking_model (object) Rerank 模型配置
+ - reranking_provider_name (string) Rerank 模型的提供商
+ - reranking_model_name (string) Rerank 模型的名称
+ - top_k (int) 召回条数
+ - score_threshold_enabled (bool)是否开启召回分数限制
+ - score_threshold (float) 召回分数限制
+ high_quality 高质量
+ - economy 经济
+ only_me 仅自己
+ - all_team_members 所有团队成员
+ - partial_members 部分团队成员
+ search_method (text) 检索方法：以下四个关键字之一，必填
+ - keyword_search 关键字检索
+ - semantic_search 语义检索
+ - full_text_search 全文检索
+ - hybrid_search 混合检索
+ - reranking_enable (bool) 是否启用 Reranking，非必填，如果检索模式为 semantic_search 模式或者 hybrid_search 则传值
+ - reranking_mode (object) Rerank 模型配置，非必填，如果启用了 reranking 则传值
+ - reranking_provider_name (string) Rerank 模型提供商
+ - reranking_model_name (string) Rerank 模型名称
+ - weights (float) 混合检索模式下语意检索的权重设置
+ - top_k (integer) 返回结果数量，非必填
+ - score_threshold_enabled (bool) 是否开启 score 阈值
+ - score_threshold (float) Score 阈值
+ search_method (text) 检索方法：以下三个关键字之一，必填
+ - search_method (text) 检索方法：以下四个关键字之一，必填
- keyword_search 关键字检索
- semantic_search 语义检索
- full_text_search 全文检索
@@ -1713,7 +1997,7 @@ import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstructi
/>
{userProfile.name}
+
+ {userProfile.name}
+ {isEducationAccount && (
+
{userProfile.email}