From 83ca7f8deba25188bc6ef9d6365044e2aff8b1dd Mon Sep 17 00:00:00 2001 From: Yeuoly Date: Mon, 26 May 2025 17:32:25 +0800 Subject: [PATCH] feat: add datasource support to PluginDeclaration and PluginCategory --- api/controllers/console/auth/oauth.py | 7 ++++--- .../console/datasets/rag_pipeline/datasource_oauth.py | 11 +---------- api/core/app/apps/pipeline/pipeline_generator.py | 2 +- .../website_crawl/website_crawl_provider.py | 6 +++++- api/core/plugin/entities/plugin.py | 6 ++++++ api/core/plugin/impl/datasource.py | 2 +- api/core/workflow/nodes/datasource/datasource_node.py | 2 +- api/models/oauth.py | 9 --------- 8 files changed, 19 insertions(+), 26 deletions(-) diff --git a/api/controllers/console/auth/oauth.py b/api/controllers/console/auth/oauth.py index d8576c3879..d5e13525d6 100644 --- a/api/controllers/console/auth/oauth.py +++ b/api/controllers/console/auth/oauth.py @@ -2,17 +2,18 @@ import logging from datetime import UTC, datetime from typing import Optional -from flask_login import current_user import requests from flask import current_app, redirect, request +from flask_login import current_user from flask_restful import Resource from sqlalchemy import select from sqlalchemy.orm import Session -from werkzeug.exceptions import Unauthorized, Forbidden, NotFound +from werkzeug.exceptions import Forbidden, NotFound, Unauthorized from configs import dify_config from constants.languages import languages from controllers.console.wraps import account_initialization_required, setup_required +from core.plugin.impl.oauth import OAuthHandler from events.tenant_event import tenant_was_created from extensions.ext_database import db from libs.helper import extract_remote_ip @@ -25,7 +26,7 @@ from services.account_service import AccountService, RegisterService, TenantServ from services.errors.account import AccountNotFoundError, AccountRegisterError from services.errors.workspace import WorkSpaceNotAllowedCreateError, WorkSpaceNotFoundError from services.feature_service import FeatureService -from core.plugin.impl.oauth import OAuthHandler + from .. import api diff --git a/api/controllers/console/datasets/rag_pipeline/datasource_oauth.py b/api/controllers/console/datasets/rag_pipeline/datasource_oauth.py index b105606c04..f4164dea7b 100644 --- a/api/controllers/console/datasets/rag_pipeline/datasource_oauth.py +++ b/api/controllers/console/datasets/rag_pipeline/datasource_oauth.py @@ -1,24 +1,15 @@ -from typing import cast from flask_login import current_user # type: ignore -from flask_restful import Resource, marshal_with, reqparse # type: ignore -from sqlalchemy.orm import Session +from flask_restful import Resource # type: ignore from werkzeug.exceptions import Forbidden from controllers.console import api -from controllers.console.datasets.wraps import get_rag_pipeline from controllers.console.wraps import ( account_initialization_required, setup_required, ) from core.plugin.impl.datasource import PluginDatasourceManager -from extensions.ext_database import db -from fields.rag_pipeline_fields import pipeline_import_check_dependencies_fields, pipeline_import_fields from libs.login import login_required -from models import Account -from models.dataset import Pipeline -from services.app_dsl_service import ImportStatus -from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineDslService class DatasourcePluginOauthApi(Resource): diff --git a/api/core/app/apps/pipeline/pipeline_generator.py b/api/core/app/apps/pipeline/pipeline_generator.py index 120927516b..9c25f8f4e6 100644 --- a/api/core/app/apps/pipeline/pipeline_generator.py +++ b/api/core/app/apps/pipeline/pipeline_generator.py @@ -30,7 +30,7 @@ from core.rag.index_processor.constant.built_in_field import BuiltInField from core.repositories import SQLAlchemyWorkflowNodeExecutionRepository from core.workflow.repository.workflow_node_execution_repository import WorkflowNodeExecutionRepository from extensions.ext_database import db -from models import Account, App, EndUser, Workflow, WorkflowNodeExecutionTriggeredFrom +from models import Account, EndUser, Workflow, WorkflowNodeExecutionTriggeredFrom from models.dataset import Document, Pipeline from models.model import AppMode from services.dataset_service import DocumentService diff --git a/api/core/datasource/website_crawl/website_crawl_provider.py b/api/core/datasource/website_crawl/website_crawl_provider.py index d9043702d2..11168b4c26 100644 --- a/api/core/datasource/website_crawl/website_crawl_provider.py +++ b/api/core/datasource/website_crawl/website_crawl_provider.py @@ -10,7 +10,11 @@ class WebsiteCrawlDatasourcePluginProviderController(DatasourcePluginProviderCon plugin_unique_identifier: str def __init__( - self, entity: DatasourceProviderEntityWithPlugin | None, plugin_id: str, plugin_unique_identifier: str, tenant_id: str + self, + entity: DatasourceProviderEntityWithPlugin | None, + plugin_id: str, + plugin_unique_identifier: str, + tenant_id: str, ) -> None: super().__init__(entity, tenant_id) self.plugin_id = plugin_id diff --git a/api/core/plugin/entities/plugin.py b/api/core/plugin/entities/plugin.py index 260d4f12db..e2ea9669fa 100644 --- a/api/core/plugin/entities/plugin.py +++ b/api/core/plugin/entities/plugin.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, Field, model_validator from werkzeug.exceptions import NotFound from core.agent.plugin_entities import AgentStrategyProviderEntity +from core.datasource.entities.datasource_entities import DatasourceProviderEntity from core.model_runtime.entities.provider_entities import ProviderEntity from core.plugin.entities.base import BasePluginEntity from core.plugin.entities.endpoint import EndpointProviderDeclaration @@ -62,6 +63,7 @@ class PluginCategory(enum.StrEnum): Model = "model" Extension = "extension" AgentStrategy = "agent-strategy" + Datasource = "datasource" class PluginDeclaration(BaseModel): @@ -69,6 +71,7 @@ class PluginDeclaration(BaseModel): tools: Optional[list[str]] = Field(default_factory=list[str]) models: Optional[list[str]] = Field(default_factory=list[str]) endpoints: Optional[list[str]] = Field(default_factory=list[str]) + datasources: Optional[list[str]] = Field(default_factory=list[str]) class Meta(BaseModel): minimum_dify_version: Optional[str] = Field(default=None, pattern=r"^\d{1,4}(\.\d{1,4}){1,3}(-\w{1,16})?$") @@ -90,6 +93,7 @@ class PluginDeclaration(BaseModel): model: Optional[ProviderEntity] = None endpoint: Optional[EndpointProviderDeclaration] = None agent_strategy: Optional[AgentStrategyProviderEntity] = None + datasource: Optional[DatasourceProviderEntity] = None meta: Meta @model_validator(mode="before") @@ -100,6 +104,8 @@ class PluginDeclaration(BaseModel): values["category"] = PluginCategory.Tool elif values.get("model"): values["category"] = PluginCategory.Model + elif values.get("datasource"): + values["category"] = PluginCategory.Datasource elif values.get("agent_strategy"): values["category"] = PluginCategory.AgentStrategy else: diff --git a/api/core/plugin/impl/datasource.py b/api/core/plugin/impl/datasource.py index 2ad5bdcfef..004cf7f9c3 100644 --- a/api/core/plugin/impl/datasource.py +++ b/api/core/plugin/impl/datasource.py @@ -236,7 +236,7 @@ class PluginDatasourceManager(BasePluginClient): response = self._request_with_plugin_daemon_response_stream( "GET", - f"plugin/datasource/oauth", + "plugin/datasource/oauth", PluginBasicBooleanResponse, params={"page": 1, "page_size": 256}, headers={ diff --git a/api/core/workflow/nodes/datasource/datasource_node.py b/api/core/workflow/nodes/datasource/datasource_node.py index 55d7ee5ccb..2f15fba3af 100644 --- a/api/core/workflow/nodes/datasource/datasource_node.py +++ b/api/core/workflow/nodes/datasource/datasource_node.py @@ -1,4 +1,4 @@ -from collections.abc import Generator, Mapping, Sequence +from collections.abc import Mapping, Sequence from typing import Any, cast from core.datasource.entities.datasource_entities import ( diff --git a/api/models/oauth.py b/api/models/oauth.py index f24c3c6723..aee45d7c41 100644 --- a/api/models/oauth.py +++ b/api/models/oauth.py @@ -1,20 +1,11 @@ from datetime import datetime -from json import JSONDecodeError -from typing import Any, cast -from sqlalchemy import func from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped -from configs import dify_config -from extensions.ext_storage import storage -from services.entities.knowledge_entities.knowledge_entities import ParentMode, Rule - -from .account import Account from .base import Base from .engine import db -from .model import App, Tag, TagBinding, UploadFile from .types import StringUUID