Merge branch 'feat/rag-2' of https://github.com/langgenius/dify into feat/rag-2

api/controllers/console/datasets/rag_pipeline/datasource_auth.py

@@ -177,7 +177,17 @@ class DatasourceAuthUpdateDeleteApi(Resource):
             raise ValueError(str(ex))
 
         return {"result": "success"}, 201
-
+    
+class DatasourceAuthListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        datasource_provider_service = DatasourceProviderService()
+        datasources = datasource_provider_service.get_all_datasource_credentials(
+            tenant_id=current_user.current_tenant_id
+        )
+        return {"result": datasources}, 200
 
 # Import Rag Pipeline
 api.add_resource(
@@ -197,3 +207,8 @@ api.add_resource(
     DatasourceAuthUpdateDeleteApi,
     "/auth/plugin/datasource/<string:auth_id>",
 )
+
+api.add_resource(
+    DatasourceAuthListApi,
+    "/auth/plugin/datasource/list",
+)

api/core/entities/provider_configuration.py

@@ -720,7 +720,7 @@ class ProviderConfiguration(BaseModel):
         """
         secret_input_form_variables = []
         for credential_form_schema in credential_form_schemas:
-            if credential_form_schema.type == FormType.SECRET_INPUT:
+            if credential_form_schema.type.value == FormType.SECRET_INPUT.value:
                 secret_input_form_variables.append(credential_form_schema.variable)
 
         return secret_input_form_variables

api/core/plugin/impl/datasource.py

@@ -56,6 +56,41 @@ class PluginDatasourceManager(BasePluginClient):
                 tool.identity.provider = provider.declaration.identity.name
 
         return all_response
+    
+    def fetch_installed_datasource_providers(self, tenant_id: str) -> list[PluginDatasourceProviderEntity]:
+        """
+        Fetch datasource providers for the given tenant.
+        """
+
+        def transformer(json_response: dict[str, Any]) -> dict:
+            if json_response.get("data"):
+                for provider in json_response.get("data", []):
+                    declaration = provider.get("declaration", {}) or {}
+                    provider_name = declaration.get("identity", {}).get("name")
+                    for datasource in declaration.get("datasources", []):
+                        datasource["identity"]["provider"] = provider_name
+
+            return json_response
+
+        response = self._request_with_plugin_daemon_response(
+            "GET",
+            f"plugin/{tenant_id}/management/datasources",
+            list[PluginDatasourceProviderEntity],
+            params={"page": 1, "page_size": 256},
+            transformer=transformer,
+        )
+
+        for provider in response:
+            ToolTransformService.repack_provider(tenant_id=tenant_id, provider=provider)
+
+        for provider in response:
+            provider.declaration.identity.name = f"{provider.plugin_id}/{provider.declaration.identity.name}"
+
+            # override the provider name for each tool to plugin_id/provider_name
+            for tool in provider.declaration.datasources:
+                tool.identity.provider = provider.declaration.identity.name
+
+        return response
 
     def fetch_datasource_provider(self, tenant_id: str, provider_id: str) -> PluginDatasourceProviderEntity:
         """

api/core/provider_manager.py

@@ -858,7 +858,7 @@ class ProviderManager:
         """
         secret_input_form_variables = []
         for credential_form_schema in credential_form_schemas:
-            if credential_form_schema.type == FormType.SECRET_INPUT:
+            if credential_form_schema.type.value == FormType.SECRET_INPUT.value:
                 secret_input_form_variables.append(credential_form_schema.variable)
 
         return secret_input_form_variables

api/core/rag/models/document.py

@@ -69,6 +69,7 @@ class QAChunk(BaseModel):
     answer: str
 
 
+
 class QAStructureChunk(BaseModel):
     """
     QAStructureChunk.

api/services/datasource_provider_service.py

@@ -84,7 +84,7 @@ class DatasourceProviderService:
         credential_form_schemas = datasource_provider.declaration.credentials_schema
         secret_input_form_variables = []
         for credential_form_schema in credential_form_schemas:
-            if credential_form_schema.type == FormType.SECRET_INPUT:
+            if credential_form_schema.type.value == FormType.SECRET_INPUT.value:
                 secret_input_form_variables.append(credential_form_schema.name)
 
         return secret_input_form_variables
@@ -131,6 +131,33 @@ class DatasourceProviderService:
             )
 
         return copy_credentials_list
+    
+
+    def get_all_datasource_credentials(self, tenant_id: str) -> list[dict]:
+        """
+        get datasource credentials.
+
+        :return:
+        """
+        # get all plugin providers
+        manager = PluginDatasourceManager()
+        datasources = manager.fetch_installed_datasource_providers(tenant_id)
+        datasource_credentials = []
+        for datasource in datasources:
+            credentials = self.get_datasource_credentials(tenant_id=tenant_id,
+                                                          provider=datasource.provider,
+                                                          plugin_id=datasource.plugin_id)
+            datasource_credentials.append({
+                "provider": datasource.provider,
+                "plugin_id": datasource.plugin_id,
+                "plugin_unique_identifier": datasource.plugin_unique_identifier,
+                "icon": datasource.declaration.identity.icon,
+                "name": datasource.declaration.identity.name,
+                "description": datasource.declaration.identity.description.model_dump(),
+                "author": datasource.declaration.identity.author,
+                "credentials": credentials,
+            })
+        return datasource_credentials
 
     def get_real_datasource_credentials(self, tenant_id: str, provider: str, plugin_id: str) -> list[dict]:
         """