Merge branch 'chore/optimize-model-selection-agent-node' into dev/plugin-deploy

api/controllers/console/datasets/datasets.py

@@ -14,6 +14,7 @@ from controllers.console.wraps import account_initialization_required, enterpris
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.indexing_runner import IndexingRunner
 from core.model_runtime.entities.model_entities import ModelType
+from core.plugin.entities.plugin import ModelProviderID
 from core.provider_manager import ProviderManager
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.extractor.entity.extract_setting import ExtractSetting
@@ -72,7 +73,9 @@ class DatasetListApi(Resource):
 
         data = marshal(datasets, dataset_detail_fields)
         for item in data:
+            # convert embedding_model_provider to plugin standard format
             if item["indexing_technique"] == "high_quality":
+                item["embedding_model_provider"] = str(ModelProviderID(item["embedding_model_provider"]))
                 item_model = f"{item['embedding_model']}:{item['embedding_model_provider']}"
                 if item_model in model_names:
                     item["embedding_available"] = True

api/core/plugin/entities/plugin.py

@@ -173,7 +173,7 @@ class ModelProviderID(GenericProviderID):
     def __init__(self, value: str, is_hardcoded: bool = False) -> None:
         super().__init__(value, is_hardcoded)
         if self.organization == "langgenius" and self.provider_name == "google":
-            self.provider_name = "gemini"
+            self.plugin_name = "gemini"
 
 
 class ToolProviderID(GenericProviderID):
@@ -181,7 +181,7 @@ class ToolProviderID(GenericProviderID):
         super().__init__(value, is_hardcoded)
         if self.organization == "langgenius":
             if self.provider_name in ["jina", "siliconflow"]:
-                self.provider_name = f"{self.provider_name}_tool"
+                self.plugin_name = f"{self.provider_name}_tool"
 
 
 class PluginDependency(BaseModel):

api/core/provider_manager.py

@@ -30,6 +30,7 @@ from core.model_runtime.entities.provider_entities import (
     ProviderEntity,
 )
 from core.model_runtime.model_providers.model_provider_factory import ModelProviderFactory
+from core.plugin.entities.plugin import ModelProviderID
 from extensions import ext_hosting_provider
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
@@ -191,7 +192,7 @@ class ProviderManager:
                 model_settings=model_settings,
             )
 
-            provider_configurations[provider_name] = provider_configuration
+            provider_configurations[str(ModelProviderID(provider_name))] = provider_configuration
 
         # Return the encapsulated object
         return provider_configurations

docker/nginx/conf.d/default.conf.template

@@ -31,6 +31,7 @@ server {
 
     location /e {
       proxy_pass http://plugin_daemon:5002;
+      proxy_set_header Dify-Hook-Url $scheme://$host$request_uri;
       include proxy.conf;
     }
 

web/app/components/header/account-setting/model-provider-page/model-parameter-modal/agent-model-trigger.tsx

@@ -1,6 +1,7 @@
 import type { FC } from 'react'
-import { useEffect, useMemo, useState } from 'react'
+import { useMemo, useState } from 'react'
 import { useTranslation } from 'react-i18next'
+import { useQuery } from '@tanstack/react-query'
 import type {
   ModelItem,
   ModelProvider,
@@ -9,7 +10,6 @@ import {
   CustomConfigurationStatusEnum,
   ModelTypeEnum,
 } from '../declarations'
-import type { PluginInfoFromMarketPlace } from '@/app/components/plugins/types'
 import { useInvalidateInstalledPluginList } from '@/service/use-plugins'
 import ConfigurationButton from './configuration-button'
 import Loading from '@/app/components/base/loading'
@@ -66,43 +66,44 @@ const AgentModelTrigger: FC<AgentModelTriggerProps> = ({
       needsConfiguration,
     }
   }, [modelProviders, providerName])
-  const [pluginInfo, setPluginInfo] = useState<PluginInfoFromMarketPlace | null>(null)
-  const [isPluginChecked, setIsPluginChecked] = useState(false)
   const [installed, setInstalled] = useState(false)
-  const [inModelList, setInModelList] = useState(false)
   const invalidateInstalledPluginList = useInvalidateInstalledPluginList()
   const handleOpenModal = useModelModalHandler()
 
-  useEffect(() => {
+  const { data: inModelList = false } = useQuery({
-    (async () => {
+    queryKey: ['modelInList', currentProvider?.provider, modelId],
-      if (modelId && currentProvider) {
+    queryFn: async () => {
-        try {
+      if (!modelId || !currentProvider) return false
-          const modelsData = await fetchModelProviderModelList(`/workspaces/current/model-providers/${currentProvider?.provider}/models`)
+      try {
-          if (modelId && modelsData.data.find(item => item.model === modelId))
+        const modelsData = await fetchModelProviderModelList(`/workspaces/current/model-providers/${currentProvider?.provider}/models`)
-            setInModelList(true)
+        return !!modelId && !!modelsData.data.find(item => item.model === modelId)
-        }
-        catch (error) {
-          // pass
-        }
       }
-      if (providerName) {
+      catch (error) {
-        const parts = providerName.split('/')
+        return false
-        const org = parts[0]
-        const name = parts[1]
-        try {
-          const pluginInfo = await fetchPluginInfoFromMarketPlace({ org, name })
-          if (pluginInfo.data.plugin.category === PluginType.model)
-            setPluginInfo(pluginInfo.data.plugin)
-        }
-        catch (error) {
-          // pass
-        }
       }
-      setIsPluginChecked(true)
+    },
-    })()
+    enabled: !!modelId && !!currentProvider,
-  }, [providerName, modelId, currentProvider])
+  })
 
-  if (modelId && !isPluginChecked)
+  const { data: pluginInfo, isLoading: isPluginLoading } = useQuery({
+    queryKey: ['pluginInfo', providerName],
+    queryFn: async () => {
+      if (!providerName) return null
+      const parts = providerName.split('/')
+      const org = parts[0]
+      const name = parts[1]
+      try {
+        const response = await fetchPluginInfoFromMarketPlace({ org, name })
+        return response.data.plugin.category === PluginType.model ? response.data.plugin : null
+      }
+      catch (error) {
+        return null
+      }
+    },
+    enabled: !!providerName,
+  })
+
+  if (modelId && isPluginLoading)
     return <Loading />
 
   return (

web/app/components/plugins/install-plugin/hooks/use-refresh-plugin-list.tsx

@@ -1,4 +1,5 @@
-import { useUpdateModelProviders } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { useModelList } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
 import { useProviderContext } from '@/context/provider-context'
 import { useInvalidateInstalledPluginList } from '@/service/use-plugins'
 import { useInvalidateAllBuiltInTools, useInvalidateAllToolProviders } from '@/service/use-tools'
@@ -8,7 +9,9 @@ import { PluginType } from '../../types'
 
 const useRefreshPluginList = () => {
   const invalidateInstalledPluginList = useInvalidateInstalledPluginList()
-  const updateModelProviders = useUpdateModelProviders()
+  const { mutate: refetchLLMModelList } = useModelList(ModelTypeEnum.textGeneration)
+  const { mutate: refetchEmbeddingModelList } = useModelList(ModelTypeEnum.textEmbedding)
+  const { mutate: refetchRerankModelList } = useModelList(ModelTypeEnum.rerank)
   const { refreshModelProviders } = useProviderContext()
 
   const invalidateAllToolProviders = useInvalidateAllToolProviders()
@@ -31,8 +34,10 @@ const useRefreshPluginList = () => {
 
       // model select
       if (PluginType.model.includes(manifest.category) || refreshAllType) {
-        updateModelProviders()
         refreshModelProviders()
+        refetchLLMModelList()
+        refetchEmbeddingModelList()
+        refetchRerankModelList()
       }
 
       // agent select

web/app/components/plugins/install-plugin/install-from-github/index.tsx

@@ -136,9 +136,10 @@ const InstallFromGitHub: React.FC<InstallFromGitHubProps> = ({ updatePayload, on
     setState(prevState => ({ ...prevState, step: InstallStepFromGitHub.uploadFailed }))
   }, [])
 
-  const handleInstalled = useCallback(() => {
+  const handleInstalled = useCallback((notRefresh?: boolean) => {
     setState(prevState => ({ ...prevState, step: InstallStepFromGitHub.installed }))
-    refreshPluginList(manifest)
+    if (!notRefresh)
+      refreshPluginList(manifest)
     setIsInstalling(false)
     onSuccess()
   }, [manifest, onSuccess, refreshPluginList, setIsInstalling])

web/app/components/plugins/install-plugin/install-from-github/steps/loaded.tsx

@@ -24,7 +24,7 @@ type LoadedProps = {
   selectedPackage: string
   onBack: () => void
   onStartToInstall?: () => void
-  onInstalled: () => void
+  onInstalled: (notRefresh?: boolean) => void
   onFailed: (message?: string) => void
 }
 
@@ -55,7 +55,7 @@ const Loaded: React.FC<LoadedProps> = ({
 
   const [isInstalling, setIsInstalling] = React.useState(false)
   const { mutateAsync: installPackageFromGitHub } = useInstallPackageFromGitHub()
-  const { handleRefetch } = usePluginTaskList()
+  const { handleRefetch } = usePluginTaskList(payload.category)
   const { check } = checkTaskStatus()
 
   useEffect(() => {
@@ -127,7 +127,7 @@ const Loaded: React.FC<LoadedProps> = ({
         onFailed(error)
         return
       }
-      onInstalled()
+      onInstalled(true)
     }
     catch (e) {
       if (typeof e === 'string') {

web/app/components/plugins/install-plugin/install-from-local-package/ready-to-install.tsx

@@ -32,9 +32,10 @@ const ReadyToInstall: FC<Props> = ({
 }) => {
   const { refreshPluginList } = useRefreshPluginList()
 
-  const handleInstalled = useCallback(() => {
+  const handleInstalled = useCallback((notRefresh?: boolean) => {
     onStepChange(InstallStep.installed)
-    refreshPluginList(manifest)
+    if (!notRefresh)
+      refreshPluginList(manifest)
     setIsInstalling(false)
   }, [manifest, onStepChange, refreshPluginList, setIsInstalling])
 

web/app/components/plugins/install-plugin/install-from-local-package/steps/install.tsx

@@ -20,7 +20,7 @@ type Props = {
   payload: PluginDeclaration
   onCancel: () => void
   onStartToInstall?: () => void
-  onInstalled: () => void
+  onInstalled: (notRefresh?: boolean) => void
   onFailed: (message?: string) => void
 }
 
@@ -62,7 +62,7 @@ const Installed: FC<Props> = ({
     onCancel()
   }
 
-  const { handleRefetch } = usePluginTaskList()
+  const { handleRefetch } = usePluginTaskList(payload.category)
   const handleInstall = async () => {
     if (isInstalling) return
     setIsInstalling(true)
@@ -92,7 +92,7 @@ const Installed: FC<Props> = ({
         onFailed(error)
         return
       }
-      onInstalled()
+      onInstalled(true)
     }
     catch (e) {
       if (typeof e === 'string') {

web/app/components/plugins/install-plugin/install-from-marketplace/index.tsx

@@ -54,9 +54,10 @@ const InstallFromMarketplace: React.FC<InstallFromMarketplaceProps> = ({
     return t(`${i18nPrefix}.installPlugin`)
   }, [isBundle, step, t])
 
-  const handleInstalled = useCallback(() => {
+  const handleInstalled = useCallback((notRefresh?: boolean) => {
     setStep(InstallStep.installed)
-    refreshPluginList(manifest)
+    if (!notRefresh)
+      refreshPluginList(manifest)
     setIsInstalling(false)
   }, [manifest, refreshPluginList, setIsInstalling])
 

web/app/components/plugins/install-plugin/install-from-marketplace/steps/install.tsx

@@ -21,7 +21,7 @@ type Props = {
   payload: PluginManifestInMarket | Plugin
   onCancel: () => void
   onStartToInstall?: () => void
-  onInstalled: () => void
+  onInstalled: (notRefresh?: boolean) => void
   onFailed: (message?: string) => void
 }
 
@@ -51,7 +51,7 @@ const Installed: FC<Props> = ({
     check,
     stop,
   } = checkTaskStatus()
-  const { handleRefetch } = usePluginTaskList()
+  const { handleRefetch } = usePluginTaskList(payload.category)
 
   useEffect(() => {
     if (hasInstalled && uniqueIdentifier === installedInfoPayload.uniqueIdentifier)
@@ -106,7 +106,7 @@ const Installed: FC<Props> = ({
         onFailed(error)
         return
       }
-      onInstalled()
+      onInstalled(true)
     }
     catch (e) {
       if (typeof e === 'string') {

web/app/components/plugins/plugin-detail-panel/detail-header.tsx

@@ -113,6 +113,7 @@ const DetailHeader = ({
         },
         payload: {
           type: PluginSource.github,
+          category: detail.declaration.category,
           github: {
             originalPackageInfo: {
               id: detail.plugin_unique_identifier,
@@ -287,6 +288,7 @@ const DetailHeader = ({
         isShowUpdateModal && (
           <UpdateFromMarketplace
             payload={{
+              category: detail.declaration.category,
               originalPackageInfo: {
                 id: detail.plugin_unique_identifier,
                 payload: detail.declaration,

web/app/components/plugins/plugin-item/action.tsx

@@ -14,6 +14,7 @@ import { useGitHubReleases } from '../install-plugin/hooks'
 import Toast from '@/app/components/base/toast'
 import { useModalContext } from '@/context/modal-context'
 import { useInvalidateInstalledPluginList } from '@/service/use-plugins'
+import type { PluginType } from '@/app/components/plugins/types'
 
 const i18nPrefix = 'plugin.action'
 
@@ -22,6 +23,7 @@ type Props = {
   installationId: string
   pluginUniqueIdentifier: string
   pluginName: string
+  category: PluginType
   usedInApps: number
   isShowFetchNewVersion: boolean
   isShowInfo: boolean
@@ -34,6 +36,7 @@ const Action: FC<Props> = ({
   installationId,
   pluginUniqueIdentifier,
   pluginName,
+  category,
   isShowFetchNewVersion,
   isShowInfo,
   isShowDelete,
@@ -67,6 +70,7 @@ const Action: FC<Props> = ({
         },
         payload: {
           type: PluginSource.github,
+          category,
           github: {
             originalPackageInfo: {
               id: pluginUniqueIdentifier,

web/app/components/plugins/plugin-item/index.tsx

@@ -20,11 +20,9 @@ import Title from '../card/base/title'
 import Action from './action'
 import cn from '@/utils/classnames'
 import { API_PREFIX, MARKETPLACE_URL_PREFIX } from '@/config'
-import { useInvalidateInstalledPluginList } from '@/service/use-plugins'
-import { useInvalidateAllBuiltInTools, useInvalidateAllToolProviders } from '@/service/use-tools'
 import { useSingleCategories } from '../hooks'
-import { useProviderContext } from '@/context/provider-context'
 import { useRenderI18nObject } from '@/hooks/use-i18n'
+import useRefreshPluginList from '@/app/components/plugins/install-plugin/hooks/use-refresh-plugin-list'
 
 type Props = {
   className?: string
@@ -39,10 +37,7 @@ const PluginItem: FC<Props> = ({
   const { categoriesMap } = useSingleCategories()
   const currentPluginID = usePluginPageContext(v => v.currentPluginID)
   const setCurrentPluginID = usePluginPageContext(v => v.setCurrentPluginID)
-  const invalidateInstalledPluginList = useInvalidateInstalledPluginList()
+  const { refreshPluginList } = useRefreshPluginList()
-  const invalidateAllToolProviders = useInvalidateAllToolProviders()
-  const invalidateAllBuiltinTools = useInvalidateAllBuiltInTools()
-  const { refreshModelProviders } = useProviderContext()
 
   const {
     source,
@@ -60,13 +55,7 @@ const PluginItem: FC<Props> = ({
   }, [source, author])
 
   const handleDelete = () => {
-    invalidateInstalledPluginList()
+    refreshPluginList({ category } as any)
-    if (PluginType.model.includes(category))
-      refreshModelProviders()
-    if (PluginType.tool.includes(category)) {
-      invalidateAllToolProviders()
-      invalidateAllBuiltinTools()
-    }
   }
   const getValueFromI18nObject = useRenderI18nObject()
   const title = getValueFromI18nObject(label)
@@ -116,6 +105,7 @@ const PluginItem: FC<Props> = ({
                   isShowDelete
                   meta={meta}
                   onDelete={handleDelete}
+                  category={category}
                 />
               </div>
             </div>

web/app/components/plugins/types.ts

@@ -151,6 +151,7 @@ export type Permissions = {
 }
 
 export type UpdateFromMarketPlacePayload = {
+  category: PluginType
   originalPackageInfo: {
     id: string
     payload: PluginDeclaration
@@ -173,6 +174,7 @@ export type UpdateFromGitHubPayload = {
 
 export type UpdatePluginPayload = {
   type: PluginSource
+  category: PluginType
   marketPlace?: UpdateFromMarketPlacePayload
   github?: UpdateFromGitHubPayload
 }

web/app/components/plugins/update-plugin/from-market-place.tsx

@@ -57,7 +57,7 @@ const UpdatePluginModal: FC<Props> = ({
   }
 
   const [uploadStep, setUploadStep] = useState<UploadStep>(UploadStep.notStarted)
-  const { handleRefetch } = usePluginTaskList()
+  const { handleRefetch } = usePluginTaskList(payload.category)
 
   const configBtnText = useMemo(() => {
     return ({

web/app/components/tools/provider-list.tsx

@@ -69,7 +69,7 @@ const ProviderList = () => {
           className='relative flex flex-col overflow-y-auto bg-background-body grow'
         >
           <div className={cn(
-            'sticky top-0 flex justify-between items-center pt-4 px-12 pb-2 leading-[56px] z-20 flex-wrap gap-y-2',
+            'sticky top-0 flex justify-between items-center pt-4 px-12 pb-2 leading-[56px] bg-background-body z-20 flex-wrap gap-y-2',
             currentProvider && 'pr-6',
           )}>
             <TabSliderNew

web/service/use-plugins.ts

@@ -11,6 +11,7 @@ import type {
   PluginDetail,
   PluginInfoFromMarketPlace,
   PluginTask,
+  PluginType,
   PluginsFromMarketplaceByInfoResponse,
   PluginsFromMarketplaceResponse,
   VersionInfo,
@@ -31,6 +32,7 @@ import {
 import { useInvalidateAllBuiltInTools } from './use-tools'
 import usePermission from '@/app/components/plugins/plugin-page/use-permission'
 import { uninstallPlugin } from '@/service/plugins'
+import useRefreshPluginList from '@/app/components/plugins/install-plugin/hooks/use-refresh-plugin-list'
 
 const NAME_SPACE = 'plugins'
 
@@ -367,10 +369,11 @@ export const useFetchPluginsInMarketPlaceByInfo = (infos: Record<string, any>[])
 }
 
 const usePluginTaskListKey = [NAME_SPACE, 'pluginTaskList']
-export const usePluginTaskList = () => {
+export const usePluginTaskList = (category?: PluginType) => {
   const {
     canManagement,
   } = usePermission()
+  const { refreshPluginList } = useRefreshPluginList()
   const {
     data,
     isFetched,
@@ -383,8 +386,12 @@ export const usePluginTaskList = () => {
     refetchInterval: (lastQuery) => {
       const lastData = lastQuery.state.data
       const taskDone = lastData?.tasks.every(task => task.status === TaskStatus.success || task.status === TaskStatus.failed)
-      if (taskDone)
+      const taskAllFailed = lastData?.tasks.every(task => task.status === TaskStatus.failed)
+      if (taskDone) {
+        if (lastData?.tasks.length && !taskAllFailed)
+          refreshPluginList(category ? { category } as any : undefined, !category)
         return false
+      }
 
       return 5000
     },