Merge branch 'fix/import-old-dsl' into dev/plugin-deploy

api/core/rag/index_processor/processor/parent_child_index_processor.py

@@ -47,6 +47,8 @@ class ParentChildIndexProcessor(BaseIndexProcessor):
                 embedding_model_instance=kwargs.get("embedding_model_instance"),
             )
             for document in documents:
+                if kwargs.get("preview") and len(all_documents) >= 10:
+                    return all_documents
                 # document clean
                 document_text = CleanProcessor.clean(document.page_content, process_rule)
                 document.page_content = document_text

api/services/app_dsl_service.py

@@ -1,5 +1,6 @@
 import logging
 import uuid
+from collections.abc import Mapping
 from enum import StrEnum
 from typing import Optional
 from urllib.parse import urlparse
@@ -256,6 +257,16 @@ class AppDslService:
             check_dependencies_pending_data = None
             if dependencies:
                 check_dependencies_pending_data = [PluginDependency.model_validate(d) for d in dependencies]
+            elif imported_version <= "0.1.5":
+                if "workflow" in data:
+                    graph = data.get("workflow", {}).get("graph", {})
+                    dependencies_list = self._extract_dependencies_from_workflow_graph(graph)
+                else:
+                    dependencies_list = self._extract_dependencies_from_model_config(data.get("model_config", {}))
+
+                check_dependencies_pending_data = DependenciesAnalysisService.generate_latest_dependencies(
+                    dependencies_list
+                )
 
             # Create or update app
             app = self._create_or_update_app(
@@ -558,7 +569,7 @@ class AppDslService:
             raise ValueError("Missing app configuration, please check.")
 
         export_data["model_config"] = app_model_config.to_dict()
-        dependencies = cls._extract_dependencies_from_model_config(app_model_config)
+        dependencies = cls._extract_dependencies_from_model_config(app_model_config.to_dict())
         export_data["dependencies"] = [
             jsonable_encoder(d.model_dump())
             for d in DependenciesAnalysisService.generate_dependencies(
@@ -574,6 +585,16 @@ class AppDslService:
         :return: dependencies list format like ["langgenius/google"]
         """
         graph = workflow.graph_dict
+        dependencies = cls._extract_dependencies_from_workflow_graph(graph)
+        return dependencies
+
+    @classmethod
+    def _extract_dependencies_from_workflow_graph(cls, graph: Mapping) -> list[str]:
+        """
+        Extract dependencies from workflow graph
+        :param graph: Workflow graph
+        :return: dependencies list format like ["langgenius/google"]
+        """
         dependencies = []
         for node in graph.get("nodes", []):
             try:
@@ -647,24 +668,24 @@ class AppDslService:
         return dependencies
 
     @classmethod
-    def _extract_dependencies_from_model_config(cls, model_config: AppModelConfig) -> list[str]:
+    def _extract_dependencies_from_model_config(cls, model_config: Mapping) -> list[str]:
         """
         Extract dependencies from model config
-        :param model_config: AppModelConfig instance
-        :return: dependencies list format like ["langgenius/google:1.0.0@abcdef1234567890"]
+        :param model_config: model config dict
+        :return: dependencies list format like ["langgenius/google"]
         """
         dependencies = []
 
         try:
             # completion model
-            model_dict = model_config.model_dict
+            model_dict = model_config.get("model", {})
             if model_dict:
                 dependencies.append(
                     DependenciesAnalysisService.analyze_model_provider_dependency(model_dict.get("provider", ""))
                 )
 
             # reranking model
-            dataset_configs = model_config.dataset_configs_dict
+            dataset_configs = model_config.get("dataset_configs", {})
             if dataset_configs:
                 for dataset_config in dataset_configs.get("datasets", {}).get("datasets", []):
                     if dataset_config.get("reranking_model"):
@@ -677,7 +698,7 @@ class AppDslService:
                         )
 
             # tools
-            agent_configs = model_config.agent_mode_dict
+            agent_configs = model_config.get("agent_mode", {})
             if agent_configs:
                 for agent_config in agent_configs.get("tools", []):
                     dependencies.append(

api/services/plugin/dependencies_analysis.py

@@ -1,3 +1,4 @@
+from core.helper import marketplace
 from core.plugin.entities.plugin import GenericProviderID, PluginDependency, PluginInstallationSource
 from core.plugin.manager.plugin import PluginInstallationManager
 
@@ -98,3 +99,18 @@ class DependenciesAnalysisService:
                 raise ValueError(f"Unknown plugin source: {plugin.source}")
 
         return result
+
+    @classmethod
+    def generate_latest_dependencies(cls, dependencies: list[str]) -> list[PluginDependency]:
+        """
+        Generate the latest version of dependencies
+        """
+        dependencies = list(set(dependencies))
+        deps = marketplace.batch_fetch_plugin_manifests(dependencies)
+        return [
+            PluginDependency(
+                type=PluginDependency.Type.Marketplace,
+                value=PluginDependency.Marketplace(marketplace_plugin_unique_identifier=dep.latest_package_identifier),
+            )
+            for dep in deps
+        ]

web/app/components/header/account-setting/model-provider-page/hooks.ts

@@ -275,6 +275,8 @@ export const useMarketplaceAllPlugins = (providers: ModelProvider[], searchText:
         category: PluginType.model,
         exclude,
         type: 'plugin',
+        sortBy: 'install_count',
+        sortOrder: 'DESC',
       })
     }
     else {
@@ -284,6 +286,8 @@ export const useMarketplaceAllPlugins = (providers: ModelProvider[], searchText:
         type: 'plugin',
         pageSize: 1000,
         exclude,
+        sortBy: 'install_count',
+        sortOrder: 'DESC',
       })
     }
   }, [queryPlugins, queryPluginsWithDebounced, searchText, exclude])

web/app/components/header/account-setting/model-provider-page/index.tsx

@@ -71,11 +71,11 @@ const ModelProviderPage = ({ searchText }: Props) => {
   const [filteredConfiguredProviders, filteredNotConfiguredProviders] = useMemo(() => {
     const filteredConfiguredProviders = configuredProviders.filter(
       provider => provider.provider.toLowerCase().includes(debouncedSearchText.toLowerCase())
-      || Object.values(provider.label).some(text => text.toLowerCase().includes(debouncedSearchText.toLowerCase())),
+        || Object.values(provider.label).some(text => text.toLowerCase().includes(debouncedSearchText.toLowerCase())),
     )
     const filteredNotConfiguredProviders = notConfiguredProviders.filter(
       provider => provider.provider.toLowerCase().includes(debouncedSearchText.toLowerCase())
-      || Object.values(provider.label).some(text => text.toLowerCase().includes(debouncedSearchText.toLowerCase())),
+        || Object.values(provider.label).some(text => text.toLowerCase().includes(debouncedSearchText.toLowerCase())),
     )
 
     return [filteredConfiguredProviders, filteredNotConfiguredProviders]
@@ -143,7 +143,7 @@ const ModelProviderPage = ({ searchText }: Props) => {
       )}
       {!!filteredNotConfiguredProviders?.length && (
         <>
-          <div className='flex items-center mb-2 pt-2 text-text-primary system-md-semibold'>{t('common.modelProvider.configureRequired')}</div>
+          <div className='flex items-center mb-2 pt-2 text-text-primary system-md-semibold'>{t('common.modelProvider.toBeConfigured')}</div>
           <div className='relative'>
             {filteredNotConfiguredProviders?.map(provider => (
               <ProviderAddedCard

web/app/components/plugins/install-plugin/hooks/use-check-installed.tsx

@@ -16,6 +16,7 @@ const useCheckInstalled = (props: Props) => {
     const res: Record<string, VersionInfo> = {}
     data?.plugins.forEach((plugin) => {
       res[plugin.plugin_id] = {
+        installedId: plugin.id,
         installedVersion: plugin.declaration.version,
         uniqueIdentifier: plugin.plugin_unique_identifier,
       }

web/app/components/plugins/install-plugin/install-bundle/steps/install-multi.tsx

@@ -26,7 +26,7 @@ const InstallByDSLList: FC<Props> = ({
   isFromMarketPlace,
 }) => {
   // DSL has id, to get plugin info to show more info
-  const { isLoading: isFetchingMarketplaceDataById, data: infoGetById, error: infoByIdError } = useFetchPluginsInMarketPlaceByIds(allPlugins.filter(d => d.type === 'marketplace').map(d => (d as GitHubItemAndMarketPlaceDependency).value.plugin_unique_identifier!))
+  const { isLoading: isFetchingMarketplaceDataById, data: infoGetById, error: infoByIdError } = useFetchPluginsInMarketPlaceByIds(allPlugins.filter(d => d.type === 'marketplace').map(d => (d as GitHubItemAndMarketPlaceDependency).value.marketplace_plugin_unique_identifier!))
   // has meta(org,name,version), to get id
   const { isLoading: isFetchingDataByMeta, data: infoByMeta, error: infoByMetaError } = useFetchPluginsInMarketPlaceByInfo(allPlugins.filter(d => d.type === 'marketplace').map(d => (d as GitHubItemAndMarketPlaceDependency).value!))
 

web/app/components/plugins/install-plugin/install-from-local-package/steps/install.tsx

@@ -8,8 +8,9 @@ import Button from '@/app/components/base/button'
 import { Trans, useTranslation } from 'react-i18next'
 import { RiLoader2Line } from '@remixicon/react'
 import checkTaskStatus from '../../base/check-task-status'
-import { useInstallPackageFromLocal, usePluginTaskList, useUpdatePackageFromMarketPlace } from '@/service/use-plugins'
+import { useInstallPackageFromLocal, usePluginTaskList } from '@/service/use-plugins'
 import useCheckInstalled from '@/app/components/plugins/install-plugin/hooks/use-check-installed'
+import { uninstallPlugin } from '@/service/plugins'
 import Version from '../../base/version'
 
 const i18nPrefix = 'plugin.installModal'
@@ -50,7 +51,6 @@ const Installed: FC<Props> = ({
 
   const [isInstalling, setIsInstalling] = React.useState(false)
   const { mutateAsync: installPackageFromLocal } = useInstallPackageFromLocal()
-  const { mutateAsync: updatePackageFromMarketPlace } = useUpdatePackageFromMarketPlace()
 
   const {
     check,
@@ -69,27 +69,15 @@ const Installed: FC<Props> = ({
     onStartToInstall?.()
 
     try {
-      let taskId
-      let isInstalled
-      if (hasInstalled) {
-        const {
-          all_installed,
-          task_id,
-        } = await updatePackageFromMarketPlace({
-          original_plugin_unique_identifier: installedInfoPayload.uniqueIdentifier,
-          new_plugin_unique_identifier: uniqueIdentifier,
-        })
-        taskId = task_id
-        isInstalled = all_installed
-      }
-      else {
-        const {
-          all_installed,
-          task_id,
-        } = await installPackageFromLocal(uniqueIdentifier)
-        taskId = task_id
-        isInstalled = all_installed
-      }
+      if (hasInstalled)
+        await uninstallPlugin(installedInfoPayload.installedId)
+
+      const {
+        all_installed,
+        task_id,
+      } = await installPackageFromLocal(uniqueIdentifier)
+      const taskId = task_id
+      const isInstalled = all_installed
 
       if (isInstalled) {
         onInstalled()

web/app/components/plugins/types.ts

@@ -373,6 +373,7 @@ export type VersionListResponse = {
 }
 
 export type VersionInfo = {
+  installedId: string, // use to uninstall
   installedVersion: string,
   uniqueIdentifier: string
 }

web/i18n/en-US/common.ts

@@ -407,7 +407,7 @@ const translation = {
     loadBalancingLeastKeyWarning: 'To enable load balancing at least 2 keys must be enabled.',
     loadBalancingInfo: 'By default, load balancing uses the Round-robin strategy. If rate limiting is triggered, a 1-minute cooldown period will be applied.',
     upgradeForLoadBalancing: 'Upgrade your plan to enable Load Balancing.',
-    configureRequired: 'Configure required',
+    toBeConfigured: 'To be configured',
     configureTip: 'Set up api-key or add model to use',
     installProvider: 'Install model providers',
     discoverMore: 'Discover more in ',

web/i18n/zh-Hans/common.ts

@@ -407,7 +407,7 @@ const translation = {
     loadBalancingInfo: '默认情况下，负载均衡使用 Round-robin 策略。如果触发速率限制，将应用 1 分钟的冷却时间',
     upgradeForLoadBalancing: '升级以解锁负载均衡功能',
     apiKey: 'API 密钥',
-    configureRequired: '尚未配置',
+    toBeConfigured: '待配置',
     configureTip: '请配置 API 密钥，添加模型。',
     installProvider: '安装模型供应商',
     discoverMore: '发现更多就在',

web/package.json

@@ -66,7 +66,7 @@
     "js-audio-recorder": "^1.0.7",
     "js-cookie": "^3.0.5",
     "jwt-decode": "^4.0.0",
-    "katex": "^0.16.11",
+    "katex": "^0.16.21",
     "ky": "^1.7.2",
     "lamejs": "^1.2.1",
     "lexical": "^0.18.0",
@@ -196,4 +196,4 @@
       "eslint --fix"
     ]
   }
-}
+}

web/pnpm-lock.yaml

@@ -140,8 +140,8 @@ importers:
         specifier: ^4.0.0
         version: 4.0.0
       katex:
-        specifier: ^0.16.11
-        version: 0.16.11
+        specifier: ^0.16.21
+        version: 0.16.21
       ky:
         specifier: ^1.7.2
         version: 1.7.2
@@ -5744,8 +5744,8 @@ packages:
     resolution: {integrity: sha512-+KJGIyHgkGuIq3IEBNftfhW/LfWhXUIY6OmyVWjliu5KH1y0fw7VQ8YndE2O4qZdMSd9SqbnC8GOcZEy0Om7sA==}
     engines: {node: '>=18'}
 
-  katex@0.16.11:
-    resolution: {integrity: sha512-RQrI8rlHY92OLf3rho/Ts8i/XvjgguEjOkO1BEXcU3N8BqPpSzBNwV/G0Ukr+P/l3ivvJUE/Fa/CwbS6HesGNQ==}
+  katex@0.16.21:
+    resolution: {integrity: sha512-XvqR7FgOHtWupfMiigNzmh+MgUVmDGU2kXZm899ZkPfcuoPuFxyHmXsgATDpFZDAXCI8tvinaVcDo8PIIJSo4A==}
     hasBin: true
 
   keyv@4.5.4:
@@ -15130,7 +15130,7 @@ snapshots:
 
   jwt-decode@4.0.0: {}
 
-  katex@0.16.11:
+  katex@0.16.21:
     dependencies:
       commander: 8.3.0
 
@@ -15558,7 +15558,7 @@ snapshots:
       dagre-d3-es: 7.0.11
       dayjs: 1.11.13
       dompurify: 3.2.3
-      katex: 0.16.11
+      katex: 0.16.21
       khroma: 2.1.0
       lodash-es: 4.17.21
       marked: 13.0.3
@@ -15652,7 +15652,7 @@ snapshots:
     dependencies:
       '@types/katex': 0.16.7
       devlop: 1.1.0
-      katex: 0.16.11
+      katex: 0.16.21
       micromark-factory-space: 2.0.0
       micromark-util-character: 2.1.0
       micromark-util-symbol: 2.0.0
@@ -16991,7 +16991,7 @@ snapshots:
       '@types/katex': 0.16.7
       hast-util-from-html-isomorphic: 2.0.0
       hast-util-to-text: 4.0.2
-      katex: 0.16.11
+      katex: 0.16.21
       unist-util-visit-parents: 6.0.1
       vfile: 6.0.3
 

web/service/use-plugins.ts

@@ -29,6 +29,8 @@ import {
   useQueryClient,
 } from '@tanstack/react-query'
 import { useInvalidateAllBuiltInTools } from './use-tools'
+import usePermission from '@/app/components/plugins/plugin-page/use-permission'
+import { uninstallPlugin } from '@/service/plugins'
 
 const NAME_SPACE = 'plugins'
 
@@ -236,10 +238,20 @@ export const useInstallOrUpdate = ({
             }
           }
           if (isInstalled) {
-            await updatePackageFromMarketPlace({
-              original_plugin_unique_identifier: installedPayload?.uniqueIdentifier,
-              new_plugin_unique_identifier: uniqueIdentifier,
-            })
+            if (item.type === 'package') {
+              await uninstallPlugin(installedPayload.installedId)
+              await post<InstallPackageResponse>('/workspaces/current/plugin/install/pkg', {
+                body: {
+                  plugin_unique_identifiers: [uniqueIdentifier],
+                },
+              })
+            }
+            else {
+              await updatePackageFromMarketPlace({
+                original_plugin_unique_identifier: installedPayload?.uniqueIdentifier,
+                new_plugin_unique_identifier: uniqueIdentifier,
+              })
+            }
           }
           return ({ success: true })
         }
@@ -356,12 +368,16 @@ export const useFetchPluginsInMarketPlaceByInfo = (infos: Record<string, any>[])
 
 const usePluginTaskListKey = [NAME_SPACE, 'pluginTaskList']
 export const usePluginTaskList = () => {
+  const {
+    canManagement,
+  } = usePermission()
   const {
     data,
     isFetched,
     refetch,
     ...rest
   } = useQuery({
+    enabled: canManagement,
     queryKey: usePluginTaskListKey,
     queryFn: () => get<{ tasks: PluginTask[] }>('/workspaces/current/plugin/tasks?page=1&page_size=100'),
     refetchInterval: (lastQuery) => {