Merge remote-tracking branch 'origin/feat/rag-2' into feat/rag-2

web/app/components/header/account-setting/data-source-page-new/card.tsx

@@ -1,20 +1,39 @@
 import { memo } from 'react'
 import Item from './item'
 import Configure from './configure'
+import type { DataSourceAuth } from './types'
+import { useRenderI18nObject } from '@/hooks/use-i18n'
+
+type CardProps = {
+  item: DataSourceAuth
+}
+const Card = ({
+  item,
+}: CardProps) => {
+  const renderI18nObject = useRenderI18nObject()
+  const {
+    icon,
+    label,
+    author,
+    provider,
+    credentials_list,
+  } = item
 
-const Card = () => {
   return (
     <div className='rounded-xl bg-background-section-burn'>
       <div className='flex items-center p-3 pb-2'>
-        <div className='mr-3 flex h-10 w-10 shrink-0 items-center justify-center'></div>
+        <img
+          src={icon}
+          className='mr-3 flex h-10 w-10 shrink-0 items-center justify-center'
+        />
         <div className='grow'>
           <div className='system-md-semibold text-text-primary'>
-            Notion Data Source
+            {renderI18nObject(label)}
           </div>
           <div className='system-xs-regular flex h-4 items-center text-text-tertiary'>
-            langgenius
+            {author}
             <div className='text-text-quaternary'>/</div>
-            notion-data-source
+            {provider}
           </div>
         </div>
         <Configure />
@@ -23,16 +42,24 @@ const Card = () => {
         Connected workspace
         <div className='ml-3 h-[1px] grow bg-divider-subtle'></div>
       </div>
-      <div className='space-y-1 p-3 pt-2'>
+      {
-        <Item />
+        !!credentials_list.length && (
-        <Item />
+          <div className='space-y-1 p-3 pt-2'>
-        <Item />
+            <Item />
-      </div>
+            <Item />
-      <div className='p-3 pt-1'>
+            <Item />
-        <div className='system-xs-regular flex h-10 items-center justify-center rounded-[10px] bg-background-section text-text-tertiary'>
+          </div>
-          Please configure authentication
+        )
-        </div>
+      }
-      </div>
+      {
+        !credentials_list.length && (
+          <div className='p-3 pt-1'>
+            <div className='system-xs-regular flex h-10 items-center justify-center rounded-[10px] bg-background-section text-text-tertiary'>
+              Please configure authentication
+            </div>
+          </div>
+        )
+      }
     </div>
   )
 }

web/app/components/header/account-setting/data-source-page-new/index.tsx

@@ -2,14 +2,23 @@ import { memo } from 'react'
 import Card from './card'
 import InstallFromMarketplace from './install-from-marketplace'
 import { useGlobalPublicStore } from '@/context/global-public-context'
+import { useGetDataSourceListAuth } from '@/service/use-datasource'
 
 const DataSourcePage = () => {
   const { enable_marketplace } = useGlobalPublicStore(s => s.systemFeatures)
+  const { data } = useGetDataSourceListAuth()
+
   return (
     <div>
       <div className='space-y-2'>
-        <Card />
+        {
-        <Card />
+          data?.result.map(item => (
+            <Card
+              key={item.plugin_unique_identifier}
+              item={item}
+            />
+          ))
+        }
       </div>
       {
         enable_marketplace && (

web/app/components/header/account-setting/data-source-page-new/types.ts

@@ -0,0 +1,22 @@
+export type DataSourceCredential = {
+  credential: Record<string, any>
+  type: string
+  name: string
+  id: string
+}
+export type DataSourceAuth = {
+  author: string
+  provider: string
+  plugin_id: string
+  plugin_unique_identifier: string
+  icon: any
+  name: string
+  label: any
+  description: any
+  credential_schema?: any[]
+  oauth_schema?: {
+    client_schema?: any[]
+    credentials_schema?: any[]
+  }
+  credentials_list: DataSourceCredential[]
+}

web/app/components/rag-pipeline/hooks/use-pipeline-config.ts

@@ -8,6 +8,7 @@ import type { FetchWorkflowDraftResponse } from '@/types/workflow'
 import { useDataSourceList } from '@/service/use-pipeline'
 import type { DataSourceItem } from '@/app/components/workflow/block-selector/types'
 import { basePath } from '@/utils/var'
+import type { FileUploadConfigResponse } from '@/models/common'
 
 export const usePipelineConfig = () => {
   const pipelineId = useStore(s => s.pipelineId)
@@ -53,5 +54,11 @@ export const usePipelineConfig = () => {
     setDataSourceList!(dataSourceList)
   }, [workflowStore])
 
+  const handleUpdateWorkflowFileUploadConfig = useCallback((config: FileUploadConfigResponse) => {
+    const { setFileUploadConfig } = workflowStore.getState()
+    setFileUploadConfig(config)
+  }, [workflowStore])
+  useWorkflowConfig('/files/upload', handleUpdateWorkflowFileUploadConfig)
+
   useDataSourceList(!!pipelineId, handleUpdateDataSourceList)
 }

web/app/components/workflow-app/hooks/use-workflow-init.ts

@@ -17,6 +17,8 @@ import {
 } from '@/service/workflow'
 import type { FetchWorkflowDraftResponse } from '@/types/workflow'
 import { useWorkflowConfig } from '@/service/use-workflow'
+import type { FileUploadConfigResponse } from '@/models/common'
+
 export const useWorkflowInit = () => {
   const workflowStore = useWorkflowStore()
   const {
@@ -38,6 +40,15 @@ export const useWorkflowInit = () => {
   }, [workflowStore])
   useWorkflowConfig(`/apps/${appDetail.id}/workflows/draft/config`, handleUpdateWorkflowConfig)
 
+  const handleUpdateWorkflowFileUploadConfig = useCallback((config: FileUploadConfigResponse) => {
+    const { setFileUploadConfig } = workflowStore.getState()
+    setFileUploadConfig(config)
+  }, [workflowStore])
+  const {
+    data: fileUploadConfigResponse,
+    isLoading: isFileUploadConfigLoading,
+  } = useWorkflowConfig('/files/upload', handleUpdateWorkflowFileUploadConfig)
+
   const handleGetInitialWorkflowData = useCallback(async () => {
     try {
       const res = await fetchWorkflowDraft(`/apps/${appDetail.id}/workflows/draft`)
@@ -117,6 +128,7 @@ export const useWorkflowInit = () => {
 
   return {
     data,
-    isLoading,
+    isLoading: isLoading || isFileUploadConfigLoading,
+    fileUploadConfigResponse,
   }
 }

web/app/components/workflow-app/index.tsx

@@ -1,7 +1,6 @@
 import {
   useMemo,
 } from 'react'
-import useSWR from 'swr'
 import {
   SupportUploadFileTypes,
 } from '@/app/components/workflow/types'
@@ -16,7 +15,6 @@ import Loading from '@/app/components/base/loading'
 import { FeaturesProvider } from '@/app/components/base/features'
 import type { Features as FeaturesData } from '@/app/components/base/features/types'
 import { FILE_EXTS } from '@/app/components/base/prompt-editor/constants'
-import { fetchFileUploadConfig } from '@/service/common'
 import WorkflowWithDefaultContext from '@/app/components/workflow'
 import {
   WorkflowContextProvider,
@@ -29,8 +27,8 @@ const WorkflowAppWithAdditionalContext = () => {
   const {
     data,
     isLoading,
+    fileUploadConfigResponse,
   } = useWorkflowInit()
-  const { data: fileUploadConfigResponse } = useSWR({ url: '/files/upload' }, fetchFileUploadConfig)
 
   const nodesData = useMemo(() => {
     if (data)

web/app/components/workflow/store/workflow/workflow-slice.ts

@@ -3,6 +3,7 @@ import type {
   Node,
   WorkflowRunningData,
 } from '@/app/components/workflow/types'
+import type { FileUploadConfigResponse } from '@/models/common'
 
 type PreviewRunningData = WorkflowRunningData & {
   resultTabActive?: boolean
@@ -32,6 +33,8 @@ export type WorkflowSliceShape = {
   setShowTips: (showTips: string) => void
   workflowConfig?: Record<string, any>
   setWorkflowConfig: (workflowConfig: Record<string, any>) => void
+  fileUploadConfig?: FileUploadConfigResponse
+  setFileUploadConfig: (fileUploadConfig: FileUploadConfigResponse) => void
 }
 
 export const createWorkflowSlice: StateCreator<WorkflowSliceShape> = set => ({
@@ -60,4 +63,6 @@ export const createWorkflowSlice: StateCreator<WorkflowSliceShape> = set => ({
   setShowTips: showTips => set(() => ({ showTips })),
   workflowConfig: undefined,
   setWorkflowConfig: workflowConfig => set(() => ({ workflowConfig })),
+  fileUploadConfig: undefined,
+  setFileUploadConfig: fileUploadConfig => set(() => ({ fileUploadConfig })),
 })

web/app/components/workflow/variable-inspect/value-content.tsx

@@ -12,7 +12,6 @@ import {
 import {
   validateJSONSchema,
 } from '@/app/components/workflow/variable-inspect/utils'
-import { useFeatures } from '@/app/components/base/features/hooks'
 import { getProcessedFiles, getProcessedFilesFromResponse } from '@/app/components/base/file-uploader/utils'
 import { JSON_SCHEMA_MAX_DEPTH } from '@/config'
 import { TransferMethod } from '@/types/app'
@@ -21,6 +20,7 @@ import { SupportUploadFileTypes } from '@/app/components/workflow/types'
 import type { VarInInspect } from '@/types/workflow'
 import { VarInInspectType } from '@/types/workflow'
 import cn from '@/utils/classnames'
+import { useStore } from '@/app/components/workflow/store'
 
 type Props = {
   currentVar: VarInInspect
@@ -40,6 +40,7 @@ const ValueContent = ({
   const showFileEditor = isSysFiles || currentVar.value_type === 'file' || currentVar.value_type === 'array[file]'
   const textEditorDisabled = currentVar.type === VarInInspectType.environment || (currentVar.type === VarInInspectType.system && currentVar.name !== 'query' && currentVar.name !== 'files')
   const JSONEditorDisabled = currentVar.value_type === 'array[any]'
+  const fileUploadConfig = useStore(s => s.fileUploadConfig)
 
   const formatFileValue = (value: VarInInspect) => {
     if (value.value_type === 'file')
@@ -53,7 +54,6 @@ const ValueContent = ({
   const [json, setJson] = useState('')
   const [parseError, setParseError] = useState<Error | null>(null)
   const [validationError, setValidationError] = useState<string>('')
-  const fileFeature = useFeatures(s => s.features.file)
   const [fileValue, setFileValue] = useState<any>(formatFileValue(currentVar))
 
   const { run: debounceValueChange } = useDebounceFn(handleValueChange, { wait: 500 })
@@ -206,8 +206,8 @@ const ValueContent = ({
                   ...FILE_EXTS[SupportUploadFileTypes.video],
                 ],
                 allowed_file_upload_methods: [TransferMethod.local_file, TransferMethod.remote_url],
-                number_limits: currentVar.value_type === 'file' ? 1 : (fileFeature as any).fileUploadConfig?.workflow_file_upload_limit || 5,
+                number_limits: currentVar.value_type === 'file' ? 1 : fileUploadConfig?.workflow_file_upload_limit || 5,
-                fileUploadConfig: (fileFeature as any).fileUploadConfig,
+                fileUploadConfig,
               }}
               isDisabled={textEditorDisabled}
             />

web/service/use-datasource.ts

@@ -0,0 +1,13 @@
+import { useQuery } from '@tanstack/react-query'
+import { get } from './base'
+import type { DataSourceAuth } from '@/app/components/header/account-setting/data-source-page-new/types'
+
+const NAME_SPACE = 'data-source-auth'
+
+export const useGetDataSourceListAuth = () => {
+  return useQuery({
+    queryKey: [NAME_SPACE, 'list'],
+    queryFn: () => get<{ result: DataSourceAuth[] }>('/auth/plugin/datasource/list'),
+    retry: 0,
+  })
+}