refactor: streamline dataset detail fetching and improve dataset list handling across components

web/app/components/datasets/create/index.tsx

@@ -10,10 +10,10 @@ import { TopBar } from './top-bar'
 import { DataSourceType } from '@/models/datasets'
 import type { CrawlOptions, CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets'
 import { fetchDataSource } from '@/service/common'
-import { fetchDatasetDetail } from '@/service/datasets'
 import { DataSourceProvider, type NotionPage } from '@/models/common'
 import { useModalContext } from '@/context/modal-context'
 import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { useDatasetDetail } from '@/service/knowledge/use-dataset'
 
 type DatasetUpdateFormProps = {
   datasetId?: string
@@ -39,7 +39,6 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
   const [retrievalMethodCache, setRetrievalMethodCache] = useState('')
   const [fileList, setFiles] = useState<FileItem[]>([])
   const [result, setResult] = useState<createDocumentResponse | undefined>()
-  const [hasError, setHasError] = useState(false)
   const { data: embeddingsDefaultModel } = useDefaultModel(ModelTypeEnum.textEmbedding)
 
   const [notionPages, setNotionPages] = useState<NotionPage[]>([])
@@ -104,21 +103,14 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => {
   }, [])
 
   const [detail, setDetail] = useState<DataSet | null>(null)
+
+  const { data: datasetDetail, error: fetchDatasetDetailError } = useDatasetDetail(datasetId || '')
   useEffect(() => {
-    (async () => {
+    if (!datasetDetail) return
-      if (datasetId) {
+    setDetail(datasetDetail)
-        try {
+  }, [datasetDetail])
-          const detail = await fetchDatasetDetail(datasetId)
+
-          setDetail(detail)
+  if (fetchDatasetDetailError)
-        }
-        catch {
-          setHasError(true)
-        }
-      }
-    })()
-  }, [datasetId])
-
-  if (hasError)
     return <AppUnavailable code={500} unknownReason={t('datasetCreation.error.unavailable') as string} />
 
   return (

web/app/components/datasets/list/datasets.tsx

@@ -22,7 +22,7 @@ const Datasets = ({
   const { t } = useTranslation()
   const isCurrentWorkspaceEditor = useAppContextWithSelector(state => state.isCurrentWorkspaceEditor)
   const {
-    data,
+    data: datasetList,
     fetchNextPage,
     hasNextPage,
     isFetching,
@@ -54,13 +54,13 @@ const Datasets = ({
       observerRef.current.observe(anchorRef.current)
     }
     return () => observerRef.current?.disconnect()
-  }, [anchorRef, data, hasNextPage, fetchNextPage])
+  }, [anchorRef, datasetList, hasNextPage, fetchNextPage])
 
   return (
     <>
       <nav className='grid shrink-0 grow grid-cols-1 content-start gap-3 px-12 pt-2 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4'>
         {isCurrentWorkspaceEditor && <NewDatasetCard />}
-        {data?.pages.map(({ data: datasets }) => datasets.map(dataset => (
+        {datasetList?.pages.map(({ data: datasets }) => datasets.map(dataset => (
           <DatasetCard key={dataset.id} dataset={dataset} onSuccess={resetDatasetList} />),
         ))}
       </nav>

web/app/components/datasets/settings/form/index.tsx

@@ -2,8 +2,6 @@
 import { useCallback, useRef, useState } from 'react'
 import { useMount } from 'ahooks'
 import { useTranslation } from 'react-i18next'
-import { useSWRConfig } from 'swr'
-import { unstable_serialize } from 'swr/infinite'
 import PermissionSelector from '../permission-selector'
 import IndexMethod from '../index-method'
 import RetrievalSettings from '../../external-knowledge-base/create/RetrievalSettings'
@@ -16,7 +14,7 @@ import Textarea from '@/app/components/base/textarea'
 import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development'
 import { updateDatasetSetting } from '@/service/datasets'
 import type { IconInfo } from '@/models/datasets'
-import { ChunkingMode, type DataSetListResponse, DatasetPermission } from '@/models/datasets'
+import { ChunkingMode, DatasetPermission } from '@/models/datasets'
 import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
 import type { AppIconType, RetrievalConfig } from '@/types/app'
 import { useSelector as useAppContextWithSelector } from '@/context/app-context'
@@ -38,16 +36,11 @@ import ChunkStructure from '../chunk-structure'
 import Toast from '@/app/components/base/toast'
 import { RiAlertFill } from '@remixicon/react'
 import { useDocLink } from '@/context/i18n'
+import { useResetDatasetList } from '@/service/knowledge/use-dataset'
 
 const rowClass = 'flex gap-x-1'
 const labelClass = 'flex items-center shrink-0 w-[180px] h-7 pt-1'
 
-const getKey = (pageIndex: number, previousPageData: DataSetListResponse) => {
-  if (!pageIndex || previousPageData.has_more)
-    return { url: 'datasets', params: { page: pageIndex + 1, limit: 30 } }
-  return null
-}
-
 const DEFAULT_APP_ICON: IconInfo = {
   icon_type: 'emoji',
   icon: '📙',
@@ -58,7 +51,6 @@ const DEFAULT_APP_ICON: IconInfo = {
 const Form = () => {
   const { t } = useTranslation()
   const docLink = useDocLink()
-  const { mutate } = useSWRConfig()
   const isCurrentWorkspaceDatasetOperator = useAppContextWithSelector(state => state.isCurrentWorkspaceDatasetOperator)
   const currentDataset = useDatasetDetailContextWithSelector(state => state.dataset)
   const mutateDatasets = useDatasetDetailContextWithSelector(state => state.mutateDatasetRes)
@@ -135,6 +127,7 @@ const Form = () => {
     getMembers()
   })
 
+  const resetDatasetList = useResetDatasetList()
   const handleSave = async () => {
     if (loading)
       return
@@ -197,7 +190,7 @@ const Form = () => {
       Toast.notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') })
       if (mutateDatasets) {
         await mutateDatasets()
-        mutate(unstable_serialize(getKey))
+        resetDatasetList()
       }
     }
     catch {

web/app/components/header/dataset-nav/index.tsx

@@ -7,39 +7,31 @@ import {
   RiBook2Fill,
   RiBook2Line,
 } from '@remixicon/react'
-import useSWR from 'swr'
-import useSWRInfinite from 'swr/infinite'
 import { flatten } from 'lodash-es'
 import Nav from '../nav'
 import type { NavItem } from '../nav/nav-selector'
-import { fetchDatasetDetail, fetchDatasets } from '@/service/datasets'
-import type { DataSetListResponse } from '@/models/datasets'
 import { basePath } from '@/utils/var'
-
+import { useDatasetDetail, useDatasetList } from '@/service/knowledge/use-dataset'
-const getKey = (pageIndex: number, previousPageData: DataSetListResponse) => {
-  if (!pageIndex || previousPageData.has_more)
-    return { url: 'datasets', params: { page: pageIndex + 1, limit: 30 } }
-  return null
-}
 
 const DatasetNav = () => {
   const { t } = useTranslation()
   const router = useRouter()
   const { datasetId } = useParams()
-  const { data: currentDataset } = useSWR(
+  const { data: currentDataset } = useDatasetDetail(datasetId as string)
-    datasetId
+  const {
-      ? {
+    data: datasetList,
-        url: 'fetchDatasetDetail',
+    fetchNextPage,
-        datasetId,
+    hasNextPage,
-      }
+  } = useDatasetList({
-      : null,
+    initialPage: 1,
-    apiParams => fetchDatasetDetail(apiParams.datasetId as string))
+    limit: 30,
-  const { data: datasetsData, setSize } = useSWRInfinite(datasetId ? getKey : () => null, fetchDatasets, { revalidateFirstPage: false, revalidateAll: true })
+  })
-  const datasetItems = flatten(datasetsData?.map(datasetData => datasetData.data))
+  const datasetItems = flatten(datasetList?.pages.map(datasetData => datasetData.data))
 
   const handleLoadMore = useCallback(() => {
-    setSize(size => size + 1)
+    if (hasNextPage)
-  }, [setSize])
+      fetchNextPage()
+  }, [hasNextPage, fetchNextPage])
 
   return (
     <Nav

web/context/dataset-detail.ts

@@ -1,11 +1,12 @@
 import { createContext, useContext, useContextSelector } from 'use-context-selector'
 import type { DataSet } from '@/models/datasets'
 import type { IndexingType } from '@/app/components/datasets/create/step-two'
+import type { QueryObserverResult, RefetchOptions } from '@tanstack/react-query'
 
 type DatasetDetailContextValue = {
   indexingTechnique?: IndexingType
   dataset?: DataSet
-  mutateDatasetRes?: () => void
+  mutateDatasetRes?: (options?: RefetchOptions | undefined) => Promise<QueryObserverResult<DataSet, Error>>
 }
 const DatasetDetailContext = createContext<DatasetDetailContextValue>({})
 

web/models/datasets.ts

@@ -178,10 +178,10 @@ export type FetchDatasetsParams = {
 
 export type DatasetListRequest = {
   initialPage: number
-  tag_ids: string[]
+  tag_ids?: string[]
   limit: number
-  include_all: boolean
+  include_all?: boolean
-  keyword: string
+  keyword?: string
 }
 
 export type DataSetListResponse = {

web/service/knowledge/use-dataset.ts

@@ -46,6 +46,7 @@ export const useDatasetDetail = (datasetId: string) => {
   return useQuery({
     queryKey: [NAME_SPACE, 'detail', datasetId],
     queryFn: () => get<DataSet>(`/datasets/${datasetId}`),
+    enabled: !!datasetId,
   })
 }