Merge branch 'main' into feat/explore

api/README.md

@@ -33,3 +33,4 @@
    flask run --host 0.0.0.0 --port=5001 --debug
    ```
 7. Setup your application by visiting http://localhost:5001/console/api/setup or other apis...
+8. If you need to debug local async processing, you can run `celery -A app.celery worker`, celery can do dataset importing and other async tasks.

api/core/completion.py

@@ -125,13 +125,17 @@ class Completion:
         pre_prompt = PromptBuilder.process_template(pre_prompt) if pre_prompt else pre_prompt
         if mode == 'completion':
             prompt_template = OutLinePromptTemplate.from_template(
-                template=("Use the following pieces of [CONTEXT] to answer the question at the end. "
-                          "If you don't know the answer, "
-                          "just say that you don't know, don't try to make up an answer. \n"
-                          "```\n"
-                          "[CONTEXT]\n"
-                          "{context}\n"
-                          "```\n" if chain_output else "")
+                template=("""Use the following CONTEXT as your learned knowledge:
+[CONTEXT]
+{context}
+[END CONTEXT]
+
+When answer to user:
+- If you don't know, just say that you don't know.
+- If you don't know when you are not sure, ask for clarification. 
+Avoid mentioning that you obtained the information from the context.
+And answer according to the language of the user's question.
+""" if chain_output else "")
                          + (pre_prompt + "\n" if pre_prompt else "")
                          + "{query}\n"
             )
@@ -153,38 +157,38 @@ class Completion:
         else:
             messages: List[BaseMessage] = []
 
-            system_message = None
-            if pre_prompt:
-                # append pre prompt as system message
-                system_message = PromptBuilder.to_system_message(pre_prompt, inputs)
+            human_inputs = {
+                "query": query
+            }
+
+            human_message_prompt = "{query}"
 
             if chain_output:
-                # append context as system message, currently only use simple stuff prompt
-                context_message = PromptBuilder.to_system_message(
-                    """Use the following pieces of [CONTEXT] to answer the users question. 
-If you don't know the answer, just say that you don't know, don't try to make up an answer.
-```
+                human_inputs['context'] = chain_output
+                human_message_instruction = """Use the following CONTEXT as your learned knowledge.
 [CONTEXT]
 {context}
-```""",
-                    {'context': chain_output}
-                )
+[END CONTEXT]
 
-                if not system_message:
-                    system_message = context_message
-                else:
-                    system_message.content = context_message.content + "\n\n" + system_message.content
-
-            if system_message:
-                messages.append(system_message)
+When answer to user:
+- If you don't know, just say that you don't know.
+- If you don't know when you are not sure, ask for clarification. 
+Avoid mentioning that you obtained the information from the context.
+And answer according to the language of the user's question.
+"""
+                if pre_prompt:
+                    human_inputs.update(inputs)
+                    human_message_instruction += pre_prompt + "\n"
 
-            human_inputs = {
-                "query": query
-            }
+                human_message_prompt = human_message_instruction + "Q:{query}\nA:"
+            else:
+                if pre_prompt:
+                    human_inputs.update(inputs)
+                    human_message_prompt = pre_prompt + "\n" + human_message_prompt
 
             # construct main prompt
             human_message = PromptBuilder.to_human_message(
-                prompt_content="{query}",
+                prompt_content=human_message_prompt,
                 inputs=human_inputs
             )
 

api/core/conversation_message_task.py

@@ -281,6 +281,9 @@ class PubHandler:
 
     @classmethod
     def generate_channel_name(cls, user: Union[Account | EndUser], task_id: str):
+        if not user:
+            raise ValueError("user is required")
+
         user_str = 'account-' + user.id if isinstance(user, Account) else 'end-user-' + user.id
         return "generate_result:{}-{}".format(user_str, task_id)
 

api/core/vector_store/weaviate_vector_store_client.py

@@ -29,7 +29,7 @@ class WeaviateVectorStoreClient(BaseVectorStoreClient):
         return weaviate.Client(
             url=endpoint,
             auth_client_secret=auth_config,
-            timeout_config=(5, 15),
+            timeout_config=(5, 60),
             startup_period=None
         )
 

web/app/(commonLayout)/_layout-client.tsx

@@ -1,5 +1,5 @@
 'use client'
-import type { FC } from 'react'
+import { FC, useRef } from 'react'
 import React, { useEffect, useState } from 'react'
 import { usePathname, useRouter, useSelectedLayoutSegments } from 'next/navigation'
 import useSWR, { SWRConfig } from 'swr'
@@ -8,7 +8,7 @@ import { fetchAppList } from '@/service/apps'
 import { fetchDatasets } from '@/service/datasets'
 import { fetchLanggeniusVersion, fetchUserProfile, logout } from '@/service/common'
 import Loading from '@/app/components/base/loading'
-import AppContext from '@/context/app-context'
+import { AppContextProvider } from '@/context/app-context'
 import DatasetsContext from '@/context/datasets-context'
 import type { LangGeniusVersionResponse, UserProfileResponse } from '@/models/common'
 
@@ -23,6 +23,7 @@ const CommonLayout: FC<ICommonLayoutProps> = ({ children }) => {
   const pattern = pathname.replace(/.*\/app\//, '')
   const [idOrMethod] = pattern.split('/')
   const isNotDetailPage = idOrMethod === 'list'
+  const pageContainerRef = useRef<HTMLDivElement>(null)
 
   const appId = isNotDetailPage ? '' : idOrMethod
 
@@ -71,14 +72,14 @@ const CommonLayout: FC<ICommonLayoutProps> = ({ children }) => {
     <SWRConfig value={{
       shouldRetryOnError: false
     }}>
-      <AppContext.Provider value={{ apps: appList.data, mutateApps, userProfile, mutateUserProfile }}>
+      <AppContextProvider value={{ apps: appList.data, mutateApps, userProfile, mutateUserProfile, pageContainerRef }}>
         <DatasetsContext.Provider value={{ datasets: datasetList?.data || [], mutateDatasets, currentDataset }}>
-          <div className='relative flex flex-col h-full overflow-scroll bg-gray-100'>
+          <div ref={pageContainerRef} className='relative flex flex-col h-full overflow-auto bg-gray-100'>
             <Header isBordered={['/apps', '/datasets'].includes(pathname)} curApp={curApp as any} appItems={appList.data} userProfile={userProfile} onLogout={onLogout} langeniusVersionInfo={langeniusVersionInfo} />
             {children}
           </div>
         </DatasetsContext.Provider>
-      </AppContext.Provider>
+      </AppContextProvider>
     </SWRConfig>
   )
 }

web/app/(commonLayout)/apps/Apps.tsx

@@ -1,23 +1,50 @@
 'use client'
 
-import { useEffect } from 'react'
+import { useEffect, useRef } from 'react'
+import useSWRInfinite from 'swr/infinite'
+import { debounce } from 'lodash-es'
 import AppCard from './AppCard'
 import NewAppCard from './NewAppCard'
-import { useAppContext } from '@/context/app-context'
+import { AppListResponse } from '@/models/app'
+import { fetchAppList } from '@/service/apps'
+import { useSelector } from '@/context/app-context'
+
+const getKey = (pageIndex: number, previousPageData: AppListResponse) => {
+  if (!pageIndex || previousPageData.has_more)
+    return { url: 'apps', params: { page: pageIndex + 1, limit: 30 } }
+  return null
+}
 
 const Apps = () => {
-  const { apps, mutateApps } = useAppContext()
+  const { data, isLoading, setSize, mutate } = useSWRInfinite(getKey, fetchAppList, { revalidateFirstPage: false })
+  const loadingStateRef = useRef(false)
+  const pageContainerRef = useSelector(state => state.pageContainerRef)
+  const anchorRef = useRef<HTMLAnchorElement>(null)
+
+  useEffect(() => {
+    loadingStateRef.current = isLoading
+  }, [isLoading])
 
   useEffect(() => {
-    mutateApps()
+    const onScroll = debounce(() => {
+      if (!loadingStateRef.current) {
+        const { scrollTop, clientHeight } = pageContainerRef.current!
+        const anchorOffset = anchorRef.current!.offsetTop
+        if (anchorOffset - scrollTop - clientHeight < 100) {
+          setSize(size => size + 1)
+        }
+      }
+    }, 50)
+
+    pageContainerRef.current?.addEventListener('scroll', onScroll)
+    return () => pageContainerRef.current?.removeEventListener('scroll', onScroll)
   }, [])
 
   return (
     <nav className='grid content-start grid-cols-1 gap-4 px-12 pt-8 sm:grid-cols-2 lg:grid-cols-4 grow shrink-0'>
-      {apps.map(app => (<AppCard key={app.id} app={app} />))}
-      <NewAppCard />
+      {data?.map(({ data: apps }) => apps.map(app => (<AppCard key={app.id} app={app} />)))}
+      <NewAppCard ref={anchorRef} onSuccess={mutate} />
     </nav>
-
   )
 }
 

web/app/(commonLayout)/apps/NewAppCard.tsx

@@ -1,16 +1,20 @@
 'use client'
 
-import { useState } from 'react'
+import { forwardRef, useState } from 'react'
 import classNames from 'classnames'
 import { useTranslation } from 'react-i18next'
 import style from '../list.module.css'
 import NewAppDialog from './NewAppDialog'
 
-const CreateAppCard = () => {
+export type CreateAppCardProps = {
+  onSuccess?: () => void
+}
+
+const CreateAppCard = forwardRef<HTMLAnchorElement, CreateAppCardProps>(({ onSuccess }, ref) => {
   const { t } = useTranslation()
   const [showNewAppDialog, setShowNewAppDialog] = useState(false)
   return (
-    <a className={classNames(style.listItem, style.newItemCard)} onClick={() => setShowNewAppDialog(true)}>
+    <a ref={ref} className={classNames(style.listItem, style.newItemCard)} onClick={() => setShowNewAppDialog(true)}>
       <div className={style.listItemTitle}>
         <span className={style.newItemIcon}>
           <span className={classNames(style.newItemIconImage, style.newItemIconAdd)} />
@@ -20,9 +24,9 @@ const CreateAppCard = () => {
         </div>
       </div>
       {/* <div className='text-xs text-gray-500'>{t('app.createFromConfigFile')}</div> */}
-      <NewAppDialog show={showNewAppDialog} onClose={() => setShowNewAppDialog(false)} />
+      <NewAppDialog show={showNewAppDialog} onSuccess={onSuccess} onClose={() => setShowNewAppDialog(false)} />
     </a>
   )
-}
+})
 
 export default CreateAppCard

web/app/(commonLayout)/apps/NewAppDialog.tsx

@@ -21,10 +21,11 @@ import EmojiPicker from '@/app/components/base/emoji-picker'
 
 type NewAppDialogProps = {
   show: boolean
+  onSuccess?: () => void
   onClose?: () => void
 }
 
-const NewAppDialog = ({ show, onClose }: NewAppDialogProps) => {
+const NewAppDialog = ({ show, onSuccess, onClose }: NewAppDialogProps) => {
   const router = useRouter()
   const { notify } = useContext(ToastContext)
   const { t } = useTranslation()
@@ -79,6 +80,8 @@ const NewAppDialog = ({ show, onClose }: NewAppDialogProps) => {
         mode: isWithTemplate ? templates.data[selectedTemplateIndex].mode : newAppMode!,
         config: isWithTemplate ? templates.data[selectedTemplateIndex].model_config : undefined,
       })
+      if (onSuccess)
+        onSuccess()
       if (onClose)
         onClose()
       notify({ type: 'success', message: t('app.newApp.appCreated') })

web/app/(commonLayout)/datasets/Datasets.tsx

@@ -1,24 +1,49 @@
 'use client'
 
-import { useEffect } from 'react'
-import useSWR from 'swr'
-import { DataSet } from '@/models/datasets';
+import { useEffect, useRef } from 'react'
+import useSWRInfinite from 'swr/infinite'
+import { debounce } from 'lodash-es';
+import { DataSetListResponse } from '@/models/datasets';
 import NewDatasetCard from './NewDatasetCard'
 import DatasetCard from './DatasetCard';
 import { fetchDatasets } from '@/service/datasets';
+import { useSelector } from '@/context/app-context';
+
+const getKey = (pageIndex: number, previousPageData: DataSetListResponse) => {
+  if (!pageIndex || previousPageData.has_more)
+    return { url: 'datasets', params: { page: pageIndex + 1, limit: 30 } }
+  return null
+}
 
 const Datasets = () => {
-  // const { datasets, mutateDatasets } = useAppContext()
-  const { data: datasetList, mutate: mutateDatasets } = useSWR({ url: '/datasets', params: { page: 1 } }, fetchDatasets)
+  const { data, isLoading, setSize, mutate } = useSWRInfinite(getKey, fetchDatasets, { revalidateFirstPage: false })
+  const loadingStateRef = useRef(false)
+  const pageContainerRef = useSelector(state => state.pageContainerRef)
+  const anchorRef = useRef<HTMLAnchorElement>(null)
+
+  useEffect(() => {
+    loadingStateRef.current = isLoading
+  }, [isLoading])
 
   useEffect(() => {
-    mutateDatasets()
+    const onScroll = debounce(() => {
+      if (!loadingStateRef.current) {
+        const { scrollTop, clientHeight } = pageContainerRef.current!
+        const anchorOffset = anchorRef.current!.offsetTop
+        if (anchorOffset - scrollTop - clientHeight < 100) {
+          setSize(size => size + 1)
+        }
+      }
+    }, 50)
+
+    pageContainerRef.current?.addEventListener('scroll', onScroll)
+    return () => pageContainerRef.current?.removeEventListener('scroll', onScroll)
   }, [])
 
   return (
     <nav className='grid content-start grid-cols-1 gap-4 px-12 pt-8 sm:grid-cols-2 lg:grid-cols-4 grow shrink-0'>
-      {datasetList?.data.map(dataset => (<DatasetCard key={dataset.id} dataset={dataset} />))}
-      <NewDatasetCard />
+      {data?.map(({ data: datasets }) => datasets.map(dataset => (<DatasetCard key={dataset.id} dataset={dataset} />)))}
+      <NewDatasetCard ref={anchorRef} />
     </nav>
   )
 }

web/app/(commonLayout)/datasets/NewDatasetCard.tsx

@@ -1,16 +1,16 @@
 'use client'
 
-import { useState } from 'react'
+import { forwardRef, useState } from 'react'
 import classNames from 'classnames'
 import { useTranslation } from 'react-i18next'
 import style from '../list.module.css'
 
-const CreateAppCard = () => {
+const CreateAppCard = forwardRef<HTMLAnchorElement>((_, ref) => {
   const { t } = useTranslation()
   const [showNewAppDialog, setShowNewAppDialog] = useState(false)
 
   return (
-    <a className={classNames(style.listItem, style.newItemCard)} href='/datasets/create'>
+    <a ref={ref} className={classNames(style.listItem, style.newItemCard)} href='/datasets/create'>
       <div className={style.listItemTitle}>
         <span className={style.newItemIcon}>
           <span className={classNames(style.newItemIconImage, style.newItemIconAdd)} />
@@ -23,6 +23,6 @@ const CreateAppCard = () => {
       {/* <div className='text-xs text-gray-500'>{t('app.createFromConfigFile')}</div> */}
     </a>
   )
-}
+})
 
 export default CreateAppCard

web/app/components/app/chat/index.tsx

@@ -291,6 +291,7 @@ const Answer: FC<IAnswerProps> = ({ item, feedbackDisabled = false, isHideFeedba
             </div>
           }
         </div>
+        <div className={s.answerWrapWrap}>
           <div className={`${s.answerWrap} ${showEdit ? 'w-full' : ''}`}>
             <div className={`${s.answer} relative text-sm text-gray-900`}>
               <div className={'ml-2 py-3 px-4 bg-gray-100 rounded-tr-2xl rounded-b-2xl'}>
@@ -362,6 +363,7 @@ const Answer: FC<IAnswerProps> = ({ item, feedbackDisabled = false, isHideFeedba
           </div>
         </div>
       </div>
+    </div>
   )
 }
 
@@ -372,7 +374,7 @@ const Question: FC<IQuestionProps> = ({ id, content, more, useCurrentUserAvatar
   const userName = userProfile?.name
   return (
     <div className='flex items-start justify-end' key={id}>
-      <div>
+      <div className={s.questionWrapWrap}>
         <div className={`${s.question} relative text-sm text-gray-900`}>
           <div
             className={'mr-2 py-3 px-4 bg-blue-500 rounded-tl-2xl rounded-b-2xl'}

web/app/components/app/chat/style.module.css

@@ -42,6 +42,23 @@
   display: none;
 }
 
+.answerWrapWrap,
+.questionWrapWrap {
+  width: 0;
+  flex-grow: 1;
+}
+
+.questionWrapWrap {
+  display: flex;
+  justify-content: flex-end;
+}
+
+.answerWrap,
+.question {
+  display: inline-block;
+  max-width: 100%;
+}
+
 .answerWrap:hover .copyBtn {
   display: block;
 }

web/app/components/base/dialog/index.tsx

@@ -33,7 +33,7 @@ const CustomDialog = ({
   const close = useCallback(() => onClose?.(), [onClose])
   return (
     <Transition appear show={show} as={Fragment}>
-      <Dialog as="div" className="relative z-10" onClose={close}>
+      <Dialog as="div" className="relative z-40" onClose={close}>
         <Transition.Child
           as={Fragment}
           enter="ease-out duration-300"

web/app/components/base/emoji-picker/index.tsx

@@ -80,6 +80,7 @@ const EmojiPicker: FC<IEmojiPickerProps> = ({
     onClose={() => { }}
     isShow
     closable={false}
+    wrapperClassName='!z-40'
     className={cn(s.container, '!w-[362px] !p-0')}
   >
     <div className='flex flex-col items-center w-full p-3'>

web/app/components/base/modal/index.tsx

@@ -26,7 +26,7 @@ export default function Modal({
 }: IModal) {
   return (
     <Transition appear show={isShow} as={Fragment}>
-      <Dialog as="div" className="relative z-10" onClose={onClose}>
+      <Dialog as="div" className={`relative z-10 ${wrapperClassName}`} onClose={onClose}>
         <Transition.Child
           as={Fragment}
           enter="ease-out duration-300"

web/app/components/datasets/create/file-preview/index.module.css

@@ -43,4 +43,7 @@
     background: #f9fafb center no-repeat url(../assets/Loading.svg);
     background-size: contain;
   }
+  .fileContent {
+    white-space: pre-line;
+  }
   

web/app/components/datasets/create/file-uploader/index.tsx

@@ -190,13 +190,15 @@ const FileUploader = ({ file, onFileUpdate }: IFileUploaderProps) => {
         onChange={fileChangeHandle}
       />
       <div className={s.title}>{t('datasetCreation.stepOne.uploader.title')}</div>
+      <div ref={dropRef}>
         {!currentFile && !file && (
-        <div ref={dropRef} className={cn(s.uploader, dragging && s.dragging)}>
+          <div className={cn(s.uploader, dragging && s.dragging)}>
             <span>{t('datasetCreation.stepOne.uploader.button')}</span>
             <label className={s.browse} onClick={selectHandle}>{t('datasetCreation.stepOne.uploader.browse')}</label>
             {dragging && <div ref={dragRef} className={s.draggingCover}/>}
           </div>
         )}
+      </div>
       {currentFile && (
         <div className={cn(s.file, uploading && s.uploading)}>
           {uploading && (

web/app/components/datasets/create/step-two/preview-item/index.tsx

@@ -41,7 +41,7 @@ const PreviewItem: FC<IPreviewItemProps> = ({
         </div>
       </div>
       <div className='mt-2 max-h-[120px] line-clamp-6 overflow-hidden text-sm text-gray-800'>
-        {content}
+        <div style={{ whiteSpace: 'pre-line'}}>{content}</div>
       </div>
     </div>
   )

web/app/components/datasets/documents/detail/completed/style.module.css

@@ -45,6 +45,7 @@
 }
 .segModalContent {
   @apply h-96 text-gray-800 text-base overflow-y-scroll;
+  white-space: pre-line;
 }
 .footer {
   @apply flex items-center justify-between box-border border-t-gray-200 border-t-[0.5px] pt-3 mt-4;

web/app/styles/markdown.scss

@@ -54,6 +54,7 @@
   font-weight: 400;
   line-height: 1.5;
   word-wrap: break-word;
+  word-break: break-all;
   user-select: text;
 }
 
@@ -593,6 +594,7 @@
 
 .markdown-body table th {
   font-weight: var(--base-text-weight-semibold, 600);
+  white-space: nowrap;
 }
 
 .markdown-body table th,

web/context/app-context.ts

@@ -1,27 +0,0 @@
-'use client'
-
-import { createContext, useContext } from 'use-context-selector'
-import type { App } from '@/types/app'
-import type { UserProfileResponse } from '@/models/common'
-
-export type AppContextValue = {
-  apps: App[]
-  mutateApps: () => void
-  userProfile: UserProfileResponse
-  mutateUserProfile: () => void
-}
-
-const AppContext = createContext<AppContextValue>({
-  apps: [],
-  mutateApps: () => { },
-  userProfile: {
-    id: '',
-    name: '',
-    email: '',
-  },
-  mutateUserProfile: () => { },
-})
-
-export const useAppContext = () => useContext(AppContext)
-
-export default AppContext

web/context/app-context.tsx

@@ -0,0 +1,45 @@
+'use client'
+
+import { createContext, useContext, useContextSelector } from 'use-context-selector'
+import type { App } from '@/types/app'
+import type { UserProfileResponse } from '@/models/common'
+import { createRef, FC, PropsWithChildren } from 'react'
+
+export const useSelector = <T extends any>(selector: (value: AppContextValue) => T): T =>
+  useContextSelector(AppContext, selector);
+
+export type AppContextValue = {
+  apps: App[]
+  mutateApps: () => void
+  userProfile: UserProfileResponse
+  mutateUserProfile: () => void
+  pageContainerRef: React.RefObject<HTMLDivElement>,
+  useSelector: typeof useSelector,
+}
+
+const AppContext = createContext<AppContextValue>({
+  apps: [],
+  mutateApps: () => { },
+  userProfile: {
+    id: '',
+    name: '',
+    email: '',
+  },
+  mutateUserProfile: () => { },
+  pageContainerRef: createRef(),
+  useSelector,
+})
+
+export type AppContextProviderProps = PropsWithChildren<{
+  value: Omit<AppContextValue, 'useSelector'>
+}>
+
+export const AppContextProvider: FC<AppContextProviderProps> = ({ value, children }) => (
+  <AppContext.Provider value={{ ...value, useSelector }}>
+    {children}
+  </AppContext.Provider>
+)
+
+export const useAppContext = () => useContext(AppContext)
+
+export default AppContext

web/middleware.ts

@@ -23,7 +23,11 @@ export const getLocale = (request: NextRequest): Locale => {
   }
 
   // match locale
-  const matchedLocale = match(languages, locales, i18n.defaultLocale) as Locale
+  let matchedLocale:Locale  = i18n.defaultLocale
+  try {
+    // If languages is ['*'], Error would happen in match function.
+    matchedLocale = match(languages, locales, i18n.defaultLocale) as Locale
+  } catch(e) {}
   return matchedLocale
 }
 

web/models/app.ts

@@ -61,6 +61,10 @@ export type SiteConfig = {
 
 export type AppListResponse = {
   data: App[]
+  has_more: boolean
+  limit: number
+  page: number
+  total: number
 }
 
 export type AppDetailResponse = App

web/models/datasets.ts

@@ -29,6 +29,10 @@ export type File = {
 
 export type DataSetListResponse = {
   data: DataSet[]
+  has_more: boolean
+  limit: number
+  page: number
+  total: number
 }
 
 export type IndexingEstimateResponse = {

web/service/apps.ts

@@ -4,8 +4,8 @@ import type { ApikeysListResponse, AppDailyConversationsResponse, AppDailyEndUse
 import type { CommonResponse } from '@/models/common'
 import type { AppMode, ModelConfig } from '@/types/app'
 
-export const fetchAppList: Fetcher<AppListResponse, { params?: Record<string, any> }> = ({ params }) => {
-  return get('apps', params) as Promise<AppListResponse>
+export const fetchAppList: Fetcher<AppListResponse, { url: string; params?: Record<string, any> }> = ({ url, params }) => {
+  return get(url, { params }) as Promise<AppListResponse>
 }
 
 export const fetchAppDetail: Fetcher<AppDetailResponse, { url: string; id: string }> = ({ url, id }) => {