merge feat/plugins

api/core/rag/extractor/word_extractor.py

@@ -50,7 +50,7 @@ class WordExtractor(BaseExtractor):
 
             self.web_path = self.file_path
             # TODO: use a better way to handle the file
-            self.temp_file = tempfile.NamedTemporaryFile()  # noqa: SIM115
+            self.temp_file = tempfile.NamedTemporaryFile()
             self.temp_file.write(r.content)
             self.file_path = self.temp_file.name
         elif not os.path.isfile(self.file_path):

web/app/components/base/markdown.tsx

@@ -9,7 +9,6 @@ import RemarkGfm from 'remark-gfm'
 import RehypeRaw from 'rehype-raw'
 import SyntaxHighlighter from 'react-syntax-highlighter'
 import { atelierHeathLight } from 'react-syntax-highlighter/dist/esm/styles/hljs'
-import type { RefObject } from 'react'
 import { Component, createContext, memo, useContext, useEffect, useMemo, useRef, useState } from 'react'
 import cn from '@/utils/classnames'
 import CopyBtn from '@/app/components/base/copy-btn'
@@ -78,28 +77,6 @@ export function PreCode(props: { children: any }) {
   )
 }
 
-const useLazyLoad = (ref: RefObject<Element>): boolean => {
-  const [isIntersecting, setIntersecting] = useState<boolean>(false)
-
-  useEffect(() => {
-    const observer = new IntersectionObserver(([entry]) => {
-      if (entry.isIntersecting) {
-        setIntersecting(true)
-        observer.disconnect()
-      }
-    })
-
-    if (ref.current)
-      observer.observe(ref.current)
-
-    return () => {
-      observer.disconnect()
-    }
-  }, [ref])
-
-  return isIntersecting
-}
-
 const PreContext = createContext({
   // if children not in PreContext, just leave inline true
   inline: true,
@@ -138,7 +115,7 @@ const CodeBlock: Components['code'] = memo(({ className, children, ...props }) =
       try {
         return JSON.parse(String(children).replace(/\n$/, ''))
       }
-      catch { }
+      catch (error) { }
     }
     return JSON.parse('{"title":{"text":"ECharts error - Wrong JSON format."}}')
   }, [language, children])
@@ -167,7 +144,7 @@ const CodeBlock: Components['code'] = memo(({ className, children, ...props }) =
     else {
       return (
         <SyntaxHighlighter
-          {...props}
+          {...props as any}
           style={atelierHeathLight}
           customStyle={{
             paddingLeft: 12,

web/app/components/base/skeleton/index.tsx

@@ -3,7 +3,7 @@ import classNames from '@/utils/classnames'
 
 type SkeletonProps = ComponentProps<'div'>
 
-export const SkeletonContanier: FC<SkeletonProps> = (props) => {
+export const SkeletonContainer: FC<SkeletonProps> = (props) => {
   const { className, children, ...rest } = props
   return (
     <div className={classNames('flex flex-col gap-1', className)} {...rest}>
@@ -24,7 +24,7 @@ export const SkeletonRow: FC<SkeletonProps> = (props) => {
 export const SkeletonRectangle: FC<SkeletonProps> = (props) => {
   const { className, children, ...rest } = props
   return (
-    <div className={classNames('h-2 rounded-sm opacity-20 bg-text-tertiary my-1', className)} {...rest}>
+    <div className={classNames('h-2 rounded-sm opacity-20 bg-text-quaternary my-1', className)} {...rest}>
       {children}
     </div>
   )

web/app/components/header/account-setting/model-provider-page/index.tsx

@@ -102,11 +102,9 @@ const ModelProviderPage = ({ searchText }: Props) => {
       onSaveCallback: () => {
         updateModelProviders()
 
-        if (configurationMethod === ConfigurationMethodEnum.predefinedModel) {
-          provider.supported_model_types.forEach((type) => {
-            updateModelList(type)
-          })
-        }
+        provider.supported_model_types.forEach((type) => {
+          updateModelList(type)
+        })
 
         if (configurationMethod === ConfigurationMethodEnum.customizableModel && provider.custom_configuration.status === CustomConfigurationStatusEnum.active) {
           eventEmitter?.emit({
@@ -218,7 +216,7 @@ const ModelProviderPage = ({ searchText }: Props) => {
         </div>
         {!collapse && (isPluginsLoading || isAllPluginsLoading) && <Loading type='area' />}
         {
-          !isPluginsLoading && (
+          !isPluginsLoading && !collapse && (
             <List
               marketplaceCollections={marketplaceCollections || []}
               marketplaceCollectionPluginsMap={marketplaceCollectionPluginsMap || {}}
@@ -231,7 +229,7 @@ const ModelProviderPage = ({ searchText }: Props) => {
           )
         }
         {
-          !isAllPluginsLoading && (
+          !isAllPluginsLoading && !collapse && (
             <List
               marketplaceCollections={[]}
               marketplaceCollectionPluginsMap={{}}

web/app/components/header/account-setting/model-provider-page/model-parameter-modal/parameter-item.tsx

@@ -148,7 +148,7 @@ const ParameterItem: FC<ParameterItemProps> = ({
           />}
           <input
             ref={numberInputRef}
-            className='shrink-0 block ml-4 pl-3 w-16 h-8 appearance-none outline-none rounded-lg bg-gray-100 text-[13px] text-gra-900'
+            className='shrink-0 block ml-4 pl-3 w-16 h-8 appearance-none outline-none rounded-lg bg-components-input-bg-normal text-components-input-text-filled system-sm-regular'
             type='number'
             max={parameterRule.max}
             min={parameterRule.min}
@@ -173,7 +173,7 @@ const ParameterItem: FC<ParameterItemProps> = ({
           />}
           <input
             ref={numberInputRef}
-            className='shrink-0 block ml-4 pl-3 w-16 h-8 appearance-none outline-none rounded-lg bg-gray-100 text-[13px] text-gra-900'
+            className='shrink-0 block ml-4 pl-3 w-16 h-8 appearance-none outline-none rounded-lg bg-components-input-bg-normal text-components-input-text-filled system-sm-regular'
             type='number'
             max={parameterRule.max}
             min={parameterRule.min}
@@ -188,12 +188,12 @@ const ParameterItem: FC<ParameterItemProps> = ({
     if (parameterRule.type === 'boolean') {
       return (
         <Radio.Group
-          className='w-[200px] flex items-center'
+          className='w-[178px] flex items-center'
           value={renderValue ? 1 : 0}
           onChange={handleRadioChange}
         >
-          <Radio value={1} className='!mr-1 w-[94px]'>True</Radio>
-          <Radio value={0} className='w-[94px]'>False</Radio>
+          <Radio value={1} className='w-[83px]'>True</Radio>
+          <Radio value={0} className='w-[83px]'>False</Radio>
         </Radio.Group>
       )
     }
@@ -201,7 +201,7 @@ const ParameterItem: FC<ParameterItemProps> = ({
     if (parameterRule.type === 'string' && !parameterRule.options?.length) {
       return (
         <input
-          className={cn(isInWorkflow ? 'w-[200px]' : 'w-full', 'ml-4 flex items-center px-3 h-8 appearance-none outline-none rounded-lg bg-gray-100 text-[13px] text-gra-900')}
+          className={cn(isInWorkflow ? 'w-[178px]' : 'w-full', 'ml-4 flex items-center px-3 h-8 appearance-none outline-none rounded-lg bg-components-input-bg-normal text-components-input-text-filled system-sm-regular')}
           value={renderValue as string}
           onChange={handleStringInputChange}
         />
@@ -211,7 +211,7 @@ const ParameterItem: FC<ParameterItemProps> = ({
     if (parameterRule.type === 'text') {
       return (
         <textarea
-          className='w-full h-20 ml-4 px-1 rounded-lg bg-gray-100 outline-none text-[12px] text-gray-900'
+          className='w-full h-20 ml-4 px-1 rounded-lg bg-components-input-bg-normal text-components-input-text-filled system-sm-regular'
           value={renderValue as string}
           onChange={handleStringInputChange}
         />
@@ -222,7 +222,7 @@ const ParameterItem: FC<ParameterItemProps> = ({
       return (
         <SimpleSelect
           className='!py-0'
-          wrapperClassName={cn(isInWorkflow ? '!w-[200px]' : 'w-full', 'ml-4 !h-8')}
+          wrapperClassName={cn('w-full !h-8')}
           defaultValue={renderValue as string}
           onSelect={handleSelect}
           items={parameterRule.options.map(option => ({ value: option, name: option }))}
@@ -232,7 +232,7 @@ const ParameterItem: FC<ParameterItemProps> = ({
 
     if (parameterRule.type === 'tag') {
       return (
-        <div className={cn(isInWorkflow ? 'w-[200px]' : 'w-full', 'ml-4')}>
+        <div className={cn('w-full !h-8')}>
           <TagInput
             items={renderValue as string[]}
             onChange={handleTagChange}
@@ -262,7 +262,7 @@ const ParameterItem: FC<ParameterItemProps> = ({
             )
           }
           <div
-            className='mr-0.5 text-[13px] font-medium text-gray-700 truncate'
+            className='mr-0.5 system-xs-regular text-text-secondary truncate'
             title={parameterRule.label[language] || parameterRule.label.en_US}
           >
             {parameterRule.label[language] || parameterRule.label.en_US}
@@ -271,7 +271,7 @@ const ParameterItem: FC<ParameterItemProps> = ({
             parameterRule.help && (
               <Tooltip
                 popupContent={(
-                  <div className='w-[200px] whitespace-pre-wrap'>{parameterRule.help[language] || parameterRule.help.en_US}</div>
+                  <div className='w-[178px] whitespace-pre-wrap'>{parameterRule.help[language] || parameterRule.help.en_US}</div>
                 )}
                 popupClassName='mr-1'
                 triggerClassName='mr-1 w-4 h-4 shrink-0'
@@ -281,7 +281,7 @@ const ParameterItem: FC<ParameterItemProps> = ({
         </div>
         {
           parameterRule.type === 'tag' && (
-            <div className={cn(!isInWorkflow && 'w-[200px]', 'text-gray-400 text-xs font-normal')}>
+            <div className={cn(!isInWorkflow && 'w-[178px]', 'text-text-tertiary system-xs-regular')}>
               {parameterRule?.tagPlaceholder?.[language]}
             </div>
           )

web/app/components/plugins/card/base/placeholder.tsx

@@ -1,5 +1,6 @@
 import { Group } from '../../../base/icons/src/vender/other'
 import Title from './title'
+import { SkeletonContainer, SkeletonPoint, SkeletonRectangle, SkeletonRow } from '@/app/components/base/skeleton'
 import cn from '@/utils/classnames'
 
 type Props = {
@@ -17,7 +18,7 @@ const Placeholder = ({
 }: Props) => {
   return (
     <div className={wrapClassName}>
-      <div className="flex">
+      <SkeletonRow>
         <div
           className='flex w-10 h-10 p-1 justify-center items-center gap-2 rounded-[10px]
               border-[0.5px] border-components-panel-border bg-background-default backdrop-blur-sm'>
@@ -25,24 +26,24 @@ const Placeholder = ({
             <Group className='text-text-tertiary' />
           </div>
         </div>
-        <div className="ml-3 grow">
-          <div className="flex items-center h-5">
-            {loadingFileName ? (
-              <Title title={loadingFileName} />
-            ) : (
-              <LoadingPlaceholder className="w-[260px]" />
-            )}
-          </div>
-          <div className={cn('flex items-center h-4 space-x-0.5')}>
-            <LoadingPlaceholder className="w-[41px]" />
-            <span className='shrink-0 text-text-quaternary system-xs-regular'>
-              ·
-            </span>
-            <LoadingPlaceholder className="w-[180px]" />
-          </div>
+        <div className="grow">
+          <SkeletonContainer>
+            <div className="flex items-center h-5">
+              {loadingFileName ? (
+                <Title title={loadingFileName} />
+              ) : (
+                <SkeletonRectangle className="w-[260px]" />
+              )}
+            </div>
+            <SkeletonRow className="h-4">
+              <SkeletonRectangle className="w-[41px]" />
+              <SkeletonPoint />
+              <SkeletonRectangle className="w-[180px]" />
+            </SkeletonRow>
+          </SkeletonContainer>
         </div>
-      </div>
-      <LoadingPlaceholder className="mt-3 w-[420px]" />
+      </SkeletonRow>
+      <SkeletonRectangle className="mt-3 w-[420px]" />
     </div>
   )
 }

web/app/components/plugins/marketplace/empty/index.tsx

@@ -29,20 +29,24 @@ const Empty = ({
               'mr-3 mb-3  h-[144px] w-[calc((100%-36px)/4)] rounded-xl bg-background-section-burn',
               index % 4 === 3 && 'mr-0',
               index > 11 && 'mb-0',
-              lightCard && 'bg-background-default-lighter',
+              lightCard && 'bg-background-default-lighter opacity-75',
             )}
           >
           </div>
         ))
       }
-      <div
-        className='absolute inset-0 bg-marketplace-plugin-empty z-[1]'
-      ></div>
+      {
+        !lightCard && (
+          <div
+            className='absolute inset-0 bg-marketplace-plugin-empty z-[1]'
+          ></div>
+        )
+      }
       <div className='absolute top-1/2 left-1/2 -translate-x-1/2 -translate-y-1/2 z-[2] flex flex-col items-center'>
         <div className='relative flex items-center justify-center mb-3 w-14 h-14 rounded-xl border border-dashed border-divider-deep bg-components-card-bg shadow-lg'>
           <Group className='w-5 h-5' />
-          <Line className='absolute -right-[1px] top-1/2 -translate-y-1/2' />
-          <Line className='absolute -left-[1px] top-1/2 -translate-y-1/2' />
+          <Line className='absolute right-[-1px] top-1/2 -translate-y-1/2' />
+          <Line className='absolute left-[-1px] top-1/2 -translate-y-1/2' />
           <Line className='absolute top-0 left-1/2 -translate-x-1/2 -translate-y-1/2 rotate-90' />
           <Line className='absolute top-full left-1/2 -translate-x-1/2 -translate-y-1/2 rotate-90' />
         </div>

web/app/components/plugins/marketplace/list/index.tsx

@@ -14,6 +14,7 @@ type ListProps = {
   locale: string
   cardContainerClassName?: string
   cardRender?: (plugin: Plugin) => JSX.Element | null
+  onMoreClick?: () => void
 }
 const List = ({
   marketplaceCollections,
@@ -23,6 +24,7 @@ const List = ({
   locale,
   cardContainerClassName,
   cardRender,
+  onMoreClick,
 }: ListProps) => {
   return (
     <>
@@ -35,6 +37,7 @@ const List = ({
             locale={locale}
             cardContainerClassName={cardContainerClassName}
             cardRender={cardRender}
+            onMoreClick={onMoreClick}
           />
         )
       }

web/app/components/plugins/marketplace/list/list-with-collection.tsx

@@ -12,6 +12,7 @@ type ListWithCollectionProps = {
   locale: string
   cardContainerClassName?: string
   cardRender?: (plugin: Plugin) => JSX.Element | null
+  onMoreClick?: () => void
 }
 const ListWithCollection = ({
   marketplaceCollections,
@@ -20,6 +21,7 @@ const ListWithCollection = ({
   locale,
   cardContainerClassName,
   cardRender,
+  // onMoreClick,
 }: ListWithCollectionProps) => {
   return (
     <>
@@ -29,8 +31,16 @@ const ListWithCollection = ({
             key={collection.name}
             className='py-3'
           >
-            <div className='title-xl-semi-bold text-text-primary'>{collection.label[getLanguage(locale)]}</div>
-            <div className='system-xs-regular text-text-tertiary'>{collection.description[getLanguage(locale)]}</div>
+            <div className='flex justify-between'>
+              <div>
+                <div className='title-xl-semi-bold text-text-primary'>{collection.label[getLanguage(locale)]}</div>
+                <div className='system-xs-regular text-text-tertiary'>{collection.description[getLanguage(locale)]}</div>
+              </div>
+              {/* <div
+                className='system-xs-regular text-text-tertiary cursor-pointer hover:underline'
+                onClick={() => onMoreClick?.()}
+              >more</div> */}
+            </div>
             <div className={cn(
               'grid grid-cols-4 gap-3 mt-2',
               cardContainerClassName,

web/app/components/tools/marketplace/hooks.ts

@@ -1,6 +1,9 @@
 import {
+  useCallback,
   useEffect,
   useMemo,
+  useRef,
+  useState,
 } from 'react'
 import {
   useMarketplaceCollectionsAndPlugins,
@@ -28,10 +31,22 @@ export const useMarketplace = (searchPluginText: string, filterPluginTags: strin
     queryPlugins,
     queryPluginsWithDebounced,
     isLoading: isPluginsLoading,
+    total: pluginsTotal,
   } = useMarketplacePlugins()
+  const [page, setPage] = useState(1)
+  const pageRef = useRef(page)
+  const searchPluginTextRef = useRef(searchPluginText)
+  const filterPluginTagsRef = useRef(filterPluginTags)
 
+  useEffect(() => {
+    searchPluginTextRef.current = searchPluginText
+    filterPluginTagsRef.current = filterPluginTags
+  }, [searchPluginText, filterPluginTags])
   useEffect(() => {
     if ((searchPluginText || filterPluginTags.length) && isSuccess) {
+      setPage(1)
+      pageRef.current = 1
+
       if (searchPluginText) {
         queryPluginsWithDebounced({
           category: PluginType.tool,
@@ -39,6 +54,7 @@ export const useMarketplace = (searchPluginText: string, filterPluginTags: strin
           tags: filterPluginTags,
           exclude,
           type: 'plugin',
+          page: pageRef.current,
         })
         return
       }
@@ -48,6 +64,7 @@ export const useMarketplace = (searchPluginText: string, filterPluginTags: strin
         tags: filterPluginTags,
         exclude,
         type: 'plugin',
+        page: pageRef.current,
       })
     }
     else {
@@ -63,10 +80,38 @@ export const useMarketplace = (searchPluginText: string, filterPluginTags: strin
     }
   }, [searchPluginText, filterPluginTags, queryPlugins, queryMarketplaceCollectionsAndPlugins, queryPluginsWithDebounced, resetPlugins, exclude, isSuccess])
 
+  const handleScroll = useCallback((e: Event) => {
+    const target = e.target as HTMLDivElement
+    const {
+      scrollTop,
+      scrollHeight,
+      clientHeight,
+    } = target
+    if (scrollTop + clientHeight >= scrollHeight - 5 && scrollTop > 0) {
+      const searchPluginText = searchPluginTextRef.current
+      const filterPluginTags = filterPluginTagsRef.current
+      if (pluginsTotal && plugins && pluginsTotal > plugins.length && (!!searchPluginText || !!filterPluginTags.length)) {
+        setPage(pageRef.current + 1)
+        pageRef.current++
+
+        queryPlugins({
+          category: PluginType.tool,
+          query: searchPluginText,
+          tags: filterPluginTags,
+          exclude,
+          type: 'plugin',
+          page: pageRef.current,
+        })
+      }
+    }
+  }, [exclude, plugins, pluginsTotal, queryPlugins])
+
   return {
     isLoading: isLoading || isPluginsLoading,
     marketplaceCollections,
     marketplaceCollectionPluginsMap,
     plugins,
+    handleScroll,
+    page,
   }
 }

web/app/components/tools/marketplace/index.tsx

@@ -1,3 +1,7 @@
+import {
+  useEffect,
+  useRef,
+} from 'react'
 import {
   RiArrowRightUpLine,
   RiArrowUpDoubleLine,
@@ -21,15 +25,33 @@ const Marketplace = ({
 }: MarketplaceProps) => {
   const locale = getLocaleOnClient()
   const { t } = useTranslation()
+
   const {
     isLoading,
     marketplaceCollections,
     marketplaceCollectionPluginsMap,
     plugins,
+    handleScroll,
+    page,
   } = useMarketplace(searchPluginText, filterPluginTags)
+  const containerRef = useRef<HTMLDivElement>(null)
+
+  useEffect(() => {
+    const container = containerRef.current
+    if (container)
+      container.addEventListener('scroll', handleScroll)
+
+    return () => {
+      if (container)
+        container.removeEventListener('scroll', handleScroll)
+    }
+  }, [handleScroll])
 
   return (
-    <div className='flex flex-col shrink-0 sticky bottom-[-442px] h-[530px] overflow-y-auto px-12 py-2 pt-0 bg-background-default-subtle'>
+    <div
+      ref={containerRef}
+      className='grow flex flex-col shrink-0 sticky bottom-[-442px] h-[530px] overflow-y-auto px-12 py-2 pt-0 bg-background-default-subtle'
+    >
       <RiArrowUpDoubleLine
         className='absolute top-2 left-1/2 -translate-x-1/2 w-4 h-4 text-text-quaternary cursor-pointer'
         onClick={() => onMarketplaceScroll()}
@@ -67,14 +89,14 @@ const Marketplace = ({
         </div>
       </div>
       {
-        isLoading && (
+        isLoading && page === 1 && (
           <div className='absolute top-1/2 left-1/2 -translate-x-1/2 -translate-y-1/2'>
             <Loading />
           </div>
         )
       }
       {
-        !isLoading && (
+        (!isLoading || page > 1) && (
           <List
             marketplaceCollections={marketplaceCollections || []}
             marketplaceCollectionPluginsMap={marketplaceCollectionPluginsMap || {}}

web/app/components/tools/provider-list.tsx

@@ -95,7 +95,7 @@ const ProviderList = () => {
           </div>
           {(filteredCollectionList.length > 0 || activeTab !== 'builtin') && (
             <div className={cn(
-              'relative grid content-start grid-cols-1 gap-4 px-12 pt-2 pb-4 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4 grow shrink-0',
+              'relative grid content-start grid-cols-1 gap-4 px-12 pt-2 pb-4 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4 shrink-0',
             )}>
               {activeTab === 'api' && <CustomCreateCard onRefreshData={refetch} />}
               {filteredCollectionList.map(collection => (
@@ -125,7 +125,7 @@ const ProviderList = () => {
             </div>
           )}
           {!filteredCollectionList.length && activeTab === 'builtin' && (
-            <Empty lightCard text={t('tools.noTools')} className='px-12' />
+            <Empty lightCard text={t('tools.noTools')} className='px-12 h-[224px]' />
           )}
           {
             enable_marketplace && activeTab === 'builtin' && (

web/package.json

@@ -54,7 +54,8 @@
     "dayjs": "^1.11.13",
     "echarts": "^5.5.1",
     "echarts-for-react": "^3.0.2",
-    "emoji-mart": "^5.6.0",
+    "elkjs": "^0.9.3",
+    "emoji-mart": "^5.5.2",
     "fast-deep-equal": "^3.1.3",
     "globals": "^15.11.0",
     "i18next": "^23.16.4",
@@ -155,7 +156,7 @@
     "@types/uuid": "^10.0.0",
     "autoprefixer": "^10.4.20",
     "bing-translate-api": "^4.0.2",
-    "code-inspector-plugin": "^0.17.4",
+    "code-inspector-plugin": "^0.18.1",
     "cross-env": "^7.0.3",
     "eslint": "^9.13.0",
     "eslint-config-next": "^15.0.0",