feat: support firecrawl frontend code (#5226)
Joel
2 years ago
committed by
GitHub
parent
8d1386df0f
commit
28554350de
@ -0,0 +1,5 @@
|
||||
<svg width="624" height="48" viewBox="0 0 624 48" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<rect x="8" y="7" width="16" height="16" rx="5" fill="#F2F4F7"/>
|
||||
<rect x="32" y="10" width="233" height="10" rx="3" fill="#EAECF0"/>
|
||||
<rect x="32" y="31" width="345" height="6" rx="3" fill="#F2F4F7"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 305 B |
@ -0,0 +1,5 @@
|
||||
<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<g id="Icon-3-dots">
|
||||
<path id="Icon" d="M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103" stroke="#374151" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 285 B |
@ -0,0 +1,56 @@
|
||||
{
|
||||
"icon": {
|
||||
"type": "element",
|
||||
"isRootNode": true,
|
||||
"name": "svg",
|
||||
"attributes": {
|
||||
"width": "624",
|
||||
"height": "48",
|
||||
"viewBox": "0 0 624 48",
|
||||
"fill": "none",
|
||||
"xmlns": "http://www.w3.org/2000/svg"
|
||||
},
|
||||
"children": [
|
||||
{
|
||||
"type": "element",
|
||||
"name": "rect",
|
||||
"attributes": {
|
||||
"x": "8",
|
||||
"y": "7",
|
||||
"width": "16",
|
||||
"height": "16",
|
||||
"rx": "5",
|
||||
"fill": "#F2F4F7"
|
||||
},
|
||||
"children": []
|
||||
},
|
||||
{
|
||||
"type": "element",
|
||||
"name": "rect",
|
||||
"attributes": {
|
||||
"x": "32",
|
||||
"y": "10",
|
||||
"width": "233",
|
||||
"height": "10",
|
||||
"rx": "3",
|
||||
"fill": "#EAECF0"
|
||||
},
|
||||
"children": []
|
||||
},
|
||||
{
|
||||
"type": "element",
|
||||
"name": "rect",
|
||||
"attributes": {
|
||||
"x": "32",
|
||||
"y": "31",
|
||||
"width": "345",
|
||||
"height": "6",
|
||||
"rx": "3",
|
||||
"fill": "#F2F4F7"
|
||||
},
|
||||
"children": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"name": "RowStruct"
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
// GENERATE BY script
|
||||
// DON NOT EDIT IT MANUALLY
|
||||
|
||||
import * as React from 'react'
|
||||
import data from './RowStruct.json'
|
||||
import IconBase from '@/app/components/base/icons/IconBase'
|
||||
import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase'
|
||||
|
||||
const Icon = React.forwardRef<React.MutableRefObject<SVGElement>, Omit<IconBaseProps, 'data'>>((
|
||||
props,
|
||||
ref,
|
||||
) => <IconBase {...props} ref={ref} data={data as IconData} />)
|
||||
|
||||
Icon.displayName = 'RowStruct'
|
||||
|
||||
export default Icon
|
||||
@ -1,2 +1,3 @@
|
||||
export { default as Icon3Dots } from './Icon3Dots'
|
||||
export { default as DefaultToolIcon } from './DefaultToolIcon'
|
||||
export { default as RowStruct } from './RowStruct'
|
||||
|
||||
@ -0,0 +1,39 @@
|
||||
{
|
||||
"icon": {
|
||||
"type": "element",
|
||||
"isRootNode": true,
|
||||
"name": "svg",
|
||||
"attributes": {
|
||||
"width": "16",
|
||||
"height": "16",
|
||||
"viewBox": "0 0 16 16",
|
||||
"fill": "none",
|
||||
"xmlns": "http://www.w3.org/2000/svg"
|
||||
},
|
||||
"children": [
|
||||
{
|
||||
"type": "element",
|
||||
"name": "g",
|
||||
"attributes": {
|
||||
"id": "Icon-3-dots"
|
||||
},
|
||||
"children": [
|
||||
{
|
||||
"type": "element",
|
||||
"name": "path",
|
||||
"attributes": {
|
||||
"id": "Icon",
|
||||
"d": "M5 6.5V5M8.93934 7.56066L10 6.5M10.0103 11.5H11.5103",
|
||||
"stroke": "currentColor",
|
||||
"stroke-width": "2",
|
||||
"stroke-linecap": "round",
|
||||
"stroke-linejoin": "round"
|
||||
},
|
||||
"children": []
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"name": "Icon3Dots"
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
// GENERATE BY script
|
||||
// DON NOT EDIT IT MANUALLY
|
||||
|
||||
import * as React from 'react'
|
||||
import data from './Icon3Dots.json'
|
||||
import IconBase from '@/app/components/base/icons/IconBase'
|
||||
import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase'
|
||||
|
||||
const Icon = React.forwardRef<React.MutableRefObject<SVGElement>, Omit<IconBaseProps, 'data'>>((
|
||||
props,
|
||||
ref,
|
||||
) => <IconBase {...props} ref={ref} data={data as IconData} />)
|
||||
|
||||
Icon.displayName = 'Icon3Dots'
|
||||
|
||||
export default Icon
|
||||
@ -0,0 +1,29 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import cn from 'classnames'
|
||||
import Checkbox from '@/app/components/base/checkbox'
|
||||
|
||||
type Props = {
|
||||
className?: string
|
||||
isChecked: boolean
|
||||
onChange: (isChecked: boolean) => void
|
||||
label: string
|
||||
labelClassName?: string
|
||||
}
|
||||
|
||||
const CheckboxWithLabel: FC<Props> = ({
|
||||
className = '',
|
||||
isChecked,
|
||||
onChange,
|
||||
label,
|
||||
labelClassName,
|
||||
}) => {
|
||||
return (
|
||||
<label className={cn(className, 'flex items-center h-7 space-x-2')}>
|
||||
<Checkbox checked={isChecked} onCheck={() => onChange(!isChecked)} />
|
||||
<div className={cn(labelClassName, 'text-sm font-normal text-gray-800')}>{label}</div>
|
||||
</label>
|
||||
)
|
||||
}
|
||||
export default React.memo(CheckboxWithLabel)
|
||||
@ -0,0 +1,30 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import cn from 'classnames'
|
||||
import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback'
|
||||
|
||||
type Props = {
|
||||
className?: string
|
||||
title: string
|
||||
errorMsg?: string
|
||||
}
|
||||
|
||||
const ErrorMessage: FC<Props> = ({
|
||||
className,
|
||||
title,
|
||||
errorMsg,
|
||||
}) => {
|
||||
return (
|
||||
<div className={cn(className, 'py-2 px-4 border-t border-gray-200 bg-[#FFFAEB]')}>
|
||||
<div className='flex items-center h-5'>
|
||||
<AlertTriangle className='mr-2 w-4 h-4 text-[#F79009]' />
|
||||
<div className='text-sm font-medium text-[#DC6803]'>{title}</div>
|
||||
</div>
|
||||
{errorMsg && (
|
||||
<div className='mt-1 pl-6 leading-[18px] text-xs font-normal text-gray-700'>{errorMsg}</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(ErrorMessage)
|
||||
@ -0,0 +1,54 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import cn from 'classnames'
|
||||
import Input from './input'
|
||||
import TooltipPlus from '@/app/components/base/tooltip-plus'
|
||||
import { HelpCircle } from '@/app/components/base/icons/src/vender/line/general'
|
||||
|
||||
type Props = {
|
||||
className?: string
|
||||
label: string
|
||||
labelClassName?: string
|
||||
value: string | number
|
||||
onChange: (value: string | number) => void
|
||||
isRequired?: boolean
|
||||
placeholder?: string
|
||||
isNumber?: boolean
|
||||
tooltip?: string
|
||||
}
|
||||
|
||||
const Field: FC<Props> = ({
|
||||
className,
|
||||
label,
|
||||
labelClassName,
|
||||
value,
|
||||
onChange,
|
||||
isRequired = false,
|
||||
placeholder = '',
|
||||
isNumber = false,
|
||||
tooltip,
|
||||
}) => {
|
||||
return (
|
||||
<div className={cn(className)}>
|
||||
<div className='flex py-[7px]'>
|
||||
<div className={cn(labelClassName, 'flex items-center h-[18px] text-[13px] font-medium text-gray-900')}>{label} </div>
|
||||
{isRequired && <span className='ml-0.5 text-xs font-semibold text-[#D92D20]'>*</span>}
|
||||
{tooltip && (
|
||||
<TooltipPlus popupContent={
|
||||
<div className='w-[200px]'>{tooltip}</div>
|
||||
}>
|
||||
<HelpCircle className='relative top-[3px] w-3 h-3 ml-1 text-gray-500' />
|
||||
</TooltipPlus>
|
||||
)}
|
||||
</div>
|
||||
<Input
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
placeholder={placeholder}
|
||||
isNumber={isNumber}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Field)
|
||||
@ -0,0 +1,58 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback } from 'react'
|
||||
|
||||
type Props = {
|
||||
value: string | number
|
||||
onChange: (value: string | number) => void
|
||||
placeholder?: string
|
||||
isNumber?: boolean
|
||||
}
|
||||
|
||||
const MIN_VALUE = 1
|
||||
|
||||
const Input: FC<Props> = ({
|
||||
value,
|
||||
onChange,
|
||||
placeholder = '',
|
||||
isNumber = false,
|
||||
}) => {
|
||||
const handleChange = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const value = e.target.value
|
||||
if (isNumber) {
|
||||
let numberValue = parseInt(value, 10) // integer only
|
||||
if (isNaN(numberValue)) {
|
||||
onChange('')
|
||||
return
|
||||
}
|
||||
if (numberValue < MIN_VALUE)
|
||||
numberValue = MIN_VALUE
|
||||
|
||||
onChange(numberValue)
|
||||
return
|
||||
}
|
||||
onChange(value)
|
||||
}, [isNumber, onChange])
|
||||
|
||||
const otherOption = (() => {
|
||||
if (isNumber) {
|
||||
return {
|
||||
min: MIN_VALUE,
|
||||
}
|
||||
}
|
||||
return {
|
||||
|
||||
}
|
||||
})()
|
||||
return (
|
||||
<input
|
||||
type={isNumber ? 'number' : 'text'}
|
||||
{...otherOption}
|
||||
value={value}
|
||||
onChange={handleChange}
|
||||
className='flex h-9 w-full py-1 px-2 rounded-lg text-xs leading-normal bg-gray-100 caret-primary-600 hover:bg-gray-100 focus:ring-1 focus:ring-inset focus:ring-gray-200 focus-visible:outline-none focus:bg-gray-50 placeholder:text-gray-400'
|
||||
placeholder={placeholder}
|
||||
/>
|
||||
)
|
||||
}
|
||||
export default React.memo(Input)
|
||||
@ -0,0 +1,55 @@
|
||||
'use client'
|
||||
import { useBoolean } from 'ahooks'
|
||||
import type { FC } from 'react'
|
||||
import React, { useEffect } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from 'classnames'
|
||||
import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
|
||||
import { ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
className?: string
|
||||
children: React.ReactNode
|
||||
controlFoldOptions?: number
|
||||
}
|
||||
|
||||
const OptionsWrap: FC<Props> = ({
|
||||
className = '',
|
||||
children,
|
||||
controlFoldOptions,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const [fold, {
|
||||
toggle: foldToggle,
|
||||
setTrue: foldHide,
|
||||
}] = useBoolean(false)
|
||||
|
||||
useEffect(() => {
|
||||
if (controlFoldOptions)
|
||||
foldHide()
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [controlFoldOptions])
|
||||
return (
|
||||
<div className={cn(className, !fold ? 'mb-0' : 'mb-3')}>
|
||||
<div
|
||||
className='flex justify-between items-center h-[26px] py-1 cursor-pointer select-none'
|
||||
onClick={foldToggle}
|
||||
>
|
||||
<div className='flex items-center text-gray-700'>
|
||||
<Settings04 className='mr-1 w-4 h-4' />
|
||||
<div className='text-[13px] font-semibold text-gray-800 uppercase'>{t(`${I18N_PREFIX}.options`)}</div>
|
||||
</div>
|
||||
<ChevronRight className={cn(!fold && 'rotate-90', 'w-4 h-4 text-gray-500')} />
|
||||
</div>
|
||||
{!fold && (
|
||||
<div className='mb-4'>
|
||||
{children}
|
||||
</div>
|
||||
)}
|
||||
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(OptionsWrap)
|
||||
@ -0,0 +1,48 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Input from './input'
|
||||
import Button from '@/app/components/base/button'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
isRunning: boolean
|
||||
onRun: (url: string) => void
|
||||
}
|
||||
|
||||
const UrlInput: FC<Props> = ({
|
||||
isRunning,
|
||||
onRun,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
const [url, setUrl] = useState('')
|
||||
const handleUrlChange = useCallback((url: string | number) => {
|
||||
setUrl(url as string)
|
||||
}, [])
|
||||
const handleOnRun = useCallback(() => {
|
||||
if (isRunning)
|
||||
return
|
||||
onRun(url)
|
||||
}, [isRunning, onRun, url])
|
||||
|
||||
return (
|
||||
<div className='flex items-center justify-between'>
|
||||
<Input
|
||||
value={url}
|
||||
onChange={handleUrlChange}
|
||||
placeholder='https://docs.dify.ai'
|
||||
/>
|
||||
<Button
|
||||
type='primary'
|
||||
onClick={handleOnRun}
|
||||
className='ml-2 !h-8 text-[13px] font-medium'
|
||||
loading={isRunning}
|
||||
>
|
||||
{!isRunning ? t(`${I18N_PREFIX}.run`) : ''}
|
||||
</Button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(UrlInput)
|
||||
@ -0,0 +1,40 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from 'classnames'
|
||||
import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
|
||||
import Checkbox from '@/app/components/base/checkbox'
|
||||
|
||||
type Props = {
|
||||
payload: CrawlResultItemType
|
||||
isChecked: boolean
|
||||
isPreview: boolean
|
||||
onCheckChange: (checked: boolean) => void
|
||||
onPreview: () => void
|
||||
}
|
||||
|
||||
const CrawledResultItem: FC<Props> = ({
|
||||
isPreview,
|
||||
payload,
|
||||
isChecked,
|
||||
onCheckChange,
|
||||
onPreview,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const handleCheckChange = useCallback(() => {
|
||||
onCheckChange(!isChecked)
|
||||
}, [isChecked, onCheckChange])
|
||||
return (
|
||||
<div className={cn(isPreview ? 'border-[#D1E0FF] bg-primary-50 shadow-xs' : 'group hover:bg-gray-100', 'rounded-md px-2 py-[5px] cursor-pointer border border-transparent')}>
|
||||
<div className='flex items-center h-5'>
|
||||
<Checkbox className='group-hover:border-2 group-hover:border-primary-600 mr-2 shrink-0' checked={isChecked} onCheck={handleCheckChange} />
|
||||
<div className='grow w-0 truncate text-sm font-medium text-gray-700' title={payload.title}>{payload.title}</div>
|
||||
<div onClick={onPreview} className='hidden group-hover:flex items-center h-6 px-2 text-xs rounded-md font-medium text-gray-500 uppercase hover:bg-gray-50'>{t('datasetCreation.stepOne.website.preview')}</div>
|
||||
</div>
|
||||
<div className='mt-0.5 truncate pl-6 leading-[18px] text-xs font-normal text-gray-500' title={payload.source_url}>{payload.source_url}</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(CrawledResultItem)
|
||||
@ -0,0 +1,87 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from 'classnames'
|
||||
import CheckboxWithLabel from './base/checkbox-with-label'
|
||||
import CrawledResultItem from './crawled-result-item'
|
||||
import type { CrawlResultItem } from '@/models/datasets'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
className?: string
|
||||
list: CrawlResultItem[]
|
||||
checkedList: CrawlResultItem[]
|
||||
onSelectedChange: (selected: CrawlResultItem[]) => void
|
||||
onPreview: (payload: CrawlResultItem) => void
|
||||
usedTime: number
|
||||
}
|
||||
|
||||
const CrawledResult: FC<Props> = ({
|
||||
className = '',
|
||||
list,
|
||||
checkedList,
|
||||
onSelectedChange,
|
||||
onPreview,
|
||||
usedTime,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const isCheckAll = checkedList.length === list.length
|
||||
|
||||
const handleCheckedAll = useCallback(() => {
|
||||
if (!isCheckAll)
|
||||
onSelectedChange(list)
|
||||
|
||||
else
|
||||
onSelectedChange([])
|
||||
}, [isCheckAll, list, onSelectedChange])
|
||||
|
||||
const handleItemCheckChange = useCallback((item: CrawlResultItem) => {
|
||||
return (checked: boolean) => {
|
||||
if (checked)
|
||||
onSelectedChange([...checkedList, item])
|
||||
|
||||
else
|
||||
onSelectedChange(checkedList.filter(checkedItem => checkedItem.source_url !== item.source_url))
|
||||
}
|
||||
}, [checkedList, onSelectedChange])
|
||||
|
||||
const [previewIndex, setPreviewIndex] = React.useState<number>(-1)
|
||||
const handlePreview = useCallback((index: number) => {
|
||||
return () => {
|
||||
setPreviewIndex(index)
|
||||
onPreview(list[index])
|
||||
}
|
||||
}, [list, onPreview])
|
||||
|
||||
return (
|
||||
<div className={cn(className, 'border-t border-gray-200')}>
|
||||
<div className='flex items-center justify-between h-[34px] px-4 bg-gray-50 shadow-xs border-b-[0.5px] border-black/8 text-xs font-normal text-gray-700'>
|
||||
<CheckboxWithLabel
|
||||
isChecked={isCheckAll}
|
||||
onChange={handleCheckedAll} label={isCheckAll ? t(`${I18N_PREFIX}.resetAll`) : t(`${I18N_PREFIX}.selectAll`)}
|
||||
labelClassName='!font-medium'
|
||||
/>
|
||||
<div>{t(`${I18N_PREFIX}.scrapTimeInfo`, {
|
||||
total: list.length,
|
||||
time: usedTime.toFixed(1),
|
||||
})}</div>
|
||||
</div>
|
||||
<div className='p-2'>
|
||||
{list.map((item, index) => (
|
||||
<CrawledResultItem
|
||||
key={item.source_url}
|
||||
isPreview={index === previewIndex}
|
||||
onPreview={handlePreview(index)}
|
||||
payload={item}
|
||||
isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)}
|
||||
onCheckChange={handleItemCheckChange(item)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(CrawledResult)
|
||||
@ -0,0 +1,37 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import cn from 'classnames'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { RowStruct } from '@/app/components/base/icons/src/public/other'
|
||||
|
||||
type Props = {
|
||||
className?: string
|
||||
crawledNum: number
|
||||
totalNum: number
|
||||
}
|
||||
|
||||
const Crawling: FC<Props> = ({
|
||||
className = '',
|
||||
crawledNum,
|
||||
totalNum,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className={cn(className, 'border-t border-gray-200')}>
|
||||
<div className='flex items-center h-[34px] px-4 bg-gray-50 shadow-xs border-b-[0.5px] border-black/8 text-xs font-normal text-gray-700'>
|
||||
{t('datasetCreation.stepOne.website.totalPageScraped')} {crawledNum}/{totalNum}
|
||||
</div>
|
||||
|
||||
<div className='p-2'>
|
||||
{['', '', '', ''].map((item, index) => (
|
||||
<div className='py-[5px]' key={index}>
|
||||
<RowStruct />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Crawling)
|
||||
@ -0,0 +1,42 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { Settings01 } from '@/app/components/base/icons/src/vender/line/general'
|
||||
import { BookOpen01 } from '@/app/components/base/icons/src/vender/line/education'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
onSetting: () => void
|
||||
}
|
||||
|
||||
const Header: FC<Props> = ({
|
||||
onSetting,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='flex h-6 items-center justify-between'>
|
||||
<div className='flex items-center'>
|
||||
<div className='text-base font-medium text-gray-700'>{t(`${I18N_PREFIX}.firecrawlTitle`)}</div>
|
||||
<div className='ml-2 mr-1 w-px h-3.5 bg-gray-200'></div>
|
||||
<div
|
||||
className='p-1 rounded-md hover:bg-black/5 cursor-pointer'
|
||||
onClick={onSetting}
|
||||
>
|
||||
<Settings01 className='w-3.5 h-3.5 text-gray-500' />
|
||||
</div>
|
||||
</div>
|
||||
<a
|
||||
href='https://docs.firecrawl.dev/introduction'
|
||||
target='_blank' rel='noopener noreferrer'
|
||||
className='flex items-center text-xs text-primary-600'
|
||||
>
|
||||
<BookOpen01 className='mr-1 w-3.5 h-3.5 text-primary-600' />
|
||||
{t(`${I18N_PREFIX}.firecrawlDoc`)}
|
||||
</a>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Header)
|
||||
@ -0,0 +1,216 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback, useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from 'classnames'
|
||||
import Header from './header'
|
||||
import UrlInput from './base/url-input'
|
||||
import OptionsWrap from './base/options-wrap'
|
||||
import Options from './options'
|
||||
import CrawledResult from './crawled-result'
|
||||
import Crawling from './crawling'
|
||||
import ErrorMessage from './base/error-message'
|
||||
import { useModalContext } from '@/context/modal-context'
|
||||
import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { checkFirecrawlTaskStatus, createFirecrawlTask } from '@/service/datasets'
|
||||
import { sleep } from '@/utils'
|
||||
|
||||
const ERROR_I18N_PREFIX = 'common.errorMsg'
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
onPreview: (payload: CrawlResultItem) => void
|
||||
checkedCrawlResult: CrawlResultItem[]
|
||||
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
|
||||
onJobIdChange: (jobId: string) => void
|
||||
crawlOptions: CrawlOptions
|
||||
onCrawlOptionsChange: (payload: CrawlOptions) => void
|
||||
}
|
||||
|
||||
enum Step {
|
||||
init = 'init',
|
||||
running = 'running',
|
||||
finished = 'finished',
|
||||
}
|
||||
|
||||
const FireCrawl: FC<Props> = ({
|
||||
onPreview,
|
||||
checkedCrawlResult,
|
||||
onCheckedCrawlResultChange,
|
||||
onJobIdChange,
|
||||
crawlOptions,
|
||||
onCrawlOptionsChange,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
const [step, setStep] = useState<Step>(Step.init)
|
||||
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
|
||||
useEffect(() => {
|
||||
if (step !== Step.init)
|
||||
setControlFoldOptions(Date.now())
|
||||
}, [step])
|
||||
const { setShowAccountSettingModal } = useModalContext()
|
||||
const handleSetting = useCallback(() => {
|
||||
setShowAccountSettingModal({
|
||||
payload: 'data-source',
|
||||
})
|
||||
}, [setShowAccountSettingModal])
|
||||
|
||||
const checkValid = useCallback((url: string) => {
|
||||
let errorMsg = ''
|
||||
if (!url) {
|
||||
errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
|
||||
field: 'url',
|
||||
})
|
||||
}
|
||||
|
||||
if (!errorMsg && !((url.startsWith('http://') || url.startsWith('https://'))))
|
||||
errorMsg = t(`${ERROR_I18N_PREFIX}.urlError`)
|
||||
|
||||
if (!errorMsg && (crawlOptions.limit === null || crawlOptions.limit === undefined || crawlOptions.limit === '')) {
|
||||
errorMsg = t(`${ERROR_I18N_PREFIX}.fieldRequired`, {
|
||||
field: t(`${I18N_PREFIX}.limit`),
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
isValid: !errorMsg,
|
||||
errorMsg,
|
||||
}
|
||||
}, [crawlOptions, t])
|
||||
|
||||
const isInit = step === Step.init
|
||||
const isCrawlFinished = step === Step.finished
|
||||
const isRunning = step === Step.running
|
||||
const [crawlResult, setCrawlResult] = useState<{
|
||||
current: number
|
||||
total: number
|
||||
data: CrawlResultItem[]
|
||||
time_consuming: number | string
|
||||
} | undefined>(undefined)
|
||||
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
|
||||
const showError = isCrawlFinished && crawlErrorMessage
|
||||
|
||||
const waitForCrawlFinished = useCallback(async (jobId: string) => {
|
||||
try {
|
||||
const res = await checkFirecrawlTaskStatus(jobId) as any
|
||||
if (res.status === 'completed') {
|
||||
return {
|
||||
isError: false,
|
||||
data: {
|
||||
...res,
|
||||
total: Math.min(res.total, parseFloat(crawlOptions.limit as string)),
|
||||
},
|
||||
}
|
||||
}
|
||||
if (res.status === 'error' || !res.status) {
|
||||
// can't get the error message from the firecrawl api
|
||||
return {
|
||||
isError: true,
|
||||
errorMessage: res.message,
|
||||
data: {
|
||||
data: [],
|
||||
},
|
||||
}
|
||||
}
|
||||
// update the progress
|
||||
setCrawlResult({
|
||||
...res,
|
||||
total: Math.min(res.total, parseFloat(crawlOptions.limit as string)),
|
||||
})
|
||||
await sleep(2500)
|
||||
return await waitForCrawlFinished(jobId)
|
||||
}
|
||||
catch (e: any) {
|
||||
const errorBody = await e.json()
|
||||
return {
|
||||
isError: true,
|
||||
errorMessage: errorBody.message,
|
||||
data: {
|
||||
data: [],
|
||||
},
|
||||
}
|
||||
}
|
||||
}, [crawlOptions.limit])
|
||||
|
||||
const handleRun = useCallback(async (url: string) => {
|
||||
const { isValid, errorMsg } = checkValid(url)
|
||||
if (!isValid) {
|
||||
Toast.notify({
|
||||
message: errorMsg!,
|
||||
type: 'error',
|
||||
})
|
||||
return
|
||||
}
|
||||
setStep(Step.running)
|
||||
try {
|
||||
const passToServerCrawlOptions: any = {
|
||||
...crawlOptions,
|
||||
}
|
||||
if (crawlOptions.max_depth === '')
|
||||
delete passToServerCrawlOptions.max_depth
|
||||
|
||||
const res = await createFirecrawlTask({
|
||||
url,
|
||||
options: passToServerCrawlOptions,
|
||||
}) as any
|
||||
const jobId = res.job_id
|
||||
onJobIdChange(jobId)
|
||||
const { isError, data, errorMessage } = await waitForCrawlFinished(jobId)
|
||||
if (isError) {
|
||||
setCrawlErrorMessage(errorMessage || t(`${I18N_PREFIX}.unknownError`))
|
||||
}
|
||||
else {
|
||||
setCrawlResult(data)
|
||||
setCrawlErrorMessage('')
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
setCrawlErrorMessage(t(`${I18N_PREFIX}.unknownError`)!)
|
||||
console.log(e)
|
||||
}
|
||||
finally {
|
||||
setStep(Step.finished)
|
||||
}
|
||||
}, [checkValid, crawlOptions, onJobIdChange, t, waitForCrawlFinished])
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Header onSetting={handleSetting} />
|
||||
<div className={cn('mt-2 p-4 pb-0 rounded-xl border border-gray-200')}>
|
||||
<UrlInput onRun={handleRun} isRunning={isRunning} />
|
||||
<OptionsWrap
|
||||
className={cn('mt-4')}
|
||||
controlFoldOptions={controlFoldOptions}
|
||||
>
|
||||
<Options className='mt-2' payload={crawlOptions} onChange={onCrawlOptionsChange} />
|
||||
</OptionsWrap>
|
||||
|
||||
{!isInit && (
|
||||
<div className='mt-3 relative left-[-16px] w-[calc(100%_+_32px)] rounded-b-xl'>
|
||||
{isRunning
|
||||
&& <Crawling
|
||||
className='mt-2'
|
||||
crawledNum={crawlResult?.current || 0}
|
||||
totalNum={crawlResult?.total || parseFloat(crawlOptions.limit as string) || 0}
|
||||
/>}
|
||||
{showError && (
|
||||
<ErrorMessage className='rounded-b-xl' title={t(`${I18N_PREFIX}.exceptionErrorTitle`)} errorMsg={crawlErrorMessage} />
|
||||
)}
|
||||
{isCrawlFinished && !showError
|
||||
&& <CrawledResult
|
||||
className='mb-2'
|
||||
list={crawlResult?.data || []}
|
||||
checkedList={checkedCrawlResult}
|
||||
onSelectedChange={onCheckedCrawlResultChange}
|
||||
onPreview={onPreview}
|
||||
usedTime={parseFloat(crawlResult?.time_consuming as string) || 0}
|
||||
/>
|
||||
}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(FireCrawl)
|
||||
@ -0,0 +1,24 @@
|
||||
import type { CrawlResultItem } from '@/models/datasets'
|
||||
|
||||
const result: CrawlResultItem[] = [
|
||||
{
|
||||
title: 'Start the frontend Docker container separately',
|
||||
markdown: 'Markdown 1',
|
||||
description: 'Description 1',
|
||||
source_url: 'https://example.com/1',
|
||||
},
|
||||
{
|
||||
title: 'Advanced Tool Integration',
|
||||
markdown: 'Markdown 2',
|
||||
description: 'Description 2',
|
||||
source_url: 'https://example.com/2',
|
||||
},
|
||||
{
|
||||
title: 'Local Source Code Start | English | Dify',
|
||||
markdown: 'Markdown 3',
|
||||
description: 'Description 3',
|
||||
source_url: 'https://example.com/3',
|
||||
},
|
||||
]
|
||||
|
||||
export default result
|
||||
@ -0,0 +1,83 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback } from 'react'
|
||||
import cn from 'classnames'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import CheckboxWithLabel from './base/checkbox-with-label'
|
||||
import Field from './base/field'
|
||||
import type { CrawlOptions } from '@/models/datasets'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
className?: string
|
||||
payload: CrawlOptions
|
||||
onChange: (payload: CrawlOptions) => void
|
||||
}
|
||||
|
||||
const Options: FC<Props> = ({
|
||||
className = '',
|
||||
payload,
|
||||
onChange,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
const handleChange = useCallback((key: keyof CrawlOptions) => {
|
||||
return (value: any) => {
|
||||
onChange({
|
||||
...payload,
|
||||
[key]: value,
|
||||
})
|
||||
}
|
||||
}, [payload, onChange])
|
||||
return (
|
||||
<div className={cn(className, ' space-y-2')}>
|
||||
<CheckboxWithLabel
|
||||
label={t(`${I18N_PREFIX}.crawlSubPage`)}
|
||||
isChecked={payload.crawl_sub_pages}
|
||||
onChange={handleChange('crawl_sub_pages')}
|
||||
/>
|
||||
<div className='flex justify-between space-x-4'>
|
||||
<Field
|
||||
className='grow shrink-0'
|
||||
label={t(`${I18N_PREFIX}.limit`)}
|
||||
value={payload.limit}
|
||||
onChange={handleChange('limit')}
|
||||
isNumber
|
||||
isRequired
|
||||
/>
|
||||
<Field
|
||||
className='grow shrink-0'
|
||||
label={t(`${I18N_PREFIX}.maxDepth`)}
|
||||
value={payload.max_depth}
|
||||
onChange={handleChange('max_depth')}
|
||||
isNumber
|
||||
tooltip={t(`${I18N_PREFIX}.maxDepthTooltip`)!}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className='flex justify-between space-x-4'>
|
||||
<Field
|
||||
className='grow shrink-0'
|
||||
label={t(`${I18N_PREFIX}.excludePaths`)}
|
||||
value={payload.excludes}
|
||||
onChange={handleChange('excludes')}
|
||||
placeholder='blog/*, /about/*'
|
||||
/>
|
||||
<Field
|
||||
className='grow shrink-0'
|
||||
label={t(`${I18N_PREFIX}.includeOnlyPaths`)}
|
||||
value={payload.includes}
|
||||
onChange={handleChange('includes')}
|
||||
placeholder='articles/*'
|
||||
/>
|
||||
</div>
|
||||
<CheckboxWithLabel
|
||||
label={t(`${I18N_PREFIX}.extractOnlyMainContent`)}
|
||||
isChecked={payload.only_main_content}
|
||||
onChange={handleChange('only_main_content')}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Options)
|
||||
@ -0,0 +1,72 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback, useEffect, useState } from 'react'
|
||||
import NoData from './no-data'
|
||||
import Firecrawl from './firecrawl'
|
||||
import { useModalContext } from '@/context/modal-context'
|
||||
import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
|
||||
import { fetchFirecrawlApiKey } from '@/service/datasets'
|
||||
import { type DataSourceWebsiteItem, WebsiteProvider } from '@/models/common'
|
||||
|
||||
type Props = {
|
||||
onPreview: (payload: CrawlResultItem) => void
|
||||
checkedCrawlResult: CrawlResultItem[]
|
||||
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
|
||||
onJobIdChange: (jobId: string) => void
|
||||
crawlOptions: CrawlOptions
|
||||
onCrawlOptionsChange: (payload: CrawlOptions) => void
|
||||
}
|
||||
|
||||
const Website: FC<Props> = ({
|
||||
onPreview,
|
||||
checkedCrawlResult,
|
||||
onCheckedCrawlResultChange,
|
||||
onJobIdChange,
|
||||
crawlOptions,
|
||||
onCrawlOptionsChange,
|
||||
}) => {
|
||||
const { setShowAccountSettingModal } = useModalContext()
|
||||
const [isLoaded, setIsLoaded] = useState(false)
|
||||
const [isSetFirecrawlApiKey, setIsSetFirecrawlApiKey] = useState(false)
|
||||
const checkSetApiKey = useCallback(async () => {
|
||||
const res = await fetchFirecrawlApiKey() as any
|
||||
const list = res.settings.filter((item: DataSourceWebsiteItem) => item.provider === WebsiteProvider.fireCrawl && !item.disabled)
|
||||
setIsSetFirecrawlApiKey(list.length > 0)
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
checkSetApiKey().then(() => {
|
||||
setIsLoaded(true)
|
||||
})
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [])
|
||||
const handleOnConfig = useCallback(() => {
|
||||
setShowAccountSettingModal({
|
||||
payload: 'data-source',
|
||||
onCancelCallback: checkSetApiKey,
|
||||
})
|
||||
}, [checkSetApiKey, setShowAccountSettingModal])
|
||||
|
||||
if (!isLoaded)
|
||||
return null
|
||||
|
||||
return (
|
||||
<div>
|
||||
{isSetFirecrawlApiKey
|
||||
? (
|
||||
<Firecrawl
|
||||
onPreview={onPreview}
|
||||
checkedCrawlResult={checkedCrawlResult}
|
||||
onCheckedCrawlResultChange={onCheckedCrawlResultChange}
|
||||
onJobIdChange={onJobIdChange}
|
||||
crawlOptions={crawlOptions}
|
||||
onCrawlOptionsChange={onCrawlOptionsChange}
|
||||
/>
|
||||
)
|
||||
: (
|
||||
<NoData onConfig={handleOnConfig} />
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Website)
|
||||
@ -0,0 +1,36 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others'
|
||||
import Button from '@/app/components/base/button'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
type Props = {
|
||||
onConfig: () => void
|
||||
}
|
||||
|
||||
const NoData: FC<Props> = ({
|
||||
onConfig,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className='max-w-[640px] p-6 rounded-2xl bg-gray-50'>
|
||||
<div className='flex w-11 h-11 items-center justify-center bg-gray-50 rounded-xl border-[0.5px] border-gray-100 shadow-lg'>
|
||||
🔥
|
||||
</div>
|
||||
<div className='my-2'>
|
||||
<span className='text-gray-700 font-semibold'>{t(`${I18N_PREFIX}.fireCrawlNotConfigured`)}<Icon3Dots className='inline relative -top-3 -left-1.5' /></span>
|
||||
<div className='mt-1 pb-3 text-gray-500 text-[13px] font-normal'>
|
||||
{t(`${I18N_PREFIX}.fireCrawlNotConfiguredDescription`)}
|
||||
</div>
|
||||
</div>
|
||||
<Button type='primary' onClick={onConfig} className='!h-8 text-[13px] font-medium ' >
|
||||
{t(`${I18N_PREFIX}.configure`)}
|
||||
</Button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(NoData)
|
||||
@ -0,0 +1,41 @@
|
||||
'use client'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from 'classnames'
|
||||
import { XMarkIcon } from '@heroicons/react/20/solid'
|
||||
import s from '../file-preview/index.module.css'
|
||||
import type { CrawlResultItem } from '@/models/datasets'
|
||||
|
||||
type IProps = {
|
||||
payload: CrawlResultItem
|
||||
hidePreview: () => void
|
||||
}
|
||||
|
||||
const WebsitePreview = ({
|
||||
payload,
|
||||
hidePreview,
|
||||
}: IProps) => {
|
||||
const { t } = useTranslation()
|
||||
|
||||
return (
|
||||
<div className={cn(s.filePreview)}>
|
||||
<div className={cn(s.previewHeader)}>
|
||||
<div className={cn(s.title)}>
|
||||
<span>{t('datasetCreation.stepOne.pagePreview')}</span>
|
||||
<div className='flex items-center justify-center w-6 h-6 cursor-pointer' onClick={hidePreview}>
|
||||
<XMarkIcon className='h-4 w-4'></XMarkIcon>
|
||||
</div>
|
||||
</div>
|
||||
<div className='leading-5 text-sm font-medium text-gray-900 break-words'>
|
||||
{payload.title}
|
||||
</div>
|
||||
<div className='truncate leading-[18px] text-xs font-normal text-gray-500' title={payload.source_url}>{payload.source_url}</div>
|
||||
</div>
|
||||
<div className={cn(s.previewContent)}>
|
||||
<div className={cn(s.fileContent)}>{payload.markdown}</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default WebsitePreview
|
||||
@ -0,0 +1,163 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import {
|
||||
PortalToFollowElem,
|
||||
PortalToFollowElemContent,
|
||||
} from '@/app/components/base/portal-to-follow-elem'
|
||||
import { Lock01 } from '@/app/components/base/icons/src/vender/solid/security'
|
||||
import Button from '@/app/components/base/button'
|
||||
import type { FirecrawlConfig } from '@/models/common'
|
||||
import Field from '@/app/components/datasets/create/website/firecrawl/base/field'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { createFirecrawlApiKey } from '@/service/datasets'
|
||||
import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general'
|
||||
type Props = {
|
||||
onCancel: () => void
|
||||
onSaved: () => void
|
||||
}
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.firecrawl'
|
||||
|
||||
const DEFAULT_BASE_URL = 'https://api.firecrawl.dev'
|
||||
|
||||
const ConfigFirecrawlModal: FC<Props> = ({
|
||||
onCancel,
|
||||
onSaved,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
const [isSaving, setIsSaving] = useState(false)
|
||||
const [config, setConfig] = useState<FirecrawlConfig>({
|
||||
api_key: '',
|
||||
base_url: '',
|
||||
})
|
||||
|
||||
const handleConfigChange = useCallback((key: string) => {
|
||||
return (value: string | number) => {
|
||||
setConfig(prev => ({ ...prev, [key]: value as string }))
|
||||
}
|
||||
}, [])
|
||||
|
||||
const handleSave = useCallback(async () => {
|
||||
if (isSaving)
|
||||
return
|
||||
let errorMsg = ''
|
||||
if (config.base_url && !((config.base_url.startsWith('http://') || config.base_url.startsWith('https://'))))
|
||||
errorMsg = t('common.errorMsg.urlError')
|
||||
if (!errorMsg) {
|
||||
if (!config.api_key) {
|
||||
errorMsg = t('common.errorMsg.fieldRequired', {
|
||||
field: 'API Key',
|
||||
})
|
||||
}
|
||||
else if (!config.api_key.startsWith('fc-')) {
|
||||
errorMsg = t(`${I18N_PREFIX}.apiKeyFormatError`)
|
||||
}
|
||||
}
|
||||
|
||||
if (errorMsg) {
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message: errorMsg,
|
||||
})
|
||||
return
|
||||
}
|
||||
const postData = {
|
||||
category: 'website',
|
||||
provider: 'firecrawl',
|
||||
credentials: {
|
||||
auth_type: 'bearer',
|
||||
config: {
|
||||
api_key: config.api_key,
|
||||
base_url: config.base_url || DEFAULT_BASE_URL,
|
||||
},
|
||||
},
|
||||
}
|
||||
try {
|
||||
setIsSaving(true)
|
||||
await createFirecrawlApiKey(postData)
|
||||
Toast.notify({
|
||||
type: 'success',
|
||||
message: t('common.api.success'),
|
||||
})
|
||||
}
|
||||
finally {
|
||||
setIsSaving(false)
|
||||
}
|
||||
|
||||
onSaved()
|
||||
}, [config.api_key, config.base_url, onSaved, t, isSaving])
|
||||
|
||||
return (
|
||||
<PortalToFollowElem open>
|
||||
<PortalToFollowElemContent className='w-full h-full z-[60]'>
|
||||
<div className='fixed inset-0 flex items-center justify-center bg-black/[.25]'>
|
||||
<div className='mx-2 w-[640px] max-h-[calc(100vh-120px)] bg-white shadow-xl rounded-2xl overflow-y-auto'>
|
||||
<div className='px-8 pt-8'>
|
||||
<div className='flex justify-between items-center mb-4'>
|
||||
<div className='text-xl font-semibold text-gray-900'>{t(`${I18N_PREFIX}.configFirecrawl`)}</div>
|
||||
</div>
|
||||
|
||||
<div className='space-y-4'>
|
||||
<Field
|
||||
label='API Key'
|
||||
labelClassName='!text-sm'
|
||||
isRequired
|
||||
value={config.api_key}
|
||||
onChange={handleConfigChange('api_key')}
|
||||
placeholder={t(`${I18N_PREFIX}.apiKeyPlaceholder`)!}
|
||||
/>
|
||||
<Field
|
||||
label='Base URL'
|
||||
labelClassName='!text-sm'
|
||||
value={config.base_url}
|
||||
onChange={handleConfigChange('base_url')}
|
||||
placeholder={DEFAULT_BASE_URL}
|
||||
/>
|
||||
</div>
|
||||
<div className='my-8 flex justify-between items-center h-8'>
|
||||
<a className='flex items-center space-x-1 leading-[18px] text-xs font-normal text-[#155EEF]' target='_blank' href='https://www.firecrawl.dev/account'>
|
||||
<span>{t(`${I18N_PREFIX}.getApiKeyLinkText`)}</span>
|
||||
<LinkExternal02 className='w-3 h-3' />
|
||||
</a>
|
||||
<div className='flex'>
|
||||
<Button
|
||||
className='mr-2 h-9 text-sm font-medium text-gray-700'
|
||||
onClick={onCancel}
|
||||
>
|
||||
{t('common.operation.cancel')}
|
||||
</Button>
|
||||
<Button
|
||||
className='h-9 text-sm font-medium'
|
||||
type='primary'
|
||||
onClick={handleSave}
|
||||
loading={isSaving}
|
||||
>
|
||||
{t('common.operation.save')}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div className='border-t-[0.5px] border-t-black/5'>
|
||||
<div className='flex justify-center items-center py-3 bg-gray-50 text-xs text-gray-500'>
|
||||
<Lock01 className='mr-1 w-3 h-3 text-gray-500' />
|
||||
{t('common.modelProvider.encrypted.front')}
|
||||
<a
|
||||
className='text-primary-600 mx-1'
|
||||
target='_blank' rel='noopener noreferrer'
|
||||
href='https://pycryptodome.readthedocs.io/en/latest/src/cipher/oaep.html'
|
||||
>
|
||||
PKCS1_OAEP
|
||||
</a>
|
||||
{t('common.modelProvider.encrypted.back')}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</PortalToFollowElemContent>
|
||||
</PortalToFollowElem>
|
||||
)
|
||||
}
|
||||
export default React.memo(ConfigFirecrawlModal)
|
||||
@ -0,0 +1,82 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React, { useCallback, useEffect, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useBoolean } from 'ahooks'
|
||||
import cn from 'classnames'
|
||||
import Panel from '../panel'
|
||||
import { DataSourceType } from '../panel/types'
|
||||
import ConfigFirecrawlModal from './config-firecrawl-modal'
|
||||
import { fetchFirecrawlApiKey, removeFirecrawlApiKey } from '@/service/datasets'
|
||||
|
||||
import type {
|
||||
DataSourceWebsiteItem,
|
||||
} from '@/models/common'
|
||||
import { useAppContext } from '@/context/app-context'
|
||||
|
||||
import {
|
||||
WebsiteProvider,
|
||||
} from '@/models/common'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
|
||||
type Props = {}
|
||||
|
||||
const DataSourceWebsite: FC<Props> = () => {
|
||||
const { t } = useTranslation()
|
||||
const { isCurrentWorkspaceManager } = useAppContext()
|
||||
const [list, setList] = useState<DataSourceWebsiteItem[]>([])
|
||||
const checkSetApiKey = useCallback(async () => {
|
||||
const res = await fetchFirecrawlApiKey() as any
|
||||
const list = res.settings.filter((item: DataSourceWebsiteItem) => item.provider === WebsiteProvider.fireCrawl && !item.disabled)
|
||||
setList(list)
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
checkSetApiKey()
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [])
|
||||
|
||||
const [isShowConfig, {
|
||||
setTrue: showConfig,
|
||||
setFalse: hideConfig,
|
||||
}] = useBoolean(false)
|
||||
|
||||
const handleAdded = useCallback(() => {
|
||||
checkSetApiKey()
|
||||
hideConfig()
|
||||
}, [checkSetApiKey, hideConfig])
|
||||
|
||||
const handleRemove = useCallback(async () => {
|
||||
await removeFirecrawlApiKey(list[0].id)
|
||||
setList([])
|
||||
Toast.notify({
|
||||
type: 'success',
|
||||
message: t('common.api.remove'),
|
||||
})
|
||||
}, [list, t])
|
||||
|
||||
return (
|
||||
<>
|
||||
<Panel
|
||||
type={DataSourceType.website}
|
||||
isConfigured={list.length > 0}
|
||||
onConfigure={showConfig}
|
||||
readonly={!isCurrentWorkspaceManager}
|
||||
configuredList={list.map(item => ({
|
||||
id: item.id,
|
||||
logo: ({ className }: { className: string }) => (
|
||||
<div className={cn(className, 'flex items-center justify-center w-5 h-5 bg-white border border-gray-100 text-xs font-medium text-gray-500 rounded ml-3')}>🔥</div>
|
||||
),
|
||||
name: 'FireCrawl',
|
||||
isActive: true,
|
||||
}))}
|
||||
onRemove={handleRemove}
|
||||
/>
|
||||
{isShowConfig && (
|
||||
<ConfigFirecrawlModal onSaved={handleAdded} onCancel={hideConfig} />
|
||||
)}
|
||||
</>
|
||||
|
||||
)
|
||||
}
|
||||
export default React.memo(DataSourceWebsite)
|
||||
@ -0,0 +1,78 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import cn from 'classnames'
|
||||
import Indicator from '../../../indicator'
|
||||
import Operate from '../data-source-notion/operate'
|
||||
import { DataSourceType } from './types'
|
||||
import s from './style.module.css'
|
||||
import { Trash03 } from '@/app/components/base/icons/src/vender/line/general'
|
||||
|
||||
export type ConfigItemType = {
|
||||
id: string
|
||||
logo: any
|
||||
name: string
|
||||
isActive: boolean
|
||||
notionConfig?: {
|
||||
total: number
|
||||
}
|
||||
}
|
||||
|
||||
type Props = {
|
||||
type: DataSourceType
|
||||
payload: ConfigItemType
|
||||
onRemove: () => void
|
||||
notionActions?: {
|
||||
onChangeAuthorizedPage: () => void
|
||||
}
|
||||
}
|
||||
|
||||
const ConfigItem: FC<Props> = ({
|
||||
type,
|
||||
payload,
|
||||
onRemove,
|
||||
notionActions,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
const isNotion = type === DataSourceType.notion
|
||||
const isWebsite = type === DataSourceType.website
|
||||
const onChangeAuthorizedPage = notionActions?.onChangeAuthorizedPage || function () { }
|
||||
|
||||
return (
|
||||
<div className={cn(s['workspace-item'], 'flex items-center mb-1 py-1 pr-1 bg-white rounded-lg')} key={payload.id}>
|
||||
<payload.logo className='ml-3 mr-1.5' />
|
||||
<div className='grow py-[7px] leading-[18px] text-[13px] font-medium text-gray-700 truncate' title={payload.name}>{payload.name}</div>
|
||||
{
|
||||
payload.isActive
|
||||
? <Indicator className='shrink-0 mr-[6px]' />
|
||||
: <Indicator className='shrink-0 mr-[6px]' color='yellow' />
|
||||
}
|
||||
<div className='shrink-0 mr-3 text-xs font-medium uppercase'>
|
||||
{
|
||||
payload.isActive
|
||||
? t(isNotion ? 'common.dataSource.notion.connected' : 'common.dataSource.website.active')
|
||||
: t(isNotion ? 'common.dataSource.notion.disconnected' : 'common.dataSource.website.inactive')
|
||||
}
|
||||
</div>
|
||||
<div className='mr-2 w-[1px] h-3 bg-gray-100' />
|
||||
{isNotion && (
|
||||
<Operate payload={{
|
||||
id: payload.id,
|
||||
total: payload.notionConfig?.total || 0,
|
||||
}} onAuthAgain={onChangeAuthorizedPage}
|
||||
/>
|
||||
)}
|
||||
|
||||
{
|
||||
isWebsite && (
|
||||
<div className='p-2 text-gray-500 cursor-pointer rounded-md hover:bg-black/5' onClick={onRemove} >
|
||||
<Trash03 className='w-4 h-4 ' />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(ConfigItem)
|
||||
@ -0,0 +1,138 @@
|
||||
'use client'
|
||||
import type { FC } from 'react'
|
||||
import React from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { PlusIcon } from '@heroicons/react/24/solid'
|
||||
import cn from 'classnames'
|
||||
import type { ConfigItemType } from './config-item'
|
||||
import ConfigItem from './config-item'
|
||||
|
||||
import s from './style.module.css'
|
||||
import { DataSourceType } from './types'
|
||||
|
||||
type Props = {
|
||||
type: DataSourceType
|
||||
isConfigured: boolean
|
||||
onConfigure: () => void
|
||||
readonly: boolean
|
||||
isSupportList?: boolean
|
||||
configuredList: ConfigItemType[]
|
||||
onRemove: () => void
|
||||
notionActions?: {
|
||||
onChangeAuthorizedPage: () => void
|
||||
}
|
||||
}
|
||||
|
||||
const Panel: FC<Props> = ({
|
||||
type,
|
||||
isConfigured,
|
||||
onConfigure,
|
||||
readonly,
|
||||
configuredList,
|
||||
isSupportList,
|
||||
onRemove,
|
||||
notionActions,
|
||||
}) => {
|
||||
const { t } = useTranslation()
|
||||
const isNotion = type === DataSourceType.notion
|
||||
const isWebsite = type === DataSourceType.website
|
||||
|
||||
return (
|
||||
<div className='mb-2 border-[0.5px] border-gray-200 bg-gray-50 rounded-xl'>
|
||||
<div className='flex items-center px-3 py-[9px]'>
|
||||
<div className={cn(s[`${type}-icon`], 'w-8 h-8 mr-3 border border-gray-100 rounded-lg')} />
|
||||
<div className='grow'>
|
||||
<div className='flex items-center h-5'>
|
||||
<div className='text-sm font-medium text-gray-800'>{t(`common.dataSource.${type}.title`)}</div>
|
||||
{isWebsite && (
|
||||
<div className='ml-1 leading-[18px] px-1.5 rounded-md bg-white border border-gray-100 text-xs font-medium text-gray-700'>
|
||||
<span className='text-gray-500'>{t('common.dataSource.website.with')}</span> 🔥 FireCrawl
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{
|
||||
!isConfigured && (
|
||||
<div className='leading-5 text-xs text-gray-500'>
|
||||
{t(`common.dataSource.${type}.description`)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
</div>
|
||||
{isNotion && (
|
||||
<>
|
||||
{
|
||||
isConfigured
|
||||
? (
|
||||
<div
|
||||
className={
|
||||
`flex items-center ml-3 px-3 h-7 bg-white border border-gray-200
|
||||
rounded-md text-xs font-medium text-gray-700
|
||||
${!readonly ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
|
||||
}
|
||||
onClick={onConfigure}
|
||||
>
|
||||
{t('common.dataSource.configure')}
|
||||
</div>
|
||||
)
|
||||
: (
|
||||
<>
|
||||
{isSupportList && <div
|
||||
className={
|
||||
`flex items-center px-3 py-1 min-h-7 bg-white border-[0.5px] border-gray-200 text-xs font-medium text-primary-600 rounded-md
|
||||
${!readonly ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
|
||||
}
|
||||
onClick={onConfigure}
|
||||
>
|
||||
<PlusIcon className='w-[14px] h-[14px] mr-[5px]' />
|
||||
{t('common.dataSource.notion.addWorkspace')}
|
||||
</div>}
|
||||
</>
|
||||
)
|
||||
}
|
||||
</>
|
||||
)}
|
||||
|
||||
{isWebsite && !isConfigured && (
|
||||
<div
|
||||
className={
|
||||
`flex items-center ml-3 px-3 h-7 bg-white border border-gray-200
|
||||
rounded-md text-xs font-medium text-gray-700
|
||||
${!readonly ? 'cursor-pointer' : 'grayscale opacity-50 cursor-default'}`
|
||||
}
|
||||
onClick={onConfigure}
|
||||
>
|
||||
{t('common.dataSource.configure')}
|
||||
</div>
|
||||
)}
|
||||
|
||||
</div>
|
||||
{
|
||||
isConfigured && (
|
||||
<div className='flex items-center px-3 h-[18px]'>
|
||||
<div className='text-xs font-medium text-gray-500'>
|
||||
{isNotion ? t('common.dataSource.notion.connectedWorkspace') : t('common.dataSource.website.configuredCrawlers')}
|
||||
</div>
|
||||
<div className='grow ml-3 border-t border-t-gray-100' />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
{
|
||||
isConfigured && (
|
||||
<div className='px-3 pt-2 pb-3'>
|
||||
{
|
||||
configuredList.map(item => (
|
||||
<ConfigItem
|
||||
key={item.id}
|
||||
type={type}
|
||||
payload={item}
|
||||
onRemove={onRemove}
|
||||
notionActions={notionActions} />
|
||||
))
|
||||
}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Panel)
|
||||
@ -0,0 +1,4 @@
|
||||
export enum DataSourceType {
|
||||
notion = 'notion',
|
||||
website = 'website',
|
||||
}
|
||||
Loading…
Reference in New Issue