Feat/dataset support api service (#1240)
Co-authored-by: Joel <iamjoel007@gmail.com> Co-authored-by: crazywoola <427733928@qq.com>pull/1248/head
zxhlyh
3 years ago
committed by
GitHub
parent
46154c6705
commit
9dbb8acd4b
@ -0,0 +1,41 @@
|
|||||||
|
'use client'
|
||||||
|
|
||||||
|
import type { FC } from 'react'
|
||||||
|
import { useTranslation } from 'react-i18next'
|
||||||
|
import CopyFeedback from '@/app/components/base/copy-feedback'
|
||||||
|
import SecretKeyButton from '@/app/components/develop/secret-key/secret-key-button'
|
||||||
|
import { randomString } from '@/utils'
|
||||||
|
|
||||||
|
type ApiServerProps = {
|
||||||
|
apiBaseUrl: string
|
||||||
|
}
|
||||||
|
const ApiServer: FC<ApiServerProps> = ({
|
||||||
|
apiBaseUrl,
|
||||||
|
}) => {
|
||||||
|
const { t } = useTranslation()
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className='flex items-center'>
|
||||||
|
<div className='flex items-center mr-2 pl-1.5 pr-1 h-8 bg-white/80 border-[0.5px] border-white rounded-lg'>
|
||||||
|
<div className='mr-0.5 px-1.5 h-5 border border-gray-200 text-[11px] text-gray-500 rounded-md'>{t('appApi.apiServer')}</div>
|
||||||
|
<div className='px-1 w-[248px] text-[13px] font-medium text-gray-800'>{apiBaseUrl}</div>
|
||||||
|
<div className='mx-1 w-[1px] h-[14px] bg-gray-200'></div>
|
||||||
|
<CopyFeedback
|
||||||
|
content={apiBaseUrl}
|
||||||
|
selectorId={randomString(8)}
|
||||||
|
className={'!w-6 !h-6 hover:bg-gray-200'}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className='flex items-center mr-2 px-3 h-8 bg-[#ECFDF3] text-xs font-semibold text-[#039855] rounded-lg border-[0.5px] border-[#D1FADF]'>
|
||||||
|
{t('appApi.ok')}
|
||||||
|
</div>
|
||||||
|
<SecretKeyButton
|
||||||
|
className='flex-shrink-0 !h-8 bg-white'
|
||||||
|
textCls='!text-gray-700 font-medium'
|
||||||
|
iconCls='stroke-[1.2px]'
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default ApiServer
|
||||||
@ -0,0 +1,60 @@
|
|||||||
|
'use client'
|
||||||
|
|
||||||
|
import { useRef, useState } from 'react'
|
||||||
|
import { useTranslation } from 'react-i18next'
|
||||||
|
import useSWR from 'swr'
|
||||||
|
import Datasets from './Datasets'
|
||||||
|
import DatasetFooter from './DatasetFooter'
|
||||||
|
import ApiServer from './ApiServer'
|
||||||
|
import Doc from './Doc'
|
||||||
|
import TabSlider from '@/app/components/base/tab-slider'
|
||||||
|
import { fetchDatasetApiBaseUrl } from '@/service/datasets'
|
||||||
|
|
||||||
|
const Container = () => {
|
||||||
|
const { t } = useTranslation()
|
||||||
|
const options = [
|
||||||
|
{
|
||||||
|
value: 'dataset',
|
||||||
|
text: t('dataset.datasets'),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
value: 'api',
|
||||||
|
text: t('dataset.datasetsApi'),
|
||||||
|
},
|
||||||
|
]
|
||||||
|
const [activeTab, setActiveTab] = useState('dataset')
|
||||||
|
const containerRef = useRef<HTMLDivElement>(null)
|
||||||
|
const { data } = useSWR(activeTab === 'dataset' ? null : '/datasets/api-base-info', fetchDatasetApiBaseUrl)
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div ref={containerRef} className='grow relative flex flex-col bg-gray-100 overflow-y-auto'>
|
||||||
|
<div className='sticky top-0 flex justify-between pt-4 px-12 pb-2 h-14 bg-gray-100 z-10'>
|
||||||
|
<TabSlider
|
||||||
|
value={activeTab}
|
||||||
|
onChange={newActiveTab => setActiveTab(newActiveTab)}
|
||||||
|
options={options}
|
||||||
|
/>
|
||||||
|
{
|
||||||
|
activeTab === 'api' && (
|
||||||
|
<ApiServer apiBaseUrl={data?.api_base_url || ''} />
|
||||||
|
)
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
{
|
||||||
|
activeTab === 'dataset' && (
|
||||||
|
<div className=''>
|
||||||
|
<Datasets containerRef={containerRef}/>
|
||||||
|
<DatasetFooter />
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
{
|
||||||
|
activeTab === 'api' && (
|
||||||
|
<Doc apiBaseUrl={data?.api_base_url || ''} />
|
||||||
|
)
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default Container
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
'use client'
|
||||||
|
|
||||||
|
import type { FC } from 'react'
|
||||||
|
import { useContext } from 'use-context-selector'
|
||||||
|
import TemplateEn from './template/template.en.mdx'
|
||||||
|
import TemplateZh from './template/template.zh.mdx'
|
||||||
|
import I18n from '@/context/i18n'
|
||||||
|
|
||||||
|
type DocProps = {
|
||||||
|
apiBaseUrl: string
|
||||||
|
}
|
||||||
|
const Doc: FC<DocProps> = ({
|
||||||
|
apiBaseUrl,
|
||||||
|
}) => {
|
||||||
|
const { locale } = useContext(I18n)
|
||||||
|
|
||||||
|
return (
|
||||||
|
<article className='mx-12 pt-16 bg-white rounded-t-xl prose prose-xl'>
|
||||||
|
{
|
||||||
|
locale === 'en'
|
||||||
|
? <TemplateEn apiBaseUrl={apiBaseUrl} />
|
||||||
|
: <TemplateZh apiBaseUrl={apiBaseUrl} />
|
||||||
|
}
|
||||||
|
</article>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default Doc
|
||||||
@ -0,0 +1,791 @@
|
|||||||
|
import { CodeGroup } from '@/app/components/develop/code.tsx'
|
||||||
|
import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from '@/app/components/develop/md.tsx'
|
||||||
|
|
||||||
|
# Dataset API
|
||||||
|
<br/>
|
||||||
|
<br/>
|
||||||
|
<Heading
|
||||||
|
url='/datasets'
|
||||||
|
method='POST'
|
||||||
|
title='Create an empty dataset'
|
||||||
|
name='#create_empty_dataset'
|
||||||
|
/>
|
||||||
|
<Row>
|
||||||
|
<Col>
|
||||||
|
### Request Body
|
||||||
|
<Properties>
|
||||||
|
<Property name='name' type='string' key='name'>
|
||||||
|
Dataset name
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
</Col>
|
||||||
|
<Col sticky>
|
||||||
|
<CodeGroup
|
||||||
|
title="Request"
|
||||||
|
tag="POST"
|
||||||
|
label="/datasets"
|
||||||
|
targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name"}'`}
|
||||||
|
>
|
||||||
|
```bash {{ title: 'cURL' }}
|
||||||
|
curl --location --request POST '${apiBaseUrl}/v1/datasets' \
|
||||||
|
--header 'Authorization: Bearer {api_key}' \
|
||||||
|
--header 'Content-Type: application/json' \
|
||||||
|
--data-raw '{
|
||||||
|
"name": "name"
|
||||||
|
}'
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
<CodeGroup title="Response">
|
||||||
|
```json {{ title: 'Response' }}
|
||||||
|
{
|
||||||
|
"id": "",
|
||||||
|
"name": "name",
|
||||||
|
"description": null,
|
||||||
|
"provider": "vendor",
|
||||||
|
"permission": "only_me",
|
||||||
|
"data_source_type": null,
|
||||||
|
"indexing_technique": null,
|
||||||
|
"app_count": 0,
|
||||||
|
"document_count": 0,
|
||||||
|
"word_count": 0,
|
||||||
|
"created_by": "",
|
||||||
|
"created_at": 1695636173,
|
||||||
|
"updated_by": "",
|
||||||
|
"updated_at": 1695636173,
|
||||||
|
"embedding_model": null,
|
||||||
|
"embedding_model_provider": null,
|
||||||
|
"embedding_available": null
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
<Heading
|
||||||
|
url='/datasets'
|
||||||
|
method='GET'
|
||||||
|
title='Dataset list'
|
||||||
|
name='#dataset_list'
|
||||||
|
/>
|
||||||
|
<Row>
|
||||||
|
<Col>
|
||||||
|
### Path Query
|
||||||
|
<Properties>
|
||||||
|
<Property name='page' type='string' key='page'>
|
||||||
|
Page number
|
||||||
|
</Property>
|
||||||
|
<Property name='limit' type='string' key='limit'>
|
||||||
|
Number of items returned, default 20, range 1-100
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
</Col>
|
||||||
|
<Col sticky>
|
||||||
|
<CodeGroup
|
||||||
|
title="Request"
|
||||||
|
tag="POST"
|
||||||
|
label="/datasets"
|
||||||
|
targetCode={`curl --location --request GET '${props.apiBaseUrl}/datasets?page=1&limit=20' \\\n--header 'Authorization: Bearer {api_key}'`}
|
||||||
|
>
|
||||||
|
```bash {{ title: 'cURL' }}
|
||||||
|
curl --location --request GET 'https://api.dify.ai/v1/datasets?page=1&limit=20' \
|
||||||
|
--header 'Authorization: Bearer {api_key}'
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
<CodeGroup title="Response">
|
||||||
|
```json {{ title: 'Response' }}
|
||||||
|
{
|
||||||
|
"data": [
|
||||||
|
{
|
||||||
|
"id": "",
|
||||||
|
"name": "name",
|
||||||
|
"description": "desc",
|
||||||
|
"permission": "only_me",
|
||||||
|
"data_source_type": "upload_file",
|
||||||
|
"indexing_technique": "",
|
||||||
|
"app_count": 2,
|
||||||
|
"document_count": 10,
|
||||||
|
"word_count": 1200,
|
||||||
|
"created_by": "",
|
||||||
|
"created_at": "",
|
||||||
|
"updated_by": "",
|
||||||
|
"updated_at": ""
|
||||||
|
},
|
||||||
|
...
|
||||||
|
],
|
||||||
|
"has_more": true,
|
||||||
|
"limit": 20,
|
||||||
|
"total": 50,
|
||||||
|
"page": 1
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
<Heading
|
||||||
|
url='/datasets/{dataset_id}/document/create_by_text'
|
||||||
|
method='POST'
|
||||||
|
title='Create a document from text'
|
||||||
|
name='#create_by_text'
|
||||||
|
/>
|
||||||
|
<Row>
|
||||||
|
<Col>
|
||||||
|
This api is based on an existing dataset and creates a new document through text based on this dataset.
|
||||||
|
|
||||||
|
### Path Params
|
||||||
|
<Properties>
|
||||||
|
<Property name='dataset_id' type='string' key='dataset_id'>
|
||||||
|
Dataset ID
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
|
||||||
|
### Request Body
|
||||||
|
<Properties>
|
||||||
|
<Property name='name' type='string' key='name'>
|
||||||
|
Document name
|
||||||
|
</Property>
|
||||||
|
<Property name='text' type='string' key='text'>
|
||||||
|
Document content
|
||||||
|
</Property>
|
||||||
|
<Property name='indexing_technique' type='string' key='indexing_technique'>
|
||||||
|
Index mode
|
||||||
|
- high_quality High quality: embedding using embedding model, built as vector database index
|
||||||
|
- economy Economy: Build using inverted index of Keyword Table Index
|
||||||
|
</Property>
|
||||||
|
<Property name='process_rule' type='object' key='process_rule'>
|
||||||
|
Processing rules
|
||||||
|
- mode (string) Cleaning, segmentation mode, automatic / custom
|
||||||
|
- rules (text) Custom rules (in automatic mode, this field is empty)
|
||||||
|
- pre_processing_rules (array[object]) Preprocessing rules
|
||||||
|
- id (string) Unique identifier for the preprocessing rule
|
||||||
|
- enumerate
|
||||||
|
- remove_extra_spaces Replace consecutive spaces, newlines, tabs
|
||||||
|
- remove_urls_emails Delete URL, email address
|
||||||
|
- enabled (bool) Whether to select this rule or not. If no document ID is passed in, it represents the default value.
|
||||||
|
- segmentation (object) segmentation rules
|
||||||
|
- separator Custom segment identifier, currently only allows one delimiter to be set. Default is \n
|
||||||
|
- max_tokens Maximum length (token) defaults to 1000
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
</Col>
|
||||||
|
<Col sticky>
|
||||||
|
<CodeGroup
|
||||||
|
title="Request"
|
||||||
|
tag="POST"
|
||||||
|
label="/datasets/{dataset_id}/document/create_by_text"
|
||||||
|
targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/document/create_by_text' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "text","text": "text","indexing_technique": "high_quality","process_rule": {"mode": "automatic"}}'`}
|
||||||
|
>
|
||||||
|
```bash {{ title: 'cURL' }}
|
||||||
|
curl --location --request POST 'https://api.dify.ai/v1/datasets/{dataset_id}/document/create_by_text' \
|
||||||
|
--header 'Authorization: Bearer {api_key}' \
|
||||||
|
--header 'Content-Type: application/json' \
|
||||||
|
--data-raw '{
|
||||||
|
"name": "text",
|
||||||
|
"text": "text",
|
||||||
|
"indexing_technique": "high_quality",
|
||||||
|
"process_rule": {
|
||||||
|
"mode": "automatic"
|
||||||
|
}
|
||||||
|
}'
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
<CodeGroup title="Response">
|
||||||
|
```json {{ title: 'Response' }}
|
||||||
|
{
|
||||||
|
"document": {
|
||||||
|
"id": "",
|
||||||
|
"position": 1,
|
||||||
|
"data_source_type": "upload_file",
|
||||||
|
"data_source_info": {
|
||||||
|
"upload_file_id": ""
|
||||||
|
},
|
||||||
|
"dataset_process_rule_id": "",
|
||||||
|
"name": "text.txt",
|
||||||
|
"created_from": "api",
|
||||||
|
"created_by": "",
|
||||||
|
"created_at": 1695690280,
|
||||||
|
"tokens": 0,
|
||||||
|
"indexing_status": "waiting",
|
||||||
|
"error": null,
|
||||||
|
"enabled": true,
|
||||||
|
"disabled_at": null,
|
||||||
|
"disabled_by": null,
|
||||||
|
"archived": false,
|
||||||
|
"display_status": "queuing",
|
||||||
|
"word_count": 0,
|
||||||
|
"hit_count": 0,
|
||||||
|
"doc_form": "text_model"
|
||||||
|
},
|
||||||
|
"batch": ""
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
<Heading
|
||||||
|
url='/datasets/{dataset_id}/document/create_by_file'
|
||||||
|
method='POST'
|
||||||
|
title='Create documents from files'
|
||||||
|
name='#create_by_file'
|
||||||
|
/>
|
||||||
|
<Row>
|
||||||
|
<Col>
|
||||||
|
This api is based on an existing dataset and creates a new document through a file based on this dataset.
|
||||||
|
### Path Params
|
||||||
|
<Properties>
|
||||||
|
<Property name='dataset_id' type='string' key='dataset_id'>
|
||||||
|
Dataset ID
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
|
||||||
|
### Request Body
|
||||||
|
<Properties>
|
||||||
|
<Property name='original_document_id' type='string' key='original_document_id'>
|
||||||
|
Source document ID (optional)
|
||||||
|
- Used to re-upload the document or modify the document cleaning and segmentation configuration. The missing information is copied from the source document
|
||||||
|
- The source document cannot be an archived document
|
||||||
|
- When original_document_id is passed in, the update operation is performed on behalf of the document. process_rule is a fillable item. If not filled in, the segmentation method of the source document will be used by defaul
|
||||||
|
- When original_document_id is not passed in, the new operation is performed on behalf of the document, and process_rule is required
|
||||||
|
</Property>
|
||||||
|
<Property name='file' type='multipart/form-data' key='file'>
|
||||||
|
Files that need to be uploaded.
|
||||||
|
</Property>
|
||||||
|
<Property name='indexing_technique' type='string' key='indexing_technique'>
|
||||||
|
Index mode
|
||||||
|
- high_quality High quality: embedding using embedding model, built as vector database index
|
||||||
|
- economy Economy: Build using inverted index of Keyword Table Index
|
||||||
|
</Property>
|
||||||
|
<Property name='process_rule' type='object' key='process_rule'>
|
||||||
|
Processing rules
|
||||||
|
- mode (string) Cleaning, segmentation mode, automatic / custom
|
||||||
|
- rules (text) Custom rules (in automatic mode, this field is empty)
|
||||||
|
- pre_processing_rules (array[object]) Preprocessing rules
|
||||||
|
- id (string) Unique identifier for the preprocessing rule
|
||||||
|
- enumerate
|
||||||
|
- remove_extra_spaces Replace consecutive spaces, newlines, tabs
|
||||||
|
- remove_urls_emails Delete URL, email address
|
||||||
|
- enabled (bool) Whether to select this rule or not. If no document ID is passed in, it represents the default value.
|
||||||
|
- segmentation (object) segmentation rules
|
||||||
|
- separator Custom segment identifier, currently only allows one delimiter to be set. Default is \n
|
||||||
|
- max_tokens Maximum length (token) defaults to 1000
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
</Col>
|
||||||
|
<Col sticky>
|
||||||
|
<CodeGroup
|
||||||
|
title="Request"
|
||||||
|
tag="POST"
|
||||||
|
label="/datasets/{dataset_id}/document/create_by_file"
|
||||||
|
targetCode={`curl --location POST '${props.apiBaseUrl}/datasets/{dataset_id}/document/create_by_file' \\\n--header 'Authorization: Bearer {api_key}' \\\n--form 'data="{"name":"Dify","indexing_technique":"high_quality","process_rule":{"rules":{"pre_processing_rules":[{"id":"remove_extra_spaces","enabled":true},{"id":"remove_urls_emails","enabled":true}],"segmentation":{"separator":"###","max_tokens":500}},"mode":"custom"}}";type=text/plain' \\\n--form 'file=@"/path/to/file"'`}
|
||||||
|
>
|
||||||
|
```bash {{ title: 'cURL' }}
|
||||||
|
curl --location POST 'https://api.dify.ai/v1/datasets/{dataset_id}/document/create_by_file' \
|
||||||
|
--header 'Authorization: Bearer {api_key}' \
|
||||||
|
--form 'data="{\"name\":\"Dify\",\"indexing_technique\":\"high_quality\",\"process_rule\":{\"rules\":{\"pre_processing_rules\":[{\"id\":\"remove_extra_spaces\",\"enabled\":true},{\"id\":\"remove_urls_emails\",\"enabled\":true}],\"segmentation\":{\"separator\":\"###\",\"max_tokens\":500}},\"mode\":\"custom\"}}";type=text/plain' \
|
||||||
|
--form 'file=@"/path/to/file"'
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
<CodeGroup title="Response">
|
||||||
|
```json {{ title: 'Response' }}
|
||||||
|
{
|
||||||
|
"document": {
|
||||||
|
"id": "",
|
||||||
|
"position": 1,
|
||||||
|
"data_source_type": "upload_file",
|
||||||
|
"data_source_info": {
|
||||||
|
"upload_file_id": ""
|
||||||
|
},
|
||||||
|
"dataset_process_rule_id": "",
|
||||||
|
"name": "Dify.txt",
|
||||||
|
"created_from": "api",
|
||||||
|
"created_by": "",
|
||||||
|
"created_at": 1695308667,
|
||||||
|
"tokens": 0,
|
||||||
|
"indexing_status": "waiting",
|
||||||
|
"error": null,
|
||||||
|
"enabled": true,
|
||||||
|
"disabled_at": null,
|
||||||
|
"disabled_by": null,
|
||||||
|
"archived": false,
|
||||||
|
"display_status": "queuing",
|
||||||
|
"word_count": 0,
|
||||||
|
"hit_count": 0,
|
||||||
|
"doc_form": "text_model"
|
||||||
|
},
|
||||||
|
"batch": ""
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
<Heading
|
||||||
|
url='/datasets/{dataset_id}/documents/{document_id}/update_by_text'
|
||||||
|
method='POST'
|
||||||
|
title='Update document via text'
|
||||||
|
name='#update_by_text'
|
||||||
|
/>
|
||||||
|
<Row>
|
||||||
|
<Col>
|
||||||
|
This api is based on an existing dataset and updates the document through text based on this dataset.
|
||||||
|
|
||||||
|
### Path Params
|
||||||
|
<Properties>
|
||||||
|
<Property name='dataset_id' type='string' key='dataset_id'>
|
||||||
|
Dataset ID
|
||||||
|
</Property>
|
||||||
|
<Property name='document_id' type='string' key='document_id'>
|
||||||
|
Document ID
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
|
||||||
|
### Request Body
|
||||||
|
<Properties>
|
||||||
|
<Property name='name' type='string' key='name'>
|
||||||
|
Document name (optional)
|
||||||
|
</Property>
|
||||||
|
<Property name='text' type='string' key='text'>
|
||||||
|
Document content (optional)
|
||||||
|
</Property>
|
||||||
|
<Property name='process_rule' type='object' key='process_rule'>
|
||||||
|
Processing rules
|
||||||
|
- mode (string) Cleaning, segmentation mode, automatic / custom
|
||||||
|
- rules (text) Custom rules (in automatic mode, this field is empty)
|
||||||
|
- pre_processing_rules (array[object]) Preprocessing rules
|
||||||
|
- id (string) Unique identifier for the preprocessing rule
|
||||||
|
- enumerate
|
||||||
|
- remove_extra_spaces Replace consecutive spaces, newlines, tabs
|
||||||
|
- remove_urls_emails Delete URL, email address
|
||||||
|
- enabled (bool) Whether to select this rule or not. If no document ID is passed in, it represents the default value.
|
||||||
|
- segmentation (object) segmentation rules
|
||||||
|
- separator Custom segment identifier, currently only allows one delimiter to be set. Default is \n
|
||||||
|
- max_tokens Maximum length (token) defaults to 1000
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
</Col>
|
||||||
|
<Col sticky>
|
||||||
|
<CodeGroup
|
||||||
|
title="Request"
|
||||||
|
tag="POST"
|
||||||
|
label="/datasets/{dataset_id}/documents/{document_id}/update_by_text"
|
||||||
|
targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/update_by_text' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name","text": "text"}'`}
|
||||||
|
>
|
||||||
|
```bash {{ title: 'cURL' }}
|
||||||
|
curl --location --request POST 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{document_id}/update_by_text' \
|
||||||
|
--header 'Authorization: Bearer {api_key}' \
|
||||||
|
--header 'Content-Type: application/json' \
|
||||||
|
--data-raw '{
|
||||||
|
"name": "name",
|
||||||
|
"text": "text"
|
||||||
|
}'
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
<CodeGroup title="Response">
|
||||||
|
```json {{ title: 'Response' }}
|
||||||
|
{
|
||||||
|
"document": {
|
||||||
|
"id": "",
|
||||||
|
"position": 1,
|
||||||
|
"data_source_type": "upload_file",
|
||||||
|
"data_source_info": {
|
||||||
|
"upload_file_id": ""
|
||||||
|
},
|
||||||
|
"dataset_process_rule_id": "",
|
||||||
|
"name": "name.txt",
|
||||||
|
"created_from": "api",
|
||||||
|
"created_by": "",
|
||||||
|
"created_at": 1695308667,
|
||||||
|
"tokens": 0,
|
||||||
|
"indexing_status": "waiting",
|
||||||
|
"error": null,
|
||||||
|
"enabled": true,
|
||||||
|
"disabled_at": null,
|
||||||
|
"disabled_by": null,
|
||||||
|
"archived": false,
|
||||||
|
"display_status": "queuing",
|
||||||
|
"word_count": 0,
|
||||||
|
"hit_count": 0,
|
||||||
|
"doc_form": "text_model"
|
||||||
|
},
|
||||||
|
"batch": ""
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
<Heading
|
||||||
|
url='/datasets/{dataset_id}/documents/{document_id}/update_by_file'
|
||||||
|
method='POST'
|
||||||
|
title='Update a document from a file'
|
||||||
|
name='#update_by_file'
|
||||||
|
/>
|
||||||
|
<Row>
|
||||||
|
<Col>
|
||||||
|
This api is based on an existing dataset, and updates documents through files based on this dataset
|
||||||
|
|
||||||
|
### Path Params
|
||||||
|
<Properties>
|
||||||
|
<Property name='dataset_id' type='string' key='dataset_id'>
|
||||||
|
Dataset ID
|
||||||
|
</Property>
|
||||||
|
<Property name='document_id' type='string' key='document_id'>
|
||||||
|
Document ID
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
|
||||||
|
### Request Body
|
||||||
|
<Properties>
|
||||||
|
<Property name='name' type='string' key='name'>
|
||||||
|
Document name (optional)
|
||||||
|
</Property>
|
||||||
|
<Property name='file' type='multipart/form-data' key='file'>
|
||||||
|
Files to be uploaded
|
||||||
|
</Property>
|
||||||
|
<Property name='process_rule' type='object' key='process_rule'>
|
||||||
|
Processing rules
|
||||||
|
- mode (string) Cleaning, segmentation mode, automatic / custom
|
||||||
|
- rules (text) Custom rules (in automatic mode, this field is empty)
|
||||||
|
- pre_processing_rules (array[object]) Preprocessing rules
|
||||||
|
- id (string) Unique identifier for the preprocessing rule
|
||||||
|
- enumerate
|
||||||
|
- remove_extra_spaces Replace consecutive spaces, newlines, tabs
|
||||||
|
- remove_urls_emails Delete URL, email address
|
||||||
|
- enabled (bool) Whether to select this rule or not. If no document ID is passed in, it represents the default value.
|
||||||
|
- segmentation (object) segmentation rules
|
||||||
|
- separator Custom segment identifier, currently only allows one delimiter to be set. Default is \n
|
||||||
|
- max_tokens Maximum length (token) defaults to 1000
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
</Col>
|
||||||
|
<Col sticky>
|
||||||
|
<CodeGroup
|
||||||
|
title="Request"
|
||||||
|
tag="POST"
|
||||||
|
label="/datasets/{dataset_id}/documents/{document_id}/update_by_file"
|
||||||
|
targetCode={`curl --location POST '${props.apiBaseUrl}/datasets/{dataset_id}/document/{document_id}/create_by_file' \\\n--header 'Authorization: Bearer {api_key}' \\\n--form 'data="{"name":"Dify","indexing_technique":"high_quality","process_rule":{"rules":{"pre_processing_rules":[{"id":"remove_extra_spaces","enabled":true},{"id":"remove_urls_emails","enabled":true}],"segmentation":{"separator":"###","max_tokens":500}},"mode":"custom"}}";type=text/plain' \\\n--form 'file=@"/path/to/file"'`}
|
||||||
|
>
|
||||||
|
```bash {{ title: 'cURL' }}
|
||||||
|
curl --location POST 'https://api.dify.ai/v1/datasets/{dataset_id}/document/{document_id}/create_by_file' \
|
||||||
|
--header 'Authorization: Bearer {api_key}' \
|
||||||
|
--form 'data="{\"name\":\"Dify\",\"indexing_technique\":\"high_quality\",\"process_rule\":{\"rules\":{\"pre_processing_rules\":[{\"id\":\"remove_extra_spaces\",\"enabled\":true},{\"id\":\"remove_urls_emails\",\"enabled\":true}],\"segmentation\":{\"separator\":\"###\",\"max_tokens\":500}},\"mode\":\"custom\"}}";type=text/plain' \
|
||||||
|
--form 'file=@"/path/to/file"'
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
<CodeGroup title="Response">
|
||||||
|
```json {{ title: 'Response' }}
|
||||||
|
{
|
||||||
|
"document": {
|
||||||
|
"id": "",
|
||||||
|
"position": 1,
|
||||||
|
"data_source_type": "upload_file",
|
||||||
|
"data_source_info": {
|
||||||
|
"upload_file_id": ""
|
||||||
|
},
|
||||||
|
"dataset_process_rule_id": "",
|
||||||
|
"name": "Dify.txt",
|
||||||
|
"created_from": "api",
|
||||||
|
"created_by": "",
|
||||||
|
"created_at": 1695308667,
|
||||||
|
"tokens": 0,
|
||||||
|
"indexing_status": "waiting",
|
||||||
|
"error": null,
|
||||||
|
"enabled": true,
|
||||||
|
"disabled_at": null,
|
||||||
|
"disabled_by": null,
|
||||||
|
"archived": false,
|
||||||
|
"display_status": "queuing",
|
||||||
|
"word_count": 0,
|
||||||
|
"hit_count": 0,
|
||||||
|
"doc_form": "text_model"
|
||||||
|
},
|
||||||
|
"batch": "20230921150427533684"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
<Heading
|
||||||
|
url='/datasets/{dataset_id}/batch/{batch}/indexing-status'
|
||||||
|
method='GET'
|
||||||
|
title='Get document embedding status (progress)'
|
||||||
|
name='#indexing_status'
|
||||||
|
/>
|
||||||
|
<Row>
|
||||||
|
<Col>
|
||||||
|
### Path Params
|
||||||
|
<Properties>
|
||||||
|
<Property name='dataset_id' type='string' key='dataset_id'>
|
||||||
|
Dataset ID
|
||||||
|
</Property>
|
||||||
|
<Property name='batch' type='string' key='batch'>
|
||||||
|
Batch number of uploaded documents
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
</Col>
|
||||||
|
<Col sticky>
|
||||||
|
<CodeGroup
|
||||||
|
title="Request"
|
||||||
|
tag="GET"
|
||||||
|
label="/datasets/{dataset_id}/batch/{batch}/indexing-status"
|
||||||
|
targetCode={`curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{batch}/indexing-status' \\\n--header 'Authorization: Bearer {api_key}'`}
|
||||||
|
>
|
||||||
|
```bash {{ title: 'cURL' }}
|
||||||
|
curl --location --request GET 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{batch}/indexing-status' \
|
||||||
|
--header 'Authorization: Bearer {api_key}' \
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
<CodeGroup title="Response">
|
||||||
|
```json {{ title: 'Response' }}
|
||||||
|
{
|
||||||
|
"data":[{
|
||||||
|
"id": "",
|
||||||
|
"indexing_status": "indexing",
|
||||||
|
"processing_started_at": 1681623462.0,
|
||||||
|
"parsing_completed_at": 1681623462.0,
|
||||||
|
"cleaning_completed_at": 1681623462.0,
|
||||||
|
"splitting_completed_at": 1681623462.0,
|
||||||
|
"completed_at": null,
|
||||||
|
"paused_at": null,
|
||||||
|
"error": null,
|
||||||
|
"stopped_at": null,
|
||||||
|
"completed_segments": 24,
|
||||||
|
"total_segments": 100
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
<Heading
|
||||||
|
url='/datasets/{dataset_id}/documents/{document_id}'
|
||||||
|
method='DELETE'
|
||||||
|
title='Delete document'
|
||||||
|
name='#delete_document'
|
||||||
|
/>
|
||||||
|
<Row>
|
||||||
|
<Col>
|
||||||
|
### Path Params
|
||||||
|
<Properties>
|
||||||
|
<Property name='dataset_id' type='string' key='dataset_id'>
|
||||||
|
Dataset ID
|
||||||
|
</Property>
|
||||||
|
<Property name='document_id' type='string' key='document_id'>
|
||||||
|
Document ID
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
</Col>
|
||||||
|
<Col sticky>
|
||||||
|
<CodeGroup
|
||||||
|
title="Request"
|
||||||
|
tag="DELETE"
|
||||||
|
label="/datasets/{dataset_id}/documents/{document_id}"
|
||||||
|
targetCode={`curl --location --request DELETE '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}' \\\n--header 'Authorization: Bearer {api_key}'`}
|
||||||
|
>
|
||||||
|
```bash {{ title: 'cURL' }}
|
||||||
|
curl --location --request DELETE 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{document_id}' \
|
||||||
|
--header 'Authorization: Bearer {api_key}' \
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
<CodeGroup title="Response">
|
||||||
|
```json {{ title: 'Response' }}
|
||||||
|
{
|
||||||
|
"result": "success"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
<Heading
|
||||||
|
url='/datasets/{dataset_id}/documents'
|
||||||
|
method='GET'
|
||||||
|
title='Dataset document list'
|
||||||
|
name='#dataset_document_list'
|
||||||
|
/>
|
||||||
|
<Row>
|
||||||
|
<Col>
|
||||||
|
### Path Params
|
||||||
|
<Properties>
|
||||||
|
<Property name='dataset_id' type='string' key='dataset_id'>
|
||||||
|
Dataset ID
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
|
||||||
|
### Path Query
|
||||||
|
<Properties>
|
||||||
|
<Property name='keyword' type='string' key='keyword'>
|
||||||
|
Search keywords, currently only search document names(optional)
|
||||||
|
</Property>
|
||||||
|
<Property name='page' type='string' key='page'>
|
||||||
|
Page number(optional)
|
||||||
|
</Property>
|
||||||
|
<Property name='limit' type='string' key='limit'>
|
||||||
|
Number of items returned, default 20, range 1-100(optional)
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
</Col>
|
||||||
|
<Col sticky>
|
||||||
|
<CodeGroup
|
||||||
|
title="Request"
|
||||||
|
tag="GET"
|
||||||
|
label="/datasets/{dataset_id}/documents"
|
||||||
|
targetCode={`curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents' \\\n--header 'Authorization: Bearer {api_key}'`}
|
||||||
|
>
|
||||||
|
```bash {{ title: 'cURL' }}
|
||||||
|
curl --location --request GET 'https://api.dify.ai/v1/datasets/{dataset_id}/documents' \
|
||||||
|
--header 'Authorization: Bearer {api_key}' \
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
<CodeGroup title="Response">
|
||||||
|
```json {{ title: 'Response' }}
|
||||||
|
{
|
||||||
|
"data": [
|
||||||
|
{
|
||||||
|
"id": "",
|
||||||
|
"position": 1,
|
||||||
|
"data_source_type": "file_upload",
|
||||||
|
"data_source_info": null,
|
||||||
|
"dataset_process_rule_id": null,
|
||||||
|
"name": "dify",
|
||||||
|
"created_from": "",
|
||||||
|
"created_by": "",
|
||||||
|
"created_at": 1681623639,
|
||||||
|
"tokens": 0,
|
||||||
|
"indexing_status": "waiting",
|
||||||
|
"error": null,
|
||||||
|
"enabled": true,
|
||||||
|
"disabled_at": null,
|
||||||
|
"disabled_by": null,
|
||||||
|
"archived": false
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"has_more": false,
|
||||||
|
"limit": 20,
|
||||||
|
"total": 9,
|
||||||
|
"page": 1
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
<Heading
|
||||||
|
url='/datasets/{dataset_id}/documents/{document_id}/segments'
|
||||||
|
method='POST'
|
||||||
|
title='Add segment'
|
||||||
|
name='#create_new_segment'
|
||||||
|
/>
|
||||||
|
<Row>
|
||||||
|
<Col>
|
||||||
|
### Path Params
|
||||||
|
<Properties>
|
||||||
|
<Property name='dataset_id' type='string' key='dataset_id'>
|
||||||
|
Dataset ID
|
||||||
|
</Property>
|
||||||
|
<Property name='document_id' type='string' key='document_id'>
|
||||||
|
Document ID
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
|
||||||
|
### Request Body
|
||||||
|
<Properties>
|
||||||
|
<Property name='segments' type='object list' key='segments'>
|
||||||
|
segments (object list) Segmented content
|
||||||
|
- content (text) Text content/question content, required
|
||||||
|
- answer(text) Answer content, if the mode of the data set is qa mode, pass the value(optional)
|
||||||
|
- keywords(list) Keywords(optional)
|
||||||
|
</Property>
|
||||||
|
</Properties>
|
||||||
|
</Col>
|
||||||
|
<Col sticky>
|
||||||
|
<CodeGroup
|
||||||
|
title="Request"
|
||||||
|
tag="POST"
|
||||||
|
label="/datasets/{dataset_id}/documents/{document_id}/segments"
|
||||||
|
targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/segments' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"segments": [{"content": "1","answer": "1","keywords": ["a"]}]}'`}
|
||||||
|
>
|
||||||
|
```bash {{ title: 'cURL' }}
|
||||||
|
curl --location --request POST 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{document_id}/segments' \
|
||||||
|
--header 'Authorization: Bearer {api_key}' \
|
||||||
|
--header 'Content-Type: application/json' \
|
||||||
|
--data-raw '{
|
||||||
|
"segments": [
|
||||||
|
{
|
||||||
|
"content": "1",
|
||||||
|
"answer": "1",
|
||||||
|
"keywords": ["a"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}'
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
<CodeGroup title="Response">
|
||||||
|
```json {{ title: 'Response' }}
|
||||||
|
{
|
||||||
|
"data": [{
|
||||||
|
"id": "",
|
||||||
|
"position": 1,
|
||||||
|
"document_id": "",
|
||||||
|
"content": "1",
|
||||||
|
"answer": "1",
|
||||||
|
"word_count": 25,
|
||||||
|
"tokens": 0,
|
||||||
|
"keywords": [
|
||||||
|
"a"
|
||||||
|
],
|
||||||
|
"index_node_id": "",
|
||||||
|
"index_node_hash": "",
|
||||||
|
"hit_count": 0,
|
||||||
|
"enabled": true,
|
||||||
|
"disabled_at": null,
|
||||||
|
"disabled_by": null,
|
||||||
|
"status": "completed",
|
||||||
|
"created_by": "",
|
||||||
|
"created_at": 1695312007,
|
||||||
|
"indexing_at": 1695312007,
|
||||||
|
"completed_at": 1695312007,
|
||||||
|
"error": null,
|
||||||
|
"stopped_at": null
|
||||||
|
}],
|
||||||
|
"doc_form": "text_model"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
</CodeGroup>
|
||||||
|
</Col>
|
||||||
|
</Row>
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
Error message
|
||||||
|
- **document_indexing**: Document indexing failed
|
||||||
|
- **provider_not_initialize**: Embedding model is not configured
|
||||||
|
- **not_found**, Document does not exist
|
||||||
|
- **dataset_name_duplicate**: Duplicate dataset name
|
||||||
|
- **provider_quota_exceeded**: Model quota exceeds limit
|
||||||
|
- **dataset_not_initialized**: The dataset has not been initialized yet
|
||||||
|
- **unsupported_file_type**: Unsupported file types.
|
||||||
|
- Currently only supports, txt, markdown, md, pdf, html, htm, xlsx, docx, csv
|
||||||
|
- **too_many_files**: There are too many files. Currently, only a single file is uploaded
|
||||||
|
- **file_too_large*: The file is too large, support below 15M based on you environment configuration
|
||||||
@ -0,0 +1,55 @@
|
|||||||
|
import type { FC } from 'react'
|
||||||
|
|
||||||
|
type Option = {
|
||||||
|
value: string
|
||||||
|
text: string
|
||||||
|
}
|
||||||
|
type TabSliderProps = {
|
||||||
|
value: string
|
||||||
|
onChange: (v: string) => void
|
||||||
|
options: Option[]
|
||||||
|
}
|
||||||
|
const TabSlider: FC<TabSliderProps> = ({
|
||||||
|
value,
|
||||||
|
onChange,
|
||||||
|
options,
|
||||||
|
}) => {
|
||||||
|
const currentIndex = options.findIndex(option => option.value === value)
|
||||||
|
const current = options[currentIndex]
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className='relative flex p-0.5 rounded-lg bg-gray-200'>
|
||||||
|
{
|
||||||
|
options.map((option, index) => (
|
||||||
|
<div
|
||||||
|
key={option.value}
|
||||||
|
className={`
|
||||||
|
flex justify-center items-center w-[118px] h-7 text-[13px]
|
||||||
|
font-semibold text-gray-600 rounded-[7px] cursor-pointer
|
||||||
|
hover:bg-gray-50
|
||||||
|
${index !== options.length - 1 && 'mr-[1px]'}
|
||||||
|
`}
|
||||||
|
onClick={() => onChange(option.value)}
|
||||||
|
>
|
||||||
|
{option.text}
|
||||||
|
</div>
|
||||||
|
))
|
||||||
|
}
|
||||||
|
{
|
||||||
|
current && (
|
||||||
|
<div
|
||||||
|
className={`
|
||||||
|
absolute flex justify-center items-center w-[118px] h-7 bg-white text-[13px] font-semibold text-primary-600
|
||||||
|
border-[0.5px] border-gray-200 rounded-[7px] shadow-xs transition-transform
|
||||||
|
`}
|
||||||
|
style={{ transform: `translateX(${currentIndex * 118 + 1}px)` }}
|
||||||
|
>
|
||||||
|
{current.text}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export default TabSlider
|
||||||
Loading…
Reference in New Issue