Merge branch 'feat/rag-pipeline' of https://github.com/langgenius/dify into feat/rag-pipeline

pull/21398/head
twwu 12 months ago
commit d725aa8791

@ -43,6 +43,7 @@ select = [
"S307", # suspicious-eval-usage, disallow use of `eval` and `ast.literal_eval` "S307", # suspicious-eval-usage, disallow use of `eval` and `ast.literal_eval`
"S301", # suspicious-pickle-usage, disallow use of `pickle` and its wrappers. "S301", # suspicious-pickle-usage, disallow use of `pickle` and its wrappers.
"S302", # suspicious-marshal-usage, disallow use of `marshal` module "S302", # suspicious-marshal-usage, disallow use of `marshal` module
"S311", # suspicious-non-cryptographic-random-usage
] ]
ignore = [ ignore = [

@ -1,5 +1,5 @@
import logging import logging
import random import secrets
from typing import cast from typing import cast
from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity from core.app.entities.app_invoke_entities import ModelConfigWithCredentialsEntity
@ -38,7 +38,7 @@ def check_moderation(tenant_id: str, model_config: ModelConfigWithCredentialsEnt
if len(text_chunks) == 0: if len(text_chunks) == 0:
return True return True
text_chunk = random.choice(text_chunks) text_chunk = secrets.choice(text_chunks)
try: try:
model_provider_factory = ModelProviderFactory(tenant_id) model_provider_factory = ModelProviderFactory(tenant_id)

@ -1,8 +1,9 @@
import base64 import base64
import json import json
import secrets
import string
from collections.abc import Mapping from collections.abc import Mapping
from copy import deepcopy from copy import deepcopy
from random import randint
from typing import Any, Literal from typing import Any, Literal
from urllib.parse import urlencode, urlparse from urllib.parse import urlencode, urlparse
@ -434,4 +435,4 @@ def _generate_random_string(n: int) -> str:
>>> _generate_random_string(5) >>> _generate_random_string(5)
'abcde' 'abcde'
""" """
return "".join([chr(randint(97, 122)) for _ in range(n)]) return "".join(secrets.choice(string.ascii_lowercase) for _ in range(n))

@ -1,7 +1,7 @@
import json import json
import logging import logging
import random
import re import re
import secrets
import string import string
import subprocess import subprocess
import time import time
@ -176,7 +176,7 @@ def generate_string(n):
letters_digits = string.ascii_letters + string.digits letters_digits = string.ascii_letters + string.digits
result = "" result = ""
for i in range(n): for i in range(n):
result += random.choice(letters_digits) result += secrets.choice(letters_digits)
return result return result

@ -1,7 +1,6 @@
import base64 import base64
import json import json
import logging import logging
import random
import secrets import secrets
import uuid import uuid
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
@ -261,7 +260,7 @@ class AccountService:
@staticmethod @staticmethod
def generate_account_deletion_verification_code(account: Account) -> tuple[str, str]: def generate_account_deletion_verification_code(account: Account) -> tuple[str, str]:
code = "".join([str(random.randint(0, 9)) for _ in range(6)]) code = "".join([str(secrets.randbelow(exclusive_upper_bound=10)) for _ in range(6)])
token = TokenManager.generate_token( token = TokenManager.generate_token(
account=account, token_type="account_deletion", additional_data={"code": code} account=account, token_type="account_deletion", additional_data={"code": code}
) )
@ -429,7 +428,7 @@ class AccountService:
additional_data: dict[str, Any] = {}, additional_data: dict[str, Any] = {},
): ):
if not code: if not code:
code = "".join([str(random.randint(0, 9)) for _ in range(6)]) code = "".join([str(secrets.randbelow(exclusive_upper_bound=10)) for _ in range(6)])
additional_data["code"] = code additional_data["code"] = code
token = TokenManager.generate_token( token = TokenManager.generate_token(
account=account, email=email, token_type="reset_password", additional_data=additional_data account=account, email=email, token_type="reset_password", additional_data=additional_data
@ -456,7 +455,7 @@ class AccountService:
raise EmailCodeLoginRateLimitExceededError() raise EmailCodeLoginRateLimitExceededError()
code = "".join([str(random.randint(0, 9)) for _ in range(6)]) code = "".join([str(secrets.randbelow(exclusive_upper_bound=10)) for _ in range(6)])
token = TokenManager.generate_token( token = TokenManager.generate_token(
account=account, email=email, token_type="email_code_login", additional_data={"code": code} account=account, email=email, token_type="email_code_login", additional_data={"code": code}
) )

@ -2,7 +2,7 @@ import copy
import datetime import datetime
import json import json
import logging import logging
import random import secrets
import time import time
import uuid import uuid
from collections import Counter from collections import Counter
@ -970,7 +970,7 @@ class DocumentService:
documents.append(document) documents.append(document)
batch = document.batch batch = document.batch
else: else:
batch = time.strftime("%Y%m%d%H%M%S") + str(random.randint(100000, 999999)) batch = time.strftime("%Y%m%d%H%M%S") + str(100000 + secrets.randbelow(exclusive_upper_bound=900000))
# save process rule # save process rule
if not dataset_process_rule: if not dataset_process_rule:
process_rule = knowledge_config.process_rule process_rule = knowledge_config.process_rule

@ -1,4 +1,4 @@
import random import secrets
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from typing import Any, Optional, cast from typing import Any, Optional, cast
@ -66,7 +66,7 @@ class WebAppAuthService:
if email is None: if email is None:
raise ValueError("Email must be provided.") raise ValueError("Email must be provided.")
code = "".join([str(random.randint(0, 9)) for _ in range(6)]) code = "".join([str(secrets.randbelow(exclusive_upper_bound=10)) for _ in range(6)])
token = TokenManager.generate_token( token = TokenManager.generate_token(
account=account, email=email, token_type="webapp_email_code_login", additional_data={"code": code} account=account, email=email, token_type="webapp_email_code_login", additional_data={"code": code}
) )

@ -1,4 +1,4 @@
import random import secrets
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
import pytest import pytest
@ -34,7 +34,7 @@ def test_retry_logic_success(mock_request):
side_effects = [] side_effects = []
for _ in range(SSRF_DEFAULT_MAX_RETRIES): for _ in range(SSRF_DEFAULT_MAX_RETRIES):
status_code = random.choice(STATUS_FORCELIST) status_code = secrets.choice(STATUS_FORCELIST)
mock_response = MagicMock() mock_response = MagicMock()
mock_response.status_code = status_code mock_response.status_code = status_code
side_effects.append(mock_response) side_effects.append(mock_response)

@ -18,9 +18,10 @@ const queryDateFormat = 'YYYY-MM-DD HH:mm'
export type IChartViewProps = { export type IChartViewProps = {
appId: string appId: string
headerRight: React.ReactNode
} }
export default function ChartView({ appId }: IChartViewProps) { export default function ChartView({ appId, headerRight }: IChartViewProps) {
const { t } = useTranslation() const { t } = useTranslation()
const appDetail = useAppStore(state => state.appDetail) const appDetail = useAppStore(state => state.appDetail)
const isChatApp = appDetail?.mode !== 'completion' && appDetail?.mode !== 'workflow' const isChatApp = appDetail?.mode !== 'completion' && appDetail?.mode !== 'workflow'
@ -46,19 +47,22 @@ export default function ChartView({ appId }: IChartViewProps) {
return ( return (
<div> <div>
<div className='system-xl-semibold mb-4 mt-8 flex flex-row items-center text-text-primary'> <div className='mb-4 flex items-center justify-between'>
<span className='mr-3'>{t('appOverview.analysis.title')}</span> <div className='system-xl-semibold flex flex-row items-center text-text-primary'>
<SimpleSelect <span className='mr-3'>{t('appOverview.analysis.title')}</span>
items={Object.entries(TIME_PERIOD_MAPPING).map(([k, v]) => ({ value: k, name: t(`appLog.filter.period.${v.name}`) }))} <SimpleSelect
className='mt-0 !w-40' items={Object.entries(TIME_PERIOD_MAPPING).map(([k, v]) => ({ value: k, name: t(`appLog.filter.period.${v.name}`) }))}
onSelect={(item) => { className='mt-0 !w-40'
const id = item.value onSelect={(item) => {
const value = TIME_PERIOD_MAPPING[id]?.value ?? '-1' const id = item.value
const name = item.name || t('appLog.filter.period.allTime') const value = TIME_PERIOD_MAPPING[id]?.value ?? '-1'
onSelect({ value, name }) const name = item.name || t('appLog.filter.period.allTime')
}} onSelect({ value, name })
defaultValue={'2'} }}
/> defaultValue={'2'}
/>
</div>
{headerRight}
</div> </div>
{!isWorkflow && ( {!isWorkflow && (
<div className='mb-6 grid w-full grid-cols-1 gap-6 xl:grid-cols-2'> <div className='mb-6 grid w-full grid-cols-1 gap-6 xl:grid-cols-2'>

@ -1,6 +1,5 @@
import React from 'react' import React from 'react'
import ChartView from './chartView' import ChartView from './chartView'
import CardView from './cardView'
import TracingPanel from './tracing/panel' import TracingPanel from './tracing/panel'
import ApikeyInfoPanel from '@/app/components/app/overview/apikey-info-panel' import ApikeyInfoPanel from '@/app/components/app/overview/apikey-info-panel'
@ -18,9 +17,10 @@ const Overview = async (props: IDevelopProps) => {
return ( return (
<div className="h-full overflow-scroll bg-chatbot-bg px-4 py-6 sm:px-12"> <div className="h-full overflow-scroll bg-chatbot-bg px-4 py-6 sm:px-12">
<ApikeyInfoPanel /> <ApikeyInfoPanel />
<TracingPanel /> <ChartView
<CardView appId={appId} /> appId={appId}
<ChartView appId={appId} /> headerRight={<TracingPanel />}
/>
</div> </div>
) )
} }

@ -154,7 +154,6 @@ const Panel: FC = () => {
if (!isLoaded) { if (!isLoaded) {
return ( return (
<div className='mb-3 flex items-center justify-between'> <div className='mb-3 flex items-center justify-between'>
<Title className='h-[41px]' />
<div className='w-[200px]'> <div className='w-[200px]'>
<Loading /> <Loading />
</div> </div>
@ -163,8 +162,7 @@ const Panel: FC = () => {
} }
return ( return (
<div className={cn('mb-3 flex items-center justify-between')}> <div className={cn('flex items-center justify-between')}>
<Title className='h-[41px]' />
<div <div
className={cn( className={cn(
'flex cursor-pointer items-center rounded-xl border-l-[0.5px] border-t border-effects-highlight bg-background-default-dodge p-2 shadow-xs hover:border-effects-highlight-lightmode-off hover:bg-background-default-lighter', 'flex cursor-pointer items-center rounded-xl border-l-[0.5px] border-t border-effects-highlight bg-background-default-dodge p-2 shadow-xs hover:border-effects-highlight-lightmode-off hover:bg-background-default-lighter',

@ -1,14 +1,60 @@
import { memo } from 'react' import {
memo,
useState,
} from 'react'
import { useStore } from '../../workflow/store' import { useStore } from '../../workflow/store'
import InputField from './input-field' import InputField from './input-field'
import RagPipelinePanel from './panel' import RagPipelinePanel from './panel'
import RagPipelineHeader from './rag-pipeline-header' import RagPipelineHeader from './rag-pipeline-header'
import type { EnvironmentVariable } from '@/app/components/workflow/types'
import { DSL_EXPORT_CHECK } from '@/app/components/workflow/constants'
import UpdateDSLModal from '@/app/components/workflow/update-dsl-modal'
import DSLExportConfirmModal from '@/app/components/workflow/dsl-export-confirm-modal'
import {
useDSL,
usePanelInteractions,
} from '@/app/components/workflow/hooks'
import { useEventEmitterContextContext } from '@/context/event-emitter'
const RagPipelineChildren = () => { const RagPipelineChildren = () => {
const { eventEmitter } = useEventEmitterContextContext()
const [secretEnvList, setSecretEnvList] = useState<EnvironmentVariable[]>([])
const showInputFieldDialog = useStore(state => state.showInputFieldDialog) const showInputFieldDialog = useStore(state => state.showInputFieldDialog)
const showImportDSLModal = useStore(s => s.showImportDSLModal)
const setShowImportDSLModal = useStore(s => s.setShowImportDSLModal)
const {
handlePaneContextmenuCancel,
} = usePanelInteractions()
const {
exportCheck,
handleExportDSL,
} = useDSL()
eventEmitter?.useSubscription((v: any) => {
if (v.type === DSL_EXPORT_CHECK)
setSecretEnvList(v.payload.data as EnvironmentVariable[])
})
return ( return (
<> <>
{
showImportDSLModal && (
<UpdateDSLModal
onCancel={() => setShowImportDSLModal(false)}
onBackup={exportCheck!}
onImport={handlePaneContextmenuCancel}
/>
)
}
{
secretEnvList.length > 0 && (
<DSLExportConfirmModal
envList={secretEnvList}
onConfirm={handleExportDSL!}
onClose={() => setSecretEnvList([])}
/>
)
}
<RagPipelineHeader /> <RagPipelineHeader />
<RagPipelinePanel /> <RagPipelinePanel />
{ {

@ -9,7 +9,10 @@ import {
RiPlayCircleLine, RiPlayCircleLine,
RiTerminalBoxLine, RiTerminalBoxLine,
} from '@remixicon/react' } from '@remixicon/react'
import { useKeyPress } from 'ahooks' import {
useBoolean,
useKeyPress,
} from 'ahooks'
import { useTranslation } from 'react-i18next' import { useTranslation } from 'react-i18next'
import { import {
useStore, useStore,
@ -29,6 +32,7 @@ import { useParams, useRouter } from 'next/navigation'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { useInvalid } from '@/service/use-base' import { useInvalid } from '@/service/use-base'
import { publishedPipelineInfoQueryKeyPrefix } from '@/service/use-pipeline' import { publishedPipelineInfoQueryKeyPrefix } from '@/service/use-pipeline'
import Confirm from '@/app/components/base/confirm'
const PUBLISH_SHORTCUT = ['⌘', '⇧', 'P'] const PUBLISH_SHORTCUT = ['⌘', '⇧', 'P']
@ -46,29 +50,52 @@ const Popup = () => {
const { mutateAsync: publishWorkflow } = usePublishWorkflow() const { mutateAsync: publishWorkflow } = usePublishWorkflow()
const { notify } = useToastContext() const { notify } = useToastContext()
const workflowStore = useWorkflowStore() const workflowStore = useWorkflowStore()
const [confirmVisible, {
setFalse: hideConfirm,
setTrue: showConfirm,
}] = useBoolean(false)
const [publishing, {
setFalse: hidePublishing,
setTrue: showPublishing,
}] = useBoolean(false)
const invalidPublishedPipelineInfo = useInvalid([...publishedPipelineInfoQueryKeyPrefix, pipelineId]) const invalidPublishedPipelineInfo = useInvalid([...publishedPipelineInfoQueryKeyPrefix, pipelineId])
const handlePublish = useCallback(async (params?: PublishWorkflowParams) => { const handlePublish = useCallback(async (params?: PublishWorkflowParams) => {
if (await handleCheckBeforePublish()) { if (publishing)
const res = await publishWorkflow({ return
url: `/rag/pipelines/${pipelineId}/workflows/publish`, try {
title: params?.title || '', const checked = await handleCheckBeforePublish()
releaseNotes: params?.releaseNotes || '',
})
setPublished(true)
if (res) { if (checked) {
notify({ type: 'success', message: t('common.api.actionSuccess') }) if (!publishedAt && !confirmVisible) {
workflowStore.getState().setPublishedAt(res.created_at) showConfirm()
mutateDatasetRes?.() return
invalidPublishedPipelineInfo() }
showPublishing()
const res = await publishWorkflow({
url: `/rag/pipelines/${pipelineId}/workflows/publish`,
title: params?.title || '',
releaseNotes: params?.releaseNotes || '',
})
setPublished(true)
if (res) {
notify({ type: 'success', message: t('common.api.actionSuccess') })
workflowStore.getState().setPublishedAt(res.created_at)
mutateDatasetRes?.()
invalidPublishedPipelineInfo()
}
} }
} }
else { catch {
throw new Error('Checklist failed') }
finally {
if (publishing)
hidePublishing()
if (confirmVisible)
hideConfirm()
} }
}, [handleCheckBeforePublish, publishWorkflow, pipelineId, notify, t, workflowStore, mutateDatasetRes, invalidPublishedPipelineInfo]) }, [handleCheckBeforePublish, publishWorkflow, pipelineId, notify, t, workflowStore, mutateDatasetRes, invalidPublishedPipelineInfo, showConfirm, publishedAt, confirmVisible, hidePublishing, showPublishing, hideConfirm, publishing])
useKeyPress(`${getKeyboardKeyCodeBySystem('ctrl')}.shift.p`, (e) => { useKeyPress(`${getKeyboardKeyCodeBySystem('ctrl')}.shift.p`, (e) => {
e.preventDefault() e.preventDefault()
@ -108,7 +135,7 @@ const Popup = () => {
variant='primary' variant='primary'
className='mt-3 w-full' className='mt-3 w-full'
onClick={() => handlePublish()} onClick={() => handlePublish()}
disabled={published} disabled={published || publishing}
> >
{ {
published published
@ -163,6 +190,18 @@ const Popup = () => {
</div> </div>
</Button> </Button>
</div> </div>
{
confirmVisible && (
<Confirm
isShow={confirmVisible}
title={t('pipeline.common.confirmPublish')}
content={t('pipeline.common.confirmPublishContent')}
onCancel={hideConfirm}
onConfirm={handlePublish}
isDisabled={publishing}
/>
)
}
</div> </div>
) )
} }

@ -6,6 +6,7 @@ import type { WorkflowProps } from '@/app/components/workflow'
import RagPipelineChildren from './rag-pipeline-children' import RagPipelineChildren from './rag-pipeline-children'
import { import {
useAvailableNodesMetaData, useAvailableNodesMetaData,
useDSL,
useGetRunAndTraceUrl, useGetRunAndTraceUrl,
useNodesSyncDraft, useNodesSyncDraft,
usePipelineRefreshDraft, usePipelineRefreshDraft,
@ -37,6 +38,10 @@ const RagPipelineMain = ({
} = usePipelineStartRun() } = usePipelineStartRun()
const availableNodesMetaData = useAvailableNodesMetaData() const availableNodesMetaData = useAvailableNodesMetaData()
const { getWorkflowRunAndTraceUrl } = useGetRunAndTraceUrl() const { getWorkflowRunAndTraceUrl } = useGetRunAndTraceUrl()
const {
exportCheck,
handleExportDSL,
} = useDSL()
const hooksStore = useMemo(() => { const hooksStore = useMemo(() => {
return { return {
@ -52,6 +57,8 @@ const RagPipelineMain = ({
handleStartWorkflowRun, handleStartWorkflowRun,
handleWorkflowStartRunInWorkflow, handleWorkflowStartRunInWorkflow,
getWorkflowRunAndTraceUrl, getWorkflowRunAndTraceUrl,
exportCheck,
handleExportDSL,
} }
}, [ }, [
availableNodesMetaData, availableNodesMetaData,
@ -66,6 +73,8 @@ const RagPipelineMain = ({
handleStartWorkflowRun, handleStartWorkflowRun,
handleWorkflowStartRunInWorkflow, handleWorkflowStartRunInWorkflow,
getWorkflowRunAndTraceUrl, getWorkflowRunAndTraceUrl,
exportCheck,
handleExportDSL,
]) ])
return ( return (

@ -5,3 +5,4 @@ export * from './use-pipeline-run'
export * from './use-pipeline-start-run' export * from './use-pipeline-start-run'
export * from './use-pipeline-init' export * from './use-pipeline-init'
export * from './use-get-run-and-trace-url' export * from './use-get-run-and-trace-url'
export * from './use-DSL'

@ -0,0 +1,81 @@
import {
useCallback,
useState,
} from 'react'
import { useTranslation } from 'react-i18next'
import {
DSL_EXPORT_CHECK,
} from '@/app/components/workflow/constants'
import { useNodesSyncDraft } from './use-nodes-sync-draft'
import { useEventEmitterContextContext } from '@/context/event-emitter'
import { fetchWorkflowDraft } from '@/service/workflow'
import { useToastContext } from '@/app/components/base/toast'
import { useWorkflowStore } from '@/app/components/workflow/store'
import { useExportPipelineDSL } from '@/service/use-pipeline'
export const useDSL = () => {
const { t } = useTranslation()
const { notify } = useToastContext()
const { eventEmitter } = useEventEmitterContextContext()
const [exporting, setExporting] = useState(false)
const { doSyncWorkflowDraft } = useNodesSyncDraft()
const workflowStore = useWorkflowStore()
const { mutateAsync: exportPipelineConfig } = useExportPipelineDSL()
const handleExportDSL = useCallback(async (include = false) => {
const { pipelineId, knowledgeName } = workflowStore.getState()
if (!pipelineId)
return
if (exporting)
return
try {
setExporting(true)
await doSyncWorkflowDraft()
const { data } = await exportPipelineConfig({
pipelineId,
include,
})
const a = document.createElement('a')
const file = new Blob([data], { type: 'application/yaml' })
a.href = URL.createObjectURL(file)
a.download = `${knowledgeName}.yml`
a.click()
}
catch {
notify({ type: 'error', message: t('app.exportFailed') })
}
finally {
setExporting(false)
}
}, [notify, t, doSyncWorkflowDraft, exporting, exportPipelineConfig, workflowStore])
const exportCheck = useCallback(async () => {
const { pipelineId } = workflowStore.getState()
if (!pipelineId)
return
try {
const workflowDraft = await fetchWorkflowDraft(`/rag/pipelines/${pipelineId}/workflows/draft`)
const list = (workflowDraft.environment_variables || []).filter(env => env.value_type === 'secret')
if (list.length === 0) {
handleExportDSL()
return
}
eventEmitter?.emit({
type: DSL_EXPORT_CHECK,
payload: {
data: list,
},
} as any)
}
catch {
notify({ type: 'error', message: t('app.exportFailed') })
}
}, [eventEmitter, handleExportDSL, notify, t, workflowStore])
return {
exportCheck,
handleExportDSL,
}
}

@ -24,10 +24,11 @@ export const usePipelineInit = () => {
const [data, setData] = useState<FetchWorkflowDraftResponse>() const [data, setData] = useState<FetchWorkflowDraftResponse>()
const [isLoading, setIsLoading] = useState(true) const [isLoading, setIsLoading] = useState(true)
const datasetId = useDatasetDetailContextWithSelector(s => s.dataset)?.pipeline_id const datasetId = useDatasetDetailContextWithSelector(s => s.dataset)?.pipeline_id
const knowledgeName = useDatasetDetailContextWithSelector(s => s.dataset)?.name
useEffect(() => { useEffect(() => {
workflowStore.setState({ pipelineId: datasetId }) workflowStore.setState({ pipelineId: datasetId, knowledgeName })
}, [datasetId, workflowStore]) }, [datasetId, workflowStore, knowledgeName])
usePipelineConfig() usePipelineConfig()

@ -8,6 +8,7 @@ import { transformDataSourceToTool } from '@/app/components/workflow/block-selec
export type RagPipelineSliceShape = { export type RagPipelineSliceShape = {
pipelineId: string pipelineId: string
knowledgeName: string
showInputFieldDialog: boolean showInputFieldDialog: boolean
setShowInputFieldDialog: (showInputFieldPanel: boolean) => void setShowInputFieldDialog: (showInputFieldPanel: boolean) => void
nodesDefaultConfigs: Record<string, any> nodesDefaultConfigs: Record<string, any>
@ -21,6 +22,7 @@ export type RagPipelineSliceShape = {
export type CreateRagPipelineSliceSlice = StateCreator<RagPipelineSliceShape> export type CreateRagPipelineSliceSlice = StateCreator<RagPipelineSliceShape>
export const createRagPipelineSliceSlice: StateCreator<RagPipelineSliceShape> = set => ({ export const createRagPipelineSliceSlice: StateCreator<RagPipelineSliceShape> = set => ({
pipelineId: '', pipelineId: '',
knowledgeName: '',
showInputFieldDialog: false, showInputFieldDialog: false,
setShowInputFieldDialog: showInputFieldDialog => set(() => ({ showInputFieldDialog })), setShowInputFieldDialog: showInputFieldDialog => set(() => ({ showInputFieldDialog })),
nodesDefaultConfigs: {}, nodesDefaultConfigs: {},

@ -46,7 +46,7 @@ const WorkflowChildren = () => {
showImportDSLModal && ( showImportDSLModal && (
<UpdateDSLModal <UpdateDSLModal
onCancel={() => setShowImportDSLModal(false)} onCancel={() => setShowImportDSLModal(false)}
onBackup={exportCheck} onBackup={exportCheck!}
onImport={handlePaneContextmenuCancel} onImport={handlePaneContextmenuCancel}
/> />
) )
@ -55,7 +55,7 @@ const WorkflowChildren = () => {
secretEnvList.length > 0 && ( secretEnvList.length > 0 && (
<DSLExportConfirmModal <DSLExportConfirmModal
envList={secretEnvList} envList={secretEnvList}
onConfirm={handleExportDSL} onConfirm={handleExportDSL!}
onClose={() => setSecretEnvList([])} onClose={() => setSecretEnvList([])}
/> />
) )

@ -8,6 +8,7 @@ import type { WorkflowProps } from '@/app/components/workflow'
import WorkflowChildren from './workflow-children' import WorkflowChildren from './workflow-children'
import { import {
useAvailableNodesMetaData, useAvailableNodesMetaData,
useDSL,
useGetRunAndTraceUrl, useGetRunAndTraceUrl,
useNodesSyncDraft, useNodesSyncDraft,
useWorkflowRefreshDraft, useWorkflowRefreshDraft,
@ -50,6 +51,10 @@ const WorkflowMain = ({
} = useWorkflowStartRun() } = useWorkflowStartRun()
const availableNodesMetaData = useAvailableNodesMetaData() const availableNodesMetaData = useAvailableNodesMetaData()
const { getWorkflowRunAndTraceUrl } = useGetRunAndTraceUrl() const { getWorkflowRunAndTraceUrl } = useGetRunAndTraceUrl()
const {
exportCheck,
handleExportDSL,
} = useDSL()
const hooksStore = useMemo(() => { const hooksStore = useMemo(() => {
return { return {
@ -66,6 +71,8 @@ const WorkflowMain = ({
handleWorkflowStartRunInWorkflow, handleWorkflowStartRunInWorkflow,
availableNodesMetaData, availableNodesMetaData,
getWorkflowRunAndTraceUrl, getWorkflowRunAndTraceUrl,
exportCheck,
handleExportDSL,
} }
}, [ }, [
syncWorkflowDraftWhenPageClose, syncWorkflowDraftWhenPageClose,
@ -81,6 +88,8 @@ const WorkflowMain = ({
handleWorkflowStartRunInWorkflow, handleWorkflowStartRunInWorkflow,
availableNodesMetaData, availableNodesMetaData,
getWorkflowRunAndTraceUrl, getWorkflowRunAndTraceUrl,
exportCheck,
handleExportDSL,
]) ])
return ( return (

@ -7,3 +7,4 @@ export * from './use-is-chat-mode'
export * from './use-available-nodes-meta-data' export * from './use-available-nodes-meta-data'
export * from './use-workflow-refresh-draft' export * from './use-workflow-refresh-draft'
export * from './use-get-run-and-trace-url' export * from './use-get-run-and-trace-url'
export * from './use-DSL'

@ -0,0 +1,79 @@
import {
useCallback,
useState,
} from 'react'
import { useTranslation } from 'react-i18next'
import {
DSL_EXPORT_CHECK,
} from '@/app/components/workflow/constants'
import { useNodesSyncDraft } from './use-nodes-sync-draft'
import { useEventEmitterContextContext } from '@/context/event-emitter'
import { fetchWorkflowDraft } from '@/service/workflow'
import { exportAppConfig } from '@/service/apps'
import { useToastContext } from '@/app/components/base/toast'
import { useStore as useAppStore } from '@/app/components/app/store'
export const useDSL = () => {
const { t } = useTranslation()
const { notify } = useToastContext()
const { eventEmitter } = useEventEmitterContextContext()
const [exporting, setExporting] = useState(false)
const { doSyncWorkflowDraft } = useNodesSyncDraft()
const appDetail = useAppStore(s => s.appDetail)
const handleExportDSL = useCallback(async (include = false) => {
if (!appDetail)
return
if (exporting)
return
try {
setExporting(true)
await doSyncWorkflowDraft()
const { data } = await exportAppConfig({
appID: appDetail.id,
include,
})
const a = document.createElement('a')
const file = new Blob([data], { type: 'application/yaml' })
a.href = URL.createObjectURL(file)
a.download = `${appDetail.name}.yml`
a.click()
}
catch {
notify({ type: 'error', message: t('app.exportFailed') })
}
finally {
setExporting(false)
}
}, [appDetail, notify, t, doSyncWorkflowDraft, exporting])
const exportCheck = useCallback(async () => {
if (!appDetail)
return
try {
const workflowDraft = await fetchWorkflowDraft(`/apps/${appDetail?.id}/workflows/draft`)
const list = (workflowDraft.environment_variables || []).filter(env => env.value_type === 'secret')
if (list.length === 0) {
handleExportDSL()
return
}
eventEmitter?.emit({
type: DSL_EXPORT_CHECK,
payload: {
data: list,
},
} as any)
}
catch {
notify({ type: 'error', message: t('app.exportFailed') })
}
}, [appDetail, eventEmitter, handleExportDSL, notify, t])
return {
exportCheck,
handleExportDSL,
}
}

@ -13,6 +13,7 @@ export const transformDataSourceToTool = (dataSourceItem: DataSourceItem) => {
type: dataSourceItem.declaration.provider_type, type: dataSourceItem.declaration.provider_type,
team_credentials: {}, team_credentials: {},
allow_delete: true, allow_delete: true,
is_team_authorization: dataSourceItem.is_authorized,
is_authorized: dataSourceItem.is_authorized, is_authorized: dataSourceItem.is_authorized,
labels: dataSourceItem.declaration.identity.tags || [], labels: dataSourceItem.declaration.identity.tags || [],
plugin_id: dataSourceItem.plugin_id, plugin_id: dataSourceItem.plugin_id,

@ -37,6 +37,8 @@ export type CommonHooksFnMap = {
handleWorkflowStartRunInChatflow: () => void handleWorkflowStartRunInChatflow: () => void
availableNodesMetaData?: AvailableNodesMetaData availableNodesMetaData?: AvailableNodesMetaData
getWorkflowRunAndTraceUrl: (runId?: string) => { runUrl: string; traceUrl: string } getWorkflowRunAndTraceUrl: (runId?: string) => { runUrl: string; traceUrl: string }
exportCheck?: () => Promise<void>
handleExportDSL?: (include?: boolean) => Promise<void>
} }
export type Shape = { export type Shape = {
@ -62,6 +64,8 @@ export const createHooksStore = ({
runUrl: '', runUrl: '',
traceUrl: '', traceUrl: '',
}), }),
exportCheck = async () => noop(),
handleExportDSL = async () => noop(),
}: Partial<Shape>) => { }: Partial<Shape>) => {
return createStore<Shape>(set => ({ return createStore<Shape>(set => ({
refreshAll: props => set(state => ({ ...state, ...props })), refreshAll: props => set(state => ({ ...state, ...props })),
@ -78,6 +82,8 @@ export const createHooksStore = ({
handleWorkflowStartRunInChatflow, handleWorkflowStartRunInChatflow,
availableNodesMetaData, availableNodesMetaData,
getWorkflowRunAndTraceUrl, getWorkflowRunAndTraceUrl,
exportCheck,
handleExportDSL,
})) }))
} }

@ -19,3 +19,4 @@ export * from './use-nodes-meta-data'
export * from './use-available-blocks' export * from './use-available-blocks'
export * from './use-workflow-refresh-draft' export * from './use-workflow-refresh-draft'
export * from './use-tool-icon' export * from './use-tool-icon'
export * from './use-DSL'

@ -0,0 +1,11 @@
import { useHooksStore } from '@/app/components/workflow/hooks-store'
export const useDSL = () => {
const exportCheck = useHooksStore(s => s.exportCheck)
const handleExportDSL = useHooksStore(s => s.handleExportDSL)
return {
exportCheck,
handleExportDSL,
}
}

@ -1,13 +1,11 @@
import { import {
useCallback, useCallback,
useState,
} from 'react' } from 'react'
import { useTranslation } from 'react-i18next'
import { useReactFlow, useStoreApi } from 'reactflow' import { useReactFlow, useStoreApi } from 'reactflow'
import produce from 'immer' import produce from 'immer'
import { useStore, useWorkflowStore } from '../store' import { useStore, useWorkflowStore } from '../store'
import { import {
CUSTOM_NODE, DSL_EXPORT_CHECK, CUSTOM_NODE,
NODE_LAYOUT_HORIZONTAL_PADDING, NODE_LAYOUT_HORIZONTAL_PADDING,
NODE_LAYOUT_VERTICAL_PADDING, NODE_LAYOUT_VERTICAL_PADDING,
WORKFLOW_DATA_UPDATE, WORKFLOW_DATA_UPDATE,
@ -30,10 +28,6 @@ import { useNodesInteractionsWithoutSync } from './use-nodes-interactions-withou
import { useNodesSyncDraft } from './use-nodes-sync-draft' import { useNodesSyncDraft } from './use-nodes-sync-draft'
import { WorkflowHistoryEvent, useWorkflowHistory } from './use-workflow-history' import { WorkflowHistoryEvent, useWorkflowHistory } from './use-workflow-history'
import { useEventEmitterContextContext } from '@/context/event-emitter' import { useEventEmitterContextContext } from '@/context/event-emitter'
import { fetchWorkflowDraft } from '@/service/workflow'
import { exportAppConfig } from '@/service/apps'
import { useToastContext } from '@/app/components/base/toast'
import { useStore as useAppStore } from '@/app/components/app/store'
export const useWorkflowInteractions = () => { export const useWorkflowInteractions = () => {
const workflowStore = useWorkflowStore() const workflowStore = useWorkflowStore()
@ -336,68 +330,3 @@ export const useWorkflowUpdate = () => {
handleUpdateWorkflowCanvas, handleUpdateWorkflowCanvas,
} }
} }
export const useDSL = () => {
const { t } = useTranslation()
const { notify } = useToastContext()
const { eventEmitter } = useEventEmitterContextContext()
const [exporting, setExporting] = useState(false)
const { doSyncWorkflowDraft } = useNodesSyncDraft()
const appDetail = useAppStore(s => s.appDetail)
const handleExportDSL = useCallback(async (include = false) => {
if (!appDetail)
return
if (exporting)
return
try {
setExporting(true)
await doSyncWorkflowDraft()
const { data } = await exportAppConfig({
appID: appDetail.id,
include,
})
const a = document.createElement('a')
const file = new Blob([data], { type: 'application/yaml' })
a.href = URL.createObjectURL(file)
a.download = `${appDetail.name}.yml`
a.click()
}
catch {
notify({ type: 'error', message: t('app.exportFailed') })
}
finally {
setExporting(false)
}
}, [appDetail, notify, t, doSyncWorkflowDraft, exporting])
const exportCheck = useCallback(async () => {
if (!appDetail)
return
try {
const workflowDraft = await fetchWorkflowDraft(`/apps/${appDetail?.id}/workflows/draft`)
const list = (workflowDraft.environment_variables || []).filter(env => env.value_type === 'secret')
if (list.length === 0) {
handleExportDSL()
return
}
eventEmitter?.emit({
type: DSL_EXPORT_CHECK,
payload: {
data: list,
},
} as any)
}
catch {
notify({ type: 'error', message: t('app.exportFailed') })
}
}, [appDetail, eventEmitter, handleExportDSL, notify, t])
return {
exportCheck,
handleExportDSL,
}
}

@ -30,7 +30,7 @@ export const COMMON_OUTPUT = [
}, },
] ]
export const FILE_OUTPUT = [ export const LOCAL_FILE_OUTPUT = [
{ {
name: 'file', name: 'file',
type: VarType.file, type: VarType.file,
@ -80,7 +80,7 @@ export const FILE_OUTPUT = [
}, },
] ]
export const WEBSITE_OUTPUT = [ export const WEBSITE_CRAWL_OUTPUT = [
{ {
name: 'source_url', name: 'source_url',
type: VarType.string, type: VarType.string,
@ -102,3 +102,21 @@ export const WEBSITE_OUTPUT = [
description: 'The description of the crawled website', description: 'The description of the crawled website',
}, },
] ]
export const ONLINE_DOCUMENT_OUTPUT = [
{
name: 'workspace_id',
type: VarType.string,
description: 'The ID of the workspace where the document is stored',
},
{
name: 'page_id',
type: VarType.string,
description: 'The ID of the page in the document',
},
{
name: 'content',
type: VarType.string,
description: 'The content of the online document',
},
]

@ -5,8 +5,9 @@ import { genNodeMetaData } from '@/app/components/workflow/utils'
import { BlockEnum } from '@/app/components/workflow/types' import { BlockEnum } from '@/app/components/workflow/types'
import { import {
COMMON_OUTPUT, COMMON_OUTPUT,
FILE_OUTPUT, LOCAL_FILE_OUTPUT,
WEBSITE_OUTPUT, ONLINE_DOCUMENT_OUTPUT,
WEBSITE_CRAWL_OUTPUT,
} from './constants' } from './constants'
import { VarType as VarKindType } from '@/app/components/workflow/nodes/tool/types' import { VarType as VarKindType } from '@/app/components/workflow/nodes/tool/types'
@ -58,18 +59,24 @@ const nodeDefault: NodeDefault<DataSourceNodeType> = {
const { const {
provider_type, provider_type,
} = payload } = payload
const isLocalFile = provider_type === DataSourceClassification.file const isLocalFile = provider_type === DataSourceClassification.localFile
const isWebsiteCrawl = provider_type === DataSourceClassification.website const isWebsiteCrawl = provider_type === DataSourceClassification.websiteCrawl
const isOnlineDocument = provider_type === DataSourceClassification.onlineDocument
return [ return [
...COMMON_OUTPUT.map(item => ({ variable: item.name, type: item.type })), ...COMMON_OUTPUT.map(item => ({ variable: item.name, type: item.type })),
...( ...(
isLocalFile isLocalFile
? FILE_OUTPUT.map(item => ({ variable: item.name, type: item.type })) ? LOCAL_FILE_OUTPUT.map(item => ({ variable: item.name, type: item.type }))
: [] : []
), ),
...( ...(
isWebsiteCrawl isWebsiteCrawl
? WEBSITE_OUTPUT.map(item => ({ variable: item.name, type: item.type })) ? WEBSITE_CRAWL_OUTPUT.map(item => ({ variable: item.name, type: item.type }))
: []
),
...(
isOnlineDocument
? ONLINE_DOCUMENT_OUTPUT.map(item => ({ variable: item.name, type: item.type }))
: [] : []
), ),
...ragVars, ...ragVars,

@ -20,8 +20,9 @@ import { useNodesReadOnly } from '@/app/components/workflow/hooks'
import { useConfig } from './hooks/use-config' import { useConfig } from './hooks/use-config'
import { import {
COMMON_OUTPUT, COMMON_OUTPUT,
FILE_OUTPUT, LOCAL_FILE_OUTPUT,
WEBSITE_OUTPUT, ONLINE_DOCUMENT_OUTPUT,
WEBSITE_CRAWL_OUTPUT,
} from './constants' } from './constants'
import { useStore } from '@/app/components/workflow/store' import { useStore } from '@/app/components/workflow/store'
import Button from '@/app/components/base/button' import Button from '@/app/components/base/button'
@ -48,8 +49,9 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
handleFileExtensionsChange, handleFileExtensionsChange,
handleParametersChange, handleParametersChange,
} = useConfig(id) } = useConfig(id)
const isLocalFile = provider_type === DataSourceClassification.file const isLocalFile = provider_type === DataSourceClassification.localFile
const isWebsiteCrawl = provider_type === DataSourceClassification.website const isWebsiteCrawl = provider_type === DataSourceClassification.websiteCrawl
const isOnlineDocument = provider_type === DataSourceClassification.onlineDocument
const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === plugin_id) const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === plugin_id)
const isAuthorized = !!currentDataSource?.is_authorized const isAuthorized = !!currentDataSource?.is_authorized
const [showAuthModal, { const [showAuthModal, {
@ -166,7 +168,7 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
)) ))
} }
{ {
isLocalFile && FILE_OUTPUT.map(item => ( isLocalFile && LOCAL_FILE_OUTPUT.map(item => (
<VarItem <VarItem
name={item.name} name={item.name}
type={item.type} type={item.type}
@ -180,7 +182,16 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
)) ))
} }
{ {
isWebsiteCrawl && WEBSITE_OUTPUT.map(item => ( isWebsiteCrawl && WEBSITE_CRAWL_OUTPUT.map(item => (
<VarItem
name={item.name}
type={item.type}
description={item.description}
/>
))
}
{
isOnlineDocument && ONLINE_DOCUMENT_OUTPUT.map(item => (
<VarItem <VarItem
name={item.name} name={item.name}
type={item.type} type={item.type}

@ -7,8 +7,9 @@ export enum VarType {
} }
export enum DataSourceClassification { export enum DataSourceClassification {
file = 'local_file', localFile = 'local_file',
website = 'website_crawl', websiteCrawl = 'website_crawl',
onlineDocument = 'online_document',
} }
export type ToolVarInputs = Record<string, { export type ToolVarInputs = Record<string, {

@ -112,7 +112,7 @@ const PanelContextmenu = () => {
<div className='p-1'> <div className='p-1'>
<div <div
className='flex h-8 cursor-pointer items-center justify-between rounded-lg px-3 text-sm text-text-secondary hover:bg-state-base-hover' className='flex h-8 cursor-pointer items-center justify-between rounded-lg px-3 text-sm text-text-secondary hover:bg-state-base-hover'
onClick={() => exportCheck()} onClick={() => exportCheck?.()}
> >
{t('app.export')} {t('app.export')}
</div> </div>

@ -2,6 +2,8 @@ const translation = {
common: { common: {
goToAddDocuments: 'Go to add documents', goToAddDocuments: 'Go to add documents',
publishAs: 'Publish as a Knowledge Pipeline', publishAs: 'Publish as a Knowledge Pipeline',
confirmPublish: 'Confirm Publish',
confirmPublishContent: 'After successfully publishing the knowledge pipeline, the chunk structure of this knowledge base cannot be modified. Are you sure you want to publish it?',
}, },
} }

@ -2,6 +2,8 @@ const translation = {
common: { common: {
goToAddDocuments: '去添加文档', goToAddDocuments: '去添加文档',
publishAs: '发布为知识管道', publishAs: '发布为知识管道',
confirmPublish: '确认发布',
confirmPublishContent: '成功发布知识管道后,此知识库的分段结构将无法修改。您确定要发布吗?',
}, },
} }

@ -5,6 +5,18 @@
"engines": { "engines": {
"node": ">=v22.11.0" "node": ">=v22.11.0"
}, },
"browserslist": [
"last 1 Chrome version",
"last 1 Firefox version",
"last 1 Edge version",
"last 1 Safari version",
"iOS >=15",
"Android >= 10",
"and_chr >= 126",
"and_ff >= 137",
"and_uc >= 15.5",
"and_qq >= 14.9"
],
"scripts": { "scripts": {
"dev": "cross-env NODE_OPTIONS='--inspect' next dev", "dev": "cross-env NODE_OPTIONS='--inspect' next dev",
"build": "next build", "build": "next build",

@ -298,3 +298,15 @@ export const usePublishedPipelinePreProcessingParams = (params: PipelinePreProce
enabled, enabled,
}) })
} }
export const useExportPipelineDSL = () => {
return useMutation({
mutationKey: [NAME_SPACE, 'export-pipeline-dsl'],
mutationFn: ({
pipelineId,
include = false,
}: { pipelineId: string; include?: boolean }) => {
return get<ExportTemplateDSLResponse>(`/rag/pipelines/${pipelineId}/export?include_secret=${include}`)
},
})
}

Loading…
Cancel
Save