diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py index 615c253967..d93ca97f46 100644 --- a/api/controllers/service_api/dataset/dataset.py +++ b/api/controllers/service_api/dataset/dataset.py @@ -145,7 +145,9 @@ class DatasetListApi(DatasetApiResource): external_knowledge_id=args["external_knowledge_id"], embedding_model_provider=args["embedding_model_provider"], embedding_model_name=args["embedding_model"], - retrieval_model=RetrievalModel(**args["retrieval_model"]), + retrieval_model=RetrievalModel(**args["retrieval_model"]) + if args["retrieval_model"] is not None + else None, ) except services.errors.dataset.DatasetNameDuplicateError: raise DatasetNameDuplicateError() diff --git a/api/core/helper/ssrf_proxy.py b/api/core/helper/ssrf_proxy.py index 969cd112ee..11f245812e 100644 --- a/api/core/helper/ssrf_proxy.py +++ b/api/core/helper/ssrf_proxy.py @@ -48,25 +48,26 @@ def make_request(method, url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs): write=dify_config.SSRF_DEFAULT_WRITE_TIME_OUT, ) + if "ssl_verify" not in kwargs: + kwargs["ssl_verify"] = HTTP_REQUEST_NODE_SSL_VERIFY + + ssl_verify = kwargs.pop("ssl_verify") + retries = 0 while retries <= max_retries: try: if dify_config.SSRF_PROXY_ALL_URL: - with httpx.Client(proxy=dify_config.SSRF_PROXY_ALL_URL, verify=HTTP_REQUEST_NODE_SSL_VERIFY) as client: + with httpx.Client(proxy=dify_config.SSRF_PROXY_ALL_URL, verify=ssl_verify) as client: response = client.request(method=method, url=url, **kwargs) elif dify_config.SSRF_PROXY_HTTP_URL and dify_config.SSRF_PROXY_HTTPS_URL: proxy_mounts = { - "http://": httpx.HTTPTransport( - proxy=dify_config.SSRF_PROXY_HTTP_URL, verify=HTTP_REQUEST_NODE_SSL_VERIFY - ), - "https://": httpx.HTTPTransport( - proxy=dify_config.SSRF_PROXY_HTTPS_URL, verify=HTTP_REQUEST_NODE_SSL_VERIFY - ), + "http://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTP_URL, verify=ssl_verify), + "https://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTPS_URL, verify=ssl_verify), } - with httpx.Client(mounts=proxy_mounts, verify=HTTP_REQUEST_NODE_SSL_VERIFY) as client: + with httpx.Client(mounts=proxy_mounts, verify=ssl_verify) as client: response = client.request(method=method, url=url, **kwargs) else: - with httpx.Client(verify=HTTP_REQUEST_NODE_SSL_VERIFY) as client: + with httpx.Client(verify=ssl_verify) as client: response = client.request(method=method, url=url, **kwargs) if response.status_code not in STATUS_FORCELIST: diff --git a/api/core/workflow/nodes/http_request/entities.py b/api/core/workflow/nodes/http_request/entities.py index 054e30f0aa..8d7ba25d47 100644 --- a/api/core/workflow/nodes/http_request/entities.py +++ b/api/core/workflow/nodes/http_request/entities.py @@ -90,6 +90,7 @@ class HttpRequestNodeData(BaseNodeData): params: str body: Optional[HttpRequestNodeBody] = None timeout: Optional[HttpRequestNodeTimeout] = None + ssl_verify: Optional[bool] = dify_config.HTTP_REQUEST_NODE_SSL_VERIFY class Response: diff --git a/api/core/workflow/nodes/http_request/executor.py b/api/core/workflow/nodes/http_request/executor.py index f7fa8d670c..5d466e645f 100644 --- a/api/core/workflow/nodes/http_request/executor.py +++ b/api/core/workflow/nodes/http_request/executor.py @@ -88,6 +88,7 @@ class Executor: self.method = node_data.method self.auth = node_data.authorization self.timeout = timeout + self.ssl_verify = node_data.ssl_verify self.params = [] self.headers = {} self.content = None @@ -316,6 +317,7 @@ class Executor: "headers": headers, "params": self.params, "timeout": (self.timeout.connect, self.timeout.read, self.timeout.write), + "ssl_verify": self.ssl_verify, "follow_redirects": True, "max_retries": self.max_retries, } diff --git a/api/core/workflow/nodes/http_request/node.py b/api/core/workflow/nodes/http_request/node.py index 467161d5ed..fd2b0f9ae8 100644 --- a/api/core/workflow/nodes/http_request/node.py +++ b/api/core/workflow/nodes/http_request/node.py @@ -51,6 +51,7 @@ class HttpRequestNode(BaseNode[HttpRequestNodeData]): "max_read_timeout": dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT, "max_write_timeout": dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT, }, + "ssl_verify": dify_config.HTTP_REQUEST_NODE_SSL_VERIFY, }, "retry_config": { "max_retries": dify_config.SSRF_DEFAULT_MAX_RETRIES, diff --git a/api/core/workflow/nodes/list_operator/node.py b/api/core/workflow/nodes/list_operator/node.py index 432c57294e..04ccfc5405 100644 --- a/api/core/workflow/nodes/list_operator/node.py +++ b/api/core/workflow/nodes/list_operator/node.py @@ -149,7 +149,10 @@ class ListOperatorNode(BaseNode[ListOperatorNodeData]): def _extract_slice( self, variable: Union[ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment] ) -> Union[ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment]: - value = int(self.graph_runtime_state.variable_pool.convert_template(self.node_data.extract_by.serial).text) - 1 + value = int(self.graph_runtime_state.variable_pool.convert_template(self.node_data.extract_by.serial).text) + if value < 1: + raise ValueError(f"Invalid serial index: must be >= 1, got {value}") + value -= 1 if len(variable.value) > int(value): result = variable.value[value] else: diff --git a/api/test_workflow_deletion.py b/api/tests/unit_tests/services/workflow/test_workflow_deletion.py similarity index 100% rename from api/test_workflow_deletion.py rename to api/tests/unit_tests/services/workflow/test_workflow_deletion.py diff --git a/docker/.env.example b/docker/.env.example index acb09c0d4f..e49e8fee89 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -174,6 +174,12 @@ CELERY_MIN_WORKERS= API_TOOL_DEFAULT_CONNECT_TIMEOUT=10 API_TOOL_DEFAULT_READ_TIMEOUT=60 +# ------------------------------- +# Datasource Configuration +# -------------------------------- +ENABLE_WEBSITE_JINAREADER=true +ENABLE_WEBSITE_FIRECRAWL=true +ENABLE_WEBSITE_WATERCRAWL=true # ------------------------------ # Database Configuration diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 86976063c3..a8f7b755fb 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -75,7 +75,9 @@ services: MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-5} - + ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} + ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} + ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} # The postgres database. db: image: postgres:15-alpine diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index e9c8c8715a..25b0c56561 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -43,6 +43,9 @@ x-shared-env: &shared-api-worker-env CELERY_MIN_WORKERS: ${CELERY_MIN_WORKERS:-} API_TOOL_DEFAULT_CONNECT_TIMEOUT: ${API_TOOL_DEFAULT_CONNECT_TIMEOUT:-10} API_TOOL_DEFAULT_READ_TIMEOUT: ${API_TOOL_DEFAULT_READ_TIMEOUT:-60} + ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} + ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} + ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} DB_USERNAME: ${DB_USERNAME:-postgres} DB_PASSWORD: ${DB_PASSWORD:-difyai123456} DB_HOST: ${DB_HOST:-db} @@ -543,7 +546,9 @@ services: MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10} MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10} MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-5} - + ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true} + ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true} + ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true} # The postgres database. db: image: postgres:15-alpine diff --git a/web/.env.example b/web/.env.example index 51dc3d6b3c..1c3f42ddfc 100644 --- a/web/.env.example +++ b/web/.env.example @@ -49,3 +49,8 @@ NEXT_PUBLIC_MAX_PARALLEL_LIMIT=10 # The maximum number of iterations for agent setting NEXT_PUBLIC_MAX_ITERATIONS_NUM=5 + +NEXT_PUBLIC_ENABLE_WEBSITE_JINAREADER=true +NEXT_PUBLIC_ENABLE_WEBSITE_FIRECRAWL=true +NEXT_PUBLIC_ENABLE_WEBSITE_WATERCRAWL=true + diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx index 6f4231bb1f..38c885ebe2 100644 --- a/web/app/components/datasets/create/step-one/index.tsx +++ b/web/app/components/datasets/create/step-one/index.tsx @@ -20,7 +20,7 @@ import { useProviderContext } from '@/context/provider-context' import VectorSpaceFull from '@/app/components/billing/vector-space-full' import classNames from '@/utils/classnames' import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others' - +import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config' type IStepOneProps = { datasetId?: string dataSourceType?: DataSourceType @@ -126,9 +126,7 @@ const StepOne = ({ return true if (files.some(file => !file.file.id)) return true - if (isShowVectorSpaceFull) - return true - return false + return isShowVectorSpaceFull }, [files, isShowVectorSpaceFull]) return ( @@ -193,7 +191,8 @@ const StepOne = ({ {t('datasetCreation.stepOne.dataSourceType.notion')} -
changeType(DataSourceType.WEB)} - > + > {t('datasetCreation.stepOne.dataSourceType.web')} -
+ + )} ) } diff --git a/web/app/components/datasets/create/website/index.tsx b/web/app/components/datasets/create/website/index.tsx index 5122ef6ed2..e2d0e2df99 100644 --- a/web/app/components/datasets/create/website/index.tsx +++ b/web/app/components/datasets/create/website/index.tsx @@ -12,6 +12,7 @@ import { useModalContext } from '@/context/modal-context' import type { CrawlOptions, CrawlResultItem } from '@/models/datasets' import { fetchDataSources } from '@/service/datasets' import { type DataSourceItem, DataSourceProvider } from '@/models/common' +import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config' type Props = { onPreview: (payload: CrawlResultItem) => void @@ -84,7 +85,7 @@ const Website: FC = ({ {t('datasetCreation.stepOne.website.chooseProvider')}
- - - + }
{source && selectedProvider === DataSourceProvider.fireCrawl && ( diff --git a/web/app/components/datasets/create/website/no-data.tsx b/web/app/components/datasets/create/website/no-data.tsx index 14be2e29f6..65a314f516 100644 --- a/web/app/components/datasets/create/website/no-data.tsx +++ b/web/app/components/datasets/create/website/no-data.tsx @@ -6,6 +6,7 @@ import s from './index.module.css' import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others' import Button from '@/app/components/base/button' import { DataSourceProvider } from '@/models/common' +import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config' const I18N_PREFIX = 'datasetCreation.stepOne.website' @@ -16,29 +17,30 @@ type Props = { const NoData: FC = ({ onConfig, - provider, }) => { const { t } = useTranslation() const providerConfig = { - [DataSourceProvider.jinaReader]: { + [DataSourceProvider.jinaReader]: ENABLE_WEBSITE_JINAREADER ? { emoji: , title: t(`${I18N_PREFIX}.jinaReaderNotConfigured`), description: t(`${I18N_PREFIX}.jinaReaderNotConfiguredDescription`), - }, - [DataSourceProvider.fireCrawl]: { + } : null, + [DataSourceProvider.fireCrawl]: ENABLE_WEBSITE_FIRECRAWL ? { emoji: '🔥', title: t(`${I18N_PREFIX}.fireCrawlNotConfigured`), description: t(`${I18N_PREFIX}.fireCrawlNotConfiguredDescription`), - }, - [DataSourceProvider.waterCrawl]: { - emoji: , + } : null, + [DataSourceProvider.waterCrawl]: ENABLE_WEBSITE_WATERCRAWL ? { + emoji: '💧', title: t(`${I18N_PREFIX}.waterCrawlNotConfigured`), description: t(`${I18N_PREFIX}.waterCrawlNotConfiguredDescription`), - }, + } : null, } - const currentProvider = providerConfig[provider] + const currentProvider = Object.values(providerConfig).find(provider => provider !== null) || providerConfig[DataSourceProvider.jinaReader] + + if (!currentProvider) return null return ( <> diff --git a/web/app/components/develop/md.tsx b/web/app/components/develop/md.tsx index 655cd08280..a9b74a389f 100644 --- a/web/app/components/develop/md.tsx +++ b/web/app/components/develop/md.tsx @@ -12,7 +12,7 @@ type IChildrenProps = { type IHeaderingProps = { url: string - method: 'PUT' | 'DELETE' | 'GET' | 'POST' + method: 'PUT' | 'DELETE' | 'GET' | 'POST' | 'PATCH' title: string name: string } @@ -34,6 +34,9 @@ export const Heading = function H2({ case 'POST': style = 'ring-sky-300 bg-sky-400/10 text-sky-500 dark:ring-sky-400/30 dark:bg-sky-400/10 dark:text-sky-400' break + case 'PATCH': + style = 'ring-violet-300 bg-violet-400/10 text-violet-500 dark:ring-violet-400/30 dark:bg-violet-400/10 dark:text-violet-400' + break default: style = 'ring-emerald-300 dark:ring-emerald-400/30 bg-emerald-400/10 text-emerald-500 dark:text-emerald-400' break diff --git a/web/app/components/header/account-setting/data-source-page/index.tsx b/web/app/components/header/account-setting/data-source-page/index.tsx index d99bd25e02..fb13813d70 100644 --- a/web/app/components/header/account-setting/data-source-page/index.tsx +++ b/web/app/components/header/account-setting/data-source-page/index.tsx @@ -3,6 +3,7 @@ import DataSourceNotion from './data-source-notion' import DataSourceWebsite from './data-source-website' import { fetchDataSource } from '@/service/common' import { DataSourceProvider } from '@/models/common' +import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config' export default function DataSourcePage() { const { data } = useSWR({ url: 'data-source/integrates' }, fetchDataSource) @@ -11,9 +12,9 @@ export default function DataSourcePage() { return (
- - - + {ENABLE_WEBSITE_JINAREADER && } + {ENABLE_WEBSITE_FIRECRAWL && } + {ENABLE_WEBSITE_WATERCRAWL && }
) } diff --git a/web/app/components/plugins/plugin-item/index.tsx b/web/app/components/plugins/plugin-item/index.tsx index 9f66e5f400..8ce26b737a 100644 --- a/web/app/components/plugins/plugin-item/index.tsx +++ b/web/app/components/plugins/plugin-item/index.tsx @@ -104,7 +104,10 @@ const PluginItem: FC = ({ {!isDifyVersionCompatible && } - +
diff --git a/web/config/index.ts b/web/config/index.ts index 2b81adb095..b164392c52 100644 --- a/web/config/index.ts +++ b/web/config/index.ts @@ -302,3 +302,15 @@ else if (globalThis.document?.body?.getAttribute('data-public-max-iterations-num maxIterationsNum = Number.parseInt(globalThis.document.body.getAttribute('data-public-max-iterations-num') as string) export const MAX_ITERATIONS_NUM = maxIterationsNum + +export const ENABLE_WEBSITE_JINAREADER = process.env.NEXT_PUBLIC_ENABLE_WEBSITE_JINAREADER !== undefined + ? process.env.NEXT_PUBLIC_ENABLE_WEBSITE_JINAREADER === 'true' + : true + +export const ENABLE_WEBSITE_FIRECRAWL = process.env.NEXT_PUBLIC_ENABLE_WEBSITE_FIRECRAWL !== undefined + ? process.env.NEXT_PUBLIC_ENABLE_WEBSITE_FIRECRAWL === 'true' + : true + +export const ENABLE_WEBSITE_WATERCRAWL = process.env.NEXT_PUBLIC_ENABLE_WEBSITE_WATERCRAWL !== undefined + ? process.env.NEXT_PUBLIC_ENABLE_WEBSITE_WATERCRAWL === 'true' + : true diff --git a/web/docker/entrypoint.sh b/web/docker/entrypoint.sh index 797b61081a..8395ac5f4d 100755 --- a/web/docker/entrypoint.sh +++ b/web/docker/entrypoint.sh @@ -28,5 +28,7 @@ export NEXT_PUBLIC_CSP_WHITELIST=${CSP_WHITELIST} export NEXT_PUBLIC_TOP_K_MAX_VALUE=${TOP_K_MAX_VALUE} export NEXT_PUBLIC_INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH} export NEXT_PUBLIC_MAX_TOOLS_NUM=${MAX_TOOLS_NUM} - +export NEXT_PUBLIC_ENABLE_WEBSITE_JINAREADER=${ENABLE_WEBSITE_JINAREADER:-true} +export NEXT_PUBLIC_ENABLE_WEBSITE_FIRECRAWL=${ENABLE_WEBSITE_FIRECRAWL:-true} +export NEXT_PUBLIC_ENABLE_WEBSITE_WATERCRAWL=${ENABLE_WEBSITE_WATERCRAWL:-true} pm2 start /app/web/server.js --name dify-web --cwd /app/web -i ${PM2_INSTANCES} --no-daemon