Merge branch 'langgenius:main' into add-document-status-update

api/controllers/service_api/dataset/dataset.py

@@ -145,7 +145,9 @@ class DatasetListApi(DatasetApiResource):
                 external_knowledge_id=args["external_knowledge_id"],
                 embedding_model_provider=args["embedding_model_provider"],
                 embedding_model_name=args["embedding_model"],
-                retrieval_model=RetrievalModel(**args["retrieval_model"]),
+                retrieval_model=RetrievalModel(**args["retrieval_model"])
+                if args["retrieval_model"] is not None
+                else None,
             )
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()

api/core/helper/ssrf_proxy.py

@@ -48,25 +48,26 @@ def make_request(method, url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs):
             write=dify_config.SSRF_DEFAULT_WRITE_TIME_OUT,
         )
 
+    if "ssl_verify" not in kwargs:
+        kwargs["ssl_verify"] = HTTP_REQUEST_NODE_SSL_VERIFY
+
+    ssl_verify = kwargs.pop("ssl_verify")
+
     retries = 0
     while retries <= max_retries:
         try:
             if dify_config.SSRF_PROXY_ALL_URL:
-                with httpx.Client(proxy=dify_config.SSRF_PROXY_ALL_URL, verify=HTTP_REQUEST_NODE_SSL_VERIFY) as client:
+                with httpx.Client(proxy=dify_config.SSRF_PROXY_ALL_URL, verify=ssl_verify) as client:
                     response = client.request(method=method, url=url, **kwargs)
             elif dify_config.SSRF_PROXY_HTTP_URL and dify_config.SSRF_PROXY_HTTPS_URL:
                 proxy_mounts = {
-                    "http://": httpx.HTTPTransport(
+                    "http://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTP_URL, verify=ssl_verify),
-                        proxy=dify_config.SSRF_PROXY_HTTP_URL, verify=HTTP_REQUEST_NODE_SSL_VERIFY
+                    "https://": httpx.HTTPTransport(proxy=dify_config.SSRF_PROXY_HTTPS_URL, verify=ssl_verify),
-                    ),
-                    "https://": httpx.HTTPTransport(
-                        proxy=dify_config.SSRF_PROXY_HTTPS_URL, verify=HTTP_REQUEST_NODE_SSL_VERIFY
-                    ),
                 }
-                with httpx.Client(mounts=proxy_mounts, verify=HTTP_REQUEST_NODE_SSL_VERIFY) as client:
+                with httpx.Client(mounts=proxy_mounts, verify=ssl_verify) as client:
                     response = client.request(method=method, url=url, **kwargs)
             else:
-                with httpx.Client(verify=HTTP_REQUEST_NODE_SSL_VERIFY) as client:
+                with httpx.Client(verify=ssl_verify) as client:
                     response = client.request(method=method, url=url, **kwargs)
 
             if response.status_code not in STATUS_FORCELIST:

api/core/workflow/nodes/http_request/entities.py

@@ -90,6 +90,7 @@ class HttpRequestNodeData(BaseNodeData):
     params: str
     body: Optional[HttpRequestNodeBody] = None
     timeout: Optional[HttpRequestNodeTimeout] = None
+    ssl_verify: Optional[bool] = dify_config.HTTP_REQUEST_NODE_SSL_VERIFY
 
 
 class Response:

api/core/workflow/nodes/http_request/executor.py

@@ -88,6 +88,7 @@ class Executor:
         self.method = node_data.method
         self.auth = node_data.authorization
         self.timeout = timeout
+        self.ssl_verify = node_data.ssl_verify
         self.params = []
         self.headers = {}
         self.content = None
@@ -316,6 +317,7 @@ class Executor:
             "headers": headers,
             "params": self.params,
             "timeout": (self.timeout.connect, self.timeout.read, self.timeout.write),
+            "ssl_verify": self.ssl_verify,
             "follow_redirects": True,
             "max_retries": self.max_retries,
         }

api/core/workflow/nodes/http_request/node.py

@@ -51,6 +51,7 @@ class HttpRequestNode(BaseNode[HttpRequestNodeData]):
                     "max_read_timeout": dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT,
                     "max_write_timeout": dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT,
                 },
+                "ssl_verify": dify_config.HTTP_REQUEST_NODE_SSL_VERIFY,
             },
             "retry_config": {
                 "max_retries": dify_config.SSRF_DEFAULT_MAX_RETRIES,

api/core/workflow/nodes/list_operator/node.py

@@ -149,7 +149,10 @@ class ListOperatorNode(BaseNode[ListOperatorNodeData]):
     def _extract_slice(
         self, variable: Union[ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment]
     ) -> Union[ArrayFileSegment, ArrayNumberSegment, ArrayStringSegment]:
-        value = int(self.graph_runtime_state.variable_pool.convert_template(self.node_data.extract_by.serial).text) - 1
+        value = int(self.graph_runtime_state.variable_pool.convert_template(self.node_data.extract_by.serial).text)
+        if value < 1:
+            raise ValueError(f"Invalid serial index: must be >= 1, got {value}")
+        value -= 1
         if len(variable.value) > int(value):
             result = variable.value[value]
         else:

docker/.env.example

@@ -174,6 +174,12 @@ CELERY_MIN_WORKERS=
 API_TOOL_DEFAULT_CONNECT_TIMEOUT=10
 API_TOOL_DEFAULT_READ_TIMEOUT=60
 
+# -------------------------------
+# Datasource Configuration
+# --------------------------------
+ENABLE_WEBSITE_JINAREADER=true
+ENABLE_WEBSITE_FIRECRAWL=true
+ENABLE_WEBSITE_WATERCRAWL=true
 
 # ------------------------------
 # Database Configuration

docker/docker-compose-template.yaml

@@ -75,7 +75,9 @@ services:
       MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10}
       MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10}
       MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-5}
-
+      ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true}
+      ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true}
+      ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true}
   # The postgres database.
   db:
     image: postgres:15-alpine

docker/docker-compose.yaml

@@ -43,6 +43,9 @@ x-shared-env: &shared-api-worker-env
   CELERY_MIN_WORKERS: ${CELERY_MIN_WORKERS:-}
   API_TOOL_DEFAULT_CONNECT_TIMEOUT: ${API_TOOL_DEFAULT_CONNECT_TIMEOUT:-10}
   API_TOOL_DEFAULT_READ_TIMEOUT: ${API_TOOL_DEFAULT_READ_TIMEOUT:-60}
+  ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true}
+  ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true}
+  ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true}
   DB_USERNAME: ${DB_USERNAME:-postgres}
   DB_PASSWORD: ${DB_PASSWORD:-difyai123456}
   DB_HOST: ${DB_HOST:-db}
@@ -543,7 +546,9 @@ services:
       MAX_TOOLS_NUM: ${MAX_TOOLS_NUM:-10}
       MAX_PARALLEL_LIMIT: ${MAX_PARALLEL_LIMIT:-10}
       MAX_ITERATIONS_NUM: ${MAX_ITERATIONS_NUM:-5}
-
+      ENABLE_WEBSITE_JINAREADER: ${ENABLE_WEBSITE_JINAREADER:-true}
+      ENABLE_WEBSITE_FIRECRAWL: ${ENABLE_WEBSITE_FIRECRAWL:-true}
+      ENABLE_WEBSITE_WATERCRAWL: ${ENABLE_WEBSITE_WATERCRAWL:-true}
   # The postgres database.
   db:
     image: postgres:15-alpine

web/.env.example

@@ -49,3 +49,8 @@ NEXT_PUBLIC_MAX_PARALLEL_LIMIT=10
 
 # The maximum number of iterations for agent setting
 NEXT_PUBLIC_MAX_ITERATIONS_NUM=5
+
+NEXT_PUBLIC_ENABLE_WEBSITE_JINAREADER=true
+NEXT_PUBLIC_ENABLE_WEBSITE_FIRECRAWL=true
+NEXT_PUBLIC_ENABLE_WEBSITE_WATERCRAWL=true
+

web/app/components/datasets/create/step-one/index.tsx

@@ -20,7 +20,7 @@ import { useProviderContext } from '@/context/provider-context'
 import VectorSpaceFull from '@/app/components/billing/vector-space-full'
 import classNames from '@/utils/classnames'
 import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others'
-
+import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
 type IStepOneProps = {
   datasetId?: string
   dataSourceType?: DataSourceType
@@ -126,9 +126,7 @@ const StepOne = ({
       return true
     if (files.some(file => !file.file.id))
       return true
-    if (isShowVectorSpaceFull)
+    return isShowVectorSpaceFull
-      return true
-    return false
   }, [files, isShowVectorSpaceFull])
 
   return (
@@ -193,7 +191,8 @@ const StepOne = ({
                         {t('datasetCreation.stepOne.dataSourceType.notion')}
                       </span>
                     </div>
-                    <div
+                    {(ENABLE_WEBSITE_FIRECRAWL || ENABLE_WEBSITE_JINAREADER || ENABLE_WEBSITE_WATERCRAWL) && (
+                      <div
                       className={cn(
                         s.dataSourceItem,
                         'system-sm-medium',
@@ -201,7 +200,7 @@ const StepOne = ({
                         dataSourceTypeDisable && dataSourceType !== DataSourceType.WEB && s.disabled,
                       )}
                       onClick={() => changeType(DataSourceType.WEB)}
-                    >
+                      >
                       <span className={cn(s.datasetIcon, s.web)} />
                       <span
                         title={t('datasetCreation.stepOne.dataSourceType.web')}
@@ -209,7 +208,8 @@ const StepOne = ({
                       >
                         {t('datasetCreation.stepOne.dataSourceType.web')}
                       </span>
-                    </div>
+                      </div>
+                    )}
                   </div>
                 )
               }

web/app/components/datasets/create/website/index.tsx

@@ -12,6 +12,7 @@ import { useModalContext } from '@/context/modal-context'
 import type { CrawlOptions, CrawlResultItem } from '@/models/datasets'
 import { fetchDataSources } from '@/service/datasets'
 import { type DataSourceItem, DataSourceProvider } from '@/models/common'
+import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
 
 type Props = {
   onPreview: (payload: CrawlResultItem) => void
@@ -84,7 +85,7 @@ const Website: FC<Props> = ({
           {t('datasetCreation.stepOne.website.chooseProvider')}
         </div>
         <div className="flex space-x-2">
-          <button
+          {ENABLE_WEBSITE_JINAREADER && <button
             className={cn('flex items-center justify-center rounded-lg px-4 py-2',
               selectedProvider === DataSourceProvider.jinaReader
                 ? 'system-sm-medium border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary'
@@ -95,8 +96,8 @@ const Website: FC<Props> = ({
           >
             <span className={cn(s.jinaLogo, 'mr-2')}/>
             <span>Jina Reader</span>
-          </button>
+          </button>}
-          <button
+         {ENABLE_WEBSITE_FIRECRAWL && <button
             className={cn('rounded-lg px-4 py-2',
               selectedProvider === DataSourceProvider.fireCrawl
                 ? 'system-sm-medium border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary'
@@ -106,8 +107,8 @@ const Website: FC<Props> = ({
             onClick={() => setSelectedProvider(DataSourceProvider.fireCrawl)}
           >
             🔥 Firecrawl
-          </button>
+          </button>}
-          <button
+          {ENABLE_WEBSITE_WATERCRAWL && <button
             className={cn('flex items-center justify-center rounded-lg px-4 py-2',
               selectedProvider === DataSourceProvider.waterCrawl
                 ? 'system-sm-medium border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary'
@@ -118,7 +119,7 @@ const Website: FC<Props> = ({
           >
             <span className={cn(s.watercrawlLogo, 'mr-2')}/>
             <span>WaterCrawl</span>
-          </button>
+          </button>}
         </div>
       </div>
       {source && selectedProvider === DataSourceProvider.fireCrawl && (

web/app/components/datasets/create/website/no-data.tsx

@@ -6,6 +6,7 @@ import s from './index.module.css'
 import { Icon3Dots } from '@/app/components/base/icons/src/vender/line/others'
 import Button from '@/app/components/base/button'
 import { DataSourceProvider } from '@/models/common'
+import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
 
 const I18N_PREFIX = 'datasetCreation.stepOne.website'
 
@@ -16,29 +17,30 @@ type Props = {
 
 const NoData: FC<Props> = ({
   onConfig,
-  provider,
 }) => {
   const { t } = useTranslation()
 
   const providerConfig = {
-    [DataSourceProvider.jinaReader]: {
+    [DataSourceProvider.jinaReader]: ENABLE_WEBSITE_JINAREADER ? {
       emoji: <span className={s.jinaLogo} />,
       title: t(`${I18N_PREFIX}.jinaReaderNotConfigured`),
       description: t(`${I18N_PREFIX}.jinaReaderNotConfiguredDescription`),
-    },
+    } : null,
-    [DataSourceProvider.fireCrawl]: {
+    [DataSourceProvider.fireCrawl]: ENABLE_WEBSITE_FIRECRAWL ? {
       emoji: '🔥',
       title: t(`${I18N_PREFIX}.fireCrawlNotConfigured`),
       description: t(`${I18N_PREFIX}.fireCrawlNotConfiguredDescription`),
-    },
+    } : null,
-    [DataSourceProvider.waterCrawl]: {
+    [DataSourceProvider.waterCrawl]: ENABLE_WEBSITE_WATERCRAWL ? {
-      emoji: <span className={s.watercrawlLogo} />,
+      emoji: '💧',
       title: t(`${I18N_PREFIX}.waterCrawlNotConfigured`),
       description: t(`${I18N_PREFIX}.waterCrawlNotConfiguredDescription`),
-    },
+    } : null,
   }
 
-  const currentProvider = providerConfig[provider]
+  const currentProvider = Object.values(providerConfig).find(provider => provider !== null) || providerConfig[DataSourceProvider.jinaReader]
+
+  if (!currentProvider) return null
 
   return (
     <>

web/app/components/develop/md.tsx

@@ -12,7 +12,7 @@ type IChildrenProps = {
 
 type IHeaderingProps = {
   url: string
-  method: 'PUT' | 'DELETE' | 'GET' | 'POST'
+  method: 'PUT' | 'DELETE' | 'GET' | 'POST' | 'PATCH'
   title: string
   name: string
 }
@@ -34,6 +34,9 @@ export const Heading = function H2({
     case 'POST':
       style = 'ring-sky-300 bg-sky-400/10 text-sky-500 dark:ring-sky-400/30 dark:bg-sky-400/10 dark:text-sky-400'
       break
+    case 'PATCH':
+      style = 'ring-violet-300 bg-violet-400/10 text-violet-500 dark:ring-violet-400/30 dark:bg-violet-400/10 dark:text-violet-400'
+      break
     default:
       style = 'ring-emerald-300 dark:ring-emerald-400/30 bg-emerald-400/10 text-emerald-500 dark:text-emerald-400'
       break

web/app/components/header/account-setting/data-source-page/index.tsx

@@ -3,6 +3,7 @@ import DataSourceNotion from './data-source-notion'
 import DataSourceWebsite from './data-source-website'
 import { fetchDataSource } from '@/service/common'
 import { DataSourceProvider } from '@/models/common'
+import { ENABLE_WEBSITE_FIRECRAWL, ENABLE_WEBSITE_JINAREADER, ENABLE_WEBSITE_WATERCRAWL } from '@/config'
 
 export default function DataSourcePage() {
   const { data } = useSWR({ url: 'data-source/integrates' }, fetchDataSource)
@@ -11,9 +12,9 @@ export default function DataSourcePage() {
   return (
     <div className='mb-8'>
       <DataSourceNotion workspaces={notionWorkspaces} />
-      <DataSourceWebsite provider={DataSourceProvider.jinaReader} />
+      {ENABLE_WEBSITE_JINAREADER && <DataSourceWebsite provider={DataSourceProvider.jinaReader} />}
-      <DataSourceWebsite provider={DataSourceProvider.fireCrawl} />
+      {ENABLE_WEBSITE_FIRECRAWL && <DataSourceWebsite provider={DataSourceProvider.fireCrawl} />}
-      <DataSourceWebsite provider={DataSourceProvider.waterCrawl} />
+      {ENABLE_WEBSITE_WATERCRAWL && <DataSourceWebsite provider={DataSourceProvider.waterCrawl} />}
     </div>
   )
 }

web/app/components/plugins/plugin-item/index.tsx

@@ -104,7 +104,10 @@ const PluginItem: FC<Props> = ({
               {!isDifyVersionCompatible && <Tooltip popupContent={
                 t('plugin.difyVersionNotCompatible', { minimalDifyVersion: declarationMeta.minimum_dify_version })
               }><RiErrorWarningLine color='red' className="ml-0.5 h-4 w-4 shrink-0 text-text-accent" /></Tooltip>}
-              <Badge className='ml-1 shrink-0' text={source === PluginSource.github ? plugin.meta!.version : plugin.version} />
+              <Badge className='ml-1 shrink-0'
+                text={source === PluginSource.github ? plugin.meta!.version : plugin.version}
+                hasRedCornerMark={(source === PluginSource.marketplace) && !!plugin.latest_unique_identifier && plugin.latest_unique_identifier !== plugin_unique_identifier}
+              />
             </div>
             <div className='flex items-center justify-between'>
               <Description text={descriptionText} descriptionLineRows={1}></Description>

web/config/index.ts

@@ -302,3 +302,15 @@ else if (globalThis.document?.body?.getAttribute('data-public-max-iterations-num
   maxIterationsNum = Number.parseInt(globalThis.document.body.getAttribute('data-public-max-iterations-num') as string)
 
 export const MAX_ITERATIONS_NUM = maxIterationsNum
+
+export const ENABLE_WEBSITE_JINAREADER = process.env.NEXT_PUBLIC_ENABLE_WEBSITE_JINAREADER !== undefined
+  ? process.env.NEXT_PUBLIC_ENABLE_WEBSITE_JINAREADER === 'true'
+  : true
+
+export const ENABLE_WEBSITE_FIRECRAWL = process.env.NEXT_PUBLIC_ENABLE_WEBSITE_FIRECRAWL !== undefined
+  ? process.env.NEXT_PUBLIC_ENABLE_WEBSITE_FIRECRAWL === 'true'
+  : true
+
+export const ENABLE_WEBSITE_WATERCRAWL = process.env.NEXT_PUBLIC_ENABLE_WEBSITE_WATERCRAWL !== undefined
+  ? process.env.NEXT_PUBLIC_ENABLE_WEBSITE_WATERCRAWL === 'true'
+  : true

web/docker/entrypoint.sh

@@ -28,5 +28,7 @@ export NEXT_PUBLIC_CSP_WHITELIST=${CSP_WHITELIST}
 export NEXT_PUBLIC_TOP_K_MAX_VALUE=${TOP_K_MAX_VALUE}
 export NEXT_PUBLIC_INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=${INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH}
 export NEXT_PUBLIC_MAX_TOOLS_NUM=${MAX_TOOLS_NUM}
-
+export NEXT_PUBLIC_ENABLE_WEBSITE_JINAREADER=${ENABLE_WEBSITE_JINAREADER:-true}
+export NEXT_PUBLIC_ENABLE_WEBSITE_FIRECRAWL=${ENABLE_WEBSITE_FIRECRAWL:-true}
+export NEXT_PUBLIC_ENABLE_WEBSITE_WATERCRAWL=${ENABLE_WEBSITE_WATERCRAWL:-true}
 pm2 start /app/web/server.js --name dify-web --cwd /app/web -i ${PM2_INSTANCES} --no-daemon