Merge branch 'main' into feat/explore

api/config.py

@@ -21,9 +21,11 @@ DEFAULTS = {
     'REDIS_HOST': 'localhost',
     'REDIS_PORT': '6379',
     'REDIS_DB': '0',
+    'REDIS_USE_SSL': 'False',
     'SESSION_REDIS_HOST': 'localhost',
     'SESSION_REDIS_PORT': '6379',
     'SESSION_REDIS_DB': '2',
+    'SESSION_REDIS_USE_SSL': 'False',
     'OAUTH_REDIRECT_PATH': '/console/api/oauth/authorize',
     'OAUTH_REDIRECT_INDEX_PATH': '/',
     'CONSOLE_URL': 'https://cloud.dify.ai',
@@ -44,6 +46,7 @@ DEFAULTS = {
     'CELERY_BACKEND': 'database',
     'PDF_PREVIEW': 'True',
     'LOG_LEVEL': 'INFO',
+    'DISABLE_PROVIDER_CONFIG_VALIDATION': 'False',
 }
 
 
@@ -105,14 +108,18 @@ class Config:
         # redis settings
         self.REDIS_HOST = get_env('REDIS_HOST')
         self.REDIS_PORT = get_env('REDIS_PORT')
+        self.REDIS_USERNAME = get_env('REDIS_USERNAME')
         self.REDIS_PASSWORD = get_env('REDIS_PASSWORD')
         self.REDIS_DB = get_env('REDIS_DB')
+        self.REDIS_USE_SSL = get_bool_env('REDIS_USE_SSL')
 
         # session redis settings
         self.SESSION_REDIS_HOST = get_env('SESSION_REDIS_HOST')
         self.SESSION_REDIS_PORT = get_env('SESSION_REDIS_PORT')
+        self.SESSION_REDIS_USERNAME = get_env('SESSION_REDIS_USERNAME')
         self.SESSION_REDIS_PASSWORD = get_env('SESSION_REDIS_PASSWORD')
         self.SESSION_REDIS_DB = get_env('SESSION_REDIS_DB')
+        self.SESSION_REDIS_USE_SSL = get_bool_env('SESSION_REDIS_USE_SSL')
 
         # storage settings
         self.STORAGE_TYPE = get_env('STORAGE_TYPE')
@@ -165,10 +172,14 @@ class Config:
         self.CELERY_BACKEND = get_env('CELERY_BACKEND')
         self.CELERY_RESULT_BACKEND = 'db+{}'.format(self.SQLALCHEMY_DATABASE_URI) \
             if self.CELERY_BACKEND == 'database' else self.CELERY_BROKER_URL
+        self.BROKER_USE_SSL = self.CELERY_BROKER_URL.startswith('rediss://')
 
         # hosted provider credentials
         self.OPENAI_API_KEY = get_env('OPENAI_API_KEY')
 
+        # By default it is False
+        # You could disable it for compatibility with certain OpenAPI providers
+        self.DISABLE_PROVIDER_CONFIG_VALIDATION = get_bool_env('DISABLE_PROVIDER_CONFIG_VALIDATION')
 
 class CloudEditionConfig(Config):
 

api/core/indexing_runner.py

@@ -343,7 +343,7 @@ class IndexingRunner:
 
             # parse document to nodes
             nodes = node_parser.get_nodes_from_documents([text_doc])
-
+            nodes = [node for node in nodes if node.text is not None and node.text.strip()]
             all_nodes.extend(nodes)
 
         return all_nodes

api/extensions/ext_celery.py

@@ -15,9 +15,24 @@ def init_app(app: Flask) -> Celery:
         backend=app.config["CELERY_BACKEND"],
         task_ignore_result=True,
     )
+    
+    # Add SSL options to the Celery configuration
+    ssl_options = {
+        "ssl_cert_reqs": None,
+        "ssl_ca_certs": None,
+        "ssl_certfile": None,
+        "ssl_keyfile": None,
+    }
+
     celery_app.conf.update(
         result_backend=app.config["CELERY_RESULT_BACKEND"],
     )
+
+    if app.config["BROKER_USE_SSL"]:
+        celery_app.conf.update(
+            broker_use_ssl=ssl_options,  # Add the SSL options to the broker configuration
+        )
+        
     celery_app.set_default()
     app.extensions["celery"] = celery_app
     return celery_app

api/extensions/ext_redis.py

@@ -1,18 +1,23 @@
 import redis
-
+from redis.connection import SSLConnection, Connection
 
 redis_client = redis.Redis()
 
 
 def init_app(app):
+    connection_class = Connection
+    if app.config.get('REDIS_USE_SSL', False):
+        connection_class = SSLConnection
+
     redis_client.connection_pool = redis.ConnectionPool(**{
         'host': app.config.get('REDIS_HOST', 'localhost'),
         'port': app.config.get('REDIS_PORT', 6379),
+        'username': app.config.get('REDIS_USERNAME', None),
         'password': app.config.get('REDIS_PASSWORD', None),
         'db': app.config.get('REDIS_DB', 0),
         'encoding': 'utf-8',
         'encoding_errors': 'strict',
         'decode_responses': False
-    })
+    }, connection_class=connection_class)
 
     app.extensions['redis'] = redis_client

api/extensions/ext_session.py

@@ -1,4 +1,5 @@
 import redis
+from redis.connection import SSLConnection, Connection
 from flask import request
 from flask_session import Session, SqlAlchemySessionInterface, RedisSessionInterface
 from flask_session.sessions import total_seconds
@@ -23,16 +24,21 @@ def init_app(app):
     if session_type == 'sqlalchemy':
         app.session_interface = sqlalchemy_session_interface
     elif session_type == 'redis':
+        connection_class = Connection
+        if app.config.get('SESSION_REDIS_USE_SSL', False):
+            connection_class = SSLConnection
+
         sess_redis_client = redis.Redis()
         sess_redis_client.connection_pool = redis.ConnectionPool(**{
             'host': app.config.get('SESSION_REDIS_HOST', 'localhost'),
             'port': app.config.get('SESSION_REDIS_PORT', 6379),
+            'username': app.config.get('SESSION_REDIS_USERNAME', None),
             'password': app.config.get('SESSION_REDIS_PASSWORD', None),
             'db': app.config.get('SESSION_REDIS_DB', 2),
             'encoding': 'utf-8',
             'encoding_errors': 'strict',
             'decode_responses': False
-        })
+        }, connection_class=connection_class)
 
         app.extensions['session_redis'] = sess_redis_client
 

api/services/provider_service.py

@@ -62,6 +62,8 @@ class ProviderService:
 
     @staticmethod
     def validate_provider_configs(tenant, provider_name: ProviderName, configs: Union[dict | str]):
+        if current_app.config['DISABLE_PROVIDER_CONFIG_VALIDATION']:
+            return
         llm_provider_service = LLMProviderService(tenant.id, provider_name.value)
         return llm_provider_service.config_validate(configs)
 

docker/docker-compose.yaml

@@ -36,14 +36,18 @@ services:
       # It is consistent with the configuration in the 'redis' service below.
       REDIS_HOST: redis
       REDIS_PORT: 6379
+      REDIS_USERNAME: ''
       REDIS_PASSWORD: difyai123456
+      REDIS_USE_SSL: 'false'
       # use redis db 0 for redis cache
       REDIS_DB: 0
       # The configurations of session, Supported values are `sqlalchemy`. `redis`
       SESSION_TYPE: redis
       SESSION_REDIS_HOST: redis
       SESSION_REDIS_PORT: 6379
+      SESSION_REDIS_USERNAME: ''
       SESSION_REDIS_PASSWORD: difyai123456
+      SESSION_REDIS_USE_SSL: 'false'
       # use redis db 2 for session store
       SESSION_REDIS_DB: 2
       # The configurations of celery broker.
@@ -129,8 +133,10 @@ services:
       # The configurations of redis cache connection.
       REDIS_HOST: redis
       REDIS_PORT: 6379
+      REDIS_USERNAME: ''
       REDIS_PASSWORD: difyai123456
       REDIS_DB: 0
+      REDIS_USE_SSL: 'false'
       # The configurations of celery broker.
       CELERY_BROKER_URL: redis://:difyai123456@redis:6379/1
       # The type of storage to use for storing user files. Supported values are `local` and `s3`, Default: `local`

web/app/components/app/configuration/config-model/index.tsx

@@ -11,6 +11,7 @@ import type { CompletionParams } from '@/models/debug'
 import { Cog8ToothIcon, InformationCircleIcon, ChevronDownIcon } from '@heroicons/react/24/outline'
 import { AppType } from '@/types/app'
 import { TONE_LIST } from '@/config'
+import Toast from '@/app/components/base/toast'
 
 export type IConifgModelProps = {
   mode: string
@@ -93,7 +94,7 @@ const ConifgModel: FC<IConifgModelProps> = ({
       key: 'max_tokens',
       tip: t('common.model.params.maxTokenTip'),
       step: 100,
-      max: 4000,
+      max: modelId === 'gpt-4' ? 8000 : 4000,
     },
   ]
 
@@ -114,6 +115,16 @@ const ConifgModel: FC<IConifgModelProps> = ({
         onShowUseGPT4Confirm()
         return
       }
+      if(id !== 'gpt-4' && completionParams.max_tokens > 4000) {
+        Toast.notify({
+          type: 'warning',
+          message: t('common.model.params.setToCurrentModelMaxTokenTip')
+        })
+        onCompletionParamsChange({
+          ...completionParams,
+          max_tokens: 4000
+        })
+      }
       setModelId(id)
     }
   }

web/app/components/app/configuration/prompt-value-panel/index.tsx

@@ -73,7 +73,7 @@ const PromptValuePanel: FC<IPromptValuePanelProps> = ({
           {
             (promptTemplate && promptTemplate?.trim()) ? (
               <div
-                className="max-h-48 overflow-y-auto text-sm text-gray-700"
+                className="max-h-48 overflow-y-auto text-sm text-gray-700 break-all"
                 dangerouslySetInnerHTML={{
                   __html: format(replaceStringWithValuesWithFormat(promptTemplate, promptVariables, inputs)),
                 }}

web/app/components/app/log/list.tsx

@@ -166,7 +166,7 @@ function DetailPanel<T extends ChatConversationFullDetailResponse | CompletionCo
     return res
   })?.name ?? 'custom'
 
-  return (<div className='rounded-xl border-[0.5px] border-gray-200 h-full flex flex-col'>
+  return (<div className='rounded-xl border-[0.5px] border-gray-200 h-full flex flex-col overflow-auto'>
     {/* Panel Header */}
     <div className='border-b border-gray-100 py-4 px-6 flex items-center justify-between'>
       <div className='flex-1'>
@@ -207,7 +207,7 @@ function DetailPanel<T extends ChatConversationFullDetailResponse | CompletionCo
       <div className='text-gray-700 font-medium text-sm mt-2'>{detail.model_config?.pre_prompt || emptyText}</div>
     </div>
     {!isChatMode
-      ? <div className="px-2.5 py-4 overflow-y-auto">
+      ? <div className="px-2.5 py-4">
         <Chat
           chatList={getFormattedChatList([detail.message])}
           isHideSendInput={true}
@@ -217,7 +217,7 @@ function DetailPanel<T extends ChatConversationFullDetailResponse | CompletionCo
         />
       </div>
       : items.length < 8
-        ? <div className="px-2.5 pt-4 mb-4 overflow-y-auto">
+        ? <div className="px-2.5 pt-4 mb-4">
           <Chat
             chatList={items}
             isHideSendInput={true}

web/app/components/base/block-input/index.tsx

@@ -63,7 +63,7 @@ const BlockInput: FC<IBlockInputProps> = ({
   }, [isEditing])
 
   const style = classNames({
-    'block px-4 py-1 w-full h-full text-sm text-gray-900 outline-0 border-0': true,
+    'block px-4 py-1 w-full h-full text-sm text-gray-900 outline-0 border-0 break-all': true,
     'block-input--editing': isEditing,
   })
 

web/app/components/base/modal/index.tsx

@@ -5,6 +5,7 @@ import { XMarkIcon } from '@heroicons/react/24/outline'
 
 type IModal = {
   className?: string
+  wrapperClassName?: string
   isShow: boolean
   onClose: () => void
   title?: React.ReactNode
@@ -15,6 +16,7 @@ type IModal = {
 
 export default function Modal({
   className,
+  wrapperClassName,
   isShow,
   onClose,
   title,
@@ -38,7 +40,7 @@ export default function Modal({
                 </Transition.Child>
 
                 <div className="fixed inset-0 overflow-y-auto">
-                    <div className="flex min-h-full items-center justify-center p-4 text-center">
+                    <div className={`flex min-h-full items-center justify-center p-4 text-center ${wrapperClassName}`}>
                         <Transition.Child
                             as={Fragment}
                             enter="ease-out duration-300"

web/app/components/header/account-setting/index.tsx

@@ -75,6 +75,7 @@ export default function AccountSetting({
       isShow
       onClose={() => { }}
       className={s.modal}
+      wrapperClassName='pt-[60px]'
     >
       <div className='flex'>
         <div className='w-[200px] p-4 border border-gray-100'>

web/i18n/lang/common.en.ts

@@ -51,6 +51,7 @@ const translation = {
       maxToken: 'Max token',
       maxTokenTip:
         'Max tokens generated is 2,048 or 4,000, depending on the model. Prompt and completion share this limit. One token is roughly 1 English character.',
+      setToCurrentModelMaxTokenTip: 'Max token is updated to the maximum token of the current model 4,000.',
     },
     tone: {
       Creative: 'Creative',

web/i18n/lang/common.zh.ts

@@ -51,6 +51,7 @@ const translation = {
       maxToken: '最大 Token',
       maxTokenTip:
         '生成的最大令牌数为 2,048 或 4,000，取决于模型。提示和完成共享令牌数限制。一个令牌约等于 1 个英文或 4 个中文字符。',
+      setToCurrentModelMaxTokenTip: '最大令牌数更新为当前模型最大的令牌数 4,000。',
     },
     tone: {
       Creative: '创意',