add create tidb serverless job control (#10467)

Co-authored-by: crazywoola <427733928@qq.com>

api/.env.example

@@ -396,3 +396,5 @@ POSITION_PROVIDER_EXCLUDES=
 
 # Reset password token expiry minutes
 RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
+
+CREATE_TIDB_SERVICE_JOB_ENABLED=false

api/configs/feature/__init__.py

@@ -611,6 +611,11 @@ class DataSetConfig(BaseSettings):
         default=500,
     )
 
+    CREATE_TIDB_SERVICE_JOB_ENABLED: bool = Field(
+        description="Enable or disable create tidb service job",
+        default=False,
+    )
+
 
 class WorkspaceConfig(BaseSettings):
     """

api/schedule/create_tidb_serverless_task.py

@@ -12,6 +12,8 @@ from models.dataset import TidbAuthBinding
 @app.celery.task(queue="dataset")
 def create_tidb_serverless_task():
     click.echo(click.style("Start create tidb serverless task.", fg="green"))
+    if not dify_config.CREATE_TIDB_SERVICE_JOB_ENABLED:
+        return
     tidb_serverless_number = dify_config.TIDB_SERVERLESS_NUMBER
     start_at = time.perf_counter()
     while True:

docker-legacy/docker-compose.yaml

@@ -384,6 +384,7 @@ services:
       NOTION_INTERNAL_SECRET: you-internal-secret
       # Indexing configuration
       INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH: 1000
+      CREATE_TIDB_SERVICE_JOB_ENABLED: false
     depends_on:
       - db
       - redis

docker/.env.example

@@ -907,3 +907,6 @@ POSITION_PROVIDER_EXCLUDES=
 
 # CSP https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP
 CSP_WHITELIST=
+
+# Enable or disable create tidb service job
+CREATE_TIDB_SERVICE_JOB_ENABLED=false

docker/docker-compose.yaml

@@ -271,6 +271,7 @@ x-shared-env: &shared-api-worker-env
   OCEANBASE_VECTOR_DATABASE: ${OCEANBASE_VECTOR_DATABASE:-test}
   OCEANBASE_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai}
   OCEANBASE_MEMORY_LIMIT: ${OCEANBASE_MEMORY_LIMIT:-6G}
+  CREATE_TIDB_SERVICE_JOB_ENABLED: ${CREATE_TIDB_SERVICE_JOB_ENABLED:-false}
 
 services:
   # API service