diff --git a/api/.env.example b/api/.env.example index 95da531a1d..695d8eab66 100644 --- a/api/.env.example +++ b/api/.env.example @@ -430,4 +430,7 @@ CREATE_TIDB_SERVICE_JOB_ENABLED=false # Maximum number of submitted thread count in a ThreadPool for parallel node execution MAX_SUBMIT_COUNT=100 # Lockout duration in seconds -LOGIN_LOCKOUT_DURATION=86400 \ No newline at end of file +LOGIN_LOCKOUT_DURATION=86400 + +# User memory generate task interval in minutes +USER_MEMORY_GENERATE_TASK_INTERVAL=5 \ No newline at end of file diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 9e2ba41780..bd7d3b435b 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -1,5 +1,6 @@ from typing import Annotated, Literal, Optional +from configs.feature.hosted_service import HostedServiceConfig from pydantic import ( AliasChoices, Field, @@ -12,8 +13,6 @@ from pydantic import ( ) from pydantic_settings import BaseSettings -from configs.feature.hosted_service import HostedServiceConfig - class SecurityConfig(BaseSettings): """ @@ -68,7 +67,7 @@ class CodeExecutionSandboxConfig(BaseSettings): Configuration for the code execution sandbox environment """ - CODE_EXECUTION_ENDPOINT: HttpUrl = Field( + CODE_EXECUTION_ENDPOINT: str = Field( description="URL endpoint for the code execution service", default="http://sandbox:8194", ) @@ -691,6 +690,11 @@ class CeleryBeatConfig(BaseSettings): default=1, ) + USER_MEMORY_GENERATE_TASK_INTERVAL: int = Field( + description="Interval in seconds for user memory generate task execution, default to 5 minutes", + default=5, + ) + class PositionConfig(BaseSettings): POSITION_PROVIDER_PINS: str = Field( diff --git a/api/extensions/ext_celery.py b/api/extensions/ext_celery.py index 26bd6b3577..5d9e850a29 100644 --- a/api/extensions/ext_celery.py +++ b/api/extensions/ext_celery.py @@ -3,7 +3,6 @@ from datetime import timedelta import pytz from celery import Celery, Task # type: ignore from celery.schedules import crontab # type: ignore - from configs import dify_config from dify_app import DifyApp @@ -68,8 +67,9 @@ def init_app(app: DifyApp) -> Celery: "schedule.clean_unused_datasets_task", "schedule.create_tidb_serverless_task", "schedule.update_tidb_serverless_status_task", - "schedule.clean_messages", - "schedule.mail_clean_document_notify_task", + # "schedule.clean_messages", + # "schedule.mail_clean_document_notify_task", + "schedule.user_memory_generate_task", ] day = dify_config.CELERY_BEAT_SCHEDULER_TIME beat_schedule = { @@ -89,14 +89,18 @@ def init_app(app: DifyApp) -> Celery: "task": "schedule.update_tidb_serverless_status_task.update_tidb_serverless_status_task", "schedule": timedelta(minutes=10), }, - "clean_messages": { - "task": "schedule.clean_messages.clean_messages", - "schedule": timedelta(days=day), - }, + # "clean_messages": { + # "task": "schedule.clean_messages.clean_messages", + # "schedule": timedelta(days=day), + # }, # every Monday - "mail_clean_document_notify_task": { - "task": "schedule.mail_clean_document_notify_task.mail_clean_document_notify_task", - "schedule": crontab(minute="0", hour="10", day_of_week="1"), + # "mail_clean_document_notify_task": { + # "task": "schedule.mail_clean_document_notify_task.mail_clean_document_notify_task", + # "schedule": crontab(minute="0", hour="10", day_of_week="1"), + # }, + "user_memory_generate_task": { + "task": "schedule.user_memory_generate_task.user_memory_generate_task", + "schedule": timedelta(minutes=dify_config.USER_MEMORY_GENERATE_TASK_INTERVAL), }, } celery_app.conf.update(beat_schedule=beat_schedule, imports=imports) diff --git a/api/schedule/user_memory_generate_task.py b/api/schedule/user_memory_generate_task.py new file mode 100644 index 0000000000..23923fcbef --- /dev/null +++ b/api/schedule/user_memory_generate_task.py @@ -0,0 +1,19 @@ +import time + +import app +import click + +# Import other necessary modules + + +@app.celery.task(queue="dataset") +def user_memory_generate_task(): + click.echo(click.style("Starting user memory generate task.", fg="green")) + start_at = time.perf_counter() + + # Your task logic here + # ... + click.echo(click.style("TODO: ytqh user memory generate task.", fg="green")) + + end_at = time.perf_counter() + click.echo(click.style(f"Task completed successfully. Latency: {end_at - start_at}", fg="green")) diff --git a/docker/.env.example b/docker/.env.example index 653360f266..e302bb35a5 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -943,3 +943,6 @@ ALLOW_REGISTER=true DEFAULT_APP_ID= DEFAULT_TENANT_ID= DEBUG_EMAIL_CODE_FOR_LOGIN= + +# User memory generate task interval in minutes +USER_MEMORY_GENERATE_TASK_INTERVAL=5 diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 16927749b0..209c34dbaf 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -49,6 +49,29 @@ services: - ssrf_proxy_network - default + scheduler: + image: akiyu303/jim-api:latest + restart: always + extra_hosts: + - "host.docker.internal:host-gateway" + environment: + # Use the shared environment variables. + <<: *shared-api-worker-env + # Startup mode, 'worker' starts the Celery worker for processing the queue. + MODE: beat + SENTRY_DSN: ${API_SENTRY_DSN:-} + SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} + SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} + depends_on: + - db + - redis + volumes: + # Mount the storage directory to the container, for storing user files. + - ./volumes/app/storage:/app/api/storage + networks: + - ssrf_proxy_network + - default + # Frontend web application. web: image: langgenius/dify-web:0.15.2 diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index db538a198a..91d4bcbd1a 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -396,6 +396,7 @@ x-shared-env: &shared-api-worker-env DEFAULT_APP_ID: ${DEFAULT_APP_ID:-} DEFAULT_TENANT_ID: ${DEFAULT_TENANT_ID:-} DEBUG_EMAIL_CODE_FOR_LOGIN: ${DEBUG_EMAIL_CODE_FOR_LOGIN:-} + USER_MEMORY_GENERATE_TASK_INTERVAL: ${USER_MEMORY_GENERATE_TASK_INTERVAL:-5} services: # API service @@ -447,6 +448,29 @@ services: - ssrf_proxy_network - default + scheduler: + image: akiyu303/jim-api:latest + restart: always + extra_hosts: + - "host.docker.internal:host-gateway" + environment: + # Use the shared environment variables. + <<: *shared-api-worker-env + # Startup mode, 'worker' starts the Celery worker for processing the queue. + MODE: beat + SENTRY_DSN: ${API_SENTRY_DSN:-} + SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0} + SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0} + depends_on: + - db + - redis + volumes: + # Mount the storage directory to the container, for storing user files. + - ./volumes/app/storage:/app/api/storage + networks: + - ssrf_proxy_network + - default + # Frontend web application. web: image: langgenius/dify-web:0.15.2