add user_memory_generate_task

api/.env.example

@@ -430,4 +430,7 @@ CREATE_TIDB_SERVICE_JOB_ENABLED=false
 # Maximum number of submitted thread count in a ThreadPool for parallel node execution
 MAX_SUBMIT_COUNT=100
 # Lockout duration in seconds
-LOGIN_LOCKOUT_DURATION=86400
+LOGIN_LOCKOUT_DURATION=86400
+
+# User memory generate task interval in minutes
+USER_MEMORY_GENERATE_TASK_INTERVAL=5

api/configs/feature/__init__.py

@@ -1,5 +1,6 @@
 from typing import Annotated, Literal, Optional
 
+from configs.feature.hosted_service import HostedServiceConfig
 from pydantic import (
     AliasChoices,
     Field,
@@ -12,8 +13,6 @@ from pydantic import (
 )
 from pydantic_settings import BaseSettings
 
-from configs.feature.hosted_service import HostedServiceConfig
-
 
 class SecurityConfig(BaseSettings):
     """
@@ -68,7 +67,7 @@ class CodeExecutionSandboxConfig(BaseSettings):
     Configuration for the code execution sandbox environment
     """
 
-    CODE_EXECUTION_ENDPOINT: HttpUrl = Field(
+    CODE_EXECUTION_ENDPOINT: str = Field(
         description="URL endpoint for the code execution service",
         default="http://sandbox:8194",
     )
@@ -691,6 +690,11 @@ class CeleryBeatConfig(BaseSettings):
         default=1,
     )
 
+    USER_MEMORY_GENERATE_TASK_INTERVAL: int = Field(
+        description="Interval in seconds for user memory generate task execution, default to 5 minutes",
+        default=5,
+    )
+
 
 class PositionConfig(BaseSettings):
     POSITION_PROVIDER_PINS: str = Field(

api/extensions/ext_celery.py

@@ -3,7 +3,6 @@ from datetime import timedelta
 import pytz
 from celery import Celery, Task  # type: ignore
 from celery.schedules import crontab  # type: ignore
-
 from configs import dify_config
 from dify_app import DifyApp
 
@@ -68,8 +67,9 @@ def init_app(app: DifyApp) -> Celery:
         "schedule.clean_unused_datasets_task",
         "schedule.create_tidb_serverless_task",
         "schedule.update_tidb_serverless_status_task",
-        "schedule.clean_messages",
-        "schedule.mail_clean_document_notify_task",
+        # "schedule.clean_messages",
+        # "schedule.mail_clean_document_notify_task",
+        "schedule.user_memory_generate_task",
     ]
     day = dify_config.CELERY_BEAT_SCHEDULER_TIME
     beat_schedule = {
@@ -89,14 +89,18 @@ def init_app(app: DifyApp) -> Celery:
             "task": "schedule.update_tidb_serverless_status_task.update_tidb_serverless_status_task",
             "schedule": timedelta(minutes=10),
         },
-        "clean_messages": {
-            "task": "schedule.clean_messages.clean_messages",
-            "schedule": timedelta(days=day),
-        },
+        # "clean_messages": {
+        #     "task": "schedule.clean_messages.clean_messages",
+        #     "schedule": timedelta(days=day),
+        # },
         # every Monday
-        "mail_clean_document_notify_task": {
-            "task": "schedule.mail_clean_document_notify_task.mail_clean_document_notify_task",
-            "schedule": crontab(minute="0", hour="10", day_of_week="1"),
+        # "mail_clean_document_notify_task": {
+        #     "task": "schedule.mail_clean_document_notify_task.mail_clean_document_notify_task",
+        #     "schedule": crontab(minute="0", hour="10", day_of_week="1"),
+        # },
+        "user_memory_generate_task": {
+            "task": "schedule.user_memory_generate_task.user_memory_generate_task",
+            "schedule": timedelta(minutes=dify_config.USER_MEMORY_GENERATE_TASK_INTERVAL),
         },
     }
     celery_app.conf.update(beat_schedule=beat_schedule, imports=imports)

api/schedule/user_memory_generate_task.py

@@ -0,0 +1,19 @@
+import time
+
+import app
+import click
+
+# Import other necessary modules
+
+
+@app.celery.task(queue="dataset")
+def user_memory_generate_task():
+    click.echo(click.style("Starting user memory generate task.", fg="green"))
+    start_at = time.perf_counter()
+
+    # Your task logic here
+    # ...
+    click.echo(click.style("TODO: ytqh user memory generate task.", fg="green"))
+
+    end_at = time.perf_counter()
+    click.echo(click.style(f"Task completed successfully. Latency: {end_at - start_at}", fg="green"))

docker/.env.example

@@ -943,3 +943,6 @@ ALLOW_REGISTER=true
 DEFAULT_APP_ID=
 DEFAULT_TENANT_ID=
 DEBUG_EMAIL_CODE_FOR_LOGIN=
+
+# User memory generate task interval in minutes
+USER_MEMORY_GENERATE_TASK_INTERVAL=5

docker/docker-compose-template.yaml

@@ -49,6 +49,29 @@ services:
       - ssrf_proxy_network
       - default
 
+  scheduler:
+    image: akiyu303/jim-api:latest
+    restart: always
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    environment:
+      # Use the shared environment variables.
+      <<: *shared-api-worker-env
+      # Startup mode, 'worker' starts the Celery worker for processing the queue.
+      MODE: beat
+      SENTRY_DSN: ${API_SENTRY_DSN:-}
+      SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0}
+      SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0}
+    depends_on:
+      - db
+      - redis
+    volumes:
+      # Mount the storage directory to the container, for storing user files.
+      - ./volumes/app/storage:/app/api/storage
+    networks:
+      - ssrf_proxy_network
+      - default
+
   # Frontend web application.
   web:
     image: langgenius/dify-web:0.15.2

docker/docker-compose.yaml

@@ -396,6 +396,7 @@ x-shared-env: &shared-api-worker-env
   DEFAULT_APP_ID: ${DEFAULT_APP_ID:-}
   DEFAULT_TENANT_ID: ${DEFAULT_TENANT_ID:-}
   DEBUG_EMAIL_CODE_FOR_LOGIN: ${DEBUG_EMAIL_CODE_FOR_LOGIN:-}
+  USER_MEMORY_GENERATE_TASK_INTERVAL: ${USER_MEMORY_GENERATE_TASK_INTERVAL:-5}
 
 services:
   # API service
@@ -447,6 +448,29 @@ services:
       - ssrf_proxy_network
       - default
 
+  scheduler:
+    image: akiyu303/jim-api:latest
+    restart: always
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    environment:
+      # Use the shared environment variables.
+      <<: *shared-api-worker-env
+      # Startup mode, 'worker' starts the Celery worker for processing the queue.
+      MODE: beat
+      SENTRY_DSN: ${API_SENTRY_DSN:-}
+      SENTRY_TRACES_SAMPLE_RATE: ${API_SENTRY_TRACES_SAMPLE_RATE:-1.0}
+      SENTRY_PROFILES_SAMPLE_RATE: ${API_SENTRY_PROFILES_SAMPLE_RATE:-1.0}
+    depends_on:
+      - db
+      - redis
+    volumes:
+      # Mount the storage directory to the container, for storing user files.
+      - ./volumes/app/storage:/app/api/storage
+    networks:
+      - ssrf_proxy_network
+      - default
+
   # Frontend web application.
   web:
     image: langgenius/dify-web:0.15.2