test(api): adjusted configuration files for integration testings

- split configuration for vdb tests into vdb.example.env.
- add a minimal configuration for dify to `.env.example`.
- copy .env.example to .env while executing integration tests.
- Loading configuration from both `.env` and `vdb.env` while executing
  integration tests
pull/20699/head
QuantumGhost 11 months ago
parent a1624acc08
commit b864653eb3

@ -83,9 +83,16 @@ jobs:
compose-file: | compose-file: |
docker/docker-compose.middleware.yaml docker/docker-compose.middleware.yaml
services: | services: |
db
redis
sandbox sandbox
ssrf_proxy ssrf_proxy
- name: setup test config
run: |
cp api/tests/integration_tests/.env.example cp api/tests/integration_tests/.env
cp api/tests/integration_tests/vdb.example.env api/tests/integration_tests/vdb.env
- name: Run Workflow - name: Run Workflow
run: uv run --project api bash dev/pytest/pytest_workflow.sh run: uv run --project api bash dev/pytest/pytest_workflow.sh

@ -84,6 +84,11 @@ jobs:
elasticsearch elasticsearch
oceanbase oceanbase
- name: setup test config
run: |
cp api/tests/integration_tests/.env.example cp api/tests/integration_tests/.env
cp api/tests/integration_tests/vdb.example.env api/tests/integration_tests/vdb.env
- name: Check VDB Ready (TiDB) - name: Check VDB Ready (TiDB)
run: uv run --project api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py run: uv run --project api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py

@ -1,107 +1,217 @@
# OpenAI API Key FLASK_APP=app.py
OPENAI_API_KEY= FLASK_DEBUG=0
SECRET_KEY='uhySf6a3aZuvRNfAlcr47paOw9TRYBY6j8ZHXpVw1yx5RP27Yj3w2uvI'
CONSOLE_API_URL=http://127.0.0.1:5001
CONSOLE_WEB_URL=http://127.0.0.1:3000
# Service API base URL
SERVICE_API_URL=http://127.0.0.1:5001
# Web APP base URL
APP_WEB_URL=http://127.0.0.1:3000
# Files URL
FILES_URL=http://127.0.0.1:5001
# The time in seconds after the signature is rejected
FILES_ACCESS_TIMEOUT=300
# Access token expiration time in minutes
ACCESS_TOKEN_EXPIRE_MINUTES=60
# Refresh token expiration time in days
REFRESH_TOKEN_EXPIRE_DAYS=30
# celery configuration
CELERY_BROKER_URL=redis://:difyai123456@localhost:6379/1
# redis configuration
REDIS_HOST=localhost
REDIS_PORT=6379
REDIS_USERNAME=
REDIS_PASSWORD=difyai123456
REDIS_USE_SSL=false
REDIS_DB=0
# PostgreSQL database configuration
DB_USERNAME=postgres
DB_PASSWORD=difyai123456
DB_HOST=localhost
DB_PORT=5432
DB_DATABASE=dify
# Storage configuration
# use for store upload files, private keys...
# storage type: opendal, s3, aliyun-oss, azure-blob, baidu-obs, google-storage, huawei-obs, oci-storage, tencent-cos, volcengine-tos, supabase
STORAGE_TYPE=opendal
# Apache OpenDAL storage configuration, refer to https://github.com/apache/opendal
OPENDAL_SCHEME=fs
OPENDAL_FS_ROOT=storage
# CORS configuration
WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
# Vector database configuration
# support: weaviate, qdrant, milvus, myscale, relyt, pgvecto_rs, pgvector, pgvector, chroma, opensearch, tidb_vector, couchbase, vikingdb, upstash, lindorm, oceanbase
VECTOR_STORE=weaviate
# Weaviate configuration
WEAVIATE_ENDPOINT=http://localhost:8080
WEAVIATE_API_KEY=WVF5YThaHlkYwhGUSmCRgsX3tD5ngdN8pkih
WEAVIATE_GRPC_ENABLED=false
WEAVIATE_BATCH_SIZE=100
# Upload configuration
UPLOAD_FILE_SIZE_LIMIT=15
UPLOAD_FILE_BATCH_LIMIT=5
UPLOAD_IMAGE_FILE_SIZE_LIMIT=10
UPLOAD_VIDEO_FILE_SIZE_LIMIT=100
UPLOAD_AUDIO_FILE_SIZE_LIMIT=50
# Model configuration
MULTIMODAL_SEND_FORMAT=base64
PROMPT_GENERATION_MAX_TOKENS=4096
CODE_GENERATION_MAX_TOKENS=1024
# Mail configuration, support: resend, smtp
MAIL_TYPE=
MAIL_DEFAULT_SEND_FROM=no-reply <no-reply@example.com>
RESEND_API_KEY=
RESEND_API_URL=https://api.resend.com
# smtp configuration
SMTP_SERVER=smtp.example.com
SMTP_PORT=465
SMTP_USERNAME=123
SMTP_PASSWORD=abc
SMTP_USE_TLS=true
SMTP_OPPORTUNISTIC_TLS=false
# Sentry configuration
SENTRY_DSN=
# DEBUG
DEBUG=false
SQLALCHEMY_ECHO=false
# Notion import configuration, support public and internal
NOTION_INTEGRATION_TYPE=public
NOTION_CLIENT_SECRET=you-client-secret
NOTION_CLIENT_ID=you-client-id
NOTION_INTERNAL_SECRET=you-internal-secret
ETL_TYPE=dify
UNSTRUCTURED_API_URL=
UNSTRUCTURED_API_KEY=
SCARF_NO_ANALYTICS=false
#ssrf
SSRF_PROXY_HTTP_URL=
SSRF_PROXY_HTTPS_URL=
SSRF_DEFAULT_MAX_RETRIES=3
SSRF_DEFAULT_TIME_OUT=5
SSRF_DEFAULT_CONNECT_TIME_OUT=5
SSRF_DEFAULT_READ_TIME_OUT=5
SSRF_DEFAULT_WRITE_TIME_OUT=5
BATCH_UPLOAD_LIMIT=10
KEYWORD_DATA_SOURCE_TYPE=database
# Workflow file upload limit
WORKFLOW_FILE_UPLOAD_LIMIT=10
# Azure OpenAI API Base Endpoint & API Key # CODE EXECUTION CONFIGURATION
AZURE_OPENAI_API_BASE= CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194
AZURE_OPENAI_API_KEY= CODE_EXECUTION_API_KEY=dify-sandbox
CODE_MAX_NUMBER=9223372036854775807
# Anthropic API Key CODE_MIN_NUMBER=-9223372036854775808
ANTHROPIC_API_KEY= CODE_MAX_STRING_LENGTH=80000
TEMPLATE_TRANSFORM_MAX_LENGTH=80000
# Replicate API Key CODE_MAX_STRING_ARRAY_LENGTH=30
REPLICATE_API_KEY= CODE_MAX_OBJECT_ARRAY_LENGTH=30
CODE_MAX_NUMBER_ARRAY_LENGTH=1000
# Hugging Face API Key
HUGGINGFACE_API_KEY= # API Tool configuration
HUGGINGFACE_TEXT_GEN_ENDPOINT_URL= API_TOOL_DEFAULT_CONNECT_TIMEOUT=10
HUGGINGFACE_TEXT2TEXT_GEN_ENDPOINT_URL= API_TOOL_DEFAULT_READ_TIMEOUT=60
HUGGINGFACE_EMBEDDINGS_ENDPOINT_URL=
# HTTP Node configuration
# Minimax Credentials HTTP_REQUEST_MAX_CONNECT_TIMEOUT=300
MINIMAX_API_KEY= HTTP_REQUEST_MAX_READ_TIMEOUT=600
MINIMAX_GROUP_ID= HTTP_REQUEST_MAX_WRITE_TIMEOUT=600
HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760
# Spark Credentials HTTP_REQUEST_NODE_MAX_TEXT_SIZE=1048576
SPARK_APP_ID=
SPARK_API_KEY= # Respect X-* headers to redirect clients
SPARK_API_SECRET= RESPECT_XFORWARD_HEADERS_ENABLED=false
# Tongyi Credentials # Log file path
TONGYI_DASHSCOPE_API_KEY= LOG_FILE=
# Log file max size, the unit is MB
# Wenxin Credentials LOG_FILE_MAX_SIZE=20
WENXIN_API_KEY= # Log file max backup count
WENXIN_SECRET_KEY= LOG_FILE_BACKUP_COUNT=5
# Log dateformat
# ZhipuAI Credentials LOG_DATEFORMAT=%Y-%m-%d %H:%M:%S
ZHIPUAI_API_KEY= # Log Timezone
LOG_TZ=UTC
# Baichuan Credentials # Log format
BAICHUAN_API_KEY= LOG_FORMAT=%(asctime)s,%(msecs)d %(levelname)-2s [%(filename)s:%(lineno)d] %(req_id)s %(message)s
BAICHUAN_SECRET_KEY=
# Indexing configuration
# ChatGLM Credentials INDEXING_MAX_SEGMENTATION_TOKENS_LENGTH=4000
CHATGLM_API_BASE=
# Workflow runtime configuration
# Xinference Credentials WORKFLOW_MAX_EXECUTION_STEPS=500
XINFERENCE_SERVER_URL= WORKFLOW_MAX_EXECUTION_TIME=1200
XINFERENCE_GENERATION_MODEL_UID= WORKFLOW_CALL_MAX_DEPTH=5
XINFERENCE_CHAT_MODEL_UID= WORKFLOW_PARALLEL_DEPTH_LIMIT=3
XINFERENCE_EMBEDDINGS_MODEL_UID= MAX_VARIABLE_SIZE=204800
XINFERENCE_RERANK_MODEL_UID=
# App configuration
# OpenLLM Credentials APP_MAX_EXECUTION_TIME=1200
OPENLLM_SERVER_URL= APP_MAX_ACTIVE_REQUESTS=0
# LocalAI Credentials # Celery beat configuration
LOCALAI_SERVER_URL= CELERY_BEAT_SCHEDULER_TIME=1
# Cohere Credentials # Position configuration
COHERE_API_KEY= POSITION_TOOL_PINS=
POSITION_TOOL_INCLUDES=
# Jina Credentials POSITION_TOOL_EXCLUDES=
JINA_API_KEY=
POSITION_PROVIDER_PINS=
# Ollama Credentials POSITION_PROVIDER_INCLUDES=
OLLAMA_BASE_URL= POSITION_PROVIDER_EXCLUDES=
# Together API Key # Plugin configuration
TOGETHER_API_KEY= PLUGIN_DAEMON_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi
PLUGIN_DAEMON_URL=http://127.0.0.1:5002
PLUGIN_REMOTE_INSTALL_PORT=5003
PLUGIN_REMOTE_INSTALL_HOST=localhost
PLUGIN_MAX_PACKAGE_SIZE=15728640
INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
# Mock Switch # Marketplace configuration
MOCK_SWITCH=false MARKETPLACE_ENABLED=true
MARKETPLACE_API_URL=https://marketplace.dify.ai
# CODE EXECUTION CONFIGURATION # Endpoint configuration
CODE_EXECUTION_ENDPOINT= ENDPOINT_URL_TEMPLATE=http://localhost:5002/e/{hook_id}
CODE_EXECUTION_API_KEY=
# Volcengine MaaS Credentials # Reset password token expiry minutes
VOLC_API_KEY= RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5
VOLC_SECRET_KEY=
VOLC_MODEL_ENDPOINT_ID=
VOLC_EMBEDDING_ENDPOINT_ID=
# 360 AI Credentials CREATE_TIDB_SERVICE_JOB_ENABLED=false
ZHINAO_API_KEY=
# Plugin configuration # Maximum number of submitted thread count in a ThreadPool for parallel node execution
PLUGIN_DAEMON_KEY= MAX_SUBMIT_COUNT=100
PLUGIN_DAEMON_URL= # Lockout duration in seconds
LOGIN_LOCKOUT_DURATION=86400
# Marketplace configuration HTTP_PROXY='http://127.0.0.1:1092'
MARKETPLACE_API_URL= HTTPS_PROXY='http://127.0.0.1:1092'
# VESSL AI Credentials NO_PROXY='localhost,127.0.0.1'
VESSL_AI_MODEL_NAME= LOG_LEVEL=INFO
VESSL_AI_API_KEY=
VESSL_AI_ENDPOINT_URL=
# GPUStack Credentials
GPUSTACK_SERVER_URL=
GPUSTACK_API_KEY=
# Gitee AI Credentials
GITEE_AI_API_KEY=
# xAI Credentials
XAI_API_KEY=
XAI_API_BASE=

@ -1,4 +1,4 @@
import os import pathlib
import random import random
import secrets import secrets
from collections.abc import Generator from collections.abc import Generator
@ -12,20 +12,22 @@ from app_factory import create_app
from models import Account, DifySetup, Tenant, TenantAccountJoin, db from models import Account, DifySetup, Tenant, TenantAccountJoin, db
from services.account_service import AccountService, RegisterService from services.account_service import AccountService, RegisterService
# Getting the absolute path of the current file's directory
ABS_PATH = os.path.dirname(os.path.abspath(__file__))
# Getting the absolute path of the project's root directory
PROJECT_DIR = os.path.abspath(os.path.join(ABS_PATH, os.pardir, os.pardir))
# Loading the .env file if it exists # Loading the .env file if it exists
def _load_env() -> None: def _load_env() -> None:
dotenv_path = os.path.join(PROJECT_DIR, "tests", "integration_tests", ".env") current_file_path = pathlib.Path(__file__).absolute()
if os.path.exists(dotenv_path): # Items later in the list have higher precedence.
files_to_load = [".env", "vdb.env"]
env_file_paths = [current_file_path.parent / i for i in files_to_load]
for path in env_file_paths:
if not path.exists():
continue
from dotenv import load_dotenv from dotenv import load_dotenv
load_dotenv(dotenv_path) # Set `override=True` to ensure values from `vdb.env` take priority over values from `.env`
load_dotenv(str(path), override=True)
_load_env() _load_env()

Loading…
Cancel
Save