diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml deleted file mode 100644 index f08befefb8..0000000000 --- a/.github/workflows/api-tests.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: Run Pytest - -on: - pull_request: - branches: - - main - paths: - - api/** - - docker/** - - .github/workflows/api-tests.yml - -concurrency: - group: api-tests-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - test: - name: API Tests - runs-on: ubuntu-latest - defaults: - run: - shell: bash - strategy: - matrix: - python-version: - - "3.11" - - "3.12" - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup UV and Python - uses: ./.github/actions/setup-uv - with: - python-version: ${{ matrix.python-version }} - uv-lockfile: api/uv.lock - - - name: Check UV lockfile - run: uv lock --project api --check - - - name: Install dependencies - run: uv sync --project api --dev - - - name: Run Unit tests - run: | - uv run --project api bash dev/pytest/pytest_unit_tests.sh - # Extract coverage percentage and create a summary - TOTAL_COVERAGE=$(python -c 'import json; print(json.load(open("coverage.json"))["totals"]["percent_covered_display"])') - - # Create a detailed coverage summary - echo "### Test Coverage Summary :test_tube:" >> $GITHUB_STEP_SUMMARY - echo "Total Coverage: ${TOTAL_COVERAGE}%" >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - uv run --project api coverage report >> $GITHUB_STEP_SUMMARY - echo "\`\`\`" >> $GITHUB_STEP_SUMMARY - - - name: Run dify config tests - run: uv run --project api dev/pytest/pytest_config_tests.py - - - name: MyPy Cache - uses: actions/cache@v4 - with: - path: api/.mypy_cache - key: mypy-${{ matrix.python-version }}-${{ runner.os }}-${{ hashFiles('api/uv.lock') }} - - - name: Run MyPy Checks - run: dev/mypy-check - - - name: Set up dotenvs - run: | - cp docker/.env.example docker/.env - cp docker/middleware.env.example docker/middleware.env - - - name: Expose Service Ports - run: sh .github/workflows/expose_service_ports.sh - - - name: Set up Sandbox - uses: hoverkraft-tech/compose-action@v2.0.2 - with: - compose-file: | - docker/docker-compose.middleware.yaml - services: | - sandbox - ssrf_proxy - - - name: Run Workflow - run: uv run --project api bash dev/pytest/pytest_workflow.sh - - - name: Run Tool - run: uv run --project api bash dev/pytest/pytest_tools.sh diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml deleted file mode 100644 index cc735ae67c..0000000000 --- a/.github/workflows/build-push.yml +++ /dev/null @@ -1,149 +0,0 @@ -name: Build and Push API & Web - -on: - push: - branches: - - "main" - - "deploy/dev" - - "deploy/enterprise" - tags: - - "*" - -concurrency: - group: build-push-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - DIFY_WEB_IMAGE_NAME: ${{ vars.DIFY_WEB_IMAGE_NAME || 'langgenius/dify-web' }} - DIFY_API_IMAGE_NAME: ${{ vars.DIFY_API_IMAGE_NAME || 'langgenius/dify-api' }} - -jobs: - build: - runs-on: ${{ matrix.platform == 'linux/arm64' && 'arm64_runner' || 'ubuntu-latest' }} - if: github.repository == 'langgenius/dify' - strategy: - matrix: - include: - - service_name: "build-api-amd64" - image_name_env: "DIFY_API_IMAGE_NAME" - context: "api" - platform: linux/amd64 - - service_name: "build-api-arm64" - image_name_env: "DIFY_API_IMAGE_NAME" - context: "api" - platform: linux/arm64 - - service_name: "build-web-amd64" - image_name_env: "DIFY_WEB_IMAGE_NAME" - context: "web" - platform: linux/amd64 - - service_name: "build-web-arm64" - image_name_env: "DIFY_WEB_IMAGE_NAME" - context: "web" - platform: linux/arm64 - - steps: - - name: Prepare - run: | - platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ env.DOCKERHUB_USER }} - password: ${{ env.DOCKERHUB_TOKEN }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Extract metadata for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env[matrix.image_name_env] }} - - - name: Build Docker image - id: build - uses: docker/build-push-action@v6 - with: - context: "{{defaultContext}}:${{ matrix.context }}" - platforms: ${{ matrix.platform }} - build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env[matrix.image_name_env] }},push-by-digest=true,name-canonical=true,push=true - cache-from: type=gha,scope=${{ matrix.service_name }} - cache-to: type=gha,mode=max,scope=${{ matrix.service_name }} - - - name: Export digest - env: - DIGEST: ${{ steps.build.outputs.digest }} - run: | - mkdir -p /tmp/digests - sanitized_digest=${DIGEST#sha256:} - touch "/tmp/digests/${sanitized_digest}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ matrix.context }}-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - create-manifest: - needs: build - runs-on: ubuntu-latest - if: github.repository == 'langgenius/dify' - strategy: - matrix: - include: - - service_name: "merge-api-images" - image_name_env: "DIFY_API_IMAGE_NAME" - context: "api" - - service_name: "merge-web-images" - image_name_env: "DIFY_WEB_IMAGE_NAME" - context: "web" - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digests-${{ matrix.context }}-* - merge-multiple: true - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ env.DOCKERHUB_USER }} - password: ${{ env.DOCKERHUB_TOKEN }} - - - name: Extract metadata for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env[matrix.image_name_env] }} - tags: | - type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') }} - type=ref,event=branch - type=sha,enable=true,priority=100,prefix=,suffix=,format=long - type=raw,value=${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }} - - - name: Create manifest list and push - working-directory: /tmp/digests - env: - IMAGE_NAME: ${{ env[matrix.image_name_env] }} - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf "$IMAGE_NAME@sha256:%s " *) - - - name: Inspect image - env: - IMAGE_NAME: ${{ env[matrix.image_name_env] }} - IMAGE_VERSION: ${{ steps.meta.outputs.version }} - run: | - docker buildx imagetools inspect "$IMAGE_NAME:$IMAGE_VERSION" diff --git a/.github/workflows/db-migration-test.yml b/.github/workflows/db-migration-test.yml deleted file mode 100644 index 5181546b4a..0000000000 --- a/.github/workflows/db-migration-test.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: DB Migration Test - -on: - pull_request: - branches: - - main - - plugins/beta - paths: - - api/migrations/** - - .github/workflows/db-migration-test.yml - -concurrency: - group: db-migration-test-${{ github.ref }} - cancel-in-progress: true - -jobs: - db-migration-test: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Setup UV and Python - uses: ./.github/actions/setup-uv - with: - uv-lockfile: api/uv.lock - - - name: Install dependencies - run: uv sync --project api - - - name: Prepare middleware env - run: | - cd docker - cp middleware.env.example middleware.env - - - name: Set up Middlewares - uses: hoverkraft-tech/compose-action@v2.0.2 - with: - compose-file: | - docker/docker-compose.middleware.yaml - services: | - db - redis - - - name: Prepare configs - run: | - cd api - cp .env.example .env - - - name: Run DB Migration - env: - DEBUG: true - run: uv run --directory api flask upgrade-db diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml deleted file mode 100644 index 47ca03c2eb..0000000000 --- a/.github/workflows/deploy-dev.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Deploy Dev - -on: - workflow_run: - workflows: ["Build and Push API & Web"] - branches: - - "deploy/dev" - types: - - completed - -jobs: - deploy: - runs-on: ubuntu-latest - if: | - github.event.workflow_run.conclusion == 'success' - steps: - - name: Deploy to server - uses: appleboy/ssh-action@v0.1.8 - with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - key: ${{ secrets.SSH_PRIVATE_KEY }} - script: | - ${{ vars.SSH_SCRIPT || secrets.SSH_SCRIPT }} diff --git a/.github/workflows/deploy-enterprise.yml b/.github/workflows/deploy-enterprise.yml deleted file mode 100644 index 98fa7c3b49..0000000000 --- a/.github/workflows/deploy-enterprise.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Deploy Enterprise - -permissions: - contents: read - -on: - workflow_run: - workflows: ["Build and Push API & Web"] - branches: - - "deploy/enterprise" - types: - - completed - -jobs: - deploy: - runs-on: ubuntu-latest - if: | - github.event.workflow_run.conclusion == 'success' && - github.event.workflow_run.head_branch == 'deploy/enterprise' - - steps: - - name: Deploy to server - uses: appleboy/ssh-action@v0.1.8 - with: - host: ${{ secrets.ENTERPRISE_SSH_HOST }} - username: ${{ secrets.ENTERPRISE_SSH_USER }} - password: ${{ secrets.ENTERPRISE_SSH_PASSWORD }} - script: | - ${{ vars.ENTERPRISE_SSH_SCRIPT || secrets.ENTERPRISE_SSH_SCRIPT }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000000..c1854e49b0 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,142 @@ +name: Reusable deployment job + +on: + workflow_call: + inputs: + APPLICATION_NAME: + default: 'oneapi' + type: string + AWS_REGION: + default: 'us-east-2' + type: string + DEPLOYMENT_ENV: + required: true + type: string + ECR_REPOSITORY: + required: true + type: string + ECS_CLUSTER: + required: true + type: string + SECURITY_GROUP_IDS_SSM: + required: true + type: string + SUBNET_IDS_SSM: + required: true + type: string + VPC_ID_SSM: + required: true + type: string + SNS_TO_LARK_LAMBDA_ARN: + required: true + type: string + CERERTIFICATE_US_EAST_2_ARN: + required: true + type: string + ECS_EXECUTION_ROLE_ARN: + required: true + type: string + DOCKER_IMAGE_NAME: + required: true + type: string + MODE: + required: true + type: string + APP_PORT: + required: true + type: number + HEALTH_CHECK: + required: true + type: string + TEMPLATE: + required: true + type: string + DOCKERFILEDIR: + required: true + type: string + + + secrets: + DEPLOYMENT_AWS_ACCESS_KEY_ID: + required: true + DEPLOYMENT_AWS_SECRET_ACCESS_KEY: + required: true + ED25519_CI_KEY: + required: true + PAT: + required: true +env: + ECS_CONTAINER_ENV_FILE_S3_PATH: ${{ inputs.DEPLOYMENT_ENV }}-dify-ai/.env +jobs: + deploy: + name: Deploy to ${{ inputs.DEPLOYMENT_ENV }} + runs-on: ubuntu-latest + concurrency: deployment-${{ inputs.APPLICATION_NAME }}-${{ inputs.DEPLOYMENT_ENV }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Fetch and checkout branch + run: | + BRANCH=${{ inputs.DEPLOYMENT_ENV }} + echo "Target branch: $BRANCH" + git fetch origin $BRANCH + git checkout $BRANCH + git reset --hard origin/$BRANCH + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.AWS_REGION }} + + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Set SHORT_SHA + id: set_short_sha + run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >>$GITHUB_ENV + - name: Set IMAGE_TAG + id: set_image_tag + run: echo "IMAGE_TAG=${{ inputs.APPLICATION_NAME }}-$SHORT_SHA" >> $GITHUB_ENV + - name: Set GIT_BRANCH + id: set_git_branch + run: echo "GIT_BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV + + - name: Build, tag, and push image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + + run: | + cd ${{ inputs.DOCKERFILEDIR }} + docker build --build-arg GIT_BRANCH=$GIT_BRANCH --build-arg GIT_PAT_TOKEN=${{ secrets.PAT }} -t $ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG . + docker push $ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG + echo "::set-output name=image::$ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG" + + - name: Deploy CloudFormation + uses: aws-actions/aws-cloudformation-github-deploy@v1 + with: + name: ${{ inputs.APPLICATION_NAME }}-${{ inputs.DEPLOYMENT_ENV }} + template: aws/${{ inputs.TEMPLATE }} + no-fail-on-empty-changeset: "1" + parameter-overrides: >- + CertificateArn=${{ inputs.CERERTIFICATE_US_EAST_2_ARN }}, + ExecutionRoleArn=${{ inputs.ECS_EXECUTION_ROLE_ARN }}, + ECSClusterName=${{ inputs.ECS_CLUSTER }}, + ApplicationName=${{ inputs.APPLICATION_NAME }}, + EnvName=${{ inputs.DEPLOYMENT_ENV }}, + SecurityGroupIDs=${{ inputs.SECURITY_GROUP_IDS_SSM }}, + SubnetIDs=${{ inputs.SUBNET_IDS_SSM }}, + VpcID=${{ inputs.VPC_ID_SSM }}, + DockerImageUrl=${{ steps.build-image.outputs.image }}, + EnvironmentFileUrl=arn:aws:s3:::${{ env.ECS_CONTAINER_ENV_FILE_S3_PATH }}, + SnsToLarkLambda=${{ inputs.SNS_TO_LARK_LAMBDA_ARN }}, + AppPort=${{ inputs.APP_PORT }}, + Mode=${{ inputs.MODE }}, + DockerImageName=${{ inputs.DOCKER_IMAGE_NAME }}, + HealthCheck=${{ inputs.HEALTH_CHECK }} diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml deleted file mode 100644 index cadc1b5507..0000000000 --- a/.github/workflows/docker-build.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Build docker image - -on: - pull_request: - branches: - - "main" - paths: - - api/Dockerfile - - web/Dockerfile - -concurrency: - group: docker-build-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - build-docker: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - service_name: "api-amd64" - platform: linux/amd64 - context: "api" - - service_name: "api-arm64" - platform: linux/arm64 - context: "api" - - service_name: "web-amd64" - platform: linux/amd64 - context: "web" - - service_name: "web-arm64" - platform: linux/arm64 - context: "web" - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build Docker Image - uses: docker/build-push-action@v6 - with: - push: false - context: "{{defaultContext}}:${{ matrix.context }}" - file: "${{ matrix.file }}" - platforms: ${{ matrix.platform }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/expose_service_ports.sh b/.github/workflows/expose_service_ports.sh deleted file mode 100755 index 10d95cb736..0000000000 --- a/.github/workflows/expose_service_ports.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -yq eval '.services.weaviate.ports += ["8080:8080"]' -i docker/docker-compose.yaml -yq eval '.services.qdrant.ports += ["6333:6333"]' -i docker/docker-compose.yaml -yq eval '.services.chroma.ports += ["8000:8000"]' -i docker/docker-compose.yaml -yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/docker-compose.yaml -yq eval '.services.pgvector.ports += ["5433:5432"]' -i docker/docker-compose.yaml -yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compose.yaml -yq eval '.services["elasticsearch"].ports += ["9200:9200"]' -i docker/docker-compose.yaml -yq eval '.services.couchbase-server.ports += ["8091-8096:8091-8096"]' -i docker/docker-compose.yaml -yq eval '.services.couchbase-server.ports += ["11210:11210"]' -i docker/docker-compose.yaml -yq eval '.services.tidb.ports += ["4000:4000"]' -i docker/tidb/docker-compose.yaml -yq eval '.services.opengauss.ports += ["6600:6600"]' -i docker/docker-compose.yaml - -echo "Ports exposed for sandbox, weaviate, tidb, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase, opengauss" diff --git a/.github/workflows/prod-api.yml b/.github/workflows/prod-api.yml new file mode 100644 index 0000000000..2685b7ea0e --- /dev/null +++ b/.github/workflows/prod-api.yml @@ -0,0 +1,36 @@ + +name: Api Prod Deployment + +on: + workflow_dispatch: + +jobs: + call-deployment-in-local-repo: + if: contains('carloschen6688,yvan-65535', github.actor) + uses: ./.github/workflows/prod-deploy.yml + with: + APPLICATION_NAME: dify-api + DEPLOYMENT_ENV: prod + ECR_REPOSITORY: ai-dify-api + ECS_CLUSTER: ai-dify + SECURITY_GROUP_IDS_SSM: /prod/security-groups + SUBNET_IDS_SSM: /prod/subnet-ids + VPC_ID_SSM: /prod/vpc-id + SNS_TO_LARK_LAMBDA_ARN: /prod/lambda/sns-to-lark + CERERTIFICATE_US_EAST_2_ARN: /prod/certificate + ECS_EXECUTION_ROLE_ARN: /prod/ecs-execution-role + APP_PORT: 5001 + MODE: api + DOCKER_IMAGE_NAME: dify-api + HEALTH_CHECK: /console/api/ping + TEMPLATE: cloudformation-ecs-service-template.yml + DOCKERFILEDIR: api + + + secrets: + DEPLOYMENT_AWS_ACCESS_KEY_ID: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID_PROD }} + DEPLOYMENT_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY_PROD }} + ED25519_CI_KEY: ${{ secrets.ED25519_CI_KEY }} + PAT: ${{ secrets.PAT }} + + diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml new file mode 100644 index 0000000000..38188a0f32 --- /dev/null +++ b/.github/workflows/prod-deploy.yml @@ -0,0 +1,134 @@ +name: Reusable deployment job + +on: + workflow_call: + inputs: + APPLICATION_NAME: + default: 'oneapi' + type: string + AWS_REGION: + default: 'us-east-2' + type: string + DEPLOYMENT_ENV: + required: true + type: string + ECR_REPOSITORY: + required: true + type: string + ECS_CLUSTER: + required: true + type: string + SECURITY_GROUP_IDS_SSM: + required: true + type: string + SUBNET_IDS_SSM: + required: true + type: string + VPC_ID_SSM: + required: true + type: string + SNS_TO_LARK_LAMBDA_ARN: + required: true + type: string + CERERTIFICATE_US_EAST_2_ARN: + required: true + type: string + ECS_EXECUTION_ROLE_ARN: + required: true + type: string + DOCKER_IMAGE_NAME: + required: true + type: string + MODE: + required: true + type: string + APP_PORT: + required: true + type: number + HEALTH_CHECK: + required: true + type: string + TEMPLATE: + required: true + type: string + DOCKERFILEDIR: + required: true + type: string + + + secrets: + DEPLOYMENT_AWS_ACCESS_KEY_ID: + required: true + DEPLOYMENT_AWS_SECRET_ACCESS_KEY: + required: true + ED25519_CI_KEY: + required: true + PAT: + required: true +env: + ECS_CONTAINER_ENV_FILE_S3_PATH: ${{ inputs.DEPLOYMENT_ENV }}-dify-ai/.env +jobs: + deploy: + name: Deploy to ${{ inputs.DEPLOYMENT_ENV }} + runs-on: ubuntu-latest + concurrency: deployment-${{ inputs.APPLICATION_NAME }}-${{ inputs.DEPLOYMENT_ENV }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ inputs.AWS_REGION }} + + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Set SHORT_SHA + id: set_short_sha + run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >>$GITHUB_ENV + - name: Set IMAGE_TAG + id: set_image_tag + run: echo "IMAGE_TAG=${{ inputs.APPLICATION_NAME }}-$SHORT_SHA" >> $GITHUB_ENV + - name: Set GIT_BRANCH + id: set_git_branch + run: echo "GIT_BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV + + - name: Build, tag, and push image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + + run: | + cd ${{ inputs.DOCKERFILEDIR }} + docker build --build-arg GIT_BRANCH=$GIT_BRANCH --build-arg GIT_PAT_TOKEN=${{ secrets.PAT }} -t $ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG . + docker push $ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG + echo "::set-output name=image::$ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG" + + - name: Deploy CloudFormation + uses: aws-actions/aws-cloudformation-github-deploy@v1 + with: + name: ${{ inputs.APPLICATION_NAME }}-${{ inputs.DEPLOYMENT_ENV }} + template: aws/${{ inputs.TEMPLATE }} + no-fail-on-empty-changeset: "1" + parameter-overrides: >- + CertificateArn=${{ inputs.CERERTIFICATE_US_EAST_2_ARN }}, + ExecutionRoleArn=${{ inputs.ECS_EXECUTION_ROLE_ARN }}, + ECSClusterName=${{ inputs.ECS_CLUSTER }}, + ApplicationName=${{ inputs.APPLICATION_NAME }}, + EnvName=${{ inputs.DEPLOYMENT_ENV }}, + SecurityGroupIDs=${{ inputs.SECURITY_GROUP_IDS_SSM }}, + SubnetIDs=${{ inputs.SUBNET_IDS_SSM }}, + VpcID=${{ inputs.VPC_ID_SSM }}, + DockerImageUrl=${{ steps.build-image.outputs.image }}, + EnvironmentFileUrl=arn:aws:s3:::${{ env.ECS_CONTAINER_ENV_FILE_S3_PATH }}, + SnsToLarkLambda=${{ inputs.SNS_TO_LARK_LAMBDA_ARN }}, + AppPort=${{ inputs.APP_PORT }}, + Mode=${{ inputs.MODE }}, + DockerImageName=${{ inputs.DOCKER_IMAGE_NAME }}, + HealthCheck=${{ inputs.HEALTH_CHECK }} diff --git a/.github/workflows/prod-web.yml b/.github/workflows/prod-web.yml new file mode 100644 index 0000000000..7a759b38f0 --- /dev/null +++ b/.github/workflows/prod-web.yml @@ -0,0 +1,34 @@ +name: Web Prod Deployment + +on: + workflow_dispatch: + +jobs: + call-deployment-in-local-repo: + if: contains('carloschen6688,yvan-65535', github.actor) + uses: ./.github/workflows/prod-deploy.yml + with: + APPLICATION_NAME: dify-web + DEPLOYMENT_ENV: prod + ECR_REPOSITORY: ai-dify-web + ECS_CLUSTER: ai-dify + SECURITY_GROUP_IDS_SSM: /prod/security-groups + SUBNET_IDS_SSM: /prod/subnet-ids + VPC_ID_SSM: /prod/vpc-id + SNS_TO_LARK_LAMBDA_ARN: /prod/lambda/sns-to-lark + CERERTIFICATE_US_EAST_2_ARN: /prod/certificate + ECS_EXECUTION_ROLE_ARN: /prod/ecs-execution-role + APP_PORT: 3000 + MODE: web + DOCKER_IMAGE_NAME: dify-web + HEALTH_CHECK: /apps + TEMPLATE: cloudformation-ecs-service-template.yml + DOCKERFILEDIR: web + + + secrets: + DEPLOYMENT_AWS_ACCESS_KEY_ID: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID_PROD }} + DEPLOYMENT_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY_PROD }} + ED25519_CI_KEY: ${{ secrets.ED25519_CI_KEY }} + PAT: ${{ secrets.PAT }} + diff --git a/.github/workflows/prod-worker.yml b/.github/workflows/prod-worker.yml new file mode 100644 index 0000000000..6767dd84a7 --- /dev/null +++ b/.github/workflows/prod-worker.yml @@ -0,0 +1,34 @@ +name: Worker Prod Deployment + +on: + workflow_dispatch: + +jobs: + call-deployment-in-local-repo: + if: contains('carloschen6688,yvan-65535', github.actor) + uses: ./.github/workflows/prod-deploy.yml + with: + APPLICATION_NAME: dify-worker + DEPLOYMENT_ENV: prod + ECR_REPOSITORY: ai-dify-worker + ECS_CLUSTER: ai-dify + SECURITY_GROUP_IDS_SSM: /prod/security-groups + SUBNET_IDS_SSM: /prod/subnet-ids + VPC_ID_SSM: /prod/vpc-id + SNS_TO_LARK_LAMBDA_ARN: /prod/lambda/sns-to-lark + CERERTIFICATE_US_EAST_2_ARN: /prod/certificate + ECS_EXECUTION_ROLE_ARN: /prod/ecs-execution-role + APP_PORT: 5001 + MODE: worker + DOCKER_IMAGE_NAME: dify-api + HEALTH_CHECK: /health + TEMPLATE: worker-cloudformation-ecs-service-template.yml + DOCKERFILEDIR: api + + + secrets: + DEPLOYMENT_AWS_ACCESS_KEY_ID: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID_PROD }} + DEPLOYMENT_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY_PROD }} + ED25519_CI_KEY: ${{ secrets.ED25519_CI_KEY }} + PAT: ${{ secrets.PAT }} + diff --git a/.github/workflows/staging-api.yml b/.github/workflows/staging-api.yml new file mode 100644 index 0000000000..85310e59b3 --- /dev/null +++ b/.github/workflows/staging-api.yml @@ -0,0 +1,36 @@ +name: Api Staging Deployment + +on: + workflow_dispatch: + repository_dispatch: + types: [deploy-staging] + +jobs: + call-deployment-in-local-repo: + uses: ./.github/workflows/deploy.yml + with: + APPLICATION_NAME: dify-api + DEPLOYMENT_ENV: staging + ECR_REPOSITORY: ai-dify-api + ECS_CLUSTER: ai-dify + SECURITY_GROUP_IDS_SSM: /staging/security-groups + SUBNET_IDS_SSM: /staging/subnet-ids + VPC_ID_SSM: /staging/vpc-id + SNS_TO_LARK_LAMBDA_ARN: /staging/lambda/sns-to-lark + CERERTIFICATE_US_EAST_2_ARN: /staging/certificate + ECS_EXECUTION_ROLE_ARN: /staging/ecs-execution-role + APP_PORT: 5001 + MODE: api + DOCKER_IMAGE_NAME: dify-api + HEALTH_CHECK: /console/api/ping + TEMPLATE: cloudformation-ecs-service-template.yml + DOCKERFILEDIR: api + + + + secrets: + DEPLOYMENT_AWS_ACCESS_KEY_ID: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID_STAGING }} + DEPLOYMENT_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY_STAGING }} + ED25519_CI_KEY: ${{ secrets.ED25519_CI_KEY }} + PAT: ${{ secrets.PAT }} + diff --git a/.github/workflows/staging-web.yml b/.github/workflows/staging-web.yml new file mode 100644 index 0000000000..c7152b3421 --- /dev/null +++ b/.github/workflows/staging-web.yml @@ -0,0 +1,34 @@ +name: Web Staging Deployment + +on: + workflow_dispatch: + +jobs: + call-deployment-in-local-repo: + uses: ./.github/workflows/deploy.yml + with: + APPLICATION_NAME: dify-web + DEPLOYMENT_ENV: staging + ECR_REPOSITORY: ai-dify-web + ECS_CLUSTER: ai-dify + SECURITY_GROUP_IDS_SSM: /staging/security-groups + SUBNET_IDS_SSM: /staging/subnet-ids + VPC_ID_SSM: /staging/vpc-id + SNS_TO_LARK_LAMBDA_ARN: /staging/lambda/sns-to-lark + CERERTIFICATE_US_EAST_2_ARN: /staging/certificate + ECS_EXECUTION_ROLE_ARN: /staging/ecs-execution-role + APP_PORT: 3000 + MODE: web + DOCKER_IMAGE_NAME: dify-web + HEALTH_CHECK: /apps + TEMPLATE: cloudformation-ecs-service-template.yml + DOCKERFILEDIR: web + + + + secrets: + DEPLOYMENT_AWS_ACCESS_KEY_ID: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID_STAGING }} + DEPLOYMENT_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY_STAGING }} + ED25519_CI_KEY: ${{ secrets.ED25519_CI_KEY }} + PAT: ${{ secrets.PAT }} + diff --git a/.github/workflows/staging-worker.yml b/.github/workflows/staging-worker.yml new file mode 100644 index 0000000000..565077c254 --- /dev/null +++ b/.github/workflows/staging-worker.yml @@ -0,0 +1,33 @@ +name: Worker Staging Deployment + +on: + workflow_dispatch: + +jobs: + call-deployment-in-local-repo: + uses: ./.github/workflows/deploy.yml + with: + APPLICATION_NAME: dify-worker + DEPLOYMENT_ENV: staging + ECR_REPOSITORY: ai-dify-worker + ECS_CLUSTER: ai-dify + SECURITY_GROUP_IDS_SSM: /staging/security-groups + SUBNET_IDS_SSM: /staging/subnet-ids + VPC_ID_SSM: /staging/vpc-id + SNS_TO_LARK_LAMBDA_ARN: /staging/lambda/sns-to-lark + CERERTIFICATE_US_EAST_2_ARN: /staging/certificate + ECS_EXECUTION_ROLE_ARN: /staging/ecs-execution-role + APP_PORT: 5001 + MODE: worker + DOCKER_IMAGE_NAME: dify-api + HEALTH_CHECK: /health + TEMPLATE: worker-cloudformation-ecs-service-template.yml + DOCKERFILEDIR: api + + + secrets: + DEPLOYMENT_AWS_ACCESS_KEY_ID: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID_STAGING }} + DEPLOYMENT_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY_STAGING }} + ED25519_CI_KEY: ${{ secrets.ED25519_CI_KEY }} + PAT: ${{ secrets.PAT }} + diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index 1870b1f670..0000000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,30 +0,0 @@ -# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. -# -# You can adjust the behavior by modifying this file. -# For more information, see: -# https://github.com/actions/stale -name: Mark stale issues and pull requests - -on: - schedule: - - cron: '0 3 * * *' - -jobs: - stale: - - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - - steps: - - uses: actions/stale@v5 - with: - days-before-issue-stale: 15 - days-before-issue-close: 3 - repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-issue-message: "Close due to it's no longer active, if you have any questions, you can reopen it." - stale-pr-message: "Close due to it's no longer active, if you have any questions, you can reopen it." - stale-issue-label: 'no-issue-activity' - stale-pr-label: 'no-pr-activity' - any-of-labels: 'duplicate,question,invalid,wontfix,no-issue-activity,no-pr-activity,enhancement,cant-reproduce,help-wanted' diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml deleted file mode 100644 index b06ab9653e..0000000000 --- a/.github/workflows/style.yml +++ /dev/null @@ -1,183 +0,0 @@ -name: Style check - -on: - pull_request: - branches: - - main - -concurrency: - group: style-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - checks: write - statuses: write - contents: read - - -jobs: - python-style: - name: Python Style - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Check changed files - id: changed-files - uses: tj-actions/changed-files@v45 - with: - files: | - api/** - .github/workflows/style.yml - - - name: Setup UV and Python - if: steps.changed-files.outputs.any_changed == 'true' - uses: ./.github/actions/setup-uv - with: - uv-lockfile: api/uv.lock - enable-cache: false - - - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' - run: uv sync --project api --dev - - - name: Ruff check - if: steps.changed-files.outputs.any_changed == 'true' - run: | - uv run --directory api ruff --version - uv run --directory api ruff check --diff ./ - uv run --directory api ruff format --check --diff ./ - - - name: Dotenv check - if: steps.changed-files.outputs.any_changed == 'true' - run: uv run --project api dotenv-linter ./api/.env.example ./web/.env.example - - - name: Lint hints - if: failure() - run: echo "Please run 'dev/reformat' to fix the fixable linting errors." - - web-style: - name: Web Style - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./web - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Check changed files - id: changed-files - uses: tj-actions/changed-files@v45 - with: - files: web/** - - - name: Install pnpm - uses: pnpm/action-setup@v4 - with: - version: 10 - run_install: false - - - name: Setup NodeJS - uses: actions/setup-node@v4 - if: steps.changed-files.outputs.any_changed == 'true' - with: - node-version: 22 - cache: pnpm - cache-dependency-path: ./web/package.json - - - name: Web dependencies - if: steps.changed-files.outputs.any_changed == 'true' - run: pnpm install --frozen-lockfile - - - name: Web style check - if: steps.changed-files.outputs.any_changed == 'true' - run: pnpm run lint - - docker-compose-template: - name: Docker Compose Template - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Check changed files - id: changed-files - uses: tj-actions/changed-files@v45 - with: - files: | - docker/generate_docker_compose - docker/.env.example - docker/docker-compose-template.yaml - docker/docker-compose.yaml - - - name: Generate Docker Compose - if: steps.changed-files.outputs.any_changed == 'true' - run: | - cd docker - ./generate_docker_compose - - - name: Check for changes - if: steps.changed-files.outputs.any_changed == 'true' - run: git diff --exit-code - - superlinter: - name: SuperLinter - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Check changed files - id: changed-files - uses: tj-actions/changed-files@v45 - with: - files: | - **.sh - **.yaml - **.yml - **Dockerfile - dev/** - - - name: Super-linter - uses: super-linter/super-linter/slim@v7 - if: steps.changed-files.outputs.any_changed == 'true' - env: - BASH_SEVERITY: warning - DEFAULT_BRANCH: main - FILTER_REGEX_INCLUDE: pnpm-lock.yaml - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - IGNORE_GENERATED_FILES: true - IGNORE_GITIGNORED_FILES: true - VALIDATE_BASH: true - VALIDATE_BASH_EXEC: true - # FIXME: temporarily disabled until api-docker.yaml's run script is fixed for shellcheck - # VALIDATE_GITHUB_ACTIONS: true - VALIDATE_DOCKERFILE_HADOLINT: true - VALIDATE_XML: true - VALIDATE_YAML: true - - - name: EditorConfig checks - uses: super-linter/super-linter/slim@v7 - env: - DEFAULT_BRANCH: main - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - IGNORE_GENERATED_FILES: true - IGNORE_GITIGNORED_FILES: true - # EditorConfig validation - VALIDATE_EDITORCONFIG: true - EDITORCONFIG_FILE_NAME: editorconfig-checker.json diff --git a/.github/workflows/tool-test-sdks.yaml b/.github/workflows/tool-test-sdks.yaml deleted file mode 100644 index b1ccd7417a..0000000000 --- a/.github/workflows/tool-test-sdks.yaml +++ /dev/null @@ -1,43 +0,0 @@ -name: Run Unit Test For SDKs - -on: - pull_request: - branches: - - main - paths: - - sdks/** - -concurrency: - group: sdk-tests-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - build: - name: unit test for Node.js SDK - runs-on: ubuntu-latest - - strategy: - matrix: - node-version: [16, 18, 20, 22] - - defaults: - run: - working-directory: sdks/nodejs-client - - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - cache: '' - cache-dependency-path: 'pnpm-lock.yaml' - - - name: Install Dependencies - run: pnpm install --frozen-lockfile - - - name: Test - run: pnpm test diff --git a/.github/workflows/translate-i18n-base-on-english.yml b/.github/workflows/translate-i18n-base-on-english.yml deleted file mode 100644 index c79d58563f..0000000000 --- a/.github/workflows/translate-i18n-base-on-english.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Check i18n Files and Create PR - -on: - pull_request: - types: [closed] - branches: [main] - -jobs: - check-and-update: - if: github.event.pull_request.merged == true - runs-on: ubuntu-latest - defaults: - run: - working-directory: web - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 2 # last 2 commits - persist-credentials: false - - - name: Check for file changes in i18n/en-US - id: check_files - run: | - recent_commit_sha=$(git rev-parse HEAD) - second_recent_commit_sha=$(git rev-parse HEAD~1) - changed_files=$(git diff --name-only $recent_commit_sha $second_recent_commit_sha -- 'i18n/en-US/*.ts') - echo "Changed files: $changed_files" - if [ -n "$changed_files" ]; then - echo "FILES_CHANGED=true" >> $GITHUB_ENV - else - echo "FILES_CHANGED=false" >> $GITHUB_ENV - fi - - - name: Install pnpm - uses: pnpm/action-setup@v4 - with: - version: 10 - run_install: false - - - name: Set up Node.js - if: env.FILES_CHANGED == 'true' - uses: actions/setup-node@v4 - with: - node-version: 'lts/*' - cache: pnpm - cache-dependency-path: ./web/package.json - - - name: Install dependencies - if: env.FILES_CHANGED == 'true' - run: pnpm install --frozen-lockfile - - - name: Run npm script - if: env.FILES_CHANGED == 'true' - run: pnpm run auto-gen-i18n - - - name: Create Pull Request - if: env.FILES_CHANGED == 'true' - uses: peter-evans/create-pull-request@v6 - with: - commit-message: Update i18n files based on en-US changes - title: 'chore: translate i18n files' - body: This PR was automatically created to update i18n files based on changes in en-US locale. - branch: chore/automated-i18n-updates diff --git a/.github/workflows/vdb-tests.yml b/.github/workflows/vdb-tests.yml deleted file mode 100644 index c784817e72..0000000000 --- a/.github/workflows/vdb-tests.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Run VDB Tests - -on: - pull_request: - branches: - - main - paths: - - api/core/rag/datasource/** - - docker/** - - .github/workflows/vdb-tests.yml - - api/uv.lock - - api/pyproject.toml - -concurrency: - group: vdb-tests-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - test: - name: VDB Tests - runs-on: ubuntu-latest - strategy: - matrix: - python-version: - - "3.11" - - "3.12" - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup UV and Python - uses: ./.github/actions/setup-uv - with: - python-version: ${{ matrix.python-version }} - uv-lockfile: api/uv.lock - - - name: Check UV lockfile - run: uv lock --project api --check - - - name: Install dependencies - run: uv sync --project api --dev - - - name: Set up dotenvs - run: | - cp docker/.env.example docker/.env - cp docker/middleware.env.example docker/middleware.env - - - name: Expose Service Ports - run: sh .github/workflows/expose_service_ports.sh - - - name: Set up Vector Store (TiDB) - uses: hoverkraft-tech/compose-action@v2.0.2 - with: - compose-file: docker/tidb/docker-compose.yaml - services: | - tidb - tiflash - - - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase) - uses: hoverkraft-tech/compose-action@v2.0.2 - with: - compose-file: | - docker/docker-compose.yaml - services: | - weaviate - qdrant - couchbase-server - etcd - minio - milvus-standalone - pgvecto-rs - pgvector - chroma - elasticsearch - - - name: Check TiDB Ready - run: uv run --project api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py - - - name: Test Vector Stores - run: uv run --project api bash dev/pytest/pytest_vdb.sh diff --git a/.github/workflows/web-tests.yml b/.github/workflows/web-tests.yml deleted file mode 100644 index 37cfdc5c1e..0000000000 --- a/.github/workflows/web-tests.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Web Tests - -on: - pull_request: - branches: - - main - paths: - - web/** - -concurrency: - group: web-tests-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - test: - name: Web Tests - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./web - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Check changed files - id: changed-files - uses: tj-actions/changed-files@v45 - with: - files: web/** - - - name: Install pnpm - if: steps.changed-files.outputs.any_changed == 'true' - uses: pnpm/action-setup@v4 - with: - version: 10 - run_install: false - - - name: Setup Node.js - uses: actions/setup-node@v4 - if: steps.changed-files.outputs.any_changed == 'true' - with: - node-version: 22 - cache: pnpm - cache-dependency-path: ./web/package.json - - - name: Install dependencies - if: steps.changed-files.outputs.any_changed == 'true' - run: pnpm install --frozen-lockfile - - - name: Run tests - if: steps.changed-files.outputs.any_changed == 'true' - run: pnpm test diff --git a/aws/cloudformation-ecs-service-template.yml b/aws/cloudformation-ecs-service-template.yml new file mode 100644 index 0000000000..c65274a78a --- /dev/null +++ b/aws/cloudformation-ecs-service-template.yml @@ -0,0 +1,347 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: The template used to create an ECS Service. +Parameters: + CertificateArn: + Type: AWS::SSM::Parameter::Value + ExecutionRoleArn: + Type: AWS::SSM::Parameter::Value + ECSClusterName: + Type: String + ApplicationName: + Type: String + EnvName: + Type: String + SecurityGroupIDs: + Type: AWS::SSM::Parameter::Value> + SubnetIDs: + Type: AWS::SSM::Parameter::Value> + VpcID: + Type: AWS::SSM::Parameter::Value + DockerImageUrl: + Type: String + EnvironmentFileUrl: + Type: String + Description: The s3 url that contains the list of ecs container environment variables + TaskDesiredCount: + Type: Number + Default: 1 + TaskCpuSetting: + Type: Number + Default: 2048 + TaskMemorySetting: + Type: Number + Default: 4096 + AppPort: + Type: Number + Default: 3000 + SnsToLarkLambda: + Type: AWS::SSM::Parameter::Value + RedisPass: + Type: AWS::SSM::Parameter::Value + Default: /ai-dify/REDIS_PASSWORD + PgPass: + Type: AWS::SSM::Parameter::Value + Default: /ai-dify/PG_PASS + OpenserchPass: + Type: AWS::SSM::Parameter::Value + Default: /ai-dify/ES_PASS + S3SecretsKey: + Type: AWS::SSM::Parameter::Value + Default: /ai-dify/S3_KEY + Mode: + Type: String + HealthCheck: + Type: String + DockerImageName: + Type: String + EnvironmentFileUrl: + Type: String + CeleryBrokerUrl: + Type: AWS::SSM::Parameter::Value + Default: /ai-dify/REDIS_URL + CeleryWorkerAmount: + Type: Number + Default: 5 + SqlalchemyPoolSize: + Type: Number + Default: 50 + ServerWorkerConnections: + Type: Number + Default: 1000 + SqlalchemyPoolRecycle: + Type: Number + Default: 1200 + ServerWorkerAmount: + Type: Number + Default: 1 + Lang: + Type: String + Default: US.UTF-8 + LcAll: + Type: String + Default: US.UTF-8 +Resources: + ECSService: + Type: 'AWS::ECS::Service' + Properties: + Cluster: !Ref ECSClusterName + CapacityProviderStrategy: + - CapacityProvider: FARGATE + Base: 0 + Weight: 1 + TaskDefinition: !Ref TaskDefinition + ServiceName: !Sub ${ApplicationName}-${EnvName} + DesiredCount: !Ref TaskDesiredCount + EnableExecuteCommand: true + LoadBalancers: + - ContainerName: !Sub ${ApplicationName}-${EnvName} + ContainerPort: !Ref AppPort + TargetGroupArn: !Ref TargetGroup + HealthCheckGracePeriodSeconds: '60' + NetworkConfiguration: + AwsvpcConfiguration: + AssignPublicIp: ENABLED + SecurityGroups: !Ref SecurityGroupIDs + Subnets: !Ref SubnetIDs + DeploymentConfiguration: + MaximumPercent: 200 + MinimumHealthyPercent: 100 + DeploymentController: + Type: ECS + DependsOn: Listener + LoadBalancer: + Type: 'AWS::ElasticLoadBalancingV2::LoadBalancer' + Properties: + Name: !Sub ${ApplicationName}-${EnvName}-alb + SecurityGroups: !Ref SecurityGroupIDs + Subnets: !Ref SubnetIDs + Type: application + Scheme: internal + TargetGroup: + Type: 'AWS::ElasticLoadBalancingV2::TargetGroup' + Properties: + VpcId: !Ref VpcID + Protocol: HTTP + Port: '443' + HealthCheckPath: !Ref HealthCheck + TargetType: ip + Listener: + Type: 'AWS::ElasticLoadBalancingV2::Listener' + Properties: + DefaultActions: + - TargetGroupArn: !Ref TargetGroup + Type: forward + LoadBalancerArn: !Ref LoadBalancer + Port: '80' + Protocol: HTTP + TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub ${ApplicationName}-${EnvName} + Cpu: !Ref TaskCpuSetting + Memory: !Ref TaskMemorySetting + NetworkMode: awsvpc + RequiresCompatibilities: + - FARGATE + ExecutionRoleArn: !Ref ExecutionRoleArn + TaskRoleArn: !Ref ExecutionRoleArn + ContainerDefinitions: + - Name: !Sub ${ApplicationName}-${EnvName} + Image: !Ref DockerImageUrl + Environment: + - Name: REDIS_PASSWORD + Value: !Ref RedisPass + - Name: DB_PASSWORD + Value: !Ref PgPass + - Name: OPENSEARCH_PASSWORD + Value: !Ref OpenserchPass + - Name: S3_SECRET_KEY + Value: !Ref S3SecretsKey + - Name: MODE + Value: !Ref Mode + - Name: CELERY_BROKER_URL + Value: !Ref CeleryBrokerUrl + - Name: CELERY_WORKER_AMOUNT + Value: !Ref CeleryWorkerAmount + - Name: SQLALCHEMY_POOL_SIZE + Value: !Ref SqlalchemyPoolSize + - Name: SERVER_WORKER_AMOUNT + Value: !Ref ServerWorkerAmount + - Name: SQLALCHEMY_POOL_RECYCLE + Value: !Ref SqlalchemyPoolRecycle + - Name: SERVER_WORKER_CONNECTIONS + Value: !Ref ServerWorkerConnections + - Name: LANG + Value: !Ref Lang + - Name: LC_ALL + Value: !Ref LcAll + + EnvironmentFiles: + - Type: s3 + Value: !Ref EnvironmentFileUrl + PortMappings: + - ContainerPort: !Ref AppPort + HostPort: !Ref AppPort + LogConfiguration: + LogDriver: "awslogs" + Options: + awslogs-create-group: true + awslogs-group: !Sub ${ApplicationName}-${EnvName} + awslogs-region: !Ref "AWS::Region" + awslogs-stream-prefix: ecs + + TooManyHttpCodeElb5XXCountAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub "TooMany_HTTPCode_ELB_5XX_Count_Alarm-${EnvName}-${ApplicationName}" + AlarmDescription: !Sub "TooMany_HTTPCode_ELB_5XX_Count_Alarm of ${ApplicationName} in environment ${EnvName}" + Namespace: AWS/ApplicationELB + MetricName: HTTPCode_ELB_5XX_Count + Dimensions: + - Name: LoadBalancer + Value: !GetAtt LoadBalancer.LoadBalancerFullName + Statistic: Sum + Period: 600 + EvaluationPeriods: 1 + Threshold: 1 + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching + AlarmActions: + - !Ref LarkTopic + OKActions: + - !Ref LarkTopic + + TargetResponseTimeTooSlowAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub "TargetResponseTime-too-slow-${EnvName}-${ApplicationName}" + AlarmDescription: !Sub "TargetResponseTime is too slow for ${ApplicationName} in environment ${EnvName}" + Namespace: AWS/ApplicationELB + MetricName: TargetResponseTime + Dimensions: + - Name: LoadBalancer + Value: !GetAtt LoadBalancer.LoadBalancerFullName + Statistic: Maximum + Period: 600 + EvaluationPeriods: 2 + Threshold: 20 + ComparisonOperator: GreaterThanOrEqualToThreshold + TreatMissingData: notBreaching + AlarmActions: + - !Ref LarkTopic + OKActions: + - !Ref LarkTopic + + HealthyHostCountTooLowAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub "HealthyHostCountTooLow-alarm-${EnvName}-${ApplicationName}" + AlarmDescription: !Sub "The HealthyHostCount is too low for ${ApplicationName} in environment ${EnvName}" + Namespace: AWS/ApplicationELB + MetricName: HealthyHostCount + Dimensions: + - Name: LoadBalancer + Value: !GetAtt LoadBalancer.LoadBalancerFullName + - Name: TargetGroup + Value: !GetAtt TargetGroup.TargetGroupFullName + Statistic: Minimum + Period: 180 + EvaluationPeriods: 1 + Threshold: !Ref TaskDesiredCount + ComparisonOperator: LessThanThreshold + TreatMissingData: breaching + AlarmActions: + - !Ref LarkTopic + OKActions: + - !Ref LarkTopic + + LarkTopic: + Type: AWS::SNS::Topic + Properties: + TopicName: !Sub ${ApplicationName}-${EnvName} + Subscription: + - Endpoint: !Ref SnsToLarkLambda + Protocol: "LAMBDA" + + +# ASG setting begin + AutoScalingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - ecs-tasks.amazonaws.com + Action: + - 'sts:AssumeRole' + Path: '/' + Policies: + - PolicyName: allow-ecs-cw-scaling + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - ecs:DescribeServices + - ecs:UpdateService + - cloudwatch:DeleteAlarms + - cloudwatch:DescribeAlarms + - cloudwatch:PutMetricAlarm + Resource: '*' + AutoScalingTarget: + Type: AWS::ApplicationAutoScaling::ScalableTarget + Properties: + MinCapacity: !Ref TaskDesiredCount + MaxCapacity: 50 + ResourceId: !Join ["/", [service, !Ref ECSClusterName, !GetAtt ECSService.Name]] + ScalableDimension: ecs:service:DesiredCount + ServiceNamespace: ecs + RoleARN: !GetAtt AutoScalingRole.Arn + AutoScalingPolicyForCPU: + Type: AWS::ApplicationAutoScaling::ScalingPolicy + Properties: + PolicyName: ecs-cpu-utilization-ScalingPolicy + PolicyType: TargetTrackingScaling + ScalingTargetId: !Ref AutoScalingTarget + TargetTrackingScalingPolicyConfiguration: + PredefinedMetricSpecification: + PredefinedMetricType: ECSServiceAverageCPUUtilization + ScaleInCooldown: 300 + ScaleOutCooldown: 120 + TargetValue: 70.0 + AutoScalingPolicyForMemory: + Type: AWS::ApplicationAutoScaling::ScalingPolicy + Properties: + PolicyName: ecs-memory-utilization-ScalingPolicy + PolicyType: TargetTrackingScaling + ScalingTargetId: !Ref AutoScalingTarget + TargetTrackingScalingPolicyConfiguration: + PredefinedMetricSpecification: + PredefinedMetricType: ECSServiceAverageMemoryUtilization + ScaleInCooldown: 300 + ScaleOutCooldown: 120 + TargetValue: 75.0 +# ASG setting end + +Outputs: + ClusterName: + Description: The cluster used to create the service. + Value: !Ref ECSClusterName + ECSService: + Description: The created service. + Value: !Ref ECSService + LoadBalancer: + Description: The created load balancer. + Value: !Ref LoadBalancer + Listener: + Description: The created listener. + Value: !Ref Listener + TargetGroup: + Description: The created target group. + Value: !Ref TargetGroup + LarkTopic: + Description: The created Lark Topic. + Value: !Ref LarkTopic diff --git a/aws/worker-cloudformation-ecs-service-template.yml b/aws/worker-cloudformation-ecs-service-template.yml new file mode 100644 index 0000000000..fa4052163b --- /dev/null +++ b/aws/worker-cloudformation-ecs-service-template.yml @@ -0,0 +1,204 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: The template used to create an ECS Service without Load Balancer. +Parameters: + CertificateArn: + Type: AWS::SSM::Parameter::Value + ExecutionRoleArn: + Type: AWS::SSM::Parameter::Value + ECSClusterName: + Type: String + ApplicationName: + Type: String + EnvName: + Type: String + SecurityGroupIDs: + Type: AWS::SSM::Parameter::Value> + SubnetIDs: + Type: AWS::SSM::Parameter::Value> + VpcID: + Type: AWS::SSM::Parameter::Value + DockerImageUrl: + Type: String + EnvironmentFileUrl: + Type: String + Description: The s3 url that contains the list of ecs container environment variables + TaskDesiredCount: + Type: Number + Default: 1 + TaskCpuSetting: + Type: Number + Default: 1024 + TaskMemorySetting: + Type: Number + Default: 4096 + AppPort: + Type: Number + Default: 3000 + SnsToLarkLambda: + Type: AWS::SSM::Parameter::Value + RedisPass: + Type: AWS::SSM::Parameter::Value + Default: /ai-dify/REDIS_PASSWORD + PgPass: + Type: AWS::SSM::Parameter::Value + Default: /ai-dify/PG_PASS + OpenserchPass: + Type: AWS::SSM::Parameter::Value + Default: /ai-dify/ES_PASS + S3SecretsKey: + Type: AWS::SSM::Parameter::Value + Default: /ai-dify/S3_KEY + Mode: + Type: String + HealthCheck: + Type: String + DockerImageName: + Type: String + CeleryBrokerUrl: + Type: AWS::SSM::Parameter::Value + Default: /ai-dify/REDIS_URL +Resources: + ECSService: + Type: 'AWS::ECS::Service' + Properties: + Cluster: !Ref ECSClusterName + CapacityProviderStrategy: + - CapacityProvider: FARGATE + Base: 0 + Weight: 1 + TaskDefinition: !Ref TaskDefinition + ServiceName: !Sub ${ApplicationName}-${EnvName} + DesiredCount: !Ref TaskDesiredCount + EnableExecuteCommand: true + NetworkConfiguration: + AwsvpcConfiguration: + AssignPublicIp: ENABLED + SecurityGroups: !Ref SecurityGroupIDs + Subnets: !Ref SubnetIDs + DeploymentConfiguration: + MaximumPercent: 200 + MinimumHealthyPercent: 100 + DeploymentController: + Type: ECS + + TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub ${ApplicationName}-${EnvName} + Cpu: !Ref TaskCpuSetting + Memory: !Ref TaskMemorySetting + NetworkMode: awsvpc + RequiresCompatibilities: + - FARGATE + ExecutionRoleArn: !Ref ExecutionRoleArn + TaskRoleArn: !Ref ExecutionRoleArn + ContainerDefinitions: + - Name: !Sub ${ApplicationName}-${EnvName} + Image: !Ref DockerImageUrl + Environment: + - Name: REDIS_PASSWORD + Value: !Ref RedisPass + - Name: DB_PASSWORD + Value: !Ref PgPass + - Name: OPENSEARCH_PASSWORD + Value: !Ref OpenserchPass + - Name: S3_SECRET_KEY + Value: !Ref S3SecretsKey + - Name: MODE + Value: !Ref Mode + - Name: CELERY_BROKER_URL + Value: !Ref CeleryBrokerUrl + EnvironmentFiles: + - Type: s3 + Value: !Ref EnvironmentFileUrl + PortMappings: + - ContainerPort: !Ref AppPort + HostPort: !Ref AppPort + LogConfiguration: + LogDriver: "awslogs" + Options: + awslogs-create-group: true + awslogs-group: !Sub ${ApplicationName}-${EnvName} + awslogs-region: !Ref "AWS::Region" + awslogs-stream-prefix: ecs + + LarkTopic: + Type: AWS::SNS::Topic + Properties: + TopicName: !Sub ${ApplicationName}-${EnvName} + Subscription: + - Endpoint: !Ref SnsToLarkLambda + Protocol: "LAMBDA" + +# ASG setting begin + AutoScalingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - ecs-tasks.amazonaws.com + Action: + - 'sts:AssumeRole' + Path: '/' + Policies: + - PolicyName: allow-ecs-cw-scaling + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - ecs:DescribeServices + - ecs:UpdateService + - cloudwatch:DeleteAlarms + - cloudwatch:DescribeAlarms + - cloudwatch:PutMetricAlarm + Resource: '*' + AutoScalingTarget: + Type: AWS::ApplicationAutoScaling::ScalableTarget + Properties: + MinCapacity: !Ref TaskDesiredCount + MaxCapacity: 10 + ResourceId: !Join ["/", [service, !Ref ECSClusterName, !GetAtt ECSService.Name]] + ScalableDimension: ecs:service:DesiredCount + ServiceNamespace: ecs + RoleARN: !GetAtt AutoScalingRole.Arn + AutoScalingPolicyForCPU: + Type: AWS::ApplicationAutoScaling::ScalingPolicy + Properties: + PolicyName: ecs-cpu-utilization-ScalingPolicy + PolicyType: TargetTrackingScaling + ScalingTargetId: !Ref AutoScalingTarget + TargetTrackingScalingPolicyConfiguration: + PredefinedMetricSpecification: + PredefinedMetricType: ECSServiceAverageCPUUtilization + ScaleInCooldown: 300 + ScaleOutCooldown: 120 + TargetValue: 70.0 + AutoScalingPolicyForMemory: + Type: AWS::ApplicationAutoScaling::ScalingPolicy + Properties: + PolicyName: ecs-memory-utilization-ScalingPolicy + PolicyType: TargetTrackingScaling + ScalingTargetId: !Ref AutoScalingTarget + TargetTrackingScalingPolicyConfiguration: + PredefinedMetricSpecification: + PredefinedMetricType: ECSServiceAverageMemoryUtilization + ScaleInCooldown: 300 + ScaleOutCooldown: 120 + TargetValue: 75.0 +# ASG setting end + +Outputs: + ClusterName: + Description: The cluster used to create the service. + Value: !Ref ECSClusterName + ECSService: + Description: The created service. + Value: !Ref ECSService + LarkTopic: + Description: The created Lark Topic. + Value: !Ref LarkTopic