action config
zyuvszqk
11 months ago
parent
f6641c0f41
commit
52f42d12fc
@ -1,93 +0,0 @@
|
||||
name: Run Pytest
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- api/**
|
||||
- docker/**
|
||||
- .github/workflows/api-tests.yml
|
||||
|
||||
concurrency:
|
||||
group: api-tests-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: API Tests
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
- "3.11"
|
||||
- "3.12"
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup UV and Python
|
||||
uses: ./.github/actions/setup-uv
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
uv-lockfile: api/uv.lock
|
||||
|
||||
- name: Check UV lockfile
|
||||
run: uv lock --project api --check
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --project api --dev
|
||||
|
||||
- name: Run Unit tests
|
||||
run: |
|
||||
uv run --project api bash dev/pytest/pytest_unit_tests.sh
|
||||
# Extract coverage percentage and create a summary
|
||||
TOTAL_COVERAGE=$(python -c 'import json; print(json.load(open("coverage.json"))["totals"]["percent_covered_display"])')
|
||||
|
||||
# Create a detailed coverage summary
|
||||
echo "### Test Coverage Summary :test_tube:" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Total Coverage: ${TOTAL_COVERAGE}%" >> $GITHUB_STEP_SUMMARY
|
||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
||||
uv run --project api coverage report >> $GITHUB_STEP_SUMMARY
|
||||
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: Run dify config tests
|
||||
run: uv run --project api dev/pytest/pytest_config_tests.py
|
||||
|
||||
- name: MyPy Cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: api/.mypy_cache
|
||||
key: mypy-${{ matrix.python-version }}-${{ runner.os }}-${{ hashFiles('api/uv.lock') }}
|
||||
|
||||
- name: Run MyPy Checks
|
||||
run: dev/mypy-check
|
||||
|
||||
- name: Set up dotenvs
|
||||
run: |
|
||||
cp docker/.env.example docker/.env
|
||||
cp docker/middleware.env.example docker/middleware.env
|
||||
|
||||
- name: Expose Service Ports
|
||||
run: sh .github/workflows/expose_service_ports.sh
|
||||
|
||||
- name: Set up Sandbox
|
||||
uses: hoverkraft-tech/compose-action@v2.0.2
|
||||
with:
|
||||
compose-file: |
|
||||
docker/docker-compose.middleware.yaml
|
||||
services: |
|
||||
sandbox
|
||||
ssrf_proxy
|
||||
|
||||
- name: Run Workflow
|
||||
run: uv run --project api bash dev/pytest/pytest_workflow.sh
|
||||
|
||||
- name: Run Tool
|
||||
run: uv run --project api bash dev/pytest/pytest_tools.sh
|
||||
@ -1,149 +0,0 @@
|
||||
name: Build and Push API & Web
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- "main"
|
||||
- "deploy/dev"
|
||||
- "deploy/enterprise"
|
||||
tags:
|
||||
- "*"
|
||||
|
||||
concurrency:
|
||||
group: build-push-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
|
||||
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
DIFY_WEB_IMAGE_NAME: ${{ vars.DIFY_WEB_IMAGE_NAME || 'langgenius/dify-web' }}
|
||||
DIFY_API_IMAGE_NAME: ${{ vars.DIFY_API_IMAGE_NAME || 'langgenius/dify-api' }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ${{ matrix.platform == 'linux/arm64' && 'arm64_runner' || 'ubuntu-latest' }}
|
||||
if: github.repository == 'langgenius/dify'
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- service_name: "build-api-amd64"
|
||||
image_name_env: "DIFY_API_IMAGE_NAME"
|
||||
context: "api"
|
||||
platform: linux/amd64
|
||||
- service_name: "build-api-arm64"
|
||||
image_name_env: "DIFY_API_IMAGE_NAME"
|
||||
context: "api"
|
||||
platform: linux/arm64
|
||||
- service_name: "build-web-amd64"
|
||||
image_name_env: "DIFY_WEB_IMAGE_NAME"
|
||||
context: "web"
|
||||
platform: linux/amd64
|
||||
- service_name: "build-web-arm64"
|
||||
image_name_env: "DIFY_WEB_IMAGE_NAME"
|
||||
context: "web"
|
||||
platform: linux/arm64
|
||||
|
||||
steps:
|
||||
- name: Prepare
|
||||
run: |
|
||||
platform=${{ matrix.platform }}
|
||||
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKERHUB_USER }}
|
||||
password: ${{ env.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Extract metadata for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env[matrix.image_name_env] }}
|
||||
|
||||
- name: Build Docker image
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: "{{defaultContext}}:${{ matrix.context }}"
|
||||
platforms: ${{ matrix.platform }}
|
||||
build-args: COMMIT_SHA=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.revision'] }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
outputs: type=image,name=${{ env[matrix.image_name_env] }},push-by-digest=true,name-canonical=true,push=true
|
||||
cache-from: type=gha,scope=${{ matrix.service_name }}
|
||||
cache-to: type=gha,mode=max,scope=${{ matrix.service_name }}
|
||||
|
||||
- name: Export digest
|
||||
env:
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
sanitized_digest=${DIGEST#sha256:}
|
||||
touch "/tmp/digests/${sanitized_digest}"
|
||||
|
||||
- name: Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digests-${{ matrix.context }}-${{ env.PLATFORM_PAIR }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
create-manifest:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
if: github.repository == 'langgenius/dify'
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- service_name: "merge-api-images"
|
||||
image_name_env: "DIFY_API_IMAGE_NAME"
|
||||
context: "api"
|
||||
- service_name: "merge-web-images"
|
||||
image_name_env: "DIFY_WEB_IMAGE_NAME"
|
||||
context: "web"
|
||||
steps:
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: /tmp/digests
|
||||
pattern: digests-${{ matrix.context }}-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ env.DOCKERHUB_USER }}
|
||||
password: ${{ env.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env[matrix.image_name_env] }}
|
||||
tags: |
|
||||
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') }}
|
||||
type=ref,event=branch
|
||||
type=sha,enable=true,priority=100,prefix=,suffix=,format=long
|
||||
type=raw,value=${{ github.ref_name }},enable=${{ startsWith(github.ref, 'refs/tags/') }}
|
||||
|
||||
- name: Create manifest list and push
|
||||
working-directory: /tmp/digests
|
||||
env:
|
||||
IMAGE_NAME: ${{ env[matrix.image_name_env] }}
|
||||
run: |
|
||||
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf "$IMAGE_NAME@sha256:%s " *)
|
||||
|
||||
- name: Inspect image
|
||||
env:
|
||||
IMAGE_NAME: ${{ env[matrix.image_name_env] }}
|
||||
IMAGE_VERSION: ${{ steps.meta.outputs.version }}
|
||||
run: |
|
||||
docker buildx imagetools inspect "$IMAGE_NAME:$IMAGE_VERSION"
|
||||
@ -1,57 +0,0 @@
|
||||
name: DB Migration Test
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- plugins/beta
|
||||
paths:
|
||||
- api/migrations/**
|
||||
- .github/workflows/db-migration-test.yml
|
||||
|
||||
concurrency:
|
||||
group: db-migration-test-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
db-migration-test:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup UV and Python
|
||||
uses: ./.github/actions/setup-uv
|
||||
with:
|
||||
uv-lockfile: api/uv.lock
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --project api
|
||||
|
||||
- name: Prepare middleware env
|
||||
run: |
|
||||
cd docker
|
||||
cp middleware.env.example middleware.env
|
||||
|
||||
- name: Set up Middlewares
|
||||
uses: hoverkraft-tech/compose-action@v2.0.2
|
||||
with:
|
||||
compose-file: |
|
||||
docker/docker-compose.middleware.yaml
|
||||
services: |
|
||||
db
|
||||
redis
|
||||
|
||||
- name: Prepare configs
|
||||
run: |
|
||||
cd api
|
||||
cp .env.example .env
|
||||
|
||||
- name: Run DB Migration
|
||||
env:
|
||||
DEBUG: true
|
||||
run: uv run --directory api flask upgrade-db
|
||||
@ -1,24 +0,0 @@
|
||||
name: Deploy Dev
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["Build and Push API & Web"]
|
||||
branches:
|
||||
- "deploy/dev"
|
||||
types:
|
||||
- completed
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
if: |
|
||||
github.event.workflow_run.conclusion == 'success'
|
||||
steps:
|
||||
- name: Deploy to server
|
||||
uses: appleboy/ssh-action@v0.1.8
|
||||
with:
|
||||
host: ${{ secrets.SSH_HOST }}
|
||||
username: ${{ secrets.SSH_USER }}
|
||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
||||
script: |
|
||||
${{ vars.SSH_SCRIPT || secrets.SSH_SCRIPT }}
|
||||
@ -1,29 +0,0 @@
|
||||
name: Deploy Enterprise
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["Build and Push API & Web"]
|
||||
branches:
|
||||
- "deploy/enterprise"
|
||||
types:
|
||||
- completed
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
if: |
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.head_branch == 'deploy/enterprise'
|
||||
|
||||
steps:
|
||||
- name: Deploy to server
|
||||
uses: appleboy/ssh-action@v0.1.8
|
||||
with:
|
||||
host: ${{ secrets.ENTERPRISE_SSH_HOST }}
|
||||
username: ${{ secrets.ENTERPRISE_SSH_USER }}
|
||||
password: ${{ secrets.ENTERPRISE_SSH_PASSWORD }}
|
||||
script: |
|
||||
${{ vars.ENTERPRISE_SSH_SCRIPT || secrets.ENTERPRISE_SSH_SCRIPT }}
|
||||
@ -0,0 +1,142 @@
|
||||
name: Reusable deployment job
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
APPLICATION_NAME:
|
||||
default: 'oneapi'
|
||||
type: string
|
||||
AWS_REGION:
|
||||
default: 'us-east-2'
|
||||
type: string
|
||||
DEPLOYMENT_ENV:
|
||||
required: true
|
||||
type: string
|
||||
ECR_REPOSITORY:
|
||||
required: true
|
||||
type: string
|
||||
ECS_CLUSTER:
|
||||
required: true
|
||||
type: string
|
||||
SECURITY_GROUP_IDS_SSM:
|
||||
required: true
|
||||
type: string
|
||||
SUBNET_IDS_SSM:
|
||||
required: true
|
||||
type: string
|
||||
VPC_ID_SSM:
|
||||
required: true
|
||||
type: string
|
||||
SNS_TO_LARK_LAMBDA_ARN:
|
||||
required: true
|
||||
type: string
|
||||
CERERTIFICATE_US_EAST_2_ARN:
|
||||
required: true
|
||||
type: string
|
||||
ECS_EXECUTION_ROLE_ARN:
|
||||
required: true
|
||||
type: string
|
||||
DOCKER_IMAGE_NAME:
|
||||
required: true
|
||||
type: string
|
||||
MODE:
|
||||
required: true
|
||||
type: string
|
||||
APP_PORT:
|
||||
required: true
|
||||
type: number
|
||||
HEALTH_CHECK:
|
||||
required: true
|
||||
type: string
|
||||
TEMPLATE:
|
||||
required: true
|
||||
type: string
|
||||
DOCKERFILEDIR:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
|
||||
secrets:
|
||||
DEPLOYMENT_AWS_ACCESS_KEY_ID:
|
||||
required: true
|
||||
DEPLOYMENT_AWS_SECRET_ACCESS_KEY:
|
||||
required: true
|
||||
ED25519_CI_KEY:
|
||||
required: true
|
||||
PAT:
|
||||
required: true
|
||||
env:
|
||||
ECS_CONTAINER_ENV_FILE_S3_PATH: ${{ inputs.DEPLOYMENT_ENV }}-dify-ai/.env
|
||||
jobs:
|
||||
deploy:
|
||||
name: Deploy to ${{ inputs.DEPLOYMENT_ENV }}
|
||||
runs-on: ubuntu-latest
|
||||
concurrency: deployment-${{ inputs.APPLICATION_NAME }}-${{ inputs.DEPLOYMENT_ENV }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Fetch and checkout branch
|
||||
run: |
|
||||
BRANCH=${{ inputs.DEPLOYMENT_ENV }}
|
||||
echo "Target branch: $BRANCH"
|
||||
git fetch origin $BRANCH
|
||||
git checkout $BRANCH
|
||||
git reset --hard origin/$BRANCH
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v1
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ inputs.AWS_REGION }}
|
||||
|
||||
|
||||
- name: Login to Amazon ECR
|
||||
id: login-ecr
|
||||
uses: aws-actions/amazon-ecr-login@v1
|
||||
|
||||
- name: Set SHORT_SHA
|
||||
id: set_short_sha
|
||||
run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >>$GITHUB_ENV
|
||||
- name: Set IMAGE_TAG
|
||||
id: set_image_tag
|
||||
run: echo "IMAGE_TAG=${{ inputs.APPLICATION_NAME }}-$SHORT_SHA" >> $GITHUB_ENV
|
||||
- name: Set GIT_BRANCH
|
||||
id: set_git_branch
|
||||
run: echo "GIT_BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Build, tag, and push image to Amazon ECR
|
||||
id: build-image
|
||||
env:
|
||||
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
|
||||
|
||||
run: |
|
||||
cd ${{ inputs.DOCKERFILEDIR }}
|
||||
docker build --build-arg GIT_BRANCH=$GIT_BRANCH --build-arg GIT_PAT_TOKEN=${{ secrets.PAT }} -t $ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG .
|
||||
docker push $ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG
|
||||
echo "::set-output name=image::$ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG"
|
||||
|
||||
- name: Deploy CloudFormation
|
||||
uses: aws-actions/aws-cloudformation-github-deploy@v1
|
||||
with:
|
||||
name: ${{ inputs.APPLICATION_NAME }}-${{ inputs.DEPLOYMENT_ENV }}
|
||||
template: aws/${{ inputs.TEMPLATE }}
|
||||
no-fail-on-empty-changeset: "1"
|
||||
parameter-overrides: >-
|
||||
CertificateArn=${{ inputs.CERERTIFICATE_US_EAST_2_ARN }},
|
||||
ExecutionRoleArn=${{ inputs.ECS_EXECUTION_ROLE_ARN }},
|
||||
ECSClusterName=${{ inputs.ECS_CLUSTER }},
|
||||
ApplicationName=${{ inputs.APPLICATION_NAME }},
|
||||
EnvName=${{ inputs.DEPLOYMENT_ENV }},
|
||||
SecurityGroupIDs=${{ inputs.SECURITY_GROUP_IDS_SSM }},
|
||||
SubnetIDs=${{ inputs.SUBNET_IDS_SSM }},
|
||||
VpcID=${{ inputs.VPC_ID_SSM }},
|
||||
DockerImageUrl=${{ steps.build-image.outputs.image }},
|
||||
EnvironmentFileUrl=arn:aws:s3:::${{ env.ECS_CONTAINER_ENV_FILE_S3_PATH }},
|
||||
SnsToLarkLambda=${{ inputs.SNS_TO_LARK_LAMBDA_ARN }},
|
||||
AppPort=${{ inputs.APP_PORT }},
|
||||
Mode=${{ inputs.MODE }},
|
||||
DockerImageName=${{ inputs.DOCKER_IMAGE_NAME }},
|
||||
HealthCheck=${{ inputs.HEALTH_CHECK }}
|
||||
@ -1,48 +0,0 @@
|
||||
name: Build docker image
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- "main"
|
||||
paths:
|
||||
- api/Dockerfile
|
||||
- web/Dockerfile
|
||||
|
||||
concurrency:
|
||||
group: docker-build-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-docker:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- service_name: "api-amd64"
|
||||
platform: linux/amd64
|
||||
context: "api"
|
||||
- service_name: "api-arm64"
|
||||
platform: linux/arm64
|
||||
context: "api"
|
||||
- service_name: "web-amd64"
|
||||
platform: linux/amd64
|
||||
context: "web"
|
||||
- service_name: "web-arm64"
|
||||
platform: linux/arm64
|
||||
context: "web"
|
||||
steps:
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build Docker Image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
push: false
|
||||
context: "{{defaultContext}}:${{ matrix.context }}"
|
||||
file: "${{ matrix.file }}"
|
||||
platforms: ${{ matrix.platform }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@ -1,15 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
yq eval '.services.weaviate.ports += ["8080:8080"]' -i docker/docker-compose.yaml
|
||||
yq eval '.services.qdrant.ports += ["6333:6333"]' -i docker/docker-compose.yaml
|
||||
yq eval '.services.chroma.ports += ["8000:8000"]' -i docker/docker-compose.yaml
|
||||
yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/docker-compose.yaml
|
||||
yq eval '.services.pgvector.ports += ["5433:5432"]' -i docker/docker-compose.yaml
|
||||
yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compose.yaml
|
||||
yq eval '.services["elasticsearch"].ports += ["9200:9200"]' -i docker/docker-compose.yaml
|
||||
yq eval '.services.couchbase-server.ports += ["8091-8096:8091-8096"]' -i docker/docker-compose.yaml
|
||||
yq eval '.services.couchbase-server.ports += ["11210:11210"]' -i docker/docker-compose.yaml
|
||||
yq eval '.services.tidb.ports += ["4000:4000"]' -i docker/tidb/docker-compose.yaml
|
||||
yq eval '.services.opengauss.ports += ["6600:6600"]' -i docker/docker-compose.yaml
|
||||
|
||||
echo "Ports exposed for sandbox, weaviate, tidb, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase, opengauss"
|
||||
@ -0,0 +1,36 @@
|
||||
|
||||
name: Api Prod Deployment
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
call-deployment-in-local-repo:
|
||||
if: contains('carloschen6688,yvan-65535', github.actor)
|
||||
uses: ./.github/workflows/prod-deploy.yml
|
||||
with:
|
||||
APPLICATION_NAME: dify-api
|
||||
DEPLOYMENT_ENV: prod
|
||||
ECR_REPOSITORY: ai-dify-api
|
||||
ECS_CLUSTER: ai-dify
|
||||
SECURITY_GROUP_IDS_SSM: /prod/security-groups
|
||||
SUBNET_IDS_SSM: /prod/subnet-ids
|
||||
VPC_ID_SSM: /prod/vpc-id
|
||||
SNS_TO_LARK_LAMBDA_ARN: /prod/lambda/sns-to-lark
|
||||
CERERTIFICATE_US_EAST_2_ARN: /prod/certificate
|
||||
ECS_EXECUTION_ROLE_ARN: /prod/ecs-execution-role
|
||||
APP_PORT: 5001
|
||||
MODE: api
|
||||
DOCKER_IMAGE_NAME: dify-api
|
||||
HEALTH_CHECK: /console/api/ping
|
||||
TEMPLATE: cloudformation-ecs-service-template.yml
|
||||
DOCKERFILEDIR: api
|
||||
|
||||
|
||||
secrets:
|
||||
DEPLOYMENT_AWS_ACCESS_KEY_ID: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID_PROD }}
|
||||
DEPLOYMENT_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY_PROD }}
|
||||
ED25519_CI_KEY: ${{ secrets.ED25519_CI_KEY }}
|
||||
PAT: ${{ secrets.PAT }}
|
||||
|
||||
|
||||
@ -0,0 +1,134 @@
|
||||
name: Reusable deployment job
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
APPLICATION_NAME:
|
||||
default: 'oneapi'
|
||||
type: string
|
||||
AWS_REGION:
|
||||
default: 'us-east-2'
|
||||
type: string
|
||||
DEPLOYMENT_ENV:
|
||||
required: true
|
||||
type: string
|
||||
ECR_REPOSITORY:
|
||||
required: true
|
||||
type: string
|
||||
ECS_CLUSTER:
|
||||
required: true
|
||||
type: string
|
||||
SECURITY_GROUP_IDS_SSM:
|
||||
required: true
|
||||
type: string
|
||||
SUBNET_IDS_SSM:
|
||||
required: true
|
||||
type: string
|
||||
VPC_ID_SSM:
|
||||
required: true
|
||||
type: string
|
||||
SNS_TO_LARK_LAMBDA_ARN:
|
||||
required: true
|
||||
type: string
|
||||
CERERTIFICATE_US_EAST_2_ARN:
|
||||
required: true
|
||||
type: string
|
||||
ECS_EXECUTION_ROLE_ARN:
|
||||
required: true
|
||||
type: string
|
||||
DOCKER_IMAGE_NAME:
|
||||
required: true
|
||||
type: string
|
||||
MODE:
|
||||
required: true
|
||||
type: string
|
||||
APP_PORT:
|
||||
required: true
|
||||
type: number
|
||||
HEALTH_CHECK:
|
||||
required: true
|
||||
type: string
|
||||
TEMPLATE:
|
||||
required: true
|
||||
type: string
|
||||
DOCKERFILEDIR:
|
||||
required: true
|
||||
type: string
|
||||
|
||||
|
||||
secrets:
|
||||
DEPLOYMENT_AWS_ACCESS_KEY_ID:
|
||||
required: true
|
||||
DEPLOYMENT_AWS_SECRET_ACCESS_KEY:
|
||||
required: true
|
||||
ED25519_CI_KEY:
|
||||
required: true
|
||||
PAT:
|
||||
required: true
|
||||
env:
|
||||
ECS_CONTAINER_ENV_FILE_S3_PATH: ${{ inputs.DEPLOYMENT_ENV }}-dify-ai/.env
|
||||
jobs:
|
||||
deploy:
|
||||
name: Deploy to ${{ inputs.DEPLOYMENT_ENV }}
|
||||
runs-on: ubuntu-latest
|
||||
concurrency: deployment-${{ inputs.APPLICATION_NAME }}-${{ inputs.DEPLOYMENT_ENV }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v1
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ inputs.AWS_REGION }}
|
||||
|
||||
|
||||
- name: Login to Amazon ECR
|
||||
id: login-ecr
|
||||
uses: aws-actions/amazon-ecr-login@v1
|
||||
|
||||
- name: Set SHORT_SHA
|
||||
id: set_short_sha
|
||||
run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >>$GITHUB_ENV
|
||||
- name: Set IMAGE_TAG
|
||||
id: set_image_tag
|
||||
run: echo "IMAGE_TAG=${{ inputs.APPLICATION_NAME }}-$SHORT_SHA" >> $GITHUB_ENV
|
||||
- name: Set GIT_BRANCH
|
||||
id: set_git_branch
|
||||
run: echo "GIT_BRANCH=${{ github.ref_name }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Build, tag, and push image to Amazon ECR
|
||||
id: build-image
|
||||
env:
|
||||
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
|
||||
|
||||
run: |
|
||||
cd ${{ inputs.DOCKERFILEDIR }}
|
||||
docker build --build-arg GIT_BRANCH=$GIT_BRANCH --build-arg GIT_PAT_TOKEN=${{ secrets.PAT }} -t $ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG .
|
||||
docker push $ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG
|
||||
echo "::set-output name=image::$ECR_REGISTRY/${{ inputs.ECR_REPOSITORY }}:$IMAGE_TAG"
|
||||
|
||||
- name: Deploy CloudFormation
|
||||
uses: aws-actions/aws-cloudformation-github-deploy@v1
|
||||
with:
|
||||
name: ${{ inputs.APPLICATION_NAME }}-${{ inputs.DEPLOYMENT_ENV }}
|
||||
template: aws/${{ inputs.TEMPLATE }}
|
||||
no-fail-on-empty-changeset: "1"
|
||||
parameter-overrides: >-
|
||||
CertificateArn=${{ inputs.CERERTIFICATE_US_EAST_2_ARN }},
|
||||
ExecutionRoleArn=${{ inputs.ECS_EXECUTION_ROLE_ARN }},
|
||||
ECSClusterName=${{ inputs.ECS_CLUSTER }},
|
||||
ApplicationName=${{ inputs.APPLICATION_NAME }},
|
||||
EnvName=${{ inputs.DEPLOYMENT_ENV }},
|
||||
SecurityGroupIDs=${{ inputs.SECURITY_GROUP_IDS_SSM }},
|
||||
SubnetIDs=${{ inputs.SUBNET_IDS_SSM }},
|
||||
VpcID=${{ inputs.VPC_ID_SSM }},
|
||||
DockerImageUrl=${{ steps.build-image.outputs.image }},
|
||||
EnvironmentFileUrl=arn:aws:s3:::${{ env.ECS_CONTAINER_ENV_FILE_S3_PATH }},
|
||||
SnsToLarkLambda=${{ inputs.SNS_TO_LARK_LAMBDA_ARN }},
|
||||
AppPort=${{ inputs.APP_PORT }},
|
||||
Mode=${{ inputs.MODE }},
|
||||
DockerImageName=${{ inputs.DOCKER_IMAGE_NAME }},
|
||||
HealthCheck=${{ inputs.HEALTH_CHECK }}
|
||||
@ -0,0 +1,34 @@
|
||||
name: Web Prod Deployment
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
call-deployment-in-local-repo:
|
||||
if: contains('carloschen6688,yvan-65535', github.actor)
|
||||
uses: ./.github/workflows/prod-deploy.yml
|
||||
with:
|
||||
APPLICATION_NAME: dify-web
|
||||
DEPLOYMENT_ENV: prod
|
||||
ECR_REPOSITORY: ai-dify-web
|
||||
ECS_CLUSTER: ai-dify
|
||||
SECURITY_GROUP_IDS_SSM: /prod/security-groups
|
||||
SUBNET_IDS_SSM: /prod/subnet-ids
|
||||
VPC_ID_SSM: /prod/vpc-id
|
||||
SNS_TO_LARK_LAMBDA_ARN: /prod/lambda/sns-to-lark
|
||||
CERERTIFICATE_US_EAST_2_ARN: /prod/certificate
|
||||
ECS_EXECUTION_ROLE_ARN: /prod/ecs-execution-role
|
||||
APP_PORT: 3000
|
||||
MODE: web
|
||||
DOCKER_IMAGE_NAME: dify-web
|
||||
HEALTH_CHECK: /apps
|
||||
TEMPLATE: cloudformation-ecs-service-template.yml
|
||||
DOCKERFILEDIR: web
|
||||
|
||||
|
||||
secrets:
|
||||
DEPLOYMENT_AWS_ACCESS_KEY_ID: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID_PROD }}
|
||||
DEPLOYMENT_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY_PROD }}
|
||||
ED25519_CI_KEY: ${{ secrets.ED25519_CI_KEY }}
|
||||
PAT: ${{ secrets.PAT }}
|
||||
|
||||
@ -0,0 +1,34 @@
|
||||
name: Worker Prod Deployment
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
call-deployment-in-local-repo:
|
||||
if: contains('carloschen6688,yvan-65535', github.actor)
|
||||
uses: ./.github/workflows/prod-deploy.yml
|
||||
with:
|
||||
APPLICATION_NAME: dify-worker
|
||||
DEPLOYMENT_ENV: prod
|
||||
ECR_REPOSITORY: ai-dify-worker
|
||||
ECS_CLUSTER: ai-dify
|
||||
SECURITY_GROUP_IDS_SSM: /prod/security-groups
|
||||
SUBNET_IDS_SSM: /prod/subnet-ids
|
||||
VPC_ID_SSM: /prod/vpc-id
|
||||
SNS_TO_LARK_LAMBDA_ARN: /prod/lambda/sns-to-lark
|
||||
CERERTIFICATE_US_EAST_2_ARN: /prod/certificate
|
||||
ECS_EXECUTION_ROLE_ARN: /prod/ecs-execution-role
|
||||
APP_PORT: 5001
|
||||
MODE: worker
|
||||
DOCKER_IMAGE_NAME: dify-api
|
||||
HEALTH_CHECK: /health
|
||||
TEMPLATE: worker-cloudformation-ecs-service-template.yml
|
||||
DOCKERFILEDIR: api
|
||||
|
||||
|
||||
secrets:
|
||||
DEPLOYMENT_AWS_ACCESS_KEY_ID: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID_PROD }}
|
||||
DEPLOYMENT_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY_PROD }}
|
||||
ED25519_CI_KEY: ${{ secrets.ED25519_CI_KEY }}
|
||||
PAT: ${{ secrets.PAT }}
|
||||
|
||||
@ -0,0 +1,36 @@
|
||||
name: Api Staging Deployment
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
repository_dispatch:
|
||||
types: [deploy-staging]
|
||||
|
||||
jobs:
|
||||
call-deployment-in-local-repo:
|
||||
uses: ./.github/workflows/deploy.yml
|
||||
with:
|
||||
APPLICATION_NAME: dify-api
|
||||
DEPLOYMENT_ENV: staging
|
||||
ECR_REPOSITORY: ai-dify-api
|
||||
ECS_CLUSTER: ai-dify
|
||||
SECURITY_GROUP_IDS_SSM: /staging/security-groups
|
||||
SUBNET_IDS_SSM: /staging/subnet-ids
|
||||
VPC_ID_SSM: /staging/vpc-id
|
||||
SNS_TO_LARK_LAMBDA_ARN: /staging/lambda/sns-to-lark
|
||||
CERERTIFICATE_US_EAST_2_ARN: /staging/certificate
|
||||
ECS_EXECUTION_ROLE_ARN: /staging/ecs-execution-role
|
||||
APP_PORT: 5001
|
||||
MODE: api
|
||||
DOCKER_IMAGE_NAME: dify-api
|
||||
HEALTH_CHECK: /console/api/ping
|
||||
TEMPLATE: cloudformation-ecs-service-template.yml
|
||||
DOCKERFILEDIR: api
|
||||
|
||||
|
||||
|
||||
secrets:
|
||||
DEPLOYMENT_AWS_ACCESS_KEY_ID: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID_STAGING }}
|
||||
DEPLOYMENT_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY_STAGING }}
|
||||
ED25519_CI_KEY: ${{ secrets.ED25519_CI_KEY }}
|
||||
PAT: ${{ secrets.PAT }}
|
||||
|
||||
@ -0,0 +1,34 @@
|
||||
name: Web Staging Deployment
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
call-deployment-in-local-repo:
|
||||
uses: ./.github/workflows/deploy.yml
|
||||
with:
|
||||
APPLICATION_NAME: dify-web
|
||||
DEPLOYMENT_ENV: staging
|
||||
ECR_REPOSITORY: ai-dify-web
|
||||
ECS_CLUSTER: ai-dify
|
||||
SECURITY_GROUP_IDS_SSM: /staging/security-groups
|
||||
SUBNET_IDS_SSM: /staging/subnet-ids
|
||||
VPC_ID_SSM: /staging/vpc-id
|
||||
SNS_TO_LARK_LAMBDA_ARN: /staging/lambda/sns-to-lark
|
||||
CERERTIFICATE_US_EAST_2_ARN: /staging/certificate
|
||||
ECS_EXECUTION_ROLE_ARN: /staging/ecs-execution-role
|
||||
APP_PORT: 3000
|
||||
MODE: web
|
||||
DOCKER_IMAGE_NAME: dify-web
|
||||
HEALTH_CHECK: /apps
|
||||
TEMPLATE: cloudformation-ecs-service-template.yml
|
||||
DOCKERFILEDIR: web
|
||||
|
||||
|
||||
|
||||
secrets:
|
||||
DEPLOYMENT_AWS_ACCESS_KEY_ID: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID_STAGING }}
|
||||
DEPLOYMENT_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY_STAGING }}
|
||||
ED25519_CI_KEY: ${{ secrets.ED25519_CI_KEY }}
|
||||
PAT: ${{ secrets.PAT }}
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
name: Worker Staging Deployment
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
call-deployment-in-local-repo:
|
||||
uses: ./.github/workflows/deploy.yml
|
||||
with:
|
||||
APPLICATION_NAME: dify-worker
|
||||
DEPLOYMENT_ENV: staging
|
||||
ECR_REPOSITORY: ai-dify-worker
|
||||
ECS_CLUSTER: ai-dify
|
||||
SECURITY_GROUP_IDS_SSM: /staging/security-groups
|
||||
SUBNET_IDS_SSM: /staging/subnet-ids
|
||||
VPC_ID_SSM: /staging/vpc-id
|
||||
SNS_TO_LARK_LAMBDA_ARN: /staging/lambda/sns-to-lark
|
||||
CERERTIFICATE_US_EAST_2_ARN: /staging/certificate
|
||||
ECS_EXECUTION_ROLE_ARN: /staging/ecs-execution-role
|
||||
APP_PORT: 5001
|
||||
MODE: worker
|
||||
DOCKER_IMAGE_NAME: dify-api
|
||||
HEALTH_CHECK: /health
|
||||
TEMPLATE: worker-cloudformation-ecs-service-template.yml
|
||||
DOCKERFILEDIR: api
|
||||
|
||||
|
||||
secrets:
|
||||
DEPLOYMENT_AWS_ACCESS_KEY_ID: ${{ secrets.DEPLOYMENT_AWS_ACCESS_KEY_ID_STAGING }}
|
||||
DEPLOYMENT_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYMENT_AWS_SECRET_ACCESS_KEY_STAGING }}
|
||||
ED25519_CI_KEY: ${{ secrets.ED25519_CI_KEY }}
|
||||
PAT: ${{ secrets.PAT }}
|
||||
|
||||
@ -1,30 +0,0 @@
|
||||
# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
|
||||
#
|
||||
# You can adjust the behavior by modifying this file.
|
||||
# For more information, see:
|
||||
# https://github.com/actions/stale
|
||||
name: Mark stale issues and pull requests
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 3 * * *'
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- uses: actions/stale@v5
|
||||
with:
|
||||
days-before-issue-stale: 15
|
||||
days-before-issue-close: 3
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
stale-issue-message: "Close due to it's no longer active, if you have any questions, you can reopen it."
|
||||
stale-pr-message: "Close due to it's no longer active, if you have any questions, you can reopen it."
|
||||
stale-issue-label: 'no-issue-activity'
|
||||
stale-pr-label: 'no-pr-activity'
|
||||
any-of-labels: 'duplicate,question,invalid,wontfix,no-issue-activity,no-pr-activity,enhancement,cant-reproduce,help-wanted'
|
||||
@ -1,183 +0,0 @@
|
||||
name: Style check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
concurrency:
|
||||
group: style-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
checks: write
|
||||
statuses: write
|
||||
contents: read
|
||||
|
||||
|
||||
jobs:
|
||||
python-style:
|
||||
name: Python Style
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@v45
|
||||
with:
|
||||
files: |
|
||||
api/**
|
||||
.github/workflows/style.yml
|
||||
|
||||
- name: Setup UV and Python
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
uses: ./.github/actions/setup-uv
|
||||
with:
|
||||
uv-lockfile: api/uv.lock
|
||||
enable-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
run: uv sync --project api --dev
|
||||
|
||||
- name: Ruff check
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
run: |
|
||||
uv run --directory api ruff --version
|
||||
uv run --directory api ruff check --diff ./
|
||||
uv run --directory api ruff format --check --diff ./
|
||||
|
||||
- name: Dotenv check
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
run: uv run --project api dotenv-linter ./api/.env.example ./web/.env.example
|
||||
|
||||
- name: Lint hints
|
||||
if: failure()
|
||||
run: echo "Please run 'dev/reformat' to fix the fixable linting errors."
|
||||
|
||||
web-style:
|
||||
name: Web Style
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ./web
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@v45
|
||||
with:
|
||||
files: web/**
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- name: Setup NodeJS
|
||||
uses: actions/setup-node@v4
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
cache-dependency-path: ./web/package.json
|
||||
|
||||
- name: Web dependencies
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Web style check
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
run: pnpm run lint
|
||||
|
||||
docker-compose-template:
|
||||
name: Docker Compose Template
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@v45
|
||||
with:
|
||||
files: |
|
||||
docker/generate_docker_compose
|
||||
docker/.env.example
|
||||
docker/docker-compose-template.yaml
|
||||
docker/docker-compose.yaml
|
||||
|
||||
- name: Generate Docker Compose
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
run: |
|
||||
cd docker
|
||||
./generate_docker_compose
|
||||
|
||||
- name: Check for changes
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
run: git diff --exit-code
|
||||
|
||||
superlinter:
|
||||
name: SuperLinter
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@v45
|
||||
with:
|
||||
files: |
|
||||
**.sh
|
||||
**.yaml
|
||||
**.yml
|
||||
**Dockerfile
|
||||
dev/**
|
||||
|
||||
- name: Super-linter
|
||||
uses: super-linter/super-linter/slim@v7
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
env:
|
||||
BASH_SEVERITY: warning
|
||||
DEFAULT_BRANCH: main
|
||||
FILTER_REGEX_INCLUDE: pnpm-lock.yaml
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
IGNORE_GENERATED_FILES: true
|
||||
IGNORE_GITIGNORED_FILES: true
|
||||
VALIDATE_BASH: true
|
||||
VALIDATE_BASH_EXEC: true
|
||||
# FIXME: temporarily disabled until api-docker.yaml's run script is fixed for shellcheck
|
||||
# VALIDATE_GITHUB_ACTIONS: true
|
||||
VALIDATE_DOCKERFILE_HADOLINT: true
|
||||
VALIDATE_XML: true
|
||||
VALIDATE_YAML: true
|
||||
|
||||
- name: EditorConfig checks
|
||||
uses: super-linter/super-linter/slim@v7
|
||||
env:
|
||||
DEFAULT_BRANCH: main
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
IGNORE_GENERATED_FILES: true
|
||||
IGNORE_GITIGNORED_FILES: true
|
||||
# EditorConfig validation
|
||||
VALIDATE_EDITORCONFIG: true
|
||||
EDITORCONFIG_FILE_NAME: editorconfig-checker.json
|
||||
@ -1,43 +0,0 @@
|
||||
name: Run Unit Test For SDKs
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- sdks/**
|
||||
|
||||
concurrency:
|
||||
group: sdk-tests-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: unit test for Node.js SDK
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
node-version: [16, 18, 20, 22]
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: sdks/nodejs-client
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Use Node.js ${{ matrix.node-version }}
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
cache: ''
|
||||
cache-dependency-path: 'pnpm-lock.yaml'
|
||||
|
||||
- name: Install Dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Test
|
||||
run: pnpm test
|
||||
@ -1,63 +0,0 @@
|
||||
name: Check i18n Files and Create PR
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
check-and-update:
|
||||
if: github.event.pull_request.merged == true
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: web
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 2 # last 2 commits
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check for file changes in i18n/en-US
|
||||
id: check_files
|
||||
run: |
|
||||
recent_commit_sha=$(git rev-parse HEAD)
|
||||
second_recent_commit_sha=$(git rev-parse HEAD~1)
|
||||
changed_files=$(git diff --name-only $recent_commit_sha $second_recent_commit_sha -- 'i18n/en-US/*.ts')
|
||||
echo "Changed files: $changed_files"
|
||||
if [ -n "$changed_files" ]; then
|
||||
echo "FILES_CHANGED=true" >> $GITHUB_ENV
|
||||
else
|
||||
echo "FILES_CHANGED=false" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Install pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- name: Set up Node.js
|
||||
if: env.FILES_CHANGED == 'true'
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 'lts/*'
|
||||
cache: pnpm
|
||||
cache-dependency-path: ./web/package.json
|
||||
|
||||
- name: Install dependencies
|
||||
if: env.FILES_CHANGED == 'true'
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Run npm script
|
||||
if: env.FILES_CHANGED == 'true'
|
||||
run: pnpm run auto-gen-i18n
|
||||
|
||||
- name: Create Pull Request
|
||||
if: env.FILES_CHANGED == 'true'
|
||||
uses: peter-evans/create-pull-request@v6
|
||||
with:
|
||||
commit-message: Update i18n files based on en-US changes
|
||||
title: 'chore: translate i18n files'
|
||||
body: This PR was automatically created to update i18n files based on changes in en-US locale.
|
||||
branch: chore/automated-i18n-updates
|
||||
@ -1,83 +0,0 @@
|
||||
name: Run VDB Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- api/core/rag/datasource/**
|
||||
- docker/**
|
||||
- .github/workflows/vdb-tests.yml
|
||||
- api/uv.lock
|
||||
- api/pyproject.toml
|
||||
|
||||
concurrency:
|
||||
group: vdb-tests-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: VDB Tests
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
python-version:
|
||||
- "3.11"
|
||||
- "3.12"
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup UV and Python
|
||||
uses: ./.github/actions/setup-uv
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
uv-lockfile: api/uv.lock
|
||||
|
||||
- name: Check UV lockfile
|
||||
run: uv lock --project api --check
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --project api --dev
|
||||
|
||||
- name: Set up dotenvs
|
||||
run: |
|
||||
cp docker/.env.example docker/.env
|
||||
cp docker/middleware.env.example docker/middleware.env
|
||||
|
||||
- name: Expose Service Ports
|
||||
run: sh .github/workflows/expose_service_ports.sh
|
||||
|
||||
- name: Set up Vector Store (TiDB)
|
||||
uses: hoverkraft-tech/compose-action@v2.0.2
|
||||
with:
|
||||
compose-file: docker/tidb/docker-compose.yaml
|
||||
services: |
|
||||
tidb
|
||||
tiflash
|
||||
|
||||
- name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch, Couchbase)
|
||||
uses: hoverkraft-tech/compose-action@v2.0.2
|
||||
with:
|
||||
compose-file: |
|
||||
docker/docker-compose.yaml
|
||||
services: |
|
||||
weaviate
|
||||
qdrant
|
||||
couchbase-server
|
||||
etcd
|
||||
minio
|
||||
milvus-standalone
|
||||
pgvecto-rs
|
||||
pgvector
|
||||
chroma
|
||||
elasticsearch
|
||||
|
||||
- name: Check TiDB Ready
|
||||
run: uv run --project api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py
|
||||
|
||||
- name: Test Vector Stores
|
||||
run: uv run --project api bash dev/pytest/pytest_vdb.sh
|
||||
@ -1,55 +0,0 @@
|
||||
name: Web Tests
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- web/**
|
||||
|
||||
concurrency:
|
||||
group: web-tests-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: Web Tests
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: ./web
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Check changed files
|
||||
id: changed-files
|
||||
uses: tj-actions/changed-files@v45
|
||||
with:
|
||||
files: web/**
|
||||
|
||||
- name: Install pnpm
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 10
|
||||
run_install: false
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
cache-dependency-path: ./web/package.json
|
||||
|
||||
- name: Install dependencies
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Run tests
|
||||
if: steps.changed-files.outputs.any_changed == 'true'
|
||||
run: pnpm test
|
||||
@ -0,0 +1,347 @@
|
||||
AWSTemplateFormatVersion: 2010-09-09
|
||||
Description: The template used to create an ECS Service.
|
||||
Parameters:
|
||||
CertificateArn:
|
||||
Type: AWS::SSM::Parameter::Value<AWS::CertificateManager::Certificate::ARN>
|
||||
ExecutionRoleArn:
|
||||
Type: AWS::SSM::Parameter::Value<AWS::IAM::Role::ARN>
|
||||
ECSClusterName:
|
||||
Type: String
|
||||
ApplicationName:
|
||||
Type: String
|
||||
EnvName:
|
||||
Type: String
|
||||
SecurityGroupIDs:
|
||||
Type: AWS::SSM::Parameter::Value<List<AWS::EC2::SecurityGroup::Id>>
|
||||
SubnetIDs:
|
||||
Type: AWS::SSM::Parameter::Value<List<AWS::EC2::Subnet::Id>>
|
||||
VpcID:
|
||||
Type: AWS::SSM::Parameter::Value<AWS::EC2::VPC::Id>
|
||||
DockerImageUrl:
|
||||
Type: String
|
||||
EnvironmentFileUrl:
|
||||
Type: String
|
||||
Description: The s3 url that contains the list of ecs container environment variables
|
||||
TaskDesiredCount:
|
||||
Type: Number
|
||||
Default: 1
|
||||
TaskCpuSetting:
|
||||
Type: Number
|
||||
Default: 2048
|
||||
TaskMemorySetting:
|
||||
Type: Number
|
||||
Default: 4096
|
||||
AppPort:
|
||||
Type: Number
|
||||
Default: 3000
|
||||
SnsToLarkLambda:
|
||||
Type: AWS::SSM::Parameter::Value<AWS::Lambda::ARN>
|
||||
RedisPass:
|
||||
Type: AWS::SSM::Parameter::Value<String>
|
||||
Default: /ai-dify/REDIS_PASSWORD
|
||||
PgPass:
|
||||
Type: AWS::SSM::Parameter::Value<String>
|
||||
Default: /ai-dify/PG_PASS
|
||||
OpenserchPass:
|
||||
Type: AWS::SSM::Parameter::Value<String>
|
||||
Default: /ai-dify/ES_PASS
|
||||
S3SecretsKey:
|
||||
Type: AWS::SSM::Parameter::Value<String>
|
||||
Default: /ai-dify/S3_KEY
|
||||
Mode:
|
||||
Type: String
|
||||
HealthCheck:
|
||||
Type: String
|
||||
DockerImageName:
|
||||
Type: String
|
||||
EnvironmentFileUrl:
|
||||
Type: String
|
||||
CeleryBrokerUrl:
|
||||
Type: AWS::SSM::Parameter::Value<String>
|
||||
Default: /ai-dify/REDIS_URL
|
||||
CeleryWorkerAmount:
|
||||
Type: Number
|
||||
Default: 5
|
||||
SqlalchemyPoolSize:
|
||||
Type: Number
|
||||
Default: 50
|
||||
ServerWorkerConnections:
|
||||
Type: Number
|
||||
Default: 1000
|
||||
SqlalchemyPoolRecycle:
|
||||
Type: Number
|
||||
Default: 1200
|
||||
ServerWorkerAmount:
|
||||
Type: Number
|
||||
Default: 1
|
||||
Lang:
|
||||
Type: String
|
||||
Default: US.UTF-8
|
||||
LcAll:
|
||||
Type: String
|
||||
Default: US.UTF-8
|
||||
Resources:
|
||||
ECSService:
|
||||
Type: 'AWS::ECS::Service'
|
||||
Properties:
|
||||
Cluster: !Ref ECSClusterName
|
||||
CapacityProviderStrategy:
|
||||
- CapacityProvider: FARGATE
|
||||
Base: 0
|
||||
Weight: 1
|
||||
TaskDefinition: !Ref TaskDefinition
|
||||
ServiceName: !Sub ${ApplicationName}-${EnvName}
|
||||
DesiredCount: !Ref TaskDesiredCount
|
||||
EnableExecuteCommand: true
|
||||
LoadBalancers:
|
||||
- ContainerName: !Sub ${ApplicationName}-${EnvName}
|
||||
ContainerPort: !Ref AppPort
|
||||
TargetGroupArn: !Ref TargetGroup
|
||||
HealthCheckGracePeriodSeconds: '60'
|
||||
NetworkConfiguration:
|
||||
AwsvpcConfiguration:
|
||||
AssignPublicIp: ENABLED
|
||||
SecurityGroups: !Ref SecurityGroupIDs
|
||||
Subnets: !Ref SubnetIDs
|
||||
DeploymentConfiguration:
|
||||
MaximumPercent: 200
|
||||
MinimumHealthyPercent: 100
|
||||
DeploymentController:
|
||||
Type: ECS
|
||||
DependsOn: Listener
|
||||
LoadBalancer:
|
||||
Type: 'AWS::ElasticLoadBalancingV2::LoadBalancer'
|
||||
Properties:
|
||||
Name: !Sub ${ApplicationName}-${EnvName}-alb
|
||||
SecurityGroups: !Ref SecurityGroupIDs
|
||||
Subnets: !Ref SubnetIDs
|
||||
Type: application
|
||||
Scheme: internal
|
||||
TargetGroup:
|
||||
Type: 'AWS::ElasticLoadBalancingV2::TargetGroup'
|
||||
Properties:
|
||||
VpcId: !Ref VpcID
|
||||
Protocol: HTTP
|
||||
Port: '443'
|
||||
HealthCheckPath: !Ref HealthCheck
|
||||
TargetType: ip
|
||||
Listener:
|
||||
Type: 'AWS::ElasticLoadBalancingV2::Listener'
|
||||
Properties:
|
||||
DefaultActions:
|
||||
- TargetGroupArn: !Ref TargetGroup
|
||||
Type: forward
|
||||
LoadBalancerArn: !Ref LoadBalancer
|
||||
Port: '80'
|
||||
Protocol: HTTP
|
||||
TaskDefinition:
|
||||
Type: AWS::ECS::TaskDefinition
|
||||
Properties:
|
||||
Family: !Sub ${ApplicationName}-${EnvName}
|
||||
Cpu: !Ref TaskCpuSetting
|
||||
Memory: !Ref TaskMemorySetting
|
||||
NetworkMode: awsvpc
|
||||
RequiresCompatibilities:
|
||||
- FARGATE
|
||||
ExecutionRoleArn: !Ref ExecutionRoleArn
|
||||
TaskRoleArn: !Ref ExecutionRoleArn
|
||||
ContainerDefinitions:
|
||||
- Name: !Sub ${ApplicationName}-${EnvName}
|
||||
Image: !Ref DockerImageUrl
|
||||
Environment:
|
||||
- Name: REDIS_PASSWORD
|
||||
Value: !Ref RedisPass
|
||||
- Name: DB_PASSWORD
|
||||
Value: !Ref PgPass
|
||||
- Name: OPENSEARCH_PASSWORD
|
||||
Value: !Ref OpenserchPass
|
||||
- Name: S3_SECRET_KEY
|
||||
Value: !Ref S3SecretsKey
|
||||
- Name: MODE
|
||||
Value: !Ref Mode
|
||||
- Name: CELERY_BROKER_URL
|
||||
Value: !Ref CeleryBrokerUrl
|
||||
- Name: CELERY_WORKER_AMOUNT
|
||||
Value: !Ref CeleryWorkerAmount
|
||||
- Name: SQLALCHEMY_POOL_SIZE
|
||||
Value: !Ref SqlalchemyPoolSize
|
||||
- Name: SERVER_WORKER_AMOUNT
|
||||
Value: !Ref ServerWorkerAmount
|
||||
- Name: SQLALCHEMY_POOL_RECYCLE
|
||||
Value: !Ref SqlalchemyPoolRecycle
|
||||
- Name: SERVER_WORKER_CONNECTIONS
|
||||
Value: !Ref ServerWorkerConnections
|
||||
- Name: LANG
|
||||
Value: !Ref Lang
|
||||
- Name: LC_ALL
|
||||
Value: !Ref LcAll
|
||||
|
||||
EnvironmentFiles:
|
||||
- Type: s3
|
||||
Value: !Ref EnvironmentFileUrl
|
||||
PortMappings:
|
||||
- ContainerPort: !Ref AppPort
|
||||
HostPort: !Ref AppPort
|
||||
LogConfiguration:
|
||||
LogDriver: "awslogs"
|
||||
Options:
|
||||
awslogs-create-group: true
|
||||
awslogs-group: !Sub ${ApplicationName}-${EnvName}
|
||||
awslogs-region: !Ref "AWS::Region"
|
||||
awslogs-stream-prefix: ecs
|
||||
|
||||
TooManyHttpCodeElb5XXCountAlarm:
|
||||
Type: AWS::CloudWatch::Alarm
|
||||
Properties:
|
||||
AlarmName: !Sub "TooMany_HTTPCode_ELB_5XX_Count_Alarm-${EnvName}-${ApplicationName}"
|
||||
AlarmDescription: !Sub "TooMany_HTTPCode_ELB_5XX_Count_Alarm of ${ApplicationName} in environment ${EnvName}"
|
||||
Namespace: AWS/ApplicationELB
|
||||
MetricName: HTTPCode_ELB_5XX_Count
|
||||
Dimensions:
|
||||
- Name: LoadBalancer
|
||||
Value: !GetAtt LoadBalancer.LoadBalancerFullName
|
||||
Statistic: Sum
|
||||
Period: 600
|
||||
EvaluationPeriods: 1
|
||||
Threshold: 1
|
||||
ComparisonOperator: GreaterThanOrEqualToThreshold
|
||||
TreatMissingData: notBreaching
|
||||
AlarmActions:
|
||||
- !Ref LarkTopic
|
||||
OKActions:
|
||||
- !Ref LarkTopic
|
||||
|
||||
TargetResponseTimeTooSlowAlarm:
|
||||
Type: AWS::CloudWatch::Alarm
|
||||
Properties:
|
||||
AlarmName: !Sub "TargetResponseTime-too-slow-${EnvName}-${ApplicationName}"
|
||||
AlarmDescription: !Sub "TargetResponseTime is too slow for ${ApplicationName} in environment ${EnvName}"
|
||||
Namespace: AWS/ApplicationELB
|
||||
MetricName: TargetResponseTime
|
||||
Dimensions:
|
||||
- Name: LoadBalancer
|
||||
Value: !GetAtt LoadBalancer.LoadBalancerFullName
|
||||
Statistic: Maximum
|
||||
Period: 600
|
||||
EvaluationPeriods: 2
|
||||
Threshold: 20
|
||||
ComparisonOperator: GreaterThanOrEqualToThreshold
|
||||
TreatMissingData: notBreaching
|
||||
AlarmActions:
|
||||
- !Ref LarkTopic
|
||||
OKActions:
|
||||
- !Ref LarkTopic
|
||||
|
||||
HealthyHostCountTooLowAlarm:
|
||||
Type: AWS::CloudWatch::Alarm
|
||||
Properties:
|
||||
AlarmName: !Sub "HealthyHostCountTooLow-alarm-${EnvName}-${ApplicationName}"
|
||||
AlarmDescription: !Sub "The HealthyHostCount is too low for ${ApplicationName} in environment ${EnvName}"
|
||||
Namespace: AWS/ApplicationELB
|
||||
MetricName: HealthyHostCount
|
||||
Dimensions:
|
||||
- Name: LoadBalancer
|
||||
Value: !GetAtt LoadBalancer.LoadBalancerFullName
|
||||
- Name: TargetGroup
|
||||
Value: !GetAtt TargetGroup.TargetGroupFullName
|
||||
Statistic: Minimum
|
||||
Period: 180
|
||||
EvaluationPeriods: 1
|
||||
Threshold: !Ref TaskDesiredCount
|
||||
ComparisonOperator: LessThanThreshold
|
||||
TreatMissingData: breaching
|
||||
AlarmActions:
|
||||
- !Ref LarkTopic
|
||||
OKActions:
|
||||
- !Ref LarkTopic
|
||||
|
||||
LarkTopic:
|
||||
Type: AWS::SNS::Topic
|
||||
Properties:
|
||||
TopicName: !Sub ${ApplicationName}-${EnvName}
|
||||
Subscription:
|
||||
- Endpoint: !Ref SnsToLarkLambda
|
||||
Protocol: "LAMBDA"
|
||||
|
||||
|
||||
# ASG setting begin
|
||||
AutoScalingRole:
|
||||
Type: AWS::IAM::Role
|
||||
Properties:
|
||||
AssumeRolePolicyDocument:
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Principal:
|
||||
Service:
|
||||
- ecs-tasks.amazonaws.com
|
||||
Action:
|
||||
- 'sts:AssumeRole'
|
||||
Path: '/'
|
||||
Policies:
|
||||
- PolicyName: allow-ecs-cw-scaling
|
||||
PolicyDocument:
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- ecs:DescribeServices
|
||||
- ecs:UpdateService
|
||||
- cloudwatch:DeleteAlarms
|
||||
- cloudwatch:DescribeAlarms
|
||||
- cloudwatch:PutMetricAlarm
|
||||
Resource: '*'
|
||||
AutoScalingTarget:
|
||||
Type: AWS::ApplicationAutoScaling::ScalableTarget
|
||||
Properties:
|
||||
MinCapacity: !Ref TaskDesiredCount
|
||||
MaxCapacity: 50
|
||||
ResourceId: !Join ["/", [service, !Ref ECSClusterName, !GetAtt ECSService.Name]]
|
||||
ScalableDimension: ecs:service:DesiredCount
|
||||
ServiceNamespace: ecs
|
||||
RoleARN: !GetAtt AutoScalingRole.Arn
|
||||
AutoScalingPolicyForCPU:
|
||||
Type: AWS::ApplicationAutoScaling::ScalingPolicy
|
||||
Properties:
|
||||
PolicyName: ecs-cpu-utilization-ScalingPolicy
|
||||
PolicyType: TargetTrackingScaling
|
||||
ScalingTargetId: !Ref AutoScalingTarget
|
||||
TargetTrackingScalingPolicyConfiguration:
|
||||
PredefinedMetricSpecification:
|
||||
PredefinedMetricType: ECSServiceAverageCPUUtilization
|
||||
ScaleInCooldown: 300
|
||||
ScaleOutCooldown: 120
|
||||
TargetValue: 70.0
|
||||
AutoScalingPolicyForMemory:
|
||||
Type: AWS::ApplicationAutoScaling::ScalingPolicy
|
||||
Properties:
|
||||
PolicyName: ecs-memory-utilization-ScalingPolicy
|
||||
PolicyType: TargetTrackingScaling
|
||||
ScalingTargetId: !Ref AutoScalingTarget
|
||||
TargetTrackingScalingPolicyConfiguration:
|
||||
PredefinedMetricSpecification:
|
||||
PredefinedMetricType: ECSServiceAverageMemoryUtilization
|
||||
ScaleInCooldown: 300
|
||||
ScaleOutCooldown: 120
|
||||
TargetValue: 75.0
|
||||
# ASG setting end
|
||||
|
||||
Outputs:
|
||||
ClusterName:
|
||||
Description: The cluster used to create the service.
|
||||
Value: !Ref ECSClusterName
|
||||
ECSService:
|
||||
Description: The created service.
|
||||
Value: !Ref ECSService
|
||||
LoadBalancer:
|
||||
Description: The created load balancer.
|
||||
Value: !Ref LoadBalancer
|
||||
Listener:
|
||||
Description: The created listener.
|
||||
Value: !Ref Listener
|
||||
TargetGroup:
|
||||
Description: The created target group.
|
||||
Value: !Ref TargetGroup
|
||||
LarkTopic:
|
||||
Description: The created Lark Topic.
|
||||
Value: !Ref LarkTopic
|
||||
@ -0,0 +1,204 @@
|
||||
AWSTemplateFormatVersion: 2010-09-09
|
||||
Description: The template used to create an ECS Service without Load Balancer.
|
||||
Parameters:
|
||||
CertificateArn:
|
||||
Type: AWS::SSM::Parameter::Value<AWS::CertificateManager::Certificate::ARN>
|
||||
ExecutionRoleArn:
|
||||
Type: AWS::SSM::Parameter::Value<AWS::IAM::Role::ARN>
|
||||
ECSClusterName:
|
||||
Type: String
|
||||
ApplicationName:
|
||||
Type: String
|
||||
EnvName:
|
||||
Type: String
|
||||
SecurityGroupIDs:
|
||||
Type: AWS::SSM::Parameter::Value<List<AWS::EC2::SecurityGroup::Id>>
|
||||
SubnetIDs:
|
||||
Type: AWS::SSM::Parameter::Value<List<AWS::EC2::Subnet::Id>>
|
||||
VpcID:
|
||||
Type: AWS::SSM::Parameter::Value<AWS::EC2::VPC::Id>
|
||||
DockerImageUrl:
|
||||
Type: String
|
||||
EnvironmentFileUrl:
|
||||
Type: String
|
||||
Description: The s3 url that contains the list of ecs container environment variables
|
||||
TaskDesiredCount:
|
||||
Type: Number
|
||||
Default: 1
|
||||
TaskCpuSetting:
|
||||
Type: Number
|
||||
Default: 1024
|
||||
TaskMemorySetting:
|
||||
Type: Number
|
||||
Default: 4096
|
||||
AppPort:
|
||||
Type: Number
|
||||
Default: 3000
|
||||
SnsToLarkLambda:
|
||||
Type: AWS::SSM::Parameter::Value<AWS::Lambda::ARN>
|
||||
RedisPass:
|
||||
Type: AWS::SSM::Parameter::Value<String>
|
||||
Default: /ai-dify/REDIS_PASSWORD
|
||||
PgPass:
|
||||
Type: AWS::SSM::Parameter::Value<String>
|
||||
Default: /ai-dify/PG_PASS
|
||||
OpenserchPass:
|
||||
Type: AWS::SSM::Parameter::Value<String>
|
||||
Default: /ai-dify/ES_PASS
|
||||
S3SecretsKey:
|
||||
Type: AWS::SSM::Parameter::Value<String>
|
||||
Default: /ai-dify/S3_KEY
|
||||
Mode:
|
||||
Type: String
|
||||
HealthCheck:
|
||||
Type: String
|
||||
DockerImageName:
|
||||
Type: String
|
||||
CeleryBrokerUrl:
|
||||
Type: AWS::SSM::Parameter::Value<String>
|
||||
Default: /ai-dify/REDIS_URL
|
||||
Resources:
|
||||
ECSService:
|
||||
Type: 'AWS::ECS::Service'
|
||||
Properties:
|
||||
Cluster: !Ref ECSClusterName
|
||||
CapacityProviderStrategy:
|
||||
- CapacityProvider: FARGATE
|
||||
Base: 0
|
||||
Weight: 1
|
||||
TaskDefinition: !Ref TaskDefinition
|
||||
ServiceName: !Sub ${ApplicationName}-${EnvName}
|
||||
DesiredCount: !Ref TaskDesiredCount
|
||||
EnableExecuteCommand: true
|
||||
NetworkConfiguration:
|
||||
AwsvpcConfiguration:
|
||||
AssignPublicIp: ENABLED
|
||||
SecurityGroups: !Ref SecurityGroupIDs
|
||||
Subnets: !Ref SubnetIDs
|
||||
DeploymentConfiguration:
|
||||
MaximumPercent: 200
|
||||
MinimumHealthyPercent: 100
|
||||
DeploymentController:
|
||||
Type: ECS
|
||||
|
||||
TaskDefinition:
|
||||
Type: AWS::ECS::TaskDefinition
|
||||
Properties:
|
||||
Family: !Sub ${ApplicationName}-${EnvName}
|
||||
Cpu: !Ref TaskCpuSetting
|
||||
Memory: !Ref TaskMemorySetting
|
||||
NetworkMode: awsvpc
|
||||
RequiresCompatibilities:
|
||||
- FARGATE
|
||||
ExecutionRoleArn: !Ref ExecutionRoleArn
|
||||
TaskRoleArn: !Ref ExecutionRoleArn
|
||||
ContainerDefinitions:
|
||||
- Name: !Sub ${ApplicationName}-${EnvName}
|
||||
Image: !Ref DockerImageUrl
|
||||
Environment:
|
||||
- Name: REDIS_PASSWORD
|
||||
Value: !Ref RedisPass
|
||||
- Name: DB_PASSWORD
|
||||
Value: !Ref PgPass
|
||||
- Name: OPENSEARCH_PASSWORD
|
||||
Value: !Ref OpenserchPass
|
||||
- Name: S3_SECRET_KEY
|
||||
Value: !Ref S3SecretsKey
|
||||
- Name: MODE
|
||||
Value: !Ref Mode
|
||||
- Name: CELERY_BROKER_URL
|
||||
Value: !Ref CeleryBrokerUrl
|
||||
EnvironmentFiles:
|
||||
- Type: s3
|
||||
Value: !Ref EnvironmentFileUrl
|
||||
PortMappings:
|
||||
- ContainerPort: !Ref AppPort
|
||||
HostPort: !Ref AppPort
|
||||
LogConfiguration:
|
||||
LogDriver: "awslogs"
|
||||
Options:
|
||||
awslogs-create-group: true
|
||||
awslogs-group: !Sub ${ApplicationName}-${EnvName}
|
||||
awslogs-region: !Ref "AWS::Region"
|
||||
awslogs-stream-prefix: ecs
|
||||
|
||||
LarkTopic:
|
||||
Type: AWS::SNS::Topic
|
||||
Properties:
|
||||
TopicName: !Sub ${ApplicationName}-${EnvName}
|
||||
Subscription:
|
||||
- Endpoint: !Ref SnsToLarkLambda
|
||||
Protocol: "LAMBDA"
|
||||
|
||||
# ASG setting begin
|
||||
AutoScalingRole:
|
||||
Type: AWS::IAM::Role
|
||||
Properties:
|
||||
AssumeRolePolicyDocument:
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Principal:
|
||||
Service:
|
||||
- ecs-tasks.amazonaws.com
|
||||
Action:
|
||||
- 'sts:AssumeRole'
|
||||
Path: '/'
|
||||
Policies:
|
||||
- PolicyName: allow-ecs-cw-scaling
|
||||
PolicyDocument:
|
||||
Version: '2012-10-17'
|
||||
Statement:
|
||||
- Effect: Allow
|
||||
Action:
|
||||
- ecs:DescribeServices
|
||||
- ecs:UpdateService
|
||||
- cloudwatch:DeleteAlarms
|
||||
- cloudwatch:DescribeAlarms
|
||||
- cloudwatch:PutMetricAlarm
|
||||
Resource: '*'
|
||||
AutoScalingTarget:
|
||||
Type: AWS::ApplicationAutoScaling::ScalableTarget
|
||||
Properties:
|
||||
MinCapacity: !Ref TaskDesiredCount
|
||||
MaxCapacity: 10
|
||||
ResourceId: !Join ["/", [service, !Ref ECSClusterName, !GetAtt ECSService.Name]]
|
||||
ScalableDimension: ecs:service:DesiredCount
|
||||
ServiceNamespace: ecs
|
||||
RoleARN: !GetAtt AutoScalingRole.Arn
|
||||
AutoScalingPolicyForCPU:
|
||||
Type: AWS::ApplicationAutoScaling::ScalingPolicy
|
||||
Properties:
|
||||
PolicyName: ecs-cpu-utilization-ScalingPolicy
|
||||
PolicyType: TargetTrackingScaling
|
||||
ScalingTargetId: !Ref AutoScalingTarget
|
||||
TargetTrackingScalingPolicyConfiguration:
|
||||
PredefinedMetricSpecification:
|
||||
PredefinedMetricType: ECSServiceAverageCPUUtilization
|
||||
ScaleInCooldown: 300
|
||||
ScaleOutCooldown: 120
|
||||
TargetValue: 70.0
|
||||
AutoScalingPolicyForMemory:
|
||||
Type: AWS::ApplicationAutoScaling::ScalingPolicy
|
||||
Properties:
|
||||
PolicyName: ecs-memory-utilization-ScalingPolicy
|
||||
PolicyType: TargetTrackingScaling
|
||||
ScalingTargetId: !Ref AutoScalingTarget
|
||||
TargetTrackingScalingPolicyConfiguration:
|
||||
PredefinedMetricSpecification:
|
||||
PredefinedMetricType: ECSServiceAverageMemoryUtilization
|
||||
ScaleInCooldown: 300
|
||||
ScaleOutCooldown: 120
|
||||
TargetValue: 75.0
|
||||
# ASG setting end
|
||||
|
||||
Outputs:
|
||||
ClusterName:
|
||||
Description: The cluster used to create the service.
|
||||
Value: !Ref ECSClusterName
|
||||
ECSService:
|
||||
Description: The created service.
|
||||
Value: !Ref ECSService
|
||||
LarkTopic:
|
||||
Description: The created Lark Topic.
|
||||
Value: !Ref LarkTopic
|
||||
Loading…
Reference in New Issue