feat: remove unused codes

api/core/model_runtime/model_providers/__base/tokenizers/gpt2/special_tokens_map.json

@@ -1,23 +0,0 @@
-{
-  "bos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  }
-}

api/core/model_runtime/model_providers/__base/tokenizers/gpt2/tokenizer_config.json

@@ -1,33 +0,0 @@
-{
-  "add_bos_token": false,
-  "add_prefix_space": false,
-  "bos_token": {
-    "__type": "AddedToken",
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "clean_up_tokenization_spaces": true,
-  "eos_token": {
-    "__type": "AddedToken",
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "errors": "replace",
-  "model_max_length": 1024,
-  "pad_token": null,
-  "tokenizer_class": "GPT2Tokenizer",
-  "unk_token": {
-    "__type": "AddedToken",
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  }
-}

api/pyproject.toml

@@ -139,8 +139,6 @@ azure-identity = "1.16.1"
 azure-storage-blob = "12.13.0"
 beautifulsoup4 = "4.12.2"
 boto3 = "1.35.17"
-sagemaker = "2.231.0"
-bs4 = "~0.0.1"
 cachetools = "~5.3.0"
 celery = "~5.3.6"
 chardet = "~5.1.0"
@@ -155,21 +153,17 @@ flask-restful = "~0.3.10"
 Flask-SQLAlchemy = "~3.1.1"
 gevent = "~23.9.1"
 gmpy2 = "~2.2.1"
-google-ai-generativelanguage = "0.6.9"
 google-api-core = "2.18.0"
 google-api-python-client = "2.90.0"
 google-auth = "2.29.0"
 google-auth-httplib2 = "0.2.0"
-google-cloud-aiplatform = "1.49.0"
 google-cloud-storage = "2.16.0"
-google-generativeai = "0.8.1"
 googleapis-common-protos = "1.63.0"
 gunicorn = "~22.0.0"
 httpx = { version = "~0.27.0", extras = ["socks"] }
 jieba = "0.42.1"
 langfuse = "^2.48.0"
 langsmith = "^0.1.77"
-mailchimp-transactional = "~1.0.50"
 markdown = "~3.5.1"
 nltk = "3.8.1"
 numpy = "~1.26.4"
@@ -193,9 +187,7 @@ resend = "~0.7.0"
 scikit-learn = "^1.5.1"
 sentry-sdk = { version = "~1.44.1", extras = ["flask"] }
 sqlalchemy = "~2.0.29"
-tencentcloud-sdk-python-hunyuan = "~3.0.1158"
 tiktoken = "~0.7.0"
-tokenizers = "~0.15.0"
 unstructured = { version = "~0.10.27", extras = ["docx", "epub", "md", "msg", "ppt", "pptx"] }
 werkzeug = "~3.0.1"
 yarl = "~1.9.4"