From 9f637ead3890ad07e6dba64d45424b5820817ae7 Mon Sep 17 00:00:00 2001 From: takatost Date: Thu, 1 Feb 2024 18:11:57 +0800 Subject: [PATCH] bump version to 0.5.3 (#2306) --- api/app.py | 2 +- api/config.py | 2 +- api/constants/languages.py | 1 + api/controllers/console/__init__.py | 4 +- api/controllers/console/admin.py | 2 +- api/controllers/console/app/app.py | 3 +- api/controllers/console/app/site.py | 2 +- api/controllers/console/auth/activate.py | 2 +- api/controllers/console/datasets/file.py | 2 +- api/controllers/console/explore/message.py | 4 +- api/controllers/console/explore/parameter.py | 4 +- .../console/explore/recommended_app.py | 2 +- api/controllers/console/init_validate.py | 1 + api/controllers/console/setup.py | 2 +- api/controllers/console/workspace/account.py | 2 +- api/controllers/console/workspace/members.py | 7 ++-- .../console/workspace/tool_providers.py | 14 +++---- api/controllers/console/wraps.py | 3 +- api/controllers/files/__init__.py | 3 +- api/controllers/files/tool_files.py | 4 +- api/controllers/service_api/app/app.py | 7 ++-- .../service_api/dataset/dataset.py | 2 +- api/controllers/service_api/index.py | 3 +- api/controllers/web/app.py | 7 ++-- api/core/app_runner/app_runner.py | 16 ++++---- api/core/app_runner/assistant_app_runner.py | 7 ++-- api/core/app_runner/basic_app_runner.py | 3 +- api/core/app_runner/generate_task_pipeline.py | 14 +++---- api/core/application_manager.py | 7 ++-- api/core/application_queue_manager.py | 8 ++-- .../agent_tool_callback_handler.py | 3 +- .../loader/unstructured/unstructured_ppt.py | 1 + api/core/embedding/cached_embedding.py | 3 +- api/core/entities/application_entities.py | 5 +-- api/core/features/agent_runner.py | 17 ++++---- api/core/features/assistant_base_runner.py | 41 ++++++++----------- api/core/features/assistant_cot_runner.py | 23 +++++------ api/core/features/assistant_fc_runner.py | 19 ++++----- api/core/file/message_file_parser.py | 4 +- api/core/hosting_configuration.py | 2 +- api/core/indexing_runner.py | 2 +- api/core/model_manager.py | 2 +- .../model_providers/__base/tts_model.py | 5 +-- .../model_providers/bedrock/llm/llm.py | 11 +++-- .../model_providers/chatglm/llm/llm.py | 3 +- .../model_providers/cohere/llm/llm.py | 17 ++++---- .../cohere/text_embedding/text_embedding.py | 5 +-- .../model_providers/minimax/llm/llm.py | 2 +- .../model_providers/ollama/llm/llm.py | 23 +++++------ .../ollama/text_embedding/text_embedding.py | 15 ++++--- .../model_providers/openai/tts/tts.py | 9 ++-- .../model_providers/tongyi/llm/llm.py | 5 +-- .../model_providers/tongyi/tts/tts.py | 11 +++-- .../model_providers/xinference/llm/llm.py | 8 ++-- .../text_embedding/text_embedding.py | 4 +- .../xinference/xinference_helper.py | 2 +- .../model_providers/zhipuai/llm/llm.py | 13 +++--- .../zhipuai/text_embedding/text_embedding.py | 2 +- .../zhipuai/zhipuai_sdk/__init__.py | 19 ++------- .../zhipuai/zhipuai_sdk/_client.py | 14 +++---- .../zhipuai_sdk/api_resource/__init__.py | 2 +- .../api_resource/chat/async_completions.py | 6 +-- .../zhipuai_sdk/api_resource/chat/chat.py | 5 ++- .../api_resource/chat/completions.py | 4 +- .../zhipuai_sdk/api_resource/embeddings.py | 4 +- .../zhipuai/zhipuai_sdk/api_resource/files.py | 6 +-- .../api_resource/fine_tuning/fine_tuning.py | 3 +- .../api_resource/fine_tuning/jobs.py | 13 ++---- .../zhipuai_sdk/api_resource/images.py | 4 +- .../zhipuai/zhipuai_sdk/core/_base_api.py | 1 + .../zhipuai/zhipuai_sdk/core/_base_type.py | 14 +------ .../zhipuai/zhipuai_sdk/core/_files.py | 7 +--- .../zhipuai/zhipuai_sdk/core/_http_client.py | 10 +---- .../zhipuai/zhipuai_sdk/core/_request_opt.py | 8 ++-- .../zhipuai/zhipuai_sdk/core/_response.py | 4 +- .../zhipuai/zhipuai_sdk/core/_sse_client.py | 2 +- .../zhipuai/zhipuai_sdk/core/_utils.py | 2 +- .../zhipuai/zhipuai_sdk/types/embeddings.py | 4 +- .../zhipuai/zhipuai_sdk/types/file_object.py | 2 +- .../types/fine_tuning/fine_tuning_job.py | 4 +- .../fine_tuning/fine_tuning_job_event.py | 4 +- .../zhipuai/zhipuai_sdk/types/image.py | 2 +- api/core/tools/entities/tool_bundle.py | 5 ++- api/core/tools/entities/tool_entities.py | 5 ++- api/core/tools/entities/user_entities.py | 5 ++- api/core/tools/model/tool_model_manager.py | 16 ++++---- api/core/tools/provider/api_tool_provider.py | 11 ++--- api/core/tools/provider/app_tool_provider.py | 11 +++-- api/core/tools/provider/builtin/_positions.py | 3 +- .../provider/builtin/azuredalle/azuredalle.py | 7 ++-- .../builtin/azuredalle/tools/dalle3.py | 8 ++-- .../tools/provider/builtin/chart/chart.py | 5 +-- .../tools/provider/builtin/chart/tools/bar.py | 9 ++-- .../provider/builtin/chart/tools/line.py | 9 ++-- .../tools/provider/builtin/chart/tools/pie.py | 9 ++-- .../tools/provider/builtin/dalle/dalle.py | 7 ++-- .../provider/builtin/dalle/tools/dalle2.py | 8 ++-- .../provider/builtin/dalle/tools/dalle3.py | 8 ++-- .../tools/provider/builtin/gaode/gaode.py | 5 ++- .../builtin/gaode/tools/gaode_weather.py | 5 ++- .../tools/provider/builtin/github/github.py | 2 +- .../github/tools/github_repositories.py | 8 ++-- .../tools/provider/builtin/google/google.py | 6 +-- .../builtin/google/tools/google_search.py | 9 ++-- .../stablediffusion/stablediffusion.py | 7 ++-- .../stablediffusion/tools/stable_diffusion.py | 26 ++++++------ api/core/tools/provider/builtin/time/time.py | 6 +-- .../builtin/time/tools/current_time.py | 6 +-- .../builtin/vectorizer/tools/vectorizer.py | 11 ++--- .../provider/builtin/vectorizer/vectorizer.py | 6 +-- .../builtin/webscraper/tools/webscraper.py | 5 ++- .../provider/builtin/webscraper/webscraper.py | 6 +-- .../wikipedia/tools/wikipedia_search.py | 9 ++-- .../provider/builtin/wikipedia/wikipedia.py | 4 +- .../wolframalpha/tools/wolframalpha.py | 8 ++-- .../builtin/wolframalpha/wolframalpha.py | 8 ++-- .../provider/builtin/yahoo/tools/analytics.py | 12 +++--- .../provider/builtin/yahoo/tools/news.py | 8 ++-- .../provider/builtin/yahoo/tools/ticker.py | 8 ++-- .../tools/provider/builtin/yahoo/yahoo.py | 4 +- .../provider/builtin/youtube/tools/videos.py | 8 ++-- .../tools/provider/builtin/youtube/youtube.py | 4 +- .../tools/provider/builtin_tool_provider.py | 22 +++++----- api/core/tools/provider/tool_provider.py | 13 +++--- api/core/tools/tool/api_tool.py | 10 ++--- api/core/tools/tool/builtin_tool.py | 13 +++--- api/core/tools/tool/dataset_retriever_tool.py | 11 ++--- api/core/tools/tool/tool.py | 14 +++---- api/core/tools/tool_file_manager.py | 24 +++++------ api/core/tools/tool_manager.py | 34 +++++++-------- api/core/tools/utils/configuration.py | 7 ++-- api/core/tools/utils/encoder.py | 4 +- api/core/tools/utils/parser.py | 16 ++++---- api/core/tools/utils/web_reader_tool.py | 13 +++--- api/libs/gmpy2_pkcs10aep_cipher.py | 15 +++---- api/libs/rsa.py | 2 +- ...ef91f18_rename_api_provider_description.py | 3 +- .../053da0c1d756_add_api_tool_privacy.py | 2 +- ...84c228_remove_tool_id_from_model_invoke.py | 2 +- ...9d_add_message_files_into_agent_thought.py | 3 +- ...a5a70d_add_tool_labels_to_agent_thought.py | 3 +- .../3ef9b2b6bee6_add_assistant_app.py | 2 +- .../versions/4823da1d26cf_add_tool_file.py | 2 +- ...fee_change_message_chain_id_to_nullable.py | 2 +- api/migrations/versions/89c7899ca936_.py | 3 +- ...daa_add_tool_conversation_variables_idx.py | 3 +- ...6f3c800_rename_api_provider_credentails.py | 3 +- ...fafbd60eca1_add_message_file_belongs_to.py | 3 +- .../ad472b61a054_add_api_provider_icon.py | 3 +- api/migrations/versions/b24be59fbb04_.py | 3 +- .../c71211c8f604_add_tool_invoke_model_log.py | 2 +- .../de95f5c77138_migration_serpapi_api_key.py | 4 +- .../versions/dfb3b7f477da_add_tool_index.py | 3 +- .../f25003750af4_add_created_updated_at.py | 3 +- api/models/model.py | 2 +- api/models/tools.py | 12 +++--- api/services/account_service.py | 2 +- api/services/audio_service.py | 6 +-- api/services/tools_manage_service.py | 25 +++++------ api/tasks/mail_invite_member_task.py | 3 +- .../model_runtime/bedrock/test_llm.py | 1 + .../model_runtime/bedrock/test_provider.py | 1 + .../model_runtime/cohere/test_llm.py | 3 +- .../model_runtime/ollama/test_llm.py | 9 ++-- .../ollama/test_text_embedding.py | 1 - .../model_runtime/zhipuai/test_llm.py | 4 +- .../tools/test_all_provider.py | 1 + docker/docker-compose.yaml | 6 +-- web/package.json | 2 +- 169 files changed, 547 insertions(+), 626 deletions(-) diff --git a/api/app.py b/api/app.py index caf4e8f459..cb3b226e4c 100644 --- a/api/app.py +++ b/api/app.py @@ -30,7 +30,7 @@ from flask import Flask, Response, request from flask_cors import CORS from libs.passport import PassportService # DO NOT REMOVE BELOW -from models import account, dataset, model, source, task, tool, web, tools +from models import account, dataset, model, source, task, tool, tools, web from services.account_service import AccountService # DO NOT REMOVE ABOVE diff --git a/api/config.py b/api/config.py index 385ea32494..66e4c90a6f 100644 --- a/api/config.py +++ b/api/config.py @@ -87,7 +87,7 @@ class Config: # ------------------------ # General Configurations. # ------------------------ - self.CURRENT_VERSION = "0.5.2" + self.CURRENT_VERSION = "0.5.3" self.COMMIT_SHA = get_env('COMMIT_SHA') self.EDITION = "SELF_HOSTED" self.DEPLOY_ENV = get_env('DEPLOY_ENV') diff --git a/api/constants/languages.py b/api/constants/languages.py index a2c5a8b683..6a6a3cf311 100644 --- a/api/constants/languages.py +++ b/api/constants/languages.py @@ -1,5 +1,6 @@ import json + from models.model import AppModelConfig languages = ['en-US', 'zh-Hans', 'pt-BR', 'es-ES', 'fr-FR', 'de-DE', 'ja-JP', 'ko-KR', 'ru-RU', 'it-IT'] diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py index 9642849732..ecfdc38612 100644 --- a/api/controllers/console/__init__.py +++ b/api/controllers/console/__init__.py @@ -11,11 +11,11 @@ from .app import (advanced_prompt_template, annotation, app, audio, completion, model_config, site, statistic) # Import auth controllers from .auth import activate, data_source_oauth, login, oauth +# Import billing controllers +from .billing import billing # Import datasets controllers from .datasets import data_source, datasets, datasets_document, datasets_segments, file, hit_testing # Import explore controllers from .explore import audio, completion, conversation, installed_app, message, parameter, recommended_app, saved_message # Import workspace controllers from .workspace import account, members, model_providers, models, tool_providers, workspace -# Import billing controllers -from .billing import billing diff --git a/api/controllers/console/admin.py b/api/controllers/console/admin.py index 699605eb13..4b8d27434d 100644 --- a/api/controllers/console/admin.py +++ b/api/controllers/console/admin.py @@ -1,12 +1,12 @@ import os from functools import wraps +from constants.languages import supported_language from controllers.console import api from controllers.console.wraps import only_edition_cloud from extensions.ext_database import db from flask import request from flask_restful import Resource, reqparse -from constants.languages import supported_language from models.model import App, InstalledApp, RecommendedApp from werkzeug.exceptions import NotFound, Unauthorized diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index e81c0dccbc..2aac27af3e 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -3,8 +3,8 @@ import json import logging from datetime import datetime -from constants.model_template import model_templates from constants.languages import demo_model_templates, languages +from constants.model_template import model_templates from controllers.console import api from controllers.console.app.error import AppNotFoundError, ProviderNotInitializeError from controllers.console.setup import setup_required @@ -26,6 +26,7 @@ from models.tools import ApiToolProvider from services.app_model_config_service import AppModelConfigService from werkzeug.exceptions import Forbidden + def _get_app(app_id, tenant_id): app = db.session.query(App).filter(App.id == app_id, App.tenant_id == tenant_id).first() if not app: diff --git a/api/controllers/console/app/site.py b/api/controllers/console/app/site.py index 0dc3ecc1e6..daba012bd9 100644 --- a/api/controllers/console/app/site.py +++ b/api/controllers/console/app/site.py @@ -1,4 +1,5 @@ # -*- coding:utf-8 -*- +from constants.languages import supported_language from controllers.console import api from controllers.console.app import _get_app from controllers.console.setup import setup_required @@ -7,7 +8,6 @@ from extensions.ext_database import db from fields.app_fields import app_site_fields from flask_login import current_user from flask_restful import Resource, marshal_with, reqparse -from constants.languages import supported_language from libs.login import login_required from models.model import Site from werkzeug.exceptions import Forbidden, NotFound diff --git a/api/controllers/console/auth/activate.py b/api/controllers/console/auth/activate.py index 41832189a8..0b3672efc9 100644 --- a/api/controllers/console/auth/activate.py +++ b/api/controllers/console/auth/activate.py @@ -2,12 +2,12 @@ import base64 import secrets from datetime import datetime +from constants.languages import supported_language from controllers.console import api from controllers.console.error import AlreadyActivateError from extensions.ext_database import db from flask_restful import Resource, reqparse from libs.helper import email, str_len, timezone -from constants.languages import supported_language from libs.password import hash_password, valid_password from models.account import AccountStatus, Tenant from services.account_service import RegisterService diff --git a/api/controllers/console/datasets/file.py b/api/controllers/console/datasets/file.py index 476249d0f3..c15b3c0cd4 100644 --- a/api/controllers/console/datasets/file.py +++ b/api/controllers/console/datasets/file.py @@ -9,7 +9,7 @@ from flask import current_app, request from flask_login import current_user from flask_restful import Resource, marshal_with from libs.login import login_required -from services.file_service import FileService, ALLOWED_EXTENSIONS, UNSTRUSTURED_ALLOWED_EXTENSIONS +from services.file_service import ALLOWED_EXTENSIONS, UNSTRUSTURED_ALLOWED_EXTENSIONS, FileService PREVIEW_WORDS_LIMIT = 3000 diff --git a/api/controllers/console/explore/message.py b/api/controllers/console/explore/message.py index 404b855bb2..684ecd8b28 100644 --- a/api/controllers/console/explore/message.py +++ b/api/controllers/console/explore/message.py @@ -17,9 +17,9 @@ from core.model_runtime.errors.invoke import InvokeError from fields.message_fields import message_infinite_scroll_pagination_fields from flask import Response, stream_with_context from flask_login import current_user -from flask_restful import marshal_with, reqparse, fields +from flask_restful import fields, marshal_with, reqparse from flask_restful.inputs import int_range -from libs.helper import uuid_value, TimestampField +from libs.helper import TimestampField, uuid_value from services.completion_service import CompletionService from services.errors.app import MoreLikeThisDisabledError from services.errors.conversation import ConversationNotExistsError diff --git a/api/controllers/console/explore/parameter.py b/api/controllers/console/explore/parameter.py index 6bdfa34095..c073ebad01 100644 --- a/api/controllers/console/explore/parameter.py +++ b/api/controllers/console/explore/parameter.py @@ -3,12 +3,12 @@ import json from controllers.console import api from controllers.console.explore.wraps import InstalledAppResource +from extensions.ext_database import db from flask import current_app from flask_restful import fields, marshal_with -from models.model import InstalledApp, AppModelConfig +from models.model import AppModelConfig, InstalledApp from models.tools import ApiToolProvider -from extensions.ext_database import db class AppParameterApi(InstalledAppResource): """Resource for app variables.""" diff --git a/api/controllers/console/explore/recommended_app.py b/api/controllers/console/explore/recommended_app.py index 2f0fa51d77..4ce8fbfbe9 100644 --- a/api/controllers/console/explore/recommended_app.py +++ b/api/controllers/console/explore/recommended_app.py @@ -1,4 +1,5 @@ # -*- coding:utf-8 -*- +from constants.languages import languages from controllers.console import api from controllers.console.app.error import AppNotFoundError from controllers.console.wraps import account_initialization_required @@ -9,7 +10,6 @@ from libs.login import login_required from models.model import App, InstalledApp, RecommendedApp from services.account_service import TenantService from sqlalchemy import and_ -from constants.languages import languages app_fields = { 'id': fields.String, diff --git a/api/controllers/console/init_validate.py b/api/controllers/console/init_validate.py index ebe8b96d61..d1994a84c9 100644 --- a/api/controllers/console/init_validate.py +++ b/api/controllers/console/init_validate.py @@ -1,4 +1,5 @@ import os + from flask import current_app, session from flask_restful import Resource, reqparse from libs.helper import str_len diff --git a/api/controllers/console/setup.py b/api/controllers/console/setup.py index 040f9f23b7..765161ff9d 100644 --- a/api/controllers/console/setup.py +++ b/api/controllers/console/setup.py @@ -10,7 +10,7 @@ from models.model import DifySetup from services.account_service import AccountService, RegisterService, TenantService from . import api -from .error import AlreadySetupError, NotSetupError, NotInitValidateError +from .error import AlreadySetupError, NotInitValidateError, NotSetupError from .init_validate import get_init_validate_status from .wraps import only_edition_self_hosted diff --git a/api/controllers/console/workspace/account.py b/api/controllers/console/workspace/account.py index ae08c3a982..1f856394e2 100644 --- a/api/controllers/console/workspace/account.py +++ b/api/controllers/console/workspace/account.py @@ -2,6 +2,7 @@ from datetime import datetime import pytz +from constants.languages import supported_language from controllers.console import api from controllers.console.setup import setup_required from controllers.console.workspace.error import (AccountAlreadyInitedError, CurrentPasswordIncorrectError, @@ -12,7 +13,6 @@ from flask import current_app, request from flask_login import current_user from flask_restful import Resource, fields, marshal_with, reqparse from libs.helper import TimestampField, timezone -from constants.languages import supported_language from libs.login import login_required from models.account import AccountIntegrate, InvitationCode from services.account_service import AccountService diff --git a/api/controllers/console/workspace/members.py b/api/controllers/console/workspace/members.py index 1b7d08a879..84be878545 100644 --- a/api/controllers/console/workspace/members.py +++ b/api/controllers/console/workspace/members.py @@ -1,13 +1,12 @@ # -*- coding:utf-8 -*- -from flask import current_app -from flask_login import current_user -from flask_restful import Resource, abort, fields, marshal_with, reqparse - import services from controllers.console import api from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required, cloud_edition_billing_resource_check from extensions.ext_database import db +from flask import current_app +from flask_login import current_user +from flask_restful import Resource, abort, fields, marshal_with, reqparse from libs.helper import TimestampField from libs.login import login_required from models.account import Account diff --git a/api/controllers/console/workspace/tool_providers.py b/api/controllers/console/workspace/tool_providers.py index 1d5fd74314..fb42146eee 100644 --- a/api/controllers/console/workspace/tool_providers.py +++ b/api/controllers/console/workspace/tool_providers.py @@ -1,18 +1,16 @@ +import io import json -from libs.login import login_required -from flask_login import current_user -from flask_restful import Resource, reqparse -from flask import send_file -from werkzeug.exceptions import Forbidden - from controllers.console import api from controllers.console.setup import setup_required from controllers.console.wraps import account_initialization_required - +from flask import send_file +from flask_login import current_user +from flask_restful import Resource, reqparse +from libs.login import login_required from services.tools_manage_service import ToolManageService +from werkzeug.exceptions import Forbidden -import io class ToolProviderListApi(Resource): @setup_required diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index 7e01b26846..7bfb064a23 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -1,10 +1,9 @@ # -*- coding:utf-8 -*- import json from functools import wraps -from flask import request from controllers.console.workspace.error import AccountNotInitializedError -from flask import abort, current_app +from flask import abort, current_app, request from flask_login import current_user from services.feature_service import FeatureService from services.operation_service import OperationService diff --git a/api/controllers/files/__init__.py b/api/controllers/files/__init__.py index 7c3e848b53..c7bc7d26d2 100644 --- a/api/controllers/files/__init__.py +++ b/api/controllers/files/__init__.py @@ -6,5 +6,4 @@ bp = Blueprint('files', __name__) api = ExternalApi(bp) -from . import image_preview -from . import tool_files \ No newline at end of file +from . import image_preview, tool_files diff --git a/api/controllers/files/tool_files.py b/api/controllers/files/tool_files.py index f8b1936fc9..b4a290ec87 100644 --- a/api/controllers/files/tool_files.py +++ b/api/controllers/files/tool_files.py @@ -1,10 +1,10 @@ from controllers.files import api +from core.tools.tool_file_manager import ToolFileManager from flask import Response from flask_restful import Resource, reqparse from libs.exception import BaseHTTPException -from werkzeug.exceptions import NotFound, Forbidden +from werkzeug.exceptions import Forbidden, NotFound -from core.tools.tool_file_manager import ToolFileManager class ToolFilePreviewApi(Resource): def get(self, file_id, extension): diff --git a/api/controllers/service_api/app/app.py b/api/controllers/service_api/app/app.py index aa4323a146..63591f8f49 100644 --- a/api/controllers/service_api/app/app.py +++ b/api/controllers/service_api/app/app.py @@ -1,15 +1,14 @@ # -*- coding:utf-8 -*- +import json + from controllers.service_api import api from controllers.service_api.wraps import AppApiResource +from extensions.ext_database import db from flask import current_app from flask_restful import fields, marshal_with from models.model import App, AppModelConfig from models.tools import ApiToolProvider -import json - -from extensions.ext_database import db - class AppParameterApi(AppApiResource): """Resource for app variables.""" diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py index 6827d47dfc..900a796674 100644 --- a/api/controllers/service_api/dataset/dataset.py +++ b/api/controllers/service_api/dataset/dataset.py @@ -1,4 +1,3 @@ -from models.dataset import Dataset import services.dataset_service from controllers.service_api import api from controllers.service_api.dataset.error import DatasetNameDuplicateError @@ -9,6 +8,7 @@ from fields.dataset_fields import dataset_detail_fields from flask import request from flask_restful import marshal, reqparse from libs.login import current_user +from models.dataset import Dataset from services.dataset_service import DatasetService diff --git a/api/controllers/service_api/index.py b/api/controllers/service_api/index.py index 932388b562..489018cf9b 100644 --- a/api/controllers/service_api/index.py +++ b/api/controllers/service_api/index.py @@ -1,8 +1,7 @@ +from controllers.service_api import api from flask import current_app from flask_restful import Resource -from controllers.service_api import api - class IndexApi(Resource): def get(self): diff --git a/api/controllers/web/app.py b/api/controllers/web/app.py index 913f7fbee1..82a9ad8683 100644 --- a/api/controllers/web/app.py +++ b/api/controllers/web/app.py @@ -1,15 +1,14 @@ # -*- coding:utf-8 -*- +import json + from controllers.web import api from controllers.web.wraps import WebApiResource +from extensions.ext_database import db from flask import current_app from flask_restful import fields, marshal_with from models.model import App, AppModelConfig from models.tools import ApiToolProvider -from extensions.ext_database import db - -import json - class AppParameterApi(WebApiResource): """Resource for app variables.""" diff --git a/api/core/app_runner/app_runner.py b/api/core/app_runner/app_runner.py index c7c5474b2a..d751e301cc 100644 --- a/api/core/app_runner/app_runner.py +++ b/api/core/app_runner/app_runner.py @@ -2,8 +2,13 @@ import time from typing import Generator, List, Optional, Tuple, Union, cast from core.application_queue_manager import ApplicationQueueManager, PublishFrom -from core.entities.application_entities import AppOrchestrationConfigEntity, ModelConfigEntity, \ - PromptTemplateEntity, ExternalDataVariableEntity, ApplicationGenerateEntity, InvokeFrom +from core.entities.application_entities import (ApplicationGenerateEntity, AppOrchestrationConfigEntity, + ExternalDataVariableEntity, InvokeFrom, ModelConfigEntity, + PromptTemplateEntity) +from core.features.annotation_reply import AnnotationReplyFeature +from core.features.external_data_fetch import ExternalDataFetchFeature +from core.features.hosting_moderation import HostingModerationFeature +from core.features.moderation import ModerationFeature from core.file.file_obj import FileObj from core.memory.token_buffer_memory import TokenBufferMemory from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage @@ -11,12 +16,9 @@ from core.model_runtime.entities.message_entities import AssistantPromptMessage, from core.model_runtime.entities.model_entities import ModelPropertyKey from core.model_runtime.errors.invoke import InvokeBadRequestError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel -from core.features.hosting_moderation import HostingModerationFeature -from core.features.moderation import ModerationFeature -from core.features.external_data_fetch import ExternalDataFetchFeature -from core.features.annotation_reply import AnnotationReplyFeature from core.prompt.prompt_transform import PromptTransform -from models.model import App, MessageAnnotation, Message +from models.model import App, Message, MessageAnnotation + class AppRunner: def get_pre_calculate_rest_tokens(self, app_record: App, diff --git a/api/core/app_runner/assistant_app_runner.py b/api/core/app_runner/assistant_app_runner.py index b7af18abf0..d0b9bb872c 100644 --- a/api/core/app_runner/assistant_app_runner.py +++ b/api/core/app_runner/assistant_app_runner.py @@ -3,11 +3,10 @@ import logging from typing import cast from core.app_runner.app_runner import AppRunner +from core.application_queue_manager import ApplicationQueueManager, PublishFrom +from core.entities.application_entities import AgentEntity, ApplicationGenerateEntity, ModelConfigEntity from core.features.assistant_cot_runner import AssistantCotApplicationRunner from core.features.assistant_fc_runner import AssistantFunctionCallApplicationRunner -from core.entities.application_entities import ApplicationGenerateEntity, ModelConfigEntity, \ - AgentEntity -from core.application_queue_manager import ApplicationQueueManager, PublishFrom from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance from core.model_runtime.entities.llm_entities import LLMUsage @@ -16,7 +15,7 @@ from core.model_runtime.model_providers.__base.large_language_model import Large from core.moderation.base import ModerationException from core.tools.entities.tool_entities import ToolRuntimeVariablePool from extensions.ext_database import db -from models.model import Conversation, Message, App, MessageChain, MessageAgentThought +from models.model import App, Conversation, Message, MessageAgentThought, MessageChain from models.tools import ToolConversationVariables logger = logging.getLogger(__name__) diff --git a/api/core/app_runner/basic_app_runner.py b/api/core/app_runner/basic_app_runner.py index 0864786530..ae2b712187 100644 --- a/api/core/app_runner/basic_app_runner.py +++ b/api/core/app_runner/basic_app_runner.py @@ -4,8 +4,7 @@ from typing import Optional from core.app_runner.app_runner import AppRunner from core.application_queue_manager import ApplicationQueueManager, PublishFrom from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler -from core.entities.application_entities import (ApplicationGenerateEntity, DatasetEntity, - InvokeFrom, ModelConfigEntity) +from core.entities.application_entities import ApplicationGenerateEntity, DatasetEntity, InvokeFrom, ModelConfigEntity from core.features.dataset_retrieval import DatasetRetrievalFeature from core.memory.token_buffer_memory import TokenBufferMemory from core.model_manager import ModelInstance diff --git a/api/core/app_runner/generate_task_pipeline.py b/api/core/app_runner/generate_task_pipeline.py index 4bb8b1dd70..964228bf9b 100644 --- a/api/core/app_runner/generate_task_pipeline.py +++ b/api/core/app_runner/generate_task_pipeline.py @@ -6,21 +6,21 @@ from typing import Generator, Optional, Union, cast from core.app_runner.moderation_handler import ModerationRule, OutputModerationHandler from core.application_queue_manager import ApplicationQueueManager, PublishFrom from core.entities.application_entities import ApplicationGenerateEntity, InvokeFrom -from core.entities.queue_entities import (AnnotationReplyEvent, QueueAgentThoughtEvent, QueueErrorEvent, - QueueMessageEndEvent, QueueMessageEvent, QueueMessageReplaceEvent, - QueuePingEvent, QueueRetrieverResourcesEvent, QueueStopEvent, - QueueMessageFileEvent, QueueAgentMessageEvent) -from core.errors.error import ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError +from core.entities.queue_entities import (AnnotationReplyEvent, QueueAgentMessageEvent, QueueAgentThoughtEvent, + QueueErrorEvent, QueueMessageEndEvent, QueueMessageEvent, + QueueMessageFileEvent, QueueMessageReplaceEvent, QueuePingEvent, + QueueRetrieverResourcesEvent, QueueStopEvent) +from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent, PromptMessage, PromptMessageContentType, PromptMessageRole, TextPromptMessageContent) from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel -from core.tools.tool_file_manager import ToolFileManager -from core.tools.tool_manager import ToolManager from core.model_runtime.utils.encoders import jsonable_encoder from core.prompt.prompt_template import PromptTemplateParser +from core.tools.tool_file_manager import ToolFileManager +from core.tools.tool_manager import ToolManager from events.message_event import message_was_created from extensions.ext_database import db from models.model import Conversation, Message, MessageAgentThought, MessageFile diff --git a/api/core/application_manager.py b/api/core/application_manager.py index 86c0bc0d0d..7f07bed3a5 100644 --- a/api/core/application_manager.py +++ b/api/core/application_manager.py @@ -9,11 +9,12 @@ from core.app_runner.basic_app_runner import BasicApplicationRunner from core.app_runner.generate_task_pipeline import GenerateTaskPipeline from core.application_queue_manager import ApplicationQueueManager, ConversationTaskStoppedException, PublishFrom from core.entities.application_entities import (AdvancedChatPromptTemplateEntity, - AdvancedCompletionPromptTemplateEntity, AgentEntity, AgentToolEntity, - ApplicationGenerateEntity, AppOrchestrationConfigEntity, DatasetEntity, + AdvancedCompletionPromptTemplateEntity, AgentEntity, AgentPromptEntity, + AgentToolEntity, ApplicationGenerateEntity, + AppOrchestrationConfigEntity, DatasetEntity, DatasetRetrieveConfigEntity, ExternalDataVariableEntity, FileUploadEntity, InvokeFrom, ModelConfigEntity, PromptTemplateEntity, - SensitiveWordAvoidanceEntity, AgentPromptEntity) + SensitiveWordAvoidanceEntity) from core.entities.model_entities import ModelStatus from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from core.file.file_obj import FileObj diff --git a/api/core/application_queue_manager.py b/api/core/application_queue_manager.py index 605255f3bf..65b52fd1f3 100644 --- a/api/core/application_queue_manager.py +++ b/api/core/application_queue_manager.py @@ -4,10 +4,10 @@ from enum import Enum from typing import Any, Generator from core.entities.application_entities import InvokeFrom -from core.entities.queue_entities import (AnnotationReplyEvent, AppQueueEvent, QueueAgentThoughtEvent, QueueErrorEvent, - QueueMessage, QueueMessageEndEvent, QueueMessageEvent, - QueueMessageReplaceEvent, QueuePingEvent, QueueRetrieverResourcesEvent, - QueueStopEvent, QueueMessageFileEvent, QueueAgentMessageEvent) +from core.entities.queue_entities import (AnnotationReplyEvent, AppQueueEvent, QueueAgentMessageEvent, + QueueAgentThoughtEvent, QueueErrorEvent, QueueMessage, QueueMessageEndEvent, + QueueMessageEvent, QueueMessageFileEvent, QueueMessageReplaceEvent, + QueuePingEvent, QueueRetrieverResourcesEvent, QueueStopEvent) from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk from extensions.ext_redis import redis_client from models.model import MessageAgentThought, MessageFile diff --git a/api/core/callback_handler/agent_tool_callback_handler.py b/api/core/callback_handler/agent_tool_callback_handler.py index c1037035ee..ae77bf6cd1 100644 --- a/api/core/callback_handler/agent_tool_callback_handler.py +++ b/api/core/callback_handler/agent_tool_callback_handler.py @@ -1,9 +1,10 @@ import os from typing import Any, Dict, Optional, Union -from pydantic import BaseModel from langchain.callbacks.base import BaseCallbackHandler from langchain.input import print_text +from pydantic import BaseModel + class DifyAgentCallbackHandler(BaseCallbackHandler, BaseModel): """Callback Handler that prints to std out.""" diff --git a/api/core/data_loader/loader/unstructured/unstructured_ppt.py b/api/core/data_loader/loader/unstructured/unstructured_ppt.py index a0d918cce7..9b1e6b5abf 100644 --- a/api/core/data_loader/loader/unstructured/unstructured_ppt.py +++ b/api/core/data_loader/loader/unstructured/unstructured_ppt.py @@ -1,5 +1,6 @@ import logging from typing import List + from langchain.document_loaders.base import BaseLoader from langchain.schema import Document diff --git a/api/core/embedding/cached_embedding.py b/api/core/embedding/cached_embedding.py index ba25eaa787..185b87b8b6 100644 --- a/api/core/embedding/cached_embedding.py +++ b/api/core/embedding/cached_embedding.py @@ -8,9 +8,8 @@ from core.model_manager import ModelInstance from core.model_runtime.entities.model_entities import ModelPropertyKey from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel from extensions.ext_database import db -from langchain.embeddings.base import Embeddings - from extensions.ext_redis import redis_client +from langchain.embeddings.base import Embeddings from libs import helper from models.dataset import Embedding from sqlalchemy.exc import IntegrityError diff --git a/api/core/entities/application_entities.py b/api/core/entities/application_entities.py index 56172188e4..6883a004e4 100644 --- a/api/core/entities/application_entities.py +++ b/api/core/entities/application_entities.py @@ -1,12 +1,11 @@ from enum import Enum -from typing import Optional, Any, cast, Literal, Union - -from pydantic import BaseModel +from typing import Any, Literal, Optional, Union, cast from core.entities.provider_configuration import ProviderModelBundle from core.file.file_obj import FileObj from core.model_runtime.entities.message_entities import PromptMessageRole from core.model_runtime.entities.model_entities import AIModelEntity +from pydantic import BaseModel class ModelConfigEntity(BaseModel): diff --git a/api/core/features/agent_runner.py b/api/core/features/agent_runner.py index 2a7e373dbf..66d41dace0 100644 --- a/api/core/features/agent_runner.py +++ b/api/core/features/agent_runner.py @@ -1,27 +1,26 @@ import logging -from typing import cast, Optional, List - -from langchain import WikipediaAPIWrapper -from langchain.callbacks.base import BaseCallbackHandler -from langchain.tools import BaseTool, WikipediaQueryRun, Tool -from pydantic import BaseModel, Field +from typing import List, Optional, cast from core.agent.agent.agent_llm_callback import AgentLLMCallback -from core.agent.agent_executor import PlanningStrategy, AgentConfiguration, AgentExecutor +from core.agent.agent_executor import AgentConfiguration, AgentExecutor, PlanningStrategy from core.application_queue_manager import ApplicationQueueManager from core.callback_handler.agent_loop_gather_callback_handler import AgentLoopGatherCallbackHandler from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler from core.callback_handler.std_out_callback_handler import DifyStdOutCallbackHandler -from core.entities.application_entities import ModelConfigEntity, InvokeFrom, \ - AgentEntity, AgentToolEntity, AppOrchestrationConfigEntity +from core.entities.application_entities import (AgentEntity, AgentToolEntity, AppOrchestrationConfigEntity, InvokeFrom, + ModelConfigEntity) from core.memory.token_buffer_memory import TokenBufferMemory from core.model_runtime.entities.model_entities import ModelFeature, ModelType from core.model_runtime.model_providers import model_provider_factory from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.tools.tool.dataset_retriever.dataset_retriever_tool import DatasetRetrieverTool from extensions.ext_database import db +from langchain import WikipediaAPIWrapper +from langchain.callbacks.base import BaseCallbackHandler +from langchain.tools import BaseTool, Tool, WikipediaQueryRun from models.dataset import Dataset from models.model import Message +from pydantic import BaseModel, Field logger = logging.getLogger(__name__) diff --git a/api/core/features/assistant_base_runner.py b/api/core/features/assistant_base_runner.py index e9ee59263c..5538918234 100644 --- a/api/core/features/assistant_base_runner.py +++ b/api/core/features/assistant_base_runner.py @@ -1,37 +1,32 @@ -import logging import json - -from typing import Optional, List, Tuple, Union, cast +import logging from datetime import datetime from mimetypes import guess_extension +from typing import List, Optional, Tuple, Union, cast from core.app_runner.app_runner import AppRunner -from extensions.ext_database import db - -from models.model import MessageAgentThought, Message, MessageFile -from models.tools import ToolConversationVariables - -from core.tools.entities.tool_entities import ToolInvokeMessage, ToolInvokeMessageBinary, \ - ToolRuntimeVariablePool, ToolParameter -from core.tools.tool.tool import Tool -from core.tools.tool_manager import ToolManager -from core.tools.tool_file_manager import ToolFileManager -from core.tools.tool.dataset_retriever_tool import DatasetRetrieverTool -from core.app_runner.app_runner import AppRunner +from core.application_queue_manager import ApplicationQueueManager from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler -from core.entities.application_entities import ModelConfigEntity, AgentEntity, AgentToolEntity -from core.application_queue_manager import ApplicationQueueManager +from core.entities.application_entities import (AgentEntity, AgentToolEntity, ApplicationGenerateEntity, + AppOrchestrationConfigEntity, InvokeFrom, ModelConfigEntity) +from core.file.message_file_parser import FileTransferMethod from core.memory.token_buffer_memory import TokenBufferMemory -from core.entities.application_entities import ModelConfigEntity, \ - AgentEntity, AppOrchestrationConfigEntity, ApplicationGenerateEntity, InvokeFrom -from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool +from core.model_manager import ModelInstance from core.model_runtime.entities.llm_entities import LLMUsage +from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageTool from core.model_runtime.entities.model_entities import ModelFeature -from core.model_runtime.utils.encoders import jsonable_encoder from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel -from core.model_manager import ModelInstance -from core.file.message_file_parser import FileTransferMethod +from core.model_runtime.utils.encoders import jsonable_encoder +from core.tools.entities.tool_entities import (ToolInvokeMessage, ToolInvokeMessageBinary, ToolParameter, + ToolRuntimeVariablePool) +from core.tools.tool.dataset_retriever_tool import DatasetRetrieverTool +from core.tools.tool.tool import Tool +from core.tools.tool_file_manager import ToolFileManager +from core.tools.tool_manager import ToolManager +from extensions.ext_database import db +from models.model import Message, MessageAgentThought, MessageFile +from models.tools import ToolConversationVariables logger = logging.getLogger(__name__) diff --git a/api/core/features/assistant_cot_runner.py b/api/core/features/assistant_cot_runner.py index f07af0e694..c9cb2ba61a 100644 --- a/api/core/features/assistant_cot_runner.py +++ b/api/core/features/assistant_cot_runner.py @@ -1,24 +1,21 @@ import json import logging import re -from typing import Literal, Union, Generator, Dict, List +from typing import Dict, Generator, List, Literal, Union -from core.entities.application_entities import AgentPromptEntity, AgentScratchpadUnit from core.application_queue_manager import PublishFrom -from core.model_runtime.utils.encoders import jsonable_encoder -from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage, \ - UserPromptMessage, SystemPromptMessage, AssistantPromptMessage -from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage, LLMResultChunk, LLMResultChunkDelta -from core.model_manager import ModelInstance - -from core.tools.errors import ToolInvokeError, ToolNotFoundError, \ - ToolNotSupportedError, ToolProviderNotFoundError, ToolParameterValidationError, \ - ToolProviderCredentialValidationError - +from core.entities.application_entities import AgentPromptEntity, AgentScratchpadUnit from core.features.assistant_base_runner import BaseAssistantApplicationRunner - +from core.model_manager import ModelInstance +from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool, + SystemPromptMessage, UserPromptMessage) +from core.model_runtime.utils.encoders import jsonable_encoder +from core.tools.errors import (ToolInvokeError, ToolNotFoundError, ToolNotSupportedError, ToolParameterValidationError, + ToolProviderCredentialValidationError, ToolProviderNotFoundError) from models.model import Conversation, Message + class AssistantCotApplicationRunner(BaseAssistantApplicationRunner): def run(self, conversation: Conversation, message: Message, diff --git a/api/core/features/assistant_fc_runner.py b/api/core/features/assistant_fc_runner.py index b3340906b5..8b42244838 100644 --- a/api/core/features/assistant_fc_runner.py +++ b/api/core/features/assistant_fc_runner.py @@ -1,20 +1,15 @@ import json import logging +from typing import Any, Dict, Generator, List, Tuple, Union -from typing import Union, Generator, Dict, Any, Tuple, List - -from core.model_runtime.entities.message_entities import PromptMessage, UserPromptMessage,\ - SystemPromptMessage, AssistantPromptMessage, ToolPromptMessage, PromptMessageTool -from core.model_runtime.entities.llm_entities import LLMResultChunk, LLMResult, LLMUsage, LLMResultChunkDelta -from core.model_manager import ModelInstance from core.application_queue_manager import PublishFrom - -from core.tools.errors import ToolInvokeError, ToolNotFoundError, \ - ToolNotSupportedError, ToolProviderNotFoundError, ToolParameterValidationError, \ - ToolProviderCredentialValidationError - from core.features.assistant_base_runner import BaseAssistantApplicationRunner - +from core.model_manager import ModelInstance +from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool, + SystemPromptMessage, ToolPromptMessage, UserPromptMessage) +from core.tools.errors import (ToolInvokeError, ToolNotFoundError, ToolNotSupportedError, ToolParameterValidationError, + ToolProviderCredentialValidationError, ToolProviderNotFoundError) from models.model import Conversation, Message, MessageAgentThought logger = logging.getLogger(__name__) diff --git a/api/core/file/message_file_parser.py b/api/core/file/message_file_parser.py index 1f35bb245d..c92f9e6950 100644 --- a/api/core/file/message_file_parser.py +++ b/api/core/file/message_file_parser.py @@ -1,11 +1,11 @@ from typing import Dict, List, Optional, Union import requests -from core.file.file_obj import FileObj, FileTransferMethod, FileType, FileBelongsTo -from services.file_service import IMAGE_EXTENSIONS +from core.file.file_obj import FileBelongsTo, FileObj, FileTransferMethod, FileType from extensions.ext_database import db from models.account import Account from models.model import AppModelConfig, EndUser, MessageFile, UploadFile +from services.file_service import IMAGE_EXTENSIONS class MessageFileParser: diff --git a/api/core/hosting_configuration.py b/api/core/hosting_configuration.py index 995bcb0622..a04eb0a5e8 100644 --- a/api/core/hosting_configuration.py +++ b/api/core/hosting_configuration.py @@ -2,7 +2,7 @@ from typing import Optional from core.entities.provider_entities import QuotaUnit, RestrictModel from core.model_runtime.entities.model_entities import ModelType -from flask import Flask, Config +from flask import Config, Flask from models.provider import ProviderQuotaType from pydantic import BaseModel diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 42c43a6643..2f1cf282f8 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -13,7 +13,7 @@ from core.docstore.dataset_docstore import DatasetDocumentStore from core.errors.error import ProviderTokenNotInitError from core.generator.llm_generator import LLMGenerator from core.index.index import IndexBuilder -from core.model_manager import ModelManager, ModelInstance +from core.model_manager import ModelInstance, ModelManager from core.model_runtime.entities.model_entities import ModelType, PriceType from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel diff --git a/api/core/model_manager.py b/api/core/model_manager.py index 20316510dd..8a622e4f5b 100644 --- a/api/core/model_manager.py +++ b/api/core/model_manager.py @@ -12,8 +12,8 @@ from core.model_runtime.model_providers.__base.large_language_model import Large from core.model_runtime.model_providers.__base.moderation_model import ModerationModel from core.model_runtime.model_providers.__base.rerank_model import RerankModel from core.model_runtime.model_providers.__base.speech2text_model import Speech2TextModel -from core.model_runtime.model_providers.__base.tts_model import TTSModel from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel +from core.model_runtime.model_providers.__base.tts_model import TTSModel from core.provider_manager import ProviderManager diff --git a/api/core/model_runtime/model_providers/__base/tts_model.py b/api/core/model_runtime/model_providers/__base/tts_model.py index f0edcff1a7..ff20cf7b9f 100644 --- a/api/core/model_runtime/model_providers/__base/tts_model.py +++ b/api/core/model_runtime/model_providers/__base/tts_model.py @@ -1,13 +1,12 @@ -import uuid import hashlib import subprocess +import uuid from abc import abstractmethod from typing import Optional +from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelType from core.model_runtime.errors.invoke import InvokeBadRequestError -from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.model_providers.__base.ai_model import AIModel -from core.model_runtime.entities.model_entities import ModelPropertyKey class TTSModel(AIModel): diff --git a/api/core/model_runtime/model_providers/bedrock/llm/llm.py b/api/core/model_runtime/model_providers/bedrock/llm/llm.py index 19c4d25fa6..6a9c695350 100644 --- a/api/core/model_runtime/model_providers/bedrock/llm/llm.py +++ b/api/core/model_runtime/model_providers/bedrock/llm/llm.py @@ -1,15 +1,14 @@ +import json import logging from typing import Generator, List, Optional, Union import boto3 -from botocore.exceptions import ClientError, EndpointConnectionError, NoRegionError, ServiceNotInRegionError, UnknownServiceError from botocore.config import Config -import json - +from botocore.exceptions import (ClientError, EndpointConnectionError, NoRegionError, ServiceNotInRegionError, + UnknownServiceError) from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta - -from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, - PromptMessageTool, SystemPromptMessage, UserPromptMessage) +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool, + SystemPromptMessage, UserPromptMessage) from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError diff --git a/api/core/model_runtime/model_providers/chatglm/llm/llm.py b/api/core/model_runtime/model_providers/chatglm/llm/llm.py index 471898fcf6..6f78f7aa88 100644 --- a/api/core/model_runtime/model_providers/chatglm/llm/llm.py +++ b/api/core/model_runtime/model_providers/chatglm/llm/llm.py @@ -5,7 +5,8 @@ from typing import Generator, List, Optional, cast from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageFunction, - PromptMessageTool, SystemPromptMessage, UserPromptMessage, ToolPromptMessage) + PromptMessageTool, SystemPromptMessage, ToolPromptMessage, + UserPromptMessage) from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError diff --git a/api/core/model_runtime/model_providers/cohere/llm/llm.py b/api/core/model_runtime/model_providers/cohere/llm/llm.py index 7b7687bc99..acff4177c3 100644 --- a/api/core/model_runtime/model_providers/cohere/llm/llm.py +++ b/api/core/model_runtime/model_providers/cohere/llm/llm.py @@ -1,19 +1,18 @@ import logging -from typing import Generator, List, Optional, Union, cast, Tuple +from typing import Generator, List, Optional, Tuple, Union, cast import cohere from cohere.responses import Chat, Generations -from cohere.responses.chat import StreamingChat, StreamTextGeneration, StreamEnd -from cohere.responses.generation import StreamingText, StreamingGenerations - +from cohere.responses.chat import StreamEnd, StreamingChat, StreamTextGeneration +from cohere.responses.generation import StreamingGenerations, StreamingText from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, - PromptMessageContentType, SystemPromptMessage, - TextPromptMessageContent, UserPromptMessage, - PromptMessageTool) + PromptMessageContentType, PromptMessageTool, + SystemPromptMessage, TextPromptMessageContent, + UserPromptMessage) from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, I18nObject, ModelType -from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeError, \ - InvokeRateLimitError, InvokeAuthorizationError, InvokeBadRequestError +from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, + InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel diff --git a/api/core/model_runtime/model_providers/cohere/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/cohere/text_embedding/text_embedding.py index a1c73ab6a0..a239727814 100644 --- a/api/core/model_runtime/model_providers/cohere/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/cohere/text_embedding/text_embedding.py @@ -4,11 +4,10 @@ from typing import Optional, Tuple import cohere import numpy as np from cohere.responses import Tokens - from core.model_runtime.entities.model_entities import PriceType from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult -from core.model_runtime.errors.invoke import InvokeConnectionError, InvokeServerUnavailableError, InvokeRateLimitError, \ - InvokeAuthorizationError, InvokeBadRequestError, InvokeError +from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, + InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel diff --git a/api/core/model_runtime/model_providers/minimax/llm/llm.py b/api/core/model_runtime/model_providers/minimax/llm/llm.py index 86a246c714..2657c85419 100644 --- a/api/core/model_runtime/model_providers/minimax/llm/llm.py +++ b/api/core/model_runtime/model_providers/minimax/llm/llm.py @@ -2,7 +2,7 @@ from typing import Generator, List from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool, - SystemPromptMessage, UserPromptMessage, ToolPromptMessage) + SystemPromptMessage, ToolPromptMessage, UserPromptMessage) from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError diff --git a/api/core/model_runtime/model_providers/ollama/llm/llm.py b/api/core/model_runtime/model_providers/ollama/llm/llm.py index 083e60f35e..27c4be125a 100644 --- a/api/core/model_runtime/model_providers/ollama/llm/llm.py +++ b/api/core/model_runtime/model_providers/ollama/llm/llm.py @@ -2,21 +2,20 @@ import json import logging import re from decimal import Decimal -from typing import Optional, Generator, Union, List, cast +from typing import Generator, List, Optional, Union, cast from urllib.parse import urljoin import requests - -from core.model_runtime.entities.message_entities import PromptMessageTool, PromptMessage, AssistantPromptMessage, \ - UserPromptMessage, PromptMessageContentType, ImagePromptMessageContent, \ - TextPromptMessageContent, SystemPromptMessage -from core.model_runtime.entities.model_entities import I18nObject, ModelType, \ - PriceConfig, AIModelEntity, FetchFrom, ModelPropertyKey, ParameterRule, ParameterType, DefaultParameterName, \ - ModelFeature -from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, \ - LLMResultChunk, LLMResultChunkDelta -from core.model_runtime.errors.invoke import InvokeError, InvokeAuthorizationError, InvokeBadRequestError, \ - InvokeRateLimitError, InvokeServerUnavailableError, InvokeConnectionError +from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent, + PromptMessage, PromptMessageContentType, PromptMessageTool, + SystemPromptMessage, TextPromptMessageContent, + UserPromptMessage) +from core.model_runtime.entities.model_entities import (AIModelEntity, DefaultParameterName, FetchFrom, I18nObject, + ModelFeature, ModelPropertyKey, ModelType, ParameterRule, + ParameterType, PriceConfig) +from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, + InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel diff --git a/api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py index f496898180..5d96ac65ff 100644 --- a/api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py @@ -1,19 +1,18 @@ +import json import logging import time from decimal import Decimal from typing import Optional from urllib.parse import urljoin -import requests -import json import numpy as np - +import requests from core.model_runtime.entities.common_entities import I18nObject -from core.model_runtime.entities.model_entities import PriceType, ModelPropertyKey, ModelType, AIModelEntity, FetchFrom, \ - PriceConfig -from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult, EmbeddingUsage -from core.model_runtime.errors.invoke import InvokeError, InvokeAuthorizationError, InvokeBadRequestError, \ - InvokeRateLimitError, InvokeServerUnavailableError, InvokeConnectionError +from core.model_runtime.entities.model_entities import (AIModelEntity, FetchFrom, ModelPropertyKey, ModelType, + PriceConfig, PriceType) +from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult +from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, + InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel diff --git a/api/core/model_runtime/model_providers/openai/tts/tts.py b/api/core/model_runtime/model_providers/openai/tts/tts.py index 013392eb3a..95a88e9bec 100644 --- a/api/core/model_runtime/model_providers/openai/tts/tts.py +++ b/api/core/model_runtime/model_providers/openai/tts/tts.py @@ -1,16 +1,15 @@ +import concurrent.futures +from functools import reduce from io import BytesIO from typing import Optional -from functools import reduce -from pydub import AudioSegment -from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.errors.invoke import InvokeBadRequestError +from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.tts_model import TTSModel from core.model_runtime.model_providers.openai._common import _CommonOpenAI - from flask import Response, stream_with_context from openai import OpenAI -import concurrent.futures +from pydub import AudioSegment class OpenAIText2SpeechModel(_CommonOpenAI, TTSModel): diff --git a/api/core/model_runtime/model_providers/tongyi/llm/llm.py b/api/core/model_runtime/model_providers/tongyi/llm/llm.py index 1e03bc801e..63f300fc19 100644 --- a/api/core/model_runtime/model_providers/tongyi/llm/llm.py +++ b/api/core/model_runtime/model_providers/tongyi/llm/llm.py @@ -1,14 +1,13 @@ from typing import Generator, List, Optional, Union -from dashscope import get_tokenizer - -from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMMode +from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool, SystemPromptMessage, UserPromptMessage) from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel +from dashscope import get_tokenizer from dashscope.api_entities.dashscope_response import DashScopeAPIResponse from dashscope.common.error import (AuthenticationError, InvalidParameter, RequestFailure, ServiceUnavailableError, UnsupportedHTTPMethod, UnsupportedModel) diff --git a/api/core/model_runtime/model_providers/tongyi/tts/tts.py b/api/core/model_runtime/model_providers/tongyi/tts/tts.py index 1dd307e28f..a6fc201080 100644 --- a/api/core/model_runtime/model_providers/tongyi/tts/tts.py +++ b/api/core/model_runtime/model_providers/tongyi/tts/tts.py @@ -1,16 +1,15 @@ +import concurrent.futures +from functools import reduce from io import BytesIO from typing import Optional -from functools import reduce -from pydub import AudioSegment -from core.model_runtime.errors.validate import CredentialsValidateFailedError +import dashscope from core.model_runtime.errors.invoke import InvokeBadRequestError +from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.tts_model import TTSModel from core.model_runtime.model_providers.tongyi._common import _CommonTongyi - -import dashscope from flask import Response, stream_with_context -import concurrent.futures +from pydub import AudioSegment class TongyiText2SpeechModel(_CommonTongyi, TTSModel): diff --git a/api/core/model_runtime/model_providers/xinference/llm/llm.py b/api/core/model_runtime/model_providers/xinference/llm/llm.py index 4f5a3b1604..58801229d2 100644 --- a/api/core/model_runtime/model_providers/xinference/llm/llm.py +++ b/api/core/model_runtime/model_providers/xinference/llm/llm.py @@ -3,15 +3,15 @@ from typing import Generator, Iterator, List, Optional, Union, cast from core.model_runtime.entities.common_entities import I18nObject from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageTool, - SystemPromptMessage, UserPromptMessage, ToolPromptMessage) -from core.model_runtime.entities.model_entities import (AIModelEntity, FetchFrom, ModelPropertyKey, ModelType, - ParameterRule, ParameterType, ModelFeature) + SystemPromptMessage, ToolPromptMessage, UserPromptMessage) +from core.model_runtime.entities.model_entities import (AIModelEntity, FetchFrom, ModelFeature, ModelPropertyKey, + ModelType, ParameterRule, ParameterType) from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.model_runtime.model_providers.xinference.xinference_helper import (XinferenceHelper, - XinferenceModelExtraParameter) + XinferenceModelExtraParameter) from core.model_runtime.utils import helper from openai import (APIConnectionError, APITimeoutError, AuthenticationError, ConflictError, InternalServerError, NotFoundError, OpenAI, PermissionDeniedError, RateLimitError, Stream, UnprocessableEntityError) diff --git a/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py index 28389db6a4..02541e310f 100644 --- a/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py @@ -2,15 +2,15 @@ import time from typing import Optional from core.model_runtime.entities.common_entities import I18nObject -from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelType, PriceType, ModelPropertyKey +from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelPropertyKey, ModelType, PriceType from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, InvokeError, InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel +from core.model_runtime.model_providers.xinference.xinference_helper import XinferenceHelper from xinference_client.client.restful.restful_client import Client, RESTfulEmbeddingModelHandle, RESTfulModelHandle -from core.model_runtime.model_providers.xinference.xinference_helper import XinferenceHelper class XinferenceTextEmbeddingModel(TextEmbeddingModel): """ diff --git a/api/core/model_runtime/model_providers/xinference/xinference_helper.py b/api/core/model_runtime/model_providers/xinference/xinference_helper.py index 1b8d1fbfd7..764ffe8b65 100644 --- a/api/core/model_runtime/model_providers/xinference/xinference_helper.py +++ b/api/core/model_runtime/model_providers/xinference/xinference_helper.py @@ -1,7 +1,7 @@ +from os import path from threading import Lock from time import time from typing import List -from os import path from requests import get from requests.adapters import HTTPAdapter diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py index eafae9ab6f..e3180ec177 100644 --- a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py +++ b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py @@ -2,16 +2,19 @@ import json from typing import Any, Dict, Generator, List, Optional, Union from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta -from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessage, PromptMessageRole, - PromptMessageTool, SystemPromptMessage, UserPromptMessage, ToolPromptMessage, - TextPromptMessageContent, ImagePromptMessageContent, PromptMessageContentType) +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent, + PromptMessage, PromptMessageContentType, PromptMessageRole, + PromptMessageTool, SystemPromptMessage, + TextPromptMessageContent, ToolPromptMessage, + UserPromptMessage) from core.model_runtime.errors.validate import CredentialsValidateFailedError -from core.model_runtime.utils import helper from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel from core.model_runtime.model_providers.zhipuai._common import _CommonZhipuaiAI from core.model_runtime.model_providers.zhipuai.zhipuai_sdk._client import ZhipuAI -from core.model_runtime.model_providers.zhipuai.zhipuai_sdk.types.chat.chat_completion_chunk import ChatCompletionChunk from core.model_runtime.model_providers.zhipuai.zhipuai_sdk.types.chat.chat_completion import Completion +from core.model_runtime.model_providers.zhipuai.zhipuai_sdk.types.chat.chat_completion_chunk import ChatCompletionChunk +from core.model_runtime.utils import helper + class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): diff --git a/api/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py index e5ecc85c47..fd39f5a7a9 100644 --- a/api/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py @@ -5,8 +5,8 @@ from core.model_runtime.entities.model_entities import PriceType from core.model_runtime.entities.text_embedding_entities import EmbeddingUsage, TextEmbeddingResult from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.__base.text_embedding_model import TextEmbeddingModel -from core.model_runtime.model_providers.zhipuai.zhipuai_sdk._client import ZhipuAI from core.model_runtime.model_providers.zhipuai._common import _CommonZhipuaiAI +from core.model_runtime.model_providers.zhipuai.zhipuai_sdk._client import ZhipuAI from langchain.schema.language_model import _get_token_ids_default_method diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/__init__.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/__init__.py index 5527c48913..8a687ef47a 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/__init__.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/__init__.py @@ -1,17 +1,6 @@ -from ._client import ZhipuAI - -from .core._errors import ( - ZhipuAIError, - APIStatusError, - APIRequestFailedError, - APIAuthenticationError, - APIReachLimitError, - APIInternalError, - APIServerFlowExceedError, - APIResponseError, - APIResponseValidationError, - APITimeoutError, -) - from .__version__ import __version__ +from ._client import ZhipuAI +from .core._errors import (APIAuthenticationError, APIInternalError, APIReachLimitError, APIRequestFailedError, + APIResponseError, APIResponseValidationError, APIServerFlowExceedError, APIStatusError, + APITimeoutError, ZhipuAIError) diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/_client.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/_client.py index e169c5485c..573f0715c4 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/_client.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/_client.py @@ -1,17 +1,17 @@ from __future__ import annotations -from typing import Union, Mapping +import os +from typing import Mapping, Union +import httpx +from httpx import Timeout from typing_extensions import override +from . import api_resource from .core import _jwt_token +from .core._base_type import NOT_GIVEN, NotGiven from .core._errors import ZhipuAIError -from .core._http_client import HttpClient, ZHIPUAI_DEFAULT_MAX_RETRIES -from .core._base_type import NotGiven, NOT_GIVEN -from . import api_resource -import os -import httpx -from httpx import Timeout +from .core._http_client import ZHIPUAI_DEFAULT_MAX_RETRIES, HttpClient class ZhipuAI(HttpClient): diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/__init__.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/__init__.py index b596702bc2..0a90e21e48 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/__init__.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/__init__.py @@ -1,5 +1,5 @@ from .chat import chat -from .images import Images from .embeddings import Embeddings from .files import Files from .fine_tuning import fine_tuning +from .images import Images diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/async_completions.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/async_completions.py index b926bd0131..16c4b54f1a 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/async_completions.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/async_completions.py @@ -1,14 +1,14 @@ from __future__ import annotations -from typing import Union, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Union import httpx from typing_extensions import Literal from ...core._base_api import BaseAPI -from ...core._base_type import NotGiven, NOT_GIVEN, Headers +from ...core._base_type import NOT_GIVEN, Headers, NotGiven from ...core._http_client import make_user_request_input -from ...types.chat.async_chat_completion import AsyncTaskStatus, AsyncCompletion +from ...types.chat.async_chat_completion import AsyncCompletion, AsyncTaskStatus if TYPE_CHECKING: from ..._client import ZhipuAI diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/chat.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/chat.py index 6a9fef671f..92362fc50a 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/chat.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/chat.py @@ -1,7 +1,8 @@ from typing import TYPE_CHECKING -from .completions import Completions -from .async_completions import AsyncCompletions + from ...core._base_api import BaseAPI +from .async_completions import AsyncCompletions +from .completions import Completions if TYPE_CHECKING: from ..._client import ZhipuAI diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/completions.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/completions.py index bec2755f93..e5bb8cdf68 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/completions.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/chat/completions.py @@ -1,12 +1,12 @@ from __future__ import annotations -from typing import Union, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Union import httpx from typing_extensions import Literal from ...core._base_api import BaseAPI -from ...core._base_type import NotGiven, NOT_GIVEN, Headers +from ...core._base_type import NOT_GIVEN, Headers, NotGiven from ...core._http_client import make_user_request_input from ...core._sse_client import StreamResponse from ...types.chat.chat_completion import Completion diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/embeddings.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/embeddings.py index b12ce95645..d5db469de4 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/embeddings.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/embeddings.py @@ -1,11 +1,11 @@ from __future__ import annotations -from typing import Union, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Union import httpx from ..core._base_api import BaseAPI -from ..core._base_type import NotGiven, NOT_GIVEN, Headers +from ..core._base_type import NOT_GIVEN, Headers, NotGiven from ..core._http_client import make_user_request_input from ..types.embeddings import EmbeddingsResponded diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/files.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/files.py index 5dde40dae0..dfe52fd54c 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/files.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/files.py @@ -5,11 +5,9 @@ from typing import TYPE_CHECKING import httpx from ..core._base_api import BaseAPI -from ..core._base_type import NOT_GIVEN, Body, Query, Headers, NotGiven, FileTypes +from ..core._base_type import NOT_GIVEN, Body, FileTypes, Headers, NotGiven, Query from ..core._files import is_file_content -from ..core._http_client import ( - make_user_request_input, -) +from ..core._http_client import make_user_request_input from ..types.file_object import FileObject, ListOfFileObject if TYPE_CHECKING: diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/fine_tuning.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/fine_tuning.py index b06274c955..dc54a9ca45 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/fine_tuning.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/fine_tuning.py @@ -1,6 +1,7 @@ from typing import TYPE_CHECKING -from .jobs import Jobs + from ...core._base_api import BaseAPI +from .jobs import Jobs if TYPE_CHECKING: from ..._client import ZhipuAI diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/jobs.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/jobs.py index 03b597ddbe..ead6cdae2f 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/jobs.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/fine_tuning/jobs.py @@ -1,20 +1,13 @@ from __future__ import annotations -from typing import Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Optional import httpx from ...core._base_api import BaseAPI from ...core._base_type import NOT_GIVEN, Headers, NotGiven -from ...core._http_client import ( - make_user_request_input, -) -from ...types.fine_tuning import ( - FineTuningJob, - job_create_params, - ListOfFineTuningJob, - FineTuningJobEvent, -) +from ...core._http_client import make_user_request_input +from ...types.fine_tuning import FineTuningJob, FineTuningJobEvent, ListOfFineTuningJob, job_create_params if TYPE_CHECKING: from ..._client import ZhipuAI diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/images.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/images.py index d245fb8abc..65ce5b246f 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/images.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/api_resource/images.py @@ -1,11 +1,11 @@ from __future__ import annotations -from typing import Union, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Union import httpx from ..core._base_api import BaseAPI -from ..core._base_type import NotGiven, NOT_GIVEN, Headers +from ..core._base_type import NOT_GIVEN, Headers, NotGiven from ..core._http_client import make_user_request_input from ..types.image import ImagesResponded diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_api.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_api.py index 21291b2a73..10b46ff8e3 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_api.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_api.py @@ -1,4 +1,5 @@ from __future__ import annotations + from typing import TYPE_CHECKING if TYPE_CHECKING: diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_type.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_type.py index cc613490dd..f3dde8461c 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_type.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_base_type.py @@ -1,20 +1,10 @@ from __future__ import annotations from os import PathLike -from typing import ( - TYPE_CHECKING, - Type, - Union, - Mapping, - TypeVar, IO, Tuple, Sequence, Any, List, -) +from typing import IO, TYPE_CHECKING, Any, List, Mapping, Sequence, Tuple, Type, TypeVar, Union import pydantic -from typing_extensions import ( - Literal, - override, -) - +from typing_extensions import Literal, override Query = Mapping[str, object] Body = object diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_files.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_files.py index 9ae372adce..e41ede128a 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_files.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_files.py @@ -5,12 +5,7 @@ import os from pathlib import Path from typing import Mapping, Sequence -from ._base_type import ( - FileTypes, - HttpxFileTypes, - HttpxRequestFiles, - RequestFiles, -) +from ._base_type import FileTypes, HttpxFileTypes, HttpxRequestFiles, RequestFiles def is_file_content(obj: object) -> bool: diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py index 0720c78e76..5227d20615 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_http_client.py @@ -2,20 +2,14 @@ from __future__ import annotations import inspect -from typing import ( - Any, - Type, - Union, - cast, - Mapping, -) +from typing import Any, Mapping, Type, Union, cast import httpx import pydantic from httpx import URL, Timeout from . import _errors -from ._base_type import NotGiven, ResponseT, Body, Headers, NOT_GIVEN, RequestFiles, Query, Data +from ._base_type import NOT_GIVEN, Body, Data, Headers, NotGiven, Query, RequestFiles, ResponseT from ._errors import APIResponseValidationError, APIStatusError, APITimeoutError from ._files import make_httpx_files from ._request_opt import ClientRequestParam, UserRequestInput diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_request_opt.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_request_opt.py index f2281c5284..3f22731de6 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_request_opt.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_request_opt.py @@ -1,15 +1,13 @@ from __future__ import annotations -from typing import Union, Any, cast +from typing import Any, Union, cast import pydantic.generics from httpx import Timeout from pydantic import ConfigDict -from typing_extensions import ( - Unpack, ClassVar, TypedDict -) +from typing_extensions import ClassVar, TypedDict, Unpack -from ._base_type import Body, NotGiven, Headers, HttpxRequestFiles, Query +from ._base_type import Body, Headers, HttpxRequestFiles, NotGiven, Query from ._utils import remove_notgiven_indict diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_response.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_response.py index 86ce50d9f4..116246e645 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_response.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_response.py @@ -1,11 +1,11 @@ from __future__ import annotations import datetime -from typing import TypeVar, Generic, cast, Any, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast import httpx import pydantic -from typing_extensions import ParamSpec, get_origin, get_args +from typing_extensions import ParamSpec, get_args, get_origin from ._base_type import NoneType from ._sse_client import StreamResponse diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_sse_client.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_sse_client.py index 83b487e871..6efe20edcb 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_sse_client.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_sse_client.py @@ -2,7 +2,7 @@ from __future__ import annotations import json -from typing import Generic, Iterator, TYPE_CHECKING, Mapping +from typing import TYPE_CHECKING, Generic, Iterator, Mapping import httpx diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_utils.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_utils.py index 6193edcbee..78c97af65b 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_utils.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/core/_utils.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Mapping, Iterable, TypeVar +from typing import Iterable, Mapping, TypeVar from ._base_type import NotGiven diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/embeddings.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/embeddings.py index 9f52d296d3..a8737cf8dc 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/embeddings.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/embeddings.py @@ -1,9 +1,11 @@ from __future__ import annotations -from typing import Optional, List +from typing import List, Optional from pydantic import BaseModel + from .chat.chat_completion import CompletionUsage + __all__ = ["Embedding", "EmbeddingsResponded"] diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/file_object.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/file_object.py index 39599786ea..94db046bd6 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/file_object.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/file_object.py @@ -1,4 +1,4 @@ -from typing import Optional, List +from typing import List, Optional from pydantic import BaseModel diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job.py index c41fd5f246..c841e1d756 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job.py @@ -1,7 +1,7 @@ -from typing import List, Union, Optional -from typing_extensions import Literal +from typing import List, Optional, Union from pydantic import BaseModel +from typing_extensions import Literal __all__ = ["FineTuningJob", "Error", "Hyperparameters", "ListOfFineTuningJob" ] diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job_event.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job_event.py index fd2a4138aa..1a70483a7b 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job_event.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/fine_tuning/fine_tuning_job_event.py @@ -1,7 +1,7 @@ -from typing import List, Union, Optional -from typing_extensions import Literal +from typing import List, Optional, Union from pydantic import BaseModel +from typing_extensions import Literal __all__ = ["FineTuningJobEvent", "Metric", "JobEvent"] diff --git a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/image.py b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/image.py index 681942c84d..429a7e25bc 100644 --- a/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/image.py +++ b/api/core/model_runtime/model_providers/zhipuai/zhipuai_sdk/types/image.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Optional, List +from typing import List, Optional from pydantic import BaseModel diff --git a/api/core/tools/entities/tool_bundle.py b/api/core/tools/entities/tool_bundle.py index 6958ccda36..e9d677cbc1 100644 --- a/api/core/tools/entities/tool_bundle.py +++ b/api/core/tools/entities/tool_bundle.py @@ -1,7 +1,8 @@ +from typing import Any, Dict, List, Optional + +from core.tools.entities.tool_entities import ToolParameter, ToolProviderType from pydantic import BaseModel -from typing import Dict, Optional, Any, List -from core.tools.entities.tool_entities import ToolProviderType, ToolParameter class ApiBasedToolBundle(BaseModel): """ diff --git a/api/core/tools/entities/tool_entities.py b/api/core/tools/entities/tool_entities.py index 6afc3e9615..d33c1ba8c3 100644 --- a/api/core/tools/entities/tool_entities.py +++ b/api/core/tools/entities/tool_entities.py @@ -1,8 +1,9 @@ -from pydantic import BaseModel, Field from enum import Enum -from typing import Optional, List, Dict, Any, Union, cast +from typing import Any, Dict, List, Optional, Union, cast from core.tools.entities.common_entities import I18nObject +from pydantic import BaseModel, Field + class ToolProviderType(Enum): """ diff --git a/api/core/tools/entities/user_entities.py b/api/core/tools/entities/user_entities.py index 99d02f32c1..265f856eb4 100644 --- a/api/core/tools/entities/user_entities.py +++ b/api/core/tools/entities/user_entities.py @@ -1,10 +1,11 @@ -from pydantic import BaseModel from enum import Enum -from typing import List, Dict, Optional +from typing import Dict, List, Optional from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_entities import ToolProviderCredentials from core.tools.tool.tool import ToolParameter +from pydantic import BaseModel + class UserToolProvider(BaseModel): class ProviderType(Enum): diff --git a/api/core/tools/model/tool_model_manager.py b/api/core/tools/model/tool_model_manager.py index 0a8e131344..ee17742161 100644 --- a/api/core/tools/model/tool_model_manager.py +++ b/api/core/tools/model/tool_model_manager.py @@ -4,23 +4,21 @@ Therefore, a model manager is needed to list/invoke/validate models. """ -from core.model_runtime.entities.message_entities import PromptMessage +import json +from typing import List, cast + +from core.model_manager import ModelManager from core.model_runtime.entities.llm_entities import LLMResult +from core.model_runtime.entities.message_entities import PromptMessage from core.model_runtime.entities.model_entities import ModelType +from core.model_runtime.errors.invoke import (InvokeAuthorizationError, InvokeBadRequestError, InvokeConnectionError, + InvokeRateLimitError, InvokeServerUnavailableError) from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel, ModelPropertyKey -from core.model_runtime.errors.invoke import InvokeRateLimitError, InvokeBadRequestError, \ - InvokeConnectionError, InvokeAuthorizationError, InvokeServerUnavailableError from core.model_runtime.utils.encoders import jsonable_encoder -from core.model_manager import ModelManager - from core.tools.model.errors import InvokeModelError - from extensions.ext_database import db - from models.tools import ToolModelInvoke -from typing import List, cast -import json class ToolModelManager: @staticmethod diff --git a/api/core/tools/provider/api_tool_provider.py b/api/core/tools/provider/api_tool_provider.py index 6472a34dda..03edb365ce 100644 --- a/api/core/tools/provider/api_tool_provider.py +++ b/api/core/tools/provider/api_tool_provider.py @@ -1,15 +1,16 @@ from typing import Any, Dict, List -from core.tools.entities.tool_entities import ToolProviderType, ApiProviderAuthType, ToolProviderCredentials, ToolCredentialsOption + from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_bundle import ApiBasedToolBundle -from core.tools.tool.tool import Tool -from core.tools.tool.api_tool import ApiTool +from core.tools.entities.tool_entities import (ApiProviderAuthType, ToolCredentialsOption, ToolProviderCredentials, + ToolProviderType) from core.tools.provider.tool_provider import ToolProviderController - +from core.tools.tool.api_tool import ApiTool +from core.tools.tool.tool import Tool from extensions.ext_database import db - from models.tools import ApiToolProvider + class ApiBasedToolProviderController(ToolProviderController): @staticmethod def from_db(db_provider: ApiToolProvider, auth_type: ApiProviderAuthType) -> 'ApiBasedToolProviderController': diff --git a/api/core/tools/provider/app_tool_provider.py b/api/core/tools/provider/app_tool_provider.py index 1b3e391d53..963c2a1b15 100644 --- a/api/core/tools/provider/app_tool_provider.py +++ b/api/core/tools/provider/app_tool_provider.py @@ -1,14 +1,13 @@ +import logging from typing import Any, Dict, List -from core.tools.entities.tool_entities import ToolProviderType, ToolParameter, ToolParameterOption -from core.tools.tool.tool import Tool + from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_entities import ToolParameter, ToolParameterOption, ToolProviderType from core.tools.provider.tool_provider import ToolProviderController - +from core.tools.tool.tool import Tool from extensions.ext_database import db -from models.tools import PublishedAppTool from models.model import App, AppModelConfig - -import logging +from models.tools import PublishedAppTool logger = logging.getLogger(__name__) diff --git a/api/core/tools/provider/builtin/_positions.py b/api/core/tools/provider/builtin/_positions.py index b8b92e3129..232b6684aa 100644 --- a/api/core/tools/provider/builtin/_positions.py +++ b/api/core/tools/provider/builtin/_positions.py @@ -1,6 +1,7 @@ -from core.tools.entities.user_entities import UserToolProvider from typing import List +from core.tools.entities.user_entities import UserToolProvider + position = { 'google': 1, 'wikipedia': 2, diff --git a/api/core/tools/provider/builtin/azuredalle/azuredalle.py b/api/core/tools/provider/builtin/azuredalle/azuredalle.py index e218d09efa..09fe5600a5 100644 --- a/api/core/tools/provider/builtin/azuredalle/azuredalle.py +++ b/api/core/tools/provider/builtin/azuredalle/azuredalle.py @@ -1,8 +1,9 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.provider.builtin.azuredalle.tools.dalle3 import DallE3Tool +from typing import Any, Dict + from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.azuredalle.tools.dalle3 import DallE3Tool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from typing import Any, Dict class AzureDALLEProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: diff --git a/api/core/tools/provider/builtin/azuredalle/tools/dalle3.py b/api/core/tools/provider/builtin/azuredalle/tools/dalle3.py index 2809247957..25fac01268 100644 --- a/api/core/tools/provider/builtin/azuredalle/tools/dalle3.py +++ b/api/core/tools/provider/builtin/azuredalle/tools/dalle3.py @@ -1,12 +1,12 @@ -from typing import Any, Dict, List, Union -from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.tool.builtin_tool import BuiltinTool - from base64 import b64decode from os.path import join +from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool from openai import AzureOpenAI + class DallE3Tool(BuiltinTool): def _invoke(self, user_id: str, diff --git a/api/core/tools/provider/builtin/chart/chart.py b/api/core/tools/provider/builtin/chart/chart.py index 679c4e05cf..77773bf6bb 100644 --- a/api/core/tools/provider/builtin/chart/chart.py +++ b/api/core/tools/provider/builtin/chart/chart.py @@ -1,9 +1,8 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController +import matplotlib.pyplot as plt from core.tools.errors import ToolProviderCredentialValidationError - from core.tools.provider.builtin.chart.tools.line import LinearChartTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -import matplotlib.pyplot as plt # use a business theme plt.style.use('seaborn-v0_8-darkgrid') diff --git a/api/core/tools/provider/builtin/chart/tools/bar.py b/api/core/tools/provider/builtin/chart/tools/bar.py index 0403430357..bfc4936140 100644 --- a/api/core/tools/provider/builtin/chart/tools/bar.py +++ b/api/core/tools/provider/builtin/chart/tools/bar.py @@ -1,10 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage -import matplotlib.pyplot as plt import io - from typing import Any, Dict, List, Union +import matplotlib.pyplot as plt +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + class BarChartTool(BuiltinTool): def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \ -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]: diff --git a/api/core/tools/provider/builtin/chart/tools/line.py b/api/core/tools/provider/builtin/chart/tools/line.py index d77ab2e43d..a2b618315b 100644 --- a/api/core/tools/provider/builtin/chart/tools/line.py +++ b/api/core/tools/provider/builtin/chart/tools/line.py @@ -1,10 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage -import matplotlib.pyplot as plt import io - from typing import Any, Dict, List, Union +import matplotlib.pyplot as plt +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + class LinearChartTool(BuiltinTool): def _invoke(self, user_id: str, diff --git a/api/core/tools/provider/builtin/chart/tools/pie.py b/api/core/tools/provider/builtin/chart/tools/pie.py index 0635b2afb3..47c2bb1976 100644 --- a/api/core/tools/provider/builtin/chart/tools/pie.py +++ b/api/core/tools/provider/builtin/chart/tools/pie.py @@ -1,10 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage -import matplotlib.pyplot as plt import io - from typing import Any, Dict, List, Union +import matplotlib.pyplot as plt +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + + class PieChartTool(BuiltinTool): def _invoke(self, user_id: str, diff --git a/api/core/tools/provider/builtin/dalle/dalle.py b/api/core/tools/provider/builtin/dalle/dalle.py index 26e09aaab6..e5d3159d93 100644 --- a/api/core/tools/provider/builtin/dalle/dalle.py +++ b/api/core/tools/provider/builtin/dalle/dalle.py @@ -1,8 +1,9 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.provider.builtin.dalle.tools.dalle2 import DallE2Tool +from typing import Any, Dict + from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.dalle.tools.dalle2 import DallE2Tool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from typing import Any, Dict class DALLEProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: diff --git a/api/core/tools/provider/builtin/dalle/tools/dalle2.py b/api/core/tools/provider/builtin/dalle/tools/dalle2.py index a542f01796..ef62faf8b8 100644 --- a/api/core/tools/provider/builtin/dalle/tools/dalle2.py +++ b/api/core/tools/provider/builtin/dalle/tools/dalle2.py @@ -1,12 +1,12 @@ -from typing import Any, Dict, List, Union -from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.tool.builtin_tool import BuiltinTool - from base64 import b64decode from os.path import join +from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool from openai import OpenAI + class DallE2Tool(BuiltinTool): def _invoke(self, user_id: str, diff --git a/api/core/tools/provider/builtin/dalle/tools/dalle3.py b/api/core/tools/provider/builtin/dalle/tools/dalle3.py index 6f75719859..43e96d7780 100644 --- a/api/core/tools/provider/builtin/dalle/tools/dalle3.py +++ b/api/core/tools/provider/builtin/dalle/tools/dalle3.py @@ -1,12 +1,12 @@ -from typing import Any, Dict, List, Union -from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.tool.builtin_tool import BuiltinTool - from base64 import b64decode from os.path import join +from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool from openai import OpenAI + class DallE3Tool(BuiltinTool): def _invoke(self, user_id: str, diff --git a/api/core/tools/provider/builtin/gaode/gaode.py b/api/core/tools/provider/builtin/gaode/gaode.py index 50433fbc9e..a89ea6579a 100644 --- a/api/core/tools/provider/builtin/gaode/gaode.py +++ b/api/core/tools/provider/builtin/gaode/gaode.py @@ -1,7 +1,8 @@ -import requests import urllib.parse -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + +import requests from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController class GaodeProvider(BuiltinToolProviderController): diff --git a/api/core/tools/provider/builtin/gaode/tools/gaode_weather.py b/api/core/tools/provider/builtin/gaode/tools/gaode_weather.py index dd50622394..527ee6ba25 100644 --- a/api/core/tools/provider/builtin/gaode/tools/gaode_weather.py +++ b/api/core/tools/provider/builtin/gaode/tools/gaode_weather.py @@ -1,8 +1,9 @@ import json +from typing import Any, Dict, List, Union + import requests -from core.tools.tool.builtin_tool import BuiltinTool from core.tools.entities.tool_entities import ToolInvokeMessage -from typing import Any, Dict, List, Union +from core.tools.tool.builtin_tool import BuiltinTool class GaodeRepositoriesTool(BuiltinTool): diff --git a/api/core/tools/provider/builtin/github/github.py b/api/core/tools/provider/builtin/github/github.py index 8376093bfe..b224a82a4b 100644 --- a/api/core/tools/provider/builtin/github/github.py +++ b/api/core/tools/provider/builtin/github/github.py @@ -1,6 +1,6 @@ import requests -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController class GihubProvider(BuiltinToolProviderController): diff --git a/api/core/tools/provider/builtin/github/tools/github_repositories.py b/api/core/tools/provider/builtin/github/tools/github_repositories.py index cb0099f6e9..10ca5a5cd7 100644 --- a/api/core/tools/provider/builtin/github/tools/github_repositories.py +++ b/api/core/tools/provider/builtin/github/tools/github_repositories.py @@ -1,11 +1,11 @@ import json -import requests from datetime import datetime +from typing import Any, Dict, List, Union from urllib.parse import quote -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage -from typing import Any, Dict, List, Union +import requests +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool class GihubRepositoriesTool(BuiltinTool): diff --git a/api/core/tools/provider/builtin/google/google.py b/api/core/tools/provider/builtin/google/google.py index 09a802757b..0ac34af33d 100644 --- a/api/core/tools/provider/builtin/google/google.py +++ b/api/core/tools/provider/builtin/google/google.py @@ -1,9 +1,9 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.errors import ToolProviderCredentialValidationError +from typing import Any, Dict, List +from core.tools.errors import ToolProviderCredentialValidationError from core.tools.provider.builtin.google.tools.google_search import GoogleSearchTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from typing import Any, Dict, List class GoogleProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: diff --git a/api/core/tools/provider/builtin/google/tools/google_search.py b/api/core/tools/provider/builtin/google/tools/google_search.py index cfc1508481..81ff20fdc1 100644 --- a/api/core/tools/provider/builtin/google/tools/google_search.py +++ b/api/core/tools/provider/builtin/google/tools/google_search.py @@ -1,13 +1,12 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - -from typing import Any, Dict, List, Union - import os import sys +from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool from serpapi import GoogleSearch + class HiddenPrints: """Context manager to hide prints.""" diff --git a/api/core/tools/provider/builtin/stablediffusion/stablediffusion.py b/api/core/tools/provider/builtin/stablediffusion/stablediffusion.py index 2e79df4ff3..f899abf00f 100644 --- a/api/core/tools/provider/builtin/stablediffusion/stablediffusion.py +++ b/api/core/tools/provider/builtin/stablediffusion/stablediffusion.py @@ -1,9 +1,8 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.errors import ToolProviderCredentialValidationError +from typing import Any, Dict +from core.tools.errors import ToolProviderCredentialValidationError from core.tools.provider.builtin.stablediffusion.tools.stable_diffusion import StableDiffusionTool - -from typing import Any, Dict +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController class StableDiffusionProvider(BuiltinToolProviderController): diff --git a/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py b/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py index b9e91698ec..2899235f76 100644 --- a/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py +++ b/api/core/tools/provider/builtin/stablediffusion/tools/stable_diffusion.py @@ -1,19 +1,17 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolParameterOption +import io +import json +from base64 import b64decode, b64encode +from copy import deepcopy +from os.path import join +from typing import Any, Dict, List, Union + from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolParameterOption from core.tools.errors import ToolProviderCredentialValidationError - -from typing import Any, Dict, List, Union -from httpx import post, get -from os.path import join -from base64 import b64decode, b64encode +from core.tools.tool.builtin_tool import BuiltinTool +from httpx import get, post from PIL import Image -import json -import io - -from copy import deepcopy - DRAW_TEXT_OPTIONS = { "prompt": "", "negative_prompt": "", @@ -70,10 +68,10 @@ class StableDiffusionTool(BuiltinTool): base_url = self.runtime.credentials.get('base_url', None) if not base_url: return self.create_text_message('Please input base_url') - + if 'model' in tool_parameters: self.runtime.credentials['model'] = tool_parameters['model'] - + model = self.runtime.credentials.get('model', None) if not model: return self.create_text_message('Please input model') diff --git a/api/core/tools/provider/builtin/time/time.py b/api/core/tools/provider/builtin/time/time.py index e2b25238e5..214f202c91 100644 --- a/api/core/tools/provider/builtin/time/time.py +++ b/api/core/tools/provider/builtin/time/time.py @@ -1,9 +1,9 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.errors import ToolProviderCredentialValidationError +from typing import Any, Dict +from core.tools.errors import ToolProviderCredentialValidationError from core.tools.provider.builtin.time.tools.current_time import CurrentTimeTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from typing import Any, Dict class WikiPediaProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: diff --git a/api/core/tools/provider/builtin/time/tools/current_time.py b/api/core/tools/provider/builtin/time/tools/current_time.py index ae4ece9aa6..07a1f94b31 100644 --- a/api/core/tools/provider/builtin/time/tools/current_time.py +++ b/api/core/tools/provider/builtin/time/tools/current_time.py @@ -1,10 +1,10 @@ +from datetime import datetime, timezone +from typing import Any, Dict, List, Union + from core.tools.entities.tool_entities import ToolInvokeMessage from core.tools.tool.builtin_tool import BuiltinTool - -from typing import Any, Dict, List, Union from pytz import timezone as pytz_timezone -from datetime import datetime, timezone class CurrentTimeTool(BuiltinTool): def _invoke(self, diff --git a/api/core/tools/provider/builtin/vectorizer/tools/vectorizer.py b/api/core/tools/provider/builtin/vectorizer/tools/vectorizer.py index 725c596e23..8051729825 100644 --- a/api/core/tools/provider/builtin/vectorizer/tools/vectorizer.py +++ b/api/core/tools/provider/builtin/vectorizer/tools/vectorizer.py @@ -1,11 +1,12 @@ -from core.tools.tool.builtin_tool import BuiltinTool +from base64 import b64decode +from typing import Any, Dict, List, Union + from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter -from core.tools.provider.builtin.vectorizer.tools.test_data import VECTORIZER_ICON_PNG from core.tools.errors import ToolProviderCredentialValidationError - -from typing import Any, Dict, List, Union +from core.tools.provider.builtin.vectorizer.tools.test_data import VECTORIZER_ICON_PNG +from core.tools.tool.builtin_tool import BuiltinTool from httpx import post -from base64 import b64decode + class VectorizerTool(BuiltinTool): def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \ diff --git a/api/core/tools/provider/builtin/vectorizer/vectorizer.py b/api/core/tools/provider/builtin/vectorizer/vectorizer.py index 6b4edd67b8..3711d2b5f1 100644 --- a/api/core/tools/provider/builtin/vectorizer/vectorizer.py +++ b/api/core/tools/provider/builtin/vectorizer/vectorizer.py @@ -1,9 +1,9 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.errors import ToolProviderCredentialValidationError +from typing import Any, Dict +from core.tools.errors import ToolProviderCredentialValidationError from core.tools.provider.builtin.vectorizer.tools.vectorizer import VectorizerTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from typing import Any, Dict class VectorizerProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: diff --git a/api/core/tools/provider/builtin/webscraper/tools/webscraper.py b/api/core/tools/provider/builtin/webscraper/tools/webscraper.py index 4d5251533b..d3129c37e2 100644 --- a/api/core/tools/provider/builtin/webscraper/tools/webscraper.py +++ b/api/core/tools/provider/builtin/webscraper/tools/webscraper.py @@ -1,8 +1,9 @@ -from core.tools.tool.builtin_tool import BuiltinTool +from typing import Any, Dict, List, Union + from core.tools.entities.tool_entities import ToolInvokeMessage from core.tools.errors import ToolInvokeError +from core.tools.tool.builtin_tool import BuiltinTool -from typing import Any, Dict, List, Union class WebscraperTool(BuiltinTool): def _invoke(self, diff --git a/api/core/tools/provider/builtin/webscraper/webscraper.py b/api/core/tools/provider/builtin/webscraper/webscraper.py index edea0384d8..aef35e02a4 100644 --- a/api/core/tools/provider/builtin/webscraper/webscraper.py +++ b/api/core/tools/provider/builtin/webscraper/webscraper.py @@ -1,9 +1,9 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.errors import ToolProviderCredentialValidationError +from typing import Any, Dict, List +from core.tools.errors import ToolProviderCredentialValidationError from core.tools.provider.builtin.webscraper.tools.webscraper import WebscraperTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from typing import Any, Dict, List class WebscraperProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: diff --git a/api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py b/api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py index 6ecd449131..3cab06fdd5 100644 --- a/api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py +++ b/api/core/tools/provider/builtin/wikipedia/tools/wikipedia_search.py @@ -1,12 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - -from pydantic import BaseModel, Field - from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool from langchain import WikipediaAPIWrapper from langchain.tools import WikipediaQueryRun +from pydantic import BaseModel, Field + class WikipediaInput(BaseModel): query: str = Field(..., description="search query.") diff --git a/api/core/tools/provider/builtin/wikipedia/wikipedia.py b/api/core/tools/provider/builtin/wikipedia/wikipedia.py index d3af7d6224..8d53852255 100644 --- a/api/core/tools/provider/builtin/wikipedia/wikipedia.py +++ b/api/core/tools/provider/builtin/wikipedia/wikipedia.py @@ -1,7 +1,7 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController from core.tools.errors import ToolProviderCredentialValidationError - from core.tools.provider.builtin.wikipedia.tools.wikipedia_search import WikiPediaSearchTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + class WikiPediaProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: dict) -> None: diff --git a/api/core/tools/provider/builtin/wolframalpha/tools/wolframalpha.py b/api/core/tools/provider/builtin/wolframalpha/tools/wolframalpha.py index 40075b3b8e..d915044d89 100644 --- a/api/core/tools/provider/builtin/wolframalpha/tools/wolframalpha.py +++ b/api/core/tools/provider/builtin/wolframalpha/tools/wolframalpha.py @@ -1,11 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.errors import ToolProviderCredentialValidationError, ToolInvokeError - from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.errors import ToolInvokeError, ToolProviderCredentialValidationError +from core.tools.tool.builtin_tool import BuiltinTool from httpx import get + class WolframAlphaTool(BuiltinTool): _base_url = 'https://api.wolframalpha.com/v2/query' diff --git a/api/core/tools/provider/builtin/wolframalpha/wolframalpha.py b/api/core/tools/provider/builtin/wolframalpha/wolframalpha.py index 9293870029..55d827093c 100644 --- a/api/core/tools/provider/builtin/wolframalpha/wolframalpha.py +++ b/api/core/tools/provider/builtin/wolframalpha/wolframalpha.py @@ -1,11 +1,11 @@ +from typing import Any, Dict, List + from core.tools.entities.tool_entities import ToolInvokeMessage, ToolProviderType -from core.tools.tool.tool import Tool -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController from core.tools.errors import ToolProviderCredentialValidationError - from core.tools.provider.builtin.wolframalpha.tools.wolframalpha import WolframAlphaTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController +from core.tools.tool.tool import Tool -from typing import Any, Dict, List class GoogleProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: Dict[str, Any]) -> None: diff --git a/api/core/tools/provider/builtin/yahoo/tools/analytics.py b/api/core/tools/provider/builtin/yahoo/tools/analytics.py index 557ea176fd..85608b5324 100644 --- a/api/core/tools/provider/builtin/yahoo/tools/analytics.py +++ b/api/core/tools/provider/builtin/yahoo/tools/analytics.py @@ -1,12 +1,12 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - -from typing import Any, Dict, List, Union -from requests.exceptions import HTTPError, ReadTimeout from datetime import datetime +from typing import Any, Dict, List, Union -from yfinance import download import pandas as pd +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool +from requests.exceptions import HTTPError, ReadTimeout +from yfinance import download + class YahooFinanceAnalyticsTool(BuiltinTool): def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \ diff --git a/api/core/tools/provider/builtin/yahoo/tools/news.py b/api/core/tools/provider/builtin/yahoo/tools/news.py index 6a8dec99d6..79de87e0f3 100644 --- a/api/core/tools/provider/builtin/yahoo/tools/news.py +++ b/api/core/tools/provider/builtin/yahoo/tools/news.py @@ -1,10 +1,10 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - from typing import Any, Dict, List, Union -from requests.exceptions import HTTPError, ReadTimeout import yfinance +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool +from requests.exceptions import HTTPError, ReadTimeout + class YahooFinanceSearchTickerTool(BuiltinTool): def _invoke(self,user_id: str, tool_parameters: Dict[str, Any]) \ diff --git a/api/core/tools/provider/builtin/yahoo/tools/ticker.py b/api/core/tools/provider/builtin/yahoo/tools/ticker.py index 3843619546..b28cee0374 100644 --- a/api/core/tools/provider/builtin/yahoo/tools/ticker.py +++ b/api/core/tools/provider/builtin/yahoo/tools/ticker.py @@ -1,11 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - from typing import Any, Dict, List, Union -from requests.exceptions import HTTPError, ReadTimeout +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool +from requests.exceptions import HTTPError, ReadTimeout from yfinance import Ticker + class YahooFinanceSearchTickerTool(BuiltinTool): def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \ -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]: diff --git a/api/core/tools/provider/builtin/yahoo/yahoo.py b/api/core/tools/provider/builtin/yahoo/yahoo.py index 4cb5d76b8b..ade33ffb63 100644 --- a/api/core/tools/provider/builtin/yahoo/yahoo.py +++ b/api/core/tools/provider/builtin/yahoo/yahoo.py @@ -1,7 +1,7 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController from core.tools.errors import ToolProviderCredentialValidationError - from core.tools.provider.builtin.yahoo.tools.ticker import YahooFinanceSearchTickerTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + class YahooFinanceProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: dict) -> None: diff --git a/api/core/tools/provider/builtin/youtube/tools/videos.py b/api/core/tools/provider/builtin/youtube/tools/videos.py index 15415f32ce..5922c37dda 100644 --- a/api/core/tools/provider/builtin/youtube/tools/videos.py +++ b/api/core/tools/provider/builtin/youtube/tools/videos.py @@ -1,11 +1,11 @@ -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.entities.tool_entities import ToolInvokeMessage - -from typing import Any, Dict, List, Union from datetime import datetime +from typing import Any, Dict, List, Union +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool from googleapiclient.discovery import build + class YoutubeVideosAnalyticsTool(BuiltinTool): def _invoke(self, user_id: str, tool_parameters: Dict[str, Any]) \ -> Union[ToolInvokeMessage, List[ToolInvokeMessage]]: diff --git a/api/core/tools/provider/builtin/youtube/youtube.py b/api/core/tools/provider/builtin/youtube/youtube.py index cdca1468e6..8cca578c46 100644 --- a/api/core/tools/provider/builtin/youtube/youtube.py +++ b/api/core/tools/provider/builtin/youtube/youtube.py @@ -1,7 +1,7 @@ -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController from core.tools.errors import ToolProviderCredentialValidationError - from core.tools.provider.builtin.youtube.tools.videos import YoutubeVideosAnalyticsTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + class YahooFinanceProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: dict) -> None: diff --git a/api/core/tools/provider/builtin_tool_provider.py b/api/core/tools/provider/builtin_tool_provider.py index 09da8860de..14eefebf49 100644 --- a/api/core/tools/provider/builtin_tool_provider.py +++ b/api/core/tools/provider/builtin_tool_provider.py @@ -1,19 +1,17 @@ +import importlib from abc import abstractmethod -from typing import List, Dict, Any - -from os import path, listdir -from yaml import load, FullLoader +from os import listdir, path +from typing import Any, Dict, List -from core.tools.entities.tool_entities import ToolProviderType, \ - ToolParameter, ToolProviderCredentials -from core.tools.tool.tool import Tool -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.provider.tool_provider import ToolProviderController +from core.tools.entities.tool_entities import ToolParameter, ToolProviderCredentials, ToolProviderType from core.tools.entities.user_entities import UserToolProviderCredentials -from core.tools.errors import ToolNotFoundError, ToolProviderNotFoundError, \ - ToolParameterValidationError, ToolProviderCredentialValidationError +from core.tools.errors import (ToolNotFoundError, ToolParameterValidationError, ToolProviderCredentialValidationError, + ToolProviderNotFoundError) +from core.tools.provider.tool_provider import ToolProviderController +from core.tools.tool.builtin_tool import BuiltinTool +from core.tools.tool.tool import Tool +from yaml import FullLoader, load -import importlib class BuiltinToolProviderController(ToolProviderController): def __init__(self, **data: Any) -> None: diff --git a/api/core/tools/provider/tool_provider.py b/api/core/tools/provider/tool_provider.py index 155f45d016..954727f774 100644 --- a/api/core/tools/provider/tool_provider.py +++ b/api/core/tools/provider/tool_provider.py @@ -1,14 +1,13 @@ from abc import ABC, abstractmethod -from typing import List, Dict, Any, Optional +from typing import Any, Dict, List, Optional +from core.tools.entities.tool_entities import (ToolParameter, ToolProviderCredentials, ToolProviderIdentity, + ToolProviderType) +from core.tools.entities.user_entities import UserToolProviderCredentials +from core.tools.errors import ToolNotFoundError, ToolParameterValidationError, ToolProviderCredentialValidationError +from core.tools.tool.tool import Tool from pydantic import BaseModel -from core.tools.entities.tool_entities import ToolProviderType, \ - ToolProviderIdentity, ToolParameter, ToolProviderCredentials -from core.tools.tool.tool import Tool -from core.tools.entities.user_entities import UserToolProviderCredentials -from core.tools.errors import ToolNotFoundError, \ - ToolParameterValidationError, ToolProviderCredentialValidationError class ToolProviderController(BaseModel, ABC): identity: Optional[ToolProviderIdentity] = None diff --git a/api/core/tools/tool/api_tool.py b/api/core/tools/tool/api_tool.py index 4d1973b1c8..2d8845e98b 100644 --- a/api/core/tools/tool/api_tool.py +++ b/api/core/tools/tool/api_tool.py @@ -1,14 +1,14 @@ -from typing import Any, Dict, List, Union +import json from json import dumps +from typing import Any, Dict, List, Union +import httpx +import requests from core.tools.entities.tool_bundle import ApiBasedToolBundle from core.tools.entities.tool_entities import ToolInvokeMessage -from core.tools.tool.tool import Tool from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.tool.tool import Tool -import httpx -import requests -import json class ApiTool(Tool): api_bundle: ApiBasedToolBundle diff --git a/api/core/tools/tool/builtin_tool.py b/api/core/tools/tool/builtin_tool.py index 11116dbad2..e862d427af 100644 --- a/api/core/tools/tool/builtin_tool.py +++ b/api/core/tools/tool/builtin_tool.py @@ -1,13 +1,12 @@ -from core.tools.tool.tool import Tool -from core.tools.model.tool_model_manager import ToolModelManager -from core.model_runtime.entities.message_entities import PromptMessage +from enum import Enum +from typing import List + from core.model_runtime.entities.llm_entities import LLMResult -from core.model_runtime.entities.message_entities import SystemPromptMessage, UserPromptMessage +from core.model_runtime.entities.message_entities import PromptMessage, SystemPromptMessage, UserPromptMessage +from core.tools.model.tool_model_manager import ToolModelManager +from core.tools.tool.tool import Tool from core.tools.utils.web_reader_tool import get_url -from typing import List -from enum import Enum - _SUMMARY_PROMPT = """You are a professional language researcher, you are interested in the language and you can quickly aimed at the main point of an webpage and reproduce it in your own words but retain the original meaning and keep the key points. diff --git a/api/core/tools/tool/dataset_retriever_tool.py b/api/core/tools/tool/dataset_retriever_tool.py index e092bcdccd..13b9a8497b 100644 --- a/api/core/tools/tool/dataset_retriever_tool.py +++ b/api/core/tools/tool/dataset_retriever_tool.py @@ -1,13 +1,14 @@ from typing import Any, Dict, List, Union -from core.features.dataset_retrieval import DatasetRetrievalFeature -from core.tools.entities.tool_entities import ToolInvokeMessage, ToolParameter, ToolIdentity, ToolDescription -from core.tools.tool.tool import Tool -from core.tools.entities.common_entities import I18nObject + from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler from core.entities.application_entities import DatasetRetrieveConfigEntity, InvokeFrom - +from core.features.dataset_retrieval import DatasetRetrievalFeature +from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_entities import ToolDescription, ToolIdentity, ToolInvokeMessage, ToolParameter +from core.tools.tool.tool import Tool from langchain.tools import BaseTool + class DatasetRetrieverTool(Tool): langchain_tool: BaseTool diff --git a/api/core/tools/tool/tool.py b/api/core/tools/tool/tool.py index fdb25b8635..95a8f8578f 100644 --- a/api/core/tools/tool/tool.py +++ b/api/core/tools/tool/tool.py @@ -1,13 +1,13 @@ -from pydantic import BaseModel - -from typing import List, Dict, Any, Union, Optional -from abc import abstractmethod, ABC +from abc import ABC, abstractmethod from enum import Enum +from typing import Any, Dict, List, Optional, Union -from core.tools.entities.tool_entities import ToolIdentity, ToolInvokeMessage,\ - ToolParameter, ToolDescription, ToolRuntimeVariablePool, ToolRuntimeVariable, ToolRuntimeImageVariable -from core.tools.tool_file_manager import ToolFileManager from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler +from core.tools.entities.tool_entities import (ToolDescription, ToolIdentity, ToolInvokeMessage, ToolParameter, + ToolRuntimeImageVariable, ToolRuntimeVariable, ToolRuntimeVariablePool) +from core.tools.tool_file_manager import ToolFileManager +from pydantic import BaseModel + class Tool(BaseModel, ABC): identity: ToolIdentity = None diff --git a/api/core/tools/tool_file_manager.py b/api/core/tools/tool_file_manager.py index 43d74622ac..c373399606 100644 --- a/api/core/tools/tool_file_manager.py +++ b/api/core/tools/tool_file_manager.py @@ -1,22 +1,19 @@ -import logging -import time -import os -import hmac import base64 import hashlib - -from typing import Union, Tuple, Generator -from uuid import uuid4 +import hmac +import logging +import os +import time from mimetypes import guess_extension, guess_type -from httpx import get - -from flask import current_app - -from models.tools import ToolFile -from models.model import MessageFile +from typing import Generator, Tuple, Union +from uuid import uuid4 from extensions.ext_database import db from extensions.ext_storage import storage +from flask import current_app +from httpx import get +from models.model import MessageFile +from models.tools import ToolFile logger = logging.getLogger(__name__) @@ -194,4 +191,5 @@ class ToolFileManager: # init tool_file_parser from core.file.tool_file_parser import tool_file_manager + tool_file_manager['manager'] = ToolFileManager diff --git a/api/core/tools/tool_manager.py b/api/core/tools/tool_manager.py index 6250f0e5f1..ee1f38a813 100644 --- a/api/core/tools/tool_manager.py +++ b/api/core/tools/tool_manager.py @@ -1,33 +1,29 @@ -from typing import List, Dict, Any, Tuple, Union +import importlib +import json +import logging +import mimetypes from os import listdir, path +from typing import Any, Dict, List, Tuple, Union -from core.tools.entities.tool_entities import ToolInvokeMessage, ApiProviderAuthType, ToolProviderCredentials -from core.tools.provider.tool_provider import ToolProviderController -from core.tools.tool.builtin_tool import BuiltinTool -from core.tools.tool.api_tool import ApiTool -from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -from core.tools.entities.constant import DEFAULT_PROVIDERS +from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler +from core.model_runtime.entities.message_entities import PromptMessage from core.tools.entities.common_entities import I18nObject +from core.tools.entities.constant import DEFAULT_PROVIDERS +from core.tools.entities.tool_entities import ApiProviderAuthType, ToolInvokeMessage, ToolProviderCredentials +from core.tools.entities.user_entities import UserToolProvider from core.tools.errors import ToolProviderNotFoundError from core.tools.provider.api_tool_provider import ApiBasedToolProviderController from core.tools.provider.app_tool_provider import AppBasedToolProviderEntity -from core.tools.entities.user_entities import UserToolProvider +from core.tools.provider.builtin._positions import BuiltinToolProviderSort +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController +from core.tools.provider.tool_provider import ToolProviderController +from core.tools.tool.api_tool import ApiTool +from core.tools.tool.builtin_tool import BuiltinTool from core.tools.utils.configuration import ToolConfiguration from core.tools.utils.encoder import serialize_base_model_dict -from core.tools.provider.builtin._positions import BuiltinToolProviderSort - -from core.model_runtime.entities.message_entities import PromptMessage -from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler - from extensions.ext_database import db - from models.tools import ApiToolProvider, BuiltinToolProvider -import importlib -import logging -import json -import mimetypes - logger = logging.getLogger(__name__) _builtin_providers = {} diff --git a/api/core/tools/utils/configuration.py b/api/core/tools/utils/configuration.py index 47583b7cae..21609ce302 100644 --- a/api/core/tools/utils/configuration.py +++ b/api/core/tools/utils/configuration.py @@ -1,9 +1,10 @@ -from typing import Dict, Any -from pydantic import BaseModel +from typing import Any, Dict +from core.helper import encrypter from core.tools.entities.tool_entities import ToolProviderCredentials from core.tools.provider.tool_provider import ToolProviderController -from core.helper import encrypter +from pydantic import BaseModel + class ToolConfiguration(BaseModel): tenant_id: str diff --git a/api/core/tools/utils/encoder.py b/api/core/tools/utils/encoder.py index eaf9b6bedc..cce50fb0ac 100644 --- a/api/core/tools/utils/encoder.py +++ b/api/core/tools/utils/encoder.py @@ -1,7 +1,9 @@ -from pydantic import BaseModel from enum import Enum from typing import List +from pydantic import BaseModel + + def serialize_base_model_array(l: List[BaseModel]) -> str: class _BaseModel(BaseModel): __root__: List[BaseModel] diff --git a/api/core/tools/utils/parser.py b/api/core/tools/utils/parser.py index ec4f3237ad..48b84d630c 100644 --- a/api/core/tools/utils/parser.py +++ b/api/core/tools/utils/parser.py @@ -1,15 +1,15 @@ -from core.tools.entities.tool_bundle import ApiBasedToolBundle -from core.tools.entities.tool_entities import ToolParameter, ToolParameterOption, ApiProviderSchemaType -from core.tools.entities.common_entities import I18nObject -from core.tools.errors import ToolProviderNotFoundError, ToolNotSupportedError, \ - ToolApiSchemaError - +from json import dumps as json_dumps +from json import loads as json_loads from typing import List, Tuple -from yaml import FullLoader, load -from json import loads as json_loads, dumps as json_dumps +from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_bundle import ApiBasedToolBundle +from core.tools.entities.tool_entities import ApiProviderSchemaType, ToolParameter, ToolParameterOption +from core.tools.errors import ToolApiSchemaError, ToolNotSupportedError, ToolProviderNotFoundError from requests import get +from yaml import FullLoader, load + class ApiBasedToolSchemaParser: @staticmethod diff --git a/api/core/tools/utils/web_reader_tool.py b/api/core/tools/utils/web_reader_tool.py index b4cfc19871..5a769f6f2d 100644 --- a/api/core/tools/utils/web_reader_tool.py +++ b/api/core/tools/utils/web_reader_tool.py @@ -7,10 +7,14 @@ import subprocess import tempfile import unicodedata from contextlib import contextmanager -from typing import Type, Any +from typing import Any, Type import requests -from bs4 import BeautifulSoup, NavigableString, Comment, CData +from bs4 import BeautifulSoup, CData, Comment, NavigableString +from core.chain.llm_chain import LLMChain +from core.data_loader import file_extractor +from core.data_loader.file_extractor import FileExtractor +from core.entities.application_entities import ModelConfigEntity from langchain.chains import RefineDocumentsChain from langchain.chains.summarize import refine_prompts from langchain.schema import Document @@ -20,11 +24,6 @@ from newspaper import Article from pydantic import BaseModel, Field from regex import regex -from core.chain.llm_chain import LLMChain -from core.data_loader import file_extractor -from core.data_loader.file_extractor import FileExtractor -from core.entities.application_entities import ModelConfigEntity - FULL_TEMPLATE = """ TITLE: {title} AUTHORS: {authors} diff --git a/api/libs/gmpy2_pkcs10aep_cipher.py b/api/libs/gmpy2_pkcs10aep_cipher.py index d2557f251a..e4af4309ab 100644 --- a/api/libs/gmpy2_pkcs10aep_cipher.py +++ b/api/libs/gmpy2_pkcs10aep_cipher.py @@ -20,16 +20,17 @@ # SOFTWARE. # =================================================================== -from Crypto.Signature.pss import MGF1 -import Crypto.Hash.SHA1 +from hashlib import sha1 -from Crypto.Util.py3compat import bord, _copy_bytes +import Crypto.Hash.SHA1 import Crypto.Util.number -from Crypto.Util.number import ceil_div, bytes_to_long, long_to_bytes -from Crypto.Util.strxor import strxor -from Crypto import Random -from hashlib import sha1 import gmpy2 +from Crypto import Random +from Crypto.Signature.pss import MGF1 +from Crypto.Util.number import bytes_to_long, ceil_div, long_to_bytes +from Crypto.Util.py3compat import _copy_bytes, bord +from Crypto.Util.strxor import strxor + class PKCS1OAEP_Cipher: """Cipher object for PKCS#1 v1.5 OAEP. diff --git a/api/libs/rsa.py b/api/libs/rsa.py index f45fb44f56..e2ffed8d05 100644 --- a/api/libs/rsa.py +++ b/api/libs/rsa.py @@ -1,13 +1,13 @@ # -*- coding:utf-8 -*- import hashlib +import libs.gmpy2_pkcs10aep_cipher as gmpy2_pkcs10aep_cipher from Crypto.Cipher import AES from Crypto.PublicKey import RSA from Crypto.Random import get_random_bytes from extensions.ext_redis import redis_client from extensions.ext_storage import storage -import libs.gmpy2_pkcs10aep_cipher as gmpy2_pkcs10aep_cipher def generate_key_pair(tenant_id): private_key = RSA.generate(2048) diff --git a/api/migrations/versions/00bacef91f18_rename_api_provider_description.py b/api/migrations/versions/00bacef91f18_rename_api_provider_description.py index a498c90460..5ae9e8769a 100644 --- a/api/migrations/versions/00bacef91f18_rename_api_provider_description.py +++ b/api/migrations/versions/00bacef91f18_rename_api_provider_description.py @@ -5,9 +5,8 @@ Revises: 8ec536f3c800 Create Date: 2024-01-07 04:07:34.482983 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '00bacef91f18' diff --git a/api/migrations/versions/053da0c1d756_add_api_tool_privacy.py b/api/migrations/versions/053da0c1d756_add_api_tool_privacy.py index 8d304ebfb9..a589f1f08b 100644 --- a/api/migrations/versions/053da0c1d756_add_api_tool_privacy.py +++ b/api/migrations/versions/053da0c1d756_add_api_tool_privacy.py @@ -5,8 +5,8 @@ Revises: 4829e54d2fee Create Date: 2024-01-12 06:47:21.656262 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/api/migrations/versions/114eed84c228_remove_tool_id_from_model_invoke.py b/api/migrations/versions/114eed84c228_remove_tool_id_from_model_invoke.py index 1b1d77055a..58863fe3a7 100644 --- a/api/migrations/versions/114eed84c228_remove_tool_id_from_model_invoke.py +++ b/api/migrations/versions/114eed84c228_remove_tool_id_from_model_invoke.py @@ -5,8 +5,8 @@ Revises: c71211c8f604 Create Date: 2024-01-10 04:40:57.257824 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/api/migrations/versions/23db93619b9d_add_message_files_into_agent_thought.py b/api/migrations/versions/23db93619b9d_add_message_files_into_agent_thought.py index 25363ca947..f3eef4681e 100644 --- a/api/migrations/versions/23db93619b9d_add_message_files_into_agent_thought.py +++ b/api/migrations/versions/23db93619b9d_add_message_files_into_agent_thought.py @@ -5,9 +5,8 @@ Revises: 8ae9bc661daa Create Date: 2024-01-18 08:46:37.302657 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '23db93619b9d' diff --git a/api/migrations/versions/380c6aa5a70d_add_tool_labels_to_agent_thought.py b/api/migrations/versions/380c6aa5a70d_add_tool_labels_to_agent_thought.py index 2031dc2ad3..6c13818463 100644 --- a/api/migrations/versions/380c6aa5a70d_add_tool_labels_to_agent_thought.py +++ b/api/migrations/versions/380c6aa5a70d_add_tool_labels_to_agent_thought.py @@ -5,9 +5,8 @@ Revises: dfb3b7f477da Create Date: 2024-01-24 10:58:15.644445 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '380c6aa5a70d' diff --git a/api/migrations/versions/3ef9b2b6bee6_add_assistant_app.py b/api/migrations/versions/3ef9b2b6bee6_add_assistant_app.py index 4a4a497993..4fbc570303 100644 --- a/api/migrations/versions/3ef9b2b6bee6_add_assistant_app.py +++ b/api/migrations/versions/3ef9b2b6bee6_add_assistant_app.py @@ -5,8 +5,8 @@ Revises: 89c7899ca936 Create Date: 2024-01-05 15:26:25.117551 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/api/migrations/versions/4823da1d26cf_add_tool_file.py b/api/migrations/versions/4823da1d26cf_add_tool_file.py index 797a9539b7..1a473a10fe 100644 --- a/api/migrations/versions/4823da1d26cf_add_tool_file.py +++ b/api/migrations/versions/4823da1d26cf_add_tool_file.py @@ -5,8 +5,8 @@ Revises: 053da0c1d756 Create Date: 2024-01-15 11:37:16.782718 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/api/migrations/versions/4829e54d2fee_change_message_chain_id_to_nullable.py b/api/migrations/versions/4829e54d2fee_change_message_chain_id_to_nullable.py index f67a18cb2b..790ade64ec 100644 --- a/api/migrations/versions/4829e54d2fee_change_message_chain_id_to_nullable.py +++ b/api/migrations/versions/4829e54d2fee_change_message_chain_id_to_nullable.py @@ -5,8 +5,8 @@ Revises: 114eed84c228 Create Date: 2024-01-12 03:42:27.362415 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/api/migrations/versions/89c7899ca936_.py b/api/migrations/versions/89c7899ca936_.py index 75cbb5c2b4..0fad39fa57 100644 --- a/api/migrations/versions/89c7899ca936_.py +++ b/api/migrations/versions/89c7899ca936_.py @@ -5,9 +5,8 @@ Revises: 187385f442fc Create Date: 2024-01-21 04:10:23.192853 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '89c7899ca936' diff --git a/api/migrations/versions/8ae9bc661daa_add_tool_conversation_variables_idx.py b/api/migrations/versions/8ae9bc661daa_add_tool_conversation_variables_idx.py index c65372419e..430a8c78c2 100644 --- a/api/migrations/versions/8ae9bc661daa_add_tool_conversation_variables_idx.py +++ b/api/migrations/versions/8ae9bc661daa_add_tool_conversation_variables_idx.py @@ -5,9 +5,8 @@ Revises: 9fafbd60eca1 Create Date: 2024-01-15 14:22:03.597692 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '8ae9bc661daa' diff --git a/api/migrations/versions/8ec536f3c800_rename_api_provider_credentails.py b/api/migrations/versions/8ec536f3c800_rename_api_provider_credentails.py index 6cab1f2092..6cafc198aa 100644 --- a/api/migrations/versions/8ec536f3c800_rename_api_provider_credentails.py +++ b/api/migrations/versions/8ec536f3c800_rename_api_provider_credentails.py @@ -5,9 +5,8 @@ Revises: ad472b61a054 Create Date: 2024-01-07 03:57:35.257545 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '8ec536f3c800' diff --git a/api/migrations/versions/9fafbd60eca1_add_message_file_belongs_to.py b/api/migrations/versions/9fafbd60eca1_add_message_file_belongs_to.py index 367c2e731f..968906bdd7 100644 --- a/api/migrations/versions/9fafbd60eca1_add_message_file_belongs_to.py +++ b/api/migrations/versions/9fafbd60eca1_add_message_file_belongs_to.py @@ -5,9 +5,8 @@ Revises: 4823da1d26cf Create Date: 2024-01-15 13:07:20.340896 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = '9fafbd60eca1' diff --git a/api/migrations/versions/ad472b61a054_add_api_provider_icon.py b/api/migrations/versions/ad472b61a054_add_api_provider_icon.py index 1328326c2d..0ddaf1eb0a 100644 --- a/api/migrations/versions/ad472b61a054_add_api_provider_icon.py +++ b/api/migrations/versions/ad472b61a054_add_api_provider_icon.py @@ -5,9 +5,8 @@ Revises: 3ef9b2b6bee6 Create Date: 2024-01-07 02:21:23.114790 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = 'ad472b61a054' diff --git a/api/migrations/versions/b24be59fbb04_.py b/api/migrations/versions/b24be59fbb04_.py index e19ea09e86..29ba859f2b 100644 --- a/api/migrations/versions/b24be59fbb04_.py +++ b/api/migrations/versions/b24be59fbb04_.py @@ -5,9 +5,8 @@ Revises: 187385f442fc Create Date: 2024-01-17 01:31:12.670556 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = 'b24be59fbb04' diff --git a/api/migrations/versions/c71211c8f604_add_tool_invoke_model_log.py b/api/migrations/versions/c71211c8f604_add_tool_invoke_model_log.py index dc96672b5e..95fb8f5d0e 100644 --- a/api/migrations/versions/c71211c8f604_add_tool_invoke_model_log.py +++ b/api/migrations/versions/c71211c8f604_add_tool_invoke_model_log.py @@ -5,8 +5,8 @@ Revises: f25003750af4 Create Date: 2024-01-09 11:42:50.664797 """ -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. diff --git a/api/migrations/versions/de95f5c77138_migration_serpapi_api_key.py b/api/migrations/versions/de95f5c77138_migration_serpapi_api_key.py index cbb3662851..f1236df316 100644 --- a/api/migrations/versions/de95f5c77138_migration_serpapi_api_key.py +++ b/api/migrations/versions/de95f5c77138_migration_serpapi_api_key.py @@ -5,10 +5,10 @@ Revises: 23db93619b9d Create Date: 2024-01-21 12:09:04.651394 """ -from alembic import op -import sqlalchemy as sa from json import dumps, loads +import sqlalchemy as sa +from alembic import op # revision identifiers, used by Alembic. revision = 'de95f5c77138' diff --git a/api/migrations/versions/dfb3b7f477da_add_tool_index.py b/api/migrations/versions/dfb3b7f477da_add_tool_index.py index 66e2a4f863..3ef03595fe 100644 --- a/api/migrations/versions/dfb3b7f477da_add_tool_index.py +++ b/api/migrations/versions/dfb3b7f477da_add_tool_index.py @@ -5,9 +5,8 @@ Revises: b24be59fbb04 Create Date: 2024-01-24 02:17:01.631635 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = 'dfb3b7f477da' diff --git a/api/migrations/versions/f25003750af4_add_created_updated_at.py b/api/migrations/versions/f25003750af4_add_created_updated_at.py index 8bdb6a0ff7..178eaf2380 100644 --- a/api/migrations/versions/f25003750af4_add_created_updated_at.py +++ b/api/migrations/versions/f25003750af4_add_created_updated_at.py @@ -5,9 +5,8 @@ Revises: 00bacef91f18 Create Date: 2024-01-07 04:53:24.441861 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = 'f25003750af4' diff --git a/api/models/model.py b/api/models/model.py index c45b0e636b..afe2bc1628 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -1,8 +1,8 @@ import json import uuid -from core.file.upload_file_parser import UploadFileParser from core.file.tool_file_parser import ToolFileParser +from core.file.upload_file_parser import UploadFileParser from extensions.ext_database import db from flask import current_app, request from flask_login import UserMixin diff --git a/api/models/tools.py b/api/models/tools.py index 68668e2343..6b074786a2 100644 --- a/api/models/tools.py +++ b/api/models/tools.py @@ -2,16 +2,14 @@ import json from enum import Enum from typing import List -from sqlalchemy.dialects.postgresql import UUID -from sqlalchemy import ForeignKey - -from extensions.ext_database import db - -from core.tools.entities.tool_bundle import ApiBasedToolBundle from core.tools.entities.common_entities import I18nObject +from core.tools.entities.tool_bundle import ApiBasedToolBundle from core.tools.entities.tool_entities import ApiProviderSchemaType, ToolRuntimeVariablePool +from extensions.ext_database import db +from models.model import Account, App, Tenant +from sqlalchemy import ForeignKey +from sqlalchemy.dialects.postgresql import UUID -from models.model import Tenant, Account, App class BuiltinToolProvider(db.Model): """ diff --git a/api/services/account_service.py b/api/services/account_service.py index 18f72f9ef7..9a97d19889 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -8,7 +8,7 @@ from datetime import datetime, timedelta from hashlib import sha256 from typing import Any, Dict, Optional -from constants.languages import languages, language_timezone_mapping +from constants.languages import language_timezone_mapping, languages from events.tenant_event import tenant_was_created from extensions.ext_redis import redis_client from flask import current_app, session diff --git a/api/services/audio_service.py b/api/services/audio_service.py index bbd5bfb52b..4cc965bacf 100644 --- a/api/services/audio_service.py +++ b/api/services/audio_service.py @@ -3,11 +3,9 @@ from typing import Optional from core.model_manager import ModelManager from core.model_runtime.entities.model_entities import ModelType -from services.errors.audio import (AudioTooLargeServiceError, - NoAudioUploadedServiceError, - ProviderNotSupportTextToSpeechServiceError, +from services.errors.audio import (AudioTooLargeServiceError, NoAudioUploadedServiceError, ProviderNotSupportSpeechToTextServiceError, - UnsupportedAudioTypeServiceError) + ProviderNotSupportTextToSpeechServiceError, UnsupportedAudioTypeServiceError) from werkzeug.datastructures import FileStorage FILE_SIZE = 15 diff --git a/api/services/tools_manage_service.py b/api/services/tools_manage_service.py index 6fb1853871..2a97b3e8b5 100644 --- a/api/services/tools_manage_service.py +++ b/api/services/tools_manage_service.py @@ -1,26 +1,23 @@ +import json from typing import List, Tuple -from flask import current_app - -from core.tools.tool_manager import ToolManager -from core.tools.entities.user_entities import UserToolProvider, UserTool -from core.tools.entities.tool_entities import ApiProviderSchemaType, ApiProviderAuthType, ToolProviderCredentials, \ - ToolCredentialsOption from core.tools.entities.common_entities import I18nObject from core.tools.entities.tool_bundle import ApiBasedToolBundle -from core.tools.provider.tool_provider import ToolProviderController +from core.tools.entities.tool_entities import (ApiProviderAuthType, ApiProviderSchemaType, ToolCredentialsOption, + ToolProviderCredentials) +from core.tools.entities.user_entities import UserTool, UserToolProvider +from core.tools.errors import ToolNotFoundError, ToolProviderCredentialValidationError, ToolProviderNotFoundError from core.tools.provider.api_tool_provider import ApiBasedToolProviderController -from core.tools.utils.parser import ApiBasedToolSchemaParser -from core.tools.utils.encoder import serialize_base_model_array, serialize_base_model_dict +from core.tools.provider.tool_provider import ToolProviderController +from core.tools.tool_manager import ToolManager from core.tools.utils.configuration import ToolConfiguration -from core.tools.errors import ToolProviderCredentialValidationError, ToolProviderNotFoundError, ToolNotFoundError - +from core.tools.utils.encoder import serialize_base_model_array, serialize_base_model_dict +from core.tools.utils.parser import ApiBasedToolSchemaParser from extensions.ext_database import db -from models.tools import BuiltinToolProvider, ApiToolProvider - +from flask import current_app from httpx import get +from models.tools import ApiToolProvider, BuiltinToolProvider -import json class ToolManageService: @staticmethod diff --git a/api/tasks/mail_invite_member_task.py b/api/tasks/mail_invite_member_task.py index bdb6dbacec..a7df7f9298 100644 --- a/api/tasks/mail_invite_member_task.py +++ b/api/tasks/mail_invite_member_task.py @@ -3,9 +3,10 @@ import time import click from celery import shared_task +from constants.languages import languages from extensions.ext_mail import mail from flask import current_app, render_template -from constants.languages import languages + @shared_task(queue='mail') def send_invite_member_mail_task(language: str, to: str, token: str, inviter_name: str, workspace_name: str): diff --git a/api/tests/integration_tests/model_runtime/bedrock/test_llm.py b/api/tests/integration_tests/model_runtime/bedrock/test_llm.py index c395a08fe2..750c049614 100644 --- a/api/tests/integration_tests/model_runtime/bedrock/test_llm.py +++ b/api/tests/integration_tests/model_runtime/bedrock/test_llm.py @@ -7,6 +7,7 @@ from core.model_runtime.entities.message_entities import AssistantPromptMessage, from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.bedrock.llm.llm import BedrockLargeLanguageModel + def test_validate_credentials(): model = BedrockLargeLanguageModel() diff --git a/api/tests/integration_tests/model_runtime/bedrock/test_provider.py b/api/tests/integration_tests/model_runtime/bedrock/test_provider.py index ce3d61c0f7..6819f8c9a1 100644 --- a/api/tests/integration_tests/model_runtime/bedrock/test_provider.py +++ b/api/tests/integration_tests/model_runtime/bedrock/test_provider.py @@ -4,6 +4,7 @@ import pytest from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.bedrock.bedrock import BedrockProvider + def test_validate_provider_credentials(): provider = BedrockProvider() diff --git a/api/tests/integration_tests/model_runtime/cohere/test_llm.py b/api/tests/integration_tests/model_runtime/cohere/test_llm.py index 613675c1fd..a3d054cacf 100644 --- a/api/tests/integration_tests/model_runtime/cohere/test_llm.py +++ b/api/tests/integration_tests/model_runtime/cohere/test_llm.py @@ -3,8 +3,7 @@ from typing import Generator import pytest from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta -from core.model_runtime.entities.message_entities import (AssistantPromptMessage, SystemPromptMessage, - UserPromptMessage) +from core.model_runtime.entities.message_entities import AssistantPromptMessage, SystemPromptMessage, UserPromptMessage from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.cohere.llm.llm import CohereLargeLanguageModel diff --git a/api/tests/integration_tests/model_runtime/ollama/test_llm.py b/api/tests/integration_tests/model_runtime/ollama/test_llm.py index 5543085a54..4265190f58 100644 --- a/api/tests/integration_tests/model_runtime/ollama/test_llm.py +++ b/api/tests/integration_tests/model_runtime/ollama/test_llm.py @@ -2,11 +2,10 @@ import os from typing import Generator import pytest - -from core.model_runtime.entities.message_entities import AssistantPromptMessage, UserPromptMessage, \ - SystemPromptMessage, TextPromptMessageContent, ImagePromptMessageContent -from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunkDelta, \ - LLMResultChunk +from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, ImagePromptMessageContent, + SystemPromptMessage, TextPromptMessageContent, + UserPromptMessage) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.ollama.llm.llm import OllamaLargeLanguageModel diff --git a/api/tests/integration_tests/model_runtime/ollama/test_text_embedding.py b/api/tests/integration_tests/model_runtime/ollama/test_text_embedding.py index c5f5918235..e305226b85 100644 --- a/api/tests/integration_tests/model_runtime/ollama/test_text_embedding.py +++ b/api/tests/integration_tests/model_runtime/ollama/test_text_embedding.py @@ -1,7 +1,6 @@ import os import pytest - from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.ollama.text_embedding.text_embedding import OllamaEmbeddingModel diff --git a/api/tests/integration_tests/model_runtime/zhipuai/test_llm.py b/api/tests/integration_tests/model_runtime/zhipuai/test_llm.py index a04e607a5d..5ca1ee44b8 100644 --- a/api/tests/integration_tests/model_runtime/zhipuai/test_llm.py +++ b/api/tests/integration_tests/model_runtime/zhipuai/test_llm.py @@ -3,8 +3,8 @@ from typing import Generator import pytest from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta -from core.model_runtime.entities.message_entities import (AssistantPromptMessage, SystemPromptMessage, - UserPromptMessage, PromptMessageTool) +from core.model_runtime.entities.message_entities import (AssistantPromptMessage, PromptMessageTool, + SystemPromptMessage, UserPromptMessage) from core.model_runtime.errors.validate import CredentialsValidateFailedError from core.model_runtime.model_providers.zhipuai.llm.llm import ZhipuAILargeLanguageModel diff --git a/api/tests/integration_tests/tools/test_all_provider.py b/api/tests/integration_tests/tools/test_all_provider.py index 83eccb1b11..c846ddecfb 100644 --- a/api/tests/integration_tests/tools/test_all_provider.py +++ b/api/tests/integration_tests/tools/test_all_provider.py @@ -1,5 +1,6 @@ from core.tools.tool_manager import ToolManager + def test_tool_providers(): """ Test that all tool providers can be loaded diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index d9d8ba8d52..cd22a73cac 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3.1' services: # API service api: - image: langgenius/dify-api:0.5.2 + image: langgenius/dify-api:0.5.3 restart: always environment: # Startup mode, 'api' starts the API server. @@ -130,7 +130,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.5.2 + image: langgenius/dify-api:0.5.3 restart: always environment: # Startup mode, 'worker' starts the Celery worker for processing the queue. @@ -201,7 +201,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.5.2 + image: langgenius/dify-web:0.5.3 restart: always environment: EDITION: SELF_HOSTED diff --git a/web/package.json b/web/package.json index 4ff7ad11d4..60a59b5061 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "dify-web", - "version": "0.5.2", + "version": "0.5.3", "private": true, "scripts": { "dev": "next dev",