From 7613d9dc33c2872e2518405035083190f09b4ba0 Mon Sep 17 00:00:00 2001 From: AichiB7A Date: Sun, 27 Apr 2025 14:39:47 +0800 Subject: [PATCH 1/5] [Observability] Convert exception logging into span in OpenTelemetry (#18821) --- api/extensions/ext_otel.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/api/extensions/ext_otel.py b/api/extensions/ext_otel.py index a2edd832ec..29fa46902e 100644 --- a/api/extensions/ext_otel.py +++ b/api/extensions/ext_otel.py @@ -36,6 +36,31 @@ from configs import dify_config from dify_app import DifyApp +class ExceptionLoggingHandler(logging.Handler): + """Custom logging handler that creates spans for logging.exception() calls""" + + def emit(self, record): + try: + if record.exc_info: + tracer = get_tracer_provider().get_tracer("dify.exception.logging") + with tracer.start_as_current_span( + "log.exception", + attributes={ + "log.level": record.levelname, + "log.message": record.getMessage(), + "log.logger": record.name, + "log.file.path": record.pathname, + "log.file.line": record.lineno, + }, + ) as span: + span.set_status(StatusCode.ERROR) + span.record_exception(record.exc_info[1]) + span.set_attribute("exception.type", record.exc_info[0].__name__) + span.set_attribute("exception.message", str(record.exc_info[1])) + except Exception: + pass + + @user_logged_in.connect @user_loaded_from_request.connect def on_user_loaded(_sender, user): @@ -103,6 +128,7 @@ def init_app(app: DifyApp): if not is_celery_worker(): init_flask_instrumentor(app) CeleryInstrumentor(tracer_provider=get_tracer_provider(), meter_provider=get_meter_provider()).instrument() + instrument_exception_logging() init_sqlalchemy_instrumentor(app) atexit.register(shutdown_tracer) @@ -111,6 +137,11 @@ def is_celery_worker(): return "celery" in sys.argv[0].lower() +def instrument_exception_logging(): + exception_handler = ExceptionLoggingHandler() + logging.getLogger().addHandler(exception_handler) + + def init_flask_instrumentor(app: DifyApp): meter = get_meter("http_metrics", version=dify_config.CURRENT_VERSION) _http_response_counter = meter.create_counter( From 7ccec5cd956ead473d6fe3ecf725c345662ad4c7 Mon Sep 17 00:00:00 2001 From: GuanMu Date: Sun, 27 Apr 2025 14:47:00 +0800 Subject: [PATCH 2/5] refactor: remove external link for dataset description guidance (#18884) --- .../configuration/dataset-config/settings-modal/index.tsx | 5 ----- 1 file changed, 5 deletions(-) diff --git a/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx b/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx index 645f6045f0..3170d33a82 100644 --- a/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx +++ b/web/app/components/app/configuration/dataset-config/settings-modal/index.tsx @@ -4,7 +4,6 @@ import { useMount } from 'ahooks' import { useTranslation } from 'react-i18next' import { isEqual } from 'lodash-es' import { RiCloseLine } from '@remixicon/react' -import { BookOpenIcon } from '@heroicons/react/24/outline' import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development' import cn from '@/utils/classnames' import IndexMethodRadio from '@/app/components/datasets/settings/index-method-radio' @@ -223,10 +222,6 @@ const SettingsModal: FC = ({ className='resize-none' placeholder={t('datasetSettings.form.descPlaceholder') || ''} /> - - - {t('datasetSettings.form.descWrite')} -
From 0e0ec4691ac027d9c4ea77575f8ebad3ce4ee777 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Sun, 27 Apr 2025 16:00:37 +0800 Subject: [PATCH 3/5] feat: add interfaces of OAuth handler methods for authorization (#18889) --- api/core/plugin/entities/plugin_daemon.py | 11 ++- api/core/plugin/impl/oauth.py | 96 ++++++++++++++++++- .../test_oauth_convert_request_to_raw_data.py | 20 ++++ 3 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 api/tests/unit_tests/utils/http_parser/test_oauth_convert_request_to_raw_data.py diff --git a/api/core/plugin/entities/plugin_daemon.py b/api/core/plugin/entities/plugin_daemon.py index 1588cbc3c7..2bea07bea0 100644 --- a/api/core/plugin/entities/plugin_daemon.py +++ b/api/core/plugin/entities/plugin_daemon.py @@ -1,6 +1,7 @@ +from collections.abc import Mapping from datetime import datetime from enum import StrEnum -from typing import Generic, Optional, TypeVar +from typing import Any, Generic, Optional, TypeVar from pydantic import BaseModel, ConfigDict, Field @@ -158,3 +159,11 @@ class PluginInstallTaskStartResponse(BaseModel): class PluginUploadResponse(BaseModel): unique_identifier: str = Field(description="The unique identifier of the plugin.") manifest: PluginDeclaration + + +class PluginOAuthAuthorizationUrlResponse(BaseModel): + authorization_url: str = Field(description="The URL of the authorization.") + + +class PluginOAuthCredentialsResponse(BaseModel): + credentials: Mapping[str, Any] = Field(description="The credentials of the OAuth.") diff --git a/api/core/plugin/impl/oauth.py b/api/core/plugin/impl/oauth.py index 1d40edb086..91774984c8 100644 --- a/api/core/plugin/impl/oauth.py +++ b/api/core/plugin/impl/oauth.py @@ -1,6 +1,98 @@ +from collections.abc import Mapping +from typing import Any + +from werkzeug import Request + +from core.plugin.entities.plugin_daemon import PluginOAuthAuthorizationUrlResponse, PluginOAuthCredentialsResponse from core.plugin.impl.base import BasePluginClient class OAuthHandler(BasePluginClient): - def get_authorization_url(self, tenant_id: str, user_id: str, provider_name: str) -> str: - return "1234567890" + def get_authorization_url( + self, + tenant_id: str, + user_id: str, + plugin_id: str, + provider: str, + system_credentials: Mapping[str, Any], + ) -> PluginOAuthAuthorizationUrlResponse: + return self._request_with_plugin_daemon_response( + "POST", + f"plugin/{tenant_id}/dispatch/oauth/get_authorization_url", + PluginOAuthAuthorizationUrlResponse, + data={ + "user_id": user_id, + "data": { + "provider": provider, + "system_credentials": system_credentials, + }, + }, + headers={ + "X-Plugin-ID": plugin_id, + "Content-Type": "application/json", + }, + ) + + def get_credentials( + self, + tenant_id: str, + user_id: str, + plugin_id: str, + provider: str, + system_credentials: Mapping[str, Any], + request: Request, + ) -> PluginOAuthCredentialsResponse: + """ + Get credentials from the given request. + """ + + # encode request to raw http request + raw_request_bytes = self._convert_request_to_raw_data(request) + + return self._request_with_plugin_daemon_response( + "POST", + f"plugin/{tenant_id}/dispatch/oauth/get_credentials", + PluginOAuthCredentialsResponse, + data={ + "user_id": user_id, + "data": { + "provider": provider, + "system_credentials": system_credentials, + "raw_request_bytes": raw_request_bytes, + }, + }, + headers={ + "X-Plugin-ID": plugin_id, + "Content-Type": "application/json", + }, + ) + + def _convert_request_to_raw_data(self, request: Request) -> bytes: + """ + Convert a Request object to raw HTTP data. + + Args: + request: The Request object to convert. + + Returns: + The raw HTTP data as bytes. + """ + # Start with the request line + method = request.method + path = request.path + protocol = request.headers.get("HTTP_VERSION", "HTTP/1.1") + raw_data = f"{method} {path} {protocol}\r\n".encode() + + # Add headers + for header_name, header_value in request.headers.items(): + raw_data += f"{header_name}: {header_value}\r\n".encode() + + # Add empty line to separate headers from body + raw_data += b"\r\n" + + # Add body if exists + body = request.get_data(as_text=False) + if body: + raw_data += body + + return raw_data diff --git a/api/tests/unit_tests/utils/http_parser/test_oauth_convert_request_to_raw_data.py b/api/tests/unit_tests/utils/http_parser/test_oauth_convert_request_to_raw_data.py new file mode 100644 index 0000000000..f788a9756b --- /dev/null +++ b/api/tests/unit_tests/utils/http_parser/test_oauth_convert_request_to_raw_data.py @@ -0,0 +1,20 @@ +from werkzeug import Request +from werkzeug.datastructures import Headers +from werkzeug.test import EnvironBuilder + +from core.plugin.impl.oauth import OAuthHandler + + +def test_oauth_convert_request_to_raw_data(): + oauth_handler = OAuthHandler() + builder = EnvironBuilder( + method="GET", + path="/test", + headers=Headers({"Content-Type": "application/json"}), + ) + request = Request(builder.get_environ()) + raw_request_bytes = oauth_handler._convert_request_to_raw_data(request) + + assert b"GET /test HTTP/1.1" in raw_request_bytes + assert b"Content-Type: application/json" in raw_request_bytes + assert b"\r\n\r\n" in raw_request_bytes From e3daef19e7ec9740286b3cbb2218b629201c5842 Mon Sep 17 00:00:00 2001 From: "Junjie.M" <118170653@qq.com> Date: Sun, 27 Apr 2025 16:12:02 +0800 Subject: [PATCH 4/5] chatflow/workflow add field required (#18892) --- .../components/workflow/nodes/_base/components/field.tsx | 6 +++++- web/app/components/workflow/nodes/agent/panel.tsx | 6 +++++- web/app/components/workflow/nodes/code/panel.tsx | 2 +- .../components/workflow/nodes/document-extractor/panel.tsx | 1 + web/app/components/workflow/nodes/http/panel.tsx | 2 ++ web/app/components/workflow/nodes/iteration/panel.tsx | 2 ++ .../components/workflow/nodes/knowledge-retrieval/panel.tsx | 2 ++ web/app/components/workflow/nodes/list-operator/panel.tsx | 1 + web/app/components/workflow/nodes/llm/panel.tsx | 1 + .../components/workflow/nodes/parameter-extractor/panel.tsx | 3 +++ .../components/workflow/nodes/question-classifier/panel.tsx | 3 +++ 11 files changed, 26 insertions(+), 3 deletions(-) diff --git a/web/app/components/workflow/nodes/_base/components/field.tsx b/web/app/components/workflow/nodes/_base/components/field.tsx index 14e850b99a..aadcea1065 100644 --- a/web/app/components/workflow/nodes/_base/components/field.tsx +++ b/web/app/components/workflow/nodes/_base/components/field.tsx @@ -17,6 +17,7 @@ type Props = { children?: React.JSX.Element | string | null operations?: React.JSX.Element inline?: boolean + required?: boolean } const Field: FC = ({ @@ -28,6 +29,7 @@ const Field: FC = ({ operations, inline, supportFold, + required, }) => { const [fold, { toggle: toggleFold, @@ -38,7 +40,9 @@ const Field: FC = ({ onClick={() => supportFold && toggleFold()} className={cn('flex items-center justify-between', supportFold && 'cursor-pointer')}>
-
{title}
+
+ {title} {required && *} +
{tooltip && ( > = (props) => { const resetEditor = useStore(s => s.setControlPromptEditorRerenderKey) return
- + > = ({ operations={ } + required > - > = ({
<> > = ({
> = ({ > = ({
Array
)} @@ -91,6 +92,7 @@ const Panel: FC> = ({
Array
)} diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx b/web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx index 20fd24e50c..3b5eefd853 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx +++ b/web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx @@ -81,6 +81,7 @@ const Panel: FC> = ({ {/* {JSON.stringify(inputs, null, 2)} */} > = ({ > = ({
> = ({
> = ({
> = ({ <> > = ({ /> > = ({
> = ({ > = ({ /> Date: Sun, 27 Apr 2025 16:13:36 +0800 Subject: [PATCH 5/5] fix: annotation update need use http put method and annotation-reply api doc parms wrong (#18891) --- api/controllers/service_api/app/annotation.py | 2 +- .../develop/template/template.zh.mdx | 32 ++++++++----------- .../template/template_advanced_chat.en.mdx | 32 ++++++++----------- .../template/template_advanced_chat.zh.mdx | 32 ++++++++----------- 4 files changed, 43 insertions(+), 55 deletions(-) diff --git a/api/controllers/service_api/app/annotation.py b/api/controllers/service_api/app/annotation.py index 522a96b791..c50f551faf 100644 --- a/api/controllers/service_api/app/annotation.py +++ b/api/controllers/service_api/app/annotation.py @@ -79,7 +79,7 @@ class AnnotationListApi(Resource): class AnnotationUpdateDeleteApi(Resource): @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON)) @marshal_with(annotation_fields) - def post(self, app_model: App, end_user: EndUser, annotation_id): + def put(self, app_model: App, end_user: EndUser, annotation_id): if not current_user.is_editor: raise Forbidden() diff --git a/web/app/components/develop/template/template.zh.mdx b/web/app/components/develop/template/template.zh.mdx index 24abb481e3..447dc08396 100755 --- a/web/app/components/develop/template/template.zh.mdx +++ b/web/app/components/develop/template/template.zh.mdx @@ -643,13 +643,11 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ```json {{ title: 'Response' }} { - { - "id": "69d48372-ad81-4c75-9c46-2ce197b4d402", - "question": "What is your name?", - "answer": "I am Dify.", - "hit_count": 0, - "created_at": 1735625869 - } + "id": "69d48372-ad81-4c75-9c46-2ce197b4d402", + "question": "What is your name?", + "answer": "I am Dify.", + "hit_count": 0, + "created_at": 1735625869 } ``` @@ -683,10 +681,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' title="Request" tag="PUT" label="/apps/annotations/{annotation_id}" - targetCode={`curl --location --request POST '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"question": "What is your name?","answer": "I am Dify."}'`} + targetCode={`curl --location --request PUT '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"question": "What is your name?","answer": "I am Dify."}'`} > ```bash {{ title: 'cURL' }} - curl --location --request POST '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \ + curl --location --request PUT '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ @@ -699,13 +697,11 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ```json {{ title: 'Response' }} { - { - "id": "69d48372-ad81-4c75-9c46-2ce197b4d402", - "question": "What is your name?", - "answer": "I am Dify.", - "hit_count": 0, - "created_at": 1735625869 - } + "id": "69d48372-ad81-4c75-9c46-2ce197b4d402", + "question": "What is your name?", + "answer": "I am Dify.", + "hit_count": 0, + "created_at": 1735625869 } ``` @@ -763,10 +759,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' 动作，只能是 'enable' 或 'disable' - + 指定的嵌入模型提供商, 必须先在系统内设定好接入的模型，对应的是provider字段 - + 指定的嵌入模型，对应的是model字段 diff --git a/web/app/components/develop/template/template_advanced_chat.en.mdx b/web/app/components/develop/template/template_advanced_chat.en.mdx index f645133030..0d10af6a71 100644 --- a/web/app/components/develop/template/template_advanced_chat.en.mdx +++ b/web/app/components/develop/template/template_advanced_chat.en.mdx @@ -1337,13 +1337,11 @@ Chat applications support session persistence, allowing previous chat history to ```json {{ title: 'Response' }} { - { - "id": "69d48372-ad81-4c75-9c46-2ce197b4d402", - "question": "What is your name?", - "answer": "I am Dify.", - "hit_count": 0, - "created_at": 1735625869 - } + "id": "69d48372-ad81-4c75-9c46-2ce197b4d402", + "question": "What is your name?", + "answer": "I am Dify.", + "hit_count": 0, + "created_at": 1735625869 } ``` @@ -1377,10 +1375,10 @@ Chat applications support session persistence, allowing previous chat history to title="Request" tag="PUT" label="/apps/annotations/{annotation_id}" - targetCode={`curl --location --request POST '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"question": "What is your name?","answer": "I am Dify."}'`} + targetCode={`curl --location --request PUT '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"question": "What is your name?","answer": "I am Dify."}'`} > ```bash {{ title: 'cURL' }} - curl --location --request POST '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \ + curl --location --request PUT '${props.apiBaseUrl}/apps/annotations/{annotation_id}' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ @@ -1393,13 +1391,11 @@ Chat applications support session persistence, allowing previous chat history to ```json {{ title: 'Response' }} { - { - "id": "69d48372-ad81-4c75-9c46-2ce197b4d402", - "question": "What is your name?", - "answer": "I am Dify.", - "hit_count": 0, - "created_at": 1735625869 - } + "id": "69d48372-ad81-4c75-9c46-2ce197b4d402", + "question": "What is your name?", + "answer": "I am Dify.", + "hit_count": 0, + "created_at": 1735625869 } ``` @@ -1457,10 +1453,10 @@ Chat applications support session persistence, allowing previous chat history to Action, can only be 'enable' or 'disable' - + Specified embedding model provider, must be set up in the system first, corresponding to the provider field(Optional) - + Specified embedding model, corresponding to the model field(Optional) diff --git a/web/app/components/develop/template/template_advanced_chat.zh.mdx b/web/app/components/develop/template/template_advanced_chat.zh.mdx index 240044a25d..e634130a4b 100755 --- a/web/app/components/develop/template/template_advanced_chat.zh.mdx +++ b/web/app/components/develop/template/template_advanced_chat.zh.mdx @@ -1361,13 +1361,11 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ```json {{ title: 'Response' }} { - { - "id": "69d48372-ad81-4c75-9c46-2ce197b4d402", - "question": "What is your name?", - "answer": "I am Dify.", - "hit_count": 0, - "created_at": 1735625869 - } + "id": "69d48372-ad81-4c75-9c46-2ce197b4d402", + "question": "What is your name?", + "answer": "I am Dify.", + "hit_count": 0, + "created_at": 1735625869 } ``` @@ -1401,10 +1399,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' title="Request" tag="PUT" label="/apps/annotations/{annotation_id}" - targetCode={`curl --location --request POST '${props.appDetail.api_base_url}/apps/annotations/{annotation_id}' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"question": "What is your name?","answer": "I am Dify."}'`} + targetCode={`curl --location --request PUT '${props.appDetail.api_base_url}/apps/annotations/{annotation_id}' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"question": "What is your name?","answer": "I am Dify."}'`} > ```bash {{ title: 'cURL' }} - curl --location --request POST '${props.appDetail.api_base_url}/apps/annotations/{annotation_id}' \ + curl --location --request PUT '${props.appDetail.api_base_url}/apps/annotations/{annotation_id}' \ --header 'Authorization: Bearer {api_key}' \ --header 'Content-Type: application/json' \ --data-raw '{ @@ -1417,13 +1415,11 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ```json {{ title: 'Response' }} { - { - "id": "69d48372-ad81-4c75-9c46-2ce197b4d402", - "question": "What is your name?", - "answer": "I am Dify.", - "hit_count": 0, - "created_at": 1735625869 - } + "id": "69d48372-ad81-4c75-9c46-2ce197b4d402", + "question": "What is your name?", + "answer": "I am Dify.", + "hit_count": 0, + "created_at": 1735625869 } ``` @@ -1481,10 +1477,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' 动作，只能是 'enable' 或 'disable' - + 指定的嵌入模型提供商, 必须先在系统内设定好接入的模型，对应的是provider字段 - + 指定的嵌入模型，对应的是model字段