fix(http): support remote_url for file in http_request node

Fixes #21913

api/core/workflow/nodes/http_request/executor.py

@@ -11,6 +11,7 @@ import httpx
 
 from configs import dify_config
 from core.file import file_manager
+from core.file.enums import FileTransferMethod
 from core.helper import ssrf_proxy
 from core.variables.segments import ArrayFileSegment, FileSegment
 from core.workflow.entities.variable_pool import VariablePool
@@ -225,28 +226,32 @@ class Executor:
                     files: dict[str, list[tuple[str | None, bytes, str]]] = {}
                     for key, files_in_segment in files_list:
                         for file in files_in_segment:
-                            if file.related_id is not None:
+                            if file.related_id is not None or file.transfer_method == FileTransferMethod.REMOTE_URL:
+                                downloaded_file = file_manager.download(file)
                                 file_tuple = (
                                     file.filename,
-                                    file_manager.download(file),
+                                    downloaded_file,
                                     file.mime_type or "application/octet-stream",
                                 )
                                 if key not in files:
                                     files[key] = []
                                 files[key].append(file_tuple)
 
-                    # convert files to list for httpx request
+                    # convert files dictionary to list for httpx request
+                    self.files = []
+                    for key, file_tuples in files.items():
+                        for file_tuple in file_tuples:
+                            self.files.append((key, file_tuple))
+
                     # If there are no actual files, we still need to force httpx to use `multipart/form-data`.
                     # This is achieved by inserting a harmless placeholder file that will be ignored by the server.
-                    if not files:
+                    if not self.files:
                         self.files = [("__multipart_placeholder__", ("", b"", "application/octet-stream"))]
-                    if files:
-                        self.files = []
-                        for key, file_tuples in files.items():
-                            for file_tuple in file_tuples:
-                                self.files.append((key, file_tuple))
 
-                    self.data = form_data
+                    # Only set self.data if there are text fields. If only files are present, self.data should be None,
+                    # which tells httpx to build the request body from self.files.
+                    if form_data:
+                        self.data = form_data
 
     def _assembling_headers(self) -> dict[str, Any]:
         authorization = deepcopy(self.auth)
@@ -390,7 +395,7 @@ class Executor:
                     body_string += content.decode("utf-8")
                 except UnicodeDecodeError:
                     # fix: decode binary content
-                    pass
+                    body_string += f"[Binary data of {len(content)} bytes]"
                 body_string += "\r\n"
             body_string += f"--{boundary}--\r\n"
         elif self.node_data.body:

api/factories/file_factory.py

@@ -1,9 +1,9 @@
 import mimetypes
+import re
 import uuid
 from collections.abc import Callable, Mapping, Sequence
-from typing import Any, cast
+from typing import Any
 
-import httpx
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
@@ -224,10 +224,16 @@ def _build_from_remote_url(
     mime_type, filename, file_size = _get_remote_file_info(url)
     extension = mimetypes.guess_extension(mime_type) or ("." + filename.split(".")[-1] if "." in filename else ".bin")
 
-    file_type = _standardize_file_type(extension=extension, mime_type=mime_type)
-    if file_type.value != mapping.get("type", "custom"):
+    specified_type = mapping.get("type")
+    detected_file_type = _standardize_file_type(extension=extension, mime_type=mime_type)
+
+    if strict_type_validation and specified_type and detected_file_type.value != specified_type:
         raise ValueError("Detected file type does not match the specified type. Please verify the file.")
 
+    file_type = (
+        FileType(specified_type) if specified_type and specified_type != FileType.CUSTOM.value else detected_file_type
+    )
+
     return File(
         id=mapping.get("id"),
         filename=filename,
@@ -244,16 +250,32 @@ def _build_from_remote_url(
 
 def _get_remote_file_info(url: str):
     file_size = -1
-    filename = url.split("/")[-1].split("?")[0] or "unknown_file"
-    mime_type = mimetypes.guess_type(filename)[0] or ""
+    filename = ""
 
     resp = ssrf_proxy.head(url, follow_redirects=True)
-    resp = cast(httpx.Response, resp)
-    if resp.status_code == httpx.codes.OK:
-        if content_disposition := resp.headers.get("Content-Disposition"):
-            filename = str(content_disposition.split("filename=")[-1].strip('"'))
-        file_size = int(resp.headers.get("Content-Length", file_size))
-        mime_type = mime_type or str(resp.headers.get("Content-Type", ""))
+    resp.raise_for_status()
+
+    content_disposition = resp.headers.get("Content-Disposition")
+    if content_disposition:
+        # Use regex to parse filename from content-disposition header
+        # RFC 2616, Section 19.5.1
+        filename_match = re.search(r'filename="([^"]+)"', content_disposition)
+        if filename_match:
+            filename = filename_match.group(1)
+
+    if not filename:
+        filename = url.split("/")[-1].split("?")[0] or "unknown_file"
+
+    mime_type = resp.headers.get("Content-Type", "")
+    if not mime_type:
+        mime_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
+    else:
+        # strip charset or other parameters from mime type
+        mime_type = mime_type.split(";")[0].strip()
+
+    content_length = resp.headers.get("Content-Length")
+    if content_length and content_length.isdigit():
+        file_size = int(content_length)
 
     return mime_type, filename, file_size
 

api/tests/unit_tests/factories/test_build_from_mapping.py

@@ -62,7 +62,7 @@ def mock_tool_file():
 @pytest.fixture
 def mock_http_head():
     def _mock_response(filename, size, content_type):
-        return Response(
+        response = Response(
             status_code=200,
             headers={
                 "Content-Disposition": f'attachment; filename="{filename}"',
@@ -70,6 +70,8 @@ def mock_http_head():
                 "Content-Type": content_type,
             },
         )
+        response.request = MagicMock()
+        return response
 
     with patch("factories.file_factory.ssrf_proxy.head") as mock_head:
         mock_head.return_value = _mock_response("remote_test.jpg", 2048, "image/jpeg")