From d51f52a649bc165a6299c2e0ca2123040e3133a8 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Tue, 7 May 2024 16:56:25 +0800 Subject: [PATCH] fix: http authorization leakage (#4146) --- .../workflow/nodes/http_request/entities.py | 13 ++++++++---- .../nodes/http_request/http_executor.py | 17 ++++++++++++++-- .../nodes/http_request/http_request_node.py | 20 +++++++++++-------- .../workflow/nodes/test_http.py | 7 +++++++ 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/api/core/workflow/nodes/http_request/entities.py b/api/core/workflow/nodes/http_request/entities.py index d88ad999b7..31d5a679b0 100644 --- a/api/core/workflow/nodes/http_request/entities.py +++ b/api/core/workflow/nodes/http_request/entities.py @@ -1,9 +1,13 @@ +import os from typing import Literal, Optional, Union from pydantic import BaseModel, validator from core.workflow.entities.base_node_data_entities import BaseNodeData +MAX_CONNECT_TIMEOUT = int(os.environ.get('HTTP_REQUEST_MAX_CONNECT_TIMEOUT', '300')) +MAX_READ_TIMEOUT = int(os.environ.get('HTTP_REQUEST_MAX_READ_TIMEOUT', '600')) +MAX_WRITE_TIMEOUT = int(os.environ.get('HTTP_REQUEST_MAX_WRITE_TIMEOUT', '600')) class HttpRequestNodeData(BaseNodeData): """ @@ -36,9 +40,9 @@ class HttpRequestNodeData(BaseNodeData): data: Union[None, str] class Timeout(BaseModel): - connect: int - read: int - write: int + connect: int = MAX_CONNECT_TIMEOUT + read: int = MAX_READ_TIMEOUT + write: int = MAX_WRITE_TIMEOUT method: Literal['get', 'post', 'put', 'patch', 'delete', 'head'] url: str @@ -46,4 +50,5 @@ class HttpRequestNodeData(BaseNodeData): headers: str params: str body: Optional[Body] - timeout: Optional[Timeout] \ No newline at end of file + timeout: Optional[Timeout] + mask_authorization_header: Optional[bool] = True diff --git a/api/core/workflow/nodes/http_request/http_executor.py b/api/core/workflow/nodes/http_request/http_executor.py index 1fb73afd12..4ca8a81d8c 100644 --- a/api/core/workflow/nodes/http_request/http_executor.py +++ b/api/core/workflow/nodes/http_request/http_executor.py @@ -19,7 +19,6 @@ READABLE_MAX_BINARY_SIZE = f'{MAX_BINARY_SIZE / 1024 / 1024:.2f}MB' MAX_TEXT_SIZE = int(os.environ.get('HTTP_REQUEST_NODE_MAX_TEXT_SIZE', str(1024 * 1024))) # 10MB # 1MB READABLE_MAX_TEXT_SIZE = f'{MAX_TEXT_SIZE / 1024 / 1024:.2f}MB' - class HttpExecutorResponse: headers: dict[str, str] response: Union[httpx.Response, requests.Response] @@ -345,10 +344,13 @@ class HttpExecutor: # validate response return self._validate_and_parse_response(response) - def to_raw_request(self) -> str: + def to_raw_request(self, mask_authorization_header: Optional[bool] = True) -> str: """ convert to raw request """ + if mask_authorization_header == None: + mask_authorization_header = True + server_url = self.server_url if self.params: server_url += f'?{urlencode(self.params)}' @@ -357,6 +359,17 @@ class HttpExecutor: headers = self._assembling_headers() for k, v in headers.items(): + if mask_authorization_header: + # get authorization header + if self.authorization.type == 'api-key': + authorization_header = 'Authorization' + if self.authorization.config and self.authorization.config.header: + authorization_header = self.authorization.config.header + + if k.lower() == authorization_header.lower(): + raw_request += f'{k}: {"*" * len(v)}\n' + continue + raw_request += f'{k}: {v}\n' raw_request += '\n' diff --git a/api/core/workflow/nodes/http_request/http_request_node.py b/api/core/workflow/nodes/http_request/http_request_node.py index cba1a11a8a..bfd686175a 100644 --- a/api/core/workflow/nodes/http_request/http_request_node.py +++ b/api/core/workflow/nodes/http_request/http_request_node.py @@ -1,5 +1,4 @@ import logging -import os from mimetypes import guess_extension from os import path from typing import cast @@ -9,14 +8,15 @@ from core.tools.tool_file_manager import ToolFileManager from core.workflow.entities.node_entities import NodeRunResult, NodeType from core.workflow.entities.variable_pool import VariablePool from core.workflow.nodes.base_node import BaseNode -from core.workflow.nodes.http_request.entities import HttpRequestNodeData +from core.workflow.nodes.http_request.entities import ( + MAX_CONNECT_TIMEOUT, + MAX_READ_TIMEOUT, + MAX_WRITE_TIMEOUT, + HttpRequestNodeData, +) from core.workflow.nodes.http_request.http_executor import HttpExecutor, HttpExecutorResponse from models.workflow import WorkflowNodeExecutionStatus -MAX_CONNECT_TIMEOUT = int(os.environ.get('HTTP_REQUEST_MAX_CONNECT_TIMEOUT', '300')) -MAX_READ_TIMEOUT = int(os.environ.get('HTTP_REQUEST_MAX_READ_TIMEOUT', '600')) -MAX_WRITE_TIMEOUT = int(os.environ.get('HTTP_REQUEST_MAX_WRITE_TIMEOUT', '600')) - HTTP_REQUEST_DEFAULT_TIMEOUT = HttpRequestNodeData.Timeout(connect=min(10, MAX_CONNECT_TIMEOUT), read=min(60, MAX_READ_TIMEOUT), write=min(20, MAX_WRITE_TIMEOUT)) @@ -63,7 +63,9 @@ class HttpRequestNode(BaseNode): process_data = {} if http_executor: process_data = { - 'request': http_executor.to_raw_request(), + 'request': http_executor.to_raw_request( + mask_authorization_header=node_data.mask_authorization_header + ), } return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, @@ -82,7 +84,9 @@ class HttpRequestNode(BaseNode): 'files': files, }, process_data={ - 'request': http_executor.to_raw_request(), + 'request': http_executor.to_raw_request( + mask_authorization_header=node_data.mask_authorization_header + ), } ) diff --git a/api/tests/integration_tests/workflow/nodes/test_http.py b/api/tests/integration_tests/workflow/nodes/test_http.py index 63b6b7d962..10e3d53608 100644 --- a/api/tests/integration_tests/workflow/nodes/test_http.py +++ b/api/tests/integration_tests/workflow/nodes/test_http.py @@ -38,6 +38,7 @@ def test_get(setup_http_mock): 'headers': 'X-Header:123', 'params': 'A:b', 'body': None, + 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -95,6 +96,7 @@ def test_custom_authorization_header(setup_http_mock): 'headers': 'X-Header:123', 'params': 'A:b', 'body': None, + 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -126,6 +128,7 @@ def test_template(setup_http_mock): 'headers': 'X-Header:123\nX-Header2:{{#a.b123.args2#}}', 'params': 'A:b\nTemplate:{{#a.b123.args2#}}', 'body': None, + 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -161,6 +164,7 @@ def test_json(setup_http_mock): 'type': 'json', 'data': '{"a": "{{#a.b123.args1#}}"}' }, + 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -193,6 +197,7 @@ def test_x_www_form_urlencoded(setup_http_mock): 'type': 'x-www-form-urlencoded', 'data': 'a:{{#a.b123.args1#}}\nb:{{#a.b123.args2#}}' }, + 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -225,6 +230,7 @@ def test_form_data(setup_http_mock): 'type': 'form-data', 'data': 'a:{{#a.b123.args1#}}\nb:{{#a.b123.args2#}}' }, + 'mask_authorization_header': False, } }, **BASIC_NODE_DATA) @@ -260,6 +266,7 @@ def test_none_data(setup_http_mock): 'type': 'none', 'data': '123123123' }, + 'mask_authorization_header': False, } }, **BASIC_NODE_DATA)