From 1b6c3870bf2b8c1a4b31e6696a62ffcbd8effc5c Mon Sep 17 00:00:00 2001 From: Philip Moore Date: Fri, 22 Aug 2025 08:26:41 -0400 Subject: [PATCH 1/3] Bugfix for header sort order for S3 Requester pays mode. --- extension/httpfs/s3fs.cpp | 147 ++++++-------------------------------- 1 file changed, 22 insertions(+), 125 deletions(-) diff --git a/extension/httpfs/s3fs.cpp b/extension/httpfs/s3fs.cpp index e149e61..f1d95d6 100644 --- a/extension/httpfs/s3fs.cpp +++ b/extension/httpfs/s3fs.cpp @@ -74,20 +74,23 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin signed_headers += "content-type;"; } signed_headers += "host;x-amz-content-sha256;x-amz-date"; + if (use_requester_pays) { + signed_headers += ";x-amz-request-payer"; + } if (auth_params.session_token.length() > 0) { signed_headers += ";x-amz-security-token"; } if (use_sse_kms) { signed_headers += ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id"; } - if (use_requester_pays) { - signed_headers += ";x-amz-request-payer"; - } auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query; if (content_type.length() > 0) { canonical_request += "\ncontent-type:" + content_type; } canonical_request += "\nhost:" + host + "\nx-amz-content-sha256:" + payload_hash + "\nx-amz-date:" + datetime_now; + if (use_requester_pays) { + canonical_request += "\nx-amz-request-payer:requester"; + } if (auth_params.session_token.length() > 0) { canonical_request += "\nx-amz-security-token:" + auth_params.session_token; } @@ -95,9 +98,6 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin canonical_request += "\nx-amz-server-side-encryption:aws:kms"; canonical_request += "\nx-amz-server-side-encryption-aws-kms-key-id:" + auth_params.kms_key_id; } - if (use_requester_pays) { - canonical_request += "\nx-amz-request-payer:requester"; - } canonical_request += "\n\n" + signed_headers + "\n" + payload_hash; sha256(canonical_request.c_str(), canonical_request.length(), canonical_request_hash); @@ -131,11 +131,6 @@ string S3FileSystem::UrlEncode(const string &input, bool encode_slash) { return StringUtil::URLEncode(input, encode_slash); } -static bool IsGCSRequest(const string &url) { - return StringUtil::StartsWith(url, "gcs://") || - StringUtil::StartsWith(url, "gs://"); -} - void AWSEnvironmentCredentialsProvider::SetExtensionOptionValue(string key, const char *env_var_name) { char *evar; @@ -214,8 +209,6 @@ S3AuthParams S3AuthParams::ReadFrom(optional_ptr opener, FileOpenerI if (result.url_style.empty() || url_style_result.GetScope() != SettingScope::SECRET) { result.url_style = "path"; } - // Read bearer token for GCS - secret_reader.TryGetSecretKey("bearer_token", result.oauth2_bearer_token); } if (!result.region.empty() && (result.endpoint.empty() || result.endpoint == "s3.amazonaws.com")) { @@ -242,13 +235,9 @@ unique_ptr CreateSecret(vector &prefix_paths_p, string & return_value->secret_map["kms_key_id"] = params.kms_key_id; return_value->secret_map["s3_url_compatibility_mode"] = params.s3_url_compatibility_mode; return_value->secret_map["requester_pays"] = params.requester_pays; - return_value->secret_map["bearer_token"] = params.oauth2_bearer_token; //! Set redact keys return_value->redact_keys = {"secret", "session_token"}; - if (!params.oauth2_bearer_token.empty()) { - return_value->redact_keys.insert("bearer_token"); - } return return_value; } @@ -683,19 +672,9 @@ unique_ptr S3FileSystem::PostRequest(FileHandle &handle, string ur auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params, http_params); - - HTTPHeaders headers; - if (IsGCSRequest(url) && !auth_params.oauth2_bearer_token.empty()) { - // Use bearer token for GCS - headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; - headers["Host"] = parsed_s3_url.host; - headers["Content-Type"] = "application/octet-stream"; - } else { - // Use existing S3 authentication - auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len); - headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "POST", auth_params, "", - "", payload_hash, "application/octet-stream"); - } + auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len); + auto headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "POST", auth_params, "", + "", payload_hash, "application/octet-stream"); return HTTPFileSystem::PostRequest(handle, http_url, headers, result, buffer_in, buffer_in_len); } @@ -706,20 +685,10 @@ unique_ptr S3FileSystem::PutRequest(FileHandle &handle, string url auto parsed_s3_url = S3UrlParse(url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params, http_params); auto content_type = "application/octet-stream"; - - HTTPHeaders headers; - if (IsGCSRequest(url) && !auth_params.oauth2_bearer_token.empty()) { - // Use bearer token for GCS - headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; - headers["Host"] = parsed_s3_url.host; - headers["Content-Type"] = content_type; - } else { - // Use existing S3 authentication - auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len); - headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "PUT", auth_params, "", - "", payload_hash, content_type); - } - + auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len); + + auto headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "PUT", auth_params, "", + "", payload_hash, content_type); return HTTPFileSystem::PutRequest(handle, http_url, headers, buffer_in, buffer_in_len); } @@ -727,18 +696,8 @@ unique_ptr S3FileSystem::HeadRequest(FileHandle &handle, string s3 auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(s3_url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params); - - HTTPHeaders headers; - if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) { - // Use bearer token for GCS - headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; - headers["Host"] = parsed_s3_url.host; - } else { - // Use existing S3 authentication - headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, - "s3", "HEAD", auth_params, "", "", "", ""); - } - + auto headers = + create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "HEAD", auth_params, "", "", "", ""); return HTTPFileSystem::HeadRequest(handle, http_url, headers); } @@ -746,18 +705,8 @@ unique_ptr S3FileSystem::GetRequest(FileHandle &handle, string s3_ auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(s3_url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params); - - HTTPHeaders headers; - if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) { - // Use bearer token for GCS - headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; - headers["Host"] = parsed_s3_url.host; - } else { - // Use existing S3 authentication - headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, - "s3", "GET", auth_params, "", "", "", ""); - } - + auto headers = + create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", ""); return HTTPFileSystem::GetRequest(handle, http_url, headers); } @@ -766,18 +715,8 @@ unique_ptr S3FileSystem::GetRangeRequest(FileHandle &handle, strin auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(s3_url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params); - - HTTPHeaders headers; - if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) { - // Use bearer token for GCS - headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; - headers["Host"] = parsed_s3_url.host; - } else { - // Use existing S3 authentication - headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, - "s3", "GET", auth_params, "", "", "", ""); - } - + auto headers = + create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", ""); return HTTPFileSystem::GetRangeRequest(handle, http_url, headers, file_offset, buffer_out, buffer_out_len); } @@ -785,18 +724,8 @@ unique_ptr S3FileSystem::DeleteRequest(FileHandle &handle, string auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(s3_url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params); - - HTTPHeaders headers; - if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) { - // Use bearer token for GCS - headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; - headers["Host"] = parsed_s3_url.host; - } else { - // Use existing S3 authentication - headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, - "s3", "DELETE", auth_params, "", "", "", ""); - } - + auto headers = + create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "DELETE", auth_params, "", "", "", ""); return HTTPFileSystem::DeleteRequest(handle, http_url, headers); } @@ -845,12 +774,7 @@ void S3FileHandle::Initialize(optional_ptr opener) { } if (entry->second == "403") { // 403: FORBIDDEN - string extra_text; - if (IsGCSRequest(path)) { - extra_text = S3FileSystem::GetGCSAuthError(auth_params); - } else { - extra_text = S3FileSystem::GetS3AuthError(auth_params); - } + auto extra_text = S3FileSystem::GetS3AuthError(auth_params); throw Exception(error.Type(), error.RawMessage() + extra_text, extra_info); } } @@ -1112,24 +1036,6 @@ string S3FileSystem::GetS3AuthError(S3AuthParams &s3_auth_params) { return extra_text; } -string S3FileSystem::GetGCSAuthError(S3AuthParams &s3_auth_params) { - string extra_text = "\n\nAuthentication Failure - GCS authentication failed."; - if (s3_auth_params.oauth2_bearer_token.empty() && - s3_auth_params.secret_access_key.empty() && - s3_auth_params.access_key_id.empty()) { - extra_text += "\n* No credentials provided."; - extra_text += "\n* For OAuth2: CREATE SECRET (TYPE GCS, bearer_token 'your-token')"; - extra_text += "\n* For HMAC: CREATE SECRET (TYPE GCS, key_id 'key', secret 'secret')"; - } else if (!s3_auth_params.oauth2_bearer_token.empty()) { - extra_text += "\n* Bearer token was provided but authentication failed."; - extra_text += "\n* Ensure your OAuth2 token is valid and not expired."; - } else { - extra_text += "\n* HMAC credentials were provided but authentication failed."; - extra_text += "\n* Ensure your HMAC key_id and secret are correct."; - } - return extra_text; -} - HTTPException S3FileSystem::GetS3Error(S3AuthParams &s3_auth_params, const HTTPResponse &response, const string &url) { string extra_text; if (response.status == HTTPStatusCode::BadRequest_400) { @@ -1145,15 +1051,6 @@ HTTPException S3FileSystem::GetS3Error(S3AuthParams &s3_auth_params, const HTTPR HTTPException S3FileSystem::GetHTTPError(FileHandle &handle, const HTTPResponse &response, const string &url) { auto &s3_handle = handle.Cast(); - - // Use GCS-specific error for GCS URLs - if (IsGCSRequest(url) && response.status == HTTPStatusCode::Forbidden_403) { - string extra_text = GetGCSAuthError(s3_handle.auth_params); - auto status_message = HTTPFSUtil::GetStatusMessage(response.status); - throw HTTPException(response, "HTTP error on '%s' (HTTP %d %s)%s", url, - response.status, status_message, extra_text); - } - return GetS3Error(s3_handle.auth_params, response, url); } string AWSListObjectV2::Request(string &path, HTTPParams &http_params, S3AuthParams &s3_auth_params, From 7573fd700709ab021a00b822f4caa8a4f748a037 Mon Sep 17 00:00:00 2001 From: Philip Moore Date: Fri, 22 Aug 2025 08:33:45 -0400 Subject: [PATCH 2/3] Revert "Bugfix for header sort order for S3 Requester pays mode." This reverts commit 1b6c3870bf2b8c1a4b31e6696a62ffcbd8effc5c. --- extension/httpfs/s3fs.cpp | 147 ++++++++++++++++++++++++++++++++------ 1 file changed, 125 insertions(+), 22 deletions(-) diff --git a/extension/httpfs/s3fs.cpp b/extension/httpfs/s3fs.cpp index f1d95d6..e149e61 100644 --- a/extension/httpfs/s3fs.cpp +++ b/extension/httpfs/s3fs.cpp @@ -74,23 +74,20 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin signed_headers += "content-type;"; } signed_headers += "host;x-amz-content-sha256;x-amz-date"; - if (use_requester_pays) { - signed_headers += ";x-amz-request-payer"; - } if (auth_params.session_token.length() > 0) { signed_headers += ";x-amz-security-token"; } if (use_sse_kms) { signed_headers += ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id"; } + if (use_requester_pays) { + signed_headers += ";x-amz-request-payer"; + } auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query; if (content_type.length() > 0) { canonical_request += "\ncontent-type:" + content_type; } canonical_request += "\nhost:" + host + "\nx-amz-content-sha256:" + payload_hash + "\nx-amz-date:" + datetime_now; - if (use_requester_pays) { - canonical_request += "\nx-amz-request-payer:requester"; - } if (auth_params.session_token.length() > 0) { canonical_request += "\nx-amz-security-token:" + auth_params.session_token; } @@ -98,6 +95,9 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin canonical_request += "\nx-amz-server-side-encryption:aws:kms"; canonical_request += "\nx-amz-server-side-encryption-aws-kms-key-id:" + auth_params.kms_key_id; } + if (use_requester_pays) { + canonical_request += "\nx-amz-request-payer:requester"; + } canonical_request += "\n\n" + signed_headers + "\n" + payload_hash; sha256(canonical_request.c_str(), canonical_request.length(), canonical_request_hash); @@ -131,6 +131,11 @@ string S3FileSystem::UrlEncode(const string &input, bool encode_slash) { return StringUtil::URLEncode(input, encode_slash); } +static bool IsGCSRequest(const string &url) { + return StringUtil::StartsWith(url, "gcs://") || + StringUtil::StartsWith(url, "gs://"); +} + void AWSEnvironmentCredentialsProvider::SetExtensionOptionValue(string key, const char *env_var_name) { char *evar; @@ -209,6 +214,8 @@ S3AuthParams S3AuthParams::ReadFrom(optional_ptr opener, FileOpenerI if (result.url_style.empty() || url_style_result.GetScope() != SettingScope::SECRET) { result.url_style = "path"; } + // Read bearer token for GCS + secret_reader.TryGetSecretKey("bearer_token", result.oauth2_bearer_token); } if (!result.region.empty() && (result.endpoint.empty() || result.endpoint == "s3.amazonaws.com")) { @@ -235,9 +242,13 @@ unique_ptr CreateSecret(vector &prefix_paths_p, string & return_value->secret_map["kms_key_id"] = params.kms_key_id; return_value->secret_map["s3_url_compatibility_mode"] = params.s3_url_compatibility_mode; return_value->secret_map["requester_pays"] = params.requester_pays; + return_value->secret_map["bearer_token"] = params.oauth2_bearer_token; //! Set redact keys return_value->redact_keys = {"secret", "session_token"}; + if (!params.oauth2_bearer_token.empty()) { + return_value->redact_keys.insert("bearer_token"); + } return return_value; } @@ -672,9 +683,19 @@ unique_ptr S3FileSystem::PostRequest(FileHandle &handle, string ur auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params, http_params); - auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len); - auto headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "POST", auth_params, "", - "", payload_hash, "application/octet-stream"); + + HTTPHeaders headers; + if (IsGCSRequest(url) && !auth_params.oauth2_bearer_token.empty()) { + // Use bearer token for GCS + headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; + headers["Host"] = parsed_s3_url.host; + headers["Content-Type"] = "application/octet-stream"; + } else { + // Use existing S3 authentication + auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len); + headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "POST", auth_params, "", + "", payload_hash, "application/octet-stream"); + } return HTTPFileSystem::PostRequest(handle, http_url, headers, result, buffer_in, buffer_in_len); } @@ -685,10 +706,20 @@ unique_ptr S3FileSystem::PutRequest(FileHandle &handle, string url auto parsed_s3_url = S3UrlParse(url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params, http_params); auto content_type = "application/octet-stream"; - auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len); - - auto headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "PUT", auth_params, "", - "", payload_hash, content_type); + + HTTPHeaders headers; + if (IsGCSRequest(url) && !auth_params.oauth2_bearer_token.empty()) { + // Use bearer token for GCS + headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; + headers["Host"] = parsed_s3_url.host; + headers["Content-Type"] = content_type; + } else { + // Use existing S3 authentication + auto payload_hash = GetPayloadHash(buffer_in, buffer_in_len); + headers = create_s3_header(parsed_s3_url.path, http_params, parsed_s3_url.host, "s3", "PUT", auth_params, "", + "", payload_hash, content_type); + } + return HTTPFileSystem::PutRequest(handle, http_url, headers, buffer_in, buffer_in_len); } @@ -696,8 +727,18 @@ unique_ptr S3FileSystem::HeadRequest(FileHandle &handle, string s3 auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(s3_url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params); - auto headers = - create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "HEAD", auth_params, "", "", "", ""); + + HTTPHeaders headers; + if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) { + // Use bearer token for GCS + headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; + headers["Host"] = parsed_s3_url.host; + } else { + // Use existing S3 authentication + headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, + "s3", "HEAD", auth_params, "", "", "", ""); + } + return HTTPFileSystem::HeadRequest(handle, http_url, headers); } @@ -705,8 +746,18 @@ unique_ptr S3FileSystem::GetRequest(FileHandle &handle, string s3_ auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(s3_url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params); - auto headers = - create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", ""); + + HTTPHeaders headers; + if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) { + // Use bearer token for GCS + headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; + headers["Host"] = parsed_s3_url.host; + } else { + // Use existing S3 authentication + headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, + "s3", "GET", auth_params, "", "", "", ""); + } + return HTTPFileSystem::GetRequest(handle, http_url, headers); } @@ -715,8 +766,18 @@ unique_ptr S3FileSystem::GetRangeRequest(FileHandle &handle, strin auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(s3_url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params); - auto headers = - create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "GET", auth_params, "", "", "", ""); + + HTTPHeaders headers; + if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) { + // Use bearer token for GCS + headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; + headers["Host"] = parsed_s3_url.host; + } else { + // Use existing S3 authentication + headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, + "s3", "GET", auth_params, "", "", "", ""); + } + return HTTPFileSystem::GetRangeRequest(handle, http_url, headers, file_offset, buffer_out, buffer_out_len); } @@ -724,8 +785,18 @@ unique_ptr S3FileSystem::DeleteRequest(FileHandle &handle, string auto auth_params = handle.Cast().auth_params; auto parsed_s3_url = S3UrlParse(s3_url, auth_params); string http_url = parsed_s3_url.GetHTTPUrl(auth_params); - auto headers = - create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, "s3", "DELETE", auth_params, "", "", "", ""); + + HTTPHeaders headers; + if (IsGCSRequest(s3_url) && !auth_params.oauth2_bearer_token.empty()) { + // Use bearer token for GCS + headers["Authorization"] = "Bearer " + auth_params.oauth2_bearer_token; + headers["Host"] = parsed_s3_url.host; + } else { + // Use existing S3 authentication + headers = create_s3_header(parsed_s3_url.path, "", parsed_s3_url.host, + "s3", "DELETE", auth_params, "", "", "", ""); + } + return HTTPFileSystem::DeleteRequest(handle, http_url, headers); } @@ -774,7 +845,12 @@ void S3FileHandle::Initialize(optional_ptr opener) { } if (entry->second == "403") { // 403: FORBIDDEN - auto extra_text = S3FileSystem::GetS3AuthError(auth_params); + string extra_text; + if (IsGCSRequest(path)) { + extra_text = S3FileSystem::GetGCSAuthError(auth_params); + } else { + extra_text = S3FileSystem::GetS3AuthError(auth_params); + } throw Exception(error.Type(), error.RawMessage() + extra_text, extra_info); } } @@ -1036,6 +1112,24 @@ string S3FileSystem::GetS3AuthError(S3AuthParams &s3_auth_params) { return extra_text; } +string S3FileSystem::GetGCSAuthError(S3AuthParams &s3_auth_params) { + string extra_text = "\n\nAuthentication Failure - GCS authentication failed."; + if (s3_auth_params.oauth2_bearer_token.empty() && + s3_auth_params.secret_access_key.empty() && + s3_auth_params.access_key_id.empty()) { + extra_text += "\n* No credentials provided."; + extra_text += "\n* For OAuth2: CREATE SECRET (TYPE GCS, bearer_token 'your-token')"; + extra_text += "\n* For HMAC: CREATE SECRET (TYPE GCS, key_id 'key', secret 'secret')"; + } else if (!s3_auth_params.oauth2_bearer_token.empty()) { + extra_text += "\n* Bearer token was provided but authentication failed."; + extra_text += "\n* Ensure your OAuth2 token is valid and not expired."; + } else { + extra_text += "\n* HMAC credentials were provided but authentication failed."; + extra_text += "\n* Ensure your HMAC key_id and secret are correct."; + } + return extra_text; +} + HTTPException S3FileSystem::GetS3Error(S3AuthParams &s3_auth_params, const HTTPResponse &response, const string &url) { string extra_text; if (response.status == HTTPStatusCode::BadRequest_400) { @@ -1051,6 +1145,15 @@ HTTPException S3FileSystem::GetS3Error(S3AuthParams &s3_auth_params, const HTTPR HTTPException S3FileSystem::GetHTTPError(FileHandle &handle, const HTTPResponse &response, const string &url) { auto &s3_handle = handle.Cast(); + + // Use GCS-specific error for GCS URLs + if (IsGCSRequest(url) && response.status == HTTPStatusCode::Forbidden_403) { + string extra_text = GetGCSAuthError(s3_handle.auth_params); + auto status_message = HTTPFSUtil::GetStatusMessage(response.status); + throw HTTPException(response, "HTTP error on '%s' (HTTP %d %s)%s", url, + response.status, status_message, extra_text); + } + return GetS3Error(s3_handle.auth_params, response, url); } string AWSListObjectV2::Request(string &path, HTTPParams &http_params, S3AuthParams &s3_auth_params, From eba5b6105f740fdda183a260d3e2a6da72c9d2eb Mon Sep 17 00:00:00 2001 From: Philip Moore Date: Fri, 22 Aug 2025 08:34:47 -0400 Subject: [PATCH 3/3] Bugfix for header sort order for S3 Requester pays mode. --- extension/httpfs/s3fs.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extension/httpfs/s3fs.cpp b/extension/httpfs/s3fs.cpp index e149e61..1238510 100644 --- a/extension/httpfs/s3fs.cpp +++ b/extension/httpfs/s3fs.cpp @@ -74,20 +74,23 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin signed_headers += "content-type;"; } signed_headers += "host;x-amz-content-sha256;x-amz-date"; + if (use_requester_pays) { + signed_headers += ";x-amz-request-payer"; + } if (auth_params.session_token.length() > 0) { signed_headers += ";x-amz-security-token"; } if (use_sse_kms) { signed_headers += ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id"; } - if (use_requester_pays) { - signed_headers += ";x-amz-request-payer"; - } auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query; if (content_type.length() > 0) { canonical_request += "\ncontent-type:" + content_type; } canonical_request += "\nhost:" + host + "\nx-amz-content-sha256:" + payload_hash + "\nx-amz-date:" + datetime_now; + if (use_requester_pays) { + canonical_request += "\nx-amz-request-payer:requester"; + } if (auth_params.session_token.length() > 0) { canonical_request += "\nx-amz-security-token:" + auth_params.session_token; } @@ -95,9 +98,6 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin canonical_request += "\nx-amz-server-side-encryption:aws:kms"; canonical_request += "\nx-amz-server-side-encryption-aws-kms-key-id:" + auth_params.kms_key_id; } - if (use_requester_pays) { - canonical_request += "\nx-amz-request-payer:requester"; - } canonical_request += "\n\n" + signed_headers + "\n" + payload_hash; sha256(canonical_request.c_str(), canonical_request.length(), canonical_request_hash);