Skip to content

[Backport 3.11] /vsis3/: fix issue when doing a new connection using EC2 credentials would go through WebIdentity #13291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 28, 2025
+83 −14
Merged

[Backport 3.11] /vsis3/: fix issue when doing a new connection using EC2 credentials would go through WebIdentity #13291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 61 additions & 8 deletions autotest/gcore/vsis3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
# SPDX-License-Identifier: MIT
###############################################################################

import copy
import json
import os.path
import stat
Expand Down Expand Up @@ -5301,7 +5302,6 @@ def test_vsis3_read_credentials_ec2_imdsv2(aws_test_config, webserver_port):
"AWS_ACCESS_KEY_ID": "",
# Disable hypervisor related check to test if we are really on EC2
"CPL_AWS_AUTODETECT_EC2": "NO",
"CPL_AWS_WEB_IDENTITY_ENABLE": "NO",
}

gdal.VSICurlClearCache()
Expand Down Expand Up @@ -5342,7 +5342,9 @@ def test_vsis3_read_credentials_ec2_imdsv2(aws_test_config, webserver_port):
custom_method=get_s3_fake_bucket_resource_method,
)
with webserver.install_http_handler(handler):
with gdaltest.config_options(options, thread_local=False):
initial_options = copy.copy(options)
initial_options["CPL_AWS_WEB_IDENTITY_ENABLE"] = "NO"
with gdaltest.config_options(initial_options, thread_local=False):
with gdaltest.config_option(
"CPL_AWS_EC2_API_ROOT_URL",
"http://localhost:%d" % webserver_port,
Expand Down Expand Up @@ -5536,14 +5538,31 @@ def test_vsis3_read_credentials_ec2_imdsv1(aws_test_config, webserver_port):
custom_method=get_s3_fake_bucket_resource_method,
)
with webserver.install_http_handler(handler):
with gdaltest.config_options(options, thread_local=False):
initial_options = copy.copy(options)
initial_options["CPL_AWS_WEB_IDENTITY_ENABLE"] = "NO"
with gdaltest.config_options(initial_options, thread_local=False):
f = open_for_read("/vsis3/s3_fake_bucket/resource")
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode("ascii")
gdal.VSIFCloseL(f)

assert data == "foo"

handler = webserver.SequentialHandler()
handler.add("GET", "/s3_fake_bucket/bar", 200, {}, "bar")
with webserver.install_http_handler(handler):
with gdaltest.config_options(options, thread_local=False):
# Set a fake URL to check that credentials re-use works
with gdaltest.config_option(
"CPL_AWS_EC2_API_ROOT_URL", "", thread_local=False
):
f = open_for_read("/vsis3/s3_fake_bucket/bar")
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode("ascii")
gdal.VSIFCloseL(f)

assert data == "bar"


###############################################################################
# Read credentials from simulated EC2 instance with expiration of the
Expand Down Expand Up @@ -5620,7 +5639,9 @@ def test_vsis3_read_credentials_ec2_expiration(aws_test_config, webserver_port):
custom_method=get_s3_fake_bucket_resource_method,
)
with webserver.install_http_handler(handler):
with gdaltest.config_options(options, thread_local=False):
initial_options = copy.copy(options)
initial_options["CPL_AWS_WEB_IDENTITY_ENABLE"] = "NO"
with gdaltest.config_options(initial_options, thread_local=False):
with gdaltest.config_option(
"CPL_AWS_EC2_API_ROOT_URL", valid_url, thread_local=False
):
Expand Down Expand Up @@ -5662,7 +5683,6 @@ def test_vsis3_read_credentials_AWS_CONTAINER_CREDENTIALS_FULL_URI(
"AWS_ACCESS_KEY_ID": "",
# Disable hypervisor related check to test if we are really on EC2
"CPL_AWS_AUTODETECT_EC2": "NO",
"CPL_AWS_WEB_IDENTITY_ENABLE": "NO",
"AWS_CONTAINER_CREDENTIALS_FULL_URI": f"http://localhost:{webserver_port}/AWS_CONTAINER_CREDENTIALS_FULL_URI",
}

Expand All @@ -5687,14 +5707,31 @@ def test_vsis3_read_credentials_AWS_CONTAINER_CREDENTIALS_FULL_URI(
custom_method=get_s3_fake_bucket_resource_method,
)
with webserver.install_http_handler(handler):
with gdaltest.config_options(options, thread_local=False):
initial_options = copy.copy(options)
initial_options["CPL_AWS_WEB_IDENTITY_ENABLE"] = "NO"
with gdaltest.config_options(initial_options, thread_local=False):
f = open_for_read("/vsis3/s3_fake_bucket/resource")
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode("ascii")
gdal.VSIFCloseL(f)

assert data == "foo"

handler = webserver.SequentialHandler()
handler.add("GET", "/s3_fake_bucket/bar", 200, {}, "bar")
with webserver.install_http_handler(handler):
with gdaltest.config_options(options, thread_local=False):
# Set a fake URL to check that credentials re-use works
with gdaltest.config_option(
"CPL_AWS_EC2_API_ROOT_URL", "", thread_local=False
):
f = open_for_read("/vsis3/s3_fake_bucket/bar")
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode("ascii")
gdal.VSIFCloseL(f)

assert data == "bar"


###############################################################################
# Read credentials from simulated instance with AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE
Expand All @@ -5711,7 +5748,6 @@ def test_vsis3_read_credentials_AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE(
"AWS_ACCESS_KEY_ID": "",
# Disable hypervisor related check to test if we are really on EC2
"CPL_AWS_AUTODETECT_EC2": "NO",
"CPL_AWS_WEB_IDENTITY_ENABLE": "NO",
"AWS_CONTAINER_CREDENTIALS_FULL_URI": f"http://localhost:{webserver_port}/AWS_CONTAINER_CREDENTIALS_FULL_URI",
"AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE": f"{tmp_vsimem}/container_authorization_token_file",
"AWS_CONTAINER_AUTHORIZATION_TOKEN": "invalid",
Expand Down Expand Up @@ -5741,14 +5777,31 @@ def test_vsis3_read_credentials_AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE(
custom_method=get_s3_fake_bucket_resource_method,
)
with webserver.install_http_handler(handler):
with gdaltest.config_options(options, thread_local=False):
initial_options = copy.copy(options)
initial_options["CPL_AWS_WEB_IDENTITY_ENABLE"] = "NO"
with gdaltest.config_options(initial_options, thread_local=False):
f = open_for_read("/vsis3/s3_fake_bucket/resource")
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode("ascii")
gdal.VSIFCloseL(f)

assert data == "foo"

handler = webserver.SequentialHandler()
handler.add("GET", "/s3_fake_bucket/bar", 200, {}, "bar")
with webserver.install_http_handler(handler):
with gdaltest.config_options(options, thread_local=False):
# Set a fake URL to check that credentials re-use works
with gdaltest.config_option(
"CPL_AWS_EC2_API_ROOT_URL", "", thread_local=False
):
f = open_for_read("/vsis3/s3_fake_bucket/bar")
assert f is not None
data = gdal.VSIFReadL(1, 4, f).decode("ascii")
gdal.VSIFCloseL(f)

assert data == "bar"


###############################################################################
# Read credentials from simulated instance with AWS_CONTAINER_AUTHORIZATION_TOKEN
Expand Down
26 changes: 20 additions & 6 deletions port/cpl_aws.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ const char *CPLGetWineVersion(); // defined in cpl_vsil_win32.cpp

#ifdef HAVE_CURL
static CPLMutex *ghMutex = nullptr;
static AWSCredentialsSource geCredentialsSource =
AWSCredentialsSource::UNINITIALIZED;
static std::string gosIAMRole;
static std::string gosGlobalAccessKeyId;
static std::string gosGlobalSecretAccessKey;
Expand Down Expand Up @@ -683,7 +685,8 @@ bool VSIS3HandleHelper::GetConfigurationFromAssumeRoleWithWebIdentity(
std::string &osSessionToken)
{
CPLMutexHolder oHolder(&ghMutex);
if (!bForceRefresh)
if (!bForceRefresh &&
geCredentialsSource == AWSCredentialsSource::WEB_IDENTITY)
{
time_t nCurTime;
time(&nCurTime);
Expand Down Expand Up @@ -796,6 +799,7 @@ bool VSIS3HandleHelper::GetConfigurationFromAssumeRoleWithWebIdentity(
!osSessionToken.empty() &&
Iso8601ToUnixTime(osExpiration.c_str(), &nExpirationUnix))
{
geCredentialsSource = AWSCredentialsSource::WEB_IDENTITY;
gosGlobalAccessKeyId = osAccessKeyId;
gosGlobalSecretAccessKey = osSecretAccessKey;
gosGlobalSessionToken = osSessionToken;
Expand All @@ -817,7 +821,7 @@ bool VSIS3HandleHelper::GetConfigurationFromEC2(
std::string &osSessionToken)
{
CPLMutexHolder oHolder(&ghMutex);
if (!bForceRefresh)
if (!bForceRefresh && geCredentialsSource == AWSCredentialsSource::EC2)
{
time_t nCurTime;
time(&nCurTime);
Expand Down Expand Up @@ -1042,6 +1046,7 @@ bool VSIS3HandleHelper::GetConfigurationFromEC2(
if (!osAccessKeyId.empty() && !osSecretAccessKey.empty() &&
Iso8601ToUnixTime(osExpiration.c_str(), &nExpirationUnix))
{
geCredentialsSource = AWSCredentialsSource::EC2;
gosGlobalAccessKeyId = osAccessKeyId;
gosGlobalSecretAccessKey = osSecretAccessKey;
gosGlobalSessionToken = osSessionToken;
Expand Down Expand Up @@ -1623,7 +1628,8 @@ bool VSIS3HandleHelper::GetOrRefreshTemporaryCredentialsForRole(
std::string &osRegion)
{
CPLMutexHolder oHolder(&ghMutex);
if (!bForceRefresh)
if (!bForceRefresh &&
geCredentialsSource == AWSCredentialsSource::ASSUMED_ROLE)
{
time_t nCurTime;
time(&nCurTime);
Expand Down Expand Up @@ -1668,6 +1674,7 @@ bool VSIS3HandleHelper::GetOrRefreshTemporaryCredentialsForRole(
gosSourceProfileSessionToken, gosGlobalSecretAccessKey,
gosGlobalAccessKeyId, gosGlobalSessionToken, osExpiration))
{
geCredentialsSource = AWSCredentialsSource::ASSUMED_ROLE;
Iso8601ToUnixTime(osExpiration.c_str(), &gnGlobalExpiration);
osAccessKeyId = gosGlobalAccessKeyId;
osSecretAccessKey = gosGlobalSecretAccessKey;
Expand All @@ -1690,7 +1697,7 @@ bool VSIS3HandleHelper::GetOrRefreshTemporaryCredentialsForSSO(
std::string &osRegion)
{
CPLMutexHolder oHolder(&ghMutex);
if (!bForceRefresh)
if (!bForceRefresh && geCredentialsSource == AWSCredentialsSource::SSO)
{
time_t nCurTime;
time(&nCurTime);
Expand All @@ -1717,6 +1724,7 @@ bool VSIS3HandleHelper::GetOrRefreshTemporaryCredentialsForSSO(
gosGlobalSecretAccessKey, gosGlobalAccessKeyId,
gosGlobalSessionToken, osExpirationEpochInMS))
{
geCredentialsSource = AWSCredentialsSource::SSO;
gnGlobalExpiration =
CPLAtoGIntBig(osExpirationEpochInMS.c_str()) / 1000;
osAccessKeyId = gosGlobalAccessKeyId;
Expand All @@ -1740,7 +1748,7 @@ bool VSIS3HandleHelper::GetConfiguration(
std::string &osSessionToken, std::string &osRegion,
AWSCredentialsSource &eCredentialsSource)
{
eCredentialsSource = AWSCredentialsSource::REGULAR;
eCredentialsSource = AWSCredentialsSource::UNINITIALIZED;

// AWS_REGION is GDAL specific. Later overloaded by standard
// AWS_DEFAULT_REGION
Expand All @@ -1752,6 +1760,7 @@ bool VSIS3HandleHelper::GetConfiguration(
if (CPLTestBool(VSIGetPathSpecificOption(osPathForOption.c_str(),
"AWS_NO_SIGN_REQUEST", "NO")))
{
eCredentialsSource = AWSCredentialsSource::NO_SIGN_REQUEST;
osSecretAccessKey.clear();
osAccessKeyId.clear();
osSessionToken.clear();
Expand All @@ -1775,6 +1784,7 @@ bool VSIS3HandleHelper::GetConfiguration(
return false;
}

eCredentialsSource = AWSCredentialsSource::REGULAR;
osSessionToken = CSLFetchNameValueDef(
papszOptions, "AWS_SESSION_TOKEN",
VSIGetPathSpecificOption(osPathForOption.c_str(),
Expand Down Expand Up @@ -1898,6 +1908,7 @@ bool VSIS3HandleHelper::GetConfiguration(
// Store global variables to be able to reuse the
// temporary credentials
CPLMutexHolder oHolder(&ghMutex);
geCredentialsSource = AWSCredentialsSource::ASSUMED_ROLE;
Iso8601ToUnixTime(osExpiration.c_str(),
&gnGlobalExpiration);
gosRoleArn = std::move(osRoleArn);
Expand Down Expand Up @@ -1938,6 +1949,7 @@ bool VSIS3HandleHelper::GetConfiguration(
// Store global variables to be able to reuse the
// temporary credentials
CPLMutexHolder oHolder(&ghMutex);
geCredentialsSource = AWSCredentialsSource::SSO;
gnGlobalExpiration =
CPLAtoGIntBig(osExpirationEpochInMS.c_str()) / 1000;
gosSSOStartURL = std::move(osSSOStartURL);
Expand Down Expand Up @@ -2019,6 +2031,7 @@ void VSIS3HandleHelper::ClearCache()
{
CPLMutexHolder oHolder(&ghMutex);

geCredentialsSource = AWSCredentialsSource::UNINITIALIZED;
gosIAMRole.clear();
gosGlobalAccessKeyId.clear();
gosGlobalSecretAccessKey.clear();
Expand Down Expand Up @@ -2056,7 +2069,8 @@ VSIS3HandleHelper *VSIS3HandleHelper::BuildFromURI(const char *pszURI,
std::string osAccessKeyId;
std::string osSessionToken;
std::string osRegion;
AWSCredentialsSource eCredentialsSource = AWSCredentialsSource::REGULAR;
AWSCredentialsSource eCredentialsSource =
AWSCredentialsSource::UNINITIALIZED;
if (!GetConfiguration(osPathForOption, papszOptions, osSecretAccessKey,
osAccessKeyId, osSessionToken, osRegion,
eCredentialsSource))
Expand Down
2 changes: 2 additions & 0 deletions port/cpl_aws.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,8 @@ class IVSIS3LikeHandleHelper

enum class AWSCredentialsSource
{
UNINITIALIZED,
NO_SIGN_REQUEST,
REGULAR, // credentials from env variables or ~/.aws/crediential
EC2, // credentials from EC2 private networking
WEB_IDENTITY, // credentials from Web Identity Token
Expand Down
Loading