diff --git a/autotest/gcore/vsis3.py b/autotest/gcore/vsis3.py index dde2272b5ec1..dfa113349a82 100755 --- a/autotest/gcore/vsis3.py +++ b/autotest/gcore/vsis3.py @@ -11,6 +11,7 @@ # SPDX-License-Identifier: MIT ############################################################################### +import copy import json import os.path import stat @@ -5301,7 +5302,6 @@ def test_vsis3_read_credentials_ec2_imdsv2(aws_test_config, webserver_port): "AWS_ACCESS_KEY_ID": "", # Disable hypervisor related check to test if we are really on EC2 "CPL_AWS_AUTODETECT_EC2": "NO", - "CPL_AWS_WEB_IDENTITY_ENABLE": "NO", } gdal.VSICurlClearCache() @@ -5342,7 +5342,9 @@ def test_vsis3_read_credentials_ec2_imdsv2(aws_test_config, webserver_port): custom_method=get_s3_fake_bucket_resource_method, ) with webserver.install_http_handler(handler): - with gdaltest.config_options(options, thread_local=False): + initial_options = copy.copy(options) + initial_options["CPL_AWS_WEB_IDENTITY_ENABLE"] = "NO" + with gdaltest.config_options(initial_options, thread_local=False): with gdaltest.config_option( "CPL_AWS_EC2_API_ROOT_URL", "http://localhost:%d" % webserver_port, @@ -5536,7 +5538,9 @@ def test_vsis3_read_credentials_ec2_imdsv1(aws_test_config, webserver_port): custom_method=get_s3_fake_bucket_resource_method, ) with webserver.install_http_handler(handler): - with gdaltest.config_options(options, thread_local=False): + initial_options = copy.copy(options) + initial_options["CPL_AWS_WEB_IDENTITY_ENABLE"] = "NO" + with gdaltest.config_options(initial_options, thread_local=False): f = open_for_read("/vsis3/s3_fake_bucket/resource") assert f is not None data = gdal.VSIFReadL(1, 4, f).decode("ascii") @@ -5544,6 +5548,21 @@ def test_vsis3_read_credentials_ec2_imdsv1(aws_test_config, webserver_port): assert data == "foo" + handler = webserver.SequentialHandler() + handler.add("GET", "/s3_fake_bucket/bar", 200, {}, "bar") + with webserver.install_http_handler(handler): + with gdaltest.config_options(options, thread_local=False): + # Set a fake URL to check that credentials re-use works + with gdaltest.config_option( + "CPL_AWS_EC2_API_ROOT_URL", "", thread_local=False + ): + f = open_for_read("/vsis3/s3_fake_bucket/bar") + assert f is not None + data = gdal.VSIFReadL(1, 4, f).decode("ascii") + gdal.VSIFCloseL(f) + + assert data == "bar" + ############################################################################### # Read credentials from simulated EC2 instance with expiration of the @@ -5620,7 +5639,9 @@ def test_vsis3_read_credentials_ec2_expiration(aws_test_config, webserver_port): custom_method=get_s3_fake_bucket_resource_method, ) with webserver.install_http_handler(handler): - with gdaltest.config_options(options, thread_local=False): + initial_options = copy.copy(options) + initial_options["CPL_AWS_WEB_IDENTITY_ENABLE"] = "NO" + with gdaltest.config_options(initial_options, thread_local=False): with gdaltest.config_option( "CPL_AWS_EC2_API_ROOT_URL", valid_url, thread_local=False ): @@ -5662,7 +5683,6 @@ def test_vsis3_read_credentials_AWS_CONTAINER_CREDENTIALS_FULL_URI( "AWS_ACCESS_KEY_ID": "", # Disable hypervisor related check to test if we are really on EC2 "CPL_AWS_AUTODETECT_EC2": "NO", - "CPL_AWS_WEB_IDENTITY_ENABLE": "NO", "AWS_CONTAINER_CREDENTIALS_FULL_URI": f"http://localhost:{webserver_port}/AWS_CONTAINER_CREDENTIALS_FULL_URI", } @@ -5687,7 +5707,9 @@ def test_vsis3_read_credentials_AWS_CONTAINER_CREDENTIALS_FULL_URI( custom_method=get_s3_fake_bucket_resource_method, ) with webserver.install_http_handler(handler): - with gdaltest.config_options(options, thread_local=False): + initial_options = copy.copy(options) + initial_options["CPL_AWS_WEB_IDENTITY_ENABLE"] = "NO" + with gdaltest.config_options(initial_options, thread_local=False): f = open_for_read("/vsis3/s3_fake_bucket/resource") assert f is not None data = gdal.VSIFReadL(1, 4, f).decode("ascii") @@ -5695,6 +5717,21 @@ def test_vsis3_read_credentials_AWS_CONTAINER_CREDENTIALS_FULL_URI( assert data == "foo" + handler = webserver.SequentialHandler() + handler.add("GET", "/s3_fake_bucket/bar", 200, {}, "bar") + with webserver.install_http_handler(handler): + with gdaltest.config_options(options, thread_local=False): + # Set a fake URL to check that credentials re-use works + with gdaltest.config_option( + "CPL_AWS_EC2_API_ROOT_URL", "", thread_local=False + ): + f = open_for_read("/vsis3/s3_fake_bucket/bar") + assert f is not None + data = gdal.VSIFReadL(1, 4, f).decode("ascii") + gdal.VSIFCloseL(f) + + assert data == "bar" + ############################################################################### # Read credentials from simulated instance with AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE @@ -5711,7 +5748,6 @@ def test_vsis3_read_credentials_AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE( "AWS_ACCESS_KEY_ID": "", # Disable hypervisor related check to test if we are really on EC2 "CPL_AWS_AUTODETECT_EC2": "NO", - "CPL_AWS_WEB_IDENTITY_ENABLE": "NO", "AWS_CONTAINER_CREDENTIALS_FULL_URI": f"http://localhost:{webserver_port}/AWS_CONTAINER_CREDENTIALS_FULL_URI", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE": f"{tmp_vsimem}/container_authorization_token_file", "AWS_CONTAINER_AUTHORIZATION_TOKEN": "invalid", @@ -5741,7 +5777,9 @@ def test_vsis3_read_credentials_AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE( custom_method=get_s3_fake_bucket_resource_method, ) with webserver.install_http_handler(handler): - with gdaltest.config_options(options, thread_local=False): + initial_options = copy.copy(options) + initial_options["CPL_AWS_WEB_IDENTITY_ENABLE"] = "NO" + with gdaltest.config_options(initial_options, thread_local=False): f = open_for_read("/vsis3/s3_fake_bucket/resource") assert f is not None data = gdal.VSIFReadL(1, 4, f).decode("ascii") @@ -5749,6 +5787,21 @@ def test_vsis3_read_credentials_AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE( assert data == "foo" + handler = webserver.SequentialHandler() + handler.add("GET", "/s3_fake_bucket/bar", 200, {}, "bar") + with webserver.install_http_handler(handler): + with gdaltest.config_options(options, thread_local=False): + # Set a fake URL to check that credentials re-use works + with gdaltest.config_option( + "CPL_AWS_EC2_API_ROOT_URL", "", thread_local=False + ): + f = open_for_read("/vsis3/s3_fake_bucket/bar") + assert f is not None + data = gdal.VSIFReadL(1, 4, f).decode("ascii") + gdal.VSIFCloseL(f) + + assert data == "bar" + ############################################################################### # Read credentials from simulated instance with AWS_CONTAINER_AUTHORIZATION_TOKEN diff --git a/port/cpl_aws.cpp b/port/cpl_aws.cpp index c4d1375229e3..609a9233e534 100644 --- a/port/cpl_aws.cpp +++ b/port/cpl_aws.cpp @@ -36,6 +36,8 @@ const char *CPLGetWineVersion(); // defined in cpl_vsil_win32.cpp #ifdef HAVE_CURL static CPLMutex *ghMutex = nullptr; +static AWSCredentialsSource geCredentialsSource = + AWSCredentialsSource::UNINITIALIZED; static std::string gosIAMRole; static std::string gosGlobalAccessKeyId; static std::string gosGlobalSecretAccessKey; @@ -683,7 +685,8 @@ bool VSIS3HandleHelper::GetConfigurationFromAssumeRoleWithWebIdentity( std::string &osSessionToken) { CPLMutexHolder oHolder(&ghMutex); - if (!bForceRefresh) + if (!bForceRefresh && + geCredentialsSource == AWSCredentialsSource::WEB_IDENTITY) { time_t nCurTime; time(&nCurTime); @@ -796,6 +799,7 @@ bool VSIS3HandleHelper::GetConfigurationFromAssumeRoleWithWebIdentity( !osSessionToken.empty() && Iso8601ToUnixTime(osExpiration.c_str(), &nExpirationUnix)) { + geCredentialsSource = AWSCredentialsSource::WEB_IDENTITY; gosGlobalAccessKeyId = osAccessKeyId; gosGlobalSecretAccessKey = osSecretAccessKey; gosGlobalSessionToken = osSessionToken; @@ -817,7 +821,7 @@ bool VSIS3HandleHelper::GetConfigurationFromEC2( std::string &osSessionToken) { CPLMutexHolder oHolder(&ghMutex); - if (!bForceRefresh) + if (!bForceRefresh && geCredentialsSource == AWSCredentialsSource::EC2) { time_t nCurTime; time(&nCurTime); @@ -1042,6 +1046,7 @@ bool VSIS3HandleHelper::GetConfigurationFromEC2( if (!osAccessKeyId.empty() && !osSecretAccessKey.empty() && Iso8601ToUnixTime(osExpiration.c_str(), &nExpirationUnix)) { + geCredentialsSource = AWSCredentialsSource::EC2; gosGlobalAccessKeyId = osAccessKeyId; gosGlobalSecretAccessKey = osSecretAccessKey; gosGlobalSessionToken = osSessionToken; @@ -1623,7 +1628,8 @@ bool VSIS3HandleHelper::GetOrRefreshTemporaryCredentialsForRole( std::string &osRegion) { CPLMutexHolder oHolder(&ghMutex); - if (!bForceRefresh) + if (!bForceRefresh && + geCredentialsSource == AWSCredentialsSource::ASSUMED_ROLE) { time_t nCurTime; time(&nCurTime); @@ -1668,6 +1674,7 @@ bool VSIS3HandleHelper::GetOrRefreshTemporaryCredentialsForRole( gosSourceProfileSessionToken, gosGlobalSecretAccessKey, gosGlobalAccessKeyId, gosGlobalSessionToken, osExpiration)) { + geCredentialsSource = AWSCredentialsSource::ASSUMED_ROLE; Iso8601ToUnixTime(osExpiration.c_str(), &gnGlobalExpiration); osAccessKeyId = gosGlobalAccessKeyId; osSecretAccessKey = gosGlobalSecretAccessKey; @@ -1690,7 +1697,7 @@ bool VSIS3HandleHelper::GetOrRefreshTemporaryCredentialsForSSO( std::string &osRegion) { CPLMutexHolder oHolder(&ghMutex); - if (!bForceRefresh) + if (!bForceRefresh && geCredentialsSource == AWSCredentialsSource::SSO) { time_t nCurTime; time(&nCurTime); @@ -1717,6 +1724,7 @@ bool VSIS3HandleHelper::GetOrRefreshTemporaryCredentialsForSSO( gosGlobalSecretAccessKey, gosGlobalAccessKeyId, gosGlobalSessionToken, osExpirationEpochInMS)) { + geCredentialsSource = AWSCredentialsSource::SSO; gnGlobalExpiration = CPLAtoGIntBig(osExpirationEpochInMS.c_str()) / 1000; osAccessKeyId = gosGlobalAccessKeyId; @@ -1740,7 +1748,7 @@ bool VSIS3HandleHelper::GetConfiguration( std::string &osSessionToken, std::string &osRegion, AWSCredentialsSource &eCredentialsSource) { - eCredentialsSource = AWSCredentialsSource::REGULAR; + eCredentialsSource = AWSCredentialsSource::UNINITIALIZED; // AWS_REGION is GDAL specific. Later overloaded by standard // AWS_DEFAULT_REGION @@ -1752,6 +1760,7 @@ bool VSIS3HandleHelper::GetConfiguration( if (CPLTestBool(VSIGetPathSpecificOption(osPathForOption.c_str(), "AWS_NO_SIGN_REQUEST", "NO"))) { + eCredentialsSource = AWSCredentialsSource::NO_SIGN_REQUEST; osSecretAccessKey.clear(); osAccessKeyId.clear(); osSessionToken.clear(); @@ -1775,6 +1784,7 @@ bool VSIS3HandleHelper::GetConfiguration( return false; } + eCredentialsSource = AWSCredentialsSource::REGULAR; osSessionToken = CSLFetchNameValueDef( papszOptions, "AWS_SESSION_TOKEN", VSIGetPathSpecificOption(osPathForOption.c_str(), @@ -1898,6 +1908,7 @@ bool VSIS3HandleHelper::GetConfiguration( // Store global variables to be able to reuse the // temporary credentials CPLMutexHolder oHolder(&ghMutex); + geCredentialsSource = AWSCredentialsSource::ASSUMED_ROLE; Iso8601ToUnixTime(osExpiration.c_str(), &gnGlobalExpiration); gosRoleArn = std::move(osRoleArn); @@ -1938,6 +1949,7 @@ bool VSIS3HandleHelper::GetConfiguration( // Store global variables to be able to reuse the // temporary credentials CPLMutexHolder oHolder(&ghMutex); + geCredentialsSource = AWSCredentialsSource::SSO; gnGlobalExpiration = CPLAtoGIntBig(osExpirationEpochInMS.c_str()) / 1000; gosSSOStartURL = std::move(osSSOStartURL); @@ -2019,6 +2031,7 @@ void VSIS3HandleHelper::ClearCache() { CPLMutexHolder oHolder(&ghMutex); + geCredentialsSource = AWSCredentialsSource::UNINITIALIZED; gosIAMRole.clear(); gosGlobalAccessKeyId.clear(); gosGlobalSecretAccessKey.clear(); @@ -2056,7 +2069,8 @@ VSIS3HandleHelper *VSIS3HandleHelper::BuildFromURI(const char *pszURI, std::string osAccessKeyId; std::string osSessionToken; std::string osRegion; - AWSCredentialsSource eCredentialsSource = AWSCredentialsSource::REGULAR; + AWSCredentialsSource eCredentialsSource = + AWSCredentialsSource::UNINITIALIZED; if (!GetConfiguration(osPathForOption, papszOptions, osSecretAccessKey, osAccessKeyId, osSessionToken, osRegion, eCredentialsSource)) diff --git a/port/cpl_aws.h b/port/cpl_aws.h index bcdf2d37ff3f..36b72b93117a 100644 --- a/port/cpl_aws.h +++ b/port/cpl_aws.h @@ -120,6 +120,8 @@ class IVSIS3LikeHandleHelper enum class AWSCredentialsSource { + UNINITIALIZED, + NO_SIGN_REQUEST, REGULAR, // credentials from env variables or ~/.aws/crediential EC2, // credentials from EC2 private networking WEB_IDENTITY, // credentials from Web Identity Token