diff --git a/ami/main/migrations/0079_s3storagesource_region.py b/ami/main/migrations/0079_s3storagesource_region.py new file mode 100644 index 000000000..c62db56f2 --- /dev/null +++ b/ami/main/migrations/0079_s3storagesource_region.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.10 on 2025-12-05 20:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("main", "0078_classification_applied_to"), + ] + + operations = [ + migrations.AddField( + model_name="s3storagesource", + name="region", + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index 515f5286a..dc12b030c 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -1394,6 +1394,7 @@ class S3StorageSource(BaseModel): name = models.CharField(max_length=255) bucket = models.CharField(max_length=255) + region = models.CharField(max_length=255, null=True, blank=True) prefix = models.CharField(max_length=255, blank=True) access_key = models.TextField() secret_key = models.TextField() @@ -1413,6 +1414,7 @@ class S3StorageSource(BaseModel): def config(self) -> ami.utils.s3.S3Config: return ami.utils.s3.S3Config( bucket_name=self.bucket, + region=self.region, prefix=self.prefix, access_key_id=self.access_key, secret_access_key=self.secret_key, diff --git a/ami/tests/fixtures/storage.py b/ami/tests/fixtures/storage.py index d53c782ff..1e23a25e9 100644 --- a/ami/tests/fixtures/storage.py +++ b/ami/tests/fixtures/storage.py @@ -15,6 +15,7 @@ access_key_id=settings.S3_TEST_KEY, secret_access_key=settings.S3_TEST_SECRET, bucket_name=settings.S3_TEST_BUCKET, + region=settings.S3_TEST_REGION, prefix="test_prefix", public_base_url=f"http://minio:9000/{settings.S3_TEST_BUCKET}/test_prefix", # public_base_url="http://minio:9001", diff --git a/ami/utils/s3.py b/ami/utils/s3.py index ce157b213..4a54c8244 100644 --- a/ami/utils/s3.py +++ b/ami/utils/s3.py @@ -10,7 +10,7 @@ from dataclasses import dataclass import boto3 -import boto3.resources.base +import boto3.session import botocore import botocore.config import botocore.exceptions @@ -37,6 +37,7 @@ class S3Config: secret_access_key: str bucket_name: str prefix: str + region: str | None = None public_base_url: str | None = None sensitive_fields = ["access_key_id", "secret_access_key"] @@ -94,26 +95,36 @@ def get_session(config: S3Config) -> boto3.session.Session: session = boto3.Session( aws_access_key_id=config.access_key_id, aws_secret_access_key=config.secret_access_key, + region_name=config.region, ) return session def get_s3_client(config: S3Config) -> S3Client: session = get_session(config) + + # Always use signature version 4 + boto_config = botocore.config.Config(signature_version="s3v4") + if config.endpoint_url: client = session.client( service_name="s3", endpoint_url=config.endpoint_url, aws_access_key_id=config.access_key_id, aws_secret_access_key=config.secret_access_key, - config=botocore.config.Config(signature_version="s3v4"), + region_name=config.region, + config=boto_config, ) else: client = session.client( service_name="s3", aws_access_key_id=config.access_key_id, aws_secret_access_key=config.secret_access_key, + region_name=config.region, + config=boto_config, ) + + client = typing.cast(S3Client, client) return client @@ -124,6 +135,7 @@ def get_resource(config: S3Config) -> S3ServiceResource: endpoint_url=config.endpoint_url, # api_version="s3v4", ) + s3 = typing.cast(S3ServiceResource, s3) return s3 @@ -584,7 +596,9 @@ def read_image(config: S3Config, key: str) -> PIL.Image.Image: obj = bucket.Object(key) logger.info(f"Fetching image {key} from S3") try: - img = PIL.Image.open(obj.get()["Body"]) + # StreamingBody inherits from io.IOBase, but type checkers don't see that + fp = obj.get()["Body"] + img = PIL.Image.open(fp) # type: ignore[arg-type] except PIL.UnidentifiedImageError: logger.error(f"Could not read image {key}") raise @@ -677,6 +691,7 @@ def test(): bucket_name="test", prefix="", public_base_url="http://minio:9000/test", + region=None, ) projects = list_projects(config) diff --git a/config/settings/base.py b/config/settings/base.py index c9a8a9681..8385c38f3 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -449,6 +449,7 @@ S3_TEST_KEY = env("MINIO_ROOT_USER", default=None) # type: ignore[no-untyped-call] S3_TEST_SECRET = env("MINIO_ROOT_PASSWORD", default=None) # type: ignore[no-untyped-call] S3_TEST_BUCKET = env("MINIO_TEST_BUCKET", default="ami-test") # type: ignore[no-untyped-call] +S3_TEST_REGION = env("MINIO_REGION", default=None) # type: ignore[no-untyped-call] # Default processing service settings