feat(preprod): Add preprod artifact upload endpoint (#92528)

Upload endpoint described in the [emerge size integration
proposal](https://www.notion.so/sentry/Emerge-Size-Integration-Technical-Proposal-1ec8b10e4b5d801d9bc4cf8c7cc5ad1b?d=1fa8b10e4b5d80df9ed5001c55e19b89#1fa8b10e4b5d80dba610cb7273434b0d)

Most of the logic is pretty similar to the debug file and sourcemap
upload endpoints

in addition to the unit tests, I've manually tested uploading works via
my [test sentry-cli
branch](getsentry/sentry-cli#2533)

Author: NicoHinderling

.github/CODEOWNERS

@@ -681,7 +681,7 @@ tests/sentry/api/endpoints/test_organization_dashboard_widget_details.py     @ge
 /src/sentry/*/migrations/                                @getsentry/owners-migrations
 
 # Preprod build artifact analysis
-/src/sentry/preprod                                   @getsentry/emerge-tool
+/src/sentry/preprod                                   @getsentry/emerge-tools
 # End of preprod
 
 ## Frontend Platform (keep last as we want highest specificity)

src/sentry/api/api_owners.py

@@ -28,3 +28,4 @@ class ApiOwner(Enum):
     UNOWNED = "unowned"
     WEB_FRONTEND_SDKS = "team-web-sdk-frontend"
     GDX = "gdx"
+    EMERGE_TOOLS = "emerge-tools"

src/sentry/api/endpoints/chunk.py

@@ -37,6 +37,7 @@
     "artifact_bundles",  # Artifact Bundles for JavaScript Source Maps
     "artifact_bundles_v2",  # The `assemble` endpoint will check for missing chunks
     "proguard",  # Chunk-uploaded proguard mappings
+    "preprod_artifacts",  # Preprod artifacts (mobile builds, etc.)
 )
 
 

src/sentry/api/urls.py

@@ -286,6 +286,9 @@
 from sentry.notifications.api.endpoints.user_notification_settings_providers import (
     UserNotificationSettingsProvidersEndpoint,
 )
+from sentry.preprod.api.endpoints.organization_preprod_artifact_assemble import (
+    ProjectPreprodArtifactAssembleEndpoint,
+)
 from sentry.relocation.api.endpoints.abort import RelocationAbortEndpoint
 from sentry.relocation.api.endpoints.artifacts.details import RelocationArtifactDetailsEndpoint
 from sentry.relocation.api.endpoints.artifacts.index import RelocationArtifactIndexEndpoint
@@ -2497,6 +2500,11 @@ def create_group_urls(name_prefix: str) -> list[URLPattern | URLResolver]:
         DifAssembleEndpoint.as_view(),
         name="sentry-api-0-assemble-dif-files",
     ),
+    re_path(
+        r"^(?P<organization_id_or_slug>[^\/]+)/(?P<project_id_or_slug>[^\/]+)/files/preprodartifacts/assemble/$",
+        ProjectPreprodArtifactAssembleEndpoint.as_view(),
+        name="sentry-api-0-assemble-preprod-artifact-files",
+    ),
     re_path(
         r"^(?P<organization_id_or_slug>[^\/]+)/(?P<project_id_or_slug>[^\/]+)/files/dsyms/unknown/$",
         UnknownDebugFilesEndpoint.as_view(),

src/sentry/preprod/api/endpoints/organization_preprod_artifact_assemble.py

@@ -0,0 +1,126 @@
+import jsonschema
+import orjson
+import sentry_sdk
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+from sentry.api.api_owners import ApiOwner
+from sentry.api.api_publish_status import ApiPublishStatus
+from sentry.api.base import region_silo_endpoint
+from sentry.api.bases.project import ProjectEndpoint, ProjectReleasePermission
+from sentry.debug_files.upload import find_missing_chunks
+from sentry.models.orgauthtoken import is_org_auth_token_auth, update_org_auth_token_last_used
+from sentry.preprod.tasks import assemble_preprod_artifact
+from sentry.tasks.assemble import (
+    AssembleTask,
+    ChunkFileState,
+    get_assemble_status,
+    set_assemble_status,
+)
+
+
+def validate_preprod_artifact_schema(request_body: bytes) -> tuple[dict, str | None]:
+    """
+    Validate the JSON schema for preprod artifact assembly requests.
+
+    Returns:
+        tuple: (parsed_data, error_message) where error_message is None if validation succeeds
+    """
+    schema = {
+        "type": "object",
+        "properties": {
+            "checksum": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
+            "chunks": {
+                "type": "array",
+                "items": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
+            },
+            # Optional metadata
+            "git_sha": {"type": "string", "pattern": "^[0-9a-f]{40}$"},
+            "build_configuration": {"type": "string"},
+        },
+        "required": ["checksum", "chunks"],
+        "additionalProperties": False,
+    }
+
+    error_messages = {
+        "checksum": "The checksum field is required and must be a 40-character hexadecimal string.",
+        "chunks": "The chunks field is required and must be provided as an array of 40-character hexadecimal strings.",
+        "git_sha": "The git_sha field must be a 40-character hexadecimal string.",
+        "build_configuration": "The build_configuration field must be a string.",
+    }
+
+    try:
+        data = orjson.loads(request_body)
+        jsonschema.validate(data, schema)
+        return data, None
+    except jsonschema.ValidationError as e:
+        error_message = e.message
+        # Get the field from the path if available
+        if e.path:
+            if field := e.path[0]:
+                error_message = error_messages.get(str(field), error_message)
+        return {}, error_message
+    except (orjson.JSONDecodeError, TypeError):
+        return {}, "Invalid json body"
+
+
+@region_silo_endpoint
+class ProjectPreprodArtifactAssembleEndpoint(ProjectEndpoint):
+    owner = ApiOwner.EMERGE_TOOLS
+    publish_status = {
+        "POST": ApiPublishStatus.EXPERIMENTAL,
+    }
+    permission_classes = (ProjectReleasePermission,)
+
+    def post(self, request: Request, project) -> Response:
+        """
+        Assembles a preprod artifact (mobile build, etc.) and stores it in the database.
+        """
+        with sentry_sdk.start_span(op="preprod_artifact.assemble"):
+            data, error_message = validate_preprod_artifact_schema(request.body)
+            if error_message:
+                return Response({"error": error_message}, status=400)
+
+            checksum = data.get("checksum")
+            chunks = data.get("chunks", [])
+
+            # Check if all requested chunks have been uploaded
+            missing_chunks = find_missing_chunks(project.organization_id, set(chunks))
+            if missing_chunks:
+                return Response(
+                    {
+                        "state": ChunkFileState.NOT_FOUND,
+                        "missingChunks": missing_chunks,
+                    }
+                )
+
+            # Check current assembly status
+            state, detail = get_assemble_status(AssembleTask.PREPROD_ARTIFACT, project.id, checksum)
+            if state is not None:
+                return Response({"state": state, "detail": detail, "missingChunks": []})
+
+            # There is neither a known file nor a cached state, so we will
+            # have to create a new file.  Assure that there are checksums.
+            # If not, we assume this is a poll and report NOT_FOUND
+            if not chunks:
+                return Response({"state": ChunkFileState.NOT_FOUND, "missingChunks": []})
+
+            set_assemble_status(
+                AssembleTask.PREPROD_ARTIFACT, project.id, checksum, ChunkFileState.CREATED
+            )
+
+            assemble_preprod_artifact.apply_async(
+                kwargs={
+                    "org_id": project.organization_id,
+                    "project_id": project.id,
+                    "checksum": checksum,
+                    "chunks": chunks,
+                    "git_sha": data.get("git_sha"),
+                    "build_configuration": data.get("build_configuration"),
+                }
+            )
+
+            if is_org_auth_token_auth(request.auth):
+                update_org_auth_token_last_used(request.auth, [project.id])
+
+        return Response({"state": ChunkFileState.CREATED, "missingChunks": []})

src/sentry/preprod/tasks.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+import datetime
+import logging
+import uuid
+
+from django.db import router, transaction
+
+from sentry.models.organization import Organization
+from sentry.models.project import Project
+from sentry.silo.base import SiloMode
+from sentry.tasks.assemble import AssembleTask, ChunkFileState, assemble_file, set_assemble_status
+from sentry.tasks.base import instrumented_task
+from sentry.taskworker.config import TaskworkerConfig
+from sentry.taskworker.namespaces import attachments_tasks
+from sentry.utils.sdk import bind_organization_context
+
+logger = logging.getLogger(__name__)
+
+
+@instrumented_task(
+    name="sentry.preprod.tasks.assemble_preprod_artifact",
+    queue="assemble",
+    silo_mode=SiloMode.REGION,
+    taskworker_config=TaskworkerConfig(
+        namespace=attachments_tasks,
+        processing_deadline_duration=30,
+    ),
+)
+def assemble_preprod_artifact(
+    org_id,
+    project_id,
+    checksum,
+    chunks,
+    git_sha=None,
+    build_configuration=None,
+    **kwargs,
+) -> None:
+    """
+    Creates a preprod artifact from uploaded chunks.
+    """
+    from sentry.preprod.models import PreprodArtifact, PreprodBuildConfiguration
+
+    logger.info(
+        "Starting preprod artifact assembly",
+        extra={
+            "timestamp": datetime.datetime.now().isoformat(),
+            "project_id": project_id,
+            "organization_id": org_id,
+        },
+    )
+
+    try:
+        organization = Organization.objects.get_from_cache(pk=org_id)
+        project = Project.objects.get(id=project_id, organization=organization)
+        bind_organization_context(organization)
+
+        set_assemble_status(
+            AssembleTask.PREPROD_ARTIFACT, org_id, checksum, ChunkFileState.ASSEMBLING
+        )
+
+        assemble_result = assemble_file(
+            task=AssembleTask.PREPROD_ARTIFACT,
+            org_or_project=organization,
+            name=f"preprod-artifact-{uuid.uuid4().hex}",
+            checksum=checksum,
+            chunks=chunks,
+            file_type="preprod.artifact",
+        )
+
+        if assemble_result is None:
+            return
+
+        with transaction.atomic(router.db_for_write(PreprodArtifact)):
+            build_config = None
+            if build_configuration:
+                build_config, _ = PreprodBuildConfiguration.objects.get_or_create(
+                    project=project,
+                    name=build_configuration,
+                )
+
+            # Create PreprodArtifact record
+            preprod_artifact = PreprodArtifact.objects.create(
+                project=project,
+                file_id=assemble_result.bundle.id,
+                build_configuration=build_config,
+                state=PreprodArtifact.ArtifactState.UPLOADED,
+            )
+
+        logger.info(
+            "Created preprod artifact",
+            extra={
+                "preprod_artifact_id": preprod_artifact.id,
+                "project_id": project_id,
+                "organization_id": org_id,
+            },
+        )
+
+        logger.info(
+            "Finished preprod artifact assembly",
+            extra={
+                "timestamp": datetime.datetime.now().isoformat(),
+                "project_id": project_id,
+                "organization_id": org_id,
+            },
+        )
+
+        # where next set of changes will happen
+        # TODO: Trigger artifact processing (size analysis, etc.)
+        # This is where you'd add logic to:
+        # 1. create_or_update a new row in the Commit table as well (once base_sha is added as a column to it)
+        # 2. Detect artifact type (iOS/Android/etc.)
+        # 3. Queue processing tasks
+        # 4. Update state to PROCESSED when done (also update the date_built value to reflect when the artifact was built, among other fields)
+
+    except Exception as e:
+        logger.exception(
+            "Failed to assemble preprod artifact",
+            extra={
+                "project_id": project_id,
+                "organization_id": org_id,
+            },
+        )
+        set_assemble_status(
+            AssembleTask.PREPROD_ARTIFACT,
+            org_id,
+            checksum,
+            ChunkFileState.ERROR,
+            detail=str(e),
+        )
+    else:
+        set_assemble_status(AssembleTask.PREPROD_ARTIFACT, org_id, checksum, ChunkFileState.OK)

src/sentry/tasks/assemble.py

@@ -60,6 +60,7 @@ class AssembleTask:
     DIF = "project.dsym"  # Debug file upload
     RELEASE_BUNDLE = "organization.artifacts"  # Release file upload
     ARTIFACT_BUNDLE = "organization.artifact_bundle"  # Artifact bundle upload
+    PREPROD_ARTIFACT = "organization.preprod_artifact_bundle"  # Preprod artifact upload
 
 
 class AssembleResult(NamedTuple):