ref(mobile-app): Wireup mobile app upload (#2542)

Adds the upload functionality to the new `mobile-app upload` command. 

Tested E2E with new API preprod/assemble endpoint added in
getsentry/sentry#92528. Also added some basic
tests to ensure a minimal APK is successfully uploaded and that it skips
uploading if the server already has the chunks.

Author: rbro112

src/api/data_types/chunking/mobile_app.rs

@@ -0,0 +1,22 @@
+use serde::{Deserialize, Serialize};
+use sha1_smol::Digest;
+
+use super::ChunkedFileState;
+
+#[derive(Debug, Serialize)]
+pub struct ChunkedMobileAppRequest<'a> {
+    pub checksum: Digest,
+    pub chunks: &'a [Digest],
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub git_sha: Option<&'a str>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub build_configuration: Option<&'a str>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct AssembleMobileAppResponse {
+    pub state: ChunkedFileState,
+    pub missing_chunks: Vec<Digest>,
+    pub detail: Option<String>,
+}

src/api/data_types/chunking/mod.rs

@@ -6,11 +6,13 @@ mod compression;
 mod dif;
 mod file_state;
 mod hash_algorithm;
+mod mobile_app;
 mod upload;
 
 pub use self::artifact::{AssembleArtifactsResponse, ChunkedArtifactRequest};
 pub use self::compression::ChunkCompression;
 pub use self::dif::{AssembleDifsRequest, AssembleDifsResponse, ChunkedDifRequest};
 pub use self::file_state::ChunkedFileState;
 pub use self::hash_algorithm::ChunkHashAlgorithm;
+pub use self::mobile_app::{AssembleMobileAppResponse, ChunkedMobileAppRequest};
 pub use self::upload::{ChunkServerOptions, ChunkUploadCapability};

src/api/data_types/chunking/upload/capability.rs

@@ -30,6 +30,9 @@ pub enum ChunkUploadCapability {
     /// Upload of il2cpp line mappings
     Il2Cpp,
 
+    /// Upload of preprod artifacts
+    PreprodArtifacts,
+
     /// Any other unsupported capability (ignored)
     Unknown,
 }
@@ -49,6 +52,7 @@ impl<'de> Deserialize<'de> for ChunkUploadCapability {
             "sources" => ChunkUploadCapability::Sources,
             "bcsymbolmaps" => ChunkUploadCapability::BcSymbolmap,
             "il2cpp" => ChunkUploadCapability::Il2Cpp,
+            "preprod_artifacts" => ChunkUploadCapability::PreprodArtifacts,
             _ => ChunkUploadCapability::Unknown,
         })
     }

src/api/mod.rs

@@ -1018,6 +1018,32 @@ impl<'a> AuthenticatedApi<'a> {
             .convert_rnf(ApiErrorKind::ReleaseNotFound)
     }
 
+    pub fn assemble_mobile_app(
+        &self,
+        org: &str,
+        project: &str,
+        checksum: Digest,
+        chunks: &[Digest],
+        git_sha: Option<&str>,
+        build_configuration: Option<&str>,
+    ) -> ApiResult<AssembleMobileAppResponse> {
+        let url = format!(
+            "/projects/{}/{}/files/preprodartifacts/assemble/",
+            PathArg(org),
+            PathArg(project)
+        );
+
+        self.request(Method::Post, &url)?
+            .with_json_body(&ChunkedMobileAppRequest {
+                checksum,
+                chunks,
+                git_sha,
+                build_configuration,
+            })?
+            .send()?
+            .convert_rnf(ApiErrorKind::ProjectNotFound)
+    }
+
     pub fn associate_proguard_mappings(
         &self,
         org: &str,

src/commands/mobile_app/upload.rs

@@ -1,16 +1,26 @@
+use std::borrow::Cow;
 use std::io::Write;
 use std::path::Path;
 
-use anyhow::{anyhow, Context as _, Result};
+use anyhow::{anyhow, bail, Context as _, Result};
 use clap::{Arg, ArgAction, ArgMatches, Command};
-use log::debug;
+use indicatif::ProgressStyle;
+use itertools::Itertools;
+use log::{debug, info, warn};
+use sha1_smol::Digest;
 use symbolic::common::ByteView;
 use zip::write::SimpleFileOptions;
-use zip::ZipWriter;
+use zip::{DateTime, ZipWriter};
 
+use crate::api::{Api, AuthenticatedApi};
+use crate::config::Config;
 use crate::utils::args::ArgExt;
+use crate::utils::chunks::{upload_chunks, Chunk, ASSEMBLE_POLL_INTERVAL};
+use crate::utils::fs::get_sha1_checksums;
 use crate::utils::fs::TempFile;
 use crate::utils::mobile_app::{is_aab_file, is_apk_file, is_xcarchive_directory, is_zip_file};
+use crate::utils::progress::ProgressBar;
+use crate::utils::vcs;
 
 pub fn make_command(command: Command) -> Command {
     command
@@ -24,13 +34,31 @@ pub fn make_command(command: Command) -> Command {
                 .num_args(1..)
                 .action(ArgAction::Append),
         )
+        .arg(
+            Arg::new("sha")
+                .long("sha")
+                .help("The git commit sha to use for the upload. If not provided, the current commit sha will be used.")
+        )
+        .arg(
+            Arg::new("build_configuration")
+                .long("build-configuration")
+                .help("The build configuration to use for the upload. If not provided, the current version will be used.")
+        )
 }
 
 pub fn execute(matches: &ArgMatches) -> Result<()> {
     let path_strings = matches
         .get_many::<String>("paths")
         .expect("paths argument is required");
 
+    let sha = matches
+        .get_one("sha")
+        .map(String::as_str)
+        .map(Cow::Borrowed)
+        .or_else(|| vcs::find_head().ok().map(Cow::Owned));
+
+    let build_configuration = matches.get_one("build_configuration").map(String::as_str);
+
     debug!(
         "Starting mobile app upload for {} paths",
         path_strings.len()
@@ -77,15 +105,54 @@ pub fn execute(matches: &ArgMatches) -> Result<()> {
             "Successfully normalized to: {}",
             normalized_zip.path().display()
         );
-        normalized_zips.push(normalized_zip);
+        normalized_zips.push((path, normalized_zip));
+    }
+
+    let config = Config::current();
+    let (org, project) = config.get_org_and_project(matches)?;
+
+    let mut uploaded_paths = vec![];
+    let mut errored_paths = vec![];
+    for (path, zip) in normalized_zips {
+        info!("Uploading file: {}", path.display());
+        let bytes = ByteView::open(zip.path())?;
+        match upload_file(&bytes, &org, &project, sha.as_deref(), build_configuration) {
+            Ok(_) => {
+                info!("Successfully uploaded file: {}", path.display());
+                uploaded_paths.push(path.to_path_buf());
+            }
+            Err(e) => {
+                debug!("Failed to upload file at path {}: {}", path.display(), e);
+                errored_paths.push(path.to_path_buf());
+            }
+        }
+    }
+
+    if !errored_paths.is_empty() {
+        warn!(
+            "Failed to upload {} file{}:",
+            errored_paths.len(),
+            if errored_paths.len() == 1 { "" } else { "s" }
+        );
+        for path in errored_paths {
+            warn!("  - {}", path.display());
+        }
     }
 
-    for zip in normalized_zips {
-        println!("Created normalized zip at: {}", zip.path().display());
-        // TODO: Upload the normalized zip to the chunked uploads API
+    println!(
+        "Successfully uploaded {} file{} to Sentry",
+        uploaded_paths.len(),
+        if uploaded_paths.len() == 1 { "" } else { "s" }
+    );
+    if uploaded_paths.len() < 3 {
+        for path in &uploaded_paths {
+            println!("  - {}", path.display());
+        }
     }
 
-    eprintln!("Uploading mobile app files to a project is not yet implemented.");
+    if uploaded_paths.is_empty() {
+        bail!("Failed to upload any files");
+    }
     Ok(())
 }
 
@@ -133,7 +200,15 @@ fn normalize_file(path: &Path, bytes: &[u8]) -> Result<TempFile> {
         .with_context(|| format!("Failed to get relative path for {}", path.display()))?;
 
     debug!("Adding file to zip: {}", file_name);
-    zip.start_file(file_name, SimpleFileOptions::default())?;
+
+    // Need to set the last modified time to a fixed value to ensure consistent checksums
+    // This is important as an optimization to avoid re-uploading the same chunks if they're already on the server
+    // but the last modified time being different will cause checksums to be different.
+    let options = SimpleFileOptions::default()
+        .compression_method(zip::CompressionMethod::Stored)
+        .last_modified_time(DateTime::default());
+
+    zip.start_file(file_name, options)?;
     zip.write_all(bytes)?;
 
     zip.finish()?;
@@ -149,24 +224,38 @@ fn normalize_directory(path: &Path) -> Result<TempFile> {
     let mut zip = ZipWriter::new(temp_file.open()?);
 
     let mut file_count = 0;
-    for entry in walkdir::WalkDir::new(path)
+
+    // Collect and sort entries for deterministic ordering
+    // This is important to ensure stable sha1 checksums for the zip file as
+    // an optimization is used to avoid re-uploading the same chunks if they're already on the server.
+    let entries = walkdir::WalkDir::new(path)
         .follow_links(true)
         .into_iter()
         .filter_map(Result::ok)
-    {
-        let entry_path = entry.path();
-        if entry_path.is_file() {
-            let relative_path = entry_path.strip_prefix(path)?;
-            debug!("Adding file to zip: {}", relative_path.display());
-
-            zip.start_file(
-                relative_path.to_string_lossy(),
-                SimpleFileOptions::default(),
-            )?;
-            let file_byteview = ByteView::open(entry_path)?;
-            zip.write_all(file_byteview.as_slice())?;
-            file_count += 1;
-        }
+        .filter(|entry| entry.path().is_file())
+        .map(|entry| {
+            let entry_path = entry.into_path();
+            let relative_path = entry_path.strip_prefix(path)?.to_owned();
+            Ok((entry_path, relative_path))
+        })
+        .collect::<Result<Vec<_>>>()?
+        .into_iter()
+        .sorted_by(|(_, a), (_, b)| a.cmp(b));
+
+    // Need to set the last modified time to a fixed value to ensure consistent checksums
+    // This is important as an optimization to avoid re-uploading the same chunks if they're already on the server
+    // but the last modified time being different will cause checksums to be different.
+    let options = SimpleFileOptions::default()
+        .compression_method(zip::CompressionMethod::Stored)
+        .last_modified_time(DateTime::default());
+
+    for (entry_path, relative_path) in entries {
+        debug!("Adding file to zip: {}", relative_path.display());
+
+        zip.start_file(relative_path.to_string_lossy(), options)?;
+        let file_byteview = ByteView::open(&entry_path)?;
+        zip.write_all(file_byteview.as_slice())?;
+        file_count += 1;
     }
 
     zip.finish()?;
@@ -176,3 +265,117 @@ fn normalize_directory(path: &Path) -> Result<TempFile> {
     );
     Ok(temp_file)
 }
+
+fn upload_file(
+    bytes: &[u8],
+    org: &str,
+    project: &str,
+    sha: Option<&str>,
+    build_configuration: Option<&str>,
+) -> Result<()> {
+    debug!(
+        "Uploading file to organization: {}, project: {}, sha: {}, build_configuration: {}",
+        org,
+        project,
+        sha.unwrap_or("unknown"),
+        build_configuration.unwrap_or("unknown")
+    );
+
+    let api = Api::current();
+    let authenticated_api = api.authenticated()?;
+
+    let chunk_upload_options = authenticated_api
+        .get_chunk_upload_options(org)?
+        .expect("Chunked uploading is not supported");
+
+    let progress_style =
+        ProgressStyle::default_spinner().template("{spinner} Optimizing bundle for upload...");
+    let pb = ProgressBar::new_spinner();
+    pb.enable_steady_tick(100);
+    pb.set_style(progress_style);
+
+    let chunk_size = chunk_upload_options.chunk_size as usize;
+    let (checksum, checksums) = get_sha1_checksums(bytes, chunk_size)?;
+    let mut chunks = bytes
+        .chunks(chunk_size)
+        .zip(checksums.iter())
+        .map(|(data, checksum)| Chunk((*checksum, data)))
+        .collect::<Vec<_>>();
+
+    pb.finish_with_duration("Finishing upload");
+
+    let response = authenticated_api.assemble_mobile_app(
+        org,
+        project,
+        checksum,
+        &checksums,
+        sha,
+        build_configuration,
+    )?;
+    chunks.retain(|Chunk((digest, _))| response.missing_chunks.contains(digest));
+
+    if !chunks.is_empty() {
+        let upload_progress_style = ProgressStyle::default_bar().template(
+            "{prefix:.dim} Uploading files...\
+             \n{wide_bar}  {bytes}/{total_bytes} ({eta})",
+        );
+        upload_chunks(&chunks, &chunk_upload_options, upload_progress_style)?;
+    } else {
+        println!("Nothing to upload, all files are on the server");
+    }
+
+    poll_assemble(
+        &authenticated_api,
+        checksum,
+        &checksums,
+        org,
+        project,
+        sha,
+        build_configuration,
+    )?;
+    Ok(())
+}
+
+fn poll_assemble(
+    api: &AuthenticatedApi,
+    checksum: Digest,
+    chunks: &[Digest],
+    org: &str,
+    project: &str,
+    sha: Option<&str>,
+    build_configuration: Option<&str>,
+) -> Result<()> {
+    debug!("Polling assemble for checksum: {}", checksum);
+
+    let progress_style = ProgressStyle::default_spinner().template("{spinner} Processing files...");
+    let pb = ProgressBar::new_spinner();
+
+    pb.enable_steady_tick(100);
+    pb.set_style(progress_style);
+
+    let response = loop {
+        let response =
+            api.assemble_mobile_app(org, project, checksum, chunks, sha, build_configuration)?;
+
+        if response.state.is_finished() {
+            break response;
+        }
+
+        std::thread::sleep(ASSEMBLE_POLL_INTERVAL);
+    };
+
+    pb.finish_with_duration("Processing");
+
+    if response.state.is_err() {
+        let message = response.detail.as_deref().unwrap_or("unknown error");
+        bail!("Failed to process uploaded files: {}", message);
+    }
+
+    if response.state.is_pending() {
+        info!("File upload complete (processing pending on server)");
+    } else {
+        info!("File processing complete");
+    }
+
+    Ok(())
+}