diff --git a/script/get-preprocessed-dataset-cognata/meta.yaml b/script/get-preprocessed-dataset-cognata/meta.yaml index 940351310..c33caf57a 100644 --- a/script/get-preprocessed-dataset-cognata/meta.yaml +++ b/script/get-preprocessed-dataset-cognata/meta.yaml @@ -38,22 +38,42 @@ variations: env: MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: val_2d MLC_DATASET_COGNATA_TAR_FILENAME: val_2d.tar.gz + validation,2d_obj_det,rclone: + env: MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_2d/<<>> + validation,2d_obj_det,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/val_2d.uri calibration,2d_obj_det: env: MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: calib_2d MLC_DATASET_COGNATA_TAR_FILENAME: calib_2d.tar.gz + calibration,2d_obj_det,rclone: + env: MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_2d/<<>> + calibration,2d_obj_det,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/calib_2d.uri validation,segmentation: env: MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: val_seg MLC_DATASET_COGNATA_TAR_FILENAME: val_seg.tar.gz + validation,segmentation,rclone: + env: MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_seg/<<>> + validation,segmentation,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/val_seg.uri calibration,segmentation: env: MLC_DATASET_COGNATA_EXTRACTED_FOLDER_NAME: calib_seg MLC_DATASET_COGNATA_TAR_FILENAME: calib_seg.tar.gz + calibration,segmentation,rclone: + env: MLC_DOWNLOAD_URL: mlc-cognata:mlc_cognata_dataset/preprocessed_seg/<<>> + calibration,segmentation,r2-downloader: + env: + MLC_DOWNLOAD_URL: https://cognata.mlcommons-storage.org/metadata/calib_seg.uri prebuilt: default: true group: dataset-src @@ -61,22 +81,13 @@ variations: MLC_NUSCENES_DATASET_TYPE: prebuilt MLC_DOWNLOAD_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_COGNATA_PATH MLC_EXTRACT_FINAL_ENV_NAME: MLC_PREPROCESSED_DATASET_COGNATA_PATH + prebuilt,rclone: + env: MLC_DOWNLOAD_EXTRA_OPTIONS: ' --include ' mlc: group: download-src default: true prehook_deps: - - tags: get,rclone - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - - tags: get,rclone-config,_config-name.mlc-cognata - force_cache: true - enable_if_env: - MLC_TMP_REQUIRE_DOWNLOAD: - - yes - env: - MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 - enable_if_env: MLC_TMP_REQUIRE_DOWNLOAD: - 'yes' @@ -92,6 +103,19 @@ variations: - MLC_DOWNLOAD_URL env: MLC_DOWNLOAD_SRC: mlcommons + mlc,rclone: + prehook_deps: + - tags: get,rclone + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + - tags: get,rclone-config,_config-name.mlc-cognata + force_cache: true + enable_if_env: + MLC_TMP_REQUIRE_DOWNLOAD: + - yes + env: + MLC_RCLONE_DRIVE_FOLDER_ID: 1u5FDoeXHVtDrd4zClE47Gmyr7iLFidz1 rclone: group: download-tool add_deps_recursive: @@ -105,10 +129,23 @@ variations: dry-run,rclone: env: MLC_DOWNLOAD_EXTRA_OPTIONS: --dry-run + r2-downloader: + group: download-tool + add_deps_recursive: + dae: + tags: _r2-downloader + dry-run,r2-downloader: + env: + MLC_DOWNLOAD_EXTRA_OPTIONS: -x tests: + needs_pat: true run_inputs: - variations_list: - validation,prebuilt,2d_obj_det,rclone,mlc,dry-run - calibration,prebuilt,2d_obj_det,rclone,mlc,dry-run - validation,prebuilt,segmentation,rclone,mlc,dry-run - - calibration,prebuilt,segmentation,rclone,mlc,dry-run \ No newline at end of file + - calibration,prebuilt,segmentation,rclone,mlc,dry-run + - validation,prebuilt,2d_obj_det,r2-downloader,mlc,dry-run + - calibration,prebuilt,2d_obj_det,r2-downloader,mlc,dry-run + - validation,prebuilt,segmentation,r2-downloader,mlc,dry-run + - calibration,prebuilt,segmentation,r2-downloader,mlc,dry-run diff --git a/script/get-preprocessed-dataset-cognata/run.sh b/script/get-preprocessed-dataset-cognata/run.sh index 0c141b8e6..4e9dbe810 100644 --- a/script/get-preprocessed-dataset-cognata/run.sh +++ b/script/get-preprocessed-dataset-cognata/run.sh @@ -3,7 +3,7 @@ if [[ "$MLC_DOWNLOAD_MODE" != "dry" && "$MLC_TMP_REQUIRE_DOWNLOAD" = "yes" ]]; then cd "${MLC_PREPROCESSED_DATASET_COGNATA_PATH}" || exit for f in *.tar.gz; do - tar -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } + tar --no-same-owner -xzvf "$f" || { echo "Failed to extract $f"; exit 1; } done cd - || exit fi \ No newline at end of file