adapt github CI and provide a minimal data example that can be run for testing purposes

Author: liellnima

.github/workflows/climateset-emulator.yml

@@ -1,4 +1,4 @@
-name: ssp package
+name: ClimateSet Emulator Checks
 
 on: [push, pull_request]
 
@@ -26,17 +26,18 @@ jobs:
     - name: Setup environment 
       run: |
         bash setup.sh
-    - name: Download data 
+    - name: Test metrics and losses
       run: |
-        bash setup.sh -d
+        cd tests
+        pytest
+    # - name: Download data from original source
+    #   run: |
+    #     bash setup.sh -d
     - name: Test if you can run one single model
       run: |
         source env_emulator/bin/activate
         python emulator/run.py experiment=github_workflow.yaml
-    - name: Test metrics and losses
-      run: |
-        cd tests
-        pytest
+    
     # - name: Pipx build
     # - name: check dist
     # - name: publish dist (on TestPyPi)

Climateset_DATA/climate_model_NorESM2-LM_num_ensembles_1_years_2020-2040_historical_years_1890-1900_variables_tas_scenarios_['ssp126']_channels_last_False_seq_to_seq_True_seq__12_train+val_target.npz

@@ -0,0 +1,29 @@
+_target_: emulator.src.datamodules.climate_datamodule.ClimateDataModule
+
+in_var_ids: ['CO2_sum', 'CH4_sum']
+out_var_ids: ['tas']
+seq_to_seq: True
+batch_size: 4
+channels_last: False
+eval_batch_size: 4
+train_historical_years: "1890-1900"
+test_years: "2040-2050"
+train_years: "2020-2040"
+val_split: 0.1
+train_scenarios: ["ssp126"]
+test_scenarios: ["ssp245"]
+train_models:  ["NorESM2-LM"]
+num_ensembles: 1
+num_workers:  0
+pin_memory: False
+emissions_tracker: True
+load_train_into_mem: True
+load_test_into_mem:  True
+verbose: True
+seed: 11
+seq_len: 12
+lat: 96  # corrected latitude after fix
+lon: 144 # corrected longitude after fix
+num_levels: 1
+#input_transform: Optional[AbstractTransform] = None,
+#normalizer: Optional[Normalizer] = None,

Climateset_DATA/years_2020-2040_historical_years_1890-1900_variables_CO2_sum_CH4_sum_scenarios_['ssp126']_channels_last_False_openburning_specs_('no-fires', 'no-fires')_seq_to_seq_True_seq_len_12_train+val_input.npz

@@ -8,7 +8,7 @@ defaults:
   - override /model: conv_lstm.yaml # put the desired model name here
   - override /callbacks: default.yaml
   - override /logger: none.yaml
-  - override /datamodule: climate.yaml # standard datamodule configurations
+  - override /datamodule: mini-climate.yaml # standard datamodule configurations
 
 # all parameters below will be merged with parameters from default configurations set above
 # this allows you to overwrite only specified parameters
@@ -21,7 +21,7 @@ seed: 42
 
 trainer:
   min_epochs: 1
-  max_epochs: 3
+  max_epochs: 10
 
 model:
   loss_function: "climax_lon_lat_rmse"
@@ -30,12 +30,12 @@ model:
   pretrained_run_id: null 
   pretrained_ckpt_dir: null 
 
-datamodule: 
-   in_var_ids: ['BC_sum', 'CO2_sum', 'SO2_sum', 'CH4_sum']
-   out_var_ids: ['tas', 'pr']
-   train_historical_years: "1850-2010"
-   train_models:  ["NorESM2-LM"]
-   seq_to_seq: True 
-   batch_size: 4
-   channels_last: False
-   eval_batch_size: 4
+# datamodule: 
+#    in_var_ids: ['BC_sum', 'CO2_sum', 'SO2_sum', 'CH4_sum']
+#    out_var_ids: ['tas', 'pr']
+#    train_historical_years: "1850-2010"
+#    train_models:  ["NorESM2-LM"]
+#    seq_to_seq: True 
+#    batch_size: 4
+#    channels_last: False
+#    eval_batch_size: 4

emulator/configs/datamodule/mini-climate.yaml

@@ -13,7 +13,7 @@ on_cluster=false
 on_windows=false
 
 # Parse command line options
-while getopts pbcnh opt; do
+while getopts dmcrwh opt; do
   case ${opt} in
     d )
       get_data=true