Adds default config for run_evaluation

config/evaluate/config_zarr2cf.yaml

@@ -0,0 +1,135 @@
+variables:
+  q:
+    var: q
+    long: specific_humidity_on_pressure_level
+    std: specific_humidity
+    wg_unit: kg kg**-1
+    std_unit: kg kg-1
+    level_type: pl
+  t:
+    var: t
+    long: temperature_on_pressure_levels
+    std: air_temperature
+    wg_unit: K
+    std_unit: K
+    level_type: pl
+  u:
+    var: u
+    long: u_wind_on_pressure_levels
+    std: x_wind
+    wg_unit: m s**-1
+    std_unit: m s-1
+    level_type: pl
+  v:
+    var: v
+    long: v_wind_on_pressure_levels
+    std: y_wind
+    wg_unit: m s**-1
+    std_unit: m s-1
+    level_type: pl
+  z:
+    var: z
+    long: geopotential_height_on_pressure_levels
+    std: geopotential_height
+    wg_unit: m**2 s**-2
+    std_unit: m
+    level_type: pl
+  10u:
+    var: u10
+    long: u_wind_at_10m
+    std: x_wind
+    wg_unit: m s**-1
+    std_unit: m s-1
+    level_type: sfc
+  10v:
+    var: v10
+    long: v_wind_at_10m
+    std: y_wind
+    wg_unit: m s**-1
+    std_unit: m s-1
+    level_type: sfc
+  2d:
+    var: d2m
+    long: 2m_dewpoint_temperature
+    std: dew_point_temperature
+    wg_unit: K
+    std_unit: K
+    level_type: sfc
+  2t:
+    var: t2m
+    long: 2m_temperature
+    std: air_temperature
+    #near-surface (usually, 2 meter) : https://pcmdi.llnl.gov/mips/cmip3/variableList.html
+    wg_unit: K
+    std_unit: K
+    level_type: sfc
+  msl:
+    var: msl
+    long: mean_sea_level_pressure
+    std: air_pressure_at_mean_sea_level
+    wg_unit: Pa
+    std_unit: Pa
+    level_type: sfc
+  skt:
+    var: skt
+    # The standard name for skin temperature is sea_surface_skin_temperature, which is defined as "The temperature of the very thin layer at the surface of the sea that is in contact with the atmosphere".
+  # It is strongly recommended that a variable with this standard name should have a units_metadata attribute, with one of the values "on-scale" or "difference", whichever is appropriate for the data, because it is essential to know whether the temperature is on-scale (meaning relative to the origin of the scale indicated by the units) or refers to temperature differences (implying that the origin of the temperature scale is irrevelant)
+    long: skin_temperature
+    std: sea_surface_skin_temperature
+    wg_unit: K
+    std_unit: K
+    level_type: sfc
+  sp:
+    var: sp
+    long: surface_pressure
+    std: surface_air_pressure
+    wg_unit: Pa
+    std_unit: Pa
+    level_type: sfc
+
+
+coordinates:
+    sfc:
+        valid_time: valid_time
+        lat: latitude
+        lon: longitude
+        stream: stream
+        forecast_step: forecast_period
+        forecast_ref_time: forecast_ref_time
+    pl:
+        pressure_level: pressure
+        valid_time: valid_time
+        lat: latitude
+        lon: longitude
+        stream: stream
+        forecast_step: forecast_period
+        forecast_ref_time: forecast_ref_time
+dimensions:
+    valid_time:
+      wg: valid_time
+      std: time
+    lat: 
+      wg: latitude
+      std: latitude
+      std_unit: degrees_north
+    lon: 
+      wg: longitude
+      std: longitude
+      std_unit: degrees_east
+    pressure_level: 
+      wg: pressure
+      std: air_pressure
+      std_unit: hPa
+    forecast_ref_time: 
+      wg: forecast_ref_time
+      std: forecast_ref_time
+    forecast_step:
+      wg: forecast_period
+      std: forecast_period
+    stream:
+      wg: stream
+      std: stream
+    #TODO maybe absorb stream as an attribute
+    # forecast_step: forecast_step
+    # don't want forecast_step anyway
+

config/evaluate/plot_config_template.yml

@@ -0,0 +1,64 @@
+image_format : "png" #options: "png", "pdf", "svg", "eps", "jpg" ..
+dpi_val : 300
+summary_plots : true
+summary_dir: "./plots/"
+print_summary: false #print out score values on screen. it can be verbose
+
+evaluation:
+  metrics  : ["rmse", "mae"]
+
+# global_plotting_options:
+#   ERA5:
+#     2t: 
+#       vmin: 250
+#       vmax: 300
+#     marker_size: 2
+
+#MTM models
+run_ids :
+
+  grwnhykd:
+     results_base_dir : "/p/scratch/weatherai/shared_work/results/"
+     epoch: 0
+     rank: 0
+     streams:
+       ERA5:
+          channels: ["2t", "t_850", "q_850"]
+          evaluation:
+            forecast_step: "all"
+            sample: "all"
+          plotting: 
+            sample: [0]
+            forecast_step: "all"
+            plot_maps: true
+            plot_histograms: false
+
+       NPPATMS:
+          channels: ["obsvalue_rawbt_1", "obsvalue_rawbt_5"]
+          evaluation: 
+            forecast_step: "all"
+            sample: "all"
+          plotting: 
+            sample: [0]
+            forecast_step: [1, 2]
+            plot_maps: false
+            plot_histograms: false
+            plot_animations: true
+
+  z51gxtjl:
+    label: "forecast ERA5"
+    epoch: 0
+    rank: 0
+    streams:
+      ERA5:
+        channels: ["2t", "t_850"]
+        evaluation:
+          forecast_step: "all"
+          sample: "all"
+        plotting: 
+          sample: [0]
+          forecast_step: [1, 2, 3, 4]
+          plot_maps: true
+          plot_histograms: true 
+          plot_animations: false
+

packages/evaluate/src/weathergen/evaluate/io_reader.py

@@ -470,10 +470,20 @@ def get_data(
                     if len(samples) == 1:
                         # Ensure sample coordinate is repeated along ipoint even if only one sample
                         da_tars_fs = da_tars_fs.assign_coords(
-                            sample=("ipoint", np.repeat(da_tars_fs.sample.values, len(da_tars_fs.ipoint)))
+                            sample=(
+                                "ipoint",
+                                np.repeat(
+                                    da_tars_fs.sample.values, len(da_tars_fs.ipoint)
+                                ),
+                            )
                         )
                         da_preds_fs = da_preds_fs.assign_coords(
-                            sample=("ipoint", np.repeat(da_preds_fs.sample.values, len(da_preds_fs.ipoint)))
+                            sample=(
+                                "ipoint",
+                                np.repeat(
+                                    da_preds_fs.sample.values, len(da_preds_fs.ipoint)
+                                ),
+                            )
                         )
 
                     if set(channels) != set(all_channels):

packages/evaluate/src/weathergen/evaluate/run_evaluation.py

@@ -30,6 +30,8 @@
 
 _DEFAULT_PLOT_DIR = _REPO_ROOT / "plots"
 
+_DEFAULT_EVAL_CONFIG_DIR = _REPO_ROOT / "config/evaluate"
+
 
 def evaluate() -> None:
     # By default, arguments from the command line are read.
@@ -47,7 +49,12 @@ def evaluate_from_args(argl: list[str]) -> None:
     )
 
     args = parser.parse_args(argl)
-    evaluate_from_config(OmegaConf.load(args.config))
+    if args.config:
+        config = Path(args.config)
+    else:
+        _logger.info("No config file provided, using the default template config.")
+        config = Path(_DEFAULT_EVAL_CONFIG_DIR / "plot_config_template.yml")
+    evaluate_from_config(OmegaConf.load(config))
 
 
 def evaluate_from_config(cfg):