Observations feature branch

.gitignore

@@ -146,3 +146,6 @@ trace.txt
 *.prof
 prof/
 *.gz
+*.odb
+*.bufr
+*.csv

src/anemoi/datasets/__init__.py

@@ -8,16 +8,16 @@
 # nor does it submit to any jurisdiction.
 
 
-from .data import MissingDateError
-from .data import add_dataset_path
-from .data import add_named_dataset
-from .data import list_dataset_names
-from .data import open_dataset
+from anemoi.datasets.use.gridded import MissingDateError
+from anemoi.datasets.use.gridded import add_dataset_path
+from anemoi.datasets.use.gridded import add_named_dataset
+from anemoi.datasets.use.gridded import list_dataset_names
+from anemoi.datasets.use.gridded import open_dataset
 
 try:
     # NOTE: the `_version.py` file must not be present in the git repository
     #   as it is generated by setuptools at install time
-    from ._version import __version__  # type: ignore
+    from anemoi.datasets._version import __version__  # type: ignore
 except ImportError:  # pragma: no cover
     # Local copy or not installed with setuptools
     __version__ = "999"

src/anemoi/datasets/commands/create.py

@@ -45,11 +45,11 @@ def task(what: str, options: dict, *args: Any, **kwargs: Any) -> Any:
     now = datetime.datetime.now()
     LOG.info(f"🎬 Task {what}({args},{kwargs}) starting")
 
-    from anemoi.datasets.create import creator_factory
+    from anemoi.datasets.create.tasks import task_factory
 
     options = {k: v for k, v in options.items() if v is not None}
 
-    c = creator_factory(what.replace("-", "_"), **options)
+    c = task_factory(what.replace("-", "_"), **options)
     result = c.run()
 
     LOG.info(f"🏁 Task {what}({args},{kwargs}) completed ({datetime.datetime.now()-now})")

src/anemoi/datasets/commands/inspect.py

@@ -27,8 +27,8 @@
 from numpy.typing import NDArray
 
 from anemoi.datasets import open_dataset
-from anemoi.datasets.data.stores import open_zarr
-from anemoi.datasets.data.stores import zarr_lookup
+from anemoi.datasets.use.gridded.stores import dataset_lookup
+from anemoi.datasets.use.gridded.stores import open_zarr
 
 from . import Command
 
@@ -810,7 +810,7 @@ def _info(self, path: str) -> Version:
         Version
             The version object of the dataset.
         """
-        z = open_zarr(zarr_lookup(path))
+        z = open_zarr(dataset_lookup(path))
 
         metadata = dict(z.attrs)
         version = metadata.get("version", "0.0.0")

src/anemoi/datasets/commands/recipe/__init__.py

@@ -15,8 +15,6 @@
 
 import yaml
 
-from anemoi.datasets.create import validate_config
-
 from .. import Command
 from .format import format_recipe
 from .migrate import migrate_recipe
@@ -37,6 +35,7 @@ def add_arguments(self, command_parser: Any) -> None:
         command_parser.add_argument("--validate", action="store_true", help="Validate recipe.")
         command_parser.add_argument("--format", action="store_true", help="Format the recipe.")
         command_parser.add_argument("--migrate", action="store_true", help="Migrate the recipe to the latest version.")
+        command_parser.add_argument("--python", action="store_true", help="Convert the recipe to a Python script.")
 
         group = command_parser.add_mutually_exclusive_group()
         group.add_argument("--inplace", action="store_true", help="Overwrite the recipe file in place.")
@@ -49,7 +48,7 @@ def add_arguments(self, command_parser: Any) -> None:
 
     def run(self, args: Any) -> None:
 
-        if not args.validate and not args.format and not args.migrate:
+        if not args.validate and not args.format and not args.migrate and not args.python:
             args.validate = True
 
         with open(args.path) as file:
@@ -58,10 +57,12 @@ def run(self, args: Any) -> None:
         assert isinstance(config, dict)
 
         if args.validate:
-            if args.inplace and (not args.format and not args.migrate):
+            from anemoi.datasets.create.gridded.tasks import validate_config
+
+            if args.inplace and (not args.format and not args.migrate and not args.python):
                 argparse.ArgumentError(None, "--inplace is not supported with --validate.")
 
-            if args.output and (not args.format and not args.migrate):
+            if args.output and (not args.format and not args.migrate and not args.python):
                 argparse.ArgumentError(None, "--output is not supported with --validate.")
 
             validate_config(config)

src/anemoi/datasets/commands/recipe/format.py

@@ -1,4 +1,4 @@
-# (C) Copyright 2025 Anemoi contributors.
+# (C) Copyright 2024 Anemoi contributors.
 #
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
@@ -11,7 +11,7 @@
 import datetime
 import logging
 
-from ...dumper import yaml_dump
+from anemoi.datasets.misc.dumper import yaml_dump
 
 LOG = logging.getLogger(__name__)
 

src/anemoi/datasets/commands/recipe/migrate.py

@@ -17,7 +17,7 @@
 from glom import delete
 from glom import glom
 
-from anemoi.datasets.create import validate_config
+from anemoi.datasets.create.gridded.tasks import validate_config
 from anemoi.datasets.dumper import yaml_dump
 
 LOG = logging.getLogger(__name__)
@@ -417,7 +417,7 @@ def _fix_some(config: dict) -> None:
         node = glom(config, ".".join(p[:-1]))
         node.update(node.pop("<<"))
         parent[node.pop("name")] = node
-        assert len(parent) == 2
+        assert len(parent) == 2, parent
         del parent["source"]
 
     paths = find_paths_in_substrees("label.mars", config)