Merge pull request #191 from xylar/setup_to_custom_suite

Make `compass setup` create a custom test suite

Author: xylar

compass/clean.py

@@ -7,7 +7,7 @@
 from compass import provenance
 
 
-def clean_cases(tests=None, numbers=None, work_dir=None):
+def clean_cases(tests=None, numbers=None, work_dir=None, suite_name='custom'):
     """
     Set up one or more test cases
 
@@ -21,13 +21,18 @@ def clean_cases(tests=None, numbers=None, work_dir=None):
 
     work_dir : str, optional
         A directory that will serve as the base for creating case directories
+
+    suite_name : str, optional
+        The name of the test suite if tests are being set up through a test
+        suite or ``'custom'`` if not
     """
 
     if tests is None and numbers is None:
         raise ValueError('At least one of tests or numbers is needed.')
 
     if work_dir is None:
         work_dir = os.getcwd()
+    work_dir = os.path.abspath(work_dir)
 
     mpas_cores = get_mpas_cores()
     all_test_cases = dict()
@@ -65,6 +70,14 @@ def clean_cases(tests=None, numbers=None, work_dir=None):
         except OSError:
             pass
 
+    # delete the pickle file for the test suite (if any)
+    pickle_file = os.path.join(work_dir, '{}.pickle'.format(suite_name))
+
+    try:
+        os.remove(pickle_file)
+    except OSError:
+        pass
+
 
 def main():
     parser = argparse.ArgumentParser(

compass/setup.py

@@ -11,7 +11,8 @@
 
 
 def setup_cases(tests=None, numbers=None, config_file=None, machine=None,
-                work_dir=None, baseline_dir=None, mpas_model_path=None):
+                work_dir=None, baseline_dir=None, mpas_model_path=None,
+                suite_name='custom'):
     """
     Set up one or more test cases
 
@@ -41,6 +42,10 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None,
         The relative or absolute path to the root of a branch where the MPAS
         model has been built
 
+    suite_name : str, optional
+        The name of the test suite if tests are being set up through a test
+        suite or ``'custom'`` if not
+
     Returns
     -------
     test_cases : dict of compass.TestCase
@@ -58,6 +63,7 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None,
 
     if work_dir is None:
         work_dir = os.getcwd()
+    work_dir = os.path.abspath(work_dir)
 
     mpas_cores = get_mpas_cores()
 
@@ -97,6 +103,26 @@ def setup_cases(tests=None, numbers=None, config_file=None, machine=None,
         setup_case(path, test_case, config_file, machine, work_dir,
                    baseline_dir, mpas_model_path)
 
+    test_suite = {'name': suite_name,
+                  'test_cases': test_cases,
+                  'work_dir': work_dir}
+
+    # pickle the test or step dictionary for use at runtime
+    pickle_file = os.path.join(test_suite['work_dir'],
+                               '{}.pickle'.format(suite_name))
+    with open(pickle_file, 'wb') as handle:
+        pickle.dump(test_suite, handle, protocol=pickle.HIGHEST_PROTOCOL)
+
+    if 'LOAD_COMPASS_ENV' in os.environ:
+        script_filename = os.environ['LOAD_COMPASS_ENV']
+        # make a symlink to the script for loading the compass conda env.
+        symlink(script_filename, os.path.join(work_dir, 'load_compass_env.sh'))
+
+    max_cores, max_of_min_cores = _get_required_cores(test_cases)
+
+    print('target cores: {}'.format(max_cores))
+    print('minimum cores: {}'.format(max_of_min_cores))
+
     return test_cases
 
 
@@ -274,6 +300,10 @@ def main():
                         help="The path to the build of the MPAS model for the "
                              "core.",
                         metavar="PATH")
+    parser.add_argument("--suite_name", dest="suite_name", default="custom",
+                        help="The name to use for the 'custom' test suite"
+                             "containing all setup test cases.",
+                        metavar="SUITE")
 
     args = parser.parse_args(sys.argv[2:])
     if args.test is None:
@@ -283,4 +313,18 @@ def main():
     setup_cases(tests=tests, numbers=args.case_num,
                 config_file=args.config_file, machine=args.machine,
                 work_dir=args.work_dir, baseline_dir=args.baseline_dir,
-                mpas_model_path=args.mpas_model)
+                mpas_model_path=args.mpas_model, suite_name=args.suite_name)
+
+
+def _get_required_cores(test_cases):
+    """ Get the maximum number of target cores and the max of min cores """
+
+    max_cores = 0
+    max_of_min_cores = 0
+    for test_case in test_cases.values():
+        for step_name in test_case.steps_to_run:
+            step = test_case.steps[step_name]
+            max_cores = max(max_cores, step.cores)
+            max_of_min_cores = max(max_of_min_cores, step.min_cores)
+
+    return max_cores, max_of_min_cores

compass/suite.py

@@ -1,11 +1,8 @@
 import argparse
 import sys
-import os
 from importlib import resources
-import pickle
 
 from compass.setup import setup_cases
-from compass.io import symlink
 from compass.clean import clean_cases
 
 
@@ -43,12 +40,6 @@ def setup_suite(mpas_core, suite_name, config_file=None, machine=None,
         The relative or absolute path to the root of a branch where the MPAS
         model has been built
     """
-    if machine is None and 'COMPASS_MACHINE' in os.environ:
-        machine = os.environ['COMPASS_MACHINE']
-
-    if config_file is None and machine is None:
-        raise ValueError('At least one of config_file and machine is needed.')
-
     text = resources.read_text('compass.{}.suites'.format(mpas_core),
                                '{}.txt'.format(suite_name))
     tests = list()
@@ -58,33 +49,9 @@ def setup_suite(mpas_core, suite_name, config_file=None, machine=None,
                 and not test.startswith('#')):
             tests.append(test)
 
-    if work_dir is None:
-        work_dir = os.getcwd()
-    work_dir = os.path.abspath(work_dir)
-
-    test_cases = setup_cases(tests, config_file=config_file, machine=machine,
-                             work_dir=work_dir, baseline_dir=baseline_dir,
-                             mpas_model_path=mpas_model_path)
-
-    test_suite = {'name': suite_name,
-                  'test_cases': test_cases,
-                  'work_dir': work_dir}
-
-    # pickle the test or step dictionary for use at runtime
-    pickle_file = os.path.join(test_suite['work_dir'],
-                               '{}.pickle'.format(suite_name))
-    with open(pickle_file, 'wb') as handle:
-        pickle.dump(test_suite, handle, protocol=pickle.HIGHEST_PROTOCOL)
-
-    if 'LOAD_COMPASS_ENV' in os.environ:
-        script_filename = os.environ['LOAD_COMPASS_ENV']
-        # make a symlink to the script for loading the compass conda env.
-        symlink(script_filename, os.path.join(work_dir, 'load_compass_env.sh'))
-
-    max_cores, max_of_min_cores = _get_required_cores(test_cases)
-
-    print('target cores: {}'.format(max_cores))
-    print('minimum cores: {}'.format(max_of_min_cores))
+    setup_cases(tests, config_file=config_file, machine=machine,
+                work_dir=work_dir, baseline_dir=baseline_dir,
+                mpas_model_path=mpas_model_path, suite_name=suite_name)
 
 
 def clean_suite(mpas_core, suite_name, work_dir=None):
@@ -111,19 +78,7 @@ def clean_suite(mpas_core, suite_name, work_dir=None):
     tests = [test.strip() for test in text.split('\n') if
              len(test.strip()) > 0 and not test.startswith('#')]
 
-    if work_dir is None:
-        work_dir = os.getcwd()
-    work_dir = os.path.abspath(work_dir)
-
-    clean_cases(tests=tests, work_dir=work_dir)
-
-    # delete the pickle file
-    pickle_file = os.path.join(work_dir, '{}.pickle'.format(suite_name))
-
-    try:
-        os.remove(pickle_file)
-    except OSError:
-        pass
+    clean_cases(tests=tests, work_dir=work_dir, suite_name=suite_name)
 
 
 def main():
@@ -173,16 +128,3 @@ def main():
                     config_file=args.config_file, machine=args.machine,
                     work_dir=args.work_dir, baseline_dir=args.baseline_dir,
                     mpas_model_path=args.mpas_model)
-
-
-def _get_required_cores(test_cases):
-    """ Get the maximum number of target cores and the max of min cores """
-
-    max_cores = 0
-    max_of_min_cores = 0
-    for test_case in test_cases.values():
-        for step in test_case.steps.values():
-            max_cores = max(max_cores, step.cores)
-            max_of_min_cores = max(max_of_min_cores, step.min_cores)
-
-    return max_cores, max_of_min_cores

docs/developers_guide/command_line.rst

@@ -95,7 +95,7 @@ The command-line options are:
 .. code-block:: none
 
     compass setup [-h] [-t PATH] [-n NUM [NUM ...]] [-f FILE] [-m MACH]
-                  [-w PATH] [-b PATH] [-p PATH]
+                  [-w PATH] [-b PATH] [-p PATH] [--suite_name SUITE]
 
 The ``-h`` or ``--help`` options will display the help message describing the
 command-line options.
@@ -142,8 +142,18 @@ previous run.  Many test cases validate variables to make sure they are
 identical between runs, compare timers to see how much performance has changed,
 or both.  See :ref:`dev_validation`.
 
-See :ref:`dev_setup` for more about the underlying framework.
+The test cases will be included in a "custom" test suite in the order they are
+named or numbered.  You can give this suite a name with ``--suite_name`` or
+leave it with the default name ``custom``.  You can run this test suite with
+``compass run [suite_name]`` as with the predefined test suites (see
+:ref:`dev_compass_suite`).
+
+Test cases within the custom suite are run in the order they are supplied to
+``compass setup``, so keep this in mind when providing the list.  Any test
+cases that depend on the output of other test cases must run afther their
+dependencies.
 
+See :ref:`dev_setup` for more about the underlying framework.
 
 .. _dev_compass_clean:
 
@@ -228,9 +238,10 @@ Whereas other ``compass`` commands are typically run in the local clone of the
 compass repo, ``compass run`` needs to be run in the appropriate work
 directory. If you are running a test suite, you may need to provide the name
 of the test suite if more than one suite has been set up in the same work
-directory (with or without the ``.pickle`` suffix that exists on the suite's
-file in the working directory).  If you are in the work directory for a test
-case or step, you do not need to provide any arguments.
+directory.  You can provide either just the suite name or
+``<suite_name>.pickle`` (the latter is convenient for tab completion).  If you
+are in the work directory for a test case or step, you do not need to provide
+any arguments.
 
 If you want to explicitly select which steps in a test case you want to run,
 you have two options.  You can either edit the ``steps_to_run`` config options

docs/users_guide/quick_start.rst

@@ -79,8 +79,8 @@ Each time you want to work with compass, you will need to run:
 
 .. _setup_overview:
 
-Setting up a test case
-----------------------
+Setting up test cases
+---------------------
 
 Before you set up a test case with ``compass``, you will need to build the
 MPAS component you wish to test with.  Since the instructions for building
@@ -213,6 +213,11 @@ in the repository.
 In order to run a bit-for-bit test with a previous test case, use
 ``-b $PREVIOUS_WORKDIR`` to specify a "baseline".
 
+When you set up one or more test cases, they will also be included in a custom
+test suite, which is called ``custom`` by default.  (You can give it another
+name with the ``--suite_name`` flag.)  You can run all the test cases in
+sequence with one command as described in :ref:`suite_overview` or run them
+one at a time as follows.
 
 Running a test case
 -------------------