From 1283707613ae207636526186faeadcd13fbc646e Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Thu, 31 Jul 2025 14:00:57 +0200 Subject: [PATCH 1/9] 1M test case which should also become CI [WIP] --- .../testsuite/tests/apps/openfoam/openfoam.py | 116 +++++++++++++++++- 1 file changed, 115 insertions(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/openfoam/openfoam.py b/eessi/testsuite/tests/apps/openfoam/openfoam.py index 7e314e08..d9d29482 100644 --- a/eessi/testsuite/tests/apps/openfoam/openfoam.py +++ b/eessi/testsuite/tests/apps/openfoam/openfoam.py @@ -53,7 +53,7 @@ def filter_scales_8M(): @rfm.simple_test -class EESSI_OPENFOAM_LID_DRIVEN_CAVITY(rfm.RunOnlyRegressionTest, EESSI_Mixin): +class EESSI_OPENFOAM_LID_DRIVEN_CAVITY_8M(rfm.RunOnlyRegressionTest, EESSI_Mixin): """ This is the main OPENFOAM class for the Lid-driven cavity test. The test consists of many steps which are run as pre-run commands and the main test with the executable `icoFoam` is measured for performance. @@ -164,3 +164,117 @@ def assert_sanity(self): self.assert_completion(), self.assert_convergence(), ]) + + +@rfm.simple_test +class EESSI_OPENFOAM_LID_DRIVEN_CAVITY_1M(rfm.RunOnlyRegressionTest, EESSI_Mixin): + """ + This is the main OPENFOAM class for the Lid-driven cavity test. The test consists of many steps which are run as + pre-run commands and the main test with the executable `icoFoam` is measured for performance. + """ + executable = 'icoFoam' + executable_opts = ['-parallel', '2>&1', '|', 'tee log.icofoam'] + time_limit = '60m' + readonly_files = [''] + device_type = parameter([DEVICE_TYPES.CPU]) + module_name = parameter(find_modules('OpenFOAM/v', name_only=False)) + valid_systems = ['*'] + scale = parameter(filter_scales_1M()) + + @run_after('init') + def set_compute_unit(self): + """ + Set the compute unit to which tasks will be assigned: + one task per CPU core for CPU runs. + """ + if self.device_type == DEVICE_TYPES.CPU: + self.compute_unit = COMPUTE_UNITS.CPU + else: + msg = f"No mapping of device type {self.device_type} to a COMPUTE_UNITS was specified in this test" + raise NotImplementedError(msg) + + def required_mem_per_node(self): + return self.num_tasks_per_node * 1700 + + @run_after('setup') + def check_launcher_options(self): + # We had to get the launcher command and prepend this to the prerun steps (func prepare_environment) because: + # 1. A typical OpenFOAM job would contain multiple mpirun steps working on the same stage directory. + # 2. We had trouble using ReFrame fixtures to separate these over multiple jobs, because we couldn't get it to + # work together with the mixin class. + if (self.job.launcher.command(self.job)[0] == 'mpirun'): + self.launcher_command = self.job.launcher.command(self.job) + self.launcher_command[-1] = str(self.num_tasks_per_node * self.num_nodes) + elif (self.job.launcher.command(self.job)[0] == 'srun'): + self.launcher_command = self.job.launcher.command(self.job) + else: + self.skip(msg="The chosen launcher for this test is different from mpirun or srun which means that the" + "test will definitely fail, therefore skipping this test.") + + @run_after('setup') + def check_maximum_cores(self): + # The 1M test case should maximally run on 128 cores to run properly. Otherwise communication overhead + # becomes too high. The number of cells per core is 1e6/128 = 7812.5, which is already a pretty low number. + # This is a limitation of the test case, not of OpenFOAM. + if self.num_tasks > 128: + self.skip(msg="The maximum number of cores this test can run on is 128. Launch on a scale with lower core" + "count.") + + @run_before('run') + def prepare_environment(self): + self.prerun_cmds = [ + 'cd ./cavity3D/1M/fixedTol', + 'source $FOAM_BASH', + f"foamDictionary -entry numberOfSubdomains -set {self.num_tasks_per_node * self.num_nodes} " + "system/decomposeParDict", + 'blockMesh 2>&1 | tee log.blockMesh', + f"{' '.join(self.launcher_command)} redistributePar -decompose -parallel 2>&1 | tee log.decompose", + f"{' '.join(self.launcher_command)} renumberMesh -parallel -overwrite 2>&1 | tee log.renumberMesh"] + + @deferrable + def check_files(self): + ''' Check for all the log files present. ''' + return (sn.path_isfile("./cavity3D/1M/fixedTol/log.blockMesh") + and sn.path_isfile("./cavity3D/1M/fixedTol/log.decompose") + and sn.path_isfile("./cavity3D/1M/fixedTol/log.renumberMesh") + and sn.path_isfile("./cavity3D/1M/fixedTol/log.icofoam")) + + @deferrable + def assert_completion(self): + n_ranks = sn.count(sn.extractall( + '^Processor (?P[0-9]+)', "./cavity3D/1M/fixedTol/log.decompose", tag='rank')) + return (sn.assert_found("^Writing polyMesh with 0 cellZones", "./cavity3D/1M/fixedTol/log.blockMesh", + msg="BlockMesh failure.") + and sn.assert_found(r"\s+nCells: 1000000", "./cavity3D/1M/fixedTol/log.blockMesh", + msg="BlockMesh failure.") + and sn.assert_eq(n_ranks, self.num_tasks) + and sn.assert_found(r"^Finalising parallel run", "./cavity3D/1M/fixedTol/log.renumberMesh", + msg="Did not reach the end of the renumberMesh run. RenumberMesh failure.") + and sn.assert_found(r"^Time = 0.0075", "./cavity3D/1M/fixedTol/log.icofoam", + msg="Did not reach the last time step. IcoFoam failure.") + and sn.assert_found(r"^Finalising parallel run", "./cavity3D/1M/fixedTol/log.icofoam", + msg="Did not reach the end of the icofoam run. IcoFoam failure.")) + + @deferrable + def assert_convergence(self): + cumulative_cont_err = sn.extractall(r'cumulative = (?P\S+)', "./cavity3D/1M/fixedTol/log.icofoam", + 'cont', float) + abs_cumulative_cont_err = sn.abs(cumulative_cont_err[-1]) + return sn.assert_le(abs_cumulative_cont_err, 1e-15, + msg="The cumulative continuity errors are high. Try varying pressure solver.") + + @performance_function('s/timestep') + def perf(self): + perftimes = sn.extractall(r'ClockTime = (?P\S+)', "./cavity3D/1M/fixedTol/log.icofoam", 'perf', + float) + seconds_per_timestep = perftimes[-1] / 15.0 + return seconds_per_timestep + + @sanity_function + def assert_sanity(self): + '''Check all sanity criteria''' + return sn.all([ + self.check_files(), + self.assert_completion(), + self.assert_convergence(), + ]) From bee36eb9b15f5147ffdae993948b622a06d574bf Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Thu, 31 Jul 2025 17:22:13 +0200 Subject: [PATCH 2/9] Added a different filter function for scales of 1M mesh --- eessi/testsuite/tests/apps/openfoam/openfoam.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/eessi/testsuite/tests/apps/openfoam/openfoam.py b/eessi/testsuite/tests/apps/openfoam/openfoam.py index d9d29482..f3e86009 100644 --- a/eessi/testsuite/tests/apps/openfoam/openfoam.py +++ b/eessi/testsuite/tests/apps/openfoam/openfoam.py @@ -51,6 +51,16 @@ def filter_scales_8M(): if (v['num_nodes'] >= 1) and (0 < v.get('node_part', 0) <= 2) ] +def filter_scales_1M(): + """ + Filtering function for filtering scales for the OpenFOAM 8M mesh test + returns all scales with at least half a node. + """ + return [ + k for (k, v) in SCALES.items() + if (v['num_nodes'] <= 2) and (v.get('node_part', 0) <= 2) and (v.get('num_cpus_per_node', 0) >= 2) + ] + @rfm.simple_test class EESSI_OPENFOAM_LID_DRIVEN_CAVITY_8M(rfm.RunOnlyRegressionTest, EESSI_Mixin): From 68bf8e9b46decb98f98c9fd11a7ac29f99c5b15d Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Thu, 31 Jul 2025 19:22:42 +0200 Subject: [PATCH 3/9] Fixed sanity issue in the 1M test. --- eessi/testsuite/tests/apps/openfoam/openfoam.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/openfoam/openfoam.py b/eessi/testsuite/tests/apps/openfoam/openfoam.py index f3e86009..68d5e0c1 100644 --- a/eessi/testsuite/tests/apps/openfoam/openfoam.py +++ b/eessi/testsuite/tests/apps/openfoam/openfoam.py @@ -260,7 +260,7 @@ def assert_completion(self): and sn.assert_eq(n_ranks, self.num_tasks) and sn.assert_found(r"^Finalising parallel run", "./cavity3D/1M/fixedTol/log.renumberMesh", msg="Did not reach the end of the renumberMesh run. RenumberMesh failure.") - and sn.assert_found(r"^Time = 0.0075", "./cavity3D/1M/fixedTol/log.icofoam", + and sn.assert_found(r"^Time = 0.015", "./cavity3D/1M/fixedTol/log.icofoam", msg="Did not reach the last time step. IcoFoam failure.") and sn.assert_found(r"^Finalising parallel run", "./cavity3D/1M/fixedTol/log.icofoam", msg="Did not reach the end of the icofoam run. IcoFoam failure.")) From 605f6eb6d744dd37354428f3cc4033682b58608b Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Thu, 14 Aug 2025 13:27:38 +0200 Subject: [PATCH 4/9] Trying to satisfy the linter --- eessi/testsuite/tests/apps/openfoam/openfoam.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eessi/testsuite/tests/apps/openfoam/openfoam.py b/eessi/testsuite/tests/apps/openfoam/openfoam.py index 68d5e0c1..41f05640 100644 --- a/eessi/testsuite/tests/apps/openfoam/openfoam.py +++ b/eessi/testsuite/tests/apps/openfoam/openfoam.py @@ -51,6 +51,7 @@ def filter_scales_8M(): if (v['num_nodes'] >= 1) and (0 < v.get('node_part', 0) <= 2) ] + def filter_scales_1M(): """ Filtering function for filtering scales for the OpenFOAM 8M mesh test From 059c1b950aa772b747c2c2b59875d29cc7be76f8 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Thu, 14 Aug 2025 14:22:03 +0200 Subject: [PATCH 5/9] Added CI tag to the 1M case --- eessi/testsuite/tests/apps/openfoam/openfoam.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eessi/testsuite/tests/apps/openfoam/openfoam.py b/eessi/testsuite/tests/apps/openfoam/openfoam.py index 41f05640..ae38c4c4 100644 --- a/eessi/testsuite/tests/apps/openfoam/openfoam.py +++ b/eessi/testsuite/tests/apps/openfoam/openfoam.py @@ -94,6 +94,7 @@ def set_compute_unit(self): def required_mem_per_node(self): return self.num_tasks_per_node * 1700 + @run_after('setup') def check_launcher_options(self): # We had to get the launcher command and prepend this to the prerun steps (func prepare_environment) because: @@ -207,6 +208,13 @@ def set_compute_unit(self): def required_mem_per_node(self): return self.num_tasks_per_node * 1700 + + @run_after('init') + def select_ci(self): + " Select the CI variants " + self.bench_name = self.bench_name_ci = 'icoFoam_1M_CI' + + @run_after('setup') def check_launcher_options(self): # We had to get the launcher command and prepend this to the prerun steps (func prepare_environment) because: From 7e3595c9701bb19a439a3d6fc6ee2c2bed30e565 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Sun, 17 Aug 2025 14:31:04 +0200 Subject: [PATCH 6/9] Corrected Scales and fixed line skips to satisfy linter --- eessi/testsuite/tests/apps/openfoam/openfoam.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/eessi/testsuite/tests/apps/openfoam/openfoam.py b/eessi/testsuite/tests/apps/openfoam/openfoam.py index ae38c4c4..a944fa73 100644 --- a/eessi/testsuite/tests/apps/openfoam/openfoam.py +++ b/eessi/testsuite/tests/apps/openfoam/openfoam.py @@ -59,7 +59,7 @@ def filter_scales_1M(): """ return [ k for (k, v) in SCALES.items() - if (v['num_nodes'] <= 2) and (v.get('node_part', 0) <= 2) and (v.get('num_cpus_per_node', 0) >= 2) + if (v['num_nodes'] <= 2) and (v.get('node_part', 0) <= 2) and (v.get('num_cpus_per_node', 0) != 1) ] @@ -94,7 +94,6 @@ def set_compute_unit(self): def required_mem_per_node(self): return self.num_tasks_per_node * 1700 - @run_after('setup') def check_launcher_options(self): # We had to get the launcher command and prepend this to the prerun steps (func prepare_environment) because: @@ -208,13 +207,11 @@ def set_compute_unit(self): def required_mem_per_node(self): return self.num_tasks_per_node * 1700 - @run_after('init') def select_ci(self): " Select the CI variants " self.bench_name = self.bench_name_ci = 'icoFoam_1M_CI' - @run_after('setup') def check_launcher_options(self): # We had to get the launcher command and prepend this to the prerun steps (func prepare_environment) because: @@ -297,3 +294,4 @@ def assert_sanity(self): self.assert_completion(), self.assert_convergence(), ]) + From 2b112384ad1506e774924d9aee0282ccac30118a Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Sun, 17 Aug 2025 14:34:52 +0200 Subject: [PATCH 7/9] Removing blank line at the end of file. --- eessi/testsuite/tests/apps/openfoam/openfoam.py | 1 - 1 file changed, 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/openfoam/openfoam.py b/eessi/testsuite/tests/apps/openfoam/openfoam.py index a944fa73..8f1d4b4f 100644 --- a/eessi/testsuite/tests/apps/openfoam/openfoam.py +++ b/eessi/testsuite/tests/apps/openfoam/openfoam.py @@ -294,4 +294,3 @@ def assert_sanity(self): self.assert_completion(), self.assert_convergence(), ]) - From 196c02995da8957311303c7b0b3ec93b966b0051 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Sun, 17 Aug 2025 20:46:45 +0200 Subject: [PATCH 8/9] Corrected scales now properly but needs to be tested. --- eessi/testsuite/tests/apps/openfoam/openfoam.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eessi/testsuite/tests/apps/openfoam/openfoam.py b/eessi/testsuite/tests/apps/openfoam/openfoam.py index 8f1d4b4f..c4758dd8 100644 --- a/eessi/testsuite/tests/apps/openfoam/openfoam.py +++ b/eessi/testsuite/tests/apps/openfoam/openfoam.py @@ -59,7 +59,8 @@ def filter_scales_1M(): """ return [ k for (k, v) in SCALES.items() - if (v['num_nodes'] <= 2) and (v.get('node_part', 0) <= 2) and (v.get('num_cpus_per_node', 0) != 1) + if ((v['num_nodes'] <= 2) and (v.get('node_part', 0) != 0)) or (v.get('num_cpus_per_node', 0) * + v.get('num_nodes') > 1) ] From 83c9f074bf35a0b15a2e542a2baf56c7274921d9 Mon Sep 17 00:00:00 2001 From: Satish Kamath Date: Sun, 17 Aug 2025 21:39:27 +0200 Subject: [PATCH 9/9] Now trying to satisfy the linter. --- eessi/testsuite/tests/apps/openfoam/openfoam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi/testsuite/tests/apps/openfoam/openfoam.py b/eessi/testsuite/tests/apps/openfoam/openfoam.py index c4758dd8..bdf5f60f 100644 --- a/eessi/testsuite/tests/apps/openfoam/openfoam.py +++ b/eessi/testsuite/tests/apps/openfoam/openfoam.py @@ -59,8 +59,8 @@ def filter_scales_1M(): """ return [ k for (k, v) in SCALES.items() - if ((v['num_nodes'] <= 2) and (v.get('node_part', 0) != 0)) or (v.get('num_cpus_per_node', 0) * - v.get('num_nodes') > 1) + if ((v['num_nodes'] <= 2) and (v.get('node_part', 0) != 0)) or (v.get('num_cpus_per_node', 0) + * v.get('num_nodes', 0) > 1) ]