Merge pull request #3398 from vkarak/bugfix/git-clone-stagedir-marks

[bugfix] Mark the stagedir after the copy or clone of resources

Author: vkarak

ci-scripts/dockerfiles/reframe-lmod77.dockerfile

@@ -7,7 +7,7 @@ FROM ghcr.io/reframe-hpc/lmod:7.7
 
 # Install ReFrame unit test requirements
 RUN apt-get -y update && \
-    apt-get -y install gcc make python3 python3-pip
+    apt-get -y install gcc git make python3 python3-pip
 
 # ReFrame user
 RUN useradd -ms /bin/bash rfmuser

reframe/core/pipeline.py

@@ -1780,6 +1780,9 @@ def setup(self, partition, environ, **job_opts):
         self._setup_container_platform()
         self._resolve_fixtures()
 
+    def _mark_stagedir(self):
+        (Path(self.stagedir) / '.rfm_mark').touch()
+
     def _requires_stagedir_contents(self):
         '''Return true if the contents of the stagedir need to be generated'''
 
@@ -1788,12 +1791,7 @@ def _requires_stagedir_contents(self):
         # `--dont-restage` is passed. In this case, we want to leave the
         # existing stagedir untouched.
 
-        mark = Path(self.stagedir) / '.rfm_mark'
-        if mark.exists():
-            return False
-        else:
-            mark.touch()
-            return True
+        return not (Path(self.stagedir) / '.rfm_mark').exists()
 
     def _copy_to_stagedir(self, path):
         self.logger.debug(f'Copying {path} to stage directory')
@@ -1804,13 +1802,16 @@ def _copy_to_stagedir(self, path):
             )
         except (OSError, ValueError, TypeError) as e:
             raise PipelineError('copying of files failed') from e
+        else:
+            self._mark_stagedir()
 
     def _clone_to_stagedir(self, url):
         self.logger.debug(f'Cloning URL {url} into stage directory')
         osext.git_clone(
             self.sourcesdir, self._stagedir,
             timeout=rt.runtime().get_option('general/0/git_timeout')
         )
+        self._mark_stagedir()
 
     @final
     def compile(self):

unittests/test_pipeline.py

@@ -763,6 +763,23 @@ def validate(self):
     _run(MyTest(), *local_exec_ctx)
 
 
+def test_sourcesdir_git(local_exec_ctx):
+    @test_util.custom_prefix('unittests/resources/checks')
+    class MyTest(rfm.RunOnlyRegressionTest):
+        sourcesdir = 'https://github.com/reframe-hpc/ci-hello-world.git'
+        executable = 'true'
+        valid_systems = ['*']
+        valid_prog_environs = ['*']
+        keep_files = ['README.md']
+
+        @sanity_function
+        def validate(self):
+            print(self.stagedir)
+            return sn.assert_true(os.path.exists('README.md'))
+
+    _run(MyTest(), *local_exec_ctx)
+
+
 def test_sourcesdir_none_generated_sources(local_exec_ctx):
     @test_util.custom_prefix('unittests/resources/checks')
     class MyTest(rfm.RegressionTest):

unittests/test_utility.py

@@ -428,8 +428,8 @@ def test_is_interactive(monkeypatch):
 
 
 def test_is_url():
-    repo_https = 'https://github.com/reframe-hpc/reframe.git'
-    repo_ssh = '[email protected]:reframe-hpc/reframe.git'
+    repo_https = 'https://github.com/reframe-hpc/ci-hello-world.git'
+    repo_ssh = '[email protected]:reframe-hpc/ci-hello-world.git'
     assert osext.is_url(repo_https)
     assert not osext.is_url(repo_ssh)