Merge pull request #225 from djarecka/enh/DirectoryType

adding hash_dir for Directory type (closes #209)

Author: djarecka

pydra/engine/helpers.py

@@ -9,8 +9,8 @@
 from hashlib import sha256
 import subprocess as sp
 
-from .specs import Runtime, File, attr_fields
-from .helpers_file import hash_file, copyfile, is_existing_file
+from .specs import Runtime, File, Directory, attr_fields
+from .helpers_file import hash_file, hash_dir, copyfile, is_existing_file
 
 
 def ensure_list(obj, tuple2list=False):
@@ -459,6 +459,12 @@ def hash_value(value, tp=None, metadata=None):
             and "container_path" not in metadata
         ):
             return hash_file(value)
+        elif (
+            (tp is File or "pydra.engine.specs.Directory" in str(tp))
+            and is_existing_file(value)
+            and "container_path" not in metadata
+        ):
+            return hash_dir(value)
         else:
             return value
 

pydra/engine/helpers_file.py

@@ -113,7 +113,7 @@ def hash_file(afile, chunk_len=8192, crypto=sha256, raise_notfound=True):
 
     if afile is None or isinstance(afile, LazyField) or isinstance(afile, list):
         return None
-    if not os.path.isfile(afile):
+    if not Path(afile).is_file():
         if raise_notfound:
             raise RuntimeError('File "%s" not found.' % afile)
         return None
@@ -128,6 +128,29 @@ def hash_file(afile, chunk_len=8192, crypto=sha256, raise_notfound=True):
     return crypto_obj.hexdigest()
 
 
+def hash_dir(dirpath, raise_notfound=True):
+    from .specs import LazyField
+
+    if dirpath is None or isinstance(dirpath, LazyField) or isinstance(dirpath, list):
+        return None
+    if not Path(dirpath).is_dir():
+        if raise_notfound:
+            raise RuntimeError(f"Directory {dirpath} not found.")
+        return None
+
+    def search_dir(path):
+        path = Path(path)
+        file_list = []
+        for el in path.iterdir():
+            if el.is_file():
+                file_list.append(hash_file(el))
+            else:
+                file_list.append(search_dir(path / el))
+        return file_list
+
+    return search_dir(dirpath)
+
+
 def _parse_mount_table(exit_code, output):
     """
     Parse the output of ``mount`` to produce (path, fs_type) pairs.

pydra/engine/tests/test_helpers.py

@@ -6,7 +6,7 @@
 from .utils import multiply
 from ..helpers import hash_value, hash_function, save, create_pyscript
 from .. import helpers_file
-from ..specs import File
+from ..specs import File, Directory
 
 
 def test_save(tmpdir):
@@ -110,6 +110,7 @@ def test_hash_value_files(tmpdir):
     assert hash_value(file_1, tp=File) == hash_value(file_2, tp=File)
     assert hash_value(file_1, tp=str) != hash_value(file_2, tp=str)
     assert hash_value(file_1) != hash_value(file_2)
+    assert hash_value(file_1, tp=File) == helpers_file.hash_file(file_1)
 
 
 def test_hash_value_files_list(tmpdir):
@@ -124,3 +125,33 @@ def test_hash_value_files_list(tmpdir):
         hash_value(file_1, tp=File),
         hash_value(file_2, tp=File),
     ]
+
+
+def test_hash_value_dir(tmpdir):
+    file_1 = tmpdir.join("file_1.txt")
+    file_2 = tmpdir.join("file_2.txt")
+    with open(file_1, "w") as f:
+        f.write("hello")
+    with open(file_2, "w") as f:
+        f.write("hi")
+
+    assert hash_value(tmpdir, tp=Directory) == hash_value([file_1, file_2], tp=File)
+    assert hash_value(tmpdir, tp=Directory) == helpers_file.hash_dir(tmpdir)
+
+
+def test_hash_value_nested(tmpdir):
+    nested = tmpdir.mkdir("nested")
+    file_1 = tmpdir.join("file_1.txt")
+    file_2 = nested.join("file_2.txt")
+    file_3 = nested.join("file_3.txt")
+    with open(file_1, "w") as f:
+        f.write("hello")
+    with open(file_2, "w") as f:
+        f.write("hi")
+    with open(file_3, "w") as f:
+        f.write("hola")
+
+    assert hash_value(tmpdir, tp=Directory) == hash_value(
+        [file_1, [file_2, file_3]], tp=File
+    )
+    assert hash_value(tmpdir, tp=Directory) == helpers_file.hash_dir(tmpdir)

pydra/engine/tests/test_helpers_file.py

@@ -107,7 +107,7 @@ def test_copyfiles(_temp_analyze_files, _temp_analyze_files_prime):
 
 
 def test_linkchain(_temp_analyze_files):
-    if os.name is not "posix":
+    if os.name != "posix":
         return
     orig_img, orig_hdr = _temp_analyze_files
     pth, fname = os.path.split(orig_img)