Merge pull request #19 from effigies/test/data

effigies · web-flow · commit 552bac51313e · 2018-10-16T07:28:05.000-07:00
TEST: Get data files for Node/Workflow tests
diff --git a/.travis.yml b/.travis.yml
@@ -6,10 +6,22 @@ python:
   - 3.6
   - 3.7
 
+env:
+  - PYDRA_TEST_DATA=/tmp/pydra_test_data
+
+before_install:
+  - travis_retry bash <(wget -q -O- http://neuro.debian.net/_files/neurodebian-travis.sh);
+  - travis_retry sudo apt-get update
+  - travis_retry sudo apt-get install fsl git-annex-standalone datalad
+  - source /etc/fsl/fsl.sh
+  - travis_retry pip install --upgrade pip setuptools wheel
+
 install:
-  - pip install --upgrade pip setuptools wheel
   - pip install ".[all]"
 
+before_script:
+  - bash tools/get_test_data.sh
+
 script:
   - py.test -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra
 
diff --git a/pydra/__about__.py b/pydra/__about__.py
@@ -63,6 +63,7 @@
 
 EXTRA_REQUIRES = {
     'tests': TESTS_REQUIRES,
+    'dev': TESTS_REQUIRES + ['yapf>=0.22'],
     'plugins': ['dask', 'distributed'],
 }
 
diff --git a/pydra/engine/tests/test_node.py b/pydra/engine/tests/test_node.py
@@ -1,13 +1,21 @@
+import sys
+import os
+import time
+import numpy as np
+from pathlib import Path
+
 from nipype.utils.filemanip import save_json, makedirs, to_str
 from nipype.interfaces import fsl
 
 from ..node import Node, Workflow
 from ..auxiliary import FunctionInterface, CurrentInterface
 from ..submitter import Submitter
 
-import sys, time, os
-import numpy as np
-import pytest, pdb
+import pytest
+import pdb
+
+TEST_DATA_DIR = Path(os.getenv('PYDRA_TEST_DATA', '/nonexistent/path'))
+DS114_DIR = TEST_DATA_DIR / 'ds000114'
 
 python35_only = pytest.mark.skipif(sys.version_info < (3, 5), reason="requires Python>3.4")
 
@@ -1441,8 +1449,7 @@ def test_workflow_16a(plugin, change_dir):
 # testing CurrentInterface that is a temporary wrapper for current interfaces
 
 
-@pytest.mark.skipif(
-    not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_current_node_1(change_dir, plugin):
@@ -1451,10 +1458,7 @@ def test_current_node_1(change_dir, plugin):
 
     nn = Node(
         name="NA",
-        inputs={
-            "in_file":
-            "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"
-        },
+        inputs={"in_file": str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz")},
         interface=interf_bet,
         workingdir="test_cnd1_{}".format(plugin),
         output_names=["out_file"])
@@ -1466,17 +1470,16 @@ def test_current_node_1(change_dir, plugin):
     assert "out_file" in nn.output.keys()
 
 
-@pytest.mark.skipif(
-    not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_current_node_2(change_dir, plugin):
     """Node with a current interface and mapper"""
     interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface")
 
     in_file_l = [
-        "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz",
-        "/Users/dorota/nipype_workshop/data/ds000114/sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz"
+        str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"),
+        str(DS114_DIR / "sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz")
     ]
     nn = Node(
         name="NA",
@@ -1496,8 +1499,7 @@ def test_current_node_2(change_dir, plugin):
     assert "NA.in_file:1" in nn.output["out_file"].keys()
 
 
-@pytest.mark.skipif(
-    not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_current_wf_1(change_dir, plugin):
@@ -1506,10 +1508,7 @@ def test_current_wf_1(change_dir, plugin):
 
     nn = Node(
         name="fsl",
-        inputs={
-            "in_file":
-            "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"
-        },
+        inputs={"in_file": str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz")},
         interface=interf_bet,
         workingdir="nn",
         output_names=["out_file"],
@@ -1529,8 +1528,7 @@ def test_current_wf_1(change_dir, plugin):
     assert "fsl_out" in wf.output.keys()
 
 
-@pytest.mark.skipif(
-    not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_current_wf_1a(change_dir, plugin):
@@ -1539,10 +1537,7 @@ def test_current_wf_1a(change_dir, plugin):
 
     nn = Node(
         name="fsl",
-        inputs={
-            "in_file":
-            "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"
-        },
+        inputs={"in_file": str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz")},
         interface=interf_bet,
         workingdir="nn",
         output_names=["out_file"],
@@ -1562,8 +1557,7 @@ def test_current_wf_1a(change_dir, plugin):
     assert "fsl_out" in wf.output.keys()
 
 
-@pytest.mark.skipif(
-    not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_current_wf_1b(change_dir, plugin):
@@ -1581,10 +1575,7 @@ def test_current_wf_1b(change_dir, plugin):
         workingdir="nn",
         output_names=["out_file"],
         write_state=False,
-        inputs={
-            "in_file":
-            "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"
-        })
+        inputs={"in_file": str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz")})
 
     sub = Submitter(plugin=plugin, runnable=wf)
     sub.run()
@@ -1593,8 +1584,7 @@ def test_current_wf_1b(change_dir, plugin):
     assert "fsl_out" in wf.output.keys()
 
 
-@pytest.mark.skipif(
-    not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_current_wf_1c(change_dir, plugin):
@@ -1611,10 +1601,7 @@ def test_current_wf_1c(change_dir, plugin):
         workingdir="nn",
         output_names=["out_file"],
         write_state=False,
-        inputs={
-            "in_file":
-            "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"
-        })
+        inputs={"in_file": str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz")})
 
     sub = Submitter(plugin=plugin, runnable=wf)
     sub.run()
@@ -1623,17 +1610,16 @@ def test_current_wf_1c(change_dir, plugin):
     assert "fsl_out" in wf.output.keys()
 
 
-@pytest.mark.skipif(
-    not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_current_wf_2(change_dir, plugin):
     """Wf with a current interface and mapper"""
     interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface")
 
     in_file_l = [
-        "/Users/dorota/nipype_workshop/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz",
-        "/Users/dorota/nipype_workshop/data/ds000114/sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz"
+        str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"),
+        str(DS114_DIR / "sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz")
     ]
 
     nn = Node(
@@ -1662,17 +1648,16 @@ def test_current_wf_2(change_dir, plugin):
     assert 'cw2.in_file:1' in wf.output["fsl_out"].keys()
 
 
-@pytest.mark.skipif(
-    not os.path.exists("/Users/dorota/nipype_workshop/data/ds000114"), reason="adding data")
+@pytest.mark.skipif(not DS114_DIR.exists(), reason="Missing $PYDRA_TEST_DATA/ds000114")
 @pytest.mark.parametrize("plugin", Plugins)
 @python35_only
 def test_current_wf_2a(change_dir, plugin):
     """Wf with a current interface and mapper"""
     interf_bet = CurrentInterface(interface=fsl.BET(), name="fsl_interface")
 
     in_file_l = [
-        "/data/ds000114/sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz",
-        "/data/ds000114/sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz"
+        str(DS114_DIR / "sub-01/ses-test/anat/sub-01_ses-test_T1w.nii.gz"),
+        str(DS114_DIR / "sub-02/ses-test/anat/sub-02_ses-test_T1w.nii.gz")
     ]
 
     nn = Node(
diff --git a/pydra/engine/tests/test_node_neuro.py b/pydra/engine/tests/test_node_neuro.py
diff --git a/tools/get_test_data.sh b/tools/get_test_data.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+mkdir -p $PYDRA_TEST_DATA
+datalad install -r -s ///openfmri/ds000114 $PYDRA_TEST_DATA/ds000114
+datalad get $PYDRA_TEST_DATA/ds000114/sub-0{1,2}/ses-test/anat/sub-0?_ses-test_T1w.nii.gz

Original file line number	Diff line number	Diff line change
`@@ -63,6 +63,7 @@`
`63`	`63`
`64`	`64`	`EXTRA_REQUIRES = {`
`65`	`65`	`'tests': TESTS_REQUIRES,`
	`66`	`+ 'dev': TESTS_REQUIRES + ['yapf>=0.22'],`
`66`	`67`	`'plugins': ['dask', 'distributed'],`
`67`	`68`	`}`
`68`	`69`