Merge pull request #157 from djarecka/singularity

I've checked it on openmind

Author: djarecka

.travis.yml

@@ -26,7 +26,6 @@ env:
     - INSTALL_TYPE="sdist"
     - INSTALL_TYPE="wheel"
     - INSTALL_DEPENDS="pip==18.1 setuptools==30.2.1"
-    - INSTALL_DEPENDS="pip==10.0.1 setuptools==30.3.0"
     - DEPENDS="-r min-requirements.txt"
     - CHECK_TYPE="style"
 
@@ -36,10 +35,15 @@ matrix:
   include:
   - python: 3.7
     env: JOBQUEUE="slurm"
-  allow_failures:
+  - language: go
+    go: 1.13
+    env:
+      - JOBQUEUE="singularity"
+      - INSTALL_TYPE="develop"
   - python: 3.7
     env: INSTALL_DEPENDS="pip==10.0.1 setuptools==30.3.0"
-  - python: 3.8
+  allow_failures:
+  - python: 3.7
     env: INSTALL_DEPENDS="pip==10.0.1 setuptools==30.3.0"
 
 

ci/none.sh

@@ -47,7 +47,7 @@ function travis_before_script {
 
 function travis_script {
     if [ "$CHECK_TYPE" = "test" ]; then
-        pytest -vs -n auto --cov pydra tutorial --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra
+        pytest -vs -n auto --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra tutorial
     elif [ "$CHECK_TYPE" = "style" ]; then
         black --check pydra tools setup.py
     fi

ci/singularity.sh

@@ -0,0 +1,36 @@
+# local build environment
+
+function travis_before_install {
+    sudo apt-get update;
+    sudo apt-get install flawfinder squashfs-tools uuid-dev libuuid1 libffi-dev libssl-dev libssl1.0.0 \
+    libarchive-dev libgpgme11-dev libseccomp-dev wget gcc make pkg-config -y;
+    export VERSION=3.5.0;
+    travis_retry wget -q https://github.com/sylabs/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz;
+    tar -xzf singularity-${VERSION}.tar.gz;
+    cd singularity;
+    ./mconfig;
+    make -C ./builddir;
+    sudo make -C ./builddir install;
+    cd -;
+    travis_retry wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh;
+    bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/miniconda;
+    eval "$($HOME/miniconda/bin/conda shell.bash hook)"
+}
+
+function travis_install {
+    python setup.py develop
+    # Verify import with bare install
+    python -c 'import pydra; print(pydra.__version__)'
+}
+
+function travis_before_script {
+    pip install -e ".[test]"
+}
+
+function travis_script {
+    pytest -vs --cov pydra --cov-config .coveragerc --cov-report xml:cov.xml --doctest-modules pydra
+}
+
+function travis_after_script {
+    codecov --file cov.xml --flags unittests -e TRAVIS_JOB_NUMBER
+}

codecov.yml

@@ -1,7 +1,14 @@
 coverage:
-  range: "0...100"
+  range: "50...100"
   ignore:          # files and folders that will be removed during processing
     - "**/tests"
     - "pydra/_version.py"
     - "setup.py"
     - "versioneer.py"
+  status:
+    project:
+      default:
+        threshold: 1
+    patch:
+      default:
+        target: 80

pydra/engine/task.py

@@ -309,12 +309,9 @@ def cmdline(self):
     def container_args(self):
         if self.inputs.container is None:
             raise AttributeError("Container software is not specified")
-        cargs = [self.inputs.container, "run"]
-        if self.inputs.container_xargs is not None:
-            cargs.extend(self.inputs.container_xargs)
+        cargs = [self.inputs.container]
         if self.inputs.image is None:
             raise AttributeError("Container image is not specified")
-        cargs.append(self.inputs.image)
         return cargs
 
     @property
@@ -351,7 +348,6 @@ def binds(self, opt):
         for (key, val) in self.bind_paths.items():
             bargs.extend([opt, "{0}:{1}:{2}".format(key, val[0], val[1])])
         # TODO: would need changes for singularity
-        bargs.extend(["-w", str(self.output_cpath)])
         return bargs
 
     def _run_task(self):
@@ -399,9 +395,14 @@ def __init__(
     def container_args(self):
         cargs = super().container_args
         assert self.inputs.container == "docker"
-        # insert bindings before image
-        idx = len(cargs) - 1
-        cargs[idx:-1] = self.binds("-v")
+        cargs.append("run")
+        if self.inputs.container_xargs is not None:
+            cargs.extend(self.inputs.container_xargs)
+
+        cargs.extend(self.binds("-v"))
+        cargs.extend(["-w", str(self.output_cpath)])
+        cargs.append(self.inputs.image)
+
         return cargs
 
 
@@ -418,14 +419,10 @@ def __init__(
         **kwargs,
     ):
         if input_spec is None:
-            field = dc.field(default_factory=list)
-            field.metadata = {}
-            fields = [("args", ty.List[str], field)]
-            input_spec = SpecInfo(
-                name="Inputs", fields=fields, bases=(SingularitySpec,)
-            )
-        super(ContainerTask, self).__init__(
+            input_spec = SpecInfo(name="Inputs", fields=[], bases=(SingularitySpec,))
+        super(SingularityTask, self).__init__(
             input_spec=input_spec,
+            output_spec=output_spec,
             audit_flags=audit_flags,
             messengers=messengers,
             messenger_args=messenger_args,
@@ -438,6 +435,11 @@ def __init__(
     def container_args(self):
         cargs = super().container_args
         assert self.inputs.container == "singularity"
+        cargs.append("exec")
+        if self.inputs.container_xargs is not None:
+            cargs.extend(self.inputs.container_xargs)
+        cargs.append(self.inputs.image)
+
         # insert bindings before image
         idx = len(cargs) - 1
         cargs[idx:-1] = self.binds("-B")

pydra/engine/tests/test_dockertask.py

@@ -1,8 +1,8 @@
 # -*- coding: utf-8 -*-
 
 import os, shutil
+import subprocess as sp
 import pytest
-from pathlib import Path
 import dataclasses as dc
 
 from ..task import DockerTask
@@ -16,7 +16,8 @@
     Plugins = ["cf"]
 
 need_docker = pytest.mark.skipif(
-    shutil.which("docker") is None, reason="no docker within the container"
+    shutil.which("docker") is None or sp.call(["docker", "info"]),
+    reason="no docker within the container",
 )
 
 
@@ -343,6 +344,9 @@ def test_wf_docker_2(plugin, tmpdir):
     assert res.output.out == "Hello!"
 
 
+# tests with customized output_spec
+
+
 @need_docker
 @pytest.mark.parametrize("plugin", Plugins)
 def test_docker_outputspec_1(plugin, tmpdir):