WIP Fix resolver for imports (#1093)

* Fix resolver for TRS

Author: denis-yuen

MANIFEST.in

@@ -9,6 +9,7 @@ include tests/wf/*
 include tests/override/*
 include tests/checker_wf/*
 include tests/subgraph/*
+include tests/trs/*
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/v1.0/*.md

README.rst

@@ -498,7 +498,7 @@ Running tests locally
 
 -  Running basic tests ``(/tests)``:
 
-To run the basis tests after installing `cwltool` execute the following:
+To run the basic tests after installing `cwltool` execute the following:
 
 .. code:: bash
 

cwltool/resolver.py

@@ -13,10 +13,17 @@
 else:
     from pathlib import Path
 
+if not getattr(__builtins__, "WindowsError", None):
+    class WindowsError(OSError): pass
 
 def resolve_local(document_loader, uri):
     pathpart, frag = urllib.parse.urldefrag(uri)
-    pathobj = Path(pathpart).resolve()
+
+    try:
+        pathobj = Path(pathpart).resolve()
+    except (WindowsError, OSError):
+        _logger.debug("local resolver could not resolve %s", uri)
+        return None
 
     if pathobj.is_file():
         if frag:
@@ -48,19 +55,35 @@ def tool_resolver(document_loader, uri):
 
 
 ga4gh_tool_registries = ["https://dockstore.org/api"]
-GA4GH_TRS = "{0}/api/ga4gh/v2/tools/{1}/versions/{2}/plain-CWL/descriptor"
+# in the TRS registry, a primary descriptor can be reached at {0}/api/ga4gh/v2/tools/{1}/versions/{2}/plain-CWL/descriptor
+# The primary descriptor is a CommandLineTool in the case that the files endpoint only describes one file
+# When the primary descriptor is a Workflow, files need to be imported without stripping off "descriptor", looking at the files endpoint is a workaround
+# tested with TRS version 2.0.0-beta.2 
+# TODO not stripping off "descriptor" when looking for local imports would also work https://github.com/ga4gh/tool-registry-service-schemas/blob/2.0.0-beta.2/src/main/resources/swagger/ga4gh-tool-discovery.yaml#L273
+GA4GH_TRS_FILES = "{0}/api/ga4gh/v2/tools/{1}/versions/{2}/CWL/files"
+GA4GH_TRS_PRIMARY_DESCRIPTOR = "{0}/api/ga4gh/v2/tools/{1}/versions/{2}/plain-CWL/descriptor/{3}"
+
 
 def resolve_ga4gh_tool(document_loader, uri):
     path, version = uri.partition(":")[::2]
     if not version:
         version = "latest"
     for reg in ga4gh_tool_registries:
-        ds = GA4GH_TRS.format(reg, urllib.parse.quote(path, ""),
-                              urllib.parse.quote(version, ""))
+        ds = GA4GH_TRS_FILES.format(reg, urllib.parse.quote(path, ""), urllib.parse.quote(version, ""))
         try:
+            _logger.debug("Head path is %s", ds)
             resp = document_loader.session.head(ds)
             resp.raise_for_status()
-            return ds
+
+            _logger.debug("Passed head path of %s", ds)
+
+            resp = document_loader.session.get(ds)
+            for file_listing in resp.json():
+                if file_listing.get('file_type') == 'PRIMARY_DESCRIPTOR':
+                    primary_path = file_listing.get('path')
+                    ds2 = GA4GH_TRS_PRIMARY_DESCRIPTOR.format(reg, urllib.parse.quote(path, ""), urllib.parse.quote(version, ""), urllib.parse.quote(primary_path, ""))
+                    _logger.debug("Resolved %s", ds2)
+                    return ds2
         except Exception:
             pass
     return None

tests/test_trs.py

@@ -0,0 +1,76 @@
+from __future__ import absolute_import
+
+import mock
+
+from cwltool.main import main
+from .util import get_data
+
+def mocked_requests_head(*args):
+
+    class MockResponse:
+        def __init__(self, json_data, status_code, raise_for_status=None):
+            self.json_data = json_data
+            self.status_code = status_code
+            self.raise_for_status = mock.Mock()
+            self.raise_for_status.side_effect = raise_for_status
+
+        def json(self):
+            return self.json_data
+
+    return MockResponse(None, 200)
+
+
+def mocked_requests_get(*args):
+
+    class MockResponse:
+        def __init__(self, json_data, status_code, raise_for_status=None):
+            self.json_data = json_data
+            self.text = json_data
+            self.status_code = status_code
+            self.raise_for_status = mock.Mock()
+            self.raise_for_status.side_effect = raise_for_status
+
+        def json(self):
+            return self.json_data
+
+    if args[0] == 'https://dockstore.org/api/api/ga4gh/v2/tools/quay.io%2Fbriandoconnor%2Fdockstore-tool-md5sum/versions/1.0.4/CWL/files':
+        return MockResponse(
+            [{"file_type": "CONTAINERFILE", "path": "Dockerfile"}, {"file_type": "PRIMARY_DESCRIPTOR", "path": "Dockstore.cwl"},
+             {"file_type": "TEST_FILE", "path": "test.json"}], 200)
+    elif args[0] == 'https://dockstore.org/api/api/ga4gh/v2/tools/quay.io%2Fbriandoconnor%2Fdockstore-tool-md5sum/versions/1.0.4/plain-CWL/descriptor/Dockstore.cwl':
+        string = open(get_data("tests/trs/Dockstore.cwl"), "r").read()
+        return MockResponse(string, 200)
+    elif args[0] == 'https://dockstore.org/api/api/ga4gh/v2/tools/%23workflow%2Fgithub.com%2Fdockstore-testing%2Fmd5sum-checker/versions/develop/plain-CWL/descriptor/md5sum-tool.cwl':
+        string = open(get_data("tests/trs/md5sum-tool.cwl"), "r").read()
+        return MockResponse(string, 200)
+    elif args[0] == 'https://dockstore.org/api/api/ga4gh/v2/tools/%23workflow%2Fgithub.com%2Fdockstore-testing%2Fmd5sum-checker/versions/develop/plain-CWL/descriptor/md5sum-workflow.cwl':
+        string = open(get_data("tests/trs/md5sum-workflow.cwl"), "r").read()
+        return MockResponse(string, 200)
+    elif args[
+        0] == 'https://dockstore.org/api/api/ga4gh/v2/tools/%23workflow%2Fgithub.com%2Fdockstore-testing%2Fmd5sum-checker/versions/develop/CWL/files':
+        return MockResponse(
+            [{"file_type": "TEST_FILE", "path": "md5sum-input-cwl.json"}, {"file_type": "SECONDARY_DESCRIPTOR", "path": "md5sum-tool.cwl"},
+             {"file_type": "PRIMARY_DESCRIPTOR", "path": "md5sum-workflow.cwl"}], 200)
+
+    print ("A mocked call to TRS missed, target was %s", args[0])
+    return MockResponse(None, 404)
+
+
+@mock.patch('requests.Session.head', side_effect=mocked_requests_head)
+@mock.patch('requests.Session.get', side_effect=mocked_requests_get)
+def test_tool_trs_template(mock_head, mock_get):
+    params = ["--make-template", r"quay.io/briandoconnor/dockstore-tool-md5sum:1.0.4"]
+    return_value = main(params)
+    mock_head.assert_called()
+    mock_get.assert_called()
+    assert return_value == 0
+
+
+@mock.patch('requests.Session.head', side_effect=mocked_requests_head)
+@mock.patch('requests.Session.get', side_effect=mocked_requests_get)
+def test_workflow_trs_template(mock_head, mock_get):
+    params = ["--make-template", r"#workflow/github.com/dockstore-testing/md5sum-checker:develop"]
+    return_value = main(params)
+    mock_head.assert_called()
+    mock_get.assert_called()
+    assert return_value == 0

tests/trs/Dockstore.cwl

@@ -0,0 +1,50 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+id: Md5sum
+label: Simple md5sum tool
+cwlVersion: v1.0
+
+$namespaces:
+  dct: http://purl.org/dc/terms/
+  foaf: http://xmlns.com/foaf/0.1/
+
+doc: |
+  [![Docker Repository on Quay.io](https://quay.io/repository/briandoconnor/dockstore-tool-md5sum/status "Docker Repository on Quay.io")](https://quay.io/repository/briandoconnor/dockstore-tool-md5sum)
+  [![Build Status](https://travis-ci.org/briandoconnor/dockstore-tool-md5sum.svg)](https://travis-ci.org/briandoconnor/dockstore-tool-md5sum)
+  A very, very simple Docker container for the md5sum command. See the [README](https://github.com/briandoconnor/dockstore-tool-md5sum/blob/master/README.md) for more information.
+
+
+#dct:creator:
+#  '@id': http://orcid.org/0000-0002-7681-6415
+#  foaf:name: Brian O'Connor
+#  foaf:mbox: [email protected]
+
+requirements:
+- class: DockerRequirement
+  dockerPull: quay.io/briandoconnor/dockstore-tool-md5sum:1.0.4
+- class: InlineJavascriptRequirement
+
+hints:
+- class: ResourceRequirement
+  # The command really requires very little resources.
+  coresMin: 1
+  ramMin: 1024
+  outdirMin: 1024
+
+inputs:
+  input_file:
+    type: File
+    inputBinding:
+      position: 1
+    doc: The file that will have its md5sum calculated.
+
+outputs:
+  output_file:
+    type: File
+    format: http://edamontology.org/data_3671
+    outputBinding:
+      glob: md5sum.txt
+    doc: A text file that contains a single line that is the md5sum of the input file.
+
+baseCommand: [/bin/my_md5sum]

tests/trs/md5sum-tool.cwl

@@ -0,0 +1,39 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+id: Md5sum
+label: Simple md5sum tool
+cwlVersion: v1.0
+
+$namespaces:
+  dct: http://purl.org/dc/terms/
+  foaf: http://xmlns.com/foaf/0.1/
+
+requirements:
+- class: DockerRequirement
+  dockerPull: quay.io/agduncan94/my-md5sum
+- class: InlineJavascriptRequirement
+
+hints:
+- class: ResourceRequirement
+  # The command really requires very little resources.
+  coresMin: 1
+  ramMin: 1024
+  outdirMin: 512
+
+inputs:
+  input_file:
+    type: File
+    inputBinding:
+      position: 1
+    doc: The file that will have its md5sum calculated.
+
+outputs:
+  output_file:
+    type: File
+    format: http://edamontology.org/data_3671
+    outputBinding:
+      glob: md5sum.txt
+    doc: A text file that contains a single line that is the md5sum of the input file.
+
+baseCommand: [/bin/my_md5sum]

tests/trs/md5sum-workflow.cwl

@@ -0,0 +1,17 @@
+cwlVersion: v1.0
+class: Workflow
+
+inputs:
+  input_file: File
+
+outputs:
+  output_file:
+    type: File
+    outputSource: md5sum/output_file
+
+steps:
+  md5sum:
+    run: md5sum-tool.cwl
+    in:
+      input_file: input_file
+    out: [output_file]