Added graph loading feature (#146)

Co-authored-by: Michael R. Crusoe <michael.crusoe@gmail.com>

Author: GlassOfWhiskey

cwl_utils/errors.py

@@ -24,3 +24,7 @@ class SubstitutionError(Exception):
 
 class WorkflowException(Exception):
     pass
+
+
+class GraphTargetMissingException(WorkflowException):
+    """When a $graph is encountered and there is no target and no main/#main."""

cwl_utils/parser/__init__.py

@@ -8,9 +8,8 @@
 from schema_salad.exceptions import ValidationException
 from schema_salad.utils import yaml_no_ts
 
-from . import cwl_v1_0 as cwl_v1_0
-from . import cwl_v1_1 as cwl_v1_1
-from . import cwl_v1_2 as cwl_v1_2
+from ..errors import GraphTargetMissingException
+from . import cwl_v1_0, cwl_v1_1, cwl_v1_2
 
 LoadingOptions = Union[
     cwl_v1_0.LoadingOptions, cwl_v1_1.LoadingOptions, cwl_v1_2.LoadingOptions
@@ -38,6 +37,18 @@
 _Loader = Union[cwl_v1_0._Loader, cwl_v1_1._Loader, cwl_v1_2._Loader]
 
 
+def _get_id_from_graph(yaml: MutableMapping[str, Any], id_: Optional[str]) -> Any:
+    if id_ is None:
+        id_ = "main"
+    for el in yaml["$graph"]:
+        if el["id"].lstrip("#") == id_:
+            return el
+    raise GraphTargetMissingException(
+        "Tool file contains graph of multiple objects, must specify "
+        "one of #%s" % ", #".join(el["id"] for el in yaml["$graph"])
+    )
+
+
 def cwl_version(yaml: Any) -> Any:
     """Return the cwlVersion of a YAML object.
 
@@ -64,49 +75,61 @@ def load_document_by_uri(
     """Load a CWL object from a URI or a path."""
     if isinstance(path, str):
         uri = urlparse(path)
+        id_ = uri.fragment or None
         if not uri.scheme or uri.scheme == "file":
             real_path = Path(unquote_plus(uri.path)).resolve().as_uri()
         else:
             real_path = path
     else:
         real_path = path.resolve().as_uri()
+        id_ = path.resolve().name.split("#")[1] if "#" in path.resolve().name else None
 
     baseuri = str(real_path)
 
     if loadingOptions is None:
         loadingOptions = cwl_v1_2.LoadingOptions(fileuri=baseuri)
 
     doc = loadingOptions.fetcher.fetch_text(real_path)
-    return load_document_by_string(doc, baseuri, loadingOptions)
+    return load_document_by_string(doc, baseuri, loadingOptions, id_)
 
 
 def load_document(
     doc: Any,
     baseuri: Optional[str] = None,
     loadingOptions: Optional[LoadingOptions] = None,
+    id_: Optional[str] = None,
 ) -> Any:
     """Load a CWL object from a serialized YAML string or a YAML object."""
     if baseuri is None:
         baseuri = cwl_v1_0.file_uri(os.getcwd()) + "/"
     if isinstance(doc, str):
-        return load_document_by_string(doc, baseuri, loadingOptions)
-    return load_document_by_yaml(doc, baseuri, loadingOptions)
+        return load_document_by_string(doc, baseuri, loadingOptions, id_)
+    return load_document_by_yaml(doc, baseuri, loadingOptions, id_)
 
 
 def load_document_by_string(
-    string: str, uri: str, loadingOptions: Optional[LoadingOptions] = None
+    string: str,
+    uri: str,
+    loadingOptions: Optional[LoadingOptions] = None,
+    id_: Optional[str] = None,
 ) -> Any:
     """Load a CWL object from a serialized YAML string."""
     yaml = yaml_no_ts()
     result = yaml.load(string)
-    return load_document_by_yaml(result, uri, loadingOptions)
+    return load_document_by_yaml(result, uri, loadingOptions, id_)
 
 
 def load_document_by_yaml(
-    yaml: Any, uri: str, loadingOptions: Optional[LoadingOptions] = None
+    yaml: Any,
+    uri: str,
+    loadingOptions: Optional[LoadingOptions] = None,
+    id_: Optional[str] = None,
 ) -> Any:
     """Load a CWL object from a YAML object."""
     version = cwl_version(yaml)
+    if "$graph" in yaml:
+        yaml = _get_id_from_graph(yaml, id_)
+        yaml["cwlVersion"] = version
     if version == "v1.0":
         result = cwl_v1_0.load_document_by_yaml(
             yaml, uri, cast(Optional[cwl_v1_0.LoadingOptions], loadingOptions)

testdata/echo-tool-packed.cwl

@@ -0,0 +1,31 @@
+cwlVersion: v1.2
+$graph:
+  - class: CommandLineTool
+    id: first
+    inputs:
+      in:
+        type: Any
+    outputs:
+      out:
+        type: string
+        outputBinding:
+          glob: out.txt
+          loadContents: true
+          outputEval: $(self[0].contents)
+    baseCommand: [ echo, first ]
+    stdout: out.txt
+  - class: CommandLineTool
+    id: main
+    inputs:
+      in:
+        type: Any
+        inputBinding: {}
+    outputs:
+      out:
+        type: string
+        outputBinding:
+          glob: out.txt
+          loadContents: true
+          outputEval: $(self[0].contents)
+    baseCommand: echo
+    stdout: out.txt

testdata/js-expr-req-wf.cwl

@@ -0,0 +1,30 @@
+cwlVersion: v1.2
+$graph:
+  - id: tool
+    class: CommandLineTool
+    requirements:
+      InlineJavascriptRequirement:
+        expressionLib:
+          - "function foo() { return 2; }"
+    inputs: []
+    outputs:
+      out: stdout
+    arguments: [echo, $(foo())]
+    stdout: whatever.txt
+
+  - id: wf
+    class: Workflow
+    requirements:
+      InlineJavascriptRequirement:
+        expressionLib:
+          - "function bar() { return 1; }"
+    inputs: []
+    outputs:
+      out:
+        type: File
+        outputSource: tool/out
+    steps:
+      tool:
+        run: "#tool"
+        in: {}
+        out: [out]

tests/test_parser.py

@@ -1,9 +1,11 @@
 """Test the load and save functions for CWL."""
 from pathlib import Path
 
+from pytest import raises
 from ruamel.yaml.main import YAML
 
 import cwl_utils.parser.latest as latest
+from cwl_utils.errors import GraphTargetMissingException
 from cwl_utils.parser import (
     cwl_v1_2,
     cwl_version,
@@ -86,3 +88,24 @@ def test_shortname() -> None:
     assert cwl_v1_2.shortname("http://example.com/foo#bar") == "bar"
     assert cwl_v1_2.shortname("http://example.com/#foo/bar") == "bar"
     assert cwl_v1_2.shortname("http://example.com/foo#bar/baz") == "baz"
+
+
+def test_get_id_from_graph() -> None:
+    """Test loading an explicit id of a CWL document with $graph property."""
+    uri = Path(HERE / "../testdata/echo-tool-packed.cwl").resolve().as_uri()
+    cwl_obj = load_document_by_uri(uri + "#main")
+    assert cwl_obj.id == uri + "#main"
+
+
+def test_get_default_id_from_graph() -> None:
+    """Test that loading the default id of a CWL document with $graph property returns the `#main` id."""
+    uri = Path(HERE / "../testdata/echo-tool-packed.cwl").resolve().as_uri()
+    cwl_obj = load_document_by_uri(uri)
+    assert cwl_obj.id == uri + "#main"
+
+
+def test_get_default_id_from_graph_without_main() -> None:
+    """Test that loading the default id of a CWL document with $graph property and no `#main` id throws an error."""
+    with raises(GraphTargetMissingException):
+        uri = Path(HERE / "../testdata/js-expr-req-wf.cwl").resolve().as_uri()
+        load_document_by_uri(uri)