AN-733 WDL 1.1: Call input passthrough syntax (#7815)

Author: jgainerdewar

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Cromwell Change Log
 
+## 92 Release Notes
+
+### Progress toward WDL 1.1 Support
+* WDL 1.1 support is in progress. Users that would like to try out the current partial support can do so by using WDL version `development-1.1`. In Cromwell 92, `development-1.1` has been enhanced to include:
+    * Support for passthrough syntax for call inputs, e.g. `{ input: foo }` rather than `{ input: foo = foo }`.
+
 ## 91 Release Notes
 
 #### Removal of Google LifeSciences backend code

docs/developers/UpdateWDLParser.md

@@ -0,0 +1,35 @@
+This page contains instructions for using [Hermes](https://github.com/scottfrazer/hermes) to regenerate the WDL parser after making changes to the WDL grammar file, which is located at `wdl/transforms/$version/src/main/resources/grammar.hgr`. Note that there is a separate grammar file and parser for each WDL version.
+
+## Install Hermes
+
+Requires Python 3 (tested with Python 3.5). [Pyenv](https://github.com/pyenv/pyenv) is the recommended way to install this older Python version. Newer Python versions may not work correctly.
+
+Hermes can be installed via pip (https://pypi.python.org/pypi/hermes-parser/2.0rc6)
+
+However, It is recommended that Hermes is installed from source into a virtual environment because PyPI isn’t updated very often.
+
+First, clone Hermes and create then activate a new Python virtual environment (in directory ./hermes-venv)
+```
+$ git clone [email protected]:scottfrazer/hermes.git
+$ cd hermes
+$ pyenv virtualenv 3.5.10 hermes-venv
+$ pyenv activate hermes-venv
+```
+
+In the Hermes repo, make sure you have the latest ‘develop’ branch checked out. Then, install Hermes into the virtual environment (need to separately run pip installs because the Python 3.5 default index URL doesn't work anymore):
+```
+$ pip install --index-url https://pypi.org/simple/ moody-templates xtermcolor nose pygments hermes_pygments
+$ python setup.py install
+$ hermes --help
+```
+
+With the virtual environment activated, navigate to `cromwell/wdl/transforms/$version` and run the below command, replacing `$version` with the correct package name (e.g. `biscayne` or `cascades`).:
+```
+hermes generate src/main/resources/grammar.hgr \
+          --language=java \
+          --directory=src/main/java \
+          --name=wdl \
+          --java-package=wdl.$version.parser \
+          --java-use-apache-commons --java-imports=org.apache.commons.lang3.StringEscapeUtils \
+          --header
+```

mkdocs.yml

@@ -75,6 +75,7 @@ nav:
     - How to Build: developers/Building.md
     - Architecture: developers/Arch.md
     - Backend Development: developers/Backend.md
+    - Update WDL Parsers: developers/UpdateWDLParser.md
     - Instrumentation: developers/Instrumentation.md
     - Integration tests: developers/Centaur.md
     - Security: developers/Security.md

wdl/transforms/biscayne/src/main/java/wdl/biscayne/parser/WdlParser.java

@@ -1,9 +1,16 @@
+# Changelog
 2023-03-16
 Synced grammar from OpenWDL `development` version, which is actually development of 2.0. There is no 1.1 Hermes grammar, develop it here.
 Changed version declaration to `development1_1`.
 This disallows `version 1.1` workflows to run with incomplete support. Once development is finished, change to `1.1`.
 
 2024-02-28
+Added support for struct literals
+
+2025-09-30
+Added support for passthrough task inputs (supply `foo` rather than `foo=foo` when a local variable and task input share the same name).
+
+# HowTo
 When changing the grammar file, generate a new parser by:
 - changing current working directory to cromwell/wdl/transforms/biscayne
 - running: hermes generate src/main/resources/grammar.hgr \

wdl/transforms/biscayne/src/main/resources/CHANGELOG.txt

@@ -429,7 +429,10 @@ grammar {
     $wf_body_element = $call | $declaration | $if_stmt | $scatter | $inputs | $outputs | $wf_parameter_meta | $wf_meta
     $call = :call :fqn optional($alias) list($call_after) optional($call_brace_block) -> Call(task=$1, alias=$2, after=$3, body=$4)
     $call_brace_block = :lbrace optional($call_body) :rbrace -> $1
-    $call_body = :input :colon list($input_kv, :comma) -> CallBody(inputs=$2)
+    $call_body = :input :colon list($call_input, :comma) -> CallBody(inputs=$2)
+    $call_input = :identifier optional($input_assignment) -> CallInputKV(key=$0, value=$1)
+    $input_assignment = :equal $e -> $1
+
     $alias = :as :identifier -> $1
     $call_after = :after :identifier -> $1
     $wf_parameter_meta = :parameter_meta $meta_map -> ParameterMeta(map=$1)
@@ -440,6 +443,5 @@ grammar {
         -> Scatter(item=$2, collection=$4, body=$7)
 
     $object_kv = :identifier :colon $e -> ObjectKV(key=$0, value=$2)
-    $input_kv = :identifier :equal $e -> ObjectKV(key=$0, value=$2)
   }
 }

wdl/transforms/biscayne/src/main/resources/grammar.hgr

@@ -0,0 +1,22 @@
+package wdl.transforms.biscayne.ast2wdlom
+
+import common.transforms.CheckedAtoB
+import wdl.model.draft3.elements.ExpressionElement
+import wdl.model.draft3.elements.ExpressionElement.KvPair
+import wdl.transforms.base.ast2wdlom.{AstNodeToKvPair, GenericAst, GenericAstNode}
+
+object BiscayneAstNodeToKvPair {
+  def astNodeToKvPair(implicit
+    astNodeToExpressionElement: CheckedAtoB[GenericAstNode, ExpressionElement]
+  ): CheckedAtoB[GenericAstNode, KvPair] = CheckedAtoB.fromErrorOr {
+    case i: GenericAst if i.getName == "CallInputKV" =>
+      // For CallInputKV, we accept either a key/value pair (`foo=foo`) or a plain identifier (`foo`).
+      // In the latter case, "foo" is both the key and the name of an in-scope variable whose value should
+      // be passed through. Due to limitations in the parser, both map to the same AST structure,
+      // with the "value" attribute missing when a passthrough is intended.
+      val valueKeyName = if (i.getAttributes.contains("value")) "value" else "key"
+      AstNodeToKvPair.validate(i.getAttributeAs[String]("key"), i.getAttributeAs[ExpressionElement](valueKeyName))
+    case a: GenericAst if a.getName == "ObjectKV" || a.getName == "MapLiteralKv" || a.getName == "RuntimeAttribute" =>
+      AstNodeToKvPair.validate(a.getAttributeAs[String]("key"), a.getAttributeAs[ExpressionElement]("value"))
+  }
+}

wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/ast2wdlom/BiscayneAstNodeToKvPair.scala

@@ -33,7 +33,7 @@ package object ast2wdlom {
       AstToNewExpressionElements.newBiscayneEngineFunctionMakers
     )
   implicit val astNodeToKvPair: CheckedAtoB[GenericAstNode, KvPair] =
-    AstNodeToKvPair.astNodeToKvPair(astNodeToExpressionElement)
+    BiscayneAstNodeToKvPair.astNodeToKvPair(astNodeToExpressionElement)
 
   implicit val astNodeToTypeElement: CheckedAtoB[GenericAstNode, TypeElement] =
     AstNodeToTypeElement.astNodeToTypeElement(Map.empty)

wdl/transforms/biscayne/src/main/scala/wdl/transforms/biscayne/ast2wdlom/ast2wdlom.scala

@@ -0,0 +1,26 @@
+version development-1.1
+
+task sayHi {
+  input {
+    String name
+    String greeting
+    String inquiry
+  }
+  command <<<
+    cat "~{greeting} ~{name} ~{inquiry}" > /tmp/helloFile
+  >>>
+}
+
+workflow callInputPassthrough {
+  String name = "friend"
+  String greeting = "hello"
+  String inquiry = "how are you?"
+
+  # These two inputs should be handled the same way
+  call sayHi {
+    input:
+      name,
+      greeting = greeting,
+      inquiry
+  }
+}

wdl/transforms/biscayne/src/test/cases/call_input_passthrough.wdl

@@ -6,7 +6,7 @@ import org.scalatest.flatspec.AnyFlatSpec
 import org.scalatest.matchers.should.Matchers
 import wdl.model.draft3.elements.CommandPartElement.{PlaceholderCommandPartElement, StringCommandPartElement}
 import wdl.model.draft3.elements.ExpressionElement._
-import wdl.model.draft3.elements.{CallElement, FileElement, _}
+import wdl.model.draft3.elements._
 import wdl.transforms.biscayne.ast2wdlom.WdlFileToWdlomSpec._
 import wom.SourceFileLocation
 import wom.types._
@@ -467,6 +467,88 @@ object WdlFileToWdlomSpec {
           Some(SourceFileLocation(13))
         )
       )
-    )
+    ),
+    "call_input_passthrough" ->
+      FileElement(
+        imports = Vector.empty,
+        structs = Vector.empty,
+        workflows = Vector(
+          WorkflowDefinitionElement(
+            name = "callInputPassthrough",
+            inputsSection = None,
+            graphElements = Set(
+              IntermediateValueDeclarationElement(
+                PrimitiveTypeElement(WomStringType),
+                "name",
+                StringLiteral("friend")
+              ),
+              IntermediateValueDeclarationElement(
+                PrimitiveTypeElement(WomStringType),
+                "greeting",
+                StringLiteral("hello")
+              ),
+              IntermediateValueDeclarationElement(
+                PrimitiveTypeElement(WomStringType),
+                "inquiry",
+                StringLiteral("how are you?")
+              ),
+              CallElement(
+                "sayHi",
+                None,
+                Vector.empty,
+                Some(
+                  CallBodyElement(
+                    Vector(
+                      KvPair("name", IdentifierLookup("name")),
+                      KvPair("greeting", IdentifierLookup("greeting")),
+                      KvPair("inquiry", IdentifierLookup("inquiry"))
+                    )
+                  )
+                ),
+                Some(SourceFileLocation(20))
+              )
+            ),
+            outputsSection = None,
+            metaSection = None,
+            parameterMetaSection = None,
+            sourceLocation = Some(SourceFileLocation(14))
+          )
+        ),
+        tasks = Vector(
+          TaskDefinitionElement(
+            name = "sayHi",
+            inputsSection = Some(
+              InputsSectionElement(
+                Vector(
+                  InputDeclarationElement(PrimitiveTypeElement(WomStringType), "name", None),
+                  InputDeclarationElement(PrimitiveTypeElement(WomStringType), "greeting", None),
+                  InputDeclarationElement(PrimitiveTypeElement(WomStringType), "inquiry", None)
+                )
+              )
+            ),
+            declarations = Vector.empty,
+            outputsSection = None,
+            commandSection = CommandSectionElement(
+              List(
+                CommandSectionLine(
+                  Vector(
+                    StringCommandPartElement("cat \""),
+                    PlaceholderCommandPartElement(IdentifierLookup("greeting"), PlaceholderAttributeSet.empty),
+                    StringCommandPartElement(" "),
+                    PlaceholderCommandPartElement(IdentifierLookup("name"), PlaceholderAttributeSet.empty),
+                    StringCommandPartElement(" "),
+                    PlaceholderCommandPartElement(IdentifierLookup("inquiry"), PlaceholderAttributeSet.empty),
+                    StringCommandPartElement("\" > /tmp/helloFile")
+                  )
+                )
+              )
+            ),
+            runtimeSection = None,
+            metaSection = None,
+            parameterMetaSection = None,
+            sourceLocation = Some(SourceFileLocation(3))
+          )
+        )
+      )
   )
 }