Support array of pair (#266)

* Handling pairs correctly in input files

RELEASE_NOTES.md

@@ -1,5 +1,40 @@
 # Release Notes
 
+## 1.10
+**Fixed**
+- Handling of pair type in a JSON input file
+- Allow overriding default values in calls from the input file. For example, in a workflow like:
+
+```wdl
+version 1.0
+
+workflow override {
+    call etl { input: a = 3 }
+    output {
+        Int result = etl.result
+    }
+}
+
+task etl {
+    input {
+        Int a
+        Int b = 10
+    }
+    command {}
+    output {
+        Int result = a + b
+    }
+}
+```
+
+We can now set b to a value other than 10, with an input file like this:
+
+```
+{
+  "override.etl.b" : 5
+}
+```
+
 ## 1.09
 **Fixed**
 - The `-p` flag was not respected

scripts/run_tests.py

@@ -49,7 +49,11 @@
     "top",
 
     # can we download from a container?
-    "download_from_container"
+    "download_from_container",
+
+    # input file with pairs
+    "echo_pairs",
+    "array_structs"
 ]
 
 # docker image tests

src/main/resources/application.conf

@@ -1,5 +1,5 @@
 dxWDL {
-    version = "1.09"
+    version = "1.10"
 }
 
 #

src/main/scala/dxWDL/compiler/InputFile.scala

@@ -71,12 +71,19 @@ case class InputFileScan(bundle: IR.Bundle,
                 val vFiles = valuesJs.flatMap(findDxFiles(valueType, _))
                 kFiles.toVector ++ vFiles.toVector
 
+                // Two ways of writing pairs: an object with left/right fields, or an array
+                // with two elements.
             case (WomPairType(lType, rType), JsObject(fields))
                     if (List("left", "right").forall(fields contains _)) =>
                 val lFiles = findDxFiles(lType, fields("left"))
                 val rFiles = findDxFiles(rType, fields("right"))
                 lFiles ++ rFiles
 
+            case (WomPairType(lType, rType), JsArray(Vector(l,r))) =>
+                val lFiles = findDxFiles(lType, l)
+                val rFiles = findDxFiles(rType, r)
+                lFiles ++ rFiles
+
             case (WomArrayType(t), JsArray(vec)) =>
                 vec.flatMap( findDxFiles(t, _ ))
 
@@ -210,12 +217,18 @@ case class InputFile(fileInfoDir: Map[DXFile, DxDescribe],
                 }.toMap
                 WomMap(WomMapType(keyType, valueType), m)
 
+            // a few ways of writing a pair: an object, or an array
             case (WomPairType(lType, rType), JsObject(fields))
                     if (List("left", "right").forall(fields contains _)) =>
                 val left = womValueFromCromwellJSON(lType, fields("left"))
                 val right = womValueFromCromwellJSON(rType, fields("right"))
                 WomPair(left, right)
 
+            case (WomPairType(lType, rType), JsArray(Vector(l,r))) =>
+                val left = womValueFromCromwellJSON(lType, l)
+                val right = womValueFromCromwellJSON(rType, r)
+                WomPair(left, right)
+
             // empty array
             case (WomArrayType(t), JsNull) =>
                 WomArray(WomArrayType(t), List.empty[WomValue])
@@ -280,6 +293,13 @@ case class InputFile(fileInfoDir: Map[DXFile, DxDescribe],
                              k -> jsValueFromCromwellJSON(valueType, v)
                          })
 
+            // a pair can be written as an object or an array with two elements.
+            case (WomPairType(lType, rType), JsObject(fields))
+                    if (List("left", "right").forall(fields contains _)) =>
+                val lJs = jsValueFromCromwellJSON(lType, fields("left"))
+                val rJs = jsValueFromCromwellJSON(rType, fields("right"))
+                JsObject("left" -> lJs, "right" -> rJs)
+
             case (WomPairType(lType, rType), JsArray(Vector(l,r))) =>
                 val lJs = jsValueFromCromwellJSON(lType, l)
                 val rJs = jsValueFromCromwellJSON(rType, r)

src/test/resources/input_file/echo_pairs.json

@@ -0,0 +1,6 @@
+{"echo_pairs.pairs_of_words":
+  [
+    ["hello", "world"],
+    {"left" : "hello_again", "right" : "world_again"}
+  ]
+}

src/test/resources/input_file/echo_pairs.wdl

@@ -0,0 +1,36 @@
+version 1.0
+
+task echo_this_pair {
+  input {
+    String first_word
+    String second_word
+  }
+  command <<<
+  set -exo
+
+  echo ~{first_word} >> output.txt
+  echo ~{second_word} >> output.txt
+  >>>
+  runtime {
+    docker: "debian:stretch-slim"
+  }
+  output {
+    String echoed_words = read_string("output.txt")
+  }
+}
+
+workflow echo_pairs {
+  input {
+    Array[Pair[String, String]] pairs_of_words
+  }
+  scatter (pair in pairs_of_words) {
+    call echo_this_pair {
+      input:
+        first_word = pair.left,
+        second_word = pair.right
+    }
+  }
+  output {
+    Array[String] all_echoed_words = echo_this_pair.echoed_words
+  }
+}

src/test/resources/input_file/override.wdl

@@ -0,0 +1,20 @@
+version 1.0
+
+workflow override {
+    call etl { input: a = 3 }
+    output {
+        Int result = etl.result
+    }
+}
+
+task etl {
+    input {
+        Int a
+        Int b = 10
+        Int? c
+    }
+    command {}
+    output {
+        Int result = a + b
+    }
+}

src/test/resources/input_file/override_input.json

@@ -0,0 +1,3 @@
+{
+  "override.etl.b" : 5
+}

src/test/resources/struct/array_of_structs.wdl

@@ -0,0 +1,35 @@
+version 1.0
+
+struct WordStruct {
+  String word
+}
+
+task echo_xx {
+  input {
+    WordStruct input_word
+  }
+  command <<<
+    echo ~{input_word.word}
+  >>>
+  runtime {
+    docker: "debian:stretch-slim"
+  }
+  output {
+    String out = read_string(stdout())
+  }
+}
+
+workflow array_of_structs {
+  input {
+    Array[WordStruct] words_to_say
+  }
+  scatter (input_word in words_to_say) {
+    call echo_xx {
+      input:
+        input_word = input_word
+    }
+  }
+  output {
+    Array[String] out_words = echo_xx.out
+  }
+}

src/test/resources/struct/array_of_structs_input.json

@@ -0,0 +1,4 @@
+{"array_of_structs.words_to_say": [
+  {"word": "hello"},
+  {"word": "world"}
+]}

src/test/scala/dxWDL/compiler/InputFileTest.scala

@@ -119,7 +119,7 @@ class InputFileTest extends FlatSpec with Matchers {
     }
 
 
-    it should "support struct inputs" taggedAs(EdgeTest) in {
+    it should "support struct inputs" in {
         val wdlCode = pathFromBasename("struct", "Person.wdl")
         val inputs = pathFromBasename("struct", "Person_input.json")
 
@@ -129,4 +129,37 @@ class InputFileTest extends FlatSpec with Matchers {
         )
         retval shouldBe a [Main.SuccessfulTerminationIR]
     }
+
+    it should "support array of pairs" in {
+        val wdlCode = pathFromBasename("input_file", "echo_pairs.wdl")
+        val inputs = pathFromBasename("input_file", "echo_pairs.json")
+
+        val retval = Main.compile(
+            List(wdlCode.toString, "-inputs", inputs.toString)
+                ++ cFlags
+        )
+        retval shouldBe a [Main.SuccessfulTerminationIR]
+    }
+
+    it should "array of structs" in {
+        val wdlCode = pathFromBasename("struct", "array_of_structs.wdl")
+        val inputs = pathFromBasename("struct", "array_of_structs_input.json")
+
+        val retval = Main.compile(
+            List(wdlCode.toString, "-inputs", inputs.toString)
+                ++ cFlags
+        )
+        retval shouldBe a [Main.SuccessfulTerminationIR]
+    }
+
+    ignore should "override default values in input file" taggedAs(EdgeTest) in {
+        val wdlCode = pathFromBasename("input_file", "override.wdl")
+        val inputs = pathFromBasename("input_file", "override_input.json")
+
+        val retval = Main.compile(
+            List(wdlCode.toString, "-inputs", inputs.toString)
+                ++ cFlags
+        )
+        retval shouldBe a [Main.SuccessfulTerminationIR]
+    }
 }

test/input_file/echo_pairs.wdl

@@ -0,0 +1,36 @@
+version 1.0
+
+task echo_this_pair {
+  input {
+    String first_word
+    String second_word
+  }
+  command <<<
+  set -exo
+
+  echo ~{first_word} >> output.txt
+  echo ~{second_word} >> output.txt
+  >>>
+  runtime {
+    docker: "debian:stretch-slim"
+  }
+  output {
+    String echoed_words = read_string("output.txt")
+  }
+}
+
+workflow echo_pairs {
+  input {
+    Array[Pair[String, String]] pairs_of_words
+  }
+  scatter (pair in pairs_of_words) {
+    call echo_this_pair {
+      input:
+        first_word = pair.left,
+        second_word = pair.right
+    }
+  }
+  output {
+    Array[String] all_echoed_words = echo_this_pair.echoed_words
+  }
+}

test/input_file/echo_pairs_input.json

@@ -0,0 +1,6 @@
+{"echo_pairs.pairs_of_words":
+  [
+    ["hello", "world"],
+    ["hello_again", "world_again"]
+  ]
+}

test/input_file/echo_pairs_results.json

@@ -0,0 +1,3 @@
+{
+  "echo_pairs.all_echoed_words" : [ "hello\nworld", "hello_again\nworld_again"]
+}

test/struct/array_structs.wdl

@@ -0,0 +1,35 @@
+version 1.0
+
+struct WordStruct {
+  String word
+}
+
+task echo_xx {
+  input {
+    WordStruct input_word
+  }
+  command <<<
+    echo ~{input_word.word}
+  >>>
+  runtime {
+    docker: "debian:stretch-slim"
+  }
+  output {
+    String out = read_string(stdout())
+  }
+}
+
+workflow array_structs {
+  input {
+    Array[WordStruct] words_to_say
+  }
+  scatter (input_word in words_to_say) {
+    call echo_xx {
+      input:
+        input_word = input_word
+    }
+  }
+  output {
+    Array[String] out_words = echo_xx.out
+  }
+}

test/struct/array_structs_input.json

@@ -0,0 +1,4 @@
+{"array_structs.words_to_say": [
+  {"word": "hello"},
+  {"word": "world"}
+]}

test/struct/array_structs_results.json

@@ -0,0 +1,3 @@
+{
+  "array_structs.out_words" : ["hello", "world"]
+}