Merge branch 'main' of https://github.com/finos/morphir

docs/spark-backend/design.md

@@ -0,0 +1,64 @@
+# Morphir to Spark mapping
+
+To understand how the Morphir to Apache Spark mapping works let's begin with an example Morphir domain model and business logic that we might want to transpile to Spark. We present the Morphir logic using Elm syntax to make it easy to read:
+
+```elm
+type alias RecordA =
+    { number1 : Int
+    , number2 : Int
+    }
+
+type alias RecordB =
+    { sum : Int
+    , product : Int
+    , ratio : Float
+    }
+
+
+job : List RecordA -> List RecordB
+job input =
+    input
+        |> List.filter (\a -> a.field2 < 100)
+        |> List.map 
+            (\a ->
+                { sum = a.number1 + a.number2
+                , product = a.number1 * a.number2
+                , ratio = a.number1 / a.number2
+                }
+            )
+```
+
+The above is a representative example of the base functionality. There are a few constraints that we put on the Elm code to make it possible to translate to Spark initially (later on we might gradually lift those constraints by automatically transforming the logic behind the scenes). Here are the initial restrictions:
+
+- Functions need to get a list of records as input and return a list of records, so that we can directly map to Spark jobs taking a DataSet and returning a DataSet
+- The record types in the input and output need to be completely flat and use only built-in SDK types or be enumerations (custom types with only no-argument constructors)
+- The input value in the function can only be passed through the following collection operations:
+  - [List.filter](https://package.elm-lang.org/packages/elm/core/latest/List#filter) to filter the data set (corresponds to a WHERE clause in SQL)
+  - [List.map](https://package.elm-lang.org/packages/elm/core/latest/List#map) to trasform each input row to an output row (corresponds to the SELECT clause in SQL) 
+- Field expressions can: 
+  - use any combination of operations from the [Morphir SDK](https://package.elm-lang.org/packages/elm/core/latest/) as long as every intermediary result within the expression is a simple type
+  - include `if-then-else` expressions
+
+## Supported Field Types
+
+- [Int](https://package.elm-lang.org/packages/elm/core/latest/Basics#Int)
+- [Float](https://package.elm-lang.org/packages/elm/core/latest/Basics#Float)
+- [Bool](https://package.elm-lang.org/packages/elm/core/latest/Basics#Bool)
+- [String](https://package.elm-lang.org/packages/elm/core/latest/String#String)
+- [Maybe](https://package.elm-lang.org/packages/elm/core/latest/Maybe#Maybe)
+
+
+## Supported Field Operations
+
+- [Basics](https://package.elm-lang.org/packages/elm/core/latest/Basics): 
+  - All number operations
+  - All comparison operations
+  - All boolean operations
+- [String](https://package.elm-lang.org/packages/elm/core/latest/String)
+  - All operations  
+
+## Supported DataSet Operations
+
+- [List.filter](https://package.elm-lang.org/packages/elm/core/latest/List#filter)
+- [List.map](https://package.elm-lang.org/packages/elm/core/latest/List#map)
+

website/yarn.lock

@@ -2411,9 +2411,9 @@ asap@~2.0.3:
   integrity sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=
 
 async@^2.6.2:
-  version "2.6.3"
-  resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff"
-  integrity sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==
+  version "2.6.4"
+  resolved "https://registry.yarnpkg.com/async/-/async-2.6.4.tgz#706b7ff6084664cd7eae713f6f965433b5504221"
+  integrity sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==
   dependencies:
     lodash "^4.17.14"
 
@@ -5225,9 +5225,9 @@ minimatch@^3.0.4:
     brace-expansion "^1.1.7"
 
 minimist@^1.2.0, minimist@^1.2.5:
-  version "1.2.5"
-  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
-  integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
+  version "1.2.6"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
+  integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
 
 mkdirp@^0.5.5:
   version "0.5.5"
@@ -5307,9 +5307,9 @@ [email protected]:
     whatwg-url "^5.0.0"
 
 node-forge@^1.2.0:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.2.1.tgz#82794919071ef2eb5c509293325cec8afd0fd53c"
-  integrity sha512-Fcvtbb+zBcZXbTTVwqGA5W+MKBj56UjVRevvchv5XrcyXbmNdesfZL37nlcWOfpgHhgmxApw3tQbTr4CqNmX4w==
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.0.tgz#37a874ea723855f37db091e6c186e5b67a01d4b2"
+  integrity sha512-08ARB91bUi6zNKzVmaj3QO7cr397uiDT2nJ63cHjyNtCTWIgvS47j3eT0WfzUwS9+6Z5YshRaoasFkXCKrIYbA==
 
 node-releases@^2.0.1:
   version "2.0.1"
@@ -5980,9 +5980,9 @@ prism-react-renderer@^1.2.1:
   integrity sha512-w23ch4f75V1Tnz8DajsYKvY5lF7H1+WvzvLUcF0paFxkTHSp42RS0H5CttdN2Q8RR3DRGZ9v5xD/h3n8C8kGmg==
 
 prismjs@^1.23.0:
-  version "1.26.0"
-  resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.26.0.tgz#16881b594828bb6b45296083a8cbab46b0accd47"
-  integrity sha512-HUoH9C5Z3jKkl3UunCyiD5jwk0+Hz0fIgQ2nbwU2Oo/ceuTAQAg+pPVnfdt2TJWRVLcxKh9iuoYDUSc8clb5UQ==
+  version "1.27.0"
+  resolved "https://registry.yarnpkg.com/prismjs/-/prismjs-1.27.0.tgz#bb6ee3138a0b438a3653dd4d6ce0cc6510a45057"
+  integrity sha512-t13BGPUlFDR7wRB5kQDG4jjl7XeuH6jbJGt11JHPL96qwsEHNX2+68tFXqc1/k+/jALsbSWJKUOT/hcYAZ5LkA==
 
 process-nextick-args@~2.0.0:
   version "2.0.1"