Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add unmarshalling to AST (Part 3/n for transpiler implementation) #7533

Closed
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
feat: add unmarshalling to AST
Signed-off-by: isubasinghe <isitha@pipekit.io>
isubasinghe committed Jan 10, 2022
commit 73b90b1fc522da7034e7f09b70dec493e82de023
303 changes: 303 additions & 0 deletions cmd/cwl2argo/transpiler/ast_cl.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,303 @@
package transpiler

// interface due to large number of child types
type CWLRequirements interface {
isCWLRequirement()
}

type DockerRequirement struct {
Class string `yaml:"class"`
DockerPull *string `yaml:"dockerPull"`
DockerLoad *string `yaml:"dockerLoad"`
DockerFile *string `yaml:"dockerFile"`
DockerImport *string `yaml:"dockerImport"`
DockerImageId *string `yaml:"dockerImageId"`
DockerOutputDirectory *string `yaml:"dockerOutputDirectory"`
}

type SoftwarePackage struct {
Package string `yaml:"package"`
Version Strings `yaml:"version"`
Specs Strings `yaml:"specs"`
}

type SoftwareRequirement struct {
Class string `yaml:"class"` // constant SoftwareRequirement
Packages []SoftwarePackage `yaml:"packages"`
}

type LoadListingRequirement struct {
Class string `yaml:"class"` // constant LoadListingRequirement
LoadListing *LoadListingEnum `yaml:"loadListing"`
}

type Dirent struct {
entry CWLExpression `yaml:"entry"`
entryName *CWLExpression `yaml:"entryName"`
writeable *bool `yaml:"writeable"`
}

type InitialWorkDirRequirementListing interface {
isInitialWorkDirRequirementListing()
}

type InitialWorkDirRequirement struct {
Class string `yaml:"class"` // constant InitialWorkDirRequirement
Listing InitialWorkDirRequirementListing `yaml:"listing"`
}

type InlineJavascriptRequirement struct {
Class string `yaml:"class"` // constant InlineJavascriptRequirement
ExpressionLib Strings `yaml:"expressionLib"`
}

type SchemaDefRequirementType interface {
isSchemaDefRequirementType()
}

type SchemaDefRequirement struct {
Class string `yaml:"class"` // constant SchemaDefRequirement
Types []SchemaDefRequirementType `yaml:"types"`
}

type EnvironmentDef struct {
EnvName string `yaml:"envName"`
EnvValue CWLExpression `yaml:"envValue"`
}

type EnvVarRequirement struct {
Class string `yaml:"class"` // constant EnvVarRequirement
EnvDef []EnvironmentDef `yaml:"envDef"`
}

type ShellCommandRequirement struct {
Class string `yaml:"class"` // constant ShellCommandRequirement
}

type WorkReuse struct {
Class string `yaml:"class"` // constant WorkReuse
EnableReuse CWLExpression `yaml:"enableReuse"`
}

type NetworkAccess struct {
Class string // constant NetworkAccess
NetworkAccess CWLExpression
}

type InplaceUpdateRequirement struct {
Class string `yaml:"class"` // constant InplaceUpdateRequirement
InplaceUpdate Bool `yaml:"inplaceUpdate"`
}

type ToolTimeLimit struct {
Class string `yaml:"class"` // constant ToolTimeLimit
TimeLimit CWLExpression `yaml:"timeLimit"`
}

type ResourceRequirement struct {
Class string `yaml:"class"` // constand ResourceRequirement
CoresMin *CWLExpression `yaml:"coresMin"`
CoresMax *CWLExpression `yaml:"coresMax"`
RamMin *CWLExpression `yaml:"ramMin"`
RamMax *CWLExpression `yaml:"ramMax"`
TmpdirMin *CWLExpression `yaml:"tmpdirMin"`
TmpdirMax *CWLExpression `yaml:"tmpdirMin"`
OutdirMin *CWLExpression `yaml:"outdirMin"`
OutdirMax *CWLExpression `yaml:"outdirMax"`
}

func (_ InlineJavascriptRequirement) isCWLRequirement() {}
func (_ SchemaDefRequirement) isCWLRequirement() {}
func (_ LoadListingRequirement) isCWLRequirement() {}
func (_ DockerRequirement) isCWLRequirement() {}
func (_ SoftwareRequirement) isCWLRequirement() {}
func (_ InitialWorkDirRequirement) isCWLRequirement() {}
func (_ EnvVarRequirement) isCWLRequirement() {}
func (_ ShellCommandRequirement) isCWLRequirement() {}
func (_ WorkReuse) isCWLRequirement() {}
func (_ NetworkAccess) isCWLRequirement() {}
func (_ InplaceUpdateRequirement) isCWLRequirement() {}
func (_ ToolTimeLimit) isCWLRequirement() {}

func (_ CommandlineInputRecordSchema) isSchemaDefRequirementType() {}
func (_ CommandlineInputEnumSchema) isSchemaDefRequirementType() {}
func (_ CommandlineInputArraySchema) isSchemaDefRequirementType() {}
func (_ DockerRequirement) isSchemaDefRequirementType() {}
func (_ SoftwareRequirement) isSchemaDefRequirementType() {}
func (_ InitialWorkDirRequirement) isSchemaDefRequirementType() {}

type CommandlineInputRecordField struct {
Name string `yaml:"name"`
Type CommandlineTypes `yaml:"type"` // len(1) represents scalar len > 1 represents array
Doc Strings `yaml:"doc"`
Label *string `yaml:"label"`
SecondaryFiles SecondaryFiles `yaml:"secondaryFiles"`
Streamable *bool `yaml:"streamable"`
Format CWLFormat `yaml:"format"`
LoadContents *bool `yaml:"loadContents"`
LoadListing LoadListingEnum `yaml:"loadListing"`
InputBinding *CommandlineBinding `yaml:"inputBinding"`
}

type CommandlineInputArraySchema struct {
Items CommandlineTypes `yaml:"items"`
Type string `yaml:"type"` // MUST be array
Label *string `yaml:"label"`
Doc Strings `yaml:"doc"`
Name *string `yaml:"name"`
InputBinding *CommandlineBinding `yaml:"inputBinding"`
}

type CommandlineInputEnumSchema struct {
Symbols Strings `yaml:"symbols"`
Type string `yaml:"type"` // MUST BE enum, only a placeholder for type verification purposes
Label *string `yaml:"label"`
Doc Strings `yaml:"doc"`
Name *string `yaml:"name"`
InputBinding *CommandlineBinding
}

type CommandlineInputRecordSchema struct {
Type string `yaml:"type"` // MUST BE "record"
Fields *[]CommandlineInputRecordField `yaml:"fields"`
Label *string `yaml:"label"`
Doc *Strings `yaml:"doc"`
Name *string `yaml:"name"`
inputBinding *CommandlineBinding `yaml:"inputBinding"`
}

func (_ CWLNull) isFlat() {}
func (_ CWLBool) isFlat() {}
func (_ CWLInt) isFlat() {}
func (_ CWLLong) isFlat() {}
func (_ CWLFloat) isFlat() {}
func (_ CWLDouble) isFlat() {}
func (_ CWLFile) isFlat() {}
func (_ CWLDirectory) isFlat() {}
func (_ CWLStdin) isFlat() {}
func (_ String) isFlat() {}
func (_ CommandlineInputEnumSchema) isFlat() {}

type Type int32

const (
CWLNullKind Type = iota
CWLBoolKind
CWLIntKind
CWLLongKind
CWLFloatKind
CWLDoubleKind
CWLFileKind
CWLDirectoryKind
CWLStdinKind
CWLStringKind
CWLRecordKind
CWLRecordFieldKind
CWLEnumKind
CWLArrayKind
)

type CommandlineType struct {
Kind Type
Record *CommandlineInputRecordSchema
Enum *CommandlineInputEnumSchema
Array *CommandlineInputArraySchema
}

type CommandlineTypes []CommandlineType

type CommandlineBinding struct {
LoadContents *bool `yaml:"loadContents"`
Position *int `yaml:"position"`
Prefix *string `yaml:"prefix"`
Seperate *bool `yaml:"seperate"`
ItemSeperator *string `yaml:"itemSeperator"`
ValueFrom CWLExpression `yaml:"valueFrom"`
ShellQuote *bool `yaml:"bool"`
}

type CommandlineInputParameter struct {
Type CommandlineTypes `yaml:"type"` // len(1) == scalar while len > 1 == array
Label *string `yaml:"label"`
SecondaryFiles SecondaryFiles `yaml:"secondaryFiles"` // len(1) == scalar while len > 1 == array
Streamable *bool `yaml:"streamable"`
Doc Strings `yaml:"doc"`
Id *string `yaml:"id"`
Format CWLFormat `yaml:"format"`
LoadContents *bool `yaml:"loadContents"`
LoadListing LoadListingEnum `yaml:"loadListing"`
Default interface{} `yaml:"default"`
InputBinding *CommandlineBinding `yaml:"inputBinding"`
}

type OutputBindingGlobKind int32

const (
GlobStringKind OutputBindingGlobKind = iota
GlobStringsKind
GlobExpressionKind
)

type CommandlineOutputBindingGlob struct {
Kind OutputBindingGlobKind
String String
Strings Strings
Expression CWLExpression
}

type CommandlineOutputBinding struct {
LoadContents *bool `yaml:"loadContents"`
LoadListing LoadListingEnum `yaml:"loadListing"`
Glob CommandlineOutputBindingGlob `yaml:"glob"`
OutputEval CWLExpression `yaml:"outputEval"`
}

type CommandlineOutputParameter struct {
Type CommandlineTypes `yaml:"type"`
Label *string `yaml:"label"`
SecondaryFiles SecondaryFiles `yaml:"secondaryFiles"`
Streamable *bool `yaml:"streamable"`
Doc Strings `yaml:"doc"`
Id *string `yaml:"id"`
Format CWLFormat `yaml:"format"`
OutputBinding *CommandlineOutputBinding `yaml:"outputBinding"`
}

type CommandlineArgumentKind int32

const (
ArguementStringKind CommandlineArgumentKind = iota
ArguementExpressionKind
ArguementCLIBindingKind
)

type CommandlineArgument struct {
Kind CommandlineArgumentKind
String String
Expression CWLExpression
CommandlineBinding CommandlineBinding
}

type Inputs []CommandlineInputParameter
type Outputs []CommandlineOutputParameter
type Requirements []CWLRequirements
type Hints []interface{}
type Arguments []CommandlineArgument

type CommandlineTool struct {
Inputs Inputs `yaml:"inputs"`
Outputs Outputs `yaml:"outputs"`
Class string `yaml:"class"` // Must be "CommandLineTool"
Id *string `yaml:"id"`
Label *string `yaml:"label"`
Doc Strings `yaml:"doc"`
Requirements Requirements `yaml:"requirements"`
Hints Hints `yaml:"hints"`
CWLVersion *string `yaml:"cwlVersion"`
Intent Strings `yaml:"intent"`
BaseCommand Strings `yaml:"baseCommand"`
Arguments Arguments `yaml:"arguments"`
Stdin CWLExpression `yaml:"stdin"`
Stderr CWLExpression `yaml:"stderr"`
Stdout CWLExpression `yaml:"stdout"`
}
89 changes: 89 additions & 0 deletions cmd/cwl2argo/transpiler/ast_common.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package transpiler

type (
String string
Bool bool
Int int
Float float32
Strings []string
SecondaryFiles []CWLSecondaryFileSchema
)

type CWLFormatKind int32

const (
FormatStringKind CWLFormatKind = iota
FormatStringsKind
FormatExpressionKind
)

type CWLFormat struct {
Kind CWLFormatKind
String String
Strings Strings
Expression CWLExpression
}

type (
CWLNull struct{}
CWLBool struct{}
CWLInt struct{}
CWLLong struct{}
CWLFloat struct{}
CWLDouble struct{}
CWLString struct{}
CWLFile struct{}
CWLDirectory struct{}
)

func (_ CWLNull) isCWLType() {}
func (_ CWLBool) isCWLType() {}
func (_ CWLInt) isCWLType() {}
func (_ CWLLong) isCWLType() {}
func (_ CWLFloat) isCWLType() {}
func (_ CWLDouble) isCWLType() {}
func (_ CWLString) isCWLType() {}
func (_ CWLFile) isCWLType() {}
func (_ CWLDirectory) isCWLType() {}

type CWLStdin struct{}

type CWLType interface {
isCWLType()
}

type LoadListingKind int32

const (
ShallowListingKind LoadListingKind = iota
DeepListingKind
NoListingKind
)

type LoadListingEnum struct {
Kind LoadListingKind
}

type CWLExpressionKind int32

const (
RawKind CWLExpressionKind = iota
ExpressionKind
BoolKind
IntKind
FloatKind
)

type CWLExpression struct {
Kind CWLExpressionKind
Raw string
Expression string
Bool bool
Int int
Float float64
}

type CWLSecondaryFileSchema struct {
Pattern CWLExpression `yaml:"pattern"`
Required CWLExpression `yaml:"required"`
}
303 changes: 303 additions & 0 deletions cmd/cwl2argo/transpiler/fill_cl.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,303 @@
package transpiler

import (
"errors"
"fmt"

"gopkg.in/yaml.v3"
)

func keyNotPresentError(key string) error {
return fmt.Errorf("Could not find %s", key)
}

// intermediate representation used to
// parse into interfaces. The class string is used
// to decode Node into a structure.
type IntermediateRepr struct {
Class *string
Node *yaml.Node
}

func getCWLExpressionInner(str string) *string {
if len(str) < 3 {
return nil
}
if str[0] != '$' {
return nil
}
if str[1] == '[' && str[len(str)-1] == ']' {
return &str
}
if str[1] == '{' && str[len(str)-1] == '}' {
return &str
}
return nil
}

func (s *Strings) UnmarshalYAML(value *yaml.Node) error {
strings := make([]string, 0)
switch value.Kind {
case yaml.ScalarNode:
var s string
if err := value.Decode(&s); err != nil {
return err
}
strings = append(strings, s)
case yaml.SequenceNode:
if err := value.Decode(&strings); err != nil {
return err
}
default:
return errors.New("string | []string expected")
}
*s = strings
return nil
}

func (tys *CommandlineTypes) UnmarshalYAML(value *yaml.Node) error {
newTys := make([]CommandlineType, 0)
switch value.Kind {
case yaml.ScalarNode:
var s string
if err := value.Decode(&s); err != nil {
return err
}
var ty CommandlineType
switch s {
case "string":
ty.Kind = CWLStringKind
case "null":
ty.Kind = CWLNullKind
case "boolean":
ty.Kind = CWLBoolKind
case "int":
ty.Kind = CWLIntKind
case "long":
ty.Kind = CWLLongKind
case "float":
ty.Kind = CWLFloatKind
case "double":
ty.Kind = CWLDoubleKind
case "File":
ty.Kind = CWLFileKind
case "Directory":
ty.Kind = CWLDirectoryKind
default:
return fmt.Errorf("%s is not a supported type", s)
}
newTys = append(newTys, ty)
case yaml.MappingNode:
return errors.New("complex types not supported yet")
case yaml.SequenceNode:
return errors.New("array types not supported yet")
default:
return errors.New("type not supported")
}
*tys = newTys
return nil
}

func (format *CWLFormat) UnmarshalYAML(value *yaml.Node) error {
switch value.Kind {
case yaml.ScalarNode:
var s string
if err := value.Decode(&s); err != nil {
return err
}
format.Kind = FormatStringKind
format.String = String(s)
return nil
case yaml.SequenceNode:
s := make([]string, 0)
if err := value.Decode(&s); err != nil {
return err
}
format.Kind = FormatStringsKind
format.Strings = s
return nil
default:
return errors.New("string | []string expected")
}
}

func (listing *LoadListingEnum) UnmarshalYAML(value *yaml.Node) error {
return errors.New("LoadListingEnum not yet supported")
}

func (input *CommandlineInputParameter) UnmarshalYAML(value *yaml.Node) error {
type rawParamType CommandlineInputParameter

err := value.Decode((*rawParamType)(input))
if err != nil {
return err
}
return nil
}

func (inp *Inputs) UnmarshalYAML(value *yaml.Node) error {
inputs := make([]CommandlineInputParameter, 0)

switch value.Kind {
case yaml.MappingNode:
m := make(map[string]CommandlineInputParameter)
err := value.Decode(&m)
if err != nil {
return err
}
for key, input := range m {
input.Id = &key
inputs = append(inputs, input)
}
case yaml.SequenceNode:
err := value.Decode(&inputs)
if err != nil {
return err
}
default:
return errors.New("sequence or mapping type expected")
}
*inp = inputs
return nil
}

func (out *Outputs) UnmarshalYAML(value *yaml.Node) error {
outputs := make([]CommandlineOutputParameter, 0)

switch value.Kind {
case yaml.MappingNode:
m := make(map[string]CommandlineOutputParameter)
err := value.Decode(&m)
if err != nil {
return err
}
for key, output := range m {
output.Id = &key
outputs = append(outputs, output)
}
case yaml.SequenceNode:
err := value.Decode(&outputs)
if err != nil {
return err
}
default:
return errors.New("Sequence or mapping type expected")
}

*out = outputs

return nil
}

func (ir *IntermediateRepr) UnmarshalYAML(value *yaml.Node) error {
m := make(map[string]interface{})
err := value.Decode(&m)
if err != nil {
return err
}
classi, ok := m["class"]
if ok {
class, ok := classi.(string)
if !ok {
return errors.New("string expected")
}
ir.Class = &class
}
ir.Node = value
return nil
}

func (reqs *Requirements) UnmarshalYAML(value *yaml.Node) error {
rs := make(map[string]IntermediateRepr, 0)
err := value.Decode(&rs)
if err != nil {
rsArray := make([]IntermediateRepr, 0)
err = value.Decode(&rsArray)
if err != nil {
return errors.New("[]requirement or map[class]requirement was expected")
}
for _, req := range rsArray {
if req.Class == nil {
return errors.New("class expected")
}
rs[*req.Class] = req
}
}

newRequests := make([]CWLRequirements, 0)
for class, req := range rs {
switch class {
case "DockerRequirement":
var d DockerRequirement
err := req.Node.Decode(&d)
if err != nil {
return err
}
newRequests = append(newRequests, d)
default:
return fmt.Errorf("%s is not implemented", class)
}
}
*reqs = newRequests
return nil
}

func (hints Hints) UnmarshalYAML(value *yaml.Node) error {
return errors.New("hints not supported yet")
}

func (args Arguments) UnmarshalYAML(value *yaml.Node) error {
return errors.New("arguments not supported yet")
}

func (expr *CWLExpression) UnmarshalYAML(value *yaml.Node) error {
if value.Kind != yaml.ScalarNode {
return errors.New("can only be string | bool | int | float")
}
var f float64
err := value.Decode(&f)
if err == nil {
expr.Kind = FloatKind
expr.Float = f
return nil
}

var i int
err = value.Decode(&i)
if err == nil {
expr.Kind = IntKind
expr.Int = i
return nil
}

var b bool
err = value.Decode(&b)
if err == nil {
expr.Kind = BoolKind
expr.Bool = b
return nil
}

var s string
err = value.Decode(&s)
if err == nil {
exprS := getCWLExpressionInner(s)
if exprS != nil {
expr.Kind = ExpressionKind
expr.Expression = *exprS
return nil
}
expr.Kind = RawKind
expr.Raw = s
return nil
}
return errors.New("can only be string | bool | int | float")
}

func (cl *CommandlineTool) UnmarshalYAML(value *yaml.Node) error {
type rawCLITool CommandlineTool
if err := value.Decode((*rawCLITool)(cl)); err != nil {
return err
}
return nil
}
31 changes: 31 additions & 0 deletions cmd/cwl2argo/transpiler/fill_cl_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package transpiler

import (
"testing"

"gopkg.in/yaml.v3"
)

func TestSimpleCWL(t *testing.T) {
data := `cwlVersion: v1.2
class: CommandLineTool
requirements:
- class: DockerRequirement
dockerPull: python:3.7
baseCommand: echo
id: echo-tool
inputs:
message:
type: string
inputBinding:
position: 1
outputs: []
`

var cliTool CommandlineTool
err := yaml.Unmarshal([]byte(data), &cliTool)
if err != nil {
t.Error(err)
}

}