diff --git a/CHANGELOG.md b/CHANGELOG.md index ca02f4f9..182d154d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,11 @@ Types of changes - `Fixed` for any bug fixes. - `Security` in case of vulnerabilities. +## [3.1.0] + +- `Added` possibility to filter columns via `select` property in ingress descriptors +- `Added` commands `set-parent-select` and `set-child-select` to `lino id` + ## [3.0.2] - `Fixed` invalid length for DATE or NUMBER bind variable, bumping oracle driver to 2.8.19 diff --git a/README.md b/README.md index 14437bd0..2886600a 100755 --- a/README.md +++ b/README.md @@ -267,6 +267,8 @@ To modify the `ingress-descriptor.yml`, some commands can be used instead of edi - `lino id set-parent-lookup ` : modify the `lookup` property of the parent object - `lino id set-child-where ` : modify the `where` property of the child object - `lino id set-parent-where ` : modify the `where` property of the parent object +- `lino id set-child-select ...` : modify the `select` property of the child object +- `lino id set-parent-select ...` : modify the `select` property of the parent object - `lino id set-start-table ` : modify the `startTable` property of the ingress descriptor Example: diff --git a/internal/app/id/cli.go b/internal/app/id/cli.go index ce2831c8..350d5b91 100755 --- a/internal/app/id/cli.go +++ b/internal/app/id/cli.go @@ -59,7 +59,9 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra cmd.AddCommand(newSetChildLookupCommand(fullName, err, out, in)) cmd.AddCommand(newSetParentLookupCommand(fullName, err, out, in)) cmd.AddCommand(newSetChildWhereCommand(fullName, err, out, in)) + cmd.AddCommand(newSetChildSelectCommand(fullName, err, out, in)) cmd.AddCommand(newSetParentWhereCommand(fullName, err, out, in)) + cmd.AddCommand(newSetParentSelectCommand(fullName, err, out, in)) cmd.PersistentFlags().StringVarP(&ingressDescriptor, "ingress-descriptor", "i", "ingress-descriptor.yaml", "Ingress descriptor filename") cmd.SetOut(out) cmd.SetErr(err) diff --git a/internal/app/id/create.go b/internal/app/id/create.go index c1327f68..6d2b4dee 100755 --- a/internal/app/id/create.go +++ b/internal/app/id/create.go @@ -45,7 +45,7 @@ func newCreateCommand(fullName string, err *os.File, out *os.File, in *os.File) reader := infra.NewRelationReader(relations) - e := id.Create(table, reader, idStorageFactory(ingressDescriptor)) + e := id.Create(table, []string{}, reader, idStorageFactory(ingressDescriptor)) if e != nil { fmt.Fprintln(err, e.Description) os.Exit(1) diff --git a/internal/app/id/set_child_select.go b/internal/app/id/set_child_select.go new file mode 100644 index 00000000..3b34949b --- /dev/null +++ b/internal/app/id/set_child_select.go @@ -0,0 +1,53 @@ +// Copyright (C) 2021 CGI France +// +// This file is part of LINO. +// +// LINO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// LINO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with LINO. If not, see . + +package id + +import ( + "fmt" + "os" + + "github.com/cgi-fr/lino/pkg/id" + "github.com/spf13/cobra" +) + +// newSetChildSelectCommand implements the cli id set-child-select command +func newSetChildSelectCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra.Command { + cmd := &cobra.Command{ + Use: "set-child-select [relation] [column1] [column2] [column3] ...", + Short: "set child select attribut for relation [relation] in ingress descriptor", + Long: "", + Example: fmt.Sprintf(" %[1]s id set-child-select public.store store_id name", fullName), + Args: cobra.MinimumNArgs(2), + Run: func(cmd *cobra.Command, args []string) { + relation := args[0] + selectColumns := args[1:] + + e := id.SetChildSelect(relation, selectColumns, idStorageFactory(ingressDescriptor)) + if e != nil { + fmt.Fprintln(err, e.Description) + os.Exit(1) + } + + fmt.Fprintf(out, "successfully update relation %s in ingress descriptor\n", relation) + }, + } + cmd.SetOut(out) + cmd.SetErr(err) + cmd.SetIn(in) + return cmd +} diff --git a/internal/app/id/set_parent_select.go b/internal/app/id/set_parent_select.go new file mode 100644 index 00000000..24b9ce60 --- /dev/null +++ b/internal/app/id/set_parent_select.go @@ -0,0 +1,53 @@ +// Copyright (C) 2021 CGI France +// +// This file is part of LINO. +// +// LINO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// LINO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with LINO. If not, see . + +package id + +import ( + "fmt" + "os" + + "github.com/cgi-fr/lino/pkg/id" + "github.com/spf13/cobra" +) + +// newSetParentSelectCommand implements the cli id set-parent-select command +func newSetParentSelectCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra.Command { + cmd := &cobra.Command{ + Use: "set-parent-select [relation] [column1] [column2] [column3] ...", + Short: "set parent select attribut for relation [relation] in ingress descriptor", + Long: "", + Example: fmt.Sprintf(" %[1]s id set-parent-select public.store store_id name", fullName), + Args: cobra.MinimumNArgs(2), + Run: func(cmd *cobra.Command, args []string) { + relation := args[0] + selectColumns := args[1:] + + e := id.SetParentSelect(relation, selectColumns, idStorageFactory(ingressDescriptor)) + if e != nil { + fmt.Fprintln(err, e.Description) + os.Exit(1) + } + + fmt.Fprintf(out, "successfully update relation %s in ingress descriptor\n", relation) + }, + } + cmd.SetOut(out) + cmd.SetErr(err) + cmd.SetIn(in) + return cmd +} diff --git a/internal/app/pull/builder.go b/internal/app/pull/builder.go index 374338b0..4713e35f 100644 --- a/internal/app/pull/builder.go +++ b/internal/app/pull/builder.go @@ -124,7 +124,8 @@ func (b builder) buildRelation(rel id.IngressRelation) ([]pull.Relation, error) Table: b.getTable(rel.Child().Name()), Keys: relyaml.Child.Keys, }, - Where: rel.WhereChild(), + Where: rel.WhereChild(), + Select: rel.SelectChild(), } } b.exrmap[name] = exrel @@ -146,7 +147,8 @@ func (b builder) buildRelation(rel id.IngressRelation) ([]pull.Relation, error) Table: b.getTable(rel.Child().Name()), Keys: relyaml.Child.Keys, }, - Where: rel.WhereParent(), + Where: rel.WhereParent(), + Select: rel.SelectParent(), } } b.exrmap[name] = exrel diff --git a/internal/app/pull/cli.go b/internal/app/pull/cli.go index 2f6d9344..f295133e 100755 --- a/internal/app/pull/cli.go +++ b/internal/app/pull/cli.go @@ -117,7 +117,7 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra os.Exit(1) } - plan, start, e2 := getPullerPlan(idStorageFactory(table, ingressDescriptor)) + plan, start, startSelect, e2 := getPullerPlan(idStorageFactory(table, ingressDescriptor)) if e2 != nil { fmt.Fprintln(err, e2.Error()) os.Exit(1) @@ -175,7 +175,7 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra } puller := pull.NewPullerParallel(plan, datasource, pullExporterFactory(out), tracer, parallel) - if e3 := puller.Pull(start, filter, filters, filtersEx); e3 != nil { + if e3 := puller.Pull(start, filter, startSelect, filters, filtersEx); e3 != nil { log.Fatal().AnErr("error", e3).Msg("Fatal error stop the pull command") os.Exit(1) } @@ -222,26 +222,26 @@ func getDataSource(dataconnectorName string, out io.Writer) (pull.DataSource, er return datasourceFactory.New(u.URL.String(), alias.Schema), nil } -func getPullerPlan(idStorage id.Storage) (pull.Plan, pull.Table, error) { - ep, err1 := id.GetPullerPlan(idStorage) +func getPullerPlan(idStorage id.Storage) (pull.Plan, pull.Table, []string, error) { + pp, err1 := id.GetPullerPlan(idStorage) if err1 != nil { - return pull.Plan{}, pull.Table{}, err1 + return pull.Plan{}, pull.Table{}, []string{}, err1 } relations, err2 := relStorage.List() if err2 != nil { - return pull.Plan{}, pull.Table{}, err2 + return pull.Plan{}, pull.Table{}, []string{}, err2 } tables, err3 := tabStorage.List() if err3 != nil { - return pull.Plan{}, pull.Table{}, err3 + return pull.Plan{}, pull.Table{}, []string{}, err3 } - builder := newBuilder(ep, relations, tables) + builder := newBuilder(pp, relations, tables) plan, startTable, err4 := builder.plan() if err4 != nil { - return pull.Plan{}, pull.Table{}, err4 + return pull.Plan{}, pull.Table{}, []string{}, err4 } // Check startTable existe in table.yaml @@ -256,8 +256,8 @@ func getPullerPlan(idStorage id.Storage) (pull.Plan, pull.Table, error) { if !tableExiste { err5 := fmt.Errorf("Table '%s' does not exist in table.yaml", string(startTable.Name)) - return pull.Plan{}, pull.Table{}, err5 + return pull.Plan{}, pull.Table{}, []string{}, err5 } - return plan, startTable, nil + return plan, startTable, pp.Select(), nil } diff --git a/internal/app/pull/http.go b/internal/app/pull/http.go index 165033db..7912b788 100644 --- a/internal/app/pull/http.go +++ b/internal/app/pull/http.go @@ -126,7 +126,7 @@ func HandlerFactory(ingressDescriptor string) func(w http.ResponseWriter, r *htt return } - plan, start, e2 := getPullerPlan(idStorageFactory(query.Get("table"), ingressDescriptor)) + plan, start, startSelect, e2 := getPullerPlan(idStorageFactory(query.Get("table"), ingressDescriptor)) if e2 != nil { log.Error().Err(e2).Msg("") w.WriteHeader(http.StatusInternalServerError) @@ -141,7 +141,7 @@ func HandlerFactory(ingressDescriptor string) func(w http.ResponseWriter, r *htt pullExporter := pullExporterFactory(w) puller := pull.NewPuller(plan, datasource, pullExporter, pull.NoTraceListener{}) - e3 := puller.Pull(start, pull.Filter{Limit: limit, Values: pull.Row{}, Where: where, Distinct: distinct}, nil, nil) + e3 := puller.Pull(start, pull.Filter{Limit: limit, Values: pull.Row{}, Where: where, Distinct: distinct}, startSelect, nil, nil) if e3 != nil { log.Error().Err(e3).Msg("") w.WriteHeader(http.StatusInternalServerError) diff --git a/internal/infra/id/table_storage.go b/internal/infra/id/table_storage.go index 9f7b0131..3170013a 100644 --- a/internal/infra/id/table_storage.go +++ b/internal/infra/id/table_storage.go @@ -38,5 +38,5 @@ func (s *TableStorage) Store(adef id.IngressDescriptor) *id.Error { // Read create new Ingress Descriptor with table as start table without relations func (s *TableStorage) Read() (id.IngressDescriptor, *id.Error) { - return id.NewIngressDescriptor(s.table, id.NewIngressRelationList([]id.IngressRelation{})), nil + return id.NewIngressDescriptor(s.table, []string{}, id.NewIngressRelationList([]id.IngressRelation{})), nil } diff --git a/internal/infra/id/yaml_storage.go b/internal/infra/id/yaml_storage.go index e28d1cf1..0443ffb3 100755 --- a/internal/infra/id/yaml_storage.go +++ b/internal/infra/id/yaml_storage.go @@ -34,6 +34,7 @@ type YAMLStructure struct { // YAMLIngressDescriptor defines how to store an ingress descriptor in YAML format. type YAMLIngressDescriptor struct { StartTable string `yaml:"startTable"` + Select []string `yaml:"select"` Relations []YAMLRelation `yaml:"relations"` } @@ -46,9 +47,10 @@ type YAMLRelation struct { // YAMLTable defines how to store a table in YAML format. type YAMLTable struct { - Name string `yaml:"name"` - Lookup bool `yaml:"lookup"` - Where string `yaml:"where,omitempty"` + Name string `yaml:"name"` + Lookup bool `yaml:"lookup"` + Where string `yaml:"where,omitempty"` + Select []string `yaml:"select,omitempty"` } // YAMLStorage provides storage in a local YAML file @@ -73,13 +75,14 @@ func (s *YAMLStorage) Store(id id.IngressDescriptor) *id.Error { relation := list.Relation(i) relations = append(relations, YAMLRelation{ Name: relation.Name(), - Parent: YAMLTable{Name: relation.Parent().Name(), Lookup: relation.LookUpParent(), Where: relation.WhereParent()}, - Child: YAMLTable{Name: relation.Child().Name(), Lookup: relation.LookUpChild(), Where: relation.WhereChild()}, + Parent: YAMLTable{Name: relation.Parent().Name(), Lookup: relation.LookUpParent(), Where: relation.WhereParent(), Select: relation.SelectParent()}, + Child: YAMLTable{Name: relation.Child().Name(), Lookup: relation.LookUpChild(), Where: relation.WhereChild(), Select: relation.SelectChild()}, }) } structure.IngressDescriptor = YAMLIngressDescriptor{ StartTable: id.StartTable().Name(), + Select: id.Select(), Relations: relations, } @@ -107,11 +110,13 @@ func (s *YAMLStorage) Read() (id.IngressDescriptor, *id.Error) { id.NewTable(relation.Child.Name), ), relation.Parent.Lookup, relation.Child.Lookup, - relation.Parent.Where, relation.Child.Where), + relation.Parent.Where, relation.Child.Where, + relation.Parent.Select, relation.Child.Select, + ), ) } - return id.NewIngressDescriptor(id.NewTable(structure.IngressDescriptor.StartTable), id.NewIngressRelationList(relations)), nil + return id.NewIngressDescriptor(id.NewTable(structure.IngressDescriptor.StartTable), structure.IngressDescriptor.Select, id.NewIngressRelationList(relations)), nil } func writeFile(structure *YAMLStructure, filename string) *id.Error { diff --git a/pkg/id/driver.go b/pkg/id/driver.go index 653969ce..99ff230f 100755 --- a/pkg/id/driver.go +++ b/pkg/id/driver.go @@ -24,7 +24,7 @@ import ( ) // Create and store ingress descriptor for the given start table and relation set. -func Create(startTable string, relReader RelationReader, storage Storage) *Error { +func Create(startTable string, selectColumns []string, relReader RelationReader, storage Storage) *Error { relations, err := relReader.Read() if err != nil { return err @@ -33,7 +33,7 @@ func Create(startTable string, relReader RelationReader, storage Storage) *Error ingressRels := []IngressRelation{} for i := uint(0); i < relations.Len(); i++ { rel := relations.Relation(i) - ingressRels = append(ingressRels, NewIngressRelation(rel, false, false, "", "")) + ingressRels = append(ingressRels, NewIngressRelation(rel, false, false, "", "", []string{}, []string{})) } fullGraph := newGraph(NewIngressRelationList(ingressRels)) @@ -57,13 +57,13 @@ func Create(startTable string, relReader RelationReader, storage Storage) *Error for i := uint(0); i < connectedGraph.relations.Len(); i++ { rel := connectedGraph.relations.Relation(i) if setLookUpChild.contains(rel.Name()) { - adrelations = append(adrelations, NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), false, true, rel.WhereParent(), rel.WhereChild())) + adrelations = append(adrelations, NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), false, true, rel.WhereParent(), rel.WhereChild(), rel.SelectParent(), rel.SelectChild())) } else { - adrelations = append(adrelations, NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), false, false, rel.WhereParent(), rel.WhereChild())) + adrelations = append(adrelations, NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), false, false, rel.WhereParent(), rel.WhereChild(), rel.SelectParent(), rel.SelectChild())) } } - id := NewIngressDescriptor(NewTable(startTable), NewIngressRelationList(adrelations)) + id := NewIngressDescriptor(NewTable(startTable), selectColumns, NewIngressRelationList(adrelations)) err = storage.Store(id) if err != nil { @@ -89,7 +89,7 @@ func SetStartTable(table Table, storage Storage) *Error { return &Error{Description: fmt.Sprintf("Table %s doesn't exist", table.Name())} } - updatedID := NewIngressDescriptor(table, id.Relations()) + updatedID := NewIngressDescriptor(table, id.Select(), id.Relations()) err = storage.Store(updatedID) if err != nil { @@ -114,12 +114,12 @@ func SetChildLookup(relation string, flag bool, storage Storage) *Error { for i := uint(0); i < id.Relations().Len(); i++ { rel := id.Relations().Relation(i) if rel.Name() == relation { - rel = NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), rel.LookUpParent(), flag, rel.WhereParent(), rel.WhereChild()) + rel = NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), rel.LookUpParent(), flag, rel.WhereParent(), rel.WhereChild(), rel.SelectParent(), rel.SelectChild()) } relations[i] = rel } - updatedID := NewIngressDescriptor(id.StartTable(), NewIngressRelationList(relations)) + updatedID := NewIngressDescriptor(id.StartTable(), id.Select(), NewIngressRelationList(relations)) err = storage.Store(updatedID) if err != nil { @@ -144,12 +144,12 @@ func SetParentLookup(relation string, flag bool, storage Storage) *Error { for i := uint(0); i < id.Relations().Len(); i++ { rel := id.Relations().Relation(i) if rel.Name() == relation { - rel = NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), flag, rel.LookUpChild(), rel.WhereParent(), rel.WhereChild()) + rel = NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), flag, rel.LookUpChild(), rel.WhereParent(), rel.WhereChild(), rel.SelectParent(), rel.SelectChild()) } relations[i] = rel } - updatedID := NewIngressDescriptor(id.StartTable(), NewIngressRelationList(relations)) + updatedID := NewIngressDescriptor(id.StartTable(), id.Select(), NewIngressRelationList(relations)) err = storage.Store(updatedID) if err != nil { @@ -174,12 +174,42 @@ func SetChildWhere(relation string, where string, storage Storage) *Error { for i := uint(0); i < id.Relations().Len(); i++ { rel := id.Relations().Relation(i) if rel.Name() == relation { - rel = NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), rel.LookUpParent(), rel.LookUpChild(), rel.WhereParent(), where) + rel = NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), rel.LookUpParent(), rel.LookUpChild(), rel.WhereParent(), where, rel.SelectParent(), rel.SelectChild()) } relations[i] = rel } - updatedID := NewIngressDescriptor(id.StartTable(), NewIngressRelationList(relations)) + updatedID := NewIngressDescriptor(id.StartTable(), id.Select(), NewIngressRelationList(relations)) + + err = storage.Store(updatedID) + if err != nil { + return err + } + return nil +} + +// SetChildSelect update child select relation's parameter in ingress descriptor +func SetChildSelect(relation string, columns []string, storage Storage) *Error { + id, err := storage.Read() + if err != nil { + return err + } + + if !id.Relations().Contains(relation) { + return &Error{Description: fmt.Sprintf("Relation %s doesn't exist", relation)} + } + + relations := make([]IngressRelation, id.Relations().Len()) + + for i := uint(0); i < id.Relations().Len(); i++ { + rel := id.Relations().Relation(i) + if rel.Name() == relation { + rel = NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), rel.LookUpParent(), rel.LookUpChild(), rel.WhereParent(), rel.WhereChild(), rel.SelectParent(), columns) + } + relations[i] = rel + } + + updatedID := NewIngressDescriptor(id.StartTable(), id.Select(), NewIngressRelationList(relations)) err = storage.Store(updatedID) if err != nil { @@ -204,12 +234,42 @@ func SetParentWhere(relation string, where string, storage Storage) *Error { for i := uint(0); i < id.Relations().Len(); i++ { rel := id.Relations().Relation(i) if rel.Name() == relation { - rel = NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), rel.LookUpParent(), rel.LookUpChild(), where, rel.WhereChild()) + rel = NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), rel.LookUpParent(), rel.LookUpChild(), where, rel.WhereChild(), rel.SelectParent(), rel.SelectChild()) + } + relations[i] = rel + } + + updatedID := NewIngressDescriptor(id.StartTable(), id.Select(), NewIngressRelationList(relations)) + + err = storage.Store(updatedID) + if err != nil { + return err + } + return nil +} + +// SetParentSelect update parent select relation's parameter in ingress descriptor +func SetParentSelect(relation string, columns []string, storage Storage) *Error { + id, err := storage.Read() + if err != nil { + return err + } + + if !id.Relations().Contains(relation) { + return &Error{Description: fmt.Sprintf("Relation %s doesn't exist", relation)} + } + + relations := make([]IngressRelation, id.Relations().Len()) + + for i := uint(0); i < id.Relations().Len(); i++ { + rel := id.Relations().Relation(i) + if rel.Name() == relation { + rel = NewIngressRelation(NewRelation(rel.Name(), rel.Parent(), rel.Child()), rel.LookUpParent(), rel.LookUpChild(), rel.WhereParent(), rel.WhereChild(), columns, rel.SelectChild()) } relations[i] = rel } - updatedID := NewIngressDescriptor(id.StartTable(), NewIngressRelationList(relations)) + updatedID := NewIngressDescriptor(id.StartTable(), id.Select(), NewIngressRelationList(relations)) err = storage.Store(updatedID) if err != nil { @@ -249,7 +309,7 @@ func GetPullerPlan(storage Storage) (PullerPlan, *Error) { startRelationsList = sg.relations } steps := []Step{ - NewStep(1, id.StartTable(), NewIngressRelation(NewRelation("", nil, nil), false, false, "", ""), startRelationsList, startTableList, startCycles, 0), + NewStep(1, id.StartTable(), NewIngressRelation(NewRelation("", nil, nil), false, false, "", "", []string{}, []string{}), startRelationsList, startTableList, startCycles, 0), } log.Debug().Msg(fmt.Sprintf("%v", steps[0])) @@ -265,7 +325,7 @@ func GetPullerPlan(storage Storage) (PullerPlan, *Error) { log.Warn().Msg(err.Error()) } - return NewPullerPlan(steps, g.relations, g.tables), nil + return NewPullerPlan(steps, g.relations, g.tables, id.Select()), nil } // Export the puller plan. diff --git a/pkg/id/driver_test.go b/pkg/id/driver_test.go index c4e2167e..1528eed5 100755 --- a/pkg/id/driver_test.go +++ b/pkg/id/driver_test.go @@ -65,7 +65,7 @@ func relationString(relation string) id.Relation { // relation help to create id.Relation object from a string representation `parent -> child`. func adRelationString(relation string, lookupParent bool, lookupChild bool) id.IngressRelation { - return id.NewIngressRelation(relationString(relation), lookupParent, lookupChild, "", "") + return id.NewIngressRelation(relationString(relation), lookupParent, lookupChild, "", "", []string{}, []string{}) } var adCreateTests = []struct { @@ -80,6 +80,7 @@ var adCreateTests = []struct { }), id.NewIngressDescriptor( id.NewTable("A"), + []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("B->A", false, false), }), @@ -98,6 +99,7 @@ var adCreateTests = []struct { }), id.NewIngressDescriptor( id.NewTable("A"), + []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("A->B", false, true), adRelationString("B->C", false, true), @@ -121,6 +123,7 @@ var adCreateTests = []struct { }), id.NewIngressDescriptor( id.NewTable("A"), + []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("A->B", false, true), adRelationString("B->C", false, true), @@ -141,7 +144,7 @@ func TestCreate(t *testing.T) { } storage := &MemoryStorage{} - err := id.Create(startTable, relReader, storage) + err := id.Create(startTable, []string{}, relReader, storage) assert.Nil(t, err) @@ -155,7 +158,7 @@ func newInitialStep(tableName string) id.Step { return id.NewStep( 1, table, - id.NewIngressRelation(id.NewRelation("", nil, nil), false, false, "", ""), + id.NewIngressRelation(id.NewRelation("", nil, nil), false, false, "", "", []string{}, []string{}), id.NewIngressRelationList([]id.IngressRelation{}), id.NewTableList([]id.Table{table}), id.NewCycleList([]id.IngressRelationList{}), @@ -200,6 +203,7 @@ var adShowTests = []struct { { id.NewIngressDescriptor( id.NewTable("A"), + []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("A->B", false, true), }), @@ -213,6 +217,7 @@ var adShowTests = []struct { { // example 1 id.NewIngressDescriptor( id.NewTable("I"), + []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("C->O", false, true), adRelationString("O->D", false, true), @@ -227,6 +232,7 @@ var adShowTests = []struct { { // example 1 id.NewIngressDescriptor( id.NewTable("I"), + []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("C->O", false, true), adRelationString("O->D", false, true), @@ -241,6 +247,7 @@ var adShowTests = []struct { { // example 1 (table C) id.NewIngressDescriptor( id.NewTable("C"), + []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("C->O", false, true), adRelationString("O->D", false, true), @@ -256,6 +263,7 @@ var adShowTests = []struct { { // example 1 (table D) id.NewIngressDescriptor( id.NewTable("D"), + []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("C->O", false, true), adRelationString("O->D", false, true), @@ -269,6 +277,7 @@ var adShowTests = []struct { { // example 1 (table O) id.NewIngressDescriptor( id.NewTable("O"), + []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("C->O", false, true), adRelationString("O->D", false, true), @@ -283,6 +292,7 @@ var adShowTests = []struct { { // example 2 id.NewIngressDescriptor( id.NewTable("C"), + []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("C->O", false, true), adRelationString("O->D", false, true), @@ -337,6 +347,7 @@ var adShowTests = []struct { { // example 3 Variant bis id.NewIngressDescriptor( id.NewTable("O"), + []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("C->O", true, false), }), @@ -386,49 +397,49 @@ var adShowTests = []struct { } func TestUpdateStartTable(t *testing.T) { - storage := &MemoryStorage{id: id.NewIngressDescriptor(id.NewTable("old"), id.NewIngressRelationList([]id.IngressRelation{ + storage := &MemoryStorage{id: id.NewIngressDescriptor(id.NewTable("old"), []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("old->new", false, true), }))} err := id.SetStartTable(id.NewTable("new"), storage) assert.Nil(t, err) - assert.Equal(t, id.NewIngressDescriptor(id.NewTable("new"), id.NewIngressRelationList([]id.IngressRelation{ + assert.Equal(t, id.NewIngressDescriptor(id.NewTable("new"), []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("old->new", false, true), })), storage.id) } func TestUpdateStartTableCheckTable(t *testing.T) { - storage := &MemoryStorage{id: id.NewIngressDescriptor(id.NewTable("old"), id.NewIngressRelationList([]id.IngressRelation{}))} + storage := &MemoryStorage{id: id.NewIngressDescriptor(id.NewTable("old"), []string{}, id.NewIngressRelationList([]id.IngressRelation{}))} err := id.SetStartTable(id.NewTable("new"), storage) assert.EqualError(t, err, "Table new doesn't exist") - assert.Equal(t, id.NewIngressDescriptor(id.NewTable("old"), id.NewIngressRelationList([]id.IngressRelation{})), storage.id) + assert.Equal(t, id.NewIngressDescriptor(id.NewTable("old"), []string{}, id.NewIngressRelationList([]id.IngressRelation{})), storage.id) } func TestUpdateParentLookup(t *testing.T) { - storage := &MemoryStorage{id: id.NewIngressDescriptor(id.NewTable("A"), id.NewIngressRelationList([]id.IngressRelation{ + storage := &MemoryStorage{id: id.NewIngressDescriptor(id.NewTable("A"), []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("A->B", false, true), }))} err := id.SetParentLookup("A_B", true, storage) assert.Nil(t, err) - assert.Equal(t, id.NewIngressDescriptor(id.NewTable("A"), id.NewIngressRelationList([]id.IngressRelation{ + assert.Equal(t, id.NewIngressDescriptor(id.NewTable("A"), []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("A->B", true, true), })), storage.id) } func TestUpdateChildLookup(t *testing.T) { - storage := &MemoryStorage{id: id.NewIngressDescriptor(id.NewTable("A"), id.NewIngressRelationList([]id.IngressRelation{ + storage := &MemoryStorage{id: id.NewIngressDescriptor(id.NewTable("A"), []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("A->B", false, true), }))} err := id.SetChildLookup("A_B", false, storage) assert.Nil(t, err) - assert.Equal(t, id.NewIngressDescriptor(id.NewTable("A"), id.NewIngressRelationList([]id.IngressRelation{ + assert.Equal(t, id.NewIngressDescriptor(id.NewTable("A"), []string{}, id.NewIngressRelationList([]id.IngressRelation{ adRelationString("A->B", false, false), })), storage.id) } diff --git a/pkg/id/model.go b/pkg/id/model.go index e19ff076..84bde93b 100755 --- a/pkg/id/model.go +++ b/pkg/id/model.go @@ -51,9 +51,11 @@ type RelationList interface { type IngressRelation interface { Relation LookUpChild() bool - LookUpParent() bool WhereChild() string + SelectChild() []string + LookUpParent() bool WhereParent() string + SelectParent() []string } // IngressRelationList involved in an puller plan. @@ -67,6 +69,7 @@ type IngressRelationList interface { // IngressDescriptor from which the puller plan will be computed. type IngressDescriptor interface { StartTable() Table + Select() []string Relations() IngressRelationList String() string } @@ -102,6 +105,7 @@ type PullerPlan interface { Relations() IngressRelationList Tables() TableList String() string + Select() []string } // Error is the error type returned by the domain diff --git a/pkg/id/model_ad.go b/pkg/id/model_ad.go index 2737cebc..38f3f985 100755 --- a/pkg/id/model_ad.go +++ b/pkg/id/model_ad.go @@ -28,20 +28,40 @@ type idrelation struct { lookUpChild bool whereParent string whereChild string + selectParent []string + selectChild []string } // NewIngressRelation initialize a new IngressRelation object -func NewIngressRelation(rel Relation, lookUpParent bool, lookUpChild bool, whereParent string, whereChild string) IngressRelation { - return idrelation{Relation: rel, lookUpParent: lookUpParent, lookUpChild: lookUpChild, whereParent: whereParent, whereChild: whereChild} +func NewIngressRelation( + rel Relation, + lookUpParent bool, + lookUpChild bool, + whereParent string, + whereChild string, + selectParent []string, + selectChild []string, +) IngressRelation { + return idrelation{ + Relation: rel, + lookUpParent: lookUpParent, + lookUpChild: lookUpChild, + whereParent: whereParent, + whereChild: whereChild, + selectParent: selectParent, + selectChild: selectChild, + } } -func (r idrelation) Name() string { return r.Relation.Name() } -func (r idrelation) Parent() Table { return r.Relation.Parent() } -func (r idrelation) Child() Table { return r.Relation.Child() } -func (r idrelation) LookUpParent() bool { return r.lookUpParent } -func (r idrelation) LookUpChild() bool { return r.lookUpChild } -func (r idrelation) WhereParent() string { return r.whereParent } -func (r idrelation) WhereChild() string { return r.whereChild } +func (r idrelation) Name() string { return r.Relation.Name() } +func (r idrelation) Parent() Table { return r.Relation.Parent() } +func (r idrelation) Child() Table { return r.Relation.Child() } +func (r idrelation) LookUpParent() bool { return r.lookUpParent } +func (r idrelation) WhereParent() string { return r.whereParent } +func (r idrelation) SelectParent() []string { return r.selectParent } +func (r idrelation) LookUpChild() bool { return r.lookUpChild } +func (r idrelation) WhereChild() string { return r.whereChild } +func (r idrelation) SelectChild() []string { return r.selectChild } func (r idrelation) String() string { switch { case r.LookUpChild() && r.LookUpParent(): @@ -89,15 +109,19 @@ func (l idrelationList) String() string { } // NewIngressDescriptor initialize a new IngressDescriptor object -func NewIngressDescriptor(start Table, relations IngressRelationList) IngressDescriptor { - return id{startTable: table{name: start.Name()}, relations: relations} +func NewIngressDescriptor(start Table, selectColumns []string, relations IngressRelationList) IngressDescriptor { + return id{startTable: table{name: start.Name()}, selectColumns: selectColumns, relations: relations} } type id struct { - startTable table - relations IngressRelationList + startTable table + selectColumns []string + relations IngressRelationList } func (id id) StartTable() Table { return id.startTable } +func (id id) Select() []string { return id.selectColumns } func (id id) Relations() IngressRelationList { return id.relations } -func (id id) String() string { return fmt.Sprintf("%v (%v)", id.startTable, id.relations) } +func (id id) String() string { + return fmt.Sprintf("%v [%v] (%v)", id.startTable, id.selectColumns, id.relations) +} diff --git a/pkg/id/model_step.go b/pkg/id/model_step.go index 7c9639ea..2c9f2e42 100755 --- a/pkg/id/model_step.go +++ b/pkg/id/model_step.go @@ -61,21 +61,23 @@ func (s step) String() string { } type pullerPlan struct { - len uint - slice []Step - relations IngressRelationList - tables TableList + len uint + slice []Step + relations IngressRelationList + tables TableList + startSelect []string } // NewPullerPlan initialize a new PullerPlan object -func NewPullerPlan(steps []Step, relations IngressRelationList, tables TableList) PullerPlan { - return pullerPlan{uint(len(steps)), steps, relations, tables} +func NewPullerPlan(steps []Step, relations IngressRelationList, tables TableList, startSelect []string) PullerPlan { + return pullerPlan{uint(len(steps)), steps, relations, tables, startSelect} } func (l pullerPlan) Len() uint { return l.len } func (l pullerPlan) Step(idx uint) Step { return l.slice[idx] } func (l pullerPlan) Relations() IngressRelationList { return l.relations } func (l pullerPlan) Tables() TableList { return l.tables } +func (l pullerPlan) Select() []string { return l.startSelect } func (l pullerPlan) String() string { switch l.len { case 0: diff --git a/pkg/pull/driven_DatasourceInMemory.go b/pkg/pull/driven_DatasourceInMemory.go index 196c5f2e..b5169ab4 100644 --- a/pkg/pull/driven_DatasourceInMemory.go +++ b/pkg/pull/driven_DatasourceInMemory.go @@ -76,7 +76,7 @@ LOOK_FOR_MATCHING_ROWS: if len(source.Columns) == 0 { result = append(result, row) } else { - copyr := make(Row, len(row)) + copyr := make(Row, len(source.Columns)) for _, columns := range source.Columns { copyr[columns.Name] = row[columns.Name] } diff --git a/pkg/pull/driver.go b/pkg/pull/driver.go index c35b582b..1efcf428 100755 --- a/pkg/pull/driver.go +++ b/pkg/pull/driver.go @@ -46,7 +46,7 @@ func NewStep(puller *puller, out ExportedRow, entry Relation) *Step { } type Puller interface { - Pull(start Table, filter Filter, filterCohort RowReader, excluded KeyStore) error + Pull(start Table, filter Filter, selectColumns []string, filterCohort RowReader, excluded KeyStore) error } type puller struct { @@ -65,7 +65,8 @@ func NewPuller(plan Plan, datasource DataSource, exporter RowExporter, diagnosti } } -func (p *puller) Pull(start Table, filter Filter, filterCohort RowReader, excluded KeyStore) error { +func (p *puller) Pull(start Table, filter Filter, selectColumns []string, filterCohort RowReader, excluded KeyStore) error { + start.selectColumns(selectColumns...) start = p.graph.addMissingColumns(start) if err := p.datasource.Open(); err != nil { diff --git a/pkg/pull/driver_parallel.go b/pkg/pull/driver_parallel.go index ea55c821..23a71b1c 100644 --- a/pkg/pull/driver_parallel.go +++ b/pkg/pull/driver_parallel.go @@ -61,7 +61,8 @@ func NewPullerParallel(plan Plan, datasource DataSource, exporter RowExporter, d return puller } -func (p *pullerParallel) Pull(start Table, filter Filter, filterCohort RowReader, excluded KeyStore) error { +func (p *pullerParallel) Pull(start Table, filter Filter, selectColumns []string, filterCohort RowReader, excluded KeyStore) error { + start.selectColumns(selectColumns...) start = p.graph.addMissingColumns(start) if err := p.datasource.Open(); err != nil { diff --git a/pkg/pull/driver_test.go b/pkg/pull/driver_test.go index b15053a1..a9bfaae9 100644 --- a/pkg/pull/driver_test.go +++ b/pkg/pull/driver_test.go @@ -32,6 +32,7 @@ import ( type Execution struct { Start pull.Table Filter pull.Filter + Select []string Result []string } @@ -73,7 +74,7 @@ func RunTest(t *testing.T, test *Test) { for _, execution := range test.Executions { collector.Reset() - assert.NoError(t, puller.Pull(execution.Start, execution.Filter, nil, nil)) + assert.NoError(t, puller.Pull(execution.Start, execution.Filter, execution.Select, nil, nil)) assert.Len(t, collector.Result, len(execution.Result)) for i := 0; i < len(execution.Result); i++ { @@ -93,7 +94,7 @@ func RunBench(b *testing.B, test *Test) { for _, execution := range test.Executions { collector.Reset() - assert.NoError(b, puller.Pull(execution.Start, execution.Filter, nil, nil)) + assert.NoError(b, puller.Pull(execution.Start, execution.Filter, execution.Select, nil, nil)) assert.Len(b, collector.Result, len(execution.Result)) } } @@ -113,12 +114,30 @@ func TestSimple(t *testing.T) { LoadAndRunTest(t, "simple.yaml") } +func TestSimpleSelect(t *testing.T) { + t.Parallel() + + LoadAndRunTest(t, "simple_select.yaml") +} + +func TestKeyNotSelected(t *testing.T) { + t.Parallel() + + LoadAndRunTest(t, "key_not_selected.yaml") +} + func TestSimpleWithExport(t *testing.T) { t.Parallel() LoadAndRunTest(t, "simple_export.yaml") } +func TestSelectOverrideColumns(t *testing.T) { + t.Parallel() + + LoadAndRunTest(t, "select_override_columns.yaml") +} + func TestCycle(t *testing.T) { t.Parallel() diff --git a/pkg/pull/model.go b/pkg/pull/model.go index 8e1e2bca..8f6a80c4 100755 --- a/pkg/pull/model.go +++ b/pkg/pull/model.go @@ -72,6 +72,7 @@ type Relation struct { Local RelationTip Foreign RelationTip Where string + Select []string } type RelationSet []Relation diff --git a/pkg/pull/model_graph.go b/pkg/pull/model_graph.go index b2556200..0f4b4d5e 100644 --- a/pkg/pull/model_graph.go +++ b/pkg/pull/model_graph.go @@ -24,6 +24,7 @@ func (g Graph) addMissingColumns(t Table) Table { if len(t.Columns) > 0 { for _, relation := range g.Relations[t.Name] { + t.selectColumns(relation.Select...) t.addMissingColumns(relation.Local.Keys...) } } diff --git a/pkg/pull/model_plan.go b/pkg/pull/model_plan.go index 88155373..cccb5a21 100755 --- a/pkg/pull/model_plan.go +++ b/pkg/pull/model_plan.go @@ -28,6 +28,7 @@ func (plan Plan) buildGraph() Graph { cached := map[TableName]bool{} for _, relation := range plan.Relations { + relation.Foreign.Table.selectColumns(relation.Select...) relation.Local.Table.addMissingColumns(relation.Local.Keys...) cached[relation.Local.Table.Name] = true @@ -37,12 +38,12 @@ func (plan Plan) buildGraph() Graph { if len(relation.Foreign.Table.Columns) > 0 { for _, follow := range relations[relation.Foreign.Table.Name] { + relation.Local.Table.selectColumns(follow.Select...) relation.Foreign.Table.addMissingColumns(follow.Local.Keys...) } } - relationsWithMissingColumns[relation.Local.Table.Name] = - append(relationsWithMissingColumns[relation.Local.Table.Name], relation) + relationsWithMissingColumns[relation.Local.Table.Name] = append(relationsWithMissingColumns[relation.Local.Table.Name], relation) } return Graph{Relations: relationsWithMissingColumns, Components: plan.Components, Cached: cached} diff --git a/pkg/pull/model_table.go b/pkg/pull/model_table.go index 98508ef8..78f31cde 100755 --- a/pkg/pull/model_table.go +++ b/pkg/pull/model_table.go @@ -127,3 +127,41 @@ func (t *Table) addMissingColumns(columnNames ...string) { } } } + +func (t *Table) selectColumns(columnNames ...string) { + if len(columnNames) == 0 { + return + } + + columns := []Column{} + + if len(t.Columns) == 0 { + for _, columnName := range columnNames { + columns = append(columns, Column{Name: columnName}) + } + } else { + for _, column := range t.Columns { + for _, columnName := range columnNames { + if column.Name == columnName { + columns = append(columns, column) + } + } + } + } + + // there are no columns to override + if len(columns) == 0 { + log.Info(). + Strs("select", columnNames). + Interface("columns", t.Columns). + Interface("table", t.Name). + Msg("there are no columns to override") + return + } + + t.Columns = columns + log.Info(). + Strs("columns", columnNames). + Interface("table", t.Name). + Msg("select only columns defined") +} diff --git a/pkg/pull/testdata/key_not_selected.yaml b/pkg/pull/testdata/key_not_selected.yaml new file mode 100644 index 00000000..10932f44 --- /dev/null +++ b/pkg/pull/testdata/key_not_selected.yaml @@ -0,0 +1,69 @@ +tables: + actors: &actors + name: actors + keys: [id] + films_actors: &films_actors + name: films_actors + keys: [id_film, id_actor] + films: &films + name: films + keys: [id] +dataset: + actors: + - { "id": 0, "first_name": "Harrison", "last_name": "Ford" } + - { "id": 1, "first_name": "Carrie", "last_name": "Fisher" } + films_actors: + - { "id_film": 0, "id_actor": 0 } + - { "id_film": 0, "id_actor": 1 } + - { "id_film": 1, "id_actor": 0 } + - { "id_film": 1, "id_actor": 1 } + - { "id_film": 2, "id_actor": 0 } + films: + - { "id": 0, "title": "Star Wars: Episode IV – A New Hope" } + - { "id": 1, "title": "Star Wars : Episode V – The Empire Strikes Back" } + - { "id": 2, "title": "Indiana Jones and the Temple of Doom" } +plan: + components: + actors: 0 + films_actors: 1 + films: 2 + relations: + - name: rel1 + cardinality: true # = many + select: [id_actor] # select only id_actor column + local: + table: *actors + keys: [id] + foreign: + table: *films_actors + keys: [id_actor] + - name: rel2 + cardinality: false # = single + select: [id] # select only id column + local: + table: *films_actors + keys: [id_film] + foreign: + table: *films + keys: [id] +executions: + - start: *actors + filter: + limit: 0 + values: { "first_name": "Harrison" } + where: "" + result: + - '{"first_name":"Harrison","id":0,"last_name":"Ford","rel1":[{"id_actor":0,"rel2":{"id":0}},{"id_actor":0,"rel2":{"id":1}},{"id_actor":0,"rel2":{"id":2}}]}' + - start: *actors + filter: + limit: 0 + values: { "last_name": "Fisher" } + where: "" + result: + - '{"first_name":"Carrie","id":1,"last_name":"Fisher","rel1":[{"id_actor":1,"rel2":{"id":0}},{"id_actor":1,"rel2":{"id":1}}]}' + - start: *actors + filter: + limit: 0 + values: { "last_name": "Reeves" } + where: "" + result: [] diff --git a/pkg/pull/testdata/select_override_columns.yaml b/pkg/pull/testdata/select_override_columns.yaml new file mode 100644 index 00000000..c8c43997 --- /dev/null +++ b/pkg/pull/testdata/select_override_columns.yaml @@ -0,0 +1,71 @@ +tables: + actors: &actors + name: actors + keys: [id] + films_actors: &films_actors + name: films_actors + keys: [id_film, id_actor] + films: &films + name: films + keys: [id] + columns: + - name: id + - name: title +dataset: + actors: + - { "id": 0, "first_name": "Harrison", "last_name": "Ford" } + - { "id": 1, "first_name": "Carrie", "last_name": "Fisher" } + films_actors: + - { "id_film": 0, "id_actor": 0 } + - { "id_film": 0, "id_actor": 1 } + - { "id_film": 1, "id_actor": 0 } + - { "id_film": 1, "id_actor": 1 } + - { "id_film": 2, "id_actor": 0 } + films: + - { "id": 0, "title": "Star Wars: Episode IV – A New Hope" } + - { "id": 1, "title": "Star Wars : Episode V – The Empire Strikes Back" } + - { "id": 2, "title": "Indiana Jones and the Temple of Doom" } +plan: + components: + actors: 0 + films_actors: 1 + films: 2 + relations: + - name: rel1 + cardinality: true # = many + local: + table: *actors + keys: [id] + foreign: + table: *films_actors + keys: [id_actor] + - name: rel2 + cardinality: false # = single + select: [id] # select only id column + local: + table: *films_actors + keys: [id_film] + foreign: + table: *films + keys: [id] +executions: + - start: *actors + filter: + limit: 0 + values: { "first_name": "Harrison" } + where: "" + result: + - '{"first_name":"Harrison","id":0,"last_name":"Ford","rel1":[{"id_actor":0,"id_film":0,"rel2":{"id":0}},{"id_actor":0,"id_film":1,"rel2":{"id":1}},{"id_actor":0,"id_film":2,"rel2":{"id":2}}]}' + - start: *actors + filter: + limit: 0 + values: { "last_name": "Fisher" } + where: "" + result: + - '{"first_name":"Carrie","id":1,"last_name":"Fisher","rel1":[{"id_actor":1,"id_film":0,"rel2":{"id":0}},{"id_actor":1,"id_film":1,"rel2":{"id":1}}]}' + - start: *actors + filter: + limit: 0 + values: { "last_name": "Reeves" } + where: "" + result: [] diff --git a/pkg/pull/testdata/simple_select.yaml b/pkg/pull/testdata/simple_select.yaml new file mode 100644 index 00000000..256144b8 --- /dev/null +++ b/pkg/pull/testdata/simple_select.yaml @@ -0,0 +1,69 @@ +tables: + actors: &actors + name: actors + keys: [id] + films_actors: &films_actors + name: films_actors + keys: [id_film, id_actor] + films: &films + name: films + keys: [id] +dataset: + actors: + - { "id": 0, "first_name": "Harrison", "last_name": "Ford" } + - { "id": 1, "first_name": "Carrie", "last_name": "Fisher" } + films_actors: + - { "id_film": 0, "id_actor": 0 } + - { "id_film": 0, "id_actor": 1 } + - { "id_film": 1, "id_actor": 0 } + - { "id_film": 1, "id_actor": 1 } + - { "id_film": 2, "id_actor": 0 } + films: + - { "id": 0, "title": "Star Wars: Episode IV – A New Hope" } + - { "id": 1, "title": "Star Wars : Episode V – The Empire Strikes Back" } + - { "id": 2, "title": "Indiana Jones and the Temple of Doom" } +plan: + components: + actors: 0 + films_actors: 1 + films: 2 + relations: + - name: rel1 + cardinality: true # = many + local: + table: *actors + keys: [id] + foreign: + table: *films_actors + keys: [id_actor] + - name: rel2 + cardinality: false # = single + select: [id] # select only id column + local: + table: *films_actors + keys: [id_film] + foreign: + table: *films + keys: [id] +executions: + - start: *actors + filter: + limit: 0 + values: { "first_name": "Harrison" } + where: "" + result: + - '{"first_name":"Harrison","id":0,"last_name":"Ford","rel1":[{"id_actor":0,"id_film":0,"rel2":{"id":0}},{"id_actor":0,"id_film":1,"rel2":{"id":1}},{"id_actor":0,"id_film":2,"rel2":{"id":2}}]}' + - start: *actors + select: [first_name, id] + filter: + limit: 0 + values: { "last_name": "Fisher" } + where: "" + result: + - '{"first_name":"Carrie","id":1,"rel1":[{"id_actor":1,"id_film":0,"rel2":{"id":0}},{"id_actor":1,"id_film":1,"rel2":{"id":1}}]}' + - start: *actors + filter: + limit: 0 + values: { "last_name": "Reeves" } + where: "" + result: [] diff --git a/tests/suites/pull/single_relation.yml b/tests/suites/pull/single_relation.yml index 4f998c0b..f6273b3d 100755 --- a/tests/suites/pull/single_relation.yml +++ b/tests/suites/pull/single_relation.yml @@ -21,7 +21,7 @@ testcases: steps: # Clean working directory - script: rm -f * - - script: lino dataconnector add source postgresql://postgres:sakila@source:5432/postgres?sslmode=disable + - script: lino dataconnector add source 'postgresql://postgres:sakila@source:5432/postgres?sslmode=disable' - script: lino relation extract source - script: lino table extract source --only-tables - script: lino id create store @@ -178,7 +178,7 @@ testcases: steps: - script: rm -f * - - script: lino dataconnector add source --schema public postgresql://postgres:sakila@source:5432/postgres?sslmode=disable + - script: lino dataconnector add source --schema public 'postgresql://postgres:sakila@source:5432/postgres?sslmode=disable' - script: lino relation extract source - script: lino table extract source --only-tables - script: lino id create customer @@ -188,3 +188,28 @@ testcases: - result.code ShouldEqual 0 - result.systemout ShouldEqual {"active":1,"activebool":true,"address_id":5,"create_date":"2006-02-14T00:00:00Z","customer_id":1,"email":"MARY.SMITH@sakilacustomer.org","first_name":"MARY","last_name":"SMITH","last_update":"2006-02-15T09:57:20Z","store_id":1} - result.systemerr ShouldContainSubstring "public"."customer" + + - name: pull with custom select columns on child in ingress descriptor + steps: + - script: lino id create store + - script: sed -i "s/true/false/g" ingress-descriptor.yaml + - script: lino id set-child-lookup staff_store_id_fkey true + - script: lino id set-child-select staff_store_id_fkey last_name first_name email + - script: lino pull source --limit 1 + assertions: + - result.code ShouldEqual 0 + - result.systemout ShouldEqual {"address_id":1,"last_update":"2006-02-15T09:57:12Z","manager_staff_id":1,"store_id":1,"staff_store_id_fkey":[{"last_name":"Hillyer","first_name":"Mike","email":"Mike.Hillyer@sakilastaff.com"}]} + - result.systemerr ShouldBeEmpty + - script: lino id set-child-lookup staff_store_id_fkey false + + - name: pull with custom select columns on parent in ingress descriptor + steps: + - script: lino id create store + - script: sed -i "s/true/false/g" ingress-descriptor.yaml + - script: lino id set-parent-lookup store_address_id_fkey true + - script: lino id set-parent-select store_address_id_fkey district address + - script: lino pull source --limit 1 + assertions: + - result.code ShouldEqual 0 + - result.systemout ShouldEqual {"address_id":1,"last_update":"2006-02-15T09:57:12Z","manager_staff_id":1,"store_id":1,"store_address_id_fkey":{"district":"Alberta","address":"47 MySakila Drive"}} + - result.systemerr ShouldBeEmpty