From ce1a479885d52b0b9eada5084247ea398de67a07 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sat, 11 May 2024 21:14:45 +0200 Subject: [PATCH 01/18] Refactor --- internal/planner/planner.go | 4 +- internal/planner/type_join.go | 268 ++++++++++++++++++---------------- 2 files changed, 145 insertions(+), 127 deletions(-) diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 13e1e0b2e9..40ce75c15b 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -362,8 +362,8 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan for subFieldName, subFieldInd := range filteredSubFields { indexes := desc.GetIndexesOnField(subFieldName) if len(indexes) > 0 && !filter.IsComplex(parentPlan.selectNode.filter) { - subInd := node.documentMapping.FirstIndexOfName(node.subTypeName) - relatedField := mapper.Field{Name: node.subTypeName, Index: subInd} + subInd := node.documentMapping.FirstIndexOfName(node.getSubTypeName()) + relatedField := mapper.Field{Name: node.getSubTypeName(), Index: subInd} fieldFilter := filter.UnwrapRelation(filter.CopyField( parentPlan.selectNode.filter, relatedField, diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index f745e3c5cf..5b2e89d9c5 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -147,6 +147,22 @@ func (n *typeIndexJoin) simpleExplain() (map[string]any, error) { // Add the type attribute. simpleExplainMap[joinTypeLabel] = n.joinPlan.Kind() + addExplainData := func(j *invertibleTypeJoin) error { + // Add the attribute(s). + simpleExplainMap[joinRootLabel] = immutable.Some(j.getRootTypeName()) + simpleExplainMap[joinSubTypeNameLabel] = j.getSubTypeName() + + subTypeExplainGraph, err := buildSimpleExplainGraph(j.subType) + if err != nil { + return err + } + + // Add the joined (subType) type's entire explain graph. + simpleExplainMap[joinSubTypeLabel] = subTypeExplainGraph + return nil + } + + var err error switch joinType := n.joinPlan.(type) { case *typeJoinOne: // Add the direction attribute. @@ -156,36 +172,16 @@ func (n *typeIndexJoin) simpleExplain() (map[string]any, error) { simpleExplainMap[joinDirectionLabel] = joinDirectionPrimaryLabel } - // Add the attribute(s). - simpleExplainMap[joinRootLabel] = joinType.rootName - simpleExplainMap[joinSubTypeNameLabel] = joinType.subTypeName - - subTypeExplainGraph, err := buildSimpleExplainGraph(joinType.subType) - if err != nil { - return nil, err - } - - // Add the joined (subType) type's entire explain graph. - simpleExplainMap[joinSubTypeLabel] = subTypeExplainGraph + err = addExplainData(&joinType.invertibleTypeJoin) case *typeJoinMany: - // Add the attribute(s). - simpleExplainMap[joinRootLabel] = joinType.rootName - simpleExplainMap[joinSubTypeNameLabel] = joinType.subTypeName - - subTypeExplainGraph, err := buildSimpleExplainGraph(joinType.subType) - if err != nil { - return nil, err - } - - // Add the joined (subType) type's entire explain graph. - simpleExplainMap[joinSubTypeLabel] = subTypeExplainGraph + err = addExplainData(&joinType.invertibleTypeJoin) default: - return simpleExplainMap, client.NewErrUnhandledType("join plan", n.joinPlan) + err = client.NewErrUnhandledType("join plan", n.joinPlan) } - return simpleExplainMap, nil + return simpleExplainMap, err } // Explain method returns a map containing all attributes of this node that @@ -239,38 +235,39 @@ func (p *Planner) makeTypeJoinOne( } // get the correct sub field schema type (collection) - subTypeFieldDesc, ok := parent.collection.Definition().GetFieldByName(subType.Name) + queriedRelFieldDesc, ok := parent.collection.Definition().GetFieldByName(subType.Name) if !ok { return nil, client.NewErrFieldNotExist(subType.Name) } - subTypeCol, err := p.db.GetCollectionByName(p.ctx, subType.CollectionName) + queriedSubTypeCol, err := p.db.GetCollectionByName(p.ctx, subType.CollectionName) if err != nil { return nil, err } - subTypeField, subTypeFieldNameFound := subTypeCol.Description().GetFieldByRelation( - subTypeFieldDesc.RelationName, + subTypesRelField, ok := queriedSubTypeCol.Description().GetFieldByRelation( + queriedRelFieldDesc.RelationName, parent.collection.Name().Value(), - subTypeFieldDesc.Name, + queriedRelFieldDesc.Name, ) - if !subTypeFieldNameFound { - return nil, client.NewErrFieldNotExist(subTypeFieldDesc.RelationName) + if !ok { + return nil, client.NewErrFieldNotExist(queriedRelFieldDesc.RelationName) } var secondaryFieldIndex immutable.Option[int] - if !subTypeFieldDesc.IsPrimaryRelation { - idFieldName := subTypeFieldDesc.Name + request.RelatedObjectID + if !queriedRelFieldDesc.IsPrimaryRelation { + idFieldName := queriedRelFieldDesc.Name secondaryFieldIndex = immutable.Some( - parent.documentMapping.FirstIndexOfName(idFieldName), + parent.documentMapping.FirstIndexOfName(idFieldName + request.RelatedObjectID), ) } dir := joinDirection{ - firstNode: source, - secondNode: selectPlan, - secondaryField: immutable.Some(subTypeField.Name + request.RelatedObjectID), - primaryField: subTypeFieldDesc.Name + request.RelatedObjectID, + firstNode: source, + secondNode: selectPlan, + topRelField: queriedRelFieldDesc.Name, + subRelField: subTypesRelField.Name, + isInvertable: true, } return &typeJoinOne{ @@ -279,10 +276,8 @@ func (p *Planner) makeTypeJoinOne( root: source, subType: selectPlan, subSelect: subType, - subSelectFieldDef: subTypeFieldDesc, - rootName: immutable.Some(subTypeField.Name), - subTypeName: subType.Name, - isSecondary: !subTypeFieldDesc.IsPrimaryRelation, + subSelectFieldDef: queriedRelFieldDesc, + isSecondary: !queriedRelFieldDesc.IsPrimaryRelation, secondaryFieldIndex: secondaryFieldIndex, secondaryFetchLimit: 1, dir: dir, @@ -369,47 +364,42 @@ func (p *Planner) makeTypeJoinMany( return nil, err } - subTypeFieldDesc, ok := parent.collection.Definition().GetFieldByName(subType.Name) + queriedRelFieldDesc, ok := parent.collection.Definition().GetFieldByName(subType.Name) if !ok { return nil, client.NewErrFieldNotExist(subType.Name) } - subTypeCol, err := p.db.GetCollectionByName(p.ctx, subType.CollectionName) + queriedSubTypeCol, err := p.db.GetCollectionByName(p.ctx, subType.CollectionName) if err != nil { return nil, err } - var secondaryFieldName immutable.Option[string] - var rootName immutable.Option[string] - if subTypeFieldDesc.RelationName != "" { - rootField, rootNameFound := subTypeCol.Description().GetFieldByRelation( - subTypeFieldDesc.RelationName, + dir := joinDirection{ + firstNode: source, + secondNode: selectPlan, + topRelField: queriedRelFieldDesc.Name, + } + + if queriedRelFieldDesc.RelationName != "" { + rootField, ok := queriedSubTypeCol.Description().GetFieldByRelation( + queriedRelFieldDesc.RelationName, parent.collection.Name().Value(), - subTypeFieldDesc.Name, + queriedRelFieldDesc.Name, ) - if rootNameFound { - rootName = immutable.Some(rootField.Name) - secondaryFieldName = immutable.Some(rootField.Name + request.RelatedObjectID) + if ok { + dir.subRelField = rootField.Name + dir.isInvertable = true } } - dir := joinDirection{ - firstNode: source, - secondNode: selectPlan, - secondaryField: secondaryFieldName, - primaryField: subTypeFieldDesc.Name + request.RelatedObjectID, - } - return &typeJoinMany{ invertibleTypeJoin: invertibleTypeJoin{ docMapper: docMapper{parent.documentMapping}, root: source, subType: selectPlan, subSelect: subType, - subSelectFieldDef: subTypeFieldDesc, - rootName: rootName, + subSelectFieldDef: queriedRelFieldDesc, isSecondary: true, - subTypeName: subType.Name, secondaryFetchLimit: 0, dir: dir, }, @@ -420,12 +410,17 @@ func (n *typeJoinMany) Kind() string { return "typeJoinMany" } -func fetchPrimaryDoc(node, subNode planNode, parentProp string) (bool, error) { +func getPrimaryDocIDFromSecondaryDoc(subNode planNode, parentProp string) string { subDoc := subNode.Value() - ind := subNode.DocumentMap().FirstIndexOfName(parentProp) + ind := subNode.DocumentMap().FirstIndexOfName(parentProp + request.RelatedObjectID) + + docIDStr, _ := subDoc.Fields[ind].(string) + return docIDStr +} - docIDStr, isStr := subDoc.Fields[ind].(string) - if !isStr { +func fetchPrimaryDoc(node, subNode planNode, parentProp string) (bool, error) { + docIDStr := getPrimaryDocIDFromSecondaryDoc(subNode, parentProp) + if docIDStr == "" { return false, nil } @@ -452,31 +447,34 @@ func fetchPrimaryDoc(node, subNode planNode, parentProp string) (bool, error) { return true, nil } +// joinDirection is a struct that holds the two nodes that are that are executed one after another +// depending on the direction of the join. type joinDirection struct { - firstNode planNode - secondNode planNode - secondaryField immutable.Option[string] - primaryField string - isInverted bool + // firstNode is the node that is executed first (usually an indexed collection). + firstNode planNode // User + // secondNode is the node that is executed second. + secondNode planNode // Device + // subRelField is a field name of a secondary doc that refers to the primary docID (like author_id). + subRelField string // owner_id + // topRelField is a field name of the primary doc that refers to the secondary docID (like author_id). + topRelField string // devices_id + // isInvertable indicates if the join can be inverted. + isInvertable bool + // isInverted indicates if the join direction is inverted. + isInverted bool } func (dir *joinDirection) invert() { - if !dir.secondaryField.HasValue() { - // If the secondary field has no value it cannot be inverted - return - } dir.isInverted = !dir.isInverted dir.firstNode, dir.secondNode = dir.secondNode, dir.firstNode - dir.secondaryField, dir.primaryField = immutable.Some(dir.primaryField), dir.secondaryField.Value() + dir.subRelField, dir.topRelField = dir.topRelField, dir.subRelField } type invertibleTypeJoin struct { docMapper - root planNode - subType planNode - rootName immutable.Option[string] - subTypeName string + root planNode + subType planNode subSelect *mapper.Select subSelectFieldDef client.FieldDefinition @@ -491,6 +489,20 @@ type invertibleTypeJoin struct { dir joinDirection } +func (join *invertibleTypeJoin) getRootTypeName() string { + if join.dir.isInverted { + return join.dir.topRelField + } + return join.dir.subRelField +} + +func (join *invertibleTypeJoin) getSubTypeName() string { + if join.dir.isInverted { + return join.dir.subRelField + } + return join.dir.topRelField +} + func (join *invertibleTypeJoin) replaceRoot(node planNode) { join.root = node if join.dir.isInverted { @@ -533,27 +545,56 @@ func (tj *invertibleTypeJoin) invert() { tj.isSecondary = !tj.isSecondary } -func (join *invertibleTypeJoin) processSecondResult(secondDocs []core.Doc) (any, any) { - var secondResult any - var secondIDResult any +// addSecondaryDocsToRootPrimaryDoc adds the second docs to the root primary doc. +// If the relations is 1-to-1 a single second doc will be added to the root primary doc. +// Otherwise, all second docs will be added as an array. +func (join *invertibleTypeJoin) addSecondaryDocsToRootPrimaryDoc(secondDocs []core.Doc) { + var secondaryResult any + var secondaryIDResult any if join.secondaryFetchLimit == 1 { if len(secondDocs) != 0 { - secondResult = secondDocs[0] - secondIDResult = secondDocs[0].GetID() + secondaryResult = secondDocs[0] + secondaryIDResult = secondDocs[0].GetID() } } else { - secondResult = secondDocs + secondaryResult = secondDocs secondDocIDs := make([]string, len(secondDocs)) for i, doc := range secondDocs { secondDocIDs[i] = doc.GetID() } - secondIDResult = secondDocIDs + secondaryIDResult = secondDocIDs } - join.root.Value().Fields[join.subSelect.Index] = secondResult + join.root.Value().Fields[join.subSelect.Index] = secondaryResult if join.secondaryFieldIndex.HasValue() { - join.root.Value().Fields[join.secondaryFieldIndex.Value()] = secondIDResult + join.root.Value().Fields[join.secondaryFieldIndex.Value()] = secondaryIDResult } - return secondResult, secondIDResult +} + +func (join *invertibleTypeJoin) fetchSecondaryDocsForPrimaryDoc(primaryDocID string) (bool, error) { + secondDocs, err := fetchDocsWithFieldValue( + join.dir.secondNode, + // As the join is from the secondary field, we know that [join.dir.secondaryField] must have a value + // otherwise the user would not have been able to request it. + join.dir.subRelField+request.RelatedObjectID, + primaryDocID, + ) + if err != nil { + return false, err + } + if join.dir.secondNode == join.root { + if len(secondDocs) == 0 { + return false, nil + } + for i := range secondDocs { + secondDocs[i].Fields[join.subSelect.Index] = join.subType.Value() + } + join.docsToYield = append(join.docsToYield, secondDocs...) + return true, nil + } else { + join.addSecondaryDocsToRootPrimaryDoc(secondDocs) + join.docsToYield = append(join.docsToYield, join.root.Value()) + } + return true, nil } func (join *invertibleTypeJoin) Next() (bool, error) { @@ -574,37 +615,11 @@ func (join *invertibleTypeJoin) Next() (bool, error) { return false, err } - firstDoc := join.dir.firstNode.Value() - if join.isSecondary { - secondDocs, err := fetchDocsWithFieldValue( - join.dir.secondNode, - // As the join is from the secondary field, we know that [join.dir.secondaryField] must have a value - // otherwise the user would not have been able to request it. - join.dir.secondaryField.Value(), - firstDoc.GetID(), - ) - if err != nil { - return false, err - } - if join.dir.secondNode == join.root { - if len(secondDocs) == 0 { - return false, nil - } - for i := range secondDocs { - secondDocs[i].Fields[join.subSelect.Index] = join.subType.Value() - } - join.docsToYield = append(join.docsToYield, secondDocs...) - return true, nil - } else { - secondResult, secondIDResult := join.processSecondResult(secondDocs) - join.dir.firstNode.Value().Fields[join.subSelect.Index] = secondResult - if join.secondaryFieldIndex.HasValue() { - join.dir.firstNode.Value().Fields[join.secondaryFieldIndex.Value()] = secondIDResult - } - } + firstDoc := join.dir.firstNode.Value() + return join.fetchSecondaryDocsForPrimaryDoc(firstDoc.GetID()) } else { - hasDoc, err := fetchPrimaryDoc(join.dir.secondNode, join.dir.firstNode, join.dir.primaryField) + hasDoc, err := fetchPrimaryDoc(join.dir.secondNode, join.dir.firstNode, join.dir.topRelField) if err != nil { return false, err } @@ -612,9 +627,9 @@ func (join *invertibleTypeJoin) Next() (bool, error) { if hasDoc { join.root.Value().Fields[join.subSelect.Index] = join.subType.Value() } - } - join.docsToYield = append(join.docsToYield, join.root.Value()) + join.docsToYield = append(join.docsToYield, join.root.Value()) + } return true, nil } @@ -630,16 +645,19 @@ func (join *invertibleTypeJoin) invertJoinDirectionWithIndex( fieldFilter *mapper.Filter, index client.IndexDescription, ) error { - if !join.rootName.HasValue() { - // If the root field has no value it cannot be inverted + if !join.dir.isInvertable { return nil } if join.subSelectFieldDef.Kind.IsArray() { // invertibleTypeJoin does not support inverting one-many relations atm return nil } + rootName := join.dir.subRelField + if join.dir.isInverted { + rootName = join.dir.topRelField + } subScan := getScanNode(join.subType) - subScan.tryAddField(join.rootName.Value() + request.RelatedObjectID) + subScan.tryAddField(rootName + request.RelatedObjectID) subScan.filter = fieldFilter subScan.initFetcher(immutable.Option[string]{}, immutable.Some(index)) From 63b3191a97aec8d64a37d67862c43d366f2070c6 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Sun, 26 May 2024 17:37:53 +0300 Subject: [PATCH 02/18] tmp-commit --- internal/db/collection_index.go | 24 +- internal/db/index.go | 2 +- internal/planner/explain.go | 16 +- internal/planner/planner.go | 6 +- internal/planner/select.go | 13 + internal/planner/type_join.go | 535 ++++++++++++++++++++++++-------- 6 files changed, 444 insertions(+), 152 deletions(-) diff --git a/internal/db/collection_index.go b/internal/db/collection_index.go index 14f9a1b805..7ded8fff0e 100644 --- a/internal/db/collection_index.go +++ b/internal/db/collection_index.go @@ -21,6 +21,7 @@ import ( "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/client/request" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/db/base" @@ -264,7 +265,7 @@ func (c *collection) createIndex( return nil, err } - err = c.checkExistingFields(desc.Fields) + err = c.checkExistingFieldsAndAdjustRelFieldNames(desc.Fields) if err != nil { return nil, err } @@ -493,20 +494,19 @@ func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, return c.Description().Indexes, nil } -func (c *collection) checkExistingFields( +// checkExistingFieldsAndAdjustRelFieldNames checks if the fields in the index description +// exist in the collection schema. +// If a field is a relation, it will be adjusted to relation id field name, a.k.a. `field_name + _id`. +func (c *collection) checkExistingFieldsAndAdjustRelFieldNames( fields []client.IndexedFieldDescription, ) error { - collectionFields := c.Schema().Fields - for _, field := range fields { - found := false - for _, colField := range collectionFields { - if field.Name == colField.Name { - found = true - break - } - } + for i := range fields { + field, found := c.Schema().GetFieldByName(fields[i].Name) if !found { - return NewErrNonExistingFieldForIndex(field.Name) + return NewErrNonExistingFieldForIndex(fields[i].Name) + } + if field.Kind.IsObject() { + fields[i].Name = fields[i].Name + request.RelatedObjectID } } return nil diff --git a/internal/db/index.go b/internal/db/index.go index 71569e64db..bd11e9f94b 100644 --- a/internal/db/index.go +++ b/internal/db/index.go @@ -41,7 +41,7 @@ func getValidateIndexFieldFunc(kind client.FieldKind) func(any) bool { } switch kind { - case client.FieldKind_NILLABLE_STRING: + case client.FieldKind_NILLABLE_STRING, client.FieldKind_DocID: return canConvertIndexFieldValue[string] case client.FieldKind_NILLABLE_INT: return canConvertIndexFieldValue[int64] diff --git a/internal/planner/explain.go b/internal/planner/explain.go index 5ab2f292f8..e68c348787 100644 --- a/internal/planner/explain.go +++ b/internal/planner/explain.go @@ -92,8 +92,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) { var explainGraphBuilder = map[string]any{} // If root is not the last child then keep walking and explaining the root graph. - if node.root != nil { - indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.root) + if node.parentPlan != nil { + indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.parentPlan) if err != nil { return nil, err } @@ -101,8 +101,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) { explainGraphBuilder[joinRootLabel] = indexJoinRootExplainGraph } - if node.subType != nil { - indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.subType) + if node.childPlan != nil { + indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.childPlan) if err != nil { return nil, err } @@ -117,8 +117,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) { var explainGraphBuilder = map[string]any{} // If root is not the last child then keep walking and explaining the root graph. - if node.root != nil { - indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.root) + if node.parentPlan != nil { + indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.parentPlan) if err != nil { return nil, err } @@ -128,8 +128,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) { explainGraphBuilder[joinRootLabel] = nil } - if node.subType != nil { - indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.subType) + if node.childPlan != nil { + indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.childPlan) if err != nil { return nil, err } diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 40ce75c15b..05e95e27e3 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -357,7 +357,7 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan parentPlan.selectNode.filter.Conditions, node.documentMapping, ) - slct := node.subType.(*selectTopNode).selectNode + slct := node.childPlan.(*selectTopNode).selectNode desc := slct.collection.Description() for subFieldName, subFieldInd := range filteredSubFields { indexes := desc.GetIndexesOnField(subFieldName) @@ -383,7 +383,7 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan // expandTypeJoin does a plan graph expansion and other optimizations on invertibleTypeJoin. func (p *Planner) expandTypeJoin(node *invertibleTypeJoin, parentPlan *selectTopNode) error { if parentPlan.selectNode.filter == nil { - return p.expandPlan(node.subType, parentPlan) + return p.expandPlan(node.childPlan, parentPlan) } err := p.tryOptimizeJoinDirection(node, parentPlan) @@ -391,7 +391,7 @@ func (p *Planner) expandTypeJoin(node *invertibleTypeJoin, parentPlan *selectTop return err } - return p.expandPlan(node.subType, parentPlan) + return p.expandPlan(node.childPlan, parentPlan) } func (p *Planner) expandGroupNodePlan(topNodeSelect *selectTopNode) error { diff --git a/internal/planner/select.go b/internal/planner/select.go index c3e7f4dd2d..60c895167d 100644 --- a/internal/planner/select.go +++ b/internal/planner/select.go @@ -315,6 +315,19 @@ func findIndexByFilteringField(scanNode *scanNode) immutable.Option[client.Index return immutable.None[client.IndexDescription]() } +func findIndexByFieldName(col client.Collection, fieldName string) immutable.Option[client.IndexDescription] { + for _, field := range col.Schema().Fields { + if field.Name != fieldName { + continue + } + indexes := col.Description().GetIndexesOnField(field.Name) + if len(indexes) > 0 { + return immutable.Some(indexes[0]) + } + } + return immutable.None[client.IndexDescription]() +} + func (n *selectNode) initFields(selectReq *mapper.Select) ([]aggregateNode, error) { aggregates := []aggregateNode{} // loop over the sub type diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index 5b2e89d9c5..8a7c9373dd 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -152,7 +152,7 @@ func (n *typeIndexJoin) simpleExplain() (map[string]any, error) { simpleExplainMap[joinRootLabel] = immutable.Some(j.getRootTypeName()) simpleExplainMap[joinSubTypeNameLabel] = j.getSubTypeName() - subTypeExplainGraph, err := buildSimpleExplainGraph(j.subType) + subTypeExplainGraph, err := buildSimpleExplainGraph(j.childPlan) if err != nil { return err } @@ -166,7 +166,7 @@ func (n *typeIndexJoin) simpleExplain() (map[string]any, error) { switch joinType := n.joinPlan.(type) { case *typeJoinOne: // Add the direction attribute. - if joinType.isSecondary { + if joinType.isChildSecondary { simpleExplainMap[joinDirectionLabel] = joinDirectionSecondaryLabel } else { simpleExplainMap[joinDirectionLabel] = joinDirectionPrimaryLabel @@ -197,10 +197,10 @@ func (n *typeIndexJoin) Explain(explainType request.ExplainType) (map[string]any } var subScan *scanNode if joinMany, isJoinMany := n.joinPlan.(*typeJoinMany); isJoinMany { - subScan = getScanNode(joinMany.subType) + subScan = getScanNode(joinMany.childPlan) } if joinOne, isJoinOne := n.joinPlan.(*typeJoinOne); isJoinOne { - subScan = getScanNode(joinOne.subType) + subScan = getScanNode(joinOne.childPlan) } if subScan != nil { subScanExplain, err := subScan.Explain(explainType) @@ -224,60 +224,67 @@ type typeJoinOne struct { func (p *Planner) makeTypeJoinOne( parent *selectNode, - source planNode, - subType *mapper.Select, + sourcePlan planNode, + subSelect *mapper.Select, ) (*typeJoinOne, error) { - prepareScanNodeFilterForTypeJoin(parent, source, subType) + prepareScanNodeFilterForTypeJoin(parent, sourcePlan, subSelect) - selectPlan, err := p.Select(subType) + subSelectPlan, err := p.Select(subSelect) if err != nil { return nil, err } // get the correct sub field schema type (collection) - queriedRelFieldDesc, ok := parent.collection.Definition().GetFieldByName(subType.Name) + parentsRelFieldDef, ok := parent.collection.Definition().GetFieldByName(subSelect.Name) if !ok { - return nil, client.NewErrFieldNotExist(subType.Name) + return nil, client.NewErrFieldNotExist(subSelect.Name) } - queriedSubTypeCol, err := p.db.GetCollectionByName(p.ctx, subType.CollectionName) + subCol, err := p.db.GetCollectionByName(p.ctx, subSelect.CollectionName) if err != nil { return nil, err } - subTypesRelField, ok := queriedSubTypeCol.Description().GetFieldByRelation( - queriedRelFieldDesc.RelationName, + childsRelFieldDesc, ok := subCol.Description().GetFieldByRelation( + parentsRelFieldDef.RelationName, parent.collection.Name().Value(), - queriedRelFieldDesc.Name, + parentsRelFieldDef.Name, ) if !ok { - return nil, client.NewErrFieldNotExist(queriedRelFieldDesc.RelationName) + return nil, client.NewErrFieldNotExist(parentsRelFieldDef.RelationName) + } + + childsRelFieldDef, ok := subCol.Definition().GetFieldByName(childsRelFieldDesc.Name) + if !ok { + return nil, client.NewErrFieldNotExist(subSelect.Name) } var secondaryFieldIndex immutable.Option[int] - if !queriedRelFieldDesc.IsPrimaryRelation { - idFieldName := queriedRelFieldDesc.Name + if !parentsRelFieldDef.IsPrimaryRelation { + idFieldName := parentsRelFieldDef.Name secondaryFieldIndex = immutable.Some( parent.documentMapping.FirstIndexOfName(idFieldName + request.RelatedObjectID), ) } dir := joinDirection{ - firstNode: source, - secondNode: selectPlan, - topRelField: queriedRelFieldDesc.Name, - subRelField: subTypesRelField.Name, - isInvertable: true, + firstNode: sourcePlan, + secondNode: subSelectPlan, + firstNodesRelFieldName: parentsRelFieldDef.Name, + secondNodesRelFieldName: childsRelFieldDesc.Name, + isInvertable: true, } return &typeJoinOne{ invertibleTypeJoin: invertibleTypeJoin{ docMapper: docMapper{parent.documentMapping}, - root: source, - subType: selectPlan, - subSelect: subType, - subSelectFieldDef: queriedRelFieldDesc, - isSecondary: !queriedRelFieldDesc.IsPrimaryRelation, + parentPlan: sourcePlan, + childPlan: subSelectPlan, + childSelect: subSelect, + parentsRelFieldDef: parentsRelFieldDef, + childsRelFieldDef: childsRelFieldDef, + parentCollection: parent.collection, + isChildSecondary: !parentsRelFieldDef.IsPrimaryRelation, secondaryFieldIndex: secondaryFieldIndex, secondaryFetchLimit: 1, dir: dir, @@ -289,30 +296,6 @@ func (n *typeJoinOne) Kind() string { return "typeJoinOne" } -func fetchDocsWithFieldValue(plan planNode, fieldName string, val any) ([]core.Doc, error) { - propIndex := plan.DocumentMap().FirstIndexOfName(fieldName) - setSubTypeFilterToScanNode(plan, propIndex, val) - - if err := plan.Init(); err != nil { - return nil, NewErrSubTypeInit(err) - } - - var docs []core.Doc - for { - next, err := plan.Next() - if err != nil { - return nil, err - } - if !next { - break - } - - docs = append(docs, plan.Value()) - } - - return docs, nil -} - type typeJoinMany struct { invertibleTypeJoin } @@ -354,40 +337,54 @@ func prepareScanNodeFilterForTypeJoin( func (p *Planner) makeTypeJoinMany( parent *selectNode, - source planNode, - subType *mapper.Select, + sourcePlan planNode, + subSelect *mapper.Select, ) (*typeJoinMany, error) { - prepareScanNodeFilterForTypeJoin(parent, source, subType) + prepareScanNodeFilterForTypeJoin(parent, sourcePlan, subSelect) - selectPlan, err := p.Select(subType) + subSelectPlan, err := p.Select(subSelect) if err != nil { return nil, err } - queriedRelFieldDesc, ok := parent.collection.Definition().GetFieldByName(subType.Name) + parentsRelFieldDef, ok := parent.collection.Definition().GetFieldByName(subSelect.Name) if !ok { - return nil, client.NewErrFieldNotExist(subType.Name) + return nil, client.NewErrFieldNotExist(subSelect.Name) } - queriedSubTypeCol, err := p.db.GetCollectionByName(p.ctx, subType.CollectionName) + subCol, err := p.db.GetCollectionByName(p.ctx, subSelect.CollectionName) if err != nil { return nil, err } + childsRelFieldDesc, ok := subCol.Description().GetFieldByRelation( + parentsRelFieldDef.RelationName, + parent.collection.Name().Value(), + parentsRelFieldDef.Name, + ) + if !ok { + return nil, client.NewErrFieldNotExist(subSelect.Name) + } + + childsRelFieldDef, ok := subCol.Definition().GetFieldByName(childsRelFieldDesc.Name) + if !ok { + return nil, client.NewErrFieldNotExist(subSelect.Name) + } + dir := joinDirection{ - firstNode: source, - secondNode: selectPlan, - topRelField: queriedRelFieldDesc.Name, + firstNode: sourcePlan, + secondNode: subSelectPlan, + firstNodesRelFieldName: parentsRelFieldDef.Name, } - if queriedRelFieldDesc.RelationName != "" { - rootField, ok := queriedSubTypeCol.Description().GetFieldByRelation( - queriedRelFieldDesc.RelationName, + if parentsRelFieldDef.RelationName != "" { + rootField, ok := subCol.Description().GetFieldByRelation( + parentsRelFieldDef.RelationName, parent.collection.Name().Value(), - queriedRelFieldDesc.Name, + parentsRelFieldDef.Name, ) if ok { - dir.subRelField = rootField.Name + dir.secondNodesRelFieldName = rootField.Name dir.isInvertable = true } } @@ -395,11 +392,13 @@ func (p *Planner) makeTypeJoinMany( return &typeJoinMany{ invertibleTypeJoin: invertibleTypeJoin{ docMapper: docMapper{parent.documentMapping}, - root: source, - subType: selectPlan, - subSelect: subType, - subSelectFieldDef: queriedRelFieldDesc, - isSecondary: true, + parentPlan: sourcePlan, + childPlan: subSelectPlan, + childSelect: subSelect, + parentsRelFieldDef: parentsRelFieldDef, + childsRelFieldDef: childsRelFieldDef, + parentCollection: parent.collection, + isChildSecondary: true, secondaryFetchLimit: 0, dir: dir, }, @@ -451,13 +450,13 @@ func fetchPrimaryDoc(node, subNode planNode, parentProp string) (bool, error) { // depending on the direction of the join. type joinDirection struct { // firstNode is the node that is executed first (usually an indexed collection). - firstNode planNode // User + firstNode planNode // secondNode is the node that is executed second. - secondNode planNode // Device - // subRelField is a field name of a secondary doc that refers to the primary docID (like author_id). - subRelField string // owner_id - // topRelField is a field name of the primary doc that refers to the secondary docID (like author_id). - topRelField string // devices_id + secondNode planNode + // secondNodesRelFieldName is a field name of a secondary doc that refers to the primary docID (like author_id). + secondNodesRelFieldName string + // firstNodesRelFieldName is a field name of the primary doc that refers to the secondary docID (like author_id). + firstNodesRelFieldName string // isInvertable indicates if the join can be inverted. isInvertable bool // isInverted indicates if the join direction is inverted. @@ -467,19 +466,21 @@ type joinDirection struct { func (dir *joinDirection) invert() { dir.isInverted = !dir.isInverted dir.firstNode, dir.secondNode = dir.secondNode, dir.firstNode - dir.subRelField, dir.topRelField = dir.topRelField, dir.subRelField + dir.secondNodesRelFieldName, dir.firstNodesRelFieldName = dir.firstNodesRelFieldName, dir.secondNodesRelFieldName } type invertibleTypeJoin struct { docMapper - root planNode - subType planNode + parentPlan planNode + childPlan planNode - subSelect *mapper.Select - subSelectFieldDef client.FieldDefinition + childSelect *mapper.Select + parentsRelFieldDef client.FieldDefinition + childsRelFieldDef client.FieldDefinition + parentCollection client.Collection - isSecondary bool + isChildSecondary bool secondaryFieldIndex immutable.Option[int] secondaryFetchLimit uint @@ -491,20 +492,20 @@ type invertibleTypeJoin struct { func (join *invertibleTypeJoin) getRootTypeName() string { if join.dir.isInverted { - return join.dir.topRelField + return join.dir.firstNodesRelFieldName } - return join.dir.subRelField + return join.dir.secondNodesRelFieldName } func (join *invertibleTypeJoin) getSubTypeName() string { if join.dir.isInverted { - return join.dir.subRelField + return join.dir.secondNodesRelFieldName } - return join.dir.topRelField + return join.dir.firstNodesRelFieldName } func (join *invertibleTypeJoin) replaceRoot(node planNode) { - join.root = node + join.parentPlan = node if join.dir.isInverted { join.dir.secondNode = node } else { @@ -513,41 +514,41 @@ func (join *invertibleTypeJoin) replaceRoot(node planNode) { } func (join *invertibleTypeJoin) Init() error { - if err := join.subType.Init(); err != nil { + if err := join.childPlan.Init(); err != nil { return err } - return join.root.Init() + return join.parentPlan.Init() } func (join *invertibleTypeJoin) Start() error { - if err := join.subType.Start(); err != nil { + if err := join.childPlan.Start(); err != nil { return err } - return join.root.Start() + return join.parentPlan.Start() } func (join *invertibleTypeJoin) Close() error { - if err := join.root.Close(); err != nil { + if err := join.parentPlan.Close(); err != nil { return err } - return join.subType.Close() + return join.childPlan.Close() } func (join *invertibleTypeJoin) Spans(spans core.Spans) { - join.root.Spans(spans) + join.parentPlan.Spans(spans) } -func (join *invertibleTypeJoin) Source() planNode { return join.root } +func (join *invertibleTypeJoin) Source() planNode { return join.parentPlan } func (tj *invertibleTypeJoin) invert() { tj.dir.invert() - tj.isSecondary = !tj.isSecondary + tj.isChildSecondary = !tj.isChildSecondary } -// addSecondaryDocsToRootPrimaryDoc adds the second docs to the root primary doc. -// If the relations is 1-to-1 a single second doc will be added to the root primary doc. -// Otherwise, all second docs will be added as an array. +// addSecondaryDocsToRootPrimaryDoc adds the secondary docs to the root primary doc. +// If the relations is 1-to-1 a single secondary doc will be added to the root primary doc. +// Otherwise, all secondary docs will be added as an array. func (join *invertibleTypeJoin) addSecondaryDocsToRootPrimaryDoc(secondDocs []core.Doc) { var secondaryResult any var secondaryIDResult any @@ -564,37 +565,263 @@ func (join *invertibleTypeJoin) addSecondaryDocsToRootPrimaryDoc(secondDocs []co } secondaryIDResult = secondDocIDs } - join.root.Value().Fields[join.subSelect.Index] = secondaryResult + join.parentPlan.Value().Fields[join.childSelect.Index] = secondaryResult if join.secondaryFieldIndex.HasValue() { - join.root.Value().Fields[join.secondaryFieldIndex.Value()] = secondaryIDResult + join.parentPlan.Value().Fields[join.secondaryFieldIndex.Value()] = secondaryIDResult + } +} + +type secondaryDocsJoiner struct { + node planNode + primaryDoc core.Doc + fieldDef client.FieldDefinition + relIDFieldDef client.FieldDefinition + parentPlan planNode + childPlan planNode + + scan *scanNode + origScanFields []client.FieldDefinition + + resultSecondaryDocIDs []string + resultSecondaryDocs []core.Doc +} + +func newSecondaryDocsJoiner( + node planNode, + primaryDoc core.Doc, + fieldDef client.FieldDefinition, + relIDFieldDef client.FieldDefinition, +) secondaryDocsJoiner { + j := secondaryDocsJoiner{ + node: node, + primaryDoc: primaryDoc, + fieldDef: fieldDef, + relIDFieldDef: relIDFieldDef, + } + return j +} + +func (j *secondaryDocsJoiner) fetchSecondaryDocsForPrimaryDoc() error { + err := j.fetchSecondaryDocIDs() + + if err != nil { + return err + } + + docs, err := j.fetchSecondaryDocsByIDs() + + if err != nil { + return err + } + + j.join(docs) + + return nil +} + +func (j *secondaryDocsJoiner) prepareScanNodeForDocIDsFetching() error { + j.scan = getScanNode(j.node) + if err := j.scan.fetcher.Close(); err != nil { + return NewErrSubTypeInit(err) } + j.scan.filter = nil + j.origScanFields = j.scan.fields + j.scan.fields = []client.FieldDefinition{j.relIDFieldDef} + j.scan.spans = core.Spans{} + return nil } -func (join *invertibleTypeJoin) fetchSecondaryDocsForPrimaryDoc(primaryDocID string) (bool, error) { +func (j *secondaryDocsJoiner) collectDocs(numDocs int) ([]core.Doc, error) { + if err := j.node.Init(); err != nil { + return nil, NewErrSubTypeInit(err) + } + + docs := make([]core.Doc, 0, numDocs) + + for { + hasValue, err := j.node.Next() + + if err != nil { + return nil, err + } + + if !hasValue { + break + } + + docs = append(docs, j.node.Value()) + } + + if err := j.node.Close(); err != nil { + return nil, NewErrSubTypeInit(err) + } + + return docs, nil +} + +func (j *secondaryDocsJoiner) fetchSecondaryDocIDs() error { + err := j.prepareScanNodeForDocIDsFetching() + if err != nil { + return err + } + + relIDPropIndex := j.node.DocumentMap().FirstIndexOfName(j.relIDFieldDef.Name) + setSubTypeFilterToScanNode(j.node, relIDPropIndex, j.primaryDoc.GetID()) + + indexOnRelation := findIndexByFieldName(j.scan.col, j.relIDFieldDef.Name) + j.scan.initFetcher(immutable.None[string](), indexOnRelation) + + docs, err := j.collectDocs(0) + + if err != nil { + return err + } + + for i := range docs { + j.resultSecondaryDocIDs = append(j.resultSecondaryDocIDs, docs[i].Fields[core.DocIDFieldIndex].(string)) + } + + return nil +} + +func (j *secondaryDocsJoiner) spansFromDocIDs() core.Spans { + colRootID := j.scan.col.Description().RootID + + spans := make([]core.Span, 0, len(j.resultSecondaryDocIDs)) + for _, secondaryID := range j.resultSecondaryDocIDs { + dsKey := core.DataStoreKey{CollectionRootID: colRootID, DocID: secondaryID} + spans = append(spans, core.NewSpan(dsKey, dsKey.PrefixEnd())) + } + return core.NewSpans(spans...) +} + +func (j *secondaryDocsJoiner) prepareScanNodeForDocsFetching() error { + if err := j.scan.fetcher.Close(); err != nil { + return NewErrSubTypeInit(err) + } + + j.scan.filter = nil + for i := range j.origScanFields { + if j.origScanFields[i].Name == j.relIDFieldDef.Name { + j.origScanFields = append(j.origScanFields[:i], j.origScanFields[i+1:]...) + break + } + } + j.scan.fields = j.origScanFields + return nil +} + +func (j *secondaryDocsJoiner) fetchSecondaryDocsByIDs() ([]core.Doc, error) { + err := j.prepareScanNodeForDocsFetching() + + if err != nil { + return nil, err + } + + j.scan.initFetcher(immutable.None[string](), immutable.None[client.IndexDescription]()) + j.node.Spans(j.spansFromDocIDs()) + + return j.collectDocs(len(j.resultSecondaryDocIDs)) +} + +func (j *secondaryDocsJoiner) join(docs []core.Doc) { + relPropIndex := j.node.DocumentMap().FirstIndexOfName(j.fieldDef.Name) + + for i := range docs { + docs[i].Fields[relPropIndex] = j.primaryDoc + } + + j.resultSecondaryDocs = docs +} + +func fetchDocsWithFieldValue(plan planNode, fieldName string, val any) ([]core.Doc, error) { + propIndex := plan.DocumentMap().FirstIndexOfName(fieldName) + setSubTypeFilterToScanNode(plan, propIndex, val) + + if err := plan.Init(); err != nil { + return nil, NewErrSubTypeInit(err) + } + + var docs []core.Doc + for { + next, err := plan.Next() + if err != nil { + return nil, err + } + if !next { + break + } + + docs = append(docs, plan.Value()) + } + + return docs, nil +} + +func (j *secondaryDocsJoiner) fetchSecondaryFullScan() error { secondDocs, err := fetchDocsWithFieldValue( - join.dir.secondNode, + j.node, // As the join is from the secondary field, we know that [join.dir.secondaryField] must have a value // otherwise the user would not have been able to request it. - join.dir.subRelField+request.RelatedObjectID, - primaryDocID, + j.relIDFieldDef.Name, + j.primaryDoc.GetID(), ) if err != nil { - return false, err + return err } - if join.dir.secondNode == join.root { + if j.node == j.parentPlan { if len(secondDocs) == 0 { - return false, nil + return nil } for i := range secondDocs { - secondDocs[i].Fields[join.subSelect.Index] = join.subType.Value() + //secondDocs[i].Fields[join.subSelect.Index] = j.subType.Value() + i = i } - join.docsToYield = append(join.docsToYield, secondDocs...) - return true, nil + j.resultSecondaryDocs = append(j.resultSecondaryDocs, secondDocs...) + return nil } else { - join.addSecondaryDocsToRootPrimaryDoc(secondDocs) - join.docsToYield = append(join.docsToYield, join.root.Value()) + //j.addSecondaryDocsToRootPrimaryDoc(secondDocs) + j.resultSecondaryDocs = append(j.resultSecondaryDocs, j.parentPlan.Value()) } - return true, nil + return nil +} + +func (j *secondaryDocsJoiner) fetchDocsWithFieldValue() error { + propIndex := j.node.DocumentMap().FirstIndexOfName(j.relIDFieldDef.Name) + setSubTypeFilterToScanNode(j.node, propIndex, j.primaryDoc.GetID()) + + if err := j.node.Init(); err != nil { + return NewErrSubTypeInit(err) + } + + j.resultSecondaryDocs = make([]core.Doc, 0, len(j.resultSecondaryDocIDs)) + for { + hasValue, err := j.node.Next() + if err != nil { + return err + } + if !hasValue { + break + } + + j.resultSecondaryDocs = append(j.resultSecondaryDocs, j.node.Value()) + } + + return nil +} + +func (join *invertibleTypeJoin) fetchSecondaryDocsForPrimaryDoc(node planNode, primaryDoc core.Doc) (bool, error) { + relIDFieldDef, ok := join.parentCollection.Definition().GetFieldByName(join.childSelect.Name + request.RelatedObjectID) + if !ok { + return false, client.NewErrFieldNotExist(join.childSelect.Name + request.RelatedObjectID) + } + + secJoiner := newSecondaryDocsJoiner(node, primaryDoc, join.parentsRelFieldDef, relIDFieldDef) + secJoiner.parentPlan = join.parentPlan + secJoiner.childPlan = join.childPlan + err := secJoiner.fetchSecondaryDocsForPrimaryDoc() + join.docsToYield = append(join.docsToYield, secJoiner.resultSecondaryDocs...) + return len(secJoiner.resultSecondaryDocs) > 0, err } func (join *invertibleTypeJoin) Next() (bool, error) { @@ -615,25 +842,81 @@ func (join *invertibleTypeJoin) Next() (bool, error) { return false, err } - if join.isSecondary { + if join.isChildSecondary { firstDoc := join.dir.firstNode.Value() - return join.fetchSecondaryDocsForPrimaryDoc(firstDoc.GetID()) + return join.fetchSecondaryDocsForPrimaryDoc(join.dir.secondNode, firstDoc) } else { - hasDoc, err := fetchPrimaryDoc(join.dir.secondNode, join.dir.firstNode, join.dir.topRelField) + hasDoc, err := fetchPrimaryDoc(join.dir.secondNode, join.dir.firstNode, join.dir.firstNodesRelFieldName) if err != nil { return false, err } if hasDoc { - join.root.Value().Fields[join.subSelect.Index] = join.subType.Value() + //join.root.Value().Fields[join.subSelect.Index] = join.fetchIndexedRelatedSecondaryDocs(join.dir.firstNode, join.dir.topRelField) + join.parentPlan.Value().Fields[join.childSelect.Index] = join.childPlan.Value() } - join.docsToYield = append(join.docsToYield, join.root.Value()) + join.docsToYield = append(join.docsToYield, join.parentPlan.Value()) } return true, nil } +func (join *invertibleTypeJoin) fetchIndexedRelatedSecondaryDocs(node planNode, field string) []any { + docIDStr := getPrimaryDocIDFromSecondaryDoc(node, field) + if docIDStr == "" { + return nil + } + var secondaryIDs []string + var spans []core.Span + scan := getScanNode(node) + + scan.initFetcher(immutable.None[string](), findIndexByFieldName(scan.col, field)) + + colRootID := scan.col.Description().RootID + + for _, secondaryID := range secondaryIDs { + dsKey := core.DataStoreKey{CollectionRootID: colRootID, DocID: secondaryID} + spans = append(spans, core.NewSpan(dsKey, dsKey.PrefixEnd())) + } + node.Spans(core.NewSpans(spans...)) + + if err := node.Init(); err != nil { + //return false, NewErrSubTypeInit(err) + return nil + } + + secondaryDocs := make([]core.Doc, 0, len(secondaryIDs)) + + for { + hasValue, err := node.Next() + + if err != nil { + //return false, err + return nil + } + + if !hasValue { + break + } + + secondaryDocs = append(secondaryDocs, node.Value()) + } + + if err := node.Close(); err != nil { + //return false, NewErrSubTypeInit(err) + return nil + } + + //return true, nil + //subTypeFieldDesc, ok := parent.collection.Definition().GetFieldByName(subType.Name) + join.parentPlan.Value().Fields[join.childSelect.Index] = secondaryDocs + if join.secondaryFieldIndex.HasValue() { + join.parentPlan.Value().Fields[join.secondaryFieldIndex.Value()] = secondaryIDs + } + return nil +} + func (join *invertibleTypeJoin) Value() core.Doc { if len(join.docsToYield) == 0 { return core.Doc{} @@ -648,16 +931,12 @@ func (join *invertibleTypeJoin) invertJoinDirectionWithIndex( if !join.dir.isInvertable { return nil } - if join.subSelectFieldDef.Kind.IsArray() { - // invertibleTypeJoin does not support inverting one-many relations atm - return nil - } - rootName := join.dir.subRelField + childsRelFieldName := join.dir.secondNodesRelFieldName if join.dir.isInverted { - rootName = join.dir.topRelField + childsRelFieldName = join.dir.firstNodesRelFieldName } - subScan := getScanNode(join.subType) - subScan.tryAddField(rootName + request.RelatedObjectID) + subScan := getScanNode(join.childPlan) + subScan.tryAddField(childsRelFieldName + request.RelatedObjectID) subScan.filter = fieldFilter subScan.initFetcher(immutable.Option[string]{}, immutable.Some(index)) From 737504b8659d720a687286baed4acbd5f6181562 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Wed, 29 May 2024 09:50:55 +0200 Subject: [PATCH 03/18] tmp2 --- internal/planner/explain.go | 16 +- internal/planner/planner.go | 6 +- internal/planner/type_join.go | 546 +++++++++++++++++++--------------- 3 files changed, 309 insertions(+), 259 deletions(-) diff --git a/internal/planner/explain.go b/internal/planner/explain.go index e68c348787..f6d3f57209 100644 --- a/internal/planner/explain.go +++ b/internal/planner/explain.go @@ -92,8 +92,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) { var explainGraphBuilder = map[string]any{} // If root is not the last child then keep walking and explaining the root graph. - if node.parentPlan != nil { - indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.parentPlan) + if node.parentSide.plan != nil { + indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.parentSide.plan) if err != nil { return nil, err } @@ -101,8 +101,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) { explainGraphBuilder[joinRootLabel] = indexJoinRootExplainGraph } - if node.childPlan != nil { - indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.childPlan) + if node.childSide.plan != nil { + indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.childSide.plan) if err != nil { return nil, err } @@ -117,8 +117,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) { var explainGraphBuilder = map[string]any{} // If root is not the last child then keep walking and explaining the root graph. - if node.parentPlan != nil { - indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.parentPlan) + if node.parentSide.plan != nil { + indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.parentSide.plan) if err != nil { return nil, err } @@ -128,8 +128,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) { explainGraphBuilder[joinRootLabel] = nil } - if node.childPlan != nil { - indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.childPlan) + if node.childSide.plan != nil { + indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.childSide.plan) if err != nil { return nil, err } diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 05e95e27e3..782992668c 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -357,7 +357,7 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan parentPlan.selectNode.filter.Conditions, node.documentMapping, ) - slct := node.childPlan.(*selectTopNode).selectNode + slct := node.childSide.plan.(*selectTopNode).selectNode desc := slct.collection.Description() for subFieldName, subFieldInd := range filteredSubFields { indexes := desc.GetIndexesOnField(subFieldName) @@ -383,7 +383,7 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan // expandTypeJoin does a plan graph expansion and other optimizations on invertibleTypeJoin. func (p *Planner) expandTypeJoin(node *invertibleTypeJoin, parentPlan *selectTopNode) error { if parentPlan.selectNode.filter == nil { - return p.expandPlan(node.childPlan, parentPlan) + return p.expandPlan(node.childSide.plan, parentPlan) } err := p.tryOptimizeJoinDirection(node, parentPlan) @@ -391,7 +391,7 @@ func (p *Planner) expandTypeJoin(node *invertibleTypeJoin, parentPlan *selectTop return err } - return p.expandPlan(node.childPlan, parentPlan) + return p.expandPlan(node.childSide.plan, parentPlan) } func (p *Planner) expandGroupNodePlan(topNodeSelect *selectTopNode) error { diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index 8a7c9373dd..d3983c7aa0 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -152,7 +152,7 @@ func (n *typeIndexJoin) simpleExplain() (map[string]any, error) { simpleExplainMap[joinRootLabel] = immutable.Some(j.getRootTypeName()) simpleExplainMap[joinSubTypeNameLabel] = j.getSubTypeName() - subTypeExplainGraph, err := buildSimpleExplainGraph(j.childPlan) + subTypeExplainGraph, err := buildSimpleExplainGraph(j.childSide.plan) if err != nil { return err } @@ -166,7 +166,7 @@ func (n *typeIndexJoin) simpleExplain() (map[string]any, error) { switch joinType := n.joinPlan.(type) { case *typeJoinOne: // Add the direction attribute. - if joinType.isChildSecondary { + if joinType.parentSide.isPrimary() { simpleExplainMap[joinDirectionLabel] = joinDirectionSecondaryLabel } else { simpleExplainMap[joinDirectionLabel] = joinDirectionPrimaryLabel @@ -197,10 +197,10 @@ func (n *typeIndexJoin) Explain(explainType request.ExplainType) (map[string]any } var subScan *scanNode if joinMany, isJoinMany := n.joinPlan.(*typeJoinMany); isJoinMany { - subScan = getScanNode(joinMany.childPlan) + subScan = getScanNode(joinMany.childSide.plan) } if joinOne, isJoinOne := n.joinPlan.(*typeJoinOne); isJoinOne { - subScan = getScanNode(joinOne.childPlan) + subScan = getScanNode(joinOne.childSide.plan) } if subScan != nil { subScanExplain, err := subScan.Explain(explainType) @@ -267,27 +267,40 @@ func (p *Planner) makeTypeJoinOne( ) } - dir := joinDirection{ - firstNode: sourcePlan, - secondNode: subSelectPlan, - firstNodesRelFieldName: parentsRelFieldDef.Name, - secondNodesRelFieldName: childsRelFieldDesc.Name, - isInvertable: true, + parentSide := joinSide{ + plan: sourcePlan, + relFieldDef: parentsRelFieldDef, + relFieldMapIndex: immutable.Some(subSelect.Index), + col: parent.collection, + requestedFields: getRequestedFields(sourcePlan), + isFirst: true, + isParent: true, + } + + childSide := joinSide{ + plan: subSelectPlan, + relFieldDef: childsRelFieldDef, + col: subCol, + requestedFields: getRequestedFields(subSelectPlan), + isFirst: false, + isParent: false, + } + + subDocMap := subSelectPlan.DocumentMap() + ind := subDocMap.IndexesByName[childsRelFieldDef.Name+request.RelatedObjectID] + if len(ind) > 0 { + childSide.relIDFieldMapIndex = immutable.Some(ind[0]) } return &typeJoinOne{ invertibleTypeJoin: invertibleTypeJoin{ docMapper: docMapper{parent.documentMapping}, - parentPlan: sourcePlan, - childPlan: subSelectPlan, + parentSide: parentSide, + childSide: childSide, childSelect: subSelect, - parentsRelFieldDef: parentsRelFieldDef, - childsRelFieldDef: childsRelFieldDef, - parentCollection: parent.collection, - isChildSecondary: !parentsRelFieldDef.IsPrimaryRelation, secondaryFieldIndex: secondaryFieldIndex, secondaryFetchLimit: 1, - dir: dir, + //dir: dir, }, }, nil } @@ -371,63 +384,131 @@ func (p *Planner) makeTypeJoinMany( return nil, client.NewErrFieldNotExist(subSelect.Name) } - dir := joinDirection{ - firstNode: sourcePlan, - secondNode: subSelectPlan, - firstNodesRelFieldName: parentsRelFieldDef.Name, + parentSide := joinSide{ + plan: sourcePlan, + relFieldDef: parentsRelFieldDef, + relFieldMapIndex: immutable.Some(subSelect.Index), + col: parent.collection, + requestedFields: getRequestedFields(sourcePlan), + isFirst: true, + isParent: true, } - if parentsRelFieldDef.RelationName != "" { - rootField, ok := subCol.Description().GetFieldByRelation( - parentsRelFieldDef.RelationName, - parent.collection.Name().Value(), - parentsRelFieldDef.Name, - ) - if ok { - dir.secondNodesRelFieldName = rootField.Name - dir.isInvertable = true - } + childSide := joinSide{ + plan: subSelectPlan, + relFieldDef: childsRelFieldDef, + col: subCol, + requestedFields: getRequestedFields(subSelectPlan), + isFirst: false, + isParent: false, + } + + subDocMap := subSelectPlan.DocumentMap() + ind := subDocMap.IndexesByName[childsRelFieldDef.Name+request.RelatedObjectID] + if len(ind) > 0 { + childSide.relIDFieldMapIndex = immutable.Some(ind[0]) } return &typeJoinMany{ invertibleTypeJoin: invertibleTypeJoin{ docMapper: docMapper{parent.documentMapping}, - parentPlan: sourcePlan, - childPlan: subSelectPlan, + parentSide: parentSide, + childSide: childSide, childSelect: subSelect, - parentsRelFieldDef: parentsRelFieldDef, - childsRelFieldDef: childsRelFieldDef, - parentCollection: parent.collection, - isChildSecondary: true, secondaryFetchLimit: 0, - dir: dir, + //dir: dir, }, }, nil } -func (n *typeJoinMany) Kind() string { - return "typeJoinMany" +func getRequestedFields(sourcePlan planNode) []string { + scan := getScanNode(sourcePlan) + if scan == nil { + return nil + } + fields := make([]string, len(scan.fields)) + for i := range scan.fields { + fields[i] = scan.fields[i].Name + } + return fields } -func getPrimaryDocIDFromSecondaryDoc(subNode planNode, parentProp string) string { - subDoc := subNode.Value() - ind := subNode.DocumentMap().FirstIndexOfName(parentProp + request.RelatedObjectID) +type joinSide struct { + plan planNode + relFieldDef client.FieldDefinition + relFieldMapIndex immutable.Option[int] + relIDFieldMapIndex immutable.Option[int] + col client.Collection + requestedFields []string + isFirst bool + isParent bool +} + +func (s *joinSide) isFieldRequested(fieldName string) bool { + for i := range s.requestedFields { + if s.requestedFields[i] == fieldName { + return true + } + } + return false - docIDStr, _ := subDoc.Fields[ind].(string) - return docIDStr } -func fetchPrimaryDoc(node, subNode planNode, parentProp string) (bool, error) { - docIDStr := getPrimaryDocIDFromSecondaryDoc(subNode, parentProp) - if docIDStr == "" { - return false, nil +func (s *joinSide) isPrimary() bool { + return s.relFieldDef.IsPrimaryRelation +} + +func (join *invertibleTypeJoin) getFirstSide() *joinSide { + if join.parentSide.isFirst { + return &join.parentSide } + return &join.childSide +} +func (join *invertibleTypeJoin) getSecondSide() *joinSide { + if !join.parentSide.isFirst { + return &join.parentSide + } + return &join.childSide +} + +func (join *invertibleTypeJoin) getOtherSide(side *joinSide) *joinSide { + if side == &join.parentSide { + return &join.childSide + } + return &join.parentSide +} + +func (join *invertibleTypeJoin) getPrimarySide() *joinSide { + if join.parentSide.isPrimary() { + return &join.parentSide + } + return &join.childSide +} + +func (join *invertibleTypeJoin) getSecondarySide() *joinSide { + if !join.parentSide.isPrimary() { + return &join.parentSide + } + return &join.childSide +} + +func (n *typeJoinMany) Kind() string { + return "typeJoinMany" +} + +func getForeignKey(node planNode, relFieldName string) string { + ind := node.DocumentMap().FirstIndexOfName(relFieldName + request.RelatedObjectID) + docIDStr, _ := node.Value().Fields[ind].(string) + return docIDStr +} + +func fetchDocWithID(node planNode, docID string) (bool, error) { scan := getScanNode(node) if scan == nil { return false, nil } - dsKey := base.MakeDataStoreKeyWithCollectionAndDocID(scan.col.Description(), docIDStr) + dsKey := base.MakeDataStoreKeyWithCollectionAndDocID(scan.col.Description(), docID) spans := core.NewSpans(core.NewSpan(dsKey, dsKey.PrefixEnd())) @@ -446,104 +527,68 @@ func fetchPrimaryDoc(node, subNode planNode, parentProp string) (bool, error) { return true, nil } -// joinDirection is a struct that holds the two nodes that are that are executed one after another -// depending on the direction of the join. -type joinDirection struct { - // firstNode is the node that is executed first (usually an indexed collection). - firstNode planNode - // secondNode is the node that is executed second. - secondNode planNode - // secondNodesRelFieldName is a field name of a secondary doc that refers to the primary docID (like author_id). - secondNodesRelFieldName string - // firstNodesRelFieldName is a field name of the primary doc that refers to the secondary docID (like author_id). - firstNodesRelFieldName string - // isInvertable indicates if the join can be inverted. - isInvertable bool - // isInverted indicates if the join direction is inverted. - isInverted bool -} - -func (dir *joinDirection) invert() { - dir.isInverted = !dir.isInverted - dir.firstNode, dir.secondNode = dir.secondNode, dir.firstNode - dir.secondNodesRelFieldName, dir.firstNodesRelFieldName = dir.firstNodesRelFieldName, dir.secondNodesRelFieldName -} - type invertibleTypeJoin struct { docMapper - parentPlan planNode - childPlan planNode + childSelect *mapper.Select - childSelect *mapper.Select - parentsRelFieldDef client.FieldDefinition - childsRelFieldDef client.FieldDefinition - parentCollection client.Collection + parentSide joinSide + childSide joinSide - isChildSecondary bool secondaryFieldIndex immutable.Option[int] secondaryFetchLimit uint // docsToYield contains documents read and ready to be yielded by this node. - docsToYield []core.Doc + docsToYield []core.Doc + encounteredDocIDs []string - dir joinDirection + //dir joinDirection } func (join *invertibleTypeJoin) getRootTypeName() string { - if join.dir.isInverted { - return join.dir.firstNodesRelFieldName - } - return join.dir.secondNodesRelFieldName + return join.getSecondSide().relFieldDef.Name } func (join *invertibleTypeJoin) getSubTypeName() string { - if join.dir.isInverted { - return join.dir.secondNodesRelFieldName - } - return join.dir.firstNodesRelFieldName + return join.getFirstSide().relFieldDef.Name } func (join *invertibleTypeJoin) replaceRoot(node planNode) { - join.parentPlan = node - if join.dir.isInverted { - join.dir.secondNode = node - } else { - join.dir.firstNode = node - } + join.parentSide.plan = node + join.getFirstSide().plan = node } func (join *invertibleTypeJoin) Init() error { - if err := join.childPlan.Init(); err != nil { + if err := join.childSide.plan.Init(); err != nil { return err } - return join.parentPlan.Init() + return join.parentSide.plan.Init() } func (join *invertibleTypeJoin) Start() error { - if err := join.childPlan.Start(); err != nil { + if err := join.childSide.plan.Start(); err != nil { return err } - return join.parentPlan.Start() + return join.parentSide.plan.Start() } func (join *invertibleTypeJoin) Close() error { - if err := join.parentPlan.Close(); err != nil { + if err := join.parentSide.plan.Close(); err != nil { return err } - return join.childPlan.Close() + return join.childSide.plan.Close() } func (join *invertibleTypeJoin) Spans(spans core.Spans) { - join.parentPlan.Spans(spans) + join.parentSide.plan.Spans(spans) } -func (join *invertibleTypeJoin) Source() planNode { return join.parentPlan } +func (join *invertibleTypeJoin) Source() planNode { return join.parentSide.plan } -func (tj *invertibleTypeJoin) invert() { - tj.dir.invert() - tj.isChildSecondary = !tj.isChildSecondary +func (join *invertibleTypeJoin) invert() { + join.childSide.isFirst = join.parentSide.isFirst + join.parentSide.isFirst = !join.parentSide.isFirst } // addSecondaryDocsToRootPrimaryDoc adds the secondary docs to the root primary doc. @@ -565,81 +610,106 @@ func (join *invertibleTypeJoin) addSecondaryDocsToRootPrimaryDoc(secondDocs []co } secondaryIDResult = secondDocIDs } - join.parentPlan.Value().Fields[join.childSelect.Index] = secondaryResult + join.parentSide.plan.Value().Fields[join.childSelect.Index] = secondaryResult if join.secondaryFieldIndex.HasValue() { - join.parentPlan.Value().Fields[join.secondaryFieldIndex.Value()] = secondaryIDResult + join.parentSide.plan.Value().Fields[join.secondaryFieldIndex.Value()] = secondaryIDResult } } type secondaryDocsJoiner struct { - node planNode - primaryDoc core.Doc - fieldDef client.FieldDefinition relIDFieldDef client.FieldDefinition - parentPlan planNode - childPlan planNode + primarySide *joinSide + secondarySide *joinSide - scan *scanNode + primaryScan *scanNode origScanFields []client.FieldDefinition - resultSecondaryDocIDs []string - resultSecondaryDocs []core.Doc + resultSecondaryDocs []core.Doc } func newSecondaryDocsJoiner( - node planNode, - primaryDoc core.Doc, - fieldDef client.FieldDefinition, - relIDFieldDef client.FieldDefinition, + primarySide, secondarySide *joinSide, ) secondaryDocsJoiner { j := secondaryDocsJoiner{ - node: node, - primaryDoc: primaryDoc, - fieldDef: fieldDef, - relIDFieldDef: relIDFieldDef, + primarySide: primarySide, + secondarySide: secondarySide, } return j } -func (j *secondaryDocsJoiner) fetchSecondaryDocsForPrimaryDoc() error { - err := j.fetchSecondaryDocIDs() +func (j *secondaryDocsJoiner) fetchPrimaryDocsReferencingSecondaryDoc() error { + relIDFieldDef, ok := j.primarySide.col.Definition().GetFieldByName( + j.primarySide.relFieldDef.Name + request.RelatedObjectID) + if !ok { + return client.NewErrFieldNotExist(j.primarySide.relFieldDef.Name + request.RelatedObjectID) + } - if err != nil { - return err + scan := getScanNode(j.primarySide.plan) + j.primaryScan = &scanNode{ + docMapper: scan.docMapper, + p: scan.p, + col: scan.col, + showDeleted: scan.showDeleted, + slct: scan.slct, + fields: scan.fields, } - docs, err := j.fetchSecondaryDocsByIDs() + j.relIDFieldDef = relIDFieldDef + + primaryDocs, err := j.fetchPrimaryDocs() if err != nil { return err } - j.join(docs) + j.join(primaryDocs) + + scan.execInfo.fetches.Add(j.primaryScan.execInfo.fetches) return nil } func (j *secondaryDocsJoiner) prepareScanNodeForDocIDsFetching() error { - j.scan = getScanNode(j.node) - if err := j.scan.fetcher.Close(); err != nil { - return NewErrSubTypeInit(err) + //if err := j.primaryScan.fetcher.Close(); err != nil { + //return NewErrSubTypeInit(err) + //} + j.primaryScan.filter = nil + j.origScanFields = j.primaryScan.fields + + found := false + for i := range j.primaryScan.fields { + if j.primaryScan.fields[i].Name == j.relIDFieldDef.Name { + found = true + break + } } - j.scan.filter = nil - j.origScanFields = j.scan.fields - j.scan.fields = []client.FieldDefinition{j.relIDFieldDef} - j.scan.spans = core.Spans{} + if !found { + j.primaryScan.fields = append(j.primaryScan.fields, j.relIDFieldDef) + } + j.primaryScan.spans = core.Spans{} return nil } +/*func (j *secondaryDocsJoiner) prepareScanNodeForDocIDsFetching() error { + //if err := j.primaryScan.fetcher.Close(); err != nil { + //return NewErrSubTypeInit(err) + //} + j.primaryScan.filter = nil + j.origScanFields = j.primaryScan.fields + j.primaryScan.fields = []client.FieldDefinition{j.relIDFieldDef} + j.primaryScan.spans = core.Spans{} + return nil +}*/ + func (j *secondaryDocsJoiner) collectDocs(numDocs int) ([]core.Doc, error) { - if err := j.node.Init(); err != nil { + if err := j.primaryScan.Init(); err != nil { return nil, NewErrSubTypeInit(err) } docs := make([]core.Doc, 0, numDocs) for { - hasValue, err := j.node.Next() + hasValue, err := j.primaryScan.Next() if err != nil { return nil, err @@ -649,94 +719,57 @@ func (j *secondaryDocsJoiner) collectDocs(numDocs int) ([]core.Doc, error) { break } - docs = append(docs, j.node.Value()) + docs = append(docs, j.primaryScan.Value()) } - if err := j.node.Close(); err != nil { + if err := j.primaryScan.Close(); err != nil { return nil, NewErrSubTypeInit(err) } return docs, nil } -func (j *secondaryDocsJoiner) fetchSecondaryDocIDs() error { +func (j *secondaryDocsJoiner) fetchPrimaryDocs() ([]core.Doc, error) { err := j.prepareScanNodeForDocIDsFetching() if err != nil { - return err + return nil, err } - relIDPropIndex := j.node.DocumentMap().FirstIndexOfName(j.relIDFieldDef.Name) - setSubTypeFilterToScanNode(j.node, relIDPropIndex, j.primaryDoc.GetID()) + relIDPropIndex := j.primarySide.plan.DocumentMap().FirstIndexOfName(j.relIDFieldDef.Name) + secondaryDoc := j.secondarySide.plan.Value() + setSubTypeFilterToScanNode(j.primaryScan, relIDPropIndex, secondaryDoc.GetID()) - indexOnRelation := findIndexByFieldName(j.scan.col, j.relIDFieldDef.Name) - j.scan.initFetcher(immutable.None[string](), indexOnRelation) + indexOnRelation := findIndexByFieldName(j.primaryScan.col, j.relIDFieldDef.Name) + j.primaryScan.initFetcher(immutable.None[string](), indexOnRelation) docs, err := j.collectDocs(0) if err != nil { - return err - } - - for i := range docs { - j.resultSecondaryDocIDs = append(j.resultSecondaryDocIDs, docs[i].Fields[core.DocIDFieldIndex].(string)) + return nil, err } - return nil -} - -func (j *secondaryDocsJoiner) spansFromDocIDs() core.Spans { - colRootID := j.scan.col.Description().RootID - - spans := make([]core.Span, 0, len(j.resultSecondaryDocIDs)) - for _, secondaryID := range j.resultSecondaryDocIDs { - dsKey := core.DataStoreKey{CollectionRootID: colRootID, DocID: secondaryID} - spans = append(spans, core.NewSpan(dsKey, dsKey.PrefixEnd())) - } - return core.NewSpans(spans...) + return docs, nil } -func (j *secondaryDocsJoiner) prepareScanNodeForDocsFetching() error { - if err := j.scan.fetcher.Close(); err != nil { - return NewErrSubTypeInit(err) - } +func (j *secondaryDocsJoiner) join(primaryDocs []core.Doc) { + if j.primarySide.relFieldMapIndex.HasValue() { + secondaryDoc := j.secondarySide.plan.Value() - j.scan.filter = nil - for i := range j.origScanFields { - if j.origScanFields[i].Name == j.relIDFieldDef.Name { - j.origScanFields = append(j.origScanFields[:i], j.origScanFields[i+1:]...) - break + if j.secondarySide.relFieldMapIndex.HasValue() { + secondaryDoc.Fields[j.secondarySide.relFieldMapIndex.Value()] = primaryDocs } - } - j.scan.fields = j.origScanFields - return nil -} - -func (j *secondaryDocsJoiner) fetchSecondaryDocsByIDs() ([]core.Doc, error) { - err := j.prepareScanNodeForDocsFetching() - - if err != nil { - return nil, err - } - j.scan.initFetcher(immutable.None[string](), immutable.None[client.IndexDescription]()) - j.node.Spans(j.spansFromDocIDs()) - - return j.collectDocs(len(j.resultSecondaryDocIDs)) -} - -func (j *secondaryDocsJoiner) join(docs []core.Doc) { - relPropIndex := j.node.DocumentMap().FirstIndexOfName(j.fieldDef.Name) - - for i := range docs { - docs[i].Fields[relPropIndex] = j.primaryDoc + for i := range primaryDocs { + primaryDocs[i].Fields[j.primarySide.relFieldMapIndex.Value()] = secondaryDoc + } } - j.resultSecondaryDocs = docs + j.resultSecondaryDocs = primaryDocs } func fetchDocsWithFieldValue(plan planNode, fieldName string, val any) ([]core.Doc, error) { propIndex := plan.DocumentMap().FirstIndexOfName(fieldName) - setSubTypeFilterToScanNode(plan, propIndex, val) + setSubTypeFilterToScanNode(getScanNode(plan), propIndex, val) if err := plan.Init(); err != nil { return nil, NewErrSubTypeInit(err) @@ -759,17 +792,18 @@ func fetchDocsWithFieldValue(plan planNode, fieldName string, val any) ([]core.D } func (j *secondaryDocsJoiner) fetchSecondaryFullScan() error { + primaryDoc := j.primarySide.plan.Value() secondDocs, err := fetchDocsWithFieldValue( - j.node, - // As the join is from the secondary field, we know that [join.dir.secondaryField] must have a value + j.primarySide.plan, + // At the join is from the secondary field, we know that [join.dir.secondaryField] must have a value // otherwise the user would not have been able to request it. j.relIDFieldDef.Name, - j.primaryDoc.GetID(), + primaryDoc.GetID(), ) if err != nil { return err } - if j.node == j.parentPlan { + if j.primarySide.isParent { if len(secondDocs) == 0 { return nil } @@ -781,22 +815,24 @@ func (j *secondaryDocsJoiner) fetchSecondaryFullScan() error { return nil } else { //j.addSecondaryDocsToRootPrimaryDoc(secondDocs) - j.resultSecondaryDocs = append(j.resultSecondaryDocs, j.parentPlan.Value()) + j.resultSecondaryDocs = append(j.resultSecondaryDocs, j.secondarySide.plan.Value()) } return nil } func (j *secondaryDocsJoiner) fetchDocsWithFieldValue() error { - propIndex := j.node.DocumentMap().FirstIndexOfName(j.relIDFieldDef.Name) - setSubTypeFilterToScanNode(j.node, propIndex, j.primaryDoc.GetID()) + node := j.primarySide.plan + propIndex := node.DocumentMap().FirstIndexOfName(j.relIDFieldDef.Name) + primaryDoc := node.Value() + setSubTypeFilterToScanNode(j.primaryScan, propIndex, primaryDoc.GetID()) - if err := j.node.Init(); err != nil { + if err := node.Init(); err != nil { return NewErrSubTypeInit(err) } - j.resultSecondaryDocs = make([]core.Doc, 0, len(j.resultSecondaryDocIDs)) + j.resultSecondaryDocs = []core.Doc{} for { - hasValue, err := j.node.Next() + hasValue, err := node.Next() if err != nil { return err } @@ -804,24 +840,16 @@ func (j *secondaryDocsJoiner) fetchDocsWithFieldValue() error { break } - j.resultSecondaryDocs = append(j.resultSecondaryDocs, j.node.Value()) + j.resultSecondaryDocs = append(j.resultSecondaryDocs, node.Value()) } return nil } -func (join *invertibleTypeJoin) fetchSecondaryDocsForPrimaryDoc(node planNode, primaryDoc core.Doc) (bool, error) { - relIDFieldDef, ok := join.parentCollection.Definition().GetFieldByName(join.childSelect.Name + request.RelatedObjectID) - if !ok { - return false, client.NewErrFieldNotExist(join.childSelect.Name + request.RelatedObjectID) - } - - secJoiner := newSecondaryDocsJoiner(node, primaryDoc, join.parentsRelFieldDef, relIDFieldDef) - secJoiner.parentPlan = join.parentPlan - secJoiner.childPlan = join.childPlan - err := secJoiner.fetchSecondaryDocsForPrimaryDoc() - join.docsToYield = append(join.docsToYield, secJoiner.resultSecondaryDocs...) - return len(secJoiner.resultSecondaryDocs) > 0, err +func (join *invertibleTypeJoin) fetchPrimaryDocsReferencingSecondaryDoc(node planNode, secondaryDoc core.Doc) ([]core.Doc, error) { + secJoiner := newSecondaryDocsJoiner(join.getPrimarySide(), join.getSecondarySide()) + err := secJoiner.fetchPrimaryDocsReferencingSecondaryDoc() + return secJoiner.resultSecondaryDocs, err } func (join *invertibleTypeJoin) Next() (bool, error) { @@ -836,34 +864,63 @@ func (join *invertibleTypeJoin) Next() (bool, error) { } } - hasFirstValue, err := join.dir.firstNode.Next() + firstSide := join.getFirstSide() + hasFirstValue, err := firstSide.plan.Next() if err != nil || !hasFirstValue { return false, err } - if join.isChildSecondary { - firstDoc := join.dir.firstNode.Value() - return join.fetchSecondaryDocsForPrimaryDoc(join.dir.secondNode, firstDoc) - } else { - hasDoc, err := fetchPrimaryDoc(join.dir.secondNode, join.dir.firstNode, join.dir.firstNodesRelFieldName) + secondSide := join.getSecondSide() + + if firstSide.isPrimary() { + secondaryDocID := getForeignKey(firstSide.plan, firstSide.relFieldDef.Name) + + if !firstSide.isParent { + for i := range join.encounteredDocIDs { + if join.encounteredDocIDs[i] == secondaryDocID { + return join.Next() + } + } + join.encounteredDocIDs = append(join.encounteredDocIDs, secondaryDocID) + } + + // check if there can ever be false (a.k.a. hasDoc = false) + _, err := fetchDocWithID(secondSide.plan, secondaryDocID) if err != nil { return false, err } - if hasDoc { - //join.root.Value().Fields[join.subSelect.Index] = join.fetchIndexedRelatedSecondaryDocs(join.dir.firstNode, join.dir.topRelField) - join.parentPlan.Value().Fields[join.childSelect.Index] = join.childPlan.Value() - } + secondaryDoc := secondSide.plan.Value() - join.docsToYield = append(join.docsToYield, join.parentPlan.Value()) + //if join.parentSide.relFieldDef.Kind.IsArray() && join.parentSide.isFieldRequested(join.parentSide.relFieldDef.Name) { + if join.parentSide.relFieldDef.Kind.IsArray() { + primaryDocs, err := join.fetchPrimaryDocsReferencingSecondaryDoc(firstSide.plan, secondaryDoc) + if err != nil { + return false, err + } + secondaryDoc.Fields[join.childSelect.Index] = primaryDocs + + join.docsToYield = append(join.docsToYield, secondaryDoc) + } else { + parentDoc := join.parentSide.plan.Value() + parentDoc.Fields[join.parentSide.relFieldMapIndex.Value()] = join.childSide.plan.Value() + join.docsToYield = append(join.docsToYield, parentDoc) + } + } else { + secondaryFirstDoc := firstSide.plan.Value() + primaryDocs, err := join.fetchPrimaryDocsReferencingSecondaryDoc(secondSide.plan, secondaryFirstDoc) + if err != nil { + return false, err + } + join.docsToYield = append(join.docsToYield, primaryDocs...) } return true, nil } func (join *invertibleTypeJoin) fetchIndexedRelatedSecondaryDocs(node planNode, field string) []any { - docIDStr := getPrimaryDocIDFromSecondaryDoc(node, field) + docIDStr := getForeignKey(node, field) if docIDStr == "" { return nil } @@ -910,9 +967,9 @@ func (join *invertibleTypeJoin) fetchIndexedRelatedSecondaryDocs(node planNode, //return true, nil //subTypeFieldDesc, ok := parent.collection.Definition().GetFieldByName(subType.Name) - join.parentPlan.Value().Fields[join.childSelect.Index] = secondaryDocs + join.parentSide.plan.Value().Fields[join.childSelect.Index] = secondaryDocs if join.secondaryFieldIndex.HasValue() { - join.parentPlan.Value().Fields[join.secondaryFieldIndex.Value()] = secondaryIDs + join.parentSide.plan.Value().Fields[join.secondaryFieldIndex.Value()] = secondaryIDs } return nil } @@ -928,25 +985,18 @@ func (join *invertibleTypeJoin) invertJoinDirectionWithIndex( fieldFilter *mapper.Filter, index client.IndexDescription, ) error { - if !join.dir.isInvertable { - return nil - } - childsRelFieldName := join.dir.secondNodesRelFieldName - if join.dir.isInverted { - childsRelFieldName = join.dir.firstNodesRelFieldName - } - subScan := getScanNode(join.childPlan) - subScan.tryAddField(childsRelFieldName + request.RelatedObjectID) - subScan.filter = fieldFilter - subScan.initFetcher(immutable.Option[string]{}, immutable.Some(index)) + p := join.childSide.plan + s := getScanNode(p) + s.tryAddField(join.childSide.relFieldDef.Name + request.RelatedObjectID) + s.filter = fieldFilter + s.initFetcher(immutable.Option[string]{}, immutable.Some(index)) join.invert() return nil } -func setSubTypeFilterToScanNode(plan planNode, propIndex int, val any) { - scan := getScanNode(plan) +func setSubTypeFilterToScanNode(scan *scanNode, propIndex int, val any) { if scan == nil { return } From f62156cec359269dc4d232b5168ce319429f7649 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 3 Jun 2024 09:04:52 +0200 Subject: [PATCH 04/18] Fix issues --- internal/planner/mapper/mapper.go | 34 +++--- internal/planner/type_join.go | 191 +++++++++++++++++------------- 2 files changed, 124 insertions(+), 101 deletions(-) diff --git a/internal/planner/mapper/mapper.go b/internal/planner/mapper/mapper.go index 07ec0db8e6..c27b11108d 100644 --- a/internal/planner/mapper/mapper.go +++ b/internal/planner/mapper/mapper.go @@ -413,25 +413,23 @@ func resolveAggregates( childMapping = childMapping.CloneWithoutRender() mapping.SetChildAt(index, childMapping) - if !childIsMapped { - filterDependencies, err := resolveFilterDependencies( - ctx, - store, - rootSelectType, - childCollectionName, - target.filter, - mapping.ChildMappings[index], - childFields, - ) - if err != nil { - return nil, err - } - childFields = append(childFields, filterDependencies...) - - // If the child was not mapped, the filter will not have been converted yet - // so we must do that now. - convertedFilter = ToFilter(target.filter.Value(), mapping.ChildMappings[index]) + filterDependencies, err := resolveFilterDependencies( + ctx, + store, + rootSelectType, + childCollectionName, + target.filter, + mapping.ChildMappings[index], + childFields, + ) + if err != nil { + return nil, err } + childFields = append(childFields, filterDependencies...) + + // If the child was not mapped, the filter will not have been converted yet + // so we must do that now. + convertedFilter = ToFilter(target.filter.Value(), mapping.ChildMappings[index]) dummyJoin := &Select{ Targetable: Targetable{ diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index d3983c7aa0..70086dd3fb 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -259,6 +259,7 @@ func (p *Planner) makeTypeJoinOne( return nil, client.NewErrFieldNotExist(subSelect.Name) } + // TODO: remove this block? var secondaryFieldIndex immutable.Option[int] if !parentsRelFieldDef.IsPrimaryRelation { idFieldName := parentsRelFieldDef.Name @@ -277,6 +278,11 @@ func (p *Planner) makeTypeJoinOne( isParent: true, } + ind := parent.documentMapping.IndexesByName[parentsRelFieldDef.Name+request.RelatedObjectID] + if len(ind) > 0 { + parentSide.relIDFieldMapIndex = immutable.Some(ind[0]) + } + childSide := joinSide{ plan: subSelectPlan, relFieldDef: childsRelFieldDef, @@ -286,8 +292,7 @@ func (p *Planner) makeTypeJoinOne( isParent: false, } - subDocMap := subSelectPlan.DocumentMap() - ind := subDocMap.IndexesByName[childsRelFieldDef.Name+request.RelatedObjectID] + ind = subSelectPlan.DocumentMap().IndexesByName[childsRelFieldDef.Name+request.RelatedObjectID] if len(ind) > 0 { childSide.relIDFieldMapIndex = immutable.Some(ind[0]) } @@ -416,7 +421,6 @@ func (p *Planner) makeTypeJoinMany( childSide: childSide, childSelect: subSelect, secondaryFetchLimit: 0, - //dir: dir, }, }, nil } @@ -472,13 +476,6 @@ func (join *invertibleTypeJoin) getSecondSide() *joinSide { return &join.childSide } -func (join *invertibleTypeJoin) getOtherSide(side *joinSide) *joinSide { - if side == &join.parentSide { - return &join.childSide - } - return &join.parentSide -} - func (join *invertibleTypeJoin) getPrimarySide() *joinSide { if join.parentSide.isPrimary() { return &join.parentSide @@ -614,30 +611,33 @@ func (join *invertibleTypeJoin) addSecondaryDocsToRootPrimaryDoc(secondDocs []co if join.secondaryFieldIndex.HasValue() { join.parentSide.plan.Value().Fields[join.secondaryFieldIndex.Value()] = secondaryIDResult } + //if join.parentSide.relIDFieldMapIndex.HasValue() { + //join.parentSide.plan.Value().Fields[join.parentSide.relIDFieldMapIndex.Value()] = secondaryIDResult + //} } -type secondaryDocsJoiner struct { +type docsJoiner struct { relIDFieldDef client.FieldDefinition primarySide *joinSide secondarySide *joinSide - primaryScan *scanNode - origScanFields []client.FieldDefinition + primaryScan *scanNode - resultSecondaryDocs []core.Doc + resultPrimaryDocs []core.Doc + resultSecondaryDoc core.Doc } func newSecondaryDocsJoiner( primarySide, secondarySide *joinSide, -) secondaryDocsJoiner { - j := secondaryDocsJoiner{ +) docsJoiner { + j := docsJoiner{ primarySide: primarySide, secondarySide: secondarySide, } return j } -func (j *secondaryDocsJoiner) fetchPrimaryDocsReferencingSecondaryDoc() error { +func (j *docsJoiner) fetchPrimaryDocsReferencingSecondaryDoc() error { relIDFieldDef, ok := j.primarySide.col.Definition().GetFieldByName( j.primarySide.relFieldDef.Name + request.RelatedObjectID) if !ok { @@ -645,14 +645,7 @@ func (j *secondaryDocsJoiner) fetchPrimaryDocsReferencingSecondaryDoc() error { } scan := getScanNode(j.primarySide.plan) - j.primaryScan = &scanNode{ - docMapper: scan.docMapper, - p: scan.p, - col: scan.col, - showDeleted: scan.showDeleted, - slct: scan.slct, - fields: scan.fields, - } + j.primaryScan = scan j.relIDFieldDef = relIDFieldDef @@ -669,13 +662,7 @@ func (j *secondaryDocsJoiner) fetchPrimaryDocsReferencingSecondaryDoc() error { return nil } -func (j *secondaryDocsJoiner) prepareScanNodeForDocIDsFetching() error { - //if err := j.primaryScan.fetcher.Close(); err != nil { - //return NewErrSubTypeInit(err) - //} - j.primaryScan.filter = nil - j.origScanFields = j.primaryScan.fields - +func (j *docsJoiner) addIDFieldToScanner() { found := false for i := range j.primaryScan.fields { if j.primaryScan.fields[i].Name == j.relIDFieldDef.Name { @@ -686,64 +673,53 @@ func (j *secondaryDocsJoiner) prepareScanNodeForDocIDsFetching() error { if !found { j.primaryScan.fields = append(j.primaryScan.fields, j.relIDFieldDef) } - j.primaryScan.spans = core.Spans{} - return nil } -/*func (j *secondaryDocsJoiner) prepareScanNodeForDocIDsFetching() error { - //if err := j.primaryScan.fetcher.Close(); err != nil { - //return NewErrSubTypeInit(err) - //} - j.primaryScan.filter = nil - j.origScanFields = j.primaryScan.fields - j.primaryScan.fields = []client.FieldDefinition{j.relIDFieldDef} - j.primaryScan.spans = core.Spans{} - return nil -}*/ - -func (j *secondaryDocsJoiner) collectDocs(numDocs int) ([]core.Doc, error) { - if err := j.primaryScan.Init(); err != nil { +func (j *docsJoiner) collectDocs(numDocs int) ([]core.Doc, error) { + p := j.primarySide.plan + if err := p.Init(); err != nil { return nil, NewErrSubTypeInit(err) } docs := make([]core.Doc, 0, numDocs) for { - hasValue, err := j.primaryScan.Next() + hasValue, err := p.Next() if err != nil { - return nil, err + return nil, p.Close() } if !hasValue { break } - docs = append(docs, j.primaryScan.Value()) - } - - if err := j.primaryScan.Close(); err != nil { - return nil, NewErrSubTypeInit(err) + docs = append(docs, p.Value()) } return docs, nil } -func (j *secondaryDocsJoiner) fetchPrimaryDocs() ([]core.Doc, error) { - err := j.prepareScanNodeForDocIDsFetching() - if err != nil { - return nil, err - } +func (j *docsJoiner) fetchPrimaryDocs() ([]core.Doc, error) { + j.addIDFieldToScanner() - relIDPropIndex := j.primarySide.plan.DocumentMap().FirstIndexOfName(j.relIDFieldDef.Name) secondaryDoc := j.secondarySide.plan.Value() - setSubTypeFilterToScanNode(j.primaryScan, relIDPropIndex, secondaryDoc.GetID()) + addFilterOnIDField(j.primaryScan, j.primarySide.relIDFieldMapIndex.Value(), secondaryDoc.GetID()) + + oldFetcher := j.primaryScan.fetcher + // TODO: check if spans are necessary to be saved + oldSpans := j.primaryScan.spans indexOnRelation := findIndexByFieldName(j.primaryScan.col, j.relIDFieldDef.Name) j.primaryScan.initFetcher(immutable.None[string](), indexOnRelation) docs, err := j.collectDocs(0) + j.primaryScan.fetcher.Close() + + j.primaryScan.spans = oldSpans + j.primaryScan.fetcher = oldFetcher + if err != nil { return nil, err } @@ -751,25 +727,52 @@ func (j *secondaryDocsJoiner) fetchPrimaryDocs() ([]core.Doc, error) { return docs, nil } -func (j *secondaryDocsJoiner) join(primaryDocs []core.Doc) { - if j.primarySide.relFieldMapIndex.HasValue() { - secondaryDoc := j.secondarySide.plan.Value() +func docsToDocIDs(docs []core.Doc) []string { + docIDs := make([]string, len(docs)) + for i, doc := range docs { + docIDs[i] = doc.GetID() + } + return docIDs +} + +func (j *docsJoiner) join(primaryDocs []core.Doc) { + secondaryDoc := j.secondarySide.plan.Value() - if j.secondarySide.relFieldMapIndex.HasValue() { + if j.secondarySide.relFieldMapIndex.HasValue() { + if j.secondarySide.relFieldDef.Kind.IsArray() { secondaryDoc.Fields[j.secondarySide.relFieldMapIndex.Value()] = primaryDocs + } else if len(primaryDocs) > 0 { + secondaryDoc.Fields[j.secondarySide.relFieldMapIndex.Value()] = primaryDocs[0] + } + } + + if j.secondarySide.relIDFieldMapIndex.HasValue() { + if j.secondarySide.relFieldDef.Kind.IsArray() { + secondaryDoc.Fields[j.secondarySide.relIDFieldMapIndex.Value()] = docsToDocIDs(primaryDocs) + } else if len(primaryDocs) > 0 { + secondaryDoc.Fields[j.secondarySide.relIDFieldMapIndex.Value()] = primaryDocs[0].GetID() } + } + if j.primarySide.relFieldMapIndex.HasValue() { for i := range primaryDocs { primaryDocs[i].Fields[j.primarySide.relFieldMapIndex.Value()] = secondaryDoc } } - j.resultSecondaryDocs = primaryDocs + if j.primarySide.relIDFieldMapIndex.HasValue() { + for i := range primaryDocs { + primaryDocs[i].Fields[j.primarySide.relIDFieldMapIndex.Value()] = secondaryDoc.GetID() + } + } + + j.resultPrimaryDocs = primaryDocs + j.resultSecondaryDoc = secondaryDoc } func fetchDocsWithFieldValue(plan planNode, fieldName string, val any) ([]core.Doc, error) { propIndex := plan.DocumentMap().FirstIndexOfName(fieldName) - setSubTypeFilterToScanNode(getScanNode(plan), propIndex, val) + addFilterOnIDField(getScanNode(plan), propIndex, val) if err := plan.Init(); err != nil { return nil, NewErrSubTypeInit(err) @@ -791,7 +794,7 @@ func fetchDocsWithFieldValue(plan planNode, fieldName string, val any) ([]core.D return docs, nil } -func (j *secondaryDocsJoiner) fetchSecondaryFullScan() error { +func (j *docsJoiner) fetchSecondaryFullScan() error { primaryDoc := j.primarySide.plan.Value() secondDocs, err := fetchDocsWithFieldValue( j.primarySide.plan, @@ -811,26 +814,26 @@ func (j *secondaryDocsJoiner) fetchSecondaryFullScan() error { //secondDocs[i].Fields[join.subSelect.Index] = j.subType.Value() i = i } - j.resultSecondaryDocs = append(j.resultSecondaryDocs, secondDocs...) + j.resultPrimaryDocs = append(j.resultPrimaryDocs, secondDocs...) return nil } else { //j.addSecondaryDocsToRootPrimaryDoc(secondDocs) - j.resultSecondaryDocs = append(j.resultSecondaryDocs, j.secondarySide.plan.Value()) + j.resultPrimaryDocs = append(j.resultPrimaryDocs, j.secondarySide.plan.Value()) } return nil } -func (j *secondaryDocsJoiner) fetchDocsWithFieldValue() error { +func (j *docsJoiner) fetchDocsWithFieldValue() error { node := j.primarySide.plan propIndex := node.DocumentMap().FirstIndexOfName(j.relIDFieldDef.Name) primaryDoc := node.Value() - setSubTypeFilterToScanNode(j.primaryScan, propIndex, primaryDoc.GetID()) + addFilterOnIDField(j.primaryScan, propIndex, primaryDoc.GetID()) if err := node.Init(); err != nil { return NewErrSubTypeInit(err) } - j.resultSecondaryDocs = []core.Doc{} + j.resultPrimaryDocs = []core.Doc{} for { hasValue, err := node.Next() if err != nil { @@ -840,16 +843,16 @@ func (j *secondaryDocsJoiner) fetchDocsWithFieldValue() error { break } - j.resultSecondaryDocs = append(j.resultSecondaryDocs, node.Value()) + j.resultPrimaryDocs = append(j.resultPrimaryDocs, node.Value()) } return nil } -func (join *invertibleTypeJoin) fetchPrimaryDocsReferencingSecondaryDoc(node planNode, secondaryDoc core.Doc) ([]core.Doc, error) { +func (join *invertibleTypeJoin) fetchPrimaryDocsReferencingSecondaryDoc() ([]core.Doc, core.Doc, error) { secJoiner := newSecondaryDocsJoiner(join.getPrimarySide(), join.getSecondarySide()) err := secJoiner.fetchPrimaryDocsReferencingSecondaryDoc() - return secJoiner.resultSecondaryDocs, err + return secJoiner.resultPrimaryDocs, secJoiner.resultSecondaryDoc, err } func (join *invertibleTypeJoin) Next() (bool, error) { @@ -875,6 +878,14 @@ func (join *invertibleTypeJoin) Next() (bool, error) { if firstSide.isPrimary() { secondaryDocID := getForeignKey(firstSide.plan, firstSide.relFieldDef.Name) + // TODO: add some tests with filter on nil relation + if secondaryDocID == "" { + if firstSide.isParent { + join.docsToYield = append(join.docsToYield, firstSide.plan.Value()) + return true, nil + } + return join.Next() + } if !firstSide.isParent { for i := range join.encounteredDocIDs { @@ -886,16 +897,24 @@ func (join *invertibleTypeJoin) Next() (bool, error) { } // check if there can ever be false (a.k.a. hasDoc = false) - _, err := fetchDocWithID(secondSide.plan, secondaryDocID) + hasDoc, err := fetchDocWithID(secondSide.plan, secondaryDocID) if err != nil { return false, err } - secondaryDoc := secondSide.plan.Value() + // TODO: add some tests that either return error if the doc is not found or return + // the related doc (without this one) and let it be filtered. + if !hasDoc { + if firstSide.isParent { + join.docsToYield = append(join.docsToYield, firstSide.plan.Value()) + return true, nil + } + return join.Next() + } //if join.parentSide.relFieldDef.Kind.IsArray() && join.parentSide.isFieldRequested(join.parentSide.relFieldDef.Name) { if join.parentSide.relFieldDef.Kind.IsArray() { - primaryDocs, err := join.fetchPrimaryDocsReferencingSecondaryDoc(firstSide.plan, secondaryDoc) + primaryDocs, secondaryDoc, err := join.fetchPrimaryDocsReferencingSecondaryDoc() if err != nil { return false, err } @@ -908,12 +927,15 @@ func (join *invertibleTypeJoin) Next() (bool, error) { join.docsToYield = append(join.docsToYield, parentDoc) } } else { - secondaryFirstDoc := firstSide.plan.Value() - primaryDocs, err := join.fetchPrimaryDocsReferencingSecondaryDoc(secondSide.plan, secondaryFirstDoc) + primaryDocs, secondaryDoc, err := join.fetchPrimaryDocsReferencingSecondaryDoc() if err != nil { return false, err } - join.docsToYield = append(join.docsToYield, primaryDocs...) + if join.parentSide.isPrimary() { + join.docsToYield = append(join.docsToYield, primaryDocs...) + } else { + join.docsToYield = append(join.docsToYield, secondaryDoc) + } } return true, nil @@ -971,6 +993,9 @@ func (join *invertibleTypeJoin) fetchIndexedRelatedSecondaryDocs(node planNode, if join.secondaryFieldIndex.HasValue() { join.parentSide.plan.Value().Fields[join.secondaryFieldIndex.Value()] = secondaryIDs } + //if join.parentSide.relIDFieldMapIndex.HasValue() { + //join.parentSide.plan.Value().Fields[join.parentSide.relIDFieldMapIndex.Value()] = secondaryIDs + //} return nil } @@ -996,7 +1021,7 @@ func (join *invertibleTypeJoin) invertJoinDirectionWithIndex( return nil } -func setSubTypeFilterToScanNode(scan *scanNode, propIndex int, val any) { +func addFilterOnIDField(scan *scanNode, propIndex int, val any) { if scan == nil { return } From d74fd657b918dab8a115b53e6c7688da4351744b Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 3 Jun 2024 12:59:52 +0200 Subject: [PATCH 05/18] Skip children fetching if not requested --- internal/planner/mapper/mapper.go | 1 + internal/planner/mapper/select.go | 4 ++ internal/planner/type_join.go | 69 ++++++++++++++++++------------- 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/internal/planner/mapper/mapper.go b/internal/planner/mapper/mapper.go index c27b11108d..be52066b54 100644 --- a/internal/planner/mapper/mapper.go +++ b/internal/planner/mapper/mapper.go @@ -987,6 +987,7 @@ func resolveInnerFilterDependencies( return nil, err } + childSelect.SkipResolve = true newFields = append(newFields, childSelect) } diff --git a/internal/planner/mapper/select.go b/internal/planner/mapper/select.go index 8b67b60937..b54edafeec 100644 --- a/internal/planner/mapper/select.go +++ b/internal/planner/mapper/select.go @@ -38,6 +38,10 @@ type Select struct { // These can include stuff such as version information, aggregates, and other // Selects. Fields []Requestable + + // SkipResolve is a flag that indicates that the fields in this select should not be resolved. + // It is used avoid resolving related objects if they are used in a filter and not requested in a response. + SkipResolve bool } func (s *Select) AsTargetable() (*Targetable, bool) { diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index 70086dd3fb..c5ca887eaf 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -370,6 +370,18 @@ func (p *Planner) makeTypeJoinMany( return nil, client.NewErrFieldNotExist(subSelect.Name) } + skipChild := false + for _, field := range parent.selectReq.Fields { + if field.GetName() == subSelect.Name { + if childSelect, ok := field.AsSelect(); ok { + if childSelect.SkipResolve { + skipChild = true + } + } + break + } + } + subCol, err := p.db.GetCollectionByName(p.ctx, subSelect.CollectionName) if err != nil { return nil, err @@ -420,6 +432,7 @@ func (p *Planner) makeTypeJoinMany( parentSide: parentSide, childSide: childSide, childSelect: subSelect, + skipChild: skipChild, secondaryFetchLimit: 0, }, }, nil @@ -528,6 +541,7 @@ type invertibleTypeJoin struct { docMapper childSelect *mapper.Select + skipChild bool parentSide joinSide childSide joinSide @@ -538,8 +552,6 @@ type invertibleTypeJoin struct { // docsToYield contains documents read and ready to be yielded by this node. docsToYield []core.Doc encounteredDocIDs []string - - //dir joinDirection } func (join *invertibleTypeJoin) getRootTypeName() string { @@ -644,8 +656,7 @@ func (j *docsJoiner) fetchPrimaryDocsReferencingSecondaryDoc() error { return client.NewErrFieldNotExist(j.primarySide.relFieldDef.Name + request.RelatedObjectID) } - scan := getScanNode(j.primarySide.plan) - j.primaryScan = scan + j.primaryScan = getScanNode(j.primarySide.plan) j.relIDFieldDef = relIDFieldDef @@ -655,9 +666,7 @@ func (j *docsJoiner) fetchPrimaryDocsReferencingSecondaryDoc() error { return err } - j.join(primaryDocs) - - scan.execInfo.fetches.Add(j.primaryScan.execInfo.fetches) + j.resultPrimaryDocs, j.resultSecondaryDoc = joinPrimaryDocs(primaryDocs, j.secondarySide, j.primarySide) return nil } @@ -687,7 +696,7 @@ func (j *docsJoiner) collectDocs(numDocs int) ([]core.Doc, error) { hasValue, err := p.Next() if err != nil { - return nil, p.Close() + return nil, err } if !hasValue { @@ -735,39 +744,38 @@ func docsToDocIDs(docs []core.Doc) []string { return docIDs } -func (j *docsJoiner) join(primaryDocs []core.Doc) { - secondaryDoc := j.secondarySide.plan.Value() +func joinPrimaryDocs(primaryDocs []core.Doc, secondarySide, primarySide *joinSide) ([]core.Doc, core.Doc) { + secondaryDoc := secondarySide.plan.Value() - if j.secondarySide.relFieldMapIndex.HasValue() { - if j.secondarySide.relFieldDef.Kind.IsArray() { - secondaryDoc.Fields[j.secondarySide.relFieldMapIndex.Value()] = primaryDocs + if secondarySide.relFieldMapIndex.HasValue() { + if secondarySide.relFieldDef.Kind.IsArray() { + secondaryDoc.Fields[secondarySide.relFieldMapIndex.Value()] = primaryDocs } else if len(primaryDocs) > 0 { - secondaryDoc.Fields[j.secondarySide.relFieldMapIndex.Value()] = primaryDocs[0] + secondaryDoc.Fields[secondarySide.relFieldMapIndex.Value()] = primaryDocs[0] } } - if j.secondarySide.relIDFieldMapIndex.HasValue() { - if j.secondarySide.relFieldDef.Kind.IsArray() { - secondaryDoc.Fields[j.secondarySide.relIDFieldMapIndex.Value()] = docsToDocIDs(primaryDocs) + if secondarySide.relIDFieldMapIndex.HasValue() { + if secondarySide.relFieldDef.Kind.IsArray() { + secondaryDoc.Fields[secondarySide.relIDFieldMapIndex.Value()] = docsToDocIDs(primaryDocs) } else if len(primaryDocs) > 0 { - secondaryDoc.Fields[j.secondarySide.relIDFieldMapIndex.Value()] = primaryDocs[0].GetID() + secondaryDoc.Fields[secondarySide.relIDFieldMapIndex.Value()] = primaryDocs[0].GetID() } } - if j.primarySide.relFieldMapIndex.HasValue() { + if primarySide.relFieldMapIndex.HasValue() { for i := range primaryDocs { - primaryDocs[i].Fields[j.primarySide.relFieldMapIndex.Value()] = secondaryDoc + primaryDocs[i].Fields[primarySide.relFieldMapIndex.Value()] = secondaryDoc } } - if j.primarySide.relIDFieldMapIndex.HasValue() { + if primarySide.relIDFieldMapIndex.HasValue() { for i := range primaryDocs { - primaryDocs[i].Fields[j.primarySide.relIDFieldMapIndex.Value()] = secondaryDoc.GetID() + primaryDocs[i].Fields[primarySide.relIDFieldMapIndex.Value()] = secondaryDoc.GetID() } } - j.resultPrimaryDocs = primaryDocs - j.resultSecondaryDoc = secondaryDoc + return primaryDocs, secondaryDoc } func fetchDocsWithFieldValue(plan planNode, fieldName string, val any) ([]core.Doc, error) { @@ -912,11 +920,16 @@ func (join *invertibleTypeJoin) Next() (bool, error) { return join.Next() } - //if join.parentSide.relFieldDef.Kind.IsArray() && join.parentSide.isFieldRequested(join.parentSide.relFieldDef.Name) { if join.parentSide.relFieldDef.Kind.IsArray() { - primaryDocs, secondaryDoc, err := join.fetchPrimaryDocsReferencingSecondaryDoc() - if err != nil { - return false, err + var primaryDocs []core.Doc + var secondaryDoc core.Doc + if join.skipChild { + primaryDocs, secondaryDoc = joinPrimaryDocs([]core.Doc{firstSide.plan.Value()}, secondSide, firstSide) + } else { + primaryDocs, secondaryDoc, err = join.fetchPrimaryDocsReferencingSecondaryDoc() + if err != nil { + return false, err + } } secondaryDoc.Fields[join.childSelect.Index] = primaryDocs From 3b23b96e86a383fe28867254764bb9247caee7d6 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 3 Jun 2024 13:02:13 +0200 Subject: [PATCH 06/18] Add tests and some comments --- .../index/query_with_relation_filter_test.go | 140 +++++++++++++----- 1 file changed, 104 insertions(+), 36 deletions(-) diff --git a/tests/integration/index/query_with_relation_filter_test.go b/tests/integration/index/query_with_relation_filter_test.go index aa49dd2623..8f508ac00a 100644 --- a/tests/integration/index/query_with_relation_filter_test.go +++ b/tests/integration/index/query_with_relation_filter_test.go @@ -17,6 +17,7 @@ import ( ) func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilter2(t *testing.T) { + // 3 users have a MacBook Pro: Islam, Shahzad, Keenan req1 := `query { User(filter: { devices: {model: {_eq: "MacBook Pro"}} @@ -24,6 +25,7 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte name } }` + // 1 user has an iPhone 10: Addo req2 := `query { User(filter: { devices: {model: {_eq: "iPhone 10"}} @@ -53,16 +55,14 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte testUtils.Request{ Request: req1, Results: []map[string]any{ - {"name": "Keenan"}, {"name": "Islam"}, {"name": "Shahzad"}, + {"name": "Keenan"}, }, }, testUtils.Request{ - Request: makeExplainQuery(req1), - // The invertable join does not support inverting one-many relations, so the index is - // not used. - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(450).WithIndexFetches(0), + Request: makeExplainQuery(req1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(6).WithIndexFetches(3), }, testUtils.Request{ Request: req2, @@ -71,10 +71,8 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte }, }, testUtils.Request{ - Request: makeExplainQuery(req2), - // The invertable join does not support inverting one-many relations, so the index is - // not used. - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(450).WithIndexFetches(0), + Request: makeExplainQuery(req2), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(1), }, }, } @@ -83,6 +81,7 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte } func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilter(t *testing.T) { + // 3 users have a MacBook Pro: Islam, Shahzad, Keenan req1 := `query { User(filter: { devices: {model: {_eq: "MacBook Pro"}} @@ -90,6 +89,7 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte name } }` + // 1 user has an iPhone 10: Addo req2 := `query { User(filter: { devices: {model: {_eq: "iPhone 10"}} @@ -119,16 +119,14 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte testUtils.Request{ Request: req1, Results: []map[string]any{ - {"name": "Keenan"}, {"name": "Islam"}, {"name": "Shahzad"}, + {"name": "Keenan"}, }, }, testUtils.Request{ - Request: makeExplainQuery(req1), - // The invertable join does not support inverting one-many relations, so the index is - // not used. - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(450).WithIndexFetches(0), + Request: makeExplainQuery(req1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(6).WithIndexFetches(3), }, testUtils.Request{ Request: req2, @@ -137,10 +135,8 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte }, }, testUtils.Request{ - Request: makeExplainQuery(req2), - // The invertable join does not support inverting one-many relations, so the index is - // not used. - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(450).WithIndexFetches(0), + Request: makeExplainQuery(req2), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(1), }, }, } @@ -149,6 +145,7 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte } func TestQueryWithIndexOnOneToOnesSecondaryRelation_IfFilterOnIndexedRelation_ShouldFilter(t *testing.T) { + // 1 user lives in Munich: Islam req1 := `query { User(filter: { address: {city: {_eq: "Munich"}} @@ -156,6 +153,7 @@ func TestQueryWithIndexOnOneToOnesSecondaryRelation_IfFilterOnIndexedRelation_Sh name } }` + // 3 users live in Montreal: Shahzad, Fred, John req2 := `query { User(filter: { address: {city: {_eq: "Montreal"}} @@ -176,7 +174,7 @@ func TestQueryWithIndexOnOneToOnesSecondaryRelation_IfFilterOnIndexedRelation_Sh type Address { user: User @primary - city: String @index + city: String @index }`, }, testUtils.CreatePredefinedDocs{ @@ -210,7 +208,79 @@ func TestQueryWithIndexOnOneToOnesSecondaryRelation_IfFilterOnIndexedRelation_Sh testUtils.ExecuteTestCase(t, test) } +func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelationAndRelation_ShouldFilter(t *testing.T) { + // 1 user lives in London: Andy + req1 := `query { + User(filter: { + address: {city: {_eq: "London"}} + }) { + name + } + }` + // 3 users live in Montreal: Shahzad, Fred, John + req2 := `query { + User(filter: { + address: {city: {_eq: "Montreal"}} + }) { + name + } + }` + test := testUtils.TestCase{ + Description: "Filter on indexed field of primary relation in 1-1 relation", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + address: Address @primary @index + } + + type Address { + user: User + city: String @index + street: String + }`, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req1, + Results: []map[string]any{ + {"name": "Andy"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req1), + // we make 2 index fetches: 1. to get the only address with city == "London" + // and 2. to get the corresponding user + // then 1 field fetch to get the name of the user + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(2), + }, + testUtils.Request{ + Request: req2, + Results: []map[string]any{ + {"name": "John"}, + {"name": "Fred"}, + {"name": "Shahzad"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req2), + // we make 3 index fetches to get the 3 address with city == "Montreal" + // and 3 more index fetches to get the corresponding users + // then 3 field fetches to get the name of each user + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(3).WithIndexFetches(6), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelation_ShouldFilter(t *testing.T) { + // 1 user lives in London: Andy req1 := `query { User(filter: { address: {city: {_eq: "London"}} @@ -218,6 +288,7 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelatio name } }` + // 3 users live in Montreal: Shahzad, Fred, John req2 := `query { User(filter: { address: {city: {_eq: "Montreal"}} @@ -256,7 +327,6 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelatio // we make 1 index fetch to get the only address with city == "London" // then we scan all 10 users to find one with matching "address_id" // after this we fetch the name of the user - // it should be optimized after this is done https://github.com/sourcenetwork/defradb/issues/2601 Asserter: testUtils.NewExplainAsserter().WithFieldFetches(11).WithIndexFetches(1), }, testUtils.Request{ @@ -272,7 +342,6 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelatio // we make 3 index fetch to get the 3 address with city == "Montreal" // then we scan all 10 users to find one with matching "address_id" for each address // after this we fetch the name of each user - // it should be optimized after this is done https://github.com/sourcenetwork/defradb/issues/2601 Asserter: testUtils.NewExplainAsserter().WithFieldFetches(33).WithIndexFetches(3), }, }, @@ -282,6 +351,7 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelatio } func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedRelationWhileIndexedForeignField_ShouldFilter(t *testing.T) { + // 1 user lives in London: Andy req := `query { User(filter: { address: {city: {_eq: "London"}} @@ -317,7 +387,7 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedRelationWhileI }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(11).WithIndexFetches(1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(2), }, }, } @@ -500,10 +570,8 @@ func TestQueryWithIndexOnOneToMany_IfFilterOnIndexedRelation_ShouldFilterWithExp }, }, testUtils.Request{ - Request: makeExplainQuery(req), - // The invertable join does not support inverting one-many relations, so the index is - // not used. - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(10).WithIndexFetches(0), + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(14).WithIndexFetches(2), }, }, } @@ -512,6 +580,7 @@ func TestQueryWithIndexOnOneToMany_IfFilterOnIndexedRelation_ShouldFilterWithExp } func TestQueryWithIndexOnOneToOne_IfFilterOnIndexedRelation_ShouldFilter(t *testing.T) { + // 1 user lives in Munich: Islam req := `query { User(filter: { address: {city: {_eq: "Munich"}} @@ -563,8 +632,8 @@ func TestQueryWithIndexOnOneToOne_IfFilterOnIndexedRelation_ShouldFilter(t *test } func TestQueryWithIndexOnManyToOne_IfFilterOnIndexedField_ShouldFilterWithExplain(t *testing.T) { - // This query will fetch first a matching device which is secondary doc and therefore - // has a reference to the primary User doc. + // This query will fetch first a matching device which is primary doc and therefore + // has a reference to the secondary User doc. req := `query { Device(filter: { year: {_eq: 2021} @@ -633,7 +702,6 @@ func TestQueryWithIndexOnManyToOne_IfFilterOnIndexedField_ShouldFilterWithExplai func TestQueryWithIndexOnManyToOne_IfFilterOnIndexedRelation_ShouldFilterWithExplain(t *testing.T) { // This query will fetch first a matching user (owner) which is primary doc and therefore // has no direct reference to secondary Device docs. - // At the moment the db has to make a full scan of the Device docs to find the matching ones. // Keenan has 3 devices. req := `query { Device(filter: { @@ -650,11 +718,11 @@ func TestQueryWithIndexOnManyToOne_IfFilterOnIndexedRelation_ShouldFilterWithExp type User { name: String @index devices: [Device] - } + } type Device { - model: String - owner: User + model: String + owner: User @index } `, }, @@ -671,10 +739,10 @@ func TestQueryWithIndexOnManyToOne_IfFilterOnIndexedRelation_ShouldFilterWithExp }, testUtils.Request{ Request: makeExplainQuery(req), - // we make only 1 index fetch to get the owner by it's name - // and 44 field fetches to get 2 fields for all 22 devices in the db. - // it should be optimized after this is done https://github.com/sourcenetwork/defradb/issues/2601 - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(44).WithIndexFetches(1), + // we make 1 index fetch to get the owner by it's name + // and 3 index fetches to get all 3 devices of the owner + // and 3 field fetches to get 1 'model' field for every fetched device. + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(3).WithIndexFetches(4), }, }, } From e266107d1bddfb934db3ec0ee61da2ddadec7acf Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 3 Jun 2024 13:19:27 +0200 Subject: [PATCH 07/18] Adjust explain tests --- internal/planner/type_join.go | 4 ++-- tests/integration/explain/execute/with_count_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index c5ca887eaf..35b58740fe 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -167,9 +167,9 @@ func (n *typeIndexJoin) simpleExplain() (map[string]any, error) { case *typeJoinOne: // Add the direction attribute. if joinType.parentSide.isPrimary() { - simpleExplainMap[joinDirectionLabel] = joinDirectionSecondaryLabel - } else { simpleExplainMap[joinDirectionLabel] = joinDirectionPrimaryLabel + } else { + simpleExplainMap[joinDirectionLabel] = joinDirectionSecondaryLabel } err = addExplainData(&joinType.invertibleTypeJoin) diff --git a/tests/integration/explain/execute/with_count_test.go b/tests/integration/explain/execute/with_count_test.go index 4a30b9f52a..43ff3d13df 100644 --- a/tests/integration/explain/execute/with_count_test.go +++ b/tests/integration/explain/execute/with_count_test.go @@ -62,7 +62,7 @@ func TestExecuteExplainRequestWithCountOnOneToManyRelation(t *testing.T) { "subTypeScanNode": dataMap{ "iterations": uint64(5), "docFetches": uint64(6), - "fieldFetches": uint64(14), + "fieldFetches": uint64(6), "indexFetches": uint64(0), }, }, From 181a1bde5f0c6cf84c6cfd1df1c13455a9773af9 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 3 Jun 2024 13:20:41 +0200 Subject: [PATCH 08/18] Remove unused code --- internal/planner/type_join.go | 180 +--------------------------------- 1 file changed, 2 insertions(+), 178 deletions(-) diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index 35b58740fe..5ab58bbe43 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -251,7 +251,7 @@ func (p *Planner) makeTypeJoinOne( parentsRelFieldDef.Name, ) if !ok { - return nil, client.NewErrFieldNotExist(parentsRelFieldDef.RelationName) + return nil, client.NewErrFieldNotExist(parentsRelFieldDef.Name) } childsRelFieldDef, ok := subCol.Definition().GetFieldByName(childsRelFieldDesc.Name) @@ -305,7 +305,6 @@ func (p *Planner) makeTypeJoinOne( childSelect: subSelect, secondaryFieldIndex: secondaryFieldIndex, secondaryFetchLimit: 1, - //dir: dir, }, }, nil } @@ -393,7 +392,7 @@ func (p *Planner) makeTypeJoinMany( parentsRelFieldDef.Name, ) if !ok { - return nil, client.NewErrFieldNotExist(subSelect.Name) + return nil, client.NewErrFieldNotExist(parentsRelFieldDef.Name) } childsRelFieldDef, ok := subCol.Definition().GetFieldByName(childsRelFieldDesc.Name) @@ -461,16 +460,6 @@ type joinSide struct { isParent bool } -func (s *joinSide) isFieldRequested(fieldName string) bool { - for i := range s.requestedFields { - if s.requestedFields[i] == fieldName { - return true - } - } - return false - -} - func (s *joinSide) isPrimary() bool { return s.relFieldDef.IsPrimaryRelation } @@ -600,34 +589,6 @@ func (join *invertibleTypeJoin) invert() { join.parentSide.isFirst = !join.parentSide.isFirst } -// addSecondaryDocsToRootPrimaryDoc adds the secondary docs to the root primary doc. -// If the relations is 1-to-1 a single secondary doc will be added to the root primary doc. -// Otherwise, all secondary docs will be added as an array. -func (join *invertibleTypeJoin) addSecondaryDocsToRootPrimaryDoc(secondDocs []core.Doc) { - var secondaryResult any - var secondaryIDResult any - if join.secondaryFetchLimit == 1 { - if len(secondDocs) != 0 { - secondaryResult = secondDocs[0] - secondaryIDResult = secondDocs[0].GetID() - } - } else { - secondaryResult = secondDocs - secondDocIDs := make([]string, len(secondDocs)) - for i, doc := range secondDocs { - secondDocIDs[i] = doc.GetID() - } - secondaryIDResult = secondDocIDs - } - join.parentSide.plan.Value().Fields[join.childSelect.Index] = secondaryResult - if join.secondaryFieldIndex.HasValue() { - join.parentSide.plan.Value().Fields[join.secondaryFieldIndex.Value()] = secondaryIDResult - } - //if join.parentSide.relIDFieldMapIndex.HasValue() { - //join.parentSide.plan.Value().Fields[join.parentSide.relIDFieldMapIndex.Value()] = secondaryIDResult - //} -} - type docsJoiner struct { relIDFieldDef client.FieldDefinition primarySide *joinSide @@ -778,85 +739,6 @@ func joinPrimaryDocs(primaryDocs []core.Doc, secondarySide, primarySide *joinSid return primaryDocs, secondaryDoc } -func fetchDocsWithFieldValue(plan planNode, fieldName string, val any) ([]core.Doc, error) { - propIndex := plan.DocumentMap().FirstIndexOfName(fieldName) - addFilterOnIDField(getScanNode(plan), propIndex, val) - - if err := plan.Init(); err != nil { - return nil, NewErrSubTypeInit(err) - } - - var docs []core.Doc - for { - next, err := plan.Next() - if err != nil { - return nil, err - } - if !next { - break - } - - docs = append(docs, plan.Value()) - } - - return docs, nil -} - -func (j *docsJoiner) fetchSecondaryFullScan() error { - primaryDoc := j.primarySide.plan.Value() - secondDocs, err := fetchDocsWithFieldValue( - j.primarySide.plan, - // At the join is from the secondary field, we know that [join.dir.secondaryField] must have a value - // otherwise the user would not have been able to request it. - j.relIDFieldDef.Name, - primaryDoc.GetID(), - ) - if err != nil { - return err - } - if j.primarySide.isParent { - if len(secondDocs) == 0 { - return nil - } - for i := range secondDocs { - //secondDocs[i].Fields[join.subSelect.Index] = j.subType.Value() - i = i - } - j.resultPrimaryDocs = append(j.resultPrimaryDocs, secondDocs...) - return nil - } else { - //j.addSecondaryDocsToRootPrimaryDoc(secondDocs) - j.resultPrimaryDocs = append(j.resultPrimaryDocs, j.secondarySide.plan.Value()) - } - return nil -} - -func (j *docsJoiner) fetchDocsWithFieldValue() error { - node := j.primarySide.plan - propIndex := node.DocumentMap().FirstIndexOfName(j.relIDFieldDef.Name) - primaryDoc := node.Value() - addFilterOnIDField(j.primaryScan, propIndex, primaryDoc.GetID()) - - if err := node.Init(); err != nil { - return NewErrSubTypeInit(err) - } - - j.resultPrimaryDocs = []core.Doc{} - for { - hasValue, err := node.Next() - if err != nil { - return err - } - if !hasValue { - break - } - - j.resultPrimaryDocs = append(j.resultPrimaryDocs, node.Value()) - } - - return nil -} - func (join *invertibleTypeJoin) fetchPrimaryDocsReferencingSecondaryDoc() ([]core.Doc, core.Doc, error) { secJoiner := newSecondaryDocsJoiner(join.getPrimarySide(), join.getSecondarySide()) err := secJoiner.fetchPrimaryDocsReferencingSecondaryDoc() @@ -954,64 +836,6 @@ func (join *invertibleTypeJoin) Next() (bool, error) { return true, nil } -func (join *invertibleTypeJoin) fetchIndexedRelatedSecondaryDocs(node planNode, field string) []any { - docIDStr := getForeignKey(node, field) - if docIDStr == "" { - return nil - } - var secondaryIDs []string - var spans []core.Span - scan := getScanNode(node) - - scan.initFetcher(immutable.None[string](), findIndexByFieldName(scan.col, field)) - - colRootID := scan.col.Description().RootID - - for _, secondaryID := range secondaryIDs { - dsKey := core.DataStoreKey{CollectionRootID: colRootID, DocID: secondaryID} - spans = append(spans, core.NewSpan(dsKey, dsKey.PrefixEnd())) - } - node.Spans(core.NewSpans(spans...)) - - if err := node.Init(); err != nil { - //return false, NewErrSubTypeInit(err) - return nil - } - - secondaryDocs := make([]core.Doc, 0, len(secondaryIDs)) - - for { - hasValue, err := node.Next() - - if err != nil { - //return false, err - return nil - } - - if !hasValue { - break - } - - secondaryDocs = append(secondaryDocs, node.Value()) - } - - if err := node.Close(); err != nil { - //return false, NewErrSubTypeInit(err) - return nil - } - - //return true, nil - //subTypeFieldDesc, ok := parent.collection.Definition().GetFieldByName(subType.Name) - join.parentSide.plan.Value().Fields[join.childSelect.Index] = secondaryDocs - if join.secondaryFieldIndex.HasValue() { - join.parentSide.plan.Value().Fields[join.secondaryFieldIndex.Value()] = secondaryIDs - } - //if join.parentSide.relIDFieldMapIndex.HasValue() { - //join.parentSide.plan.Value().Fields[join.parentSide.relIDFieldMapIndex.Value()] = secondaryIDs - //} - return nil -} - func (join *invertibleTypeJoin) Value() core.Doc { if len(join.docsToYield) == 0 { return core.Doc{} From 5ef897be4ee6a6b8ef7c25011d97b298d01cefc7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 3 Jun 2024 13:23:42 +0200 Subject: [PATCH 09/18] Remove useless field --- internal/planner/type_join.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index 5ab58bbe43..bf1bd24c39 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -259,15 +259,6 @@ func (p *Planner) makeTypeJoinOne( return nil, client.NewErrFieldNotExist(subSelect.Name) } - // TODO: remove this block? - var secondaryFieldIndex immutable.Option[int] - if !parentsRelFieldDef.IsPrimaryRelation { - idFieldName := parentsRelFieldDef.Name - secondaryFieldIndex = immutable.Some( - parent.documentMapping.FirstIndexOfName(idFieldName + request.RelatedObjectID), - ) - } - parentSide := joinSide{ plan: sourcePlan, relFieldDef: parentsRelFieldDef, @@ -303,7 +294,6 @@ func (p *Planner) makeTypeJoinOne( parentSide: parentSide, childSide: childSide, childSelect: subSelect, - secondaryFieldIndex: secondaryFieldIndex, secondaryFetchLimit: 1, }, }, nil @@ -535,7 +525,6 @@ type invertibleTypeJoin struct { parentSide joinSide childSide joinSide - secondaryFieldIndex immutable.Option[int] secondaryFetchLimit uint // docsToYield contains documents read and ready to be yielded by this node. From dc87c676d6472664e26bdfba3e00bb81efd29681 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 3 Jun 2024 16:15:14 +0200 Subject: [PATCH 10/18] Extract logic of creation invertableTypeJoin --- internal/planner/type_join.go | 120 ++++++++++------------------------ 1 file changed, 35 insertions(+), 85 deletions(-) diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index bf1bd24c39..3428bf4e1b 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -227,76 +227,12 @@ func (p *Planner) makeTypeJoinOne( sourcePlan planNode, subSelect *mapper.Select, ) (*typeJoinOne, error) { - prepareScanNodeFilterForTypeJoin(parent, sourcePlan, subSelect) - - subSelectPlan, err := p.Select(subSelect) - if err != nil { - return nil, err - } - - // get the correct sub field schema type (collection) - parentsRelFieldDef, ok := parent.collection.Definition().GetFieldByName(subSelect.Name) - if !ok { - return nil, client.NewErrFieldNotExist(subSelect.Name) - } - - subCol, err := p.db.GetCollectionByName(p.ctx, subSelect.CollectionName) + invertibleTypeJoin, err := p.newInvertableTypeJoin(parent, sourcePlan, subSelect) if err != nil { return nil, err } - - childsRelFieldDesc, ok := subCol.Description().GetFieldByRelation( - parentsRelFieldDef.RelationName, - parent.collection.Name().Value(), - parentsRelFieldDef.Name, - ) - if !ok { - return nil, client.NewErrFieldNotExist(parentsRelFieldDef.Name) - } - - childsRelFieldDef, ok := subCol.Definition().GetFieldByName(childsRelFieldDesc.Name) - if !ok { - return nil, client.NewErrFieldNotExist(subSelect.Name) - } - - parentSide := joinSide{ - plan: sourcePlan, - relFieldDef: parentsRelFieldDef, - relFieldMapIndex: immutable.Some(subSelect.Index), - col: parent.collection, - requestedFields: getRequestedFields(sourcePlan), - isFirst: true, - isParent: true, - } - - ind := parent.documentMapping.IndexesByName[parentsRelFieldDef.Name+request.RelatedObjectID] - if len(ind) > 0 { - parentSide.relIDFieldMapIndex = immutable.Some(ind[0]) - } - - childSide := joinSide{ - plan: subSelectPlan, - relFieldDef: childsRelFieldDef, - col: subCol, - requestedFields: getRequestedFields(subSelectPlan), - isFirst: false, - isParent: false, - } - - ind = subSelectPlan.DocumentMap().IndexesByName[childsRelFieldDef.Name+request.RelatedObjectID] - if len(ind) > 0 { - childSide.relIDFieldMapIndex = immutable.Some(ind[0]) - } - - return &typeJoinOne{ - invertibleTypeJoin: invertibleTypeJoin{ - docMapper: docMapper{parent.documentMapping}, - parentSide: parentSide, - childSide: childSide, - childSelect: subSelect, - secondaryFetchLimit: 1, - }, - }, nil + invertibleTypeJoin.secondaryFetchLimit = 1 + return &typeJoinOne{invertibleTypeJoin: invertibleTypeJoin}, nil } func (n *typeJoinOne) Kind() string { @@ -307,6 +243,19 @@ type typeJoinMany struct { invertibleTypeJoin } +func (p *Planner) makeTypeJoinMany( + parent *selectNode, + sourcePlan planNode, + subSelect *mapper.Select, +) (*typeJoinMany, error) { + invertibleTypeJoin, err := p.newInvertableTypeJoin(parent, sourcePlan, subSelect) + if err != nil { + return nil, err + } + invertibleTypeJoin.secondaryFetchLimit = 0 + return &typeJoinMany{invertibleTypeJoin: invertibleTypeJoin}, nil +} + func prepareScanNodeFilterForTypeJoin( parent *selectNode, source planNode, @@ -342,21 +291,21 @@ func prepareScanNodeFilterForTypeJoin( } } -func (p *Planner) makeTypeJoinMany( +func (p *Planner) newInvertableTypeJoin( parent *selectNode, sourcePlan planNode, subSelect *mapper.Select, -) (*typeJoinMany, error) { +) (invertibleTypeJoin, error) { prepareScanNodeFilterForTypeJoin(parent, sourcePlan, subSelect) subSelectPlan, err := p.Select(subSelect) if err != nil { - return nil, err + return invertibleTypeJoin{}, err } parentsRelFieldDef, ok := parent.collection.Definition().GetFieldByName(subSelect.Name) if !ok { - return nil, client.NewErrFieldNotExist(subSelect.Name) + return invertibleTypeJoin{}, client.NewErrFieldNotExist(subSelect.Name) } skipChild := false @@ -373,7 +322,7 @@ func (p *Planner) makeTypeJoinMany( subCol, err := p.db.GetCollectionByName(p.ctx, subSelect.CollectionName) if err != nil { - return nil, err + return invertibleTypeJoin{}, err } childsRelFieldDesc, ok := subCol.Description().GetFieldByRelation( @@ -382,12 +331,12 @@ func (p *Planner) makeTypeJoinMany( parentsRelFieldDef.Name, ) if !ok { - return nil, client.NewErrFieldNotExist(parentsRelFieldDef.Name) + return invertibleTypeJoin{}, client.NewErrFieldNotExist(parentsRelFieldDef.Name) } childsRelFieldDef, ok := subCol.Definition().GetFieldByName(childsRelFieldDesc.Name) if !ok { - return nil, client.NewErrFieldNotExist(subSelect.Name) + return invertibleTypeJoin{}, client.NewErrFieldNotExist(subSelect.Name) } parentSide := joinSide{ @@ -400,6 +349,11 @@ func (p *Planner) makeTypeJoinMany( isParent: true, } + ind := parent.documentMapping.IndexesByName[parentsRelFieldDef.Name+request.RelatedObjectID] + if len(ind) > 0 { + parentSide.relIDFieldMapIndex = immutable.Some(ind[0]) + } + childSide := joinSide{ plan: subSelectPlan, relFieldDef: childsRelFieldDef, @@ -409,21 +363,17 @@ func (p *Planner) makeTypeJoinMany( isParent: false, } - subDocMap := subSelectPlan.DocumentMap() - ind := subDocMap.IndexesByName[childsRelFieldDef.Name+request.RelatedObjectID] + ind = subSelectPlan.DocumentMap().IndexesByName[childsRelFieldDef.Name+request.RelatedObjectID] if len(ind) > 0 { childSide.relIDFieldMapIndex = immutable.Some(ind[0]) } - return &typeJoinMany{ - invertibleTypeJoin: invertibleTypeJoin{ - docMapper: docMapper{parent.documentMapping}, - parentSide: parentSide, - childSide: childSide, - childSelect: subSelect, - skipChild: skipChild, - secondaryFetchLimit: 0, - }, + return invertibleTypeJoin{ + docMapper: docMapper{parent.documentMapping}, + parentSide: parentSide, + childSide: childSide, + childSelect: subSelect, + skipChild: skipChild, }, nil } From 830d66a94ea2c3eb393d3304d8d8420b9d89d4b9 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 3 Jun 2024 17:46:38 +0200 Subject: [PATCH 11/18] Polish --- internal/planner/mapper/select.go | 4 +- internal/planner/planner.go | 4 +- internal/planner/type_join.go | 199 +++++++++++++----------------- 3 files changed, 92 insertions(+), 115 deletions(-) diff --git a/internal/planner/mapper/select.go b/internal/planner/mapper/select.go index b54edafeec..2c245c7287 100644 --- a/internal/planner/mapper/select.go +++ b/internal/planner/mapper/select.go @@ -39,8 +39,8 @@ type Select struct { // Selects. Fields []Requestable - // SkipResolve is a flag that indicates that the fields in this select should not be resolved. - // It is used avoid resolving related objects if they are used in a filter and not requested in a response. + // SkipResolve is a flag that indicates that the fields in this Select don't need to be resolved. + // It is used to avoid resolving related objects if they are used only in a filter and not requested in a response. SkipResolve bool } diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 782992668c..b76c55bbb9 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -362,8 +362,8 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan for subFieldName, subFieldInd := range filteredSubFields { indexes := desc.GetIndexesOnField(subFieldName) if len(indexes) > 0 && !filter.IsComplex(parentPlan.selectNode.filter) { - subInd := node.documentMapping.FirstIndexOfName(node.getSubTypeName()) - relatedField := mapper.Field{Name: node.getSubTypeName(), Index: subInd} + subInd := node.documentMapping.FirstIndexOfName(node.parentSide.relFieldDef.Name) + relatedField := mapper.Field{Name: node.parentSide.relFieldDef.Name, Index: subInd} fieldFilter := filter.UnwrapRelation(filter.CopyField( parentPlan.selectNode.filter, relatedField, diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index 3428bf4e1b..8c64ed7eb2 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -149,8 +149,8 @@ func (n *typeIndexJoin) simpleExplain() (map[string]any, error) { addExplainData := func(j *invertibleTypeJoin) error { // Add the attribute(s). - simpleExplainMap[joinRootLabel] = immutable.Some(j.getRootTypeName()) - simpleExplainMap[joinSubTypeNameLabel] = j.getSubTypeName() + simpleExplainMap[joinRootLabel] = immutable.Some(j.childSide.relFieldDef.Name) + simpleExplainMap[joinSubTypeNameLabel] = j.parentSide.relFieldDef.Name subTypeExplainGraph, err := buildSimpleExplainGraph(j.childSide.plan) if err != nil { @@ -344,7 +344,6 @@ func (p *Planner) newInvertableTypeJoin( relFieldDef: parentsRelFieldDef, relFieldMapIndex: immutable.Some(subSelect.Index), col: parent.collection, - requestedFields: getRequestedFields(sourcePlan), isFirst: true, isParent: true, } @@ -355,12 +354,11 @@ func (p *Planner) newInvertableTypeJoin( } childSide := joinSide{ - plan: subSelectPlan, - relFieldDef: childsRelFieldDef, - col: subCol, - requestedFields: getRequestedFields(subSelectPlan), - isFirst: false, - isParent: false, + plan: subSelectPlan, + relFieldDef: childsRelFieldDef, + col: subCol, + isFirst: false, + isParent: false, } ind = subSelectPlan.DocumentMap().IndexesByName[childsRelFieldDef.Name+request.RelatedObjectID] @@ -369,33 +367,19 @@ func (p *Planner) newInvertableTypeJoin( } return invertibleTypeJoin{ - docMapper: docMapper{parent.documentMapping}, - parentSide: parentSide, - childSide: childSide, - childSelect: subSelect, - skipChild: skipChild, + docMapper: docMapper{parent.documentMapping}, + parentSide: parentSide, + childSide: childSide, + skipChild: skipChild, }, nil } -func getRequestedFields(sourcePlan planNode) []string { - scan := getScanNode(sourcePlan) - if scan == nil { - return nil - } - fields := make([]string, len(scan.fields)) - for i := range scan.fields { - fields[i] = scan.fields[i].Name - } - return fields -} - type joinSide struct { plan planNode relFieldDef client.FieldDefinition relFieldMapIndex immutable.Option[int] relIDFieldMapIndex immutable.Option[int] col client.Collection - requestedFields []string isFirst bool isParent bool } @@ -436,12 +420,14 @@ func (n *typeJoinMany) Kind() string { return "typeJoinMany" } +// getForeignKey returns the docID of the related object referenced by the given relation field. func getForeignKey(node planNode, relFieldName string) string { ind := node.DocumentMap().FirstIndexOfName(relFieldName + request.RelatedObjectID) docIDStr, _ := node.Value().Fields[ind].(string) return docIDStr } +// fetchDocWithID fetches a document with the given docID from the given planNode. func fetchDocWithID(node planNode, docID string) (bool, error) { scan := getScanNode(node) if scan == nil { @@ -469,8 +455,7 @@ func fetchDocWithID(node planNode, docID string) (bool, error) { type invertibleTypeJoin struct { docMapper - childSelect *mapper.Select - skipChild bool + skipChild bool parentSide joinSide childSide joinSide @@ -482,16 +467,7 @@ type invertibleTypeJoin struct { encounteredDocIDs []string } -func (join *invertibleTypeJoin) getRootTypeName() string { - return join.getSecondSide().relFieldDef.Name -} - -func (join *invertibleTypeJoin) getSubTypeName() string { - return join.getFirstSide().relFieldDef.Name -} - func (join *invertibleTypeJoin) replaceRoot(node planNode) { - join.parentSide.plan = node join.getFirstSide().plan = node } @@ -523,12 +499,7 @@ func (join *invertibleTypeJoin) Spans(spans core.Spans) { func (join *invertibleTypeJoin) Source() planNode { return join.parentSide.plan } -func (join *invertibleTypeJoin) invert() { - join.childSide.isFirst = join.parentSide.isFirst - join.parentSide.isFirst = !join.parentSide.isFirst -} - -type docsJoiner struct { +type primaryObjectsFetcher struct { relIDFieldDef client.FieldDefinition primarySide *joinSide secondarySide *joinSide @@ -539,17 +510,17 @@ type docsJoiner struct { resultSecondaryDoc core.Doc } -func newSecondaryDocsJoiner( +func newPrimaryObjectsFetcher( primarySide, secondarySide *joinSide, -) docsJoiner { - j := docsJoiner{ +) primaryObjectsFetcher { + j := primaryObjectsFetcher{ primarySide: primarySide, secondarySide: secondarySide, } return j } -func (j *docsJoiner) fetchPrimaryDocsReferencingSecondaryDoc() error { +func (j *primaryObjectsFetcher) fetchPrimaryDocsReferencingSecondaryDoc() error { relIDFieldDef, ok := j.primarySide.col.Definition().GetFieldByName( j.primarySide.relFieldDef.Name + request.RelatedObjectID) if !ok { @@ -571,7 +542,7 @@ func (j *docsJoiner) fetchPrimaryDocsReferencingSecondaryDoc() error { return nil } -func (j *docsJoiner) addIDFieldToScanner() { +func (j *primaryObjectsFetcher) addIDFieldToScanner() { found := false for i := range j.primaryScan.fields { if j.primaryScan.fields[i].Name == j.relIDFieldDef.Name { @@ -584,7 +555,7 @@ func (j *docsJoiner) addIDFieldToScanner() { } } -func (j *docsJoiner) collectDocs(numDocs int) ([]core.Doc, error) { +func (j *primaryObjectsFetcher) collectDocs(numDocs int) ([]core.Doc, error) { p := j.primarySide.plan if err := p.Init(); err != nil { return nil, NewErrSubTypeInit(err) @@ -609,15 +580,13 @@ func (j *docsJoiner) collectDocs(numDocs int) ([]core.Doc, error) { return docs, nil } -func (j *docsJoiner) fetchPrimaryDocs() ([]core.Doc, error) { +func (j *primaryObjectsFetcher) fetchPrimaryDocs() ([]core.Doc, error) { j.addIDFieldToScanner() secondaryDoc := j.secondarySide.plan.Value() addFilterOnIDField(j.primaryScan, j.primarySide.relIDFieldMapIndex.Value(), secondaryDoc.GetID()) oldFetcher := j.primaryScan.fetcher - // TODO: check if spans are necessary to be saved - oldSpans := j.primaryScan.spans indexOnRelation := findIndexByFieldName(j.primaryScan.col, j.relIDFieldDef.Name) j.primaryScan.initFetcher(immutable.None[string](), indexOnRelation) @@ -626,7 +595,6 @@ func (j *docsJoiner) fetchPrimaryDocs() ([]core.Doc, error) { j.primaryScan.fetcher.Close() - j.primaryScan.spans = oldSpans j.primaryScan.fetcher = oldFetcher if err != nil { @@ -679,7 +647,7 @@ func joinPrimaryDocs(primaryDocs []core.Doc, secondarySide, primarySide *joinSid } func (join *invertibleTypeJoin) fetchPrimaryDocsReferencingSecondaryDoc() ([]core.Doc, core.Doc, error) { - secJoiner := newSecondaryDocsJoiner(join.getPrimarySide(), join.getSecondarySide()) + secJoiner := newPrimaryObjectsFetcher(join.getPrimarySide(), join.getSecondarySide()) err := secJoiner.fetchPrimaryDocsReferencingSecondaryDoc() return secJoiner.resultPrimaryDocs, secJoiner.resultSecondaryDoc, err } @@ -703,75 +671,83 @@ func (join *invertibleTypeJoin) Next() (bool, error) { return false, err } - secondSide := join.getSecondSide() - if firstSide.isPrimary() { - secondaryDocID := getForeignKey(firstSide.plan, firstSide.relFieldDef.Name) - // TODO: add some tests with filter on nil relation - if secondaryDocID == "" { - if firstSide.isParent { - join.docsToYield = append(join.docsToYield, firstSide.plan.Value()) - return true, nil - } - return join.Next() + return join.nextJoinedSecondaryDoc() + } else { + primaryDocs, secondaryDoc, err := join.fetchPrimaryDocsReferencingSecondaryDoc() + if err != nil { + return false, err } - - if !firstSide.isParent { - for i := range join.encounteredDocIDs { - if join.encounteredDocIDs[i] == secondaryDocID { - return join.Next() - } - } - join.encounteredDocIDs = append(join.encounteredDocIDs, secondaryDocID) + if join.parentSide.isPrimary() { + join.docsToYield = append(join.docsToYield, primaryDocs...) + } else { + join.docsToYield = append(join.docsToYield, secondaryDoc) } + } - // check if there can ever be false (a.k.a. hasDoc = false) - hasDoc, err := fetchDocWithID(secondSide.plan, secondaryDocID) - if err != nil { - return false, err + return true, nil +} + +func (join *invertibleTypeJoin) nextJoinedSecondaryDoc() (bool, error) { + firstSide := join.getFirstSide() + secondSide := join.getSecondSide() + + secondaryDocID := getForeignKey(firstSide.plan, firstSide.relFieldDef.Name) + // TODO: add some tests with filter on nil relation + if secondaryDocID == "" { + if firstSide.isParent { + join.docsToYield = append(join.docsToYield, firstSide.plan.Value()) + return true, nil } + return join.Next() + } - // TODO: add some tests that either return error if the doc is not found or return - // the related doc (without this one) and let it be filtered. - if !hasDoc { - if firstSide.isParent { - join.docsToYield = append(join.docsToYield, firstSide.plan.Value()) - return true, nil + if !firstSide.isParent { + for i := range join.encounteredDocIDs { + if join.encounteredDocIDs[i] == secondaryDocID { + return join.Next() } - return join.Next() } + join.encounteredDocIDs = append(join.encounteredDocIDs, secondaryDocID) + } - if join.parentSide.relFieldDef.Kind.IsArray() { - var primaryDocs []core.Doc - var secondaryDoc core.Doc - if join.skipChild { - primaryDocs, secondaryDoc = joinPrimaryDocs([]core.Doc{firstSide.plan.Value()}, secondSide, firstSide) - } else { - primaryDocs, secondaryDoc, err = join.fetchPrimaryDocsReferencingSecondaryDoc() - if err != nil { - return false, err - } - } - secondaryDoc.Fields[join.childSelect.Index] = primaryDocs + // check if there can ever be false (a.k.a. hasDoc = false) + hasDoc, err := fetchDocWithID(secondSide.plan, secondaryDocID) + if err != nil { + return false, err + } - join.docsToYield = append(join.docsToYield, secondaryDoc) - } else { - parentDoc := join.parentSide.plan.Value() - parentDoc.Fields[join.parentSide.relFieldMapIndex.Value()] = join.childSide.plan.Value() - join.docsToYield = append(join.docsToYield, parentDoc) - } - } else { - primaryDocs, secondaryDoc, err := join.fetchPrimaryDocsReferencingSecondaryDoc() - if err != nil { - return false, err + // TODO: add some tests that either return error if the doc is not found or return + // the related doc (without this one) and let it be filtered. + if !hasDoc { + if firstSide.isParent { + join.docsToYield = append(join.docsToYield, firstSide.plan.Value()) + return true, nil } - if join.parentSide.isPrimary() { - join.docsToYield = append(join.docsToYield, primaryDocs...) + return join.Next() + } + + if join.parentSide.relFieldDef.Kind.IsArray() { + var primaryDocs []core.Doc + var secondaryDoc core.Doc + // if child is not requested as part of the response, we just add the existing one (fetched by the secondary index + // on a filtered value) so that top select node that runs the filter again can yield it. + if join.skipChild { + primaryDocs, secondaryDoc = joinPrimaryDocs([]core.Doc{firstSide.plan.Value()}, secondSide, firstSide) } else { - join.docsToYield = append(join.docsToYield, secondaryDoc) + primaryDocs, secondaryDoc, err = join.fetchPrimaryDocsReferencingSecondaryDoc() + if err != nil { + return false, err + } } - } + secondaryDoc.Fields[join.parentSide.relFieldMapIndex.Value()] = primaryDocs + join.docsToYield = append(join.docsToYield, secondaryDoc) + } else { + parentDoc := join.parentSide.plan.Value() + parentDoc.Fields[join.parentSide.relFieldMapIndex.Value()] = join.childSide.plan.Value() + join.docsToYield = append(join.docsToYield, parentDoc) + } return true, nil } @@ -792,7 +768,8 @@ func (join *invertibleTypeJoin) invertJoinDirectionWithIndex( s.filter = fieldFilter s.initFetcher(immutable.Option[string]{}, immutable.Some(index)) - join.invert() + join.childSide.isFirst = join.parentSide.isFirst + join.parentSide.isFirst = !join.parentSide.isFirst return nil } From 0b493aa78dba3f5b812650530cf0bb75ac9ab761 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Mon, 3 Jun 2024 19:08:05 +0200 Subject: [PATCH 12/18] Fix lint --- internal/db/collection_index.go | 2 +- internal/planner/type_join.go | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/db/collection_index.go b/internal/db/collection_index.go index 7ded8fff0e..c2f02bf3bf 100644 --- a/internal/db/collection_index.go +++ b/internal/db/collection_index.go @@ -496,7 +496,7 @@ func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, // checkExistingFieldsAndAdjustRelFieldNames checks if the fields in the index description // exist in the collection schema. -// If a field is a relation, it will be adjusted to relation id field name, a.k.a. `field_name + _id`. +// If a field is a relation, it will be adjusted to relation id field name, a.k.a. `field_name + _id`. func (c *collection) checkExistingFieldsAndAdjustRelFieldNames( fields []client.IndexedFieldDescription, ) error { diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index 8c64ed7eb2..b6ce65dbdd 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -592,15 +592,17 @@ func (j *primaryObjectsFetcher) fetchPrimaryDocs() ([]core.Doc, error) { j.primaryScan.initFetcher(immutable.None[string](), indexOnRelation) docs, err := j.collectDocs(0) + if err != nil { + return nil, err + } - j.primaryScan.fetcher.Close() - - j.primaryScan.fetcher = oldFetcher - + err = j.primaryScan.fetcher.Close() if err != nil { return nil, err } + j.primaryScan.fetcher = oldFetcher + return docs, nil } From 8b147bb23ccec00e79fefa3a180bbb5cf9adac31 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 4 Jun 2024 11:22:38 +0200 Subject: [PATCH 13/18] Polish --- internal/planner/mapper/select.go | 3 ++- internal/planner/planner.go | 2 ++ internal/planner/select.go | 2 ++ internal/planner/type_join.go | 25 ++++++++++++------------- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/internal/planner/mapper/select.go b/internal/planner/mapper/select.go index 2c245c7287..eebb4033f2 100644 --- a/internal/planner/mapper/select.go +++ b/internal/planner/mapper/select.go @@ -39,7 +39,8 @@ type Select struct { // Selects. Fields []Requestable - // SkipResolve is a flag that indicates that the fields in this Select don't need to be resolved. + // SkipResolve is a flag that indicates that the fields in this Select don't need to be resolved, + // i.e. it's value doesn't need to be fetched and provided to the user. // It is used to avoid resolving related objects if they are used only in a filter and not requested in a response. SkipResolve bool } diff --git a/internal/planner/planner.go b/internal/planner/planner.go index b76c55bbb9..f7a875af70 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -369,6 +369,8 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan relatedField, mapper.Field{Name: subFieldName, Index: subFieldInd}, ), relatedField) + // At the moment we just take the first index, but later we want to run some kind of analysis to + // determine which index is best to use. https://github.com/sourcenetwork/defradb/issues/2680 err := node.invertJoinDirectionWithIndex(fieldFilter, indexes[0]) if err != nil { return err diff --git a/internal/planner/select.go b/internal/planner/select.go index 60c895167d..5d7e448c73 100644 --- a/internal/planner/select.go +++ b/internal/planner/select.go @@ -322,6 +322,8 @@ func findIndexByFieldName(col client.Collection, fieldName string) immutable.Opt } indexes := col.Description().GetIndexesOnField(field.Name) if len(indexes) > 0 { + // At the moment we just take the first index, but later we want to run some kind of analysis to + // determine which index is best to use. https://github.com/sourcenetwork/defradb/issues/2680 return immutable.Some(indexes[0]) } } diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index b6ce65dbdd..0b6dd5a57e 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -499,7 +499,7 @@ func (join *invertibleTypeJoin) Spans(spans core.Spans) { func (join *invertibleTypeJoin) Source() planNode { return join.parentSide.plan } -type primaryObjectsFetcher struct { +type primaryObjectsRetriever struct { relIDFieldDef client.FieldDefinition primarySide *joinSide secondarySide *joinSide @@ -510,17 +510,17 @@ type primaryObjectsFetcher struct { resultSecondaryDoc core.Doc } -func newPrimaryObjectsFetcher( +func newPrimaryObjectsRetriever( primarySide, secondarySide *joinSide, -) primaryObjectsFetcher { - j := primaryObjectsFetcher{ +) primaryObjectsRetriever { + j := primaryObjectsRetriever{ primarySide: primarySide, secondarySide: secondarySide, } return j } -func (j *primaryObjectsFetcher) fetchPrimaryDocsReferencingSecondaryDoc() error { +func (j *primaryObjectsRetriever) retrievePrimaryDocsReferencingSecondaryDoc() error { relIDFieldDef, ok := j.primarySide.col.Definition().GetFieldByName( j.primarySide.relFieldDef.Name + request.RelatedObjectID) if !ok { @@ -531,7 +531,7 @@ func (j *primaryObjectsFetcher) fetchPrimaryDocsReferencingSecondaryDoc() error j.relIDFieldDef = relIDFieldDef - primaryDocs, err := j.fetchPrimaryDocs() + primaryDocs, err := j.retrievePrimaryDocs() if err != nil { return err @@ -542,7 +542,7 @@ func (j *primaryObjectsFetcher) fetchPrimaryDocsReferencingSecondaryDoc() error return nil } -func (j *primaryObjectsFetcher) addIDFieldToScanner() { +func (j *primaryObjectsRetriever) addIDFieldToScanner() { found := false for i := range j.primaryScan.fields { if j.primaryScan.fields[i].Name == j.relIDFieldDef.Name { @@ -555,7 +555,7 @@ func (j *primaryObjectsFetcher) addIDFieldToScanner() { } } -func (j *primaryObjectsFetcher) collectDocs(numDocs int) ([]core.Doc, error) { +func (j *primaryObjectsRetriever) collectDocs(numDocs int) ([]core.Doc, error) { p := j.primarySide.plan if err := p.Init(); err != nil { return nil, NewErrSubTypeInit(err) @@ -580,7 +580,7 @@ func (j *primaryObjectsFetcher) collectDocs(numDocs int) ([]core.Doc, error) { return docs, nil } -func (j *primaryObjectsFetcher) fetchPrimaryDocs() ([]core.Doc, error) { +func (j *primaryObjectsRetriever) retrievePrimaryDocs() ([]core.Doc, error) { j.addIDFieldToScanner() secondaryDoc := j.secondarySide.plan.Value() @@ -649,9 +649,9 @@ func joinPrimaryDocs(primaryDocs []core.Doc, secondarySide, primarySide *joinSid } func (join *invertibleTypeJoin) fetchPrimaryDocsReferencingSecondaryDoc() ([]core.Doc, core.Doc, error) { - secJoiner := newPrimaryObjectsFetcher(join.getPrimarySide(), join.getSecondarySide()) - err := secJoiner.fetchPrimaryDocsReferencingSecondaryDoc() - return secJoiner.resultPrimaryDocs, secJoiner.resultSecondaryDoc, err + retriever := newPrimaryObjectsRetriever(join.getPrimarySide(), join.getSecondarySide()) + err := retriever.retrievePrimaryDocsReferencingSecondaryDoc() + return retriever.resultPrimaryDocs, retriever.resultSecondaryDoc, err } func (join *invertibleTypeJoin) Next() (bool, error) { @@ -713,7 +713,6 @@ func (join *invertibleTypeJoin) nextJoinedSecondaryDoc() (bool, error) { join.encounteredDocIDs = append(join.encounteredDocIDs, secondaryDocID) } - // check if there can ever be false (a.k.a. hasDoc = false) hasDoc, err := fetchDocWithID(secondSide.plan, secondaryDocID) if err != nil { return false, err From 770ded36fa42734ed19b9265897b5bf5dbec1321 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 4 Jun 2024 15:45:54 +0200 Subject: [PATCH 14/18] Add change detector record --- docs/data_format_changes/i2670-sec-index-on-relations.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 docs/data_format_changes/i2670-sec-index-on-relations.md diff --git a/docs/data_format_changes/i2670-sec-index-on-relations.md b/docs/data_format_changes/i2670-sec-index-on-relations.md new file mode 100644 index 0000000000..4f56429166 --- /dev/null +++ b/docs/data_format_changes/i2670-sec-index-on-relations.md @@ -0,0 +1,3 @@ +# Enable secondary index on relations + +This naturally caused some explain metrics to change and change detector complain about it. \ No newline at end of file From 76cfa778d9b8bdaf285f2f1ab31cde811f99e0df Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 4 Jun 2024 18:03:38 +0200 Subject: [PATCH 15/18] Add tests on nil relations --- client/normal_nil.go | 2 +- internal/planner/type_join.go | 3 - .../index/query_with_relation_filter_test.go | 161 ++++++++++++++++++ 3 files changed, 162 insertions(+), 4 deletions(-) diff --git a/client/normal_nil.go b/client/normal_nil.go index 7cd2df3f16..f78a0cc63e 100644 --- a/client/normal_nil.go +++ b/client/normal_nil.go @@ -30,7 +30,7 @@ func NewNormalNil(kind FieldKind) (NormalValue, error) { return NewNormalNillableFloat(immutable.None[float64]()), nil case FieldKind_NILLABLE_DATETIME: return NewNormalNillableTime(immutable.None[time.Time]()), nil - case FieldKind_NILLABLE_STRING, FieldKind_NILLABLE_JSON: + case FieldKind_NILLABLE_STRING, FieldKind_NILLABLE_JSON, FieldKind_DocID: return NewNormalNillableString(immutable.None[string]()), nil case FieldKind_NILLABLE_BLOB: return NewNormalNillableBytes(immutable.None[[]byte]()), nil diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index 0b6dd5a57e..cd20ad3c8d 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -695,7 +695,6 @@ func (join *invertibleTypeJoin) nextJoinedSecondaryDoc() (bool, error) { secondSide := join.getSecondSide() secondaryDocID := getForeignKey(firstSide.plan, firstSide.relFieldDef.Name) - // TODO: add some tests with filter on nil relation if secondaryDocID == "" { if firstSide.isParent { join.docsToYield = append(join.docsToYield, firstSide.plan.Value()) @@ -718,8 +717,6 @@ func (join *invertibleTypeJoin) nextJoinedSecondaryDoc() (bool, error) { return false, err } - // TODO: add some tests that either return error if the doc is not found or return - // the related doc (without this one) and let it be filtered. if !hasDoc { if firstSide.isParent { join.docsToYield = append(join.docsToYield, firstSide.plan.Value()) diff --git a/tests/integration/index/query_with_relation_filter_test.go b/tests/integration/index/query_with_relation_filter_test.go index 8f508ac00a..94160a5e3c 100644 --- a/tests/integration/index/query_with_relation_filter_test.go +++ b/tests/integration/index/query_with_relation_filter_test.go @@ -749,3 +749,164 @@ func TestQueryWithIndexOnManyToOne_IfFilterOnIndexedRelation_ShouldFilterWithExp testUtils.ExecuteTestCase(t, test) } + +func TestQueryWithIndexOnOneToMany_IfIndexedRelationIsNil_NeNilFilterShouldUseIndex(t *testing.T) { + req := `query { + Device(filter: { + owner_id: {_ne: null} + }) { + model + } + }` + test := testUtils.TestCase{ + Description: "Filter on indexed relation field in 1-N relations", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + devices: [Device] + } + + type Device { + model: String + manufacturer: String + owner: User @index + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "Chris" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "Walkman", + "manufacturer": "Sony", + "owner": "bae-403d7337-f73e-5c81-8719-e853938c8985" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "iPhone", + "manufacturer": "Apple", + "owner": "bae-403d7337-f73e-5c81-8719-e853938c8985" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "Running Man", + "manufacturer": "Braveworld Productions" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "PlayStation 5", + "manufacturer": "Sony" + }`, + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"model": "iPhone"}, + {"model": "Walkman"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + // we make 4 index fetches to find 2 devices with owner_id != null + // and 2 field fetches to get 1 'model' field for every fetched device + // plus 2 more field fetches to get related User docs + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(4), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithIndexOnOneToMany_IfIndexedRelationIsNil_EqNilFilterShouldUseIndex(t *testing.T) { + req := `query { + Device(filter: { + owner_id: {_eq: null} + }) { + model + } + }` + test := testUtils.TestCase{ + Description: "Filter on indexed relation field in 1-N relations", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + devices: [Device] + } + + type Device { + model: String + manufacturer: String + owner: User @index + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "Chris" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "Walkman", + "manufacturer": "Sony", + "owner": "bae-403d7337-f73e-5c81-8719-e853938c8985" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "iPhone", + "manufacturer": "Apple", + "owner": "bae-403d7337-f73e-5c81-8719-e853938c8985" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "Running Man", + "manufacturer": "Braveworld Productions" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "PlayStation 5", + "manufacturer": "Sony" + }`, + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"model": "Running Man"}, + {"model": "PlayStation 5"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + // we make 2 index fetches to get all 2 devices with owner_id == null + // and 2 field fetches to get 1 'model' field for every fetched device. + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(2), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} From fe8a7d42c6d1d876cc7464accc829bc0d63848c4 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 4 Jun 2024 18:12:34 +0200 Subject: [PATCH 16/18] Format --- internal/planner/mapper/select.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/planner/mapper/select.go b/internal/planner/mapper/select.go index eebb4033f2..5d9e8d39f0 100644 --- a/internal/planner/mapper/select.go +++ b/internal/planner/mapper/select.go @@ -39,7 +39,7 @@ type Select struct { // Selects. Fields []Requestable - // SkipResolve is a flag that indicates that the fields in this Select don't need to be resolved, + // SkipResolve is a flag that indicates that the fields in this Select don't need to be resolved, // i.e. it's value doesn't need to be fetched and provided to the user. // It is used to avoid resolving related objects if they are used only in a filter and not requested in a response. SkipResolve bool From 4feb9771129d2fd8dd9ecdae43f20802dc7ba6f7 Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 4 Jun 2024 22:03:39 +0200 Subject: [PATCH 17/18] Make docID field nillable --- client/normal_value_test.go | 2 +- client/schema_field_description.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/normal_value_test.go b/client/normal_value_test.go index 33cd20c46e..046899e57b 100644 --- a/client/normal_value_test.go +++ b/client/normal_value_test.go @@ -1404,7 +1404,7 @@ func TestNormalValue_NewNormalNil(t *testing.T) { assert.True(t, v.IsNil()) } else { _, err := NewNormalNil(kind) - require.Error(t, err) + require.Error(t, err, "field kind: " + kind.String()) } } } diff --git a/client/schema_field_description.go b/client/schema_field_description.go index 87ee843ec8..cad233b67c 100644 --- a/client/schema_field_description.go +++ b/client/schema_field_description.go @@ -104,7 +104,7 @@ func (k ScalarKind) Underlying() string { } func (k ScalarKind) IsNillable() bool { - return k != FieldKind_DocID + return true } func (k ScalarKind) IsObject() bool { From d1450329f146247d53a9f10c60231e8c6422abae Mon Sep 17 00:00:00 2001 From: Islam Aleiv Date: Tue, 4 Jun 2024 22:06:23 +0200 Subject: [PATCH 18/18] Format --- client/normal_value_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/normal_value_test.go b/client/normal_value_test.go index 046899e57b..73e9def5d6 100644 --- a/client/normal_value_test.go +++ b/client/normal_value_test.go @@ -1404,7 +1404,7 @@ func TestNormalValue_NewNormalNil(t *testing.T) { assert.True(t, v.IsNil()) } else { _, err := NewNormalNil(kind) - require.Error(t, err, "field kind: " + kind.String()) + require.Error(t, err, "field kind: "+kind.String()) } } }