diff --git a/client/normal_nil.go b/client/normal_nil.go index 7cd2df3f16..f78a0cc63e 100644 --- a/client/normal_nil.go +++ b/client/normal_nil.go @@ -30,7 +30,7 @@ func NewNormalNil(kind FieldKind) (NormalValue, error) { return NewNormalNillableFloat(immutable.None[float64]()), nil case FieldKind_NILLABLE_DATETIME: return NewNormalNillableTime(immutable.None[time.Time]()), nil - case FieldKind_NILLABLE_STRING, FieldKind_NILLABLE_JSON: + case FieldKind_NILLABLE_STRING, FieldKind_NILLABLE_JSON, FieldKind_DocID: return NewNormalNillableString(immutable.None[string]()), nil case FieldKind_NILLABLE_BLOB: return NewNormalNillableBytes(immutable.None[[]byte]()), nil diff --git a/client/normal_value_test.go b/client/normal_value_test.go index 33cd20c46e..73e9def5d6 100644 --- a/client/normal_value_test.go +++ b/client/normal_value_test.go @@ -1404,7 +1404,7 @@ func TestNormalValue_NewNormalNil(t *testing.T) { assert.True(t, v.IsNil()) } else { _, err := NewNormalNil(kind) - require.Error(t, err) + require.Error(t, err, "field kind: "+kind.String()) } } } diff --git a/client/schema_field_description.go b/client/schema_field_description.go index 87ee843ec8..cad233b67c 100644 --- a/client/schema_field_description.go +++ b/client/schema_field_description.go @@ -104,7 +104,7 @@ func (k ScalarKind) Underlying() string { } func (k ScalarKind) IsNillable() bool { - return k != FieldKind_DocID + return true } func (k ScalarKind) IsObject() bool { diff --git a/docs/data_format_changes/i2670-sec-index-on-relations.md b/docs/data_format_changes/i2670-sec-index-on-relations.md new file mode 100644 index 0000000000..4f56429166 --- /dev/null +++ b/docs/data_format_changes/i2670-sec-index-on-relations.md @@ -0,0 +1,3 @@ +# Enable secondary index on relations + +This naturally caused some explain metrics to change and change detector complain about it. \ No newline at end of file diff --git a/internal/db/collection_index.go b/internal/db/collection_index.go index 14f9a1b805..c2f02bf3bf 100644 --- a/internal/db/collection_index.go +++ b/internal/db/collection_index.go @@ -21,6 +21,7 @@ import ( "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/client/request" "github.com/sourcenetwork/defradb/datastore" "github.com/sourcenetwork/defradb/internal/core" "github.com/sourcenetwork/defradb/internal/db/base" @@ -264,7 +265,7 @@ func (c *collection) createIndex( return nil, err } - err = c.checkExistingFields(desc.Fields) + err = c.checkExistingFieldsAndAdjustRelFieldNames(desc.Fields) if err != nil { return nil, err } @@ -493,20 +494,19 @@ func (c *collection) GetIndexes(ctx context.Context) ([]client.IndexDescription, return c.Description().Indexes, nil } -func (c *collection) checkExistingFields( +// checkExistingFieldsAndAdjustRelFieldNames checks if the fields in the index description +// exist in the collection schema. +// If a field is a relation, it will be adjusted to relation id field name, a.k.a. `field_name + _id`. +func (c *collection) checkExistingFieldsAndAdjustRelFieldNames( fields []client.IndexedFieldDescription, ) error { - collectionFields := c.Schema().Fields - for _, field := range fields { - found := false - for _, colField := range collectionFields { - if field.Name == colField.Name { - found = true - break - } - } + for i := range fields { + field, found := c.Schema().GetFieldByName(fields[i].Name) if !found { - return NewErrNonExistingFieldForIndex(field.Name) + return NewErrNonExistingFieldForIndex(fields[i].Name) + } + if field.Kind.IsObject() { + fields[i].Name = fields[i].Name + request.RelatedObjectID } } return nil diff --git a/internal/db/index.go b/internal/db/index.go index 71569e64db..bd11e9f94b 100644 --- a/internal/db/index.go +++ b/internal/db/index.go @@ -41,7 +41,7 @@ func getValidateIndexFieldFunc(kind client.FieldKind) func(any) bool { } switch kind { - case client.FieldKind_NILLABLE_STRING: + case client.FieldKind_NILLABLE_STRING, client.FieldKind_DocID: return canConvertIndexFieldValue[string] case client.FieldKind_NILLABLE_INT: return canConvertIndexFieldValue[int64] diff --git a/internal/planner/explain.go b/internal/planner/explain.go index 5ab2f292f8..f6d3f57209 100644 --- a/internal/planner/explain.go +++ b/internal/planner/explain.go @@ -92,8 +92,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) { var explainGraphBuilder = map[string]any{} // If root is not the last child then keep walking and explaining the root graph. - if node.root != nil { - indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.root) + if node.parentSide.plan != nil { + indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.parentSide.plan) if err != nil { return nil, err } @@ -101,8 +101,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) { explainGraphBuilder[joinRootLabel] = indexJoinRootExplainGraph } - if node.subType != nil { - indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.subType) + if node.childSide.plan != nil { + indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.childSide.plan) if err != nil { return nil, err } @@ -117,8 +117,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) { var explainGraphBuilder = map[string]any{} // If root is not the last child then keep walking and explaining the root graph. - if node.root != nil { - indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.root) + if node.parentSide.plan != nil { + indexJoinRootExplainGraph, err := buildDebugExplainGraph(node.parentSide.plan) if err != nil { return nil, err } @@ -128,8 +128,8 @@ func buildDebugExplainGraph(source planNode) (map[string]any, error) { explainGraphBuilder[joinRootLabel] = nil } - if node.subType != nil { - indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.subType) + if node.childSide.plan != nil { + indexJoinSubTypeExplainGraph, err := buildDebugExplainGraph(node.childSide.plan) if err != nil { return nil, err } diff --git a/internal/planner/mapper/mapper.go b/internal/planner/mapper/mapper.go index 07ec0db8e6..be52066b54 100644 --- a/internal/planner/mapper/mapper.go +++ b/internal/planner/mapper/mapper.go @@ -413,25 +413,23 @@ func resolveAggregates( childMapping = childMapping.CloneWithoutRender() mapping.SetChildAt(index, childMapping) - if !childIsMapped { - filterDependencies, err := resolveFilterDependencies( - ctx, - store, - rootSelectType, - childCollectionName, - target.filter, - mapping.ChildMappings[index], - childFields, - ) - if err != nil { - return nil, err - } - childFields = append(childFields, filterDependencies...) - - // If the child was not mapped, the filter will not have been converted yet - // so we must do that now. - convertedFilter = ToFilter(target.filter.Value(), mapping.ChildMappings[index]) + filterDependencies, err := resolveFilterDependencies( + ctx, + store, + rootSelectType, + childCollectionName, + target.filter, + mapping.ChildMappings[index], + childFields, + ) + if err != nil { + return nil, err } + childFields = append(childFields, filterDependencies...) + + // If the child was not mapped, the filter will not have been converted yet + // so we must do that now. + convertedFilter = ToFilter(target.filter.Value(), mapping.ChildMappings[index]) dummyJoin := &Select{ Targetable: Targetable{ @@ -989,6 +987,7 @@ func resolveInnerFilterDependencies( return nil, err } + childSelect.SkipResolve = true newFields = append(newFields, childSelect) } diff --git a/internal/planner/mapper/select.go b/internal/planner/mapper/select.go index 8b67b60937..5d9e8d39f0 100644 --- a/internal/planner/mapper/select.go +++ b/internal/planner/mapper/select.go @@ -38,6 +38,11 @@ type Select struct { // These can include stuff such as version information, aggregates, and other // Selects. Fields []Requestable + + // SkipResolve is a flag that indicates that the fields in this Select don't need to be resolved, + // i.e. it's value doesn't need to be fetched and provided to the user. + // It is used to avoid resolving related objects if they are used only in a filter and not requested in a response. + SkipResolve bool } func (s *Select) AsTargetable() (*Targetable, bool) { diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 13e1e0b2e9..f7a875af70 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -357,18 +357,20 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan parentPlan.selectNode.filter.Conditions, node.documentMapping, ) - slct := node.subType.(*selectTopNode).selectNode + slct := node.childSide.plan.(*selectTopNode).selectNode desc := slct.collection.Description() for subFieldName, subFieldInd := range filteredSubFields { indexes := desc.GetIndexesOnField(subFieldName) if len(indexes) > 0 && !filter.IsComplex(parentPlan.selectNode.filter) { - subInd := node.documentMapping.FirstIndexOfName(node.subTypeName) - relatedField := mapper.Field{Name: node.subTypeName, Index: subInd} + subInd := node.documentMapping.FirstIndexOfName(node.parentSide.relFieldDef.Name) + relatedField := mapper.Field{Name: node.parentSide.relFieldDef.Name, Index: subInd} fieldFilter := filter.UnwrapRelation(filter.CopyField( parentPlan.selectNode.filter, relatedField, mapper.Field{Name: subFieldName, Index: subFieldInd}, ), relatedField) + // At the moment we just take the first index, but later we want to run some kind of analysis to + // determine which index is best to use. https://github.com/sourcenetwork/defradb/issues/2680 err := node.invertJoinDirectionWithIndex(fieldFilter, indexes[0]) if err != nil { return err @@ -383,7 +385,7 @@ func (p *Planner) tryOptimizeJoinDirection(node *invertibleTypeJoin, parentPlan // expandTypeJoin does a plan graph expansion and other optimizations on invertibleTypeJoin. func (p *Planner) expandTypeJoin(node *invertibleTypeJoin, parentPlan *selectTopNode) error { if parentPlan.selectNode.filter == nil { - return p.expandPlan(node.subType, parentPlan) + return p.expandPlan(node.childSide.plan, parentPlan) } err := p.tryOptimizeJoinDirection(node, parentPlan) @@ -391,7 +393,7 @@ func (p *Planner) expandTypeJoin(node *invertibleTypeJoin, parentPlan *selectTop return err } - return p.expandPlan(node.subType, parentPlan) + return p.expandPlan(node.childSide.plan, parentPlan) } func (p *Planner) expandGroupNodePlan(topNodeSelect *selectTopNode) error { diff --git a/internal/planner/select.go b/internal/planner/select.go index c3e7f4dd2d..5d7e448c73 100644 --- a/internal/planner/select.go +++ b/internal/planner/select.go @@ -315,6 +315,21 @@ func findIndexByFilteringField(scanNode *scanNode) immutable.Option[client.Index return immutable.None[client.IndexDescription]() } +func findIndexByFieldName(col client.Collection, fieldName string) immutable.Option[client.IndexDescription] { + for _, field := range col.Schema().Fields { + if field.Name != fieldName { + continue + } + indexes := col.Description().GetIndexesOnField(field.Name) + if len(indexes) > 0 { + // At the moment we just take the first index, but later we want to run some kind of analysis to + // determine which index is best to use. https://github.com/sourcenetwork/defradb/issues/2680 + return immutable.Some(indexes[0]) + } + } + return immutable.None[client.IndexDescription]() +} + func (n *selectNode) initFields(selectReq *mapper.Select) ([]aggregateNode, error) { aggregates := []aggregateNode{} // loop over the sub type diff --git a/internal/planner/type_join.go b/internal/planner/type_join.go index f745e3c5cf..cd20ad3c8d 100644 --- a/internal/planner/type_join.go +++ b/internal/planner/type_join.go @@ -147,45 +147,41 @@ func (n *typeIndexJoin) simpleExplain() (map[string]any, error) { // Add the type attribute. simpleExplainMap[joinTypeLabel] = n.joinPlan.Kind() - switch joinType := n.joinPlan.(type) { - case *typeJoinOne: - // Add the direction attribute. - if joinType.isSecondary { - simpleExplainMap[joinDirectionLabel] = joinDirectionSecondaryLabel - } else { - simpleExplainMap[joinDirectionLabel] = joinDirectionPrimaryLabel - } - + addExplainData := func(j *invertibleTypeJoin) error { // Add the attribute(s). - simpleExplainMap[joinRootLabel] = joinType.rootName - simpleExplainMap[joinSubTypeNameLabel] = joinType.subTypeName + simpleExplainMap[joinRootLabel] = immutable.Some(j.childSide.relFieldDef.Name) + simpleExplainMap[joinSubTypeNameLabel] = j.parentSide.relFieldDef.Name - subTypeExplainGraph, err := buildSimpleExplainGraph(joinType.subType) + subTypeExplainGraph, err := buildSimpleExplainGraph(j.childSide.plan) if err != nil { - return nil, err + return err } // Add the joined (subType) type's entire explain graph. simpleExplainMap[joinSubTypeLabel] = subTypeExplainGraph + return nil + } - case *typeJoinMany: - // Add the attribute(s). - simpleExplainMap[joinRootLabel] = joinType.rootName - simpleExplainMap[joinSubTypeNameLabel] = joinType.subTypeName - - subTypeExplainGraph, err := buildSimpleExplainGraph(joinType.subType) - if err != nil { - return nil, err + var err error + switch joinType := n.joinPlan.(type) { + case *typeJoinOne: + // Add the direction attribute. + if joinType.parentSide.isPrimary() { + simpleExplainMap[joinDirectionLabel] = joinDirectionPrimaryLabel + } else { + simpleExplainMap[joinDirectionLabel] = joinDirectionSecondaryLabel } - // Add the joined (subType) type's entire explain graph. - simpleExplainMap[joinSubTypeLabel] = subTypeExplainGraph + err = addExplainData(&joinType.invertibleTypeJoin) + + case *typeJoinMany: + err = addExplainData(&joinType.invertibleTypeJoin) default: - return simpleExplainMap, client.NewErrUnhandledType("join plan", n.joinPlan) + err = client.NewErrUnhandledType("join plan", n.joinPlan) } - return simpleExplainMap, nil + return simpleExplainMap, err } // Explain method returns a map containing all attributes of this node that @@ -201,10 +197,10 @@ func (n *typeIndexJoin) Explain(explainType request.ExplainType) (map[string]any } var subScan *scanNode if joinMany, isJoinMany := n.joinPlan.(*typeJoinMany); isJoinMany { - subScan = getScanNode(joinMany.subType) + subScan = getScanNode(joinMany.childSide.plan) } if joinOne, isJoinOne := n.joinPlan.(*typeJoinOne); isJoinOne { - subScan = getScanNode(joinOne.subType) + subScan = getScanNode(joinOne.childSide.plan) } if subScan != nil { subScanExplain, err := subScan.Explain(explainType) @@ -228,100 +224,38 @@ type typeJoinOne struct { func (p *Planner) makeTypeJoinOne( parent *selectNode, - source planNode, - subType *mapper.Select, + sourcePlan planNode, + subSelect *mapper.Select, ) (*typeJoinOne, error) { - prepareScanNodeFilterForTypeJoin(parent, source, subType) - - selectPlan, err := p.Select(subType) + invertibleTypeJoin, err := p.newInvertableTypeJoin(parent, sourcePlan, subSelect) if err != nil { return nil, err } - - // get the correct sub field schema type (collection) - subTypeFieldDesc, ok := parent.collection.Definition().GetFieldByName(subType.Name) - if !ok { - return nil, client.NewErrFieldNotExist(subType.Name) - } - - subTypeCol, err := p.db.GetCollectionByName(p.ctx, subType.CollectionName) - if err != nil { - return nil, err - } - - subTypeField, subTypeFieldNameFound := subTypeCol.Description().GetFieldByRelation( - subTypeFieldDesc.RelationName, - parent.collection.Name().Value(), - subTypeFieldDesc.Name, - ) - if !subTypeFieldNameFound { - return nil, client.NewErrFieldNotExist(subTypeFieldDesc.RelationName) - } - - var secondaryFieldIndex immutable.Option[int] - if !subTypeFieldDesc.IsPrimaryRelation { - idFieldName := subTypeFieldDesc.Name + request.RelatedObjectID - secondaryFieldIndex = immutable.Some( - parent.documentMapping.FirstIndexOfName(idFieldName), - ) - } - - dir := joinDirection{ - firstNode: source, - secondNode: selectPlan, - secondaryField: immutable.Some(subTypeField.Name + request.RelatedObjectID), - primaryField: subTypeFieldDesc.Name + request.RelatedObjectID, - } - - return &typeJoinOne{ - invertibleTypeJoin: invertibleTypeJoin{ - docMapper: docMapper{parent.documentMapping}, - root: source, - subType: selectPlan, - subSelect: subType, - subSelectFieldDef: subTypeFieldDesc, - rootName: immutable.Some(subTypeField.Name), - subTypeName: subType.Name, - isSecondary: !subTypeFieldDesc.IsPrimaryRelation, - secondaryFieldIndex: secondaryFieldIndex, - secondaryFetchLimit: 1, - dir: dir, - }, - }, nil + invertibleTypeJoin.secondaryFetchLimit = 1 + return &typeJoinOne{invertibleTypeJoin: invertibleTypeJoin}, nil } func (n *typeJoinOne) Kind() string { return "typeJoinOne" } -func fetchDocsWithFieldValue(plan planNode, fieldName string, val any) ([]core.Doc, error) { - propIndex := plan.DocumentMap().FirstIndexOfName(fieldName) - setSubTypeFilterToScanNode(plan, propIndex, val) - - if err := plan.Init(); err != nil { - return nil, NewErrSubTypeInit(err) - } - - var docs []core.Doc - for { - next, err := plan.Next() - if err != nil { - return nil, err - } - if !next { - break - } - - docs = append(docs, plan.Value()) - } - - return docs, nil -} - type typeJoinMany struct { invertibleTypeJoin } +func (p *Planner) makeTypeJoinMany( + parent *selectNode, + sourcePlan planNode, + subSelect *mapper.Select, +) (*typeJoinMany, error) { + invertibleTypeJoin, err := p.newInvertableTypeJoin(parent, sourcePlan, subSelect) + if err != nil { + return nil, err + } + invertibleTypeJoin.secondaryFetchLimit = 0 + return &typeJoinMany{invertibleTypeJoin: invertibleTypeJoin}, nil +} + func prepareScanNodeFilterForTypeJoin( parent *selectNode, source planNode, @@ -357,83 +291,149 @@ func prepareScanNodeFilterForTypeJoin( } } -func (p *Planner) makeTypeJoinMany( +func (p *Planner) newInvertableTypeJoin( parent *selectNode, - source planNode, - subType *mapper.Select, -) (*typeJoinMany, error) { - prepareScanNodeFilterForTypeJoin(parent, source, subType) + sourcePlan planNode, + subSelect *mapper.Select, +) (invertibleTypeJoin, error) { + prepareScanNodeFilterForTypeJoin(parent, sourcePlan, subSelect) - selectPlan, err := p.Select(subType) + subSelectPlan, err := p.Select(subSelect) if err != nil { - return nil, err + return invertibleTypeJoin{}, err } - subTypeFieldDesc, ok := parent.collection.Definition().GetFieldByName(subType.Name) + parentsRelFieldDef, ok := parent.collection.Definition().GetFieldByName(subSelect.Name) if !ok { - return nil, client.NewErrFieldNotExist(subType.Name) + return invertibleTypeJoin{}, client.NewErrFieldNotExist(subSelect.Name) } - subTypeCol, err := p.db.GetCollectionByName(p.ctx, subType.CollectionName) + skipChild := false + for _, field := range parent.selectReq.Fields { + if field.GetName() == subSelect.Name { + if childSelect, ok := field.AsSelect(); ok { + if childSelect.SkipResolve { + skipChild = true + } + } + break + } + } + + subCol, err := p.db.GetCollectionByName(p.ctx, subSelect.CollectionName) if err != nil { - return nil, err + return invertibleTypeJoin{}, err } - var secondaryFieldName immutable.Option[string] - var rootName immutable.Option[string] - if subTypeFieldDesc.RelationName != "" { - rootField, rootNameFound := subTypeCol.Description().GetFieldByRelation( - subTypeFieldDesc.RelationName, - parent.collection.Name().Value(), - subTypeFieldDesc.Name, - ) - if rootNameFound { - rootName = immutable.Some(rootField.Name) - secondaryFieldName = immutable.Some(rootField.Name + request.RelatedObjectID) - } + childsRelFieldDesc, ok := subCol.Description().GetFieldByRelation( + parentsRelFieldDef.RelationName, + parent.collection.Name().Value(), + parentsRelFieldDef.Name, + ) + if !ok { + return invertibleTypeJoin{}, client.NewErrFieldNotExist(parentsRelFieldDef.Name) } - dir := joinDirection{ - firstNode: source, - secondNode: selectPlan, - secondaryField: secondaryFieldName, - primaryField: subTypeFieldDesc.Name + request.RelatedObjectID, - } - - return &typeJoinMany{ - invertibleTypeJoin: invertibleTypeJoin{ - docMapper: docMapper{parent.documentMapping}, - root: source, - subType: selectPlan, - subSelect: subType, - subSelectFieldDef: subTypeFieldDesc, - rootName: rootName, - isSecondary: true, - subTypeName: subType.Name, - secondaryFetchLimit: 0, - dir: dir, - }, + childsRelFieldDef, ok := subCol.Definition().GetFieldByName(childsRelFieldDesc.Name) + if !ok { + return invertibleTypeJoin{}, client.NewErrFieldNotExist(subSelect.Name) + } + + parentSide := joinSide{ + plan: sourcePlan, + relFieldDef: parentsRelFieldDef, + relFieldMapIndex: immutable.Some(subSelect.Index), + col: parent.collection, + isFirst: true, + isParent: true, + } + + ind := parent.documentMapping.IndexesByName[parentsRelFieldDef.Name+request.RelatedObjectID] + if len(ind) > 0 { + parentSide.relIDFieldMapIndex = immutable.Some(ind[0]) + } + + childSide := joinSide{ + plan: subSelectPlan, + relFieldDef: childsRelFieldDef, + col: subCol, + isFirst: false, + isParent: false, + } + + ind = subSelectPlan.DocumentMap().IndexesByName[childsRelFieldDef.Name+request.RelatedObjectID] + if len(ind) > 0 { + childSide.relIDFieldMapIndex = immutable.Some(ind[0]) + } + + return invertibleTypeJoin{ + docMapper: docMapper{parent.documentMapping}, + parentSide: parentSide, + childSide: childSide, + skipChild: skipChild, }, nil } -func (n *typeJoinMany) Kind() string { - return "typeJoinMany" +type joinSide struct { + plan planNode + relFieldDef client.FieldDefinition + relFieldMapIndex immutable.Option[int] + relIDFieldMapIndex immutable.Option[int] + col client.Collection + isFirst bool + isParent bool } -func fetchPrimaryDoc(node, subNode planNode, parentProp string) (bool, error) { - subDoc := subNode.Value() - ind := subNode.DocumentMap().FirstIndexOfName(parentProp) +func (s *joinSide) isPrimary() bool { + return s.relFieldDef.IsPrimaryRelation +} - docIDStr, isStr := subDoc.Fields[ind].(string) - if !isStr { - return false, nil +func (join *invertibleTypeJoin) getFirstSide() *joinSide { + if join.parentSide.isFirst { + return &join.parentSide } + return &join.childSide +} + +func (join *invertibleTypeJoin) getSecondSide() *joinSide { + if !join.parentSide.isFirst { + return &join.parentSide + } + return &join.childSide +} +func (join *invertibleTypeJoin) getPrimarySide() *joinSide { + if join.parentSide.isPrimary() { + return &join.parentSide + } + return &join.childSide +} + +func (join *invertibleTypeJoin) getSecondarySide() *joinSide { + if !join.parentSide.isPrimary() { + return &join.parentSide + } + return &join.childSide +} + +func (n *typeJoinMany) Kind() string { + return "typeJoinMany" +} + +// getForeignKey returns the docID of the related object referenced by the given relation field. +func getForeignKey(node planNode, relFieldName string) string { + ind := node.DocumentMap().FirstIndexOfName(relFieldName + request.RelatedObjectID) + docIDStr, _ := node.Value().Fields[ind].(string) + return docIDStr +} + +// fetchDocWithID fetches a document with the given docID from the given planNode. +func fetchDocWithID(node planNode, docID string) (bool, error) { scan := getScanNode(node) if scan == nil { return false, nil } - dsKey := base.MakeDataStoreKeyWithCollectionAndDocID(scan.col.Description(), docIDStr) + dsKey := base.MakeDataStoreKeyWithCollectionAndDocID(scan.col.Description(), docID) spans := core.NewSpans(core.NewSpan(dsKey, dsKey.PrefixEnd())) @@ -452,108 +452,206 @@ func fetchPrimaryDoc(node, subNode planNode, parentProp string) (bool, error) { return true, nil } -type joinDirection struct { - firstNode planNode - secondNode planNode - secondaryField immutable.Option[string] - primaryField string - isInverted bool -} - -func (dir *joinDirection) invert() { - if !dir.secondaryField.HasValue() { - // If the secondary field has no value it cannot be inverted - return - } - dir.isInverted = !dir.isInverted - dir.firstNode, dir.secondNode = dir.secondNode, dir.firstNode - dir.secondaryField, dir.primaryField = immutable.Some(dir.primaryField), dir.secondaryField.Value() -} - type invertibleTypeJoin struct { docMapper - root planNode - subType planNode - rootName immutable.Option[string] - subTypeName string + skipChild bool - subSelect *mapper.Select - subSelectFieldDef client.FieldDefinition + parentSide joinSide + childSide joinSide - isSecondary bool - secondaryFieldIndex immutable.Option[int] secondaryFetchLimit uint // docsToYield contains documents read and ready to be yielded by this node. - docsToYield []core.Doc - - dir joinDirection + docsToYield []core.Doc + encounteredDocIDs []string } func (join *invertibleTypeJoin) replaceRoot(node planNode) { - join.root = node - if join.dir.isInverted { - join.dir.secondNode = node - } else { - join.dir.firstNode = node - } + join.getFirstSide().plan = node } func (join *invertibleTypeJoin) Init() error { - if err := join.subType.Init(); err != nil { + if err := join.childSide.plan.Init(); err != nil { return err } - return join.root.Init() + return join.parentSide.plan.Init() } func (join *invertibleTypeJoin) Start() error { - if err := join.subType.Start(); err != nil { + if err := join.childSide.plan.Start(); err != nil { return err } - return join.root.Start() + return join.parentSide.plan.Start() } func (join *invertibleTypeJoin) Close() error { - if err := join.root.Close(); err != nil { + if err := join.parentSide.plan.Close(); err != nil { return err } - return join.subType.Close() + return join.childSide.plan.Close() } func (join *invertibleTypeJoin) Spans(spans core.Spans) { - join.root.Spans(spans) + join.parentSide.plan.Spans(spans) +} + +func (join *invertibleTypeJoin) Source() planNode { return join.parentSide.plan } + +type primaryObjectsRetriever struct { + relIDFieldDef client.FieldDefinition + primarySide *joinSide + secondarySide *joinSide + + primaryScan *scanNode + + resultPrimaryDocs []core.Doc + resultSecondaryDoc core.Doc +} + +func newPrimaryObjectsRetriever( + primarySide, secondarySide *joinSide, +) primaryObjectsRetriever { + j := primaryObjectsRetriever{ + primarySide: primarySide, + secondarySide: secondarySide, + } + return j } -func (join *invertibleTypeJoin) Source() planNode { return join.root } +func (j *primaryObjectsRetriever) retrievePrimaryDocsReferencingSecondaryDoc() error { + relIDFieldDef, ok := j.primarySide.col.Definition().GetFieldByName( + j.primarySide.relFieldDef.Name + request.RelatedObjectID) + if !ok { + return client.NewErrFieldNotExist(j.primarySide.relFieldDef.Name + request.RelatedObjectID) + } + + j.primaryScan = getScanNode(j.primarySide.plan) + + j.relIDFieldDef = relIDFieldDef -func (tj *invertibleTypeJoin) invert() { - tj.dir.invert() - tj.isSecondary = !tj.isSecondary + primaryDocs, err := j.retrievePrimaryDocs() + + if err != nil { + return err + } + + j.resultPrimaryDocs, j.resultSecondaryDoc = joinPrimaryDocs(primaryDocs, j.secondarySide, j.primarySide) + + return nil } -func (join *invertibleTypeJoin) processSecondResult(secondDocs []core.Doc) (any, any) { - var secondResult any - var secondIDResult any - if join.secondaryFetchLimit == 1 { - if len(secondDocs) != 0 { - secondResult = secondDocs[0] - secondIDResult = secondDocs[0].GetID() +func (j *primaryObjectsRetriever) addIDFieldToScanner() { + found := false + for i := range j.primaryScan.fields { + if j.primaryScan.fields[i].Name == j.relIDFieldDef.Name { + found = true + break } - } else { - secondResult = secondDocs - secondDocIDs := make([]string, len(secondDocs)) - for i, doc := range secondDocs { - secondDocIDs[i] = doc.GetID() + } + if !found { + j.primaryScan.fields = append(j.primaryScan.fields, j.relIDFieldDef) + } +} + +func (j *primaryObjectsRetriever) collectDocs(numDocs int) ([]core.Doc, error) { + p := j.primarySide.plan + if err := p.Init(); err != nil { + return nil, NewErrSubTypeInit(err) + } + + docs := make([]core.Doc, 0, numDocs) + + for { + hasValue, err := p.Next() + + if err != nil { + return nil, err } - secondIDResult = secondDocIDs + + if !hasValue { + break + } + + docs = append(docs, p.Value()) } - join.root.Value().Fields[join.subSelect.Index] = secondResult - if join.secondaryFieldIndex.HasValue() { - join.root.Value().Fields[join.secondaryFieldIndex.Value()] = secondIDResult + + return docs, nil +} + +func (j *primaryObjectsRetriever) retrievePrimaryDocs() ([]core.Doc, error) { + j.addIDFieldToScanner() + + secondaryDoc := j.secondarySide.plan.Value() + addFilterOnIDField(j.primaryScan, j.primarySide.relIDFieldMapIndex.Value(), secondaryDoc.GetID()) + + oldFetcher := j.primaryScan.fetcher + + indexOnRelation := findIndexByFieldName(j.primaryScan.col, j.relIDFieldDef.Name) + j.primaryScan.initFetcher(immutable.None[string](), indexOnRelation) + + docs, err := j.collectDocs(0) + if err != nil { + return nil, err } - return secondResult, secondIDResult + + err = j.primaryScan.fetcher.Close() + if err != nil { + return nil, err + } + + j.primaryScan.fetcher = oldFetcher + + return docs, nil +} + +func docsToDocIDs(docs []core.Doc) []string { + docIDs := make([]string, len(docs)) + for i, doc := range docs { + docIDs[i] = doc.GetID() + } + return docIDs +} + +func joinPrimaryDocs(primaryDocs []core.Doc, secondarySide, primarySide *joinSide) ([]core.Doc, core.Doc) { + secondaryDoc := secondarySide.plan.Value() + + if secondarySide.relFieldMapIndex.HasValue() { + if secondarySide.relFieldDef.Kind.IsArray() { + secondaryDoc.Fields[secondarySide.relFieldMapIndex.Value()] = primaryDocs + } else if len(primaryDocs) > 0 { + secondaryDoc.Fields[secondarySide.relFieldMapIndex.Value()] = primaryDocs[0] + } + } + + if secondarySide.relIDFieldMapIndex.HasValue() { + if secondarySide.relFieldDef.Kind.IsArray() { + secondaryDoc.Fields[secondarySide.relIDFieldMapIndex.Value()] = docsToDocIDs(primaryDocs) + } else if len(primaryDocs) > 0 { + secondaryDoc.Fields[secondarySide.relIDFieldMapIndex.Value()] = primaryDocs[0].GetID() + } + } + + if primarySide.relFieldMapIndex.HasValue() { + for i := range primaryDocs { + primaryDocs[i].Fields[primarySide.relFieldMapIndex.Value()] = secondaryDoc + } + } + + if primarySide.relIDFieldMapIndex.HasValue() { + for i := range primaryDocs { + primaryDocs[i].Fields[primarySide.relIDFieldMapIndex.Value()] = secondaryDoc.GetID() + } + } + + return primaryDocs, secondaryDoc +} + +func (join *invertibleTypeJoin) fetchPrimaryDocsReferencingSecondaryDoc() ([]core.Doc, core.Doc, error) { + retriever := newPrimaryObjectsRetriever(join.getPrimarySide(), join.getSecondarySide()) + err := retriever.retrievePrimaryDocsReferencingSecondaryDoc() + return retriever.resultPrimaryDocs, retriever.resultSecondaryDoc, err } func (join *invertibleTypeJoin) Next() (bool, error) { @@ -568,54 +666,86 @@ func (join *invertibleTypeJoin) Next() (bool, error) { } } - hasFirstValue, err := join.dir.firstNode.Next() + firstSide := join.getFirstSide() + hasFirstValue, err := firstSide.plan.Next() if err != nil || !hasFirstValue { return false, err } - firstDoc := join.dir.firstNode.Value() - - if join.isSecondary { - secondDocs, err := fetchDocsWithFieldValue( - join.dir.secondNode, - // As the join is from the secondary field, we know that [join.dir.secondaryField] must have a value - // otherwise the user would not have been able to request it. - join.dir.secondaryField.Value(), - firstDoc.GetID(), - ) + if firstSide.isPrimary() { + return join.nextJoinedSecondaryDoc() + } else { + primaryDocs, secondaryDoc, err := join.fetchPrimaryDocsReferencingSecondaryDoc() if err != nil { return false, err } - if join.dir.secondNode == join.root { - if len(secondDocs) == 0 { - return false, nil - } - for i := range secondDocs { - secondDocs[i].Fields[join.subSelect.Index] = join.subType.Value() - } - join.docsToYield = append(join.docsToYield, secondDocs...) - return true, nil + if join.parentSide.isPrimary() { + join.docsToYield = append(join.docsToYield, primaryDocs...) } else { - secondResult, secondIDResult := join.processSecondResult(secondDocs) - join.dir.firstNode.Value().Fields[join.subSelect.Index] = secondResult - if join.secondaryFieldIndex.HasValue() { - join.dir.firstNode.Value().Fields[join.secondaryFieldIndex.Value()] = secondIDResult - } + join.docsToYield = append(join.docsToYield, secondaryDoc) } - } else { - hasDoc, err := fetchPrimaryDoc(join.dir.secondNode, join.dir.firstNode, join.dir.primaryField) - if err != nil { - return false, err + } + + return true, nil +} + +func (join *invertibleTypeJoin) nextJoinedSecondaryDoc() (bool, error) { + firstSide := join.getFirstSide() + secondSide := join.getSecondSide() + + secondaryDocID := getForeignKey(firstSide.plan, firstSide.relFieldDef.Name) + if secondaryDocID == "" { + if firstSide.isParent { + join.docsToYield = append(join.docsToYield, firstSide.plan.Value()) + return true, nil } + return join.Next() + } - if hasDoc { - join.root.Value().Fields[join.subSelect.Index] = join.subType.Value() + if !firstSide.isParent { + for i := range join.encounteredDocIDs { + if join.encounteredDocIDs[i] == secondaryDocID { + return join.Next() + } } + join.encounteredDocIDs = append(join.encounteredDocIDs, secondaryDocID) } - join.docsToYield = append(join.docsToYield, join.root.Value()) + hasDoc, err := fetchDocWithID(secondSide.plan, secondaryDocID) + if err != nil { + return false, err + } + if !hasDoc { + if firstSide.isParent { + join.docsToYield = append(join.docsToYield, firstSide.plan.Value()) + return true, nil + } + return join.Next() + } + + if join.parentSide.relFieldDef.Kind.IsArray() { + var primaryDocs []core.Doc + var secondaryDoc core.Doc + // if child is not requested as part of the response, we just add the existing one (fetched by the secondary index + // on a filtered value) so that top select node that runs the filter again can yield it. + if join.skipChild { + primaryDocs, secondaryDoc = joinPrimaryDocs([]core.Doc{firstSide.plan.Value()}, secondSide, firstSide) + } else { + primaryDocs, secondaryDoc, err = join.fetchPrimaryDocsReferencingSecondaryDoc() + if err != nil { + return false, err + } + } + secondaryDoc.Fields[join.parentSide.relFieldMapIndex.Value()] = primaryDocs + + join.docsToYield = append(join.docsToYield, secondaryDoc) + } else { + parentDoc := join.parentSide.plan.Value() + parentDoc.Fields[join.parentSide.relFieldMapIndex.Value()] = join.childSide.plan.Value() + join.docsToYield = append(join.docsToYield, parentDoc) + } return true, nil } @@ -630,26 +760,19 @@ func (join *invertibleTypeJoin) invertJoinDirectionWithIndex( fieldFilter *mapper.Filter, index client.IndexDescription, ) error { - if !join.rootName.HasValue() { - // If the root field has no value it cannot be inverted - return nil - } - if join.subSelectFieldDef.Kind.IsArray() { - // invertibleTypeJoin does not support inverting one-many relations atm - return nil - } - subScan := getScanNode(join.subType) - subScan.tryAddField(join.rootName.Value() + request.RelatedObjectID) - subScan.filter = fieldFilter - subScan.initFetcher(immutable.Option[string]{}, immutable.Some(index)) + p := join.childSide.plan + s := getScanNode(p) + s.tryAddField(join.childSide.relFieldDef.Name + request.RelatedObjectID) + s.filter = fieldFilter + s.initFetcher(immutable.Option[string]{}, immutable.Some(index)) - join.invert() + join.childSide.isFirst = join.parentSide.isFirst + join.parentSide.isFirst = !join.parentSide.isFirst return nil } -func setSubTypeFilterToScanNode(plan planNode, propIndex int, val any) { - scan := getScanNode(plan) +func addFilterOnIDField(scan *scanNode, propIndex int, val any) { if scan == nil { return } diff --git a/tests/integration/explain/execute/with_count_test.go b/tests/integration/explain/execute/with_count_test.go index 4a30b9f52a..43ff3d13df 100644 --- a/tests/integration/explain/execute/with_count_test.go +++ b/tests/integration/explain/execute/with_count_test.go @@ -62,7 +62,7 @@ func TestExecuteExplainRequestWithCountOnOneToManyRelation(t *testing.T) { "subTypeScanNode": dataMap{ "iterations": uint64(5), "docFetches": uint64(6), - "fieldFetches": uint64(14), + "fieldFetches": uint64(6), "indexFetches": uint64(0), }, }, diff --git a/tests/integration/index/query_with_relation_filter_test.go b/tests/integration/index/query_with_relation_filter_test.go index aa49dd2623..94160a5e3c 100644 --- a/tests/integration/index/query_with_relation_filter_test.go +++ b/tests/integration/index/query_with_relation_filter_test.go @@ -17,6 +17,7 @@ import ( ) func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilter2(t *testing.T) { + // 3 users have a MacBook Pro: Islam, Shahzad, Keenan req1 := `query { User(filter: { devices: {model: {_eq: "MacBook Pro"}} @@ -24,6 +25,7 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte name } }` + // 1 user has an iPhone 10: Addo req2 := `query { User(filter: { devices: {model: {_eq: "iPhone 10"}} @@ -53,16 +55,14 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte testUtils.Request{ Request: req1, Results: []map[string]any{ - {"name": "Keenan"}, {"name": "Islam"}, {"name": "Shahzad"}, + {"name": "Keenan"}, }, }, testUtils.Request{ - Request: makeExplainQuery(req1), - // The invertable join does not support inverting one-many relations, so the index is - // not used. - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(450).WithIndexFetches(0), + Request: makeExplainQuery(req1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(6).WithIndexFetches(3), }, testUtils.Request{ Request: req2, @@ -71,10 +71,8 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte }, }, testUtils.Request{ - Request: makeExplainQuery(req2), - // The invertable join does not support inverting one-many relations, so the index is - // not used. - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(450).WithIndexFetches(0), + Request: makeExplainQuery(req2), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(1), }, }, } @@ -83,6 +81,7 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte } func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilter(t *testing.T) { + // 3 users have a MacBook Pro: Islam, Shahzad, Keenan req1 := `query { User(filter: { devices: {model: {_eq: "MacBook Pro"}} @@ -90,6 +89,7 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte name } }` + // 1 user has an iPhone 10: Addo req2 := `query { User(filter: { devices: {model: {_eq: "iPhone 10"}} @@ -119,16 +119,14 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte testUtils.Request{ Request: req1, Results: []map[string]any{ - {"name": "Keenan"}, {"name": "Islam"}, {"name": "Shahzad"}, + {"name": "Keenan"}, }, }, testUtils.Request{ - Request: makeExplainQuery(req1), - // The invertable join does not support inverting one-many relations, so the index is - // not used. - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(450).WithIndexFetches(0), + Request: makeExplainQuery(req1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(6).WithIndexFetches(3), }, testUtils.Request{ Request: req2, @@ -137,10 +135,8 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte }, }, testUtils.Request{ - Request: makeExplainQuery(req2), - // The invertable join does not support inverting one-many relations, so the index is - // not used. - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(450).WithIndexFetches(0), + Request: makeExplainQuery(req2), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(1), }, }, } @@ -149,6 +145,7 @@ func TestQueryWithIndexOnOneToManyRelation_IfFilterOnIndexedRelation_ShouldFilte } func TestQueryWithIndexOnOneToOnesSecondaryRelation_IfFilterOnIndexedRelation_ShouldFilter(t *testing.T) { + // 1 user lives in Munich: Islam req1 := `query { User(filter: { address: {city: {_eq: "Munich"}} @@ -156,6 +153,7 @@ func TestQueryWithIndexOnOneToOnesSecondaryRelation_IfFilterOnIndexedRelation_Sh name } }` + // 3 users live in Montreal: Shahzad, Fred, John req2 := `query { User(filter: { address: {city: {_eq: "Montreal"}} @@ -176,7 +174,7 @@ func TestQueryWithIndexOnOneToOnesSecondaryRelation_IfFilterOnIndexedRelation_Sh type Address { user: User @primary - city: String @index + city: String @index }`, }, testUtils.CreatePredefinedDocs{ @@ -210,7 +208,79 @@ func TestQueryWithIndexOnOneToOnesSecondaryRelation_IfFilterOnIndexedRelation_Sh testUtils.ExecuteTestCase(t, test) } +func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelationAndRelation_ShouldFilter(t *testing.T) { + // 1 user lives in London: Andy + req1 := `query { + User(filter: { + address: {city: {_eq: "London"}} + }) { + name + } + }` + // 3 users live in Montreal: Shahzad, Fred, John + req2 := `query { + User(filter: { + address: {city: {_eq: "Montreal"}} + }) { + name + } + }` + test := testUtils.TestCase{ + Description: "Filter on indexed field of primary relation in 1-1 relation", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + age: Int + address: Address @primary @index + } + + type Address { + user: User + city: String @index + street: String + }`, + }, + testUtils.CreatePredefinedDocs{ + Docs: getUserDocs(), + }, + testUtils.Request{ + Request: req1, + Results: []map[string]any{ + {"name": "Andy"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req1), + // we make 2 index fetches: 1. to get the only address with city == "London" + // and 2. to get the corresponding user + // then 1 field fetch to get the name of the user + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(2), + }, + testUtils.Request{ + Request: req2, + Results: []map[string]any{ + {"name": "John"}, + {"name": "Fred"}, + {"name": "Shahzad"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req2), + // we make 3 index fetches to get the 3 address with city == "Montreal" + // and 3 more index fetches to get the corresponding users + // then 3 field fetches to get the name of each user + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(3).WithIndexFetches(6), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelation_ShouldFilter(t *testing.T) { + // 1 user lives in London: Andy req1 := `query { User(filter: { address: {city: {_eq: "London"}} @@ -218,6 +288,7 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelatio name } }` + // 3 users live in Montreal: Shahzad, Fred, John req2 := `query { User(filter: { address: {city: {_eq: "Montreal"}} @@ -256,7 +327,6 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelatio // we make 1 index fetch to get the only address with city == "London" // then we scan all 10 users to find one with matching "address_id" // after this we fetch the name of the user - // it should be optimized after this is done https://github.com/sourcenetwork/defradb/issues/2601 Asserter: testUtils.NewExplainAsserter().WithFieldFetches(11).WithIndexFetches(1), }, testUtils.Request{ @@ -272,7 +342,6 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelatio // we make 3 index fetch to get the 3 address with city == "Montreal" // then we scan all 10 users to find one with matching "address_id" for each address // after this we fetch the name of each user - // it should be optimized after this is done https://github.com/sourcenetwork/defradb/issues/2601 Asserter: testUtils.NewExplainAsserter().WithFieldFetches(33).WithIndexFetches(3), }, }, @@ -282,6 +351,7 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedFieldOfRelatio } func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedRelationWhileIndexedForeignField_ShouldFilter(t *testing.T) { + // 1 user lives in London: Andy req := `query { User(filter: { address: {city: {_eq: "London"}} @@ -317,7 +387,7 @@ func TestQueryWithIndexOnOneToOnePrimaryRelation_IfFilterOnIndexedRelationWhileI }, testUtils.Request{ Request: makeExplainQuery(req), - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(11).WithIndexFetches(1), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(1).WithIndexFetches(2), }, }, } @@ -500,10 +570,8 @@ func TestQueryWithIndexOnOneToMany_IfFilterOnIndexedRelation_ShouldFilterWithExp }, }, testUtils.Request{ - Request: makeExplainQuery(req), - // The invertable join does not support inverting one-many relations, so the index is - // not used. - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(10).WithIndexFetches(0), + Request: makeExplainQuery(req), + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(14).WithIndexFetches(2), }, }, } @@ -512,6 +580,7 @@ func TestQueryWithIndexOnOneToMany_IfFilterOnIndexedRelation_ShouldFilterWithExp } func TestQueryWithIndexOnOneToOne_IfFilterOnIndexedRelation_ShouldFilter(t *testing.T) { + // 1 user lives in Munich: Islam req := `query { User(filter: { address: {city: {_eq: "Munich"}} @@ -563,8 +632,8 @@ func TestQueryWithIndexOnOneToOne_IfFilterOnIndexedRelation_ShouldFilter(t *test } func TestQueryWithIndexOnManyToOne_IfFilterOnIndexedField_ShouldFilterWithExplain(t *testing.T) { - // This query will fetch first a matching device which is secondary doc and therefore - // has a reference to the primary User doc. + // This query will fetch first a matching device which is primary doc and therefore + // has a reference to the secondary User doc. req := `query { Device(filter: { year: {_eq: 2021} @@ -633,7 +702,6 @@ func TestQueryWithIndexOnManyToOne_IfFilterOnIndexedField_ShouldFilterWithExplai func TestQueryWithIndexOnManyToOne_IfFilterOnIndexedRelation_ShouldFilterWithExplain(t *testing.T) { // This query will fetch first a matching user (owner) which is primary doc and therefore // has no direct reference to secondary Device docs. - // At the moment the db has to make a full scan of the Device docs to find the matching ones. // Keenan has 3 devices. req := `query { Device(filter: { @@ -650,11 +718,11 @@ func TestQueryWithIndexOnManyToOne_IfFilterOnIndexedRelation_ShouldFilterWithExp type User { name: String @index devices: [Device] - } + } type Device { - model: String - owner: User + model: String + owner: User @index } `, }, @@ -671,10 +739,171 @@ func TestQueryWithIndexOnManyToOne_IfFilterOnIndexedRelation_ShouldFilterWithExp }, testUtils.Request{ Request: makeExplainQuery(req), - // we make only 1 index fetch to get the owner by it's name - // and 44 field fetches to get 2 fields for all 22 devices in the db. - // it should be optimized after this is done https://github.com/sourcenetwork/defradb/issues/2601 - Asserter: testUtils.NewExplainAsserter().WithFieldFetches(44).WithIndexFetches(1), + // we make 1 index fetch to get the owner by it's name + // and 3 index fetches to get all 3 devices of the owner + // and 3 field fetches to get 1 'model' field for every fetched device. + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(3).WithIndexFetches(4), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithIndexOnOneToMany_IfIndexedRelationIsNil_NeNilFilterShouldUseIndex(t *testing.T) { + req := `query { + Device(filter: { + owner_id: {_ne: null} + }) { + model + } + }` + test := testUtils.TestCase{ + Description: "Filter on indexed relation field in 1-N relations", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + devices: [Device] + } + + type Device { + model: String + manufacturer: String + owner: User @index + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "Chris" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "Walkman", + "manufacturer": "Sony", + "owner": "bae-403d7337-f73e-5c81-8719-e853938c8985" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "iPhone", + "manufacturer": "Apple", + "owner": "bae-403d7337-f73e-5c81-8719-e853938c8985" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "Running Man", + "manufacturer": "Braveworld Productions" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "PlayStation 5", + "manufacturer": "Sony" + }`, + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"model": "iPhone"}, + {"model": "Walkman"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + // we make 4 index fetches to find 2 devices with owner_id != null + // and 2 field fetches to get 1 'model' field for every fetched device + // plus 2 more field fetches to get related User docs + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(4).WithIndexFetches(4), + }, + }, + } + + testUtils.ExecuteTestCase(t, test) +} + +func TestQueryWithIndexOnOneToMany_IfIndexedRelationIsNil_EqNilFilterShouldUseIndex(t *testing.T) { + req := `query { + Device(filter: { + owner_id: {_eq: null} + }) { + model + } + }` + test := testUtils.TestCase{ + Description: "Filter on indexed relation field in 1-N relations", + Actions: []any{ + testUtils.SchemaUpdate{ + Schema: ` + type User { + name: String + devices: [Device] + } + + type Device { + model: String + manufacturer: String + owner: User @index + } + `, + }, + testUtils.CreateDoc{ + CollectionID: 0, + Doc: `{ + "name": "Chris" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "Walkman", + "manufacturer": "Sony", + "owner": "bae-403d7337-f73e-5c81-8719-e853938c8985" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "iPhone", + "manufacturer": "Apple", + "owner": "bae-403d7337-f73e-5c81-8719-e853938c8985" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "Running Man", + "manufacturer": "Braveworld Productions" + }`, + }, + testUtils.CreateDoc{ + CollectionID: 1, + Doc: `{ + "model": "PlayStation 5", + "manufacturer": "Sony" + }`, + }, + testUtils.Request{ + Request: req, + Results: []map[string]any{ + {"model": "Running Man"}, + {"model": "PlayStation 5"}, + }, + }, + testUtils.Request{ + Request: makeExplainQuery(req), + // we make 2 index fetches to get all 2 devices with owner_id == null + // and 2 field fetches to get 1 'model' field for every fetched device. + Asserter: testUtils.NewExplainAsserter().WithFieldFetches(2).WithIndexFetches(2), }, }, }