From 7567552972887491058e15b69441e2df689f19c2 Mon Sep 17 00:00:00 2001 From: Xiaoli Zhou Date: Wed, 18 Sep 2024 16:38:50 +0800 Subject: [PATCH] fix(interactive): Fix Bugs of `Pattern Not Found` in Glogue (#4235) ## What do these changes do? Some Patterns are not reordered after creation which leads to the `pattern not found` error in glogue, this PR mainly fix this. ## Related issue number Fixes --- .../ir/meta/glogue/ExtendWeightEstimator.java | 1 + .../meta/glogue/PrimitiveCountEstimator.java | 10 +++-- .../common/ir/meta/glogue/Utils.java | 7 +++- .../common/ir/planner/GraphIOProcessor.java | 2 + .../common/ir/rel/metadata/glogue/Glogue.java | 8 ++++ .../GlogueBasicCardinalityEstimationImpl.java | 9 +++++ .../ir/rel/metadata/glogue/GlogueQuery.java | 7 +++- .../glogue/pattern/ElementDetails.java | 22 +++------- .../common/ir/planner/cbo/LdbcTest.java | 40 +++++++++---------- .../common/ir/planner/neo4j/CBOTest.java | 2 +- 10 files changed, 62 insertions(+), 46 deletions(-) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/ExtendWeightEstimator.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/ExtendWeightEstimator.java index 5922e761918b..232e3f5c1f22 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/ExtendWeightEstimator.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/ExtendWeightEstimator.java @@ -65,6 +65,7 @@ public double estimate(List edges, PatternVertex target) { pattern.addVertex(edge.getSrcVertex()); pattern.addVertex(edge.getDstVertex()); pattern.addEdge(edge.getSrcVertex(), edge.getDstVertex(), edge); + pattern.reordering(); extendFromVertices.add(Utils.getExtendFromVertex(edge, target)); double weight = (edges.size() == 1) diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/PrimitiveCountEstimator.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/PrimitiveCountEstimator.java index f9156072fc57..792121c4898f 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/PrimitiveCountEstimator.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/PrimitiveCountEstimator.java @@ -33,7 +33,10 @@ public PrimitiveCountEstimator(GlogueQuery gq) { public @Nullable Double estimate(Pattern pattern) { if (Utils.canLookUpFromGlogue(pattern, gq.getMaxPatternSize())) { - return gq.getRowCount(pattern); + Double countFromGlogue = gq.getRowCount(pattern, true); + if (countFromGlogue != null) { + return countFromGlogue; + } } // estimate the pattern graph with intersect, i.e. a->b, c->b, d->b PatternVertex intersect = getIntersectVertex(pattern); @@ -56,7 +59,7 @@ public PrimitiveCountEstimator(GlogueQuery gq) { public double estimate(PatternVertex vertex) { double sum = 0.0d; for (Integer typeId : vertex.getVertexTypeIds()) { - sum += gq.getRowCount(new Pattern(new SinglePatternVertex(typeId))); + sum += gq.getRowCount(new Pattern(new SinglePatternVertex(typeId)), false); } return sum * vertex.getElementDetails().getSelectivity(); } @@ -106,7 +109,8 @@ public double estimate(PatternEdge edge, EdgeTypeId typeId, int hops) { edgePattern.addVertex(dstVertex); edgePattern.addEdge( srcVertex, dstVertex, new SinglePatternEdge(srcVertex, dstVertex, typeId, 0)); - return Math.pow(gq.getRowCount(edgePattern), hops) + edgePattern.reordering(); + return Math.pow(gq.getRowCount(edgePattern, false), hops) / Math.pow(estimate(dstVertex), hops - 1); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/Utils.java index f3a5b0ec3290..a8f283798eaa 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/glogue/Utils.java @@ -109,7 +109,9 @@ public static boolean canLookUpFromGlogue(Pattern pattern, int maxPatternSizeInG return false; } ElementDetails details = vertex.getElementDetails(); - if (details != null && Double.compare(details.getSelectivity(), 1.0d) != 0) { + if (details != null + && (Double.compare(details.getSelectivity(), 1.0d) != 0 + || details.isOptional())) { return false; } } @@ -121,7 +123,8 @@ public static boolean canLookUpFromGlogue(Pattern pattern, int maxPatternSizeInG ElementDetails details = edge.getElementDetails(); if (details != null && (Double.compare(details.getSelectivity(), 1.0d) != 0 - || details.getRange() != null)) { + || details.getRange() != null + || details.isOptional())) { return false; } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphIOProcessor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphIOProcessor.java index 6d2af7f93997..4a775c7ecc9a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphIOProcessor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphIOProcessor.java @@ -622,6 +622,7 @@ public double labelConstraintsDeltaCost( original.addVertex(dst); } original.addEdge(src, dst, patternEdge); + original.reordering(); return cost; } @@ -639,6 +640,7 @@ public RelNode visit(GraphExtendIntersect intersect) { List extendEdges = extendStep.getExtendEdges(); RelNode child = visitChildren(intersect).getInput(0); Pattern original = new Pattern(intersect.getGlogueEdge().getSrcPattern()); + original.reordering(); // convert to GraphLogicalExpand if only one extend edge if (extendEdges.size() == 1) { return addCachedCost( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/Glogue.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/Glogue.java index b2b85f3e9611..b89e7c18293a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/Glogue.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/Glogue.java @@ -21,6 +21,7 @@ import com.alibaba.graphscope.common.ir.rel.metadata.glogue.pattern.SinglePatternVertex; import com.alibaba.graphscope.common.ir.rel.metadata.schema.GlogueSchema; +import org.checkerframework.checker.nullness.qual.Nullable; import org.jgrapht.Graph; import org.jgrapht.graph.DirectedPseudograph; import org.slf4j.Logger; @@ -122,6 +123,13 @@ public Double getRowCount(Pattern pattern) { return this.glogueCardinalityEstimation.getCardinality(pattern); } + public @Nullable Double getRowCount(Pattern pattern, boolean allowsNull) { + return (glogueCardinalityEstimation instanceof GlogueBasicCardinalityEstimationImpl) + ? ((GlogueBasicCardinalityEstimationImpl) glogueCardinalityEstimation) + .getCardinality(pattern, allowsNull) + : getRowCount(pattern); + } + public int getMaxPatternSize() { return maxPatternSize; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/GlogueBasicCardinalityEstimationImpl.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/GlogueBasicCardinalityEstimationImpl.java index 702d0a1fcdfd..db0bca5c43df 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/GlogueBasicCardinalityEstimationImpl.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/GlogueBasicCardinalityEstimationImpl.java @@ -21,6 +21,7 @@ import com.alibaba.graphscope.common.ir.rel.metadata.glogue.pattern.PatternVertex; import com.alibaba.graphscope.common.ir.rel.metadata.schema.GlogueSchema; +import org.checkerframework.checker.nullness.qual.Nullable; import org.javatuples.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -156,12 +157,20 @@ private boolean containsPattern(Pattern pattern) { @Override public Double getCardinality(Pattern queryPattern) { + return getCardinality(queryPattern, false); + } + + public @Nullable Double getCardinality(Pattern queryPattern, boolean allowsNull) { for (Pattern pattern : this.patternCardinality.keySet()) { if (pattern.equals(queryPattern)) { return this.patternCardinality.get(pattern); } } + if (allowsNull) { + return null; + } // if not exist, return 1.0 + logger.warn("pattern {} not found in glogue, return count = 1.0", queryPattern); return 1.0; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/GlogueQuery.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/GlogueQuery.java index b523e719e00b..96b6c3d2ce22 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/GlogueQuery.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/GlogueQuery.java @@ -18,6 +18,8 @@ import com.alibaba.graphscope.common.ir.rel.metadata.glogue.pattern.*; +import org.checkerframework.checker.nullness.qual.Nullable; + import java.util.Set; public class GlogueQuery { @@ -48,10 +50,11 @@ public Set getOutEdges(Pattern pattern) { /** * get pattern count * @param pattern + * @param allowsNull if the flag is set to true, return null when the pattern is not found, otherwise return 1.0d * @return */ - public Double getRowCount(Pattern pattern) { - return glogue.getRowCount(pattern); + public @Nullable Double getRowCount(Pattern pattern, boolean allowsNull) { + return glogue.getRowCount(pattern, allowsNull); } /** diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/pattern/ElementDetails.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/pattern/ElementDetails.java index ac686dceb9a0..6cfd1b592c85 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/pattern/ElementDetails.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/pattern/ElementDetails.java @@ -79,12 +79,14 @@ public boolean equals(Object o) { return Double.compare(details.selectivity, selectivity) == 0 && Objects.equals(range, details.range) && Objects.equals(pxdInnerGetVTypes, details.pxdInnerGetVTypes) - && optional == details.optional; + && optional == details.optional + && Objects.equals(resultOpt, details.resultOpt) + && Objects.equals(pathOpt, details.pathOpt); } @Override public int hashCode() { - return Objects.hash(selectivity, range, optional); + return Objects.hash(selectivity, range, optional, pxdInnerGetVTypes, resultOpt, pathOpt); } public double getSelectivity() { @@ -117,20 +119,6 @@ public GraphOpt.PathExpandPath getPathOpt() { @Override public int compareTo(ElementDetails o) { - int compare = Double.compare(this.selectivity, o.selectivity); - if (compare != 0) { - return compare; - } - compare = Boolean.compare(this.optional, o.optional); - if (compare != 0) { - return compare; - } - if (!this.pxdInnerGetVTypes.equals(o.pxdInnerGetVTypes)) { - return -1; - } - if (this.range != null && o.range != null) { - return this.range.compareTo(o.range); - } - return this.range == null && o.range == null ? 0 : this.range == null ? -1 : 1; + return o == null ? -1 : this.hashCode() - o.hashCode(); } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java index d48d4fb60b2a..ea08d443639f 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java @@ -691,46 +691,44 @@ public void ldbc10_test() { + " LogicalJoin(condition=[AND(=(person, person), =(friend, friend))]," + " joinType=[anti])\n" + " LogicalFilter(condition=[<>(friend, person)])\n" - + " MultiJoin(joinFilter=[=(post1, post1)]," + + " GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST]}]," + + " alias=[post], opt=[START], physicalOpt=[ITSELF])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR," + + " POST, PERSON)]], alias=[_], startAlias=[friend], opt=[IN]," + + " physicalOpt=[VERTEX], optional=[true])\n" + + " MultiJoin(joinFilter=[=(tag, tag)]," + " isFullOuterJoin=[false], joinTypes=[[INNER, INNER]]," + " outerJoinConditions=[[NULL, NULL]], projFields=[[ALL, ALL]])\n" - + " GraphPhysicalGetV(tableConfig=[{isAll=false," - + " tables=[POST]}], alias=[post1], opt=[START], physicalOpt=[ITSELF])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR," - + " POST, PERSON)]], alias=[_], startAlias=[friend], opt=[IN]," + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASTAG," + + " POST, TAG)]], alias=[tag], startAlias=[post1], opt=[OUT]," + " physicalOpt=[VERTEX], optional=[true])\n" - + " CommonTableScan(table=[[common#-1626533514]])\n" - + " GraphPhysicalGetV(tableConfig=[{isAll=false," - + " tables=[POST]}], alias=[post1], opt=[START], physicalOpt=[ITSELF])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASTAG, POST," - + " TAG)]], alias=[_], startAlias=[tag], opt=[IN], physicalOpt=[VERTEX]," - + " optional=[true])\n" - + " CommonTableScan(table=[[common#-1626533514]])\n" + + " CommonTableScan(table=[[common#-2135802270]])\n" + + " GraphPhysicalExpand(tableConfig=[{isAll=false," + + " tables=[HASINTEREST]}], alias=[tag], startAlias=[person], opt=[OUT]," + + " physicalOpt=[VERTEX], optional=[true])\n" + + " CommonTableScan(table=[[common#-2135802270]])\n" + " GraphPhysicalExpand(tableConfig=[{isAll=false, tables=[KNOWS]}]," + " alias=[friend], startAlias=[person], opt=[BOTH], physicalOpt=[VERTEX])\n" + " GraphLogicalSource(tableConfig=[{isAll=false," + " tables=[PERSON]}], alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id," + " ?0)])\n" - + "common#-1626533514:\n" - + "GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST]}], alias=[post]," + + "common#-2135802270:\n" + + "GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST]}], alias=[post1]," + " opt=[START], physicalOpt=[ITSELF])\n" + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR, POST, PERSON)]]," + " alias=[_], startAlias=[friend], opt=[IN], physicalOpt=[VERTEX]," + " optional=[true])\n" - + " GraphPhysicalExpand(tableConfig=[{isAll=false, tables=[HASINTEREST]}]," - + " alias=[tag], startAlias=[person], opt=[OUT], physicalOpt=[VERTEX]," - + " optional=[true])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN, PERSON," + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN, PERSON," + " PLACE)]], alias=[city], startAlias=[friend], opt=[OUT]," + " physicalOpt=[VERTEX])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," + + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," + " alias=[friend], opt=[END])\n" - + " " + + " " + " GraphLogicalPathExpand(fused=[GraphPhysicalExpand(tableConfig=[{isAll=false," + " tables=[KNOWS]}], alias=[_], opt=[BOTH], physicalOpt=[VERTEX])\n" + "], offset=[2], fetch=[1], path_opt=[ARBITRARY], result_opt=[END_V]," + " alias=[_], start_alias=[person])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}]," + + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}]," + " alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id, ?0)])", com.alibaba.graphscope.common.ir.tools.Utils.toString(after).trim()); } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/neo4j/CBOTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/neo4j/CBOTest.java index a065f80cef4d..101cdfaf7476 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/neo4j/CBOTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/neo4j/CBOTest.java @@ -397,7 +397,7 @@ public void CBO_4_3_test() { + " 90491.99999999999\n" + " GraphPhysicalExpand(tableConfig=[{isAll=false, tables=[HASINTEREST]}]," + " alias=[post], startAlias=[person1], opt=[OUT], physicalOpt=[VERTEX]):" - + " rowcount = 1.0\n" + + " rowcount = 446.2587786663215\n" + " CommonTableScan(table=[[common#114550231]]): rowcount =" + " 90491.99999999999\n" + "common#114550231:\n"