From 910dec05656263fd754109af61b594be570f21ce Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Tue, 19 Dec 2023 17:46:38 -0800 Subject: [PATCH] Add EXCLUDE to partiql-eval --- .../org/partiql/eval/internal/Compiler.kt | 8 + .../internal/exclude/CompiledExcludeItem.kt | 229 ++++++++++++++++ .../eval/internal/operator/rel/RelExclude.kt | 212 +++++++++++++++ .../eval/internal/PartiQLEngineDefaultTest.kt | 29 ++ .../exclude/CompiledExcludeItemTest.kt | 251 +++++++++++++++++ .../eval/EvaluatingCompilerExcludeTests.kt | 257 ------------------ 6 files changed, 729 insertions(+), 257 deletions(-) create mode 100644 partiql-eval/src/main/kotlin/org/partiql/eval/internal/exclude/CompiledExcludeItem.kt create mode 100644 partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExclude.kt create mode 100644 partiql-eval/src/test/kotlin/org/partiql/eval/internal/exclude/CompiledExcludeItemTest.kt diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt index d0df01b071..dbcb22ee75 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt @@ -1,6 +1,7 @@ package org.partiql.eval.internal import org.partiql.eval.internal.operator.Operator +import org.partiql.eval.internal.operator.rel.RelExclude import org.partiql.eval.internal.operator.rel.RelFilter import org.partiql.eval.internal.operator.rel.RelJoinInner import org.partiql.eval.internal.operator.rel.RelJoinLeft @@ -8,6 +9,7 @@ import org.partiql.eval.internal.operator.rel.RelJoinOuterFull import org.partiql.eval.internal.operator.rel.RelJoinRight import org.partiql.eval.internal.operator.rel.RelProject import org.partiql.eval.internal.operator.rel.RelScan +import org.partiql.eval.internal.operator.rel.compileExcludeItems import org.partiql.eval.internal.operator.rex.ExprCase import org.partiql.eval.internal.operator.rex.ExprCollection import org.partiql.eval.internal.operator.rex.ExprLiteral @@ -82,6 +84,12 @@ internal object Compiler { return RelFilter(input, condition) } + override fun visitRelOpExclude(node: Rel.Op.Exclude, ctx: Unit): Operator { + val input = visitRel(node.input, ctx) + val compiledExcludeExprs = compileExcludeItems(node.items) + return RelExclude(input, compiledExcludeExprs) + } + override fun visitRex(node: Rex, ctx: Unit): Operator.Expr { return super.visitRexOp(node.op, ctx) as Operator.Expr } diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/exclude/CompiledExcludeItem.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/exclude/CompiledExcludeItem.kt new file mode 100644 index 0000000000..48ffdb52ad --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/exclude/CompiledExcludeItem.kt @@ -0,0 +1,229 @@ +package org.partiql.eval.internal.exclude + +/** + * Internal representation of an `EXCLUDE` expr step. + */ +internal sealed class ExcludeStep { + internal data class StructField(val attr: String, val caseSensitivity: ExcludeFieldCase) : ExcludeStep() + internal object StructWildcard : ExcludeStep() + internal data class CollIndex(val index: Int) : ExcludeStep() + internal object CollWildcard : ExcludeStep() +} + +/** + * Internal representation of an `EXCLUDE` struct attribute case-sensitivity. + */ +internal enum class ExcludeFieldCase { + INSENSITIVE, SENSITIVE +} + +/** + * Represents all the compiled `EXCLUDE` paths that start with the same [CompiledExcludeItem.root]. This variant of + * [ExcludeNode] represents the top-level root node of the exclude tree. + * + * Notably, redundant paths (i.e. exclude paths that exclude values already excluded by other paths) will be removed. + */ +internal data class CompiledExcludeItem( + val root: Int, + override val leaves: MutableSet, + override val branches: MutableSet +) : ExcludeNode(leaves, branches) { + companion object { + fun empty(root: Int): CompiledExcludeItem { + return CompiledExcludeItem(root, mutableSetOf(), mutableSetOf()) + } + } +} + +/** + * Represent all the `EXCLUDE` paths that start with the same [ExcludeBranch.step] that also have additional steps + * (i.e. final step is at a deeper level). This variant of [ExcludeNode] represents inner nodes (i.e. non-top-level) + * nodes of the exclude tree. + */ +internal data class ExcludeBranch( + val step: ExcludeStep, + override val leaves: MutableSet, + override val branches: MutableSet +) : ExcludeNode(leaves, branches) { + companion object { + fun empty(step: ExcludeStep): ExcludeBranch { + return ExcludeBranch(step, mutableSetOf(), mutableSetOf()) + } + } +} + +/** + * Represents all the `EXCLUDE` paths that have a final exclude step at the current level. This variant of [ExcludeNode] + * represents the leaves in our exclude tree. + */ +internal data class ExcludeLeaf( + val step: ExcludeStep, +) : ExcludeNode(mutableSetOf(), mutableSetOf()) + +/** + * A tree representation of the exclude paths that will eliminate redundant paths (i.e. exclude paths that exclude + * values already excluded by other paths). + * + * The idea behind this tree representation is that at a current level (i.e. path step index), we keep track of the + * - Exclude paths that have a final exclude step at the current level. This set of struct attributes and collection + * indexes to remove at the current level is modeled as a set of leaves (i.e. [ExcludeLeaf]). + * - Exclude paths that have additional steps (their final step is at a deeper level). This is modeled as a set of + * branches [ExcludeBranch] to group all exclude paths that share the same current step. + * + * For example, let's say we have exclude paths + * a.b, -- assuming root resolves to 0 + * x.y.z1, -- assuming root resolves to 1 + * x.y.z2 -- assuming root resolves to 1 + * ^ ^ ^ + * Level 1 2 3 + * + * These exclude paths would be converted to the following [CompiledExcludeItem]s in [ExcludeNode]s: + * ``` + * // For demonstration purposes, the syntax '' corresponds to the exclude struct attribute step of + * CompiledExcludeItem( // Root 0 (i.e. 'a') + * root = 0, + * leaves = mutableSetOf( + * ExcludeLeaf(step = 'b') // Exclude 'b' at level 2 + * ), + * branches = mutableSetOf() // No further exclusions + * ), + * CompiledExcludeItem( // Root 1 (i.e. 'x') + * root = 1, + * leaves = mutableSetOf(), // No exclusions at level 2 + * branches = mutableSetOf( + * ExcludeBranch( + * step = 'y', + * leaves = mutableSetOf( + * ExcludeLeaf(step = 'z1'), // Exclude 'z1` at level 3 + * ExcludeLeaf(step = 'z2') // Exclude `z2` at level 3 + * ) + * branches = mutableSetOf() // No further exclusions + * ) + * ) + * ) + */ +internal sealed class ExcludeNode( + open val leaves: MutableSet, + open val branches: MutableSet +) { + private fun addLeaf(step: ExcludeStep) { + when (step) { + is ExcludeStep.StructField -> { + if (leaves.contains(ExcludeLeaf(ExcludeStep.StructWildcard))) { + // leaves contain wildcard; do not add; e.g. a.* and a.b -> keep a.* + } else { + // add to leaves + leaves.add(ExcludeLeaf(step)) + // remove from branches; e.g. a.b.c and a.b -> keep a.b + branches.removeIf { subBranch -> + step == subBranch.step + } + } + } + is ExcludeStep.StructWildcard -> { + leaves.add(ExcludeLeaf(step)) + // remove all struct attribute exclude steps from leaves + leaves.removeIf { subLeaf -> + subLeaf.step is ExcludeStep.StructField + } + // remove all struct attribute/wildcard exclude steps from branches + branches.removeIf { subBranch -> + subBranch.step is ExcludeStep.StructField || subBranch.step is ExcludeStep.StructWildcard + } + } + is ExcludeStep.CollIndex -> { + if (leaves.contains(ExcludeLeaf(ExcludeStep.CollWildcard))) { + // leaves contains wildcard; do not add; e.g a[*] and a[1] -> keep a[*] + } else { + // add to leaves + leaves.add(ExcludeLeaf(step)) + // remove from branches; e.g. a.b[2].c and a.b[2] -> keep a.b[2] + branches.removeIf { subBranch -> + step == subBranch.step + } + } + } + is ExcludeStep.CollWildcard -> { + leaves.add(ExcludeLeaf(step)) + // remove all collection index exclude steps from leaves + leaves.removeIf { subLeaf -> + subLeaf.step is ExcludeStep.CollIndex + } + // remove all collection index/wildcard exclude steps from branches + branches.removeIf { subBranch -> + subBranch.step is ExcludeStep.CollIndex || subBranch.step is ExcludeStep.CollWildcard + } + } + } + } + + private fun addBranch(steps: List) { + val head = steps.first() + val tail = steps.drop(1) + when (head) { + is ExcludeStep.StructField -> { + if (leaves.contains(ExcludeLeaf(ExcludeStep.StructWildcard)) || leaves.contains( + ExcludeLeaf(head) + ) + ) { + // leaves contains struct wildcard or attr; do not add to branches + // e.g. a.* and a.b.c -> a.* + } else { + val existingBranch = branches.find { subBranch -> + head == subBranch.step + } ?: ExcludeBranch.empty(head) + branches.remove(existingBranch) + existingBranch.addNode(tail) + branches.add(existingBranch) + } + } + is ExcludeStep.StructWildcard -> { + if (leaves.any { it.step is ExcludeStep.StructWildcard }) { + // struct wildcard in leaves; do nothing + } else { + val existingBranch = branches.find { subBranch -> + head == subBranch.step + } ?: ExcludeBranch.empty(head) + branches.remove(existingBranch) + existingBranch.addNode(tail) + branches.add(existingBranch) + } + } + is ExcludeStep.CollIndex -> { + if (leaves.contains(ExcludeLeaf(ExcludeStep.CollWildcard)) || leaves.contains( + ExcludeLeaf(head) + ) + ) { + // leaves contains collection wildcard or index; do not add to branches + // e.g. a[*] and a[*][1] -> a[*] + } else { + val existingBranch = branches.find { subBranch -> + head == subBranch.step + } ?: ExcludeBranch.empty(head) + branches.remove(existingBranch) + existingBranch.addNode(tail) + branches.add(existingBranch) + } + } + is ExcludeStep.CollWildcard -> { + if (leaves.any { it.step is ExcludeStep.CollWildcard }) { + // collection wildcard in leaves; do nothing + } else { + val existingBranch = branches.find { subBranch -> + head == subBranch.step + } ?: ExcludeBranch.empty(head) + branches.remove(existingBranch) + existingBranch.addNode(tail) + branches.add(existingBranch) + } + } + } + } + + internal fun addNode(steps: List) { + when (steps.size) { + 1 -> this.addLeaf(steps.first()) + else -> this.addBranch(steps) + } + } +} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExclude.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExclude.kt new file mode 100644 index 0000000000..9bb159d877 --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelExclude.kt @@ -0,0 +1,212 @@ +package org.partiql.eval.internal.operator.rel + +import org.partiql.eval.internal.Record +import org.partiql.eval.internal.exclude.CompiledExcludeItem +import org.partiql.eval.internal.exclude.ExcludeFieldCase +import org.partiql.eval.internal.exclude.ExcludeNode +import org.partiql.eval.internal.exclude.ExcludeStep +import org.partiql.eval.internal.operator.Operator +import org.partiql.plan.Identifier +import org.partiql.plan.Rel +import org.partiql.value.BagValue +import org.partiql.value.CollectionValue +import org.partiql.value.ListValue +import org.partiql.value.PartiQLValue +import org.partiql.value.PartiQLValueExperimental +import org.partiql.value.PartiQLValueType +import org.partiql.value.SexpValue +import org.partiql.value.StructValue +import org.partiql.value.bagValue +import org.partiql.value.listValue +import org.partiql.value.sexpValue +import org.partiql.value.structValue + +internal class RelExclude( + val input: Operator.Relation, + private val compiledExcludeItems: List +) : Operator.Relation { + + override fun open() { + input.open() + } + + override fun next(): Record? { + while (true) { + val row = input.next() ?: return null + val newRecord = compiledExcludeItems.fold(row) { curRecord, expr -> + excludeOnRecord(curRecord, expr) + } + return newRecord + } + } + + override fun close() { + input.close() + } +} + +@OptIn(PartiQLValueExperimental::class) +private fun excludeOnRecord( + record: Record, + exclusions: CompiledExcludeItem +): Record { + val values = record.values + val value = values.getOrNull(exclusions.root) + val newValues = if (value != null) { + values[exclusions.root] = excludeOnPartiQLValue(value, exclusions) + values + } else { + values + } + return Record(newValues) +} + +@OptIn(PartiQLValueExperimental::class) +private fun excludeOnStructValue( + structValue: StructValue<*>, + exclusions: ExcludeNode +): PartiQLValue { + val leavesSteps = exclusions.leaves.map { leaf -> leaf.step } + val branches = exclusions.branches + if (leavesSteps.any { it is ExcludeStep.StructWildcard }) { + // tuple wildcard at current level. return empty struct + return structValue() + } + val attrsToRemove = leavesSteps.filterIsInstance() + .map { it.attr } + .toSet() + val entriesWithRemoved = structValue.entries.filter { structField -> + !attrsToRemove.contains(structField.first) + } + val finalStruct = entriesWithRemoved.map { structField -> + val name = structField.first + var expr = structField.second + // apply case-sensitive tuple attr exclusions + val structFieldCaseSensitiveKey = ExcludeStep.StructField(name, ExcludeFieldCase.SENSITIVE) + branches.find { + it.step == structFieldCaseSensitiveKey + }?.let { + expr = excludeOnPartiQLValue(expr, it) + } + // apply case-insensitive tuple attr exclusions + val structFieldCaseInsensitiveKey = ExcludeStep.StructField(name, ExcludeFieldCase.INSENSITIVE) + branches.find { + it.step == structFieldCaseInsensitiveKey + }?.let { + expr = excludeOnPartiQLValue(expr, it) + } + // apply tuple wildcard exclusions + val tupleWildcardKey = ExcludeStep.StructWildcard + branches.find { + it.step == tupleWildcardKey + }?.let { + expr = excludeOnPartiQLValue(expr, it) + } + Pair(name, expr) + } + return structValue(finalStruct) +} + +/** + * Returns a [PartiQLValue] created from an iterable of [coll]. Requires [type] to be a collection type + * (i.e. [PartiQLValueType.LIST], [PartiQLValueType.BAG], or [PartiQLValueType.SEXP]). + */ +@OptIn(PartiQLValueExperimental::class) +private fun newCollValue(type: PartiQLValueType, coll: Iterable): PartiQLValue { + return when (type) { + PartiQLValueType.LIST -> listValue(coll) + PartiQLValueType.BAG -> bagValue(coll) + PartiQLValueType.SEXP -> sexpValue(coll) + else -> error("Collection type required") + } +} + +@OptIn(PartiQLValueExperimental::class) +private fun excludeOnCollValue( + coll: CollectionValue<*>, + type: PartiQLValueType, + exclusions: ExcludeNode +): PartiQLValue { + val leavesSteps = exclusions.leaves.map { leaf -> leaf.step } + val branches = exclusions.branches + if (leavesSteps.any { it is ExcludeStep.CollWildcard }) { + // collection wildcard at current level. return empty collection + return newCollValue(type, emptyList()) + } else { + val indexesToRemove = leavesSteps.filterIsInstance() + .map { it.index } + .toSet() + val collWithRemoved = when (coll) { + is BagValue -> coll + is ListValue, is SexpValue -> coll.filterIndexed { index, _ -> + !indexesToRemove.contains(index) + } + } + val finalColl = collWithRemoved.mapIndexed { index, element -> + var expr = element + if (coll is ListValue || coll is SexpValue) { + // apply collection index exclusions for lists and sexps + val elementKey = ExcludeStep.CollIndex(index) + branches.find { + it.step == elementKey + }?.let { + expr = excludeOnPartiQLValue(element, it) + } + } + // apply collection wildcard exclusions for lists, bags, and sexps + val collectionWildcardKey = ExcludeStep.CollWildcard + branches.find { + it.step == collectionWildcardKey + }?.let { + expr = excludeOnPartiQLValue(expr, it) + } + expr + } + return newCollValue(type, finalColl) + } +} + +@OptIn(PartiQLValueExperimental::class) +private fun excludeOnPartiQLValue(initialPartiQLValue: PartiQLValue, exclusions: ExcludeNode): PartiQLValue { + return when (initialPartiQLValue) { + is StructValue<*> -> excludeOnStructValue(initialPartiQLValue, exclusions) + is BagValue<*> -> excludeOnCollValue(initialPartiQLValue, PartiQLValueType.BAG, exclusions) + is ListValue<*> -> excludeOnCollValue(initialPartiQLValue, PartiQLValueType.LIST, exclusions) + is SexpValue<*> -> excludeOnCollValue(initialPartiQLValue, PartiQLValueType.SEXP, exclusions) + else -> { + initialPartiQLValue + } + } +} + +/** + * Creates a list of [CompiledExcludeItem] with each index of the resulting list corresponding to a different + * exclude path root. + */ +internal fun compileExcludeItems(excludeExprs: List): List { + val compiledExcludeItems = excludeExprs + .groupBy { it.root } + .map { (root, exclusions) -> + exclusions.fold(CompiledExcludeItem.empty(root.ref)) { acc, exclusion -> + acc.addNode(exclusion.steps.map { it.toCompiledExcludeStep() }) + acc + } + } + return compiledExcludeItems +} + +private fun Rel.Op.Exclude.Step.toCompiledExcludeStep(): ExcludeStep { + return when (this) { + is Rel.Op.Exclude.Step.StructField -> ExcludeStep.StructField(this.symbol.symbol, this.symbol.caseSensitivity.toCompiledExcludeStepCase()) + is Rel.Op.Exclude.Step.StructWildcard -> ExcludeStep.StructWildcard + is Rel.Op.Exclude.Step.CollIndex -> ExcludeStep.CollIndex(this.index) + is Rel.Op.Exclude.Step.CollWildcard -> ExcludeStep.CollWildcard + } +} + +private fun Identifier.CaseSensitivity.toCompiledExcludeStepCase(): ExcludeFieldCase { + return when (this) { + Identifier.CaseSensitivity.SENSITIVE -> ExcludeFieldCase.SENSITIVE + Identifier.CaseSensitivity.INSENSITIVE -> ExcludeFieldCase.INSENSITIVE + } +} diff --git a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt index e57da2baf5..7566e900ac 100644 --- a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt +++ b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt @@ -264,6 +264,35 @@ class PartiQLEngineDefaultTest { assertEquals(expected, output) } + @OptIn(PartiQLValueExperimental::class) + @Test + fun testExclude() { + val statement = + parser.parse("SELECT t EXCLUDE t.a.b FROM <<{'a': {'b': 2}, 'foo': 'bar', 'foo2': 'bar2'}>> AS t;").root + val session = PartiQLPlanner.Session("q", "u") + val plan = planner.plan(statement, session) + val prepared = engine.prepare(plan.plan) + val result = engine.execute(prepared) + if (result is PartiQLResult.Error) { + throw result.cause + } + result as PartiQLResult.Value + val output = result.value as BagValue<*> + + val expected = bagValue( + structValue( + "t" to structValue( + "a" to structValue( + // field `b` excluded + ), + "foo" to stringValue("bar"), + "foo2" to stringValue("bar2") + ) + ), + ) + assertEquals(expected, output, comparisonString(expected, output)) + } + @OptIn(PartiQLValueExperimental::class) private fun comparisonString(expected: PartiQLValue, actual: PartiQLValue): String { val expectedBuffer = ByteArrayOutputStream() diff --git a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/exclude/CompiledExcludeItemTest.kt b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/exclude/CompiledExcludeItemTest.kt new file mode 100644 index 0000000000..ac9a703267 --- /dev/null +++ b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/exclude/CompiledExcludeItemTest.kt @@ -0,0 +1,251 @@ +package org.partiql.eval.internal.exclude + +import org.junit.Assert.assertEquals +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ArgumentsSource +import org.partiql.eval.internal.operator.rel.compileExcludeItems +import org.partiql.lang.util.ArgumentsProviderBase +import org.partiql.parser.PartiQLParser +import org.partiql.plan.Rel +import org.partiql.plan.Rex +import org.partiql.plan.Statement +import org.partiql.planner.PartiQLPlanner + +class CompiledExcludeItemTest { + private val planner = org.partiql.planner.PartiQLPlannerBuilder().build() + private val parser = PartiQLParser.default() + + private fun getExcludeClause(statement: Statement): Rel.Op.Exclude { + val queryExpr = (statement as Statement.Query).root + val relProject = ((queryExpr.op as Rex.Op.Select).rel).op as Rel.Op.Project + return ((relProject.input).op) as Rel.Op.Exclude + } + + private fun testExcludeExprSubsumption(tc: SubsumptionTC) { + val statement = parser.parse("SELECT * EXCLUDE ${tc.excludeExprStr} FROM <<>> AS s, <<>> AS t;").root + val session = PartiQLPlanner.Session("q", "u") + val plan = planner.plan(statement, session) + val excludeClause = getExcludeClause(plan.plan.statement) + val actualExcludeExprs = compileExcludeItems(excludeClause.items) + assertEquals(tc.expectedExcludeExprs, actualExcludeExprs) + } + + internal data class SubsumptionTC(val excludeExprStr: String, val expectedExcludeExprs: List) + + @ParameterizedTest + @ArgumentsSource(ExcludeSubsumptionTests::class) + internal fun subsumptionTests(tc: SubsumptionTC) = testExcludeExprSubsumption(tc) + + internal class ExcludeSubsumptionTests : ArgumentsProviderBase() { + override fun getParameters(): List = listOf( + SubsumptionTC( + "s.a, t.a, t.b, s.b", + listOf( + CompiledExcludeItem( + root = 0, + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.StructField("a", ExcludeFieldCase.INSENSITIVE)), ExcludeLeaf(ExcludeStep.StructField("b", ExcludeFieldCase.INSENSITIVE))), + branches = mutableSetOf() + ), + CompiledExcludeItem( + root = 1, + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.StructField("a", ExcludeFieldCase.INSENSITIVE)), ExcludeLeaf(ExcludeStep.StructField("b", ExcludeFieldCase.INSENSITIVE))), + branches = mutableSetOf() + ), + ) + ), + SubsumptionTC( + "t.a, t.b", + listOf( + CompiledExcludeItem( + root = 1, + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.StructField("a", ExcludeFieldCase.INSENSITIVE)), ExcludeLeaf(ExcludeStep.StructField("b", ExcludeFieldCase.INSENSITIVE))), + branches = mutableSetOf() + ), + ) + ), + SubsumptionTC( + "t.a, t.b, t.a, t.b, t.b", // duplicates subsumed + listOf( + CompiledExcludeItem( + root = 1, + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.StructField("a", ExcludeFieldCase.INSENSITIVE)), ExcludeLeaf(ExcludeStep.StructField("b", ExcludeFieldCase.INSENSITIVE))), + branches = mutableSetOf() + ), + ) + ), + SubsumptionTC( + "t.a, t.b, t.*", // struct wildcard subsumes struct attr + listOf( + CompiledExcludeItem( + root = 1, + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.StructWildcard)), + branches = mutableSetOf() + ), + ) + ), + SubsumptionTC( // removal at earlier step subsumes + """ + t.a, t.a.a1, -- t.a.a1 subsumed + t.b.b1.b2, t.b.b1, -- t.b.b1.b2 subsumed + t.c, t.c.c1[2].c3[*].* -- t.c.c1[2].c3[*].* subsumed + """, + listOf( + CompiledExcludeItem( + root = 1, + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.StructField("a", ExcludeFieldCase.INSENSITIVE)), ExcludeLeaf(ExcludeStep.StructField("c", ExcludeFieldCase.INSENSITIVE))), + branches = mutableSetOf( + ExcludeBranch( + step = ExcludeStep.StructField("b", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.StructField("b1", ExcludeFieldCase.INSENSITIVE))), + branches = mutableSetOf() + ) + ) + ) + ) + ), + SubsumptionTC( // exclude collection index + """ + t.a, t.a[1], + t.b[1], t.b + """, + listOf( + CompiledExcludeItem( + root = 1, + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.StructField("a", ExcludeFieldCase.INSENSITIVE)), ExcludeLeaf(ExcludeStep.StructField("b", ExcludeFieldCase.INSENSITIVE))), + branches = mutableSetOf() + ) + ) + ), + SubsumptionTC( // exclude collection index, collection wildcard + """ + t.a[*], t.a[1], + t.b[1], t.b[*], + t.c[*], t.c[1].c1, + t.d[1].d1, t.d[*] + """, + listOf( + CompiledExcludeItem( + root = 1, + leaves = mutableSetOf(), + branches = mutableSetOf( + ExcludeBranch( + step = ExcludeStep.StructField("a", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.CollWildcard)), + branches = mutableSetOf() + ), + ExcludeBranch( + step = ExcludeStep.StructField("b", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.CollWildcard)), + branches = mutableSetOf() + ), + ExcludeBranch( + step = ExcludeStep.StructField("c", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.CollWildcard)), + branches = mutableSetOf() + ), + ExcludeBranch( + step = ExcludeStep.StructField("d", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.CollWildcard)), + branches = mutableSetOf() + ) + ) + ) + ) + ), + SubsumptionTC( + """ + t.a[1].a1, t.a[1], + t.b[1], t.b[1].b1, + t.c[*], t.c[*].c1, + t.d[*].d1, t.d[*], + t.e[1], t.e[*].e1, -- keep both + t.f[*].f1, t.f[1], -- keep both + t.g[*], t.g[1].e1, + t.h[1].f1, t.h[*] + """, + listOf( + CompiledExcludeItem( + root = 1, + leaves = mutableSetOf(), + branches = mutableSetOf( + ExcludeBranch( + step = ExcludeStep.StructField("a", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.CollIndex(1))), + branches = mutableSetOf(), + ), + ExcludeBranch( + step = ExcludeStep.StructField("b", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.CollIndex(1))), + branches = mutableSetOf(), + ), + ExcludeBranch( + step = ExcludeStep.StructField("c", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.CollWildcard)), + branches = mutableSetOf(), + ), + ExcludeBranch( + step = ExcludeStep.StructField("d", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.CollWildcard)), + branches = mutableSetOf(), + ), + ExcludeBranch( + step = ExcludeStep.StructField("e", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.CollIndex(1))), + branches = mutableSetOf( + ExcludeBranch( + step = ExcludeStep.CollWildcard, + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.StructField("e1", ExcludeFieldCase.INSENSITIVE))), + branches = mutableSetOf(), + ) + ), + ), + ExcludeBranch( + step = ExcludeStep.StructField("f", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.CollIndex(1))), + branches = mutableSetOf( + ExcludeBranch( + step = ExcludeStep.CollWildcard, + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.StructField("f1", ExcludeFieldCase.INSENSITIVE))), + branches = mutableSetOf(), + ) + ), + ), + ExcludeBranch( + step = ExcludeStep.StructField("g", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.CollWildcard)), + branches = mutableSetOf(), + ), + ExcludeBranch( + step = ExcludeStep.StructField("h", ExcludeFieldCase.INSENSITIVE), + leaves = mutableSetOf(ExcludeLeaf(ExcludeStep.CollWildcard)), + branches = mutableSetOf(), + ), + ), + ) + ) + ), + SubsumptionTC( // case sensitive + """ + t.a, "t".a, + "t".b, t.b, + t."c", t.c, + t.d, t."d" + """, + listOf( + CompiledExcludeItem( + root = 1, + leaves = mutableSetOf( + ExcludeLeaf(ExcludeStep.StructField("a", ExcludeFieldCase.INSENSITIVE)), + ExcludeLeaf(ExcludeStep.StructField("b", ExcludeFieldCase.INSENSITIVE)), + ExcludeLeaf(ExcludeStep.StructField("c", ExcludeFieldCase.INSENSITIVE)), + ExcludeLeaf(ExcludeStep.StructField("d", ExcludeFieldCase.INSENSITIVE)), + ExcludeLeaf(ExcludeStep.StructField("c", ExcludeFieldCase.SENSITIVE)), + ExcludeLeaf(ExcludeStep.StructField("d", ExcludeFieldCase.SENSITIVE)), + ), + branches = mutableSetOf(), + ), + ) + ), + ) + } +} diff --git a/partiql-eval/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerExcludeTests.kt b/partiql-eval/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerExcludeTests.kt index d5cea88e99..931539de15 100644 --- a/partiql-eval/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerExcludeTests.kt +++ b/partiql-eval/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerExcludeTests.kt @@ -814,261 +814,4 @@ class EvaluatingCompilerExcludeTests : EvaluatorTestBase() { tc, EvaluationSession.standard() ) -// TODO: subsumption tests to be added back in https://github.com/partiql/partiql-lang-kotlin/pull/1280 -// -// private fun testExcludeExprSubsumption(tc: SubsumptionTC) { -// val parser = PartiQLParserBuilder.standard().build() -// val parsedSFW = parser.parseAstStatement("SELECT * EXCLUDE ${tc.excludeExprStr} FROM t") -// val exclude = (((parsedSFW as PartiqlAst.Statement.Query).expr) as PartiqlAst.Expr.Select).excludeClause!! -// val eC = EvaluatingCompiler( -// emptyList(), -// emptyMap(), -// emptyMap(), -// ) -// val actualExcludeExprs = eC.compileExcludeClause(exclude) -// assertEquals(tc.expectedExcludeExprs, actualExcludeExprs) -// } -// -// internal data class SubsumptionTC(val excludeExprStr: String, val expectedExcludeExprs: List) -// -// @ParameterizedTest -// @ArgumentsSource(ExcludeSubsumptionTests::class) -// internal fun subsumptionTests(tc: SubsumptionTC) = testExcludeExprSubsumption(tc) -// -// internal class ExcludeSubsumptionTests : ArgumentsProviderBase() { -// private fun caseSensitiveId(id: String): PartiqlAst.Identifier { -// return PartiqlAst.build { identifier(name = id, case = caseSensitive(emptyMetaContainer())) } -// } -// private fun caseInsensitiveId(id: String): PartiqlAst.Identifier { -// return PartiqlAst.build { identifier(name = id, case = caseInsensitive(emptyMetaContainer())) } -// } -// private fun exTupleAttr(id: PartiqlAst.Identifier): PartiqlAst.ExcludeStep { -// return PartiqlAst.ExcludeStep.ExcludeTupleAttr(id) -// } -// private fun exTupleWildcard(): PartiqlAst.ExcludeStep { -// return PartiqlAst.ExcludeStep.ExcludeTupleWildcard() -// } -// private fun exCollIndex(i: Int): PartiqlAst.ExcludeStep { -// return PartiqlAst.ExcludeStep.ExcludeCollectionIndex(index = LongPrimitive(i.toLong(), emptyMetaContainer())) -// } -// private fun exCollWildcard(): PartiqlAst.ExcludeStep { -// return PartiqlAst.ExcludeStep.ExcludeCollectionWildcard() -// } -// -// override fun getParameters(): List = listOf( -// SubsumptionTC( -// "s.a, t.a, t.b, s.b", -// listOf( -// EvaluatingCompiler.CompiledExcludeExpr( -// root = caseInsensitiveId("s"), -// exclusions = EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))), -// steps = emptyMap(), -// ) -// ), -// EvaluatingCompiler.CompiledExcludeExpr( -// root = caseInsensitiveId("t"), -// exclusions = EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))), -// steps = emptyMap(), -// ) -// ) -// ) -// ), -// SubsumptionTC( -// "t.a, t.b", -// listOf( -// EvaluatingCompiler.CompiledExcludeExpr( -// root = caseInsensitiveId("t"), -// exclusions = EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))), -// steps = emptyMap(), -// ) -// ) -// ) -// ), -// SubsumptionTC( -// "t.a, t.b, t.a, t.b, t.b", // duplicates subsumed -// listOf( -// EvaluatingCompiler.CompiledExcludeExpr( -// root = caseInsensitiveId("t"), -// exclusions = EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))), -// steps = emptyMap(), -// ) -// ) -// ) -// ), -// SubsumptionTC( -// "t.a, t.b, t.*", // tuple wildcard subsumes tuple attr -// listOf( -// EvaluatingCompiler.CompiledExcludeExpr( -// root = caseInsensitiveId("t"), -// exclusions = EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exTupleWildcard()), -// steps = emptyMap(), -// ) -// ) -// ) -// ), -// SubsumptionTC( // removal at earlier step subsumes -// """ -// t.a, t.a.a1, -- t.a.a1 subsumed -// t.b.b1.b2, t.b.b1, -- t.b.b1.b2 subsumed -// t.c, t.c.c1[2].c3[*].* -- t.c.c1[2].c3[*].* subsumed -// """, -// listOf( -// EvaluatingCompiler.CompiledExcludeExpr( -// root = caseInsensitiveId("t"), -// exclusions = EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("c"))), -// steps = mapOf( -// exTupleAttr(caseInsensitiveId("b")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exTupleAttr(caseInsensitiveId("b1"))), -// steps = emptyMap() -// ) -// ), -// ) -// ) -// ) -// ), -// SubsumptionTC( // exclude collection index -// """ -// t.a, t.a[1], -// t.b[1], t.b -// """, -// listOf( -// EvaluatingCompiler.CompiledExcludeExpr( -// root = caseInsensitiveId("t"), -// exclusions = EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))), -// steps = emptyMap() -// ) -// ) -// ) -// ), -// SubsumptionTC( // exclude collection index, collection wildcard -// """ -// t.a[*], t.a[1], -// t.b[1], t.b[*], -// t.c[*], t.c[1].c1, -// t.d[1].d1, t.d[*] -// """, -// listOf( -// EvaluatingCompiler.CompiledExcludeExpr( -// root = caseInsensitiveId("t"), -// exclusions = EvaluatingCompiler.RemoveAndOtherSteps( -// remove = emptySet(), -// steps = mapOf( -// exTupleAttr(caseInsensitiveId("a")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exCollWildcard()), -// steps = emptyMap() -// ), -// exTupleAttr(caseInsensitiveId("b")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exCollWildcard()), -// steps = emptyMap() -// ), -// exTupleAttr(caseInsensitiveId("c")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exCollWildcard()), -// steps = emptyMap() -// ), -// exTupleAttr(caseInsensitiveId("d")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exCollWildcard()), -// steps = emptyMap() -// ), -// ) -// ) -// ) -// ) -// ), -// SubsumptionTC( -// """ -// t.a[1].a1, t.a[1], -// t.b[1], t.b[1].b1, -// t.c[*], t.c[*].c1, -// t.d[*].d1, t.d[*], -// t.e[1], t.e[*].e1, -- keep both -// t.f[*].f1, t.f[1], -- keep both -// t.g[*], t.g[1].e1, -// t.h[1].f1, t.h[*] -// """, -// listOf( -// EvaluatingCompiler.CompiledExcludeExpr( -// root = caseInsensitiveId("t"), -// exclusions = EvaluatingCompiler.RemoveAndOtherSteps( -// remove = emptySet(), -// steps = mapOf( -// exTupleAttr(caseInsensitiveId("a")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exCollIndex(1)), -// steps = emptyMap() -// ), -// exTupleAttr(caseInsensitiveId("b")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exCollIndex(1)), -// steps = emptyMap() -// ), -// exTupleAttr(caseInsensitiveId("c")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exCollWildcard()), -// steps = emptyMap() -// ), -// exTupleAttr(caseInsensitiveId("d")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exCollWildcard()), -// steps = emptyMap() -// ), -// exTupleAttr(caseInsensitiveId("e")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exCollIndex(1)), -// steps = mapOf( -// exCollWildcard() to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exTupleAttr(caseInsensitiveId("e1"))), -// steps = emptyMap(), -// ) -// ) -// ), -// exTupleAttr(caseInsensitiveId("f")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exCollIndex(1)), -// steps = mapOf( -// exCollWildcard() to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exTupleAttr(caseInsensitiveId("f1"))), -// steps = emptyMap(), -// ) -// ) -// ), -// exTupleAttr(caseInsensitiveId("g")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exCollWildcard()), -// steps = emptyMap() -// ), -// exTupleAttr(caseInsensitiveId("h")) to EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exCollWildcard()), -// steps = emptyMap() -// ), -// ) -// ) -// ) -// ) -// ), -// SubsumptionTC( // case sensitive -// """ -// t.a, "t".a, -- "t".a in case-sensitive list -// "t".b, t.b, -- "t".b in case-sensitive list -// t."c", t.c, -// t.d, t."d" -// """, -// listOf( -// EvaluatingCompiler.CompiledExcludeExpr( -// root = caseInsensitiveId("t"), -// exclusions = EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b")), exTupleAttr(caseInsensitiveId("c")), exTupleAttr(caseInsensitiveId("d")), exTupleAttr(caseSensitiveId("c")), exTupleAttr(caseSensitiveId("d"))), -// steps = emptyMap(), -// ) -// ), -// EvaluatingCompiler.CompiledExcludeExpr( -// root = caseSensitiveId("t"), -// exclusions = EvaluatingCompiler.RemoveAndOtherSteps( -// remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))), -// steps = emptyMap(), -// ) -// ), -// ) -// ), -// ) -// } }