From a455541996125fe8b3ba01e1a38d13145ec020ee Mon Sep 17 00:00:00 2001 From: David Sisson Date: Wed, 14 Aug 2024 21:29:22 -0700 Subject: [PATCH] [GLUTEN-6834][CORE] Remove unused DDL plan that doesn't correspond to Substrait spec (#6833) --- .../gluten/substrait/ddlplan/DllNode.java | 39 ------------- .../gluten/substrait/ddlplan/DllPlanNode.java | 29 ---------- .../ddlplan/DllTransformContext.scala | 28 ---------- .../ddlplan/InsertOutputBuilder.java | 26 --------- .../substrait/ddlplan/InsertOutputNode.java | 56 ------------------- .../substrait/ddlplan/InsertPlanNode.java | 50 ----------------- .../substrait/proto/substrait/ddl.proto | 25 --------- .../gluten/substrait/SubstraitContext.scala | 9 --- 8 files changed, 262 deletions(-) delete mode 100644 gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllNode.java delete mode 100644 gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllPlanNode.java delete mode 100644 gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllTransformContext.scala delete mode 100644 gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputBuilder.java delete mode 100644 gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputNode.java delete mode 100644 gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertPlanNode.java delete mode 100644 gluten-core/src/main/resources/substrait/proto/substrait/ddl.proto diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllNode.java b/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllNode.java deleted file mode 100644 index e5878e0e3387..000000000000 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllNode.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.substrait.ddlplan; - -import io.substrait.proto.Dll; - -import java.io.Serializable; -import java.util.List; - -public class DllNode implements Serializable { - - private final List dllPlans; - - public DllNode(List dllPlans) { - this.dllPlans = dllPlans; - } - - public Dll toProtobuf() { - Dll.Builder dllBuilder = Dll.newBuilder(); - for (DllPlanNode dllPlanNode : dllPlans) { - dllBuilder.addDllPlan(dllPlanNode.toProtobuf()); - } - return dllBuilder.build(); - } -} diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllPlanNode.java b/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllPlanNode.java deleted file mode 100644 index 80c57eaa8cb1..000000000000 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllPlanNode.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.substrait.ddlplan; - -import io.substrait.proto.DllPlan; - -/** Contains helper functions for constructing substrait relations. */ -public interface DllPlanNode { - /** - * Converts a Expression into a protobuf. - * - * @return A rel protobuf - */ - DllPlan toProtobuf(); -} diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllTransformContext.scala b/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllTransformContext.scala deleted file mode 100644 index d63bb839f986..000000000000 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/DllTransformContext.scala +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.gluten.substrait.ddlplan - -import org.apache.gluten.substrait.SubstraitContext - -import org.apache.spark.sql.catalyst.expressions.Attribute - -case class DllTransformContext( - inputAttributes: Seq[Attribute], - outputAttributes: Seq[Attribute], - root: DllNode, - substraitContext: SubstraitContext = null) diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputBuilder.java b/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputBuilder.java deleted file mode 100644 index 75146de0c58a..000000000000 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputBuilder.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.substrait.ddlplan; - -public class InsertOutputBuilder { - private InsertOutputBuilder() {} - - public static InsertOutputNode makeInsertOutputNode( - Long partsNum, String database, String tableName, String relativePath) { - return new InsertOutputNode(partsNum, database, tableName, relativePath); - } -} diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputNode.java b/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputNode.java deleted file mode 100644 index c5804e4e2a10..000000000000 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertOutputNode.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.substrait.ddlplan; - -import com.google.protobuf.Any; -import com.google.protobuf.ByteString; -import io.substrait.proto.ReadRel; - -import java.io.Serializable; - -public class InsertOutputNode implements Serializable { - private static final String MERGE_TREE = "MergeTree;"; - private Long partsNum; - private String database = null; - private String tableName = null; - private String relativePath = null; - private StringBuffer extensionTableStr = new StringBuffer(MERGE_TREE); - - InsertOutputNode(Long partsNum, String database, String tableName, String relativePath) { - this.partsNum = partsNum; - this.database = database; - this.tableName = tableName; - this.relativePath = relativePath; - // MergeTree;{database}\n{table}\n{relative_path}\n{min_part}\n{max_part}\n - extensionTableStr - .append(database) - .append("\n") - .append(tableName) - .append("\n") - .append(relativePath) - .append("\n") - .append(this.partsNum) - .append("\n"); - } - - public ReadRel.ExtensionTable toProtobuf() { - ReadRel.ExtensionTable.Builder extensionTableBuilder = ReadRel.ExtensionTable.newBuilder(); - extensionTableBuilder.setDetail( - Any.newBuilder().setValue(ByteString.copyFromUtf8(extensionTableStr.toString()))); - return extensionTableBuilder.build(); - } -} diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertPlanNode.java b/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertPlanNode.java deleted file mode 100644 index c6afa65780c2..000000000000 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/ddlplan/InsertPlanNode.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.substrait.ddlplan; - -import org.apache.gluten.substrait.SubstraitContext; -import org.apache.gluten.substrait.plan.PlanNode; - -import io.substrait.proto.DllPlan; -import io.substrait.proto.InsertPlan; - -import java.io.Serializable; - -public class InsertPlanNode implements DllPlanNode, Serializable { - - private final PlanNode inputNode; - - private final SubstraitContext context; - - public InsertPlanNode(SubstraitContext context, PlanNode inputNode) { - this.inputNode = inputNode; - this.context = context; - } - - @Override - public DllPlan toProtobuf() { - InsertPlan.Builder insertBuilder = InsertPlan.newBuilder(); - insertBuilder.setInput(inputNode.toProtobuf()); - if (context.getInsertOutputNode() != null) { - insertBuilder.setOutput(context.getInsertOutputNode().toProtobuf()); - } - - DllPlan.Builder dllBuilder = DllPlan.newBuilder(); - dllBuilder.setInsertPlan(insertBuilder.build()); - return dllBuilder.build(); - } -} diff --git a/gluten-core/src/main/resources/substrait/proto/substrait/ddl.proto b/gluten-core/src/main/resources/substrait/proto/substrait/ddl.proto deleted file mode 100644 index 833ec87369ae..000000000000 --- a/gluten-core/src/main/resources/substrait/proto/substrait/ddl.proto +++ /dev/null @@ -1,25 +0,0 @@ -syntax = "proto3"; - -package substrait; - -import "substrait/plan.proto"; -import "substrait/algebra.proto"; - -option java_multiple_files = true; -option java_package = "io.substrait.proto"; -option csharp_namespace = "Substrait.Protobuf"; - -message DllPlan { - oneof dll_type { - InsertPlan insert_plan = 1; - } -} - -message InsertPlan { - Plan input = 1; - ReadRel.ExtensionTable output = 2; -} - -message Dll { - repeated DllPlan dll_plan = 1; -} \ No newline at end of file diff --git a/gluten-core/src/main/scala/org/apache/gluten/substrait/SubstraitContext.scala b/gluten-core/src/main/scala/org/apache/gluten/substrait/SubstraitContext.scala index 6cc35bd16c6e..79148d9f3093 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/substrait/SubstraitContext.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/substrait/SubstraitContext.scala @@ -16,8 +16,6 @@ */ package org.apache.gluten.substrait -import org.apache.gluten.substrait.ddlplan.InsertOutputNode - import java.lang.{Long => JLong} import java.security.InvalidParameterException import java.util.{ArrayList => JArrayList, HashMap => JHashMap, List => JList, Map => JMap} @@ -64,16 +62,9 @@ class SubstraitContext extends Serializable { private val aggregationParamsMap = new JHashMap[JLong, AggregationParams]() private var iteratorIndex: JLong = 0L - private var insertOutputNode: InsertOutputNode = _ private var operatorId: JLong = 0L private var relId: JLong = 0L - def getInsertOutputNode: InsertOutputNode = insertOutputNode - - def setInsertOutputNode(insertOutputNode: InsertOutputNode): Unit = { - this.insertOutputNode = insertOutputNode - } - def registerFunction(funcName: String): JLong = { if (!functionMap.containsKey(funcName)) { val newFunctionId: JLong = functionMap.size.toLong