Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Core] Spark version function support #6469

Merged
merged 5 commits into from
Jul 17, 2024
Merged

[Core] Spark version function support #6469

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fa0fc77
spark version function support
gaoyangxiaozhu Jul 16, 2024
0d9ab16
Merge branch 'apache:main' into gayangya/version_function
gaoyangxiaozhu Jul 16, 2024
278a48a
Merge branch 'main' of https://github.com/gaoyangxiaozhu/gluten into …
gaoyangxiaozhu Jul 17, 2024
3569790
Merge branch 'gayangya/version_function' of https://github.com/gaoyan…
gaoyangxiaozhu Jul 17, 2024
7f72e9d
Merge branch 'apache:main' into gayangya/version_function
gaoyangxiaozhu Jul 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat
import org.apache.gluten.substrait.rel.LocalFilesNode.ReadFileFormat.{DwrfReadFormat, OrcReadFormat, ParquetReadFormat}

import org.apache.spark.sql.catalyst.catalog.BucketSpec
import org.apache.spark.sql.catalyst.expressions.{Alias, CumeDist, DenseRank, Descending, EulerNumber, Expression, Lag, Lead, Literal, MakeYMInterval, NamedExpression, NthValue, NTile, PercentRank, Pi, Rand, RangeFrame, Rank, RowNumber, SortOrder, SparkPartitionID, SpecialFrameBoundary, SpecifiedWindowFrame, Uuid}
import org.apache.spark.sql.catalyst.expressions.{Alias, CumeDist, DenseRank, Descending, EulerNumber, Expression, Lag, Lead, Literal, MakeYMInterval, NamedExpression, NthValue, NTile, PercentRank, Pi, Rand, RangeFrame, Rank, RowNumber, SortOrder, SparkPartitionID, SparkVersion, SpecialFrameBoundary, SpecifiedWindowFrame, Uuid}
import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, ApproximatePercentile, Count, Sum}
import org.apache.spark.sql.catalyst.plans.{JoinType, LeftOuter, RightOuter}
import org.apache.spark.sql.catalyst.util.CharVarcharUtils
Expand Down Expand Up @@ -430,7 +430,8 @@ object VeloxBackendSettings extends BackendSettingsApi {
expr match {
// Block directly falling back the below functions by FallbackEmptySchemaRelation.
case alias: Alias => checkExpr(alias.child)
case _: Rand | _: Uuid | _: MakeYMInterval | _: SparkPartitionID | _: EulerNumber | _: Pi =>
case _: Rand | _: Uuid | _: MakeYMInterval | _: SparkPartitionID | _: EulerNumber | _: Pi |
_: SparkVersion =>
true
case _ => false
}
Expand Down
10 changes: 10 additions & 0 deletions backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -716,6 +716,16 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest {
}
}

test("Test version function") {
runQueryAndCompare("""SELECT version() from lineitem limit 100""".stripMargin) {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
runQueryAndCompare("""SELECT version(), l_orderkey
| from lineitem limit 100""".stripMargin) {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
}

test("Test spark_partition_id function") {
runQueryAndCompare("""SELECT spark_partition_id(), l_orderkey
| from lineitem limit 100""".stripMargin) {
Expand Down
3 changes: 3 additions & 0 deletions gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import org.apache.gluten.sql.shims.SparkShimLoader
import org.apache.gluten.test.TestStats
import org.apache.gluten.utils.DecimalArithmeticUtil

import org.apache.spark.{SPARK_REVISION, SPARK_VERSION_SHORT}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.SQLConfHelper
import org.apache.spark.sql.catalyst.expressions._
Expand Down Expand Up @@ -686,6 +687,8 @@ object ExpressionConverter extends SQLConfHelper with Logging {
LiteralTransformer(Literal(Math.E))
case p: Pi =>
LiteralTransformer(Literal(Math.PI))
case v: SparkVersion =>
LiteralTransformer(SPARK_VERSION_SHORT + " " + SPARK_REVISION)
case dateAdd: DateAdd =>
BackendsApiManager.getSparkPlanExecApiInstance.genDateAddTransformer(
attributeSeq,
Expand Down
1 change: 1 addition & 0 deletions gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionMappings.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,7 @@ object ExpressionMappings {
Sig[WidthBucket](WIDTH_BUCKET),
Sig[ReplicateRows](REPLICATE_ROWS),
Sig[RaiseError](RAISE_ERROR),
Sig[SparkVersion](VERSION),
// Decimal
Sig[UnscaledValue](UNSCALED_VALUE),
// Generator function
Expand Down
1 change: 1 addition & 0 deletions shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,7 @@ object ExpressionNames {
final val WIDTH_BUCKET = "width_bucket"
final val REPLICATE_ROWS = "replicaterows"
final val RAISE_ERROR = "raise_error"
final val VERSION = "version"

// Directly use child expression transformer
final val KNOWN_NULLABLE = "known_nullable"
Expand Down
Loading