-
Notifications
You must be signed in to change notification settings - Fork 453
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[GLUTEN-3951][CH]Bug fix floor diff (#3956)
What changes were proposed in this pull request? (Please fill in changes proposed in this fix) (Fixes: #3951) How was this patch tested? TEST BY UT 端到端性能测试 数据类型为Int64，表结构 test_tbl(d Int64), 测试SQL: select count(1) from test_tbl where floor(d) > 1 数据总量3000W， 测试三次 PR 改动前：1.13s, 0.92s, 0.985s PR 改动后： 1.064s， 1.077s, 0.984s 数据类型为Float64， 表结构为test_tbl(d float64) , 测试SQL select count(1) from test_tbl where floor(d) > 1 数据总量3000W， 测试三次 PR 改动前： 1.417s， 1.386s 1.426s PR 改动后：1.568s， 1.476s， 1.508s 可见对于Int64类型来说，改动前后性能基本持平；对于float64类型来说，大约有7.6%的性能回退，主要是来自于针对数据中可能出现NaN 以及INF 的情况进行了判断和赋值。 benchmark 性能测试 使用开发的 benchmark_spark_floor_function.cpp 来测试 Int64类型测试 对于CH 的Floor函数, 结果如下 image 对于新开发的Floor函数，结果如下 image Float64类型测试 对于CH的Floor函数，结果如下 image 对于新开发的Floor函数，结果如下 image 可见对于Int64，大概有 3%左右的回退，对于Float64类型 大概有70%左右的回退 Spark UT 关于Floor 函数的测试，会通过 org.apache.spark.sql.GlutenMathFunctionsSuite 这个测试来完成，已开启
- Loading branch information
1 parent
fcb31fc
commit 9b6beac
Showing
8 changed files
with
292 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| #include <Functions/SparkFunctionFloor.h> | ||
|
|
||
| namespace local_engine | ||
| { | ||
|
|
||
| REGISTER_FUNCTION(SparkFunctionFloor) | ||
| { | ||
| factory.registerFunction<SparkFunctionFloor>(); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,143 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| #include <Functions/FunctionsRound.h> | ||
| #include <Functions/FunctionFactory.h> | ||
| #include <Columns/ColumnNullable.h> | ||
| #include <Columns/ColumnVector.h> | ||
| #include <DataTypes/IDataType.h> | ||
| #include <DataTypes/DataTypesNumber.h> | ||
| #include <bit> | ||
|
|
||
| using namespace DB; | ||
|
|
||
| namespace local_engine | ||
| { | ||
|
|
||
| template <typename T, ScaleMode scale_mode> | ||
| struct SparkFloatFloorImpl | ||
| { | ||
| private: | ||
| static_assert(!is_decimal<T>); | ||
| using Op = FloatRoundingComputation<T, RoundingMode::Floor, scale_mode>; | ||
| using Data = std::array<T, Op::data_count>; | ||
| public: | ||
| static NO_INLINE void apply(const PaddedPODArray<T> & in, size_t scale, PaddedPODArray<T> & out, PaddedPODArray<UInt8> & null_map) | ||
| { | ||
| auto mm_scale = Op::prepare(scale); | ||
| const size_t data_count = std::tuple_size<Data>(); | ||
| const T* end_in = in.data() + in.size(); | ||
| const T* limit = in.data() + in.size() / data_count * data_count; | ||
| const T* __restrict p_in = in.data(); | ||
| T* __restrict p_out = out.data(); | ||
| while (p_in < limit) | ||
| { | ||
| Op::compute(p_in, mm_scale, p_out); | ||
| p_in += data_count; | ||
| p_out += data_count; | ||
| } | ||
|
|
||
| if (p_in < end_in) | ||
| { | ||
| Data tmp_src{{}}; | ||
| Data tmp_dst; | ||
| size_t tail_size_bytes = (end_in - p_in) * sizeof(*p_in); | ||
| memcpy(&tmp_src, p_in, tail_size_bytes); | ||
| Op::compute(reinterpret_cast<T *>(&tmp_src), mm_scale, reinterpret_cast<T *>(&tmp_dst)); | ||
| memcpy(p_out, &tmp_dst, tail_size_bytes); | ||
| } | ||
| for (size_t i = 0; i < out.size(); ++i) | ||
| checkAndSetNullable(out[i], null_map[i]); | ||
| } | ||
|
|
||
| static void checkAndSetNullable(T & t, UInt8 & null_flag) | ||
| { | ||
| UInt8 is_nan = (t != t); | ||
| UInt8 is_inf = 0; | ||
| if constexpr (std::is_same_v<T, float>) | ||
| is_inf = ((*reinterpret_cast<const uint32_t *>(&t) & 0b01111111111111111111111111111111) == 0b01111111100000000000000000000000); | ||
| else if constexpr (std::is_same_v<T, double>) | ||
| is_inf | ||
| = ((*reinterpret_cast<const uint64_t *>(&t) & 0b0111111111111111111111111111111111111111111111111111111111111111) | ||
| == 0b0111111111110000000000000000000000000000000000000000000000000000); | ||
|
|
||
| null_flag = is_nan | is_inf; | ||
| if (null_flag) t = 0; | ||
| } | ||
| }; | ||
|
|
||
| class SparkFunctionFloor : public DB::FunctionFloor | ||
| { | ||
| public: | ||
| static constexpr auto name = "sparkFloor"; | ||
| static DB::FunctionPtr create(DB::ContextPtr) { return std::make_shared<SparkFunctionFloor>(); } | ||
| SparkFunctionFloor() = default; | ||
| ~SparkFunctionFloor() override = default; | ||
| DB::String getName() const override { return name; } | ||
|
|
||
| DB::DataTypePtr getReturnTypeImpl(const DB::DataTypes & arguments) const override | ||
| { | ||
| auto result_type = DB::FunctionFloor::getReturnTypeImpl(arguments); | ||
| return makeNullable(result_type); | ||
| } | ||
|
|
||
| DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments, const DB::DataTypePtr & result_type, size_t input_rows) const override | ||
| { | ||
| const ColumnWithTypeAndName & first_arg = arguments[0]; | ||
| Scale scale_arg = getScaleArg(arguments); | ||
| switch(first_arg.type->getTypeId()) | ||
| { | ||
| case TypeIndex::Float32: | ||
| return executeInternal<Float32>(first_arg.column, scale_arg); | ||
| case TypeIndex::Float64: | ||
| return executeInternal<Float64>(first_arg.column, scale_arg); | ||
| default: | ||
| DB::ColumnPtr res = DB::FunctionFloor::executeImpl(arguments, result_type, input_rows); | ||
| DB::MutableColumnPtr null_map_col = DB::ColumnUInt8::create(first_arg.column->size(), 0); | ||
| return DB::ColumnNullable::create(std::move(res), std::move(null_map_col)); | ||
| } | ||
| } | ||
|
|
||
| template<typename T> | ||
| static ColumnPtr executeInternal(const ColumnPtr & col_arg, const Scale & scale_arg) | ||
| { | ||
| const auto * col = checkAndGetColumn<ColumnVector<T>>(col_arg.get()); | ||
| auto col_res = ColumnVector<T>::create(col->size()); | ||
| MutableColumnPtr null_map_col = DB::ColumnUInt8::create(col->size(), 0); | ||
| PaddedPODArray<T> & vec_res = col_res->getData(); | ||
| PaddedPODArray<UInt8> & null_map_data = assert_cast<ColumnVector<UInt8> *>(null_map_col.get())->getData(); | ||
| if (!vec_res.empty()) | ||
| { | ||
| if (scale_arg == 0) | ||
| { | ||
| size_t scale = 1; | ||
| SparkFloatFloorImpl<T, ScaleMode::Zero>::apply(col->getData(), scale, vec_res, null_map_data); | ||
| } | ||
| else if (scale_arg > 0) | ||
| { | ||
| size_t scale = intExp10(scale_arg); | ||
| SparkFloatFloorImpl<T, ScaleMode::Positive>::apply(col->getData(), scale, vec_res, null_map_data); | ||
| } | ||
| else | ||
| { | ||
| size_t scale = intExp10(-scale_arg); | ||
| SparkFloatFloorImpl<T, ScaleMode::Negative>::apply(col->getData(), scale, vec_res, null_map_data); | ||
| } | ||
| } | ||
| return DB::ColumnNullable::create(std::move(col_res), std::move(null_map_col)); | ||
| } | ||
| }; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
98 changes: 98 additions & 0 deletions
98
cpp-ch/local-engine/tests/benchmark_spark_floor_function.cpp
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one or more | ||
| * contributor license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright ownership. | ||
| * The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| * (the "License"); you may not use this file except in compliance with | ||
| * the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| #include <Core/Block.h> | ||
| #include <Columns/IColumn.h> | ||
| #include <DataTypes/IDataType.h> | ||
| #include <DataTypes/DataTypeFactory.h> | ||
| #include <Functions/FunctionFactory.h> | ||
| #include <Functions/SparkFunctionFloor.h> | ||
| #include <Functions/FunctionsRound.h> | ||
| #include <Parser/SerializedPlanParser.h> | ||
| #include <benchmark/benchmark.h> | ||
|
|
||
| using namespace DB; | ||
|
|
||
| static Block createDataBlock(String type_str, size_t rows) | ||
| { | ||
| auto type = DataTypeFactory::instance().get(type_str); | ||
| auto column = type->createColumn(); | ||
| for (size_t i = 0; i < rows; ++i) | ||
| { | ||
| if (isInt(type)) | ||
| { | ||
| column->insert(i); | ||
| } | ||
| else if (isFloat(type)) | ||
| { | ||
| double d = i * 1.0; | ||
| column->insert(d); | ||
| } | ||
| } | ||
| Block block; | ||
| block.insert(ColumnWithTypeAndName(std::move(column), type, "d")); | ||
| return std::move(block); | ||
| } | ||
|
|
||
| static void BM_CHFloorFunction_For_Int64(benchmark::State & state) | ||
| { | ||
| using namespace DB; | ||
| auto & factory = FunctionFactory::instance(); | ||
| auto function = factory.get("floor", local_engine::SerializedPlanParser::global_context); | ||
| Block int64_block = createDataBlock("Int64", 30000000); | ||
| auto executable = function->build(int64_block.getColumnsWithTypeAndName()); | ||
| for (auto _ : state)[[maybe_unused]] | ||
| auto result = executable->execute(int64_block.getColumnsWithTypeAndName(), executable->getResultType(), int64_block.rows()); | ||
| } | ||
|
|
||
| static void BM_CHFloorFunction_For_Float64(benchmark::State & state) | ||
| { | ||
| using namespace DB; | ||
| auto & factory = FunctionFactory::instance(); | ||
| auto function = factory.get("floor", local_engine::SerializedPlanParser::global_context); | ||
| Block float64_block = createDataBlock("Float64", 30000000); | ||
| auto executable = function->build(float64_block.getColumnsWithTypeAndName()); | ||
| for (auto _ : state)[[maybe_unused]] | ||
| auto result = executable->execute(float64_block.getColumnsWithTypeAndName(), executable->getResultType(), float64_block.rows()); | ||
| } | ||
|
|
||
| static void BM_SparkFloorFunction_For_Int64(benchmark::State & state) | ||
| { | ||
| using namespace DB; | ||
| auto & factory = FunctionFactory::instance(); | ||
| auto function = factory.get("sparkFloor", local_engine::SerializedPlanParser::global_context); | ||
| Block int64_block = createDataBlock("Int64", 30000000); | ||
| auto executable = function->build(int64_block.getColumnsWithTypeAndName()); | ||
| for (auto _ : state) [[maybe_unused]] | ||
| auto result = executable->execute(int64_block.getColumnsWithTypeAndName(), executable->getResultType(), int64_block.rows()); | ||
| } | ||
|
|
||
| static void BM_SparkFloorFunction_For_Float64(benchmark::State & state) | ||
| { | ||
| using namespace DB; | ||
| auto & factory = FunctionFactory::instance(); | ||
| auto function = factory.get("sparkFloor", local_engine::SerializedPlanParser::global_context); | ||
| Block float64_block = createDataBlock("Float64", 30000000); | ||
| auto executable = function->build(float64_block.getColumnsWithTypeAndName()); | ||
| for (auto _ : state) [[maybe_unused]] | ||
| auto result = executable->execute(float64_block.getColumnsWithTypeAndName(), executable->getResultType(), float64_block.rows()); | ||
| } | ||
|
|
||
| BENCHMARK(BM_CHFloorFunction_For_Int64)->Unit(benchmark::kMillisecond)->Iterations(10); | ||
| BENCHMARK(BM_CHFloorFunction_For_Float64)->Unit(benchmark::kMillisecond)->Iterations(10); | ||
| BENCHMARK(BM_SparkFloorFunction_For_Int64)->Unit(benchmark::kMillisecond)->Iterations(10); | ||
| BENCHMARK(BM_SparkFloorFunction_For_Float64)->Unit(benchmark::kMillisecond)->Iterations(10); |