-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support expression transformers in expression fuzzer (#11264)
Summary: Pull Request resolved: #11264 Some Velox functions returns Array-typed results with different order of elements from Presto, such as array_intersect, array_except, etc. This mismatch of element order is cosmetic because the order is not defined. To tolerate this mismatch in fuzzer, this diff introduces an SortArrayTransformer that wraps these array functions in array_sort when generating the random expressions. There is a limitation with the current SortArrayTransformer that it doesn't support arrays that contain maps because map is not an orderable type. In this situation, SortArrayTransformer transform the expression into a NULL constant. This can be extended later to apply `array_sort(a, (x, y) -> if(json_format(cast(x as json)) < json_format(cast(y as json)), 1, if(json_format(cast(x as json)) = json_format(cast(y as json)), 0, -1)))` if the array contains map. Reviewed By: bikramSingh91 Differential Revision: D64358222 fbshipit-source-id: 12b35def70b1fb8297499a09324ad3292b3052c5
- Loading branch information
1 parent
6440a44
commit e67f11b
Showing
8 changed files
with
175 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| /* | ||
| * Copyright (c) Facebook, Inc. and its affiliates. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| #pragma once | ||
|
|
||
| #include "velox/core/Expressions.h" | ||
|
|
||
| namespace facebook::velox::exec::test { | ||
|
|
||
| class ExprTransformer { | ||
| public: | ||
| virtual ~ExprTransformer() = default; | ||
|
|
||
| /// Transforms the given expression into a new expression. This should be | ||
| /// called during the expression generation in expression fuzzer. | ||
| virtual core::TypedExprPtr transform(core::TypedExprPtr) const = 0; | ||
|
|
||
| /// Returns the additional number of levels of nesting introduced by the | ||
| /// transformation. | ||
| virtual int32_t extraLevelOfNesting() const = 0; | ||
| }; | ||
|
|
||
| } // namespace facebook::velox::exec::test |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| /* | ||
| * Copyright (c) Facebook, Inc. and its affiliates. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| #pragma once | ||
|
|
||
| #include "velox/core/Expressions.h" | ||
| #include "velox/exec/fuzzer/ExprTransformer.h" | ||
| #include "velox/type/Type.h" | ||
|
|
||
| namespace facebook::velox::exec::test { | ||
|
|
||
| using facebook::velox::TypePtr; | ||
| using facebook::velox::core::TypedExprPtr; | ||
| using facebook::velox::exec::test::ExprTransformer; | ||
|
|
||
| class SortArrayTransformer : public ExprTransformer { | ||
| public: | ||
| ~SortArrayTransformer() override = default; | ||
|
|
||
| /// Wraps 'expr' in a call to array_sort. If the type of 'expr' contains a | ||
| /// map, array_sort doesn't support this type, so we return a constant null | ||
| /// instead. | ||
| TypedExprPtr transform(TypedExprPtr expr) const override { | ||
| facebook::velox::TypePtr type = expr->type(); | ||
| if (containsMap(type)) { | ||
| // TODO: support map type by using array_sort with a lambda that casts | ||
| // array elements to JSON before comparison. | ||
| return std::make_shared<facebook::velox::core::ConstantTypedExpr>( | ||
| type, facebook::velox::variant::null(type->kind())); | ||
| } else { | ||
| return std::make_shared<facebook::velox::core::CallTypedExpr>( | ||
| type, std::vector<TypedExprPtr>{std::move(expr)}, "array_sort"); | ||
| } | ||
| } | ||
|
|
||
| int32_t extraLevelOfNesting() const override { | ||
| return 1; | ||
| } | ||
|
|
||
| private: | ||
| bool containsMap(const TypePtr& type) const { | ||
| if (type->isMap()) { | ||
| return true; | ||
| } else if (type->isArray()) { | ||
| return containsMap(type->asArray().elementType()); | ||
| } else if (type->isRow()) { | ||
| for (const auto& child : type->asRow().children()) { | ||
| if (containsMap(child)) { | ||
| return true; | ||
| } | ||
| } | ||
| } | ||
| return false; | ||
| } | ||
| }; | ||
|
|
||
| } // namespace facebook::velox::exec::test |