forked from cad-audio/executorch
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adding permute_copy operator kernel optimization (#21)
* Adding permute_copy operator kernel optimization * Adding permute_copy operator kernel optimization * Code cleanup --------- Co-authored-by: dijopaul <[email protected]>
- Loading branch information
Showing
3 changed files
with
191 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,189 @@ | ||
| /* | ||
| * Copyright (c) Meta Platforms, Inc. and affiliates. | ||
| * All rights reserved. | ||
| * | ||
| * This source code is licensed under the BSD-style license found in the | ||
| * LICENSE file in the root directory of this source tree. | ||
| */ | ||
|
|
||
| #include <executorch/backends/cadence/hifi/kernels/kernels.h> | ||
| #include <executorch/kernels/portable/cpu/util/copy_ops_util.h> | ||
| #include <executorch/runtime/kernel/kernel_includes.h> | ||
|
|
||
| using exec_aten::ScalarType; | ||
| using exec_aten::SizesType; | ||
| using exec_aten::Tensor; | ||
| using executorch::runtime::IntArrayRef; | ||
| using executorch::runtime::KernelRuntimeContext; | ||
| using executorch::runtime::kTensorDimensionLimit; | ||
| using torch::executor::Error; | ||
|
|
||
| namespace impl { | ||
| namespace HiFi { | ||
| namespace native { | ||
|
|
||
| namespace { | ||
|
|
||
| void increment_coordinate_permuted( | ||
| const Tensor& tensor, | ||
| size_t* const coordinate, | ||
| IntArrayRef dims) { | ||
| for (int i = dims.size() - 1; i >= 0; i--) { | ||
| size_t d = dims[i] >= 0 ? dims[i] : dims[i] + tensor.dim(); | ||
| coordinate[d]++; | ||
| if (coordinate[d] == tensor.size(d)) { | ||
| coordinate[d] = 0; | ||
| } else { | ||
| return; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| } // namespace | ||
|
|
||
| Tensor& permute_copy_out( | ||
| KernelRuntimeContext& ctx, | ||
| const Tensor& in, | ||
| IntArrayRef dims, | ||
| Tensor& out) { | ||
| (void)ctx; | ||
|
|
||
| ET_KERNEL_CHECK( | ||
| ctx, check_permute_copy_args(in, dims, out), InvalidArgument, out); | ||
|
|
||
| ET_KERNEL_CHECK( | ||
| ctx, tensors_have_same_dim_order(in, out), InvalidArgument, out); | ||
|
|
||
| Tensor::SizesType expected_out_size[kTensorDimensionLimit]; | ||
| size_t expected_out_dim = 0; | ||
| get_permute_copy_out_target_size( | ||
| in, dims, expected_out_size, &expected_out_dim); | ||
| ET_KERNEL_CHECK( | ||
| ctx, | ||
| resize_tensor(out, {expected_out_size, expected_out_dim}) == Error::Ok, | ||
| InvalidArgument, | ||
| out); | ||
|
|
||
| const auto in_type = out.scalar_type(); | ||
|
|
||
| constexpr auto name = "permute_copy.out"; | ||
| constexpr int kNnlibMaxDim = 16; | ||
|
|
||
| bool optimized = 0; | ||
|
|
||
| if (out.scalar_type() == ScalarType::Float) | ||
| optimized = 1; | ||
| else if (out.scalar_type() == ScalarType::Char) | ||
| optimized = 1; | ||
| else if (out.scalar_type() == ScalarType::Byte) | ||
| optimized = 1; | ||
|
|
||
| if (in.dim() > kNnlibMaxDim) | ||
| optimized = 0; | ||
|
|
||
| if (optimized) { | ||
| if (in_type == ScalarType::Float) { | ||
| WORD32* p_inp = (WORD32*)in.const_data_ptr<float>(); | ||
| WORD32* p_out = (WORD32*)out.mutable_data_ptr<float>(); | ||
|
|
||
| WORD32 num_inp_dims = in.dim(); | ||
| WORD32 num_out_dims = num_inp_dims; | ||
|
|
||
| WORD32 p_inp_shape[kNnlibMaxDim]; | ||
| WORD32 p_out_shape[kNnlibMaxDim]; | ||
| WORD32 p_permute_vec[kNnlibMaxDim]; | ||
|
|
||
| for (int i = 0; i < num_inp_dims; i++) { | ||
| p_inp_shape[i] = in.size(i); | ||
| p_out_shape[i] = in.size(dims[i]); | ||
| p_permute_vec[i] = dims[i]; | ||
| } | ||
|
|
||
| xa_nn_transpose_32_32( | ||
| p_out, | ||
| p_out_shape, | ||
| p_inp, | ||
| p_inp_shape, | ||
| p_permute_vec, | ||
| num_out_dims, | ||
| num_inp_dims); | ||
|
|
||
| return out; | ||
| } else if (in_type == ScalarType::Char) { | ||
| WORD8* p_inp = (WORD8*)in.const_data_ptr<char>(); | ||
| WORD8* p_out = (WORD8*)out.mutable_data_ptr<char>(); | ||
|
|
||
| WORD32 num_inp_dims = in.dim(); | ||
| WORD32 num_out_dims = num_inp_dims; | ||
|
|
||
| WORD32 p_inp_shape[kNnlibMaxDim]; | ||
| WORD32 p_out_shape[kNnlibMaxDim]; | ||
| WORD32 p_permute_vec[kNnlibMaxDim]; | ||
|
|
||
| for (int i = 0; i < num_inp_dims; i++) { | ||
| p_inp_shape[i] = in.size(i); | ||
| p_out_shape[i] = in.size(dims[i]); | ||
| p_permute_vec[i] = dims[i]; | ||
| } | ||
|
|
||
| xa_nn_transpose_8_8( | ||
| p_out, | ||
| p_out_shape, | ||
| p_inp, | ||
| p_inp_shape, | ||
| p_permute_vec, | ||
| num_out_dims, | ||
| num_inp_dims); | ||
|
|
||
| } else if (in_type == ScalarType::Byte) { | ||
| WORD8* p_inp = (WORD8*)in.const_data_ptr<uint8_t>(); | ||
| WORD8* p_out = (WORD8*)out.mutable_data_ptr<uint8_t>(); | ||
|
|
||
| WORD32 num_inp_dims = in.dim(); | ||
| WORD32 num_out_dims = num_inp_dims; | ||
|
|
||
| WORD32 p_inp_shape[kNnlibMaxDim]; | ||
| WORD32 p_out_shape[kNnlibMaxDim]; | ||
| WORD32 p_permute_vec[kNnlibMaxDim]; | ||
|
|
||
| for (int i = 0; i < num_inp_dims; i++) { | ||
| p_inp_shape[i] = in.size(i); | ||
| p_out_shape[i] = in.size(dims[i]); | ||
| p_permute_vec[i] = dims[i]; | ||
| } | ||
|
|
||
| xa_nn_transpose_8_8( | ||
| p_out, | ||
| p_out_shape, | ||
| p_inp, | ||
| p_inp_shape, | ||
| p_permute_vec, | ||
| num_out_dims, | ||
| num_inp_dims); | ||
| } | ||
| return out; | ||
| } | ||
|
|
||
| size_t in_coord[kTensorDimensionLimit] = {0}; | ||
| size_t trailing_dims_memo[kTensorDimensionLimit]; | ||
| executorch::runtime::memoizeTrailingDims(in, trailing_dims_memo); | ||
|
|
||
| // in and out must be the same dtype | ||
| ET_SWITCH_ALL_TYPES(in_type, ctx, name, CTYPE, [&] { | ||
| const CTYPE* const in_data = in.const_data_ptr<CTYPE>(); | ||
| CTYPE* const out_data = out.mutable_data_ptr<CTYPE>(); | ||
|
|
||
| for (size_t i = 0; i < out.numel(); ++i) { | ||
| out_data[i] = | ||
| in_data[executorch::runtime::coordinateToIndexWithTrailingDimsMemo( | ||
| in, in_coord, trailing_dims_memo)]; | ||
| increment_coordinate_permuted(in, in_coord, dims); | ||
| } | ||
| }); | ||
|
|
||
| return out; | ||
| } | ||
|
|
||
| } // namespace native | ||
| } // namespace HiFi | ||
| } // namespace impl |