From 20c8e8c14a6e9fd7a5d9fb10ee9ec46443e96bf8 Mon Sep 17 00:00:00 2001 From: pytorchbot Date: Mon, 25 Nov 2024 15:11:15 -0800 Subject: [PATCH] Fix test with resources (#7071) Fix test failure due to resources not handled correctly by ios tests. Differential Revision: [D66392647](https://our.internmc.facebook.com/intern/diff/D66392647/) ghstack-source-id: 255370795 Pull Request resolved: https://github.com/pytorch/executorch/pull/7062 Co-authored-by: Mengwei Liu --- .../llama/tokenizer/test/test_tiktoken.cpp | 15 ++++++- .../llm/tokenizer/test/test_bpe_tokenizer.cpp | 8 ++++ .../llm/tokenizer/test/test_tiktoken.cpp | 40 ++++++++++--------- 3 files changed, 42 insertions(+), 21 deletions(-) diff --git a/examples/models/llama/tokenizer/test/test_tiktoken.cpp b/examples/models/llama/tokenizer/test/test_tiktoken.cpp index b9309f9921..442da62174 100644 --- a/examples/models/llama/tokenizer/test/test_tiktoken.cpp +++ b/examples/models/llama/tokenizer/test/test_tiktoken.cpp @@ -14,6 +14,10 @@ #include +#ifdef EXECUTORCH_FB_BUCK +#include +#endif + using namespace ::testing; using ::example::Version; @@ -21,13 +25,20 @@ using ::executorch::extension::llm::Tokenizer; using ::executorch::runtime::Error; using ::executorch::runtime::Result; +static std::string get_resource_path(const std::string& name) { +#ifdef EXECUTORCH_FB_BUCK + return facebook::xplat::testing::getPathForTestResource("resources/" + name); +#else + return std::getenv("RESOURCES_PATH") + std::string("/") + name; +#endif +} + class MultimodalTiktokenV5ExtensionTest : public Test { public: void SetUp() override { executorch::runtime::runtime_init(); tokenizer_ = get_tiktoken_for_llama(Version::Multimodal); - modelPath_ = std::getenv("RESOURCES_PATH") + - std::string("/test_tiktoken_tokenizer.model"); + modelPath_ = get_resource_path("test_tiktoken_tokenizer.model"); } std::unique_ptr tokenizer_; diff --git a/extension/llm/tokenizer/test/test_bpe_tokenizer.cpp b/extension/llm/tokenizer/test/test_bpe_tokenizer.cpp index c553fe59f9..d207578de1 100644 --- a/extension/llm/tokenizer/test/test_bpe_tokenizer.cpp +++ b/extension/llm/tokenizer/test/test_bpe_tokenizer.cpp @@ -6,6 +6,9 @@ * LICENSE file in the root directory of this source tree. */ +#ifdef EXECUTORCH_FB_BUCK +#include +#endif #include #include #include @@ -23,8 +26,13 @@ class TokenizerExtensionTest : public Test { void SetUp() override { executorch::runtime::runtime_init(); tokenizer_ = std::make_unique(); +#ifdef EXECUTORCH_FB_BUCK + modelPath_ = facebook::xplat::testing::getPathForTestResource( + "resources/test_bpe_tokenizer.bin"); +#else modelPath_ = std::getenv("RESOURCES_PATH") + std::string("/test_bpe_tokenizer.bin"); +#endif } std::unique_ptr tokenizer_; diff --git a/extension/llm/tokenizer/test/test_tiktoken.cpp b/extension/llm/tokenizer/test/test_tiktoken.cpp index ce2a781aa1..3132170683 100644 --- a/extension/llm/tokenizer/test/test_tiktoken.cpp +++ b/extension/llm/tokenizer/test/test_tiktoken.cpp @@ -6,11 +6,13 @@ * LICENSE file in the root directory of this source tree. */ +#ifdef EXECUTORCH_FB_BUCK +#include +#endif #include #include #include #include -#include #include using namespace ::testing; @@ -47,6 +49,15 @@ static inline std::unique_ptr> _get_special_tokens() { } return special_tokens; } + +static inline std::string _get_resource_path(const std::string& name) { +#ifdef EXECUTORCH_FB_BUCK + return facebook::xplat::testing::getPathForTestResource("resources/" + name); +#else + return std::getenv("RESOURCES_PATH") + std::string("/") + name; +#endif +} + } // namespace class TiktokenExtensionTest : public Test { @@ -55,8 +66,7 @@ class TiktokenExtensionTest : public Test { executorch::runtime::runtime_init(); tokenizer_ = std::make_unique( _get_special_tokens(), kBOSTokenIndex, kEOSTokenIndex); - modelPath_ = std::getenv("RESOURCES_PATH") + - std::string("/test_tiktoken_tokenizer.model"); + modelPath_ = _get_resource_path("test_tiktoken_tokenizer.model"); } std::unique_ptr tokenizer_; @@ -144,44 +154,36 @@ TEST_F(TiktokenExtensionTest, ConstructionWithInvalidEOSIndex) { } TEST_F(TiktokenExtensionTest, LoadWithInvalidPath) { - auto invalidModelPath = - std::getenv("RESOURCES_PATH") + std::string("/nonexistent.model"); - - Error res = tokenizer_->load(invalidModelPath.c_str()); + auto invalidModelPath = "./nonexistent.model"; + Error res = tokenizer_->load(invalidModelPath); EXPECT_EQ(res, Error::InvalidArgument); } TEST_F(TiktokenExtensionTest, LoadTiktokenFileWithInvalidRank) { - auto invalidModelPath = std::getenv("RESOURCES_PATH") + - std::string("/test_tiktoken_invalid_rank.model"); - + auto invalidModelPath = + _get_resource_path("test_tiktoken_invalid_rank.model"); Error res = tokenizer_->load(invalidModelPath.c_str()); EXPECT_EQ(res, Error::InvalidArgument); } TEST_F(TiktokenExtensionTest, LoadTiktokenFileWithInvalidBase64) { - auto invalidModelPath = std::getenv("RESOURCES_PATH") + - std::string("/test_tiktoken_invalid_base64.model"); - + auto invalidModelPath = + _get_resource_path("test_tiktoken_invalid_base64.model"); Error res = tokenizer_->load(invalidModelPath.c_str()); EXPECT_EQ(res, Error::InvalidArgument); } TEST_F(TiktokenExtensionTest, LoadTiktokenFileWithNoSpace) { - auto invalidModelPath = std::getenv("RESOURCES_PATH") + - std::string("/test_tiktoken_no_space.model"); - + auto invalidModelPath = _get_resource_path("test_tiktoken_no_space.model"); Error res = tokenizer_->load(invalidModelPath.c_str()); EXPECT_EQ(res, Error::InvalidArgument); } TEST_F(TiktokenExtensionTest, LoadTiktokenFileWithBPEFile) { - auto invalidModelPath = - std::getenv("RESOURCES_PATH") + std::string("/test_bpe_tokenizer.bin"); - + auto invalidModelPath = _get_resource_path("test_bpe_tokenizer.bin"); Error res = tokenizer_->load(invalidModelPath.c_str()); EXPECT_EQ(res, Error::InvalidArgument);