From 3038049d4eac1d7c4995573122d11109c50af440 Mon Sep 17 00:00:00 2001 From: Robin Holzinger Date: Wed, 1 May 2024 15:55:11 +0200 Subject: [PATCH] fix: Load file labels with appropriate `byteorder` (#398) --- .gitignore | 6 + .../storage/integrationtest_storage_binary.py | 1 + .../file_wrapper/binary_file_wrapper.hpp | 7 +- .../file_wrapper/binary_file_wrapper.cpp | 40 +++--- .../file_wrapper/binary_file_wrapper_test.cpp | 127 ++++++++++++++++-- .../file_wrapper/csv_file_wrapper_test.cpp | 8 ++ modyn/tests/storage/storage_test_utils.cpp | 6 +- modyn/tests/storage/storage_test_utils.hpp | 4 +- 8 files changed, 166 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 16fb50465..e8cf2aa96 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,9 @@ plots/ cmake-build-debug/ clang-tidy-build/ libbuild/ + + +# Data & config files + +.data/ +.env diff --git a/integrationtests/storage/integrationtest_storage_binary.py b/integrationtests/storage/integrationtest_storage_binary.py index 58f6ffb96..ebb5010ce 100644 --- a/integrationtests/storage/integrationtest_storage_binary.py +++ b/integrationtests/storage/integrationtest_storage_binary.py @@ -45,6 +45,7 @@ def register_new_dataset() -> None: description="Test dataset for integration tests of binary wrapper.", file_wrapper_config=json.dumps( { + "byteorder": "little", "file_extension": ".bin", "label_size": 4, "record_size": 10, diff --git a/modyn/storage/include/internal/file_wrapper/binary_file_wrapper.hpp b/modyn/storage/include/internal/file_wrapper/binary_file_wrapper.hpp index e2d9b191d..235442a91 100644 --- a/modyn/storage/include/internal/file_wrapper/binary_file_wrapper.hpp +++ b/modyn/storage/include/internal/file_wrapper/binary_file_wrapper.hpp @@ -24,6 +24,7 @@ class BinaryFileWrapper : public FileWrapper { sample_size_ = record_size_ - label_size_; validate_file_extension(); file_size_ = filesystem_wrapper_->get_file_size(path); + little_endian_ = fw_config["byteorder"].as() == "little"; ASSERT(static_cast(record_size_ - label_size_) >= 1, "Each record must have at least 1 byte of data other than the label."); @@ -53,12 +54,16 @@ class BinaryFileWrapper : public FileWrapper { private: static void validate_request_indices(uint64_t total_samples, const std::vector& indices); - static int64_t int_from_bytes(const unsigned char* begin, const unsigned char* end); + static int64_t int_from_bytes_little_endian(const unsigned char* begin, const unsigned char* end); + static int64_t int_from_bytes_big_endian(const unsigned char* begin, const unsigned char* end); std::ifstream* get_stream(); uint64_t record_size_; uint64_t label_size_; uint64_t file_size_; uint64_t sample_size_; + bool little_endian_; std::shared_ptr stream_; + + friend class BinaryFileWrapperTest; // let gtest access private members }; } // namespace modyn::storage diff --git a/modyn/storage/src/internal/file_wrapper/binary_file_wrapper.cpp b/modyn/storage/src/internal/file_wrapper/binary_file_wrapper.cpp index 22619d3a5..20b04bb32 100644 --- a/modyn/storage/src/internal/file_wrapper/binary_file_wrapper.cpp +++ b/modyn/storage/src/internal/file_wrapper/binary_file_wrapper.cpp @@ -6,19 +6,16 @@ using namespace modyn::storage; -int64_t BinaryFileWrapper::int_from_bytes(const unsigned char* begin, const unsigned char* end) { - int64_t value = 0; - -#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - value = std::accumulate(begin, end, 0LL, [](uint64_t acc, unsigned char byte) { return (acc << 8u) | byte; }); -#elif __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +int64_t BinaryFileWrapper::int_from_bytes_little_endian(const unsigned char* begin, const unsigned char* end) { const std::reverse_iterator rbegin(end); const std::reverse_iterator rend(begin); - value = std::accumulate(rbegin, rend, 0LL, [](uint64_t acc, unsigned char byte) { return (acc << 8u) | byte; }); -#else -#error "Unknown byte order" -#endif - return value; + return static_cast( + std::accumulate(rbegin, rend, 0LL, [](uint64_t acc, unsigned char byte) { return (acc << 8u) | byte; })); +} + +int64_t BinaryFileWrapper::int_from_bytes_big_endian(const unsigned char* begin, const unsigned char* end) { + return static_cast( + std::accumulate(begin, end, 0LL, [](uint64_t acc, unsigned char byte) { return (acc << 8u) | byte; })); } uint64_t BinaryFileWrapper::get_number_of_samples() { return file_size_ / record_size_; } @@ -40,10 +37,11 @@ int64_t BinaryFileWrapper::get_label(uint64_t index) { get_stream()->seekg(static_cast(label_start), std::ios::beg); - std::vector label_vec(label_size_); - get_stream()->read(reinterpret_cast(label_vec.data()), static_cast(label_size_)); + std::vector tmp_label_buffer(label_size_); + get_stream()->read(reinterpret_cast(tmp_label_buffer.data()), static_cast(label_size_)); - return int_from_bytes(label_vec.data(), label_vec.data() + label_size_); + auto func_int_from_bytes = little_endian_ ? int_from_bytes_little_endian : int_from_bytes_big_endian; + return func_int_from_bytes(tmp_label_buffer.data(), tmp_label_buffer.data() + label_size_); } std::ifstream* BinaryFileWrapper::get_stream() { @@ -61,13 +59,15 @@ std::vector BinaryFileWrapper::get_all_labels() { std::vector labels = std::vector(); labels.reserve(num_samples); - for (uint64_t i = 0; i < num_samples; ++i) { - get_stream()->seekg(static_cast(i * record_size_), std::ios::beg); + std::vector tmp_label_buffer(label_size_); - std::vector label_vec(label_size_); - get_stream()->read(reinterpret_cast(label_vec.data()), static_cast(label_size_)); + auto func_int_from_bytes = little_endian_ ? int_from_bytes_little_endian : int_from_bytes_big_endian; - labels.push_back(int_from_bytes(label_vec.data(), label_vec.data() + label_size_)); + for (uint64_t i = 0; i < num_samples; ++i) { + get_stream()->seekg(static_cast(i * record_size_), std::ios::beg); + tmp_label_buffer.assign(label_size_, 0); + get_stream()->read(reinterpret_cast(tmp_label_buffer.data()), static_cast(label_size_)); + labels.push_back(func_int_from_bytes(tmp_label_buffer.data(), tmp_label_buffer.data() + label_size_)); } return labels; @@ -162,4 +162,4 @@ void BinaryFileWrapper::set_file_path(const std::string& path) { } } -FileWrapperType BinaryFileWrapper::get_type() { return FileWrapperType::BINARY; } \ No newline at end of file +FileWrapperType BinaryFileWrapper::get_type() { return FileWrapperType::BINARY; } diff --git a/modyn/tests/storage/internal/file_wrapper/binary_file_wrapper_test.cpp b/modyn/tests/storage/internal/file_wrapper/binary_file_wrapper_test.cpp index eaeb07577..1e15446a5 100644 --- a/modyn/tests/storage/internal/file_wrapper/binary_file_wrapper_test.cpp +++ b/modyn/tests/storage/internal/file_wrapper/binary_file_wrapper_test.cpp @@ -5,17 +5,18 @@ #include #include +#include #include #include "internal/filesystem_wrapper/mock_filesystem_wrapper.hpp" #include "storage_test_utils.hpp" #include "test_utils.hpp" -using namespace modyn::storage; +namespace modyn::storage { class BinaryFileWrapperTest : public ::testing::Test { protected: - std::string file_name_; + std::string file_name_, file_name_endian_; YAML::Node config_; std::shared_ptr filesystem_wrapper_; std::string tmp_dir_ = modyn::test::TestUtils::get_tmp_testdir("binary_file_wrapper_test"); @@ -24,30 +25,73 @@ class BinaryFileWrapperTest : public ::testing::Test { : config_{StorageTestUtils::get_dummy_file_wrapper_config()}, filesystem_wrapper_{std::make_shared()} { file_name_ = tmp_dir_ + "/test.bin"; + file_name_endian_ = tmp_dir_ + "/test_endian.bin"; } void SetUp() override { std::filesystem::create_directory(tmp_dir_); std::ofstream file(file_name_, std::ios::binary); - const std::vector> data = {{42, 12}, {43, 13}, {44, 14}, {45, 15}}; + const std::vector> data = {{42, 12}, {43, 13}, {44, 14}, {45, 15}}; for (const auto& [payload, label] : data) { - payload_to_file(file, payload, label); + payload_to_file_little_endian(file, payload); + payload_to_file_little_endian(file, label); } file.close(); + + // tmp test file with 2 byte labels + std::ofstream file_endian(file_name_endian_, std::ios::binary); + const std::vector> data_endian = { + {(1u << 8u) + 2u, 0}, {(3u << 8u) + 4u, 1}, {(5u << 8u) + 6u, 2}, {(7u << 8u) + 8u, 3}}; + for (const auto& [payload, label] : data_endian) { + payload_to_file_little_endian(file_endian, payload); + payload_to_file_little_endian(file_endian, label); + } + file_endian.close(); + + ASSERT_TRUE(std::filesystem::exists(file_name_)); + ASSERT_TRUE(std::filesystem::exists(file_name_endian_)); + } + + static void payload_to_file_little_endian(std::ofstream& file, uint16_t data) { + auto tmp = static_cast(data & 0x00FFu); // least significant byte + file.write(reinterpret_cast(&tmp), 1); + tmp = static_cast(data >> 8u); // most significant byte + file.write(reinterpret_cast(&tmp), 1); + } + + void TearDown() override { + std::filesystem::remove_all(file_name_); + std::filesystem::remove_all(file_name_endian_); } - static void payload_to_file(std::ofstream& file, uint16_t payload, uint16_t label) { - file.write(reinterpret_cast(&payload), sizeof(uint16_t)); - file.write(reinterpret_cast(&label), sizeof(uint16_t)); + // tests needs to be in class context to access private members + constexpr static std::array BYTES1{0b00000001}; + constexpr static std::array BYTES2{0b00000001, 0b00000010}; + constexpr static std::array BYTES4{0b00000001, 0b00000010, 0b00000011, 0b00000100}; + constexpr static std::array BYTES8{0b00000001, 0, 0, 0, 0, 0, 0, 0b00001000}; + + static void test_int_from_bytes_little_endian() { + ASSERT_EQ(BinaryFileWrapper::int_from_bytes_little_endian(BYTES1.data(), BYTES1.data() + 1u), 1); + ASSERT_EQ(BinaryFileWrapper::int_from_bytes_little_endian(BYTES2.data(), BYTES2.data() + 2u), (2u << 8u) + 1u); + ASSERT_EQ(BinaryFileWrapper::int_from_bytes_little_endian(BYTES4.data(), BYTES4.data() + 4u), + (4ull << 24u) + (3u << 16u) + (2u << 8u) + 1u); + ASSERT_EQ(BinaryFileWrapper::int_from_bytes_little_endian(BYTES8.data(), BYTES8.data() + 8u), (8ull << 56u) + 1u); } - void TearDown() override { std::filesystem::remove_all(file_name_); } + static void test_int_from_bytes_big_endian() { + ASSERT_EQ(BinaryFileWrapper::int_from_bytes_big_endian(BYTES1.data(), BYTES1.data() + 1u), 1); + ASSERT_EQ(BinaryFileWrapper::int_from_bytes_big_endian(BYTES2.data(), BYTES2.data() + 2u), (1u << 8u) + 2u); + ASSERT_EQ(BinaryFileWrapper::int_from_bytes_big_endian(BYTES4.data(), BYTES4.data() + 4u), + (1u << 24u) + (2u << 16u) + (3u << 8u) + 4u); + ASSERT_EQ(BinaryFileWrapper::int_from_bytes_big_endian(BYTES8.data(), BYTES8.data() + 8u), (1ull << 56u) + 8u); + } }; TEST_F(BinaryFileWrapperTest, TestGetNumberOfSamples) { const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillOnce(testing::Return(stream_ptr)); EXPECT_CALL(*filesystem_wrapper_, get_file_size(testing::_)).WillOnce(testing::Return(16)); @@ -61,6 +105,7 @@ TEST_F(BinaryFileWrapperTest, TestGetNumberOfSamples) { TEST_F(BinaryFileWrapperTest, TestValidateFileExtension) { const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillOnce(testing::Return(stream_ptr)); EXPECT_CALL(*filesystem_wrapper_, get_file_size(testing::_)).WillOnce(testing::Return(16)); @@ -71,6 +116,7 @@ TEST_F(BinaryFileWrapperTest, TestValidateRequestIndices) { EXPECT_CALL(*filesystem_wrapper_, get_file_size(testing::_)).WillOnce(testing::Return(16)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillRepeatedly(testing::Return(stream_ptr)); @@ -85,10 +131,14 @@ TEST_F(BinaryFileWrapperTest, TestValidateRequestIndices) { ASSERT_THROW(file_wrapper2.get_sample(8), modyn::utils::ModynException); } +TEST_F(BinaryFileWrapperTest, TestIntFromBytesLittleEndian) { test_int_from_bytes_little_endian(); } +TEST_F(BinaryFileWrapperTest, TestIntFromBytesBigEndian) { test_int_from_bytes_big_endian(); } + TEST_F(BinaryFileWrapperTest, TestGetLabel) { EXPECT_CALL(*filesystem_wrapper_, get_file_size(testing::_)).WillOnce(testing::Return(16)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillOnce(testing::Return(stream_ptr)); @@ -99,10 +149,36 @@ TEST_F(BinaryFileWrapperTest, TestGetLabel) { ASSERT_EQ(file_wrapper.get_label(3), 45); } +TEST_F(BinaryFileWrapperTest, TestGetLabelEndian) { + EXPECT_CALL(*filesystem_wrapper_, get_file_size(testing::_)).Times(2).WillRepeatedly(testing::Return(16)); + const std::shared_ptr stream_ptr = std::make_shared(); + stream_ptr->open(file_name_endian_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); + + EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).Times(2).WillRepeatedly(testing::Return(stream_ptr)); + + // [LITTLE ENDIAN] + const YAML::Node little_endian_config = StorageTestUtils::get_dummy_file_wrapper_config("little"); + BinaryFileWrapper file_wrapper_little_endian(file_name_endian_, little_endian_config, filesystem_wrapper_); + ASSERT_EQ(file_wrapper_little_endian.get_label(0), (1u << 8u) + 2u); + ASSERT_EQ(file_wrapper_little_endian.get_label(1), (3u << 8u) + 4u); + ASSERT_EQ(file_wrapper_little_endian.get_label(2), (5u << 8u) + 6u); + ASSERT_EQ(file_wrapper_little_endian.get_label(3), (7u << 8u) + 8u); + + // [BIG ENDIAN] + const YAML::Node big_endian_config = StorageTestUtils::get_dummy_file_wrapper_config("big"); + BinaryFileWrapper file_wrapper_big_endian(file_name_, big_endian_config, filesystem_wrapper_); + ASSERT_EQ(file_wrapper_big_endian.get_label(0), (2u << 8u) + 1u); + ASSERT_EQ(file_wrapper_big_endian.get_label(1), (4u << 8u) + 3u); + ASSERT_EQ(file_wrapper_big_endian.get_label(2), (6u << 8u) + 5u); + ASSERT_EQ(file_wrapper_big_endian.get_label(3), (8u << 8u) + 7u); +} + TEST_F(BinaryFileWrapperTest, TestGetAllLabels) { EXPECT_CALL(*filesystem_wrapper_, get_file_size(testing::_)).WillOnce(testing::Return(16)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillOnce(testing::Return(stream_ptr)); @@ -115,10 +191,40 @@ TEST_F(BinaryFileWrapperTest, TestGetAllLabels) { ASSERT_EQ((labels)[3], 45); } +TEST_F(BinaryFileWrapperTest, TestGetAllLabelsEndian) { + EXPECT_CALL(*filesystem_wrapper_, get_file_size(testing::_)).Times(2).WillRepeatedly(testing::Return(16)); + const std::shared_ptr stream_ptr = std::make_shared(); + stream_ptr->open(file_name_endian_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); + + EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).Times(2).WillRepeatedly(testing::Return(stream_ptr)); + + // [LITTLE ENDIAN] + const YAML::Node little_endian_config = StorageTestUtils::get_dummy_file_wrapper_config("little"); + BinaryFileWrapper file_wrapper_little_endian(file_name_, little_endian_config, filesystem_wrapper_); + std::vector labels_little = file_wrapper_little_endian.get_all_labels(); + ASSERT_EQ(labels_little.size(), 4); + ASSERT_EQ((labels_little)[0], (1u << 8u) + 2u); + ASSERT_EQ((labels_little)[1], (3u << 8u) + 4u); + ASSERT_EQ((labels_little)[2], (5u << 8u) + 6u); + ASSERT_EQ((labels_little)[3], (7u << 8u) + 8u); + + // [BIG ENDIAN] + const YAML::Node big_endian_config = StorageTestUtils::get_dummy_file_wrapper_config("big"); + BinaryFileWrapper file_wrapper_big_endian(file_name_, big_endian_config, filesystem_wrapper_); + std::vector labels_big = file_wrapper_big_endian.get_all_labels(); + ASSERT_EQ(labels_big.size(), 4); + ASSERT_EQ((labels_big)[0], (2u << 8u) + 1u); + ASSERT_EQ((labels_big)[1], (4u << 8u) + 3u); + ASSERT_EQ((labels_big)[2], (6u << 8u) + 5u); + ASSERT_EQ((labels_big)[3], (8u << 8u) + 7u); +} + TEST_F(BinaryFileWrapperTest, TestGetSample) { EXPECT_CALL(*filesystem_wrapper_, get_file_size(testing::_)).WillRepeatedly(testing::Return(16)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillRepeatedly(testing::Return(stream_ptr)); @@ -144,6 +250,7 @@ TEST_F(BinaryFileWrapperTest, TestGetSamples) { EXPECT_CALL(*filesystem_wrapper_, get_file_size(testing::_)).WillRepeatedly(testing::Return(16)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillRepeatedly(testing::Return(stream_ptr)); @@ -182,6 +289,7 @@ TEST_F(BinaryFileWrapperTest, TestGetSamplesFromIndices) { EXPECT_CALL(*filesystem_wrapper_, get_file_size(testing::_)).WillRepeatedly(testing::Return(16)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillRepeatedly(testing::Return(stream_ptr)); @@ -223,6 +331,7 @@ TEST_F(BinaryFileWrapperTest, TestGetSamplesFromIndices) { TEST_F(BinaryFileWrapperTest, TestDeleteSamples) { const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillOnce(testing::Return(stream_ptr)); EXPECT_CALL(*filesystem_wrapper_, get_file_size(testing::_)).WillOnce(testing::Return(16)); @@ -233,3 +342,5 @@ TEST_F(BinaryFileWrapperTest, TestDeleteSamples) { ASSERT_NO_THROW(file_wrapper.delete_samples(label_indices)); } + +} // namespace modyn::storage diff --git a/modyn/tests/storage/internal/file_wrapper/csv_file_wrapper_test.cpp b/modyn/tests/storage/internal/file_wrapper/csv_file_wrapper_test.cpp index bf10ddbaa..e19bd998b 100644 --- a/modyn/tests/storage/internal/file_wrapper/csv_file_wrapper_test.cpp +++ b/modyn/tests/storage/internal/file_wrapper/csv_file_wrapper_test.cpp @@ -36,6 +36,7 @@ class CsvFileWrapperTest : public ::testing::Test { file << "2,Jane,Smith,30\n"; file << "3,Michael,Johnson,35\n"; file.close(); + ASSERT_TRUE(std::filesystem::exists(file_name_)); } void TearDown() override { std::filesystem::remove_all(file_name_); } @@ -45,6 +46,7 @@ TEST_F(CsvFileWrapperTest, TestGetNumberOfSamples) { EXPECT_CALL(*filesystem_wrapper_, exists(testing::_)).WillOnce(testing::Return(true)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillOnce(testing::Return(stream_ptr)); CsvFileWrapper file_wrapper{file_name_, config_, filesystem_wrapper_}; @@ -59,6 +61,7 @@ TEST_F(CsvFileWrapperTest, TestGetLabel) { EXPECT_CALL(*filesystem_wrapper_, exists(testing::_)).WillOnce(testing::Return(true)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillOnce(testing::Return(stream_ptr)); CsvFileWrapper file_wrapper{file_name_, config_, filesystem_wrapper_}; @@ -80,6 +83,7 @@ TEST_F(CsvFileWrapperTest, TestGetAllLabels) { EXPECT_CALL(*filesystem_wrapper_, exists(testing::_)).WillOnce(testing::Return(true)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillOnce(testing::Return(stream_ptr)); CsvFileWrapper file_wrapper{file_name_, config_, filesystem_wrapper_}; @@ -94,6 +98,7 @@ TEST_F(CsvFileWrapperTest, TestGetSamples) { EXPECT_CALL(*filesystem_wrapper_, exists(testing::_)).WillOnce(testing::Return(true)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillOnce(testing::Return(stream_ptr)); CsvFileWrapper file_wrapper{file_name_, config_, filesystem_wrapper_}; @@ -113,6 +118,7 @@ TEST_F(CsvFileWrapperTest, TestGetSample) { EXPECT_CALL(*filesystem_wrapper_, exists(testing::_)).WillOnce(testing::Return(true)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillOnce(testing::Return(stream_ptr)); CsvFileWrapper file_wrapper{file_name_, config_, filesystem_wrapper_}; @@ -128,6 +134,7 @@ TEST_F(CsvFileWrapperTest, TestGetSamplesFromIndices) { EXPECT_CALL(*filesystem_wrapper_, exists(testing::_)).WillOnce(testing::Return(true)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillOnce(testing::Return(stream_ptr)); CsvFileWrapper file_wrapper{file_name_, config_, filesystem_wrapper_}; @@ -146,6 +153,7 @@ TEST_F(CsvFileWrapperTest, TestDeleteSamples) { EXPECT_CALL(*filesystem_wrapper_, exists(testing::_)).WillOnce(testing::Return(true)); const std::shared_ptr stream_ptr = std::make_shared(); stream_ptr->open(file_name_, std::ios::binary); + ASSERT_TRUE(stream_ptr->is_open()); EXPECT_CALL(*filesystem_wrapper_, get_stream(testing::_)).WillOnce(testing::Return(stream_ptr)); CsvFileWrapper file_wrapper{file_name_, config_, filesystem_wrapper_}; diff --git a/modyn/tests/storage/storage_test_utils.cpp b/modyn/tests/storage/storage_test_utils.cpp index 78981bf83..335f09522 100644 --- a/modyn/tests/storage/storage_test_utils.cpp +++ b/modyn/tests/storage/storage_test_utils.cpp @@ -2,8 +2,9 @@ using namespace modyn::storage; -YAML::Node StorageTestUtils::get_dummy_file_wrapper_config() { +YAML::Node StorageTestUtils::get_dummy_file_wrapper_config(const std::string& byteorder) { YAML::Node config; + config["byteorder"] = byteorder; config["file_extension"] = ".txt"; config["label_file_extension"] = ".json"; config["label_size"] = 2; @@ -16,8 +17,9 @@ YAML::Node StorageTestUtils::get_dummy_file_wrapper_config() { return config; } -std::string StorageTestUtils::get_dummy_file_wrapper_config_inline() { +std::string StorageTestUtils::get_dummy_file_wrapper_config_inline(const std::string& byteorder) { std::string test_config = R"( +byteorder: ")" + byteorder + R"(" file_extension: ".txt" label_file_extension: ".lbl" label_size: 1 diff --git a/modyn/tests/storage/storage_test_utils.hpp b/modyn/tests/storage/storage_test_utils.hpp index 1dd6bfc04..29e43fab9 100644 --- a/modyn/tests/storage/storage_test_utils.hpp +++ b/modyn/tests/storage/storage_test_utils.hpp @@ -10,8 +10,8 @@ namespace modyn::storage { class StorageTestUtils { public: - static YAML::Node get_dummy_file_wrapper_config(); - static std::string get_dummy_file_wrapper_config_inline(); + static YAML::Node get_dummy_file_wrapper_config(const std::string& byteorder = "little"); + static std::string get_dummy_file_wrapper_config_inline(const std::string& byteorder = "little"); }; template