Move utils

eth-easl · Sep 27, 2023 · 25f2308 · 25f2308
1 parent 7edd8b4
commit 25f2308
Show file tree

Hide file tree

Showing 11 changed files with 62 additions and 46 deletions.
diff --git a/modyn/storage/include/internal/file_watcher/file_watcher.hpp b/modyn/storage/include/internal/file_watcher/file_watcher.hpp
@@ -15,7 +15,7 @@
 #include "internal/filesystem_wrapper/filesystem_wrapper.hpp"
 #include "internal/utils/utils.hpp"
 
-namespace storage {
+namespace storage::file_wrapper {
 class FileWatcher {
  public:
   std::atomic<bool>* stop_file_watcher_;

diff --git a/modyn/storage/include/internal/file_watcher/file_watcher_watchdog.hpp b/modyn/storage/include/internal/file_watcher/file_watcher_watchdog.hpp
@@ -13,7 +13,7 @@
 #include "internal/utils/utils.hpp"
 #include "internal/database/storage_database_connection.hpp"
 
-namespace storage {
+namespace storage::file_wrapper {
 class FileWatcherWatchdog {
  public:
   FileWatchdog(

diff --git a/modyn/storage/include/internal/file_wrapper/file_wrapper.hpp b/modyn/storage/include/internal/file_wrapper/file_wrapper.hpp
@@ -5,6 +5,7 @@
 #include <string>
 
 #include "internal/filesystem_wrapper/filesystem_wrapper.hpp"
+#include "internal/file_wrapper/file_wraper_utils.hpp"
 
 namespace storage {
 

diff --git a/modyn/storage/include/internal/file_wrapper/file_wrapper_utils.hpp b/modyn/storage/include/internal/file_wrapper/file_wrapper_utils.hpp
@@ -0,0 +1,27 @@
+#pragma once
+
+#include "internal/file_wrapper/binary_file_wrapper.hpp"
+#include "internal/file_wrapper/file_wrapper.hpp"
+#include "internal/file_wrapper/single_sample_file_wrapper.hpp"
+#include "internal/utils/utils.hpp"
+
+namespace storage::file_wrapper {
+
+static std::unique_ptr<FileWrapper> get_file_wrapper(const std::string& path, const FileWrapperType& type,
+                                                     const YAML::Node& file_wrapper_config,
+                                                     const std::shared_ptr<FilesystemWrapper>& filesystem_wrapper) {
+  ASSERT(filesystem_wrapper != nullptr, "Filesystem wrapper is nullptr");
+  ASSERT(!path.empty(), "Path is empty");
+  ASSERT(filesystem_wrapper->exists(path), "Path does not exist");
+
+  std::unique_ptr<FileWrapper> file_wrapper;
+  if (type == FileWrapperType::BINARY) {
+    file_wrapper = std::make_unique<BinaryFileWrapper>(path, file_wrapper_config, filesystem_wrapper);
+  } else if (type == FileWrapperType::SINGLE_SAMPLE) {
+    file_wrapper = std::make_unique<SingleSampleFileWrapper>(path, file_wrapper_config, filesystem_wrapper);
+  } else {
+    FAIL("Unknown file wrapper type");
+  }
+  return file_wrapper;
+}
+} // namespace storage::file_wrapper
diff --git a/modyn/storage/include/internal/filesystem_wrapper/filesystem_wrapper.hpp b/modyn/storage/include/internal/filesystem_wrapper/filesystem_wrapper.hpp
@@ -6,7 +6,9 @@
 #include <unordered_map>
 #include <vector>
 
-namespace storage {
+#include "internal/filesystem_wrapper/filesystem_wrapper_utils.hpp"
+
+namespace storage::filesystem_wrapper {
 
 enum FilesystemWrapperType { LOCAL };
 

diff --git a/modyn/storage/include/internal/filesystem_wrapper/filesystem_wrapper_utils.hpp b/modyn/storage/include/internal/filesystem_wrapper/filesystem_wrapper_utils.hpp
@@ -0,0 +1,19 @@
+#pragma once
+
+#include "internal/filesystem_wrapper/filesystem_wrapper.hpp"
+#include "internal/filesystem_wrapper/local_filesystem_wrapper.hpp"
+#include "internal/utils/utils.hpp"
+
+namespace storage::filesystem_wrapper {
+
+static std::shared_ptr<FilesystemWrapper> get_filesystem_wrapper(const std::string& path,
+                                                                 const FilesystemWrapperType& type) {
+  std::shared_ptr<FilesystemWrapper> filesystem_wrapper;
+  if (type == FilesystemWrapperType::LOCAL) {
+    filesystem_wrapper = std::make_shared<LocalFilesystemWrapper>(path);
+  } else {
+    FAIL("Unknown filesystem wrapper type");
+  }
+  return filesystem_wrapper;
+}
+}  // namespace storage::filesystem_wrapper
diff --git a/modyn/storage/include/internal/filesystem_wrapper/local_filesystem_wrapper.hpp b/modyn/storage/include/internal/filesystem_wrapper/local_filesystem_wrapper.hpp
@@ -2,7 +2,7 @@
 
 #include "internal/filesystem_wrapper/filesystem_wrapper.hpp"
 
-namespace storage {
+namespace storage::filesystem_wrapper {
 class LocalFilesystemWrapper : public FilesystemWrapper {  // NOLINT
  public:
   explicit LocalFilesystemWrapper(const std::string& path) : FilesystemWrapper(path) {}

diff --git a/modyn/storage/include/internal/utils/utils.hpp b/modyn/storage/include/internal/utils/utils.hpp
@@ -9,13 +9,7 @@
 #include <random>
 #include <string>
 
-#include "internal/file_wrapper/binary_file_wrapper.hpp"
-#include "internal/file_wrapper/file_wrapper.hpp"
-#include "internal/file_wrapper/single_sample_file_wrapper.hpp"
-#include "internal/filesystem_wrapper/filesystem_wrapper.hpp"
-#include "internal/filesystem_wrapper/local_filesystem_wrapper.hpp"
-
-#define FAIL(msg)                                                                                          \
+#define FAIL(msg) \
   throw storage::utils::ModynException("ERROR at " __FILE__ ":" + std::to_string(__LINE__) + " " + (msg) + \
                                        "\nExecution failed.")
 
@@ -36,33 +30,4 @@ class ModynException : public std::exception {
   const std::string msg_;
 };
 
-static std::shared_ptr<FilesystemWrapper> get_filesystem_wrapper(const std::string& path,
-                                                                 const FilesystemWrapperType& type) {
-  std::shared_ptr<FilesystemWrapper> filesystem_wrapper;
-  if (type == FilesystemWrapperType::LOCAL) {
-    filesystem_wrapper = std::make_shared<LocalFilesystemWrapper>(path);
-  } else {
-    FAIL("Unknown filesystem wrapper type");
-  }
-  return filesystem_wrapper;
-}
-
-static std::unique_ptr<FileWrapper> get_file_wrapper(const std::string& path, const FileWrapperType& type,
-                                                     const YAML::Node& file_wrapper_config,
-                                                     const std::shared_ptr<FilesystemWrapper>& filesystem_wrapper) {
-  assert(filesystem_wrapper != nullptr);
-  assert(!path.empty());
-  assert(filesystem_wrapper->exists(path));
-
-  std::unique_ptr<FileWrapper> file_wrapper;
-  if (type == FileWrapperType::BINARY) {
-    file_wrapper = std::make_unique<BinaryFileWrapper>(path, file_wrapper_config, filesystem_wrapper);
-  } else if (type == FileWrapperType::SINGLE_SAMPLE) {
-    file_wrapper = std::make_unique<SingleSampleFileWrapper>(path, file_wrapper_config, filesystem_wrapper);
-  } else {
-    FAIL("Unknown file wrapper type");
-  }
-  return file_wrapper;
-}
-
 }  // namespace storage::utils
diff --git a/modyn/storage/src/CMakeLists.txt b/modyn/storage/src/CMakeLists.txt
@@ -20,8 +20,10 @@ set(MODYNSTORAGE_HEADERS
     ../include/internal/file_wrapper/binary_file_wrapper.hpp
     ../include/internal/file_wrapper/single_sample_file_wrapper.hpp
     ../include/internal/file_wrapper/csv_file_wrapper.hpp
+    ../include/internal/file_wrapper/file_wrapper_utils.hpp
     ../include/internal/filesystem_wrapper/filesystem_wrapper.hpp
     ../include/internal/filesystem_wrapper/local_filesystem_wrapper.hpp
+    ../include/internal/filesystem_wrapper/filesystem_wrapper_utils.hpp
     ../include/internal/grpc/storage_grpc_server.hpp
     ../include/internal/utils/utils.hpp
     )

diff --git a/modyn/storage/src/internal/filesystem_wrapper/local_filesystem_wrapper.cpp b/modyn/storage/src/internal/filesystem_wrapper/local_filesystem_wrapper.cpp
@@ -24,7 +24,7 @@ const char path_separator =
     '/';
 #endif
 
-using namespace storage;
+using namespace storage::filesystem_wrapper;
 
 std::vector<unsigned char> LocalFilesystemWrapper::get(const std::string& path) {
   std::ifstream file;

diff --git a/modyn/storage/src/internal/grpc/storage_service_impl.cpp b/modyn/storage/src/internal/grpc/storage_service_impl.cpp
@@ -51,7 +51,7 @@ grpc::Status StorageServiceImpl::Get(  // NOLINT (readability-identifier-naming)
   }
 
   auto filesystem_wrapper =
-      storage::utils::get_filesystem_wrapper(base_path, static_cast<FilesystemWrapperType>(filesystem_wrapper_type));
+      storage::filesystem_wrapper::get_filesystem_wrapper(base_path, static_cast<FilesystemWrapperType>(filesystem_wrapper_type));
   const YAML::Node file_wrapper_config_node = YAML::Load(file_wrapper_config);
 
   if (file_id_to_sample_data.size() == 0) {
@@ -98,7 +98,7 @@ void StorageServiceImpl::send_get_response(grpc::ServerWriter<modyn::storage::Ge
   std::string file_path;
   session << "SELECT path FROM files WHERE file_id = :file_id", soci::into(file_path), soci::use(file_id);
 
-  auto file_wrapper = storage::utils::get_file_wrapper(file_path, static_cast<FileWrapperType>(file_wrapper_type),
+  auto file_wrapper = storage::file_wrapper::get_file_wrapper(file_path, static_cast<FileWrapperType>(file_wrapper_type),
                                               file_wrapper_config, filesystem_wrapper);
 
   std::vector<std::vector<unsigned char>> samples = file_wrapper->get_samples_from_indices(sample_data.indices);
@@ -346,7 +346,7 @@ grpc::Status StorageServiceImpl::DeleteDataset(              // NOLINT (readabil
   session << "SELECT base_path, filesystem_wrapper_type FROM datasets WHERE name = :name", soci::into(base_path),
       soci::into(filesystem_wrapper_type), soci::use(request->dataset_id());
 
-  auto filesystem_wrapper = storage::utils::get_filesystem_wrapper(base_path, static_cast<FilesystemWrapperType>(filesystem_wrapper_type));
+  auto filesystem_wrapper = storage::filesystem_wrapper::get_filesystem_wrapper(base_path, static_cast<FilesystemWrapperType>(filesystem_wrapper_type));
 
   int64_t number_of_files = 0;
   session << "SELECT COUNT(*) FROM files WHERE dataset_id = :dataset_id", soci::into(number_of_files),
@@ -433,7 +433,7 @@ grpc::Status StorageServiceImpl::DeleteData(           // NOLINT (readability-id
     return {grpc::StatusCode::NOT_FOUND, "No files found."};
   }
 
-  auto filesystem_wrapper = storage::utils::get_filesystem_wrapper(base_path, static_cast<FilesystemWrapperType>(filesystem_wrapper_type));
+  auto filesystem_wrapper = storage::filesystem_wrapper::get_filesystem_wrapper(base_path, static_cast<FilesystemWrapperType>(filesystem_wrapper_type));
   YAML::Node file_wrapper_config_node = YAML::Load(file_wrapper_config);
   std::string file_placeholders = fmt::format("({})", fmt::join(file_ids, ","));
   std::string index_placeholders;
@@ -447,7 +447,7 @@ grpc::Status StorageServiceImpl::DeleteData(           // NOLINT (readability-id
       return {grpc::StatusCode::INTERNAL, "Error deleting data."};
     }
 
-    auto file_wrapper = storage::utils::get_file_wrapper(file_paths.front(), static_cast<FileWrapperType>(file_wrapper_type),
+    auto file_wrapper = storage::file_wrapper::get_file_wrapper(file_paths.front(), static_cast<FileWrapperType>(file_wrapper_type),
                                                 file_wrapper_config_node, filesystem_wrapper);
     for (size_t i = 0; i < file_paths.size(); ++i) {
       const auto& file_id = file_ids[i];