From ae28f8aa0d667e555187e57ede45ebbdfb097931 Mon Sep 17 00:00:00 2001 From: Robert Underwood Date: Wed, 17 Feb 2021 14:52:56 -0500 Subject: [PATCH] libpressio version 0.57.0 Major Changes: + BREAKING CHANGE - removed the ftk plugin in favor of using it from the external metrics plugin. This change reduces complexity and code duplication between this library and the ftk command line application this also prevents the frequent internals changes in FTK from breaking libpressio's build. closes #12 + libpressio now has JSON support with the following features: + pressio_options structures can be serialized to and from JSON. NOTE when serializing options that use pressio_option_userptr_type are omitted because by definition they cannot be serialized. + external metrics can now read a JSON based format when using the `external:api=json:1` version. This allows returning complex datatypes from metrics. + an experimental hdf5_filter which provides another way to use compressors supported with libpressio. The JSON support is used in serializing options where relevant. Minor Changes: + Moved the python bindings to `tools/swig` + Added the ability to not produce man page links and install docs when documentation is built + renamed the imagemagick check to be imagemagick_longlong to more clear about what it was testing for. Bug Fixes: + Added missing headers to CMakeLists to improve clangd and clion autocompetion. + Fixed all current documentation warnings and compiler warnings --- CMakeLists.txt | 55 ++-- ...imagemagick.cc => imagemagick_longlong.cc} | 0 include/libpressio_ext/cpp/compressor.h | 6 +- include/libpressio_ext/cpp/configurable.h | 5 + include/libpressio_ext/cpp/data.h | 5 + .../libpressio_ext/cpp/distributed_manager.h | 65 ++++ include/libpressio_ext/cpp/io.h | 6 +- include/libpressio_ext/cpp/json.h | 11 + include/libpressio_ext/cpp/metrics.h | 1 + include/libpressio_ext/cpp/options.h | 24 ++ include/libpressio_ext/cpp/pressio.h | 21 ++ include/libpressio_ext/cpp/printers.h | 16 +- include/libpressio_ext/cpp/serializable.h | 108 +++++++ include/libpressio_ext/cpp/subgroup_manager.h | 29 ++ .../json/pressio_options_json.h | 32 ++ .../libpressio_ext/launch/external_launch.h | 89 +++++- src/plugins/metrics/external.cc | 50 ++- src/plugins/metrics/ftk_critical_points.cc | 268 ---------------- src/pressio_data.cc | 16 + src/pressio_options_json.cc | 284 +++++++++++++++++ src/pressio_version.h.in | 1 + test/CMakeLists.txt | 11 +- test/test_hdffilter.cc | 112 +++++++ tools/hdf5_filter/CMakeLists.txt | 31 ++ .../include/libpressio_hdf5_filter.h | 21 ++ .../hdf5_filter/src/libpressio_hdf5_filter.cc | 300 ++++++++++++++++++ .../hdf5_filter/src/libpressio_hdf5_props.cc | 236 ++++++++++++++ {swig => tools/swig}/CMakeLists.txt | 0 {swig => tools/swig}/libpressio.py | 0 {swig => tools/swig}/numpy.i | 0 {swig => tools/swig}/pressio.i | 0 {swig => tools/swig}/pressio_sz.i | 0 {swig => tools/swig}/pressio_zfp.i | 0 {swig => tools/swig}/pypressio.cc | 0 {swig => tools/swig}/pypressio.h | 0 35 files changed, 1485 insertions(+), 318 deletions(-) rename checks/{imagemagick.cc => imagemagick_longlong.cc} (100%) create mode 100644 include/libpressio_ext/cpp/json.h create mode 100644 include/libpressio_ext/json/pressio_options_json.h delete mode 100644 src/plugins/metrics/ftk_critical_points.cc create mode 100644 src/pressio_options_json.cc create mode 100644 test/test_hdffilter.cc create mode 100644 tools/hdf5_filter/CMakeLists.txt create mode 100644 tools/hdf5_filter/include/libpressio_hdf5_filter.h create mode 100644 tools/hdf5_filter/src/libpressio_hdf5_filter.cc create mode 100644 tools/hdf5_filter/src/libpressio_hdf5_props.cc rename {swig => tools/swig}/CMakeLists.txt (100%) rename {swig => tools/swig}/libpressio.py (100%) rename {swig => tools/swig}/numpy.i (100%) rename {swig => tools/swig}/pressio.i (100%) rename {swig => tools/swig}/pressio_sz.i (100%) rename {swig => tools/swig}/pressio_zfp.i (100%) rename {swig => tools/swig}/pypressio.cc (100%) rename {swig => tools/swig}/pypressio.h (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8651e47..a8eface 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.13 FATAL_ERROR) -project(libpressio VERSION "0.56.2" LANGUAGES CXX C) +project(libpressio VERSION "0.57.0" LANGUAGES CXX C) #correct was to set a default build type # https://blog.kitware.com/cmake-and-the-default-build-type/ @@ -77,18 +77,18 @@ add_library(libpressio #public headers include/libpressio.h - include/libpressio_ext/cpp/configurable.h - include/libpressio_ext/cpp/versionable.h - include/libpressio_ext/cpp/errorable.h include/libpressio_ext/cpp/compressor.h + include/libpressio_ext/cpp/configurable.h include/libpressio_ext/cpp/data.h + include/libpressio_ext/cpp/errorable.h + include/libpressio_ext/cpp/io.h include/libpressio_ext/cpp/libpressio.h include/libpressio_ext/cpp/metrics.h include/libpressio_ext/cpp/options.h include/libpressio_ext/cpp/pressio.h include/libpressio_ext/cpp/printers.h include/libpressio_ext/cpp/subgroup_manager.h - include/libpressio_ext/cpp/io.h + include/libpressio_ext/cpp/versionable.h include/libpressio_ext/io/posix.h include/libpressio_ext/io/pressio_io.h include/pressio.h @@ -241,7 +241,7 @@ if(LIBPRESSIO_HAS_MAGICK) try_compile( LIBPRESSIO_COMPAT_HAS_IMAGEMAGICK_LONGLONG ${CMAKE_BINARY_DIR} - SOURCES "${CMAKE_SOURCE_DIR}/checks/imagemagick.cc" + SOURCES "${CMAKE_SOURCE_DIR}/checks/imagemagick_longlong.cc" LINK_LIBRARIES PkgConfig::Magick++ ) message(STATUS "Checking for Magick::LongLong: ${LIBPRESSIO_COMPAT_HAS_IMAGEMAGICK_LONGLONG}") @@ -291,17 +291,6 @@ if(LIBPRESSIO_HAS_MPI) target_link_libraries(libpressio PRIVATE MPI::MPI_CXX) endif() -option(LIBPRESSIO_HAS_FTK "build the FTK metrics modules" OFF) -if(LIBPRESSIO_HAS_FTK) - find_package(FTK) - target_sources( - libpressio - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/src/plugins/metrics/ftk_critical_points.cc - ) - target_include_directories(libpressio PRIVATE ${FTK_INCLUDE_DIRS}) -endif() - option(LIBPRESSIO_HAS_PETSC "build the petsc io plugin" OFF) if(LIBPRESSIO_HAS_PETSC) @@ -342,10 +331,25 @@ if(LIBPRESSIO_HAS_LIBDISTRIBUTED) ${CMAKE_CURRENT_SOURCE_DIR}/src/serializable.cc ${CMAKE_CURRENT_SOURCE_DIR}/src/pressio_distributed_manager.cc ${CMAKE_CURRENT_SOURCE_DIR}/include/libpressio_ext/cpp/distributed_manager.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/libpressio_ext/cpp/serializable.h ) endif() +option(LIBPRESSIO_HAS_JSON "build the options json serialization support" OFF) +if(LIBPRESSIO_HAS_JSON) + set(LIBPRESSIO_FEATURES "${LIBPRESSIO_FEATURES} json") + find_package(nlohmann_json REQUIRED) + target_link_libraries(libpressio PRIVATE nlohmann_json::nlohmann_json) + target_sources(libpressio + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/pressio_options_json.cc + ${CMAKE_CURRENT_SOURCE_DIR}/include/libpressio_ext/json/pressio_options_json.h + ${CMAKE_CURRENT_SOURCE_DIR}/include/libpressio_ext/cpp/json.h + ) + +endif() + option(LIBPRESSIO_HAS_REMOTELAUNCH "build the remote external launch plugin" OFF) if(LIBPRESSIO_HAS_REMOTELAUNCH) set(LIBPRESSIO_FEATURES "${LIBPRESSIO_FEATURES} remotelaunch") @@ -368,6 +372,10 @@ configure_file( ${CMAKE_CURRENT_BINARY_DIR}/libpressio.pc @ONLY ) +target_sources(libpressio + PRIVATE + ${CMAKE_CURRENT_BINARY_DIR}/include/pressio_version.h +) option(USE_CLANG_TIDY "include clang-tidy warnings in the build log" OFF) if(USE_CLANG_TIDY) @@ -401,13 +409,22 @@ endif() option(BUILD_PYTHON_WRAPPER "build python wrapper" OFF) if(BUILD_PYTHON_WRAPPER) - add_subdirectory(swig) + add_subdirectory(tools/swig) +endif() + +if(LIBPRESSIO_HAS_JSON AND LIBPRESSIO_HAS_HDF) + add_subdirectory(tools/hdf5_filter) endif() option(BUILD_DOCS "build the documetation" OFF) if(BUILD_DOCS) + option(BUILD_DOCS_LINKS "build the documetation" ON) find_package(Doxygen REQUIRED dot) - set(DOXYGEN_MAN_LINKS YES) + if(BUILD_DOCS_LINKS) + set(DOXYGEN_MAN_LINKS YES) + else() + set(DOXYGEN_MAN_LINKS NO) + endif() set(DOXYGEN_GENERATE_MAN YES) set(DOXYGEN_GENERATE_HTML YES) set(DOXYGEN_EXTRACT_LOCAL_METHODS YES) diff --git a/checks/imagemagick.cc b/checks/imagemagick_longlong.cc similarity index 100% rename from checks/imagemagick.cc rename to checks/imagemagick_longlong.cc diff --git a/include/libpressio_ext/cpp/compressor.h b/include/libpressio_ext/cpp/compressor.h index f5b1cca..4688fbe 100644 --- a/include/libpressio_ext/cpp/compressor.h +++ b/include/libpressio_ext/cpp/compressor.h @@ -117,10 +117,8 @@ class libpressio_compressor_plugin :public pressio_configurable, public pressio_ */ int compress(struct pressio_data const*input, struct pressio_data* output); /** decompress a pressio_data buffer - * \param[in] in_begin iterator to the beginning of the inputs - * \param[in] in_end iterator to the end of the inputs - * \param[in,out] out_begin iterator to the beginning of the outputs - * \param[in,out] out_end iterator to the end of the outputs + * \param[in] input input buffer to compress + * \param[in,out] output buffer to decompress * \see pressio_compressor_decompress for the semantics this function should obey */ int decompress(struct pressio_data const*input, struct pressio_data* output); diff --git a/include/libpressio_ext/cpp/configurable.h b/include/libpressio_ext/cpp/configurable.h index 3d241aa..307902b 100644 --- a/include/libpressio_ext/cpp/configurable.h +++ b/include/libpressio_ext/cpp/configurable.h @@ -237,10 +237,15 @@ class pressio_configurable { protected: + /** + * \returns the string the metrics key name + * \internal + */ std::string get_metrics_key_name() const { return std::string(prefix()) + ":metric"; } + /** the name of the configurable used in nested hierarchies*/ std::string name; }; diff --git a/include/libpressio_ext/cpp/data.h b/include/libpressio_ext/cpp/data.h index 74add8c..6f74670 100644 --- a/include/libpressio_ext/cpp/data.h +++ b/include/libpressio_ext/cpp/data.h @@ -31,6 +31,7 @@ namespace { size_t data_size_in_bytes(pressio_dtype type, T const dimensions, T const dims[]) { return data_size_in_elements(dimensions, dims) * pressio_dtype_size(type); } + } /** @@ -477,6 +478,10 @@ struct pressio_data { */ pressio_data transpose(std::vector const& axis = {}) const; + /** + * \param[in] rhs the object to compare against + * \returns true if the options are equal */ + bool operator==(pressio_data const& rhs) const; private: /** diff --git a/include/libpressio_ext/cpp/distributed_manager.h b/include/libpressio_ext/cpp/distributed_manager.h index bb7816d..e9bdf44 100644 --- a/include/libpressio_ext/cpp/distributed_manager.h +++ b/include/libpressio_ext/cpp/distributed_manager.h @@ -22,6 +22,10 @@ * build an array of workers, tries to be fault tolerant in degenerate cases */ compat::optional> distributed_build_groups(const unsigned int size, const unsigned int n_workers_groups, const unsigned int n_masters, const unsigned int root); + +/** + * \returns the size of MPI_COMM_WORLD + */ int distributed_world_size(); /** @@ -29,7 +33,16 @@ int distributed_world_size(); */ class pressio_distributed_manager: public pressio_configurable, public pressio_errorable { public: + /** + * a variable used to indicate there is no bound applied + */ static size_t unlimited; + /** + * constructs a distributed_manager + * + * \param[in] max_ranks_per_worker the maximum numbers of workers to use + * \param[in] max_masters the maximum number master processes to use + */ pressio_distributed_manager(unsigned int max_ranks_per_worker = 1, unsigned int max_masters = 1): groups(*distributed_build_groups(distributed_world_size(), 0, 0, 0)), max_masters(max_masters), @@ -38,6 +51,13 @@ class pressio_distributed_manager: public pressio_configurable, public pressio_e n_masters(0) {} + /** + * create a work_queue from the provided configuration + * \param[in] begin the first task in the queue + * \param[in] end the last task in the queue + * \param[in] workerfn the function worker tasks should execute + * \param[in] masterfn the function master tasks should execute + */ template void work_queue(TaskRandomIt begin, TaskRandomIt end, WorkerFn&& workerfn, MasterFn&& masterfn) { distributed::queue::work_queue_options::type> options(comm); @@ -46,38 +66,76 @@ class pressio_distributed_manager: public pressio_configurable, public pressio_e distributed::queue::work_queue( options, begin, end, std::forward(workerfn), std::forward(masterfn)); } + /** + * helper function for send + * \param[in] t the value to send + * \param[in] dest the destination + * \param[in] tag the tag to use + * \returns an error code + */ template int send(T const& t, int dest, int tag=0) { return distributed::comm::send(t, dest, tag, comm); } + + /** + * helper function for recv + * \param[out] t the value to recv + * \param[in] source the source + * \param[in] tag the tag to use + * \param[out] s the optional status value to use + * \returns an error code + */ template int recv(T& t, int source, int tag=0, MPI_Status* s=nullptr) { return distributed::comm::recv(t, source, tag, comm, s); } + + /** + * helper function for recv + * \param[in,out] t the value to bcast + * \param[in] bcast_root the destination + * \returns an error code + */ template int bcast(T& t, int bcast_root) { return distributed::comm::bcast(t, bcast_root, comm); } + /** + * helper function for recv + * \param[in,out] t the value to bcast + * \returns an error code + */ template int bcast(T& t) { return distributed::comm::bcast(t, root, comm); } + /** + * \returns the size of the managed communicator + */ int comm_size() const { int size; MPI_Comm_size(comm, &size); return size; } + /** + * \returns the rank of the managed communicator + */ int comm_rank() const { int rank; MPI_Comm_rank(comm, &rank); return rank; } + /** + * provides options for the manager for the user to configure + * \returns the options + */ struct pressio_options get_options () const override { pressio_options opts; set(opts, "distributed:mpi_comm", (void*)comm); @@ -92,6 +150,10 @@ class pressio_distributed_manager: public pressio_configurable, public pressio_e } return opts; } + /** + * sets options for the manager for the user to configure + * \returns an error code + */ virtual int set_options (struct pressio_options const &options) override { get(options, "distributed:root", &root); get(options, "distributed:mpi_comm", (void**)&comm); @@ -129,6 +191,9 @@ class pressio_distributed_manager: public pressio_configurable, public pressio_e return 0; } + /** + * the prefix "distributed" used in introspection + */ const char* prefix() const override { return "distributed"; } private: diff --git a/include/libpressio_ext/cpp/io.h b/include/libpressio_ext/cpp/io.h index 7fc4ca0..9621aa4 100644 --- a/include/libpressio_ext/cpp/io.h +++ b/include/libpressio_ext/cpp/io.h @@ -139,7 +139,8 @@ struct libpressio_io_plugin: public pressio_configurable, public pressio_version * \param[in] data data object to populate, or nullptr to allocate it from the file if supported * \see pressio_io_read for the semantics this function should obey */ - virtual int read_many_impl(compat::span const&) { + virtual int read_many_impl(compat::span const& data) { + (void)data; return set_error(1, "read many not supported"); } @@ -147,7 +148,8 @@ struct libpressio_io_plugin: public pressio_configurable, public pressio_version * \param[in] data data to write * \see pressio_io_write for the semantics this function should obey */ - virtual int write_many_impl(compat::span const&) { + virtual int write_many_impl(compat::span const& data) { + (void)data; return set_error(1, "write many not supported"); } diff --git a/include/libpressio_ext/cpp/json.h b/include/libpressio_ext/cpp/json.h new file mode 100644 index 0000000..aa9d873 --- /dev/null +++ b/include/libpressio_ext/cpp/json.h @@ -0,0 +1,11 @@ +#include +class pressio_data; +class pressio_option; +class pressio_options; + +void to_json(nlohmann::json& j, pressio_data const& data); +void to_json(nlohmann::json& j, pressio_option const& option); +void to_json(nlohmann::json& j, pressio_options const& options); +void from_json(nlohmann::json const& j, pressio_data& data); +void from_json(nlohmann::json const& j, pressio_option& option); +void from_json(nlohmann::json const& j, pressio_options& options); diff --git a/include/libpressio_ext/cpp/metrics.h b/include/libpressio_ext/cpp/metrics.h index e949c34..87f1121 100644 --- a/include/libpressio_ext/cpp/metrics.h +++ b/include/libpressio_ext/cpp/metrics.h @@ -141,6 +141,7 @@ struct pressio_metrics { /** construct a metrics wrapper*/ pressio_metrics(std::unique_ptr&& metrics): plugin(std::move(metrics)) {} + /** construct a metrics wrapper*/ pressio_metrics(std::shared_ptr&& metrics): plugin(std::move(metrics)) {} /** allow default construction*/ diff --git a/include/libpressio_ext/cpp/options.h b/include/libpressio_ext/cpp/options.h index 4e5fbed..3a6f7c0 100644 --- a/include/libpressio_ext/cpp/options.h +++ b/include/libpressio_ext/cpp/options.h @@ -265,6 +265,13 @@ struct pressio_option final { } } + /** + * \param[in] rhs the object to compare against + * \returns true if the options are equal */ + bool operator==(pressio_option const& rhs) const { + return option == rhs.option; + } + /** * converts rhs according the conversion safety specified to the type of the option and stores the result if the cast succeeds * \param[in] rhs the option to assign to this option @@ -596,6 +603,10 @@ struct pressio_options final { options.clear(); } + /** + * copies all of the options from o + * \param[in] o the options to copy from + */ void copy_from(pressio_options const& o) { insert(o.begin(), o.end()); } @@ -631,6 +642,12 @@ struct pressio_options final { return options.insert(it, value); } + /** + * insert a collection of iterator + * + * \param[in] begin the iterator to the beginning + * \param[in] end the iterator to the end + */ template void insert(InputIt begin, InputIt end){ options.insert(begin, end); @@ -696,6 +713,13 @@ struct pressio_options final { return options.insert(value); } + /** + * \param[in] rhs the object to compare against + * \returns true if the options are equal */ + bool operator==(pressio_options const& rhs) const { + return options == rhs.options; + } + /**\returns the number of set options*/ size_t num_set() const { return std::count_if(std::begin(options), std::end(options), [](decltype(options)::value_type const& key_option){ diff --git a/include/libpressio_ext/cpp/pressio.h b/include/libpressio_ext/cpp/pressio.h index 24512d1..bd67f93 100644 --- a/include/libpressio_ext/cpp/pressio.h +++ b/include/libpressio_ext/cpp/pressio.h @@ -47,8 +47,11 @@ struct pressio_registry { std::map> factories; public: + /** the value type the registry constructs*/ using value_type = T; + /** the reference type the registry constructs*/ using reference = T&; + /** the const reference type the registry constructs*/ using const_reference = T const; /** * \returns an begin iterator over the registered types @@ -59,14 +62,32 @@ struct pressio_registry { */ auto end() const -> decltype(factories.end()) { return std::end(factories); } + /** + * checks if the name is registered + * + * \param[in] key the key to search for + * \returns true if present + */ bool contains(std::string const& key) const { return factories.find(key) != factories.end(); } + /** + * checks if the name is registered + * + * \param[in] key the key to search for + * \returns an iterator if the entry is found; else end() + */ auto find(std::string const& key) const -> decltype(factories.find(key)) { return factories.find(key); } + /** + * checks if the name is registered + * + * \param[in] key the key to search for + * \returns an iterator if the entry is found; else end() + */ auto find(std::string const& key) -> decltype(factories.find(key)) { return factories.find(key); } diff --git a/include/libpressio_ext/cpp/printers.h b/include/libpressio_ext/cpp/printers.h index a938b16..cdd6a9a 100644 --- a/include/libpressio_ext/cpp/printers.h +++ b/include/libpressio_ext/cpp/printers.h @@ -14,18 +14,30 @@ * \brief C++ stream compatible IO functions * */ +/** + * print the elements of an iterable container + * \internal + */ template > struct print_elements_helper{ template + /** + * prints the underlying container + */ int operator()(T const* begin, T const* end) { out << '['; std::copy( begin, end, std::ostream_iterator(out, ", ")); out << ']'; return 0; } + /** the stream to output to */ std::basic_ostream& out; }; +/** + * helper to construct the print_elements_helper for printing iterable collections + * \internal + */ template > print_elements_helper print_elements(std::basic_ostream &out) { return print_elements_helper{out}; @@ -114,9 +126,9 @@ operator<<(std::basic_ostream& out, pressio_option const& option) switch(type) { case pressio_option_int8_type: - return out << option.get_value(); + return out << int16_t{option.get_value()}; case pressio_option_uint8_type: - return out << option.get_value(); + return out << uint16_t{option.get_value()}; case pressio_option_int16_type: return out << option.get_value(); case pressio_option_uint16_type: diff --git a/include/libpressio_ext/cpp/serializable.h b/include/libpressio_ext/cpp/serializable.h index 83e1265..2faac18 100644 --- a/include/libpressio_ext/cpp/serializable.h +++ b/include/libpressio_ext/cpp/serializable.h @@ -22,11 +22,38 @@ namespace serializer { */ template <> struct serializer { + /** is the type representable with mpi_types */ using mpi_type = std::false_type; + /** type MPI_Datatype when mpi_type is true */ static MPI_Datatype dtype() { return MPI_INT; } + /** name of the data type */ static std::string name() { return "pressio_data"; } + /** + * send the value + * \param[in] dtype the datatype to serialize + * \param[in] dest the destination to send to + * \param[in] tag the tag to use + * \param[in] comm the comm to serialize to + * \returns an error code + * */ static int send(pressio_dtype const& dtype, int dest, int tag, MPI_Comm comm); + /** + * recv the value + * \param[out] dtype the datatype to serialize + * \param[in] source the destination to send to + * \param[in] tag the tag to use + * \param[in] comm the comm to serialize to + * \param[out] status the optional status to receive + * \returns an error code + * */ static int recv(pressio_dtype& dtype, int source, int tag, MPI_Comm comm, MPI_Status* status); + /** + * broadcast a value + * + * \param[in] dtype the datatype to broadcast + * \param[in] root the root to broadcast from + * \param[in] comm the communicator to use + */ static int bcast(pressio_dtype& dtype, int root, MPI_Comm comm); }; @@ -35,11 +62,38 @@ struct serializer { */ template <> struct serializer { + /** is the type representable with mpi_types */ using mpi_type = std::false_type; + /** type MPI_Datatype when mpi_type is true */ static MPI_Datatype dtype() { return MPI_INT; } + /** name of the data type */ static std::string name() { return "pressio_data"; } + /** + * send the value + * \param[in] data the data to serialize + * \param[in] dest the destination to send to + * \param[in] tag the tag to use + * \param[in] comm the comm to serialize to + * \returns an error code + * */ static int send(pressio_data const& data, int dest, int tag, MPI_Comm comm); + /** + * recv the value + * \param[out] data the datatype to serialize + * \param[in] source the destination to send to + * \param[in] tag the tag to use + * \param[in] comm the comm to serialize to + * \param[out] status the optional status to receive + * \returns an error code + * */ static int recv(pressio_data& data, int source, int tag, MPI_Comm comm, MPI_Status* status); + /** + * broadcast a value + * + * \param[in,out] data the datatype to broadcast + * \param[in] root the root to broadcast from + * \param[in] comm the communicator to use + */ static int bcast(pressio_data& data, int root, MPI_Comm comm); }; @@ -48,11 +102,38 @@ struct serializer { */ template <> struct serializer { + /** is the type representable with mpi_types */ using mpi_type = std::false_type; + /** type MPI_Datatype when mpi_type is true */ static MPI_Datatype dtype() { return MPI_INT; } + /** name of the data type */ static std::string name() { return "pressio_data"; } + /** + * send the value + * \param[in] data the data to serialize + * \param[in] dest the destination to send to + * \param[in] tag the tag to use + * \param[in] comm the comm to serialize to + * \returns an error code + * */ static int send(pressio_option const& data, int dest, int tag, MPI_Comm comm); + /** + * recv the value + * \param[out] data the datatype to serialize + * \param[in] source the destination to send to + * \param[in] tag the tag to use + * \param[in] comm the comm to serialize to + * \param[out] status the optional status to receive + * \returns an error code + * */ static int recv(pressio_option& data, int source, int tag, MPI_Comm comm, MPI_Status* status); + /** + * broadcast a value + * + * \param[in] data the datatype to broadcast + * \param[in] root the root to broadcast from + * \param[in] comm the communicator to use + */ static int bcast(pressio_option& data, int root, MPI_Comm comm); }; @@ -61,11 +142,38 @@ struct serializer { */ template <> struct serializer { + /** is the type representable with mpi_types */ using mpi_type = std::false_type; + /** type MPI_Datatype when mpi_type is true */ static MPI_Datatype dtype() { return MPI_INT; } + /** name of the data type */ static std::string name() { return "pressio_data"; } + /** + * send the value + * \param[in] data the data to serialize + * \param[in] dest the destination to send to + * \param[in] tag the tag to use + * \param[in] comm the comm to serialize to + * \returns an error code + * */ static int send(pressio_options const& data, int dest, int tag, MPI_Comm comm); + /** + * recv the value + * \param[out] data the datatype to serialize + * \param[in] source the destination to send to + * \param[in] tag the tag to use + * \param[in] comm the comm to serialize to + * \param[out] status the optional status to receive + * \returns an error code + * */ static int recv(pressio_options& data, int source, int tag, MPI_Comm comm, MPI_Status* status); + /** + * broadcast a value + * + * \param[in] data the datatype to broadcast + * \param[in] root the root to broadcast from + * \param[in] comm the communicator to use + */ static int bcast(pressio_options& data, int root, MPI_Comm comm); }; diff --git a/include/libpressio_ext/cpp/subgroup_manager.h b/include/libpressio_ext/cpp/subgroup_manager.h index 9bfeae5..26f60ab 100644 --- a/include/libpressio_ext/cpp/subgroup_manager.h +++ b/include/libpressio_ext/cpp/subgroup_manager.h @@ -21,6 +21,13 @@ #include #include +/** \file + * \brief helper for subgroups in meta-objects which support multiple inputs + * */ + +/** + * a helper class to map input and output buffers to a meta-compressor + */ class pressio_subgroup_manager: public pressio_configurable, public pressio_errorable { public: int set_options(const struct pressio_options &options) override { @@ -44,6 +51,12 @@ class pressio_subgroup_manager: public pressio_configurable, public pressio_erro return "subgroups"; } + /** + * makes the input and output groups match sizes and other sanity tests + * + * \param[in] inputs the inputs groups + * \param[in] outputs the output groups + */ template int normalize_and_validate(compat::span const& inputs, compat::span const& outputs) { effective_input_group = normalize_data_group(input_data_groups, inputs.size()); @@ -62,20 +75,36 @@ class pressio_subgroup_manager: public pressio_configurable, public pressio_erro return 0; } + /** + * \param[in] inputs the actual inputs + * \param[in] idx which input group to retrieve + * \returns the input group based on internal configuration + */ template std::vector get_input_group(Span const& inputs, int idx) const { return make_data_group(inputs, idx, effective_input_group); } + /** + * \param[in] inputs the actual inputs + * \param[in] idx which input group to retrieve + * \returns the input group based on internal configuration + */ template std::vector get_output_group(Span const& inputs, int idx) const { return make_data_group(inputs, idx, effective_output_group); } + /** + * \returns the effective input groups + */ std::vector const& effective_input_groups() const { return effective_input_group; } + /** + * \returns the effective outputs groups + */ std::vector const& effective_output_groups() const { return effective_output_group; } diff --git a/include/libpressio_ext/json/pressio_options_json.h b/include/libpressio_ext/json/pressio_options_json.h new file mode 100644 index 0000000..12bba69 --- /dev/null +++ b/include/libpressio_ext/json/pressio_options_json.h @@ -0,0 +1,32 @@ +#ifndef __cplusplus +extern "C" { +#endif + + /** + * \file + * \brief JSON serialization and deserialization for options structures + */ + + /** + * Converts JSON to a pressio_options structure. Intended for serialization and deserialization and the format + * is unstable. boolean entries in the JSON are not supported. + * + * \param[in] library optional argument, if the parse fails, the error message is stored in the library object + * \param[in] json c-style string containing json to be deserialized to a pressio_options type + * \return a pressio_options that needs to be freed with pressio_options_free + */ + struct pressio_options* pressio_options_new_json(struct pressio* library, const char* json); + + /** + * Converts pressio_options to JSON structure. Intended for serialization and deserialization and the format + * is unstable. Entries of type pressio_option_userptr_type are omitted. + * + * \param[in] library optional argument, if the parse fails, the error message is stored in the library object + * \param[in] options the options structure to serialize + * \return a string that needs to be freed with free() + */ + char* pressio_options_to_json(struct pressio* library, struct pressio_options const* options); + +#ifndef __cplusplus +} +#endif diff --git a/include/libpressio_ext/launch/external_launch.h b/include/libpressio_ext/launch/external_launch.h index 04bae9c..793a723 100644 --- a/include/libpressio_ext/launch/external_launch.h +++ b/include/libpressio_ext/launch/external_launch.h @@ -2,6 +2,11 @@ #define LIBPRESSIO_EXTERNAL_LAUNCH_H +/** + * \file + * \brief interface for external launch methods + */ + #include "libpressio_ext/cpp/compressor.h" #include "libpressio_ext/cpp/configurable.h" @@ -10,54 +15,124 @@ #include #include +/** + * error codes for extern_proc_results + */ enum extern_proc_error_codes { - success=0, - pipe_error=1, + /**the launch was successful */ + success=0, + /** there was a failure to create the pipe */ + pipe_error=1, + /** there was a failure to fork process */ fork_error=2, + /** there was a failure to exec the process */ exec_error=3, + /** there was a failure parsing the format */ format_error=4 }; +/** + * results from launching a process + */ struct extern_proc_results { - std::string proc_stdout; //stdout from the command - std::string proc_stderr; //stdin from the command - int return_code = 0; //the return code from the external process - int error_code = success; //used to report errors with run_command + /** stdout from the command */ + std::string proc_stdout; + /** stderr from the command */ + std::string proc_stderr; + /** the return code from the external process */ + int return_code = 0; + /** used to report errors with run_command */ + int error_code = success; }; +/** + * an extension point for launching processes + */ struct libpressio_launch_plugin: public pressio_configurable, public pressio_errorable { virtual ~libpressio_launch_plugin()=default; - virtual extern_proc_results launch(std::vector const&) const =0; + /** + * launch the process + * + * \param[in] args the arguments to launch the process + * \returns results of running the process + */ + virtual extern_proc_results launch(std::vector const& args) const =0; + /** + * clones the launch method + * + * \returns a clone of the launcher + */ virtual std::unique_ptr clone() const = 0; }; +/** + * wrapper for launching processes + */ struct pressio_launcher { + /** + * launch methods are default constructible + */ pressio_launcher()=default; + /** + * launch methods are constructible from a unique_ptr to a plugin + * + * \param[in] ptr the pointer to move from + */ pressio_launcher(std::unique_ptr&& ptr): plugin(std::move(ptr)) {} + /** + * launch methods are copy constructible and have the effect of cloning the plugin + * + * \param[in] launcher the launcher to clone + */ pressio_launcher(pressio_launcher const& launcher): plugin(launcher.plugin->clone()) {} + /** + * launch methods are move constructible + * + * \param[in] compressor the launcher to clone + */ pressio_launcher(pressio_launcher&& compressor)=default; + /** + * launch methods are move assignable + * + * \param[in] launcher the launcher to clone + */ pressio_launcher& operator=(pressio_launcher&& launcher) noexcept=default; + /** + * \returns true if the plugin is not a nullptr + */ operator bool() const { return bool(plugin); } + /** + * pressio_launcher are dereference-able + */ libpressio_launch_plugin& operator*() const noexcept { return *plugin; } + /** + * pressio_launcher are dereference-able + */ libpressio_launch_plugin* operator->() const noexcept { return plugin.get(); } + /** + * the underlying plugin + */ std::unique_ptr plugin; }; +/** + * the registry for launch plugins + */ pressio_registry>& launch_plugins(); #endif /* end of include guard: LIBPRESSIO_EXTERNAL_LAUNCH_H */ diff --git a/src/plugins/metrics/external.cc b/src/plugins/metrics/external.cc index 32eea2b..6c3f688 100644 --- a/src/plugins/metrics/external.cc +++ b/src/plugins/metrics/external.cc @@ -20,6 +20,12 @@ #include "std_compat/language.h" #include "libpressio_ext/launch/external_launch.h" +#include "pressio_version.h" + +#if LIBPRESSIO_HAS_JSON +#include +#include "libpressio_ext/cpp/json.h" +#endif @@ -142,7 +148,7 @@ class external_metric_plugin : public libpressio_metrics_plugin { //returns the version number parsed, starts at 1, zero means error - size_t api_version_number(std::istringstream& stdout_stream) const { + std::string api_version_number(std::istringstream& stdout_stream) const { std::string version_line; std::getline(stdout_stream, version_line); auto eq_pos = version_line.find('=') + 1; @@ -150,25 +156,24 @@ class external_metric_plugin : public libpressio_metrics_plugin { //report error return 0; } - return stoull(version_line.substr(eq_pos)); + return version_line.substr(eq_pos); } size_t parse_result(extern_proc_results& proc_results, pressio_options& results) const { try{ std::istringstream stdout_stream(proc_results.proc_stdout); - size_t api_version = api_version_number(stdout_stream); - switch(api_version) { - case 1: - case 2: - case 3: - case 4: - case 5: + auto api_version = api_version_number(stdout_stream); + if(api_version == "1" || api_version == "2" || api_version == "3" || api_version == "4" || api_version == "5") { parse_v1(stdout_stream, proc_results, results); - return api_version; - default: - (void)0; + return stoull(api_version); +#if LIBPRESSIO_HAS_JSON + } else if(api_version == "json:1") { + parse_json(stdout_stream, proc_results, results); + return 1; +#endif } + } catch(...) {} //swallow all errors and set error information results.clear(); @@ -179,7 +184,7 @@ class external_metric_plugin : public libpressio_metrics_plugin { return 0; } - void parse_v1(std::istringstream& stdout_stream, extern_proc_results& input, pressio_options& results) const { + void parse_v1(std::istringstream& stdout_stream, extern_proc_results const& input, pressio_options& results) const { results.clear(); for (std::string line; std::getline(stdout_stream, line); ) { @@ -195,6 +200,25 @@ class external_metric_plugin : public libpressio_metrics_plugin { set(results, "external:runtime", duration); } +#if LIBPRESSIO_HAS_JSON + void parse_json(std::istringstream& stdout_stream, extern_proc_results const& input, pressio_options& results) const { + results.clear(); + + nlohmann::json j; + stdout_stream >> j; + pressio_options options = j; + + for (auto const& item : options) { + results.set("external:results:"+item.first, item.second); + } + + set(results, "external:stderr", input.proc_stderr); + set(results, "external:return_code", input.return_code); + set(results, "external:error_code", input.return_code); + set(results, "external:runtime", duration); + } +#endif + std::vector build_command(std::vector> const& filenames, compat::span const& input_datasets) const { std::vector full_command; full_command.push_back("--api"); diff --git a/src/plugins/metrics/ftk_critical_points.cc b/src/plugins/metrics/ftk_critical_points.cc deleted file mode 100644 index 39603dd..0000000 --- a/src/plugins/metrics/ftk_critical_points.cc +++ /dev/null @@ -1,268 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "pressio_data.h" -#include "pressio_options.h" -#include "libpressio_ext/cpp/data.h" -#include "libpressio_ext/cpp/metrics.h" -#include "libpressio_ext/cpp/options.h" -#include "libpressio_ext/cpp/pressio.h" -#include "std_compat/memory.h" -#include -#include -#include -#include - -enum class pressio_ftk_field_type { - scalar = 0, - vector = 1, -}; - -struct pressio_ftk_result { - compat::optional critical_points; - compat::optional num_connected_components; -}; - -template -struct pressio_ftk_critical_point_tracker: public Tracker { - //inherit constructors from the parent class - using Tracker::Tracker; - - - pressio_data pressio_get_critical_points() const { - auto const ftk_critical_points = this->get_critical_points(); - size_t num_points = 0; - std::vector critical_points; - const size_t extent = this->cpdims(); - for (auto const& point : ftk_critical_points) { - critical_points.emplace_back(point.t); - critical_points.insert( - critical_points.end(), - std::begin(point.x), - std::next(std::begin(point.x), extent) - ); - num_points++; - } - - return pressio_data::copy( - pressio_double_dtype, - critical_points.data(), - {extent + 1, num_points} - ); - - } - - size_t pressio_num_critial_points() const { - return this->connected_components.size(); - } - -}; - -struct run_ftk_tracker { - template - pressio_ftk_result run_tracker(Tracker& tracker, double const* begin) const { - std::vector origin(global_dims.size(), 0); - - - tracker.set_communicator(comm); - tracker.use_accelerator(accelerator); - tracker.set_number_of_threads(nthreads); - tracker.set_domain(domain); - tracker.set_array_domain(ftk::lattice(origin, global_dims)); - tracker.set_input_array_partial(false); - tracker.set_scalar_field_source( - (input_type == pressio_ftk_field_type::scalar) ? ftk::SOURCE_GIVEN : ftk::SOURCE_DERIVED); - tracker.set_vector_field_source( - (input_type == pressio_ftk_field_type::vector) ? ftk::SOURCE_GIVEN : ftk::SOURCE_DERIVED); - tracker.set_jacobian_field_source(ftk::SOURCE_DERIVED); - if(input_type == pressio_ftk_field_type::scalar) { - tracker.set_jacobian_symmetric(true); - } - tracker.initialize(); - - ftk::ndarray source(begin, global_dims); - switch(input_type) { - case pressio_ftk_field_type::scalar: - tracker.push_scalar_field_snapshot(source); - break; - case pressio_ftk_field_type::vector: - tracker.push_vector_field_snapshot(source); - break; - } - tracker.advance_timestep(); - tracker.finalize(); - - return {tracker.pressio_get_critical_points(), tracker.pressio_num_critial_points()}; - - } - - template - pressio_ftk_result operator()(T const* begin, T const* end) const { - std::vector converted(begin, end); - - if(global_dims.size() == 2) { - auto tracker = pressio_ftk_critical_point_tracker(); - return run_tracker(tracker, converted.data()); - } else if(global_dims.size() == 3) { - auto tracker = pressio_ftk_critical_point_tracker(); - return run_tracker(tracker, converted.data()); - } else { - return {}; - } - } - std::vector const& global_dims; - ftk::lattice const& domain; - pressio_ftk_field_type const &input_type; - const size_t nthreads; - int accelerator; - MPI_Comm comm; - -}; - -class pressio_ftk_critical_points_plugin : public libpressio_metrics_plugin { - -public: - void begin_compress(const struct pressio_data* input, - struct pressio_data const*) override - { - const std::vector global_dims = input->dimensions(); - std::vector domain_latice_start, domain_latice_sizes; - uncomp_results = pressio_data_for_each(*input, run_ftk_tracker{ - global_dims, - ftk_domain(input), - field_type, - nthreads, - accelerator, - comm - }); - } - void end_decompress(struct pressio_data const*, - struct pressio_data const* output, int) override - { - std::vector global_dims = output->dimensions(); - std::vector domain_latice_start, domain_latice_sizes; - decomp_results = pressio_data_for_each(*output, run_ftk_tracker{ - global_dims, - ftk_domain(output), - field_type, - nthreads, - accelerator, - comm - }); - } - - struct pressio_options get_metrics_results() const override - { - pressio_options opt; - set(opt, "ftk_critical_points:uncompressed_critical_points", uncomp_results.critical_points); - set(opt, "ftk_critical_points:decompressed_critical_points", decomp_results.critical_points); - return opt; - } - - int set_options(struct pressio_options const& opts) override - { - { - int tmp_i; - std::string tmp_s; - if(get(opts, "ftk_critical_points:field_type_str", &tmp_i) == pressio_options_key_set) { - field_type = pressio_ftk_field_type(tmp_i); - } else if(get(opts, "ftk_critical_points:field_type", &tmp_s) == pressio_options_key_set) { - auto it = pressio_ftk_field_type_str.find(tmp_s); - if(it != pressio_ftk_field_type_str.end()) { - field_type = it->second; - } - } - } - - get(opts, "ftk_critical_points:ftk_accelerator", &accelerator); - get(opts, "ftk_critical_points:ftk_nthreads", &nthreads); - get(opts, "ftk_critical_points:mpi_comm", (void**)&comm); - - return 0; - } - - pressio_options get_options() const override - { - pressio_options opts{}; - set(opts, "ftk_critical_points:field_type", int(field_type)); - set_type(opts, "ftk_critical_points:field_type_str", pressio_option_charptr_type); - - set(opts, "ftk_critical_points:ftk_accelerator", accelerator); - set(opts, "ftk_critical_points:ftk_nthreads", nthreads); - set(opts, "ftk_critical_points:mpi_comm", (void*)comm); - - return opts; - } - - pressio_options get_configuration() const override - { - pressio_options opts{}; - std::vector keys; - std::transform( - std::begin(pressio_ftk_field_type_str), - std::end(pressio_ftk_field_type_str), - std::back_inserter(keys), - [](decltype(pressio_ftk_field_type_str)::const_reference c) { - return c.first; - }); - - set(opts, "ftk_critical_points:field_type_str", keys); - - return opts; - } - - - std::unique_ptr clone() override { - return compat::make_unique(*this); - } - - const char* prefix() const override { - return "ftk_critical_points"; - } - -private: - ftk::lattice ftk_domain(pressio_data const* data) const { - if(field_type == pressio_ftk_field_type::scalar) { - std::vector origin(data->num_dimensions(), 2); - std::vector size = data->dimensions(); - for (auto& i : size) { - i -= 3; - } - return ftk::lattice(origin, size); - } if(field_type == pressio_ftk_field_type::vector) { - std::vector origin(data->num_dimensions(), 1); - std::vector size = data->dimensions(); - for (auto& i : size) { - i -= 2; - } - return ftk::lattice(origin, size); - } - return ftk::lattice{}; - } - - pressio_ftk_result uncomp_results,decomp_results; - pressio_ftk_field_type field_type = pressio_ftk_field_type::scalar; - unsigned int nthreads = 1; - int accelerator = ftk::FTK_XL_NONE; - MPI_Comm comm = MPI_COMM_WORLD; - static const std::map pressio_ftk_field_type_str ; -}; - -const std::map pressio_ftk_critical_points_plugin::pressio_ftk_field_type_str { - {"scalar", pressio_ftk_field_type::scalar}, - {"vector", pressio_ftk_field_type::vector}, -}; - -static pressio_register metrics_ftk_critcal_points_plugin(metrics_plugins(), "ftk_critical_points", []() { - return compat::make_unique(); -}); diff --git a/src/pressio_data.cc b/src/pressio_data.cc index 7b1b473..362ed27 100644 --- a/src/pressio_data.cc +++ b/src/pressio_data.cc @@ -122,6 +122,16 @@ namespace { return 0; } }; + + + struct data_all_equal { + + template + bool operator()(T const* lhs_begin, T const* lhs_end, V const* rhs_begin, V const*) { + return std::equal(lhs_begin, lhs_end, rhs_begin); + } + + }; } @@ -369,4 +379,10 @@ pressio_data pressio_data::cast(pressio_dtype const dtype) const { return data; } +bool pressio_data::operator==(pressio_data const& rhs) const { + if(data_dtype != rhs.data_dtype) return false; + if(dims != rhs.dims) return false; + return pressio_data_for_each(*this, rhs, data_all_equal{}); +} + } diff --git a/src/pressio_options_json.cc b/src/pressio_options_json.cc new file mode 100644 index 0000000..9553359 --- /dev/null +++ b/src/pressio_options_json.cc @@ -0,0 +1,284 @@ +#include +#include +#include +#include +#include +#include + +void to_json(nlohmann::json& j, pressio_data const& data){ + j["dims"] = data.dimensions(); + j["dtype"] = uint32_t{data.dtype()}; + j["values"] = data.cast(pressio_double_dtype).to_vector(); +} + +void to_json(nlohmann::json& j, pressio_option const& option){ + j["type"] = uint32_t{option.type()}; + if(not option.has_value()) { + j["value"] = nullptr; + } + switch(option.type()) { + case pressio_option_data_type: + j["value"] = option.get_value(); + break; + case pressio_option_int8_type: + j["value"] = option.get_value(); + break; + case pressio_option_int16_type: + j["value"] = option.get_value(); + break; + case pressio_option_int32_type: + j["value"] = option.get_value(); + break; + case pressio_option_int64_type: + j["value"] = option.get_value(); + break; + case pressio_option_uint8_type: + j["value"] = option.get_value(); + break; + case pressio_option_uint16_type: + j["value"] = option.get_value(); + break; + case pressio_option_uint32_type: + j["value"] = option.get_value(); + break; + case pressio_option_uint64_type: + j["value"] = option.get_value(); + break; + case pressio_option_float_type: + j["value"] = option.get_value(); + break; + case pressio_option_double_type: + //javascript numbers are IEEE 754 double precision values + //store them directly + j = option.get_value(); + break; + case pressio_option_charptr_type: + //strings can be represented directly in javascript + //store them directly + j = option.get_value(); + break; + case pressio_option_charptr_array_type: + //lists of strings can be represented directly in javascript + //store them directly + j = option.get_value>(); + break; + case pressio_option_unset_type: + j["value"] = nullptr; + break; + case pressio_option_userptr_type: + throw std::runtime_error("userptr and unset ptr types are not convertible to JSON"); + } +} +void to_json(nlohmann::json& j, pressio_options const& options){ + j = {}; + for(auto const& [key, value]: options) { + if(value.type() != pressio_option_userptr_type) { + j[key] = value; + } + } +} + +void from_json(nlohmann::json const& j, pressio_data& data) { + std::vector values = j.at("values"); + std::vector dims = j.at("dims"); + auto dtype = static_cast(j.at("dtype")); + + data = pressio_data::nonowning( + pressio_double_dtype, + values.data(), + dims).cast(dtype); +} + +//returns the type of a nlohmann::json array by looking for the first entry and returning it's type +static nlohmann::json::value_t array_type(nlohmann::json const& j) { + auto type = j.type(); + switch(type) { + case nlohmann::json::value_t::array: + return array_type(j.at(0)); + default: + return type; + } +} + +template +static void flatten(nlohmann::json const& j, std::vector& values, std::vector& dims, size_t depth) { + + switch(j.type()) { + case nlohmann::json::value_t::array: + { + size_t entries{0}; + if(dims.size() <= depth) { + dims.emplace_back(0); + } + for (auto const& i : j) { + flatten(i, values, dims, depth+1); + ++entries; + } + dims[depth] = entries; + } + break; + case nlohmann::json::value_t::number_float: + case nlohmann::json::value_t::number_unsigned: + case nlohmann::json::value_t::number_integer: + values.emplace_back(j); + break; + default: + throw std::runtime_error("other types are not supported"); + break; + } +} + + +static void from_json_array(nlohmann::json const& j, pressio_option& option) { + if(j.type() != nlohmann::json::value_t::array) { + throw std::runtime_error("expected an array input"); + } + const auto j_type = array_type(j); + switch(j_type) { + case nlohmann::json::value_t::string: + case nlohmann::json::value_t::binary: + option = std::vector(j.begin(), j.end()); + break; + case nlohmann::json::value_t::number_integer: + case nlohmann::json::value_t::number_unsigned: + case nlohmann::json::value_t::number_float: + { + std::vector values; + std::vector dims; + flatten(j, values, dims, 0); + option = pressio_data::copy( + pressio_double_dtype, + values.data(), + dims + ); + } + break; + default: + throw std::runtime_error("unexpected array type"); + break; + + } + +} + +void from_json(nlohmann::json const& j, pressio_option& option) { + switch(j.type()) { + case nlohmann::json::value_t::binary: + case nlohmann::json::value_t::string: + option = j.get(); + break; + case nlohmann::json::value_t::object: + { + pressio_option_type dt = j.at("type").get(); + switch(dt) { + case pressio_option_data_type: + { + pressio_data d; + from_json(j.at("value"), d); + option = d; + } + break; + case pressio_option_int8_type: + option = int8_t(j.at("value")); + break; + case pressio_option_int16_type: + option = int16_t(j.at("value")); + break; + case pressio_option_int32_type: + option = int32_t(j.at("value")); + break; + case pressio_option_int64_type: + option = int64_t(j.at("value")); + break; + case pressio_option_uint8_type: + option = uint8_t(j.at("value")); + break; + case pressio_option_uint16_type: + option = uint16_t(j.at("value")); + break; + case pressio_option_uint32_type: + option = uint32_t(j.at("value")); + break; + case pressio_option_uint64_type: + option = uint64_t(j.at("value")); + break; + case pressio_option_float_type: + option = float(j.at("value")); + break; + case pressio_option_double_type: + option = double(j.at("value")); + break; + case pressio_option_charptr_type: + option = std::string(j.at("value")); + break; + case pressio_option_charptr_array_type: + option = std::vector(j.at("value")); + break; + case pressio_option_unset_type: + option = {}; + break; + case pressio_option_userptr_type: + throw std::runtime_error("userptr and unset ptr types are not convertible to nlohmann::json"); + } + } + break; + case nlohmann::json::value_t::number_float: + case nlohmann::json::value_t::number_integer: + case nlohmann::json::value_t::number_unsigned: + option = j.get(); + break; + case nlohmann::json::value_t::array: + from_json_array(j, option); + break; + case nlohmann::json::value_t::null: + option = {}; + break; + case nlohmann::json::value_t::boolean: + case nlohmann::json::value_t::discarded: + break; + + } +} + + +void from_json(nlohmann::json const& j, pressio_options& options) { + pressio_option o; + for (auto const& [key, value]: j.items()) { + from_json(value, o); + options.set(key, o); + } +} + + + +extern "C" { + struct pressio_options* pressio_options_new_json(struct pressio* library, const char* json) { + nlohmann::json parsed = nlohmann::json::parse(json); + pressio_options* options = nullptr; + try { + options = new pressio_options(parsed); + } catch (std::runtime_error& ex) { + if(library) { + library->set_error(1, ex.what()); + } + } + return options; + } + + char* pressio_options_to_json(struct pressio* library, struct pressio_options const* options) { + char* ret = nullptr; + try { + nlohmann::json jstr = *options; + std::string str = jstr.dump(); + ret = strdup(str.c_str()); + } catch (std::runtime_error& ex) { + if(library) { + library->set_error(1, ex.what()); + } + } + return ret; + } + + +} + diff --git a/src/pressio_version.h.in b/src/pressio_version.h.in index 29ad2d9..292c26f 100644 --- a/src/pressio_version.h.in +++ b/src/pressio_version.h.in @@ -13,6 +13,7 @@ #cmakedefine01 LIBPRESSIO_HAS_MPI #cmakedefine01 LIBPRESSIO_HAS_MPI4PY #cmakedefine01 LIBPRESSIO_HAS_LUA +#cmakedefine01 LIBPRESSIO_HAS_JSON #cmakedefine01 LIBPRESSIO_COMPAT_HAS_IMAGEMAGICK_LONGLONG #cmakedefine01 LIBPRESSIO_MGARD_NEED_FLOAT_HEADER diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 496cfe7..76ec577 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -114,16 +114,21 @@ if(BUILD_PYTHON_WRAPPER AND LIBPRESSIO_HAS_SZ) add_test(test_python_wrapper ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_python.py - ${CMAKE_BINARY_DIR}/swig + ${CMAKE_BINARY_DIR}/tools/swig ) add_test(test_python_io_wrapper ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/test_pythonio.py - ${CMAKE_BINARY_DIR}/swig + ${CMAKE_BINARY_DIR}/tools/swig ) add_test(test_python_numpy_conversions ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/from_numpy.py - ${CMAKE_BINARY_DIR}/swig + ${CMAKE_BINARY_DIR}/tools/swig ) endif() + +if(LIBPRESSIO_HAS_JSON AND LIBPRESSIO_HAS_HDF) + add_gtest(test_hdffilter.cc) + target_link_libraries(test_hdffilter PRIVATE hdf5::hdf5 libpressio_hdf5_filter) +endif() \ No newline at end of file diff --git a/test/test_hdffilter.cc b/test/test_hdffilter.cc new file mode 100644 index 0000000..b64d088 --- /dev/null +++ b/test/test_hdffilter.cc @@ -0,0 +1,112 @@ +#include +#include +#include +#include +#include +#include +#include +#include "libpressio_hdf5_filter.h" + + +struct cleanup { + template + cleanup(Func&& f): f(f) {} + cleanup(cleanup && rhs) : f(compat::exchange(rhs.f, []{})) {} + cleanup& operator=(cleanup && rhs) { + if (this == &rhs) return *this; + f = compat::exchange(rhs.f, []{}); + return *this; + } + cleanup& operator=(cleanup const&)=delete; + cleanup(cleanup const&)=delete; + cleanup() : f([]{}) {} + ~cleanup() { + f(); + } + + std::function f; +}; + + + +TEST(hdffilter, filter_runs) { + const hsize_t dims[] = {30,30,30}; + const auto vector_len = std::accumulate(std::begin(dims), std::end(dims), 1, compat::multiplies<>{}); + std::vector v(vector_len); + std::vector v2(vector_len); + + if(H5Zregister(H5Z_LIBPRESSIO) < 0) { + FAIL() << "failed to register plugin"; + } + + hid_t fapl = H5Pcreate(H5P_FILE_ACCESS); + if(fapl < 0) { + FAIL() << "failed to create fapl"; + } + cleanup fapl_cleanup ([&fapl]{ H5Pclose(fapl);}); + if(H5Pset_fapl_core(fapl, /*increment*/1024, /*backingstore*/false)) { + FAIL() << "failed to set core driver"; + } + + hid_t file = H5Fcreate("hdf5filter", H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + if(file < 0) { + FAIL() << "failed to create in-memory file"; + } + cleanup file_cleanup([&file]{H5Fclose(file);}); + + hid_t space = H5Screate_simple(3, dims, nullptr); + if(space < 0) { + FAIL() << "failed to create dataspace"; + } + cleanup space_cleanup([&space]{H5Sclose(space);}); + + hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); + if(dcpl < 0) { + FAIL() << "failed to create dataset creation property list"; + } + cleanup cleanup_dcpl([&dcpl]{H5Pclose(dcpl);}); + + if(H5Pset_chunk(dcpl, 3, dims) < 0 ) { + FAIL() << "failure to configure chunk size"; + } + + const char* compressor_id = "sz"; + pressio_options options { + {"sz:abs_err_bound", 1e-4}, + {"sz:error_bound_mode_str", "abs"}, + }; + + if(H5Pset_libpressio(dcpl, compressor_id, &options) < 0) { + FAIL() << "failed to configure libpressio settings"; + } + + + { + hid_t dataset = H5Dcreate(file, "testing", H5T_NATIVE_FLOAT, space, H5P_DEFAULT, dcpl, H5P_DEFAULT); + if(dataset < 0) { + FAIL() << "failed to create dataset"; + } + cleanup dataset_cleanup([&dataset]{ H5Dclose(dataset);}); + + std::iota(v.begin(), v.end(), 1.0f); + + H5Dwrite(dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, v.data()); + } + + //force a flush to ensure the data is written and we don't just get a cached copy + { + hid_t dataset = H5Dopen(file, "testing", H5P_DEFAULT); + if(dataset < 0) { + FAIL() << "failed to open dataset"; + } + cleanup dataset_cleanup([&dataset]{ H5Dclose(dataset);}); + + H5Dread(dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, v2.data()); + } + + for (size_t i = 0; i < vector_len; ++i) { + EXPECT_LE(std::abs(v[i] - v2[i]) , 1e-4); + } + + +} diff --git a/tools/hdf5_filter/CMakeLists.txt b/tools/hdf5_filter/CMakeLists.txt new file mode 100644 index 0000000..245082d --- /dev/null +++ b/tools/hdf5_filter/CMakeLists.txt @@ -0,0 +1,31 @@ +add_library(libpressio_hdf5_filter + ./src/libpressio_hdf5_filter.cc + ./src/libpressio_hdf5_props.cc + ./include/libpressio_hdf5_filter.h + ) +target_link_libraries(libpressio_hdf5_filter PRIVATE + libpressio + hdf5::hdf5 + nlohmann_json::nlohmann_json + MPI::MPI_CXX + ) +target_include_directories(libpressio_hdf5_filter + PUBLIC + $ + $ + $ + ) +target_compile_options(libpressio_hdf5_filter PRIVATE + $<$: -Wall -Wextra -Wpedantic> + $<$: ${NO_OMIT_FRAME_POINTER_FLAG}> + ) + +export(TARGETS libpressio_hdf5_filter NAMESPACE libpressio_hdf5_filter:: FILE libpressio_hdf5_filter.cmake) +install(TARGETS libpressio_hdf5_filter EXPORT libpressio_hdf5_filter + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + ) +install(EXPORT libpressio_hdf5_filter NAMESPACE libpressio_hdf5_filter:: DESTINATION share/libpressio_hdf5_filter/cmake) + + diff --git a/tools/hdf5_filter/include/libpressio_hdf5_filter.h b/tools/hdf5_filter/include/libpressio_hdf5_filter.h new file mode 100644 index 0000000..82a64aa --- /dev/null +++ b/tools/hdf5_filter/include/libpressio_hdf5_filter.h @@ -0,0 +1,21 @@ +#include + +#ifdef __cplusplus +extern "C" { +#endif + +//a made up unique hdf5 filter value +//TODO replace with an actual value +#define H5Z_FILTER_LIBPRESSIO 21023 + +H5PL_type_t H5PLget_plugin_type(); + +herr_t H5Pset_libpressio(hid_t dcpl, const char* compressor_id, struct pressio_options const* options); + +const void* H5PLget_plugin_info(); + +extern const H5Z_class2_t H5Z_LIBPRESSIO[1]; + +#ifdef __cplusplus +} +#endif diff --git a/tools/hdf5_filter/src/libpressio_hdf5_filter.cc b/tools/hdf5_filter/src/libpressio_hdf5_filter.cc new file mode 100644 index 0000000..2cf4e75 --- /dev/null +++ b/tools/hdf5_filter/src/libpressio_hdf5_filter.cc @@ -0,0 +1,300 @@ +#include "libpressio_hdf5_filter.h" +#include +#include +#include +#include +#include +#include +#include + +struct compression_options { + pressio_dtype dtype; + std::vector dims; + std::string compressor_id; + pressio_options options; +}; + +compression_options get_options_from_cd_values(size_t cd_nelmts, const unsigned int* cd_values) { + compression_options options; + size_t current = 0; + compat::span vec(cd_values, cd_nelmts); + auto pop_word = [¤t, &vec]() { + const uint64_t* ret = reinterpret_cast(&vec[current]); + current += 2; + return *ret; + }; + options.dtype = pressio_dtype(vec[current++]); + options.dims.resize(pop_word()); + for (size_t i = 0; i < options.dims.size(); ++i) { + options.dims[i] = pop_word(); + } + + options.compressor_id.resize(pop_word()); + memcpy(&options.compressor_id[0], &vec[current], options.compressor_id.size()); + current += ((options.compressor_id.size() /4) + 1); + + std::string op_str(pop_word(), 0); + memcpy(&op_str[0], &vec[current], op_str.size()); + nlohmann::json j = nlohmann::json::from_msgpack(op_str); + options.options = j; + + return options; +} + +static std::vector get_cd_values_from_options(compression_options const& options) { + std::vector ret; + auto push_word = [&ret](uint64_t value) { + ret.push_back(value & 0x0000FFFF); + ret.push_back(value >> 32); + }; + auto push_bytes = [&ret, &push_word](const char* bytes, size_t len) { + push_word(len); + const size_t current_size = ret.size(); + ret.resize(current_size + (len/4) +1); + memcpy(&ret[current_size], bytes, len); + }; + + ret.push_back(options.dtype); + push_word(options.dims.size()); + for (auto i : options.dims) { + push_word(i); + } + + push_bytes(options.compressor_id.c_str(), options.compressor_id.size()); + + nlohmann::json j = options.options; + std::vector msgpk = nlohmann::json::to_msgpack(j); + push_bytes(reinterpret_cast(msgpk.data()), msgpk.size()); + + return ret; +} + +extern "C" { + +#define H5Z_LIBPRESSIO_PUSH_AND_GOTO(MAJ, MIN, RET, MSG) \ + do { \ + H5Epush(H5E_DEFAULT, __FILE__, _funcname_, __LINE__, H5E_ERR_CLS, MAJ, \ + MIN, MSG); \ + retval = RET; \ + goto done; \ + } while (0) + + + /** + * apply a libpressio compressor to an HDF5 chunk + * + * \param[in] flags - options passed to the HDF5 filter plugin such as + * H5Z_FLAG_REVERSE which indicates decompression + * \param[in] cd_nelmts - the number of auxiliary values passed to the filter + * \param[in] cd_values - auxiliary values passed to the filter + * \param[in] n_bytes - the size of the input buffer in bytes + * \param[out] buf_size - the size of the output buffer in bytes + * \param[in,out] buf - on input, the actual memory to be compressed/decompressed; on output, the + * compressed/decompressed memory; we can't assume the operation can be done + * in-place, so buf needs to be freed before re-assigned + * \return + */ + static size_t H5Z_filter_libpressio(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], + size_t n_bytes, size_t* buf_size, + void** buf) + { + //parse compression_options from cd_values, cd_nelmts + auto comp_options = get_options_from_cd_values(cd_nelmts, cd_values); + + //instantiate the data + + //instantiate the compressor from cd_values + pressio library; + pressio_compressor compressor = library.get_compressor(comp_options.compressor_id); + compressor->set_options(comp_options.options); + + if (flags & H5Z_FLAG_REVERSE) { + // preform decompression + pressio_data const& input = pressio_data::nonowning(pressio_byte_dtype, *buf, {n_bytes}); + pressio_data output = pressio_data::owning(comp_options.dtype, comp_options.dims); + int rc = compressor->decompress(&input, &output); + if(rc == 0) { + H5free_memory(*buf); + *buf = pressio_data_copy(&output, buf_size); + } else { + *buf_size = 0; + } + } else { + // preform compression + pressio_data const& input = pressio_data::nonowning(comp_options.dtype, *buf, comp_options.dims); + pressio_data output = pressio_data::empty(pressio_byte_dtype, {}); + int rc = compressor->compress(&input, &output); + if(rc == 0) { + H5free_memory(*buf); + *buf = pressio_data_copy(&output, buf_size); + } else { + *buf_size = 0; + } + } + + return *buf_size; + } + + /** + * apply local options to this libpressio compressor + * + * \param[in] dcpl_id the dataset creation property list + * \param[in] type_id datatype identifier passed in to H5Dcreate. should not be modified + * \param[in] chunk_space_id a dataspace describing the chunk. should not be modified + * \returns non-negative value on success and a negative value for an error + */ + static herr_t H5Z_libpressio_set_local(hid_t dcpl_id, hid_t type_id, + hid_t chunk_space_id) + { + static char const*_funcname_ = "H5Z_libpressio_can_apply"; + compression_options opts; + const int h_ndims = H5Sget_simple_extent_ndims(chunk_space_id); + const hid_t native_type = H5Tget_native_type(type_id, H5T_DIR_DEFAULT); + std::vector h_dims; + std::vector cd_vals; + unsigned int flags; + herr_t retval = 1; + pressio_options* p_opts = nullptr; + std::string* p_string = nullptr; + + if(h_ndims < 0) { + H5Z_LIBPRESSIO_PUSH_AND_GOTO(H5E_ARGS, H5E_BADTYPE, -1, "not a data space"); + } else { + h_dims.resize(h_ndims); + } + + if(H5Sget_simple_extent_dims(chunk_space_id, h_dims.data(), nullptr) > 0) { + opts.dims = std::move(std::vector(h_dims.begin(), h_dims.end())); + } else { + H5Z_LIBPRESSIO_PUSH_AND_GOTO(H5E_PLINE, H5E_BADTYPE, -1, "bad chunk data space"); + } + + if(native_type < 0) { + H5Z_LIBPRESSIO_PUSH_AND_GOTO(H5E_ARGS, H5E_BADTYPE, -1, "not a datatype"); + } else { + opts.dtype = [&native_type]{ + if(H5Tequal(native_type, H5T_NATIVE_INT8) > 0) return pressio_int8_dtype; + if(H5Tequal(native_type, H5T_NATIVE_INT16) > 0) return pressio_int16_dtype; + if(H5Tequal(native_type, H5T_NATIVE_INT32) > 0) return pressio_int32_dtype; + if(H5Tequal(native_type, H5T_NATIVE_INT64) > 0) return pressio_int64_dtype; + if(H5Tequal(native_type, H5T_NATIVE_UINT8) > 0) return pressio_uint8_dtype; + if(H5Tequal(native_type, H5T_NATIVE_UINT16) > 0) return pressio_uint16_dtype; + if(H5Tequal(native_type, H5T_NATIVE_UINT32) > 0) return pressio_uint32_dtype; + if(H5Tequal(native_type, H5T_NATIVE_UINT64) > 0) return pressio_uint64_dtype; + if(H5Tequal(native_type, H5T_NATIVE_FLOAT) > 0) return pressio_float_dtype; + if(H5Tequal(native_type, H5T_NATIVE_DOUBLE) > 0) return pressio_double_dtype; + return pressio_byte_dtype; + }(); + H5Tclose(native_type); + } + //get the compressor id and configuration from the dcpl + if(H5Pexist(dcpl_id, "libpressio_compressor_options") > 0) { + if(H5Pget(dcpl_id, "libpressio_compressor_options", &p_opts) > 0){ + H5Z_LIBPRESSIO_PUSH_AND_GOTO(H5E_PLINE, H5E_CANTGET, 0, "unable to get libpressio controls"); + } else { + opts.options = *p_opts; + delete p_opts; + } + } + if(H5Pexist(dcpl_id, "libpressio_compressor_id") > 0) { + if(H5Pget(dcpl_id, "libpressio_compressor_id", &p_string) > 0){ + H5Z_LIBPRESSIO_PUSH_AND_GOTO(H5E_PLINE, H5E_CANTGET, 0, "unable to get libpressio compressor_id"); + } else { + opts.compressor_id = *p_string; + delete p_string; + } + } + + //convert the options to cd_values + cd_vals = get_cd_values_from_options(opts); + + if(H5Pget_filter_by_id(dcpl_id, H5Z_FILTER_LIBPRESSIO, &flags, 0, nullptr, 0, nullptr, nullptr) < 0){ + H5Z_LIBPRESSIO_PUSH_AND_GOTO(H5E_PLINE, H5E_CANTGET, 0, "unable to get current LIBPRESSIO flags"); + } + + //overwrite the plugin with the new configuration + if(H5Pmodify_filter(dcpl_id, H5Z_FILTER_LIBPRESSIO, flags, cd_vals.size(), cd_vals.data()) < 0) { + H5Z_LIBPRESSIO_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, 0, "failed to modify cd_values"); + } + + + +done: + return retval; + } + + /** + * checks if the input data format is supported by the compressor + * + * since we don't know what the underlying compressor fully supports + * we can only check some basics things here + * + * \param[in] type_id datatype identifier passed in to H5Dcreate. should not be modified + * \returns non-negative value on success and a negative value for an error + */ + static htri_t H5Z_libpressio_can_apply(hid_t , hid_t type_id, + hid_t ) + { + static char const*_funcname_ = "H5Z_libpressio_can_apply"; + htri_t retval = 0; + H5T_class_t dclass; + size_t dsize; + hid_t native_type_id; + + if((dclass = H5Tget_class(type_id)) == H5T_NO_CLASS) { + H5Z_LIBPRESSIO_PUSH_AND_GOTO(H5E_PLINE, H5E_BADTYPE, -1, "bad datatype class"); + } + + if((dsize = H5Tget_size(type_id)) == 0) { + H5Z_LIBPRESSIO_PUSH_AND_GOTO(H5E_PLINE, H5E_BADTYPE, -1, "bad datatype size"); + } + + if(dclass == H5T_FLOAT) { + switch(dsize) { + case 4: + case 8: + break; + default: + H5Z_LIBPRESSIO_PUSH_AND_GOTO(H5E_PLINE, H5E_BADTYPE, 0, "only 32 and 64 bit floats supported"); + } + } else if(dclass == H5T_INTEGER) { + switch(dsize) { + case 1: + case 2: + case 4: + case 8: + break; + default: + H5Z_LIBPRESSIO_PUSH_AND_GOTO(H5E_PLINE, H5E_BADTYPE, 0, "only 8, 16, 32, and 64 bit integers supported"); + } + } + + native_type_id = H5Tget_native_type(type_id, H5T_DIR_ASCEND); + if (H5Tget_order(type_id) != H5Tget_order(native_type_id)) { + H5Z_LIBPRESSIO_PUSH_AND_GOTO(H5E_PLINE, H5E_BADTYPE, 0, + "endian targetting is not currently supported with libpressio"); + } + H5Tclose(native_type_id); + + retval = 1; +done: + return retval; + } + + const H5Z_class_t H5Z_LIBPRESSIO[1] = { { + H5Z_CLASS_T_VERS, /* class_t version*/ + (H5Z_filter_t)H5Z_FILTER_LIBPRESSIO, /*filter id number*/ + 1, /*has an encoder phase*/ + 1, /*has an decoder phase*/ + "libpressio", /*filter name for debugging*/ + H5Z_libpressio_can_apply, /* can apply callback??*/ + H5Z_libpressio_set_local, /* set local callback??*/ + H5Z_filter_libpressio /*actual filter function*/ + } }; + + H5PL_type_t H5PLget_plugin_type() { return H5PL_TYPE_FILTER; } + + const void* H5PLget_plugin_info() { return H5Z_LIBPRESSIO; } + +} diff --git a/tools/hdf5_filter/src/libpressio_hdf5_props.cc b/tools/hdf5_filter/src/libpressio_hdf5_props.cc new file mode 100644 index 0000000..1a2e9c1 --- /dev/null +++ b/tools/hdf5_filter/src/libpressio_hdf5_props.cc @@ -0,0 +1,236 @@ +#include "H5Ppublic.h" +#include +#include +#include +#include "libpressio_hdf5_filter.h" + +extern "C" { + + + /** + * @brief creates a compressor_id property list item + * + * @param[in] name name of the property being created + * @param[in] size size of the property in bytes + * @param[in,out] inital_value default value of the property which will be passed to H5Pregsiter2 + * @return herr_t negative on error + */ + herr_t libpressio_compressor_create(const char* name, size_t size, void* inital_value) { + if(strcmp(name,"libpressio_compressor_options") == 0) { + if(size != sizeof(pressio_options**)) { + return -1; + } + pressio_options** initial_op_value = static_cast(inital_value); + *initial_op_value = new pressio_options(); + return 0; + } else if(strcmp(name, "libpressio_compressor_id") == 0) { + if(size != sizeof(std::string**)) { + return -1; + } + std::string** initial_op_value = static_cast(inital_value); + *initial_op_value = new std::string(); + return 0; + + return 0; + } + + return -2; + } + + herr_t libpressio_compressor_get(hid_t, const char* name, size_t size, void* new_value) { + if(strcmp(name,"libpressio_compressor_options") == 0) { + if(size != sizeof(pressio_options**)) { + return -1; + } + pressio_options** initial_op_value = static_cast(new_value); + *initial_op_value = new pressio_options(**initial_op_value); + return 0; + } else if(strcmp(name, "libpressio_compressor_id") == 0) { + if(size != sizeof(std::string**)) { + return -1; + } + std::string** initial_op_value = static_cast(new_value); + *initial_op_value = new std::string(**initial_op_value); + return 0; + + return 0; + } + + return -2; + } + + herr_t libpressio_compressor_set(hid_t, const char* name, size_t size, void* new_value) { + if(strcmp(name,"libpressio_compressor_options") == 0) { + if(size != sizeof(pressio_options**)) { + return -1; + } + pressio_options** initial_op_value = static_cast(new_value); + *initial_op_value = new pressio_options(**initial_op_value); + return 0; + } else if(strcmp(name, "libpressio_compressor_id") == 0) { + if(size != sizeof(std::string**)) { + return -1; + } + std::string** initial_op_value = static_cast(new_value); + *initial_op_value = new std::string(**initial_op_value); + return 0; + + return 0; + } + + return -2; + } + + herr_t libpressio_compressor_delete(hid_t, const char* name, size_t size, void* value) { + if(strcmp(name,"libpressio_compressor_options") == 0) { + if(size != sizeof(pressio_options**)) { + return -1; + } + pressio_options** initial_op_value = static_cast(value); + delete *initial_op_value; + *initial_op_value = nullptr; + return 0; + } else if(strcmp(name, "libpressio_compressor_id") == 0) { + if(size != sizeof(std::string*)) { + return -1; + } + std::string** initial_op_value = static_cast(value); + delete *initial_op_value; + *initial_op_value = nullptr; + return 0; + + return 0; + } + return 0; + + } + herr_t libpressio_compressor_copy(const char* name, size_t size, void* new_value) { + if(strcmp(name,"libpressio_compressor_options") == 0) { + if(size != sizeof(pressio_options**)) { + return -1; + } + pressio_options** initial_op_value = static_cast(new_value); + *initial_op_value = new pressio_options(**initial_op_value); + return 0; + } else if(strcmp(name, "libpressio_compressor_id") == 0) { + if(size != sizeof(std::string**)) { + return -1; + } + std::string** initial_op_value = static_cast(new_value); + *initial_op_value = new std::string(**initial_op_value); + return 0; + + return 0; + } + + return -2; + } + + herr_t libpressio_compressor_options_compare(const void* value1, const void* value2, size_t) { + auto lhs = static_cast(value1); + auto rhs = static_cast(value2); + if ((**lhs) == (**rhs)) { + return 0; + } else { + return 1; + } + } + herr_t libpressio_compressor_ids_compare(const void* value1, const void* value2, size_t) { + auto lhs = static_cast(value1); + auto rhs = static_cast(value2); + if ((**lhs) == (**rhs)) { + return 0; + } else if((**lhs) < (**rhs)) { + return -1; + } else { + return 1; + } + } + herr_t libpressio_compressor_close(const char* name, size_t size, void* value) { + if(strcmp(name,"libpressio_compressor_options") == 0) { + if(size != sizeof(pressio_options**)) { + return -1; + } + pressio_options** initial_op_value = static_cast(value); + delete *initial_op_value; + *initial_op_value = nullptr; + return 0; + } else if(strcmp(name, "libpressio_compressor_id") == 0) { + if(size != sizeof(std::string*)) { + return -1; + } + std::string** initial_op_value = static_cast(value); + delete *initial_op_value; + *initial_op_value = nullptr; + return 0; + + return 0; + } + return 0; + } + + herr_t H5Pset_libpressio(hid_t dcpl, const char* compressor_id, pressio_options const* options) { + herr_t error = 0; + + if((error = H5Pset_filter(dcpl, H5Z_FILTER_LIBPRESSIO, 0, 0, nullptr)) < 0 ) { + return error; + } + + std::string** compressor_id_str = new std::string*(new std::string(compressor_id)); + pressio_options** options_ptr = new pressio_options*(new pressio_options(*options)); + + if(H5Pexist(dcpl,"libpressio_compressor_options") > 0) { + error = H5Pset(dcpl, "libpressio_compressor_options", &options); + } else { + error = H5Pinsert2( + dcpl, + "libpressio_compressor_options", + sizeof(pressio_options**), + options_ptr, + libpressio_compressor_set, + libpressio_compressor_get, + libpressio_compressor_delete, + libpressio_compressor_copy, + libpressio_compressor_options_compare, + libpressio_compressor_close + ); + } + if(error < 0) { + delete *compressor_id_str; + delete compressor_id_str; + delete *options_ptr; + delete options_ptr; + return error; + } + + if(H5Pexist(dcpl,"libpressio_compressor_id") > 0 ) { + error = H5Pset(dcpl, "libpressio_compressor_id", &compressor_id); + } else { + error = H5Pinsert2( + dcpl, + "libpressio_compressor_id", + sizeof(std::string**), + compressor_id_str, + libpressio_compressor_set, + libpressio_compressor_get, + libpressio_compressor_delete, + libpressio_compressor_copy, + libpressio_compressor_ids_compare, + libpressio_compressor_close + ); + } + + if(error < 0) { + delete *compressor_id_str; + delete compressor_id_str; + delete *options_ptr; + delete options_ptr; + return error; + } else { + delete compressor_id_str; + delete options_ptr; + } + + return error; + } +} diff --git a/swig/CMakeLists.txt b/tools/swig/CMakeLists.txt similarity index 100% rename from swig/CMakeLists.txt rename to tools/swig/CMakeLists.txt diff --git a/swig/libpressio.py b/tools/swig/libpressio.py similarity index 100% rename from swig/libpressio.py rename to tools/swig/libpressio.py diff --git a/swig/numpy.i b/tools/swig/numpy.i similarity index 100% rename from swig/numpy.i rename to tools/swig/numpy.i diff --git a/swig/pressio.i b/tools/swig/pressio.i similarity index 100% rename from swig/pressio.i rename to tools/swig/pressio.i diff --git a/swig/pressio_sz.i b/tools/swig/pressio_sz.i similarity index 100% rename from swig/pressio_sz.i rename to tools/swig/pressio_sz.i diff --git a/swig/pressio_zfp.i b/tools/swig/pressio_zfp.i similarity index 100% rename from swig/pressio_zfp.i rename to tools/swig/pressio_zfp.i diff --git a/swig/pypressio.cc b/tools/swig/pypressio.cc similarity index 100% rename from swig/pypressio.cc rename to tools/swig/pypressio.cc diff --git a/swig/pypressio.h b/tools/swig/pypressio.h similarity index 100% rename from swig/pypressio.h rename to tools/swig/pypressio.h