-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Major Changes + new autocorr and diff_pdf metrics modules to compute the 1d autocorrelation of the errors and the distributions of the differences respectively + added new pkg-config for libpressio_cxx that depends on std_compat_cxx to make it easier to develop compressor modules without cmake Minor Changes + allow more moves for options for performance Bug Fixes + correct incorrect forward declarations in the JSON modules that could cause ABI failures + throw an error instead of returning a deceptive nullptr in external metrics module + include missing return for empty options
- Loading branch information
Showing
9 changed files
with
307 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| includedir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@/libpressio | ||
| libdir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@ | ||
|
|
||
| Name: Libpressio | ||
| Description: An abstraction between different pressio compressors | ||
| Version: @PROJECT_VERSION@ | ||
| Cflags: -I${includedir} | ||
| Libs: -L${libdir} -llibpressio | ||
| Requires: std_compat_cxx |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,165 @@ | ||
| #include <cmath> | ||
| #include "pressio_data.h" | ||
| #include "pressio_options.h" | ||
| #include "libpressio_ext/cpp/data.h" | ||
| #include "libpressio_ext/cpp/metrics.h" | ||
| #include "libpressio_ext/cpp/options.h" | ||
| #include "libpressio_ext/cpp/pressio.h" | ||
| #include "std_compat/memory.h" | ||
|
|
||
| namespace autocorr { | ||
| struct metrics { | ||
| pressio_data autocorr; | ||
| }; | ||
| struct compute_metrics{ | ||
| template <class ForwardIt1, class ForwardIt2> | ||
| metrics operator()(ForwardIt1 input_begin, ForwardIt1 input_end, ForwardIt2 input2_begin, ForwardIt2 input2_end) const | ||
| { | ||
| metrics m; | ||
| const size_t num_elements = std::min(input_end - input_begin, input2_end- input2_begin); | ||
| if(num_elements == 0) { | ||
| return m; | ||
| } | ||
|
|
||
| std::vector<double> errors(num_elements); | ||
|
|
||
| m.autocorr = pressio_data::owning(pressio_double_dtype, {static_cast<size_t>(autocorr_lags + 1)}); | ||
| double average_error=0; | ||
| for (size_t i = 0; i < num_elements; ++i) { | ||
| errors[i] = input_begin[i] - input2_begin[i]; | ||
| average_error += errors[i]; | ||
| } | ||
| average_error /= static_cast<double>(num_elements); | ||
| auto autocorr = static_cast<double*>(m.autocorr.data()); | ||
|
|
||
|
|
||
| if (num_elements > 4096) | ||
| { | ||
| double cov = 0; | ||
| for (size_t i = 0; i < num_elements; i++) { | ||
| cov += (errors[i] - average_error)*(errors[i] - average_error); | ||
| } | ||
|
|
||
| cov = cov/num_elements; | ||
|
|
||
| if (cov == 0) | ||
| { | ||
| for (size_t delta = 1; delta <= autocorr_lags; delta++) | ||
| autocorr[delta] = 0; | ||
| } | ||
| else | ||
| { | ||
| for(size_t delta = 1; delta <= autocorr_lags; delta++) | ||
| { | ||
| double sum = 0; | ||
|
|
||
| for (size_t i = 0; i < num_elements-delta; i++) | ||
| sum += (errors[i]-average_error)*(errors[i+delta]-average_error); | ||
|
|
||
| autocorr[delta] = sum/(num_elements-delta)/cov; | ||
| } | ||
| } | ||
| } | ||
| else | ||
| { | ||
| for (size_t delta = 1; delta <= autocorr_lags; delta++) | ||
| { | ||
| double avg_0 = 0; | ||
| double avg_1 = 0; | ||
|
|
||
| for (size_t i = 0; i < num_elements-delta; i++) | ||
| { | ||
| avg_0 += errors[i]; | ||
| avg_1 += errors[i+delta]; | ||
| } | ||
|
|
||
| avg_0 = avg_0 / (num_elements-delta); | ||
| avg_1 = avg_1 / (num_elements-delta); | ||
|
|
||
| double cov_0 = 0; | ||
| double cov_1 = 0; | ||
|
|
||
| for (size_t i = 0; i < num_elements-delta; i++) | ||
| { | ||
| cov_0 += (errors[i]-avg_0)*(errors[i]-avg_0); | ||
| cov_1 += (errors[i+delta]-avg_1)*(errors[i+delta]-avg_1); | ||
| } | ||
|
|
||
| cov_0 = cov_0/(num_elements-delta); | ||
| cov_1 = cov_1/(num_elements-delta); | ||
|
|
||
| cov_0 = sqrt(cov_0); | ||
| cov_1 = sqrt(cov_1); | ||
|
|
||
| if (cov_0*cov_1 == 0) | ||
| { | ||
| for (delta = 1; delta <= autocorr_lags; delta++) | ||
| autocorr[delta] = 1; | ||
| } | ||
| else | ||
| { | ||
| double sum = 0; | ||
|
|
||
| for (size_t i = 0; i < num_elements-delta; i++) | ||
| sum += (errors[i]-avg_0)*(errors[i+delta]-avg_1); | ||
|
|
||
| autocorr[delta] = sum/(num_elements-delta)/(cov_0*cov_1); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| autocorr[0] = 1; | ||
|
|
||
|
|
||
| return m; | ||
| } | ||
|
|
||
| uint64_t autocorr_lags; | ||
| }; | ||
| } | ||
|
|
||
| class autocorr_plugin : public libpressio_metrics_plugin { | ||
|
|
||
| public: | ||
| void begin_compress(const struct pressio_data * input, struct pressio_data const * ) override { | ||
| input_data = pressio_data::clone(*input); | ||
| } | ||
| void end_decompress(struct pressio_data const*, struct pressio_data const* output, int ) override { | ||
| err_metrics = pressio_data_for_each<autocorr::metrics>(input_data, *output, autocorr::compute_metrics{autocorr_lags}); | ||
| } | ||
|
|
||
| int set_options(pressio_options const& opts) override { | ||
| get(opts, "autocorr:autocorr_lags", &autocorr_lags); | ||
| return 0; | ||
| } | ||
| pressio_options get_options() const override { | ||
| pressio_options opts; | ||
| set(opts, "autocorr:autocorr_lags", autocorr_lags); | ||
| return opts; | ||
| } | ||
|
|
||
| struct pressio_options get_metrics_results() const override { | ||
| pressio_options opt; | ||
| if(err_metrics) { | ||
| set(opt, "autocorr:autocorr", err_metrics->autocorr); | ||
| } else { | ||
| set_type(opt, "autocorr:autocorr", pressio_option_data_type); | ||
| } | ||
| return opt; | ||
| } | ||
| std::unique_ptr<libpressio_metrics_plugin> clone() override { | ||
| return compat::make_unique<autocorr_plugin>(*this); | ||
| } | ||
|
|
||
| const char* prefix() const override { | ||
| return "autocorr"; | ||
| } | ||
|
|
||
|
|
||
| private: | ||
| uint64_t autocorr_lags = 100; | ||
| pressio_data input_data = pressio_data::empty(pressio_byte_dtype, {}); | ||
| compat::optional<autocorr::metrics> err_metrics; | ||
| }; | ||
|
|
||
| static pressio_register metrics_autocorr_plugin(metrics_plugins(), "autocorr", [](){ return compat::make_unique<autocorr_plugin>(); }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| #include "pressio_data.h" | ||
| #include "pressio_options.h" | ||
| #include "libpressio_ext/cpp/data.h" | ||
| #include "libpressio_ext/cpp/metrics.h" | ||
| #include "libpressio_ext/cpp/options.h" | ||
| #include "libpressio_ext/cpp/pressio.h" | ||
| #include "std_compat/memory.h" | ||
|
|
||
| namespace diff_pdf { | ||
| static const uint64_t zero = 0; | ||
| struct metrics { | ||
| pressio_data histogram = pressio_data::copy(pressio_uint64_dtype, &zero, {1}); | ||
| double interval = 0; | ||
| double min_diff = 0; | ||
| double max_diff = 0; | ||
| }; | ||
| struct compute_metrics{ | ||
| template <class ForwardIt1, class ForwardIt2> | ||
| metrics operator()(ForwardIt1 input_begin, ForwardIt1 input_end, ForwardIt2 input2_begin, ForwardIt2 input2_end) const | ||
| { | ||
| metrics m; | ||
| size_t num_elements = std::min(input_end-input_begin, input2_end-input2_begin); | ||
| if (num_elements == 0) { | ||
| return m; | ||
| } | ||
|
|
||
| m.max_diff = input_begin[0] - input2_begin[0]; | ||
| m.min_diff = m.max_diff; | ||
| for (size_t i = 1; i < num_elements; ++i) { | ||
| double diff = input_begin[i] - input2_begin[i]; | ||
| m.max_diff = std::max(m.max_diff, diff); | ||
| m.min_diff = std::min(m.min_diff, diff); | ||
| } | ||
| m.interval = (m.max_diff - m.min_diff)/pdf_intervals; | ||
| if (m.interval == 0) { | ||
| return m; | ||
| } | ||
|
|
||
|
|
||
| m.histogram = pressio_data::owning(pressio_uint64_dtype, {static_cast<size_t>(pdf_intervals)}); | ||
| const auto dist_ptr = static_cast<uint64_t*>(m.histogram.data()); | ||
| memset(dist_ptr, 0, m.histogram.size_in_bytes()); | ||
|
|
||
| for (size_t i = 0; i < num_elements; ++i) { | ||
| auto diff = input_begin[i] - input2_begin[i]; | ||
| auto idx = std::min(static_cast<size_t>((diff-m.min_diff)/m.interval), static_cast<size_t>(pdf_intervals - 1)); | ||
| dist_ptr[idx]++; | ||
| } | ||
|
|
||
| return m; | ||
|
|
||
| } | ||
|
|
||
| uint64_t pdf_intervals; | ||
| }; | ||
| } | ||
|
|
||
| class diff_pdf_plugin : public libpressio_metrics_plugin { | ||
|
|
||
| public: | ||
| void begin_compress(const struct pressio_data * input, struct pressio_data const * ) override { | ||
| input_data = pressio_data::clone(*input); | ||
| } | ||
| void end_decompress(struct pressio_data const*, struct pressio_data const* output, int ) override { | ||
| err_metrics = pressio_data_for_each<diff_pdf::metrics>(input_data, *output, diff_pdf::compute_metrics{pdf_intervals}); | ||
| } | ||
|
|
||
| int set_options(pressio_options const& opts) override { | ||
| get(opts, "diff_pdf:intervals", &pdf_intervals); | ||
| return 0; | ||
| } | ||
| pressio_options get_options() const override { | ||
| pressio_options opts; | ||
| set(opts, "diff_pdf:intervals", pdf_intervals); | ||
| return opts; | ||
| } | ||
|
|
||
| struct pressio_options get_metrics_results() const override { | ||
| pressio_options opt; | ||
| if(err_metrics) { | ||
| set(opt, "diff_pdf:histogram", err_metrics->histogram); | ||
| set(opt, "diff_pdf:interval", err_metrics->interval); | ||
| set(opt, "diff_pdf:min_diff", err_metrics->min_diff); | ||
| set(opt, "diff_pdf:max_diff", err_metrics->max_diff); | ||
| } else { | ||
| set_type(opt, "diff_pdf:pdf", pressio_option_data_type); | ||
| } | ||
| return opt; | ||
| } | ||
| std::unique_ptr<libpressio_metrics_plugin> clone() override { | ||
| return compat::make_unique<diff_pdf_plugin>(*this); | ||
| } | ||
|
|
||
| const char* prefix() const override { | ||
| return "diff_pdf"; | ||
| } | ||
|
|
||
|
|
||
| private: | ||
| uint64_t pdf_intervals = 2000; | ||
| pressio_data input_data = pressio_data::empty(pressio_byte_dtype, {}); | ||
| compat::optional<diff_pdf::metrics> err_metrics; | ||
| }; | ||
|
|
||
| static pressio_register metrics_diff_pdf_plugin(metrics_plugins(), "diff_pdf", [](){ return compat::make_unique<diff_pdf_plugin>(); }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters