Skip to content

Commit

Permalink
libpressio version 0.34.1
Browse files Browse the repository at this point in the history 
Bug Fixes

+ WARNING: the fpzip previously had several flaws that would cause it to
  not produce correct compression results and not be able to decompress
  data.  These have been resolved, but all results produced with the
  previous versions of this plugin should be discarded.
+ the CSV generic IO modules is now orders of magnitude faster by using
  more efficient methods to construct the buffer.
  • Loading branch information
@robertu94
robertu94 committed Feb 20, 2020
1 parent 976dd6a commit 51da773
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 137 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
project(libpressio VERSION "0.34.0" LANGUAGES CXX C)
project(libpressio VERSION "0.34.1" LANGUAGES CXX C)

#correct was to set a default build type
# https://blog.kitware.com/cmake-and-the-default-build-type/
Expand Down
2 changes: 1 addition & 1 deletion COPYRIGHT.txt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
Copyright © 2020 , UChicago Argonne, LLC
All Rights Reserved
[libpressio, Version 0.31.1]
[libpressio, Version 0.34.1]
Robert Underwood
Argonne National Laboratory

Expand Down
60 changes: 44 additions & 16 deletions src/plugins/compressors/fpzip_plugin.cc
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#include <memory>
#include <string>
#include <sstream>
#include <algorithm>
#include <fpzip.h>
#include "pressio_data.h"
#include "pressio_compressor.h"
Expand All @@ -12,34 +13,37 @@

namespace {
constexpr int INVALID_TYPE = 8;
auto check_dim = [](size_t dim) {
if(dim == 0) return 1ul;
else return dim;
};
}

class fpzip_plugin: public libpressio_compressor_plugin {

struct pressio_options get_options_impl () const override {
struct pressio_options options = pressio_options();
options.set_type("fpzip:has_header", pressio_option_uint32_type);
options.set_type("fpzip:prec", pressio_option_uint32_type);
options.set("fpzip:has_header", has_header);
options.set("fpzip:prec", prec);
return options;
};

struct pressio_options get_configuration_impl () const override {
struct pressio_options options = pressio_options();
pressio_options_set_integer(&options, "pressio:thread_safe", pressio_thread_safety_multiple);
pressio_options_set_uinteger(&options, "fpzip:codec_version", fpzip_codec_version);
pressio_options_set_uinteger(&options, "fpzip:library_version", fpzip_library_version);
pressio_options_set_uinteger(&options, "fpzip:data_model", fpzip_data_model);
return options;
};

int set_options_impl (struct pressio_options const& options) override {
int tmp;
if( options.get("fpzip:has_header", &tmp) != pressio_options_key_set) {
has_header = tmp != 0;
} else {
return set_error(7, "fpzip:has_header is required");
}
if(options.get("fpzip:prec", &prec) != pressio_options_key_set) {
return set_error(7, "fpzip:prec is required");
}

options.get("fpzip:prec", &prec);
return 0;
}

Expand All @@ -61,10 +65,11 @@ class fpzip_plugin: public libpressio_compressor_plugin {
pressio_data_get_bytes(output)
);

fpz->nx = input->get_dimension(0);
fpz->ny = input->get_dimension(1);
fpz->nz = input->get_dimension(2);
fpz->nf = input->get_dimension(3);

fpz->nx = check_dim(input->get_dimension(0));
fpz->ny = check_dim(input->get_dimension(1));
fpz->nz = check_dim(input->get_dimension(2));
fpz->nf = check_dim(input->get_dimension(3));
fpz->type = type;
fpz->prec = prec;

Expand All @@ -75,14 +80,37 @@ class fpzip_plugin: public libpressio_compressor_plugin {
}
}
size_t outsize = fpzip_write(fpz, input->data());
if(outsize == 0) {
return fpzip_error();
}
fpzip_write_close(fpz);
output->set_dimensions({outsize});

return 0;
};

int decompress_impl (const pressio_data *input, struct pressio_data *output) override {
(void)input;
(void)output;
int type = pressio_type_to_fpzip_type(output);
if(type == INVALID_TYPE) {
return INVALID_TYPE;
}

FPZ* fpz = fpzip_read_from_buffer(
pressio_data_ptr(input, nullptr)
);
if(has_header) {
fpzip_read_header(fpz);
} else {
fpz->nx = check_dim(output->get_dimension(0));
fpz->ny = check_dim(output->get_dimension(1));
fpz->nz = check_dim(output->get_dimension(2));
fpz->nf = check_dim(output->get_dimension(3));
fpz->type = type;
fpz->prec = prec;
}
fpzip_read(fpz, pressio_data_ptr(output, nullptr));
fpzip_read_close(fpz);

return 0;
}

Expand Down Expand Up @@ -124,7 +152,7 @@ class fpzip_plugin: public libpressio_compressor_plugin {
}

const char* version() const override {
return version_str.c_str();
return fpzip_version_string;
}
const char* prefix() const noexcept override {
return "fpzip";
Expand All @@ -135,8 +163,8 @@ class fpzip_plugin: public libpressio_compressor_plugin {

private:
std::string version_str;
bool has_header;
int prec;
int has_header = 0;
int prec = 0;

};

Expand Down
124 changes: 5 additions & 119 deletions src/plugins/io/csv.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,118 +35,6 @@ namespace {
const size_t rows, columns;
std::ofstream& outfile;
};

template <class Value>
struct csv_builder {

csv_builder()=default;
~csv_builder() {
if(this->value) free(this->value);
}
csv_builder(csv_builder &&)=delete;
csv_builder& operator=(csv_builder &&)=delete;
csv_builder(csv_builder const&)=delete;
csv_builder& operator=(csv_builder const&)=delete;

void resize(size_t new_rows, size_t new_columns) {
Value* tmp = static_cast<Value*>(malloc(sizeof(Value) * new_rows * new_columns));
copy_resized(
value, this->n_rows, this->n_columns,
tmp, new_rows, new_columns,
row_major
);

this->n_rows = new_rows;
this->n_columns = new_columns;
free(this->value);
this->value = std::move(tmp);
}

void reserve(size_t rows, size_t columns, size_t)
{
auto new_rows = std::max(this->n_rows, rows);
auto new_columns = std::max(this->n_columns, rows);
if(new_rows == this->n_rows && new_columns == this->n_columns) return;
else resize(rows, columns);
}

void set_entry(size_t row, size_t column, Value const& value)
{
if(row >= this->n_rows || column >= this->n_columns) {
resize(row >= this->n_rows ? row + 1 : this->n_rows, column >= this->n_columns ? column + 1: this->n_columns);
}
if(row_major) {
this->value[this->n_columns * row + column] = value;
} else {
this->value[this->n_rows * column + row] = value;
}
}

Value* build() {
auto tmp = this->value;
this->value = nullptr;
this->n_rows = 0;
this->n_columns = 0;
this->row_major = true;
return tmp;
}


size_t n_rows=0, n_columns=0;
bool row_major=true;
private:
Value* value=nullptr;

/**
* Copies or Zero initializes memory
*
* \param[in] old_values the memory to copy from
* \param[in] old_rows the old number of rows
* \param[in] old_columns the old number of columns
* \param[in] new_values the memory to fill/allocate
* \param[in] new_rows the new number of rows
* \param[in] new_columns the new number of columns
* \param[in] use_row_order -- the elements are in row-major order
*
* \tparam T the type of elements to copy
*/
void copy_resized(Value* old_values, size_t old_rows, size_t old_columns,
Value* new_values, size_t new_rows, size_t new_columns,
bool use_row_order
)
{
if(old_rows == new_rows && old_columns == new_columns) return;
if(use_row_order)
{
for (size_t i = 0; i < std::min(new_rows,old_rows); ++i) {
for (size_t j = 0; j < std::min(new_columns,old_columns); ++j) {
new_values[i*new_columns + j] = old_values[i*old_columns + j];
}
}

//zero initialize new values not set by previous code;
for (size_t i = std::min(new_rows, old_rows); i < new_rows; ++i) {
for (size_t j = std::min(new_columns, old_columns); j < new_columns; ++j) {
new_values[i*new_columns + j] = 0;
}
}

} else {
for (size_t j = 0; j < std::min(new_columns,old_columns); ++j) {
for (size_t i = 0; i < std::min(new_rows,old_rows); ++i) {
new_values[i*new_rows + j] = old_values[i*old_rows + j];
}
}
//zero initialize new values not set by previous code;
for (size_t j = std::min(new_columns, old_columns); j < new_columns; ++j) {
for (size_t i = std::min(new_rows, old_rows); i < new_rows; ++i) {
new_values[i*new_rows + j] = 0;
}
}
}

}
};
}

struct csv_io : public libpressio_io_plugin
Expand All @@ -160,24 +48,22 @@ struct csv_io : public libpressio_io_plugin
return nullptr;
}
size_t sizes[2] = {0,0};
csv_builder<double> builder;
std::vector<double> builder;
for(std::string line; std::getline(in, line); sizes[0]++) {
if(sizes[0] < skip_rows) continue;
std::istringstream line_ss(line);
size_t column = 0;
for(std::string value; std::getline(line_ss, value,','); ++column) {
builder.set_entry(sizes[0] - skip_rows, column, std::stold(value));
builder.emplace_back(std::stold(value));
}
sizes[1] = column;
}
sizes[0] -= skip_rows;
return pressio_data_new_move(
return pressio_data_new_copy(
pressio_double_dtype,
builder.build(),
builder.data(),
2,
sizes,
pressio_data_libc_free_fn,
nullptr
sizes
);
}

Expand Down

0 comments on commit 51da773

Please sign in to comment.