diff --git a/include/erasurecode/erasurecode.h b/include/erasurecode/erasurecode.h index 0a5f98e..45a9203 100644 --- a/include/erasurecode/erasurecode.h +++ b/include/erasurecode/erasurecode.h @@ -50,6 +50,8 @@ typedef enum { EC_BACKEND_LIBERASURECODE_RS_VAND = 6, EC_BACKEND_ISA_L_RS_CAUCHY = 7, EC_BACKEND_LIBPHAZR = 8, + EC_BACKEND_QUADIRON_FNT_SYS = 9, + EC_BACKEND_QUADIRON_FNT_NSYS = 10, EC_BACKENDS_MAX, } ec_backend_id_t; diff --git a/include/erasurecode/erasurecode_helpers_ext.h b/include/erasurecode/erasurecode_helpers_ext.h index 8041a57..7bbedf0 100644 --- a/include/erasurecode/erasurecode_helpers_ext.h +++ b/include/erasurecode/erasurecode_helpers_ext.h @@ -47,6 +47,9 @@ void init_fragment_header(char *buf) char *alloc_fragment_buffer(int size); int free_fragment_buffer(char *buf); int get_aligned_data_size(ec_backend_t instance, int data_len); +char *_get_data_ptr_from_fragment(char *buf); +int _get_data_ptr_array_from_fragments(char **data_array, char **fragments, + int num_fragments); char *get_data_ptr_from_fragment(char *buf); int get_data_ptr_array_from_fragments(char **data_array, char **fragments, int num_fragments); diff --git a/include/erasurecode/erasurecode_preprocessing.h b/include/erasurecode/erasurecode_preprocessing.h index 7ca891d..746140b 100644 --- a/include/erasurecode/erasurecode_preprocessing.h +++ b/include/erasurecode/erasurecode_preprocessing.h @@ -41,7 +41,7 @@ int prepare_fragments_for_decode( char **data, char **parity, int *missing_idxs, int *orig_size, int *fragment_payload_size, int fragment_size, - uint64_t *realloc_bm); + int *realloc_bv); int get_fragment_partition( int k, int m, diff --git a/include/quadiron/quadiron_fnt_common.h b/include/quadiron/quadiron_fnt_common.h new file mode 100644 index 0000000..9c0aac1 --- /dev/null +++ b/include/quadiron/quadiron_fnt_common.h @@ -0,0 +1,67 @@ +/* + * Copyright 2018 Scality + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY + * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * quadiron_fnt backend implementation + * + * vi: set noai tw=79 ts=4 sw=4: + */ + +/* Forward declarations */ +typedef void *(*quadiron_fnt32_new_func)(int word_size, int n_data, int n_parities, int systematic); +typedef void (*quadiron_fnt32_delete_func)(void *fec); +typedef int (*quadiron_fnt32_encode_func)(void *fec, char **data, char **parity, int *qmissing_idxs, int block_size); +typedef int (*quadiron_fnt32_decode_func)(void *fec, char **data, char **parity, int *qmissing_idxs, int block_size); +typedef int (*quadiron_fnt32_reconstruct_func)(void *fec, char **data, char **parity, int *qmissing_idxs, unsigned int destination_idx, int block_size); +typedef int (*quadiron_fnt32_get_metadata_size_func)(void *fec, int block_size); + +typedef struct { + int k; + int m; + int w; + + void *fec; // the instance + + quadiron_fnt32_new_func fnt32_new_func; + quadiron_fnt32_delete_func fnt32_delete_func; + quadiron_fnt32_encode_func fnt32_encode_func; + quadiron_fnt32_decode_func fnt32_decode_func; + quadiron_fnt32_reconstruct_func fnt32_reconstruct_func; + quadiron_fnt32_get_metadata_size_func fnt32_get_metadata_size_func; + +} quadiron_fnt_descriptor; + +int quadiron_fnt_encode(void *desc, char **data, char **parity, int blocksize); +int quadiron_fnt_decode(void *desc, char **data, char **parity, int *missing_idxs, + int blocksize); +int quadiron_fnt_reconstruct(void *desc, char **data, char **parity, + int *missing_idxs, int destination_idx, int blocksize); +int quadiron_fnt_min_fragments(void *desc, int *missing_idxs, + int *fragments_to_exclude, int *fragments_needed); +int quadiron_fnt_element_size(void* desc); +int quadiron_fnt_exit(void *desc); +void * quadiron_fnt_common_init(struct ec_backend_args *args, void *backend_sohandle, + int systematic); +size_t quadiron_fnt_common_get_metadata_size(void *desc, int block_size); +size_t quadiron_fnt_common_get_encode_offset(void *desc, int metadata_size); + +extern void (*quadiron_hex_dump)(char *buf, int buf_size); diff --git a/src/Makefile.am b/src/Makefile.am index 693809e..95209c4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,6 +7,7 @@ INCLUDE = \ -I$(abs_top_srcdir)/include/xor_codes \ -I$(abs_top_srcdir)/include/rs_vand \ -I$(abs_top_srcdir)/include/isa_l \ + -I$(abs_top_srcdir)/include/quadiron \ -I$(abs_top_srcdir)/include/shss AM_CPPFLAGS = $(CPPFLAGS) $(INCLUDE) @@ -30,7 +31,10 @@ liberasurecode_la_SOURCES = \ backends/rs_vand/liberasurecode_rs_vand.c \ builtin/rs_vand/rs_galois.c \ backends/shss/shss.c \ - backends/phazrio/libphazr.c + backends/phazrio/libphazr.c \ + backends/quadiron/quadiron_fnt_common.c \ + backends/quadiron/quadiron_fnt_nsys.c \ + backends/quadiron/quadiron_fnt_sys.c liberasurecode_la_CPPFLAGS = -Werror @GCOV_FLAGS@ liberasurecode_la_LIBADD = \ diff --git a/src/backends/quadiron/quadiron_fnt_common.c b/src/backends/quadiron/quadiron_fnt_common.c new file mode 100644 index 0000000..6b668af --- /dev/null +++ b/src/backends/quadiron/quadiron_fnt_common.c @@ -0,0 +1,238 @@ +/* + * Copyright 2018 Scality + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY + * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * quadiron_fnt backend implementation + * + * vi: set noai tw=79 ts=4 sw=4: + */ + +#include +#include + +#include "erasurecode.h" +#include "erasurecode_backend.h" +#include "erasurecode_helpers.h" +#include "erasurecode_helpers_ext.h" +#include "quadiron_fnt_common.h" + +typedef void (*quadiron_hex_dump_func)(char *buf, int buf_size); +quadiron_hex_dump_func quadiron_hex_dump; + +int quadiron_fnt_encode(void *desc, char **data, char **parity, + int blocksize) +{ + quadiron_fnt_descriptor *qdesc = (quadiron_fnt_descriptor*) desc; + int qwanted_idxs[qdesc->k + qdesc->m]; + unsigned int i; + + for (i = 0; i < qdesc->k + qdesc->m; i++) { + qwanted_idxs[i] = 1; + } + return qdesc->fnt32_encode_func(qdesc->fec, + data, parity, qwanted_idxs, blocksize); +} + +int quadiron_fnt_decode(void *desc, char **data, char **parity, + int *missing_idxs, int blocksize) +{ + quadiron_fnt_descriptor *qdesc = (quadiron_fnt_descriptor*)desc; + int qmissing_idxs[qdesc->k + qdesc->m]; + + convert_idx_list_to_bitvalues(missing_idxs, qmissing_idxs, + qdesc->k + qdesc->m); + return qdesc->fnt32_decode_func(qdesc->fec, + data, parity, qmissing_idxs, + blocksize); +} + +int quadiron_fnt_reconstruct(void *desc, char **data, char **parity, + int *missing_idxs, int destination_idx, int blocksize) +{ + quadiron_fnt_descriptor *qdesc = (quadiron_fnt_descriptor*)desc; + int qmissing_idxs[qdesc->k + qdesc->m]; + + convert_idx_list_to_bitvalues(missing_idxs, qmissing_idxs, + qdesc->k + qdesc->m); + return qdesc->fnt32_reconstruct_func(qdesc->fec, + data, parity, qmissing_idxs, + destination_idx, + blocksize); +} + +int quadiron_fnt_min_fragments(void *desc, int *missing_idxs, + int *fragments_to_exclude, int *fragments_needed) +{ + quadiron_fnt_descriptor *qdesc = (quadiron_fnt_descriptor*) desc; + int qexclude[qdesc->k + qdesc->m]; + int qmissing[qdesc->k + qdesc->m]; + int i; + int j = 0; + int ret = -1; + + convert_idx_list_to_bitvalues(fragments_to_exclude, qexclude, qdesc->k + qdesc->m); + convert_idx_list_to_bitvalues(missing_idxs, qmissing, qdesc->k + qdesc->m); + + for (i = 0; i < (qdesc->k + qdesc->m); i++) { + if (!(qmissing[i])) { + fragments_needed[j] = i; + j++; + } + if (j == qdesc->k) { + ret = 0; + fragments_needed[j] = -1; + break; + } + } + + return ret; +} + +/** + * Return the element-size, which is the number of bits stored + * on a given device, per codeword. + * + * Returns the size in bits! + */ +int quadiron_fnt_element_size(void* desc) +{ + return 8; +} + +int quadiron_fnt_exit(void *desc) +{ + quadiron_fnt_descriptor *quadiron_fnt_desc = NULL; + + quadiron_fnt_desc = (quadiron_fnt_descriptor*) desc; + + quadiron_fnt_desc->fnt32_delete_func(quadiron_fnt_desc->fec); + + free(quadiron_fnt_desc); + + return 0; +} + +size_t quadiron_fnt_common_get_metadata_size(void *desc, int block_size) +{ + quadiron_fnt_descriptor *quadiron_fnt_desc = NULL; + + quadiron_fnt_desc = (quadiron_fnt_descriptor*) desc; + + return quadiron_fnt_desc->fnt32_get_metadata_size_func(quadiron_fnt_desc->fec, block_size); +} + +size_t quadiron_fnt_common_get_encode_offset(void *desc, int metadata_size) +{ + return metadata_size; +} + +void * quadiron_fnt_common_init(struct ec_backend_args *args, void *backend_sohandle, + int systematic) +{ + quadiron_fnt_descriptor *desc = NULL; + + desc = (quadiron_fnt_descriptor *)malloc(sizeof(quadiron_fnt_descriptor)); + if (NULL == desc) { + return NULL; + } + + desc->k = args->uargs.k; + desc->m = args->uargs.m; + desc->w = args->uargs.w; + + /* validate EC arguments */ + + /* + * ISO C forbids casting a void* to a function pointer. + * Since dlsym return returns a void*, we use this union to + * "transform" the void* to a function pointer. + */ + union { + quadiron_fnt32_new_func quadiron_fnt32_new_funcp; + quadiron_fnt32_delete_func quadiron_fnt32_delete_funcp; + quadiron_fnt32_encode_func quadiron_fnt32_encode_funcp; + quadiron_fnt32_decode_func quadiron_fnt32_decode_funcp; + quadiron_fnt32_reconstruct_func quadiron_fnt32_reconstruct_funcp; + quadiron_fnt32_get_metadata_size_func quadiron_fnt32_get_metadata_size_funcp; + quadiron_hex_dump_func quadiron_hex_dump_funcp; + void *vptr; + } func_handle = {.vptr = NULL}; + + /* fill in function addresses */ + func_handle.vptr = NULL; + func_handle.vptr = dlsym(backend_sohandle, "quadiron_fnt32_new"); + desc->fnt32_new_func = func_handle.quadiron_fnt32_new_funcp; + if (NULL == desc->fnt32_new_func) { + goto error; + } + + func_handle.vptr = NULL; + func_handle.vptr = dlsym(backend_sohandle, "quadiron_fnt32_delete"); + desc->fnt32_delete_func = func_handle.quadiron_fnt32_delete_funcp; + if (NULL == desc->fnt32_delete_func) { + goto error; + } + + func_handle.vptr = NULL; + func_handle.vptr = dlsym(backend_sohandle, "quadiron_fnt32_encode"); + desc->fnt32_encode_func = func_handle.quadiron_fnt32_encode_funcp; + if (NULL == desc->fnt32_encode_func) { + goto error; + } + + func_handle.vptr = NULL; + func_handle.vptr = dlsym(backend_sohandle, "quadiron_fnt32_decode"); + desc->fnt32_decode_func = func_handle.quadiron_fnt32_decode_funcp; + if (NULL == desc->fnt32_decode_func) { + goto error; + } + + func_handle.vptr = NULL; + func_handle.vptr = dlsym(backend_sohandle, "quadiron_fnt32_reconstruct"); + desc->fnt32_reconstruct_func = func_handle.quadiron_fnt32_reconstruct_funcp; + if (NULL == desc->fnt32_reconstruct_func) { + goto error; + } + + func_handle.vptr = NULL; + func_handle.vptr = dlsym(backend_sohandle, "quadiron_fnt32_get_metadata_size"); + desc->fnt32_get_metadata_size_func = func_handle.quadiron_fnt32_get_metadata_size_funcp; + if (NULL == desc->fnt32_get_metadata_size_func) { + goto error; + } + + func_handle.vptr = dlsym(backend_sohandle, "quadiron_hex_dump"); + quadiron_hex_dump = func_handle.quadiron_hex_dump_funcp; + + desc->fec = desc->fnt32_new_func(args->uargs.w / 8, args->uargs.k, + args->uargs.m, systematic); + if (NULL == desc->fec) { + goto error; + } + + return desc; + +error: + free(desc); + + return NULL; +} diff --git a/src/backends/quadiron/quadiron_fnt_nsys.c b/src/backends/quadiron/quadiron_fnt_nsys.c new file mode 100644 index 0000000..dca8bf4 --- /dev/null +++ b/src/backends/quadiron/quadiron_fnt_nsys.c @@ -0,0 +1,85 @@ +/* + * Copyright 2018 Scality + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY + * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * quadiron_fnt_nsys backend implementation + * + * vi: set noai tw=79 ts=4 sw=4: + */ + +#include +#include "erasurecode_backend.h" +#include "quadiron_fnt_common.h" + +#define QUADIRON_FNT_NSYS_LIB_MAJOR 2 +#define QUADIRON_FNT_NSYS_LIB_MINOR 13 +#define QUADIRON_FNT_NSYS_LIB_REV 0 +#define QUADIRON_FNT_NSYS_LIB_VER_STR "2.13" +#define QUADIRON_FNT_NSYS_LIB_NAME "quadiron_fnt_nsys" +#if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__) +#define QUADIRON_FNT_NSYS_SO_NAME "libquadiron.dylib" +#else +#define QUADIRON_FNT_NSYS_SO_NAME "libquadiron.so" +#endif + +/* Forward declarations */ +struct ec_backend_op_stubs quadiron_fnt_nsys_ops; +struct ec_backend quadiron_fnt_nsys; +struct ec_backend_common backend_quadiron_fnt_nsys; + +static void * quadiron_fnt_nsys_init(struct ec_backend_args *args, + void *backend_sohandle) +{ + return quadiron_fnt_common_init(args, backend_sohandle, 0); +} + +/* + * For the time being, we only claim compatibility with versions that + * match exactly + */ +static bool quadiron_fnt_nsys_is_compatible_with(uint32_t version) { + return version == backend_quadiron_fnt_nsys.ec_backend_version; +} + +struct ec_backend_op_stubs quadiron_fnt_nsys_op_stubs = { + .INIT = quadiron_fnt_nsys_init, + .EXIT = quadiron_fnt_exit, + .ENCODE = quadiron_fnt_encode, + .DECODE = quadiron_fnt_decode, + .FRAGSNEEDED = quadiron_fnt_min_fragments, + .RECONSTRUCT = quadiron_fnt_reconstruct, + .ELEMENTSIZE = quadiron_fnt_element_size, + .ISCOMPATIBLEWITH = quadiron_fnt_nsys_is_compatible_with, + .GETMETADATASIZE = quadiron_fnt_common_get_metadata_size, + .GETENCODEOFFSET = quadiron_fnt_common_get_encode_offset, +}; + +struct ec_backend_common backend_quadiron_fnt_nsys = { + .id = EC_BACKEND_QUADIRON_FNT_NSYS, + .name = QUADIRON_FNT_NSYS_LIB_NAME, + .soname = QUADIRON_FNT_NSYS_SO_NAME, + .soversion = QUADIRON_FNT_NSYS_LIB_VER_STR, + .ops = &quadiron_fnt_nsys_op_stubs, + .ec_backend_version = _VERSION(QUADIRON_FNT_NSYS_LIB_MAJOR, + QUADIRON_FNT_NSYS_LIB_MINOR, + QUADIRON_FNT_NSYS_LIB_REV), +}; diff --git a/src/backends/quadiron/quadiron_fnt_sys.c b/src/backends/quadiron/quadiron_fnt_sys.c new file mode 100644 index 0000000..50ef627 --- /dev/null +++ b/src/backends/quadiron/quadiron_fnt_sys.c @@ -0,0 +1,85 @@ +/* + * Copyright 2018 Scality + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, this + * list of conditions and the following disclaimer in the documentation and/or + * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY + * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO + * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * quadiron_fnt_sys backend implementation + * + * vi: set noai tw=79 ts=4 sw=4: + */ + +#include +#include "erasurecode_backend.h" +#include "quadiron_fnt_common.h" + +#define QUADIRON_FNT_SYS_LIB_MAJOR 2 +#define QUADIRON_FNT_SYS_LIB_MINOR 13 +#define QUADIRON_FNT_SYS_LIB_REV 0 +#define QUADIRON_FNT_SYS_LIB_VER_STR "2.13" +#define QUADIRON_FNT_SYS_LIB_NAME "quadiron_fnt_sys" +#if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__) +#define QUADIRON_FNT_SYS_SO_NAME "libquadiron.dylib" +#else +#define QUADIRON_FNT_SYS_SO_NAME "libquadiron.so" +#endif + +/* Forward declarations */ +struct ec_backend_op_stubs quadiron_fnt_sys_ops; +struct ec_backend quadiron_fnt_sys; +struct ec_backend_common backend_quadiron_fnt_sys; + +static void * quadiron_fnt_sys_init(struct ec_backend_args *args, + void *backend_sohandle) +{ + return quadiron_fnt_common_init(args, backend_sohandle, 1); +} + +/* + * For the time being, we only claim compatibility with versions that + * match exactly + */ +static bool quadiron_fnt_sys_is_compatible_with(uint32_t version) { + return version == backend_quadiron_fnt_sys.ec_backend_version; +} + +struct ec_backend_op_stubs quadiron_fnt_sys_op_stubs = { + .INIT = quadiron_fnt_sys_init, + .EXIT = quadiron_fnt_exit, + .ENCODE = quadiron_fnt_encode, + .DECODE = quadiron_fnt_decode, + .FRAGSNEEDED = quadiron_fnt_min_fragments, + .RECONSTRUCT = quadiron_fnt_reconstruct, + .ELEMENTSIZE = quadiron_fnt_element_size, + .ISCOMPATIBLEWITH = quadiron_fnt_sys_is_compatible_with, + .GETMETADATASIZE = quadiron_fnt_common_get_metadata_size, + .GETENCODEOFFSET = quadiron_fnt_common_get_encode_offset, +}; + +struct ec_backend_common backend_quadiron_fnt_sys = { + .id = EC_BACKEND_QUADIRON_FNT_SYS, + .name = QUADIRON_FNT_SYS_LIB_NAME, + .soname = QUADIRON_FNT_SYS_SO_NAME, + .soversion = QUADIRON_FNT_SYS_LIB_VER_STR, + .ops = &quadiron_fnt_sys_op_stubs, + .ec_backend_version = _VERSION(QUADIRON_FNT_SYS_LIB_MAJOR, + QUADIRON_FNT_SYS_LIB_MINOR, + QUADIRON_FNT_SYS_LIB_REV), +}; diff --git a/src/erasurecode.c b/src/erasurecode.c index dd59ae5..550f33d 100644 --- a/src/erasurecode.c +++ b/src/erasurecode.c @@ -52,6 +52,8 @@ extern struct ec_backend_common backend_shss; extern struct ec_backend_common backend_liberasurecode_rs_vand; extern struct ec_backend_common backend_isa_l_rs_cauchy; extern struct ec_backend_common backend_libphazr; +extern struct ec_backend_common backend_quadiron_fnt_sys; +extern struct ec_backend_common backend_quadiron_fnt_nsys; ec_backend_t ec_backends_supported[] = { (ec_backend_t) &backend_null, @@ -63,6 +65,8 @@ ec_backend_t ec_backends_supported[] = { (ec_backend_t) &backend_liberasurecode_rs_vand, (ec_backend_t) &backend_isa_l_rs_cauchy, (ec_backend_t) &backend_libphazr, + (ec_backend_t) &backend_quadiron_fnt_sys, + (ec_backend_t) &backend_quadiron_fnt_nsys, NULL, }; @@ -276,7 +280,7 @@ int liberasurecode_instance_create(const ec_backend_id_t id, if (args->k < 0 || args->m < 0) return -EINVALIDPARAMS; - if ((args->k + args->m) > EC_MAX_FRAGMENTS) { + if ((args->k + args->m) > EC_MAX_FRAGMENTS && id != EC_BACKEND_QUADIRON_FNT_SYS) { log_error("Total number of fragments (k + m) must be less than %d\n", EC_MAX_FRAGMENTS); return -EINVALIDPARAMS; @@ -555,8 +559,7 @@ int liberasurecode_decode(int desc, char **data_segments = NULL; char **parity_segments = NULL; int *missing_idxs = NULL; - - uint64_t realloc_bm = 0; + int *realloc_bv = NULL; ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc); if (NULL == instance) { @@ -585,6 +588,9 @@ int liberasurecode_decode(int desc, k = instance->args.uargs.k; m = instance->args.uargs.m; + realloc_bv = alloca((k + m) * sizeof (int)); + memset(realloc_bv, 0, (k + m) * sizeof (int)); + if (num_fragments < k) { log_error("Not enough fragments to decode, got %d, need %d!", num_fragments, k); @@ -609,8 +615,9 @@ int liberasurecode_decode(int desc, } } - if (instance->common.id != EC_BACKEND_SHSS && instance->common.id != EC_BACKEND_LIBPHAZR) { - /* shss (ntt_backend) & libphazr backend must force to decode */ + if (instance->common.id != EC_BACKEND_SHSS && instance->common.id != EC_BACKEND_LIBPHAZR && + instance->common.id != EC_BACKEND_QUADIRON_FNT_NSYS) { + /* shss (ntt_backend) & libphazr & quadiron-nsys backend must force to decode */ // TODO: Add a frag and function to handle whether the backend want to decode or not. /* * Try to re-assebmle the original data before attempting a decode @@ -677,13 +684,13 @@ int liberasurecode_decode(int desc, * Preparing the fragments for decode. This will alloc aligned buffers * when unaligned buffers were passed in available_fragments. It passes * back a bitmap telling us which buffers need to be freed by us - * (realloc_bm). + * (realloc_bv). * */ ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs, &orig_data_size, &blocksize, - fragment_len, &realloc_bm); + fragment_len, realloc_bv); if (ret < 0) { log_error("Could not prepare fragments for decode!"); goto out; @@ -732,18 +739,18 @@ int liberasurecode_decode(int desc, out: /* Free the buffers allocated in prepare_fragments_for_decode */ - if (realloc_bm != 0) { - for (i = 0; i < k; i++) { - if (realloc_bm & (1 << i)) { - free(data[i]); - } + if (realloc_bv) { + for (i = 0; i < k; i++) { + if (realloc_bv[i]) { + free(data[i]); } - - for (i = 0; i < m; i++) { - if (realloc_bm & (1 << (i + k))) { - free(parity[i]); - } + } + + for (i = 0; i < m; i++) { + if (realloc_bv[i + k]) { + free(parity[i]); } + } } free(data); @@ -784,11 +791,11 @@ int liberasurecode_reconstruct_fragment(int desc, int k = -1; int m = -1; int i; - uint64_t realloc_bm = 0; char **data_segments = NULL; char **parity_segments = NULL; int set_chksum = 1; - + int *realloc_bv = NULL; + ec_backend_t instance = liberasurecode_backend_instance_get_by_desc(desc); if (NULL == instance) { ret = -EBACKENDNOTAVAIL; @@ -810,6 +817,9 @@ int liberasurecode_reconstruct_fragment(int desc, k = instance->args.uargs.k; m = instance->args.uargs.m; + realloc_bv = alloca((k + m) * sizeof (int)); + memset(realloc_bv, 0, (k + m) * sizeof (int)); + for (i = 0; i < num_fragments; i++) { /* Verify metadata checksum */ if (is_invalid_fragment_header( @@ -887,11 +897,11 @@ int liberasurecode_reconstruct_fragment(int desc, * Preparing the fragments for reconstruction. This will alloc aligned * buffers when unaligned buffers were passed in available_fragments. * It passes back a bitmap telling us which buffers need to be freed by - * us (realloc_bm). + * us (realloc_bv). */ ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs, &orig_data_size, &blocksize, - fragment_len, &realloc_bm); + fragment_len, realloc_bv); if (ret < 0) { log_error("Could not prepare fragments for reconstruction!"); goto out; @@ -935,18 +945,18 @@ int liberasurecode_reconstruct_fragment(int desc, out: /* Free the buffers allocated in prepare_fragments_for_decode */ - if (realloc_bm != 0) { - for (i = 0; i < k; i++) { - if (realloc_bm & (1 << i)) { - free(data[i]); - } + if (realloc_bv) { + for (i = 0; i < k; i++) { + if (realloc_bv[i]) { + free(data[i]); } - - for (i = 0; i < m; i++) { - if (realloc_bm & (1 << (i + k))) { - free(parity[i]); - } + } + + for (i = 0; i < m; i++) { + if (realloc_bv[i + k]) { + free(parity[i]); } + } } free(data); diff --git a/src/erasurecode_helpers.c b/src/erasurecode_helpers.c index 1494587..e3816d8 100644 --- a/src/erasurecode_helpers.c +++ b/src/erasurecode_helpers.c @@ -208,13 +208,44 @@ int get_aligned_data_size(ec_backend_t instance, int data_len) /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */ -char *get_data_ptr_from_fragment(char *buf) +char *_get_data_ptr_from_fragment(char *buf) { - buf += sizeof(fragment_header_t); + fragment_header_t *header = (fragment_header_t *) buf; + + assert(NULL != header); + if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) { + log_error("Invalid fragment header (size check)!"); + return NULL; + } + + buf += sizeof(fragment_header_t) + header->meta.frag_backend_metadata_size; return buf; } +int _get_data_ptr_array_from_fragments(char **data_array, char **fragments, + int num_fragments) +{ + int i = 0, num = 0; + for (i = 0; i < num_fragments; i++) { + char *frag = fragments[i]; + if (frag == NULL) { + data_array[i] = NULL; + continue; + } + data_array[i] = _get_data_ptr_from_fragment(frag); + num++; + } + return num; +} + +char *get_data_ptr_from_fragment(char *buf) +{ + buf += sizeof(fragment_header_t); + + return buf; +} + int get_data_ptr_array_from_fragments(char **data_array, char **fragments, int num_fragments) { diff --git a/src/erasurecode_preprocessing.c b/src/erasurecode_preprocessing.c index 26ad261..20ff6ce 100644 --- a/src/erasurecode_preprocessing.c +++ b/src/erasurecode_preprocessing.c @@ -111,7 +111,7 @@ int prepare_fragments_for_encode(ec_backend_t instance, } /* - * Note that the caller should always check realloc_bm during success or + * Note that the caller should always check realloc_bv during success or * failure to free buffers allocated here. We could free up in this function, * but it is internal to this library and only used in a few places. In any * case, the caller has to free up in the success case, so it may as well do @@ -122,14 +122,14 @@ int prepare_fragments_for_decode( char **data, char **parity, int *missing_idxs, int *orig_size, int *fragment_payload_size, int fragment_size, - uint64_t *realloc_bm) + int *realloc_bv) { int i; /* a counter */ - unsigned long long missing_bm; /* bitmap form of missing indexes list */ + int missing_bv[k + m]; /* bitvalues form of missing indexes list */ int orig_data_size = -1; int payload_size = -1; - missing_bm = convert_list_to_bitmap(missing_idxs); + convert_idx_list_to_bitvalues(missing_idxs, missing_bv, k + m); /* * Determine if each data fragment is: @@ -150,7 +150,7 @@ int prepare_fragments_for_decode( log_error("Could not allocate data buffer!"); return -ENOMEM; } - *realloc_bm = *realloc_bm | (1 << i); + realloc_bv[i] = 1; } else if (!is_addr_aligned((unsigned long)data[i], 16)) { char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t)); if (NULL == tmp_buf) { @@ -159,11 +159,11 @@ int prepare_fragments_for_decode( } memcpy(tmp_buf, data[i], fragment_size); data[i] = tmp_buf; - *realloc_bm = *realloc_bm | (1 << i); + realloc_bv[i] = 1; } /* Need to determine the size of the original data */ - if (((missing_bm & (1 << i)) == 0) && orig_data_size < 0) { + if (!missing_bv[i] && orig_data_size < 0) { orig_data_size = get_orig_data_size(data[i]); if (orig_data_size < 0) { log_error("Invalid orig_data_size in fragment header!"); @@ -189,7 +189,7 @@ int prepare_fragments_for_decode( log_error("Could not allocate parity buffer!"); return -ENOMEM; } - *realloc_bm = *realloc_bm | (1 << (k + i)); + realloc_bv[k + i] = 1; } else if (!is_addr_aligned((unsigned long)parity[i], 16)) { char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t)); if (NULL == tmp_buf) { @@ -198,11 +198,11 @@ int prepare_fragments_for_decode( } memcpy(tmp_buf, parity[i], fragment_size); parity[i] = tmp_buf; - *realloc_bm = *realloc_bm | (1 << (k + i)); + realloc_bv[k + i] = 1; } /* Need to determine the size of the original data */ - if (((missing_bm & (1 << (k + i))) == 0) && orig_data_size < 0) { + if (!missing_bv[k + i] && orig_data_size < 0) { orig_data_size = get_orig_data_size(parity[i]); if (orig_data_size < 0) { log_error("Invalid orig_data_size in fragment header!"); @@ -361,7 +361,7 @@ int fragments_to_string(int k, int m, /* Copy fragment data into cstring (fragments should be in index order) */ for (i = 0; i < num_data && orig_data_size > 0; i++) { - char* fragment_data = get_data_ptr_from_fragment(data[i]); + char* fragment_data = _get_data_ptr_from_fragment(data[i]); int fragment_size = get_fragment_payload_size(data[i]); int payload_size = orig_data_size > fragment_size ? fragment_size : orig_data_size; memcpy(internal_payload + string_off, fragment_data, payload_size); diff --git a/test/liberasurecode_test.c b/test/liberasurecode_test.c index d3ca200..01c7255 100644 --- a/test/liberasurecode_test.c +++ b/test/liberasurecode_test.c @@ -44,6 +44,8 @@ #define SHSS_BACKEND "shss" #define RS_VAND_BACKEND "liberasurecode_rs_vand" #define LIBPHAZR_BACKEND "libphazr" +#define QUADIRON_FNT_SYS_BACKEND "quadiron_fnt_sys" +#define QUADIRON_FNT_NSYS_BACKEND "quadiron_fnt_nsys" typedef void (*TEST_FUNC)(); @@ -231,6 +233,22 @@ struct ec_args libphazr_args = { struct ec_args *libphazr_test_args[] = { &libphazr_args, NULL }; +struct ec_args quadiron_fnt_sys_args = { + .k = 3, + .m = 3, + .w = 16, +}; + +struct ec_args *quadiron_fnt_sys_test_args[] = { &quadiron_fnt_sys_args, NULL }; + +struct ec_args quadiron_fnt_nsys_args = { + .k = 3, + .m = 3, + .w = 16, +}; + +struct ec_args *quadiron_fnt_nsys_test_args[] = { &quadiron_fnt_nsys_args, NULL }; + struct ec_args **all_backend_tests[] = { null_test_args, flat_xor_test_args, @@ -240,6 +258,8 @@ struct ec_args **all_backend_tests[] = { shss_test_args, liberasurecode_rs_vand_test_args, libphazr_test_args, + quadiron_fnt_sys_test_args, + quadiron_fnt_nsys_test_args, NULL}; int num_backends() @@ -300,7 +320,11 @@ char * get_name_from_backend_id(ec_backend_id_t be) { return RS_VAND_BACKEND; case EC_BACKEND_LIBPHAZR: return LIBPHAZR_BACKEND; - default: + case EC_BACKEND_QUADIRON_FNT_SYS: + return QUADIRON_FNT_SYS_BACKEND; + case EC_BACKEND_QUADIRON_FNT_NSYS: + return QUADIRON_FNT_NSYS_BACKEND; + default: return "UNKNOWN"; } } @@ -340,6 +364,12 @@ struct ec_args *create_ec_args(ec_backend_id_t be, ec_checksum_type_t ct, int ba case EC_BACKEND_LIBPHAZR: backend_args_array = libphazr_test_args; break; + case EC_BACKEND_QUADIRON_FNT_SYS: + backend_args_array = quadiron_fnt_sys_test_args; + break; + case EC_BACKEND_QUADIRON_FNT_NSYS: + backend_args_array = quadiron_fnt_nsys_test_args; + break; default: return NULL; } @@ -366,7 +396,9 @@ struct ec_args *create_ec_args(ec_backend_id_t be, ec_checksum_type_t ct, int ba char *create_buffer(int size, int fill) { char *buf = malloc(size); - memset(buf, fill, size); + int i; + for (i = 0;i < size;i++) + buf[i] = rand() % 256; return buf; } @@ -1010,7 +1042,8 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id, if (-EBACKENDNOTAVAIL == desc) { fprintf(stderr, "Backend library not available!\n"); return; - } else if ((args->k + args->m) > EC_MAX_FRAGMENTS) { + } else if ((args->k + args->m) > EC_MAX_FRAGMENTS && + be_id != EC_BACKEND_QUADIRON_FNT_SYS) { assert(-EINVALIDPARAMS == desc); return; } else @@ -1040,10 +1073,11 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id, assert(metadata.orig_data_size == orig_data_size); assert(metadata.backend_id == be_id); assert(metadata.chksum_mismatch == 0); - data_ptr = frag + frag_header_size; + data_ptr = frag + frag_header_size + metadata.frag_backend_metadata_size; cmp_size = remaining >= metadata.size ? metadata.size : remaining; // shss & libphazr doesn't keep original data on data fragments - if (be_id != EC_BACKEND_SHSS && be_id != EC_BACKEND_LIBPHAZR) { + if (be_id != EC_BACKEND_SHSS && be_id != EC_BACKEND_LIBPHAZR && + be_id != EC_BACKEND_QUADIRON_FNT_NSYS) { assert(memcmp(data_ptr, orig_data_ptr, cmp_size) == 0); } remaining -= cmp_size; @@ -1059,7 +1093,7 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id, assert(0 == rc); assert(decoded_data_len == orig_data_size); assert(memcmp(decoded_data, orig_data, orig_data_size) == 0); - + rc = liberasurecode_encode_cleanup(desc, encoded_data, encoded_parity); assert(rc == 0); @@ -1908,6 +1942,9 @@ struct testcase testcases[] = { TEST_SUITE(EC_BACKEND_LIBERASURECODE_RS_VAND), // libphazr backend tests TEST_SUITE(EC_BACKEND_LIBPHAZR), + // quadiron backend tests + TEST_SUITE(EC_BACKEND_QUADIRON_FNT_SYS), + TEST_SUITE(EC_BACKEND_QUADIRON_FNT_NSYS), { NULL, NULL, 0, 0, false }, };