quadiron integration

include/erasurecode/erasurecode.h

@@ -50,6 +50,8 @@ typedef enum {
     EC_BACKEND_LIBERASURECODE_RS_VAND = 6,
     EC_BACKEND_ISA_L_RS_CAUCHY        = 7,
     EC_BACKEND_LIBPHAZR               = 8,
+    EC_BACKEND_QUADIRON_FNT_SYS       = 9,
+    EC_BACKEND_QUADIRON_FNT_NSYS      = 10,
     EC_BACKENDS_MAX,
 } ec_backend_id_t;
 

include/erasurecode/erasurecode_helpers_ext.h

@@ -47,6 +47,9 @@ void init_fragment_header(char *buf)
 char *alloc_fragment_buffer(int size);
 int free_fragment_buffer(char *buf);
 int get_aligned_data_size(ec_backend_t instance, int data_len);
+char *_get_data_ptr_from_fragment(char *buf);
+int _get_data_ptr_array_from_fragments(char **data_array, char **fragments,
+        int num_fragments);
 char *get_data_ptr_from_fragment(char *buf);
 int get_data_ptr_array_from_fragments(char **data_array, char **fragments,
         int num_fragments);

include/erasurecode/erasurecode_preprocessing.h

@@ -41,7 +41,7 @@ int prepare_fragments_for_decode(
         char **data, char **parity,
         int *missing_idxs,
         int *orig_size, int *fragment_payload_size, int fragment_size,
-        uint64_t *realloc_bm);
+        int *realloc_bv);
 
 int get_fragment_partition(
         int k, int m,

include/quadiron/quadiron_fnt_common.h

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2018 Scality
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice, this
+ * list of conditions and the following disclaimer in the documentation and/or
+ * other materials provided with the distribution.  THIS SOFTWARE IS PROVIDED BY
+ * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * quadiron_fnt backend implementation
+ *
+ * vi: set noai tw=79 ts=4 sw=4:
+ */
+
+/* Forward declarations */
+typedef void *(*quadiron_fnt32_new_func)(int word_size, int n_data, int n_parities, int systematic);
+typedef void (*quadiron_fnt32_delete_func)(void *fec);
+typedef int (*quadiron_fnt32_encode_func)(void *fec, char **data, char **parity, int *qmissing_idxs, int block_size);
+typedef int (*quadiron_fnt32_decode_func)(void *fec, char **data, char **parity, int *qmissing_idxs, int block_size);
+typedef int (*quadiron_fnt32_reconstruct_func)(void *fec, char **data, char **parity, int *qmissing_idxs, unsigned int destination_idx, int block_size);
+typedef int (*quadiron_fnt32_get_metadata_size_func)(void *fec, int block_size);
+
+typedef struct {
+  int k;
+  int m;
+  int w;
+
+  void *fec; // the instance
+
+  quadiron_fnt32_new_func fnt32_new_func;
+  quadiron_fnt32_delete_func fnt32_delete_func;
+  quadiron_fnt32_encode_func fnt32_encode_func;
+  quadiron_fnt32_decode_func fnt32_decode_func;
+  quadiron_fnt32_reconstruct_func fnt32_reconstruct_func;
+  quadiron_fnt32_get_metadata_size_func fnt32_get_metadata_size_func;
+
+} quadiron_fnt_descriptor;
+
+int quadiron_fnt_encode(void *desc, char **data, char **parity, int blocksize);
+int quadiron_fnt_decode(void *desc, char **data, char **parity, int *missing_idxs,
+        int blocksize);
+int quadiron_fnt_reconstruct(void *desc, char **data, char **parity,
+        int *missing_idxs, int destination_idx, int blocksize);
+int quadiron_fnt_min_fragments(void *desc, int *missing_idxs,
+        int *fragments_to_exclude, int *fragments_needed);
+int quadiron_fnt_element_size(void* desc);
+int quadiron_fnt_exit(void *desc);
+void * quadiron_fnt_common_init(struct ec_backend_args *args, void *backend_sohandle,
+                                int systematic);
+size_t quadiron_fnt_common_get_metadata_size(void *desc, int block_size);
+size_t quadiron_fnt_common_get_encode_offset(void *desc, int metadata_size);
+
+extern void (*quadiron_hex_dump)(char *buf, int buf_size);

src/Makefile.am

@@ -7,6 +7,7 @@ INCLUDE = \
 		-I$(abs_top_srcdir)/include/xor_codes \
 		-I$(abs_top_srcdir)/include/rs_vand \
 		-I$(abs_top_srcdir)/include/isa_l \
+		-I$(abs_top_srcdir)/include/quadiron \
 		-I$(abs_top_srcdir)/include/shss
 
 AM_CPPFLAGS = $(CPPFLAGS) $(INCLUDE)
@@ -30,7 +31,10 @@ liberasurecode_la_SOURCES = \
 		backends/rs_vand/liberasurecode_rs_vand.c \
 		builtin/rs_vand/rs_galois.c \
 		backends/shss/shss.c \
-		backends/phazrio/libphazr.c
+		backends/phazrio/libphazr.c \
+		backends/quadiron/quadiron_fnt_common.c \
+		backends/quadiron/quadiron_fnt_nsys.c \
+		backends/quadiron/quadiron_fnt_sys.c
 
 liberasurecode_la_CPPFLAGS = -Werror @GCOV_FLAGS@
 liberasurecode_la_LIBADD = \

src/backends/quadiron/quadiron_fnt_common.c

@@ -0,0 +1,239 @@
+/*
+ * Copyright 2018 Scality
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice, this
+ * list of conditions and the following disclaimer in the documentation and/or
+ * other materials provided with the distribution.  THIS SOFTWARE IS PROVIDED BY
+ * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * quadiron_fnt backend implementation
+ *
+ * vi: set noai tw=79 ts=4 sw=4:
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "erasurecode.h"
+#include "erasurecode_backend.h"
+#include "erasurecode_helpers.h"
+#include "erasurecode_helpers_ext.h"
+#include "quadiron_fnt_common.h"
+
+typedef void (*quadiron_hex_dump_func)(char *buf, int buf_size);
+quadiron_hex_dump_func quadiron_hex_dump;
+
+int quadiron_fnt_encode(void *desc, char **data, char **parity,
+        int blocksize)
+{
+  quadiron_fnt_descriptor *qdesc = (quadiron_fnt_descriptor*) desc;
+  int qwanted_idxs[qdesc->k + qdesc->m];
+  unsigned int i;
+
+  for (i = 0; i < qdesc->k + qdesc->m; i++) {
+    qwanted_idxs[i] = 1;
+  }
+  return qdesc->fnt32_encode_func(qdesc->fec,
+                                  data, parity, qwanted_idxs, blocksize);
+}
+
+
+int quadiron_fnt_decode(void *desc, char **data, char **parity,
+        int *missing_idxs, int blocksize)
+{
+  quadiron_fnt_descriptor *qdesc = (quadiron_fnt_descriptor*)desc;
+  int qmissing_idxs[qdesc->k + qdesc->m];
+
+  convert_idx_list_to_bitvalues(missing_idxs, qmissing_idxs,
+                                qdesc->k + qdesc->m);
+  return qdesc->fnt32_decode_func(qdesc->fec,
+                                  data, parity, qmissing_idxs,
+                                  blocksize);
+}
+
+int quadiron_fnt_reconstruct(void *desc, char **data, char **parity,
+        int *missing_idxs, int destination_idx, int blocksize)
+{
+  quadiron_fnt_descriptor *qdesc = (quadiron_fnt_descriptor*)desc;
+  int qmissing_idxs[qdesc->k + qdesc->m];
+
+  convert_idx_list_to_bitvalues(missing_idxs, qmissing_idxs,
+                                qdesc->k + qdesc->m);
+  return qdesc->fnt32_reconstruct_func(qdesc->fec,
+                                       data, parity, qmissing_idxs,
+                                       destination_idx,
+                                       blocksize);
+}
+
+int quadiron_fnt_min_fragments(void *desc, int *missing_idxs,
+        int *fragments_to_exclude, int *fragments_needed)
+{
+  quadiron_fnt_descriptor *qdesc = (quadiron_fnt_descriptor*) desc;
+  int qexclude[qdesc->k + qdesc->m];
+  int qmissing[qdesc->k + qdesc->m];
+  int i;
+  int j = 0;
+  int ret = -1;
+
+  convert_idx_list_to_bitvalues(fragments_to_exclude, qexclude, qdesc->k + qdesc->m);
+  convert_idx_list_to_bitvalues(missing_idxs, qmissing, qdesc->k + qdesc->m);
+
+  for (i = 0; i < (qdesc->k + qdesc->m); i++) {
+    if (!(qmissing[i])) {
+      fragments_needed[j] = i;
+      j++;
+    }
+    if (j == qdesc->k) {
+      ret = 0;
+      fragments_needed[j] = -1;
+      break;
+    }
+  }
+
+  return ret;
+}
+
+/**
+ * Return the element-size, which is the number of bits stored
+ * on a given device, per codeword.
+ *
+ * Returns the size in bits!
+ */
+int quadiron_fnt_element_size(void* desc)
+{
+  return 8;
+}
+
+int quadiron_fnt_exit(void *desc)
+{
+    quadiron_fnt_descriptor *quadiron_fnt_desc = NULL;
+
+    quadiron_fnt_desc = (quadiron_fnt_descriptor*) desc;
+
+    quadiron_fnt_desc->fnt32_delete_func(quadiron_fnt_desc->fec);
+
+    free(quadiron_fnt_desc);
+
+    return 0;
+}
+
+size_t quadiron_fnt_common_get_metadata_size(void *desc, int block_size)
+{
+    quadiron_fnt_descriptor *quadiron_fnt_desc = NULL;
+
+    quadiron_fnt_desc = (quadiron_fnt_descriptor*) desc;
+
+    return quadiron_fnt_desc->fnt32_get_metadata_size_func(quadiron_fnt_desc->fec, block_size);
+}
+
+size_t quadiron_fnt_common_get_encode_offset(void *desc, int metadata_size)
+{
+  return metadata_size;
+}
+
+void * quadiron_fnt_common_init(struct ec_backend_args *args, void *backend_sohandle,
+                                int systematic)
+{
+    quadiron_fnt_descriptor *desc = NULL;
+
+    desc = (quadiron_fnt_descriptor *)malloc(sizeof(quadiron_fnt_descriptor));
+    if (NULL == desc) {
+        return NULL;
+    }
+
+    desc->k = args->uargs.k;
+    desc->m = args->uargs.m;
+    desc->w = args->uargs.w;
+
+    /* validate EC arguments */
+
+     /*
+     * ISO C forbids casting a void* to a function pointer.
+     * Since dlsym return returns a void*, we use this union to
+     * "transform" the void* to a function pointer.
+     */
+    union {
+      quadiron_fnt32_new_func quadiron_fnt32_new_funcp;
+      quadiron_fnt32_delete_func quadiron_fnt32_delete_funcp;
+      quadiron_fnt32_encode_func quadiron_fnt32_encode_funcp;
+      quadiron_fnt32_decode_func quadiron_fnt32_decode_funcp;
+      quadiron_fnt32_reconstruct_func quadiron_fnt32_reconstruct_funcp;
+      quadiron_fnt32_get_metadata_size_func quadiron_fnt32_get_metadata_size_funcp;
+      quadiron_hex_dump_func quadiron_hex_dump_funcp;
+      void *vptr;
+    } func_handle = {.vptr = NULL};
+
+    /* fill in function addresses */
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "quadiron_fnt32_new");
+    desc->fnt32_new_func = func_handle.quadiron_fnt32_new_funcp;
+    if (NULL == desc->fnt32_new_func) {
+        goto error;
+    }
+
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "quadiron_fnt32_delete");
+    desc->fnt32_delete_func = func_handle.quadiron_fnt32_delete_funcp;
+    if (NULL == desc->fnt32_delete_func) {
+      goto error;
+    }
+
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "quadiron_fnt32_encode");
+    desc->fnt32_encode_func = func_handle.quadiron_fnt32_encode_funcp;
+    if (NULL == desc->fnt32_encode_func) {
+      goto error;
+    }
+
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "quadiron_fnt32_decode");
+    desc->fnt32_decode_func = func_handle.quadiron_fnt32_decode_funcp;
+    if (NULL == desc->fnt32_decode_func) {
+      goto error;
+    }
+
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "quadiron_fnt32_reconstruct");
+    desc->fnt32_reconstruct_func = func_handle.quadiron_fnt32_reconstruct_funcp;
+    if (NULL == desc->fnt32_reconstruct_func) {
+      goto error;
+    }
+
+    func_handle.vptr = NULL;
+    func_handle.vptr = dlsym(backend_sohandle, "quadiron_fnt32_get_metadata_size");
+    desc->fnt32_get_metadata_size_func = func_handle.quadiron_fnt32_get_metadata_size_funcp;
+    if (NULL == desc->fnt32_get_metadata_size_func) {
+      goto error;
+    }
+
+    func_handle.vptr = dlsym(backend_sohandle, "quadiron_hex_dump");
+    quadiron_hex_dump = func_handle.quadiron_hex_dump_funcp;
+
+    desc->fec = desc->fnt32_new_func(args->uargs.w / 8, args->uargs.k,
+                                     args->uargs.m, systematic);
+    if (NULL == desc->fec) {
+        goto error;
+    }
+
+    return desc;
+
+error:
+    free(desc);
+
+    return NULL;
+}