Systematic FNT

benchmark/benchmark.cpp

@@ -174,7 +174,12 @@ int Benchmark<T>::init()
         fec = new quadiron::fec::RsNf4<T>(word_size, k, m, pkt_size);
         break;
     case EC_TYPE_RS_FNT:
-        fec = new quadiron::fec::RsFnt<T>(word_size, k, m, pkt_size);
+        fec = new quadiron::fec::RsFnt<T>(
+            quadiron::fec::FecType::NON_SYSTEMATIC, word_size, k, m, pkt_size);
+        break;
+    case EC_TYPE_RS_FNT_SYS:
+        fec = new quadiron::fec::RsFnt<T>(
+            quadiron::fec::FecType::SYSTEMATIC, word_size, k, m, pkt_size);
         break;
     default:
         return ERR_FEC_TYPE_NOT_SUPPORTED;
@@ -282,12 +287,13 @@ int Benchmark<T>::check_params()
     if (sizeof(T) < word_size) {
         return ERR_COMPT_WORD_SIZE_T;
     }
-    if (fec_type == EC_TYPE_RS_FNT || fec_type == EC_TYPE_RS_GFP_FFT) {
+    if (fec_type == EC_TYPE_RS_FNT || fec_type == EC_TYPE_RS_FNT_SYS
+        || fec_type == EC_TYPE_RS_GFP_FFT) {
         if (sizeof(T) <= word_size) {
             return ERR_COMPT_WORD_SIZE_T;
         }
     }
-    if (fec_type == EC_TYPE_RS_FNT) {
+    if (fec_type == EC_TYPE_RS_FNT || fec_type == EC_TYPE_RS_FNT_SYS) {
         if (word_size > 2)
             return ERR_WORD_SIZE;
     }
@@ -639,6 +645,7 @@ void xusage()
               << ec_desc.at(EC_TYPE_RS_GF2N_FFT_ADD) << '\n'
               << "\t\t\trs-gfp-fft: " << ec_desc.at(EC_TYPE_RS_GFP_FFT) << '\n'
               << "\t\t\trs-fnt: " << ec_desc.at(EC_TYPE_RS_FNT) << '\n'
+              << "\t\t\trs-fnt-sys: " << ec_desc.at(EC_TYPE_RS_FNT_SYS) << '\n'
               << "\t\t\trs-nf4: " << ec_desc.at(EC_TYPE_RS_NF4) << '\n'
               << "\t\t\tall: All available Reed-solomon codes\n"
               << "\t-s \tScenario for benchmark, either\n"
@@ -788,6 +795,7 @@ int main(int argc, char** argv)
 
     // Currently support operating on packet:RS_FNT
     if (params->fec_type != EC_TYPE_RS_FNT
+        && params->fec_type != EC_TYPE_RS_FNT_SYS
         && params->fec_type != EC_TYPE_RS_NF4) {
         params->operation_on_packet = false;
     }

benchmark/benchmark.h

@@ -47,6 +47,7 @@
 enum ec_type {
     EC_TYPE_ALL = 0,
     EC_TYPE_RS_FNT,
+    EC_TYPE_RS_FNT_SYS,
     EC_TYPE_RS_NF4,
     EC_TYPE_RS_GFP_FFT,
     EC_TYPE_RS_GF2N_FFT_ADD,
@@ -66,7 +67,10 @@ const std::map<ec_type, std::string> ec_desc = {
     {EC_TYPE_RS_GF2N_FFT_ADD,
      "Reed-solomon codes over GF(2^n) using additive FFT"},
     {EC_TYPE_RS_GFP_FFT, "Reed-solomon codes over GF(p) using FFT"},
-    {EC_TYPE_RS_FNT, "Reed-solomon codes over GF(p = Fermat number) using FFT"},
+    {EC_TYPE_RS_FNT,
+     "Non-systematic Reed-solomon codes over GF(p = Fermat number) using FFT"},
+    {EC_TYPE_RS_FNT_SYS,
+     "Systematic Reed-solomon codes over GF(p = Fermat number) using FFT"},
     {EC_TYPE_RS_NF4,
      "Reed-solomon codes over GF(65537) using FFT on pack of codewords"},
 };
@@ -80,6 +84,7 @@ const std::map<ec_type, std::string> ec_desc_short = {
     {EC_TYPE_RS_GF2N_FFT_ADD, "rs-gf2n-fft-add"},
     {EC_TYPE_RS_GFP_FFT, "rs-gfp-fft"},
     {EC_TYPE_RS_FNT, "rs-fnt"},
+    {EC_TYPE_RS_FNT_SYS, "rs-fnt-sys"},
     {EC_TYPE_RS_NF4, "rs-nf4"},
 };
 
@@ -120,6 +125,7 @@ const std::map<std::string, ec_type> fec_type_map = {
     {"rs-gf2n-fft-add", EC_TYPE_RS_GF2N_FFT_ADD},
     {"rs-gfp-fft", EC_TYPE_RS_GFP_FFT},
     {"rs-fnt", EC_TYPE_RS_FNT},
+    {"rs-fnt-sys", EC_TYPE_RS_FNT_SYS},
     {"rs-nf4", EC_TYPE_RS_NF4},
 };
 

scripts/benchmark.sh

@@ -44,13 +44,14 @@ fi
 
 chunk_size=51200
 # for rs-fnt with different packet sizes
-ec_type=rs-fnt
 word_size=2
-for k in 16; do
-  for n in 256 1024; do
-    m=$((n-k))
-    for pkt_size in 256 512 1024; do
-      ${bin} -e ${ec_type} -w ${word_size} -k ${k} -m ${m} -c ${chunk_size} -s ${sce_type} -g ${threads_nb} -f ${show_type} -p ${pkt_size}
+for ec_type in rs-fnt rs-fnt-sys; do
+  for k in 16; do
+    for n in 256 1024; do
+      m=$((n-k))
+      for pkt_size in 256 512 1024; do
+        ${bin} -e ${ec_type} -w ${word_size} -k ${k} -m ${m} -c ${chunk_size} -s ${sce_type} -g ${threads_nb} -f ${show_type} -p ${pkt_size} -n ${samples_nb}
+      done
     done
   done
 done

scripts/test_ec.sh

@@ -34,7 +34,7 @@
 #valgrind="gdb --args"
 
 bin=$1
-bs=50K
+bs=51200
 
 if [ -z $1 ]
 then
@@ -83,7 +83,7 @@ do_test()
         md5sum foo.c${i} > foo.c${i}.md5sum.1
     done
 
-    if [ "${type}" = "type_2" ]
+    if [ "${type}" = "NON_SYSTEMATIC" ]
     then
         #remove all data
         shopt -s extglob
@@ -98,7 +98,7 @@ do_test()
     j=0
     for i in $data_loss
     do
-        if [ "${type}" = "type_2" ]
+        if [ "${type}" = "NON_SYSTEMATIC" ]
         then
             mv foo.c${j} foo.c${j}.1
             mv foo.c${j}.props foo.c${j}.props.1
@@ -110,7 +110,7 @@ do_test()
 
     for i in $coding_loss
     do
-        if [ "${type}" = "type_2" ]
+        if [ "${type}" = "NON_SYSTEMATIC" ]
         then
             mv foo.c${j} foo.c${j}.1
             mv foo.c${j}.props foo.c${j}.props.1
@@ -147,7 +147,7 @@ do_test()
     echo
 }
 
-for i in rs-fnt_1 rs-fnt_2 rs-nf4_2 rs-nf4_4 rs-nf4_8 rs-gfp-fft_1 rs-gfp-fft_2 rs-gfp-fft_4 rs-gf2n-fft_1 rs-gf2n-fft_2 rs-gf2n-fft_4 rs-gf2n-fft_8 rs-gf2n-fft-add_1 rs-gf2n-fft-add_2 rs-gf2n-fft-add_4 rs-gf2n-fft-add_8 rs-gf2n-v_1 rs-gf2n-v_2 rs-gf2n-c_1 rs-gf2n-c_2 rs-gf2n-v_4 rs-gf2n-v_8 rs-gf2n-v_16 rs-gf2n-c_4 rs-gf2n-c_8 rs-gf2n-c_16
+for i in rs-fnt_1 rs-fnt_2 rs-fnt-sys_1 rs-fnt-sys_2 rs-nf4_2 rs-nf4_4 rs-nf4_8 rs-gfp-fft_1 rs-gfp-fft_2 rs-gfp-fft_4 rs-gf2n-fft_1 rs-gf2n-fft_2 rs-gf2n-fft_4 rs-gf2n-fft_8 rs-gf2n-fft-add_1 rs-gf2n-fft-add_2 rs-gf2n-fft-add_4 rs-gf2n-fft-add_8 rs-gf2n-v_1 rs-gf2n-v_2 rs-gf2n-c_1 rs-gf2n-c_2 rs-gf2n-v_4 rs-gf2n-v_8 rs-gf2n-v_16 rs-gf2n-c_4 rs-gf2n-c_8 rs-gf2n-c_16
 do
     fec_type=$(echo $i|cut -d_ -f1)
     word_size=$(echo $i|cut -d_ -f2)

src/fec_base.h

@@ -239,6 +239,8 @@ class FecCode {
     std::unique_ptr<vec::Vector<T>> inv_r_powers = nullptr;
     // This vector MUST be initialized by derived Class using multiplicative FFT
     std::unique_ptr<vec::Vector<T>> r_powers = nullptr;
+    // buffers for intermediate symbols used for systematic FNT
+    std::unique_ptr<vec::Buffers<T>> dec_inter_codeword;
 
     // pure abstract methods that will be defined in derived class
     virtual void check_params() = 0;
@@ -915,7 +917,6 @@ bool FecCode<T>::decode_packet(
         if (fragment_index == n_data)
             return true;
     }
-    fragments_ids.sort();
 
     vec::Vector<T> avail_parity_ids(*(this->gf), n_data - avail_data_nb);
 
@@ -938,6 +939,7 @@ bool FecCode<T>::decode_packet(
         if (fragment_index < n_data)
             return false;
     }
+    fragments_ids.sort();
 
     decode_build();
 
@@ -1041,6 +1043,13 @@ void FecCode<T>::decode(
 
     // Lagrange interpolation
     decode_apply(context, output, words);
+
+    if (type == FecType::SYSTEMATIC) {
+        this->fft->fft(*dec_inter_codeword, output);
+        for (unsigned i = 0; i < this->n_data; i++) {
+            output.copy(i, dec_inter_codeword->get(i));
+        }
+    }
 }
 
 /* Prepare for decoding
@@ -1060,8 +1069,14 @@ void FecCode<T>::decode_prepare(
 
     T thres = (this->gf->card() - 1);
     for (unsigned i = 0; i < this->n_data; i++) {
-        const int frag_id = fragments_ids.get(i);
+        unsigned frag_id = fragments_ids.get(i);
+        if (type == FecType::SYSTEMATIC && frag_id < this->n_data) {
+            continue;
+        }
         T* chunk = words.get(i);
+        if (type == FecType::SYSTEMATIC) {
+            frag_id -= this->n_data;
+        }
         // loop over marked symbols
         for (auto const& data : props[frag_id].get_map()) {
             off_t loc_offset = data.first;

src/fec_rs_fnt.h

@@ -50,18 +50,24 @@ namespace fec {
  */
 template <typename T>
 class RsFnt : public FecCode<T> {
+  private:
+    // buffers for intermediate symbols used for systematic FNT
+    std::unique_ptr<vec::Buffers<T>> inter_words;
+    // buffers for suffix symbols of codewords used for systematic FNT
+    std::unique_ptr<vec::Buffers<T>> suffix_words;
+    // received fragments id for encoding of systematic FNT
+    std::unique_ptr<vec::Vector<T>> enc_frag_ids;
+    // decoding context used in encoding of systematic FNT
+    std::unique_ptr<DecodeContext<T>> enc_context;
+
   public:
     RsFnt(
+        FecType type,
         unsigned word_size,
         unsigned n_data,
         unsigned n_parities,
         size_t pkt_size = 8)
-        : FecCode<T>(
-              FecType::NON_SYSTEMATIC,
-              word_size,
-              n_data,
-              n_parities,
-              pkt_size)
+        : FecCode<T>(type, word_size, n_data, n_parities, pkt_size)
     {
         this->fec_init();
     }
@@ -94,34 +100,55 @@ class RsFnt : public FecCode<T> {
         this->r = this->gf->get_nth_root(this->n);
 
         int m = arith::get_smallest_power_of_2<int>(this->n_data);
-        this->fft = std::unique_ptr<fft::Radix2<T>>(
-            new fft::Radix2<T>(*(this->gf), this->n, m, this->pkt_size));
+        this->fft = std::make_unique<fft::Radix2<T>>(
+            *(this->gf), this->n, m, this->pkt_size);
 
         unsigned len_2k = this->gf->get_code_len_high_compo(2 * this->n_data);
-        this->fft_2k = std::unique_ptr<fft::Radix2<T>>(
-            new fft::Radix2<T>(*(this->gf), len_2k, len_2k, this->pkt_size));
+        this->fft_2k = std::make_unique<fft::Radix2<T>>(
+            *(this->gf), len_2k, len_2k, this->pkt_size);
     }
 
     inline void init_others() override
     {
         // vector stores r^{-i} for i = 0, ... , k
         T inv_r = this->gf->inv(this->r);
-        this->inv_r_powers = std::unique_ptr<vec::Vector<T>>(
-            new vec::Vector<T>(*(this->gf), this->n_data + 1));
+        this->inv_r_powers =
+            std::make_unique<vec::Vector<T>>(*(this->gf), this->n_data + 1);
         for (unsigned i = 0; i <= this->n_data; i++)
             this->inv_r_powers->set(i, this->gf->exp(inv_r, i));
 
         // vector stores r^{i} for i = 0, ... , n-1
-        this->r_powers = std::unique_ptr<vec::Vector<T>>(
-            new vec::Vector<T>(*(this->gf), this->n));
+        this->r_powers = std::make_unique<vec::Vector<T>>(*(this->gf), this->n);
         for (unsigned i = 0; i < this->n; i++) {
             this->r_powers->set(i, this->gf->exp(this->r, i));
         }
+
+        if (this->type == FecType::SYSTEMATIC) {
+            // for encoding
+            enc_frag_ids =
+                std::make_unique<vec::Vector<T>>(*(this->gf), this->n_data);
+            // ids of received fragments, from 0 to codelen-1
+            for (unsigned i = 0; i < this->n_data; i++) {
+                enc_frag_ids->set(i, i);
+            }
+
+            inter_words =
+                std::make_unique<vec::Buffers<T>>(this->n_data, this->pkt_size);
+            suffix_words = std::make_unique<vec::Buffers<T>>(
+                this->n - this->n_data - this->n_outputs, this->pkt_size);
+
+            enc_context = this->init_context_dec(
+                *enc_frag_ids, this->pkt_size, inter_words.get());
+
+            // for decoding
+            this->dec_inter_codeword =
+                std::make_unique<vec::Buffers<T>>(this->n, this->pkt_size);
+        }
     }
 
     int get_n_outputs() override
     {
-        return this->n;
+        return (this->type == FecType::SYSTEMATIC) ? this->n_parities : this->n;
     }
 
     /**
@@ -150,7 +177,7 @@ class RsFnt : public FecCode<T> {
         // max_value = 2^x
         T thres = this->gf->card() - 1;
         // check for out of range value in output
-        for (unsigned i = 0; i < this->code_len; i++) {
+        for (unsigned i = 0; i < this->n_outputs; i++) {
             if (output.get(i) & thres) {
                 props[i].add(offset, OOR_MARK);
                 output.set(i, 0);
@@ -164,7 +191,14 @@ class RsFnt : public FecCode<T> {
         off_t offset,
         vec::Buffers<T>& words) override
     {
-        this->fft->fft(output, words);
+        if (this->type == FecType::SYSTEMATIC) {
+            this->decode_apply(*enc_context, *inter_words, words);
+            vec::Buffers<T> _tmp(words, output);
+            vec::Buffers<T> _output(_tmp, *suffix_words);
+            this->fft->fft(_output, *inter_words);
+        } else {
+            this->fft->fft(output, words);
+        }
         encode_post_process(output, props, offset);
     }
 
@@ -176,7 +210,7 @@ class RsFnt : public FecCode<T> {
         // check for out of range value in output
         unsigned size = output.get_size();
         T thres = (this->gf->card() - 1);
-        for (unsigned i = 0; i < this->code_len; ++i) {
+        for (unsigned i = 0; i < this->n_outputs; ++i) {
             T* chunk = output.get(i);
             for (unsigned j = 0; j < size; ++j) {
                 if (chunk[j] & thres) {
@@ -186,11 +220,7 @@ class RsFnt : public FecCode<T> {
         }
     }
 
-    void decode_add_data(int fragment_index, int row) override
-    {
-        // not applicable
-        assert(false);
-    }
+    void decode_add_data(int fragment_index, int row) override {}
 
     void decode_add_parities(int fragment_index, int row) override
     {

src/fec_vectorisation.cpp

@@ -50,7 +50,7 @@ void RsFnt<uint16_t>::encode_post_process(
 {
     size_t size = this->pkt_size;
     uint16_t threshold = this->gf->card_minus_one();
-    unsigned code_len = this->code_len;
+    unsigned code_len = this->n_outputs;
 
     // number of elements per vector register
     unsigned vec_size = ALIGN_SIZE / sizeof(uint16_t);
@@ -67,7 +67,7 @@ void RsFnt<uint16_t>::encode_post_process(
             uint16_t* chunk = output.get(i);
             for (size_t j = vecs_nb * vec_size; j < size; ++j) {
                 if (chunk[j] == threshold) {
-                    props[i].add(offset + j, 1);
+                    props[i].add(offset + j, OOR_MARK);
                 }
             }
         }
@@ -82,7 +82,7 @@ void RsFnt<uint32_t>::encode_post_process(
 {
     const size_t size = this->pkt_size;
     const uint32_t threshold = this->gf->card_minus_one();
-    const unsigned code_len = this->code_len;
+    const unsigned code_len = this->n_outputs;
 
     // number of elements per vector register
     const unsigned vec_size = ALIGN_SIZE / sizeof(uint32_t);
@@ -99,7 +99,7 @@ void RsFnt<uint32_t>::encode_post_process(
             uint32_t* chunk = output.get(i);
             for (size_t j = vecs_nb * vec_size; j < size; ++j) {
                 if (chunk[j] == threshold) {
-                    props[i].add(offset + j, 1);
+                    props[i].add(offset + j, OOR_MARK);
                 }
             }
         }