LIC compress script

examples/20_encode_lic/CMakeLists.txt

@@ -0,0 +1,6 @@
+recoil_example_add_executable(
+        encode_lic encode_lic.cpp
+        ${RECOIL_EXAMPLES_COMMON_SOURCE_DIR}/cdf_utils.cpp
+        ${RECOIL_EXAMPLES_COMMON_SOURCE_DIR}/file.cpp
+        ${RECOIL_EXAMPLES_COMMON_SOURCE_DIR}/profiling.cpp
+)

examples/20_encode_lic/encode_lic.cpp

@@ -0,0 +1,66 @@
+#include "cdf_utils.h"
+#include "file.h"
+#include "profiling.h"
+#include "params.h"
+
+#include "recoil/symbol_lookup/cdf_lut_pool.h"
+#include "recoil/split/rans_split_encoder.h"
+#include "recoil/split/bitstream_generation/splits_metadata_encoder.h"
+
+#include <iostream>
+#include <vector>
+#include <cstdint>
+#include <array>
+
+using namespace Recoil;
+using namespace Recoil::Examples;
+
+using CdfType = uint16_t;
+using ValueType = uint16_t;
+
+int main(int argc, const char **argv) {
+    if (argc != 6) {
+        std::cerr << "Usage: " << argv[0] << " [img_data.txt] [cdf_indices.txt] [cdf.txt] [n_splits] [output_bin]" << std::endl;
+        return 1;
+    }
+
+    auto imgSymbols = readVectorFromTextFile<ValueType>(argv[1]);
+    auto rawCdfIndices = readVectorFromTextFile<ValueType>(argv[2]);
+
+    CdfLutPool<CdfType, ValueType, ProbBits, LutGranularity> pool(40000, (1 << (ProbBits - LutGranularity + 1)) * 65); // Just give it enough memory
+
+    std::vector<CdfLutOffsetType> cdfIndicesMap, lutIndicesMap;
+    std::ifstream cdfTxt(argv[3]);
+    if (!cdfTxt.good()) [[unlikely]] throw std::runtime_error("Error reading cdf text file");
+    while (!cdfTxt.eof()) {
+        int count;
+        cdfTxt >> count;
+
+        auto cdfVec = readVectorFromTextStream<CdfType>(cdfTxt, count);
+        auto lutVec = LutBuilder<CdfType, ValueType, ProbBits, LutGranularity>::buildLut(std::span{cdfVec});
+
+        auto cdfIndex = pool.insertCdf(cdfVec);
+        auto lutIndex = pool.insertLut(lutVec);
+
+        cdfIndicesMap.push_back(cdfIndex);
+        lutIndicesMap.push_back(lutIndex);
+    }
+    std::vector<CdfLutOffsetType> cdfIndices(rawCdfIndices.size());
+    std::transform(rawCdfIndices.begin(), rawCdfIndices.end(), cdfIndices.begin(), [&cdfIndicesMap](auto v) {
+        return cdfIndicesMap[v];
+    });
+
+    auto nSplits = std::stoull(argv[4]);
+    auto outputPrefix = std::string(argv[5]);
+
+    RansSplitEncoder enc((std::array<Rans32<ValueType, ProbBits>, NInterleaved>{}), pool);
+    enc.getEncoder().buffer(imgSymbols, cdfIndices);
+    auto result = enc.flushSplits(nSplits);
+
+    SplitsMetadataEncoder metadataEnc(result.first, result.second);
+    auto bitstream = metadataEnc.combine();
+
+    writeSpanToFile(outputPrefix, std::span{bitstream});
+
+    return 0;
+}

examples/CMakeLists.txt

@@ -26,6 +26,8 @@ foreach(EXAMPLE
         15_decode_textfile_split
         16_decode_textfile_split_cuda
         17_decode_textfile_symbolsplit
-        18_decode_textfile_symbolsplit_cuda)
+        18_decode_textfile_symbolsplit_cuda
+
+        20_encode_lic)
     add_subdirectory(${EXAMPLE})
 endforeach()

examples/common/file.h

@@ -37,6 +37,31 @@ namespace Recoil::Examples {
         return vec;
     }
 
+    template<typename T>
+    std::vector<T> readVectorFromTextStream(std::ifstream &fin, size_t count) {
+        std::vector<T> result(count);
+        for (auto i = 0; i < count; i++) fin >> result[i];
+        return result;
+    }
+
+    template<typename T>
+    std::vector<T> readVectorFromTextStream(std::ifstream &fin) {
+        std::vector<T> result;
+        while (!fin.eof()) {
+            T temp;
+            fin >> temp;
+            result.push_back(temp);
+        }
+        return result;
+    }
+
+    template<typename T>
+    std::vector<T> readVectorFromTextFile(const std::string &name) {
+        std::ifstream fin(name);
+        if (!fin.good()) [[unlikely]] throw std::runtime_error("Error reading vector text file");
+        return readVectorFromTextStream<T>(fin);
+    }
+
     template<std::unsigned_integral ValueType>
     std::vector<ValueType> stringToSymbols(const std::string &str) {
         std::vector<ValueType> symbols;