Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@shuwens
shuwens committed Oct 28, 2024
1 parent 4af07f5 commit 5d38a67
Showing 1 changed file with 88 additions and 91 deletions.
179 changes: 88 additions & 91 deletions src/tests/map_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,48 +5,40 @@
#include <string>
#include <tuple>

// Basic types
using namespace std;

// Type definitions
using ObjectKey = std::string;
using ObjectKeyHash = unsigned char *; // Assuming a fixed-length hash
using ObjectKeyHash = unsigned char *;
using TargetDev = std::string;
using Lba = uint64_t;
using Length = uint32_t;

// Device types
// Device types and map entry
using TargetLbaTuple = std::tuple<TargetDev, Lba, Length>;
using DevTuple =
std::tuple<std::pair<TargetDev, uint32_t>, std::pair<TargetDev, uint32_t>,
std::pair<TargetDev, uint32_t>>;

// MapEntry definition
using MapEntry = std::tuple<TargetLbaTuple, TargetLbaTuple, TargetLbaTuple>;

// Alias for the concurrent_flat_map
// using MapType = boost::container::concurrent_flat_map<ObjectKeyHash,
// MapEntry>;

// Fixed hash length for serialization (e.g., 32 bytes for SHA-256)
constexpr size_t HASH_LENGTH = 32;

// Helper function to write a string to a binary file
// Helper function to write a single string to the file
void writeString(std::ofstream &outFile, const std::string &str)
{
size_t length = str.size();
outFile.write(reinterpret_cast<const char *>(&length), sizeof(length));
outFile.write(str.data(), length);
size_t size = str.size();
outFile.write(reinterpret_cast<const char *>(&size), sizeof(size));
outFile.write(str.data(), size);
}

// Helper function to read a string from a binary file
// Helper function to read a single string from the file
std::string readString(std::ifstream &inFile)
{
size_t length;
inFile.read(reinterpret_cast<char *>(&length), sizeof(length));
std::string str(length, '\0');
inFile.read(&str[0], length);
size_t size;
inFile.read(reinterpret_cast<char *>(&size), sizeof(size));
std::string str(size, '\0');
inFile.read(&str[0], size);
return str;
}

// Function to serialize a concurrent_flat_map to a file
// Function to write the map to a file
void writeMapToFile(const zstore_map &map, const std::string &filename)
{
std::ofstream outFile(filename, std::ios::binary);
Expand All @@ -56,33 +48,33 @@ void writeMapToFile(const zstore_map &map, const std::string &filename)
return;
}

// Write the size of the map
size_t mapSize = map.size();
size_t mapSize = 0;
map.visit_all([&mapSize](auto &x) { ++mapSize; });

// Write the map size
outFile.write(reinterpret_cast<const char *>(&mapSize), sizeof(mapSize));

// Write each key-value pair
for (const auto &[key, value] : map) {
// Serialize the key (fixed-length hash)
outFile.write(reinterpret_cast<const char *>(key), HASH_LENGTH);

// Serialize the value (MapEntry)
for (const auto &targetTuple :
{std::get<0>(value), std::get<1>(value), std::get<2>(value)}) {
// Write TargetDev
writeString(outFile, std::get<0>(targetTuple));

// Write Lba and Length
outFile.write(
reinterpret_cast<const char *>(&std::get<1>(targetTuple)),
sizeof(Lba));
outFile.write(
reinterpret_cast<const char *>(&std::get<2>(targetTuple)),
sizeof(Length));
}
}
// Iterate over each key-value pair and write them to the file
map.visit_all([&outFile](auto &x) {
// Serialize the key
outFile.write(reinterpret_cast<const char *>(x.first),
32); // Assuming 32-byte hash

// Serialize the entry
auto writeTargetLbaTuple = [&outFile](const TargetLbaTuple &tuple) {
writeString(outFile, std::get<0>(tuple)); // TargetDev
outFile.write(reinterpret_cast<const char *>(&std::get<1>(tuple)),
sizeof(Lba)); // Lba
outFile.write(reinterpret_cast<const char *>(&std::get<2>(tuple)),
sizeof(Length)); // Length
};
writeTargetLbaTuple(std::get<0>(x.second));
writeTargetLbaTuple(std::get<1>(x.second));
writeTargetLbaTuple(std::get<2>(x.second));
});
}

// Function to deserialize a file into a concurrent_flat_map
// Function to read the map from a file
zstore_map readMapFromFile(const std::string &filename)
{
std::ifstream inFile(filename, std::ios::binary);
Expand All @@ -93,73 +85,78 @@ zstore_map readMapFromFile(const std::string &filename)
}

zstore_map map;

// Read the size of the map
size_t mapSize;
inFile.read(reinterpret_cast<char *>(&mapSize), sizeof(mapSize));

// Read each key-value pair
for (size_t i = 0; i < mapSize; ++i) {
// Deserialize the key (fixed-length hash)
unsigned char key[HASH_LENGTH];
inFile.read(reinterpret_cast<char *>(key), HASH_LENGTH);

// Deserialize the value (MapEntry)
TargetLbaTuple t1, t2, t3;
for (auto &targetTuple : {std::ref(t1), std::ref(t2), std::ref(t3)}) {
// Read TargetDev
std::get<0>(targetTuple.get()) = readString(inFile);

// Read Lba and Length
inFile.read(
reinterpret_cast<char *>(&std::get<1>(targetTuple.get())),
sizeof(Lba));
inFile.read(
reinterpret_cast<char *>(&std::get<2>(targetTuple.get())),
sizeof(Length));
}

// Insert the key-value pair into the map
map.emplace(key, std::make_tuple(t1, t2, t3));
// Deserialize the key
unsigned char *key = new unsigned char[32];
inFile.read(reinterpret_cast<char *>(key), 32);

// Deserialize the entry
auto readTargetLbaTuple = [&inFile]() -> TargetLbaTuple {
std::string dev = readString(inFile);
Lba lba;
Length length;
inFile.read(reinterpret_cast<char *>(&lba), sizeof(Lba));
inFile.read(reinterpret_cast<char *>(&length), sizeof(Length));
return std::make_tuple(dev, lba, length);
};

MapEntry entry = std::make_tuple(
readTargetLbaTuple(), readTargetLbaTuple(), readTargetLbaTuple());
map.emplace(key, entry);
}

return map;
}

int main()
{
// Sample map to serialize
zstore_map mMap;

// Example data (assuming the hash is 32 bytes for the key)
unsigned char key1[HASH_LENGTH] = "0123456789abcdef0123456789abcde";
MapEntry entry1 = {std::make_tuple("DeviceA", 12345, 64),
std::make_tuple("DeviceB", 67890, 128),
std::make_tuple("DeviceC", 111213, 256)};
// Sample data
unsigned char key1[32] = "sample_key_1";
unsigned char key2[32] = "sample_key_2";
MapEntry entry1 = {std::make_tuple("device1", 1000, 256),
std::make_tuple("device2", 2000, 512),
std::make_tuple("device3", 3000, 128)};
MapEntry entry2 = {std::make_tuple("device1", 1500, 128),
std::make_tuple("device2", 2500, 256),
std::make_tuple("device3", 3500, 64)};
mMap.emplace(key1, entry1);
mMap.emplace(key2, entry2);

// Write the map to a file
std::string filename = "map_data.bin";
std::string filename = "mMap_data.bin";
writeMapToFile(mMap, filename);

// Read the map back from the file
zstore_map loadedMap = readMapFromFile(filename);

// Print the loaded map
for (const auto &[key, value] : loadedMap) {
std::cout << "Key: "
<< std::string(reinterpret_cast<const char *>(key),
HASH_LENGTH)
<< "\n";
std::cout << "Value: ";
for (const auto &targetTuple :
{std::get<0>(value), std::get<1>(value), std::get<2>(value)}) {
std::cout << "(" << std::get<0>(targetTuple) << ", "
<< std::get<1>(targetTuple) << ", "
<< std::get<2>(targetTuple) << ") ";
}
std::cout << std::endl;
}
loadedMap.visit_all([](auto &x) {
std::cout << "Key: " << x.first << "\n";

// for (size_t i = 0; i < 3; ++i) {
// // auto &tuple = x.second;
// std::cout << " Device: " << std::get<0>(x.second)
// << ", LBA: " << std::get<1>(x.second)
// << ", Length: " << std::get<2>(x.second) << "\n";
// }
});

// loadedMap.visit_all([](ObjectKeyHash key, const MapEntry &entry) {
// std::cout << "Key: " << std::string(reinterpret_cast<char
// *>(key), 32)
// << "\n";
// for (size_t i = 0; i < 3; ++i) {
// const auto &tuple = std::get<i>(entry);
// std::cout << " Device: " << std::get<0>(tuple)
// << ", LBA: " << std::get<1>(tuple)
// << ", Length: " << std::get<2>(tuple) << "\n";
// }
// });

return 0;
}

0 comments on commit 5d38a67

Please sign in to comment.