Skip to content

Commit

Permalink
Make serialization handle mappish types in addition to just std::map (#…
Browse files Browse the repository at this point in the history 
…9368)

Summary: Pull Request resolved: #9368

Differential Revision: D55771042
  • Loading branch information
@facebook-github-bot
Zhenyuan Zhao authored and facebook-github-bot committed Apr 11, 2024
1 parent d4f8d85 commit 16da397
Show file tree
Hide file tree
Showing 2 changed files with 81 additions and 46 deletions.
104 changes: 58 additions & 46 deletions velox/common/serialization/Serializable.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include "folly/json.h"
#include "velox/common/base/Exceptions.h"
#include "velox/common/serialization/DeserializationRegistry.h"
#include "velox/core/Metaprogramming.h"

namespace facebook {
namespace velox {
Expand Down Expand Up @@ -95,19 +96,27 @@ class ISerializable {

template <
typename T,
typename = std::enable_if_t<has_serialize_type<T>::value>>
static folly::dynamic serialize(T& obj) {
std::enable_if_t<
has_serialize_type<T>::value ||
std::is_base_of_v<ISerializable, T>>* = nullptr>
static folly::dynamic serialize(const T& obj) {
return obj.serialize();
}

template <
typename T,
typename = std::enable_if_t<
is_any_of<T, int64_t, double, std::string, bool>::value>>
std::enable_if_t<std::is_same_v<T, std::string>>* = nullptr>
static folly::dynamic serialize(const T& val) {
return val;
}

template <
typename T,
typename = std::enable_if_t<is_any_of<T, int64_t, double, bool>::value>>
static folly::dynamic serialize(T val) {
return val;
}

static folly::dynamic serialize(int32_t val) {
return folly::dynamic{(int64_t)val};
}
Expand All @@ -132,7 +141,7 @@ class ISerializable {
}

template <
class T,
typename T,
std::enable_if_t<
std::is_same_v<T, folly::Optional<typename T::value_type>>>>
static folly::dynamic serialize(const folly::Optional<T>& val) {
Expand All @@ -143,8 +152,8 @@ class ISerializable {
return serialize(val.value());
}

template <class K, class U>
static folly::dynamic serialize(const std::map<K, U>& map) {
template <typename T, std::enable_if_t<util::is_mappish<T>::value>* = nullptr>
static folly::dynamic serialize(const T& map) {
folly::dynamic keys = folly::dynamic::array;
folly::dynamic values = folly::dynamic::array;
for (auto& pair : map) {
Expand All @@ -160,11 +169,11 @@ class ISerializable {
}

template <
class T,
typename T,
typename = std::enable_if_t<std::is_base_of_v<ISerializable, T>>>
static std::shared_ptr<const T> deserialize(
const folly::dynamic& obj,
void* context) {
void* context = nullptr) {
VELOX_USER_CHECK(obj.isObject());
// use the key to lookup creator and call it.
// creator generally be a static method in the class.
Expand Down Expand Up @@ -192,20 +201,13 @@ class ISerializable {
return std::dynamic_pointer_cast<const T>(registry.Create(name, obj));
}

template <
class T,
typename = std::enable_if_t<std::is_base_of_v<ISerializable, T>>>
static std::shared_ptr<const T> deserialize(const folly::dynamic& obj) {
return deserialize<T>(obj, nullptr);
}

template <
typename T,
typename = std::enable_if_t<has_static_obj_create_type<T>::value>>
using createReturnType = decltype(T::create(std::declval<folly::dynamic>()));

template <
class T,
typename T,
typename = std::enable_if_t<has_static_obj_create_type<T>::value>>
static createReturnType<T> deserialize(
const folly::dynamic& obj,
Expand All @@ -214,7 +216,7 @@ class ISerializable {
}

template <
class T,
typename T,
typename =
std::enable_if_t<std::is_integral_v<T> && !std::is_same_v<T, bool>>>
static T deserialize(const folly::dynamic& obj, void* context = nullptr) {
Expand All @@ -224,20 +226,20 @@ class ISerializable {
return (T)raw;
}

template <class T, typename = std::enable_if_t<std::is_same_v<T, bool>>>
template <typename T, typename = std::enable_if_t<std::is_same_v<T, bool>>>
static bool deserialize(const folly::dynamic& obj, void* context = nullptr) {
return obj.asBool();
}

template <class T, typename = std::enable_if_t<std::is_same_v<T, double>>>
template <typename T, typename = std::enable_if_t<std::is_same_v<T, double>>>
static double deserialize(
const folly::dynamic& obj,
void* context = nullptr) {
return obj.asDouble();
}

template <
class T,
typename T,
typename = std::enable_if_t<std::is_same_v<T, std::string>>>
static std::string deserialize(
const folly::dynamic& obj,
Expand All @@ -246,7 +248,7 @@ class ISerializable {
}

template <
class T,
typename T,
typename = std::enable_if_t<
std::is_same_v<T, folly::Optional<typename T::value_type>>>>
static folly::Optional<
Expand All @@ -268,13 +270,13 @@ class ISerializable {
using deserializeType =
decltype(ISerializable::deserialize<T>(std::declval<folly::dynamic>()));

template <class T, typename = std::enable_if_t<is_vector_type<T>::value>>
template <typename T, std::enable_if_t<is_vector_type<T>::value>* = nullptr>
static auto deserialize(
const folly::dynamic& array,
void* context = nullptr) {
using deserializeValType =
decltype(ISerializable::deserialize<typename T::value_type>(
std::declval<folly::dynamic>(), context));
std::declval<folly::dynamic>()));

VELOX_USER_CHECK(array.isArray());
std::vector<deserializeValType> exprs;
Expand All @@ -286,34 +288,44 @@ class ISerializable {
}

template <
class T,
typename = std::enable_if_t<std::is_same_v<
typename T,
std::enable_if_t<std::is_same_v<
T,
std::map<typename T::key_type, typename T::mapped_type>>>>
static std::map<
decltype(ISerializable::deserialize<typename T::key_type>(
std::declval<folly::dynamic>())),
decltype(ISerializable::deserialize<typename T::mapped_type>(
std::declval<folly::dynamic>()))>
deserialize(const folly::dynamic& obj, void* context = nullptr) {
using deserializeKeyType =
decltype(ISerializable::deserialize<typename T::key_type>(
std::declval<folly::dynamic>()));

using deserializeMappedType =
decltype(ISerializable::deserialize<typename T::mapped_type>(
std::declval<folly::dynamic>()));
std::map<typename T::key_type, typename T::mapped_type>>>* = nullptr>
static auto deserialize(const folly::dynamic& obj, void* context = nullptr) {
return deserialize<std::map, typename T::key_type, typename T::mapped_type>(
obj, context);
}

std::map<deserializeKeyType, deserializeMappedType> map;
template <
template <typename, typename, typename...>
typename TMap,
typename TKey,
typename TMapped,
typename... TArgs,
typename = std::enable_if_t<
util::is_mappish<TMap<TKey, TMapped, TArgs...>>::value &&
std::is_same_v<
typename TMap<TKey, TMapped, TArgs...>::key_type,
TKey> &&
std::is_same_v<
typename TMap<TKey, TMapped, TArgs...>::mapped_type,
TMapped>>>
static auto deserialize(const folly::dynamic& obj, void* context = nullptr) {
using deserializeKeyType = decltype(ISerializable::deserialize<TKey>(
std::declval<folly::dynamic>()));

using deserializeMappedType = decltype(ISerializable::deserialize<TMapped>(
std::declval<folly::dynamic>()));

TMap<deserializeKeyType, deserializeMappedType, TArgs...> map;
const folly::dynamic& keys = obj["keys"];
const folly::dynamic& values = obj["values"];
VELOX_USER_CHECK(keys.isArray() && values.isArray());
VELOX_USER_CHECK_EQ(keys.size(), values.size());
for (size_t idx = 0; idx < keys.size(); ++idx) {
auto first =
ISerializable::deserialize<typename T::key_type>(keys[idx], context);
auto second = ISerializable::deserialize<typename T::mapped_type>(
values[idx], context);
auto first = ISerializable::deserialize<TKey>(keys[idx], context);
auto second = ISerializable::deserialize<TMapped>(values[idx], context);
map.insert({first, second});
}
return map;
Expand All @@ -323,7 +335,7 @@ class ISerializable {

private:
template <
class T,
typename T,
typename = std::enable_if_t<std::is_base_of_v<ISerializable, T>>>
static auto deserializeAsUniquePtr(const folly::dynamic& obj) {
auto name = obj["name"].asString();
Expand Down
23 changes: 23 additions & 0 deletions velox/common/serialization/tests/SerializableTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

#include "velox/common/serialization/Serializable.h"
#include <gtest/gtest.h>
#include "folly/container/F14Map.h"
#include "folly/json.h"

using namespace ::facebook::velox;
Expand Down Expand Up @@ -158,4 +159,26 @@ TEST(SerializableTest, context) {
}
}

template <
template <typename, typename, typename...>
typename TMap,
typename TKey,
typename TMapped,
typename TIt,
typename... TArgs>
void testMap(TIt first, TIt last) {
TMap<TKey, TMapped, TArgs...> map{first, last};
auto serialized = ISerializable::serialize(map);
auto copy = ISerializable::deserialize<TMap, TKey, TMapped>(serialized);
ASSERT_EQ(map, copy);
}

TEST(SerializableTest, map) {
std::vector<std::pair<int32_t, std::string>> vals{
{1, "a"}, {2, "b"}, {3, "c"}};
testMap<std::map, int32_t, std::string>(vals.begin(), vals.end());
testMap<std::unordered_map, int32_t, std::string>(vals.begin(), vals.end());
testMap<folly::F14FastMap, int32_t, std::string>(vals.begin(), vals.end());
}

} // namespace

0 comments on commit 16da397

Please sign in to comment.