From d0260f730d1e81a8730eb9126a839e3d3bc63ff3 Mon Sep 17 00:00:00 2001 From: Hoyt Koepke Date: Sat, 21 Dec 2019 16:26:44 -0700 Subject: [PATCH 1/5] Working prototype of alternate model management system. This class proposes a new method for managing extensions. It is meant to replace the bulky and unweildy macro system in use currently. The new code is contained in model_server_v2. --- src/CMakeLists.txt | 3 +- src/model_server/lib/variant.hpp | 8 +- src/model_server_v2/CMakeLists.txt | 17 + src/model_server_v2/demo.cpp | 105 +++++ src/model_server_v2/method_parameters.hpp | 112 +++++ src/model_server_v2/method_registry.hpp | 137 ++++++ src/model_server_v2/method_wrapper.hpp | 398 ++++++++++++++++++ src/model_server_v2/model_base.cpp | 17 + src/model_server_v2/model_base.hpp | 143 +++++++ src/model_server_v2/model_server.cpp | 67 +++ src/model_server_v2/model_server.hpp | 288 +++++++++++++ src/model_server_v2/registration.hpp | 68 +++ .../annotation/object_detection.hpp | 2 +- 13 files changed, 1362 insertions(+), 3 deletions(-) create mode 100644 src/model_server_v2/CMakeLists.txt create mode 100644 src/model_server_v2/demo.cpp create mode 100644 src/model_server_v2/method_parameters.hpp create mode 100644 src/model_server_v2/method_registry.hpp create mode 100644 src/model_server_v2/method_wrapper.hpp create mode 100644 src/model_server_v2/model_base.cpp create mode 100644 src/model_server_v2/model_base.hpp create mode 100644 src/model_server_v2/model_server.cpp create mode 100644 src/model_server_v2/model_server.hpp create mode 100644 src/model_server_v2/registration.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7a68f607aa..de8679d3c5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,7 +7,8 @@ subdirs( ml toolkits visualization - model_server) + model_server + model_server_v2) if(TC_BUILD_PYTHON) diff --git a/src/model_server/lib/variant.hpp b/src/model_server/lib/variant.hpp index 8b14a629db..bd58c6358f 100644 --- a/src/model_server/lib/variant.hpp +++ b/src/model_server/lib/variant.hpp @@ -14,7 +14,6 @@ #include #include #include -#include namespace turi { class model_base; @@ -308,6 +307,13 @@ template inline variant_type to_variant(const T& f) { return variant_converter::type>().set(f); } + +/** Overload for the case when we're trying to wrap a void return + * type in a template. + */ +inline variant_type to_variant() { + return variant_type(); +} } // namespace turi namespace turi { diff --git a/src/model_server_v2/CMakeLists.txt b/src/model_server_v2/CMakeLists.txt new file mode 100644 index 0000000000..778460b573 --- /dev/null +++ b/src/model_server_v2/CMakeLists.txt @@ -0,0 +1,17 @@ +make_library( model_server_v2 + SOURCES + model_server.cpp + model_base.cpp + REQUIRES + unity_shared) + + +make_executable(demo + SOURCES demo.cpp + REQUIRES model_server_v2 unity_shared) + + + + + + diff --git a/src/model_server_v2/demo.cpp b/src/model_server_v2/demo.cpp new file mode 100644 index 0000000000..76039f1ed0 --- /dev/null +++ b/src/model_server_v2/demo.cpp @@ -0,0 +1,105 @@ +#include +#include +#include + +using namespace turi; +using namespace turi::v2; + +/** Demo model. + * + */ +class demo_model : public turi::v2::model_base { + + public: + + /** The name of the model. + * + */ + const char* name() const { return "demo_model"; } + + /** Sets up the options and the registration. + * + * The registration is done in the constructor, without the use of macros. + */ + demo_model() { + register_method("add", &demo_model::add, "x", "y"); + register_method("concat_strings", &demo_model::append_strings, "s1", "s2"); + + // Defaults are specified inline + register_method("increment", &demo_model::increment, "x", Parameter("delta", 1)); + } + + /** Add two numbers. + * + * Const is fine. + */ + size_t add(size_t x, size_t y) const { + return x + y; + } + + /** Append two strings with a + + */ + std::string append_strings(const std::string& s1, const std::string& s2) const + { + return s1 + "+" + s2; + } + + /** Incerment a value. + */ + size_t increment(size_t x, size_t delta) const { + return x + delta; + } + +}; + +/** Registration for a model is just a single macro in the header. + * + * This automatically loads and registers the model when the library is loaded. + * This registration is trivially cheap. + */ +REGISTER_MODEL(demo_model); + + +void hello_world(const std::string& greeting) { + std::cout << "Hello, world!! " << greeting << std::endl; +} + +/** Registration for a function is just a single macro in a source file or header. + * + * This automatically loads and registers the function when the library is loaded. + */ +REGISTER_FUNCTION(hello_world, "greeting"); + + + +int main(int argc, char** argv) { + + + auto dm = model_server().create_model("demo_model"); + + std::string name = variant_get_value(dm->call_method("name")); + + std::cout << "Demoing model = " << name << std::endl; + + size_t result = variant_get_value(dm->call_method("add", 5, 9)); + + std::cout << "5 + 9 = " << result << std::endl; + + std::string s_res = variant_get_value(dm->call_method("concat_strings", "A", "B")); + + std::cout << "Concat A, +, B: " << s_res << std::endl; + + // Delta default is 1 + size_t inc_value = variant_get_value(dm->call_method("increment", 5)); + + std::cout << "Incremented 5: " << inc_value << std::endl; + + + // Call the registered function. + std::cout << "Calling hello_world." << std::endl; + model_server().call_function("hello_world", "This works!"); + + + return 0; + +} diff --git a/src/model_server_v2/method_parameters.hpp b/src/model_server_v2/method_parameters.hpp new file mode 100644 index 0000000000..b5b83132b0 --- /dev/null +++ b/src/model_server_v2/method_parameters.hpp @@ -0,0 +1,112 @@ +/* Copyright © 2019 Apple Inc. All rights reserved. + * + * Use of this source code is governed by a BSD-3-clause license that can + * be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause + */ +#ifndef TURI_METHOD_PARAMETERS_HPP_ +#define TURI_METHOD_PARAMETERS_HPP_ + +#include +#include +#include + +namespace turi { +namespace v2 { + + +/** Struct to hold information about the user specified parameter of a method. + * + * Includes information about a possible default value. + */ +struct Parameter { + + Parameter() {} + + // Allow implicit cast from string here.. + Parameter(const std::string& n) : name(n) {} + Parameter(std::string&& n) : name(std::move(n)) {} + + // Specify parameter with a default value + Parameter(const std::string& n, const variant_type& v) + : name(n), has_default(true), default_value(v) + {} + + Parameter(std::string&& n, variant_type&& v) + : name(std::move(n)), has_default(true), default_value(std::move(v)) + {} + + + // TODO: expand this out into a proper container class. + + + // Name + std::string name; + + // Optional default value + bool has_default = false; + variant_type default_value; +}; + + + +template +void validate_parameter_list(const std::vector& params) { + + // Validates that the parameter list works with the given types of + // the function. + if(sizeof...(FuncParams) != params.size()) { + throw std::invalid_argument("Mismatch in number of specified parameters."); + } + + // TODO: validate uniqueness of names. + // TODO: validate defaults can be cast to proper types. +} + + +//////////////////////////////////////////////////////////////////////////////// + +// How the arguments are bundled up and packaged. +struct argument_pack { + std::vector ordered_arguments; + variant_map_type named_arguments; +}; + +/** Method for resolving incoming arguments to a method. + * + */ +template +void resolve_method_arguments(std::array& arg_v, + const std::vector& parameter_list, + const argument_pack& args) { + + size_t n_ordered = args.ordered_arguments.size(); + for(size_t i = 0; i < n_ordered; ++i) { + arg_v[i] = &args.ordered_arguments[i]; + } + + // TODO: check if more ordered arguments given than are + // possible here. + size_t used_counter = n_ordered; + for(size_t i = n_ordered; i < n; ++i) { + auto it = args.named_arguments.find(parameter_list[i].name); + if(it == args.named_arguments.end()) { + if(parameter_list[i].has_default) { + arg_v[i] = &(parameter_list[i].default_value); + } else { + // TODO: intelligent error message. + throw std::string("Missing argument."); + } + } else { + arg_v[i] = &(it->second); + ++used_counter; + } + } + + // TODO: check that all the arguments have been used up. If not, + // generate a good error message. +} + +} +} + +#endif diff --git a/src/model_server_v2/method_registry.hpp b/src/model_server_v2/method_registry.hpp new file mode 100644 index 0000000000..a6472d306c --- /dev/null +++ b/src/model_server_v2/method_registry.hpp @@ -0,0 +1,137 @@ +/* Copyright © 2017 Apple Inc. All rights reserved. + * + * Use of this source code is governed by a BSD-3-clause license that can + * be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause + */ +#ifndef TURI_METHOD_REGISTRY_HPP_ +#define TURI_METHOD_REGISTRY_HPP_ + +#include +#include +#include +#include +#include + +namespace turi { +namespace v2 { + + +/** Manager all the methods in a given class / model. + * + * This class exists to manage a collection of methods associated with a given + * class. It provides an interface to call previously registered methods on + * this class by name, along with helpful error messages if the call is wrong. + * + * If the BaseClass is void, it provides a registry for standalone functions. + * TODO: implement this. + */ +template +class method_registry { + public: + + method_registry() + : m_class_name() + {} + + method_registry(const std::string& _name) + : m_class_name(_name) + {} + + /** Register a new method. + * + * See method_wrapper::create for an explanation of the arguments. + * + */ + template + void register_method(const std::string& name, RegisterMethodArgs&&... rmargs) { + + try { + + auto wrapper = method_wrapper::create(rmargs...); + + m_method_lookup[name] = wrapper; + } catch(...) { + // TODO: Expand these exceptions to make them informative. + process_exception(std::current_exception()); + } + } + + // Lookup a call function information. + std::shared_ptr > lookup(const std::string& name) const { + + // TODO: proper error message here + return m_method_lookup.at(name); + } + + /** Call a given const method registered previously. + */ + variant_type call_method(const BaseClass* inst, const std::string& name, + const argument_pack& arguments) const { + + try { + return lookup(name)->call(inst, arguments); + + } catch(...) { + process_exception(std::current_exception()); + } + } + + /** Call a given const or non-const method registered previously. + */ + variant_type call_method(BaseClass* inst, const std::string& name, + const argument_pack& arguments) const { + + try { + return lookup(name)->call(inst, arguments); + } catch(...) { + process_exception(std::current_exception()); + } + } + + private: + + [[noreturn]] void process_exception(std::exception_ptr e) const { + // TODO: Expand these exceptions to make them informative. + + std::rethrow_exception(e); + } + + // Unpack arguments. + template = 0> + inline void _arg_unpack(std::vector& dest, const Tuple& t) const { + dest[i] = to_variant(std::get(t)); + _arg_unpack(dest, t); + } + + template = 0> + inline void _arg_unpack(std::vector& dest, const Tuple& t) const { + } + + + public: + + // Call a method with the arguments explicitly. + template ::type, BaseClass>::value> = 0> + variant_type call_method(BC* inst, const std::string& name, const Args&... args) const { + + argument_pack arg_list; + arg_list.ordered_arguments.resize(sizeof...(Args)); + + _arg_unpack<0, sizeof...(Args)>(arg_list.ordered_arguments, std::make_tuple(args...)); + + return call_method(inst, name, arg_list); + } + + private: + + std::string m_class_name; + + std::unordered_map > > + m_method_lookup; +}; + +} +} + +#endif diff --git a/src/model_server_v2/method_wrapper.hpp b/src/model_server_v2/method_wrapper.hpp new file mode 100644 index 0000000000..25189eefd7 --- /dev/null +++ b/src/model_server_v2/method_wrapper.hpp @@ -0,0 +1,398 @@ +/* Copyright © 2019 Apple Inc. All rights reserved. + * + * Use of this source code is governed by a BSD-3-clause license that can + * be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause + */ +#ifndef TURI_METHOD_WRAPPER_HPP_ +#define TURI_METHOD_WRAPPER_HPP_ + +#include +#include +#include + +namespace turi { + +// Helper for enable if in templates. +// TODO: move this to a common location +template using enable_if_ = typename std::enable_if::type; + +namespace v2 { + + +/** Base class for wrapper around a specific method. + * + * This class provides an interface to call a method + * or function using generic arguments. The interface is contained in a + * templated instance of the class derived from this. It's created + * through the `create` static factory method. + * + * This class is meant to be a member of the + * + */ +template class method_wrapper { + + public: + + virtual ~method_wrapper(){} + + /** The type of the Class on which our method rests. + * + * May be , in which case it's a standalone function. + */ + typedef BaseClass class_type; + + /// Calling method for const base classes. + virtual variant_type call(const BaseClass* _inst, const argument_pack& args) const = 0; + + /// Calling method for non-const base classes. + virtual variant_type call(BaseClass* C, const argument_pack& args) const = 0; + + /// Calling method for standalone functions. + variant_type call(const argument_pack& args) const { + return call(nullptr, args); + } + + /// Returns the parameter info struct for a particular parameter + inline const Parameter& parameter_info(size_t n) const { + return m_parameter_list.at(n); + } + + /// Returns the name of the parameter + inline const std::string& parameter_name(size_t n) const { + return parameter_info(n).name; + } + + /** Factory method. + * + * Call this method to create the interface wrapper around + * the method. + */ + template + static std::shared_ptr + create(RetType(Class::*method)(FuncParams...) const, const ParamDefs&...); + + /** Overload of above factory method for non-const methods + * + */ + template + static std::shared_ptr + create(RetType(Class::*method)(FuncParams...), const ParamDefs&...); + + + /** Factory method for non-method functions. + * + */ + template + static std::shared_ptr + create(RetType(*function)(FuncParams...), const ParamDefs&...); + + protected: + + // To be called only from the instantiating class + method_wrapper(const std::vector& _parameter_list) + : m_parameter_list(_parameter_list) + { } + + // Information about the function / method + std::vector m_parameter_list; + }; + + +///////////////////////////////////////////////////////////////////////////////// +// +// Implementation details of the above. +// + +// If there is no class, then this is used instead of Class +// to denote that there is no class. +struct __NoClass {}; + +/** Child class for resolving the arguments passed into a function call. + * + * This class is mainly a container to define the types present + * during the recursive parameter expansion stage. + * + */ +template + class method_wrapper_impl : public method_wrapper { + private: + + /// The number of parameters required for the function. + static constexpr size_t N = sizeof...(FuncParams); + + /// Are we in a class instance or just a standalone function? + static constexpr bool is_method = !std::is_same::value; + + // logic below requires is_const_method to be false when is_method is false + static_assert(is_method || !is_const_method, + "is_const_method=1 when is_method =0"); + + /// Set the method type -- general class vs standalone function + template struct method_type_impl {}; + + // Non-method case + template <> struct method_type_impl<0> { + typedef RetType (*type)(FuncParams...); + }; + + // Method non-const case. + template <> struct method_type_impl<1> { + typedef RetType (Class::*type)(FuncParams...); + }; + + // Method const case. + template <> struct method_type_impl<2> { + typedef RetType (Class::*type)(FuncParams...) const; + }; + + typedef typename method_type_impl::type method_type; + + /// Function pointer to the method we're calling. + method_type m_method; + + public: + /** Constructor. + */ + method_wrapper_impl( + std::vector&& _parameter_list, method_type method) + : method_wrapper(std::move(_parameter_list)) + , m_method(method) + { + validate_parameter_list(this->m_parameter_list); + } + + private: + + + + ////////////////////////////////////////////////////////////////////////////// + // + // Calling methods + + /** A handy way to refer to the type of the nth argument. + */ + template + struct nth_param_type { + typedef typename std::tuple_element >::type raw_type; + typedef typename std::decay::type type; + }; + + /// Container for passing the calling arguments around after unpacking from argument_list + typedef std::array arg_v_type; + + ////////////////////////////////////////////////////////////////////////////// + // Entrance methods to this process. + + /** Non-const calling method. + * + */ + variant_type call(BaseClass* inst, const argument_pack& args) const override { + return _choose_call_path(inst, args); + } + + /** Const calling method. + * + */ + variant_type call(const BaseClass* inst, const argument_pack& args) const override { + return _choose_call_path(inst, args); + } + + ////////////////////////////////////////////////////////////////////////////// + // Step 1: Determine the evaluation path and Class pointer type depending + // on whether it's a const method, regular method, or function. + + template + struct _call_chooser { + static constexpr bool func_path = !is_method; + static constexpr bool const_method = is_const_method; + static constexpr bool _non_const_method = is_method && !is_const_method; + static constexpr bool bad_const_call = _non_const_method && std::is_const::value; + static constexpr bool method_path = _non_const_method && !std::is_const::value; + }; + + // If it's a regular function. + template ::func_path> = 0> + variant_type _choose_call_path(C* inst, const argument_pack& args) const { + return _call(nullptr, args); + } + + // If it's a bad const call. + template ::bad_const_call> = 0> + [[noreturn]] + variant_type _choose_call_path(C* inst, const argument_pack& args) const { + // Null implementation of the above to intercept compilation of the in-formed + // case where it's a const class and the method is non-const. + throw std::invalid_argument("Non-const method call attempted on const class pointer."); + } + + // Const method. + template ::const_method> = 0> + variant_type _choose_call_path(C* inst, const argument_pack& args) const { + return _call(dynamic_cast(inst), args); + } + + // Non-const method. + template ::method_call> = 0> + variant_type _choose_call_path(C* inst, const argument_pack& args) const { + return _call(dynamic_cast(inst), args); + } + + ////////////////////////////////////////////////////////////////////////////// + // Step 2: Unpack and resolve arguments. + + template + variant_type _call(C* inst, const argument_pack& args) const { + arg_v_type arg_v; + + // Resolve and unpack the incoming arguments. + resolve_method_arguments(arg_v, this->m_parameter_list, args); + + // Now that the argument list arg_v is filled out, we can call the + // recursive calling function and return the value. + return __call<0>(inst, arg_v); + } + + ////////////////////////////////////////////////////////////////////////////// + // Step 3: Recursively unpack the parameters into a parameter pack with the + // correct values. Checks performed here. + + template = 0> + variant_type __call(C* inst, const arg_v_type& arg_v, const Expanded&... args) const { + + // TODO: Separate out the case where the unpacking can be done by + // reference. + typedef typename nth_param_type::type arg_type; + + // TODO: Add intelligent error messages here on failure + arg_type next_arg = variant_get_value(*(arg_v[arg_idx])); + + // Call the next unpacking routine. + return __call(inst, arg_v, args..., next_arg); + } + + ////////////////////////////////////////////////////////////////////////////// + // Step 4: Call the function / method. + // + // This is the stopping case of expansion -- we've unpacked and translated all the + // arguments, now it's time to actually call the method. + + // First case: class method, void return type. + template ::value> = 0> + variant_type __call(C* inst, const arg_v_type& arg_v, const Expanded&... args) const { + (inst->*m_method)(args...); + return variant_type(); + } + + // Second case: class method, non-void return type. + template ::value> = 0> + variant_type __call(C* inst, const arg_v_type& arg_v, const Expanded&... args) const { + + return to_variant( (inst->*m_method)(args...) ); + } + + // Third case: standalone function, void return type + template ::value> = 0> + variant_type __call(C*, const arg_v_type& arg_v, const Expanded&... args) const { + + m_method(args...); + return variant_type(); + } + + // Fourth case: standalone function, non-void return type. + template ::value> = 0> + variant_type __call(C*, const arg_v_type& arg_v, const Expanded&... args) const { + + return to_variant(m_method(args...) ); + } +}; + + +////////////////////////////////////////////////////////// +// Some utility functions to help unpack the variadic arguments into +// a vector of parameters. + +template +void __unpack_parameters( + std::vector& dest, + const std::vector& vp) { + dest = vp; +} + +/** Recursive function to unpack the parameter list. + */ +template = 0> +void __unpack_parameters(std::vector& dest, const Params&... pv) { + dest[idx] = Parameter(std::get(std::make_tuple(pv...))); + __unpack_parameters(dest, pv...); +} + +/** Stopping case of recursive unpack function. + * + */ +template = 0> +void __unpack_parameters(std::vector& dest, const Params&... pv) { +} + +/** Implementation of the factory method for non-const methods. + */ +template +template +std::shared_ptr > method_wrapper::create( + RetType(Class::*method)(FuncParams...), + const ParamDefs&... param_defs) { + + std::vector params; + params.resize(sizeof...(ParamDefs)); + __unpack_parameters<0>(params, param_defs...); + + return std::make_shared > + (std::move(params), method); + }; + +/** Const overload of the method interface factory method. + */ +template +template +std::shared_ptr > method_wrapper::create( + RetType(Class::*method)(FuncParams...) const, + const ParamDefs&... param_defs) { + + std::vector params; + params.resize(sizeof...(ParamDefs)); + __unpack_parameters<0>(params, param_defs...); + + return std::make_shared > + (std::move(params), method); + }; + +/** Factory method for non-method functions. + * + */ +template + template + std::shared_ptr > + method_wrapper::create( + RetType(*function)(FuncParams...), + const ParamDefs&... param_defs) { + + std::vector params; + params.resize(sizeof...(ParamDefs)); + __unpack_parameters<0>(params, param_defs...); + + return std::make_shared > + (std::move(params), function); + + } + + +} +} + +#endif diff --git a/src/model_server_v2/model_base.cpp b/src/model_server_v2/model_base.cpp new file mode 100644 index 0000000000..0f9ebe9859 --- /dev/null +++ b/src/model_server_v2/model_base.cpp @@ -0,0 +1,17 @@ +#include + +namespace turi { +namespace v2 { + +model_base::model_base() + : m_registry(new method_registry()) +{ + register_method("name", &model_base::name); +} + + +model_base::~model_base() { } + +} +} + diff --git a/src/model_server_v2/model_base.hpp b/src/model_server_v2/model_base.hpp new file mode 100644 index 0000000000..658be4dceb --- /dev/null +++ b/src/model_server_v2/model_base.hpp @@ -0,0 +1,143 @@ +/* Copyright © 2019 Apple Inc. All rights reserved. + * + * Use of this source code is governed by a BSD-3-clause license that can + * be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause + */ +#ifndef TURI_MODEL_BASE_V2_HPP +#define TURI_MODEL_BASE_V2_HPP + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace turi { + namespace v2 { + + +/** + * The base class from which all new models must inherit. + * + * This class defines a generic object interface, listing properties and + * callable methods, so that instances can be naturally wrapped and exposed to + * other languages, such as Python. + * + * Subclasses that wish to support saving and loading should also override the + * save_impl, load_version, and get_version functions below. + */ +class EXPORT model_base { + public: + + model_base(); + + virtual ~model_base(); + + // These public member functions define the communication between model_base + // instances and the unity runtime. Subclasses define the behavior of their + // instances using the protected interface below. + + /** + * Returns the name of the toolkit class, as exposed to client code. For + * example, the Python proxy for this instance will have a type with this + * name. + * + */ + virtual const char* name() const = 0; + + /** Sets up the class given the options present. + * + * TODO: implement all of this. + */ + virtual void setup(const variant_map_type& options) { + // option_manager.update(options); + } + + /** Call one of the const methods registered using the configure() method above. + * + * `args` may either be explicit arguments or an instance of + * the argument_pack class. + */ + template + variant_type call_method(const std::string& name, const Args&... args) const; + + + /** Call one of the methods registered using the configure() method above. + * + * `args` may either be explicit arguments or an instance of + * the argument_pack class. + */ + template + variant_type call_method(const std::string& name, const Args&... args); + + + /** Register a method that can be called by name using the registry above. + * + * The format for calling this is the function name, the pointer to the method, + * then a list of names or Parameter class instances giving the names of the + * parameters. + * + * Example: + * + * // For a method "add" in class C that derives from model_base. + * register_method("add", &C::add, "x", "y"); + * + * // For a method "inc" in class C that derives from model_base, + * // with a default parameter of 1. + * register_method("inc", &C::inc, Parameter("delta", 1) ); + * + * + * See the documentation on method_wrapper<...>::create to see the format of args. + */ + template + void register_method(const std::string& name, Method&&, const Args&... args); + + + // TODO: add back in load and save routines. + + private: + std::shared_ptr > m_registry; + +}; + +/////////////////////////////////////////////////////////////////////// +// +// Implementation of above template functions. + +/** Call one of the methods registered using the configure() method above. + * + */ +template +variant_type model_base::call_method( + const std::string& name, const Args&... args) { + + return m_registry->call_method(this, name, args...); +} + +/** Const overload of the above. + * + */ +template +variant_type model_base::call_method( + const std::string& name, const Args&... args) const { + + return m_registry->call_method(this, name, args...); +} + +/** Register a method that can be called by name using the registry above. + */ +template +void model_base::register_method( + const std::string& name, Method&& m, const Args&... args) { + + m_registry->register_method(name, m, args...); +} + +} +} + +#endif diff --git a/src/model_server_v2/model_server.cpp b/src/model_server_v2/model_server.cpp new file mode 100644 index 0000000000..4c16321c0c --- /dev/null +++ b/src/model_server_v2/model_server.cpp @@ -0,0 +1,67 @@ +/* Copyright © 2017 Apple Inc. All rights reserved. + * + * Use of this source code is governed by a BSD-3-clause license that can + * be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause + */ +#include + + +namespace turi { +namespace v2 { + +EXPORT model_server_impl& model_server() { + static model_server_impl global_model_server; + return global_model_server; +} + + +model_server_impl::model_server_impl() + : m_function_registry(new method_registry) +{ + + +} + +/** Does the work of registering things with the callbacks. + */ +void model_server_impl::_process_registered_callbacks_internal() { + + std::lock_guard _lg(m_model_registration_lock); + + size_t cur_idx; + + while( (cur_idx = m_callback_last_processed_index) != m_callback_pushback_index) { + + + // Call the callback function to perform the registration, simultaneously + // zeroing out the pointer. + _registration_callback reg_f = nullptr; + size_t idx = cur_idx % m_registration_callback_list.size(); + std::swap(reg_f, m_registration_callback_list[idx]); + reg_f(*this); + + // We're done here; advance. + ++m_callback_last_processed_index; + } +} + +/** Instantiate a previously registered model by name. + */ +std::shared_ptr model_server_impl::create_model(const std::string& model_name) { + + // Make sure there aren't new models waiting on the horizon. + check_registered_callback_queue(); + + auto it = m_model_by_name.find(model_name); + + if(it == m_model_by_name.end()) { + // TODO: make this more informative. + throw std::invalid_argument("Model not recognized."); + } + + return it->second(); +} + + +} +} diff --git a/src/model_server_v2/model_server.hpp b/src/model_server_v2/model_server.hpp new file mode 100644 index 0000000000..3e7871ca35 --- /dev/null +++ b/src/model_server_v2/model_server.hpp @@ -0,0 +1,288 @@ +/* Copyright © 2017 Apple Inc. All rights reserved. + * + * Use of this source code is governed by a BSD-3-clause license that can + * be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause + */ +#ifndef TURI_MODEL_SERVER_HPP +#define TURI_MODEL_SERVER_HPP + +#include +#include +#include +#include +#include +#include +#include + + +namespace turi { +namespace v2 { + +class model_server_impl; + +/** Returns the singleton version of the model server. + * + */ +EXPORT model_server_impl& model_server(); + + +EXPORT class model_server_impl { + private: + + // Disable instantiation outside of the global instance. + model_server_impl(); + friend model_server_impl& model_server(); + + // Explicitly disable copying, etc. + model_server_impl(const model_server_impl&) = delete; + model_server_impl(model_server_impl&&) = delete; + + public: + + //////////////////////////////////////////////////////////////////////////// + // Calling models. + + /** Instantiate a previously registered model by name. + * + */ + std::shared_ptr create_model(const std::string& model_name); + + + /** Instantiate a model by type. + */ + template + std::shared_ptr create_model(); + + + /** Call a previously registered function. + */ + template + variant_type call_function( + const std::string& function_name, FunctionArgs&&... args); + + public: + + /** Registration of a function. + * + * Registers a new function that can be called through the call_function + * call above. + * + * The format of the call is name of the function, the function, and then + * a list of 0 or more parameter specs. + * + * Example: + * + * void f(int x, int y); + * register_new_function("f", f, "x", "y"); + * + * + * + * \param name The name of the function. + * \param function A pointer to the function itself. + * \param param_specs The parameter specs + */ + template + void register_new_function(const std::string& name, Function&& function, ParamSpecs&&...); + + + /** Registration of new models. + * + * A model is registered through a call to register new model, which + * instantiates it and populates the required options and method call + * lookups. Copies of these options and method call lookups are stored + * internally in a registry here so new models can be instantiated quickly. + * + * + * The new model's name() method provides the name of the model being + * registered. + * + * This method can be called at any point. + * + */ + template void register_new_model(); + + /** Fast on-load model registration. + * + * The callbacks below provide a fast method for registering new models + * on library load time. This works by first registering a callback + * using a simple callback function. + * + */ + typedef void (*_registration_callback)(model_server_impl&); + + /** Register a callback function to be processed when a model is served. + * + * Function is reentrant and fast enough to be called from a static initializer. + */ + inline void add_registration_callback(_registration_callback callback) GL_HOT_INLINE_FLATTEN; + + private: + + + ////////////////////////////////////////////////////////////////////////////// + // Registered model lookups. + typedef std::function()> model_creation_function; + std::unordered_map m_model_by_name; + + + ////////////////////////////////////////////////////////////////////////////// + // Registered function lookups. + + std::unique_ptr > m_function_registry; + + + /////////////////////////////////////////////////////////////////////////////// + // TODO: Registered function lookups. + // + + /** Lock to ensure that model registration is queued correctly. + */ + std::mutex m_model_registration_lock; + std::type_index m_last_model_registered = std::type_index(typeid(void)); + + /** An intermediate buffer of registration callbacks. + * + * These queues are used on library load to register callback functions, which + * are then processed when any model is requested to ensure that library loading + * is done efficiently. check_registered_callback_queue() should be called + * before any lookups are done to ensure that all possible lookups have been + * registered. + * + */ + std::array<_registration_callback, 512> m_registration_callback_list; + std::atomic m_callback_pushback_index; + std::atomic m_callback_last_processed_index; + + /** Process the registered callbacks. + * + * First performs a fast inline check to see if it's needed, so + * this function can be called easily. + */ + inline void check_registered_callback_queue(); + + /** Does the work of registering things with the callbacks. + */ + void _process_registered_callbacks_internal(); +}; + +///////////////////////////////////////////////////////////////////////////////// +// +// Implementations of inline functions for the model server class +// + +/** Fast inline check + */ +inline void model_server_impl::check_registered_callback_queue() { + if(m_callback_last_processed_index < m_callback_pushback_index) { + _process_registered_callbacks_internal(); + } +} + +/** Add the callback to the registration function. + * + * This works by putting the callback function into a round-robin queue to avoid + * potential allocations or deallocations during library load time and to + * preserve thread safety. + */ +inline void model_server_impl::add_registration_callback( + model_server_impl::_registration_callback callback) { + + + size_t insert_index_raw = (m_callback_pushback_index++); + + do { + // Check to make sure this can be safely inserted. + size_t processed_index_raw = m_callback_last_processed_index; + + // Check to make sure we aren't so far behind the number of actually + // registered callbacks that we're out of space. + if(processed_index_raw + m_registration_callback_list.size() > insert_index_raw) { + break; + } else { + // This will process the next block of insertions. + _process_registered_callbacks_internal(); + } + + } while(true); + + size_t insert_index = insert_index_raw % m_registration_callback_list.size(); + + ASSERT_TRUE(m_registration_callback_list[insert_index] == nullptr); + m_registration_callback_list[insert_index] = callback; +} + + +/** Registration of new models. +* +* A model is registered through a call to register new model, which +* instantiates it and populates the required options and method call +* lookups. Copies of these options and method call lookups are stored +* internally in a registry here so new models can be instantiated quickly. +* +* +* The new model's name() method provides the name of the model being +* registered. +* +* This method can be called at any point. +* +*/ +template void model_server_impl::register_new_model() { + + // Quick check to cut out duplicate registrations. This can + // happen, e.g. if the class or the function macros appear in a header, + // which is fine and something we are designed to handle. + // However, this means that multiple registration calls can occur for the same + // class, and this quickly filters those registrations out. + if(std::type_index(typeid(ModelClass)) == m_last_model_registered) { + return; + } + m_last_model_registered = std::type_index(typeid(ModelClass)); + + // TODO: As the registration is now performed in the constructor, + // a base instantiated version of the class should be held, then + // subsequent model creations should simply use the copy constructor to + // instantiate them. This means the entire method registry is not + // duplicated. For now, just go through this way. + const std::string& name = ModelClass().name(); + + model_creation_function mcf = [=](){ return this->create_model(); }; + + m_model_by_name.insert({name, mcf}); +} + + +/** Instantiate a previously registered model by type. + */ +template + std::shared_ptr model_server_impl::create_model() { + + // Make sure there aren't new models waiting on the horizon. + check_registered_callback_queue(); + + return std::make_shared(); +} + + +/** Register a new function. + */ +template + void model_server_impl::register_new_function( + const std::string& name, Function&& function, ParamSpecs&&... param_specs) { + m_function_registry->register_method(name, function, param_specs...); +} + +/** Call the function. + */ +template + variant_type model_server_impl::call_function( + const std::string& function_name, FunctionArgs&&... args) { + + // Make sure there aren't new functions waiting on the horizon. + check_registered_callback_queue(); + + return m_function_registry->call_method(nullptr, function_name, args...); +} + +} +} // End turi namespace +#endif diff --git a/src/model_server_v2/registration.hpp b/src/model_server_v2/registration.hpp new file mode 100644 index 0000000000..82b9246170 --- /dev/null +++ b/src/model_server_v2/registration.hpp @@ -0,0 +1,68 @@ +/* Copyright © 2017 Apple Inc. All rights reserved. + * + * Use of this source code is governed by a BSD-3-clause license that can + * be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause + */ +#ifndef TURI_MODEL_SERVER_V2_REGSTRATION_HPP_ +#define TURI_MODEL_SERVER_V2_REGSTRATION_HPP_ + + +#include +#include +#include +#include + +namespace turi { + namespace v2 { + +// A helper class to use a static initializer to do a lightweight registration +// of class loading at library load time. Intended to be used as a component of +// the registration macro. +class __model_server_static_class_registration_hook { + public: + inline __model_server_static_class_registration_hook( + model_server_impl::_registration_callback f) { + model_server().add_registration_callback(f); + } +}; + + + +#define REGISTER_MODEL(model) \ + static void __register_##model(model_server_impl& server) { \ + server.template register_new_model(); \ + } \ + \ + static turi::v2::__model_server_static_class_registration_hook \ + __register_##model##_hook(__register_##model) + + +// A helper class to use a static initializer to do a lightweight registration +// of class loading at library load time. Intended to be used as a component of +// the +class __model_server_static_function_registration_hook { + public: + inline __model_server_static_function_registration_hook( + model_server_impl::_registration_callback f) { + model_server().add_registration_callback(f); + } +}; + + + +#define REGISTER_NAMED_FUNCTION(name, function, ...) \ +\ + static void register_function_##function(model_server_impl& server) {\ + server.register_new_function(name, function, __VA_ARGS__);\ + } \ + __model_server_static_function_registration_hook \ +__register_function_##function##_hook(register_function_##function) + +#define REGISTER_FUNCTION(function, ...) \ + REGISTER_NAMED_FUNCTION(#function, function, __VA_ARGS__) + + +} +} + +#endif diff --git a/src/visualization/annotation/object_detection.hpp b/src/visualization/annotation/object_detection.hpp index df45e3c656..650aece20c 100644 --- a/src/visualization/annotation/object_detection.hpp +++ b/src/visualization/annotation/object_detection.hpp @@ -60,4 +60,4 @@ create_object_detection_annotation(const std::shared_ptr &data, } // namespace annotate } // namespace turi -#endif \ No newline at end of file +#endif From 35d6b0fd9eb863c328676deb639fe159f3c9f86c Mon Sep 17 00:00:00 2001 From: Hoyt Koepke Date: Sat, 21 Dec 2019 16:46:10 -0700 Subject: [PATCH 2/5] Adjust comment. --- src/model_server_v2/method_registry.hpp | 1 - src/model_server_v2/method_wrapper.hpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/model_server_v2/method_registry.hpp b/src/model_server_v2/method_registry.hpp index a6472d306c..c217ebbd49 100644 --- a/src/model_server_v2/method_registry.hpp +++ b/src/model_server_v2/method_registry.hpp @@ -23,7 +23,6 @@ namespace v2 { * this class by name, along with helpful error messages if the call is wrong. * * If the BaseClass is void, it provides a registry for standalone functions. - * TODO: implement this. */ template class method_registry { diff --git a/src/model_server_v2/method_wrapper.hpp b/src/model_server_v2/method_wrapper.hpp index 25189eefd7..dbe9a2a8f2 100644 --- a/src/model_server_v2/method_wrapper.hpp +++ b/src/model_server_v2/method_wrapper.hpp @@ -304,7 +304,7 @@ template ::value> = 0> variant_type __call(C*, const arg_v_type& arg_v, const Expanded&... args) const { From 03bcd61bb0f99d8ab59835b5a6bdd9b7a1f58e9c Mon Sep 17 00:00:00 2001 From: Hoyt Koepke Date: Mon, 23 Dec 2019 20:23:31 -0700 Subject: [PATCH 3/5] Bugfixes on model server v2. --- src/model_server_v2/demo.cpp | 1 + src/model_server_v2/method_parameters.hpp | 1 + src/model_server_v2/method_registry.hpp | 1 + src/model_server_v2/method_wrapper.hpp | 15 ++++++++------- src/model_server_v2/model_base.cpp | 1 + src/model_server_v2/model_base.hpp | 1 + src/model_server_v2/model_server.cpp | 1 + src/model_server_v2/model_server.hpp | 6 ++++-- src/model_server_v2/registration.hpp | 13 +++++++------ 9 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/model_server_v2/demo.cpp b/src/model_server_v2/demo.cpp index 76039f1ed0..f346a9a40d 100644 --- a/src/model_server_v2/demo.cpp +++ b/src/model_server_v2/demo.cpp @@ -1,4 +1,5 @@ #include +#include #include #include diff --git a/src/model_server_v2/method_parameters.hpp b/src/model_server_v2/method_parameters.hpp index b5b83132b0..430edcddb7 100644 --- a/src/model_server_v2/method_parameters.hpp +++ b/src/model_server_v2/method_parameters.hpp @@ -6,6 +6,7 @@ #ifndef TURI_METHOD_PARAMETERS_HPP_ #define TURI_METHOD_PARAMETERS_HPP_ +#include #include #include #include diff --git a/src/model_server_v2/method_registry.hpp b/src/model_server_v2/method_registry.hpp index c217ebbd49..588f6fdd00 100644 --- a/src/model_server_v2/method_registry.hpp +++ b/src/model_server_v2/method_registry.hpp @@ -6,6 +6,7 @@ #ifndef TURI_METHOD_REGISTRY_HPP_ #define TURI_METHOD_REGISTRY_HPP_ +#include #include #include #include diff --git a/src/model_server_v2/method_wrapper.hpp b/src/model_server_v2/method_wrapper.hpp index dbe9a2a8f2..b675f985b0 100644 --- a/src/model_server_v2/method_wrapper.hpp +++ b/src/model_server_v2/method_wrapper.hpp @@ -6,6 +6,7 @@ #ifndef TURI_METHOD_WRAPPER_HPP_ #define TURI_METHOD_WRAPPER_HPP_ +#include #include #include #include @@ -206,11 +207,11 @@ template struct _call_chooser { - static constexpr bool func_path = !is_method; - static constexpr bool const_method = is_const_method; - static constexpr bool _non_const_method = is_method && !is_const_method; - static constexpr bool bad_const_call = _non_const_method && std::is_const::value; - static constexpr bool method_path = _non_const_method && !std::is_const::value; + static constexpr int func_path = !is_method; + static constexpr int const_method_path = is_const_method; + static constexpr int _non_const_method = is_method && !is_const_method; + static constexpr int bad_const_call = _non_const_method && std::is_const::value; + static constexpr int method_path = _non_const_method && !std::is_const::value; }; // If it's a regular function. @@ -229,13 +230,13 @@ template ::const_method> = 0> + template ::const_method_path> = 0> variant_type _choose_call_path(C* inst, const argument_pack& args) const { return _call(dynamic_cast(inst), args); } // Non-const method. - template ::method_call> = 0> + template ::method_path> = 0> variant_type _choose_call_path(C* inst, const argument_pack& args) const { return _call(dynamic_cast(inst), args); } diff --git a/src/model_server_v2/model_base.cpp b/src/model_server_v2/model_base.cpp index 0f9ebe9859..7ec1d28911 100644 --- a/src/model_server_v2/model_base.cpp +++ b/src/model_server_v2/model_base.cpp @@ -1,3 +1,4 @@ +#include #include namespace turi { diff --git a/src/model_server_v2/model_base.hpp b/src/model_server_v2/model_base.hpp index 658be4dceb..2dfe5569e5 100644 --- a/src/model_server_v2/model_base.hpp +++ b/src/model_server_v2/model_base.hpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include diff --git a/src/model_server_v2/model_server.cpp b/src/model_server_v2/model_server.cpp index 4c16321c0c..09fdb75a73 100644 --- a/src/model_server_v2/model_server.cpp +++ b/src/model_server_v2/model_server.cpp @@ -3,6 +3,7 @@ * Use of this source code is governed by a BSD-3-clause license that can * be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause */ +#include #include diff --git a/src/model_server_v2/model_server.hpp b/src/model_server_v2/model_server.hpp index 3e7871ca35..42845d4429 100644 --- a/src/model_server_v2/model_server.hpp +++ b/src/model_server_v2/model_server.hpp @@ -3,9 +3,10 @@ * Use of this source code is governed by a BSD-3-clause license that can * be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause */ -#ifndef TURI_MODEL_SERVER_HPP -#define TURI_MODEL_SERVER_HPP +#ifndef TURI_MODEL_SERVER_V2_HPP +#define TURI_MODEL_SERVER_V2_HPP +#include #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include namespace turi { diff --git a/src/model_server_v2/registration.hpp b/src/model_server_v2/registration.hpp index 82b9246170..db112f0640 100644 --- a/src/model_server_v2/registration.hpp +++ b/src/model_server_v2/registration.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include namespace turi { @@ -21,15 +22,15 @@ namespace turi { class __model_server_static_class_registration_hook { public: inline __model_server_static_class_registration_hook( - model_server_impl::_registration_callback f) { - model_server().add_registration_callback(f); + turi::v2::model_server_impl::_registration_callback f) { + turi::v2::model_server().add_registration_callback(f); } }; #define REGISTER_MODEL(model) \ - static void __register_##model(model_server_impl& server) { \ + static void __register_##model(turi::v2::model_server_impl& server) { \ server.template register_new_model(); \ } \ \ @@ -43,8 +44,8 @@ class __model_server_static_class_registration_hook { class __model_server_static_function_registration_hook { public: inline __model_server_static_function_registration_hook( - model_server_impl::_registration_callback f) { - model_server().add_registration_callback(f); + turi::v2::model_server_impl::_registration_callback f) { + turi::v2::model_server().add_registration_callback(f); } }; @@ -52,7 +53,7 @@ class __model_server_static_function_registration_hook { #define REGISTER_NAMED_FUNCTION(name, function, ...) \ \ - static void register_function_##function(model_server_impl& server) {\ + static void register_function_##function(turi::v2::model_server_impl& server) {\ server.register_new_function(name, function, __VA_ARGS__);\ } \ __model_server_static_function_registration_hook \ From 222ba5bf55899cd0389294a772e9df8682f38705 Mon Sep 17 00:00:00 2001 From: Hoyt Koepke Date: Mon, 23 Dec 2019 20:30:01 -0700 Subject: [PATCH 4/5] [WIP] New build system. Rearchitecture of the the build system. A vast simplification. Changes - ./configure can produce an XCode project of the Turi source code. - deps/ can be compiled and installed out of the regular build process. Helpful for the XCode project. - All packages in src/external/ and src/visualization compile into static libraries. - The rest of the code base compiles into a single shared library. This means that most everything occurs in one single CMakeLists.txt file using standard cmake commands. - All headers are installed into targets/include when `make install` is called. The main library is installed into targets/lib. - The code definitions that reflect compiler oddities (e.g. is std::hash defined) are dumped into a single header file, src/turi_common.h. All source files and headers include this file first. This allows other programs to link against this library / headers reliably. - The python part of TuriCreate now builds through the standard setup.py method. Cython / pybind11 extensions are compiled and packaged using the standard extensions mechanisms and linking against the installed Turi headers. WIP TODO: the python installation, deployment. --- .gitignore | 1 + CMakeLists.txt | 419 +++++++++--------- build_python_wheel.sh | 114 +++++ cmake/CompilerFlags.cmake | 10 + cmake/CompilerOddities.cmake | 10 +- configure | 86 +++- deps/src/libxml2-2.9.1/ChangeLog | 2 +- deps/src/libxml2-2.9.1/bakefile/libxml2.bkl | 4 +- deps/src/libxml2-2.9.1/doc/APIchunk17.html | 2 +- deps/src/libxml2-2.9.1/doc/APIchunk9.html | 2 +- deps/src/libxml2-2.9.1/doc/APIfiles.html | 2 +- .../libxml2-2.9.1/doc/devhelp/general.html | 2 +- .../doc/devhelp/libxml2-xinclude.html | 2 +- deps/src/libxml2-2.9.1/doc/html/book1.html | 2 +- deps/src/libxml2-2.9.1/doc/html/index.html | 2 +- .../libxml2-2.9.1/doc/html/libxml-lib.html | 2 +- .../libxml2-2.9.1/doc/html/libxml-valid.html | 2 +- .../libxml2-2.9.1/doc/html/libxml-xlink.html | 2 +- deps/src/libxml2-2.9.1/doc/xmllint.html | 2 +- deps/src/libxml2-2.9.1/doc/xmllint.xml | 4 +- deps/src/libxml2-2.9.1/regressions.xml | 4 +- scripts/repo.py | 101 ++++- src/CMakeLists.txt | 20 - src/capi/TuriCreate.h | 1 + src/capi/TuriCreateEnums.h | 1 + src/capi/default_server_initializer.cpp | 1 + src/capi/impl/capi_datetime.cpp | 1 + src/capi/impl/capi_error_handling.cpp | 1 + src/capi/impl/capi_error_handling.hpp | 1 + src/capi/impl/capi_flex_dict.cpp | 1 + src/capi/impl/capi_flex_enum_list.cpp | 1 + src/capi/impl/capi_flex_image.cpp | 1 + src/capi/impl/capi_flex_list.cpp | 1 + src/capi/impl/capi_flexible_type.cpp | 1 + src/capi/impl/capi_functions.cpp | 1 + src/capi/impl/capi_initialization.cpp | 1 + src/capi/impl/capi_initialization.hpp | 1 + .../impl/capi_initialization_internal.hpp | 1 + src/capi/impl/capi_memory_management.cpp | 1 + src/capi/impl/capi_models.cpp | 1 + src/capi/impl/capi_ndarray.cpp | 1 + src/capi/impl/capi_operators.cpp | 3 +- src/capi/impl/capi_parameters.cpp | 1 + src/capi/impl/capi_sarray.cpp | 1 + src/capi/impl/capi_sframe.cpp | 1 + src/capi/impl/capi_variant.cpp | 1 + src/capi/impl/capi_visualization.cpp | 1 + src/capi/impl/capi_wrapper_structs.cpp | 1 + src/capi/impl/capi_wrapper_structs.hpp | 1 + src/core/CMakeLists.txt | 9 - src/core/data/CMakeLists.txt | 6 - src/core/data/flexible_type/CMakeLists.txt | 19 - src/core/data/flexible_type/flexible_type.cpp | 1 + src/core/data/flexible_type/flexible_type.hpp | 1 + .../flexible_type_base_types.hpp | 1 + .../flexible_type_conversion_utilities.cpp | 1 + .../flexible_type_conversion_utilities.hpp | 1 + .../flexible_type/flexible_type_converter.hpp | 1 + .../flexible_type/flexible_type_detail.hpp | 1 + .../flexible_type_spirit_parser.cpp | 1 + .../flexible_type_spirit_parser.hpp | 1 + src/core/data/flexible_type/json_util.hpp | 1 + src/core/data/flexible_type/ndarray.hpp | 1 + src/core/data/flexible_type/string_escape.cpp | 1 + src/core/data/flexible_type/string_escape.hpp | 1 + src/core/data/flexible_type/string_parser.hpp | 1 + src/core/data/flexible_type/type_traits.hpp | 1 + src/core/data/image/CMakeLists.txt | 46 -- src/core/data/image/image_type.cpp | 1 + src/core/data/image/image_type.hpp | 1 + src/core/data/image/image_util_impl.cpp | 1 + src/core/data/image/image_util_impl.hpp | 1 + src/core/data/image/io.cpp | 1 + src/core/data/image/io.hpp | 1 + .../image/{io_example.cpp => io_example.cxx} | 0 src/core/data/image/io_impl.hpp | 1 + src/core/data/image/jpeg_io.cpp | 1 + .../data/image/numeric_extension/affine.hpp | 1 + .../image/numeric_extension/algorithm.hpp | 1 + .../channel_numeric_operations.hpp | 1 + .../data/image/numeric_extension/convolve.hpp | 1 + .../data/image/numeric_extension/kernel.hpp | 1 + .../perspective_projection.hpp | 1 + .../pixel_numeric_operations.hpp | 1 + .../data/image/numeric_extension/resample.hpp | 1 + .../data/image/numeric_extension/sampler.hpp | 1 + src/core/data/image/png_io.cpp | 1 + src/core/data/json/CMakeLists.txt | 1 - src/core/data/json/json_include.hpp | 1 + src/core/data/sframe/CMakeLists.txt | 9 - .../data/sframe/gl_datatype_registration.cpp | 3 +- src/core/data/sframe/gl_gframe.cpp | 1 + src/core/data/sframe/gl_gframe.hpp | 1 + src/core/data/sframe/gl_sarray.cpp | 1 + src/core/data/sframe/gl_sarray.hpp | 1 + src/core/data/sframe/gl_sframe.cpp | 1 + src/core/data/sframe/gl_sframe.hpp | 1 + src/core/data/sframe/gl_sgraph.cpp | 11 +- src/core/data/sframe/gl_sgraph.hpp | 1 + src/core/export.hpp | 1 + src/core/generics/CMakeLists.txt | 1 - src/core/generics/blob.hpp | 1 + src/core/generics/bloom_filter.hpp | 1 + src/core/generics/cuckoo_map.hpp | 1 + src/core/generics/cuckoo_map_pow2.hpp | 1 + src/core/generics/cuckoo_set_pow2.hpp | 1 + src/core/generics/gl_string.hpp | 1 + src/core/generics/gl_vector.hpp | 1 + src/core/generics/hopscotch_map.hpp | 1 + src/core/generics/hopscotch_set.hpp | 1 + src/core/generics/hopscotch_table.hpp | 1 + src/core/generics/inplace_lf_queue2.hpp | 1 + src/core/generics/is_memmovable.hpp | 1 + src/core/generics/lock_free_internal.hpp | 1 + src/core/generics/lock_free_pool.hpp | 1 + src/core/generics/remove_member_pointer.hpp | 1 + src/core/generics/small_map.hpp | 1 + src/core/generics/small_set.hpp | 1 + .../generics/sparse_parallel_2d_array.hpp | 1 + .../generics/string_conversion_internals.hpp | 1 + src/core/generics/string_internals.hpp | 1 + src/core/generics/string_serialization.hpp | 1 + src/core/generics/string_stream_internals.hpp | 1 + src/core/generics/symmetric_2d_array.hpp | 1 + src/core/generics/type_trait_def.hpp | 1 + src/core/generics/type_trait_undef.hpp | 1 + src/core/generics/value_container_mapper.hpp | 1 + .../value_container_mapper_internal.hpp | 1 + src/core/generics/vector_internals.hpp | 1 + src/core/generics/vector_serialization.hpp | 1 + src/core/globals/CMakeLists.txt | 11 - src/core/globals/global_constants.hpp | 1 + src/core/globals/globals.cpp | 1 + src/core/globals/globals.hpp | 1 + src/core/logging/CMakeLists.txt | 24 - src/core/logging/assertions.hpp | 1 + src/core/logging/backtrace.cpp | 1 + src/core/logging/backtrace.hpp | 1 + src/core/logging/error.cpp | 1 + src/core/logging/error.hpp | 1 + src/core/logging/fail_method.hpp | 1 + src/core/logging/log_level_setter.cpp | 1 + src/core/logging/log_level_setter.hpp | 1 + src/core/logging/log_rotate.cpp | 3 +- src/core/logging/log_rotate.hpp | 1 + src/core/logging/logger.cpp | 1 + src/core/logging/logger.hpp | 8 +- src/core/logging/logger_includes.hpp | 1 + src/core/logging/table_printer/CMakeLists.txt | 19 - .../table_printer/table_element_printers.cpp | 1 + .../table_printer/table_element_printers.hpp | 1 + .../logging/table_printer/table_printer.cpp | 3 +- .../logging/table_printer/table_printer.hpp | 3 +- ...xamples.cpp => table_printer_examples.cxx} | 2 +- src/core/parallel/CMakeLists.txt | 12 - src/core/parallel/atomic.hpp | 1 + src/core/parallel/atomic_ops.hpp | 1 + src/core/parallel/deferred_rwlock.hpp | 1 + .../execute_task_in_native_thread.cpp | 1 + .../execute_task_in_native_thread.hpp | 1 + src/core/parallel/lambda_omp.hpp | 1 + src/core/parallel/lockfree_push_back.hpp | 1 + src/core/parallel/mutex.hpp | 1 + src/core/parallel/parallel_includes.hpp | 1 + src/core/parallel/pthread_h.h | 1 + src/core/parallel/pthread_tools.cpp | 3 +- src/core/parallel/pthread_tools.hpp | 1 + src/core/parallel/queued_rwlock.hpp | 1 + src/core/parallel/thread_pool.cpp | 1 + src/core/parallel/thread_pool.hpp | 1 + src/core/parallel/winpthreadsll.h | 3 +- src/core/random/CMakeLists.txt | 24 - src/core/random/alias.cpp | 3 +- src/core/random/alias.hpp | 1 + src/core/random/random.cpp | 5 +- src/core/random/random.hpp | 3 +- src/core/storage/CMakeLists.txt | 7 - src/core/storage/fileio/CMakeLists.txt | 54 --- src/core/storage/fileio/block_cache.cpp | 1 + src/core/storage/fileio/block_cache.hpp | 1 + src/core/storage/fileio/buffered_writer.hpp | 7 +- src/core/storage/fileio/cache_stream.hpp | 5 +- src/core/storage/fileio/cache_stream_sink.cpp | 1 + src/core/storage/fileio/cache_stream_sink.hpp | 1 + .../storage/fileio/cache_stream_source.cpp | 1 + .../storage/fileio/cache_stream_source.hpp | 1 + src/core/storage/fileio/curl_downloader.cpp | 1 + src/core/storage/fileio/curl_downloader.hpp | 1 + src/core/storage/fileio/dmlcio/filesys.h | 1 + src/core/storage/fileio/dmlcio/io.h | 1 + .../dmlcio/{s3_filesys.cc => s3_filesys.cpp} | 2 +- src/core/storage/fileio/dmlcio/s3_filesys.h | 1 + .../storage/fileio/file_download_cache.cpp | 1 + .../storage/fileio/file_download_cache.hpp | 1 + src/core/storage/fileio/file_handle_pool.cpp | 1 + src/core/storage/fileio/file_handle_pool.hpp | 1 + .../storage/fileio/file_ownership_handle.hpp | 1 + src/core/storage/fileio/fileio_constants.cpp | 1 + src/core/storage/fileio/fileio_constants.hpp | 1 + .../fileio/fixed_size_cache_manager.cpp | 1 + .../fileio/fixed_size_cache_manager.hpp | 1 + src/core/storage/fileio/fs_utils.cpp | 1 + src/core/storage/fileio/fs_utils.hpp | 1 + src/core/storage/fileio/general_fstream.cpp | 1 + src/core/storage/fileio/general_fstream.hpp | 1 + .../storage/fileio/general_fstream_sink.cpp | 1 + .../storage/fileio/general_fstream_sink.hpp | 1 + .../storage/fileio/general_fstream_source.cpp | 1 + .../storage/fileio/general_fstream_source.hpp | 1 + src/core/storage/fileio/get_s3_endpoint.cpp | 1 + src/core/storage/fileio/get_s3_endpoint.hpp | 1 + src/core/storage/fileio/hdfs.cpp | 1 + src/core/storage/fileio/hdfs.hpp | 1 + src/core/storage/fileio/libhdfs_shim.cpp | 5 +- .../storage/fileio/read_caching_device.hpp | 1 + src/core/storage/fileio/s3_api.cpp | 1 + src/core/storage/fileio/s3_api.hpp | 1 + src/core/storage/fileio/s3_fstream.cpp | 1 + src/core/storage/fileio/s3_fstream.hpp | 1 + src/core/storage/fileio/sanitize_url.cpp | 1 + src/core/storage/fileio/sanitize_url.hpp | 1 + src/core/storage/fileio/set_curl_options.cpp | 1 + src/core/storage/fileio/set_curl_options.hpp | 1 + src/core/storage/fileio/temp_files.cpp | 7 +- src/core/storage/fileio/temp_files.hpp | 1 + src/core/storage/fileio/union_fstream.cpp | 1 + src/core/storage/fileio/union_fstream.hpp | 1 + .../storage/lazy_eval/lazy_eval_operation.hpp | 1 + .../lazy_eval/lazy_eval_operation_dag.hpp | 1 + src/core/storage/query_engine/CMakeLists.txt | 23 - .../query_engine/algorithm/ec_permute.cpp | 1 + .../query_engine/algorithm/ec_permute.hpp | 1 + .../query_engine/algorithm/ec_sort.cpp | 1 + .../query_engine/algorithm/ec_sort.hpp | 1 + .../algorithm/groupby_aggregate.cpp | 1 + .../algorithm/groupby_aggregate.hpp | 1 + .../storage/query_engine/algorithm/sort.cpp | 1 + .../storage/query_engine/algorithm/sort.hpp | 1 + .../query_engine/algorithm/sort_and_merge.cpp | 15 +- .../query_engine/algorithm/sort_and_merge.hpp | 1 + .../algorithm/sort_comparator.hpp | 1 + .../query_engine/execution/execution_node.cpp | 1 + .../query_engine/execution/execution_node.hpp | 1 + .../query_engine/execution/query_context.cpp | 1 + .../query_engine/execution/query_context.hpp | 1 + .../execution/subplan_executor.cpp | 1 + .../execution/subplan_executor.hpp | 1 + .../query_engine/operators/all_operators.hpp | 1 + .../storage/query_engine/operators/append.hpp | 1 + .../operators/binary_transform.hpp | 1 + .../query_engine/operators/constant.hpp | 1 + .../operators/generalized_transform.hpp | 1 + .../operators/generalized_union_project.hpp | 1 + .../operators/lambda_transform.hpp | 1 + .../query_engine/operators/logical_filter.hpp | 1 + .../query_engine/operators/operator.hpp | 1 + .../operators/operator_properties.cpp | 1 + .../operators/operator_properties.hpp | 1 + .../operators/operator_transformations.cpp | 1 + .../operators/operator_transformations.hpp | 1 + .../operators/optonly_identity_operator.hpp | 1 + .../query_engine/operators/project.hpp | 1 + .../storage/query_engine/operators/range.hpp | 1 + .../storage/query_engine/operators/reduce.hpp | 1 + .../query_engine/operators/sarray_source.hpp | 1 + .../query_engine/operators/sframe_source.hpp | 1 + .../operators/ternary_operator.hpp | 1 + .../query_engine/operators/transform.hpp | 1 + .../storage/query_engine/operators/union.hpp | 1 + .../planning/materialize_options.hpp | 1 + .../planning/optimization_engine.cpp | 1 + .../planning/optimization_engine.hpp | 1 + .../planning/optimization_node_info.hpp | 1 + .../optimizations/append_transforms.hpp | 1 + .../general_union_project_transforms.hpp | 1 + .../logical_filter_transforms.hpp | 1 + .../optimizations/optimization_transforms.cpp | 1 + .../optimizations/optimization_transforms.hpp | 1 + .../optimizations/project_transforms.hpp | 1 + .../optimizations/source_transforms.hpp | 1 + .../optimizations/union_transforms.hpp | 1 + .../storage/query_engine/planning/planner.cpp | 1 + .../storage/query_engine/planning/planner.hpp | 1 + .../query_engine/planning/planner_node.cpp | 1 + .../query_engine/planning/planner_node.hpp | 1 + .../query_engine/query_engine_lock.cpp | 1 + .../query_engine/query_engine_lock.hpp | 1 + .../storage/query_engine/util/aggregates.hpp | 1 + .../query_engine/util/broadcast_queue.hpp | 1 + src/core/storage/serialization/CMakeLists.txt | 12 - .../storage/serialization/basic_types.hpp | 1 + .../serialization/conditional_serialize.hpp | 1 + .../storage/serialization/dir_archive.cpp | 1 + .../storage/serialization/dir_archive.hpp | 1 + src/core/storage/serialization/has_load.hpp | 1 + src/core/storage/serialization/has_save.hpp | 1 + src/core/storage/serialization/iarchive.hpp | 1 + src/core/storage/serialization/is_pod.hpp | 1 + src/core/storage/serialization/iterator.hpp | 1 + src/core/storage/serialization/list.hpp | 1 + src/core/storage/serialization/map.hpp | 1 + src/core/storage/serialization/oarchive.hpp | 1 + .../serialization/rcpp_serialization.hpp | 1 + .../serialization/serializable_concept.hpp | 1 + .../serialization/serializable_pod.hpp | 1 + .../serialization/serialization_includes.hpp | 1 + src/core/storage/serialization/serialize.hpp | 1 + .../storage/serialization/serialize_eigen.hpp | 1 + .../serialize_to_from_string.hpp | 1 + src/core/storage/serialization/set.hpp | 1 + .../storage/serialization/unordered_map.hpp | 1 + .../storage/serialization/unordered_set.hpp | 1 + .../serialization/unsupported_serialize.hpp | 1 + src/core/storage/serialization/vector.hpp | 1 + src/core/storage/sframe_data/CMakeLists.txt | 40 -- src/core/storage/sframe_data/algorithm.hpp | 1 + .../sframe_data/comma_escape_string.hpp | 1 + .../sframe_data/csv_line_tokenizer.cpp | 1 + .../sframe_data/csv_line_tokenizer.hpp | 1 + src/core/storage/sframe_data/csv_writer.cpp | 1 + src/core/storage/sframe_data/csv_writer.hpp | 1 + src/core/storage/sframe_data/dataframe.cpp | 1 + src/core/storage/sframe_data/dataframe.hpp | 1 + .../sframe_data/group_aggregate_value.cpp | 1 + .../sframe_data/group_aggregate_value.hpp | 1 + src/core/storage/sframe_data/groupby.cpp | 9 +- src/core/storage/sframe_data/groupby.hpp | 9 +- .../storage/sframe_data/groupby_aggregate.cpp | 1 + .../storage/sframe_data/groupby_aggregate.hpp | 1 + .../sframe_data/groupby_aggregate_impl.cpp | 1 + .../sframe_data/groupby_aggregate_impl.hpp | 1 + .../groupby_aggregate_operators.hpp | 1 + src/core/storage/sframe_data/integer_pack.hpp | 1 + .../storage/sframe_data/integer_pack_impl.hpp | 1 + .../storage/sframe_data/is_sarray_like.hpp | 1 + src/core/storage/sframe_data/is_siterable.hpp | 1 + .../storage/sframe_data/is_swriter_base.hpp | 1 + src/core/storage/sframe_data/join.cpp | 1 + src/core/storage/sframe_data/join.hpp | 1 + src/core/storage/sframe_data/join_impl.cpp | 1 + src/core/storage/sframe_data/join_impl.hpp | 1 + .../storage/sframe_data/output_iterator.hpp | 1 + .../sframe_data/parallel_csv_parser.cpp | 1 + .../sframe_data/parallel_csv_parser.hpp | 1 + .../storage/sframe_data/rolling_aggregate.cpp | 1 + .../storage/sframe_data/rolling_aggregate.hpp | 1 + src/core/storage/sframe_data/sarray.hpp | 1 + .../sarray_file_format_interface.hpp | 1 + .../sframe_data/sarray_file_format_v2.hpp | 1 + .../storage/sframe_data/sarray_index_file.cpp | 1 + .../storage/sframe_data/sarray_index_file.hpp | 1 + .../storage/sframe_data/sarray_iterators.hpp | 1 + .../storage/sframe_data/sarray_reader.hpp | 1 + .../sframe_data/sarray_reader_buffer.hpp | 1 + .../storage/sframe_data/sarray_saving.hpp | 1 + .../sframe_data/sarray_sorted_buffer.cpp | 9 +- .../sframe_data/sarray_sorted_buffer.hpp | 13 +- .../sframe_data/sarray_v2_block_manager.cpp | 1 + .../sframe_data/sarray_v2_block_manager.hpp | 1 + .../sframe_data/sarray_v2_block_types.hpp | 1 + .../sframe_data/sarray_v2_block_writer.cpp | 1 + .../sframe_data/sarray_v2_block_writer.hpp | 1 + .../sframe_data/sarray_v2_encoded_block.cpp | 1 + .../sframe_data/sarray_v2_encoded_block.hpp | 1 + .../sframe_data/sarray_v2_type_encoding.cpp | 1 + .../sframe_data/sarray_v2_type_encoding.hpp | 1 + src/core/storage/sframe_data/sframe.cpp | 1 + src/core/storage/sframe_data/sframe.hpp | 1 + .../storage/sframe_data/sframe_compact.cpp | 1 + .../storage/sframe_data/sframe_compact.hpp | 1 + .../sframe_data/sframe_compact_impl.hpp | 1 + .../storage/sframe_data/sframe_config.cpp | 1 + .../storage/sframe_data/sframe_config.hpp | 1 + .../storage/sframe_data/sframe_constants.cpp | 1 + .../storage/sframe_data/sframe_constants.hpp | 1 + .../storage/sframe_data/sframe_index_file.cpp | 1 + .../storage/sframe_data/sframe_index_file.hpp | 1 + src/core/storage/sframe_data/sframe_io.cpp | 7 +- src/core/storage/sframe_data/sframe_io.hpp | 5 +- .../storage/sframe_data/sframe_iterators.cpp | 1 + .../storage/sframe_data/sframe_iterators.hpp | 1 + .../storage/sframe_data/sframe_reader.cpp | 1 + .../storage/sframe_data/sframe_reader.hpp | 1 + .../sframe_data/sframe_reader_buffer.hpp | 1 + src/core/storage/sframe_data/sframe_rows.cpp | 1 + src/core/storage/sframe_data/sframe_rows.hpp | 1 + .../storage/sframe_data/sframe_saving.cpp | 1 + .../storage/sframe_data/sframe_saving.hpp | 1 + .../sframe_data/sframe_saving_impl.cpp | 1 + .../sframe_data/sframe_saving_impl.hpp | 1 + src/core/storage/sframe_data/shuffle.cpp | 9 +- src/core/storage/sframe_data/shuffle.hpp | 1 + src/core/storage/sframe_data/siterable.hpp | 1 + src/core/storage/sframe_data/swriter_base.hpp | 1 + .../storage/sframe_data/testing_utils.cpp | 1 + .../storage/sframe_data/testing_utils.hpp | 1 + src/core/storage/sframe_data/unfair_lock.cpp | 1 + src/core/storage/sframe_data/unfair_lock.hpp | 1 + .../storage/sframe_interface/CMakeLists.txt | 9 - .../storage/sframe_interface/unity_sarray.cpp | 1 + .../storage/sframe_interface/unity_sarray.hpp | 1 + .../unity_sarray_binary_operations.cpp | 1 + .../unity_sarray_binary_operations.hpp | 1 + .../sframe_interface/unity_sarray_builder.cpp | 1 + .../sframe_interface/unity_sarray_builder.hpp | 1 + .../storage/sframe_interface/unity_sframe.cpp | 1 + .../storage/sframe_interface/unity_sframe.hpp | 1 + .../sframe_interface/unity_sframe_builder.cpp | 1 + .../sframe_interface/unity_sframe_builder.hpp | 1 + .../storage/sframe_interface/unity_sgraph.cpp | 1 + .../storage/sframe_interface/unity_sgraph.hpp | 1 + .../unity_sgraph_lazy_ops.hpp | 3 +- src/core/storage/sgraph_data/CMakeLists.txt | 13 - .../storage/sgraph_data/hilbert_curve.hpp | 1 + .../sgraph_data/hilbert_parallel_for.hpp | 1 + src/core/storage/sgraph_data/sgraph.cpp | 1 + src/core/storage/sgraph_data/sgraph.hpp | 1 + .../storage/sgraph_data/sgraph_compute.hpp | 1 + .../sgraph_compute_vertex_block.hpp | 1 + .../storage/sgraph_data/sgraph_constants.cpp | 1 + .../storage/sgraph_data/sgraph_constants.hpp | 1 + .../storage/sgraph_data/sgraph_edge_apply.hpp | 1 + .../storage/sgraph_data/sgraph_engine.hpp | 1 + .../sgraph_data/sgraph_fast_triple_apply.cpp | 1 + .../sgraph_data/sgraph_fast_triple_apply.hpp | 7 +- src/core/storage/sgraph_data/sgraph_io.cpp | 1 + src/core/storage/sgraph_data/sgraph_io.hpp | 3 +- .../sgraph_data/sgraph_synchronize.hpp | 5 +- .../sgraph_synchronize_interface.hpp | 3 +- .../sgraph_data/sgraph_triple_apply.cpp | 1 + .../sgraph_data/sgraph_triple_apply.hpp | 7 +- src/core/storage/sgraph_data/sgraph_types.hpp | 3 +- .../sgraph_data/sgraph_vertex_apply.hpp | 1 + src/core/storage/storage.hpp | 1 + src/core/storage/storage_forward.hpp | 1 + src/core/system/CMakeLists.txt | 6 - src/core/system/cppipc/CMakeLists.txt | 30 -- src/core/system/cppipc/client/comm_client.cpp | 3 +- src/core/system/cppipc/client/comm_client.hpp | 1 + .../cppipc/client/console_cancel_handler.hpp | 1 + .../client/console_cancel_handler_unix.cpp | 6 + .../client/console_cancel_handler_unix.hpp | 1 + .../client/console_cancel_handler_win.cpp | 8 +- .../client/console_cancel_handler_win.hpp | 5 + src/core/system/cppipc/client/issue.hpp | 1 + .../system/cppipc/client/object_proxy.hpp | 1 + .../cppipc/common/authentication_base.hpp | 1 + .../common/authentication_token_method.hpp | 1 + .../system/cppipc/common/ipc_deserializer.cpp | 1 + .../system/cppipc/common/ipc_deserializer.hpp | 1 + .../common/ipc_deserializer_minimal.hpp | 1 + .../system/cppipc/common/message_types.cpp | 1 + .../system/cppipc/common/message_types.hpp | 1 + .../system/cppipc/common/object_factory.cpp | 1 + .../cppipc/common/object_factory_base.hpp | 1 + .../cppipc/common/object_factory_impl.cpp | 1 + .../cppipc/common/object_factory_impl.hpp | 1 + .../cppipc/common/object_factory_proxy.hpp | 1 + .../system/cppipc/common/status_types.hpp | 1 + src/core/system/cppipc/cppipc.hpp | 1 + src/core/system/cppipc/ipc_object_base.cpp | 1 + src/core/system/cppipc/ipc_object_base.hpp | 1 + src/core/system/cppipc/magic_macros.hpp | 1 + .../system/cppipc/registration_macros.hpp | 1 + src/core/system/cppipc/server/cancel_ops.cpp | 1 + src/core/system/cppipc/server/cancel_ops.hpp | 1 + src/core/system/cppipc/server/comm_server.cpp | 1 + src/core/system/cppipc/server/comm_server.hpp | 1 + src/core/system/cppipc/server/dispatch.hpp | 1 + .../system/cppipc/server/dispatch_impl.hpp | 1 + .../generics/member_function_return_type.hpp | 1 + .../system/cppipc/util/generics/tuple.hpp | 1 + src/core/system/exceptions/CMakeLists.txt | 1 - src/core/system/exceptions/error_types.hpp | 1 + src/core/system/lambda/CMakeLists.txt | 30 -- .../system/lambda/graph_lambda_interface.hpp | 1 + src/core/system/lambda/graph_pylambda.cpp | 1 + src/core/system/lambda/graph_pylambda.hpp | 1 + .../system/lambda/graph_pylambda_master.cpp | 1 + .../system/lambda/graph_pylambda_master.hpp | 5 +- src/core/system/lambda/lambda_constants.cpp | 1 + src/core/system/lambda/lambda_constants.hpp | 1 + src/core/system/lambda/lambda_interface.hpp | 1 + src/core/system/lambda/lambda_master.cpp | 3 +- src/core/system/lambda/lambda_master.hpp | 1 + src/core/system/lambda/lambda_utils.hpp | 3 +- src/core/system/lambda/pylambda.cpp | 3 +- src/core/system/lambda/pylambda.hpp | 1 + src/core/system/lambda/pylambda_function.cpp | 1 + src/core/system/lambda/pylambda_function.hpp | 7 +- src/core/system/lambda/pylambda_worker.cpp | 5 +- src/core/system/lambda/pylambda_worker.hpp | 1 + src/core/system/lambda/python_callbacks.cpp | 1 + src/core/system/lambda/python_callbacks.hpp | 1 + src/core/system/lambda/worker_pool.cpp | 1 + src/core/system/lambda/worker_pool.hpp | 21 +- src/core/system/nanosockets/CMakeLists.txt | 21 - .../system/nanosockets/async_reply_socket.cpp | 3 +- .../system/nanosockets/async_reply_socket.hpp | 1 + .../nanosockets/async_request_socket.cpp | 1 + .../nanosockets/async_request_socket.hpp | 1 + .../nanosockets/get_next_port_number.cpp | 1 + .../nanosockets/get_next_port_number.hpp | 1 + .../system/nanosockets/print_zmq_error.cpp | 1 + .../system/nanosockets/print_zmq_error.hpp | 1 + .../system/nanosockets/publish_socket.cpp | 3 +- .../system/nanosockets/publish_socket.hpp | 1 + src/core/system/nanosockets/socket_config.cpp | 1 + src/core/system/nanosockets/socket_config.hpp | 1 + src/core/system/nanosockets/socket_errors.hpp | 1 + .../system/nanosockets/subscribe_socket.cpp | 3 +- .../system/nanosockets/subscribe_socket.hpp | 1 + .../system/nanosockets/zmq_msg_vector.cpp | 1 + .../system/nanosockets/zmq_msg_vector.hpp | 1 + src/core/system/platform/CMakeLists.txt | 13 - .../system/platform/config/CMakeLists.txt | 19 - .../system/platform/config/apple_config.hpp | 1 + .../platform/crash_handler/CMakeLists.txt | 3 - .../platform/crash_handler/crash_handler.cpp | 1 + .../platform/crash_handler/crash_handler.hpp | 1 + .../cross_platform/windows_wrapper.hpp | 1 + .../system/platform/minipsutil/CMakeLists.txt | 4 - .../system/platform/minipsutil/minipsutil.c | 3 +- .../system/platform/minipsutil/minipsutil.h | 1 + .../system/platform/network/CMakeLists.txt | 8 - src/core/system/platform/network/net_util.cpp | 3 +- src/core/system/platform/network/net_util.hpp | 1 + src/core/system/platform/perf/CMakeLists.txt | 9 - src/core/system/platform/perf/memory_info.hpp | 1 + src/core/system/platform/perf/tracepoint.cpp | 3 +- src/core/system/platform/perf/tracepoint.hpp | 5 +- .../system/platform/process/CMakeLists.txt | 17 - src/core/system/platform/process/process.cpp | 3 +- src/core/system/platform/process/process.hpp | 3 +- .../system/platform/process/process_unix.cpp | 7 +- .../system/platform/process/process_util.hpp | 5 +- .../platform/process/process_util_unix.cpp | 7 +- .../platform/process/process_util_win.cpp | 10 +- .../system/platform/process/process_win.cpp | 7 +- .../system/platform/shmipc/CMakeLists.txt | 9 - .../system/platform/shmipc/boost_alt_winapi.h | 3 +- src/core/system/platform/shmipc/shmipc.cpp | 7 +- src/core/system/platform/shmipc/shmipc.hpp | 1 + .../shmipc/shmipc_garbage_collect.cpp | 3 +- .../shmipc/shmipc_garbage_collect.hpp | 1 + .../system/platform/so_utils/CMakeLists.txt | 10 - .../system/platform/so_utils/so_utils.cpp | 3 +- .../system/platform/so_utils/so_utils.hpp | 7 +- src/core/system/platform/timer/CMakeLists.txt | 16 - src/core/system/platform/timer/timer.cpp | 3 +- src/core/system/platform/timer/timer.hpp | 3 +- .../system/startup_teardown/CMakeLists.txt | 8 - .../startup_teardown/startup_teardown.cpp | 5 +- .../startup_teardown/startup_teardown.hpp | 1 + src/core/util/CMakeLists.txt | 19 - src/core/util/any.cpp | 1 + src/core/util/any.hpp | 1 + src/core/util/basic_types.hpp | 1 + src/core/util/bitops.hpp | 1 + src/core/util/blocking_queue.hpp | 1 + src/core/util/boost_property_tree_utils.hpp | 1 + src/core/util/branch_hints.hpp | 1 + src/core/util/buffer_pool.hpp | 1 + src/core/util/charstream.hpp | 1 + src/core/util/cityhash_tc.cpp | 1 + src/core/util/cityhash_tc.hpp | 1 + src/core/util/code_optimization.hpp | 1 + src/core/util/coro.hpp | 1 + src/core/util/dense_bitset.hpp | 1 + .../util/dot_graph_printer/CMakeLists.txt | 1 - src/core/util/dot_graph_printer/dot_graph.hpp | 1 + src/core/util/fast_integer_power.hpp | 1 + src/core/util/fast_multinomial.hpp | 1 + src/core/util/fast_set.hpp | 3 +- src/core/util/fast_top_k.hpp | 1 + src/core/util/file_line_count_estimator.hpp | 1 + src/core/util/fs_util.cpp | 1 + src/core/util/fs_util.hpp | 1 + src/core/util/function_output_iterator.hpp | 1 + src/core/util/hash_value.hpp | 1 + src/core/util/int128_types.hpp | 1 + src/core/util/logit_math.hpp | 1 + src/core/util/lru.hpp | 1 + src/core/util/md5.cpp | 1 + src/core/util/md5.hpp | 1 + src/core/util/mutable_queue.hpp | 1 + src/core/util/resizing_array_sink.hpp | 1 + src/core/util/safe_circular_char_buffer.cpp | 1 + src/core/util/safe_circular_char_buffer.hpp | 1 + src/core/util/stl_util.hpp | 1 + src/core/util/string_util.cpp | 1 + src/core/util/string_util.hpp | 1 + src/core/util/sys_util.hpp | 1 + src/core/util/syserr_reporting.cpp | 3 +- src/core/util/syserr_reporting.hpp | 1 + src/core/util/test_macros.hpp | 1 + src/core/util/testing_utils.cpp | 1 + src/core/util/testing_utils.hpp | 1 + src/core/util/try_finally.hpp | 1 + src/core/util/web_util.cpp | 1 + src/core/util/web_util.hpp | 1 + src/deployment/AudioPreprocessing.h | 1 + src/deployment/TuriCreateObjC.h | 1 + src/deployment/recommender_initialization.cpp | 1 + src/external/CMakeLists.txt | 2 + src/external/Eigen/CMakeLists.txt | 1 - src/external/coremltools_wrap/CMakeLists.txt | 1 - src/external/google/protobuf/CMakeLists.txt | 5 +- src/external/libjson/CMakeLists.txt | 1 - src/external/lz4/CMakeLists.txt | 2 +- src/external/uuid_p/CMakeLists.txt | 2 +- src/external/xgboost/CMakeLists.txt | 4 +- src/external/xgboost/src/gbm/gbm.h | 2 +- src/external/xgboost/src/io/dmlc_simple.cpp | 2 +- src/external/xgboost/src/io/io.cpp | 2 +- src/external/xgboost/src/learner/dmatrix.h | 2 +- .../xgboost/src/learner/learner-inl.hpp | 2 +- src/external/xgboost/src/tree/model.h | 2 +- .../xgboost/src/tree/updater_distcol-inl.hpp | 2 +- src/external/xgboost/src/xgboost_main.cpp | 2 +- .../rabit/include/rabit_serializable.h | 2 +- .../subtree/rabit/src/allreduce_robust.cc | 2 +- .../xgboost/wrapper/xgboost_wrapper.cpp | 2 +- src/ml/CMakeLists.txt | 4 - .../coreml_export/coreml_export_utils.cpp | 5 +- .../coreml_export/coreml_export_utils.hpp | 3 +- .../coreml_export/linear_models_exporter.cpp | 9 +- .../coreml_export/linear_models_exporter.hpp | 3 +- .../coreml_export/mldata_exporter.cpp | 5 +- .../coreml_export/mldata_exporter.hpp | 3 +- .../coreml_export/mlmodel_include.hpp | 1 + .../coreml_export/mlmodel_wrapper.cpp | 7 +- .../coreml_export/mlmodel_wrapper.hpp | 1 + .../neural_net_models_exporter.cpp | 5 +- .../neural_net_models_exporter.hpp | 3 +- .../coreml_export/xgboost_exporter.cpp | 9 +- .../coreml_export/xgboost_exporter.hpp | 3 +- .../evaluation/evaluation_constants.hpp | 1 + .../evaluation/evaluation_interface-inl.hpp | 1 + src/{toolkits => ml}/evaluation/metrics.cpp | 11 +- src/{toolkits => ml}/evaluation/metrics.hpp | 5 +- .../evaluation/unity_evaluation.cpp | 7 +- .../evaluation/unity_evaluation.hpp | 1 + src/{toolkits => ml}/factorization/als.hpp | 5 +- .../factorization/factorization_model.cpp | 9 +- .../factorization/factorization_model.hpp | 3 +- .../factorization_model_impl.hpp | 9 +- .../factorization_model_options.cpp | 7 +- .../factorization_model_serialization.cpp | 5 +- .../factorization_model_sgd_interface.hpp | 5 +- .../factorization_model_training.cpp | 11 +- .../factorization/factors_to_sframe.hpp | 3 +- .../basic_sgd_logistic.cpp | 3 +- .../basic_sgd_squared_error.cpp | 3 +- .../ranking_logistic.cpp | 3 +- .../ranking_squared_error.cpp | 3 +- .../factorization/loss_model_profiles.cpp | 3 +- .../factorization/loss_model_profiles.hpp | 1 + .../factorization/model_factory.cpp | 11 +- .../factorization/model_factory.hpp | 13 +- .../factorization/ranking_sgd_solver_base.hpp | 9 +- .../ranking_sgd_solver_explicit.hpp | 5 +- .../ranking_sgd_solver_implicit.hpp | 5 +- .../factorization/sgd_ranking_interface.hpp | 3 +- src/ml/ml_data/CMakeLists.txt | 21 - src/ml/ml_data/column_indexer.cpp | 1 + src/ml/ml_data/column_indexer.hpp | 1 + src/ml/ml_data/column_statistics.cpp | 1 + src/ml/ml_data/column_statistics.hpp | 1 + .../data_storage/internal_metadata.cpp | 1 + .../data_storage/internal_metadata.hpp | 1 + .../data_storage/ml_data_block_manager.cpp | 1 + .../data_storage/ml_data_block_manager.hpp | 1 + .../data_storage/ml_data_row_format.cpp | 1 + .../data_storage/ml_data_row_format.hpp | 1 + .../data_storage/ml_data_row_translation.cpp | 1 + .../data_storage/ml_data_row_translation.hpp | 1 + src/ml/ml_data/data_storage/util.cpp | 1 + src/ml/ml_data/data_storage/util.hpp | 1 + src/ml/ml_data/metadata.cpp | 1 + src/ml/ml_data/metadata.hpp | 1 + src/ml/ml_data/metadata_impl.hpp | 1 + src/ml/ml_data/ml_data.cpp | 1 + src/ml/ml_data/ml_data.hpp | 1 + src/ml/ml_data/ml_data_column_modes.cpp | 1 + src/ml/ml_data/ml_data_column_modes.hpp | 1 + src/ml/ml_data/ml_data_entry.hpp | 1 + src/ml/ml_data/ml_data_iterator.cpp | 1 + src/ml/ml_data/ml_data_iterator.hpp | 1 + src/ml/ml_data/row_reference.cpp | 1 + src/ml/ml_data/row_reference.hpp | 1 + src/ml/ml_data/testing_utils.cpp | 1 + src/ml/ml_data/testing_utils.hpp | 1 + .../ml_model/python_model.cpp | 1 + .../ml_model/python_model.hpp | 1 + src/ml/neural_net/CMakeLists.txt | 92 ---- src/ml/neural_net/TCMPSImageAugmenting.h | 1 + src/ml/neural_net/compute_context.cpp | 1 + src/ml/neural_net/compute_context.hpp | 1 + src/ml/neural_net/float_array.cpp | 1 + src/ml/neural_net/float_array.hpp | 1 + src/ml/neural_net/image_augmentation.cpp | 1 + src/ml/neural_net/image_augmentation.hpp | 1 + src/ml/neural_net/model_backend.hpp | 1 + src/ml/neural_net/model_spec.cpp | 3 +- src/ml/neural_net/model_spec.hpp | 1 + src/ml/neural_net/mps_cnnmodule.h | 1 + src/ml/neural_net/mps_command_queue.hpp | 1 + src/ml/neural_net/mps_compute_context.hpp | 1 + src/ml/neural_net/mps_descriptor_utils.h | 2 +- src/ml/neural_net/mps_descriptor_utils.m | 2 +- src/ml/neural_net/mps_device_manager.h | 1 + src/ml/neural_net/mps_graph_cnnmodule.h | 1 + src/ml/neural_net/mps_graph_layers.h | 1 + src/ml/neural_net/mps_graph_networks.h | 1 + src/ml/neural_net/mps_graph_trainer.h | 1 + src/ml/neural_net/mps_graph_trainer.mm | 2 +- src/ml/neural_net/mps_image_augmentation.hpp | 1 + src/ml/neural_net/mps_layer_conv_padding.h | 1 + src/ml/neural_net/mps_layer_helper.h | 1 + .../mps_layer_instance_norm_data_loader.h | 1 + src/ml/neural_net/mps_layers.h | 1 + src/ml/neural_net/mps_lstm_helper.h | 1 + src/ml/neural_net/mps_networks.h | 1 + src/ml/neural_net/mps_node_handle.h | 3 +- src/ml/neural_net/mps_node_handle.m | 2 +- src/ml/neural_net/mps_trainer.h | 1 + src/ml/neural_net/mps_updater.h | 1 + src/ml/neural_net/mps_utils.h | 1 + src/ml/neural_net/mps_weight.h | 1 + .../style_transfer/mps_style_transfer.h | 3 +- .../style_transfer/mps_style_transfer.m | 2 +- .../mps_style_transfer_backend.hpp | 3 +- .../mps_style_transfer_backend.mm | 2 +- .../mps_style_transfer_decoding_node.h | 3 +- .../mps_style_transfer_decoding_node.mm | 2 +- .../mps_style_transfer_encoding_node.h | 3 +- .../mps_style_transfer_encoding_node.mm | 2 +- .../mps_style_transfer_pre_processing.h | 3 +- .../mps_style_transfer_pre_processing.m | 2 +- .../mps_style_transfer_residual_node.h | 3 +- .../mps_style_transfer_residual_node.mm | 2 +- .../mps_style_transfer_transformer_network.h | 3 +- .../mps_style_transfer_transformer_network.mm | 2 +- .../style_transfer/mps_style_transfer_utils.h | 3 +- .../style_transfer/mps_style_transfer_utils.m | 2 +- .../mps_style_transfer_vgg_16_block_1_node.h | 3 +- .../mps_style_transfer_vgg_16_block_1_node.mm | 2 +- .../mps_style_transfer_vgg_16_block_2_node.h | 3 +- .../mps_style_transfer_vgg_16_block_2_node.mm | 2 +- .../mps_style_transfer_vgg_16_network.h | 3 +- .../mps_style_transfer_vgg_16_network.m | 2 +- .../mps_style_transfer_weights.h | 1 + src/ml/neural_net/weight_init.cpp | 1 + src/ml/neural_net/weight_init.hpp | 1 + src/ml/optimization/CMakeLists.txt | 12 - .../optimization/accelerated_gradient-inl.hpp | 1 + src/ml/optimization/constraint_interface.hpp | 1 + src/ml/optimization/constraints-inl.hpp | 1 + src/ml/optimization/gradient_descent-inl.hpp | 1 + src/ml/optimization/lbfgs.cpp | 1 + src/ml/optimization/lbfgs.hpp | 1 + src/ml/optimization/line_search-inl.hpp | 1 + src/ml/optimization/newton_method-inl.hpp | 1 + .../optimization/optimization_interface.cpp | 1 + .../optimization/optimization_interface.hpp | 1 + src/ml/optimization/regularizer_interface.hpp | 1 + src/ml/optimization/regularizers-inl.hpp | 1 + src/ml/optimization/utils.cpp | 1 + src/ml/optimization/utils.hpp | 1 + src/{toolkits => ml}/sgd/basic_sgd_solver.hpp | 9 +- src/{toolkits => ml}/sgd/sgd_interface.hpp | 1 + src/{toolkits => ml}/sgd/sgd_solver_base.cpp | 5 +- src/{toolkits => ml}/sgd/sgd_solver_base.hpp | 3 +- src/ml/sketches/CMakeLists.txt | 7 - src/ml/sketches/countmin.hpp | 1 + src/ml/sketches/countsketch.hpp | 1 + src/ml/sketches/hyperloglog.hpp | 1 + src/ml/sketches/quantile_sketch.hpp | 1 + src/ml/sketches/space_saving.hpp | 1 + src/ml/sketches/space_saving_flextype.hpp | 1 + src/ml/sketches/streaming_quantile_sketch.hpp | 1 + src/ml/sketches/unity_sketch.cpp | 1 + src/ml/sketches/unity_sketch.hpp | 1 + .../sparse_similarity/index_mapper.hpp | 1 + .../sparse_similarity/item_processing.hpp | 5 +- .../sparse_similarity/neighbor_search.hpp | 5 +- .../sparse_similarity/similarities.hpp | 1 + .../sliced_itemitem_matrix.cpp | 3 +- .../sliced_itemitem_matrix.hpp | 1 + .../sparse_similarity_lookup.cpp | 7 +- .../sparse_similarity_lookup.hpp | 1 + .../sparse_similarity_lookup_impl.hpp | 15 +- .../sparse_similarity/utilities.hpp | 1 + .../automatic_model_creation.cpp | 19 +- .../automatic_model_creation.hpp | 3 +- .../supervised_learning/boosted_trees.cpp | 5 +- .../supervised_learning/boosted_trees.hpp | 5 +- .../class_registrations.cpp | 21 +- .../class_registrations.hpp | 1 + .../classifier_evaluations.cpp | 3 +- .../classifier_evaluations.hpp | 5 +- .../supervised_learning/decision_tree.cpp | 5 +- .../supervised_learning/decision_tree.hpp | 5 +- .../supervised_learning/linear_regression.cpp | 7 +- .../supervised_learning/linear_regression.hpp | 5 +- .../linear_regression_opt_interface.cpp | 9 +- .../linear_regression_opt_interface.hpp | 7 +- .../supervised_learning/linear_svm.cpp | 9 +- .../supervised_learning/linear_svm.hpp | 5 +- .../linear_svm_opt_interface.cpp | 11 +- .../linear_svm_opt_interface.hpp | 7 +- .../logistic_regression.cpp | 7 +- .../logistic_regression.hpp | 5 +- .../logistic_regression_opt_interface.cpp | 7 +- .../logistic_regression_opt_interface.hpp | 7 +- .../supervised_learning/random_forest.cpp | 5 +- .../supervised_learning/random_forest.hpp | 5 +- .../standardization-inl.hpp | 1 + .../supervised_learning.cpp | 13 +- .../supervised_learning.hpp | 3 +- .../supervised_learning_utils-inl.hpp | 3 +- .../supervised_learning/xgboost.cpp | 13 +- .../supervised_learning/xgboost.hpp | 7 +- .../supervised_learning/xgboost_error.cpp | 1 + .../supervised_learning/xgboost_extension.cpp | 5 +- .../supervised_learning/xgboost_iterator.cpp | 7 +- .../supervised_learning/xgboost_iterator.hpp | 3 +- .../util/algorithmic_utils.hpp | 1 + .../util/class_registrations.cpp | 5 +- .../util/class_registrations.hpp | 1 + src/{toolkits => ml}/util/data_generators.cpp | 3 +- src/{toolkits => ml}/util/data_generators.hpp | 1 + .../util/indexed_sframe_tools.cpp | 3 +- .../util/indexed_sframe_tools.hpp | 1 + .../util/precision_recall.cpp | 5 +- .../util/precision_recall.hpp | 1 + .../util/random_sframe_generation.cpp | 3 +- .../util/random_sframe_generation.hpp | 1 + src/{toolkits => ml}/util/sframe_utils.cpp | 3 +- src/{toolkits => ml}/util/sframe_utils.hpp | 1 + src/{toolkits => ml}/util/spmat.hpp | 1 + src/{toolkits => ml}/util/training_utils.cpp | 5 +- src/{toolkits => ml}/util/training_utils.hpp | 3 +- src/{toolkits => ml}/util/util.dox | 0 .../additional_sframe_utilities.cpp | 1 + .../additional_sframe_utilities.hpp | 1 + .../content_interpretation_extensions.cpp | 1 + .../extensions}/coreml_extension.cpp | 22 +- src/model_server/extensions/gpu_count.cpp | 17 - .../extensions/grouped_sframe.cpp | 1 + .../extensions/grouped_sframe.hpp | 1 + src/model_server/extensions/internal_demo.cpp | 1 + src/model_server/extensions/json/decoder.cpp | 1 + src/model_server/extensions/json/decoder.hpp | 1 + src/model_server/extensions/json/encoder.cpp | 1 + src/model_server/extensions/json/encoder.hpp | 1 + src/model_server/extensions/json/json.cpp | 1 + src/model_server/extensions/json/types.hpp | 1 + .../timeseries/grouped_timeseries.cpp | 1 + .../timeseries/grouped_timeseries.hpp | 1 + .../timeseries/interpolate_value.hpp | 1 + .../extensions/timeseries/registration.cpp | 1 + .../extensions/timeseries/registration.hpp | 1 + .../extensions/timeseries/timeseries.cpp | 1 + .../extensions/timeseries/timeseries.hpp | 1 + .../transform_to_flat_dict_extensions.cpp | 1 + .../lib/api/client_base_types.hpp | 1 + .../lib/api/function_closure_info.cpp | 1 + .../lib/api/function_closure_info.hpp | 1 + .../lib/api/unity_global_interface.hpp | 1 + .../lib/api/unity_graph_interface.hpp | 1 + .../api/unity_sarray_builder_interface.hpp | 1 + .../lib/api/unity_sarray_interface.hpp | 1 + .../api/unity_sframe_builder_interface.hpp | 1 + .../lib/api/unity_sframe_interface.hpp | 1 + .../lib/api/unity_sketch_interface.hpp | 1 + src/model_server/lib/auto_close_sarray.hpp | 1 + src/model_server/lib/extensions/ml_model.cpp | 1 + src/model_server/lib/extensions/ml_model.hpp | 1 + .../lib/extensions/model_base.cpp | 1 + .../lib/extensions/model_base.hpp | 1 + .../lib/extensions/option_info.cpp | 1 + .../lib/extensions/option_info.hpp | 1 + .../lib/extensions/option_manager.cpp | 1 + .../lib/extensions/option_manager.hpp | 1 + src/model_server/lib/flex_dict_view.cpp | 1 + src/model_server/lib/flex_dict_view.hpp | 1 + .../lib/get_toolkit_function_from_closure.cpp | 1 + src/model_server/lib/image_util.cpp | 1 + src/model_server/lib/image_util.hpp | 1 + src/model_server/lib/options_map.hpp | 1 + .../lib/sdk_registration_function_types.hpp | 1 + .../lib/sgraph_triple_apply_typedefs.hpp | 5 +- src/model_server/lib/simple_model.cpp | 1 + src/model_server/lib/simple_model.hpp | 1 + src/model_server/lib/toolkit_class_macros.hpp | 1 + .../lib/toolkit_class_registry.cpp | 1 + .../lib/toolkit_class_registry.hpp | 1 + .../lib/toolkit_class_specification.hpp | 1 + .../lib/toolkit_class_wrapper_impl.hpp | 1 + .../lib/toolkit_function_invocation.hpp | 1 + .../lib/toolkit_function_macros.hpp | 1 + .../lib/toolkit_function_registry.cpp | 1 + .../lib/toolkit_function_registry.hpp | 1 + .../lib/toolkit_function_response.hpp | 1 + .../lib/toolkit_function_specification.hpp | 1 + .../lib/toolkit_function_wrapper_impl.hpp | 1 + src/model_server/lib/toolkit_util.hpp | 1 + src/model_server/lib/unity_base_types.hpp | 1 + src/model_server/lib/unity_global.cpp | 5 +- src/model_server/lib/unity_global.hpp | 1 + .../lib/unity_global_singleton.cpp | 1 + .../lib/unity_global_singleton.hpp | 1 + src/model_server/lib/variant.cpp | 1 + src/model_server/lib/variant.hpp | 1 + src/model_server/lib/variant_converter.cpp | 1 + src/model_server/lib/variant_converter.hpp | 1 + .../lib/variant_deep_serialize.cpp | 1 + .../lib/variant_deep_serialize.hpp | 1 + src/model_server/lib/version.hpp | 1 + src/model_server/lib/version_number.hpp | 1 + src/model_server/server/registration.cpp | 13 +- src/model_server/server/registration.hpp | 1 + src/model_server/server/unity_server.cpp | 1 + src/model_server/server/unity_server.hpp | 1 + .../server/unity_server_control.cpp | 1 + .../server/unity_server_control.hpp | 1 + src/model_server/server/unity_server_init.cpp | 1 + src/model_server/server/unity_server_init.hpp | 1 + .../server/unity_server_options.hpp | 1 + src/python/dummy.c | 1 + src/python/setup.py | 26 +- src/python/turicreate/_cython/cy_graph.pxd | 2 +- src/python/turicreate/_cython/cy_sarray.pxd | 2 +- src/python/turicreate/_cython/cy_sframe.pxd | 2 +- .../_cython/cython_cpp_error_handler.hpp | 1 + .../turicreate/cpp}/tf_compute_context.cpp | 1 + .../turicreate/cpp}/tf_compute_context.hpp | 1 + .../test/test_activity_classifier.py | 50 --- .../test/test_audio_functionality.py | 26 -- .../test/test_drawing_classifier.py | 20 - .../turicreate/test/test_image_classifier.py | 19 - .../turicreate/test/test_image_similarity.py | 19 - .../turicreate/test/test_object_detector.py | 19 - .../test/test_one_shot_object_detector.py | 12 - .../turicreate/test/test_style_transfer.py | 23 - src/python/turicreate/toolkits/_model.py | 27 +- .../_activity_classifier.py | 8 - .../image_classifier/image_classifier.py | 2 +- .../one_shot_object_detector.py | 39 +- src/toolkits/CMakeLists.txt | 23 - .../activity_classification/CMakeLists.txt | 11 - .../ac_data_iterator.cpp | 7 +- .../ac_data_iterator.hpp | 1 + .../activity_classifier.cpp | 9 +- .../activity_classifier.hpp | 3 +- .../class_registrations.cpp | 1 + .../class_registrations.hpp | 1 + src/toolkits/clustering/CMakeLists.txt | 12 - .../clustering/class_registrations.cpp | 1 + .../clustering/class_registrations.hpp | 1 + src/toolkits/clustering/include.hpp | 1 + src/toolkits/clustering/kmeans.cpp | 1 + src/toolkits/clustering/kmeans.hpp | 7 +- src/toolkits/clustering/unity_kmeans.cpp | 1 + src/toolkits/clustering/unity_kmeans.hpp | 1 + src/toolkits/coreml_export/CMakeLists.txt | 14 - .../drawing_classifier/CMakeLists.txt | 12 - .../class_registrations.cpp | 1 + .../class_registrations.hpp | 1 + .../drawing_classifier/data_preparation.cpp | 1 + .../drawing_classifier/data_preparation.hpp | 1 + .../drawing_classifier/dc_data_iterator.cpp | 3 +- .../drawing_classifier/dc_data_iterator.hpp | 1 + .../drawing_classifier/drawing_classifier.cpp | 9 +- .../drawing_classifier/drawing_classifier.hpp | 5 +- src/toolkits/evaluation/CMakeLists.txt | 11 - src/toolkits/factorization/CMakeLists.txt | 18 - .../feature_engineering/CMakeLists.txt | 31 -- src/toolkits/feature_engineering/bm25.cpp | 1 + src/toolkits/feature_engineering/bm25.hpp | 1 + .../categorical_imputer.cpp | 1 + .../categorical_imputer.hpp | 1 + .../class_registrations.cpp | 1 + .../class_registrations.hpp | 1 + .../content_interpretation.cpp | 1 + .../content_interpretation.hpp | 1 + .../feature_engineering/count_featurizer.cpp | 1 + .../feature_engineering/count_featurizer.hpp | 1 + .../feature_engineering/count_thresholder.cpp | 1 + .../feature_engineering/count_thresholder.hpp | 1 + .../dict_transform_utils.cpp | 1 + .../dict_transform_utils.hpp | 1 + .../dimension_reduction.cpp | 3 +- .../dimension_reduction.hpp | 1 + .../feature_engineering/feature_binner.cpp | 1 + .../feature_engineering/feature_binner.hpp | 1 + .../feature_engineering/mean_imputer.cpp | 1 + .../feature_engineering/mean_imputer.hpp | 1 + .../feature_engineering/ngram_counter.cpp | 1 + .../feature_engineering/ngram_counter.hpp | 1 + .../feature_engineering/one_hot_encoder.cpp | 1 + .../feature_engineering/one_hot_encoder.hpp | 1 + .../quadratic_features.cpp | 1 + .../quadratic_features.hpp | 1 + .../sample_transformer.cpp | 1 + .../sample_transformer.hpp | 1 + .../statistics_tracker.cpp | 1 + .../statistics_tracker.hpp | 1 + src/toolkits/feature_engineering/tfidf.cpp | 1 + src/toolkits/feature_engineering/tfidf.hpp | 1 + .../feature_engineering/tokenizer.cpp | 1 + .../feature_engineering/tokenizer.hpp | 1 + .../feature_engineering/topk_indexer.cpp | 1 + .../feature_engineering/topk_indexer.hpp | 1 + .../transform_to_flat_dictionary.cpp | 1 + .../transform_to_flat_dictionary.hpp | 1 + .../feature_engineering/transform_utils.hpp | 1 + .../feature_engineering/transformer_base.hpp | 1 + .../feature_engineering/word_counter.cpp | 1 + .../feature_engineering/word_counter.hpp | 1 + .../feature_engineering/word_trimmer.cpp | 1 + .../feature_engineering/word_trimmer.hpp | 1 + src/toolkits/graph_analytics/CMakeLists.txt | 15 - .../graph_analytics/connected_component.hpp | 1 + .../connected_component_sgraph.cpp | 1 + src/toolkits/graph_analytics/degree_count.cpp | 1 + src/toolkits/graph_analytics/degree_count.hpp | 1 + .../graph_analytics/graph_coloring.hpp | 1 + .../graph_analytics/graph_coloring_sgraph.cpp | 1 + src/toolkits/graph_analytics/include.hpp | 1 + src/toolkits/graph_analytics/kcore.hpp | 1 + src/toolkits/graph_analytics/kcore_sgraph.cpp | 1 + .../graph_analytics/label_propagation.cpp | 1 + .../graph_analytics/label_propagation.hpp | 1 + src/toolkits/graph_analytics/pagerank.hpp | 1 + .../graph_analytics/pagerank_sgraph.cpp | 1 + src/toolkits/graph_analytics/sssp.hpp | 1 + src/toolkits/graph_analytics/sssp_sgraph.cpp | 1 + .../graph_analytics/triangle_counting.hpp | 1 + .../triangle_counting_sgraph.cpp | 1 + src/toolkits/image/CMakeLists.txt | 9 - src/toolkits/image/image_fn_export.cpp | 1 + src/toolkits/image/image_fn_export.hpp | 1 + .../CMakeLists.txt | 31 -- .../class_registrations.cpp | 1 + .../class_registrations.hpp | 1 + .../image_deep_feature_extractor_toolkit.cpp | 1 + .../image_deep_feature_extractor_toolkit.hpp | 1 + .../image_feature_extractor.hpp | 3 +- .../mlmodel_image_feature_extractor.hpp | 1 + .../mlmodel_image_feature_extractor.mm | 2 +- src/toolkits/ml_data_2/CMakeLists.txt | 38 -- src/toolkits/ml_model/CMakeLists.txt | 10 - src/toolkits/nearest_neighbors/CMakeLists.txt | 17 - .../nearest_neighbors/ball_tree_neighbors.cpp | 11 +- .../nearest_neighbors/ball_tree_neighbors.hpp | 1 + .../brute_force_neighbors.cpp | 9 +- .../brute_force_neighbors.hpp | 1 + .../nearest_neighbors/class_registrations.cpp | 1 + .../nearest_neighbors/class_registrations.hpp | 1 + .../nearest_neighbors/distance_functions.hpp | 3 +- src/toolkits/nearest_neighbors/distances.hpp | 3 +- .../nearest_neighbors/hash_map_container.hpp | 1 + src/toolkits/nearest_neighbors/lsh_family.cpp | 3 +- src/toolkits/nearest_neighbors/lsh_family.hpp | 1 + .../nearest_neighbors/lsh_neighbors.cpp | 9 +- .../nearest_neighbors/lsh_neighbors.hpp | 1 + .../nearest_neighbors/nearest_neighbors.cpp | 5 +- .../nearest_neighbors/nearest_neighbors.hpp | 11 +- .../unity_nearest_neighbors.cpp | 1 + .../unity_nearest_neighbors.hpp | 1 + src/toolkits/object_detection/CMakeLists.txt | 19 - .../object_detection/class_registrations.cpp | 1 + .../object_detection/class_registrations.hpp | 1 + .../object_detection/object_detector.cpp | 9 +- .../object_detection/object_detector.hpp | 3 +- .../object_detection/od_data_iterator.cpp | 1 + .../object_detection/od_data_iterator.hpp | 1 + .../object_detection/od_evaluation.cpp | 1 + .../object_detection/od_evaluation.hpp | 1 + .../object_detection/od_serialization.cpp | 1 + .../object_detection/od_serialization.hpp | 1 + src/toolkits/object_detection/od_yolo.cpp | 1 + src/toolkits/object_detection/od_yolo.hpp | 1 + .../one_shot_object_detection/CMakeLists.txt | 14 - .../class_registrations.cpp | 1 + .../class_registrations.hpp | 1 + .../one_shot_object_detector.cpp | 1 + .../one_shot_object_detector.hpp | 3 +- .../util/color_convert.hpp | 1 + .../util/mapping_function.hpp | 1 + .../util/parameter_sampler.cpp | 1 + .../util/parameter_sampler.hpp | 1 + .../util/superposition.cpp | 1 + .../util/superposition.hpp | 1 + src/toolkits/pattern_mining/CMakeLists.txt | 16 - .../pattern_mining/class_registrations.cpp | 1 + .../pattern_mining/class_registrations.hpp | 1 + src/toolkits/pattern_mining/fp_growth.cpp | 3 +- src/toolkits/pattern_mining/fp_growth.hpp | 1 + src/toolkits/pattern_mining/fp_node.cpp | 1 + src/toolkits/pattern_mining/fp_node.hpp | 1 + .../pattern_mining/fp_results_tree.cpp | 1 + .../pattern_mining/fp_results_tree.hpp | 1 + src/toolkits/pattern_mining/fp_tree.cpp | 1 + src/toolkits/pattern_mining/fp_tree.hpp | 1 + .../pattern_mining/fp_tree_header.cpp | 1 + .../pattern_mining/fp_tree_header.hpp | 1 + src/toolkits/pattern_mining/rule_mining.cpp | 1 + src/toolkits/pattern_mining/rule_mining.hpp | 1 + src/toolkits/prototype/CMakeLists.txt | 9 - .../prototype/class_registrations.cpp | 1 + .../prototype/class_registrations.hpp | 1 + src/toolkits/prototype/sparse_nn.cpp | 1 + src/toolkits/prototype/sparse_nn.hpp | 1 + src/toolkits/recsys/CMakeLists.txt | 25 -- src/toolkits/recsys/class_registrations.cpp | 1 + src/toolkits/recsys/class_registrations.hpp | 1 + .../data_storage/internal_metadata.cpp | 7 +- .../data_storage/internal_metadata.hpp | 5 +- .../data_storage/ml_data_block_manager.cpp | 5 +- .../data_storage/ml_data_block_manager.hpp | 3 +- .../data_storage/ml_data_row_format.cpp | 7 +- .../data_storage/ml_data_row_format.hpp | 9 +- .../data_storage/ml_data_row_translation.cpp | 9 +- .../data_storage/ml_data_row_translation.hpp | 11 +- .../ml_data_side_feature_translation.hpp | 9 +- .../ml_data_2/data_storage/util.cpp | 5 +- .../ml_data_2/data_storage/util.hpp | 3 +- .../ml_data_2/indexing/column_indexer.cpp | 5 +- .../ml_data_2/indexing/column_indexer.hpp | 3 +- .../indexing/column_unique_indexer.cpp | 3 +- .../indexing/column_unique_indexer.hpp | 3 +- .../iterators/composite_row_type.cpp | 5 +- .../iterators/composite_row_type.hpp | 5 +- .../iterators/ml_data_block_iterator.cpp | 3 +- .../iterators/ml_data_block_iterator.hpp | 3 +- .../ml_data_2/iterators/ml_data_iterator.hpp | 3 +- .../iterators/ml_data_iterator_base.cpp | 3 +- .../iterators/ml_data_iterator_base.hpp | 13 +- .../ml_data_2/iterators/row_reference.hpp | 9 +- .../{ => recsys}/ml_data_2/metadata.cpp | 3 +- .../{ => recsys}/ml_data_2/metadata.hpp | 7 +- .../{ => recsys}/ml_data_2/metadata_impl.hpp | 3 +- .../{ => recsys}/ml_data_2/ml_data.cpp | 15 +- .../{ => recsys}/ml_data_2/ml_data.hpp | 11 +- .../ml_data_2/ml_data_column_modes.cpp | 3 +- .../ml_data_2/ml_data_column_modes.hpp | 1 + .../{ => recsys}/ml_data_2/ml_data_entry.hpp | 1 + .../ml_data_2/ml_data_iterators.hpp | 5 +- .../ml_data_2/ml_data_missing_values.cpp | 3 +- .../ml_data_2/ml_data_missing_values.hpp | 1 + .../ml_data_2/ml_data_options.cpp | 3 +- .../{ => recsys}/ml_data_2/ml_data_setup.cpp | 9 +- .../ml_data_2/ml_data_sorting.cpp | 11 +- .../ml_data_2/row_slicing_utilities.cpp | 3 +- .../ml_data_2/row_slicing_utilities.hpp | 3 +- .../ml_data_2/sframe_index_mapping.cpp | 5 +- .../ml_data_2/sframe_index_mapping.hpp | 5 +- .../{ => recsys}/ml_data_2/side_features.cpp | 7 +- .../{ => recsys}/ml_data_2/side_features.hpp | 9 +- .../ml_data_2/standardization-inl.hpp | 5 +- .../statistics/basic_column_statistics.cpp | 5 +- .../statistics/basic_column_statistics.hpp | 3 +- .../statistics/column_statistics.cpp | 5 +- .../statistics/column_statistics.hpp | 3 +- .../{ => recsys}/ml_data_2/testing_utils.cpp | 3 +- .../{ => recsys}/ml_data_2/testing_utils.hpp | 5 +- src/toolkits/recsys/models.hpp | 1 + .../recsys/models/factorization_models.cpp | 5 +- .../recsys/models/factorization_models.hpp | 1 + .../models/item_content_recommender.hpp | 1 + src/toolkits/recsys/models/itemcf.cpp | 21 +- src/toolkits/recsys/models/itemcf.hpp | 3 +- src/toolkits/recsys/models/popularity.cpp | 13 +- src/toolkits/recsys/models/popularity.hpp | 1 + src/toolkits/recsys/recsys_model_base.cpp | 19 +- src/toolkits/recsys/recsys_model_base.hpp | 7 +- src/toolkits/recsys/train_test_split.cpp | 7 +- src/toolkits/recsys/train_test_split.hpp | 1 + src/toolkits/recsys/user_item_graph.cpp | 7 +- src/toolkits/recsys/user_item_graph.hpp | 3 +- src/toolkits/recsys/user_item_lists.cpp | 5 +- src/toolkits/recsys/user_item_lists.hpp | 3 +- src/toolkits/sgd/CMakeLists.txt | 8 - src/toolkits/sparse_similarity/CMakeLists.txt | 10 - src/toolkits/style_transfer/CMakeLists.txt | 13 - .../style_transfer/class_registrations.cpp | 3 +- .../style_transfer/class_registrations.hpp | 1 + .../style_transfer/style_transfer.cpp | 5 +- .../style_transfer/style_transfer.hpp | 5 +- .../style_transfer_data_iterator.cpp | 3 +- .../style_transfer_data_iterator.hpp | 1 + .../style_transfer_model_definition.cpp | 5 +- .../style_transfer_model_definition.hpp | 3 +- .../supervised_learning/CMakeLists.txt | 31 -- src/toolkits/text/CMakeLists.txt | 18 - src/toolkits/text/alias.cpp | 15 +- src/toolkits/text/alias.hpp | 3 +- src/toolkits/text/cgs.cpp | 13 +- src/toolkits/text/cgs.hpp | 3 +- src/toolkits/text/class_registrations.cpp | 1 + src/toolkits/text/class_registrations.hpp | 1 + src/toolkits/text/perplexity.cpp | 3 +- src/toolkits/text/perplexity.hpp | 1 + src/toolkits/text/scvb.hpp | 1 + src/toolkits/text/topic_model.cpp | 15 +- src/toolkits/text/topic_model.hpp | 5 +- src/toolkits/text/unity_text.cpp | 3 +- src/toolkits/text/unity_text.hpp | 1 + src/toolkits/util/CMakeLists.txt | 14 - src/visualization/annotation/CMakeLists.txt | 4 +- .../annotation/annotation_base.cpp | 1 + .../annotation/annotation_base.hpp | 1 + .../annotation/build/format/cpp/annotate.pb.h | 1 + .../annotation/build/format/cpp/data.pb.h | 1 + .../annotation/build/format/cpp/message.pb.h | 1 + .../annotation/build/format/cpp/meta.pb.h | 1 + .../annotation/build/format/cpp/progress.pb.h | 1 + .../build/format/cpp/similarity.pb.h | 1 + .../annotation/class_registrations.cpp | 1 + .../annotation/class_registrations.hpp | 1 + .../annotation/image_classification.cpp | 1 + .../annotation/image_classification.hpp | 1 + .../annotation/object_detection.cpp | 1 + .../annotation/object_detection.hpp | 3 +- src/visualization/annotation/utils.cpp | 1 + src/visualization/annotation/utils.hpp | 1 + src/visualization/client/src/V8Handler.h | 1 + src/visualization/client/src/handler.cpp | 1 + src/visualization/client/src/handler.h | 1 + .../client/src/handler_linux.cpp | 1 + .../client/src/javascript_caller.cpp | 1 + .../client/src/javascript_caller.h | 1 + src/visualization/client/src/layer.cpp | 1 + src/visualization/client/src/layer.h | 1 + src/visualization/client/src/main.cpp | 1 + src/visualization/client/src/pipe.cpp | 1 + src/visualization/client/src/pipe.h | 1 + src/visualization/server/CMakeLists.txt | 4 +- src/visualization/server/batch_size.cpp | 1 + src/visualization/server/batch_size.hpp | 1 + .../server/boxes_and_whiskers.cpp | 1 + .../server/boxes_and_whiskers.hpp | 1 + .../server/categorical_heatmap.cpp | 1 + .../server/categorical_heatmap.hpp | 1 + .../server/columnwise_summary.cpp | 1 + .../server/columnwise_summary.hpp | 1 + src/visualization/server/dark_mode.cpp | 1 + src/visualization/server/dark_mode.hpp | 1 + src/visualization/server/escape.cpp | 1 + src/visualization/server/escape.hpp | 1 + src/visualization/server/extrema.hpp | 1 + src/visualization/server/groupby.cpp | 1 + src/visualization/server/groupby.hpp | 1 + src/visualization/server/heatmap.cpp | 1 + src/visualization/server/heatmap.hpp | 1 + src/visualization/server/histogram.cpp | 1 + src/visualization/server/histogram.hpp | 1 + src/visualization/server/io_buffer.cpp | 1 + src/visualization/server/io_buffer.hpp | 1 + src/visualization/server/item_frequency.cpp | 1 + src/visualization/server/item_frequency.hpp | 1 + src/visualization/server/plot.cpp | 1 + src/visualization/server/plot.hpp | 1 + src/visualization/server/process_wrapper.cpp | 1 + src/visualization/server/process_wrapper.hpp | 3 +- src/visualization/server/registration.cpp | 1 + src/visualization/server/scatter.cpp | 1 + src/visualization/server/scatter.hpp | 1 + src/visualization/server/server.cpp | 1 + src/visualization/server/server.hpp | 1 + src/visualization/server/show.cpp | 1 + src/visualization/server/show.hpp | 1 + src/visualization/server/summary_view.cpp | 1 + src/visualization/server/summary_view.hpp | 1 + src/visualization/server/table.cpp | 3 +- src/visualization/server/table.hpp | 1 + src/visualization/server/thread.cpp | 1 + src/visualization/server/thread.hpp | 1 + src/visualization/server/transformation.cpp | 1 + src/visualization/server/transformation.hpp | 1 + src/visualization/server/vega_data.cpp | 1 + src/visualization/server/vega_data.hpp | 1 + src/visualization/server/vega_spec.cpp | 1 + src/visualization/server/vega_spec.hpp | 1 + .../vega_renderer/CMakeLists.txt | 26 +- src/visualization/vega_renderer/JSConsole.h | 2 +- src/visualization/vega_renderer/LogProxy.h | 1 + .../vega_renderer/LogProxyHandler.h | 2 +- .../vega_renderer/TCVegaCGColorMap.h | 1 + .../vega_renderer/TCVegaCSSCursorMap.h | 1 + .../vega_renderer/TCVegaHTMLElement.h | 1 + .../vega_renderer/TCVegaJSCanvas.h | 1 + .../vega_renderer/TCVegaJSConsole.h | 1 + .../vega_renderer/TCVegaJSDocument.h | 1 + .../vega_renderer/TCVegaJSMouseEvent.h | 1 + .../vega_renderer/TCVegaLogProxy.h | 1 + .../vega_renderer/TCVegaLogProxyHandling.h | 1 + .../vega_renderer/TCVegaLogger.h | 1 + .../vega_renderer/TCVegaPortabilityTypes.h | 1 + .../vega_renderer/TCVegaRenderer.h | 1 + test/annotation/object_detector_tests.cxx | 2 +- test/benchmark/sframe_bench.cpp | 2 +- test/capi/capi_sframe.cxx | 2 +- test/fileio/fstream_bench.cpp | 2 +- .../flexible_type/flexible_datatype_bench.cpp | 2 +- test/lambda/dummy_worker.cpp | 2 +- test/ml_data/dml_load_from_ml_data_2.cxx | 2 +- test/network/get_ip.cpp | 2 +- test/parallel/thread_tools.cxx | 2 +- test/process/process_launch_test.cxx | 2 +- test/random/test_alias.cpp | 2 +- test/sframe/sarray_file_format_v2_test.cxx | 2 +- test/sframe/shuffle_test.cxx | 2 +- test/sgraph/sgraph_bench.cpp | 4 +- test/shmipc/shm_ping_client_test.cpp | 2 +- test/shmipc/shm_ping_server_test.cpp | 2 +- test/shmipc/shmipc_test.cxx | 2 +- test/sketches/count_sketches_test.cxx | 2 +- test/timer/timer_test.cxx | 2 +- .../test_style_transfer_data_iterator.cxx | 2 +- .../test_style_transfer_model_definition.cxx | 2 +- test/toolkits/style_transfer/utils.cpp | 4 +- .../boosted_trees_classifier_tests.cxx | 4 +- .../classifier_evaluation.cxx | 2 +- .../linear_regression_bench.cxx | 4 +- .../linear_regression_tests.cxx | 4 +- .../supervised_learning/linear_svm_tests.cxx | 8 +- .../logistic_regression_tests.cxx | 4 +- .../random_forest_tests.cxx | 2 +- .../supervised_learning_mock_test_model.cxx | 2 +- .../test_neural_nets_model_exporter.cxx | 4 +- .../drawing_classifier/dc_data_utils.hpp | 2 +- .../drawing_classifier/test_dc_evaluation.cxx | 6 +- .../drawing_classifier/test_dc_prediction.cxx | 2 +- .../test_dc_serialization.cxx | 4 +- .../toolkits/ml_data_2/basic_iteration.cxx | 12 +- .../toolkits/ml_data_2/basic_storage.cxx | 12 +- .../ml_data_2/composite_row_tools.cxx | 6 +- .../toolkits/ml_data_2/creation_methods.cxx | 12 +- .../unity/toolkits/ml_data_2/metadata_api.cxx | 12 +- .../ml_data_2/metadata_column_operations.cxx | 12 +- .../toolkits/ml_data_2/missing_values.cxx | 12 +- test/unity/toolkits/ml_data_2/numerics.cxx | 12 +- .../toolkits/ml_data_2/row_references.cxx | 12 +- test/unity/toolkits/ml_data_2/row_slicing.cxx | 6 +- test/unity/toolkits/ml_data_2/save_load.cxx | 12 +- .../toolkits/ml_data_2/schema_errors.cxx | 12 +- .../toolkits/ml_data_2/side_features.cxx | 6 +- .../toolkits/ml_data_2/sorting_and_blocks.cxx | 12 +- .../standardization_interface_tests.cxx | 4 +- .../unity/toolkits/ml_data_2/time_ml_data.cpp | 12 +- .../ml_data_2/untranslated_columns.cxx | 12 +- test/unity/toolkits/nearest_neighbors.cxx | 12 +- .../toolkits/neural_net/test_model_spec.cxx | 2 +- .../test_od_serialization.cxx | 2 +- .../object_detection/test_od_yolo.cxx | 2 +- test/unity/toolkits/recsys/algo_utils.cxx | 2 +- .../toolkits/recsys/factorization_machine.cxx | 6 +- .../recsys/factorization_test_helpers.hpp | 6 +- .../toolkits/recsys/get_similar_tests.cxx | 2 +- test/unity/toolkits/recsys/itemcf.cxx | 8 +- .../toolkits/recsys/itemcf_stress_test.cpp | 8 +- .../toolkits/recsys/matrix_factorization.cxx | 6 +- test/unity/toolkits/recsys/new_user_tests.cxx | 8 +- test/unity/toolkits/recsys/popularity.cxx | 4 +- .../toolkits/recsys/ranking_model_tests.cxx | 6 +- .../toolkits/recsys/recommend_function.cxx | 4 +- .../recsys/test_memory_consumption.cpp | 6 +- .../toolkits/recsys/train_test_split.cxx | 4 +- .../unity/toolkits/recsys/user_item_lists.cxx | 8 +- test/unity/toolkits/recsys/weird_segments.cxx | 6 +- .../test_brute_force_all_pairs.cxx | 2 +- .../test_item_similarity_consistency.cxx | 4 +- .../test_itemitem_matrix.cxx | 2 +- .../test_matrix_utilities.cxx | 2 +- .../sparse_similarity/test_transpose.cxx | 2 +- .../synthetic_timings/time_data_loading.cpp | 8 +- .../time_nearest_neighbors.cpp | 12 +- .../synthetic_timings/time_recsys_model.hpp | 6 +- test/unity/toolkits/test_evaluation.cxx | 2 +- test/unity/toolkits/topic_model.cxx | 2 +- test/util/test_crash_handler.cpp | 2 +- test/util/time_fast_power.cpp | 2 +- 1387 files changed, 2599 insertions(+), 2579 deletions(-) create mode 100755 build_python_wheel.sh delete mode 100644 src/CMakeLists.txt delete mode 100644 src/core/CMakeLists.txt delete mode 100644 src/core/data/CMakeLists.txt delete mode 100644 src/core/data/flexible_type/CMakeLists.txt delete mode 100644 src/core/data/image/CMakeLists.txt rename src/core/data/image/{io_example.cpp => io_example.cxx} (100%) delete mode 100644 src/core/data/json/CMakeLists.txt delete mode 100644 src/core/data/sframe/CMakeLists.txt delete mode 100644 src/core/generics/CMakeLists.txt delete mode 100644 src/core/globals/CMakeLists.txt delete mode 100644 src/core/logging/CMakeLists.txt delete mode 100644 src/core/logging/table_printer/CMakeLists.txt rename src/core/logging/table_printer/{table_printer_examples.cpp => table_printer_examples.cxx} (98%) delete mode 100644 src/core/parallel/CMakeLists.txt delete mode 100644 src/core/random/CMakeLists.txt delete mode 100644 src/core/storage/CMakeLists.txt delete mode 100644 src/core/storage/fileio/CMakeLists.txt rename src/core/storage/fileio/dmlcio/{s3_filesys.cc => s3_filesys.cpp} (99%) delete mode 100644 src/core/storage/query_engine/CMakeLists.txt delete mode 100644 src/core/storage/serialization/CMakeLists.txt delete mode 100644 src/core/storage/sframe_data/CMakeLists.txt delete mode 100644 src/core/storage/sframe_interface/CMakeLists.txt delete mode 100644 src/core/storage/sgraph_data/CMakeLists.txt delete mode 100644 src/core/system/CMakeLists.txt delete mode 100644 src/core/system/cppipc/CMakeLists.txt delete mode 100644 src/core/system/exceptions/CMakeLists.txt delete mode 100644 src/core/system/lambda/CMakeLists.txt delete mode 100644 src/core/system/nanosockets/CMakeLists.txt delete mode 100644 src/core/system/platform/CMakeLists.txt delete mode 100644 src/core/system/platform/config/CMakeLists.txt delete mode 100644 src/core/system/platform/crash_handler/CMakeLists.txt delete mode 100644 src/core/system/platform/minipsutil/CMakeLists.txt delete mode 100644 src/core/system/platform/network/CMakeLists.txt delete mode 100644 src/core/system/platform/perf/CMakeLists.txt delete mode 100644 src/core/system/platform/process/CMakeLists.txt delete mode 100644 src/core/system/platform/shmipc/CMakeLists.txt delete mode 100644 src/core/system/platform/so_utils/CMakeLists.txt delete mode 100644 src/core/system/platform/timer/CMakeLists.txt delete mode 100644 src/core/system/startup_teardown/CMakeLists.txt delete mode 100644 src/core/util/CMakeLists.txt delete mode 100644 src/core/util/dot_graph_printer/CMakeLists.txt delete mode 100644 src/ml/CMakeLists.txt rename src/{toolkits => ml}/coreml_export/coreml_export_utils.cpp (91%) rename src/{toolkits => ml}/coreml_export/coreml_export_utils.hpp (90%) rename src/{toolkits => ml}/coreml_export/linear_models_exporter.cpp (97%) rename src/{toolkits => ml}/coreml_export/linear_models_exporter.hpp (96%) rename src/{toolkits => ml}/coreml_export/mldata_exporter.cpp (98%) rename src/{toolkits => ml}/coreml_export/mldata_exporter.hpp (92%) rename src/{toolkits => ml}/coreml_export/mlmodel_include.hpp (98%) rename src/{toolkits => ml}/coreml_export/mlmodel_wrapper.cpp (75%) rename src/{toolkits => ml}/coreml_export/mlmodel_wrapper.hpp (97%) rename src/{toolkits => ml}/coreml_export/neural_net_models_exporter.cpp (99%) rename src/{toolkits => ml}/coreml_export/neural_net_models_exporter.hpp (97%) rename src/{toolkits => ml}/coreml_export/xgboost_exporter.cpp (97%) rename src/{toolkits => ml}/coreml_export/xgboost_exporter.hpp (93%) rename src/{toolkits => ml}/evaluation/evaluation_constants.hpp (95%) rename src/{toolkits => ml}/evaluation/evaluation_interface-inl.hpp (99%) rename src/{toolkits => ml}/evaluation/metrics.cpp (98%) rename src/{toolkits => ml}/evaluation/metrics.hpp (96%) rename src/{toolkits => ml}/evaluation/unity_evaluation.cpp (95%) rename src/{toolkits => ml}/evaluation/unity_evaluation.hpp (97%) rename src/{toolkits => ml}/factorization/als.hpp (99%) rename src/{toolkits => ml}/factorization/factorization_model.cpp (95%) rename src/{toolkits => ml}/factorization/factorization_model.hpp (99%) rename src/{toolkits => ml}/factorization/factorization_model_impl.hpp (99%) rename src/{toolkits => ml}/factorization/factorization_model_options.cpp (98%) rename src/{toolkits => ml}/factorization/factorization_model_serialization.cpp (97%) rename src/{toolkits => ml}/factorization/factorization_model_sgd_interface.hpp (99%) rename src/{toolkits => ml}/factorization/factorization_model_training.cpp (91%) rename src/{toolkits => ml}/factorization/factors_to_sframe.hpp (98%) rename src/{toolkits => ml}/factorization/factory_instantiations/basic_sgd_logistic.cpp (83%) rename src/{toolkits => ml}/factorization/factory_instantiations/basic_sgd_squared_error.cpp (83%) rename src/{toolkits => ml}/factorization/factory_instantiations/ranking_logistic.cpp (84%) rename src/{toolkits => ml}/factorization/factory_instantiations/ranking_squared_error.cpp (84%) rename src/{toolkits => ml}/factorization/loss_model_profiles.cpp (93%) rename src/{toolkits => ml}/factorization/loss_model_profiles.hpp (99%) rename src/{toolkits => ml}/factorization/model_factory.cpp (84%) rename src/{toolkits => ml}/factorization/model_factory.hpp (97%) rename src/{toolkits => ml}/factorization/ranking_sgd_solver_base.hpp (99%) rename src/{toolkits => ml}/factorization/ranking_sgd_solver_explicit.hpp (99%) rename src/{toolkits => ml}/factorization/ranking_sgd_solver_implicit.hpp (98%) rename src/{toolkits => ml}/factorization/sgd_ranking_interface.hpp (94%) delete mode 100644 src/ml/ml_data/CMakeLists.txt rename src/{toolkits => ml}/ml_model/python_model.cpp (99%) rename src/{toolkits => ml}/ml_model/python_model.hpp (96%) delete mode 100644 src/ml/neural_net/CMakeLists.txt delete mode 100644 src/ml/optimization/CMakeLists.txt rename src/{toolkits => ml}/sgd/basic_sgd_solver.hpp (96%) rename src/{toolkits => ml}/sgd/sgd_interface.hpp (99%) rename src/{toolkits => ml}/sgd/sgd_solver_base.cpp (99%) rename src/{toolkits => ml}/sgd/sgd_solver_base.hpp (99%) delete mode 100644 src/ml/sketches/CMakeLists.txt rename src/{toolkits => ml}/sparse_similarity/index_mapper.hpp (99%) rename src/{toolkits => ml}/sparse_similarity/item_processing.hpp (98%) rename src/{toolkits => ml}/sparse_similarity/neighbor_search.hpp (99%) rename src/{toolkits => ml}/sparse_similarity/similarities.hpp (99%) rename src/{toolkits => ml}/sparse_similarity/sliced_itemitem_matrix.cpp (96%) rename src/{toolkits => ml}/sparse_similarity/sliced_itemitem_matrix.hpp (99%) rename src/{toolkits => ml}/sparse_similarity/sparse_similarity_lookup.cpp (95%) rename src/{toolkits => ml}/sparse_similarity/sparse_similarity_lookup.hpp (99%) rename src/{toolkits => ml}/sparse_similarity/sparse_similarity_lookup_impl.hpp (99%) rename src/{toolkits => ml}/sparse_similarity/utilities.hpp (99%) rename src/{toolkits => ml}/supervised_learning/automatic_model_creation.cpp (95%) rename src/{toolkits => ml}/supervised_learning/automatic_model_creation.hpp (93%) rename src/{toolkits => ml}/supervised_learning/boosted_trees.cpp (98%) rename src/{toolkits => ml}/supervised_learning/boosted_trees.hpp (96%) rename src/{toolkits => ml}/supervised_learning/class_registrations.cpp (70%) rename src/{toolkits => ml}/supervised_learning/class_registrations.hpp (94%) rename src/{toolkits => ml}/supervised_learning/classifier_evaluations.cpp (97%) rename src/{toolkits => ml}/supervised_learning/classifier_evaluations.hpp (75%) rename src/{toolkits => ml}/supervised_learning/decision_tree.cpp (97%) rename src/{toolkits => ml}/supervised_learning/decision_tree.hpp (95%) rename src/{toolkits => ml}/supervised_learning/linear_regression.cpp (98%) rename src/{toolkits => ml}/supervised_learning/linear_regression.hpp (96%) rename src/{toolkits => ml}/supervised_learning/linear_regression_opt_interface.cpp (97%) rename src/{toolkits => ml}/supervised_learning/linear_regression_opt_interface.hpp (96%) rename src/{toolkits => ml}/supervised_learning/linear_svm.cpp (97%) rename src/{toolkits => ml}/supervised_learning/linear_svm.hpp (97%) rename src/{toolkits => ml}/supervised_learning/linear_svm_opt_interface.cpp (96%) rename src/{toolkits => ml}/supervised_learning/linear_svm_opt_interface.hpp (96%) rename src/{toolkits => ml}/supervised_learning/logistic_regression.cpp (99%) rename src/{toolkits => ml}/supervised_learning/logistic_regression.hpp (97%) rename src/{toolkits => ml}/supervised_learning/logistic_regression_opt_interface.cpp (98%) rename src/{toolkits => ml}/supervised_learning/logistic_regression_opt_interface.hpp (96%) rename src/{toolkits => ml}/supervised_learning/random_forest.cpp (98%) rename src/{toolkits => ml}/supervised_learning/random_forest.hpp (96%) rename src/{toolkits => ml}/supervised_learning/standardization-inl.hpp (99%) rename src/{toolkits => ml}/supervised_learning/supervised_learning.cpp (99%) rename src/{toolkits => ml}/supervised_learning/supervised_learning.hpp (99%) rename src/{toolkits => ml}/supervised_learning/supervised_learning_utils-inl.hpp (99%) rename src/{toolkits => ml}/supervised_learning/xgboost.cpp (99%) rename src/{toolkits => ml}/supervised_learning/xgboost.hpp (98%) rename src/{toolkits => ml}/supervised_learning/xgboost_error.cpp (98%) rename src/{toolkits => ml}/supervised_learning/xgboost_extension.cpp (93%) rename src/{toolkits => ml}/supervised_learning/xgboost_iterator.cpp (99%) rename src/{toolkits => ml}/supervised_learning/xgboost_iterator.hpp (94%) rename src/{toolkits => ml}/util/algorithmic_utils.hpp (99%) rename src/{toolkits => ml}/util/class_registrations.cpp (86%) rename src/{toolkits => ml}/util/class_registrations.hpp (96%) rename src/{toolkits => ml}/util/data_generators.cpp (99%) rename src/{toolkits => ml}/util/data_generators.hpp (99%) rename src/{toolkits => ml}/util/indexed_sframe_tools.cpp (98%) rename src/{toolkits => ml}/util/indexed_sframe_tools.hpp (98%) rename src/{toolkits => ml}/util/precision_recall.cpp (97%) rename src/{toolkits => ml}/util/precision_recall.hpp (99%) rename src/{toolkits => ml}/util/random_sframe_generation.cpp (99%) rename src/{toolkits => ml}/util/random_sframe_generation.hpp (97%) rename src/{toolkits => ml}/util/sframe_utils.cpp (97%) rename src/{toolkits => ml}/util/sframe_utils.hpp (99%) rename src/{toolkits => ml}/util/spmat.hpp (99%) rename src/{toolkits => ml}/util/training_utils.cpp (92%) rename src/{toolkits => ml}/util/training_utils.hpp (90%) rename src/{toolkits => ml}/util/util.dox (100%) rename src/{toolkits/feature_engineering => model_server/extensions}/content_interpretation_extensions.cpp (97%) rename src/{toolkits/coreml_export => model_server/extensions}/coreml_extension.cpp (89%) delete mode 100644 src/model_server/extensions/gpu_count.cpp rename src/{toolkits/feature_engineering => model_server/extensions}/transform_to_flat_dict_extensions.cpp (98%) rename src/{ml/neural_net => python/turicreate/cpp}/tf_compute_context.cpp (99%) rename src/{ml/neural_net => python/turicreate/cpp}/tf_compute_context.hpp (98%) delete mode 100644 src/toolkits/CMakeLists.txt delete mode 100644 src/toolkits/activity_classification/CMakeLists.txt delete mode 100644 src/toolkits/clustering/CMakeLists.txt delete mode 100644 src/toolkits/coreml_export/CMakeLists.txt delete mode 100644 src/toolkits/drawing_classifier/CMakeLists.txt delete mode 100644 src/toolkits/evaluation/CMakeLists.txt delete mode 100644 src/toolkits/factorization/CMakeLists.txt delete mode 100644 src/toolkits/feature_engineering/CMakeLists.txt delete mode 100644 src/toolkits/graph_analytics/CMakeLists.txt delete mode 100644 src/toolkits/image/CMakeLists.txt delete mode 100644 src/toolkits/image_deep_feature_extractor/CMakeLists.txt delete mode 100644 src/toolkits/ml_data_2/CMakeLists.txt delete mode 100644 src/toolkits/ml_model/CMakeLists.txt delete mode 100644 src/toolkits/nearest_neighbors/CMakeLists.txt delete mode 100644 src/toolkits/object_detection/CMakeLists.txt delete mode 100644 src/toolkits/object_detection/one_shot_object_detection/CMakeLists.txt delete mode 100644 src/toolkits/pattern_mining/CMakeLists.txt delete mode 100644 src/toolkits/prototype/CMakeLists.txt delete mode 100644 src/toolkits/recsys/CMakeLists.txt rename src/toolkits/{ => recsys}/ml_data_2/data_storage/internal_metadata.cpp (97%) rename src/toolkits/{ => recsys}/ml_data_2/data_storage/internal_metadata.hpp (97%) rename src/toolkits/{ => recsys}/ml_data_2/data_storage/ml_data_block_manager.cpp (96%) rename src/toolkits/{ => recsys}/ml_data_2/data_storage/ml_data_block_manager.hpp (96%) rename src/toolkits/{ => recsys}/ml_data_2/data_storage/ml_data_row_format.cpp (99%) rename src/toolkits/{ => recsys}/ml_data_2/data_storage/ml_data_row_format.hpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/data_storage/ml_data_row_translation.cpp (95%) rename src/toolkits/{ => recsys}/ml_data_2/data_storage/ml_data_row_translation.hpp (95%) rename src/toolkits/{ => recsys}/ml_data_2/data_storage/ml_data_side_feature_translation.hpp (87%) rename src/toolkits/{ => recsys}/ml_data_2/data_storage/util.cpp (95%) rename src/toolkits/{ => recsys}/ml_data_2/data_storage/util.hpp (91%) rename src/toolkits/{ => recsys}/ml_data_2/indexing/column_indexer.cpp (94%) rename src/toolkits/{ => recsys}/ml_data_2/indexing/column_indexer.hpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/indexing/column_unique_indexer.cpp (99%) rename src/toolkits/{ => recsys}/ml_data_2/indexing/column_unique_indexer.hpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/iterators/composite_row_type.cpp (97%) rename src/toolkits/{ => recsys}/ml_data_2/iterators/composite_row_type.hpp (97%) rename src/toolkits/{ => recsys}/ml_data_2/iterators/ml_data_block_iterator.cpp (94%) rename src/toolkits/{ => recsys}/ml_data_2/iterators/ml_data_block_iterator.hpp (97%) rename src/toolkits/{ => recsys}/ml_data_2/iterators/ml_data_iterator.hpp (92%) rename src/toolkits/{ => recsys}/ml_data_2/iterators/ml_data_iterator_base.cpp (97%) rename src/toolkits/{ => recsys}/ml_data_2/iterators/ml_data_iterator_base.hpp (96%) rename src/toolkits/{ => recsys}/ml_data_2/iterators/row_reference.hpp (96%) rename src/toolkits/{ => recsys}/ml_data_2/metadata.cpp (99%) rename src/toolkits/{ => recsys}/ml_data_2/metadata.hpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/metadata_impl.hpp (99%) rename src/toolkits/{ => recsys}/ml_data_2/ml_data.cpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/ml_data.hpp (99%) rename src/toolkits/{ => recsys}/ml_data_2/ml_data_column_modes.cpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/ml_data_column_modes.hpp (99%) rename src/toolkits/{ => recsys}/ml_data_2/ml_data_entry.hpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/ml_data_iterators.hpp (73%) rename src/toolkits/{ => recsys}/ml_data_2/ml_data_missing_values.cpp (93%) rename src/toolkits/{ => recsys}/ml_data_2/ml_data_missing_values.hpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/ml_data_options.cpp (77%) rename src/toolkits/{ => recsys}/ml_data_2/ml_data_setup.cpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/ml_data_sorting.cpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/row_slicing_utilities.cpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/row_slicing_utilities.hpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/sframe_index_mapping.cpp (99%) rename src/toolkits/{ => recsys}/ml_data_2/sframe_index_mapping.hpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/side_features.cpp (99%) rename src/toolkits/{ => recsys}/ml_data_2/side_features.hpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/standardization-inl.hpp (99%) rename src/toolkits/{ => recsys}/ml_data_2/statistics/basic_column_statistics.cpp (99%) rename src/toolkits/{ => recsys}/ml_data_2/statistics/basic_column_statistics.hpp (99%) rename src/toolkits/{ => recsys}/ml_data_2/statistics/column_statistics.cpp (95%) rename src/toolkits/{ => recsys}/ml_data_2/statistics/column_statistics.hpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/testing_utils.cpp (98%) rename src/toolkits/{ => recsys}/ml_data_2/testing_utils.hpp (96%) delete mode 100644 src/toolkits/sgd/CMakeLists.txt delete mode 100644 src/toolkits/sparse_similarity/CMakeLists.txt delete mode 100644 src/toolkits/style_transfer/CMakeLists.txt delete mode 100644 src/toolkits/supervised_learning/CMakeLists.txt delete mode 100644 src/toolkits/text/CMakeLists.txt delete mode 100644 src/toolkits/util/CMakeLists.txt diff --git a/.gitignore b/.gitignore index 38bd96332d..20ac95d238 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ syntax: glob debug/* release/* profile/* +src/turi_common.h dist/turicreateapi* configure.deps config.log diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f6fe426d3..fa17c2378f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,7 @@ cmake_policy(SET CMP0005 NEW) cmake_policy(SET CMP0045 OLD) cmake_policy(SET CMP0046 OLD) cmake_policy(SET CMP0042 NEW) + # Generate a compilation database for use with automated tools like IDE/editor # plugins. See http://clang.llvm.org/docs/JSONCompilationDatabase.html @@ -40,11 +41,8 @@ set(CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_SOURCE_DIR}/cmake") # Configure the compiler include(SetupCompiler) SetupCompiler() +include(CompilerFlags) -if(WIN32) - add_definitions(-DWINVER=0x0600) - add_definitions(-D_WIN32_WINNT=0x0600) -endif() if(${TC_BUILD_IOS}) add_definitions(-DTC_BUILD_IOS) @@ -56,7 +54,7 @@ if(${TC_BUILD_IOS}) endif() if(NOT ${TC_BUILD_REMOTEFS}) - add_definitions(-DTC_DISABLE_REMOTEFS) + add_compiler_source_define(TC_DISABLE_REMOTEFS) endif() # Determine where additional Turi specific cmake modules are @@ -86,28 +84,29 @@ else() endif() +# + + + # Add global defines add_definitions(-DTC_BUILD_PATH_BASE="${CMAKE_SOURCE_DIR}") add_definitions(-DCURL_STATICLIB) add_definitions(-DIN_TURI_SOURCE_TREE) -add_definitions(-DFUSION_MAX_VECTOR_SIZE=20) -add_definitions(-DBOOST_SPIRIT_THREAD_SAFE) -add_definitions(-DBOOST_THREAD_ONCE_ATOMIC) - -# These determine the maximum number of arguments for extension functions -add_definitions(-DBOOST_FUSION_INVOKE_MAX_ARITY=12) -add_definitions(-DBOOST_FUSION_INVOKE_PROCEDURE_MAX_ARITY=12) -add_definitions(-DBOOST_FUSION_INVOKE_FUNCTION_OBJECT_MAX_ARITY=12) -# Workaround for https://svn.boost.org/trac10/ticket/10443 in Boost 1.68.0 -add_definitions(-DBOOST_RESULT_OF_USE_TR1_WITH_DECLTYPE_FALLBACK) +# Compile specif defines for boost. +add_compiler_source_define(FUSION_MAX_VECTOR_SIZE=20) +add_compiler_source_define(BOOST_SPIRIT_THREAD_SAFE) +add_compiler_source_define(BOOST_THREAD_ONCE_ATOMIC) +add_compiler_source_define(BOOST_FUSION_INVOKE_MAX_ARITY=12) +add_compiler_source_define(BOOST_FUSION_INVOKE_PROCEDURE_MAX_ARITY=12) +add_compiler_source_define(BOOST_FUSION_INVOKE_FUNCTION_OBJECT_MAX_ARITY=12) +add_compiler_source_define(BOOST_RESULT_OF_USE_TR1_WITH_DECLTYPE_FALLBACK) # Legal define for Eigen. -add_definitions(-DEIGEN_MPL2_ONLY) +add_compiler_source_define(EIGEN_MPL2_ONLY) # Defines to avoid symbol collisions -add_definitions(-Dgoogle=_tc_google) -add_definitions(-DCoreML=_tc_CoreML) +add_compiler_source_define(google=_tc_google) #**************************************************************************/ #* */ @@ -175,6 +174,14 @@ endif() include(FindInt128) Find_Int128_Types() +set(_compiler_int128_defs ${INT128_FLAGS}) +separate_arguments(_compiler_int128_defs) +list(FILTER _compiler_int128_defs INCLUDE REGEX "^-D.*") +foreach(_def ${_compiler_int128_defs}) + string(REGEX REPLACE "^-D(.*)$" "\\1" _set_arg ${_def}) + add_compiler_source_define(${_set_arg}) +endforeach() + # Set up defines around a few compiler and language oddities that we have to understand. include(CompilerOddities) Set_Compiler_Specific_Flags() @@ -185,6 +192,7 @@ set(CMAKE_THREAD_PREFER_PTHREAD TRUE) find_package(Threads REQUIRED) if(CMAKE_USE_PTHREADS_INIT) add_definitions(-DHAVE_PTHREAD) + add_compiler_source_define(HAVE_PTHREAD) endif() # Get framework dependencies. @@ -198,13 +206,13 @@ if (APPLE) find_library(CORE_VIDEO CoreVideo) find_library(METAL NAMES Metal) find_library(METAL_PERFORMANCE_SHADERS NAMES MetalPerformanceShaders) - set(_TC_APPLE_DEPENDENCIES + set(TC_APPLE_DEPENDENCIES ${ACCELERATE} ${CORE_GRAPHICS} ${JAVASCRIPT_CORE} ${FOUNDATION} ${CORE_IMAGE} ${CORE_ML} ${CORE_VIDEO} ${METAL} ${METAL_PERFORMANCE_SHADERS}) if(NOT ${TC_BUILD_IOS}) find_library(APPKIT AppKit) - set(_TC_APPLE_DEPENDENCIES ${_TC_APPLE_DEPENDENCIES} ${APPKIT}) + set(TC_APPLE_DEPENDENCIES ${TC_APPLE_DEPENDENCIES} ${APPKIT}) endif() endif() @@ -236,9 +244,9 @@ elseif (APPLE) set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,-rpath,@loader_path -Wl,-rpath,@loader_path/..") else() # LINUX - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-rpath,\\$ORIGIN") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-rpath,\\$ORIGIN -Wl,-rpath,\\$ORIGIN/..") - set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,-rpath,\\$ORIGIN -Wl,-rpath,\\$ORIGIN/..") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-rpath,$ORIGIN") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-rpath,$ORIGIN -Wl,-rpath,$ORIGIN/..") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,-rpath,$ORIGIN -Wl,-rpath,$ORIGIN/..") endif() # Common install locations for deps linked libraries @@ -297,55 +305,63 @@ include(MakeExecutable) include(MakeTest) include(BuildUtilities) -include_directories(SYSTEM - ${CMAKE_SOURCE_DIR}/deps/local/include) -include_directories(SYSTEM - ${CMAKE_SOURCE_DIR}/src/external/boost/boost_1_68_0/) -include_directories(SYSTEM - ${CMAKE_SOURCE_DIR}/src/external/libpng/libpng-1.6.37/) -include_directories(SYSTEM - ${CMAKE_SOURCE_DIR}/src/external/libjpeg/jpeg-8d/) -include_directories(SYSTEM - ${CMAKE_SOURCE_DIR}/src/external/bzip2/) -include_directories(SYSTEM - ${CMAKE_SOURCE_DIR}/src/external/zlib/zlib-1.2.11/) -include_directories(SYSTEM - ${CMAKE_SOURCE_DIR}/src/external/coremltools_wrap/) + link_directories(${CMAKE_SOURCE_DIR}/deps/local/lib) link_directories(${CMAKE_SOURCE_DIR}/deps/local/lib64) +include_directories(${CMAKE_SOURCE_DIR}/deps/local/include/) include_directories(src) -include_directories(src/core/system/platform) -include_directories(SYSTEM src/external) +include_directories(${CMAKE_BINARY_DIR}/src) + +set(TC_EXTERNAL_INCLUDE_SUBDIRS + ./ + boost/boost_1_68_0/ + libpng/libpng-1.6.37/ + libjpeg/jpeg-8d/ + bzip2/ + google + nanomsg/nanomsg-1.0.0/src + zlib/zlib-1.2.11/ + coremltools_wrap/) if(${TC_BUILD_REMOTEFS}) - include_directories(SYSTEM src/external/aws-sdk-cpp/aws-cpp-sdk-core/include) - include_directories(SYSTEM src/external/aws-sdk-cpp/aws-cpp-sdk-s3/include) + set(TC_EXTERNAL_INCLUDE_SUBDIRS + ${TC_EXTERNAL_INCLUDE_SUBDIRS} + aws-sdk-cpp/aws-cpp-sdk-core/include + aws-sdk-cpp/aws-cpp-sdk-s3/include) endif() -include_directories(SYSTEM src/external/google) -include_directories(SYSTEM src/external/nanomsg/nanomsg-1.0.0/src) - +foreach(_dir ${TC_EXTERNAL_INCLUDE_SUBDIRS}) + include_directories(SYSTEM ${CMAKE_SOURCE_DIR}/src/external/${_dir}) +endforeach() -# Include all the external dependencies. -include(ExternalProject) add_custom_target(external_dependencies) -file(GLOB packages "${DEPS_CMAKE}/ExternalProject*.cmake") -foreach(package ${packages}) - get_filename_component(packagename "${package}" NAME_WE) - #package is of the form ExternalProjectXXX" - include(${package}) - STRING(SUBSTRING "${packagename}" 15 -1 depname) - string(TOLOWER ${depname} depname) - set(package_${depname} requires_${depname} CACHE STRING "Package map") - add_dependencies(external_dependencies ex_${depname}) -endforeach() +if(${TC_BUILD_REMOTEFS}) + if(NOT ${TC_EXTERNAL_DEPS_PREBUILT}) -set(_TC_EXTERNAL_DEPENDENCIES - openssl libxml2 curl +# Include all the external dependencies. + include(ExternalProject) + + file(GLOB packages "${DEPS_CMAKE}/ExternalProject*.cmake") + foreach(package ${packages}) + get_filename_component(packagename "${package}" NAME_WE) + #package is of the form ExternalProjectXXX" + include(${package}) + STRING(SUBSTRING "${packagename}" 15 -1 depname) + string(TOLOWER ${depname} depname) + set(package_${depname} requires_${depname} CACHE STRING "Package map") + add_dependencies(external_dependencies ex_${depname}) + endforeach() + endif() + + set(TC_EXTERNAL_DEPENDENCIES openssl libxml2 curl) +endif() + +set(TC_EXTERNAL_DEPENDENCIES + ${TC_EXTERNAL_DEPENDENCIES} Threads::Threads ${CMAKE_DL_LIBS}) @@ -353,178 +369,153 @@ set(_TC_EXTERNAL_DEPENDENCIES # Collate all the object targets shared among static and shared library targets # These are used by C API, unity_shared, etc. -if(NOT _TC_DEFAULT_SERVER_INITIALIZER) - set(_TC_DEFAULT_SERVER_INITIALIZER +if(NOT TC_DEFAULT_SERVER_INITIALIZER) + set(TC_DEFAULT_SERVER_INITIALIZER "${CMAKE_SOURCE_DIR}/src/capi/default_server_initializer.cpp" ) endif() -if(NOT TC_DISABLE_OBJECT_BUILDS) - -set(_TC_COMMON_OBJECTS - - # External dependencies (built from src/external) - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - - # Then our own source code - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" - "$" -) - -# TODO we can remove the requirements from here when target_link_libraries -# works with OBJECT library targets (requires CMake 3.12) -# See https://gitlab.kitware.com/cmake/cmake/issues/14778 -set(_TC_COMMON_REQUIREMENTS - # External dependencies (from deps) - ${_TC_EXTERNAL_DEPENDENCIES} - nanomsg - boost - libbz2 - z -) +################################################################################ +# +# Write out all the definitions into a common header file. -if(APPLE) - set(_TC_COMMON_REQUIREMENTS - # External dependencies (from deps) - ${_TC_COMMON_REQUIREMENTS} - ${_TC_APPLE_DEPENDENCIES}) -endif() - - set(_TC_COMMON_OBJECTS - ${_TC_COMMON_OBJECTS} - "$" - "$") +set(TC_COMMON_HEADER_FILE ${CMAKE_SOURCE_DIR}/src/turi_common.h) -if(APPLE) - set(_TC_COMMON_OBJECTS +# Write out all the definitions into a single file +separate_arguments(TC_COMPILER_SOURCE_DEFINES) - ${_TC_COMMON_OBJECTS} - "$" - "$" - ) - set(_TC_COMMON_REQUIREMENTS - ${_TC_COMMON_REQUIREMENTS} - ${CORE_ML} - ${CORE_VIDEO} - ${FOUNDATION} - ) - if(NOT ${TC_BUILD_IOS}) - set(_TC_COMMON_OBJECTS - ${_TC_COMMON_OBJECTS} - "$" - ) - set(_TC_COMMON_REQUIREMENTS - ${_TC_COMMON_REQUIREMENTS} - ${APPKIT} - ${CORE_GRAPHICS} - ${JAVASCRIPT_CORE} - ) - if(HAS_MPS) - set(_TC_COMMON_OBJECTS - ${_TC_COMMON_OBJECTS} - "$" - ) - set(_TC_COMMON_REQUIREMENTS - ${_TC_COMMON_REQUIREMENTS} - ${ACCELERATE} - ${CORE_IMAGE} - ${METAL} - ${METAL_PERFORMANCE_SHADERS} - ) - endif() - endif() -endif() +set(_file_contents " +// AUTOGENERATED DEFINITIONS +// DO NOT EDIT +#ifndef TC_CODE_DEFINES +#define TC_CODE_DEFINES -if(${TC_BUILD_REMOTEFS}) - # object targets that should be included in all APPLE and non-APPLE - # platforms except for iOS - set(_TC_COMMON_OBJECTS - ${_TC_COMMON_OBJECTS} - "$" - ) -endif() +") + +foreach(_def ${TC_COMPILER_SOURCE_DEFINES}) -endif() # End disable object builds + string(REGEX REPLACE "^([^\\=]+)(=|$)((.*)$|)" "\\1" _variable ${_def}) + string(REGEX REPLACE "^([^\\=]+)(=|$)((.*)$|)" "#define \\1 \\3" _expression ${_def}) + + set(_file_contents "${_file_contents} +#ifndef ${_variable} +${_expression} +#endif +") + +endforeach() +set(_file_contents "${_file_contents} +#endif // End ifdef TC_CODE_DEFINES +") +file(WRITE ${TC_COMMON_HEADER_FILE} "${_file_contents}") -if(EXISTS ${CMAKE_SOURCE_DIR}/extensions) - include_directories(SYSTEM extensions) - add_subdirectory(extensions) + +################################################################################ + +# Add in src/external as a subdirectory. The rest of the directories are +# handled directly below +add_subdirectory(src/external) + +set(TC_DEPENDENCIES + z libjson libjpeg libpng eigen sparsehash protobuf libbz2 + lz4 uuid xgboost coremltools_mlmodel boost nanomsg) + + +if(${TC_BUILD_REMOTEFS}) + set(TC_DEPENDENCIES ${TC_DEPENDENCIES} aws-sdk-cpp) endif() -# for build-time generated source code -include_directories(SYSTEM ${CMAKE_BINARY_DIR}/src) +############################################################################### +# Also set up the visualization. This contains a lot of custom scripts, +# so we preserve it. -if(EXISTS ${CMAKE_SOURCE_DIR}/subtree) - include_directories(SYSTEM subtree) - add_subdirectory(subtree) +add_subdirectory(src/visualization) +set(TC_DEPENDENCIES ${TC_DEPENDENCIES} + visualization vega_renderer annotation) + +################################################################################ +# Add all the framework dependencies + +if(APPLE) + macro(AddFrameworkDependency _name) + find_library(_fwrk_path_${_name} ${_name} REQUIRED) + set(TC_DEPENDENCIES ${TC_DEPENDENCIES} "-framework ${_name}") + endmacro() + + AddFrameworkDependency(Accelerate) + AddFrameworkDependency(CoreGraphics) + AddFrameworkDependency(JavaScriptCore) + AddFrameworkDependency(Foundation) + AddFrameworkDependency(CoreImage) + AddFrameworkDependency(CoreML) + AddFrameworkDependency(CoreVideo) + AddFrameworkDependency(Metal) + AddFrameworkDependency(MetalPerformanceShaders) + if(NOT ${TC_BUILD_IOS}) + AddFrameworkDependency(AppKit) + endif() endif() -add_subdirectory(src) -add_subdirectory(test) +FILE(GLOB_RECURSE _tc_core_files + CONFIGURE_DEPENDS + ${CMAKE_SOURCE_DIR}/src/core/*.h + ${CMAKE_SOURCE_DIR}/src/core/*.c + ${CMAKE_SOURCE_DIR}/src/core/*.hpp + ${CMAKE_SOURCE_DIR}/src/core/*.cpp + ${CMAKE_SOURCE_DIR}/src/core/*.m + ${CMAKE_SOURCE_DIR}/src/core/*.mm + + ${CMAKE_SOURCE_DIR}/src/ml/*.h + ${CMAKE_SOURCE_DIR}/src/ml/*.c + ${CMAKE_SOURCE_DIR}/src/ml/*.hpp + ${CMAKE_SOURCE_DIR}/src/ml/*.cpp + ${CMAKE_SOURCE_DIR}/src/ml/*.m + ${CMAKE_SOURCE_DIR}/src/ml/*.mm + + ${CMAKE_SOURCE_DIR}/src/model_server/*.h + ${CMAKE_SOURCE_DIR}/src/model_server/*.c + ${CMAKE_SOURCE_DIR}/src/model_server/*.hpp + ${CMAKE_SOURCE_DIR}/src/model_server/*.cpp + ${CMAKE_SOURCE_DIR}/src/model_server/*.m + ${CMAKE_SOURCE_DIR}/src/model_server/*.mm + + ${CMAKE_SOURCE_DIR}/src/toolkits/*.h + ${CMAKE_SOURCE_DIR}/src/toolkits/*.c + ${CMAKE_SOURCE_DIR}/src/toolkits/*.hpp + ${CMAKE_SOURCE_DIR}/src/toolkits/*.cpp + ${CMAKE_SOURCE_DIR}/src/toolkits/*.m + ${CMAKE_SOURCE_DIR}/src/toolkits/*.mm + + + ${CMAKE_SOURCE_DIR}/src/capi/*.h + ${CMAKE_SOURCE_DIR}/src/capi/*.c + ${CMAKE_SOURCE_DIR}/src/capi/*.hpp + ${CMAKE_SOURCE_DIR}/src/capi/*.cpp + ${CMAKE_SOURCE_DIR}/src/capi/*.m + ${CMAKE_SOURCE_DIR}/src/capi/*.mm) + +list(FILTER _tc_core_files EXCLUDE REGEX ".*/model_server/extensions/.*") + + +add_library(Core SHARED ${_tc_core_files}) +target_link_libraries(Core ${TC_DEPENDENCIES} ${TC_EXTERNAL_DEPENDENCIES}) +add_dependencies(Core external_dependencies ${TC_EXTERNAL_DEPENDENCIES}) +# set_property(TARGET Core APPEND_STRING PROPERTY LINK_FLAGS " -undefined dynamic_lookup") +source_group(TREE ${CMAKE_SOURCE_DIR} FILES ${_tc_core_files}) + +set(TC_TARGET_DIR "${CMAKE_SOURCE_DIR}/targets/") + +INSTALL(TARGETS Core + LIBRARY DESTINATION ${TC_TARGET_DIR}/lib/) +INSTALL(DIRECTORY src/ + DESTINATION ${TC_TARGET_DIR}/include/ + MESSAGE_NEVER + FILES_MATCHING REGEX "^.*\.((h)|(hpp))$") +INSTALL(CODE "execute_process( \ + COMMAND ${CMAKE_COMMAND} -E create_symlink \ + ${TC_TARGET_DIR}/include/external/boost/boost_1_68_0/boost \ + ${TC_TARGET_DIR}/include/boost)") diff --git a/build_python_wheel.sh b/build_python_wheel.sh new file mode 100755 index 0000000000..c78d84ec00 --- /dev/null +++ b/build_python_wheel.sh @@ -0,0 +1,114 @@ +#!/bin/bash -e + + +function print_help { + echo "Configures and builds a specified target." + echo + echo "Usage: ./build.sh [build options] [configure options]" + echo + echo "Common Options:" + echo " --target-dir, -t The target directory to install artifact to." + echo " default: `pwd`/targets." + echo + echo " --release Build in release mode." + echo " --debug Build in debug mode (default)." + echo + echo " --jobs,-j The number of parallel jobs to run." + echo + echo " --cleanup,-c Clean up everything before building." + echo + echo " --skip-configure,-s Skip running ./configure." + echo + echo " --build-number Set build number. " + echo " Defaults to part of git commit hash. " + echo + echo " All additional options passed through to ./configure." + echo + exit 1 +} # end of print help + +function unknown_option { + echo "Unrecognized option: $1" + echo "To get help, run ./configure --help" + exit 1 +} # end of unknown option + +if [[ ${OSTYPE} == darwin* ]] ; then + apple=1 +else + apple=0 +fi + + +# command flag options +cleanup=0 +skip_configure=0 +jobs=4 +configure_options="" +build_mode="release" +target_dir=`pwd`/targets +install_sysroot="" +no_sudo=0 +copy_links=0 +build_number=`git rev-parse --short HEAD || echo "NULL"` + +############################################################################### +# +# Parse command line configure flags ------------------------------------------ +# +while [ $# -gt 0 ] + do case $1 in + + --cleanup|-c) cleanup=1;; + + --skip-configure|-s) skip_configure=1;; + + --copy-links) copy_links=1;; + + --build-number=*) build_number=${1##--build-number=} ;; + --build-number) build_number="$2"; shift;; + + --target-dir=*) target_dir="${1##--target-dir=}" ;; + --target-dir|-t) target_dir="$2"; shift ;; + + --jobs=*) jobs=${1##--jobs=} ;; + --jobs|-j) jobs=$2; shift ;; + + --help) print_help; exit 0;; + + -D) configure_options="${configure_options} -D $2"; shift ;; + + *) configure_options="${configure_options} $1";; + esac + shift +done + +build_dir=`pwd`/${build_mode} +src_dir=`pwd` + + +echo "Setting up build:" +echo "build_mode = ${build_mode}" +echo "target_dir = ${target_dir}" +echo "target = ${target}" +echo "build_dir = ${build_dir}" + + +if [[ ${cleanup} -eq 1 ]]; then + ./configure --cleanup --yes || exit 1 +fi + +if [[ ${skip_configure} -eq 0 ]] ; then + ./configure ${configure_options} --with-python || exit 1 +else + echo "skipping configure script as requested." +fi + + +install_dir=${target_dir}/python +rm -rf ${target_dir}/python +mkdir -p ${target_dir}/python + +bash scripts/make_wheel.sh --skip_test --skip_cpp_test --build_number="$build_number" --num_procs=${jobs} --${build_mode} --target-dir="${install_dir}" + + diff --git a/cmake/CompilerFlags.cmake b/cmake/CompilerFlags.cmake index 3452338308..a57a789273 100644 --- a/cmake/CompilerFlags.cmake +++ b/cmake/CompilerFlags.cmake @@ -2,6 +2,16 @@ include(CheckCXXCompilerFlag) include(CheckCCompilerFlag) include(CMakeParseArguments) +# Set a define to be dumped into turi_common.h at the end. +# +# A compiler source define is one that determines or switches some part of the +# compilation, but isn't related to the particular build being asked for. These are dumped into +# turi_common.h at the end. +# +macro(add_compiler_source_define FLAG) + add_definitions(-D${FLAG}) + set(TC_COMPILER_SOURCE_DEFINES "${TC_COMPILER_SOURCE_DEFINES} ${FLAG}") +endmacro() # check_and_set_compiler_flag # diff --git a/cmake/CompilerOddities.cmake b/cmake/CompilerOddities.cmake index 60bbabf058..392f8d19fc 100644 --- a/cmake/CompilerOddities.cmake +++ b/cmake/CompilerOddities.cmake @@ -1,4 +1,6 @@ include(CheckCXXSourceCompiles) +include(CompilerFlags) + macro(Set_Compiler_Specific_Flags) @@ -16,7 +18,7 @@ macro(Set_Compiler_Specific_Flags) if(COMPILER_HAS_IOS_BASE_FAILURE_WITH_ERROR_CODE) message(STATUS "Compiler supports ios_base::failure(str, error_code)") - add_definitions(-DCOMPILER_HAS_IOS_BASE_FAILURE_WITH_ERROR_CODE) + add_compiler_source_define(COMPILER_HAS_IOS_BASE_FAILURE_WITH_ERROR_CODE) else() message(STATUS "Compiler does not support ios_base::failure(str, error_code)") endif() @@ -60,11 +62,11 @@ macro(Set_Compiler_Specific_Flags) message(FATAL_ERROR "Cannot determine noexcept fladg on std::ios_base::failure. See log.") elseif(COMPILER_HAS_NOEXCEPT_WHAT_ON_EXCEPTIONS_V1) - add_definitions(-DCOMPILER_MODIFIER_ON_EXCEPTION_WHAT=noexcept) + add_compiler_source_define(COMPILER_MODIFIER_ON_EXCEPTION_WHAT=noexcept) elseif(COMPILER_HAS_NOEXCEPT_WHAT_ON_EXCEPTIONS_V2) - add_definitions(-DCOMPILER_MODIFIER_ON_EXCEPTION_WHAT=_NOEXCEPT) + add_compiler_source_define(DCOMPILER_MODIFIER_ON_EXCEPTION_WHAT=_NOEXCEPT) else() - add_definitions(-DCOMPILER_MODIFIER_ON_EXCEPTION_WHAT="") + add_compiler_source_define(COMPILER_MODIFIER_ON_EXCEPTION_WHAT=) endif() endmacro() diff --git a/configure b/configure index f13139eeb5..109a4f2904 100755 --- a/configure +++ b/configure @@ -75,6 +75,13 @@ function print_help { echo " using the [SOURCE:] format. " echo echo " -D var=value Specify FLAGS definitions to be passed on to cmake." + echo + echo " --create-xcode-project Create an xcode project file as xcode/Turi.xcodeproj." + echo + echo " --build-dependencies Build the external dependencies as part of configure and install them into deps/local." + echo " Needed for the xcode project build." + echo + echo " --skip-dependencies Skip building the dependencies." echo echo "Relevant environment variables: " echo @@ -151,12 +158,14 @@ with_release=1 with_debug=1 release_opt_for_size=0 debug_opt_for_size=0 +create_xcode_project=0 arch_list="" target="" builder="make" min_ios_version=${IOS_DEFAULT_MIN_VERSION} min_macos_version=${MACOS_DEFAULT_MIN_VERSION} enable_codesign=0 +build_dependencies=0 list_source_files=0 DEBUG_DIR="${TURI_HOME}/debug" @@ -230,6 +239,11 @@ while [ $# -gt 0 ] --min-macos-version=*) min_macos_version=${1##--min-macos-version=};; --min-macos-version) min_macos_version=${2}; shift ;; + --create-xcode-project) create_xcode_project=1; build_dependencies=1; builder=xcode;; + + --build-dependencies) build_dependencies=1;; + --skip-dependencies) skip_dependencies=1;; + --list-source-files) list_source_files=1 ;; *) unknown_option $1 ;; @@ -335,6 +349,8 @@ mkdir -p ${PWD}/deps/local/bin CCCMD=`./scripts/find_compiler.sh cc --ccache=$with_ccache --script-dir=${PWD}/deps/local/bin/` CXXCMD=`./scripts/find_compiler.sh cxx --ccache=$with_ccache --script-dir=${PWD}/deps/local/bin/` +CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DCMAKE_C_COMPILER=$CCCMD -DCMAKE_CXX_COMPILER=$CXXCMD" + echo "Setting C compiler to $CCCMD." echo "Setting C++ compiler to $CXXCMD." @@ -346,16 +362,21 @@ CMAKE=`./scripts/find_cmake.sh` ######################################################## # Prepare to build + + +# Configuration flags that are specific to the xcode builder +CMAKE_XCODE_CONFIG_FLAGS="" + set -e set -o pipefail case $builder in xcode) - CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -GXcode" - CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DCMAKE_CXX_STANDARD=11 -DCMAKE_CXX_STANDARD_REQUIRED=1" + CMAKE_XCODE_CONFIG_FLAGS="${CMAKE_XCODE_CONFIG_FLAGS} -GXcode" + CMAKE_XCODE_CONFIG_FLAGS="${CMAKE_XCODE_CONFIG_FLAGS} -DCMAKE_CXX_STANDARD=11 -DCMAKE_CXX_STANDARD_REQUIRED=1" if [[ ! -z ${arch_list} ]] ; then - CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DCMAKE_OSX_ARCHITECTURES='${arch_list}'" + CMAKE_XCODE_CONFIG_FLAGS="${CMAKE_XCODE_CONFIG_FLAGS} -DCMAKE_OSX_ARCHITECTURES='${arch_list}'" fi ;; @@ -375,10 +396,10 @@ if [[ ${enable_codesign} == 1 ]] ; then elif [[ ${builder} == xcode ]] ; then # Deal with the code signing issues. - CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY= -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=0 -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM= " + CMAKE_XCODE_CONFIG_FLAGS="${CMAKE_XCODE_CONFIG_FLAGS} -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY= -DCMAKE_XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED=0 -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM= " # This also requires us to skip the code signing when trying to compile targets to check definitions. - CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY" + CMAKE_XCODE_CONFIG_FLAGS="${CMAKE_XCODE_CONFIG_FLAGS} -DCMAKE_TRY_COMPILE_TARGET_TYPE=STATIC_LIBRARY" echo "Skipping code signing." fi @@ -424,25 +445,42 @@ if [[ ${build_platform} == macosx ]] ; then exit 1 fi fi - -build_cmd="$CMAKE \ - $GENERATOR \ - -D CMAKE_C_COMPILER=$CCCMD \ - -D CMAKE_CXX_COMPILER=$CXXCMD \ - ${CMAKE_CONFIG_FLAGS}" - -if [[ $with_debug == 1 ]] ; then - set -x - mkdir -p ${DEBUG_DIR} - cd ${DEBUG_DIR} && $build_cmd -DCMAKE_BUILD_TYPE=Debug --config Debug -D CMAKE_CONFIGURATION_TYPES='Debug;Release' ${TURI_HOME} - set +x -fi -if [[ $with_release == 1 ]] ; then - set -x - mkdir -p ${RELEASE_DIR} - cd ${RELEASE_DIR} && $build_cmd -DCMAKE_BUILD_TYPE=Release -D CMAKE_CONFIGURATION_TYPES='Release;Debug' --config Release ${TURI_HOME} - set +x -fi +if [[ $build_dependencies == 1 ]] ; then + # Build the dependencies through the make build scripts so they don't have to + # be part of the XCode project, which doesn't work. + + deps_build_dir=${TURI_HOME}/deps/build + mkdir -p $deps_build_dir + cd ${deps_build_dir} + $CMAKE ${CMAKE_CONFIG_FLAGS} -DCMAKE_BUILD_TYPE=Release --config Release ${TURI_HOME} + make external_dependencies + CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DTC_EXTERNAL_DEPS_PREBUILT=1" +else + CMAKE_CONFIG_FLAGS="${CMAKE_CONFIG_FLAGS} -DTC_EXTERNAL_DEPS_PREBUILT=0" +fi + + +if [[ $create_xcode_project == 1 ]] ; then + xcode_proj_dir=${TURI_HOME}/xcode/ + mkdir -p $xcode_proj_dir + cd ${xcode_proj_dir} + $CMAKE ${CMAKE_CONFIG_FLAGS} ${CMAKE_XCODE_CONFIG_FLAGS} -DTC_EXTERNAL_DEPS_PREBUILT=1 -DCMAKE_BUILD_TYPE=Debug --config Debug -D CMAKE_CONFIGURATION_TYPES='Debug;Release' ${TURI_HOME} + +else + if [[ $with_debug == 1 ]] ; then + set -x + mkdir -p ${DEBUG_DIR} + cd ${DEBUG_DIR} && $CMAKE ${CMAKE_CONFIG_FLAGS} ${CMAKE_XCODE_CONFIG_FLAGS} -DCMAKE_BUILD_TYPE=Debug --config Debug -D CMAKE_CONFIGURATION_TYPES='Debug;Release' ${TURI_HOME} + set +x + fi + + if [[ $with_release == 1 ]] ; then + set -x + mkdir -p ${RELEASE_DIR} + cd ${RELEASE_DIR} && $CMAKE ${CMAKE_CONFIG_FLAGS} ${CMAKE_XCODE_CONFIG_FLAGS} -DCMAKE_BUILD_TYPE=Release -D CMAKE_CONFIGURATION_TYPES='Release;Debug' --config Release ${TURI_HOME} + set +x + fi +fi diff --git a/deps/src/libxml2-2.9.1/ChangeLog b/deps/src/libxml2-2.9.1/ChangeLog index e06a71da72..ce13596b3a 100644 --- a/deps/src/libxml2-2.9.1/ChangeLog +++ b/deps/src/libxml2-2.9.1/ChangeLog @@ -13956,7 +13956,7 @@ Fri Mar 15 19:41:25 CET 2002 Daniel Veillard xml2-config --cflags should not output -I$includeprefix/libxml2/libxml because libxml2 header names clashes with existing names like list.h from C++ stl. - Includes should be #include so ... + Includes should be #include so ... Fri Mar 15 10:41:50 CET 2002 Daniel Veillard diff --git a/deps/src/libxml2-2.9.1/bakefile/libxml2.bkl b/deps/src/libxml2-2.9.1/bakefile/libxml2.bkl index 4efbb73f26..1ec753b062 100644 --- a/deps/src/libxml2-2.9.1/bakefile/libxml2.bkl +++ b/deps/src/libxml2-2.9.1/bakefile/libxml2.bkl @@ -517,8 +517,8 @@ $(TAB)copy "$(DOLLAR)(InputPath)" ..\$(CONFIG_DSTNAME)