Merge branch 'feature/mpt-stissue' into feature/mpt-v2a

Builds/levelization/results/loops.txt

@@ -4,9 +4,6 @@ Loop: test.jtx test.toplevel
 Loop: test.jtx test.unit_test
   test.unit_test == test.jtx
 
-Loop: xrpl.basics xrpl.json
-  xrpl.json == xrpl.basics
-
 Loop: xrpld.app xrpld.core
   xrpld.app > xrpld.core
 

Builds/levelization/results/ordering.txt

@@ -1,5 +1,4 @@
 libxrpl.basics > xrpl.basics
-libxrpl.basics > xrpl.protocol
 libxrpl.crypto > xrpl.basics
 libxrpl.json > xrpl.basics
 libxrpl.json > xrpl.json
@@ -130,6 +129,7 @@ test.shamap > xrpl.protocol
 test.toplevel > test.csf
 test.toplevel > xrpl.json
 test.unit_test > xrpl.basics
+xrpl.json > xrpl.basics
 xrpl.protocol > xrpl.basics
 xrpl.protocol > xrpl.json
 xrpl.resource > xrpl.basics

cmake/RippledCore.cmake

@@ -47,6 +47,57 @@ target_link_libraries(xrpl.libpb
     gRPC::grpc++
 )
 
+# TODO: Clean up the number of library targets later.
+add_library(xrpl.imports.main INTERFACE)
+target_link_libraries(xrpl.imports.main INTERFACE
+    LibArchive::LibArchive
+    OpenSSL::Crypto
+    Ripple::boost
+    Ripple::opts
+    Ripple::syslibs
+    absl::random_random
+    date::date
+    ed25519::ed25519
+    secp256k1::secp256k1
+    xxHash::xxhash
+)
+
+include(add_module)
+include(target_link_modules)
+
+# Level 01
+add_module(xrpl beast)
+target_link_libraries(xrpl.libxrpl.beast PUBLIC
+  xrpl.imports.main
+  xrpl.libpb
+)
+
+# Level 02
+add_module(xrpl basics)
+target_link_libraries(xrpl.libxrpl.basics PUBLIC xrpl.libxrpl.beast)
+
+# Level 03
+add_module(xrpl json)
+target_link_libraries(xrpl.libxrpl.json PUBLIC xrpl.libxrpl.basics)
+
+add_module(xrpl crypto)
+target_link_libraries(xrpl.libxrpl.crypto PUBLIC xrpl.libxrpl.basics)
+
+# Level 04
+add_module(xrpl protocol)
+target_link_libraries(xrpl.libxrpl.protocol PUBLIC
+  xrpl.libxrpl.crypto
+  xrpl.libxrpl.json
+)
+
+# Level 05
+add_module(xrpl resource)
+target_link_libraries(xrpl.libxrpl.resource PUBLIC xrpl.libxrpl.protocol)
+
+add_module(xrpl server)
+target_link_libraries(xrpl.libxrpl.server PUBLIC xrpl.libxrpl.protocol)
+
+
 add_library(xrpl.libxrpl)
 set_target_properties(xrpl.libxrpl PROPERTIES OUTPUT_NAME xrpl)
 if(unity)
@@ -60,10 +111,24 @@ file(GLOB_RECURSE sources CONFIGURE_DEPENDS
 )
 target_sources(xrpl.libxrpl PRIVATE ${sources})
 
-target_include_directories(xrpl.libxrpl
-  PUBLIC
-    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-    $<INSTALL_INTERFACE:include>)
+target_link_modules(xrpl PUBLIC
+  basics
+  beast
+  crypto
+  json
+  protocol
+  resource
+  server
+)
+
+# All headers in libxrpl are in modules.
+# Uncomment this stanza if you have not yet moved new headers into a module.
+# target_include_directories(xrpl.libxrpl
+#   PRIVATE
+#     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>
+#   PUBLIC
+#     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+#     $<INSTALL_INTERFACE:include>)
 
 target_compile_definitions(xrpl.libxrpl
   PUBLIC

cmake/RippledInstall.cmake

@@ -2,13 +2,23 @@
    install stuff
 #]===================================================================]
 
+include(create_symbolic_link)
+
 install (
   TARGETS
     common
     opts
     ripple_syslibs
     ripple_boost
+    xrpl.imports.main
     xrpl.libpb
+    xrpl.libxrpl.basics
+    xrpl.libxrpl.beast
+    xrpl.libxrpl.crypto
+    xrpl.libxrpl.json
+    xrpl.libxrpl.protocol
+    xrpl.libxrpl.resource
+    xrpl.libxrpl.server
     xrpl.libxrpl
     antithesis-sdk-cpp
   EXPORT RippleExports
@@ -22,12 +32,12 @@ install(
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
 )
 
-if(NOT WIN32)
-  install(
-    CODE "file(CREATE_LINK xrpl \
-      \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}/ripple SYMBOLIC)"
-  )
-endif()
+install(CODE "
+  set(CMAKE_MODULE_PATH \"${CMAKE_MODULE_PATH}\")
+  include(create_symbolic_link)
+  create_symbolic_link(xrpl \
+    \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}/ripple)
+")
 
 install (EXPORT RippleExports
   FILE RippleTargets.cmake
@@ -56,12 +66,12 @@ if (is_root_project AND TARGET rippled)
     copy_if_not_exists(\"${CMAKE_CURRENT_SOURCE_DIR}/cfg/rippled-example.cfg\" etc rippled.cfg)
     copy_if_not_exists(\"${CMAKE_CURRENT_SOURCE_DIR}/cfg/validators-example.txt\" etc validators.txt)
   ")
-  if(NOT WIN32)
-    install(
-      CODE "file(CREATE_LINK rippled${suffix} \
-        \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/xrpld${suffix} SYMBOLIC)"
-    )
-  endif()
+  install(CODE "
+    set(CMAKE_MODULE_PATH \"${CMAKE_MODULE_PATH}\")
+    include(create_symbolic_link)
+    create_symbolic_link(rippled${suffix} \
+      \${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/xrpld${suffix})
+  ")
 endif ()
 
 install (

cmake/add_module.cmake

@@ -0,0 +1,37 @@
+include(isolate_headers)
+
+# Create an OBJECT library target named
+#
+#   ${PROJECT_NAME}.lib${parent}.${name}
+#
+# with sources in src/lib${parent}/${name}
+# and headers in include/${parent}/${name}
+# that cannot include headers from other directories in include/
+# unless they come through linked libraries.
+#
+# add_module(parent a)
+# add_module(parent b)
+# target_link_libraries(project.libparent.b PUBLIC project.libparent.a)
+function(add_module parent name)
+  set(target ${PROJECT_NAME}.lib${parent}.${name})
+  add_library(${target} OBJECT)
+  file(GLOB_RECURSE sources CONFIGURE_DEPENDS
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/lib${parent}/${name}/*.cpp"
+  )
+  target_sources(${target} PRIVATE ${sources})
+  target_include_directories(${target} PUBLIC
+    "$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>"
+  )
+  isolate_headers(
+    ${target}
+    "${CMAKE_CURRENT_SOURCE_DIR}/include"
+    "${CMAKE_CURRENT_SOURCE_DIR}/include/${parent}/${name}"
+    PUBLIC
+  )
+  isolate_headers(
+    ${target}
+    "${CMAKE_CURRENT_SOURCE_DIR}/src"
+    "${CMAKE_CURRENT_SOURCE_DIR}/src/lib${parent}/${name}"
+    PRIVATE
+  )
+endfunction()

cmake/create_symbolic_link.cmake

@@ -0,0 +1,20 @@
+# file(CREATE_SYMLINK) only works on Windows with administrator privileges.
+# https://stackoverflow.com/a/61244115/618906
+function(create_symbolic_link target link)
+  if(WIN32)
+    if(NOT IS_SYMLINK "${link}")
+      if(NOT IS_ABSOLUTE "${target}")
+        # Relative links work do not work on Windows.
+        set(target "${link}/../${target}")
+      endif()
+      file(TO_NATIVE_PATH "${target}" target)
+      file(TO_NATIVE_PATH "${link}" link)
+      execute_process(COMMAND cmd.exe /c mklink /J "${link}" "${target}")
+    endif()
+  else()
+    file(CREATE_LINK "${target}" "${link}" SYMBOLIC)
+  endif()
+  if(NOT IS_SYMLINK "${link}")
+    message(ERROR "failed to create symlink: <${link}>")
+  endif()
+endfunction()

cmake/isolate_headers.cmake

@@ -0,0 +1,48 @@
+include(create_symbolic_link)
+
+# Consider include directory B nested under prefix A:
+#
+#     /path/to/A/then/to/B/...
+#
+# Call C the relative path from A to B.
+# C is what we want to write in `#include` directives:
+#
+#     #include <then/to/B/...>
+#
+# Examples, all from the `jobqueue` module:
+#
+#   - Library public headers:
+#     B = /include/xrpl/jobqueue
+#     A = /include/
+#     C = xrpl/jobqueue
+#
+#   - Library private headers:
+#     B = /src/libxrpl/jobqueue
+#     A = /src/
+#     C = libxrpl/jobqueue
+#
+#   - Test private headers:
+#     B = /tests/jobqueue
+#     A = /
+#     C = tests/jobqueue
+#
+# To isolate headers from each other,
+# we want to create a symlink Y that points to B,
+# within a subdirectory X of the `CMAKE_BINARY_DIR`,
+# that has the same relative path C between X and Y,
+# and then add X as an include directory of the target,
+# sometimes `PUBLIC` and sometimes `PRIVATE`.
+# The Cs are all guaranteed to be unique.
+# We can guarantee a unique X per target by using
+# `${CMAKE_CURRENT_BINARY_DIR}/include/${target}`.
+#
+# isolate_headers(target A B scope)
+function(isolate_headers target A B scope)
+  file(RELATIVE_PATH C "${A}" "${B}")
+  set(X "${CMAKE_CURRENT_BINARY_DIR}/modules/${target}")
+  set(Y "${X}/${C}")
+  cmake_path(GET Y PARENT_PATH parent)
+  file(MAKE_DIRECTORY "${parent}")
+  create_symbolic_link("${B}" "${Y}")
+  target_include_directories(${target} ${scope} "$<BUILD_INTERFACE:${X}>")
+endfunction()

cmake/target_link_modules.cmake

@@ -0,0 +1,24 @@
+# Link a library to its modules (see: `add_module`)
+# and remove the module sources from the library's sources.
+#
+# add_module(parent a)
+# add_module(parent b)
+# target_link_libraries(project.libparent.b PUBLIC project.libparent.a)
+# add_library(project.libparent)
+# target_link_modules(parent PUBLIC a b)
+function(target_link_modules parent scope)
+  set(library ${PROJECT_NAME}.lib${parent})
+  foreach(name ${ARGN})
+    set(module ${library}.${name})
+    get_target_property(sources ${library} SOURCES)
+    list(LENGTH sources before)
+    get_target_property(dupes ${module} SOURCES)
+    list(LENGTH dupes expected)
+    list(REMOVE_ITEM sources ${dupes})
+    list(LENGTH sources after)
+    math(EXPR actual "${before} - ${after}")
+    message(STATUS "${module} with ${expected} sources took ${actual} sources from ${library}")
+    set_target_properties(${library} PROPERTIES SOURCES "${sources}")
+    target_link_libraries(${library} ${scope} ${module})
+  endforeach()
+endfunction()

include/xrpl/basics/Number.h

@@ -20,8 +20,6 @@
 #ifndef RIPPLE_BASICS_NUMBER_H_INCLUDED
 #define RIPPLE_BASICS_NUMBER_H_INCLUDED
 
-#include <xrpl/basics/MPTAmount.h>
-#include <xrpl/basics/XRPAmount.h>
 #include <cstdint>
 #include <limits>
 #include <ostream>
@@ -60,9 +58,6 @@ class Number
     explicit Number(rep mantissa, int exponent);
     explicit constexpr Number(rep mantissa, int exponent, unchecked) noexcept;
 
-    Number(XRPAmount const& x);
-    Number(MPTAmount const& x);
-
     constexpr rep
     mantissa() const noexcept;
     constexpr int
@@ -104,10 +99,6 @@ class Number
      *  "mixed mode" more convenient, e.g. MPTAmount + Number.
      */
     explicit
-    operator XRPAmount() const;  // round to nearest, even on tie
-    explicit
-    operator MPTAmount() const;  // round to nearest, even on tie
-    explicit
     operator rep() const;  // round to nearest, even on tie
 
     friend constexpr bool
@@ -217,14 +208,6 @@ inline Number::Number(rep mantissa) : Number{mantissa, 0}
 {
 }
 
-inline Number::Number(XRPAmount const& x) : Number{x.drops()}
-{
-}
-
-inline Number::Number(MPTAmount const& x) : Number{x.value()}
-{
-}
-
 inline constexpr Number::rep
 Number::mantissa() const noexcept
 {

include/xrpl/basics/SHAMapHash.h

@@ -21,6 +21,7 @@
 #define RIPPLE_BASICS_SHAMAP_HASH_H_INCLUDED
 
 #include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/partitioned_unordered_map.h>
 
 #include <ostream>
 
@@ -108,6 +109,13 @@ operator!=(SHAMapHash const& x, SHAMapHash const& y)
     return !(x == y);
 }
 
+template <>
+inline std::size_t
+extract(SHAMapHash const& key)
+{
+    return *reinterpret_cast<std::size_t const*>(key.as_uint256().data());
+}
+
 }  // namespace ripple
 
 #endif  // RIPPLE_BASICS_SHAMAP_HASH_H_INCLUDED

include/xrpl/basics/base_uint.h

@@ -29,6 +29,7 @@
 #include <xrpl/basics/Slice.h>
 #include <xrpl/basics/contract.h>
 #include <xrpl/basics/hardened_hash.h>
+#include <xrpl/basics/partitioned_unordered_map.h>
 #include <xrpl/basics/strHex.h>
 #include <xrpl/beast/utility/Zero.h>
 #include <xrpl/beast/utility/instrumentation.h>
@@ -637,6 +638,17 @@ operator<<(std::ostream& out, base_uint<Bits, Tag> const& u)
     return out << to_string(u);
 }
 
+template <>
+inline std::size_t
+extract(uint256 const& key)
+{
+    std::size_t result;
+    // Use memcpy to avoid unaligned UB
+    // (will optimize to equivalent code)
+    std::memcpy(&result, key.data(), sizeof(std::size_t));
+    return result;
+}
+
 #ifndef __INTELLISENSE__
 static_assert(sizeof(uint128) == 128 / 8, "There should be no padding bytes");
 static_assert(sizeof(uint160) == 160 / 8, "There should be no padding bytes");