Merge branch 'develop' into price-oracle-dev

API-CHANGELOG.md

@@ -141,6 +141,31 @@ In API version 2, the following methods are no longer available:
 - `tx_history` - Instead, use other methods such as `account_tx` or `ledger` with the `transactions` field set to `true`.
 - `ledger_header` - Instead, use the `ledger` method.
 
+#### Modifications to JSON transaction element in V2
+
+In API version 2, JSON elements for transaction output have been changed and made consistent for all methods which output transactions:
+
+- JSON transaction element is named `tx_json`
+- Binary transaction element is named `tx_blob`
+- JSON transaction metadata element is named `meta`
+- Binary transaction metadata element is named `meta_blob`
+
+Additionally, these elements are now consistently available next to `tx_json` (i.e. sibling elements), where possible:
+
+- `hash` - Transaction ID. This data was stored inside transaction output in API version 1, but in API version 2 is a sibling element.
+- `ledger_index` - Ledger index (only set on validated ledgers)
+- `ledger_hash` - Ledger hash (only set on closed or validated ledgers)
+- `close_time_iso` - Ledger close time expressed in ISO 8601 time format (only set on validated ledgers)
+- `validated` - Bool element set to `true` if the transaction is in a validated ledger, otherwise `false`
+
+This change affects the following methods:
+
+- `tx` - Transaction data moved into element `tx_json` (was inline inside `result`) or, if binary output was requested, moved from `tx` to `tx_blob`. Renamed binary transaction metadata element (if it was requested) from `meta` to `meta_blob`. Changed location of `hash` and added new elements
+- `account_tx` - Renamed transaction element from `tx` to `tx_json`. Renamed binary transaction metadata element (if it was requested) from `meta` to `meta_blob`. Changed location of `hash` and added new elements
+- `transaction_entry` - Renamed transaction metadata element from `metadata` to `meta`. Changed location of `hash` and added new elements
+- `subscribe` - Renamed transaction element from `transaction` to `tx_json`. Changed location of `hash` and added new elements
+- `sign`, `sign_for`, `submit` and `submit_multisigned` - Changed location of `hash` element.
+
 #### Modifications to account_info response in V2
 
 - `signer_lists` is returned in the root of the response. Previously, in API version 1, it was nested under `account_data`. (https://github.com/XRPLF/rippled/pull/3770)

Builds/CMake/RippledCompiler.cmake

@@ -131,7 +131,16 @@ else ()
       >)
 endif ()
 
-if (use_gold AND is_gcc)
+if (use_mold)
+  # use mold linker if available
+  execute_process (
+    COMMAND ${CMAKE_CXX_COMPILER} -fuse-ld=mold -Wl,--version
+    ERROR_QUIET OUTPUT_VARIABLE LD_VERSION)
+  if ("${LD_VERSION}" MATCHES "mold")
+    target_link_libraries (common INTERFACE -fuse-ld=mold)
+  endif ()
+  unset (LD_VERSION)
+elseif (use_gold AND is_gcc)
   # use gold linker if available
   execute_process (
     COMMAND ${CMAKE_CXX_COMPILER} -fuse-ld=gold -Wl,--version
@@ -163,9 +172,7 @@ if (use_gold AND is_gcc)
         $<$<NOT:$<BOOL:${static}>>:-Wl,--disable-new-dtags>)
   endif ()
   unset (LD_VERSION)
-endif ()
-
-if (use_lld)
+elseif (use_lld)
   # use lld linker if available
   execute_process (
     COMMAND ${CMAKE_CXX_COMPILER} -fuse-ld=lld -Wl,--version
@@ -176,6 +183,7 @@ if (use_lld)
   unset (LD_VERSION)
 endif()
 
+
 if (assert)
   foreach (var_ CMAKE_C_FLAGS_RELEASE CMAKE_CXX_FLAGS_RELEASE)
     STRING (REGEX REPLACE "[-/]DNDEBUG" "" ${var_} "${${var_}}")

Builds/CMake/RippledSettings.cmake

@@ -37,6 +37,7 @@ if (is_linux)
   option (static "link protobuf, openssl, libc++, and boost statically" ON)
   option (perf "Enables flags that assist with perf recording" OFF)
   option (use_gold "enables detection of gold (binutils) linker" ON)
+  option (use_mold "enables detection of mold (binutils) linker" ON)
 else ()
   # we are not ready to allow shared-libs on windows because it would require
   # export declarations. On macos it's more feasible, but static openssl
@@ -45,6 +46,7 @@ else ()
   set (static ON CACHE BOOL "static link, linux only. ON for WIN/macos" FORCE)
   set (perf OFF CACHE BOOL "perf flags, linux only" FORCE)
   set (use_gold OFF CACHE BOOL "gold linker, linux only" FORCE)
+  set (use_mold OFF CACHE BOOL "mold linker, linux only" FORCE)
 endif ()
 if (is_clang)
   option (use_lld "enables detection of lld linker" ON)

src/ripple/app/ledger/LedgerMaster.h

@@ -215,6 +215,8 @@ class LedgerMaster : public AbstractFetchPackContainer
     void
     clearLedger(std::uint32_t seq);
     bool
+    isValidated(ReadView const& ledger);
+    bool
     getValidatedRange(std::uint32_t& minVal, std::uint32_t& maxVal);
     bool
     getFullValidatedRange(std::uint32_t& minVal, std::uint32_t& maxVal);

src/ripple/app/ledger/LedgerToJson.h

@@ -21,8 +21,10 @@
 #define RIPPLE_APP_LEDGER_LEDGERTOJSON_H_INCLUDED
 
 #include <ripple/app/ledger/Ledger.h>
+#include <ripple/app/ledger/LedgerMaster.h>
 #include <ripple/app/misc/TxQ.h>
 #include <ripple/basics/StringUtilities.h>
+#include <ripple/basics/chrono.h>
 #include <ripple/json/Object.h>
 #include <ripple/protocol/STTx.h>
 #include <ripple/protocol/jss.h>
@@ -41,6 +43,8 @@ struct LedgerFill
         LedgerEntryType t = ltANY)
         : ledger(l), options(o), txQueue(std::move(q)), type(t), context(ctx)
     {
+        if (context)
+            closeTime = context->ledgerMaster.getCloseTimeBySeq(ledger.seq());
     }
 
     enum Options {
@@ -58,6 +62,7 @@ struct LedgerFill
     std::vector<TxQ::TxDetails> txQueue;
     LedgerEntryType type;
     RPC::Context* context;
+    std::optional<NetClock::time_point> closeTime;
 };
 
 /** Given a Ledger and options, fill a Json::Object or Json::Value with a

src/ripple/app/ledger/impl/LedgerMaster.cpp

@@ -603,6 +603,54 @@ LedgerMaster::clearLedger(std::uint32_t seq)
     mCompleteLedgers.erase(seq);
 }
 
+bool
+LedgerMaster::isValidated(ReadView const& ledger)
+{
+    if (app_.config().reporting())
+        return true;  // Reporting mode only supports validated ledger
+
+    if (ledger.open())
+        return false;
+
+    if (ledger.info().validated)
+        return true;
+
+    auto const seq = ledger.info().seq;
+    try
+    {
+        // Use the skip list in the last validated ledger to see if ledger
+        // comes before the last validated ledger (and thus has been
+        // validated).
+        auto const hash = walkHashBySeq(seq, InboundLedger::Reason::GENERIC);
+
+        if (!hash || ledger.info().hash != *hash)
+        {
+            // This ledger's hash is not the hash of the validated ledger
+            if (hash)
+            {
+                assert(hash->isNonZero());
+                uint256 valHash =
+                    app_.getRelationalDatabase().getHashByIndex(seq);
+                if (valHash == ledger.info().hash)
+                {
+                    // SQL database doesn't match ledger chain
+                    clearLedger(seq);
+                }
+            }
+            return false;
+        }
+    }
+    catch (SHAMapMissingNode const& mn)
+    {
+        JLOG(m_journal.warn()) << "Ledger #" << seq << ": " << mn.what();
+        return false;
+    }
+
+    // Mark ledger as validated to save time if we see it again.
+    ledger.info().validated = true;
+    return true;
+}
+
 // returns Ledgers we have all the nodes for
 bool
 LedgerMaster::getFullValidatedRange(

src/ripple/app/ledger/impl/LedgerToJson.cpp

@@ -17,12 +17,14 @@
 */
 //==============================================================================
 
+#include <ripple/app/ledger/LedgerMaster.h>
 #include <ripple/app/ledger/LedgerToJson.h>
 #include <ripple/app/main/Application.h>
 #include <ripple/app/misc/DeliverMax.h>
 #include <ripple/app/misc/TxQ.h>
 #include <ripple/basics/base_uint.h>
 #include <ripple/core/Pg.h>
+#include <ripple/protocol/jss.h>
 #include <ripple/rpc/Context.h>
 #include <ripple/rpc/DeliveredAmount.h>
 
@@ -83,6 +85,7 @@ fillJson(Object& json, bool closed, LedgerInfo const& info, bool bFull)
         json[jss::close_time_human] = to_string(info.closeTime);
         if (!getCloseAgree(info))
             json[jss::close_time_estimated] = true;
+        json[jss::close_time_iso] = to_string_iso(info.closeTime);
     }
 }
 
@@ -118,8 +121,48 @@ fillJsonTx(
     if (bBinary)
     {
         txJson[jss::tx_blob] = serializeHex(*txn);
+        if (fill.context->apiVersion > 1)
+            txJson[jss::hash] = to_string(txn->getTransactionID());
+
+        auto const json_meta =
+            (fill.context->apiVersion > 1 ? jss::meta_blob : jss::meta);
         if (stMeta)
-            txJson[jss::meta] = serializeHex(*stMeta);
+            txJson[json_meta] = serializeHex(*stMeta);
+    }
+    else if (fill.context->apiVersion > 1)
+    {
+        copyFrom(
+            txJson[jss::tx_json],
+            txn->getJson(JsonOptions::disable_API_prior_V2, false));
+        txJson[jss::hash] = to_string(txn->getTransactionID());
+        RPC::insertDeliverMax(
+            txJson[jss::tx_json], txnType, fill.context->apiVersion);
+
+        if (stMeta)
+        {
+            txJson[jss::meta] = stMeta->getJson(JsonOptions::none);
+
+            // If applicable, insert delivered amount
+            if (txnType == ttPAYMENT || txnType == ttCHECK_CASH)
+                RPC::insertDeliveredAmount(
+                    txJson[jss::meta],
+                    fill.ledger,
+                    txn,
+                    {txn->getTransactionID(), fill.ledger.seq(), *stMeta});
+        }
+
+        if (!fill.ledger.open())
+            txJson[jss::ledger_hash] = to_string(fill.ledger.info().hash);
+
+        const bool validated =
+            fill.context->ledgerMaster.isValidated(fill.ledger);
+        txJson[jss::validated] = validated;
+        if (validated)
+        {
+            txJson[jss::ledger_index] = to_string(fill.ledger.seq());
+            if (fill.closeTime)
+                txJson[jss::close_time_iso] = to_string_iso(*fill.closeTime);
+        }
     }
     else
     {
@@ -254,7 +297,11 @@ fillJsonQueue(Object& json, LedgerFill const& fill)
         if (tx.lastResult)
             txJson["last_result"] = transToken(*tx.lastResult);
 
-        txJson[jss::tx] = fillJsonTx(fill, bBinary, bExpanded, tx.txn, nullptr);
+        auto&& temp = fillJsonTx(fill, bBinary, bExpanded, tx.txn, nullptr);
+        if (fill.context->apiVersion > 1)
+            copyFrom(txJson, temp);
+        else
+            copyFrom(txJson[jss::tx], temp);
     }
 }
 

src/ripple/app/misc/NetworkOPs.cpp

@@ -63,6 +63,7 @@
 #include <ripple/protocol/BuildInfo.h>
 #include <ripple/protocol/Feature.h>
 #include <ripple/protocol/STParsedJSON.h>
+#include <ripple/protocol/jss.h>
 #include <ripple/resource/Fees.h>
 #include <ripple/resource/ResourceManager.h>
 #include <ripple/rpc/BookChanges.h>
@@ -74,6 +75,7 @@
 
 #include <algorithm>
 #include <mutex>
+#include <optional>
 #include <string>
 #include <tuple>
 #include <unordered_map>
@@ -3101,7 +3103,11 @@ NetworkOPsImp::transJson(
     transResultInfo(result, sToken, sHuman);
 
     jvObj[jss::type] = "transaction";
-    jvObj[jss::transaction] = transaction->getJson(JsonOptions::none);
+    // NOTE jvObj is not a finished object for either API version. After
+    // it's populated, we need to finish it for a specific API version. This is
+    // done in a loop, near the end of this function.
+    jvObj[jss::transaction] =
+        transaction->getJson(JsonOptions::disable_API_prior_V2, false);
 
     if (meta)
     {
@@ -3110,13 +3116,16 @@ NetworkOPsImp::transJson(
             jvObj[jss::meta], *ledger, transaction, meta->get());
     }
 
+    if (!ledger->open())
+        jvObj[jss::ledger_hash] = to_string(ledger->info().hash);
+
     if (validated)
     {
         jvObj[jss::ledger_index] = ledger->info().seq;
-        jvObj[jss::ledger_hash] = to_string(ledger->info().hash);
         jvObj[jss::transaction][jss::date] =
             ledger->info().closeTime.time_since_epoch().count();
         jvObj[jss::validated] = true;
+        jvObj[jss::close_time_iso] = to_string_iso(ledger->info().closeTime);
 
         // WRITEME: Put the account next seq here
     }
@@ -3149,27 +3158,38 @@ NetworkOPsImp::transJson(
         }
     }
 
+    std::string const hash = to_string(transaction->getTransactionID());
     MultiApiJson multiObj({jvObj, jvObj});
     // Minimum supported API version must match index 0 in MultiApiJson
     static_assert(apiVersionSelector(RPC::apiMinimumSupportedVersion)() == 0);
-    // Beta API version must match last index in MultiApiJson
+    // Last valid (possibly beta) API ver. must match last index in MultiApiJson
     static_assert(
-        apiVersionSelector(RPC::apiBetaVersion)() + 1  //
+        apiVersionSelector(RPC::apiMaximumValidVersion)() + 1  //
         == MultiApiJson::size);
     for (unsigned apiVersion = RPC::apiMinimumSupportedVersion,
                   lastIndex = MultiApiJson::size;
-         apiVersion <= RPC::apiBetaVersion;
+         apiVersion <= RPC::apiMaximumValidVersion;
          ++apiVersion)
     {
         unsigned const index = apiVersionSelector(apiVersion)();
         assert(index < MultiApiJson::size);
         if (index != lastIndex)
         {
-            RPC::insertDeliverMax(
-                multiObj.val[index][jss::transaction],
-                transaction->getTxnType(),
-                apiVersion);
             lastIndex = index;
+
+            Json::Value& jvTx = multiObj.val[index];
+            RPC::insertDeliverMax(
+                jvTx[jss::transaction], transaction->getTxnType(), apiVersion);
+
+            if (apiVersion > 1)
+            {
+                jvTx[jss::tx_json] = jvTx.removeMember(jss::transaction);
+                jvTx[jss::hash] = hash;
+            }
+            else
+            {
+                jvTx[jss::transaction][jss::hash] = hash;
+            }
         }
     }