Merge branch 'develop' into fix-fillorkill

gregtatcam · Oct 23, 2023 · 7a7c9cc · 7a7c9cc
2 parents 2834d46 + c915984
commit 7a7c9cc
Show file tree

Hide file tree

Showing 329 changed files with 72,140 additions and 11,084 deletions.
diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml
@@ -10,16 +10,12 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: dependencies
-      uses: ./.github/actions/dependencies
-      with:
-        configuration: ${{ inputs.configuration }}
     - name: configure
       shell: bash
       run: |
         cd ${build_dir}
         cmake \
-          ${{ inputs.generator && format('-G {0}', inputs.generator) || '' }} \
+          ${{ inputs.generator && format('-G "{0}"', inputs.generator) || '' }} \
           -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake \
           -DCMAKE_BUILD_TYPE=${{ inputs.configuration }} \
           ${{ inputs.cmake-args }} \

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -37,6 +37,10 @@ jobs:
         run : |
           conan profile get env.CXXFLAGS default || true
           conan profile update 'conf.tools.build:cxxflags+=["-DBOOST_ASIO_DISABLE_CONCEPTS"]' default
+      - name: dependencies
+        uses: ./.github/actions/dependencies
+        with:
+          configuration: ${{ matrix.configuration }}
       - name: build
         uses: ./.github/actions/build
         with:

diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml
@@ -79,7 +79,7 @@ jobs:
       - name: try to authenticate to ripple Conan remote
         id: remote
         run: |
-          echo outcome=$(conan user --remote ripple ${{ secrets.CONAN_USERNAME }} --password ${{ secrets.CONAN_TOKEN }} && echo success || echo failure) | tee ${GITHUB_OUTPUT}
+          echo outcome=$(conan user --remote ripple ${{ secrets.CONAN_USERNAME }} --password ${{ secrets.CONAN_TOKEN }} >&2 && echo success || echo failure) | tee ${GITHUB_OUTPUT}
       - name: archive profile
         # Create this archive before dependencies are added to the local cache.
         run: tar -czf conan.tar -C ~/.conan .
@@ -88,7 +88,7 @@ jobs:
         # Print the list of dependencies that would need to be built locally.
         # A non-empty list means we have "failed" to cache binaries remotely.
         run: |
-          echo missing=$(conan info . --build missing --json 2>/dev/null  | grep '^\[') | tee ${GITHUB_OUTPUT}
+          echo missing=$(conan info . --build missing --settings build_type=${{ matrix.configuration }} --json 2>/dev/null  | grep '^\[') | tee ${GITHUB_OUTPUT}
       - name: build dependencies
         if: (steps.binaries.outputs.missing != '[]')
         uses: ./.github/actions/dependencies
@@ -146,6 +146,10 @@ jobs:
           ls ~/.conan
       - name: checkout
         uses: actions/checkout@v3
+      - name: dependencies
+        uses: ./.github/actions/dependencies
+        with:
+          configuration: ${{ matrix.configuration }}
       - name: build
         uses: ./.github/actions/build
         with:

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -1,19 +1,11 @@
 name: windows
-# We have disabled this workflow because it fails in our CI Windows
-# environment, but we cannot replicate the failure in our personal Windows
-# test environments, nor have we gone through the trouble of setting up an
-# interactive CI Windows environment.
-# We welcome contributions to diagnose or debug the problems on Windows. Until
-# then, we leave this tombstone as a reminder that we have tried (but failed)
-# to write a reliable test for Windows.
-# on: [push, pull_request]
-on:
-  workflow_dispatch:
-  push:
-    branches:
-      - 'action'
-    paths:
-      - '.github/workflow/windows.yml'
+
+on: [push, pull_request]
+
+# https://docs.github.com/en/actions/using-jobs/using-concurrency
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
 
@@ -25,6 +17,11 @@ jobs:
           - Visual Studio 16 2019
         configuration:
           - Release
+          # Github hosted runners tend to hang when running Debug unit tests.
+          # Instead of trying to work around it, disable the Debug job until
+          # something beefier (i.e. a heavy self-hosted runner) becomes
+          # available.
+          # - Debug
     runs-on: windows-2019
     env:
       build_dir: .build
@@ -37,11 +34,12 @@ jobs:
           python-version: 3.9
       - name: learn Python cache directory
         id: pip-cache
+        shell: bash
         run: |
           pip install --upgrade pip
-          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+          echo "dir=$(pip cache dir)" | tee ${GITHUB_OUTPUT}
       - name: restore Python cache directory
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
             path: ${{ steps.pip-cache.outputs.dir }}
             key: ${{ runner.os }}-${{ hashFiles('.github/workflows/windows.yml') }}
@@ -55,45 +53,48 @@ jobs:
           cmake --version
           dir env:
       - name: configure Conan
+        shell: bash
+        env:
+          CONAN_URL: http://18.143.149.228:8081/artifactory/api/conan/conan-non-prod
         run: |
           conan profile new default --detect
-          conan profile update settings.compiler.cppstd=20 default
-          conan profile update settings.compiler.runtime=MT default
-          conan profile update settings.compiler.toolset=v141 default
-      - name: learn Conan cache directory
-        id: conan-cache
-        run: |
-          echo "dir=$(conan config get storage.path)" >> $GITHUB_OUTPUT
-      - name: restore Conan cache directory
-        uses: actions/cache@v2
-        with:
-            path: ${{ steps.conan-cache.outputs.dir }}
-            key: ${{ hashFiles('~/.conan/profiles/default', 'conanfile.py', 'external/rocksdb/*', '.github/workflows/windows.yml') }}
-      - name: export custom recipes
+          conan profile update settings.compiler.runtime=MT${{ matrix.configuration == 'Debug' && 'd' || '' }} default
+          # Do not quote the URL. An empty string will be accepted (with
+          # a non-fatal warning), but a missing argument will not.
+          conan remote add ripple ${{ env.CONAN_URL }} --insert 0
+      - name: try to authenticate to ripple Conan remote
+        shell: bash
+        id: remote
         run: |
-          conan export external/snappy snappy/1.1.9@
-          conan export external/soci soci/4.0.3@
-      - name: install dependencies
+          echo outcome=$(conan user --remote ripple ${{ secrets.CONAN_USERNAME }} \
+            --password ${{ secrets.CONAN_TOKEN }} >&2 && echo success || \
+            echo failure) | tee ${GITHUB_OUTPUT}
+      - name: list missing binaries
+        id: binaries
+        shell: bash
+        # Print the list of dependencies that would need to be built locally.
+        # A non-empty list means we have "failed" to cache binaries remotely.
         run: |
-          mkdir $env:build_dir
-          cd $env:build_dir
-          conan install .. --build missing --settings build_type=${{ matrix.configuration }}
-      - name: configure
-        run: |
-          $env:build_dir
-          cd $env:build_dir
-          pwd
-          ls
-          cmake `
-            -G "${{ matrix.generator }}" `
-            -DCMAKE_TOOLCHAIN_FILE:FILEPATH=build/generators/conan_toolchain.cmake `
-            -Dassert=ON `
-            -Dreporting=OFF `
-            -Dunity=OFF `
-            ..
+          echo missing=$(conan info . --build missing --settings build_type=${{ matrix.configuration }} --json 2>/dev/null  | grep '^\[') | tee ${GITHUB_OUTPUT}
+      - name: build dependencies
+        uses: ./.github/actions/dependencies
+        with:
+          configuration: ${{ matrix.configuration }}
+      - name: upload dependencies to remote
+        if: (steps.binaries.outputs.missing != '[]') && (steps.remote.outputs.outcome == 'success')
+        run: conan upload --remote ripple '*' --all --parallel --confirm
       - name: build
+        uses: ./.github/actions/build
+        with:
+          generator: '${{ matrix.generator }}'
+          configuration: ${{ matrix.configuration }}
+          # Hard code for now. Move to the matrix if varied options are needed
+          cmake-args: '-Dassert=ON -Dreporting=OFF -Dunity=OFF'
+      - name: test (permitted to silently fail)
+        shell: bash
+        # Github runners are resource limited, which causes unit tests to fail
+        # (e.g. OOM). To allow forward progress until self-hosted runners are
+        # up and running reliably, allow the job to succeed even if tests fail.
+        continue-on-error: true
         run: |
-          cmake --build $env:build_dir --target rippled --config ${{ matrix.configuration }} --parallel $env:NUMBER_OF_PROCESSORS
-      - name: test
-        run: |
-          & "$env:build_dir\${{ matrix.configuration }}\rippled.exe" --unittest
+          ${build_dir}/${{ matrix.configuration }}/rippled --unittest --unittest-jobs $(nproc)
diff --git a/.gitignore b/.gitignore
@@ -109,3 +109,6 @@ pkg
 CMakeUserPresets.json
 bld.rippled/
 .vscode
+
+# Suggested in-tree build directory
+/.build/
diff --git a/API-CHANGELOG.md b/API-CHANGELOG.md
@@ -1,36 +1,34 @@
 # API Changelog
 
-This changelog is intended to list all updates to the API.
+This changelog is intended to list all updates to the [public API methods](https://xrpl.org/public-api-methods.html).
 
-For info about how API versioning works, view the [XLS-22d spec](https://github.com/XRPLF/XRPL-Standards/discussions/54). For details about the implementation of API versioning, view the [implementation PR](https://github.com/XRPLF/rippled/pull/3155).
+For info about how [API versioning](https://xrpl.org/request-formatting.html#api-versioning) works, including examples, please view the [XLS-22d spec](https://github.com/XRPLF/XRPL-Standards/discussions/54). For details about the implementation of API versioning, view the [implementation PR](https://github.com/XRPLF/rippled/pull/3155). API versioning ensures existing integrations and users continue to receive existing behavior, while those that request a higher API version will experience new behavior.
 
 The API version controls the API behavior you see. This includes what properties you see in responses, what parameters you're permitted to send in requests, and so on. You specify the API version in each of your requests. When a breaking change is introduced to the `rippled` API, a new version is released. To avoid breaking your code, you should set (or increase) your version when you're ready to upgrade.
 
-For a log of breaking changes, see the **API Version [number]** headings. Breaking changes are associated with a particular API Version number. For non-breaking changes, scroll to the **XRP Ledger version [x.y.z]** headings. Non-breaking changes are associated with a particular XRP Ledger (`rippled`) release.
-
-## API Version 2
-This version will be supported by `rippled` version 1.12.
-
-#### V2 account_info response
-
-`signer_lists` is returned in the root of the response, instead of being nested under `account_data` (as it was in API version 1). ([#3770](https://github.com/XRPLF/rippled/pull/3770))
+For a log of breaking changes, see the **API Version [number]** headings. In general, breaking changes are associated with a particular API Version number. For non-breaking changes, scroll to the **XRP Ledger version [x.y.z]** headings. Non-breaking changes are associated with a particular XRP Ledger (`rippled`) release.
 
 ## API Version 1
-This version is supported by `rippled` version 1.11.
+
+This version is supported by all `rippled` versions. At time of writing, it is the default API version, used when no `api_version` is specified. When a new API version is introduced, the command line interface will default to the latest API version. The command line is intended for ad-hoc usage by humans, not programs or automated scripts. The command line is not meant for use in production code.
 
 ### Idiosyncrasies
 
 #### V1 account_info response
 
-In [the response to the `account_info` command](https://xrpl.org/account_info.html#response-format), there is `account_data` - which is supposed to be an `AccountRoot` object - and `signer_lists` is in this object. However, the docs say that `signer_lists` should be at the root level of the reponse - and this makes sense, since signer lists are not part of the AccountRoot object. (First reported in [xrpl-dev-portal#938](https://github.com/XRPLF/xrpl-dev-portal/issues/938).) Thanks to [rippled#3770](https://github.com/XRPLF/rippled/pull/3770), this field will be moved in `api_version: 2`, to the root level of the response.
+In [the response to the `account_info` command](https://xrpl.org/account_info.html#response-format), there is `account_data` - which is supposed to be an `AccountRoot` object - and `signer_lists` is returned in this object. However, the docs say that `signer_lists` should be at the root level of the reponse.
+
+It makes sense for `signer_lists` to be at the root level because signer lists are not part of the AccountRoot object. (First reported in [xrpl-dev-portal#938](https://github.com/XRPLF/xrpl-dev-portal/issues/938).)
+
+In `api_version: 2`, the `signer_lists` field [will be moved](#modifications-to-account_info-response-in-v2) to the root level of the account_info response. (https://github.com/XRPLF/rippled/pull/3770)
 
 #### server_info - network_id
 
 The `network_id` field was added in the `server_info` response in version 1.5.0 (2019), but it was not returned in [reporting mode](https://xrpl.org/rippled-server-modes.html#reporting-mode).
 
 ## XRP Ledger version 1.12.0
 
-Version 1.12.0 is in development.
+[Version 1.12.0](https://github.com/XRPLF/rippled/releases/tag/1.12.0) was released on Sep 6, 2023.
 
 ### Additions in 1.12
 
@@ -76,9 +74,11 @@ Additions are intended to be non-breaking (because they are purely additive).
 
 ### Breaking changes in 1.11
 
-- Added the ability to mark amendments as obsolete. For the `feature` admin API, there is a new possible value for the `vetoed` field. ([#4291](https://github.com/XRPLF/rippled/pull/4291))
-- The API now won't accept seeds or public keys in place of account addresses. ([#4404](https://github.com/XRPLF/rippled/pull/4404))
-- For the `ledger_data` method, when all entries are filtered out, the API now returns an empty list (an empty array, `[]`). (Previously, it would return `null`.) While this is technically a breaking change, the new behavior is consistent with the documentation, so this is considered only a bug fix. ([#4398](https://github.com/XRPLF/rippled/pull/4398))
+- Added the ability to mark amendments as obsolete. For the `feature` admin API, there is a new possible value for the `vetoed` field. (https://github.com/XRPLF/rippled/pull/4291)
+  - The value of `vetoed` can now be `true`, `false`, or `"Obsolete"`.
+- Removed the acceptance of seeds or public keys in place of account addresses. (https://github.com/XRPLF/rippled/pull/4404)
+  - This simplifies the API and encourages better security practices (i.e. seeds should never be sent over the network).
+- For the `ledger_data` method, when all entries are filtered out, the `state` field of the response is now an empty list (in other words, an empty array, `[]`). (Previously, it would return `null`.) While this is technically a breaking change, the new behavior is consistent with the documentation, so this is considered only a bug fix. (https://github.com/XRPLF/rippled/pull/4398)
 - If and when the `fixNFTokenRemint` amendment activates, there will be a new AccountRoot field, `FirstNFTSequence`. This field is set to the current account sequence when the account issues their first NFT. If an account has not issued any NFTs, then the field is not set. ([#4406](https://github.com/XRPLF/rippled/pull/4406))
   - There is a new account deletion restriction: an account can only be deleted if `FirstNFTSequence` + `MintedNFTokens` + `256` is less than the current ledger sequence.
   - This is potentially a breaking change if clients have logic for determining whether an account can be deleted.
@@ -97,3 +97,62 @@ Additions are intended to be non-breaking (because they are purely additive).
 - Added an `account_flags` object to the `account_info` method response. (https://github.com/XRPLF/rippled/pull/4459)
 - Added `NFTokenPages` to the `account_objects` RPC. (https://github.com/XRPLF/rippled/pull/4352)
 - Fixed: `marker` returned from the `account_lines` command would not work on subsequent commands. (https://github.com/XRPLF/rippled/pull/4361)
+
+## XRP Ledger version 1.10.0
+
+[Version 1.10.0](https://github.com/XRPLF/rippled/releases/tag/1.10.0)
+was released on Mar 14, 2023.
+
+### Breaking changes in 1.10
+
+- If the `XRPFees` feature is enabled, the `fee_ref` field will be
+  removed from the [ledger subscription stream](https://xrpl.org/subscribe.html#ledger-stream), because it will no longer
+  have any meaning.
+
+# In development
+
+Changes below this point are in development.
+
+## API Version 2
+
+At the time of writing, this version is expected to be introduced in `rippled` version 2.0.
+
+Currently (prior to the release of 2.0), it is available as a "beta" version, meaning it can be enabled with a config setting in `rippled.cfg`:
+```
+[beta_rpc_api]
+1
+```
+
+Since `api_version` 2 is in "beta", breaking changes to V2 can currently be made at any time.
+
+#### Modifications to account_info response in V2
+
+- `signer_lists` is returned in the root of the response. Previously, in API version 1, it was nested under `account_data`. (https://github.com/XRPLF/rippled/pull/3770)
+- When using an invalid `signer_lists` value, the API now returns an "invalidParams" error. (https://github.com/XRPLF/rippled/pull/4585)
+  - (`signer_lists` must be a boolean. In API version 1, strings are accepted and may return a normal response - as if `signer_lists` were `true`.)
+
+#### Modifications to [account_tx](https://xrpl.org/account_tx.html#account_tx) response in V2
+
+- Using `ledger_index_min`, `ledger_index_max`, and `ledger_index` returns `invalidParams` because if you use `ledger_index_min` or `ledger_index_max`, then it does not make sense to also specify `ledger_index`. Previously, in API version 1, no error was returned. (https://github.com/XRPLF/rippled/pull/4571)
+  - The same applies for `ledger_index_min`, `ledger_index_max`, and `ledger_hash`. (https://github.com/XRPLF/rippled/issues/4545#issuecomment-1565065579)
+- Using a `ledger_index_min` or `ledger_index_max` beyond the range of ledgers that the server has:
+  - returns `lgrIdxMalformed` in API version 2. (https://github.com/XRPLF/rippled/issues/4288)
+  - Previously, in API version 1, no error was returned.
+
+##### In progress
+
+- Attempting to use a non-boolean value (such as a string) for the `binary` or `forward` parameters returns `invalidParams` (`rpcINVALID_PARAMS`). Previously, in API version 1, no error was returned. (https://github.com/XRPLF/rippled/pull/4620)
+
+#### Modifications to [noripple_check](https://xrpl.org/noripple_check.html#noripple_check) response in V2
+
+##### In progress
+
+- Attempting to use a non-boolean value (such as a string) for the `transactions` parameter returns `invalidParams` (`rpcINVALID_PARAMS`). Previously, in API version 1, no error was returned. (https://github.com/XRPLF/rippled/pull/4620)
+
+# Unit tests for API changes
+
+The following information is useful to developers contributing to this project:
+
+The purpose of unit tests is to catch bugs and prevent regressions. In general, it often makes sense to create a test function when there is a breaking change to the API. For APIs that have changed in a new API version, the tests should be modified so that both the prior version and the new version are properly tested.
+
+To take one example: for `account_info` version 1, WebSocket and JSON-RPC behavior should be tested. The latest API version, i.e. API version 2, should be tested over WebSocket, JSON-RPC, and command line.