Merge pull request nervosnetwork#4224 from EthanYuan/indexer-r

feat: add rich-indexer which is another built-in indexer based on relational database

Cargo.toml

@@ -70,7 +70,9 @@ members = [
     "db-migration",
     "util/network-alert",
     "store",
+    "util/indexer-sync",
     "util/indexer",
+    "util/rich-indexer",
     "util/chain-iter",
     "util/dao",
     "util/test-chain-utils",

ckb-bin/src/subcommand/reset_data.rs

@@ -20,6 +20,10 @@ pub fn reset_data(args: ResetDataArgs) -> Result<(), ExitCode> {
         target_dirs.push(args.indexer_path);
     }
 
+    if args.rich_indexer {
+        target_dirs.push(args.rich_indexer_path);
+    }
+
     if args.network {
         target_dirs.push(args.network_dir);
     }

ckb-bin/src/subcommand/run.rs

@@ -17,6 +17,8 @@ pub fn run(args: RunArgs, version: Version, async_handle: Handle) -> Result<(),
     let block_assembler_config = launcher.sanitize_block_assembler_config()?;
     let miner_enable = block_assembler_config.is_some();
 
+    launcher.check_indexer_config()?;
+
     let (shared, mut pack) = launcher.build_shared(block_assembler_config)?;
 
     // spawn freezer background process

deny.toml

@@ -5,7 +5,11 @@ yanked = "deny"
 notice = "deny"
 ignore = [
     # waiting https://github.com/bheisler/criterion.rs/pull/628 bump release
-    "RUSTSEC-2021-0145"
+    "RUSTSEC-2021-0145",
+    # The CVE can be kept under control for its triggering.
+    # See https://github.com/launchbadge/sqlx/pull/2455#issuecomment-1507657825 for more information.
+    # Meanwhile, awaiting SQLx's new version (> 0.7.3) for full support of any DB driver.
+    "RUSTSEC-2022-0090"
 ]
 
 [licenses]

devtools/doc/rpc-gen/src/utils.rs

@@ -28,6 +28,7 @@ pub(crate) fn all_rpc_docs() -> Vec<(String, Value)> {
         stats_rpc_doc,
         integration_test_rpc_doc,
         indexer_rpc_doc,
+        rich_indexer_rpc_doc,
         experiment_rpc_doc,
     )
     .into()

resource/ckb.toml

@@ -115,7 +115,7 @@ listen_address = "127.0.0.1:8114" # {{
 # Default is 10MiB = 10 * 1024 * 1024
 max_request_body_size = 10485760
 
-# List of API modules: ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Experiment", "Debug", "Indexer"]
+# List of API modules: ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Experiment", "Debug", "Indexer", "RichIndexer"]
 modules = ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Experiment"] # {{
 # dev => modules = ["Net", "Pool", "Miner", "Chain", "Stats", "Subscription", "Experiment", "Debug"]
 # integration => modules = ["Net", "Pool", "Miner", "Chain", "Experiment", "Stats", "IntegrationTest"]
@@ -196,7 +196,8 @@ block_uncles_cache_size    = 30
 # # Or you may want use more flexible scripts, block template as arg.
 # notify_scripts = ["{cmd} {blocktemplate}"]
 #
-# # CKB built-in indexer settings. Existing indexes can be cleaned up using the `ckb reset-data --indexer` subcommand.
+# # CKB built-in indexer/rich-indexer settings.
+# # Utilize the `ckb reset-data --indexer` and `ckb reset-data --rich-indexer` subcommands to efficiently clean existing indexes.
 # [indexer_v2]
 # # Indexing the pending txs in the ckb tx-pool
 # index_tx_pool = false
@@ -206,3 +207,14 @@ block_uncles_cache_size    = 30
 # cell_filter = "let script = output.type;script!=() && script.code_hash == \"0x00000000000000000000000000000000000000000000000000545950455f4944\""
 # # The initial tip can be set higher than the current indexer tip as the starting height for indexing.
 # init_tip_hash = "0x8fbd0ec887159d2814cee475911600e3589849670f5ee1ed9798b38fdeef4e44"
+#
+# # CKB rich-indexer has its unique configuration.
+# [indexer_v2.rich_indexer]
+# # By default, it uses an embedded SQLite database.
+# # Alternatively, you can set up a PostgreSQL database service and provide the connection parameters.
+# db_type = "postgres"
+# db_name = "ckb-rich-indexer"
+# db_host = "127.0.0.1"
+# db_port = 5432
+# db_user = "postgres"
+# db_password = "123456"

rpc/Cargo.toml

@@ -38,6 +38,8 @@ ckb-tx-pool = { path = "../tx-pool", version = "= 0.115.0-pre" }
 ckb-memory-tracker = { path = "../util/memory-tracker", version = "= 0.115.0-pre" }
 ckb-pow = { path = "../pow", version = "= 0.115.0-pre" }
 ckb-indexer = { path = "../util/indexer", version = "= 0.115.0-pre" }
+ckb-indexer-sync = { path = "../util/indexer-sync", version = "= 0.115.0-pre" }
+ckb-rich-indexer = { path = "../util/rich-indexer", version = "= 0.115.0-pre" }
 ckb-stop-handler = { path = "../util/stop-handler", version = "= 0.115.0-pre" }
 itertools.workspace = true
 tokio = "1"

rpc/README.md

@@ -110,6 +110,12 @@ The crate `ckb-rpc`'s minimum supported rustc version is 1.71.1.
         * [Method `get_raw_tx_pool`](#pool-get_raw_tx_pool)
         * [Method `get_pool_tx_detail_info`](#pool-get_pool_tx_detail_info)
         * [Method `tx_pool_ready`](#pool-tx_pool_ready)
+    * [Module Rich_indexer](#module-rich_indexer) [👉 OpenRPC spec](http://playground.open-rpc.org/?uiSchema[appBar][ui:title]=CKB-Rich_indexer&uiSchema[appBar][ui:splitView]=false&uiSchema[appBar][ui:examplesDropdown]=false&uiSchema[appBar][ui:logoUrl]=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/ckb-logo.jpg&schemaUrl=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/json/rich_indexer_rpc_doc.json)
+
+        * [Method `get_indexer_tip`](#rich_indexer-get_indexer_tip)
+        * [Method `get_cells`](#rich_indexer-get_cells)
+        * [Method `get_transactions`](#rich_indexer-get_transactions)
+        * [Method `get_cells_capacity`](#rich_indexer-get_cells_capacity)
     * [Module Stats](#module-stats) [👉 OpenRPC spec](http://playground.open-rpc.org/?uiSchema[appBar][ui:title]=CKB-Stats&uiSchema[appBar][ui:splitView]=false&uiSchema[appBar][ui:examplesDropdown]=false&uiSchema[appBar][ui:logoUrl]=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/ckb-logo.jpg&schemaUrl=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/json/stats_rpc_doc.json)
 
         * [Method `get_blockchain_info`](#stats-get_blockchain_info)
@@ -4650,6 +4656,163 @@ Response
 }
 ```
 
+### Module `Rich_indexer`
+- [👉 OpenRPC spec](http://playground.open-rpc.org/?uiSchema[appBar][ui:title]=CKB-Rich_indexer&uiSchema[appBar][ui:splitView]=false&uiSchema[appBar][ui:examplesDropdown]=false&uiSchema[appBar][ui:logoUrl]=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/ckb-logo.jpg&schemaUrl=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/json/rich_indexer_rpc_doc.json)
+
+
+RPC Module Rich Indexer.
+
+<a id="rich_indexer-get_indexer_tip"></a>
+#### Method `get_indexer_tip`
+* `get_indexer_tip()`
+
+* result: [`IndexerTip`](#type-indexertip) `|` `null`
+
+Returns the indexed tip.
+
+###### Returns
+  * block_hash - indexed tip block hash
+  * block_number - indexed tip block number
+
+###### Examples
+
+Same as CKB Indexer.
+
+<a id="rich_indexer-get_cells"></a>
+#### Method `get_cells`
+* `get_cells(search_key, order, limit, after)`
+    * `search_key`: [`IndexerSearchKey`](#type-indexersearchkey)
+    * `order`: [`IndexerOrder`](#type-indexerorder)
+    * `limit`: [`Uint32`](#type-uint32)
+    * `after`: [`JsonBytes`](#type-jsonbytes) `|` `null`
+* result: [`IndexerPagination_for_IndexerCell`](#type-indexerpagination_for_indexercell)
+
+Returns the live cells collection by the lock or type script.
+
+The difference from the original CKB Indexer is that the `script_search_mode` parameter accepts the `partial` enumeration value. This implies that a partial search can be conducted on the `args` of the `script`.
+
+###### Params
+
+* search_key:
+    - script - Script, supports prefix search
+    - script_type - enum, lock | type
+    - script_search_mode - enum, prefix | exact | partial
+    - filter - filter cells by following conditions, all conditions are optional
+         - script: if search script type is lock, filter cells by type script prefix, and vice versa
+         - script_len_range: [u64; 2], filter cells by script len range, [inclusive, exclusive]
+         - output_data: filter cells by output data
+         - output_data_filter_mode: enum, prefix | exact | partial
+         - output_data_len_range: [u64; 2], filter cells by output data len range, [inclusive, exclusive]
+         - output_capacity_range: [u64; 2], filter cells by output capacity range, [inclusive, exclusive]
+         - block_range: [u64; 2], filter cells by block number range, [inclusive, exclusive]
+    - with_data - bool, optional default is `true`, if with_data is set to false, the field of returning cell.output_data is null in the result
+* order: enum, asc | desc
+* limit: result size limit
+* after: pagination parameter, optional
+
+###### Returns
+
+If the number of objects is less than the requested `limit`, it indicates that these are the last page of get_cells.
+
+* objects:
+    - output: the fields of an output cell
+    - output_data: the cell data
+    - out_point: reference to a cell via transaction hash and output index
+    - block_number: the number of the transaction committed in the block
+    - tx_index: the position index of the transaction committed in the block
+* last_cursor: pagination parameter
+
+###### Examples
+
+Same as CKB Indexer.
+
+<a id="rich_indexer-get_transactions"></a>
+#### Method `get_transactions`
+* `get_transactions(search_key, order, limit, after)`
+    * `search_key`: [`IndexerSearchKey`](#type-indexersearchkey)
+    * `order`: [`IndexerOrder`](#type-indexerorder)
+    * `limit`: [`Uint32`](#type-uint32)
+    * `after`: [`JsonBytes`](#type-jsonbytes) `|` `null`
+* result: [`IndexerPagination_for_IndexerTx`](#type-indexerpagination_for_indexertx)
+
+Returns the transactions collection by the lock or type script.
+
+The difference from the original CKB Indexer is that both the `script_search_mode` and `output_data_filter_mode` in `filter` can accept the `partial` enumeration value. This implies that a partial search can be conducted on both the `args` of the `script` and the cell `output_data`.
+
+* search_key:
+    - script - Script, supports prefix search when group_by_transaction is false
+    - script_type - enum, lock | type
+    - script_search_mode - enum, prefix | exact | partial
+    - filter - filter cells by following conditions, all conditions are optional
+        - script: if search script type is lock, filter cells by type script, and vice versa
+        - script_len_range: [u64; 2], filter cells by script len range, [inclusive, exclusive]
+        - output_data: filter cells by output data
+        - output_data_filter_mode: enum, prefix | exact | partial
+        - output_data_len_range: [u64; 2], filter cells by output data len range, [inclusive, exclusive]
+        - output_capacity_range: [u64; 2], filter cells by output capacity range, [inclusive, exclusive]
+        - block_range: [u64; 2], filter cells by block number range, [inclusive, exclusive]
+    - group_by_transaction - bool, optional default is `false`, if group_by_transaction is set to true, the returning objects will be grouped by the tx hash
+* order: enum, asc | desc
+* limit: result size limit
+* after: pagination parameter, optional
+
+###### Returns
+
+If the number of objects is less than the requested `limit`, it indicates that these are the last page of get_transactions.
+
+ * objects - enum, ungrouped TxWithCell | grouped TxWithCells
+    - TxWithCell:
+        - tx_hash: transaction hash,
+        - block_number: the number of the transaction committed in the block
+        - tx_index: the position index of the transaction committed in the block
+        - io_type: enum, input | output
+        - io_index: the position index of the cell in the transaction inputs or outputs
+    - TxWithCells:
+        - tx_hash: transaction hash,
+        - block_number: the number of the transaction committed in the block
+        - tx_index: the position index of the transaction committed in the block
+        - cells: Array [[io_type, io_index]]
+ * last_cursor - pagination parameter
+
+###### Examples
+
+Same as CKB Indexer.
+
+<a id="rich_indexer-get_cells_capacity"></a>
+#### Method `get_cells_capacity`
+* `get_cells_capacity(search_key)`
+    * `search_key`: [`IndexerSearchKey`](#type-indexersearchkey)
+* result: [`IndexerCellsCapacity`](#type-indexercellscapacity) `|` `null`
+
+Returns the live cells capacity by the lock or type script.
+
+The difference from the original CKB Indexer is that the `script_search_mode` parameter accepts the `partial` enumeration value. This implies that a partial search can be conducted on the `args` of the `script`.
+
+###### Parameters
+
+* search_key:
+    - script - Script
+    - script_type - enum, lock | type
+    - script_search_mode - enum, prefix | exact | partial
+    - filter - filter cells by following conditions, all conditions are optional
+        - script: if search script type is lock, filter cells by type script prefix, and vice versa
+        - script_len_range: [u64; 2], filter cells by script len range, [inclusive, exclusive]
+        - output_data: filter cells by output data
+        - output_data_filter_mode: enum, prefix | exact | partial
+        - output_data_len_range: [u64; 2], filter cells by output data len range, [inclusive, exclusive]
+        - output_capacity_range: [u64; 2], filter cells by output capacity range, [inclusive, exclusive]
+        - block_range: [u64; 2], filter cells by block number range, [inclusive, exclusive]
+
+###### Returns
+
+ * capacity - total capacity
+ * block_hash - indexed tip block hash
+ * block_number - indexed tip block number
+
+###### Examples
+
+Same as CKB Indexer.
+
 ### Module `Stats`
 - [👉 OpenRPC spec](http://playground.open-rpc.org/?uiSchema[appBar][ui:title]=CKB-Stats&uiSchema[appBar][ui:splitView]=false&uiSchema[appBar][ui:examplesDropdown]=false&uiSchema[appBar][ui:logoUrl]=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/ckb-logo.jpg&schemaUrl=https://raw.githubusercontent.com/nervosnetwork/ckb-rpc-resources/develop/json/stats_rpc_doc.json)
 

rpc/src/module/mod.rs

@@ -118,6 +118,7 @@ mod indexer;
 mod miner;
 mod net;
 pub(crate) mod pool;
+mod rich_indexer;
 mod stats;
 mod subscription;
 mod test;
@@ -130,6 +131,7 @@ pub(crate) use self::indexer::IndexerRpcImpl;
 pub(crate) use self::miner::MinerRpcImpl;
 pub(crate) use self::net::NetRpcImpl;
 pub(crate) use self::pool::PoolRpcImpl;
+pub(crate) use self::rich_indexer::RichIndexerRpcImpl;
 pub(crate) use self::stats::StatsRpcImpl;
 pub(crate) use self::subscription::SubscriptionRpcImpl;
 pub(crate) use self::test::IntegrationTestRpcImpl;
@@ -142,6 +144,7 @@ pub use self::indexer::{add_indexer_rpc_methods, indexer_rpc_doc, IndexerRpc};
 pub use self::miner::{add_miner_rpc_methods, miner_rpc_doc, MinerRpc};
 pub use self::net::{add_net_rpc_methods, net_rpc_doc, NetRpc};
 pub use self::pool::{add_pool_rpc_methods, pool_rpc_doc, PoolRpc};
+pub use self::rich_indexer::{add_rich_indexer_rpc_methods, rich_indexer_rpc_doc, RichIndexerRpc};
 pub use self::stats::{add_stats_rpc_methods, stats_rpc_doc, StatsRpc};
 pub use self::subscription::{add_subscription_rpc_methods, subscription_rpc_doc, SubscriptionRpc};
 pub use self::test::{