diff --git a/util/indexer-sync/src/lib.rs b/util/indexer-sync/src/lib.rs index b4558b71d5..135d516e17 100644 --- a/util/indexer-sync/src/lib.rs +++ b/util/indexer-sync/src/lib.rs @@ -161,7 +161,9 @@ impl IndexerSyncService { block.number(), block.hash() ); - indexer.append(&block).expect("append block should be OK"); + if let Err(e) = indexer.append(&block) { + error!("Failed to append block: {}. Will attempt to retry.", e); + } } else { info!( "{} rollback {}, {}", @@ -178,7 +180,11 @@ impl IndexerSyncService { } } Ok(None) => match self.get_block_by_number(0) { - Some(block) => indexer.append(&block).expect("append block should be OK"), + Some(block) => { + if let Err(e) = indexer.append(&block) { + error!("Failed to append block: {}. Will attempt to retry.", e); + } + } None => { error!("CKB node returns an empty genesis block"); break; diff --git a/util/rich-indexer/README.md b/util/rich-indexer/README.md index da67f9e0bb..9579fed5f8 100644 --- a/util/rich-indexer/README.md +++ b/util/rich-indexer/README.md @@ -13,6 +13,7 @@ In order to run a CKB node with the Rich-Indexer enabled, it is recommended to a - Processor: 4 core - RAM: 8 GB +- Disk: Solid State Drive (SSD) to ensure performance ## Quick Start @@ -57,7 +58,7 @@ Note that CKB starting options `--indexer` and `--rich-indexer` can only be used ## Enabling Rich Indexer with PostgreSQL -To enable PostgreSQL, you must first set up a functional PostgreSQL service on your own. Please refer to [Server Administration](https://www.postgresql.org/docs/16/admin.html) for guidance. It is recommended to install version 12 or above. +To enable PostgreSQL, you must first set up a functional PostgreSQL service on your own. Please refer to [Server Administration](https://www.postgresql.org/docs/16/admin.html) for guidance. It is recommended to install version 16 or above. For hardware with 4 cores and 8 GB of RAM, it is recommended to make the following two configuration parameter adjustments in PostgreSQL to achieve optimal query performance. diff --git a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells.rs b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells.rs index f66eaa28b6..63a89d4800 100644 --- a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells.rs +++ b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells.rs @@ -169,9 +169,18 @@ impl AsyncRichIndexerHandle { .bind(search_key.script.args.as_bytes()) .bind(get_binary_upper_boundary(search_key.script.args.as_bytes())); } - Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => { + Some(IndexerSearchMode::Exact) => { query = query.bind(search_key.script.args.as_bytes()); } + Some(IndexerSearchMode::Partial) => match self.store.db_driver { + DBDriver::Postgres => { + let new_args = escape_and_wrap_for_postgres_like(&search_key.script.args); + query = query.bind(new_args); + } + DBDriver::Sqlite => { + query = query.bind(search_key.script.args.as_bytes()); + } + }, } if let Some(filter) = search_key.filter.as_ref() { if let Some(script) = filter.script.as_ref() { @@ -190,9 +199,18 @@ impl AsyncRichIndexerHandle { .bind(data.as_bytes()) .bind(get_binary_upper_boundary(data.as_bytes())); } - Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => { + Some(IndexerSearchMode::Exact) => { query = query.bind(data.as_bytes()); } + Some(IndexerSearchMode::Partial) => match self.store.db_driver { + DBDriver::Postgres => { + let new_data = escape_and_wrap_for_postgres_like(data); + query = query.bind(new_data); + } + DBDriver::Sqlite => { + query = query.bind(data.as_bytes()); + } + }, } } } diff --git a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells_capacity.rs b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells_capacity.rs index dca41b72df..5a14d90592 100644 --- a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells_capacity.rs +++ b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_cells_capacity.rs @@ -129,9 +129,18 @@ impl AsyncRichIndexerHandle { .bind(search_key.script.args.as_bytes()) .bind(get_binary_upper_boundary(search_key.script.args.as_bytes())); } - Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => { + Some(IndexerSearchMode::Exact) => { query = query.bind(search_key.script.args.as_bytes()); } + Some(IndexerSearchMode::Partial) => match self.store.db_driver { + DBDriver::Postgres => { + let new_args = escape_and_wrap_for_postgres_like(&search_key.script.args); + query = query.bind(new_args); + } + DBDriver::Sqlite => { + query = query.bind(search_key.script.args.as_bytes()); + } + }, } if let Some(filter) = search_key.filter.as_ref() { if let Some(script) = filter.script.as_ref() { @@ -150,9 +159,18 @@ impl AsyncRichIndexerHandle { .bind(data.as_bytes()) .bind(get_binary_upper_boundary(data.as_bytes())); } - Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => { + Some(IndexerSearchMode::Exact) => { query = query.bind(data.as_bytes()); } + Some(IndexerSearchMode::Partial) => match self.store.db_driver { + DBDriver::Postgres => { + let new_data = escape_and_wrap_for_postgres_like(data); + query = query.bind(new_data); + } + DBDriver::Sqlite => { + query = query.bind(data.as_bytes()); + } + }, } } } diff --git a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_transactions.rs b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_transactions.rs index e4de2a9c80..d8d58e0d68 100644 --- a/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_transactions.rs +++ b/util/rich-indexer/src/indexer_handle/async_indexer_handle/get_transactions.rs @@ -194,9 +194,18 @@ pub async fn get_tx_with_cell( .bind(search_key.script.args.as_bytes()) .bind(get_binary_upper_boundary(search_key.script.args.as_bytes())); } - Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => { + Some(IndexerSearchMode::Exact) => { query = query.bind(search_key.script.args.as_bytes()); } + Some(IndexerSearchMode::Partial) => match db_driver { + DBDriver::Postgres => { + let new_args = escape_and_wrap_for_postgres_like(&search_key.script.args); + query = query.bind(new_args); + } + DBDriver::Sqlite => { + query = query.bind(search_key.script.args.as_bytes()); + } + }, } if let Some(filter) = search_key.filter.as_ref() { if let Some(script) = filter.script.as_ref() { @@ -215,9 +224,18 @@ pub async fn get_tx_with_cell( .bind(data.as_bytes()) .bind(get_binary_upper_boundary(data.as_bytes())); } - Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => { + Some(IndexerSearchMode::Exact) => { query = query.bind(data.as_bytes()); } + Some(IndexerSearchMode::Partial) => match db_driver { + DBDriver::Postgres => { + let new_data = escape_and_wrap_for_postgres_like(data); + query = query.bind(new_data); + } + DBDriver::Sqlite => { + query = query.bind(data.as_bytes()); + } + }, } } } @@ -326,9 +344,18 @@ pub async fn get_tx_with_cells( .bind(search_key.script.args.as_bytes()) .bind(get_binary_upper_boundary(search_key.script.args.as_bytes())); } - Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => { + Some(IndexerSearchMode::Exact) => { query = query.bind(search_key.script.args.as_bytes()); } + Some(IndexerSearchMode::Partial) => match db_driver { + DBDriver::Postgres => { + let new_args = escape_and_wrap_for_postgres_like(&search_key.script.args); + query = query.bind(new_args); + } + DBDriver::Sqlite => { + query = query.bind(search_key.script.args.as_bytes()); + } + }, } if let Some(filter) = search_key.filter.as_ref() { if let Some(script) = filter.script.as_ref() { @@ -347,9 +374,18 @@ pub async fn get_tx_with_cells( .bind(data.as_bytes()) .bind(get_binary_upper_boundary(data.as_bytes())); } - Some(IndexerSearchMode::Exact) | Some(IndexerSearchMode::Partial) => { + Some(IndexerSearchMode::Exact) => { query = query.bind(data.as_bytes()); } + Some(IndexerSearchMode::Partial) => match db_driver { + DBDriver::Postgres => { + let new_data = escape_and_wrap_for_postgres_like(data); + query = query.bind(new_data); + } + DBDriver::Sqlite => { + query = query.bind(data.as_bytes()); + } + }, } } } @@ -566,10 +602,7 @@ fn build_filter( Some(IndexerSearchMode::Partial) => { match db_driver { DBDriver::Postgres => { - query_builder.and_where(format!( - "position(${} in output.data) > 0", - param_index - )); + query_builder.and_where(format!("output.data LIKE ${}", param_index)); } DBDriver::Sqlite => { query_builder diff --git a/util/rich-indexer/src/indexer_handle/async_indexer_handle/mod.rs b/util/rich-indexer/src/indexer_handle/async_indexer_handle/mod.rs index c0fb7c50a6..7f6032ef25 100644 --- a/util/rich-indexer/src/indexer_handle/async_indexer_handle/mod.rs +++ b/util/rich-indexer/src/indexer_handle/async_indexer_handle/mod.rs @@ -9,7 +9,7 @@ use ckb_app_config::DBDriver; use ckb_indexer_sync::{Error, Pool}; use ckb_jsonrpc_types::{ IndexerRange, IndexerScriptType, IndexerSearchKey, IndexerSearchKeyFilter, IndexerSearchMode, - IndexerTip, + IndexerTip, JsonBytes, }; use ckb_types::H256; use num_bigint::BigUint; @@ -99,7 +99,7 @@ fn build_query_script_sql( Some(IndexerSearchMode::Partial) => { match db_driver { DBDriver::Postgres => { - query_builder.and_where(format!("position(${} in args) > 0", param_index)); + query_builder.and_where(format!("args LIKE ${}", param_index)); } DBDriver::Sqlite => { query_builder.and_where(format!("instr(args, ${}) > 0", param_index)); @@ -140,7 +140,7 @@ fn build_query_script_id_sql( Some(IndexerSearchMode::Partial) => { match db_driver { DBDriver::Postgres => { - query_builder.and_where(format!("position(${} in args) > 0", param_index)); + query_builder.and_where(format!("args LIKE ${}", param_index)); } DBDriver::Sqlite => { query_builder.and_where(format!("instr(args, ${}) > 0", param_index)); @@ -229,10 +229,7 @@ fn build_cell_filter( Some(IndexerSearchMode::Partial) => { match db_driver { DBDriver::Postgres => { - query_builder.and_where(format!( - "position(${} in output.data) > 0", - param_index - )); + query_builder.and_where(format!("output.data LIKE ${}", param_index)); } DBDriver::Sqlite => { query_builder @@ -316,6 +313,35 @@ pub(crate) fn convert_max_values_in_search_filter( }) } +/// Escapes special characters and wraps data with '%' for PostgreSQL LIKE queries. +/// +/// This function escapes the characters '%', '\' and '_' in the input `JsonBytes` by prefixing them with '\'. +/// It then wraps the processed data with '%' at both the start and end for use in PostgreSQL LIKE queries. +/// Note: This function is not suitable for SQLite queries if the data contains NUL characters (0x00), +/// as SQLite treats NUL as the end of the string. +fn escape_and_wrap_for_postgres_like(data: &JsonBytes) -> Vec { + // 0x5c is the default escape character '\' + // 0x25 is the '%' wildcard + // 0x5f is the '_' wildcard + + let mut new_data: Vec = data + .as_bytes() + .iter() + .flat_map(|&b| { + if b == 0x25 || b == 0x5c || b == 0x5f { + vec![0x5c, b] + } else { + vec![b] + } + }) + .collect(); + + new_data.insert(0, 0x25); // Start with % + new_data.push(0x25); // End with % + + new_data +} + #[cfg(test)] mod tests { use super::*; diff --git a/util/rich-indexer/src/tests/query.rs b/util/rich-indexer/src/tests/query.rs index d9e0c52a26..80731de569 100644 --- a/util/rich-indexer/src/tests/query.rs +++ b/util/rich-indexer/src/tests/query.rs @@ -1566,7 +1566,7 @@ async fn output_data_filter_mode_rpc() { .unwrap(); assert_eq!(1, cells.objects.len(),); - // test get_cells_capacity rpc with output_data Partial search mode + // test get_cells_capacity rpc with output_data Prefix search mode let cells = rpc .get_cells_capacity(IndexerSearchKey { script: lock_script11.clone().into(),