From 343eb094c9e5a93997379c64a3d25c83df74c5ec Mon Sep 17 00:00:00 2001 From: Kai Peacock Date: Wed, 23 Oct 2024 12:57:03 -0700 Subject: [PATCH] feat: adds management canister support for canister snapshots (#917) * feat: adds management canister support for canister snapshots * chore: change ic spec source back to main branch * chore: add in schnorr signing * chore: update management canister interface --------- Co-authored-by: Jason I Co-authored-by: Jason <98767015+dfx-json@users.noreply.github.com> --- docs/CHANGELOG.md | 24 +++--- packages/agent/src/canisters/management.did | 80 ++++++++++++++++--- .../agent/src/canisters/management_idl.ts | 63 ++++++++++++++- .../agent/src/canisters/management_service.ts | 59 +++++++++++++- 4 files changed, 197 insertions(+), 29 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 88825d2a..5ab233db 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -59,6 +59,8 @@ AgentError: Call failed: - docs: clarifies meaning of `effectiveCanisterId` in `CallOptions` ### Changed + +- feat: adds management canister support for canister snapshots - feat: replaces hdkey and bip32 implementations with `@scure/bip39` and `@scure/bip32` due to vulnerability and lack of maintenance for `elliptic` - chore: bumps dev dependency versions to remove warnings - chore: addresses eslint errors uncovered by bumping eslint version @@ -241,8 +243,8 @@ const result = await management.bitcoin_get_balance_query({ ### Added * feat: uses expirable map for subnet keys in `agent-js`, with a timeout of 1 hour -* **feat!: node signature verification** -This feature includes additional changes in support of testing and releasing the feature: +* **feat!: node signature verification** +This feature includes additional changes in support of testing and releasing the feature: * Mainnet e2e tests for queries and calls * published counter canister * New `HttpAgent` option - `verifyQuerySignatures`. Defaults to true, but allows you to opt out of verification. Useful for testing against older replica versions @@ -374,7 +376,7 @@ Changes default stored key for `auth-client` to use ECDSAKey* Also updates the * _Breaking change:_ Moves `Secp256k1KeyIdentity` to its own package. `@dfinity/identity-secp256k1` * _Breaking change:_ Deprecates `@dfinity/authentication`. If you relied on the `isDelegationValid` check, it has been moved to `@dfinity/identity` - + * Deprecates `@dfinity/identity-ledgerhq`. Use `@zondax/ledger-icp` instead. * chore: links assets docs in index * chore: sets up new size-limit job for packages, in preparation for CI @@ -467,9 +469,9 @@ Changes default stored key for `auth-client` to use ECDSAKey* Also updates the ### Changed * Adds a default callback to the `IdleManager` that will refresh the page after clearing the storage -* Adds a new utility method, `canisterStatus`, to `agent-js`. Canister status now allows you to query paths from the canister certificate with a simple interface, using the API from the[interface specification](https://internetcomputer.org/docs/current/references/ic-interface-spec#state-tree-canister-information) +* Adds a new utility method, `canisterStatus`, to `agent-js`. Canister status now allows you to query paths from the canister certificate with a simple interface, using the API from the[interface specification](https://internetcomputer.org/docs/current/references/ic-interface-spec#state-tree-canister-information) Comes with nicely configured options for - + `time`, `controllers`, `subnet`, `module_hash`, `candid`. Additionally, has a utility for reading custom MetaData set using [ic-wasm](https://github.com/dfinity/ic-wasm), as well as generic custom paths in the format of ArrayBuffers. * updates to package.json files for metadata in npm @@ -489,9 +491,9 @@ Changes default stored key for `auth-client` to use ECDSAKey* Also updates the * Versioning tool now sets patch version to 0 for minor version updates, or patch and minor versions to 0 for major version updates * Removes jest-expect-message, which was making test error messages less useful * `HttpAgent` now generates a nonce to ensure that calls are unique by default. If you want to opt out or provide your own nonce logic, you can now pass an option of `disableNonce: true`during the agent initialization. - + If you are currently using `agent.addTransform(makeNonceTransform())` , please note that you should remove that logic, or add the `disableNonce` option to your agent when upgrading. - + ## [0.10.3] @@ -506,13 +508,13 @@ Changes default stored key for `auth-client` to use ECDSAKey* Also updates the ### Changed * Security enhancement - adds a rewrite for urls to subdomains of - + `\*.ic0.app/api` - + to - + `ic0.app/api` - + * Improves error messages for when `HttpAgent` cannot infer `fetch` implementation ## [0.10.1] diff --git a/packages/agent/src/canisters/management.did b/packages/agent/src/canisters/management.did index d3acb2af..0fecfa23 100644 --- a/packages/agent/src/canisters/management.did +++ b/packages/agent/src/canisters/management.did @@ -1,5 +1,6 @@ type canister_id = principal; type wasm_module = blob; +type snapshot_id = blob; type log_visibility = variant { controllers; @@ -45,6 +46,11 @@ type change_details = variant { mode : variant { install; reinstall; upgrade }; module_hash : blob; }; + load_snapshot : record { + canister_version : nat64; + snapshot_id : snapshot_id; + taken_at_timestamp : nat64; + }; controllers_change : record { controllers : vec principal; }; @@ -90,7 +96,13 @@ type bitcoin_network = variant { type bitcoin_address = text; -type block_hash = blob; +type bitcoin_block_hash = blob; + +type bitcoin_block_header = blob; + +type millisatoshi_per_byte = nat64; + +type bitcoin_block_height = nat32; type outpoint = record { txid : blob; @@ -112,14 +124,10 @@ type bitcoin_get_utxos_args = record { }; }; -type bitcoin_get_current_fee_percentiles_args = record { - network : bitcoin_network; -}; - type bitcoin_get_utxos_result = record { utxos : vec utxo; - tip_block_hash : block_hash; - tip_height : nat32; + tip_block_hash : bitcoin_block_hash; + tip_height : bitcoin_block_height; next_page : opt blob; }; @@ -129,12 +137,29 @@ type bitcoin_get_balance_args = record { min_confirmations : opt nat32; }; +type bitcoin_get_balance_result = satoshi; + +type bitcoin_get_current_fee_percentiles_args = record { + network : bitcoin_network; +}; + +type bitcoin_get_current_fee_percentiles_result = vec millisatoshi_per_byte; + type bitcoin_send_transaction_args = record { transaction : blob; network : bitcoin_network; }; -type millisatoshi_per_byte = nat64; +type bitcoin_get_block_headers_args = record { + start_height : bitcoin_block_height; + end_height : opt bitcoin_block_height; + network : bitcoin_network; +}; + +type bitcoin_get_block_headers_result = record { + tip_height : bitcoin_block_height; + block_headers : vec bitcoin_block_header; +}; type node_metrics = record { node_id : principal; @@ -339,9 +364,35 @@ type stored_chunks_result = vec chunk_hash; type upload_chunk_result = chunk_hash; -type bitcoin_get_balance_result = satoshi; +type snapshot = record { + id : snapshot_id; + taken_at_timestamp : nat64; + total_size : nat64; +}; -type bitcoin_get_current_fee_percentiles_result = vec millisatoshi_per_byte; +type take_canister_snapshot_args = record { + canister_id : canister_id; + replace_snapshot : opt snapshot_id; +}; + +type take_canister_snapshot_result = snapshot; + +type load_canister_snapshot_args = record { + canister_id : canister_id; + snapshot_id : snapshot_id; + sender_canister_version : opt nat64; +}; + +type list_canister_snapshots_args = record { + canister_id : canister_id; +}; + +type list_canister_snapshots_result = vec snapshot; + +type delete_canister_snapshot_args = record { + canister_id : canister_id; + snapshot_id : snapshot_id; +}; type fetch_canister_logs_args = record { canister_id : canister_id; @@ -385,11 +436,10 @@ service ic : { // bitcoin interface bitcoin_get_balance : (bitcoin_get_balance_args) -> (bitcoin_get_balance_result); - bitcoin_get_utxos : (bitcoin_get_utxos_args) -> (bitcoin_get_utxos_result); - bitcoin_send_transaction : (bitcoin_send_transaction_args) -> (); bitcoin_get_current_fee_percentiles : (bitcoin_get_current_fee_percentiles_args) -> (bitcoin_get_current_fee_percentiles_result); + bitcoin_get_block_headers : (bitcoin_get_block_headers_args) -> (bitcoin_get_block_headers_result); // metrics interface node_metrics_history : (node_metrics_history_args) -> (node_metrics_history_result); @@ -398,6 +448,12 @@ service ic : { provisional_create_canister_with_cycles : (provisional_create_canister_with_cycles_args) -> (provisional_create_canister_with_cycles_result); provisional_top_up_canister : (provisional_top_up_canister_args) -> (); + // Canister snapshots + take_canister_snapshot : (take_canister_snapshot_args) -> (take_canister_snapshot_result); + load_canister_snapshot : (load_canister_snapshot_args) -> (); + list_canister_snapshots : (list_canister_snapshots_args) -> (list_canister_snapshots_result); + delete_canister_snapshot : (delete_canister_snapshot_args) -> (); + // canister logging fetch_canister_logs : (fetch_canister_logs_args) -> (fetch_canister_logs_result) query; }; diff --git a/packages/agent/src/canisters/management_idl.ts b/packages/agent/src/canisters/management_idl.ts index 07f15d1d..41e27dea 100644 --- a/packages/agent/src/canisters/management_idl.ts +++ b/packages/agent/src/canisters/management_idl.ts @@ -18,6 +18,17 @@ export default ({ IDL }) => { }); const satoshi = IDL.Nat64; const bitcoin_get_balance_result = satoshi; + const bitcoin_block_height = IDL.Nat32; + const bitcoin_get_block_headers_args = IDL.Record({ + start_height: bitcoin_block_height, + end_height: IDL.Opt(bitcoin_block_height), + network: bitcoin_network, + }); + const bitcoin_block_header = IDL.Vec(IDL.Nat8); + const bitcoin_get_block_headers_result = IDL.Record({ + tip_height: bitcoin_block_height, + block_headers: IDL.Vec(bitcoin_block_header), + }); const bitcoin_get_current_fee_percentiles_args = IDL.Record({ network: bitcoin_network, }); @@ -33,7 +44,7 @@ export default ({ IDL }) => { ), address: bitcoin_address, }); - const block_hash = IDL.Vec(IDL.Nat8); + const bitcoin_block_hash = IDL.Vec(IDL.Nat8); const outpoint = IDL.Record({ txid: IDL.Vec(IDL.Nat8), vout: IDL.Nat32, @@ -45,8 +56,8 @@ export default ({ IDL }) => { }); const bitcoin_get_utxos_result = IDL.Record({ next_page: IDL.Opt(IDL.Vec(IDL.Nat8)), - tip_height: IDL.Nat32, - tip_block_hash: block_hash, + tip_height: bitcoin_block_height, + tip_block_hash: bitcoin_block_hash, utxos: IDL.Vec(utxo), }); const bitcoin_send_transaction_args = IDL.Record({ @@ -65,6 +76,7 @@ export default ({ IDL }) => { canister_id: IDL.Principal, }), }); + const snapshot_id = IDL.Vec(IDL.Nat8); const change_details = IDL.Variant({ creation: IDL.Record({ controllers: IDL.Vec(IDL.Principal) }), code_deployment: IDL.Record({ @@ -75,6 +87,11 @@ export default ({ IDL }) => { }), module_hash: IDL.Vec(IDL.Nat8), }), + load_snapshot: IDL.Record({ + canister_version: IDL.Nat64, + taken_at_timestamp: IDL.Nat64, + snapshot_id: snapshot_id, + }), controllers_change: IDL.Record({ controllers: IDL.Vec(IDL.Principal), }), @@ -141,6 +158,10 @@ export default ({ IDL }) => { }); const create_canister_result = IDL.Record({ canister_id: canister_id }); const delete_canister_args = IDL.Record({ canister_id: canister_id }); + const delete_canister_snapshot_args = IDL.Record({ + canister_id: canister_id, + snapshot_id: snapshot_id, + }); const deposit_cycles_args = IDL.Record({ canister_id: canister_id }); const ecdsa_curve = IDL.Variant({ secp256k1: IDL.Null }); const ecdsa_public_key_args = IDL.Record({ @@ -221,6 +242,20 @@ export default ({ IDL }) => { canister_id: canister_id, sender_canister_version: IDL.Opt(IDL.Nat64), }); + const list_canister_snapshots_args = IDL.Record({ + canister_id: canister_id, + }); + const snapshot = IDL.Record({ + id: snapshot_id, + total_size: IDL.Nat64, + taken_at_timestamp: IDL.Nat64, + }); + const list_canister_snapshots_result = IDL.Vec(snapshot); + const load_canister_snapshot_args = IDL.Record({ + canister_id: canister_id, + sender_canister_version: IDL.Opt(IDL.Nat64), + snapshot_id: snapshot_id, + }); const node_metrics_history_args = IDL.Record({ start_at_timestamp_nanos: IDL.Nat64, subnet_id: IDL.Principal, @@ -289,6 +324,11 @@ export default ({ IDL }) => { const stop_canister_args = IDL.Record({ canister_id: canister_id }); const stored_chunks_args = IDL.Record({ canister_id: canister_id }); const stored_chunks_result = IDL.Vec(chunk_hash); + const take_canister_snapshot_args = IDL.Record({ + replace_snapshot: IDL.Opt(snapshot_id), + canister_id: canister_id, + }); + const take_canister_snapshot_result = snapshot; const uninstall_code_args = IDL.Record({ canister_id: canister_id, sender_canister_version: IDL.Opt(IDL.Nat64), @@ -305,6 +345,11 @@ export default ({ IDL }) => { const upload_chunk_result = chunk_hash; return IDL.Service({ bitcoin_get_balance: IDL.Func([bitcoin_get_balance_args], [bitcoin_get_balance_result], []), + bitcoin_get_block_headers: IDL.Func( + [bitcoin_get_block_headers_args], + [bitcoin_get_block_headers_result], + [], + ), bitcoin_get_current_fee_percentiles: IDL.Func( [bitcoin_get_current_fee_percentiles_args], [bitcoin_get_current_fee_percentiles_result], @@ -317,6 +362,7 @@ export default ({ IDL }) => { clear_chunk_store: IDL.Func([clear_chunk_store_args], [], []), create_canister: IDL.Func([create_canister_args], [create_canister_result], []), delete_canister: IDL.Func([delete_canister_args], [], []), + delete_canister_snapshot: IDL.Func([delete_canister_snapshot_args], [], []), deposit_cycles: IDL.Func([deposit_cycles_args], [], []), ecdsa_public_key: IDL.Func([ecdsa_public_key_args], [ecdsa_public_key_result], []), fetch_canister_logs: IDL.Func( @@ -327,6 +373,12 @@ export default ({ IDL }) => { http_request: IDL.Func([http_request_args], [http_request_result], []), install_chunked_code: IDL.Func([install_chunked_code_args], [], []), install_code: IDL.Func([install_code_args], [], []), + list_canister_snapshots: IDL.Func( + [list_canister_snapshots_args], + [list_canister_snapshots_result], + [], + ), + load_canister_snapshot: IDL.Func([load_canister_snapshot_args], [], []), node_metrics_history: IDL.Func([node_metrics_history_args], [node_metrics_history_result], []), provisional_create_canister_with_cycles: IDL.Func( [provisional_create_canister_with_cycles_args], @@ -341,6 +393,11 @@ export default ({ IDL }) => { start_canister: IDL.Func([start_canister_args], [], []), stop_canister: IDL.Func([stop_canister_args], [], []), stored_chunks: IDL.Func([stored_chunks_args], [stored_chunks_result], []), + take_canister_snapshot: IDL.Func( + [take_canister_snapshot_args], + [take_canister_snapshot_result], + [], + ), uninstall_code: IDL.Func([uninstall_code_args], [], []), update_settings: IDL.Func([update_settings_args], [], []), upload_chunk: IDL.Func([upload_chunk_args], [upload_chunk_result], []), diff --git a/packages/agent/src/canisters/management_service.ts b/packages/agent/src/canisters/management_service.ts index fa4ed243..3b239fed 100644 --- a/packages/agent/src/canisters/management_service.ts +++ b/packages/agent/src/canisters/management_service.ts @@ -10,12 +10,24 @@ import type { ActorMethod } from '@dfinity/agent'; import type { IDL } from '@dfinity/candid'; export type bitcoin_address = string; +export type bitcoin_block_hash = Uint8Array | number[]; +export type bitcoin_block_header = Uint8Array | number[]; +export type bitcoin_block_height = number; export interface bitcoin_get_balance_args { network: bitcoin_network; address: bitcoin_address; min_confirmations: [] | [number]; } export type bitcoin_get_balance_result = satoshi; +export interface bitcoin_get_block_headers_args { + start_height: bitcoin_block_height; + end_height: [] | [bitcoin_block_height]; + network: bitcoin_network; +} +export interface bitcoin_get_block_headers_result { + tip_height: bitcoin_block_height; + block_headers: Array; +} export interface bitcoin_get_current_fee_percentiles_args { network: bitcoin_network; } @@ -27,8 +39,8 @@ export interface bitcoin_get_utxos_args { } export interface bitcoin_get_utxos_result { next_page: [] | [Uint8Array | number[]]; - tip_height: number; - tip_block_hash: block_hash; + tip_height: bitcoin_block_height; + tip_block_hash: bitcoin_block_hash; utxos: Array; } export type bitcoin_network = { mainnet: null } | { testnet: null }; @@ -36,7 +48,6 @@ export interface bitcoin_send_transaction_args { transaction: Uint8Array | number[]; network: bitcoin_network; } -export type block_hash = Uint8Array | number[]; export type canister_id = Principal; export interface canister_info_args { canister_id: canister_id; @@ -109,6 +120,13 @@ export type change_details = module_hash: Uint8Array | number[]; }; } + | { + load_snapshot: { + canister_version: bigint; + taken_at_timestamp: bigint; + snapshot_id: snapshot_id; + }; + } | { controllers_change: { controllers: Array } } | { code_uninstall: null }; export type change_origin = @@ -144,6 +162,10 @@ export interface definite_canister_settings { export interface delete_canister_args { canister_id: canister_id; } +export interface delete_canister_snapshot_args { + canister_id: canister_id; + snapshot_id: snapshot_id; +} export interface deposit_cycles_args { canister_id: canister_id; } @@ -196,6 +218,15 @@ export interface install_code_args { canister_id: canister_id; sender_canister_version: [] | [bigint]; } +export interface list_canister_snapshots_args { + canister_id: canister_id; +} +export type list_canister_snapshots_result = Array; +export interface load_canister_snapshot_args { + canister_id: canister_id; + sender_canister_version: [] | [bigint]; + snapshot_id: snapshot_id; +} export type log_visibility = { controllers: null } | { public: null }; export type millisatoshi_per_byte = bigint; export interface node_metrics { @@ -256,6 +287,12 @@ export interface sign_with_schnorr_args { export interface sign_with_schnorr_result { signature: Uint8Array | number[]; } +export interface snapshot { + id: snapshot_id; + total_size: bigint; + taken_at_timestamp: bigint; +} +export type snapshot_id = Uint8Array | number[]; export interface start_canister_args { canister_id: canister_id; } @@ -266,6 +303,11 @@ export interface stored_chunks_args { canister_id: canister_id; } export type stored_chunks_result = Array; +export interface take_canister_snapshot_args { + replace_snapshot: [] | [snapshot_id]; + canister_id: canister_id; +} +export type take_canister_snapshot_result = snapshot; export interface uninstall_code_args { canister_id: canister_id; sender_canister_version: [] | [bigint]; @@ -288,6 +330,10 @@ export interface utxo { export type wasm_module = Uint8Array | number[]; export default interface _SERVICE { bitcoin_get_balance: ActorMethod<[bitcoin_get_balance_args], bitcoin_get_balance_result>; + bitcoin_get_block_headers: ActorMethod< + [bitcoin_get_block_headers_args], + bitcoin_get_block_headers_result + >; bitcoin_get_current_fee_percentiles: ActorMethod< [bitcoin_get_current_fee_percentiles_args], bitcoin_get_current_fee_percentiles_result @@ -299,12 +345,18 @@ export default interface _SERVICE { clear_chunk_store: ActorMethod<[clear_chunk_store_args], undefined>; create_canister: ActorMethod<[create_canister_args], create_canister_result>; delete_canister: ActorMethod<[delete_canister_args], undefined>; + delete_canister_snapshot: ActorMethod<[delete_canister_snapshot_args], undefined>; deposit_cycles: ActorMethod<[deposit_cycles_args], undefined>; ecdsa_public_key: ActorMethod<[ecdsa_public_key_args], ecdsa_public_key_result>; fetch_canister_logs: ActorMethod<[fetch_canister_logs_args], fetch_canister_logs_result>; http_request: ActorMethod<[http_request_args], http_request_result>; install_chunked_code: ActorMethod<[install_chunked_code_args], undefined>; install_code: ActorMethod<[install_code_args], undefined>; + list_canister_snapshots: ActorMethod< + [list_canister_snapshots_args], + list_canister_snapshots_result + >; + load_canister_snapshot: ActorMethod<[load_canister_snapshot_args], undefined>; node_metrics_history: ActorMethod<[node_metrics_history_args], node_metrics_history_result>; provisional_create_canister_with_cycles: ActorMethod< [provisional_create_canister_with_cycles_args], @@ -318,6 +370,7 @@ export default interface _SERVICE { start_canister: ActorMethod<[start_canister_args], undefined>; stop_canister: ActorMethod<[stop_canister_args], undefined>; stored_chunks: ActorMethod<[stored_chunks_args], stored_chunks_result>; + take_canister_snapshot: ActorMethod<[take_canister_snapshot_args], take_canister_snapshot_result>; uninstall_code: ActorMethod<[uninstall_code_args], undefined>; update_settings: ActorMethod<[update_settings_args], undefined>; upload_chunk: ActorMethod<[upload_chunk_args], upload_chunk_result>;