From 8caa1bddb98e8bab97fea52977fa3ac629545e49 Mon Sep 17 00:00:00 2001 From: Kukks Date: Wed, 4 Oct 2023 13:32:14 +0200 Subject: [PATCH 01/16] NBX enhancements This PR makes use of the enhanced flexivility of the postgres indexer and exposes them over the current API. Notably it: * has a new tracked source, Wallet, which can act as a "bag" for individual scripts, or as a parent to other wallets. * the generate wallet and track wallet now support specifying a parent wallet * there is an "associate scripts" endpoint which adds specific scripts/addresses to a tracked source to be watched. * there is an "import coins" endpoint which allows you to tell NBX that a wallet has existing coins that it has not managed to track (as they were possibly created before nbx started tracking the wallet). This endpoint verifies the coins are real by calling gettxout on bitcoin core. If you have a local state of coins, this is a much lighter and scalable version of resotring a wallet's utxos on nbx than the scanutxoset option (which can only run exclusively and takes a lot of resources). This can be improved by also adding a utxoproof that it was mined in a specific block. --- NBXplorer.Client/ExplorerClient.cs | 81 ++++---- .../Models/GenerateWalletRequest.cs | 2 + NBXplorer.Client/Models/ImportUTXORequest.cs | 9 + NBXplorer.Client/Models/TrackWalletRequest.cs | 2 + NBXplorer.Client/Models/TrackedSource.cs | 40 ++++ NBXplorer.Tests/UnitTest1.cs | 128 +++++++++++++ NBXplorer.sln | 1 + .../Backends/Postgres/PostgresRepository.cs | 97 +++++++++- NBXplorer/Controllers/ControllerBase.cs | 4 +- NBXplorer/Controllers/MainController.cs | 65 +++++-- .../Controllers/PostgresMainController.cs | 128 ++++++++++++- NBXplorer/TrackedTransaction.cs | 2 +- docker-compose.regtest.yml | 4 +- docs/API.md | 180 ++++++++---------- 14 files changed, 576 insertions(+), 167 deletions(-) create mode 100644 NBXplorer.Client/Models/ImportUTXORequest.cs diff --git a/NBXplorer.Client/ExplorerClient.cs b/NBXplorer.Client/ExplorerClient.cs index ddc2d9f07..9707fc810 100644 --- a/NBXplorer.Client/ExplorerClient.cs +++ b/NBXplorer.Client/ExplorerClient.cs @@ -269,7 +269,7 @@ public void Track(DerivationStrategyBase strategy, CancellationToken cancellatio } public Task TrackAsync(DerivationStrategyBase strategy, CancellationToken cancellation = default) { - return TrackAsync(TrackedSource.Create(strategy), cancellation); + return TrackAsync(TrackedSource.Create(strategy), cancellation: cancellation); } public void Track(DerivationStrategyBase strategy, TrackWalletRequest trackDerivationRequest, CancellationToken cancellation = default) @@ -285,14 +285,13 @@ public async Task TrackAsync(DerivationStrategyBase strategy, TrackWalletRequest public void Track(TrackedSource trackedSource, CancellationToken cancellation = default) { - TrackAsync(trackedSource, cancellation).GetAwaiter().GetResult(); + TrackAsync(trackedSource, cancellation: cancellation).GetAwaiter().GetResult(); } - public Task TrackAsync(TrackedSource trackedSource, CancellationToken cancellation = default) + public Task TrackAsync(TrackedSource trackedSource, TrackWalletRequest trackDerivationRequest = null, CancellationToken cancellation = default) { if (trackedSource == null) throw new ArgumentNullException(nameof(trackedSource)); - - return SendAsync(HttpMethod.Post, null, GetBasePath(trackedSource), cancellation); + return SendAsync(HttpMethod.Post, trackDerivationRequest, GetBasePath(trackedSource), cancellation); } private Exception UnSupported(TrackedSource trackedSource) @@ -311,19 +310,20 @@ public GetBalanceResponse GetBalance(DerivationStrategyBase userDerivationScheme } public Task GetBalanceAsync(DerivationStrategyBase userDerivationScheme, CancellationToken cancellation = default) { - return SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/derivations/{userDerivationScheme}/balance", cancellation); + return GetBalanceAsync(TrackedSource.Create(userDerivationScheme), cancellation); } - - public GetBalanceResponse GetBalance(BitcoinAddress address, CancellationToken cancellation = default) { return GetBalanceAsync(address, cancellation).GetAwaiter().GetResult(); } public Task GetBalanceAsync(BitcoinAddress address, CancellationToken cancellation = default) { - return SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/addresses/{address}/balance", cancellation); + return GetBalanceAsync(TrackedSource.Create(address), cancellation); + } + public Task GetBalanceAsync(TrackedSource trackedSource, CancellationToken cancellation = default) + { + return SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/balance", cancellation); } - public Task CancelReservationAsync(DerivationStrategyBase strategy, KeyPath[] keyPaths, CancellationToken cancellation = default) { return SendAsync(HttpMethod.Post, keyPaths, $"v1/cryptos/{CryptoCode}/derivations/{strategy}/addresses/cancelreservation", cancellation); @@ -365,19 +365,8 @@ public Task GetTransactionsAsync(TrackedSource trackedS { if (trackedSource == null) throw new ArgumentNullException(nameof(trackedSource)); - if (trackedSource is DerivationSchemeTrackedSource dsts) - { - return SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/derivations/{dsts.DerivationStrategy}/transactions", cancellation); - } - else if (trackedSource is AddressTrackedSource asts) - { - return SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/addresses/{asts.Address}/transactions", cancellation); - } - else - throw UnSupported(trackedSource); + return SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/transactions", cancellation); } - - public TransactionInformation GetTransaction(TrackedSource trackedSource, uint256 txId, CancellationToken cancellation = default) { return this.GetTransactionAsync(trackedSource, txId, cancellation).GetAwaiter().GetResult(); @@ -399,16 +388,23 @@ public Task GetTransactionAsync(TrackedSource trackedSou throw new ArgumentNullException(nameof(txId)); if (trackedSource == null) throw new ArgumentNullException(nameof(trackedSource)); - if (trackedSource is DerivationSchemeTrackedSource dsts) - { - return SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/derivations/{dsts.DerivationStrategy}/transactions/{txId}", cancellation); - } - else if (trackedSource is AddressTrackedSource asts) - { - return SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/addresses/{asts.Address}/transactions/{txId}", cancellation); - } - else - throw UnSupported(trackedSource); + return SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/transactions/{txId}", cancellation); + } + public async Task AssociateScriptsAsync(TrackedSource trackedSource, Dictionary scripts, CancellationToken cancellation = default) + { + if (scripts == null) + throw new ArgumentNullException(nameof(scripts)); + if (trackedSource == null) + throw new ArgumentNullException(nameof(trackedSource)); + await SendAsync(HttpMethod.Post, scripts, $"{GetBasePath(trackedSource)}/associate", cancellation); + } + public async Task ImportUTXOs(TrackedSource trackedSource, ImportUTXORequest[] utxos, CancellationToken cancellation = default) + { + if (utxos == null) + throw new ArgumentNullException(nameof(utxos)); + if (trackedSource == null) + throw new ArgumentNullException(nameof(trackedSource)); + await SendAsync(HttpMethod.Post, utxos, $"{GetBasePath(trackedSource)}/import-utxos", cancellation); } public Task RescanAsync(RescanRequest rescanRequest, CancellationToken cancellation = default) @@ -653,12 +649,30 @@ internal async Task SendAsync(HttpMethod method, object body, FormattableS if (Auth.RefreshCache()) { message = CreateMessage(method, body, relativePath); - result = await Client.SendAsync(message).ConfigureAwait(false); + result = await Client.SendAsync(message, cancellation).ConfigureAwait(false); } } return await ParseResponse(result).ConfigureAwait(false); } + internal async Task SendAsync(HttpMethod method, object body, FormattableString relativePath, CancellationToken cancellation) + { + var message = CreateMessage(method, body, relativePath); + var result = await Client.SendAsync(message, cancellation).ConfigureAwait(false); + if (result.StatusCode == HttpStatusCode.GatewayTimeout || result.StatusCode == HttpStatusCode.RequestTimeout) + { + throw new HttpRequestException($"HTTP error {(int)result.StatusCode}", new TimeoutException()); + } + if ((int)result.StatusCode == 401) + { + if (Auth.RefreshCache()) + { + message = CreateMessage(method, body, relativePath); + result = await Client.SendAsync(message, cancellation).ConfigureAwait(false); + } + } + await ParseResponse(result).ConfigureAwait(false); + } internal HttpRequestMessage CreateMessage(HttpMethod method, object body, FormattableString relativePath) { var uri = GetFullUri(relativePath); @@ -721,6 +735,7 @@ private FormattableString GetBasePath(TrackedSource trackedSource) { DerivationSchemeTrackedSource dsts => $"v1/cryptos/{CryptoCode}/derivations/{dsts.DerivationStrategy}", AddressTrackedSource asts => $"v1/cryptos/{CryptoCode}/addresses/{asts.Address}", + WalletTrackedSource wts => $"v1/cryptos/{CryptoCode}/wallets/{wts.WalletId}", _ => throw UnSupported(trackedSource) }; } diff --git a/NBXplorer.Client/Models/GenerateWalletRequest.cs b/NBXplorer.Client/Models/GenerateWalletRequest.cs index 666987fb0..baaea9658 100644 --- a/NBXplorer.Client/Models/GenerateWalletRequest.cs +++ b/NBXplorer.Client/Models/GenerateWalletRequest.cs @@ -17,5 +17,7 @@ public class GenerateWalletRequest public bool ImportKeysToRPC { get; set; } public bool SavePrivateKeys { get; set; } public Dictionary AdditionalOptions { get; set; } + + public TrackedSource ParentWallet { get; set; } } } diff --git a/NBXplorer.Client/Models/ImportUTXORequest.cs b/NBXplorer.Client/Models/ImportUTXORequest.cs new file mode 100644 index 000000000..ce53edbaf --- /dev/null +++ b/NBXplorer.Client/Models/ImportUTXORequest.cs @@ -0,0 +1,9 @@ +using NBitcoin; + +namespace NBXplorer.Models; + +public class ImportUTXORequest +{ + public Coin Coin { get; set; } + public MerkleBlock Proof { get; set; } +} \ No newline at end of file diff --git a/NBXplorer.Client/Models/TrackWalletRequest.cs b/NBXplorer.Client/Models/TrackWalletRequest.cs index 19ed4f196..01567fecb 100644 --- a/NBXplorer.Client/Models/TrackWalletRequest.cs +++ b/NBXplorer.Client/Models/TrackWalletRequest.cs @@ -7,6 +7,8 @@ public class TrackWalletRequest { public TrackDerivationOption[] DerivationOptions { get; set; } public bool Wait { get; set; } = false; + + public TrackedSource ParentWallet { get; set; } = null; } public class TrackDerivationOption diff --git a/NBXplorer.Client/Models/TrackedSource.cs b/NBXplorer.Client/Models/TrackedSource.cs index c170a95df..b4f1557f6 100644 --- a/NBXplorer.Client/Models/TrackedSource.cs +++ b/NBXplorer.Client/Models/TrackedSource.cs @@ -26,6 +26,12 @@ public static bool TryParse(string str, out TrackedSource trackedSource, NBXplor return false; trackedSource = addressTrackedSource; } + else if (strSpan.StartsWith("WALLET:".AsSpan(), StringComparison.Ordinal)) + { + if (!WalletTrackedSource.TryParse(strSpan, out var walletTrackedSource)) + return false; + trackedSource = walletTrackedSource; + } else { return false; @@ -97,6 +103,40 @@ public static TrackedSource Parse(string str, NBXplorerNetwork network) } } + public class WalletTrackedSource : TrackedSource + { + public string WalletId { get; } + + public WalletTrackedSource(string walletId) + { + WalletId = walletId; + } + + public static bool TryParse(ReadOnlySpan strSpan, out WalletTrackedSource walletTrackedSource) + { + if (strSpan == null) + throw new ArgumentNullException(nameof(strSpan)); + walletTrackedSource = null; + if (!strSpan.StartsWith("WALLET:".AsSpan(), StringComparison.Ordinal)) + return false; + try + { + walletTrackedSource = new WalletTrackedSource(strSpan.Slice("WALLET:".Length).ToString()); + return true; + } + catch { return false; } + } + + public override string ToString() + { + return "WALLET:" + WalletId; + } + public override string ToPrettyString() + { + return WalletId; + } + } + public class AddressTrackedSource : TrackedSource, IDestination { // Note that we should in theory access BitcoinAddress. But parsing BitcoinAddress is very expensive, so we keep storing plain strings diff --git a/NBXplorer.Tests/UnitTest1.cs b/NBXplorer.Tests/UnitTest1.cs index e1fca3cb1..9b917b15c 100644 --- a/NBXplorer.Tests/UnitTest1.cs +++ b/NBXplorer.Tests/UnitTest1.cs @@ -4523,5 +4523,133 @@ public async Task CanUseRPCProxy(Backend backend) await tester.Client.RPCClient.GetTxOutAsync(uint256.One, 0); } } + + + private async Task Eventually(Func tsk) + { + var i = 0; + while (i <10) + { + try + { + + await tsk.Invoke(); + break; + + } + catch (Exception e) + { + await Task.Delay(500); + } + + i++; + } + } + + [Theory] + [InlineData(Backend.Postgres)] + [InlineData(Backend.DBTrie)] + public async Task CanAssociateIndependentScripts(Backend backend) + { + using var tester = ServerTester.Create(backend); + + var wallet1 = Guid.NewGuid().ToString(); + var wallet1TS = new WalletTrackedSource(wallet1); + var parentWallet = Guid.NewGuid().ToString(); + var parentWalletTS = new WalletTrackedSource(parentWallet); + + + //this should create both wallets + if (backend == Backend.DBTrie) + { + await Assert.ThrowsAsync(async () => await tester.Client.TrackAsync(wallet1TS, + new TrackWalletRequest() + { + ParentWallet = parentWalletTS + }, CancellationToken.None)); + + await Assert.ThrowsAsync(async () =>await tester.Client.GenerateWalletAsync(new GenerateWalletRequest() + { + ParentWallet = parentWalletTS + })); + + await Assert.ThrowsAsync(async () =>await tester.Client.AssociateScriptsAsync(parentWalletTS, new Dictionary())); + await Assert.ThrowsAsync(async () =>await tester.Client.ImportUTXOs(parentWalletTS, Array.Empty())); + return; + } + + await tester.Client.TrackAsync(wallet1TS, new TrackWalletRequest() + { + ParentWallet = parentWalletTS + }, CancellationToken.None); + + GenerateWalletResponse derivationWallet = null; + + derivationWallet = await tester.Client.GenerateWalletAsync(new GenerateWalletRequest() + { + ParentWallet = parentWalletTS + }); + + var derivationWalletTS = TrackedSource.Create(derivationWallet.DerivationScheme); + + var address = await tester.Client.GetUnusedAsync(derivationWallet.DerivationScheme, + DerivationFeature.Deposit, 0, true); + await tester.RPC.SendToAddressAsync(address.Address, Money.FromUnit(0.1m, MoneyUnit.BTC)); + var b1 = await tester.Client.GetBalanceAsync(derivationWalletTS); + var b2 = await tester.Client.GetBalanceAsync(parentWalletTS); + Assert.Equal(b1.Total, b2.Total); + + var derivationUtxos = await tester.Client.GetUTXOsAsync(derivationWalletTS); + var parentWalletUtxos = await tester.Client.GetUTXOsAsync(parentWalletTS); + Assert.Equal(derivationUtxos.GetUnspentUTXOs().Count(), parentWalletUtxos.GetUnspentUTXOs().Count()); + + var newAddr = await tester.RPC.GetNewAddressAsync(); + var newAddr2 = await tester.RPC.GetNewAddressAsync(); + var udetectedTxId = await tester.RPC.SendToAddressAsync(newAddr, Money.FromUnit(0.1m, MoneyUnit.BTC)); + await Task.Delay(3000); + var utxos = Assert.Single(await tester.RPC.ListUnspentAsync(0, 0, newAddr)); + await tester.Client.AssociateScriptsAsync(wallet1TS, new Dictionary() + { + {newAddr2.ToString(), true} + }); + + + await tester.RPC.SendToAddressAsync(newAddr2, Money.FromUnit(0.2m, MoneyUnit.BTC)); + + derivationUtxos = await tester.Client.GetUTXOsAsync(derivationWalletTS); + UTXOChanges scriptBagUtxos = null; + //1 utxo was before we started tracking + + await Eventually(async () => + { + parentWalletUtxos = await tester.Client.GetUTXOsAsync(parentWalletTS); + scriptBagUtxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Equal(1, scriptBagUtxos.GetUnspentUTXOs().Length); + Assert.Equal(2, parentWalletUtxos.GetUnspentUTXOs().Length); + Assert.Equal(derivationUtxos.GetUnspentUTXOs().Count() + scriptBagUtxos.GetUnspentUTXOs().Length, + parentWalletUtxos.GetUnspentUTXOs().Count()); + }); + + + await tester.Client.AssociateScriptsAsync(wallet1TS, new Dictionary() + { + {newAddr.ToString(), true} + }); + + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest[] + { + new ImportUTXORequest() + { + Coin = utxos.AsCoin(), + Proof = null + } + }); + + await Eventually(async () => + { + scriptBagUtxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Equal(2, scriptBagUtxos.GetUnspentUTXOs().Length); + }); + } } } diff --git a/NBXplorer.sln b/NBXplorer.sln index a5624256f..74d67a6f1 100644 --- a/NBXplorer.sln +++ b/NBXplorer.sln @@ -19,6 +19,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "SolutionFiles", "SolutionFi Dockerfile.linuxarm32v7 = Dockerfile.linuxarm32v7 Dockerfile.linuxarm64v8 = Dockerfile.linuxarm64v8 .circleci\run-tests.sh = .circleci\run-tests.sh + docs\Postgres-Schema.md = docs\Postgres-Schema.md EndProjectSection EndProject Global diff --git a/NBXplorer/Backends/Postgres/PostgresRepository.cs b/NBXplorer/Backends/Postgres/PostgresRepository.cs index b900b01e9..dbb1b85fc 100644 --- a/NBXplorer/Backends/Postgres/PostgresRepository.cs +++ b/NBXplorer/Backends/Postgres/PostgresRepository.cs @@ -4,6 +4,7 @@ using NBXplorer.DerivationStrategy; using NBXplorer.Models; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -216,6 +217,14 @@ WalletKey GetWalletKey(IDestination destination) m.Add(new JProperty("address", new JValue(address.ToString()))); return new WalletKey(hash, m.ToString(Formatting.None)); } + + WalletKey GetWalletKey(WalletTrackedSource walletTrackedSource) + { + var m = new JObject {new JProperty("type", new JValue("Wallet"))}; + var res = new WalletKey(walletTrackedSource.WalletId, m.ToString(Formatting.None)); + return res; + } + internal WalletKey GetWalletKey(TrackedSource source) { if (source is null) @@ -224,10 +233,36 @@ internal WalletKey GetWalletKey(TrackedSource source) { DerivationSchemeTrackedSource derivation => GetWalletKey(derivation.DerivationStrategy), AddressTrackedSource addr => GetWalletKey(addr.Address), + WalletTrackedSource wallet => GetWalletKey(wallet), + + _ => throw new NotSupportedException(source.GetType().ToString()) }; } + public async Task AssociateScriptsToWalletExplicitly(TrackedSource trackedSource, + Dictionary scripts) + { + var walletKey = GetWalletKey(trackedSource); + await using var conn = await GetConnection(); + var scriptsRecords = scripts.Select(pair => new ScriptInsert(this.Network.CryptoCode, walletKey.wid, + pair.Key.ScriptPubKey.ToHex(), pair.Key.ToString(), pair.Value)).ToArray(); + { + + await conn.Connection.ExecuteAsync( + + "INSERT INTO wallets VALUES (@wid, @metadata::JSONB) ON CONFLICT DO NOTHING", new + { + walletKey.wid, + walletKey.metadata + }); + await conn.Connection.ExecuteAsync( + "INSERT INTO scripts (code, script, addr, used) VALUES(@code, @script, @addr, @used) ON CONFLICT DO NOTHING;" + + "INSERT INTO wallets_scripts (code, wallet_id, script) VALUES(@code, @wallet_id, @script) ON CONFLICT DO NOTHING;" + , scriptsRecords); + } + } + internal record ScriptInsert(string code, string wallet_id, string script, string addr, bool used); internal record DescriptorScriptInsert(string descriptor, int idx, string script, string metadata, string addr, bool used); public async Task GenerateAddresses(DerivationStrategyBase strategy, DerivationFeature derivationFeature, GenerateAddressQuery query = null) { @@ -448,12 +483,26 @@ async Task> GetKeyInformations( if (scripts.Count == 0) return result; string additionalColumn = Network.IsElement ? ", ts.blinded_addr" : ""; - var rows = await connection.QueryAsync($"SELECT ts.script, ts.addr, ts.derivation, ts.keypath, ts.redeem {additionalColumn} FROM " + - "unnest(@records) AS r (script)," + - " LATERAL (" + - " SELECT script, addr, descriptor_metadata->>'derivation' derivation, keypath, descriptors_scripts_metadata->>'redeem' redeem, descriptors_scripts_metadata->>'blindedAddress' blinded_addr " + - " FROM nbxv1_keypath_info ki " + - " WHERE ki.code=@code AND ki.script=r.script) ts;", new { code = Network.CryptoCode, records = scripts.Select(s => s.ToHex()).ToArray() }); + var rows = await connection.QueryAsync($@" + SELECT DISTINCT ts.script, ts.addr, ts.derivation, ts.keypath, ts.redeem , + COALESCE(wd.wallet_id, ws.wallet_id) AS wallet_id, + COALESCE(wd_wallet.metadata->>'type', ws_wallet.metadata->>'type') AS wallet_type + FROM unnest(@records) AS r (script), + LATERAL ( + SELECT script, addr, descriptor_metadata->>'derivation' derivation, + keypath, descriptors_scripts_metadata->>'redeem' redeem, + descriptors_scripts_metadata->>'blindedAddress' blinded_addr, + descriptor_metadata->>'descriptor' descriptor + FROM nbxv1_keypath_info ki + WHERE ki.code=@code AND ki.script=r.script + ) ts + LEFT JOIN wallets_descriptors wd ON wd.descriptor = ts.descriptor AND wd.code = @code + LEFT JOIN wallets wd_wallet ON wd_wallet.wallet_id = wd.wallet_id + LEFT JOIN wallets_scripts ws ON ws.script = ts.script AND ws.code = @code + LEFT JOIN wallets ws_wallet ON ws_wallet.wallet_id = ws.wallet_id + WHERE COALESCE(wd.wallet_id, ws.wallet_id) IS NOT NULL;", + new { code = Network.CryptoCode, records = scripts.Select(s => s.ToHex()).ToArray() }); + foreach (var r in rows) { // This might be the case for a derivation added by a different indexer @@ -466,14 +515,21 @@ async Task> GetKeyInformations( var derivationStrategy = isDescriptor ? Network.DerivationStrategyFactory.Parse(r.derivation) : null; var keypath = isDescriptor ? KeyPath.Parse(r.keypath) : null; var redeem = (string)r.redeem; + string walletType = r.wallet_type; + string walletId = r.wallet_id; + + var trackedSource = derivationStrategy is not null + ? + new DerivationSchemeTrackedSource(derivationStrategy) + : walletType == "Wallet" ? walletId is null ? (TrackedSource)null : new WalletTrackedSource(walletId) : new AddressTrackedSource(addr); + result.Add(script, new KeyPathInformation() { Address = addr, DerivationStrategy = isDescriptor ? derivationStrategy : null, KeyPath = isDescriptor ? keypath : null, ScriptPubKey = script, - TrackedSource = isDescriptor && derivationStrategy is not null ? new DerivationSchemeTrackedSource(derivationStrategy) : - isExplicit ? new AddressTrackedSource(addr) : null, + TrackedSource = trackedSource, Feature = keypath is null ? DerivationFeature.Deposit : KeyPathTemplates.GetDerivationFeature(keypath), Redeem = redeem is null ? null : Script.FromHex(redeem) }); @@ -1251,8 +1307,29 @@ public async Task SaveBlocks(IList slimBlocks) public async Task EnsureWalletCreated(DerivationStrategyBase strategy) { - using var connection = await ConnectionFactory.CreateConnection(); - await connection.ExecuteAsync("INSERT INTO wallets VALUES (@wid, @metadata::JSONB) ON CONFLICT DO NOTHING", GetWalletKey(strategy)); + await EnsureWalletCreated(GetWalletKey(strategy)); + } + + public async Task EnsureWalletCreated(TrackedSource trackedSource, TrackedSource[] parentTrackedSource = null) + { + await EnsureWalletCreated(GetWalletKey(trackedSource), parentTrackedSource?.Select(GetWalletKey).ToArray()); + } + + record WalletHierarchyInsert(string child, string parent); + public async Task EnsureWalletCreated(WalletKey walletKey, WalletKey[] parentWallets = null) + { + await using var connection = await ConnectionFactory.CreateConnection(); + parentWallets ??= Array.Empty(); + var walletRecords = new[] {walletKey}.Concat(parentWallets).ToArray(); + var parentsRecords = parentWallets.Select(key => new WalletHierarchyInsert(walletKey.wid, key.wid)).ToArray(); + + + await connection.ExecuteAsync( + "INSERT INTO wallets (wallet_id, metadata) VALUES (@wid, @metadata::JSONB) ON CONFLICT DO NOTHING" + , walletRecords); + await connection.ExecuteAsync( + "INSERT INTO wallets_wallets (wallet_id, parent_id) VALUES (@child, @parent)ON CONFLICT DO NOTHING" + , parentsRecords); } } diff --git a/NBXplorer/Controllers/ControllerBase.cs b/NBXplorer/Controllers/ControllerBase.cs index 5a361e709..4ba7e7ec8 100644 --- a/NBXplorer/Controllers/ControllerBase.cs +++ b/NBXplorer/Controllers/ControllerBase.cs @@ -27,13 +27,15 @@ public ControllerBase( public IRepositoryProvider RepositoryProvider { get; } public IIndexers Indexers { get; } - internal static TrackedSource GetTrackedSource(DerivationStrategyBase derivationScheme, BitcoinAddress address) + internal static TrackedSource GetTrackedSource(DerivationStrategyBase derivationScheme, BitcoinAddress address, string walletId) { TrackedSource trackedSource = null; if (address != null) trackedSource = new AddressTrackedSource(address); if (derivationScheme != null) trackedSource = new DerivationSchemeTrackedSource(derivationScheme); + if (walletId != null) + trackedSource = new WalletTrackedSource(walletId); return trackedSource; } internal NBXplorerNetwork GetNetwork(string cryptoCode, bool checkRPC) diff --git a/NBXplorer/Controllers/MainController.cs b/NBXplorer/Controllers/MainController.cs index bf468fbff..f0ba44a5f 100644 --- a/NBXplorer/Controllers/MainController.cs +++ b/NBXplorer/Controllers/MainController.cs @@ -25,6 +25,7 @@ using NBXplorer.Backends; using NBitcoin.Scripting; using System.Globalization; +using NBXplorer.Backends.Postgres; namespace NBXplorer.Controllers { @@ -75,7 +76,9 @@ public AddressPoolService AddressPoolService "getrawtransaction", "gettxout", "estimatesmartfee", - "getmempoolinfo" + "getmempoolinfo", + "gettxoutproof", + "verifytxoutproof" }; private Exception JsonRPCNotExposed() { @@ -509,18 +512,31 @@ private bool HasTxIndex(string cryptoCode) [HttpPost] [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}")] [Route("cryptos/{cryptoCode}/addresses/{address}")] + [Route("cryptos/{cryptoCode}/wallets/{walletId}")] public async Task TrackWallet( string cryptoCode, [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] DerivationStrategyBase derivationScheme, [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address, [FromBody] TrackWalletRequest request = null) + BitcoinAddress address, + string walletId, + [FromBody] JObject rawRequest = null) { - request = request ?? new TrackWalletRequest(); - TrackedSource trackedSource = GetTrackedSource(derivationScheme, address); + var request = ParseJObject(rawRequest ?? new JObject(), GetNetwork(cryptoCode, false)); + TrackedSource trackedSource = GetTrackedSource(derivationScheme, address, walletId); if (trackedSource == null) return NotFound(); var network = GetNetwork(cryptoCode, false); + var repo = RepositoryProvider.GetRepository(network); + if (repo is PostgresRepository postgresRepository && + (trackedSource is WalletTrackedSource || + request?.ParentWallet is not null)) + { + await postgresRepository.EnsureWalletCreated(trackedSource, request?.ParentWallet is null? null: new []{request?.ParentWallet }); + } + if (repo is not PostgresRepository && request.ParentWallet is not null) + throw new NBXplorerException(new NBXplorerError(400, "parent-wallet-not-supported", + "Parent wallet is only supported with Postgres")); if (trackedSource is DerivationSchemeTrackedSource dts) { if (request.Wait) @@ -566,17 +582,20 @@ private GenerateAddressQuery GenerateAddressQuery(TrackWalletRequest request, De [HttpGet] [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/transactions/{txId?}")] [Route("cryptos/{cryptoCode}/addresses/{address}/transactions/{txId?}")] + [Route("cryptos/{cryptoCode}/wallets/{walletId}/transactions/{txId?}")] + public async Task GetTransactions( string cryptoCode, [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] DerivationStrategyBase derivationScheme, [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] BitcoinAddress address, + string walletId, [ModelBinder(BinderType = typeof(UInt256ModelBinding))] uint256 txId = null, bool includeTransaction = true) { - var trackedSource = GetTrackedSource(derivationScheme, address); + var trackedSource = GetTrackedSource(derivationScheme, address, walletId); if (trackedSource == null) throw new ArgumentNullException(nameof(trackedSource)); TransactionInformation fetchedTransactionInfo = null; @@ -859,7 +878,7 @@ public async Task GetBalance(string cryptoCode, [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] BitcoinAddress address) { - var getTransactionsResult = await GetTransactions(cryptoCode, derivationScheme, address, includeTransaction: false); + var getTransactionsResult = await GetTransactions(cryptoCode, derivationScheme, address, null, includeTransaction: false); var jsonResult = getTransactionsResult as JsonResult; var transactions = jsonResult?.Value as GetTransactionsResponse; if (transactions == null) @@ -892,15 +911,18 @@ private IMoney CalculateBalance(NBXplorerNetwork network, TransactionInformation [HttpGet] [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/utxos")] [Route("cryptos/{cryptoCode}/addresses/{address}/utxos")] + [Route("cryptos/{cryptoCode}/wallets/{walletId}/utxos")] + [PostgresImplementationActionConstraint(false)] public async Task GetUTXOs( string cryptoCode, [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] DerivationStrategyBase derivationScheme, [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address) + BitcoinAddress address, + string walletId) { - var trackedSource = GetTrackedSource(derivationScheme, address); + var trackedSource = GetTrackedSource(derivationScheme, address, walletId); UTXOChanges changes = null; if (trackedSource == null) throw new ArgumentNullException(nameof(trackedSource)); @@ -980,10 +1002,12 @@ public async Task Broadcast( [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] DerivationStrategyBase derivationScheme, [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address, bool testMempoolAccept = false) + BitcoinAddress address, + string walletId, + bool testMempoolAccept = false) { var network = GetNetwork(cryptoCode, true); - var trackedSource = GetTrackedSource(derivationScheme ?? extPubKey, address); + var trackedSource = GetTrackedSource(derivationScheme ?? extPubKey, address, walletId); var tx = network.NBitcoinNetwork.Consensus.ConsensusFactory.CreateTransaction(); var buffer = new MemoryStream(); await Request.Body.CopyToAsync(buffer); @@ -1070,8 +1094,10 @@ public async Task Broadcast( [HttpPost] [Route("cryptos/{cryptoCode}/derivations")] - public async Task GenerateWallet(string cryptoCode, [FromBody] GenerateWalletRequest request) + public async Task GenerateWallet(string cryptoCode, [FromBody] JObject rawRequest = null) { + var request = ParseJObject(rawRequest, GetNetwork(cryptoCode, false)); + if (request == null) request = new GenerateWalletRequest(); var network = GetNetwork(cryptoCode, request.ImportKeysToRPC); @@ -1089,6 +1115,9 @@ public async Task GenerateWallet(string cryptoCode, [FromBody] Ge throw new NBXplorerException(new NBXplorerError(400, "segwit-not-supported", "Segwit is not supported, please explicitely set scriptPubKeyType to Legacy")); var repo = RepositoryProvider.GetRepository(network); + if (repo is not PostgresRepository && request.ParentWallet is not null) + throw new NBXplorerException(new NBXplorerError(400, "parent-wallet-not-supported", + "Parent wallet is only supported with Postgres")); Mnemonic mnemonic = null; if (request.ExistingMnemonic != null) { @@ -1113,8 +1142,14 @@ public async Task GenerateWallet(string cryptoCode, [FromBody] Ge ScriptPubKeyType = request.ScriptPubKeyType.Value, AdditionalOptions = request.AdditionalOptions is not null ? new System.Collections.ObjectModel.ReadOnlyDictionary(request.AdditionalOptions) : null }); - - await RepositoryProvider.GetRepository(network).EnsureWalletCreated(derivation); + if (request.ParentWallet is not null && repo is PostgresRepository postgresRepository) + { + await postgresRepository.EnsureWalletCreated(TrackedSource.Create(derivation), new[] {request.ParentWallet}); + } + else + { + await repo.EnsureWalletCreated(derivation); + } var derivationTrackedSource = new DerivationSchemeTrackedSource(derivation); List saveMetadata = new List(); if (request.SavePrivateKeys) @@ -1135,7 +1170,7 @@ public async Task GenerateWallet(string cryptoCode, [FromBody] Ge saveMetadata.Add(repo.SaveMetadata(derivationTrackedSource, WellknownMetadataKeys.AccountDescriptor, descriptor)); await Task.WhenAll(saveMetadata.ToArray()); - await TrackWallet(cryptoCode, derivation, null); + await TrackWallet(cryptoCode, derivation,null, null); return Json(new GenerateWalletResponse() { MasterHDKey = masterKey, @@ -1289,7 +1324,7 @@ await repo.Prune(trackedSource, prunableIds #if SUPPORT_DBTRIE public Task GetUTXOs(string cryptoCode, DerivationStrategyBase derivationStrategy) { - return this.GetUTXOs(cryptoCode, derivationStrategy, null); + return this.GetUTXOs(cryptoCode, derivationStrategy, null, null); } #else public Task GetUTXOs(string cryptoCode, DerivationStrategyBase derivationStrategy) diff --git a/NBXplorer/Controllers/PostgresMainController.cs b/NBXplorer/Controllers/PostgresMainController.cs index 75c14decd..568d6d803 100644 --- a/NBXplorer/Controllers/PostgresMainController.cs +++ b/NBXplorer/Controllers/PostgresMainController.cs @@ -8,8 +8,15 @@ using NBXplorer.ModelBinders; using NBXplorer.Models; using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; using System.Linq; using System.Threading.Tasks; +using NBitcoin.DataEncoders; +using NBitcoin.RPC; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; namespace NBXplorer.Controllers { @@ -35,14 +42,16 @@ public PostgresMainController( [HttpGet] [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/balance")] [Route("cryptos/{cryptoCode}/addresses/{address}/balance")] + [Route("cryptos/{cryptoCode}/wallets/{walletId}/balance")] [PostgresImplementationActionConstraint(true)] public async Task GetBalance(string cryptoCode, [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] DerivationStrategyBase derivationScheme, [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address) + BitcoinAddress address, + string walletId) { - var trackedSource = GetTrackedSource(derivationScheme, address); + var trackedSource = GetTrackedSource(derivationScheme, address, walletId); if (trackedSource == null) throw new ArgumentNullException(nameof(trackedSource)); var network = GetNetwork(cryptoCode, false); @@ -86,6 +95,113 @@ public async Task GetBalance(string cryptoCode, }; balance.Total = balance.Confirmed.Add(balance.Unconfirmed); return Json(balance, network.JsonSerializerSettings); + } + + [HttpPost] + [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/associate")] + [Route("cryptos/{cryptoCode}/addresses/{address}/associate")] + [Route("cryptos/{cryptoCode}/wallets/{walletId}/associate")] + [PostgresImplementationActionConstraint(true)] + public async Task AssociateScripts(string cryptoCode, + [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] + DerivationStrategyBase derivationScheme, + [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] + BitcoinAddress address, + string walletId, + [FromBody] Dictionary scripts) + { + var trackedSource = GetTrackedSource(derivationScheme, address, walletId); + if (trackedSource == null) + throw new ArgumentNullException(nameof(trackedSource)); + var network = GetNetwork(cryptoCode, false); + var repo = (PostgresRepository)RepositoryProvider.GetRepository(cryptoCode); + + await repo.AssociateScriptsToWalletExplicitly(trackedSource, + scripts.ToDictionary(pair => (IDestination) BitcoinAddress.Create(pair.Key, network.NBitcoinNetwork), + pair => pair.Value)); + return Ok(); + } + + + + [HttpPost] + [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/import-utxos")] + [Route("cryptos/{cryptoCode}/addresses/{address}/import-utxos")] + [Route("cryptos/{cryptoCode}/wallets/{walletId}/import-utxos")] + [PostgresImplementationActionConstraint(true)] + public async Task ImportUTXOs(string cryptoCode, + [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] + DerivationStrategyBase derivationScheme, + [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] + BitcoinAddress address, + string walletId, + [FromBody] JArray rawRequest) + { + var network = GetNetwork(cryptoCode, true); + var jsonSerializer = JsonSerializer.Create(network.JsonSerializerSettings); + var coins = rawRequest.ToObject(jsonSerializer)?.Where(c => c.Coin != null).ToArray(); + if (coins?.Any() is not true) + throw new ArgumentNullException(nameof(coins)); + + var trackedSource = GetTrackedSource(derivationScheme, address, walletId); + if (trackedSource == null) + throw new ArgumentNullException(nameof(trackedSource)); + var repo = (PostgresRepository) RepositoryProvider.GetRepository(cryptoCode); + + var rpc = RPCClients.Get(network); + + var clientBatch = rpc.PrepareBatch(); + var coinToTxOut = new ConcurrentDictionary(); + var coinToBlock = new ConcurrentDictionary(); + await Task.WhenAll(coins.SelectMany(o => + { + return new[] + { + Task.Run(async () => + { + var txOutResponse = + await clientBatch.GetTxOutAsync(o.Coin.Outpoint.Hash, (int) o.Coin.Outpoint.N); + if (txOutResponse is not null) + coinToTxOut.TryAdd(o.Coin, txOutResponse); + }), + Task.Run(async () => + { + if (o.Proof is not null && o.Proof.PartialMerkleTree.Hashes.Contains(o.Coin.Outpoint.Hash)) + { + // var merkleBLockProofBytes = Encoders.Hex.DecodeData(o.TxOutProof); + // var mb = new MerkleBlock(); + // mb.FromBytes(merkleBLockProofBytes); + // mb.ReadWrite(merkleBLockProofBytes, network.NBitcoinNetwork); + + var txoutproofResult = + await clientBatch.SendCommandAsync("verifytxoutproof", o.Proof); + + var txHash = o.Coin.Outpoint.Hash.ToString(); + if (txoutproofResult.Error is not null && txoutproofResult.Result is JArray prooftxs && + prooftxs.Any(token => + token.Value() + ?.Equals(txHash, StringComparison.InvariantCultureIgnoreCase) is true)) + { + coinToBlock.TryAdd(o.Coin, o.Proof.Header); + } + } + }) + }; + }).Concat(new[] {clientBatch.SendBatchAsync()}).ToArray()); + + DateTimeOffset now = DateTimeOffset.UtcNow; + await repo.SaveMatches(coinToTxOut.Select(pair => + { + coinToBlock.TryGetValue(pair.Key, out var blockHeader); + var ttx = repo.CreateTrackedTransaction(trackedSource, + new TrackedTransactionKey(pair.Key.Outpoint.Hash, blockHeader?.GetHash(), true){}, + new[] {pair.Key}, null); + ttx.Inserted = now; + ttx.FirstSeen = blockHeader?.BlockTime?? NBitcoin.Utils.UnixTimeToDateTime(0);; + return ttx; + }).ToArray()); + + return Ok(); } private IMoney Format(NBXplorerNetwork network, MoneyBag bag) @@ -109,15 +225,17 @@ private static MoneyBag RemoveZeros(MoneyBag bag) [HttpGet] [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/utxos")] [Route("cryptos/{cryptoCode}/addresses/{address}/utxos")] + [Route("cryptos/{cryptoCode}/wallets/{walletId}/utxos")] [PostgresImplementationActionConstraint(true)] public async Task GetUTXOs( string cryptoCode, [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] DerivationStrategyBase derivationScheme, [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address) + BitcoinAddress address, + string walletId) { - var trackedSource = GetTrackedSource(derivationScheme, address); + var trackedSource = GetTrackedSource(derivationScheme, address, walletId); if (trackedSource == null) throw new ArgumentNullException(nameof(trackedSource)); var network = GetNetwork(cryptoCode, false); @@ -202,7 +320,7 @@ public async Task GetUTXOs( public Task GetUTXOs(string cryptoCode, DerivationStrategyBase derivationStrategy) { - return this.GetUTXOs(cryptoCode, derivationStrategy, null); + return this.GetUTXOs(cryptoCode, derivationStrategy, null, null); } } } diff --git a/NBXplorer/TrackedTransaction.cs b/NBXplorer/TrackedTransaction.cs index 9a4589c1d..a079747ba 100644 --- a/NBXplorer/TrackedTransaction.cs +++ b/NBXplorer/TrackedTransaction.cs @@ -151,7 +151,7 @@ public IEnumerable GetReceivedOutputs() Output: o, KeyPath: KnownKeyPathMapping.TryGet(o.TxOut.ScriptPubKey), Address: KnownKeyPathInformation.TryGet(o.TxOut.ScriptPubKey)?.Address)) - .Where(o => o.KeyPath != null || o.Output.TxOut.ScriptPubKey == (TrackedSource as IDestination)?.ScriptPubKey) + .Where(o => o.KeyPath != null || o.Output.TxOut.ScriptPubKey == (TrackedSource as IDestination)?.ScriptPubKey || TrackedSource is WalletTrackedSource) .Select(o => new MatchedOutput() { Index = o.Index, diff --git a/docker-compose.regtest.yml b/docker-compose.regtest.yml index fa248696c..d8ac279e3 100644 --- a/docker-compose.regtest.yml +++ b/docker-compose.regtest.yml @@ -33,7 +33,7 @@ services: bitcoind: restart: always container_name: btcpayserver_bitcoind - image: btcpayserver/bitcoin:0.18.0-1 + image: btcpayserver/bitcoin:0.21.1 environment: BITCOIN_NETWORK: ${NBITCOIN_NETWORK:-regtest} BITCOIN_EXTRA_ARGS: | @@ -63,7 +63,7 @@ services: elementsd-liquid: restart: always container_name: btcpayserver_elementsd_liquid - image: btcpayserver/elements:0.21.0.1 + image: btcpayserver/elements:0.21.0.2-4 environment: ELEMENTS_CHAIN: liquidv1test ELEMENTS_EXTRA_ARGS: | diff --git a/docs/API.md b/docs/API.md index 644d9ff10..f5c41bb65 100644 --- a/docs/API.md +++ b/docs/API.md @@ -8,19 +8,19 @@ NBXplorer does not index the whole blockchain, rather, it listens transactions a * [Configuration](#configuration) * [Authentication](#authentication) +* [Tracked Sources](#tracked-source) * [Derivation Scheme Format](#derivationScheme) * [Tracking a Derivation Scheme](#track) * [Track a specific address](#address) -* [Query transactions associated to a Derivation Scheme](#transactions) -* [Query transactions associated to a specific address](#address-transactions) -* [Query a single transaction associated to a address or derivation scheme](#singletransaction) +* [Track a wallet](#wallet) +* [Query transactions associated to a tracked source](#transactions) +* [Query a single transaction associated to a tracked source](#singletransaction) * [Get current balance](#balance) * [Get a transaction](#gettransaction) * [Get connection status to the chain](#status) * [Get a new unused address](#unused) * [Get scriptPubKey information of a Derivation Scheme](#scriptPubKey) * [Get available Unspent Transaction Outputs (UTXOs)](#utxos) -* [Get available Unspent Transaction Outputs of a specific address](#address-utxos) * [Notifications via websocket](#websocket) * [Broadcast a transaction](#broadcast) * [Rescan a transaction](#rescan) @@ -90,9 +90,27 @@ This can be disabled with `--noauth`. Also, NBXPlorer listen by default on `127.0.0.1`, if you want to access it from another machine, run `--bind "0.0.0.0"`. +## Tracked Sources + +A tracked source is a generic way to track a set of scripts (addresses) and its UTXOs, transactions, and balances. + +Currently, a tracked source can be: +* A derivation scheme, often referred to as an xpub. This method allows automatic, deterministic generation of scripts to track. The format for a derivation scheme tracked source identifier is `DERIVATIIONSCHEME:xpub1`, where xpub1 follows the [derivation scheme format](#derivationScheme). +* A specific address. This method allows tracking of a single bitcoin address and its script. The format for an address tracked source identifier is `ADDRESS:address1`, where address1 is a valid bitcoin address. +* A wallet. Postgres only. This method allows tracking of a set of scripts, but does not automatically generate new scripts, so they must be [added manually](#associate-scripts). The format for a wallet tracked source identifier is `WALLET:wallet1`, where wallet1 is an arbitrary string. + +Postgres only features: +While each of these has its own logic around what scripts to follow, they all allow adding additional, seemingly unrelated scripts to track, through the use of the [associate scripts API](#associate-scripts). + +## Child Wallets + +When using Postgres, the feature of child wallets allows you to link a tracked source as a child of another tracked source. +Every script generated by a child wallet will be tracked as part of its parent wallet, including all related UTXOs and transactions. +A parent can have an unlimited number of child wallets, and child wallets themselves can act as parents to other wallets. + ## Derivation Scheme Format -A derivation scheme, also called derivationStrategy in the code, is a flexible way to define how to generate address of a wallet. +A derivation scheme, also called derivationStrategy in the code, is a flexible way to define how to generate deterministic addresses for a wallet. NBXplorer will track any addresses on the `0/x`, `1/x` and `x` path. Here a documentation of the different derivation scheme supported: @@ -134,15 +152,17 @@ Optionally, you can attach a json body: "maxAddresses": null } ], - "wait": true + "wait": true, + "parentWallet": "DERIVATIONSCHEME:xpub" } ``` * `wait`: Optional. If `true` the call will return when all addresses has been generated, addresses will be generated in the background (default: `false`) * `derivationOptions`: Optional. Options to manually start the address generation process. (default: empty) -* `derivationOptions.feature`: Optional. Define to which feature this option should be used. (defaut: null, which match all feature) +* `derivationOptions.feature`: Optional. Define to which feature this option should be used. (default: null, which match all feature) * `derivationOptions.minAddresses`: Optional. The minimum addresses that need to be generated with this call. (default: null, make sure the number of address in the pool is between MinGap and MaxGap) * `derivationOptions.maxAddresses`: Optional. The maximum addresses that need to be generated with this call. (default: null, make sure the number of address in the pool is between MinGap and MaxGap) +* `parentWallet`: Optional. Postgres only. If specified, the derivation scheme will be tracked as a [child wallet](#child-wallet) of the specified [tracked source](#tracked-source). (default: null) ## Track a specific address @@ -152,15 +172,43 @@ After this call, the specified address will be tracked by NBXplorer Returns nothing. -## Query transactions associated to a derivationScheme +Optionally, you can attach a json body: + +```json +{ + "parentWallet": "DERIVATIONSCHEME:xpub" +} +``` -To query all transactions of a `derivation scheme`: +* `parentWallet`: Optional. Postgres only. If specified, the address will be tracked as a [child wallet](#child-wallet) of the specified [tracked source](#tracked-source). (default: null) -`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/transactions` -To query a specific transaction: +## Track a wallet -`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/transactions/{txId}` +When using Postgres, you can define wallets using an arbitrary wallet id, that tracks nothing by default. You can then add [tracked sources](#tracked-source) as child wallets of this wallet, which will make it inherit all their scripts and utxos, or you can [associate scripts](#associate-scripts) to it manually. + +`HTTP POST v1/cryptos/{cryptoCode}/wallets/{walletId}` + +Returns nothing. + +Optionally, you can attach a json body: + +```json +{ + "parentWallet": "DERIVATIONSCHEME:xpub" +} +``` + + +* `parentWallet`: Optional. Postgres only. If specified, the wallet will be tracked as a [child wallet](#child-wallet) of the specified [tracked source](#tracked-source). (default: null) + +## Query transactions associated to a tracked source + +To query all transactions of a `tracked source`: + +`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/transactions` +`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/transactions` +`HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/transactions` Optional Parameters: @@ -262,73 +310,12 @@ Returns: Note for liquid, `balanceChange` is an array of [AssetMoney](#liquid). Note that the list of confirmed transaction also include immature transactions. -## Query transactions associated to a specific address - -Query all transactions of a tracked address. (Only work if you called the Track operation on this specific address) - -`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/transactions` - -Optional Parameters: - -* `includeTransaction` includes the hex of the transaction, not only information (default: true) -Returns: - -```json -{ - "height": 104, - "confirmedTransactions": { - "transactions": [ - { - "blockHash": "3e7bcca309f92ab78a47c1cdd1166de9190fa49e97165c93e2b10ae1a14b99eb", - "confirmations": 1, - "height": 104, - "transactionId": "cc33dfaf2ed794b11af83dc6e29303e2d8ff9e5e29303153dad1a1d3d8b43e40", - "transaction": "020000000166d6befa387fd646f77a10e4b0f0e66b3569f18a83f77104a0c440e4156f80890000000048473044022064b1398653171440d3e79924cb6593633e7b2c3d80b60a2e21d6c6e287ee785a02203899009df443d0a0a1b06cb970aee0158d35166fd3e26d4e3e85570738e706d101feffffff028c02102401000000160014ee0a1889783da2e1f9bba47be4184b6610efd00400e1f5050000000016001452f88af314ef3b6d03d40a5fd1f2c906188a477567000000", - "outputs": [ - { - "scriptPubKey": "001452f88af314ef3b6d03d40a5fd1f2c906188a4775", - "index": 1, - "value": 100000000 - } - ], - "inputs": [], - "timestamp": 1540381888, - "balanceChange": 100000000 - } - ] - }, - "unconfirmedTransactions": { - "transactions": [ - { - "blockHash": null, - "confirmations": 0, - "height": null, - "transactionId": "7ec0bcbd3b7685b6bbdb4287a250b64bfcb799dbbbcffa78c00e6cc11185e5f1", - "transaction": null, - "outputs": [ - { - "scriptPubKey": "0014b39fc4eb5c6dd238d39449b70a2e30d575426d99", - "index": 1, - "value": 100000000 - } - ], - "inputs": [], - "timestamp": 1540381889, - "balanceChange": 100000000 - } - ] - }, - "replacedTransactions": { - "transactions": [] - } -} -``` - -## Query a single transaction associated to a address or derivation scheme +## Query a single transaction associated to a tracked source `HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/transactions/{txId}` `HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/transactions/{txId}` +`HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/transactions/{txId}` Error codes: @@ -363,6 +350,8 @@ Returns: ## Get current balance `HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/balance` +`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/balance` +`HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/balance` Returns: @@ -507,6 +496,8 @@ Returns: ## Get available Unspent Transaction Outputs (UTXOs) `HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/utxos` +`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/utxos` +`HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/utxos` Error: @@ -520,7 +511,7 @@ Result: "derivationStrategy": "tpubD6NzVbkrYhZ4XQVi1sSEDBWTcicDqVSCTnYDxpwGwcSZVbPii2b7baRg57YfL64ed36sBRe6GviihHwhy3D1cnBe5uXb27DjrDZCKUA7PQi", "currentHeight": 107, "unconfirmed": { - "utxOs": [ + "utxos": [ { "feature": "Deposit", "outpoint": "10ba4bcadd03130b1bd98b0bc7aea9910f871b25b87ec06e484456e84440c88a01000000", @@ -538,8 +529,8 @@ Result: "hasChanges": true }, "confirmed": { - "utxOs": [ - { + "utxos": [ + {// when it is a derivation scheme "feature": "Deposit", "outpoint": "29ca6590f3f03a6523ad79975392e74e385bf2b7dafe6c537ffa12f9e124348800000000", "index": 0, @@ -550,6 +541,8 @@ Result: "keyPath": "0/3", "timestamp": 1540376174, "confirmations": 1 + }, + {// when it is an address } ], "spentOutpoints": [ @@ -616,27 +609,11 @@ Result: "value": 100000000, "timestamp": 1540390664, "confirmations": 2 - }, - { - "outpoint": "a470a71144d4cdaef2b9bd8d24f20ebc8d6548bae523869f8cceb2cef5b4538a01000000", - "index": 1, - "transactionHash": "8a53b4f5ceb2ce8c9f8623e5ba48658dbc0ef2248dbdb9f2aecdd44411a770a4", - "scriptPubKey": "76a9145461f6c342451142e07d95dd2a42b48af9114cea88ac", - "value": 100000000, - "timestamp": 1540390666, - "confirmations": 1 - }, - { - "outpoint": "1710a1b61cb1f988182347be52a16502bae5a78fa9740a68107f9ddc6e30896a00000000", - "index": 0, - "transactionHash": "6a89306edc9d7f10680a74a98fa7e5ba0265a152be47231888f9b11cb6a11017", - "scriptPubKey": "76a9145461f6c342451142e07d95dd2a42b48af9114cea88ac", - "value": 60000000, - "timestamp": 1540390666, - "confirmations": 1 } ], - "spentOutpoints": [], + "spentOutpoints": [ + "9345f9585d643a31202e686ec7a4c2fe17917a5e7731a79d2327d24d25c0339f01000000" + ], "hasChanges": true }, "hasChanges": true @@ -644,6 +621,7 @@ Result: ``` This call does not returns conflicted unconfirmed UTXOs. +Note that confirmed utxo, do not include immature UTXOs. (ie. UTXOs belonging to a coinbase transaction with less than 100 confirmations) ## Notifications via websocket @@ -973,7 +951,7 @@ The smallest `eventId` is 1. "value": 100000000 } ], - "cryptoCode": "BTC", + "cryptoCode": "BTC" } } ] @@ -1067,7 +1045,7 @@ Fields: * `destinations[].destination`: Required, the destination address * `destinations[].amount` Send this amount to the destination (Mutually exclusive with: sweepAll) * `destinations[].substractFees` Default to false, will substract the fees of this transaction to this destination (Mutually exclusive with: sweepAll) -* `destinations[].sweepAll` Deault to false, will sweep all the balance of your wallet to this destination (Mutually exclusive with: amount, substractFees) +* `destinations[].sweepAll` Default to false, will sweep all the balance of your wallet to this destination (Mutually exclusive with: amount, substractFees) * `feePreference`: Optional, determines how fees for the transaction are calculated, default to the full node estimation for 1 block target. * `feePreference.explicitFeeRate`: An explicit fee rate for the transaction in Satoshi per vBytes (Mutually exclusive with: blockTarget, explicitFee, fallbackFeeRate) * `feePreference.explicitFee`: An explicit fee for the transaction in Satoshi (Mutually exclusive with: blockTarget, explicitFeeRate, fallbackFeeRate) @@ -1227,7 +1205,8 @@ Request: "passphrase": "hello", "importKeysToRPC": true, "savePrivateKeys": true, - "additionalOptions": { "slip77": "6c2de18eabeff3f7822bc724ad482bef0557f3e1c1e1c75b7a393a5ced4de616"} + "additionalOptions": { "slip77": "6c2de18eabeff3f7822bc724ad482bef0557f3e1c1e1c75b7a393a5ced4de616"}, + "parentWallet": "WALLET:xyz" } ``` @@ -1240,6 +1219,7 @@ Request: * `importKeysToRPC`: Optional, if true, every times a call to [get a new unused address](#unused) is called, the private key will be imported into the underlying node via RPC's `importprivkey`. (Default: `false`) * `savePrivateKeys`: If true, private keys will be saved inside the following metadata `Mnemonic`, `MasterHDKey` and `AccountHDKey`. * `additionalOptions`: Optional, additional options that a derivation scheme of some networks may support, such as [Liquid](#liquid) +* `parentWallet`: Optional. Postgres only. If specified, the derivation scheme will be tracked as a [child wallet](#child-wallet) of the specified [tracked source](#tracked-source). (default: null) The `importKeysToRPC` is only useful if one need to manage his wallet via the node's cli tooling. From c8dc3e4bf9a9737c3ab947ef76f7f8872f490266 Mon Sep 17 00:00:00 2001 From: Kukks Date: Thu, 16 Nov 2023 13:15:17 +0100 Subject: [PATCH 02/16] introduce tracked source context --- NBXplorer.Tests/UnitTest1.cs | 2 + NBXplorer/Controllers/CommonRoutes.cs | 12 + NBXplorer/Controllers/ControllerBase.cs | 12 +- NBXplorer/Controllers/MainController.PSBT.cs | 47 +- NBXplorer/Controllers/MainController.cs | 402 ++++++++---------- .../Controllers/PostgresMainController.cs | 160 ++----- NBXplorer/Controllers/TrackedSourceContext.cs | 135 ++++++ NBXplorer/IUTXOService.cs | 4 +- NBXplorer/NBXplorer.csproj | 2 +- global.json | 7 + 10 files changed, 396 insertions(+), 387 deletions(-) create mode 100644 NBXplorer/Controllers/CommonRoutes.cs create mode 100644 NBXplorer/Controllers/TrackedSourceContext.cs create mode 100644 global.json diff --git a/NBXplorer.Tests/UnitTest1.cs b/NBXplorer.Tests/UnitTest1.cs index 9b917b15c..5e37c2b92 100644 --- a/NBXplorer.Tests/UnitTest1.cs +++ b/NBXplorer.Tests/UnitTest1.cs @@ -4548,7 +4548,9 @@ private async Task Eventually(Func tsk) [Theory] [InlineData(Backend.Postgres)] +#if SUPPORT_DBTRIE [InlineData(Backend.DBTrie)] +#endif public async Task CanAssociateIndependentScripts(Backend backend) { using var tester = ServerTester.Create(backend); diff --git a/NBXplorer/Controllers/CommonRoutes.cs b/NBXplorer/Controllers/CommonRoutes.cs new file mode 100644 index 000000000..c33773fd6 --- /dev/null +++ b/NBXplorer/Controllers/CommonRoutes.cs @@ -0,0 +1,12 @@ +namespace NBXplorer.Controllers; + +public static class CommonRoutes +{ + public const string BaseCryptoEndpoint = "cryptos/{cryptoCode}"; + public const string BaseDerivationEndpoint = $"{BaseCryptoEndpoint}/derivations"; + public const string DerivationEndpoint = $"{BaseCryptoEndpoint}/derivations/{{derivationScheme}}"; + public const string AddressEndpoint = $"{BaseCryptoEndpoint}/addresses/{{address}}"; + public const string WalletEndpoint = $"{BaseCryptoEndpoint}/wallets/{{walletId}}"; + public const string TrackedSourceEndpoint = $"{BaseCryptoEndpoint}/tracked-sources/{{trackedSource}}"; + public const string TransactionsPath = "transactions/{txId?}"; +} \ No newline at end of file diff --git a/NBXplorer/Controllers/ControllerBase.cs b/NBXplorer/Controllers/ControllerBase.cs index 4ba7e7ec8..f6c52348b 100644 --- a/NBXplorer/Controllers/ControllerBase.cs +++ b/NBXplorer/Controllers/ControllerBase.cs @@ -27,17 +27,7 @@ public ControllerBase( public IRepositoryProvider RepositoryProvider { get; } public IIndexers Indexers { get; } - internal static TrackedSource GetTrackedSource(DerivationStrategyBase derivationScheme, BitcoinAddress address, string walletId) - { - TrackedSource trackedSource = null; - if (address != null) - trackedSource = new AddressTrackedSource(address); - if (derivationScheme != null) - trackedSource = new DerivationSchemeTrackedSource(derivationScheme); - if (walletId != null) - trackedSource = new WalletTrackedSource(walletId); - return trackedSource; - } + internal NBXplorerNetwork GetNetwork(string cryptoCode, bool checkRPC) { if (cryptoCode == null) diff --git a/NBXplorer/Controllers/MainController.PSBT.cs b/NBXplorer/Controllers/MainController.PSBT.cs index 6704b2ef4..5c934d105 100644 --- a/NBXplorer/Controllers/MainController.PSBT.cs +++ b/NBXplorer/Controllers/MainController.PSBT.cs @@ -17,12 +17,10 @@ namespace NBXplorer.Controllers public partial class MainController { [HttpPost] - [Route("cryptos/{network}/derivations/{strategy}/psbt/create")] + [Route($"{CommonRoutes.DerivationEndpoint}/psbt/create")] + [TrackedSourceContext.TrackedSourceContextRequirement(allowedTrackedSourceTypes:typeof(DerivationSchemeTrackedSource))] public async Task CreatePSBT( - [ModelBinder(BinderType = typeof(NetworkModelBinder))] - NBXplorerNetwork network, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase strategy, + TrackedSourceContext trackedSourceContext, [FromBody] JObject body, [FromServices] @@ -30,17 +28,15 @@ public async Task CreatePSBT( { if (body == null) throw new ArgumentNullException(nameof(body)); - CreatePSBTRequest request = ParseJObject(body, network); - if (strategy == null) - throw new ArgumentNullException(nameof(strategy)); - - var repo = RepositoryProvider.GetRepository(network); - var txBuilder = request.Seed is int s ? network.NBitcoinNetwork.CreateTransactionBuilder(s) - : network.NBitcoinNetwork.CreateTransactionBuilder(); + CreatePSBTRequest request = ParseJObject(body, trackedSourceContext.Network); + var repo = RepositoryProvider.GetRepository(trackedSourceContext.Network); + var txBuilder = request.Seed is int s ? trackedSourceContext.Network.NBitcoinNetwork.CreateTransactionBuilder(s) + : trackedSourceContext.Network.NBitcoinNetwork.CreateTransactionBuilder(); + var strategy = ((DerivationSchemeTrackedSource) trackedSourceContext.TrackedSource).DerivationStrategy; CreatePSBTSuggestions suggestions = null; if (!(request.DisableFingerprintRandomization is true) && - fingerprintService.GetDistribution(network) is FingerprintDistribution distribution) + fingerprintService.GetDistribution(trackedSourceContext.Network) is FingerprintDistribution distribution) { suggestions ??= new CreatePSBTSuggestions(); var known = new List<(Fingerprint feature, bool value)>(); @@ -97,8 +93,7 @@ public async Task CreatePSBT( suggestions.ShouldEnforceLowR = fingerprint.HasFlag(Fingerprint.LowR); } - var indexer = Indexers.GetIndexer(network); - if (indexer.NetworkInfo?.GetRelayFee() is FeeRate feeRate) + if (trackedSourceContext.Indexer.NetworkInfo?.GetRelayFee() is FeeRate feeRate) { txBuilder.StandardTransactionPolicy.MinRelayTxFee = feeRate; } @@ -135,7 +130,7 @@ public async Task CreatePSBT( // nLockTime that preclude a fix later. else if (!(request.DiscourageFeeSniping is false)) { - if (indexer.State is BitcoinDWaiterState.Ready) + if (trackedSourceContext.Indexer.State is BitcoinDWaiterState.Ready) { int blockHeight = (await repo.GetTip()).Height; // Secondly occasionally randomly pick a nLockTime even further back, so @@ -153,7 +148,7 @@ public async Task CreatePSBT( txBuilder.SetLockTime(new LockTime(0)); } } - var utxoChanges = (await utxoService.GetUTXOs(network.CryptoCode, strategy)).As(); + var utxoChanges = (await utxoService.GetUTXOs(trackedSourceContext)).As(); var utxos = utxoChanges.GetUnspentUTXOs(request.MinConfirmations); var availableCoinsByOutpoint = utxos.ToDictionary(o => o.Outpoint); if (request.IncludeOnlyOutpoints != null) @@ -194,10 +189,10 @@ public async Task CreatePSBT( // We remove unconf utxos with too many ancestors, as it will result in a transaction // that can't be broadcasted. // We do only for BTC, as this isn't a shitcoin issue. - if (network.CryptoCode == "BTC" && unconfUtxos.Count > 0 && request.MinConfirmations == 0) + if (trackedSourceContext.Network.CryptoCode == "BTC" && unconfUtxos.Count > 0 && request.MinConfirmations == 0) { HashSet requestedTxs = new HashSet(); - var rpc = RPCClients.Get(network); + var rpc = trackedSourceContext.RpcClient; rpc = rpc.PrepareBatch(); var mempoolEntries = unconfUtxos @@ -270,7 +265,7 @@ public async Task CreatePSBT( bool hasChange = false; if (request.ExplicitChangeAddress == null) { - var keyInfo = (await GetUnusedAddress(network.CryptoCode, strategy, DerivationFeature.Change, autoTrack: true)).As(); + var keyInfo = (await GetUnusedAddress(trackedSourceContext, DerivationFeature.Change, autoTrack: true)).As(); change = (keyInfo.ScriptPubKey, keyInfo.KeyPath); } else @@ -296,7 +291,7 @@ public async Task CreatePSBT( { try { - var rate = await GetFeeRate(blockTarget, network.CryptoCode); + var rate = await GetFeeRate(blockTarget, trackedSourceContext.Network.CryptoCode); txBuilder.SendEstimatedFees(rate.FeeRate); } catch (NBXplorerException e) when (e.Error.Code == "fee-estimation-unavailable" && request.FeePreference?.FallbackFeeRate is FeeRate fallbackFeeRate) @@ -312,7 +307,7 @@ public async Task CreatePSBT( { try { - var rate = await GetFeeRate(1, network.CryptoCode); + var rate = await GetFeeRate(1, trackedSourceContext.Network.CryptoCode); txBuilder.SendEstimatedFees(rate.FeeRate); } catch (NBXplorerException e) when (e.Error.Code == "fee-estimation-unavailable" && request.FeePreference?.FallbackFeeRate is FeeRate fallbackFeeRate) @@ -351,7 +346,7 @@ public async Task CreatePSBT( // We made sure we can build the PSBT, so now we can reserve the change address if we need to if (hasChange && request.ExplicitChangeAddress == null && request.ReserveChangeAddress) { - var derivation = (await GetUnusedAddress(network.CryptoCode, strategy, DerivationFeature.Change, reserve: true, autoTrack: true)).As(); + var derivation = (await GetUnusedAddress(trackedSourceContext, DerivationFeature.Change, reserve: true, autoTrack: true)).As(); // In most of the time, this is the same as previously, so no need to rebuild PSBT if (derivation.ScriptPubKey != change.ScriptPubKey) { @@ -375,14 +370,14 @@ public async Task CreatePSBT( AlwaysIncludeNonWitnessUTXO = request.AlwaysIncludeNonWitnessUTXO, IncludeGlobalXPub = request.IncludeGlobalXPub }; - await UpdatePSBTCore(update, network); + await UpdatePSBTCore(update, trackedSourceContext.Network); var resp = new CreatePSBTResponse() { PSBT = update.PSBT, - ChangeAddress = hasChange ? change.ScriptPubKey.GetDestinationAddress(network.NBitcoinNetwork) : null, + ChangeAddress = hasChange ? change.ScriptPubKey.GetDestinationAddress(trackedSourceContext.Network.NBitcoinNetwork) : null, Suggestions = suggestions }; - return Json(resp, network.JsonSerializerSettings); + return Json(resp, trackedSourceContext.Network.JsonSerializerSettings); } [HttpPost] diff --git a/NBXplorer/Controllers/MainController.cs b/NBXplorer/Controllers/MainController.cs index f0ba44a5f..1ee4cb363 100644 --- a/NBXplorer/Controllers/MainController.cs +++ b/NBXplorer/Controllers/MainController.cs @@ -86,7 +86,7 @@ private Exception JsonRPCNotExposed() } [HttpPost] - [Route("cryptos/{cryptoCode}/rpc")] + [Route($"{CommonRoutes.BaseCryptoEndpoint}/rpc")] [Consumes("application/json", "application/json-rpc")] public async Task RPCProxy(string cryptoCode) { @@ -125,7 +125,7 @@ public async Task RPCProxy(string cryptoCode) } [HttpGet] - [Route("cryptos/{cryptoCode}/fees/{blockCount}")] + [Route($"{CommonRoutes.BaseCryptoEndpoint}/fees/{{blockCount}}")] public async Task GetFeeRate(int blockCount, string cryptoCode) { var network = GetNetwork(cryptoCode, true); @@ -144,30 +144,28 @@ public async Task GetFeeRate(int blockCount, string cryptoCode } [HttpGet] - [Route("cryptos/{cryptoCode}/derivations/{strategy}/addresses/unused")] + [Route($"{CommonRoutes.DerivationEndpoint}/addresses/unused")] + [TrackedSourceContext.TrackedSourceContextRequirement(allowedTrackedSourceTypes: typeof(DerivationSchemeTrackedSource))] public async Task GetUnusedAddress( - string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase strategy, DerivationFeature feature = DerivationFeature.Deposit, int skip = 0, bool reserve = false, bool autoTrack = false) + TrackedSourceContext trackedSourceContext, DerivationFeature feature = DerivationFeature.Deposit, int skip = 0, bool reserve = false, bool autoTrack = false) { - if (strategy == null) - throw new ArgumentNullException(nameof(strategy)); - var network = GetNetwork(cryptoCode, false); - var repository = RepositoryProvider.GetRepository(network); + var derivationScheme = ((DerivationSchemeTrackedSource ) trackedSourceContext.TrackedSource).DerivationStrategy; + var network = trackedSourceContext.Network; + var repository = trackedSourceContext.Repository; if (skip >= repository.MinPoolSize) throw new NBXplorerError(404, "strategy-not-found", $"This strategy is not tracked, or you tried to skip too much unused addresses").AsException(); try { - var result = await repository.GetUnused(strategy, feature, skip, reserve); + var result = await repository.GetUnused(derivationScheme, feature, skip, reserve); if (reserve || autoTrack) { while (result == null) { - await AddressPoolService.GenerateAddresses(network, strategy, feature, new GenerateAddressQuery(1, null)); - result = await repository.GetUnused(strategy, feature, skip, reserve); + await AddressPoolService.GenerateAddresses(network, derivationScheme, feature, new GenerateAddressQuery(1, null)); + result = await repository.GetUnused(derivationScheme, feature, skip, reserve); } if (reserve) - _ = AddressPoolService.GenerateAddresses(network, strategy, feature); + _ = AddressPoolService.GenerateAddresses(network, derivationScheme, feature); } return Json(result, network.Serializer.Settings); } @@ -178,19 +176,17 @@ public async Task GetUnusedAddress( } [HttpPost] - [Route("cryptos/{cryptoCode}/derivations/{strategy}/addresses/cancelreservation")] - public async Task CancelReservation(string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase strategy, [FromBody] KeyPath[] keyPaths) + [Route($"{CommonRoutes.DerivationEndpoint}/addresses/cancelreservation")] + [TrackedSourceContext.TrackedSourceContextRequirement( allowedTrackedSourceTypes:typeof(DerivationSchemeTrackedSource))] + public async Task CancelReservation(TrackedSourceContext trackedSourceContext, [FromBody] KeyPath[] keyPaths) { - var network = GetNetwork(cryptoCode, false); - var repo = RepositoryProvider.GetRepository(network); - await repo.CancelReservation(strategy, keyPaths); + var derivationScheme = ((DerivationSchemeTrackedSource ) trackedSourceContext.TrackedSource).DerivationStrategy; + await trackedSourceContext.Repository.CancelReservation(derivationScheme, keyPaths); return Ok(); } [HttpGet] - [Route("cryptos/{cryptoCode}/scripts/{script}")] + [Route($"{CommonRoutes.BaseCryptoEndpoint}/scripts/{{script}}")] public async Task GetKeyInformations(string cryptoCode, [ModelBinder(BinderType = typeof(ScriptModelBinder))] Script script) { @@ -203,25 +199,21 @@ public async Task GetKeyInformations(string cryptoCode, } [HttpGet] - [Route("cryptos/{cryptoCode}/derivations/{strategy}/scripts/{script}")] - public async Task GetKeyInformations(string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase strategy, - [ModelBinder(BinderType = typeof(ScriptModelBinder))] Script script) + [Route($"{CommonRoutes.DerivationEndpoint}/scripts/{{script}}")] + [TrackedSourceContext.TrackedSourceContextRequirement(allowedTrackedSourceTypes:typeof(DerivationSchemeTrackedSource))] + public async Task GetKeyInformations(TrackedSourceContext trackedSourceContext, [ModelBinder(BinderType = typeof(ScriptModelBinder))] Script script) { - var network = GetNetwork(cryptoCode, false); - var repo = RepositoryProvider.GetRepository(network); - var result = (await repo.GetKeyInformations(new[] { script })) - .SelectMany(k => k.Value) - .Where(k => k.DerivationStrategy == strategy) - .FirstOrDefault(); + var derivationScheme = ((DerivationSchemeTrackedSource ) trackedSourceContext.TrackedSource).DerivationStrategy; + var result = (await trackedSourceContext.Repository.GetKeyInformations(new[] { script })) + .SelectMany(k => k.Value) + .FirstOrDefault(k => k.DerivationStrategy == derivationScheme); if (result == null) throw new NBXplorerError(404, "script-not-found", "The script does not seem to be tracked").AsException(); - return Json(result, network.Serializer.Settings); + return Json(result, trackedSourceContext.Network.Serializer.Settings); } [HttpGet] - [Route("cryptos/{cryptoCode}/status")] + [Route($"{CommonRoutes.BaseCryptoEndpoint}/status")] public async Task GetStatus(string cryptoCode) { var network = GetNetwork(cryptoCode, false); @@ -299,7 +291,7 @@ public async Task GetStatus(string cryptoCode) } [HttpGet] - [Route("cryptos/{cryptoCode}/connect")] + [Route($"{CommonRoutes.BaseCryptoEndpoint}/connect")] public async Task ConnectWebSocket( string cryptoCode, bool includeTransaction = true, @@ -419,7 +411,7 @@ private JsonSerializerSettings GetSerializerSettings(string cryptoCode) return this.GetNetwork(cryptoCode, false).JsonSerializerSettings; } - [Route("cryptos/{cryptoCode}/events")] + [Route($"{CommonRoutes.BaseCryptoEndpoint}/events")] public async Task GetEvents(string cryptoCode, int lastEventId = 0, int? limit = null, bool longPolling = false, CancellationToken cancellationToken = default) { if (limit != null && limit.Value < 1) @@ -455,7 +447,7 @@ public async Task GetEvents(string cryptoCode, int lastEventId = 0, int? } - [Route("cryptos/{cryptoCode}/events/latest")] + [Route($"{CommonRoutes.BaseCryptoEndpoint}/events/latest")] public async Task GetLatestEvents(string cryptoCode, int limit = 10) { if (limit < 1) @@ -468,7 +460,7 @@ public async Task GetLatestEvents(string cryptoCode, int limit = 10) [HttpGet] - [Route("cryptos/{cryptoCode}/transactions/{txId}")] + [Route($"{CommonRoutes.BaseCryptoEndpoint}/transactions/{{txId}}")] public async Task GetTransaction( [ModelBinder(BinderType = typeof(UInt256ModelBinding))] uint256 txId, @@ -510,57 +502,51 @@ private bool HasTxIndex(string cryptoCode) } [HttpPost] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}")] - [Route("cryptos/{cryptoCode}/addresses/{address}")] - [Route("cryptos/{cryptoCode}/wallets/{walletId}")] + + [Route($"{CommonRoutes.DerivationEndpoint}")] + [Route($"{CommonRoutes.AddressEndpoint}")] + [Route($"{CommonRoutes.WalletEndpoint}")] + [Route($"{CommonRoutes.TrackedSourceEndpoint}")] public async Task TrackWallet( - string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, - [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address, - string walletId, + TrackedSourceContext trackedSourceContext, [FromBody] JObject rawRequest = null) { - var request = ParseJObject(rawRequest ?? new JObject(), GetNetwork(cryptoCode, false)); - TrackedSource trackedSource = GetTrackedSource(derivationScheme, address, walletId); - if (trackedSource == null) - return NotFound(); - var network = GetNetwork(cryptoCode, false); - var repo = RepositoryProvider.GetRepository(network); + var request = ParseJObject(rawRequest ?? new JObject(), trackedSourceContext.Network); + + var repo = RepositoryProvider.GetRepository(trackedSourceContext.Network); if (repo is PostgresRepository postgresRepository && - (trackedSource is WalletTrackedSource || + (trackedSourceContext.TrackedSource is WalletTrackedSource || request?.ParentWallet is not null)) { - await postgresRepository.EnsureWalletCreated(trackedSource, request?.ParentWallet is null? null: new []{request?.ParentWallet }); + await postgresRepository.EnsureWalletCreated(trackedSourceContext.TrackedSource, request?.ParentWallet is null? null: new []{request?.ParentWallet }); } if (repo is not PostgresRepository && request.ParentWallet is not null) throw new NBXplorerException(new NBXplorerError(400, "parent-wallet-not-supported", "Parent wallet is only supported with Postgres")); - if (trackedSource is DerivationSchemeTrackedSource dts) + if (trackedSourceContext.TrackedSource is DerivationSchemeTrackedSource dts) { if (request.Wait) { foreach (var feature in keyPathTemplates.GetSupportedDerivationFeatures()) { - await RepositoryProvider.GetRepository(network).GenerateAddresses(dts.DerivationStrategy, feature, GenerateAddressQuery(request, feature)); + await RepositoryProvider.GetRepository(trackedSourceContext.Network).GenerateAddresses(dts.DerivationStrategy, feature, GenerateAddressQuery(request, feature)); } } else { foreach (var feature in keyPathTemplates.GetSupportedDerivationFeatures()) { - await RepositoryProvider.GetRepository(network).GenerateAddresses(dts.DerivationStrategy, feature, new GenerateAddressQuery(minAddresses: 3, null)); + await repo.GenerateAddresses(dts.DerivationStrategy, feature, new GenerateAddressQuery(minAddresses: 3, null)); } foreach (var feature in keyPathTemplates.GetSupportedDerivationFeatures()) { - _ = AddressPoolService.GenerateAddresses(network, dts.DerivationStrategy, feature, GenerateAddressQuery(request, feature)); + _ = AddressPoolService.GenerateAddresses(trackedSourceContext.Network, dts.DerivationStrategy, feature, GenerateAddressQuery(request, feature)); } } } - else if (trackedSource is IDestination ats) + else if (trackedSourceContext.TrackedSource is IDestination ats) { - await RepositoryProvider.GetRepository(network).Track(ats); + await repo.Track(ats); } return Ok(); } @@ -580,33 +566,27 @@ private GenerateAddressQuery GenerateAddressQuery(TrackWalletRequest request, De } [HttpGet] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/transactions/{txId?}")] - [Route("cryptos/{cryptoCode}/addresses/{address}/transactions/{txId?}")] - [Route("cryptos/{cryptoCode}/wallets/{walletId}/transactions/{txId?}")] + + + [Route($"{CommonRoutes.DerivationEndpoint}/{CommonRoutes.TransactionsPath}")] + [Route($"{CommonRoutes.AddressEndpoint}/{CommonRoutes.TransactionsPath}")] + [Route($"{CommonRoutes.WalletEndpoint}/{CommonRoutes.TransactionsPath}")] + [Route($"{CommonRoutes.TrackedSourceEndpoint}/{CommonRoutes.TransactionsPath}")] public async Task GetTransactions( - string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, - [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address, - string walletId, + TrackedSourceContext trackedSourceContext, [ModelBinder(BinderType = typeof(UInt256ModelBinding))] uint256 txId = null, bool includeTransaction = true) { - var trackedSource = GetTrackedSource(derivationScheme, address, walletId); - if (trackedSource == null) - throw new ArgumentNullException(nameof(trackedSource)); TransactionInformation fetchedTransactionInfo = null; - var network = GetNetwork(cryptoCode, false); - var repo = RepositoryProvider.GetRepository(network); + var repo = RepositoryProvider.GetRepository(trackedSourceContext.Network); var response = new GetTransactionsResponse(); int currentHeight = (await repo.GetTip()).Height; response.Height = currentHeight; - var txs = await GetAnnotatedTransactions(repo, trackedSource, includeTransaction, txId); + var txs = await GetAnnotatedTransactions(repo, trackedSourceContext.TrackedSource, includeTransaction, txId); foreach (var item in new[] { new @@ -654,7 +634,7 @@ public async Task GetTransactions( if (txId != null && txId == txInfo.TransactionId) fetchedTransactionInfo = txInfo; - if (network.NBitcoinNetwork.NetworkSet == NBitcoin.Altcoins.Liquid.Instance) + if (trackedSourceContext.Network.NBitcoinNetwork.NetworkSet == NBitcoin.Altcoins.Liquid.Instance) { txInfo.BalanceChange = new MoneyBag(txInfo.Outputs.Select(o => o.Value).OfType().ToArray()) - new MoneyBag(txInfo.Inputs.Select(o => o.Value).OfType().ToArray()); @@ -684,26 +664,28 @@ public async Task GetTransactions( } [HttpPost] - [Route("cryptos/{cryptoCode}/rescan")] - public async Task Rescan(string cryptoCode, [FromBody] JObject body) + [Route($"{CommonRoutes.BaseCryptoEndpoint}/rescan")] + [TrackedSourceContext.TrackedSourceContextRequirement(false, false, true)] + public async Task Rescan(TrackedSourceContext trackedSourceContext, [FromBody] JObject body) { if (body == null) throw new ArgumentNullException(nameof(body)); - var rescanRequest = ParseJObject(body, GetNetwork(cryptoCode, false)); + var rescanRequest = ParseJObject(body, trackedSourceContext.Network); if (rescanRequest == null) throw new ArgumentNullException(nameof(rescanRequest)); if (rescanRequest?.Transactions == null) throw new NBXplorerException(new NBXplorerError(400, "transactions-missing", "You must specify 'transactions'")); bool willFetchTransactions = rescanRequest.Transactions.Any(t => t.Transaction == null); - bool needTxIndex = rescanRequest.Transactions.Any(t => t.Transaction == null && t.BlockId == null); - var network = GetNetwork(cryptoCode, willFetchTransactions); + if (willFetchTransactions && trackedSourceContext.RpcClient is null) + { + TrackedSourceContext.TrackedSourceContextModelBinder.ThrowRpcUnavailableException(); + } - var rpc = RPCClients.Get(network).PrepareBatch(); - var repo = RepositoryProvider.GetRepository(network); + var rpc = trackedSourceContext.RpcClient!.PrepareBatch(); var fetchingTransactions = rescanRequest .Transactions - .Select(t => FetchTransaction(rpc, HasTxIndex(network), t)) + .Select(t => FetchTransaction(rpc, HasTxIndex(trackedSourceContext.Network), t)) .ToArray(); await rpc.SendBatchAsync(); @@ -723,17 +705,17 @@ public async Task Rescan(string cryptoCode, [FromBody] JObject bo } } await batch.SendBatchAsync(); - await repo.SaveBlocks(blocks.Select(b => b.Value.Result).ToList()); + await trackedSourceContext.Repository.SaveBlocks(blocks.Select(b => b.Value.Result).ToList()); foreach (var txs in transactions.GroupBy(t => t.BlockId, t => (t.Transaction, t.BlockTime)) .OrderBy(t => t.First().BlockTime)) { blocks.TryGetValue(txs.Key, out var slimBlock); - await repo.SaveTransactions(txs.First().BlockTime, txs.Select(t => t.Transaction).ToArray(), slimBlock.Result); + await trackedSourceContext.Repository.SaveTransactions(txs.First().BlockTime, txs.Select(t => t.Transaction).ToArray(), slimBlock.Result); foreach (var tx in txs) { - var matches = await repo.GetMatches(tx.Transaction, slimBlock.Result, tx.BlockTime, false); - await repo.SaveMatches(matches); - _ = AddressPoolService.GenerateAddresses(network, matches); + var matches = await trackedSourceContext.Repository.GetMatches(tx.Transaction, slimBlock.Result, tx.BlockTime, false); + await trackedSourceContext.Repository.SaveMatches(matches); + _ = AddressPoolService.GenerateAddresses(trackedSourceContext.Network, matches); } } return Ok(); @@ -787,59 +769,37 @@ public async Task Rescan(string cryptoCode, [FromBody] JObject bo } [HttpPost] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/metadata/{key}")] - public async Task SetMetadata(string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, string key, - [FromBody] - JToken value = null) + [Route($"{CommonRoutes.DerivationEndpoint}/metadata/{{key}}")] + public async Task SetMetadata(TrackedSourceContext trackedSourceContext, string key, [FromBody] JToken value = null) { - var network = this.GetNetwork(cryptoCode, true); - var trackedSource = new DerivationSchemeTrackedSource(derivationScheme); - var repo = this.RepositoryProvider.GetRepository(network); - await repo.SaveMetadata(trackedSource, key, value); + await trackedSourceContext.Repository.SaveMetadata(trackedSourceContext.TrackedSource, key, value); return Ok(); } [HttpGet] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/metadata/{key}")] - public async Task GetMetadata(string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, string key) + [Route($"{CommonRoutes.DerivationEndpoint}/metadata/{{key}}")] + public async Task GetMetadata(TrackedSourceContext trackedSourceContext, string key) { - var network = this.GetNetwork(cryptoCode, false); - var trackedSource = new DerivationSchemeTrackedSource(derivationScheme); - var repo = this.RepositoryProvider.GetRepository(network); - var result = await repo.GetMetadata(trackedSource, key); - return result == null ? (IActionResult)NotFound() : Json(result, repo.Serializer.Settings); + var result = await trackedSourceContext.Repository.GetMetadata(trackedSourceContext.TrackedSource, key); + return result == null ? NotFound() : Json(result, trackedSourceContext.Repository.Serializer.Settings); } [HttpPost] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/utxos/wipe")] - public async Task Wipe( - string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme) + [Route($"{CommonRoutes.DerivationEndpoint}/utxos/wipe")] + public async Task Wipe(TrackedSourceContext trackedSourceContext) { - var network = this.GetNetwork(cryptoCode, true); - var repo = RepositoryProvider.GetRepository(network); - var ts = new DerivationSchemeTrackedSource(derivationScheme); - var txs = await repo.GetTransactions(ts); - await repo.Prune(ts, txs); + var txs = await trackedSourceContext.Repository.GetTransactions(trackedSourceContext.TrackedSource); + await trackedSourceContext.Repository.Prune(trackedSourceContext.TrackedSource, txs); return Ok(); } [HttpPost] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/utxos/scan")] - public IActionResult ScanUTXOSet( - string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, int? batchSize = null, int? gapLimit = null, int? from = null) + [Route($"{CommonRoutes.DerivationEndpoint}/utxos/scan")] + [TrackedSourceContext.TrackedSourceContextRequirement(requireRPC:true,allowedTrackedSourceTypes: new []{typeof(DerivationSchemeTrackedSource)})] + public IActionResult ScanUTXOSet(TrackedSourceContext trackedSourceContext, int? batchSize = null, int? gapLimit = null, int? from = null) { - var network = this.GetNetwork(cryptoCode, true); - var rpc = GetAvailableRPC(network); - if (!rpc.Capabilities.SupportScanUTXOSet) + if (!trackedSourceContext.RpcClient.Capabilities.SupportScanUTXOSet) throw new NBXplorerError(405, "scanutxoset-not-suported", "ScanUTXOSet is not supported for this currency").AsException(); ScanUTXOSetOptions options = new ScanUTXOSetOptions(); @@ -849,47 +809,40 @@ public IActionResult ScanUTXOSet( options.GapLimit = gapLimit.Value; if (from != null) options.From = from.Value; - if (!ScanUTXOSetService.EnqueueScan(network, derivationScheme, options)) + if (!ScanUTXOSetService.EnqueueScan(trackedSourceContext.Network, ((DerivationSchemeTrackedSource) trackedSourceContext.TrackedSource).DerivationStrategy, options)) throw new NBXplorerError(409, "scanutxoset-in-progress", "ScanUTXOSet has already been called for this derivationScheme").AsException(); return Ok(); } [HttpGet] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/utxos/scan")] - public IActionResult GetScanUTXOSetInfromation( - string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme) + [Route($"{CommonRoutes.DerivationEndpoint}/utxos/scan")] + [TrackedSourceContext.TrackedSourceContextRequirement(allowedTrackedSourceTypes: new []{typeof(DerivationSchemeTrackedSource)})] + public IActionResult GetScanUTXOSetInfromation(TrackedSourceContext trackedSourceContext) { - var network = this.GetNetwork(cryptoCode, false); - var info = ScanUTXOSetService.GetInformation(network, derivationScheme); + var info = ScanUTXOSetService.GetInformation(trackedSourceContext.Network, ((DerivationSchemeTrackedSource) trackedSourceContext.TrackedSource).DerivationStrategy); if (info == null) throw new NBXplorerError(404, "scanutxoset-info-not-found", "ScanUTXOSet has not been called with this derivationScheme of the result has expired").AsException(); - return Json(info, network.Serializer.Settings); + return Json(info, trackedSourceContext.Network.Serializer.Settings); } #if SUPPORT_DBTRIE [HttpGet] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/balance")] - [Route("cryptos/{cryptoCode}/addresses/{address}/balance")] + [Route($"{CommonRoutes.DerivationEndpoint}/balance")] + [Route($"{CommonRoutes.AddressEndpoint}/balance")] + [TrackedSourceContext.TrackedSourceContextRequirement(allowedTrackedSourceTypes: new []{typeof(DerivationSchemeTrackedSource),typeof(AddressTrackedSource)})] [PostgresImplementationActionConstraint(false)] - public async Task GetBalance(string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, - [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address) + public async Task GetBalance(TrackedSourceContext trackedSourceContext) { - var getTransactionsResult = await GetTransactions(cryptoCode, derivationScheme, address, null, includeTransaction: false); + var getTransactionsResult = await GetTransactions(trackedSourceContext, includeTransaction: false); var jsonResult = getTransactionsResult as JsonResult; var transactions = jsonResult?.Value as GetTransactionsResponse; if (transactions == null) return getTransactionsResult; - var network = this.GetNetwork(cryptoCode, false); var balance = new GetBalanceResponse() { - Confirmed = CalculateBalance(network, transactions.ConfirmedTransactions), - Unconfirmed = CalculateBalance(network, transactions.UnconfirmedTransactions), - Immature = CalculateBalance(network, transactions.ImmatureTransactions) + Confirmed = CalculateBalance(trackedSourceContext.Network, transactions.ConfirmedTransactions), + Unconfirmed = CalculateBalance(trackedSourceContext.Network, transactions.UnconfirmedTransactions), + Immature = CalculateBalance(trackedSourceContext.Network, transactions.ImmatureTransactions) }; balance.Total = balance.Confirmed.Add(balance.Unconfirmed); balance.Available = balance.Total.Sub(balance.Immature); @@ -909,31 +862,19 @@ private IMoney CalculateBalance(NBXplorerNetwork network, TransactionInformation } [HttpGet] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/utxos")] - [Route("cryptos/{cryptoCode}/addresses/{address}/utxos")] - [Route("cryptos/{cryptoCode}/wallets/{walletId}/utxos")] - + [Route($"{CommonRoutes.DerivationEndpoint}/utxos")] + [Route($"{CommonRoutes.AddressEndpoint}/utxos")] [PostgresImplementationActionConstraint(false)] - public async Task GetUTXOs( - string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, - [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address, - string walletId) + [TrackedSourceContext.TrackedSourceContextRequirement(allowedTrackedSourceTypes: new []{typeof(DerivationSchemeTrackedSource),typeof(AddressTrackedSource)})] + public async Task GetUTXOs(TrackedSourceContext trackedSourceContext) { - var trackedSource = GetTrackedSource(derivationScheme, address, walletId); UTXOChanges changes = null; - if (trackedSource == null) - throw new ArgumentNullException(nameof(trackedSource)); - - var network = GetNetwork(cryptoCode, false); - var repo = RepositoryProvider.GetRepository(network); + var repo = RepositoryProvider.GetRepository(trackedSourceContext.Network); changes = new UTXOChanges(); changes.CurrentHeight = (await repo.GetTip()).Height; - var transactions = await GetAnnotatedTransactions(repo, trackedSource, false); + var transactions = await GetAnnotatedTransactions(repo, trackedSourceContext.TrackedSource, false); changes.Confirmed = ToUTXOChange(transactions.ConfirmedState); changes.Confirmed.SpentOutpoints.Clear(); @@ -942,8 +883,8 @@ public async Task GetUTXOs( FillUTXOsInformation(changes.Confirmed.UTXOs, transactions, changes.CurrentHeight); FillUTXOsInformation(changes.Unconfirmed.UTXOs, transactions, changes.CurrentHeight); - changes.TrackedSource = trackedSource; - changes.DerivationStrategy = (trackedSource as DerivationSchemeTrackedSource)?.DerivationStrategy; + changes.TrackedSource = trackedSourceContext.TrackedSource; + changes.DerivationStrategy = (trackedSourceContext.TrackedSource as DerivationSchemeTrackedSource)?.DerivationStrategy; return Json(changes, repo.Serializer.Settings); } @@ -994,37 +935,28 @@ private async Task GetAnnotatedTransactions(IRep } [HttpPost] - [Route("cryptos/{cryptoCode}/transactions")] + [Route($"{CommonRoutes.BaseCryptoEndpoint}/transactions")] + [TrackedSourceContext.TrackedSourceContextRequirement(true, false)] public async Task Broadcast( - string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase extPubKey, // For back compat - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, - [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address, - string walletId, + TrackedSourceContext trackedSourceContext, bool testMempoolAccept = false) { - var network = GetNetwork(cryptoCode, true); - var trackedSource = GetTrackedSource(derivationScheme ?? extPubKey, address, walletId); - var tx = network.NBitcoinNetwork.Consensus.ConsensusFactory.CreateTransaction(); + + var tx = trackedSourceContext.Network.NBitcoinNetwork.Consensus.ConsensusFactory.CreateTransaction(); var buffer = new MemoryStream(); await Request.Body.CopyToAsync(buffer); buffer.Position = 0; tx.FromBytes(buffer.ToArrayEfficient()); - var rpc = GetAvailableRPC(network); - if (testMempoolAccept && !rpc.Capabilities.SupportTestMempoolAccept) + if (testMempoolAccept && !trackedSourceContext.RpcClient.Capabilities.SupportTestMempoolAccept) throw new NBXplorerException(new NBXplorerError(400, "not-supported", "This feature is not supported for this crypto currency")); - var repo = RepositoryProvider.GetRepository(network); - var indexer = Indexers.GetIndexer(network); + var repo = RepositoryProvider.GetRepository(trackedSourceContext.Network); RPCException rpcEx = null; try { if (testMempoolAccept) { - var mempoolAccept = await rpc.TestMempoolAcceptAsync(tx, default); + var mempoolAccept = await trackedSourceContext.RpcClient.TestMempoolAcceptAsync(tx, default); if (mempoolAccept.IsAllowed) return new BroadcastResult(true); var rpcCode = GetRPCCodeFromReason(mempoolAccept.RejectReason); @@ -1035,18 +967,18 @@ public async Task Broadcast( RPCCodeMessage = mempoolAccept.RejectReason, }; } - await rpc.SendRawTransactionAsync(tx); - await indexer.SaveMatches(tx); + await trackedSourceContext.RpcClient.SendRawTransactionAsync(tx); + await trackedSourceContext.Indexer.SaveMatches(tx); return new BroadcastResult(true); } catch (RPCException ex) when (!testMempoolAccept) { rpcEx = ex; - Logs.Explorer.LogInformation($"{network.CryptoCode}: Transaction {tx.GetHash()} failed to broadcast (Code: {ex.RPCCode}, Message: {ex.RPCCodeMessage}, Details: {ex.Message} )"); - if (trackedSource != null && ex.Message.StartsWith("Missing inputs", StringComparison.OrdinalIgnoreCase)) + Logs.Explorer.LogInformation($"{trackedSourceContext.Network.CryptoCode}: Transaction {tx.GetHash()} failed to broadcast (Code: {ex.RPCCode}, Message: {ex.RPCCodeMessage}, Details: {ex.Message} )"); + if (trackedSourceContext.TrackedSource != null && ex.Message.StartsWith("Missing inputs", StringComparison.OrdinalIgnoreCase)) { - Logs.Explorer.LogInformation($"{network.CryptoCode}: Trying to broadcast unconfirmed of the wallet"); - var transactions = await GetAnnotatedTransactions(repo, trackedSource, true); + Logs.Explorer.LogInformation($"{trackedSourceContext.Network.CryptoCode}: Trying to broadcast unconfirmed of the wallet"); + var transactions = await GetAnnotatedTransactions(repo, trackedSourceContext.TrackedSource, true); foreach (var existing in transactions.UnconfirmedTransactions) { var t = existing.Record.Transaction ?? (await repo.GetSavedTransactions(existing.Record.TransactionHash)).Select(c => c.Transaction).FirstOrDefault(); @@ -1054,21 +986,21 @@ public async Task Broadcast( continue; try { - await rpc.SendRawTransactionAsync(t); + await trackedSourceContext.RpcClient.SendRawTransactionAsync(t); } catch { } } try { - await rpc.SendRawTransactionAsync(tx); - Logs.Explorer.LogInformation($"{network.CryptoCode}: Broadcast success"); - await indexer.SaveMatches(tx); + await trackedSourceContext.RpcClient.SendRawTransactionAsync(tx); + Logs.Explorer.LogInformation($"{trackedSourceContext.Network.CryptoCode}: Broadcast success"); + await trackedSourceContext.Indexer.SaveMatches(tx); return new BroadcastResult(true); } catch (RPCException) { - Logs.Explorer.LogInformation($"{network.CryptoCode}: Transaction {tx.GetHash()} failed to broadcast (Code: {ex.RPCCode}, Message: {ex.RPCCodeMessage}, Details: {ex.Message} )"); + Logs.Explorer.LogInformation($"{trackedSourceContext.Network.CryptoCode}: Transaction {tx.GetHash()} failed to broadcast (Code: {ex.RPCCode}, Message: {ex.RPCCodeMessage}, Details: {ex.Message} )"); } } return new BroadcastResult(false) @@ -1093,15 +1025,18 @@ public async Task Broadcast( } [HttpPost] - [Route("cryptos/{cryptoCode}/derivations")] - public async Task GenerateWallet(string cryptoCode, [FromBody] JObject rawRequest = null) + + [Route($"{CommonRoutes.BaseDerivationEndpoint}")] + [TrackedSourceContext.TrackedSourceContextRequirement(false, false)] + public async Task GenerateWallet(TrackedSourceContext trackedSourceContext, [FromBody] JObject rawRequest = null) { - var request = ParseJObject(rawRequest, GetNetwork(cryptoCode, false)); - - if (request == null) - request = new GenerateWalletRequest(); - var network = GetNetwork(cryptoCode, request.ImportKeysToRPC); - if (network.CoinType == null) + var request = ParseJObject(rawRequest, trackedSourceContext.Network) ?? new GenerateWalletRequest(); + + if (request.ImportKeysToRPC && trackedSourceContext.RpcClient is null) + { + TrackedSourceContext.TrackedSourceContextModelBinder.ThrowRpcUnavailableException(); + } + if (trackedSourceContext.Network.CoinType == null) // Don't document, only shitcoins nobody use goes into this throw new NBXplorerException(new NBXplorerError(400, "not-supported", "This feature is not supported for this coin because we don't have CoinType information")); request.WordList ??= Wordlist.English; @@ -1109,12 +1044,12 @@ public async Task GenerateWallet(string cryptoCode, [FromBody] JO request.ScriptPubKeyType ??= ScriptPubKeyType.Segwit; if (request.ScriptPubKeyType is null) { - request.ScriptPubKeyType = network.NBitcoinNetwork.Consensus.SupportSegwit ? ScriptPubKeyType.Segwit : ScriptPubKeyType.Legacy; + request.ScriptPubKeyType = trackedSourceContext.Network.NBitcoinNetwork.Consensus.SupportSegwit ? ScriptPubKeyType.Segwit : ScriptPubKeyType.Legacy; } - if (!network.NBitcoinNetwork.Consensus.SupportSegwit && request.ScriptPubKeyType != ScriptPubKeyType.Legacy) + if (!trackedSourceContext.Network.NBitcoinNetwork.Consensus.SupportSegwit && request.ScriptPubKeyType != ScriptPubKeyType.Legacy) throw new NBXplorerException(new NBXplorerError(400, "segwit-not-supported", "Segwit is not supported, please explicitely set scriptPubKeyType to Legacy")); - var repo = RepositoryProvider.GetRepository(network); + var repo = RepositoryProvider.GetRepository(trackedSourceContext.Network); if (repo is not PostgresRepository && request.ParentWallet is not null) throw new NBXplorerException(new NBXplorerError(400, "parent-wallet-not-supported", "Parent wallet is only supported with Postgres")); @@ -1134,10 +1069,10 @@ public async Task GenerateWallet(string cryptoCode, [FromBody] JO { mnemonic = new Mnemonic(request.WordList, request.WordCount.Value); } - var masterKey = mnemonic.DeriveExtKey(request.Passphrase).GetWif(network.NBitcoinNetwork); - var keyPath = GetDerivationKeyPath(request.ScriptPubKeyType.Value, request.AccountNumber, network); + var masterKey = mnemonic.DeriveExtKey(request.Passphrase).GetWif(trackedSourceContext.Network.NBitcoinNetwork); + var keyPath = GetDerivationKeyPath(request.ScriptPubKeyType.Value, request.AccountNumber, trackedSourceContext.Network); var accountKey = masterKey.Derive(keyPath); - DerivationStrategyBase derivation = network.DerivationStrategyFactory.CreateDirectDerivationStrategy(accountKey.Neuter(), new DerivationStrategyOptions() + DerivationStrategyBase derivation = trackedSourceContext.Network.DerivationStrategyFactory.CreateDirectDerivationStrategy(accountKey.Neuter(), new DerivationStrategyOptions() { ScriptPubKeyType = request.ScriptPubKeyType.Value, AdditionalOptions = request.AdditionalOptions is not null ? new System.Collections.ObjectModel.ReadOnlyDictionary(request.AdditionalOptions) : null @@ -1164,13 +1099,20 @@ public async Task GenerateWallet(string cryptoCode, [FromBody] JO } var accountKeyPath = new RootedKeyPath(masterKey.GetPublicKey().GetHDFingerPrint(), keyPath); saveMetadata.Add(repo.SaveMetadata(derivationTrackedSource, WellknownMetadataKeys.AccountKeyPath, accountKeyPath)); - var importAddressToRPC = await GetImportAddressToRPC(request, network); + var importAddressToRPC = await GetImportAddressToRPC(request, trackedSourceContext.Network); saveMetadata.Add(repo.SaveMetadata(derivationTrackedSource, WellknownMetadataKeys.ImportAddressToRPC, (importAddressToRPC?.ToString() ?? "False"))); var descriptor = GetDescriptor(accountKeyPath, accountKey.Neuter(), request.ScriptPubKeyType.Value); saveMetadata.Add(repo.SaveMetadata(derivationTrackedSource, WellknownMetadataKeys.AccountDescriptor, descriptor)); await Task.WhenAll(saveMetadata.ToArray()); - await TrackWallet(cryptoCode, derivation,null, null); + + await TrackWallet(new TrackedSourceContext() + { + Indexer = trackedSourceContext.Indexer, + Network = trackedSourceContext.Network, + RpcClient = trackedSourceContext.RpcClient, + TrackedSource = new DerivationSchemeTrackedSource(derivation) + }); return Json(new GenerateWalletResponse() { MasterHDKey = masterKey, @@ -1182,7 +1124,7 @@ public async Task GenerateWallet(string cryptoCode, [FromBody] JO Passphrase = request.Passphrase ?? string.Empty, WordCount = request.WordCount.Value, WordList = request.WordList - }, network.Serializer.Settings); + }, trackedSourceContext.Network.Serializer.Settings); } private async Task GetImportAddressToRPC(GenerateWalletRequest request, NBXplorerNetwork network) @@ -1258,23 +1200,18 @@ private KeyPath GetDerivationKeyPath(ScriptPubKeyType scriptPubKeyType, int acco } [HttpPost] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/prune")] - public async Task Prune( - string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, [FromBody] PruneRequest request) + [Route($"{CommonRoutes.DerivationEndpoint}/prune")] + [TrackedSourceContext.TrackedSourceContextRequirement(allowedTrackedSourceTypes: new []{typeof(DerivationSchemeTrackedSource)})] + public async Task Prune(TrackedSourceContext trackedSourceContext ,[FromBody] PruneRequest request) { request ??= new PruneRequest(); request.DaysToKeep ??= 1.0; - var trackedSource = new DerivationSchemeTrackedSource(derivationScheme); - var network = GetNetwork(cryptoCode, false); - var repo = RepositoryProvider.GetRepository(network); - var transactions = await GetAnnotatedTransactions(repo, trackedSource, false); + var transactions = await GetAnnotatedTransactions(trackedSourceContext.Repository, trackedSourceContext.TrackedSource, false); var state = transactions.ConfirmedState; var prunableIds = new HashSet(); - var keepConfMax = network.NBitcoinNetwork.Consensus.GetExpectedBlocksFor(TimeSpan.FromDays(request.DaysToKeep.Value)); - var tip = (await repo.GetTip()).Height; + var keepConfMax = trackedSourceContext.Network.NBitcoinNetwork.Consensus.GetExpectedBlocksFor(TimeSpan.FromDays(request.DaysToKeep.Value)); + var tip = (await trackedSourceContext.Repository.GetTip()).Height; // Step 1. We can prune if all UTXOs are spent foreach (var tx in transactions.ConfirmedTransactions) { @@ -1314,22 +1251,17 @@ public async Task Prune( if (prunableIds.Count != 0) { - await repo.Prune(trackedSource, prunableIds + await trackedSourceContext.Repository.Prune(trackedSourceContext.TrackedSource, prunableIds .Select(id => transactions.GetByTxId(id).Record) .ToList()); - Logs.Explorer.LogInformation($"{network.CryptoCode}: Pruned {prunableIds.Count} transactions"); + Logs.Explorer.LogInformation($"{trackedSourceContext.Network.CryptoCode}: Pruned {prunableIds.Count} transactions"); } return new PruneResponse() { TotalPruned = prunableIds.Count }; } -#if SUPPORT_DBTRIE - public Task GetUTXOs(string cryptoCode, DerivationStrategyBase derivationStrategy) - { - return this.GetUTXOs(cryptoCode, derivationStrategy, null, null); - } -#else - public Task GetUTXOs(string cryptoCode, DerivationStrategyBase derivationStrategy) +#if !SUPPORT_DBTRIE + public async Task GetUTXOs(TrackedSourceContext trackedSourceContext) { - throw new NotSupportedException("This should never be called"); + throw new NotImplementedException(); } #endif } diff --git a/NBXplorer/Controllers/PostgresMainController.cs b/NBXplorer/Controllers/PostgresMainController.cs index 568d6d803..d8fa4f4a5 100644 --- a/NBXplorer/Controllers/PostgresMainController.cs +++ b/NBXplorer/Controllers/PostgresMainController.cs @@ -1,36 +1,32 @@ -using Dapper; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Dapper; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using NBitcoin; -using NBXplorer.Backends; +using NBitcoin.RPC; using NBXplorer.Backends.Postgres; using NBXplorer.DerivationStrategy; -using NBXplorer.ModelBinders; using NBXplorer.Models; -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using NBitcoin.DataEncoders; -using NBitcoin.RPC; using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace NBXplorer.Controllers { - [Route("v1")] + [PostgresImplementationActionConstraint(true)] + [Route($"v1/{CommonRoutes.DerivationEndpoint}")] + [Route($"v1/{CommonRoutes.AddressEndpoint}")] + [Route($"v1/{CommonRoutes.WalletEndpoint}")] + [Route($"v1/{CommonRoutes.TrackedSourceEndpoint}")] [Authorize] - public class PostgresMainController : ControllerBase, IUTXOService + public class PostgresMainController :Controller, IUTXOService { public PostgresMainController( DbConnectionFactory connectionFactory, - NBXplorerNetworkProvider networkProvider, - IRPCClients rpcClients, - IIndexers indexers, - KeyPathTemplates keyPathTemplates, - IRepositoryProvider repositoryProvider) : base(networkProvider, rpcClients, repositoryProvider, indexers) + KeyPathTemplates keyPathTemplates) { ConnectionFactory = connectionFactory; KeyPathTemplates = keyPathTemplates; @@ -39,25 +35,12 @@ public PostgresMainController( public DbConnectionFactory ConnectionFactory { get; } public KeyPathTemplates KeyPathTemplates { get; } - [HttpGet] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/balance")] - [Route("cryptos/{cryptoCode}/addresses/{address}/balance")] - [Route("cryptos/{cryptoCode}/wallets/{walletId}/balance")] - [PostgresImplementationActionConstraint(true)] - public async Task GetBalance(string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, - [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address, - string walletId) + [HttpGet("balance")] + public async Task GetBalance( TrackedSourceContext trackedSourceContext) { - var trackedSource = GetTrackedSource(derivationScheme, address, walletId); - if (trackedSource == null) - throw new ArgumentNullException(nameof(trackedSource)); - var network = GetNetwork(cryptoCode, false); - var repo = (PostgresRepository)RepositoryProvider.GetRepository(cryptoCode); + var repo = (PostgresRepository)trackedSourceContext.Repository; await using var conn = await ConnectionFactory.CreateConnection(); - var b = await conn.QueryAsync("SELECT * FROM wallets_balances WHERE code=@code AND wallet_id=@walletId", new { code = network.CryptoCode, walletId = repo.GetWalletKey(trackedSource).wid }); + var b = await conn.QueryAsync("SELECT * FROM wallets_balances WHERE code=@code AND wallet_id=@walletId", new { code = trackedSourceContext.Network.CryptoCode, walletId = repo.GetWalletKey(trackedSourceContext.TrackedSource).wid }); MoneyBag available = new MoneyBag(), confirmed = new MoneyBag(), @@ -87,68 +70,39 @@ public async Task GetBalance(string cryptoCode, var balance = new GetBalanceResponse() { - Confirmed = Format(network, confirmed), - Unconfirmed = Format(network, unconfirmed), - Available = Format(network, available), - Total = Format(network, total), - Immature = Format(network, immature) + Confirmed = Format(trackedSourceContext.Network, confirmed), + Unconfirmed = Format(trackedSourceContext.Network, unconfirmed), + Available = Format(trackedSourceContext.Network, available), + Total = Format(trackedSourceContext.Network, total), + Immature = Format(trackedSourceContext.Network, immature) }; balance.Total = balance.Confirmed.Add(balance.Unconfirmed); - return Json(balance, network.JsonSerializerSettings); + return Json(balance, trackedSourceContext.Network.JsonSerializerSettings); } - [HttpPost] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/associate")] - [Route("cryptos/{cryptoCode}/addresses/{address}/associate")] - [Route("cryptos/{cryptoCode}/wallets/{walletId}/associate")] + [HttpPost("associate")] [PostgresImplementationActionConstraint(true)] - public async Task AssociateScripts(string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, - [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address, - string walletId, + public async Task AssociateScripts( TrackedSourceContext trackedSourceContext, [FromBody] Dictionary scripts) { - var trackedSource = GetTrackedSource(derivationScheme, address, walletId); - if (trackedSource == null) - throw new ArgumentNullException(nameof(trackedSource)); - var network = GetNetwork(cryptoCode, false); - var repo = (PostgresRepository)RepositoryProvider.GetRepository(cryptoCode); - - await repo.AssociateScriptsToWalletExplicitly(trackedSource, - scripts.ToDictionary(pair => (IDestination) BitcoinAddress.Create(pair.Key, network.NBitcoinNetwork), + var repo = (PostgresRepository)trackedSourceContext.Repository; + await repo.AssociateScriptsToWalletExplicitly(trackedSourceContext.TrackedSource, + scripts.ToDictionary(pair => (IDestination) BitcoinAddress.Create(pair.Key, trackedSourceContext.Network.NBitcoinNetwork), pair => pair.Value)); return Ok(); } - - - [HttpPost] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/import-utxos")] - [Route("cryptos/{cryptoCode}/addresses/{address}/import-utxos")] - [Route("cryptos/{cryptoCode}/wallets/{walletId}/import-utxos")] - [PostgresImplementationActionConstraint(true)] - public async Task ImportUTXOs(string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, - [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address, - string walletId, - [FromBody] JArray rawRequest) + [HttpPost("import-utxos")] + [TrackedSourceContext.TrackedSourceContextRequirement(true)] + public async Task ImportUTXOs( TrackedSourceContext trackedSourceContext, [FromBody] JArray rawRequest) { - var network = GetNetwork(cryptoCode, true); - var jsonSerializer = JsonSerializer.Create(network.JsonSerializerSettings); + var jsonSerializer = JsonSerializer.Create(trackedSourceContext.Network.JsonSerializerSettings); var coins = rawRequest.ToObject(jsonSerializer)?.Where(c => c.Coin != null).ToArray(); if (coins?.Any() is not true) throw new ArgumentNullException(nameof(coins)); - var trackedSource = GetTrackedSource(derivationScheme, address, walletId); - if (trackedSource == null) - throw new ArgumentNullException(nameof(trackedSource)); - var repo = (PostgresRepository) RepositoryProvider.GetRepository(cryptoCode); - - var rpc = RPCClients.Get(network); + var repo = (PostgresRepository)trackedSourceContext.Repository; + var rpc = trackedSourceContext.RpcClient; var clientBatch = rpc.PrepareBatch(); var coinToTxOut = new ConcurrentDictionary(); @@ -193,7 +147,7 @@ await Task.WhenAll(coins.SelectMany(o => await repo.SaveMatches(coinToTxOut.Select(pair => { coinToBlock.TryGetValue(pair.Key, out var blockHeader); - var ttx = repo.CreateTrackedTransaction(trackedSource, + var ttx = repo.CreateTrackedTransaction(trackedSourceContext.TrackedSource, new TrackedTransactionKey(pair.Key.Outpoint.Hash, blockHeader?.GetHash(), true){}, new[] {pair.Key}, null); ttx.Inserted = now; @@ -222,32 +176,17 @@ private static MoneyBag RemoveZeros(MoneyBag bag) return new MoneyBag(bag.Where(a => !a.Negate().Equals(a)).ToArray()); } - [HttpGet] - [Route("cryptos/{cryptoCode}/derivations/{derivationScheme}/utxos")] - [Route("cryptos/{cryptoCode}/addresses/{address}/utxos")] - [Route("cryptos/{cryptoCode}/wallets/{walletId}/utxos")] - [PostgresImplementationActionConstraint(true)] - public async Task GetUTXOs( - string cryptoCode, - [ModelBinder(BinderType = typeof(DerivationStrategyModelBinder))] - DerivationStrategyBase derivationScheme, - [ModelBinder(BinderType = typeof(BitcoinAddressModelBinder))] - BitcoinAddress address, - string walletId) + [HttpGet("utxos")] + public async Task GetUTXOs( TrackedSourceContext trackedSourceContext) { - var trackedSource = GetTrackedSource(derivationScheme, address, walletId); - if (trackedSource == null) - throw new ArgumentNullException(nameof(trackedSource)); - var network = GetNetwork(cryptoCode, false); - var repo = (PostgresRepository)RepositoryProvider.GetRepository(cryptoCode); - + var repo = (PostgresRepository)trackedSourceContext.Repository; await using var conn = await ConnectionFactory.CreateConnection(); - var height = await conn.ExecuteScalarAsync("SELECT height FROM get_tip(@code)", new { code = network.CryptoCode }); - - + var height = await conn.ExecuteScalarAsync("SELECT height FROM get_tip(@code)", new { code = trackedSourceContext.Network.CryptoCode }); // On elements, we can't get blinded address from the scriptPubKey, so we need to fetch it rather than compute it string addrColumns = "NULL as address"; - if (network.IsElement && !derivationScheme.Unblinded()) + var derivationScheme = (trackedSourceContext.TrackedSource as DerivationSchemeTrackedSource) + ?.DerivationStrategy; + if (trackedSourceContext.Network.IsElement && derivationScheme?.Unblinded() is true) { addrColumns = "ds.metadata->>'blindedAddress' as address"; } @@ -274,11 +213,11 @@ public async Task GetUTXOs( bool input_mempool, DateTime tx_seen_at)>( $"SELECT blk_height, tx_id, wu.idx, value, script, {addrColumns}, {descriptorColumns}, mempool, input_mempool, seen_at " + - $"FROM wallets_utxos wu{descriptorJoin} WHERE code=@code AND wallet_id=@walletId AND immature IS FALSE", new { code = network.CryptoCode, walletId = repo.GetWalletKey(trackedSource).wid })); + $"FROM wallets_utxos wu{descriptorJoin} WHERE code=@code AND wallet_id=@walletId AND immature IS FALSE", new { code =trackedSourceContext.Network.CryptoCode, walletId = repo.GetWalletKey(trackedSourceContext.TrackedSource).wid })); UTXOChanges changes = new UTXOChanges() { CurrentHeight = (int)height, - TrackedSource = trackedSource, + TrackedSource = trackedSourceContext.TrackedSource, DerivationStrategy = derivationScheme }; foreach (var utxo in utxos.OrderBy(u => u.tx_seen_at)) @@ -303,7 +242,7 @@ public async Task GetUTXOs( u.KeyPath = KeyPath.Parse(utxo.keypath); u.Feature = Enum.Parse(utxo.feature); } - u.Address = utxo.address is null ? u.ScriptPubKey.GetDestinationAddress(network.NBitcoinNetwork) : BitcoinAddress.Create(utxo.address, network.NBitcoinNetwork); + u.Address = utxo.address is null ? u.ScriptPubKey.GetDestinationAddress(trackedSourceContext.Network.NBitcoinNetwork) : BitcoinAddress.Create(utxo.address, trackedSourceContext.Network.NBitcoinNetwork); if (!utxo.mempool) { changes.Confirmed.UTXOs.Add(u); @@ -315,12 +254,7 @@ public async Task GetUTXOs( else // (utxo.mempool && utxo.input_mempool) changes.SpentUnconfirmed.Add(u); } - return Json(changes, network.JsonSerializerSettings); - } - - public Task GetUTXOs(string cryptoCode, DerivationStrategyBase derivationStrategy) - { - return this.GetUTXOs(cryptoCode, derivationStrategy, null, null); + return Json(changes, trackedSourceContext.Network.JsonSerializerSettings); } } } diff --git a/NBXplorer/Controllers/TrackedSourceContext.cs b/NBXplorer/Controllers/TrackedSourceContext.cs new file mode 100644 index 000000000..502dc99ee --- /dev/null +++ b/NBXplorer/Controllers/TrackedSourceContext.cs @@ -0,0 +1,135 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Controllers; +using Microsoft.AspNetCore.Mvc.ModelBinding; +using Microsoft.Extensions.DependencyInjection; +using NBitcoin; +using NBitcoin.RPC; +using NBXplorer.Backends; +using NBXplorer.Models; + +namespace NBXplorer.Controllers; + +[ModelBinder] +public class TrackedSourceContext +{ + public TrackedSource TrackedSource { get; set; } + public NBXplorerNetwork Network { get; set; } + public RPCClient RpcClient { get; set; } + public IIndexer Indexer { get; set; } + public IRepository Repository { get; set; } + + public class TrackedSourceContextRequirementAttribute: Attribute + { + public bool RequireRpc { get; } + public bool RequireTrackedSource { get; } + public bool DisallowTrackedSource { get; } + public Type[] AllowedTrackedSourceTypes { get; } + + public TrackedSourceContextRequirementAttribute(bool requireRPC = false, bool requireTrackedSource = true, bool disallowTrackedSource = false, params Type[] allowedTrackedSourceTypes) + { + RequireRpc = requireRPC; + RequireTrackedSource = requireTrackedSource; + DisallowTrackedSource = disallowTrackedSource; + AllowedTrackedSourceTypes = allowedTrackedSourceTypes; + + } + } + + public class TrackedSourceContextModelBinder : IModelBinder + { + public Task BindModelAsync(ModelBindingContext bindingContext) + { + var cryptoCode = bindingContext.ValueProvider.GetValue("cryptoCode").FirstValue?.ToUpperInvariant(); + + if (cryptoCode == null) + throw new ArgumentNullException(nameof(cryptoCode)); + + var addressValue = bindingContext.ValueProvider.GetValue("address").FirstValue; + var derivationSchemeValue = bindingContext.ValueProvider.GetValue("derivationScheme").FirstValue; + derivationSchemeValue??= bindingContext.ValueProvider.GetValue("extPubKey").FirstValue; + var walletIdValue = bindingContext.ValueProvider.GetValue("walletId").FirstValue; + var trackedSourceValue = bindingContext.ValueProvider.GetValue("trackedSource").FirstValue; + + var networkProvider = bindingContext.HttpContext.RequestServices.GetService(); + var indexers = bindingContext.HttpContext.RequestServices.GetService(); + var repositoryProvider = bindingContext.HttpContext.RequestServices.GetService(); + + var network = networkProvider.GetFromCryptoCode(cryptoCode); + + var indexer = network is null ? null : indexers.GetIndexer(network); + if (network is null || indexer is null) + { + throw new NBXplorerException(new NBXplorerError(404, "cryptoCode-not-supported", + $"{cryptoCode} is not supported")); + } + + var requirements = ((ControllerActionDescriptor) bindingContext.ActionContext.ActionDescriptor) + .MethodInfo.GetCustomAttributes().FirstOrDefault(); + + + var rpcClient = indexer.GetConnectedClient(); + if (rpcClient?.Capabilities == null) + { + rpcClient = null; + } + + if (requirements?.RequireRpc is true && rpcClient is null) + { + ThrowRpcUnavailableException(); + } + + var ts = GetTrackedSource(derivationSchemeValue, addressValue, walletIdValue, + trackedSourceValue, + network); + if (ts is null && requirements?.RequireTrackedSource is true) + { + + throw new NBXplorerException(new NBXplorerError(400, "tracked-source-required", + $"A tracked source is required for this endpoint.")); + } + if ( ts is not null && requirements?.DisallowTrackedSource is true) + { + throw new NBXplorerException(new NBXplorerError(400, "tracked-source-unwanted", + $"This endpoint does not tracked sources..")); + } + if(ts is not null && requirements?.AllowedTrackedSourceTypes?.Any() is true && !requirements.AllowedTrackedSourceTypes.Any(t => t.IsInstanceOfType(ts))) + { + throw new NBXplorerException(new NBXplorerError(400, "tracked-source-invalid", + $"The tracked source provided is not valid for this endpoint.")); + } + + bindingContext.Result = ModelBindingResult.Success(new TrackedSourceContext() + { + Indexer = indexer, + Network = network, + TrackedSource = ts , + RpcClient = rpcClient, + Repository = repositoryProvider.GetRepository(network) + }); + return Task.CompletedTask; + } + public static void ThrowRpcUnavailableException() + { + throw new NBXplorerError(400, "rpc-unavailable", $"The RPC interface is currently not available.").AsException(); + } + + public static TrackedSource GetTrackedSource(string derivationScheme, string address, string walletId, + string trackedSource, NBXplorerNetwork network) + { + if (trackedSource != null) + return TrackedSource.Parse(trackedSource, network); + if (address != null) + return new AddressTrackedSource(BitcoinAddress.Create(address, network.NBitcoinNetwork)); + if (derivationScheme != null) + return new DerivationSchemeTrackedSource(network.DerivationStrategyFactory.Parse(derivationScheme)); + if (walletId != null) + return new WalletTrackedSource(walletId); + return null; + } + } + +} \ No newline at end of file diff --git a/NBXplorer/IUTXOService.cs b/NBXplorer/IUTXOService.cs index ba78cadfe..fdfd881a1 100644 --- a/NBXplorer/IUTXOService.cs +++ b/NBXplorer/IUTXOService.cs @@ -1,11 +1,13 @@ using Microsoft.AspNetCore.Mvc; using System.Threading.Tasks; +using NBXplorer.Controllers; +using NBXplorer.DerivationStrategy; namespace NBXplorer { // Big hack to make CreatePSBT of MainController pick PostgresController as implementation for getting utxos. public interface IUTXOService { - Task GetUTXOs(string cryptoCode, DerivationStrategy.DerivationStrategyBase derivationStrategy); + Task GetUTXOs(TrackedSourceContext trackedSourceContext); } } diff --git a/NBXplorer/NBXplorer.csproj b/NBXplorer/NBXplorer.csproj index 3c0a63d2e..75d8403d3 100644 --- a/NBXplorer/NBXplorer.csproj +++ b/NBXplorer/NBXplorer.csproj @@ -6,7 +6,7 @@ 2.4.0 bin\$(Configuration)\$(TargetFramework)\NBXplorer.xml 1701;1702;1705;1591;CS1591 - 10.0 + 11 true $(DefineConstants);SUPPORT_DBTRIE diff --git a/global.json b/global.json new file mode 100644 index 000000000..dad2db5ef --- /dev/null +++ b/global.json @@ -0,0 +1,7 @@ +{ + "sdk": { + "version": "8.0.0", + "rollForward": "latestMajor", + "allowPrerelease": true + } +} \ No newline at end of file From 369147ea973fdc974fb24180e6282a03d7a3c925 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 17 Nov 2023 16:03:45 +0900 Subject: [PATCH 03/16] Simplify code --- NBXplorer/Backends/Postgres/PostgresRepository.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/NBXplorer/Backends/Postgres/PostgresRepository.cs b/NBXplorer/Backends/Postgres/PostgresRepository.cs index dbb1b85fc..439b8045f 100644 --- a/NBXplorer/Backends/Postgres/PostgresRepository.cs +++ b/NBXplorer/Backends/Postgres/PostgresRepository.cs @@ -484,7 +484,7 @@ async Task> GetKeyInformations( return result; string additionalColumn = Network.IsElement ? ", ts.blinded_addr" : ""; var rows = await connection.QueryAsync($@" - SELECT DISTINCT ts.script, ts.addr, ts.derivation, ts.keypath, ts.redeem , + SELECT DISTINCT ts.script, ts.addr, ts.derivation, ts.keypath, ts.redeem{additionalColumn}, COALESCE(wd.wallet_id, ws.wallet_id) AS wallet_id, COALESCE(wd_wallet.metadata->>'type', ws_wallet.metadata->>'type') AS wallet_type FROM unnest(@records) AS r (script), @@ -509,11 +509,9 @@ FROM nbxv1_keypath_info ki if (r.derivation is not null && r.keypath is null) continue; var addr = GetAddress(r); - bool isExplicit = r.derivation is null; - bool isDescriptor = !isExplicit; var script = Script.FromHex(r.script); - var derivationStrategy = isDescriptor ? Network.DerivationStrategyFactory.Parse(r.derivation) : null; - var keypath = isDescriptor ? KeyPath.Parse(r.keypath) : null; + var derivationStrategy = r.derivation is not null ? Network.DerivationStrategyFactory.Parse(r.derivation) : null; + var keypath = r.keypath is not null ? KeyPath.Parse(r.keypath) : null; var redeem = (string)r.redeem; string walletType = r.wallet_type; string walletId = r.wallet_id; @@ -526,12 +524,12 @@ FROM nbxv1_keypath_info ki result.Add(script, new KeyPathInformation() { Address = addr, - DerivationStrategy = isDescriptor ? derivationStrategy : null, - KeyPath = isDescriptor ? keypath : null, + DerivationStrategy = r.derivation is not null ? derivationStrategy : null, + KeyPath = keypath, ScriptPubKey = script, TrackedSource = trackedSource, Feature = keypath is null ? DerivationFeature.Deposit : KeyPathTemplates.GetDerivationFeature(keypath), - Redeem = redeem is null ? null : Script.FromHex(redeem) + Redeem = redeem is null ? null : Script.FromHex(redeem), }); } return result; From e5bbc7f5e762ef3fae6f69a09a1897a992d0f8ae Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Fri, 17 Nov 2023 17:28:20 +0900 Subject: [PATCH 04/16] Simplify query --- .../Backends/Postgres/PostgresRepository.cs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/NBXplorer/Backends/Postgres/PostgresRepository.cs b/NBXplorer/Backends/Postgres/PostgresRepository.cs index 439b8045f..e909af0b3 100644 --- a/NBXplorer/Backends/Postgres/PostgresRepository.cs +++ b/NBXplorer/Backends/Postgres/PostgresRepository.cs @@ -484,23 +484,21 @@ async Task> GetKeyInformations( return result; string additionalColumn = Network.IsElement ? ", ts.blinded_addr" : ""; var rows = await connection.QueryAsync($@" - SELECT DISTINCT ts.script, ts.addr, ts.derivation, ts.keypath, ts.redeem{additionalColumn}, - COALESCE(wd.wallet_id, ws.wallet_id) AS wallet_id, - COALESCE(wd_wallet.metadata->>'type', ws_wallet.metadata->>'type') AS wallet_type + SELECT ts.code, ts.script, ts.addr, ts.derivation, ts.keypath, ts.redeem{additionalColumn}, + ws.wallet_id, + w.metadata->>'type' AS wallet_type FROM unnest(@records) AS r (script), LATERAL ( - SELECT script, addr, descriptor_metadata->>'derivation' derivation, + SELECT code, script, addr, descriptor_metadata->>'derivation' derivation, keypath, descriptors_scripts_metadata->>'redeem' redeem, descriptors_scripts_metadata->>'blindedAddress' blinded_addr, descriptor_metadata->>'descriptor' descriptor FROM nbxv1_keypath_info ki WHERE ki.code=@code AND ki.script=r.script ) ts - LEFT JOIN wallets_descriptors wd ON wd.descriptor = ts.descriptor AND wd.code = @code - LEFT JOIN wallets wd_wallet ON wd_wallet.wallet_id = wd.wallet_id - LEFT JOIN wallets_scripts ws ON ws.script = ts.script AND ws.code = @code - LEFT JOIN wallets ws_wallet ON ws_wallet.wallet_id = ws.wallet_id - WHERE COALESCE(wd.wallet_id, ws.wallet_id) IS NOT NULL;", + LEFT JOIN wallets_scripts ws USING(code, script) + LEFT JOIN wallets w USING(wallet_id) + WHERE ws.wallet_id IS NOT NULL;", new { code = Network.CryptoCode, records = scripts.Select(s => s.ToHex()).ToArray() }); foreach (var r in rows) From da6b9639e22095e7425ddd7e3244bb7f07fbe670 Mon Sep 17 00:00:00 2001 From: Kukks Date: Fri, 17 Nov 2023 15:37:58 +0100 Subject: [PATCH 05/16] add more complex test --- NBXplorer.Client/ExplorerClient.cs | 6 ++++- NBXplorer.Tests/UnitTest1.cs | 30 +++++++++++++++++++++++-- NBXplorer/Controllers/MainController.cs | 12 +++++++--- docs/API.md | 7 ++++-- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/NBXplorer.Client/ExplorerClient.cs b/NBXplorer.Client/ExplorerClient.cs index 9707fc810..0b70a092f 100644 --- a/NBXplorer.Client/ExplorerClient.cs +++ b/NBXplorer.Client/ExplorerClient.cs @@ -443,7 +443,11 @@ public KeyPathInformation GetKeyInformation(DerivationStrategyBase strategy, Scr public async Task GetKeyInformationAsync(DerivationStrategyBase strategy, Script script, CancellationToken cancellation = default) { - return await SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/derivations/{strategy}/scripts/{script.ToHex()}", cancellation).ConfigureAwait(false); + return await GetKeyInformationAsync(new DerivationSchemeTrackedSource(strategy), script, cancellation).ConfigureAwait(false); + } + public async Task GetKeyInformationAsync(TrackedSource trackedSource, Script script, CancellationToken cancellation = default) + { + return await SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/scripts/{script.ToHex()}", cancellation).ConfigureAwait(false); } [Obsolete("Use GetKeyInformationAsync(DerivationStrategyBase strategy, Script script) instead")] diff --git a/NBXplorer.Tests/UnitTest1.cs b/NBXplorer.Tests/UnitTest1.cs index 5e37c2b92..3ffc035a2 100644 --- a/NBXplorer.Tests/UnitTest1.cs +++ b/NBXplorer.Tests/UnitTest1.cs @@ -4560,7 +4560,7 @@ public async Task CanAssociateIndependentScripts(Backend backend) var parentWallet = Guid.NewGuid().ToString(); var parentWalletTS = new WalletTrackedSource(parentWallet); - +#if SUPPORT_DBTRIE //this should create both wallets if (backend == Backend.DBTrie) { @@ -4579,7 +4579,7 @@ await Assert.ThrowsAsync(async () =>await tester.Client.Gene await Assert.ThrowsAsync(async () =>await tester.Client.ImportUTXOs(parentWalletTS, Array.Empty())); return; } - +#endif await tester.Client.TrackAsync(wallet1TS, new TrackWalletRequest() { ParentWallet = parentWalletTS @@ -4652,6 +4652,32 @@ await Eventually(async () => scriptBagUtxos = await tester.Client.GetUTXOsAsync(wallet1TS); Assert.Equal(2, scriptBagUtxos.GetUnspentUTXOs().Length); }); + + //create wallet A + //create wallet b using generate and make it child of A + // create address using unused on B + // creat wallet C tracking address from B, make it child of A, B + var walletA = new WalletTrackedSource(Guid.NewGuid().ToString()); + await tester.Client.TrackAsync(walletA); + var generatResponse = await tester.Client.GenerateWalletAsync(new GenerateWalletRequest() + { + ParentWallet = walletA + }); + var walletB = TrackedSource.Create(generatResponse.DerivationScheme); + var addressA = await tester.Client.GetUnusedAsync(generatResponse.DerivationScheme, DerivationFeature.Deposit, 0, true); + var walletC = AddressTrackedSource.Create(addressA.Address); + await tester.Client.TrackAsync(walletC, new TrackWalletRequest() + { + ParentWallet = walletB + }); + await tester.Client.TrackAsync(walletC, new TrackWalletRequest() + { + ParentWallet = walletA + }); + + var kpi = await tester.Client.GetKeyInformationsAsync(addressA.ScriptPubKey, CancellationToken.None); + var tss = kpi.Select(information => information.TrackedSource); + Assert.True(tss.Distinct().Count() == tss.Count(), "The result should only distinct tracked source matches. While this endpoint is marked obsolete, the same logic is used to trigger events, which means there will be duplicated events when the script is matched against"); } } } diff --git a/NBXplorer/Controllers/MainController.cs b/NBXplorer/Controllers/MainController.cs index 1ee4cb363..6c1cea33a 100644 --- a/NBXplorer/Controllers/MainController.cs +++ b/NBXplorer/Controllers/MainController.cs @@ -200,13 +200,14 @@ public async Task GetKeyInformations(string cryptoCode, [HttpGet] [Route($"{CommonRoutes.DerivationEndpoint}/scripts/{{script}}")] - [TrackedSourceContext.TrackedSourceContextRequirement(allowedTrackedSourceTypes:typeof(DerivationSchemeTrackedSource))] + [Route($"{CommonRoutes.AddressEndpoint}/scripts/{{script}}")] + [Route($"{CommonRoutes.WalletEndpoint}/scripts/{{script}}")] + [Route($"{CommonRoutes.TrackedSourceEndpoint}/scripts/{{script}}")] public async Task GetKeyInformations(TrackedSourceContext trackedSourceContext, [ModelBinder(BinderType = typeof(ScriptModelBinder))] Script script) { - var derivationScheme = ((DerivationSchemeTrackedSource ) trackedSourceContext.TrackedSource).DerivationStrategy; var result = (await trackedSourceContext.Repository.GetKeyInformations(new[] { script })) .SelectMany(k => k.Value) - .FirstOrDefault(k => k.DerivationStrategy == derivationScheme); + .FirstOrDefault(k => k.TrackedSource == trackedSourceContext.TrackedSource); if (result == null) throw new NBXplorerError(404, "script-not-found", "The script does not seem to be tracked").AsException(); return Json(result, trackedSourceContext.Network.Serializer.Settings); @@ -518,6 +519,11 @@ public async Task TrackWallet( (trackedSourceContext.TrackedSource is WalletTrackedSource || request?.ParentWallet is not null)) { + if (request?.ParentWallet == trackedSourceContext.TrackedSource) + { + throw new NBXplorerException(new NBXplorerError(400, "parent-wallet-same-as-tracked-source", + "Parent wallets cannot be the same as the tracked source")); + } await postgresRepository.EnsureWalletCreated(trackedSourceContext.TrackedSource, request?.ParentWallet is null? null: new []{request?.ParentWallet }); } if (repo is not PostgresRepository && request.ParentWallet is not null) diff --git a/docs/API.md b/docs/API.md index f5c41bb65..fd50bbea6 100644 --- a/docs/API.md +++ b/docs/API.md @@ -472,10 +472,12 @@ Returns: Note: `redeem` is returning the segwit redeem if the derivation scheme is a P2SH-P2WSH or P2WSH, or the p2sh redeem if just a p2sh. -## Get scriptPubKey information of a Derivation Scheme +## Get scriptPubKey information of a tracked source `HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/scripts/{script}` - +`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/scripts/{script}` +`HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/scripts/{script}` +`HTTP GET v1/cryptos/{cryptoCode}/tracked-sources/{trackedSource}/scripts/{script}` Error codes: * HTTP 404: `cryptoCode-not-supported` @@ -498,6 +500,7 @@ Returns: `HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/utxos` `HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/utxos` `HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/utxos` +`HTTP GET v1/cryptos/{cryptoCode}/tracked-sources/{trackedSource}/utxos` Error: From a1a2ffbdbfffca4909e5950c51263531174031d6 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Wed, 22 Nov 2023 16:34:34 +0900 Subject: [PATCH 06/16] Fix query --- NBXplorer.Client/ExplorerClient.cs | 2 -- NBXplorer.Tests/UnitTest1.cs | 4 ++-- NBXplorer/Backends/Postgres/PostgresRepository.cs | 8 +++----- .../DBScripts/021.KeyPathInfoReturnsWalletId.sql | 13 +++++++++++++ NBXplorer/NBXplorer.csproj | 1 + 5 files changed, 19 insertions(+), 9 deletions(-) create mode 100644 NBXplorer/DBScripts/021.KeyPathInfoReturnsWalletId.sql diff --git a/NBXplorer.Client/ExplorerClient.cs b/NBXplorer.Client/ExplorerClient.cs index 0b70a092f..d284d561c 100644 --- a/NBXplorer.Client/ExplorerClient.cs +++ b/NBXplorer.Client/ExplorerClient.cs @@ -450,13 +450,11 @@ public async Task GetKeyInformationAsync(TrackedSource track return await SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/scripts/{script.ToHex()}", cancellation).ConfigureAwait(false); } - [Obsolete("Use GetKeyInformationAsync(DerivationStrategyBase strategy, Script script) instead")] public async Task GetKeyInformationsAsync(Script script, CancellationToken cancellation = default) { return await SendAsync(HttpMethod.Get, null, $"v1/cryptos/{CryptoCode}/scripts/{script.ToHex()}", cancellation).ConfigureAwait(false); } - [Obsolete("Use GetKeyInformation(DerivationStrategyBase strategy, Script script) instead")] public KeyPathInformation[] GetKeyInformations(Script script, CancellationToken cancellation = default) { return GetKeyInformationsAsync(script, cancellation).GetAwaiter().GetResult(); diff --git a/NBXplorer.Tests/UnitTest1.cs b/NBXplorer.Tests/UnitTest1.cs index 3ffc035a2..68b0a07e9 100644 --- a/NBXplorer.Tests/UnitTest1.cs +++ b/NBXplorer.Tests/UnitTest1.cs @@ -4537,7 +4537,7 @@ private async Task Eventually(Func tsk) break; } - catch (Exception e) + catch (Exception) { await Task.Delay(500); } @@ -4626,7 +4626,7 @@ await Eventually(async () => { parentWalletUtxos = await tester.Client.GetUTXOsAsync(parentWalletTS); scriptBagUtxos = await tester.Client.GetUTXOsAsync(wallet1TS); - Assert.Equal(1, scriptBagUtxos.GetUnspentUTXOs().Length); + Assert.Single(scriptBagUtxos.GetUnspentUTXOs()); Assert.Equal(2, parentWalletUtxos.GetUnspentUTXOs().Length); Assert.Equal(derivationUtxos.GetUnspentUTXOs().Count() + scriptBagUtxos.GetUnspentUTXOs().Length, parentWalletUtxos.GetUnspentUTXOs().Count()); diff --git a/NBXplorer/Backends/Postgres/PostgresRepository.cs b/NBXplorer/Backends/Postgres/PostgresRepository.cs index e909af0b3..79d2777e3 100644 --- a/NBXplorer/Backends/Postgres/PostgresRepository.cs +++ b/NBXplorer/Backends/Postgres/PostgresRepository.cs @@ -485,20 +485,18 @@ async Task> GetKeyInformations( string additionalColumn = Network.IsElement ? ", ts.blinded_addr" : ""; var rows = await connection.QueryAsync($@" SELECT ts.code, ts.script, ts.addr, ts.derivation, ts.keypath, ts.redeem{additionalColumn}, - ws.wallet_id, + ts.wallet_id, w.metadata->>'type' AS wallet_type FROM unnest(@records) AS r (script), LATERAL ( - SELECT code, script, addr, descriptor_metadata->>'derivation' derivation, + SELECT code, script, wallet_id, addr, descriptor_metadata->>'derivation' derivation, keypath, descriptors_scripts_metadata->>'redeem' redeem, descriptors_scripts_metadata->>'blindedAddress' blinded_addr, descriptor_metadata->>'descriptor' descriptor FROM nbxv1_keypath_info ki WHERE ki.code=@code AND ki.script=r.script ) ts - LEFT JOIN wallets_scripts ws USING(code, script) - LEFT JOIN wallets w USING(wallet_id) - WHERE ws.wallet_id IS NOT NULL;", + JOIN wallets w USING(wallet_id)", new { code = Network.CryptoCode, records = scripts.Select(s => s.ToHex()).ToArray() }); foreach (var r in rows) diff --git a/NBXplorer/DBScripts/021.KeyPathInfoReturnsWalletId.sql b/NBXplorer/DBScripts/021.KeyPathInfoReturnsWalletId.sql new file mode 100644 index 000000000..f8d6563a2 --- /dev/null +++ b/NBXplorer/DBScripts/021.KeyPathInfoReturnsWalletId.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE VIEW nbxv1_keypath_info AS + SELECT ws.code, + ws.script, + s.addr, + d.metadata AS descriptor_metadata, + nbxv1_get_keypath(d.metadata, ds.idx) AS keypath, + ds.metadata AS descriptors_scripts_metadata, + ws.wallet_id + FROM ((wallets_scripts ws + JOIN scripts s ON (((s.code = ws.code) AND (s.script = ws.script)))) + LEFT JOIN ((wallets_descriptors wd + JOIN descriptors_scripts ds ON (((ds.code = wd.code) AND (ds.descriptor = wd.descriptor)))) + JOIN descriptors d ON (((d.code = ds.code) AND (d.descriptor = ds.descriptor)))) ON (((wd.wallet_id = ws.wallet_id) AND (wd.code = ws.code) AND (ds.script = ws.script)))); \ No newline at end of file diff --git a/NBXplorer/NBXplorer.csproj b/NBXplorer/NBXplorer.csproj index 75d8403d3..603f0722b 100644 --- a/NBXplorer/NBXplorer.csproj +++ b/NBXplorer/NBXplorer.csproj @@ -28,6 +28,7 @@ + From 3c39ea86d82776f280703cab39e91ac29f0ad1d4 Mon Sep 17 00:00:00 2001 From: Kukks Date: Wed, 22 Nov 2023 10:53:40 +0100 Subject: [PATCH 07/16] add hierarchy management API --- NBXplorer.Client/ExplorerClient.cs | 49 ++++++++++++++- .../Models/TrackedSourceRequest.cs | 6 ++ NBXplorer.Tests/UnitTest1.cs | 48 +++++++++++++++ .../Backends/Postgres/PostgresRepository.cs | 33 +++++++++- NBXplorer/Controllers/MainController.PSBT.cs | 11 +--- NBXplorer/Controllers/MainController.cs | 13 ++-- .../Controllers/PostgresMainController.cs | 60 +++++++++++++++++++ NBXplorer/DBScripts/FullSchema.sql | 4 +- NBXplorer/Extensions.cs | 8 +++ 9 files changed, 213 insertions(+), 19 deletions(-) create mode 100644 NBXplorer.Client/Models/TrackedSourceRequest.cs diff --git a/NBXplorer.Client/ExplorerClient.cs b/NBXplorer.Client/ExplorerClient.cs index d284d561c..46e4d207b 100644 --- a/NBXplorer.Client/ExplorerClient.cs +++ b/NBXplorer.Client/ExplorerClient.cs @@ -561,6 +561,53 @@ public GenerateWalletResponse GenerateWallet(GenerateWalletRequest request = nul return GenerateWalletAsync(request, cancellationToken).GetAwaiter().GetResult(); } + public async Task GetChildWallets(TrackedSource trackedSource, + CancellationToken cancellation = default) + { + return await GetAsync( $"{GetBasePath(trackedSource)}/children", cancellation); + } + public async Task GetParentWallets(TrackedSource trackedSource, + CancellationToken cancellation = default) + { + return await GetAsync( $"{GetBasePath(trackedSource)}/parents", cancellation); + } + public async Task AddChildWallet(TrackedSource trackedSource, TrackedSource childWallet, CancellationToken cancellation = default) + { + var request = new TrackedSourceRequest() + { + TrackedSource = childWallet + }; + await SendAsync(HttpMethod.Post, request, $"{GetBasePath(trackedSource)}/children", cancellation); + } + + public async Task AddParentWallet(TrackedSource trackedSource, TrackedSource parentWallet, + CancellationToken cancellation = default) + { + var request = new TrackedSourceRequest() + { + TrackedSource = parentWallet + }; + await SendAsync(HttpMethod.Post, request, $"{GetBasePath(trackedSource)}/parents", cancellation); + } + public async Task RemoveChildWallet(TrackedSource trackedSource, TrackedSource childWallet, CancellationToken cancellation = default) + { + var request = new TrackedSourceRequest() + { + TrackedSource = childWallet + }; + await SendAsync(HttpMethod.Delete, request, $"{GetBasePath(trackedSource)}/children", cancellation); + } + + public async Task RemoveParentWallet(TrackedSource trackedSource, TrackedSource parentWallet, + CancellationToken cancellation = default) + { + var request = new TrackedSourceRequest() + { + TrackedSource = parentWallet + }; + await SendAsync(HttpMethod.Delete, request, $"{GetBasePath(trackedSource)}/parents", cancellation); + } + private static readonly HttpClient SharedClient = new HttpClient(); internal HttpClient Client = SharedClient; @@ -738,7 +785,7 @@ private FormattableString GetBasePath(TrackedSource trackedSource) DerivationSchemeTrackedSource dsts => $"v1/cryptos/{CryptoCode}/derivations/{dsts.DerivationStrategy}", AddressTrackedSource asts => $"v1/cryptos/{CryptoCode}/addresses/{asts.Address}", WalletTrackedSource wts => $"v1/cryptos/{CryptoCode}/wallets/{wts.WalletId}", - _ => throw UnSupported(trackedSource) + _ => $"v1/cryptos/{CryptoCode}/tracked-sources/{trackedSource}", }; } } diff --git a/NBXplorer.Client/Models/TrackedSourceRequest.cs b/NBXplorer.Client/Models/TrackedSourceRequest.cs new file mode 100644 index 000000000..86f2845a0 --- /dev/null +++ b/NBXplorer.Client/Models/TrackedSourceRequest.cs @@ -0,0 +1,6 @@ +namespace NBXplorer.Models; + +public class TrackedSourceRequest +{ + public TrackedSource TrackedSource { get; set; } +} \ No newline at end of file diff --git a/NBXplorer.Tests/UnitTest1.cs b/NBXplorer.Tests/UnitTest1.cs index 68b0a07e9..9cb70abef 100644 --- a/NBXplorer.Tests/UnitTest1.cs +++ b/NBXplorer.Tests/UnitTest1.cs @@ -4678,6 +4678,54 @@ await Eventually(async () => var kpi = await tester.Client.GetKeyInformationsAsync(addressA.ScriptPubKey, CancellationToken.None); var tss = kpi.Select(information => information.TrackedSource); Assert.True(tss.Distinct().Count() == tss.Count(), "The result should only distinct tracked source matches. While this endpoint is marked obsolete, the same logic is used to trigger events, which means there will be duplicated events when the script is matched against"); + + var parentsOfC = await tester.Client.GetParentWallets(walletC); + Assert.Equal(2, parentsOfC.Length); + Assert.Contains(parentsOfC, w => w == walletA); + Assert.Contains(parentsOfC, w => w == walletB); + + var parentsOfB = await tester.Client.GetParentWallets(walletB); + Assert.Equal(1, parentsOfB.Length); + Assert.Contains(parentsOfB, w => w == walletA); + + var parentsOfA = await tester.Client.GetParentWallets(walletA); + Assert.Empty(parentsOfA); + + var childrenOfA= await tester.Client.GetChildWallets(walletA); + Assert.Equal(2, childrenOfA.Length); + + Assert.Contains(childrenOfA, w => w == walletB); + Assert.Contains(childrenOfA, w => w == walletC); + + var childrenOfB= await tester.Client.GetChildWallets(walletB); + Assert.Equal(1, childrenOfB.Length); + Assert.Contains(childrenOfB, w => w == walletC); + + var childrenOfC = await tester.Client.GetChildWallets(walletC); + Assert.Empty(childrenOfC); + + await tester.Client.RemoveParentWallet(walletB, walletA); + await tester.Client.RemoveChildWallet(walletB, walletC); + + parentsOfB = await tester.Client.GetParentWallets(walletB); + Assert.Empty(parentsOfB); + + childrenOfB = await tester.Client.GetChildWallets(walletB); + Assert.Empty(childrenOfB); + + + await tester.Client.AddParentWallet(walletB, walletA); + await tester.Client.AddChildWallet(walletB, walletC); + + + childrenOfB= await tester.Client.GetChildWallets(walletB); + Assert.Equal(1, childrenOfB.Length); + Assert.Contains(childrenOfB, w => w == walletC); + + parentsOfB = await tester.Client.GetParentWallets(walletB); + Assert.Equal(1, parentsOfB.Length); + Assert.Contains(parentsOfB, w => w == walletA); + } } } diff --git a/NBXplorer/Backends/Postgres/PostgresRepository.cs b/NBXplorer/Backends/Postgres/PostgresRepository.cs index 79d2777e3..c192b8750 100644 --- a/NBXplorer/Backends/Postgres/PostgresRepository.cs +++ b/NBXplorer/Backends/Postgres/PostgresRepository.cs @@ -239,6 +239,36 @@ internal WalletKey GetWalletKey(TrackedSource source) _ => throw new NotSupportedException(source.GetType().ToString()) }; } + + internal TrackedSource GetTrackedSource(WalletKey walletKey) + { + var metadata = JObject.Parse(walletKey.metadata); + if (metadata.TryGetValue("type", StringComparison.OrdinalIgnoreCase, out JToken typeJToken) && + typeJToken.Value() is { } type) + { + if ((metadata.TryGetValue("code", StringComparison.OrdinalIgnoreCase, out JToken codeJToken) && + codeJToken.Value() is { } code) && !code.Equals(Network.CryptoCode, + StringComparison.InvariantCultureIgnoreCase)) + { + return null; + } + + switch (type) + { + case "NBXv1-Derivation": + var derivation = metadata["derivation"].Value(); + return new DerivationSchemeTrackedSource(Network.DerivationStrategyFactory.Parse(derivation)); + case "NBXv1-Address": + var address = metadata["address"].Value(); + return new AddressTrackedSource(BitcoinAddress.Create(address, Network.NBitcoinNetwork)); + case "Wallet": + return new WalletTrackedSource(walletKey.wid); + } + } + + return null; + } + public async Task AssociateScriptsToWalletExplicitly(TrackedSource trackedSource, Dictionary scripts) { @@ -1304,8 +1334,9 @@ public async Task EnsureWalletCreated(DerivationStrategyBase strategy) await EnsureWalletCreated(GetWalletKey(strategy)); } - public async Task EnsureWalletCreated(TrackedSource trackedSource, TrackedSource[] parentTrackedSource = null) + public async Task EnsureWalletCreated(TrackedSource trackedSource, params TrackedSource[] parentTrackedSource) { + parentTrackedSource = parentTrackedSource.Where(source => source is not null).ToArray(); await EnsureWalletCreated(GetWalletKey(trackedSource), parentTrackedSource?.Select(GetWalletKey).ToArray()); } diff --git a/NBXplorer/Controllers/MainController.PSBT.cs b/NBXplorer/Controllers/MainController.PSBT.cs index 5c934d105..f121a1e97 100644 --- a/NBXplorer/Controllers/MainController.PSBT.cs +++ b/NBXplorer/Controllers/MainController.PSBT.cs @@ -28,7 +28,7 @@ public async Task CreatePSBT( { if (body == null) throw new ArgumentNullException(nameof(body)); - CreatePSBTRequest request = ParseJObject(body, trackedSourceContext.Network); + CreatePSBTRequest request = trackedSourceContext.Network.ParseJObject(body); var repo = RepositoryProvider.GetRepository(trackedSourceContext.Network); var txBuilder = request.Seed is int s ? trackedSourceContext.Network.NBitcoinNetwork.CreateTransactionBuilder(s) @@ -388,7 +388,7 @@ public async Task UpdatePSBT( [FromBody] JObject body) { - var update = ParseJObject(body, network); + var update = network.ParseJObject(body); if (update.PSBT == null) throw new NBXplorerException(new NBXplorerError(400, "missing-parameter", "'psbt' is missing")); await UpdatePSBTCore(update, network); @@ -586,11 +586,6 @@ await Task.WhenAll(update.PSBT.Inputs } } - protected T ParseJObject(JObject requestObj, NBXplorerNetwork network) - { - if (requestObj == null) - return default; - return network.Serializer.ToObject(requestObj); - } + } } diff --git a/NBXplorer/Controllers/MainController.cs b/NBXplorer/Controllers/MainController.cs index 6c1cea33a..ed3ca723d 100644 --- a/NBXplorer/Controllers/MainController.cs +++ b/NBXplorer/Controllers/MainController.cs @@ -503,7 +503,6 @@ private bool HasTxIndex(string cryptoCode) } [HttpPost] - [Route($"{CommonRoutes.DerivationEndpoint}")] [Route($"{CommonRoutes.AddressEndpoint}")] [Route($"{CommonRoutes.WalletEndpoint}")] @@ -512,7 +511,7 @@ public async Task TrackWallet( TrackedSourceContext trackedSourceContext, [FromBody] JObject rawRequest = null) { - var request = ParseJObject(rawRequest ?? new JObject(), trackedSourceContext.Network); + var request = trackedSourceContext.Network.ParseJObject(rawRequest ?? new JObject()); var repo = RepositoryProvider.GetRepository(trackedSourceContext.Network); if (repo is PostgresRepository postgresRepository && @@ -524,7 +523,7 @@ public async Task TrackWallet( throw new NBXplorerException(new NBXplorerError(400, "parent-wallet-same-as-tracked-source", "Parent wallets cannot be the same as the tracked source")); } - await postgresRepository.EnsureWalletCreated(trackedSourceContext.TrackedSource, request?.ParentWallet is null? null: new []{request?.ParentWallet }); + await postgresRepository.EnsureWalletCreated(trackedSourceContext.TrackedSource, request?.ParentWallet); } if (repo is not PostgresRepository && request.ParentWallet is not null) throw new NBXplorerException(new NBXplorerError(400, "parent-wallet-not-supported", @@ -586,9 +585,7 @@ public async Task GetTransactions( bool includeTransaction = true) { TransactionInformation fetchedTransactionInfo = null; - - var repo = RepositoryProvider.GetRepository(trackedSourceContext.Network); - + var repo = trackedSourceContext.Repository; var response = new GetTransactionsResponse(); int currentHeight = (await repo.GetTip()).Height; response.Height = currentHeight; @@ -676,7 +673,7 @@ public async Task Rescan(TrackedSourceContext trackedSourceContex { if (body == null) throw new ArgumentNullException(nameof(body)); - var rescanRequest = ParseJObject(body, trackedSourceContext.Network); + var rescanRequest = trackedSourceContext.Network.ParseJObject(body); if (rescanRequest == null) throw new ArgumentNullException(nameof(rescanRequest)); if (rescanRequest?.Transactions == null) @@ -1036,7 +1033,7 @@ public async Task Broadcast( [TrackedSourceContext.TrackedSourceContextRequirement(false, false)] public async Task GenerateWallet(TrackedSourceContext trackedSourceContext, [FromBody] JObject rawRequest = null) { - var request = ParseJObject(rawRequest, trackedSourceContext.Network) ?? new GenerateWalletRequest(); + var request = trackedSourceContext.Network.ParseJObject(rawRequest ) ?? new GenerateWalletRequest(); if (request.ImportKeysToRPC && trackedSourceContext.RpcClient is null) { diff --git a/NBXplorer/Controllers/PostgresMainController.cs b/NBXplorer/Controllers/PostgresMainController.cs index d8fa4f4a5..fbf337c83 100644 --- a/NBXplorer/Controllers/PostgresMainController.cs +++ b/NBXplorer/Controllers/PostgresMainController.cs @@ -13,6 +13,7 @@ using NBXplorer.Models; using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Serialization; namespace NBXplorer.Controllers { @@ -256,5 +257,64 @@ public async Task GetUTXOs( TrackedSourceContext trackedSourceCon } return Json(changes, trackedSourceContext.Network.JsonSerializerSettings); } + + + [HttpGet("children")] + public async Task GetWalletChildren( TrackedSourceContext trackedSourceContext) + { + var repo = (PostgresRepository)trackedSourceContext.Repository; + await using var conn = await ConnectionFactory.CreateConnection(); + var children = await conn.QueryAsync($"SELECT w.wallet_id, w.metadata FROM wallets_wallets ww JOIN wallets w ON ww.wallet_id = w.wallet_id WHERE ww.parent_id=@walletId", new { walletId = repo.GetWalletKey(trackedSourceContext.TrackedSource).wid }); + + return Json(children.Select(c => repo.GetTrackedSource(new PostgresRepository.WalletKey(c.wallet_id, c.metadata)) ).ToArray(), trackedSourceContext.Network.JsonSerializerSettings); + } + [HttpGet("parents")] + public async Task GetWalletParents( TrackedSourceContext trackedSourceContext) + { + var repo = (PostgresRepository)trackedSourceContext.Repository; + await using var conn = await ConnectionFactory.CreateConnection(); + var children = await conn.QueryAsync($"SELECT w.wallet_id, w.metadata FROM wallets_wallets ww JOIN wallets w ON ww.parent_id = w.wallet_id WHERE ww.wallet_id=@walletId", new { walletId = repo.GetWalletKey(trackedSourceContext.TrackedSource).wid }); + + return Json(children.Select(c => repo.GetTrackedSource(new PostgresRepository.WalletKey(c.wallet_id, c.metadata)) ).ToArray(), trackedSourceContext.Network.JsonSerializerSettings); + } + [HttpPost("children")] + public async Task AddWalletChild( TrackedSourceContext trackedSourceContext, [FromBody] JObject request) + { + var trackedSource = trackedSourceContext.Network.ParseJObject(request).TrackedSource; + var repo = (PostgresRepository)trackedSourceContext.Repository; + await repo.EnsureWalletCreated(trackedSource, trackedSourceContext.TrackedSource); + return Ok(); + } + [HttpPost("parents")] + public async Task AddWalletParent( TrackedSourceContext trackedSourceContext, [FromBody] JObject request) + { + var trackedSource = trackedSourceContext.Network.ParseJObject(request).TrackedSource; + var repo = (PostgresRepository)trackedSourceContext.Repository; + await repo.EnsureWalletCreated(trackedSourceContext.TrackedSource, trackedSource); + return Ok(); + } + [HttpDelete("children")] + public async Task RemoveWalletChild( TrackedSourceContext trackedSourceContext, [FromBody] JObject request) + { + var repo = (PostgresRepository)trackedSourceContext.Repository; + + var trackedSource = repo.GetWalletKey(trackedSourceContext.Network + .ParseJObject(request).TrackedSource); + var conn = await ConnectionFactory.CreateConnection(); + await conn.ExecuteAsync($"DELETE FROM wallets_wallets WHERE wallet_id=@walletId AND parent_id=@parentId", new { walletId = trackedSource.wid, parentId = repo.GetWalletKey(trackedSourceContext.TrackedSource).wid }); + return Ok(); + } + [HttpDelete("parents")] + public async Task RemoveWalletParent( TrackedSourceContext trackedSourceContext, [FromBody] JObject request) + { + var repo = (PostgresRepository)trackedSourceContext.Repository; + + var trackedSource = repo.GetWalletKey(trackedSourceContext.Network + .ParseJObject(request).TrackedSource); + var conn = await ConnectionFactory.CreateConnection(); + await conn.ExecuteAsync($"DELETE FROM wallets_wallets WHERE wallet_id=@walletId AND parent_id=@parentId", new { walletId = repo.GetWalletKey(trackedSourceContext.TrackedSource).wid, parentId = trackedSource.wid }); + return Ok(); + } } + } diff --git a/NBXplorer/DBScripts/FullSchema.sql b/NBXplorer/DBScripts/FullSchema.sql index 040bfd826..c2c736fb0 100644 --- a/NBXplorer/DBScripts/FullSchema.sql +++ b/NBXplorer/DBScripts/FullSchema.sql @@ -958,7 +958,8 @@ CREATE VIEW nbxv1_keypath_info AS s.addr, d.metadata AS descriptor_metadata, nbxv1_get_keypath(d.metadata, ds.idx) AS keypath, - ds.metadata AS descriptors_scripts_metadata + ds.metadata AS descriptors_scripts_metadata, + ws.wallet_id FROM ((wallets_scripts ws JOIN scripts s ON (((s.code = ws.code) AND (s.script = ws.script)))) LEFT JOIN ((wallets_descriptors wd @@ -1362,6 +1363,7 @@ INSERT INTO nbxv1_migrations VALUES ('017.FixDoubleSpendDetection'); INSERT INTO nbxv1_migrations VALUES ('018.FastWalletRecent'); INSERT INTO nbxv1_migrations VALUES ('019.FixDoubleSpendDetection2'); INSERT INTO nbxv1_migrations VALUES ('020.ReplacingShouldBeIdempotent'); +INSERT INTO nbxv1_migrations VALUES ('021.KeyPathInfoReturnsWalletId'); ALTER TABLE ONLY nbxv1_migrations ADD CONSTRAINT nbxv1_migrations_pkey PRIMARY KEY (script_name); \ No newline at end of file diff --git a/NBXplorer/Extensions.cs b/NBXplorer/Extensions.cs index dca02252a..a3df78380 100644 --- a/NBXplorer/Extensions.cs +++ b/NBXplorer/Extensions.cs @@ -31,11 +31,19 @@ using Npgsql; using NBitcoin.Altcoins; using System.Threading; +using Newtonsoft.Json.Linq; namespace NBXplorer { public static class Extensions { + public static T ParseJObject(this NBXplorerNetwork network, JObject requestObj) + { + if (requestObj == null) + return default; + return network.Serializer.ToObject(requestObj); + } + public static async Task ReliableOpenConnectionAsync(this NpgsqlDataSource ds, CancellationToken cancellationToken = default) { int maxRetries = 10; From dce4f429de604359483e9b3f6cceba193ff8107f Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Mon, 27 Nov 2023 18:10:25 +0900 Subject: [PATCH 08/16] Fix doc --- NBXplorer.Tests/xunit.runner.json | 3 ++- NBXplorer/Startup.cs | 2 +- docs/API.md | 8 +++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/NBXplorer.Tests/xunit.runner.json b/NBXplorer.Tests/xunit.runner.json index dc14f82b1..6dae53c9c 100644 --- a/NBXplorer.Tests/xunit.runner.json +++ b/NBXplorer.Tests/xunit.runner.json @@ -1,3 +1,4 @@ { - "parallelizeTestCollections": false + "parallelizeTestCollections": false, + "methodDisplay": "method" } \ No newline at end of file diff --git a/NBXplorer/Startup.cs b/NBXplorer/Startup.cs index eb5035c60..85ef8e1cb 100644 --- a/NBXplorer/Startup.cs +++ b/NBXplorer/Startup.cs @@ -70,7 +70,7 @@ public void Configure(IApplicationBuilder app, IServiceProvider prov, app.UseAuthentication(); app.UseAuthorization(); app.UseWebSockets(); - //app.UseMiddleware(); + app.UseMiddleware(); app.UseEndpoints(endpoints => { endpoints.MapHealthChecks("health", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions() diff --git a/docs/API.md b/docs/API.md index fd50bbea6..47be9c90a 100644 --- a/docs/API.md +++ b/docs/API.md @@ -514,7 +514,7 @@ Result: "derivationStrategy": "tpubD6NzVbkrYhZ4XQVi1sSEDBWTcicDqVSCTnYDxpwGwcSZVbPii2b7baRg57YfL64ed36sBRe6GviihHwhy3D1cnBe5uXb27DjrDZCKUA7PQi", "currentHeight": 107, "unconfirmed": { - "utxos": [ + "utxOs": [ { "feature": "Deposit", "outpoint": "10ba4bcadd03130b1bd98b0bc7aea9910f871b25b87ec06e484456e84440c88a01000000", @@ -532,8 +532,8 @@ Result: "hasChanges": true }, "confirmed": { - "utxos": [ - {// when it is a derivation scheme + "utxOs": [ + { "feature": "Deposit", "outpoint": "29ca6590f3f03a6523ad79975392e74e385bf2b7dafe6c537ffa12f9e124348800000000", "index": 0, @@ -544,8 +544,6 @@ Result: "keyPath": "0/3", "timestamp": 1540376174, "confirmations": 1 - }, - {// when it is an address } ], "spentOutpoints": [ From a98869e293a83306162d23b966b54100a8de4891 Mon Sep 17 00:00:00 2001 From: "nicolas.dorier" Date: Mon, 27 Nov 2023 18:12:11 +0900 Subject: [PATCH 09/16] Fix warning --- NBXplorer.Tests/UnitTest1.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/NBXplorer.Tests/UnitTest1.cs b/NBXplorer.Tests/UnitTest1.cs index 9cb70abef..930080570 100644 --- a/NBXplorer.Tests/UnitTest1.cs +++ b/NBXplorer.Tests/UnitTest1.cs @@ -4685,7 +4685,7 @@ await Eventually(async () => Assert.Contains(parentsOfC, w => w == walletB); var parentsOfB = await tester.Client.GetParentWallets(walletB); - Assert.Equal(1, parentsOfB.Length); + Assert.Single(parentsOfB); Assert.Contains(parentsOfB, w => w == walletA); var parentsOfA = await tester.Client.GetParentWallets(walletA); @@ -4698,7 +4698,7 @@ await Eventually(async () => Assert.Contains(childrenOfA, w => w == walletC); var childrenOfB= await tester.Client.GetChildWallets(walletB); - Assert.Equal(1, childrenOfB.Length); + Assert.Single(childrenOfB); Assert.Contains(childrenOfB, w => w == walletC); var childrenOfC = await tester.Client.GetChildWallets(walletC); @@ -4719,11 +4719,11 @@ await Eventually(async () => childrenOfB= await tester.Client.GetChildWallets(walletB); - Assert.Equal(1, childrenOfB.Length); + Assert.Single(childrenOfB); Assert.Contains(childrenOfB, w => w == walletC); parentsOfB = await tester.Client.GetParentWallets(walletB); - Assert.Equal(1, parentsOfB.Length); + Assert.Single(parentsOfB); Assert.Contains(parentsOfB, w => w == walletA); } From 6f4b30ea96cc66f48f94c4053fca762fc17b8d94 Mon Sep 17 00:00:00 2001 From: Kukks Date: Wed, 29 Nov 2023 14:59:39 +0100 Subject: [PATCH 10/16] some fixes and test --- NBXplorer.Client/ExplorerClient.cs | 2 +- NBXplorer.Tests/UnitTest1.cs | 166 ++++++++++++++++-- .../Controllers/PostgresMainController.cs | 20 ++- 3 files changed, 170 insertions(+), 18 deletions(-) diff --git a/NBXplorer.Client/ExplorerClient.cs b/NBXplorer.Client/ExplorerClient.cs index 46e4d207b..ef2025fa3 100644 --- a/NBXplorer.Client/ExplorerClient.cs +++ b/NBXplorer.Client/ExplorerClient.cs @@ -390,7 +390,7 @@ public Task GetTransactionAsync(TrackedSource trackedSou throw new ArgumentNullException(nameof(trackedSource)); return SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/transactions/{txId}", cancellation); } - public async Task AssociateScriptsAsync(TrackedSource trackedSource, Dictionary scripts, CancellationToken cancellation = default) + public async Task AssociateScriptsAsync(TrackedSource trackedSource, Dictionary scripts, CancellationToken cancellation = default) { if (scripts == null) throw new ArgumentNullException(nameof(scripts)); diff --git a/NBXplorer.Tests/UnitTest1.cs b/NBXplorer.Tests/UnitTest1.cs index 930080570..527835905 100644 --- a/NBXplorer.Tests/UnitTest1.cs +++ b/NBXplorer.Tests/UnitTest1.cs @@ -26,6 +26,7 @@ using NBitcoin.Tests; using System.Globalization; using System.Net; +using NBitcoin.DataEncoders; using NBXplorer.HostedServices; using System.Reflection; @@ -4528,14 +4529,12 @@ public async Task CanUseRPCProxy(Backend backend) private async Task Eventually(Func tsk) { var i = 0; - while (i <10) + while (i < 10) { try { - - await tsk.Invoke(); - break; - + await tsk.Invoke(); + break; } catch (Exception) { @@ -4575,7 +4574,7 @@ await Assert.ThrowsAsync(async () =>await tester.Client.Gene ParentWallet = parentWalletTS })); - await Assert.ThrowsAsync(async () =>await tester.Client.AssociateScriptsAsync(parentWalletTS, new Dictionary())); + await Assert.ThrowsAsync(async () =>await tester.Client.AssociateScriptsAsync(parentWalletTS, new Dictionary())); await Assert.ThrowsAsync(async () =>await tester.Client.ImportUTXOs(parentWalletTS, Array.Empty())); return; } @@ -4610,9 +4609,9 @@ await Assert.ThrowsAsync(async () =>await tester.Client.Gene var udetectedTxId = await tester.RPC.SendToAddressAsync(newAddr, Money.FromUnit(0.1m, MoneyUnit.BTC)); await Task.Delay(3000); var utxos = Assert.Single(await tester.RPC.ListUnspentAsync(0, 0, newAddr)); - await tester.Client.AssociateScriptsAsync(wallet1TS, new Dictionary() + await tester.Client.AssociateScriptsAsync(wallet1TS, new Dictionary() { - {newAddr2.ToString(), true} + {newAddr2, true} }); @@ -4633,9 +4632,9 @@ await Eventually(async () => }); - await tester.Client.AssociateScriptsAsync(wallet1TS, new Dictionary() + await tester.Client.AssociateScriptsAsync(wallet1TS, new Dictionary() { - {newAddr.ToString(), true} + {newAddr, true} }); await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest[] @@ -4726,6 +4725,153 @@ await Eventually(async () => Assert.Single(parentsOfB); Assert.Contains(parentsOfB, w => w == walletA); + } + + + + + [Theory] + [InlineData(Backend.Postgres)] + public async Task CanImportUTXOs(Backend backend) + { + using var tester = ServerTester.Create(backend); + + var wallet1 = Guid.NewGuid().ToString(); + var wallet1TS = new WalletTrackedSource(wallet1); + + var k = new Key(); + var kAddress = k.GetAddress(ScriptPubKeyType.Segwit, tester.Network); + var kScript = kAddress.ScriptPubKey; + + // test 1: create a script and send 2 utxos to it(from diff txs), without confirming + // import the first one, verify it is unconfirmed, confirm, then the second one and see it is confirmed + + var tx = await tester.RPC.SendToAddressAsync(kAddress, Money.Coins(1.0m)); + var tx2 = await tester.RPC.SendToAddressAsync(kAddress, Money.Coins(1.0m)); + var rawTx = await tester.RPC.GetRawTransactionAsync(tx); + var rawTx2 = await tester.RPC.GetRawTransactionAsync(tx2); + var utxo = rawTx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kScript); + var utxo2 = rawTx2.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kScript); + + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest[] + { + new() + { + Coin = new Coin(utxo), + Proof = null + } + }); + + var utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + var matched = Assert.Single(utxos.Unconfirmed.UTXOs); + Assert.Equal(kAddress, matched.Address); + + await tester.RPC.GenerateAsync(1); + await Eventually(async () => + { + + var utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Single(utxos.Confirmed.UTXOs); + }); + + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest[] + { + new() + { + Coin = new Coin(utxo2), + Proof = null + } + }); + + await Eventually(async () => + { + var utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Equal(2, utxos.Confirmed.UTXOs.Count); + + //utxo2 may be confirmed but we dont know much about it I guess? + var utxoInfo = utxos.Confirmed.UTXOs.First(u => u.ScriptPubKey== utxo2.TxOut.ScriptPubKey); + Assert.Equal( NBitcoin.Utils.UnixTimeToDateTime(0) , utxoInfo.Timestamp); + }); + + //test2: try adding in fake utxos or spent ones + var fakescript = new Key().PubKey.GetAddress(ScriptPubKeyType.Segwit, tester.Network).ScriptPubKey; + var fakeUtxo = new Coin(new OutPoint(uint256.One, 1), new TxOut(Money.Coins(1.0m), fakescript)); + var kToSpend = new Key(); + var kToSpendAddress = kToSpend.GetAddress(ScriptPubKeyType.Segwit, tester.Network); + var tospendtx = await tester.RPC.SendToAddressAsync(kToSpendAddress, Money.Coins(1.0m)); + var tospendrawtx = await tester.RPC.GetRawTransactionAsync(tospendtx); + var tospendutxo = tospendrawtx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kToSpendAddress.ScriptPubKey); + var validScript = new Key().PubKey.GetAddress(ScriptPubKeyType.Segwit, tester.Network).ScriptPubKey; + var spendingtx = tester.Network.CreateTransactionBuilder() + .AddKeys(kToSpend) + .AddCoins(new Coin(tospendutxo)) + .SendEstimatedFees(new FeeRate(100m)) + .SendAll(validScript).BuildTransaction(true); + await tester.RPC.SendRawTransactionAsync(spendingtx); + + var validScriptUtxo = spendingtx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == validScript); + + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest[] + { + new() + { + Coin = fakeUtxo, + }, + new() + { + Coin = new Coin(tospendutxo) + }, + new() + { + Coin = new Coin(validScriptUtxo) + } + }); + + await Eventually(async () => + { + var utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + Assert.Equal(validScript, Assert.Single(utxos.Unconfirmed.UTXOs).ScriptPubKey); + + }); + + + // let's test out proofs + + var yoScript= new Key().PubKey.GetAddress(ScriptPubKeyType.Segwit, tester.Network).ScriptPubKey; + var yoTxId = await tester.SendToAddressAsync(yoScript, Money.Coins(1.0m)); + var yoTx = await tester.RPC.GetRawTransactionAsync(yoTxId); + var yoUtxo = yoTx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == yoScript); + var blockHash = await tester.RPC.GenerateAsync(1); + var proofResult = await tester.RPC.SendCommandAsync(RPCOperations.gettxoutproof, new [] {yoTxId.ToString() }, blockHash[0].ToString()); + + + var merkleBLockProofBytes = Encoders.Hex.DecodeData(proofResult.ResultString); + var mb = new MerkleBlock(); + mb.FromBytes(merkleBLockProofBytes); + + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest[] + { + new() + { + Coin = new Coin(yoUtxo), + Proof = mb + } + }); + + await Eventually(async () => + { + var utxos = await tester.Client.GetUTXOsAsync(wallet1TS); + var importedUtxoWithProof = utxos.Confirmed.UTXOs.Single(utxo1 => utxo1.ScriptPubKey == yoScript); + + Assert.NotEqual( NBitcoin.Utils.UnixTimeToDateTime(0) , importedUtxoWithProof.Timestamp); + + }); + + + + + + } } } diff --git a/NBXplorer/Controllers/PostgresMainController.cs b/NBXplorer/Controllers/PostgresMainController.cs index fbf337c83..1354e40e0 100644 --- a/NBXplorer/Controllers/PostgresMainController.cs +++ b/NBXplorer/Controllers/PostgresMainController.cs @@ -7,6 +7,7 @@ using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using NBitcoin; +using NBitcoin.DataEncoders; using NBitcoin.RPC; using NBXplorer.Backends.Postgres; using NBXplorer.DerivationStrategy; @@ -97,12 +98,14 @@ await repo.AssociateScriptsToWalletExplicitly(trackedSourceContext.TrackedSource [TrackedSourceContext.TrackedSourceContextRequirement(true)] public async Task ImportUTXOs( TrackedSourceContext trackedSourceContext, [FromBody] JArray rawRequest) { + + var repo = (PostgresRepository)trackedSourceContext.Repository; + await repo.EnsureWalletCreated(trackedSourceContext.TrackedSource); var jsonSerializer = JsonSerializer.Create(trackedSourceContext.Network.JsonSerializerSettings); var coins = rawRequest.ToObject(jsonSerializer)?.Where(c => c.Coin != null).ToArray(); if (coins?.Any() is not true) throw new ArgumentNullException(nameof(coins)); - var repo = (PostgresRepository)trackedSourceContext.Repository; var rpc = trackedSourceContext.RpcClient; var clientBatch = rpc.PrepareBatch(); @@ -123,13 +126,8 @@ await Task.WhenAll(coins.SelectMany(o => { if (o.Proof is not null && o.Proof.PartialMerkleTree.Hashes.Contains(o.Coin.Outpoint.Hash)) { - // var merkleBLockProofBytes = Encoders.Hex.DecodeData(o.TxOutProof); - // var mb = new MerkleBlock(); - // mb.FromBytes(merkleBLockProofBytes); - // mb.ReadWrite(merkleBLockProofBytes, network.NBitcoinNetwork); - var txoutproofResult = - await clientBatch.SendCommandAsync("verifytxoutproof", o.Proof); + await clientBatch.SendCommandAsync("verifytxoutproof", Encoders.Hex.EncodeData(o.Proof.ToBytes())); var txHash = o.Coin.Outpoint.Hash.ToString(); if (txoutproofResult.Error is not null && txoutproofResult.Result is JArray prooftxs && @@ -145,6 +143,13 @@ await Task.WhenAll(coins.SelectMany(o => }).Concat(new[] {clientBatch.SendBatchAsync()}).ToArray()); DateTimeOffset now = DateTimeOffset.UtcNow; + + var scripts = coinToTxOut + .Select(pair => + pair.Key.ScriptPubKey.GetDestinationAddress(trackedSourceContext.Network.NBitcoinNetwork)) + .Where(address => address is not null).ToDictionary(address => (IDestination)address, _ => true); + + await repo.AssociateScriptsToWalletExplicitly(trackedSourceContext.TrackedSource,scripts); await repo.SaveMatches(coinToTxOut.Select(pair => { coinToBlock.TryGetValue(pair.Key, out var blockHeader); @@ -152,6 +157,7 @@ await repo.SaveMatches(coinToTxOut.Select(pair => new TrackedTransactionKey(pair.Key.Outpoint.Hash, blockHeader?.GetHash(), true){}, new[] {pair.Key}, null); ttx.Inserted = now; + ttx.Immature = pair.Value.IsCoinBase && pair.Value.Confirmations <= 100; ttx.FirstSeen = blockHeader?.BlockTime?? NBitcoin.Utils.UnixTimeToDateTime(0);; return ttx; }).ToArray()); From 1d6d6c8f3a728286358cb37d2e36063e9dbc92ed Mon Sep 17 00:00:00 2001 From: Kukks Date: Thu, 30 Nov 2023 09:19:38 +0100 Subject: [PATCH 11/16] make associate handle metadata and import features --- NBXplorer.Client/ExplorerClient.cs | 2 +- NBXplorer.Client/Models/ImportUTXORequest.cs | 8 ++++ NBXplorer.Tests/UnitTest1.cs | 24 +++++----- .../Backends/Postgres/PostgresRepository.cs | 47 ++++++++++++------- .../Controllers/PostgresMainController.cs | 23 +++++---- 5 files changed, 64 insertions(+), 40 deletions(-) diff --git a/NBXplorer.Client/ExplorerClient.cs b/NBXplorer.Client/ExplorerClient.cs index ef2025fa3..0ad44ae40 100644 --- a/NBXplorer.Client/ExplorerClient.cs +++ b/NBXplorer.Client/ExplorerClient.cs @@ -390,7 +390,7 @@ public Task GetTransactionAsync(TrackedSource trackedSou throw new ArgumentNullException(nameof(trackedSource)); return SendAsync(HttpMethod.Get, null, $"{GetBasePath(trackedSource)}/transactions/{txId}", cancellation); } - public async Task AssociateScriptsAsync(TrackedSource trackedSource, Dictionary scripts, CancellationToken cancellation = default) + public async Task AssociateScriptsAsync(TrackedSource trackedSource, AssociateScriptRequest[] scripts, CancellationToken cancellation = default) { if (scripts == null) throw new ArgumentNullException(nameof(scripts)); diff --git a/NBXplorer.Client/Models/ImportUTXORequest.cs b/NBXplorer.Client/Models/ImportUTXORequest.cs index ce53edbaf..aa7208185 100644 --- a/NBXplorer.Client/Models/ImportUTXORequest.cs +++ b/NBXplorer.Client/Models/ImportUTXORequest.cs @@ -1,4 +1,5 @@ using NBitcoin; +using Newtonsoft.Json.Linq; namespace NBXplorer.Models; @@ -6,4 +7,11 @@ public class ImportUTXORequest { public Coin Coin { get; set; } public MerkleBlock Proof { get; set; } +} + +public class AssociateScriptRequest +{ + public IDestination Destination { get; set; } + public bool Used { get; set; } + public JObject Metadata { get; set; } } \ No newline at end of file diff --git a/NBXplorer.Tests/UnitTest1.cs b/NBXplorer.Tests/UnitTest1.cs index 527835905..bc2527a9e 100644 --- a/NBXplorer.Tests/UnitTest1.cs +++ b/NBXplorer.Tests/UnitTest1.cs @@ -4574,7 +4574,7 @@ await Assert.ThrowsAsync(async () =>await tester.Client.Gene ParentWallet = parentWalletTS })); - await Assert.ThrowsAsync(async () =>await tester.Client.AssociateScriptsAsync(parentWalletTS, new Dictionary())); + await Assert.ThrowsAsync(async () =>await tester.Client.AssociateScriptsAsync(parentWalletTS, Array.Empty())); await Assert.ThrowsAsync(async () =>await tester.Client.ImportUTXOs(parentWalletTS, Array.Empty())); return; } @@ -4609,9 +4609,12 @@ await Assert.ThrowsAsync(async () =>await tester.Client.Gene var udetectedTxId = await tester.RPC.SendToAddressAsync(newAddr, Money.FromUnit(0.1m, MoneyUnit.BTC)); await Task.Delay(3000); var utxos = Assert.Single(await tester.RPC.ListUnspentAsync(0, 0, newAddr)); - await tester.Client.AssociateScriptsAsync(wallet1TS, new Dictionary() + await tester.Client.AssociateScriptsAsync(wallet1TS, new[] { - {newAddr2, true} + new AssociateScriptRequest() + { + Destination = newAddr2, Used = true + } }); @@ -4632,12 +4635,15 @@ await Eventually(async () => }); - await tester.Client.AssociateScriptsAsync(wallet1TS, new Dictionary() + await tester.Client.AssociateScriptsAsync(wallet1TS, new[] { - {newAddr, true} + new AssociateScriptRequest() + { + Destination = newAddr, Used = true + } }); - await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest[] + await tester.Client.ImportUTXOs(wallet1TS, new[] { new ImportUTXORequest() { @@ -4866,12 +4872,6 @@ await Eventually(async () => Assert.NotEqual( NBitcoin.Utils.UnixTimeToDateTime(0) , importedUtxoWithProof.Timestamp); }); - - - - - - } } } diff --git a/NBXplorer/Backends/Postgres/PostgresRepository.cs b/NBXplorer/Backends/Postgres/PostgresRepository.cs index c192b8750..4c9bb1160 100644 --- a/NBXplorer/Backends/Postgres/PostgresRepository.cs +++ b/NBXplorer/Backends/Postgres/PostgresRepository.cs @@ -270,28 +270,36 @@ internal TrackedSource GetTrackedSource(WalletKey walletKey) } public async Task AssociateScriptsToWalletExplicitly(TrackedSource trackedSource, - Dictionary scripts) + AssociateScriptRequest[] scripts) { var walletKey = GetWalletKey(trackedSource); - await using var conn = await GetConnection(); + + var importMode = GetImportRPCMode(conn, walletKey); var scriptsRecords = scripts.Select(pair => new ScriptInsert(this.Network.CryptoCode, walletKey.wid, - pair.Key.ScriptPubKey.ToHex(), pair.Key.ToString(), pair.Value)).ToArray(); + pair.Destination.ScriptPubKey.ToHex(), pair.Destination.ToString(), pair.Used)).ToArray(); { - - await conn.Connection.ExecuteAsync( - - "INSERT INTO wallets VALUES (@wid, @metadata::JSONB) ON CONFLICT DO NOTHING", new - { - walletKey.wid, - walletKey.metadata - }); + await conn.Connection.ExecuteAsync(WalletInsertQuery, new {walletKey.wid, walletKey.metadata}); await conn.Connection.ExecuteAsync( "INSERT INTO scripts (code, script, addr, used) VALUES(@code, @script, @addr, @used) ON CONFLICT DO NOTHING;" + - "INSERT INTO wallets_scripts (code, wallet_id, script) VALUES(@code, @wallet_id, @script) ON CONFLICT DO NOTHING;" + "INSERT INTO wallets_scripts (code, wallet_id, script) VALUES(@code, @wallet_id, @script) ON CONFLICT DO NOTHING;" , scriptsRecords); + var descriptScriptInsert = scripts.Where(request => request.Metadata is not null).Select(request => + new DescriptorScriptInsert(request.Destination.ScriptPubKey.ToHex(), 0, + request.Destination.ScriptPubKey.ToHex(), request.Metadata.ToString(Formatting.None), + request.Destination.ToString(), request.Used)).ToList(); + await InsertDescriptorsScripts(conn.Connection, descriptScriptInsert); + if (ImportRPCMode.Legacy == await importMode) + { + foreach (var scriptsRecord in scriptsRecords) + { + await ImportAddressToRPC(null, + BitcoinAddress.Create(scriptsRecord.addr, Network.NBitcoinNetwork), null); + } + } } } + internal record ScriptInsert(string code, string wallet_id, string script, string addr, bool used); internal record DescriptorScriptInsert(string descriptor, int idx, string script, string metadata, string addr, bool used); public async Task GenerateAddresses(DerivationStrategyBase strategy, DerivationFeature derivationFeature, GenerateAddressQuery query = null) @@ -1021,10 +1029,14 @@ await connection.ExecuteAsync( "INSERT INTO wallets_scripts VALUES (@code, @script, @walletid) ON CONFLICT DO NOTHING;", inserts); } + private async Task GetImportRPCMode(DbConnectionHelper connection, WalletKey walletKey) + { + return ImportRPCMode.Parse((await connection.GetMetadata(walletKey.wid, WellknownMetadataKeys.ImportAddressToRPC))); + } private async Task ImportAddressToRPC(DbConnectionHelper connection, TrackedSource trackedSource, BitcoinAddress address, KeyPath keyPath) { var k = GetWalletKey(trackedSource); - var shouldImportRPC = ImportRPCMode.Parse((await connection.GetMetadata(k.wid, WellknownMetadataKeys.ImportAddressToRPC))); + var shouldImportRPC = await GetImportRPCMode(connection, k); if (shouldImportRPC != ImportRPCMode.Legacy) return; var accountKey = await connection.GetMetadata(k.wid, WellknownMetadataKeys.AccountHDKey); @@ -1243,7 +1255,7 @@ public async Task Track(IDestination address) await using var conn = await GetConnection(); var walletKey = GetWalletKey(address); await conn.Connection.ExecuteAsync( - "INSERT INTO wallets VALUES (@wid, @metadata::JSONB) ON CONFLICT DO NOTHING;" + + WalletInsertQuery + "INSERT INTO scripts VALUES (@code, @script, @addr) ON CONFLICT DO NOTHING;" + "INSERT INTO wallets_scripts VALUES (@code, @script, @wid) ON CONFLICT DO NOTHING" , new { code = Network.CryptoCode, script = address.ScriptPubKey.ToHex(), addr = address.ScriptPubKey.GetDestinationAddress(Network.NBitcoinNetwork).ToString(), walletKey.wid, walletKey.metadata }); @@ -1349,14 +1361,15 @@ public async Task EnsureWalletCreated(WalletKey walletKey, WalletKey[] parentWal var parentsRecords = parentWallets.Select(key => new WalletHierarchyInsert(walletKey.wid, key.wid)).ToArray(); - await connection.ExecuteAsync( - "INSERT INTO wallets (wallet_id, metadata) VALUES (@wid, @metadata::JSONB) ON CONFLICT DO NOTHING" - , walletRecords); + await connection.ExecuteAsync(WalletInsertQuery, walletRecords); await connection.ExecuteAsync( "INSERT INTO wallets_wallets (wallet_id, parent_id) VALUES (@child, @parent)ON CONFLICT DO NOTHING" , parentsRecords); } + + private readonly string WalletInsertQuery = "INSERT INTO wallets (wallet_id, metadata) VALUES (@wid, @metadata::JSONB) ON CONFLICT DO NOTHING;"; } + public class LegacyDescriptorMetadata { diff --git a/NBXplorer/Controllers/PostgresMainController.cs b/NBXplorer/Controllers/PostgresMainController.cs index 1354e40e0..abcd532f6 100644 --- a/NBXplorer/Controllers/PostgresMainController.cs +++ b/NBXplorer/Controllers/PostgresMainController.cs @@ -84,13 +84,12 @@ public async Task GetBalance( TrackedSourceContext trackedSourceC [HttpPost("associate")] [PostgresImplementationActionConstraint(true)] - public async Task AssociateScripts( TrackedSourceContext trackedSourceContext, - [FromBody] Dictionary scripts) + public async Task AssociateScripts( TrackedSourceContext trackedSourceContext, [FromBody] JArray rawRequest) { var repo = (PostgresRepository)trackedSourceContext.Repository; - await repo.AssociateScriptsToWalletExplicitly(trackedSourceContext.TrackedSource, - scripts.ToDictionary(pair => (IDestination) BitcoinAddress.Create(pair.Key, trackedSourceContext.Network.NBitcoinNetwork), - pair => pair.Value)); + var jsonSerializer = JsonSerializer.Create(trackedSourceContext.Network.JsonSerializerSettings); + var requests = rawRequest.ToObject(jsonSerializer); + await repo.AssociateScriptsToWalletExplicitly(trackedSourceContext.TrackedSource, requests); return Ok(); } @@ -100,7 +99,6 @@ public async Task ImportUTXOs( TrackedSourceContext trackedSource { var repo = (PostgresRepository)trackedSourceContext.Repository; - await repo.EnsureWalletCreated(trackedSourceContext.TrackedSource); var jsonSerializer = JsonSerializer.Create(trackedSourceContext.Network.JsonSerializerSettings); var coins = rawRequest.ToObject(jsonSerializer)?.Where(c => c.Coin != null).ToArray(); if (coins?.Any() is not true) @@ -142,12 +140,17 @@ await Task.WhenAll(coins.SelectMany(o => }; }).Concat(new[] {clientBatch.SendBatchAsync()}).ToArray()); - DateTimeOffset now = DateTimeOffset.UtcNow; + var now = DateTimeOffset.UtcNow; var scripts = coinToTxOut - .Select(pair => - pair.Key.ScriptPubKey.GetDestinationAddress(trackedSourceContext.Network.NBitcoinNetwork)) - .Where(address => address is not null).ToDictionary(address => (IDestination)address, _ => true); + .Select(pair => ( + pair.Key.ScriptPubKey.GetDestinationAddress(trackedSourceContext.Network.NBitcoinNetwork), pair)) + .Where(pair => pair.Item1 is not null).Select(tuple => new AssociateScriptRequest() + { + Destination = tuple.Item1, + Used = tuple.pair.Value is not null, + Metadata = null + }).ToArray(); await repo.AssociateScriptsToWalletExplicitly(trackedSourceContext.TrackedSource,scripts); await repo.SaveMatches(coinToTxOut.Select(pair => From e6f1deb9ff21e784382b3404b9c4a14ffa379da9 Mon Sep 17 00:00:00 2001 From: Kukks Date: Thu, 30 Nov 2023 09:39:13 +0100 Subject: [PATCH 12/16] add docs --- .../Controllers/PostgresMainController.cs | 2 - docs/API.md | 137 ++++++++++++++++-- 2 files changed, 125 insertions(+), 14 deletions(-) diff --git a/NBXplorer/Controllers/PostgresMainController.cs b/NBXplorer/Controllers/PostgresMainController.cs index abcd532f6..9ff36dae4 100644 --- a/NBXplorer/Controllers/PostgresMainController.cs +++ b/NBXplorer/Controllers/PostgresMainController.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Concurrent; -using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Dapper; @@ -14,7 +13,6 @@ using NBXplorer.Models; using Newtonsoft.Json; using Newtonsoft.Json.Linq; -using Newtonsoft.Json.Serialization; namespace NBXplorer.Controllers { diff --git a/docs/API.md b/docs/API.md index 47be9c90a..57c2df3ed 100644 --- a/docs/API.md +++ b/docs/API.md @@ -9,6 +9,7 @@ NBXplorer does not index the whole blockchain, rather, it listens transactions a * [Configuration](#configuration) * [Authentication](#authentication) * [Tracked Sources](#tracked-source) +* [Child Wallets](#child-wallets) * [Derivation Scheme Format](#derivationScheme) * [Tracking a Derivation Scheme](#track) * [Track a specific address](#address) @@ -39,6 +40,7 @@ NBXplorer does not index the whole blockchain, rather, it listens transactions a * [Node RPC Proxy](#rpc-proxy) * [Health check](#health) * [Liquid integration](#liquid) +* [Hierarchy APIs](#hierarchy) ## Configuration @@ -107,6 +109,7 @@ While each of these has its own logic around what scripts to follow, they all al When using Postgres, the feature of child wallets allows you to link a tracked source as a child of another tracked source. Every script generated by a child wallet will be tracked as part of its parent wallet, including all related UTXOs and transactions. A parent can have an unlimited number of child wallets, and child wallets themselves can act as parents to other wallets. +You can manage and view the hierarchy of tracked sources using the [hierarchy APIs](#hierarchy). ## Derivation Scheme Format @@ -206,8 +209,8 @@ Optionally, you can attach a json body: To query all transactions of a `tracked source`: -`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/transactions` -`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/transactions` +`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/transactions`
+`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/transactions`
`HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/transactions` Optional Parameters: @@ -313,8 +316,8 @@ Note that the list of confirmed transaction also include immature transactions. ## Query a single transaction associated to a tracked source -`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/transactions/{txId}` -`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/transactions/{txId}` +`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/transactions/{txId}`
+`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/transactions/{txId}`
`HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/transactions/{txId}` Error codes: @@ -349,8 +352,8 @@ Returns: ## Get current balance -`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/balance` -`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/balance` +`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/balance`
+`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/balance`
`HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/balance` Returns: @@ -474,9 +477,9 @@ Note: `redeem` is returning the segwit redeem if the derivation scheme is a P2SH ## Get scriptPubKey information of a tracked source -`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/scripts/{script}` -`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/scripts/{script}` -`HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/scripts/{script}` +`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/scripts/{script}`
+`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/scripts/{script}`
+`HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/scripts/{script}`
`HTTP GET v1/cryptos/{cryptoCode}/tracked-sources/{trackedSource}/scripts/{script}` Error codes: @@ -497,9 +500,9 @@ Returns: ## Get available Unspent Transaction Outputs (UTXOs) -`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/utxos` -`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/utxos` -`HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/utxos` +`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/utxos`
+`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/utxos`
+`HTTP GET v1/cryptos/{cryptoCode}/wallets/{walletId}/utxos`
`HTTP GET v1/cryptos/{cryptoCode}/tracked-sources/{trackedSource}/utxos` Error: @@ -1298,6 +1301,115 @@ Response: NOTE: Batch commands are also supported by sending the JSON-RPC requests in an array. The result is also returned in an array. + +## Hierarchy APIs + +Note: These APIs are only available for Postgres. +### View wallet parents + +`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/parents`
+`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/parents`
+`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/parents`
+`HTTP GET v1/cryptos/{cryptoCode}/wallets/{wallet}/parents`
+`HTTP GET v1/cryptos/{cryptoCode}/tracked-sources/{trackedSource}/parents` + +Response: an array of tracked sources + +```json +[ + "WALLET:xyz", + "DERIVATIONSCHEME:xpub...", + "ADDRESS:xyz" +] +``` + + +### Add wallet parent + +`HTTP POST v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/parents`
+`HTTP POST v1/cryptos/{cryptoCode}/addresses/{address}/parents`
+`HTTP POST v1/cryptos/{cryptoCode}/addresses/{address}/parents`
+`HTTP POST v1/cryptos/{cryptoCode}/wallets/{wallet}/parents`
+`HTTP POST v1/cryptos/{cryptoCode}/tracked-sources/{trackedSource}/parents` + +Request: a json string of the tracked source to add + +```json +"WALLET:xyz" +``` + +No response body + + +### Remove wallet parent + +`HTTP DELETE v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/parents`
+`HTTP DELETE v1/cryptos/{cryptoCode}/addresses/{address}/parents`
+`HTTP DELETE v1/cryptos/{cryptoCode}/addresses/{address}/parents`
+`HTTP DELETE v1/cryptos/{cryptoCode}/wallets/{wallet}/parents`
+`HTTP DELETE v1/cryptos/{cryptoCode}/tracked-sources/{trackedSource}/parents` + +Request: a json string of the tracked source to remove + +```json +"WALLET:xyz" +``` + +No response body + + + +### View wallet children + +`HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/children`
+`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/children`
+`HTTP GET v1/cryptos/{cryptoCode}/addresses/{address}/children`
+`HTTP GET v1/cryptos/{cryptoCode}/wallets/{wallet}/children`
+`HTTP GET v1/cryptos/{cryptoCode}/tracked-sources/{trackedSource}/children` + +Response: an array of tracked sources + +```json +[ + "WALLET:xyz", + "DERIVATIONSCHEME:xpub...", + "ADDRESS:xyz" +] +``` + +### Add wallet child + +`HTTP POST v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/children`
+`HTTP POST v1/cryptos/{cryptoCode}/addresses/{address}/children`
+`HTTP POST v1/cryptos/{cryptoCode}/addresses/{address}/children`
+`HTTP POST v1/cryptos/{cryptoCode}/wallets/{wallet}/children`
+`HTTP POST v1/cryptos/{cryptoCode}/tracked-sources/{trackedSource}/children` + +Request: a json string of the tracked source to add + +```json +"WALLET:xyz" +``` + +No response body + +### Remove wallet child + +`HTTP DELETE v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/children`
+`HTTP DELETE v1/cryptos/{cryptoCode}/addresses/{address}/children`
+`HTTP DELETE v1/cryptos/{cryptoCode}/addresses/{address}/children`
+`HTTP DELETE v1/cryptos/{cryptoCode}/wallets/{wallet}/children`
+`HTTP DELETE v1/cryptos/{cryptoCode}/tracked-sources/{trackedSource}/children` + +Request: a json string of the tracked source to remove + +```json +"WALLET:xyz" +``` + +No response body + + ## Health check A endpoint that can be used without the need for [authentication](#auth) which will returns HTTP 200 only if all nodes connected to NBXplorer are ready. @@ -1347,3 +1459,4 @@ In order to send in and out of liquid, we advise you to rely on the RPC command For doing this you need to [Generate a wallet](#wallet) with `importAddressToRPC` and `savePrivateKeys` set to `true`. Be careful to not expose your NBXplorer server on internet, your private keys can be [retrieved trivially](#getmetadata). + From acc8fe29491d0b206dce13e5b6012f5137a00aaa Mon Sep 17 00:00:00 2001 From: Kukks Date: Thu, 30 Nov 2023 11:06:09 +0100 Subject: [PATCH 13/16] updates --- NBXplorer.Client/Models/ImportUTXORequest.cs | 4 +- NBXplorer.Tests/UnitTest1.cs | 14 ++--- .../Backends/Postgres/PostgresRepository.cs | 37 +++++++----- .../Controllers/PostgresMainController.cs | 24 ++++---- NBXplorer/Extensions.cs | 56 ++++++++++++------ docs/API.md | 58 +++++++++++++++++++ 6 files changed, 142 insertions(+), 51 deletions(-) diff --git a/NBXplorer.Client/Models/ImportUTXORequest.cs b/NBXplorer.Client/Models/ImportUTXORequest.cs index aa7208185..4d3ce57e8 100644 --- a/NBXplorer.Client/Models/ImportUTXORequest.cs +++ b/NBXplorer.Client/Models/ImportUTXORequest.cs @@ -5,8 +5,10 @@ namespace NBXplorer.Models; public class ImportUTXORequest { - public Coin Coin { get; set; } + public OutPoint Utxo { get; set; } + public MerkleBlock Proof { get; set; } + } public class AssociateScriptRequest diff --git a/NBXplorer.Tests/UnitTest1.cs b/NBXplorer.Tests/UnitTest1.cs index bc2527a9e..0a1e40b99 100644 --- a/NBXplorer.Tests/UnitTest1.cs +++ b/NBXplorer.Tests/UnitTest1.cs @@ -4647,7 +4647,7 @@ await tester.Client.ImportUTXOs(wallet1TS, new[] { new ImportUTXORequest() { - Coin = utxos.AsCoin(), + Utxo = utxos.OutPoint, Proof = null } }); @@ -4763,7 +4763,7 @@ public async Task CanImportUTXOs(Backend backend) { new() { - Coin = new Coin(utxo), + Utxo = utxo.ToCoin().Outpoint, Proof = null } }); @@ -4784,7 +4784,7 @@ await Eventually(async () => { new() { - Coin = new Coin(utxo2), + Utxo = utxo2.ToCoin().Outpoint, Proof = null } }); @@ -4821,15 +4821,15 @@ await Eventually(async () => { new() { - Coin = fakeUtxo, + Utxo = fakeUtxo.Outpoint }, new() { - Coin = new Coin(tospendutxo) + Utxo = new Coin(tospendutxo).Outpoint }, new() { - Coin = new Coin(validScriptUtxo) + Utxo = new Coin(validScriptUtxo).Outpoint } }); @@ -4859,7 +4859,7 @@ await Eventually(async () => { new() { - Coin = new Coin(yoUtxo), + Utxo = new Coin(yoUtxo).Outpoint, Proof = mb } }); diff --git a/NBXplorer/Backends/Postgres/PostgresRepository.cs b/NBXplorer/Backends/Postgres/PostgresRepository.cs index 4c9bb1160..1b7295763 100644 --- a/NBXplorer/Backends/Postgres/PostgresRepository.cs +++ b/NBXplorer/Backends/Postgres/PostgresRepository.cs @@ -530,6 +530,7 @@ FROM unnest(@records) AS r (script), SELECT code, script, wallet_id, addr, descriptor_metadata->>'derivation' derivation, keypath, descriptors_scripts_metadata->>'redeem' redeem, descriptors_scripts_metadata->>'blindedAddress' blinded_addr, + descriptors_scripts_metadata->>'blindingKey' blindingKey, descriptor_metadata->>'descriptor' descriptor FROM nbxv1_keypath_info ki WHERE ki.code=@code AND ki.script=r.script @@ -551,23 +552,31 @@ JOIN wallets w USING(wallet_id)", string walletId = r.wallet_id; var trackedSource = derivationStrategy is not null - ? - new DerivationSchemeTrackedSource(derivationStrategy) - : walletType == "Wallet" ? walletId is null ? (TrackedSource)null : new WalletTrackedSource(walletId) : new AddressTrackedSource(addr); - - result.Add(script, new KeyPathInformation() - { - Address = addr, - DerivationStrategy = r.derivation is not null ? derivationStrategy : null, - KeyPath = keypath, - ScriptPubKey = script, - TrackedSource = trackedSource, - Feature = keypath is null ? DerivationFeature.Deposit : KeyPathTemplates.GetDerivationFeature(keypath), - Redeem = redeem is null ? null : Script.FromHex(redeem), - }); + ? new DerivationSchemeTrackedSource(derivationStrategy) + : walletType == "Wallet" + ? walletId is null ? (TrackedSource) null : new WalletTrackedSource(walletId) + : new AddressTrackedSource(addr); + var keyPathInformation = Network.IsElement && r.blindingKey is not null + ? new LiquidKeyPathInformation() + { + BlindingKey = Key.Parse(r.blindingKey, Network.NBitcoinNetwork) + } + : new KeyPathInformation(); + keyPathInformation.Address = addr; + keyPathInformation.DerivationStrategy = r.derivation is not null ? derivationStrategy : null; + keyPathInformation.KeyPath = keypath; + keyPathInformation.ScriptPubKey = script; + keyPathInformation.TrackedSource = trackedSource; + keyPathInformation.Feature = keypath is null ? DerivationFeature.Deposit : KeyPathTemplates.GetDerivationFeature(keypath); + keyPathInformation.Redeem = redeem is null ? null : Script.FromHex(redeem); + result.Add(script, keyPathInformation); } return result; } + public class LiquidKeyPathInformation : KeyPathInformation + { + public Key BlindingKey { get; set; } + } private BitcoinAddress GetAddress(dynamic r) { diff --git a/NBXplorer/Controllers/PostgresMainController.cs b/NBXplorer/Controllers/PostgresMainController.cs index 9ff36dae4..218e7f6c4 100644 --- a/NBXplorer/Controllers/PostgresMainController.cs +++ b/NBXplorer/Controllers/PostgresMainController.cs @@ -98,15 +98,15 @@ public async Task ImportUTXOs( TrackedSourceContext trackedSource var repo = (PostgresRepository)trackedSourceContext.Repository; var jsonSerializer = JsonSerializer.Create(trackedSourceContext.Network.JsonSerializerSettings); - var coins = rawRequest.ToObject(jsonSerializer)?.Where(c => c.Coin != null).ToArray(); + var coins = rawRequest.ToObject(jsonSerializer)?.Where(c => c.Utxo != null).ToArray(); if (coins?.Any() is not true) throw new ArgumentNullException(nameof(coins)); var rpc = trackedSourceContext.RpcClient; var clientBatch = rpc.PrepareBatch(); - var coinToTxOut = new ConcurrentDictionary(); - var coinToBlock = new ConcurrentDictionary(); + var coinToTxOut = new ConcurrentDictionary(); + var coinToBlock = new ConcurrentDictionary(); await Task.WhenAll(coins.SelectMany(o => { return new[] @@ -114,24 +114,24 @@ await Task.WhenAll(coins.SelectMany(o => Task.Run(async () => { var txOutResponse = - await clientBatch.GetTxOutAsync(o.Coin.Outpoint.Hash, (int) o.Coin.Outpoint.N); + await clientBatch.GetTxOutAsync(o.Utxo.Hash, (int) o.Utxo.N); if (txOutResponse is not null) - coinToTxOut.TryAdd(o.Coin, txOutResponse); + coinToTxOut.TryAdd(o.Utxo, txOutResponse); }), Task.Run(async () => { - if (o.Proof is not null && o.Proof.PartialMerkleTree.Hashes.Contains(o.Coin.Outpoint.Hash)) + if (o.Proof is not null && o.Proof.PartialMerkleTree.Hashes.Contains(o.Utxo.Hash)) { var txoutproofResult = await clientBatch.SendCommandAsync("verifytxoutproof", Encoders.Hex.EncodeData(o.Proof.ToBytes())); - var txHash = o.Coin.Outpoint.Hash.ToString(); + var txHash = o.Utxo.Hash.ToString(); if (txoutproofResult.Error is not null && txoutproofResult.Result is JArray prooftxs && prooftxs.Any(token => token.Value() ?.Equals(txHash, StringComparison.InvariantCultureIgnoreCase) is true)) { - coinToBlock.TryAdd(o.Coin, o.Proof.Header); + coinToBlock.TryAdd(o.Utxo, o.Proof.Header); } } }) @@ -142,7 +142,7 @@ await Task.WhenAll(coins.SelectMany(o => var scripts = coinToTxOut .Select(pair => ( - pair.Key.ScriptPubKey.GetDestinationAddress(trackedSourceContext.Network.NBitcoinNetwork), pair)) + pair.Value.TxOut.ScriptPubKey.GetDestinationAddress(repo.Network.NBitcoinNetwork), pair)) .Where(pair => pair.Item1 is not null).Select(tuple => new AssociateScriptRequest() { Destination = tuple.Item1, @@ -154,9 +154,11 @@ await Task.WhenAll(coins.SelectMany(o => await repo.SaveMatches(coinToTxOut.Select(pair => { coinToBlock.TryGetValue(pair.Key, out var blockHeader); + + var coin = new Coin(pair.Key, pair.Value.TxOut); var ttx = repo.CreateTrackedTransaction(trackedSourceContext.TrackedSource, - new TrackedTransactionKey(pair.Key.Outpoint.Hash, blockHeader?.GetHash(), true){}, - new[] {pair.Key}, null); + new TrackedTransactionKey(pair.Key.Hash, blockHeader?.GetHash(), true){}, + new[] {coin}, null); ttx.Inserted = now; ttx.Immature = pair.Value.IsCoinBase && pair.Value.Confirmations <= 100; ttx.FirstSeen = blockHeader?.BlockTime?? NBitcoin.Utils.UnixTimeToDateTime(0);; diff --git a/NBXplorer/Extensions.cs b/NBXplorer/Extensions.cs index a3df78380..a0af35d56 100644 --- a/NBXplorer/Extensions.cs +++ b/NBXplorer/Extensions.cs @@ -97,27 +97,47 @@ public static T As(this IActionResult actionResult) return default; } - public static async Task UnblindTransaction(this RPCClient rpc, TrackedTransaction tx, IEnumerable keyInfos) + public static async Task UnblindTransaction(this RPCClient rpc, TrackedTransaction tx, + IEnumerable keyInfos) { - if (tx.TrackedSource is DerivationSchemeTrackedSource ts && - !ts.DerivationStrategy.Unblinded() && - tx.Transaction is ElementsTransaction elementsTransaction) - { - var keys = keyInfos - .Select(kv => (KeyPath: kv.KeyPath, - Address: kv.Address as BitcoinBlindedAddress, - BlindingKey: NBXplorerNetworkProvider.LiquidNBXplorerNetwork.GenerateBlindingKey(ts.DerivationStrategy, kv.KeyPath, kv.ScriptPubKey, rpc.Network))) - .Where(o => o.Address != null) - .Select(o => new UnblindTransactionBlindingAddressKey() - { - Address = o.Address, - BlindingKey = o.BlindingKey - }).ToList(); - if (keys.Count != 0) + var elementsTransaction = tx.Transaction as ElementsTransaction; + if (elementsTransaction == null) + return null; + + var keys = keyInfos + .Select(kv => (KeyPath: kv.KeyPath, + Address: kv.Address as BitcoinBlindedAddress, + BlindingKey: GetBlindingKey(tx.TrackedSource, kv, rpc.Network))) + .Where(o => o.Address != null && o.BlindingKey != null) + .Select(o => new UnblindTransactionBlindingAddressKey() { - return await rpc.UnblindTransaction(keys, elementsTransaction, rpc.Network); - } + Address = o.Address, + BlindingKey = o.BlindingKey + }).ToList(); + if (keys.Count != 0) + { + return await rpc.UnblindTransaction(keys, elementsTransaction, rpc.Network); } + + return null; + } + + private static Key GetBlindingKey(TrackedSource trackedSource, KeyPathInformation keyPathInformation, Network network) + { + if(keyPathInformation.Address is not BitcoinBlindedAddress) + { + return null; + } + if (keyPathInformation is PostgresRepository.LiquidKeyPathInformation {BlindingKey: not null} liquidKeyPathInformation) + { + return liquidKeyPathInformation.BlindingKey; + } + if(trackedSource is DerivationSchemeTrackedSource derivationSchemeTrackedSource && !derivationSchemeTrackedSource.DerivationStrategy.Unblinded()) + { + return NBXplorerNetworkProvider.LiquidNBXplorerNetwork.GenerateBlindingKey( + derivationSchemeTrackedSource.DerivationStrategy, keyPathInformation.KeyPath, keyPathInformation.ScriptPubKey, network); + } + return null; } public static async Task GetBlockTimeAsync(this RPCClient client, uint256 blockId, bool throwIfNotFound = true) diff --git a/docs/API.md b/docs/API.md index 57c2df3ed..34f4b668a 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1301,10 +1301,68 @@ Response: NOTE: Batch commands are also supported by sending the JSON-RPC requests in an array. The result is also returned in an array. +## Associate scripts to wallet + +Note: This API is only available for Postgres. + +`HTTP POST v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/associate`
+`HTTP POST v1/cryptos/{cryptoCode}/addresses/{address}/associate`
+`HTTP POST v1/cryptos/{cryptoCode}/addresses/{address}/associate`
+`HTTP POST v1/cryptos/{cryptoCode}/wallets/{wallet}/associate`
+`HTTP POST v1/cryptos/{cryptoCode}/tracked-sources/{trackedSource}/associate` + +Request: + + +* `destinatiom`: Mandatory, the address to add to the wallet +* `used`: Optional, indicate that the address was used already (default: `false`) +* `metadata`: Optional, a json object that will be attached to the script as part of the descriptor (default: null) (note: for liquid, you will want to use the properties `blinded_address` and `blindingKey` to handle confidential addresses) + +```json +[ + { + "destinatiom": "bc1q...", + "used": true, + "metadata": { ...} + } +] +``` + + +## Import UTXOs to wallet + +Note: This API is only available for Postgres. + +In the case where you start tracking a wallet that already has UTXOs, you can import them to NBXplorer so that it can be aware of them. NBXplorer will validate against the bitcoin node's utxoset that the UTXOs are valid. + +`HTTP POST v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/import-utxos`
+`HTTP POST v1/cryptos/{cryptoCode}/addresses/{address}/import-utxos`
+`HTTP POST v1/cryptos/{cryptoCode}/addresses/{address}/import-utxos`
+`HTTP POST v1/cryptos/{cryptoCode}/wallets/{wallet}/import-utxos`
+`HTTP POST v1/cryptos/{cryptoCode}/tracked-sources/{trackedSource}/import-utxos` + +Request: + + +* `utxo`: Mandatory, the utxo to import, in the format `txid-vout`, where `txid` is the transaction id and `vout` is the output index +* `proof`: Optional, a merkle proof that the utxo being imported was included in a block (as provided by Bitcoin Core's gettxoutproof) + +```json +[ + { + "utxo": "txid-vout", + "proof": {} + } +] +``` + +No response body + ## Hierarchy APIs Note: These APIs are only available for Postgres. + ### View wallet parents `HTTP GET v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/parents`
From 249010e16aecbd2cd549a897162b8b482bdf60b1 Mon Sep 17 00:00:00 2001 From: Kukks Date: Thu, 30 Nov 2023 13:26:21 +0100 Subject: [PATCH 14/16] fix conn conflict --- NBXplorer/Backends/Postgres/PostgresRepository.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NBXplorer/Backends/Postgres/PostgresRepository.cs b/NBXplorer/Backends/Postgres/PostgresRepository.cs index 1b7295763..ff5f5d792 100644 --- a/NBXplorer/Backends/Postgres/PostgresRepository.cs +++ b/NBXplorer/Backends/Postgres/PostgresRepository.cs @@ -275,7 +275,6 @@ public async Task AssociateScriptsToWalletExplicitly(TrackedSource trackedSource var walletKey = GetWalletKey(trackedSource); await using var conn = await GetConnection(); - var importMode = GetImportRPCMode(conn, walletKey); var scriptsRecords = scripts.Select(pair => new ScriptInsert(this.Network.CryptoCode, walletKey.wid, pair.Destination.ScriptPubKey.ToHex(), pair.Destination.ToString(), pair.Used)).ToArray(); { @@ -289,7 +288,8 @@ await conn.Connection.ExecuteAsync( request.Destination.ScriptPubKey.ToHex(), request.Metadata.ToString(Formatting.None), request.Destination.ToString(), request.Used)).ToList(); await InsertDescriptorsScripts(conn.Connection, descriptScriptInsert); - if (ImportRPCMode.Legacy == await importMode) + + if (ImportRPCMode.Legacy == await GetImportRPCMode(conn, walletKey)) { foreach (var scriptsRecord in scriptsRecords) { From 3c998b0672dd0297dcc00a9ac2b32b98ef68849a Mon Sep 17 00:00:00 2001 From: Kukks Date: Mon, 4 Dec 2023 14:16:28 +0100 Subject: [PATCH 15/16] adjust so that we fetch block hash and time with best effort (first check proof, then db, then getblock) --- NBXplorer.Client/ExplorerClient.cs | 8 +- NBXplorer.Client/Models/ImportUTXORequest.cs | 7 +- NBXplorer.Tests/UnitTest1.cs | 48 ++--- .../Controllers/PostgresMainController.cs | 185 ++++++++++++++---- docs/API.md | 8 +- 5 files changed, 175 insertions(+), 81 deletions(-) diff --git a/NBXplorer.Client/ExplorerClient.cs b/NBXplorer.Client/ExplorerClient.cs index 0ad44ae40..98b6671cc 100644 --- a/NBXplorer.Client/ExplorerClient.cs +++ b/NBXplorer.Client/ExplorerClient.cs @@ -398,13 +398,13 @@ public async Task AssociateScriptsAsync(TrackedSource trackedSource, AssociateSc throw new ArgumentNullException(nameof(trackedSource)); await SendAsync(HttpMethod.Post, scripts, $"{GetBasePath(trackedSource)}/associate", cancellation); } - public async Task ImportUTXOs(TrackedSource trackedSource, ImportUTXORequest[] utxos, CancellationToken cancellation = default) + public async Task ImportUTXOs(TrackedSource trackedSource, ImportUTXORequest request, CancellationToken cancellation = default) { - if (utxos == null) - throw new ArgumentNullException(nameof(utxos)); + if (request == null) + throw new ArgumentNullException(nameof(request)); if (trackedSource == null) throw new ArgumentNullException(nameof(trackedSource)); - await SendAsync(HttpMethod.Post, utxos, $"{GetBasePath(trackedSource)}/import-utxos", cancellation); + await SendAsync(HttpMethod.Post, request, $"{GetBasePath(trackedSource)}/import-utxos", cancellation); } public Task RescanAsync(RescanRequest rescanRequest, CancellationToken cancellation = default) diff --git a/NBXplorer.Client/Models/ImportUTXORequest.cs b/NBXplorer.Client/Models/ImportUTXORequest.cs index 4d3ce57e8..3f363fe68 100644 --- a/NBXplorer.Client/Models/ImportUTXORequest.cs +++ b/NBXplorer.Client/Models/ImportUTXORequest.cs @@ -5,10 +5,9 @@ namespace NBXplorer.Models; public class ImportUTXORequest { - public OutPoint Utxo { get; set; } - - public MerkleBlock Proof { get; set; } - + public OutPoint[] Utxos { get; set; } + + public MerkleBlock[] Proofs { get; set; } } public class AssociateScriptRequest diff --git a/NBXplorer.Tests/UnitTest1.cs b/NBXplorer.Tests/UnitTest1.cs index 0a1e40b99..57fe3bf18 100644 --- a/NBXplorer.Tests/UnitTest1.cs +++ b/NBXplorer.Tests/UnitTest1.cs @@ -4575,7 +4575,7 @@ await Assert.ThrowsAsync(async () =>await tester.Client.Gene })); await Assert.ThrowsAsync(async () =>await tester.Client.AssociateScriptsAsync(parentWalletTS, Array.Empty())); - await Assert.ThrowsAsync(async () =>await tester.Client.ImportUTXOs(parentWalletTS, Array.Empty())); + await Assert.ThrowsAsync(async () =>await tester.Client.ImportUTXOs(parentWalletTS, new ImportUTXORequest())); return; } #endif @@ -4643,12 +4643,11 @@ await tester.Client.AssociateScriptsAsync(wallet1TS, new[] } }); - await tester.Client.ImportUTXOs(wallet1TS, new[] + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest() { - new ImportUTXORequest() + Utxos = new[] { - Utxo = utxos.OutPoint, - Proof = null + utxos.OutPoint, } }); @@ -4759,12 +4758,11 @@ public async Task CanImportUTXOs(Backend backend) var utxo = rawTx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kScript); var utxo2 = rawTx2.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == kScript); - await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest[] + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest() { - new() + Utxos = new[] { - Utxo = utxo.ToCoin().Outpoint, - Proof = null + utxo.ToCoin().Outpoint, } }); @@ -4780,12 +4778,11 @@ await Eventually(async () => Assert.Single(utxos.Confirmed.UTXOs); }); - await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest[] + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest() { - new() + Utxos = new[] { - Utxo = utxo2.ToCoin().Outpoint, - Proof = null + utxo2.ToCoin().Outpoint, } }); @@ -4816,20 +4813,14 @@ await Eventually(async () => await tester.RPC.SendRawTransactionAsync(spendingtx); var validScriptUtxo = spendingtx.Outputs.AsIndexedOutputs().First(o => o.TxOut.ScriptPubKey == validScript); - - await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest[] + + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest() { - new() - { - Utxo = fakeUtxo.Outpoint - }, - new() - { - Utxo = new Coin(tospendutxo).Outpoint - }, - new() + Utxos = new[] { - Utxo = new Coin(validScriptUtxo).Outpoint + fakeUtxo.Outpoint, + new Coin(tospendutxo).Outpoint, + new Coin(validScriptUtxo).Outpoint } }); @@ -4855,12 +4846,11 @@ await Eventually(async () => var mb = new MerkleBlock(); mb.FromBytes(merkleBLockProofBytes); - await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest[] + await tester.Client.ImportUTXOs(wallet1TS, new ImportUTXORequest() { - new() + Utxos = new[] { - Utxo = new Coin(yoUtxo).Outpoint, - Proof = mb + new Coin(yoUtxo).Outpoint } }); diff --git a/NBXplorer/Controllers/PostgresMainController.cs b/NBXplorer/Controllers/PostgresMainController.cs index 218e7f6c4..6cf0ba0d4 100644 --- a/NBXplorer/Controllers/PostgresMainController.cs +++ b/NBXplorer/Controllers/PostgresMainController.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Dapper; @@ -93,56 +94,151 @@ public async Task AssociateScripts( TrackedSourceContext trackedS [HttpPost("import-utxos")] [TrackedSourceContext.TrackedSourceContextRequirement(true)] - public async Task ImportUTXOs( TrackedSourceContext trackedSourceContext, [FromBody] JArray rawRequest) + public async Task ImportUTXOs( TrackedSourceContext trackedSourceContext, [FromBody] JObject rawRequest) { var repo = (PostgresRepository)trackedSourceContext.Repository; var jsonSerializer = JsonSerializer.Create(trackedSourceContext.Network.JsonSerializerSettings); - var coins = rawRequest.ToObject(jsonSerializer)?.Where(c => c.Utxo != null).ToArray(); - if (coins?.Any() is not true) - throw new ArgumentNullException(nameof(coins)); + var request = rawRequest.ToObject(jsonSerializer); + + if (request.Utxos?.Any() is not true) + return Ok(); var rpc = trackedSourceContext.RpcClient; var clientBatch = rpc.PrepareBatch(); - var coinToTxOut = new ConcurrentDictionary(); - var coinToBlock = new ConcurrentDictionary(); - await Task.WhenAll(coins.SelectMany(o => + var coinToTxOut = new Dictionary>(); + var txsToBlockHash = new ConcurrentDictionary(); + var blockHeaders = new ConcurrentDictionary>(); + var allUtxoTransactionHashes = request.Utxos.Select(u => u.Hash).Distinct().ToArray(); + foreach (var importUtxoRequest in request.Utxos) { - return new[] + coinToTxOut.TryAdd(importUtxoRequest, clientBatch.GetTxOutAsync(importUtxoRequest.Hash, (int)importUtxoRequest.N)); + } + request.Proofs ??= Array.Empty(); + var verifyTasks = request.Proofs + .Where(p => p is not null && p.PartialMerkleTree.Hashes.Any(uint256 => allUtxoTransactionHashes.Contains(uint256))) + .Select(async proof => { - Task.Run(async () => - { - var txOutResponse = - await clientBatch.GetTxOutAsync(o.Utxo.Hash, (int) o.Utxo.N); - if (txOutResponse is not null) - coinToTxOut.TryAdd(o.Utxo, txOutResponse); - }), - Task.Run(async () => + var txoutproofResult = await clientBatch.SendCommandAsync("verifytxoutproof", + Encoders.Hex.EncodeData(proof.ToBytes())); + if (txoutproofResult.Error is not null && txoutproofResult.Result is JArray prooftxs) { - if (o.Proof is not null && o.Proof.PartialMerkleTree.Hashes.Contains(o.Utxo.Hash)) + foreach (var txProof in prooftxs) { - var txoutproofResult = - await clientBatch.SendCommandAsync("verifytxoutproof", Encoders.Hex.EncodeData(o.Proof.ToBytes())); - - var txHash = o.Utxo.Hash.ToString(); - if (txoutproofResult.Error is not null && txoutproofResult.Result is JArray prooftxs && - prooftxs.Any(token => - token.Value() - ?.Equals(txHash, StringComparison.InvariantCultureIgnoreCase) is true)) - { - coinToBlock.TryAdd(o.Utxo, o.Proof.Header); - } + var txId = uint256.Parse(txProof.Value()); + blockHeaders.TryAdd(proof.Header.GetHash(), Task.FromResult((proof.Header.GetHash(), proof.Header.BlockTime))); + txsToBlockHash.TryAdd(txId, proof.Header.GetHash()); } - }) - }; - }).Concat(new[] {clientBatch.SendBatchAsync()}).ToArray()); + } + }); + + await clientBatch.SendBatchAsync(); + await Task.WhenAll(verifyTasks.Concat(coinToTxOut.Values)); + + + coinToTxOut = coinToTxOut.Where(c => c.Value.Result is not null).ToDictionary(pair => pair.Key, pair => pair.Value); + + await using var conn = await repo.ConnectionFactory.CreateConnection(); + + var blockTasks = new ConcurrentDictionary>(); + var blocksToRequest = new HashSet(); + foreach (var cTxOut in coinToTxOut) + { + var result = await cTxOut.Value; + if (result.Confirmations == 1) + { + txsToBlockHash.TryAdd(cTxOut.Key.Hash, result.BestBlock); + continue; + } + blocksToRequest.Add(result.BestBlock); + } + + var res = await conn.QueryAsync( + $"SELECT blk_id, height FROM blks WHERE code=@code AND blk_id IN (SELECT unnest(@blkIds)) ", + new + { + code = trackedSourceContext.Network.CryptoCode, + blkIds = blocksToRequest.Select(uint256 => uint256.ToString()).ToArray() + }); + + foreach (var r in res) + { + var blockHash = uint256.Parse((string)r.blk_id); + var height = (int)r.height; + blockTasks.TryAdd(blockHash, Task.FromResult(height)); + blocksToRequest.Remove(blockHash); + } + + clientBatch = rpc.PrepareBatch(); + foreach (var bh in blocksToRequest) + { + blockTasks.TryAdd(bh, clientBatch.GetBlockAsync(bh, GetBlockVerbosity.WithOnlyTxId).ContinueWith(task => task.Result.Height)); + } + await clientBatch.SendBatchAsync(); + await Task.WhenAll(blockTasks.Values); + var heightToBlockHash = new ConcurrentDictionary>(); + var heightsToFetch = new HashSet(); + foreach (var cTxOut in coinToTxOut) + { + var result = await cTxOut.Value; + + if (result.Confirmations <= 1) + continue; + + blockTasks.TryGetValue(result.BestBlock, out var blockTask); + var b = await blockTask; + + var heightToFetch = b - result.Confirmations - 1; + heightsToFetch.Add(heightToFetch); + } + + res = await conn.QueryAsync( + $"SELECT blk_id, height, indexed_at FROM blks WHERE code=@code AND height IN (SELECT unnest(@heights)) ", + new + { + code = trackedSourceContext.Network.CryptoCode, + heights = heightsToFetch.ToArray() + }); + + foreach (var r in res) + { + var blockHash = uint256.Parse((string)r.blk_id); + var height = (int)r.height; + var blockTime = (DateTimeOffset)r.indexed_at; + blockTasks.TryAdd(blockHash, Task.FromResult(height)); + heightToBlockHash.TryAdd(height, Task.FromResult((blockHash, blockTime))); + heightsToFetch.Remove((int)r.height); + } + + foreach (var heightToFetch in heightsToFetch) + { + heightToBlockHash.TryAdd(heightToFetch, clientBatch.GetBlockHeaderAsync(heightToFetch).ContinueWith(task => (task.Result.GetHash(), task.Result.BlockTime))); + } + + clientBatch = rpc.PrepareBatch(); + + await clientBatch.SendBatchAsync(); + foreach (var htbh in heightToBlockHash.Values) + { + var result = await htbh; + blockHeaders.TryAdd(result.hash, Task.FromResult(result)); + + foreach (var cto in coinToTxOut) + { + var result2 = await cto.Value; + if (result2.Confirmations <= 1) + continue; + + txsToBlockHash.TryAdd(cto.Key.Hash, result.hash); + } + } var now = DateTimeOffset.UtcNow; var scripts = coinToTxOut .Select(pair => ( - pair.Value.TxOut.ScriptPubKey.GetDestinationAddress(repo.Network.NBitcoinNetwork), pair)) + pair.Value.Result.TxOut.ScriptPubKey.GetDestinationAddress(repo.Network.NBitcoinNetwork), pair)) .Where(pair => pair.Item1 is not null).Select(tuple => new AssociateScriptRequest() { Destination = tuple.Item1, @@ -151,19 +247,30 @@ await Task.WhenAll(coins.SelectMany(o => }).ToArray(); await repo.AssociateScriptsToWalletExplicitly(trackedSourceContext.TrackedSource,scripts); - await repo.SaveMatches(coinToTxOut.Select(pair => + + + var trackedTransactions = coinToTxOut.Select(async pair => { - coinToBlock.TryGetValue(pair.Key, out var blockHeader); + var txOutResult = await pair.Value; + txsToBlockHash.TryGetValue(pair.Key.Hash, out var blockHash); + (uint256 hash, DateTimeOffset time)? blockHeader = null; + if (blockHash is not null && blockHeaders.TryGetValue(blockHash, out var blockHeaderT)) + { + blockHeader = await blockHeaderT; + }; + + var coin = new Coin(pair.Key, txOutResult.TxOut); - var coin = new Coin(pair.Key, pair.Value.TxOut); var ttx = repo.CreateTrackedTransaction(trackedSourceContext.TrackedSource, - new TrackedTransactionKey(pair.Key.Hash, blockHeader?.GetHash(), true){}, + new TrackedTransactionKey(pair.Key.Hash, blockHash, true){}, new[] {coin}, null); ttx.Inserted = now; - ttx.Immature = pair.Value.IsCoinBase && pair.Value.Confirmations <= 100; - ttx.FirstSeen = blockHeader?.BlockTime?? NBitcoin.Utils.UnixTimeToDateTime(0);; + ttx.Immature =txOutResult.IsCoinBase && txOutResult.Confirmations <= 100; + ttx.FirstSeen = blockHeader?.time?? NBitcoin.Utils.UnixTimeToDateTime(0);; return ttx; - }).ToArray()); + }); + + await repo.SaveMatches(await Task.WhenAll(trackedTransactions)); return Ok(); } diff --git a/docs/API.md b/docs/API.md index 34f4b668a..22fd51f80 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1333,7 +1333,7 @@ Request: Note: This API is only available for Postgres. -In the case where you start tracking a wallet that already has UTXOs, you can import them to NBXplorer so that it can be aware of them. NBXplorer will validate against the bitcoin node's utxoset that the UTXOs are valid. +In the case where you start tracking a wallet that already has UTXOs, you can import them to NBXplorer so that it can be aware of them. NBXplorer will validate against the bitcoin node's utxoset that the UTXOs are valid. Additionally, you can also provide merkle proofs that the UTXOs were included in a block. `HTTP POST v1/cryptos/{cryptoCode}/derivations/{derivationScheme}/import-utxos`
`HTTP POST v1/cryptos/{cryptoCode}/addresses/{address}/import-utxos`
@@ -1348,12 +1348,10 @@ Request: * `proof`: Optional, a merkle proof that the utxo being imported was included in a block (as provided by Bitcoin Core's gettxoutproof) ```json -[ { - "utxo": "txid-vout", - "proof": {} + "utxos": ["txid-vout"], + "proofs": [] } -] ``` No response body From 44f0b69085946c2fd26417e0f4e847ccc6547b78 Mon Sep 17 00:00:00 2001 From: Kukks Date: Tue, 5 Dec 2023 08:10:19 +0100 Subject: [PATCH 16/16] Remove dependency on getblock --- NBXplorer/Controllers/PostgresMainController.cs | 2 +- NBXplorer/RPCClientExtensions.cs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/NBXplorer/Controllers/PostgresMainController.cs b/NBXplorer/Controllers/PostgresMainController.cs index 6cf0ba0d4..def4d7f29 100644 --- a/NBXplorer/Controllers/PostgresMainController.cs +++ b/NBXplorer/Controllers/PostgresMainController.cs @@ -173,7 +173,7 @@ public async Task ImportUTXOs( TrackedSourceContext trackedSource clientBatch = rpc.PrepareBatch(); foreach (var bh in blocksToRequest) { - blockTasks.TryAdd(bh, clientBatch.GetBlockAsync(bh, GetBlockVerbosity.WithOnlyTxId).ContinueWith(task => task.Result.Height)); + blockTasks.TryAdd(bh, clientBatch.GetBlockHeaderAsyncEx(bh).ContinueWith(task => task.Result.Height)); } await clientBatch.SendBatchAsync(); await Task.WhenAll(blockTasks.Values); diff --git a/NBXplorer/RPCClientExtensions.cs b/NBXplorer/RPCClientExtensions.cs index b99f547cb..988f0e315 100644 --- a/NBXplorer/RPCClientExtensions.cs +++ b/NBXplorer/RPCClientExtensions.cs @@ -263,6 +263,12 @@ public static async Task GetBlockHeaderAsyncEx(this NBitcoin.R return new SlimChainedBlock(blk, prev is null ? null : new uint256(prev), response["height"].Value()); } + public static async Task GetBlockHeaderAsyncEx(this NBitcoin.RPC.RPCClient rpc, int height) + { + var hash = await rpc.GetBlockHashAsync(height); + return await rpc.GetBlockHeaderAsyncEx(hash); + } + public static async Task TryGetRawTransaction(this RPCClient client, uint256 txId) { var request = new RPCRequest(RPCOperations.getrawtransaction, new object[] { txId, true }) { ThrowIfRPCError = false };