Skip to content

Commit

Permalink
Refactor transaction matching, fix elements (#489)
Browse files Browse the repository at this point in the history 
Co-authored-by: Kukks <[email protected]>
  • Loading branch information
@NicolasDorier @Kukks
NicolasDorier and Kukks authored Nov 28, 2024
1 parent 654b103 commit 531f288
Show file tree
Hide file tree
Showing 17 changed files with 423 additions and 488 deletions.
14 changes: 9 additions & 5 deletions NBXplorer.Tests/TrackedTransactionBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
using System.Collections.Generic;
using NBitcoin;
using NBXplorer.Models;
using static NBXplorer.Backend.DbConnectionHelper;

namespace NBXplorer.Tests
{
Expand All @@ -21,17 +22,20 @@ public class TransactionContext

public TrackedTransaction Build()
{
var tx = new TrackedTransaction(new TrackedTransactionKey(_TransactionId, _BlockId, true), _Parent._TrackedSource, null as Coin[], null)
{
FirstSeen = _TimeStamp
};
var record = new SaveTransactionRecord(null, _TransactionId, _BlockId, null, null, false, _TimeStamp);
var tx = TrackedTransaction.Create(_Parent._TrackedSource, record);
foreach (var input in _Inputs)
{
tx.SpentOutpoints.Add(input.Coin.Outpoint, 0);
}
foreach (var output in _Outputs)
{
tx.ReceivedCoins.Add(output.Coin);
tx.MatchedOutputs.Add(new MatchedOutput()
{
Index = (int)output.Coin.Outpoint.N,
Value = output.Coin.Amount,
ScriptPubKey = output.Coin.ScriptPubKey
});
}
return tx;
}
Expand Down
43 changes: 23 additions & 20 deletions NBXplorer.Tests/UnitTest1.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
using System.Net;
using NBXplorer.HostedServices;
using NBitcoin.Altcoins;
using static NBXplorer.Backend.DbConnectionHelper;

namespace NBXplorer.Tests
{
Expand Down Expand Up @@ -1167,7 +1168,7 @@ public async Task ShowRBFedTransaction3(bool cancelB)

var bobW = await tester.Client.GenerateWalletAsync();
var bob = bobW.DerivationScheme;

var bobAddr = await tester.Client.GetUnusedAsync(bob, DerivationFeature.Deposit, 0);
var bobAddr1 = await tester.Client.GetUnusedAsync(bob, DerivationFeature.Deposit, 1);

Expand Down Expand Up @@ -2013,11 +2014,11 @@ public async Task CanUseWebSockets()
// but make test flaky.
if (blockEvent.Hash != expectedBlockId)
blockEvent = (Models.NewBlockEvent)connected.NextEvent(Cancel);

Assert.True(blockEvent.EventId != 0);
Assert.Equal(expectedBlockId, blockEvent.Hash);
Assert.NotEqual(0, blockEvent.Height);

Assert.Equal(1, blockEvent.Confirmations);

connected.ListenDerivationSchemes(new[] { pubkey });
Expand Down Expand Up @@ -2097,7 +2098,7 @@ public async Task CanUseWebSockets2()
{
tester.Client.WaitServerStarted();
var key = new BitcoinExtKey(new ExtKey(), tester.Network);

var wLegacy = await tester.Client.GenerateWalletAsync(new GenerateWalletRequest() { ScriptPubKeyType = ScriptPubKeyType.Legacy });
var wSegwit = await tester.Client.GenerateWalletAsync(new GenerateWalletRequest() { ScriptPubKeyType = ScriptPubKeyType.Segwit });

Expand Down Expand Up @@ -2137,7 +2138,7 @@ public async Task CanUseWebSockets2()

// Here, we will try to spend the coins of the segwit wallet
var psbt = await tester.Client.CreatePSBTAsync(pubkey2, new CreatePSBTRequest()
{
{
Destinations = [
new ()
{
Expand All @@ -2156,7 +2157,7 @@ public async Task CanUseWebSockets2()
txEvent = (Models.NewTransactionEvent)await connected.NextEventAsync(Cancel);
if (txEvent.TrackedSource == TrackedSource.Parse(wSegwit.TrackedSource, tester.NBXplorerNetwork))
{

void AssertInputs(List<MatchedInput> inputs)
{
Assert.Equal(2, inputs.Count);
Expand Down Expand Up @@ -2213,8 +2214,11 @@ public async Task DoNotLoseTimestampForLongConfirmations()
var transactions = await repo.GetTransactions(new DerivationSchemeTrackedSource(bobPubKey), id);
var tx = Assert.Single(transactions);
var timestamp = tx.FirstSeen;
var match = (await repo.GetMatches(tx.Transaction, null, DateTimeOffset.UtcNow + TimeSpan.FromSeconds(2), false));
await repo.SaveMatches(match);
var query = MatchQuery.FromTransactions(new[] { tx.Transaction }, null);
var records = new SaveTransactionRecord[] { SaveTransactionRecord.Create(tx: tx.Transaction, seenAt: DateTimeOffset.UtcNow + TimeSpan.FromSeconds(2)) };
var tracked = await repo.SaveMatches(query, records);
Assert.Single(tracked);
Assert.Equal(timestamp, tracked[0].FirstSeen);
transactions = await repo.GetTransactions(new DerivationSchemeTrackedSource(bobPubKey), id);
tx = Assert.Single(transactions);
Assert.Equal(timestamp, tx.FirstSeen);
Expand Down Expand Up @@ -2555,7 +2559,7 @@ public async Task Test()
{
var rpc = new RPCClient(new RPCCredentialString()
{
UserPassword = new NetworkCredential("dashrpc","PQQgOzs1jN7q2SWQ6TpBNLm9j"),
UserPassword = new NetworkCredential("dashrpc", "PQQgOzs1jN7q2SWQ6TpBNLm9j"),
}, "https://dash-testnet.nodes.m3t4c0.xyz", AltNetworkSets.Dash.Testnet);
var b1 = await rpc.GetBlockAsync(new uint256("000001f02c1623e0bb12b54ac505cefdfca3f0f664bf333fc73ae5eafe34b830"));
}
Expand Down Expand Up @@ -2737,7 +2741,7 @@ public async Task CanGetStatus()
}
}

public CancellationToken Timeout => new CancellationTokenSource(10000).Token;
public CancellationToken Timeout => new CancellationTokenSource(10_000).Token;


[FactWithTimeout]
Expand Down Expand Up @@ -2958,7 +2962,7 @@ public async Task CanTrack()
var utxo = await tester.Client.GetUTXOsAsync(pubkey);
Assert.Equal(tester.Network.Consensus.CoinbaseMaturity + 1, utxo.CurrentHeight);
Assert.Single(utxo.Unconfirmed.UTXOs);

Assert.Equal(tester.AddressOf(key, "0/0"), utxo.Unconfirmed.UTXOs[0].Address);
Assert.Equal(txId, utxo.Unconfirmed.UTXOs[0].Outpoint.Hash);
var unconfTimestamp = utxo.Unconfirmed.UTXOs[0].Timestamp;
Expand Down Expand Up @@ -3253,7 +3257,7 @@ public void CanTopologicalSortRecords()

var outpoint = new OutPoint(tx2.Record.Key.TxId, 0);
tx1.Record.SpentOutpoints.Add(outpoint, 0);
tx2.Record.ReceivedCoins.Add(new Coin(outpoint, new TxOut()));
tx2.Record.MatchedOutputs.Add(new MatchedOutput() { Index = (int)outpoint.N });
AssertExpectedOrder(new[] { tx2, tx1 }, true); // tx1 depends on tx2 so even if tx1 has been seen first, topological sort should be used

List<AnnotatedTransaction> txs = new List<AnnotatedTransaction>();
Expand Down Expand Up @@ -3298,12 +3302,11 @@ private void AssertExpectedOrder(AnnotatedTransaction[] annotatedTransaction, bo

private static AnnotatedTransaction CreateRandomAnnotatedTransaction(DerivationSchemeTrackedSource trackedSource, int? height = null, int? seen = null)
{
var a = new AnnotatedTransaction(height, new TrackedTransaction(new TrackedTransactionKey(RandomUtils.GetUInt256(), null, true), trackedSource, null as Coin[], null), true);
if (seen is int v)
{
a.Record.FirstSeen = NBitcoin.Utils.UnixTimeToDateTime(v);
}
return a;
var record = new SaveTransactionRecord(null, RandomUtils.GetUInt256(), null, null, height, false, NBitcoin.Utils.UnixTimeToDateTime(seen ?? 0));
return new AnnotatedTransaction(
record.BlockHeight,
TrackedTransaction.Create(trackedSource, record),
!record.Immature);
}

[Fact]
Expand Down Expand Up @@ -4254,15 +4257,15 @@ public async Task IsTrackedTests()
Assert.False(await tester.Client.IsTrackedAsync(xpub, Cancel));
await tester.Client.TrackAsync(xpub, new TrackWalletRequest(), Cancel);
Assert.True(await tester.Client.IsTrackedAsync(xpub, Cancel));

var address = new AddressTrackedSource(new Key().GetAddress(ScriptPubKeyType.Legacy, tester.Network));
Assert.False(await tester.Client.IsTrackedAsync(address, Cancel));
await tester.Client.TrackAsync(address, new TrackWalletRequest(), Cancel);
Assert.True(await tester.Client.IsTrackedAsync(address, Cancel));

var group = new GroupTrackedSource("lolno");
Assert.False(await tester.Client.IsTrackedAsync(group, Cancel));

group = new GroupTrackedSource((await tester.Client.CreateGroupAsync(Cancel)).GroupId);
Assert.True(await tester.Client.IsTrackedAsync(group, Cancel));
}
Expand Down
2 changes: 1 addition & 1 deletion NBXplorer/AddressPoolService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ internal Task GenerateAddresses(NBXplorerNetwork network, TrackedTransaction[] m
var derivationStrategy = (m.TrackedSource as Models.DerivationSchemeTrackedSource)?.DerivationStrategy;
if (derivationStrategy == null)
continue;
foreach (var feature in m.KnownKeyPathMapping.Select(kv => keyPathTemplates.GetDerivationFeature(kv.Value)))
foreach (var feature in m.InOuts.Select(kv => keyPathTemplates.GetDerivationFeature(kv.KeyPath)).Distinct())
{
refill.Add(GenerateAddresses(network, derivationStrategy, feature));
}
Expand Down
18 changes: 11 additions & 7 deletions NBXplorer/Backend/DbConnectionHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,11 @@ public ValueTask DisposeAsync()
return Connection.DisposeAsync();
}

public record NewOut(uint256 txId, int idx, Script script, IMoney value);
public record NewOut(uint256 txId, int idx, Script script, IMoney value)
{
public static NewOut FromCoin(ICoin c)
=> new(c.Outpoint.Hash, (int)c.Outpoint.N, c.TxOut.ScriptPubKey, c.Amount);
}
public record NewIn(uint256 txId, int idx, uint256 spentTxId, int spentIdx);
public record NewOutRaw(string tx_id, long idx, string script, long value, string asset_id);
public record NewInRaw(string tx_id, long idx, string spent_tx_id, long spent_idx);
Expand Down Expand Up @@ -92,19 +96,20 @@ public async Task<bool> FetchMatches(MatchQuery matchQuery)
await Connection.QueryAsync<int>("fetch_matches", parameters, commandType: CommandType.StoredProcedure);
return parameters.Get<bool>("has_match");
}

public record SaveTransactionRecord(Transaction? Transaction, uint256 Id, uint256? BlockId, int? BlockIndex, long? BlockHeight, bool Immature, DateTimeOffset SeenAt)
{
public static SaveTransactionRecord Create(TrackedTransaction t) => new SaveTransactionRecord(t.Transaction, t.TransactionHash, t.BlockHash, t.BlockIndex, t.BlockHeight, t.IsCoinBase, t.FirstSeen);
public static SaveTransactionRecord Create(SlimChainedBlock slimBlock, Transaction tx, int? blockIndex, DateTimeOffset now) => new SaveTransactionRecord(
public static SaveTransactionRecord Create(Transaction? tx = null, uint256? txHash = null, SlimChainedBlock? slimBlock = null, int? blockIndex = null, DateTimeOffset? seenAt = null) => new SaveTransactionRecord(
tx,
tx.GetHash(),
txHash ?? tx?.GetHash() ?? throw new ArgumentException("tx or txHash is expected"),
slimBlock?.Hash,
blockIndex,
slimBlock?.Height,
tx.IsCoinBase,
now
tx?.IsCoinBase is true,
seenAt ?? DateTimeOffset.UtcNow
);
}

public async Task SaveTransactions(IEnumerable<SaveTransactionRecord> transactions)
{
var parameters = transactions
Expand Down Expand Up @@ -188,7 +193,6 @@ public async Task<bool> SetMetadata<TMetadata>(string walletId, string key, TMet
return null;
return Network.Serializer.ToObject<TMetadata>(result);
}

public async Task<HashSet<uint256>> GetUnconfirmedTxs()
{
var txs = await Connection.QueryAsync<string>("SELECT tx_id FROM txs WHERE code=@code AND mempool IS TRUE;", new { code = Network.CryptoCode });
Expand Down
63 changes: 63 additions & 0 deletions NBXplorer/Backend/GetTransactionQuery.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#nullable enable
using Dapper;
using NBitcoin;
using NBXplorer.Models;
using System;
using System.Collections.Generic;
using System.Linq;

namespace NBXplorer.Backend
{
public abstract record GetTransactionQuery
{
public static GetTransactionQuery Create(TrackedSource TrackedSource, uint256 TxId) => new TrackedSourceTxId(TrackedSource, TxId);
public static GetTransactionQuery Create(KeyPathInformation[] KeyInfos, uint256[] TxIds) => new ScriptsTxIds(KeyInfos, TxIds);
public record TrackedSourceTxId(TrackedSource TrackedSource, uint256? TxId) : GetTransactionQuery
{
string? walletId;
public override string GetSql(DynamicParameters parameters, NBXplorerNetwork network)
{
var txIdCond = string.Empty;
if (TxId is not null)
{
txIdCond = " AND tx_id=@tx_id";
parameters.Add("@tx_id", TxId.ToString());
}
walletId = Repository.GetWalletKey(TrackedSource, network).wid;
parameters.Add("@walletId", walletId);
parameters.Add("@code", network.CryptoCode);
return """
SELECT wallet_id, tx_id, idx, blk_id, blk_height, blk_idx, is_out, spent_tx_id, spent_idx, script, s.addr, value, asset_id, immature, keypath, seen_at
FROM nbxv1_tracked_txs LEFT JOIN scripts s USING (code, script)
WHERE code=@code AND wallet_id=@walletId
""" + txIdCond;
}
public override TrackedSource? GetTrackedSource(string wallet_id) => walletId == wallet_id ? TrackedSource : null;
}

public record ScriptsTxIds(KeyPathInformation[] KeyInfos, uint256[] TxIds) : GetTransactionQuery
{
Dictionary<string, TrackedSource> widToTrackedSource = new Dictionary<string, TrackedSource>();
public override string GetSql(DynamicParameters parameters, NBXplorerNetwork network)
{
widToTrackedSource.Clear();
foreach (var k in KeyInfos)
{
widToTrackedSource.TryAdd(Repository.GetWalletKey(k.TrackedSource, network).wid, k.TrackedSource);
}
parameters.Add("@code", network.CryptoCode);
parameters.Add("@tx_ids", TxIds.Select(t => t.ToString()).ToArray());
return """
SELECT wallet_id, t.tx_id, idx, blk_id, blk_height, blk_idx, is_out, spent_tx_id, spent_idx, script, s.addr, value, asset_id, immature, keypath, seen_at
FROM nbxv1_tracked_txs LEFT JOIN scripts s USING (code, script)
JOIN unnest(@tx_ids) t(tx_id) USING (tx_id)
WHERE code=@code
""";
}
public override TrackedSource? GetTrackedSource(string wallet_id) => widToTrackedSource.TryGetValue(wallet_id, out var trackedSource) ? trackedSource : null;
}

public abstract string GetSql(DynamicParameters parameters, NBXplorerNetwork network);
public abstract TrackedSource? GetTrackedSource(string wallet_id);
}
}
6 changes: 3 additions & 3 deletions NBXplorer/Backend/Indexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -464,7 +464,7 @@ private async Task SaveMatches(DbConnectionHelper conn, List<Transaction> transa
{
await conn.NewBlock(slimChainedBlock);
}
var matches = await Repository.GetMatchesAndSave(conn, transactions, slimChainedBlock, now, true);
var matches = await Repository.GetMatches(conn, transactions, slimChainedBlock, now, blockMatch: true, useCache: true);
_ = AddressPoolService.GenerateAddresses(Network, matches);

long confirmations = 0;
Expand Down Expand Up @@ -505,8 +505,8 @@ private async Task SaveMatches(DbConnectionHelper conn, List<Transaction> transa
Transaction = matches[i].Transaction,
TransactionHash = matches[i].TransactionHash
},
Inputs = matches[i].MatchedInputs.OrderBy(m => m.InputIndex).ToList(),
Outputs = matches[i].GetReceivedOutputs().ToList(),
Inputs = matches[i].MatchedInputs,
Outputs = matches[i].MatchedOutputs,
Replacing = matches[i].Replacing.ToList()
};

Expand Down
49 changes: 12 additions & 37 deletions NBXplorer/Backend/MatchQuery.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#nullable enable
using NBitcoin;
using NBitcoin.Altcoins.Elements;
using System.Collections.Generic;
using System.Linq;

Expand All @@ -12,44 +13,13 @@ public MatchQuery(List<DbConnectionHelper.NewIn> ins, List<DbConnectionHelper.Ne
Ins = ins;
Outs = outs;
}
public List<DbConnectionHelper.NewIn> Ins { get; }
public List<DbConnectionHelper.NewOut> Outs { get; }

public static MatchQuery FromTransactions(IEnumerable<TrackedTransaction> txs, Money? minUtxoValue)
public MatchQuery(IEnumerable<ICoin> coins)
{
var outCount = txs.Select(t => t.ReceivedCoins.Count).Sum();
var inCount = txs.Select(t => t.SpentOutpoints.Count).Sum();
List<DbConnectionHelper.NewOut> outs = new List<DbConnectionHelper.NewOut>(outCount);
List<DbConnectionHelper.NewIn> ins = new List<DbConnectionHelper.NewIn>(inCount);
foreach (var tx in txs)
{
if (!tx.IsCoinBase)
{
foreach (var input in tx.SpentOutpoints)
{
ins.Add(new DbConnectionHelper.NewIn(
tx.TransactionHash,
input.InputIndex,
input.Outpoint.Hash,
(int)input.Outpoint.N
));
}
}

foreach (var output in tx.ReceivedCoins)
{
if (minUtxoValue != null && (Money)output.Amount < minUtxoValue)
continue;
outs.Add(new DbConnectionHelper.NewOut(
tx.TransactionHash,
(int)output.Outpoint.N,
output.TxOut.ScriptPubKey,
(Money)output.Amount
));
}
}
return new MatchQuery(ins, outs);
Outs = coins.Select(c => DbConnectionHelper.NewOut.FromCoin(c)).ToList();
Ins = new List<DbConnectionHelper.NewIn>();
}
public List<DbConnectionHelper.NewIn> Ins { get; }
public List<DbConnectionHelper.NewOut> Outs { get; }

public static MatchQuery FromTransactions(IEnumerable<Transaction> txs, Money? minUtxoValue)
{
Expand All @@ -75,7 +45,12 @@ public static MatchQuery FromTransactions(IEnumerable<Transaction> txs, Money? m
io++;
if (minUtxoValue != null && output.Value < minUtxoValue)
continue;
outs.Add(new DbConnectionHelper.NewOut(hash, io, output.ScriptPubKey, output.Value));
IMoney val = output switch
{
ElementsTxOut { Asset: { AssetId: { } assetId } } el => new AssetMoney(assetId, el.Value),
_ => output.Value
};
outs.Add(new DbConnectionHelper.NewOut(hash, io, output.ScriptPubKey, val));
}
}
return new MatchQuery(ins, outs);
Expand Down
Loading

0 comments on commit 531f288

Please sign in to comment.