From 38f4780c8c6d5e180c77494896b3fd7b70718cbb Mon Sep 17 00:00:00 2001 From: Warren He Date: Thu, 12 Oct 2023 17:09:01 -0700 Subject: [PATCH 1/4] analyzer: rename some upsert queries --- analyzer/consensus/consensus.go | 2 +- analyzer/queries/queries.go | 4 ++-- analyzer/runtime/runtime.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/analyzer/consensus/consensus.go b/analyzer/consensus/consensus.go index 7bc9a3a8a..f8f6be684 100644 --- a/analyzer/consensus/consensus.go +++ b/analyzer/consensus/consensus.go @@ -479,7 +479,7 @@ func (m *processor) queueEntityEvents(batch *storage.QueryBatch, data *storage.R node.String(), ) } - batch.Queue(queries.ConsensusEntityInsert, + batch.Queue(queries.ConsensusEntityUpsert, entityID, staking.NewAddress(entityEvent.Entity.ID).String(), ) diff --git a/analyzer/queries/queries.go b/analyzer/queries/queries.go index 884010a36..41a3c29bd 100644 --- a/analyzer/queries/queries.go +++ b/analyzer/queries/queries.go @@ -226,7 +226,7 @@ var ( INSERT INTO chain.claimed_nodes (entity_id, node_id) VALUES ($1, $2) ON CONFLICT (entity_id, node_id) DO NOTHING` - ConsensusEntityInsert = ` + ConsensusEntityUpsert = ` INSERT INTO chain.entities (id, address) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET @@ -527,7 +527,7 @@ var ( ON CONFLICT (runtime, token_address, account_address) DO UPDATE SET balance = chain.evm_token_balances.balance + $4` - RuntimeEVMTokenBalanceAnalysisInsert = ` + RuntimeEVMTokenBalanceAnalysisUpsert = ` INSERT INTO analysis.evm_token_balances (runtime, token_address, account_address, last_mutate_round) VALUES diff --git a/analyzer/runtime/runtime.go b/analyzer/runtime/runtime.go index f490bb348..6667e95f0 100644 --- a/analyzer/runtime/runtime.go +++ b/analyzer/runtime/runtime.go @@ -359,7 +359,7 @@ func (m *processor) queueDbUpdates(batch *storage.QueryBatch, data *BlockData) { } // Even for a (suspected) non-change, notify the evm_token_balances analyzer to // verify the correct balance by querying the EVM. - batch.Queue(queries.RuntimeEVMTokenBalanceAnalysisInsert, m.runtime, key.TokenAddress, key.AccountAddress, data.Header.Round) + batch.Queue(queries.RuntimeEVMTokenBalanceAnalysisUpsert, m.runtime, key.TokenAddress, key.AccountAddress, data.Header.Round) } // Insert NFTs. From 46b2bc5831776732441c6437f2f17c95c264b329 Mon Sep 17 00:00:00 2001 From: Warren He Date: Mon, 25 Sep 2023 15:47:12 -0700 Subject: [PATCH 2/4] storage: add NFT ownership --- storage/migrations/21_evm_nfts.up.sql | 3 +++ .../{22_evm_nfts_grant.up.sql => 22_evm_nfts.up.sql} | 3 +++ 2 files changed, 6 insertions(+) rename storage/migrations/{22_evm_nfts_grant.up.sql => 22_evm_nfts.up.sql} (72%) diff --git a/storage/migrations/21_evm_nfts.up.sql b/storage/migrations/21_evm_nfts.up.sql index f7281ef27..ededdf43b 100644 --- a/storage/migrations/21_evm_nfts.up.sql +++ b/storage/migrations/21_evm_nfts.up.sql @@ -6,6 +6,9 @@ CREATE TABLE chain.evm_nfts ( nft_id uint_numeric NOT NULL, PRIMARY KEY (runtime, token_address, nft_id), + -- Added in 22_evm_nfts_2.up.sql + -- owner oasis_addr, + last_want_download_round UINT63 NOT NULL, last_download_round UINT63, diff --git a/storage/migrations/22_evm_nfts_grant.up.sql b/storage/migrations/22_evm_nfts.up.sql similarity index 72% rename from storage/migrations/22_evm_nfts_grant.up.sql rename to storage/migrations/22_evm_nfts.up.sql index 3bb402ba9..7696480b8 100644 --- a/storage/migrations/22_evm_nfts_grant.up.sql +++ b/storage/migrations/22_evm_nfts.up.sql @@ -1,5 +1,8 @@ BEGIN; +ALTER TABLE chain.evm_nfts + ADD COLUMN owner oasis_addr; + -- Grant others read-only use. GRANT SELECT ON ALL TABLES IN SCHEMA chain TO PUBLIC; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA chain TO PUBLIC; From e47cefdc01aa8d178d14f306cf65c366bbb0fb59 Mon Sep 17 00:00:00 2001 From: Warren He Date: Mon, 25 Sep 2023 15:49:14 -0700 Subject: [PATCH 3/4] storage: add NFT num transfers --- storage/migrations/21_evm_nfts.up.sql | 1 + storage/migrations/22_evm_nfts.up.sql | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/migrations/21_evm_nfts.up.sql b/storage/migrations/21_evm_nfts.up.sql index ededdf43b..5d0009b07 100644 --- a/storage/migrations/21_evm_nfts.up.sql +++ b/storage/migrations/21_evm_nfts.up.sql @@ -8,6 +8,7 @@ CREATE TABLE chain.evm_nfts ( -- Added in 22_evm_nfts_2.up.sql -- owner oasis_addr, + -- num_transfers INT NOT NULL, last_want_download_round UINT63 NOT NULL, last_download_round UINT63, diff --git a/storage/migrations/22_evm_nfts.up.sql b/storage/migrations/22_evm_nfts.up.sql index 7696480b8..f31b67ff3 100644 --- a/storage/migrations/22_evm_nfts.up.sql +++ b/storage/migrations/22_evm_nfts.up.sql @@ -1,7 +1,8 @@ BEGIN; ALTER TABLE chain.evm_nfts - ADD COLUMN owner oasis_addr; + ADD COLUMN owner oasis_addr, + ADD COLUMN num_transfers INT NOT NULL DEFAULT 0; -- Grant others read-only use. GRANT SELECT ON ALL TABLES IN SCHEMA chain TO PUBLIC; From 71599facbfdde4692acd82669fafa5ef469d4cdc Mon Sep 17 00:00:00 2001 From: Warren He Date: Mon, 28 Aug 2023 16:07:17 -0700 Subject: [PATCH 4/4] runtime: extract NFT transfers --- analyzer/queries/queries.go | 13 ++++++++----- analyzer/runtime/extract.go | 38 +++++++++++++++++++++++++++++-------- analyzer/runtime/runtime.go | 12 ++++++++++-- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/analyzer/queries/queries.go b/analyzer/queries/queries.go index 41a3c29bd..6d6f95595 100644 --- a/analyzer/queries/queries.go +++ b/analyzer/queries/queries.go @@ -624,12 +624,15 @@ var ( token_address = $2 AND nft_id = $3` - RuntimeEVMNFTInsert = ` - INSERT INTO chain.evm_nfts - (runtime, token_address, nft_id, last_want_download_round) + RuntimeEVMNFTUpsert = ` + INSERT INTO chain.evm_nfts AS old + (runtime, token_address, nft_id, owner, num_transfers, last_want_download_round) VALUES - ($1, $2, $3, $4) - ON CONFLICT (runtime, token_address, nft_id) DO NOTHING` + ($1, $2, $3, $4, $5, $6) + ON CONFLICT (runtime, token_address, nft_id) DO UPDATE + SET + owner = COALESCE(excluded.owner, old.owner), + num_transfers = old.num_transfers + excluded.num_transfers` RuntimeEVMNFTAnalysisStale = ` SELECT diff --git a/analyzer/runtime/extract.go b/analyzer/runtime/extract.go index a4a3a7475..203a43b9a 100644 --- a/analyzer/runtime/extract.go +++ b/analyzer/runtime/extract.go @@ -120,6 +120,11 @@ type NFTKey struct { TokenID *big.Int } +type PossibleNFT struct { + NewOwner *apiTypes.Address + NumTransfers int +} + type BlockData struct { Header nodeapi.RuntimeBlockHeader NumTransactions int // Might be different from len(TransactionData) if some transactions are malformed. @@ -130,7 +135,7 @@ type BlockData struct { AddressPreimages map[apiTypes.Address]*AddressPreimageData TokenBalanceChanges map[TokenChangeKey]*big.Int PossibleTokens map[apiTypes.Address]*evm.EVMPossibleToken // key is oasis bech32 address - PossibleNFTs map[NFTKey]struct{} + PossibleNFTs map[NFTKey]*PossibleNFT } // Function naming conventions in this file: @@ -248,11 +253,24 @@ func registerRelatedEthAddress(addressPreimages map[apiTypes.Address]*AddressPre return addr, nil } -func registerNFTExist(possibleNFTs map[NFTKey]struct{}, contractAddr apiTypes.Address, tokenID *big.Int) { +func findPossibleNFT(possibleNFTs map[NFTKey]*PossibleNFT, contractAddr apiTypes.Address, tokenID *big.Int) *PossibleNFT { key := NFTKey{contractAddr, tokenID} - if _, ok := possibleNFTs[key]; !ok { - possibleNFTs[key] = struct{}{} + possibleNFT, ok := possibleNFTs[key] + if !ok { + possibleNFT = &PossibleNFT{} + possibleNFTs[key] = possibleNFT } + return possibleNFT +} + +func registerNFTExist(nftChanges map[NFTKey]*PossibleNFT, contractAddr apiTypes.Address, tokenID *big.Int) { + findPossibleNFT(nftChanges, contractAddr, tokenID) +} + +func registerNFTTransfer(nftChanges map[NFTKey]*PossibleNFT, contractAddr apiTypes.Address, tokenID *big.Int, newOwner apiTypes.Address) { + possibleNFT := findPossibleNFT(nftChanges, contractAddr, tokenID) + possibleNFT.NumTransfers++ + possibleNFT.NewOwner = &newOwner } func findTokenChange(tokenChanges map[TokenChangeKey]*big.Int, contractAddr apiTypes.Address, accountAddr apiTypes.Address) *big.Int { @@ -284,7 +302,7 @@ func ExtractRound(blockHeader nodeapi.RuntimeBlockHeader, txrs []nodeapi.Runtime AddressPreimages: map[apiTypes.Address]*AddressPreimageData{}, TokenBalanceChanges: map[TokenChangeKey]*big.Int{}, PossibleTokens: map[apiTypes.Address]*evm.EVMPossibleToken{}, - PossibleNFTs: map[NFTKey]struct{}{}, + PossibleNFTs: map[NFTKey]*PossibleNFT{}, } // Extract info from non-tx events. @@ -930,8 +948,10 @@ func extractEvents(blockData *BlockData, relatedAccountAddresses map[apiTypes.Ad tokenID.SetBytes(tokenIDU256) fromZero := bytes.Equal(fromEthAddr, uncategorized.ZeroEthAddr) toZero := bytes.Equal(toEthAddr, uncategorized.ZeroEthAddr) + var fromAddr, toAddr apiTypes.Address if !fromZero { - fromAddr, err2 := registerRelatedEthAddress(blockData.AddressPreimages, relatedAccountAddresses, fromEthAddr) + var err2 error + fromAddr, err2 = registerRelatedEthAddress(blockData.AddressPreimages, relatedAccountAddresses, fromEthAddr) if err2 != nil { return fmt.Errorf("from: %w", err2) } @@ -939,14 +959,14 @@ func extractEvents(blockData *BlockData, relatedAccountAddresses map[apiTypes.Ad registerTokenDecrease(blockData.TokenBalanceChanges, eventAddr, fromAddr, big.NewInt(1)) } if !toZero { - toAddr, err2 := registerRelatedEthAddress(blockData.AddressPreimages, relatedAccountAddresses, toEthAddr) + var err2 error + toAddr, err2 = registerRelatedEthAddress(blockData.AddressPreimages, relatedAccountAddresses, toEthAddr) if err2 != nil { return fmt.Errorf("to: %w", err2) } eventData.RelatedAddresses[toAddr] = true registerTokenIncrease(blockData.TokenBalanceChanges, eventAddr, toAddr, big.NewInt(1)) } - // TODO: Reckon ownership. if _, ok := blockData.PossibleTokens[eventAddr]; !ok { blockData.PossibleTokens[eventAddr] = &evm.EVMPossibleToken{} } @@ -967,6 +987,8 @@ func extractEvents(blockData *BlockData, relatedAccountAddresses map[apiTypes.Ad pt.Mutated = true } registerNFTExist(blockData.PossibleNFTs, eventAddr, tokenID) + // Mints, burns, and zero-value transfers all count as transfers. + registerNFTTransfer(blockData.PossibleNFTs, eventAddr, tokenID, toAddr) eventData.EvmLogName = evmabi.ERC721.Events["Transfer"].Name eventData.EvmLogSignature = ethCommon.BytesToHash(event.Topics[0]) eventData.EvmLogParams = []*apiTypes.EvmEventParam{ diff --git a/analyzer/runtime/runtime.go b/analyzer/runtime/runtime.go index 6667e95f0..da32792c8 100644 --- a/analyzer/runtime/runtime.go +++ b/analyzer/runtime/runtime.go @@ -363,7 +363,15 @@ func (m *processor) queueDbUpdates(batch *storage.QueryBatch, data *BlockData) { } // Insert NFTs. - for key := range data.PossibleNFTs { - batch.Queue(queries.RuntimeEVMNFTInsert, m.runtime, key.TokenAddress, key.TokenID, data.Header.Round) + for key, possibleNFT := range data.PossibleNFTs { + batch.Queue( + queries.RuntimeEVMNFTUpsert, + m.runtime, + key.TokenAddress, + key.TokenID, + possibleNFT.NewOwner, + possibleNFT.NumTransfers, + data.Header.Round, + ) } }