Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

usdc_reader benchmark #15303

Open
wants to merge 4 commits into
base: develop
Choose a base branch
from
+178 −12
Open

usdc_reader benchmark #15303

Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d494a3c
first attempt
0xnogo Nov 19, 2024
5d17af2
benchmark of usdc query
0xnogo Nov 20, 2024
5cb19cd
addressing comment + lint
0xnogo Nov 22, 2024
58f2246
more lint fixing
0xnogo Nov 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
177 changes: 168 additions & 9 deletions core/capabilities/ccip/ccip_integration_tests/usdcreader/usdcreader_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import (
"context"
"encoding/binary"
"math/big"
"testing"
"time"
Expand All @@ -11,13 +12,17 @@
gethtypes "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethclient/simulated"

"github.com/jmoiron/sqlx"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zapcore"

sel "github.com/smartcontractkit/chain-selectors"

"github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller"
ubig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big"
"github.com/smartcontractkit/chainlink/v2/core/utils/testutils/heavyweight"

"github.com/smartcontractkit/chainlink-ccip/pkg/contractreader"
"github.com/smartcontractkit/chainlink-ccip/pkg/reader"
cciptypes "github.com/smartcontractkit/chainlink-ccip/pkg/types/ccipocr3"
Expand All @@ -27,7 +32,6 @@
evmconfig "github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/configs/evm"
"github.com/smartcontractkit/chainlink/v2/core/chains/evm/client"
"github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker"
"github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller"
"github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils"
"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/usdc_reader_tester"
"github.com/smartcontractkit/chainlink/v2/core/internal/testutils"
Expand All @@ -37,6 +41,8 @@
evmtypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/types"
)

const ChainID = 1337

func Test_USDCReader_MessageHashes(t *testing.T) {
finalityDepth := 5

Expand All @@ -48,7 +54,7 @@
polygonChain := cciptypes.ChainSelector(sel.POLYGON_MAINNET.Selector)
polygonDomainCCTP := reader.CCTPDestDomains[uint64(polygonChain)]

ts := testSetup(ctx, t, ethereumChain, evmconfig.USDCReaderConfig, finalityDepth)
ts := testSetup(ctx, t, ethereumChain, evmconfig.USDCReaderConfig, finalityDepth, false)

usdcReader, err := reader.NewUSDCMessageReader(
ctx,
Expand Down Expand Up @@ -201,7 +207,147 @@
})
}
}
func Benchmark_MessageHashes(b *testing.B) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we add the gobench results above the test as a comment? (as a reference)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ideally we want to track the performance regressions somehow (maybe it's not in the scope for 1st iteration, but wanted to call it out anyway). Some time ago, I saw this tool https://pkg.go.dev/golang.org/x/perf/cmd/benchstat, but not sure how much it fits our use case

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree, benchmark tests provide very few impact if not compared. We could add a tool like benchstat in the CI and fail if we noticed a reg.

finalityDepth := 5

// Adding a new parameter: tokenCount
testCases := []struct {
name string
msgCount int
startNonce uint64
tokenCount int
}{
{"Small_Dataset", 100, 1, 5},
{"Medium_Dataset", 10_000, 1, 10},
{"Large_Dataset", 100_000, 1, 50},
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice, was there any major difference between these 3 datasets?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

// Benchmark_MessageHashes/Small_Dataset-14       3723        272421 ns/op       126949 B/op      2508 allocs/op
// Benchmark_MessageHashes/Medium_Dataset-14       196       6164706 ns/op      1501435 B/op     20274 allocs/op
// Benchmark_MessageHashes/Large_Dataset-14          7     163930268 ns/op     37193160 B/op    463954 allocs/op

It's going linearly. I think that the remappings we have (first the messageSentEvents and then out) are a big offender (apart for the GetQuery).

}

for _, tc := range testCases {
b.Run(tc.name, func(b *testing.B) {
ctx := testutils.Context(b)
sourceChain := cciptypes.ChainSelector(sel.ETHEREUM_MAINNET_OPTIMISM_1.Selector)
sourceDomainCCTP := reader.CCTPDestDomains[uint64(sourceChain)]
destChain := cciptypes.ChainSelector(sel.AVALANCHE_MAINNET.Selector)
destDomainCCTP := reader.CCTPDestDomains[uint64(destChain)]

ts := testSetup(ctx, b, sourceChain, evmconfig.USDCReaderConfig, finalityDepth, true)

usdcReader, err := reader.NewUSDCMessageReader(
ctx,
logger.TestLogger(b),
map[cciptypes.ChainSelector]pluginconfig.USDCCCTPTokenConfig{
sourceChain: {
SourceMessageTransmitterAddr: ts.contractAddr.String(),
},
},
map[cciptypes.ChainSelector]contractreader.ContractReaderFacade{
sourceChain: ts.reader,
})
require.NoError(b, err)

// Populate the database with the specified number of logs
populateDatabase(b, ts, sourceChain, sourceDomainCCTP, destDomainCCTP, tc.startNonce, tc.msgCount, finalityDepth)

// Create a map of tokens to query for, with the specified tokenCount
tokens := make(map[reader.MessageTokenID]cciptypes.RampTokenAmount)
for i := 1; i <= tc.tokenCount; i++ {
tokens[reader.NewMessageTokenID(cciptypes.SeqNum(i), 1)] = cciptypes.RampTokenAmount{

Check failure on line 254 in core/capabilities/ccip/ccip_integration_tests/usdcreader/usdcreader_test.go

View workflow job for this annotation

GitHub Actions / lint 

G115: integer overflow conversion int -> uint64 (gosec)
ExtraData: reader.NewSourceTokenDataPayload(tc.startNonce+uint64(i), sourceDomainCCTP).ToBytes(),

Check failure on line 255 in core/capabilities/ccip/ccip_integration_tests/usdcreader/usdcreader_test.go

View workflow job for this annotation

GitHub Actions / lint 

G115: integer overflow conversion int -> uint64 (gosec)
}
}

b.ResetTimer()

for i := 0; i < b.N; i++ {
hashes, err := usdcReader.MessageHashes(ctx, sourceChain, destChain, tokens)
require.NoError(b, err)
require.Len(b, hashes, tc.tokenCount) // Ensure the number of matches is as expected
}
})
}
}

func populateDatabase(b *testing.B,
testEnv *testSetupData,
source cciptypes.ChainSelector,
sourceDomainCCTP uint32,
destDomainCCTP uint32,
startNonce uint64,
numOfMessages int,
finalityDepth int) {
ctx := testutils.Context(b)

abi, err := usdc_reader_tester.USDCReaderTesterMetaData.GetAbi()
require.NoError(b, err)

var logs []logpoller.Log
messageSentEventSig := abi.Events["MessageSent"].ID
require.NoError(b, err)
messageTransmitterAddress := testEnv.contractAddr

for i := 0; i < numOfMessages; i++ {
nonce := startNonce + uint64(i)

Check failure on line 289 in core/capabilities/ccip/ccip_integration_tests/usdcreader/usdcreader_test.go

View workflow job for this annotation

GitHub Actions / lint 

G115: integer overflow conversion int -> uint64 (gosec)

// Pack the message following exact CCTP format
var buf []byte
0xnogo marked this conversation as resolved.
Show resolved Hide resolved
// _msgVersion (uint32)
buf = binary.BigEndian.AppendUint32(buf, reader.CCTPMessageVersion)
// _msgSourceDomain (uint32)
buf = binary.BigEndian.AppendUint32(buf, sourceDomainCCTP)
// _msgDestinationDomain (uint32)
buf = binary.BigEndian.AppendUint32(buf, destDomainCCTP)
// _msgNonce (uint64)
buf = binary.BigEndian.AppendUint64(buf, nonce)
// First 12 bytes of the sender address are always empty for EVM
senderBytes := [12]byte{}
buf = append(buf, senderBytes[:]...)

// Convert to 32 bytes
var message [32]byte
copy(message[:], buf)

// Encode the data following Solidity's encoding for bytes
data := make([]byte, 0)

// This says "actual data starts 32 bytes from the beginning"
offsetBytes := make([]byte, 32)
binary.BigEndian.PutUint64(offsetBytes[24:], 32)
data = append(data, offsetBytes...)

// This is the length of our message
lengthBytes := make([]byte, 32)
binary.BigEndian.PutUint64(lengthBytes[24:], uint64(len(message)))
data = append(data, lengthBytes...)

// Add message
data = append(data, message[:]...)

// Create topics array with just the event signature
topics := [][]byte{
messageSentEventSig[:], // Topic[0] is event signature
}

// Create log entry
logs = append(logs, logpoller.Log{
EvmChainId: ubig.New(big.NewInt(int64(uint64(source)))),

Check failure on line 332 in core/capabilities/ccip/ccip_integration_tests/usdcreader/usdcreader_test.go

View workflow job for this annotation

GitHub Actions / lint 

G115: integer overflow conversion uint64 -> int64 (gosec)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmm, is that right? We should use chain_id, not the chain selectors which is stored in source, right?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The simulated backend is created with the source chain. The only part we are using the ChainID is for the transactor creation (as we don't have any other choice here, geth is complaining about it).

LogIndex: int64(i + 1),
BlockHash: utils.NewHash(),
BlockNumber: int64(i + 1),
BlockTimestamp: time.Now(),
EventSig: messageSentEventSig,
Topics: topics,
Address: messageTransmitterAddress,
TxHash: utils.NewHash(),
Data: data,
CreatedAt: time.Now(),
})
}

require.NoError(b, testEnv.orm.InsertLogs(ctx, logs))
require.NoError(b, testEnv.orm.InsertBlock(ctx, utils.RandomHash(), int64(numOfMessages+finalityDepth), time.Now(), int64(numOfMessages+finalityDepth)))
}

// we might want to use batching (evm/batching or evm/batching) but might be slow
func emitMessageSent(t *testing.T, testEnv *testSetupData, source, dest uint32, nonce uint64) {
payload := utils.RandomBytes32()
_, err := testEnv.contract.EmitMessageSent(
Expand All @@ -219,9 +365,7 @@
testEnv.sb.Commit()
}

func testSetup(ctx context.Context, t *testing.T, readerChain cciptypes.ChainSelector, cfg evmtypes.ChainReaderConfig, depth int) *testSetupData {
const chainID = 1337

func testSetup(ctx context.Context, t testing.TB, readerChain cciptypes.ChainSelector, cfg evmtypes.ChainReaderConfig, depth int, useHeavyDB bool) *testSetupData {
// Generate a new key pair for the simulated account
privateKey, err := crypto.GenerateKey()
assert.NoError(t, err)
Expand All @@ -232,7 +376,7 @@
simulatedBackend := simulated.NewBackend(alloc, simulated.WithBlockGasLimit(0))
// Create a transactor

auth, err := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(chainID))
auth, err := bind.NewKeyedTransactorWithChainID(privateKey, big.NewInt(ChainID))
assert.NoError(t, err)
auth.GasLimit = uint64(0)

Expand All @@ -248,7 +392,15 @@

lggr := logger.TestLogger(t)
lggr.SetLogLevel(zapcore.ErrorLevel)
db := pgtest.NewSqlxDB(t)

// Parameterize database selection
var db *sqlx.DB
if useHeavyDB {
_, db = heavyweight.FullTestDBV2(t, nil) // Use heavyweight database for benchmarks
} else {
db = pgtest.NewSqlxDB(t) // Use simple in-memory DB for tests
}

lpOpts := logpoller.Opts{
PollPeriod: time.Millisecond,
FinalityDepth: int64(depth),
Expand All @@ -258,7 +410,10 @@
}
cl := client.NewSimulatedBackendClient(t, simulatedBackend, big.NewInt(0).SetUint64(uint64(readerChain)))
headTracker := headtracker.NewSimulatedHeadTracker(cl, lpOpts.UseFinalityTag, lpOpts.FinalityDepth)
lp := logpoller.NewLogPoller(logpoller.NewORM(big.NewInt(0).SetUint64(uint64(readerChain)), db, lggr),
orm := logpoller.NewORM(big.NewInt(0).SetUint64(uint64(readerChain)), db, lggr)

lp := logpoller.NewLogPoller(
orm,
cl,
lggr,
headTracker,
Expand All @@ -285,6 +440,8 @@
auth: auth,
cl: cl,
reader: cr,
orm: orm,
db: db,
lp: lp,
}
}
Expand All @@ -296,5 +453,7 @@
auth *bind.TransactOpts
cl client.Client
reader types.ContractReader
orm logpoller.ORM
db *sqlx.DB
lp logpoller.LogPoller
}
Loading