From 53cf782a13b7f4b370109504f26c94b9a7efd8fb Mon Sep 17 00:00:00 2001 From: vanderheijden86 Date: Thu, 8 Sep 2022 00:29:25 +0200 Subject: [PATCH] feat: first version of websocket for sending mempool TXs --- .gitignore | 4 +- README.md | 13 ++--- .../datacollection/datacollection.go | 18 +++---- .../datacollection/datacollection_test.go | 24 +++++----- .../datacollection/ethclients.go | 11 +++-- example.env => cmd/datacollection/example.env | 0 cmd/websocketserver/example.env | 1 + cmd/websocketserver/main.go | 47 +++++++++++++++++++ cmd/websocketserver/main_test.go | 10 ++++ go.mod | 4 +- go.sum | 2 + 11 files changed, 100 insertions(+), 34 deletions(-) rename mempoolexplorer.go => cmd/datacollection/datacollection.go (69%) rename mempoolexplorer_test.go => cmd/datacollection/datacollection_test.go (62%) rename ethclients.go => cmd/datacollection/ethclients.go (72%) rename example.env => cmd/datacollection/example.env (100%) create mode 100644 cmd/websocketserver/example.env create mode 100644 cmd/websocketserver/main.go create mode 100644 cmd/websocketserver/main_test.go diff --git a/.gitignore b/.gitignore index ba43bb0..0714c2f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.dll *.so *.dylib +/cmd/main # Test binary, built with `go test -c` *.test @@ -18,4 +19,5 @@ go.work # Env files with sensitve data -.env \ No newline at end of file +/cmd/datacollection/.env +/cmd/websocketserver/.env diff --git a/README.md b/README.md index e6f077a..19313b6 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,23 @@ # Ethereum Mempool Explorer -Golang implementation of an ethereum mempool explorer. For now it's mainly a way to gain experience with accessing +Golang implementation of an ethereum mempool explorer. For now it's mainly a way to gain experience with accessing the ethereum mempool (pending transactions), refresh my understanding of the go programming language and work with the go-ethereum `ethclient` and `gethclient` packages. ## Setup -In the project directory, you need to place a file called `.env` in which you place your Infura key for ethereum -mainnet. You can rename the `example.env` file to `.env` and paste in your own key. +In the project's `cmd/datacollection/` and `cmd/websocketserver/` directories, you need to place a file called `. +env` in which to put your Infura key for ethereum mainnet. Easiest is to rename the `example.env` files to `.env` +and paste in your own key. ```bash INFURA_KEY= ``` - ## Running -You run the tests in `mempoolexplorer_tests.go` to see the functionality to get the mempool TX hashes and transaction -details. + +You run the tests in `datacollection_test.go` to see the functionality to get the mempool TX hashes and transaction +details. ```text === RUN TestMain_streamMemPoolTxs diff --git a/mempoolexplorer.go b/cmd/datacollection/datacollection.go similarity index 69% rename from mempoolexplorer.go rename to cmd/datacollection/datacollection.go index 4584ce9..14bc982 100644 --- a/mempoolexplorer.go +++ b/cmd/datacollection/datacollection.go @@ -1,4 +1,4 @@ -package mempoolexplorer +package datacollection import ( "context" @@ -9,11 +9,11 @@ import ( "log" ) -var missingTxs = make([]common.Hash, 0, 20) -var txs = make(map[common.Hash]*types.Transaction) +var MissingTxs = make([]common.Hash, 0, 20) +var Txs = make(map[common.Hash]*types.Transaction) -// streamMemPoolTxHashes listens to all pending TXs that underlying ethereum node receives from incoming RPC requests and other nodes -func streamMemPoolTxHashes(geth *gethclient.Client, bufferLength int) chan common.Hash { +// StreamMemPoolTxHashes listens to all pending TXs that underlying ethereum node receives from incoming RPC requests and other nodes +func StreamMemPoolTxHashes(geth *gethclient.Client, bufferLength int) chan common.Hash { pendingTxs := make(chan common.Hash, bufferLength) _, err := geth.SubscribePendingTransactions(context.Background(), pendingTxs) if err != nil { @@ -22,8 +22,8 @@ func streamMemPoolTxHashes(geth *gethclient.Client, bufferLength int) chan commo return pendingTxs } -func storeTxDetails(txHash common.Hash) { - client := createEthClient() +func StoreTxDetails(txHash common.Hash) { + client := CreateEthClient() tx, _, err := client.TransactionByHash(context.Background(), txHash) // If TX is not found, print the error, store the hash and return so the rest of the program can continue. @@ -34,9 +34,9 @@ func storeTxDetails(txHash common.Hash) { } else if err != nil { log.Fatal(err) } - txs[txHash] = tx + Txs[txHash] = tx } func storeMissingTxHashes(txHash common.Hash) { - missingTxs = append(missingTxs, txHash) + MissingTxs = append(MissingTxs, txHash) } diff --git a/mempoolexplorer_test.go b/cmd/datacollection/datacollection_test.go similarity index 62% rename from mempoolexplorer_test.go rename to cmd/datacollection/datacollection_test.go index c3e0b2c..5aa39a3 100644 --- a/mempoolexplorer_test.go +++ b/cmd/datacollection/datacollection_test.go @@ -1,4 +1,4 @@ -package mempoolexplorer +package datacollection import ( "fmt" @@ -6,8 +6,8 @@ import ( "testing" ) -func TestMempoolExplorer_streamMemPoolTxs(t *testing.T) { - pendingTxs := streamMemPoolTxHashes(createGethClient(), 100) +func TestDataCollection_streamMemPoolTxs(t *testing.T) { + pendingTxs := StreamMemPoolTxHashes(CreateGethClient(), 100) for { fmt.Println("Channel length: ", len(pendingTxs)) fmt.Println("Channel capacity: ", cap(pendingTxs)) @@ -16,21 +16,21 @@ func TestMempoolExplorer_streamMemPoolTxs(t *testing.T) { } } -func TestMempoolExplorer_storeTxDetails(t *testing.T) { +func TestDataCollection_storeTxDetails(t *testing.T) { // 0xaf745220755919ee3386ca28cc207e87388841832ee4bd67d7260b06b914af85 - storeTxDetails(common.HexToHash("0xaf745220755919ee3386ca28cc207e87388841832ee4bd67d7260b06b914af85")) + StoreTxDetails(common.HexToHash("0xaf745220755919ee3386ca28cc207e87388841832ee4bd67d7260b06b914af85")) } -func TestMempoolExplorer_storeTxDetails_Live(t *testing.T) { - pendingTxs := streamMemPoolTxHashes(createGethClient(), 10) +func TestDataCollection_storeTxDetails_Live(t *testing.T) { + pendingTxs := StreamMemPoolTxHashes(CreateGethClient(), 10) for i := 1; i < 25; i++ { currentTxHash := <-pendingTxs fmt.Println(currentTxHash) - storeTxDetails(currentTxHash) + StoreTxDetails(currentTxHash) } - fmt.Println(len(txs), " stored mempool TXs found on geth node:") - for _, tx := range txs { + fmt.Println(len(Txs), " stored mempool TXs found on geth node:") + for _, tx := range Txs { fmt.Println("------------------------------------------------------") fmt.Println(tx.Hash().Hex()) // 0x5d49fcaa394c97ec8a9c3e7bd9e8388d420fb050a52083ca52ff24b3b65bc9c2 fmt.Println(tx.Value().String()) // 10000000000000000 @@ -41,8 +41,8 @@ func TestMempoolExplorer_storeTxDetails_Live(t *testing.T) { fmt.Println(tx.To().Hex()) // 0x55fE59D8Ad77035154dDd0AD0388D09Dd4047A8e } - fmt.Println(len(missingTxs), " pending TXs not found on geth node:") - for _, txHash := range missingTxs { + fmt.Println(len(MissingTxs), " pending TXs not found on geth node:") + for _, txHash := range MissingTxs { fmt.Println(txHash.Hex()) } } diff --git a/ethclients.go b/cmd/datacollection/ethclients.go similarity index 72% rename from ethclients.go rename to cmd/datacollection/ethclients.go index 3cf37fe..1e8d4ae 100644 --- a/ethclients.go +++ b/cmd/datacollection/ethclients.go @@ -1,4 +1,4 @@ -package mempoolexplorer +package datacollection import ( "github.com/ethereum/go-ethereum/ethclient" @@ -12,11 +12,14 @@ import ( var infuraKey string func init() { - godotenv.Load() + err := godotenv.Load() + if err != nil { + log.Fatal("Error loading .env file", err) + } infuraKey = os.Getenv("INFURA_KEY") } -func createEthClient() *ethclient.Client { +func CreateEthClient() *ethclient.Client { client, err := ethclient.Dial("https://mainnet.infura.io/v3/" + infuraKey) if err != nil { log.Fatal(err) @@ -24,7 +27,7 @@ func createEthClient() *ethclient.Client { return client } -func createGethClient() *gethclient.Client { +func CreateGethClient() *gethclient.Client { rpcClient, _ := rpc.Dial("wss://mainnet.infura.io/ws/v3/" + infuraKey) client := gethclient.New(rpcClient) return client diff --git a/example.env b/cmd/datacollection/example.env similarity index 100% rename from example.env rename to cmd/datacollection/example.env diff --git a/cmd/websocketserver/example.env b/cmd/websocketserver/example.env new file mode 100644 index 0000000..0a7801b --- /dev/null +++ b/cmd/websocketserver/example.env @@ -0,0 +1 @@ +INFURA_KEY= \ No newline at end of file diff --git a/cmd/websocketserver/main.go b/cmd/websocketserver/main.go new file mode 100644 index 0000000..e3be78a --- /dev/null +++ b/cmd/websocketserver/main.go @@ -0,0 +1,47 @@ +package main + +import ( + "flag" + "fmt" + "github.com/gorilla/websocket" + "github.com/vanderheijden86/mempoolexplorer/cmd/datacollection" + "log" + "net/http" +) + +var addr = flag.String("addr", "localhost:8080", "http service address") + +var upgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, +} + +func sendMempoolTxs(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + log.Print("upgrade:", err) + return + } + defer conn.Close() + + for _, tx := range datacollection.Txs { + err = conn.WriteJSON(tx) + if err != nil { + log.Println("write:", err) + break + } + } +} + +func main() { + pendingTxs := datacollection.StreamMemPoolTxHashes(datacollection.CreateGethClient(), 10) + for i := 1; i < 25; i++ { + currentTxHash := <-pendingTxs + fmt.Println(currentTxHash) + datacollection.StoreTxDetails(currentTxHash) + } + flag.Parse() + log.SetFlags(0) + http.HandleFunc("/mempooltxs", sendMempoolTxs) + log.Fatal(http.ListenAndServe(*addr, nil)) +} diff --git a/cmd/websocketserver/main_test.go b/cmd/websocketserver/main_test.go new file mode 100644 index 0000000..67fafe3 --- /dev/null +++ b/cmd/websocketserver/main_test.go @@ -0,0 +1,10 @@ +package main + +import ( + "testing" +) + +func TestMain_main(t *testing.T) { + main() + t.Logf("bullshit") +} diff --git a/go.mod b/go.mod index cf1d046..b74ed6a 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/vanderheijden86/eth-mempool-explorer +module github.com/vanderheijden86/mempoolexplorer go 1.19 @@ -15,7 +15,7 @@ require ( github.com/go-ole/go-ole v1.2.1 // indirect github.com/go-stack/stack v1.8.0 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/gorilla/websocket v1.4.2 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/huin/goupnp v1.0.3 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect diff --git a/go.sum b/go.sum index ec740e0..7d9c838 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=