Support remote RPC client to access LevelDB (#83)

* Implement use of custom chain config to test locally

* Use leveldb-ethdb-rpc instead of leveldb path

* Implement flag for leveldb access mode

* Remove custom chain config json

* Move database client to leveldb-ethdb-rpc

* Update leveldb-ethdb-rpc version

README.md

@@ -2,11 +2,11 @@
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/eth-statediff-service)](https://goreportcard.com/report/github.com/vulcanize/eth-statediff-service)
 
->> standalone statediffing service ontop of leveldb
+>> standalone statediffing service ontop of LevelDB
 
 Purpose:
 
-Stand up a statediffing service directly on top of a go-ethereum leveldb instance.
+Stand up a statediffing service directly on top of a go-ethereum LevelDB instance.
 This service can serve historical state data over the same rpc interface as
 [statediffing geth](https://github.com/vulcanize/go-ethereum/releases/tag/v1.9.11-statediff-0.0.5) without needing to run a full node
 
@@ -46,12 +46,13 @@ make build
 
     ```toml
     [leveldb]
-        # Path to geth leveldb data
+        mode = "local"
+        # Path to geth LevelDB data
         path = "/path-to-local-geth-data/chaindata"
         ancient = "/path-to-local-geth-data/chaindata/ancient"
 
     [ethereum]
-        chainConfig = "./chain.json" # Path to custom chain  config file
+        chainConfig = "./chain.json" # Path to custom chain config file
         chainID = 41337 # Same chain ID as in chain.json
 
     [database]
@@ -73,6 +74,13 @@ make build
         ]
     ```
 
+* To use remote LevelDB RPC endpoint change the following in [config file](./environments/config.toml)
+    ```toml
+    [leveldb]
+        mode = "remote"
+        url = "http://127.0.0.1:8082/" # Remote LevelDB RPC url
+    ```
+
 ## Usage
 
 ### `serve`
@@ -106,8 +114,12 @@ An example config file:
 
 ```toml
 [leveldb]
+    mode = "local"
+    # path and ancient LevelDB paths required in local mode
     path = "/Users/user/Library/Ethereum/geth/chaindata"
     ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
+    # url for leveldb-ethdb-rpc endpoint required in remote mode
+    url = "http://127.0.0.1:8082/"
 
 [server]
     ipcPath = ".ipc"

cmd/env.go

@@ -30,8 +30,10 @@ const (
 
 	DB_CACHE_SIZE_MB   = "DB_CACHE_SIZE_MB"
 	TRIE_CACHE_SIZE_MB = "TRIE_CACHE_SIZE_MB"
+	LVLDB_MODE         = "LVLDB_MODE"
 	LVLDB_PATH         = "LVLDB_PATH"
 	LVLDB_ANCIENT      = "LVLDB_ANCIENT"
+	LVLDB_URL          = "LVLDB_URL"
 
 	STATEDIFF_PRERUN            = "STATEDIFF_PRERUN"
 	STATEDIFF_TRIE_WORKERS      = "STATEDIFF_TRIE_WORKERS"
@@ -112,8 +114,10 @@ func init() {
 	viper.BindEnv("cache.database", DB_CACHE_SIZE_MB)
 	viper.BindEnv("cache.trie", TRIE_CACHE_SIZE_MB)
 
+	viper.BindEnv("leveldb.mode", LVLDB_MODE)
 	viper.BindEnv("leveldb.path", LVLDB_PATH)
 	viper.BindEnv("leveldb.ancient", LVLDB_ANCIENT)
+	viper.BindEnv("leveldb.url", LVLDB_URL)
 
 	viper.BindEnv("prom.metrics", PROM_METRICS)
 	viper.BindEnv("prom.http", PROM_HTTP)

cmd/root.go

@@ -116,8 +116,10 @@ func init() {
 	rootCmd.PersistentFlags().String("log-level", log.InfoLevel.String(),
 		"log level (trace, debug, info, warn, error, fatal, panic")
 
+	rootCmd.PersistentFlags().String("leveldb-mode", "local", "LevelDB access mode (local, remote)")
 	rootCmd.PersistentFlags().String("leveldb-path", "", "path to primary datastore")
 	rootCmd.PersistentFlags().String("ancient-path", "", "path to ancient datastore")
+	rootCmd.PersistentFlags().String("leveldb-url", "", "url to primary leveldb-ethdb-rpc server")
 
 	rootCmd.PersistentFlags().Bool("prerun", false, "turn on prerun of toml configured ranges")
 	rootCmd.PersistentFlags().Int("service-workers", 0, "number of range requests to process concurrently")
@@ -177,8 +179,10 @@ func init() {
 	viper.BindPFlag("statediff.trieWorkers", rootCmd.PersistentFlags().Lookup("trie-workers"))
 	viper.BindPFlag("statediff.workerQueueSize", rootCmd.PersistentFlags().Lookup("worker-queue-size"))
 
+	viper.BindPFlag("leveldb.mode", rootCmd.PersistentFlags().Lookup("leveldb-mode"))
 	viper.BindPFlag("leveldb.path", rootCmd.PersistentFlags().Lookup("leveldb-path"))
 	viper.BindPFlag("leveldb.ancient", rootCmd.PersistentFlags().Lookup("ancient-path"))
+	viper.BindPFlag("leveldb.url", rootCmd.PersistentFlags().Lookup("leveldb-url"))
 
 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
 	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))

cmd/serve.go

@@ -32,7 +32,7 @@ import (
 // serveCmd represents the serve command
 var serveCmd = &cobra.Command{
 	Use:   "serve",
-	Short: "Stand up a standalone statediffing RPC service on top of leveldb",
+	Short: "Stand up a standalone statediffing RPC service on top of LevelDB",
 	Long: `Usage
 
 ./eth-statediff-service serve --config={path to toml config file}`,

cmd/util.go

@@ -19,10 +19,21 @@ type blockRange [2]uint64
 func createStateDiffService() (sd.StateDiffService, error) {
 	// load some necessary params
 	logWithCommand.Info("Loading statediff service parameters")
+	mode := viper.GetString("leveldb.mode")
 	path := viper.GetString("leveldb.path")
 	ancientPath := viper.GetString("leveldb.ancient")
-	if path == "" || ancientPath == "" {
-		logWithCommand.Fatal("require a valid eth leveldb primary datastore path and ancient datastore path")
+	url := viper.GetString("leveldb.url")
+
+	if mode == "local" {
+		if path == "" || ancientPath == "" {
+			logWithCommand.Fatal("Require a valid eth LevelDB primary datastore path and ancient datastore path")
+		}
+	} else if mode == "remote" {
+		if url == "" {
+			logWithCommand.Fatal("Require a valid RPC url for accessing LevelDB")
+		}
+	} else {
+		logWithCommand.Fatal("Invalid mode provided for LevelDB access")
 	}
 
 	nodeInfo := getEthNodeInfo()
@@ -41,17 +52,19 @@ func createStateDiffService() (sd.StateDiffService, error) {
 		logWithCommand.Fatal(err)
 	}
 
-	// create leveldb reader
-	logWithCommand.Info("Creating leveldb reader")
+	// create LevelDB reader
+	logWithCommand.Info("Creating LevelDB reader")
 	readerConf := sd.LvLDBReaderConfig{
 		TrieConfig: &trie.Config{
 			Cache:     viper.GetInt("cache.trie"),
 			Journal:   "",
 			Preimages: false,
 		},
 		ChainConfig: chainConf,
+		Mode:        mode,
 		Path:        path,
 		AncientPath: ancientPath,
+		Url:         url,
 		DBCacheSize: viper.GetInt("cache.database"),
 	}
 	lvlDBReader, err := sd.NewLvlDBReader(readerConf)

environments/config.toml

@@ -1,6 +1,8 @@
 [leveldb]
+    mode = "local"
     path = "/app/geth-rw/chaindata"
     ancient = "/app/geth-rw/chaindata/ancient"
+    url = "http://127.0.0.1:8082/"
 
 [server]
     ipcPath = ""

environments/example.toml

@@ -1,6 +1,8 @@
 [leveldb]
+    mode = "local"
     path = "/Users/user/Library/Ethereum/geth/chaindata"
     ancient = "/Users/user/Library/Ethereum/geth/chaindata/ancient"
+    url = "http://127.0.0.1:8082/"
 
 [server]
     ipcPath = ".ipc"

go.mod

@@ -3,14 +3,14 @@ module github.com/vulcanize/eth-statediff-service
 go 1.16
 
 require (
-	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
 	github.com/ethereum/go-ethereum v1.10.17
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/prometheus/client_golang v1.4.0
 	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/cobra v1.3.0
 	github.com/spf13/viper v1.10.1
 	github.com/vulcanize/go-eth-state-node-iterator v1.0.1
+	github.com/vulcanize/leveldb-ethdb-rpc v0.1.1
 )
 
 replace github.com/ethereum/go-ethereum v1.10.17 => github.com/vulcanize/go-ethereum v1.10.17-statediff-3.2.1