From 79983a63e95f5a08a09e6783c921021351ed90ae Mon Sep 17 00:00:00 2001 From: Jason Paryani Date: Wed, 27 Oct 2021 16:10:05 -0400 Subject: [PATCH] Flashbots change up to v0.3 --- .github/workflows/go.yml | 64 ++ README.md | 369 +---------- README.original.md | 363 +++++++++++ cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 7 + core/tx_pool.go | 64 +- core/types/transaction.go | 8 + eth/api_backend.go | 4 + infra/Dockerfile.node | 23 + infra/Dockerfile.updater | 23 + infra/mev-geth-nodes-arm64.yaml | 979 +++++++++++++++++++++++++++++ infra/mev-geth-nodes-x86-64.yaml | 972 ++++++++++++++++++++++++++++ infra/mev-geth-updater-arm64.yaml | 749 ++++++++++++++++++++++ infra/mev-geth-updater-x86-64.yaml | 737 ++++++++++++++++++++++ infra/start-mev-geth-node.sh | 96 +++ infra/start-mev-geth-updater.sh | 181 ++++++ internal/ethapi/api.go | 51 ++ internal/ethapi/backend.go | 6 + internal/web3ext/web3ext.go | 5 + les/api_backend.go | 3 + light/txpool.go | 11 + miner/miner.go | 31 +- miner/multi_worker.go | 118 ++++ miner/worker.go | 414 +++++++++++- miner/worker_test.go | 5 +- 26 files changed, 4899 insertions(+), 386 deletions(-) create mode 100644 .github/workflows/go.yml create mode 100644 README.original.md create mode 100644 infra/Dockerfile.node create mode 100644 infra/Dockerfile.updater create mode 100644 infra/mev-geth-nodes-arm64.yaml create mode 100644 infra/mev-geth-nodes-x86-64.yaml create mode 100644 infra/mev-geth-updater-arm64.yaml create mode 100644 infra/mev-geth-updater-x86-64.yaml create mode 100755 infra/start-mev-geth-node.sh create mode 100755 infra/start-mev-geth-updater.sh create mode 100644 miner/multi_worker.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 000000000000..3fc1f2ff8c68 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,64 @@ +name: Go + +on: + push: + pull_request: + branches: [ master ] + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.13 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Test + run: go test ./core ./miner/... ./internal/ethapi/... ./les/... + + - name: Build + run: make geth + + e2e: + name: End to End + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ^1.13 + id: go + + - name: Use Node.js 12.x + uses: actions/setup-node@v1 + with: + node-version: 12.x + + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Build + run: make geth + + - name: Check out the e2e code repo + uses: actions/checkout@v2 + with: + repository: flashbots/mev-geth-demo + path: e2e + + - run: cd e2e && yarn install + - run: | + cd e2e + GETH=`pwd`/../build/bin/geth ./run.sh & + sleep 15 + yarn run demo-simple + yarn run demo-contract diff --git a/README.md b/README.md index 81b7215ba85d..8fe7df9e4255 100644 --- a/README.md +++ b/README.md @@ -1,363 +1,30 @@ -## Go Ethereum +# MEV-geth -Official Golang implementation of the Ethereum protocol. +This is a fork of go-ethereum, [the original README is here](README.original.md). -[![API Reference]( -https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 -)](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc) -[![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) -[![Travis](https://travis-ci.com/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.com/ethereum/go-ethereum) -[![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv) +Flashbots is a research and development organization formed to mitigate the negative externalities and existential risks posed by miner-extractable value (MEV) to smart-contract blockchains. We propose a permissionless, transparent, and fair ecosystem for MEV extraction that reinforce the Ethereum ideals. -Automated builds are available for stable releases and the unstable master branch. Binary -archives are published at https://geth.ethereum.org/downloads/. +## Quick start -## Building the source - -For prerequisites and detailed build instructions please read the [Installation Instructions](https://geth.ethereum.org/docs/install-and-build/installing-geth). - -Building `geth` requires both a Go (version 1.14 or later) and a C compiler. You can install -them using your favourite package manager. Once the dependencies are installed, run - -```shell -make geth -``` - -or, to build the full suite of utilities: - -```shell -make all -``` - -## Executables - -The go-ethereum project comes with several wrappers/executables found in the `cmd` -directory. - -| Command | Description | -| :-----------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/interface/command-line-options) for command line options. | -| `clef` | Stand-alone signing tool, which can be used as a backend signer for `geth`. | -| `devp2p` | Utilities to interact with nodes on the networking layer, without running a full blockchain. | -| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | -| `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | -| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | -| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://eth.wiki/en/fundamentals/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | -| `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | - -## Running `geth` - -Going through all the possible command line flags is out of scope here (please consult our -[CLI Wiki page](https://geth.ethereum.org/docs/interface/command-line-options)), -but we've enumerated a few common parameter combos to get you up to speed quickly -on how you can run your own `geth` instance. - -### Full node on the main Ethereum network - -By far the most common scenario is people wanting to simply interact with the Ethereum -network: create accounts; transfer funds; deploy and interact with contracts. For this -particular use-case the user doesn't care about years-old historical data, so we can -sync quickly to the current state of the network. To do so: - -```shell -$ geth console -``` - -This command will: - * Start `geth` in snap sync mode (default, can be changed with the `--syncmode` flag), - causing it to download more data in exchange for avoiding processing the entire history - of the Ethereum network, which is very CPU intensive. - * Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), - (via the trailing `console` subcommand) through which you can interact using [`web3` methods](https://github.com/ChainSafe/web3.js/blob/0.20.7/DOCUMENTATION.md) - (note: the `web3` version bundled within `geth` is very old, and not up to date with official docs), - as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server). - This tool is optional and if you leave it out you can always attach to an already running - `geth` instance with `geth attach`. - -### A Full node on the Görli test network - -Transitioning towards developers, if you'd like to play around with creating Ethereum -contracts, you almost certainly would like to do that without any real money involved until -you get the hang of the entire system. In other words, instead of attaching to the main -network, you want to join the **test** network with your node, which is fully equivalent to -the main network, but with play-Ether only. - -```shell -$ geth --goerli console -``` - -The `console` subcommand has the exact same meaning as above and they are equally -useful on the testnet too. Please, see above for their explanations if you've skipped here. - -Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a bit: - - * Instead of connecting the main Ethereum network, the client will connect to the Görli - test network, which uses different P2P bootnodes, different network IDs and genesis - states. - * Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth` - will nest itself one level deeper into a `goerli` subfolder (`~/.ethereum/goerli` on - Linux). Note, on OSX and Linux this also means that attaching to a running testnet node - requires the use of a custom endpoint since `geth attach` will try to attach to a - production node endpoint by default, e.g., - `geth attach /goerli/geth.ipc`. Windows users are not affected by - this. - -*Note: Although there are some internal protective measures to prevent transactions from -crossing over between the main network and test network, you should make sure to always -use separate accounts for play-money and real-money. Unless you manually move -accounts, `geth` will by default correctly separate the two networks and will not make any -accounts available between them.* - -### Full node on the Rinkeby test network - -Go Ethereum also supports connecting to the older proof-of-authority based test network -called [*Rinkeby*](https://www.rinkeby.io) which is operated by members of the community. - -```shell -$ geth --rinkeby console -``` - -### Full node on the Ropsten test network - -In addition to Görli and Rinkeby, Geth also supports the ancient Ropsten testnet. The -Ropsten test network is based on the Ethash proof-of-work consensus algorithm. As such, -it has certain extra overhead and is more susceptible to reorganization attacks due to the -network's low difficulty/security. - -```shell -$ geth --ropsten console ``` - -*Note: Older Geth configurations store the Ropsten database in the `testnet` subdirectory.* - -### Configuration - -As an alternative to passing the numerous flags to the `geth` binary, you can also pass a -configuration file via: - -```shell -$ geth --config /path/to/your_config.toml -``` - -To get an idea how the file should look like you can use the `dumpconfig` subcommand to -export your existing configuration: - -```shell -$ geth --your-favourite-flags dumpconfig -``` - -*Note: This works only with `geth` v1.6.0 and above.* - -#### Docker quick start - -One of the quickest ways to get Ethereum up and running on your machine is by using -Docker: - -```shell -docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \ - -p 8545:8545 -p 30303:30303 \ - ethereum/client-go -``` - -This will start `geth` in snap-sync mode with a DB memory allowance of 1GB just as the -above command does. It will also create a persistent volume in your home directory for -saving your blockchain as well as map the default ports. There is also an `alpine` tag -available for a slim version of the image. - -Do not forget `--http.addr 0.0.0.0`, if you want to access RPC from other containers -and/or hosts. By default, `geth` binds to the local interface and RPC endpoints are not -accessible from the outside. - -### Programmatically interfacing `geth` nodes - -As a developer, sooner rather than later you'll want to start interacting with `geth` and the -Ethereum network via your own programs and not manually through the console. To aid -this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://eth.wiki/json-rpc/API) -and [`geth` specific APIs](https://geth.ethereum.org/docs/rpc/server)). -These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based -platforms, and named pipes on Windows). - -The IPC interface is enabled by default and exposes all the APIs supported by `geth`, -whereas the HTTP and WS interfaces need to manually be enabled and only expose a -subset of APIs due to security reasons. These can be turned on/off and configured as -you'd expect. - -HTTP based JSON-RPC API options: - - * `--http` Enable the HTTP-RPC server - * `--http.addr` HTTP-RPC server listening interface (default: `localhost`) - * `--http.port` HTTP-RPC server listening port (default: `8545`) - * `--http.api` API's offered over the HTTP-RPC interface (default: `eth,net,web3`) - * `--http.corsdomain` Comma separated list of domains from which to accept cross origin requests (browser enforced) - * `--ws` Enable the WS-RPC server - * `--ws.addr` WS-RPC server listening interface (default: `localhost`) - * `--ws.port` WS-RPC server listening port (default: `8546`) - * `--ws.api` API's offered over the WS-RPC interface (default: `eth,net,web3`) - * `--ws.origins` Origins from which to accept websockets requests - * `--ipcdisable` Disable the IPC-RPC server - * `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,shh,txpool,web3`) - * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it) - -You'll need to use your own programming environments' capabilities (libraries, tools, etc) to -connect via HTTP, WS or IPC to a `geth` node configured with the above flags and you'll -need to speak [JSON-RPC](https://www.jsonrpc.org/specification) on all transports. You -can reuse the same connection for multiple requests! - -**Note: Please understand the security implications of opening up an HTTP/WS based -transport before doing so! Hackers on the internet are actively trying to subvert -Ethereum nodes with exposed APIs! Further, all browser tabs can access locally -running web servers, so malicious web pages could try to subvert locally available -APIs!** - -### Operating a private network - -Maintaining your own private network is more involved as a lot of configurations taken for -granted in the official networks need to be manually set up. - -#### Defining the private genesis state - -First, you'll need to create the genesis state of your networks, which all nodes need to be -aware of and agree upon. This consists of a small JSON file (e.g. call it `genesis.json`): - -```json -{ - "config": { - "chainId": , - "homesteadBlock": 0, - "eip150Block": 0, - "eip155Block": 0, - "eip158Block": 0, - "byzantiumBlock": 0, - "constantinopleBlock": 0, - "petersburgBlock": 0, - "istanbulBlock": 0, - "berlinBlock": 0, - "londonBlock": 0 - }, - "alloc": {}, - "coinbase": "0x0000000000000000000000000000000000000000", - "difficulty": "0x20000", - "extraData": "", - "gasLimit": "0x2fefd8", - "nonce": "0x0000000000000042", - "mixhash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "timestamp": "0x00" -} -``` - -The above fields should be fine for most purposes, although we'd recommend changing -the `nonce` to some random value so you prevent unknown remote nodes from being able -to connect to you. If you'd like to pre-fund some accounts for easier testing, create -the accounts and populate the `alloc` field with their addresses. - -```json -"alloc": { - "0x0000000000000000000000000000000000000001": { - "balance": "111111111" - }, - "0x0000000000000000000000000000000000000002": { - "balance": "222222222" - } -} -``` - -With the genesis state defined in the above JSON file, you'll need to initialize **every** -`geth` node with it prior to starting it up to ensure all blockchain parameters are correctly -set: - -```shell -$ geth init path/to/genesis.json -``` - -#### Creating the rendezvous point - -With all nodes that you want to run initialized to the desired genesis state, you'll need to -start a bootstrap node that others can use to find each other in your network and/or over -the internet. The clean way is to configure and run a dedicated bootnode: - -```shell -$ bootnode --genkey=boot.key -$ bootnode --nodekey=boot.key -``` - -With the bootnode online, it will display an [`enode` URL](https://eth.wiki/en/fundamentals/enode-url-format) -that other nodes can use to connect to it and exchange peer information. Make sure to -replace the displayed IP address information (most probably `[::]`) with your externally -accessible IP to get the actual `enode` URL. - -*Note: You could also use a full-fledged `geth` node as a bootnode, but it's the less -recommended way.* - -#### Starting up your member nodes - -With the bootnode operational and externally reachable (you can try -`telnet ` to ensure it's indeed reachable), start every subsequent `geth` -node pointed to the bootnode for peer discovery via the `--bootnodes` flag. It will -probably also be desirable to keep the data directory of your private network separated, so -do also specify a custom `--datadir` flag. - -```shell -$ geth --datadir=path/to/custom/data/folder --bootnodes= -``` - -*Note: Since your network will be completely cut off from the main and test networks, you'll -also need to configure a miner to process transactions and create new blocks for you.* - -#### Running a private miner - -Mining on the public Ethereum network is a complex task as it's only feasible using GPUs, -requiring an OpenCL or CUDA enabled `ethminer` instance. For information on such a -setup, please consult the [EtherMining subreddit](https://www.reddit.com/r/EtherMining/) -and the [ethminer](https://github.com/ethereum-mining/ethminer) repository. - -In a private network setting, however a single CPU miner instance is more than enough for -practical purposes as it can produce a stable stream of blocks at the correct intervals -without needing heavy resources (consider running on a single thread, no need for multiple -ones either). To start a `geth` instance for mining, run it with all your usual flags, extended -by: - -```shell -$ geth --mine --miner.threads=1 --miner.etherbase=0x0000000000000000000000000000000000000000 +git clone https://github.com/flashbots/mev-geth +cd mev-geth +make geth ``` -Which will start mining blocks and transactions on a single CPU thread, crediting all -proceedings to the account specified by `--miner.etherbase`. You can further tune the mining -by changing the default gas limit blocks converge to (`--miner.targetgaslimit`) and the price -transactions are accepted at (`--miner.gasprice`). - -## Contribution - -Thank you for considering to help out with the source code! We welcome contributions -from anyone on the internet, and are grateful for even the smallest of fixes! - -If you'd like to contribute to go-ethereum, please fork, fix, commit and send a pull request -for the maintainers to review and merge into the main code base. If you wish to submit -more complex changes though, please check up with the core devs first on [our Discord Server](https://discord.gg/invite/nthXNEv) -to ensure those changes are in line with the general philosophy of the project and/or get -some early feedback which can make both your efforts much lighter as well as our review -and merge procedures quick and simple. - -Please make sure your contributions adhere to our coding guidelines: +See [here](https://geth.ethereum.org/docs/install-and-build/installing-geth#build-go-ethereum-from-source-code) for further info on building MEV-geth from source. - * Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) - guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)). - * Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) - guidelines. - * Pull requests need to be based on and opened against the `master` branch. - * Commit messages should be prefixed with the package(s) they modify. - * E.g. "eth, rpc: make trace configs optional" +## Documentation -Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/devguide) -for more details on configuring your environment, managing project dependencies, and -testing procedures. +See [here](https://docs.flashbots.net) for Flashbots documentation. -## License +| Version | Spec | +| ------- | ------------------------------------------------------------------------------------------- | +| v0.4 | [MEV-Geth Spec v0.4](https://docs.flashbots.net/flashbots-auction/miners/mev-geth-spec/v04) | +| v0.3 | [MEV-Geth Spec v0.3](https://docs.flashbots.net/flashbots-auction/miners/mev-geth-spec/v03) | +| v0.2 | [MEV-Geth Spec v0.2](https://docs.flashbots.net/flashbots-auction/miners/mev-geth-spec/v02) | +| v0.1 | [MEV-Geth Spec v0.1](https://docs.flashbots.net/flashbots-auction/miners/mev-geth-spec/v01) | -The go-ethereum library (i.e. all code outside of the `cmd` directory) is licensed under the -[GNU Lesser General Public License v3.0](https://www.gnu.org/licenses/lgpl-3.0.en.html), -also included in our repository in the `COPYING.LESSER` file. +### Feature requests and bug reports -The go-ethereum binaries (i.e. all code inside of the `cmd` directory) is licensed under the -[GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html), also -included in our repository in the `COPYING` file. +If you are a user of MEV-Geth and have suggestions on how to make integration with your current setup easier, or would like to submit a bug report, we encourage you to open an issue in this repository with the `enhancement` or `bug` labels respectively. If you need help getting started, please ask in the dedicated [#⛏️miners](https://discord.gg/rcgADN9qFX) channel in our Discord. diff --git a/README.original.md b/README.original.md new file mode 100644 index 000000000000..81b7215ba85d --- /dev/null +++ b/README.original.md @@ -0,0 +1,363 @@ +## Go Ethereum + +Official Golang implementation of the Ethereum protocol. + +[![API Reference]( +https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 +)](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc) +[![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) +[![Travis](https://travis-ci.com/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.com/ethereum/go-ethereum) +[![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv) + +Automated builds are available for stable releases and the unstable master branch. Binary +archives are published at https://geth.ethereum.org/downloads/. + +## Building the source + +For prerequisites and detailed build instructions please read the [Installation Instructions](https://geth.ethereum.org/docs/install-and-build/installing-geth). + +Building `geth` requires both a Go (version 1.14 or later) and a C compiler. You can install +them using your favourite package manager. Once the dependencies are installed, run + +```shell +make geth +``` + +or, to build the full suite of utilities: + +```shell +make all +``` + +## Executables + +The go-ethereum project comes with several wrappers/executables found in the `cmd` +directory. + +| Command | Description | +| :-----------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/interface/command-line-options) for command line options. | +| `clef` | Stand-alone signing tool, which can be used as a backend signer for `geth`. | +| `devp2p` | Utilities to interact with nodes on the networking layer, without running a full blockchain. | +| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | +| `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | +| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | +| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://eth.wiki/en/fundamentals/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | +| `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | + +## Running `geth` + +Going through all the possible command line flags is out of scope here (please consult our +[CLI Wiki page](https://geth.ethereum.org/docs/interface/command-line-options)), +but we've enumerated a few common parameter combos to get you up to speed quickly +on how you can run your own `geth` instance. + +### Full node on the main Ethereum network + +By far the most common scenario is people wanting to simply interact with the Ethereum +network: create accounts; transfer funds; deploy and interact with contracts. For this +particular use-case the user doesn't care about years-old historical data, so we can +sync quickly to the current state of the network. To do so: + +```shell +$ geth console +``` + +This command will: + * Start `geth` in snap sync mode (default, can be changed with the `--syncmode` flag), + causing it to download more data in exchange for avoiding processing the entire history + of the Ethereum network, which is very CPU intensive. + * Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), + (via the trailing `console` subcommand) through which you can interact using [`web3` methods](https://github.com/ChainSafe/web3.js/blob/0.20.7/DOCUMENTATION.md) + (note: the `web3` version bundled within `geth` is very old, and not up to date with official docs), + as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server). + This tool is optional and if you leave it out you can always attach to an already running + `geth` instance with `geth attach`. + +### A Full node on the Görli test network + +Transitioning towards developers, if you'd like to play around with creating Ethereum +contracts, you almost certainly would like to do that without any real money involved until +you get the hang of the entire system. In other words, instead of attaching to the main +network, you want to join the **test** network with your node, which is fully equivalent to +the main network, but with play-Ether only. + +```shell +$ geth --goerli console +``` + +The `console` subcommand has the exact same meaning as above and they are equally +useful on the testnet too. Please, see above for their explanations if you've skipped here. + +Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a bit: + + * Instead of connecting the main Ethereum network, the client will connect to the Görli + test network, which uses different P2P bootnodes, different network IDs and genesis + states. + * Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth` + will nest itself one level deeper into a `goerli` subfolder (`~/.ethereum/goerli` on + Linux). Note, on OSX and Linux this also means that attaching to a running testnet node + requires the use of a custom endpoint since `geth attach` will try to attach to a + production node endpoint by default, e.g., + `geth attach /goerli/geth.ipc`. Windows users are not affected by + this. + +*Note: Although there are some internal protective measures to prevent transactions from +crossing over between the main network and test network, you should make sure to always +use separate accounts for play-money and real-money. Unless you manually move +accounts, `geth` will by default correctly separate the two networks and will not make any +accounts available between them.* + +### Full node on the Rinkeby test network + +Go Ethereum also supports connecting to the older proof-of-authority based test network +called [*Rinkeby*](https://www.rinkeby.io) which is operated by members of the community. + +```shell +$ geth --rinkeby console +``` + +### Full node on the Ropsten test network + +In addition to Görli and Rinkeby, Geth also supports the ancient Ropsten testnet. The +Ropsten test network is based on the Ethash proof-of-work consensus algorithm. As such, +it has certain extra overhead and is more susceptible to reorganization attacks due to the +network's low difficulty/security. + +```shell +$ geth --ropsten console +``` + +*Note: Older Geth configurations store the Ropsten database in the `testnet` subdirectory.* + +### Configuration + +As an alternative to passing the numerous flags to the `geth` binary, you can also pass a +configuration file via: + +```shell +$ geth --config /path/to/your_config.toml +``` + +To get an idea how the file should look like you can use the `dumpconfig` subcommand to +export your existing configuration: + +```shell +$ geth --your-favourite-flags dumpconfig +``` + +*Note: This works only with `geth` v1.6.0 and above.* + +#### Docker quick start + +One of the quickest ways to get Ethereum up and running on your machine is by using +Docker: + +```shell +docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \ + -p 8545:8545 -p 30303:30303 \ + ethereum/client-go +``` + +This will start `geth` in snap-sync mode with a DB memory allowance of 1GB just as the +above command does. It will also create a persistent volume in your home directory for +saving your blockchain as well as map the default ports. There is also an `alpine` tag +available for a slim version of the image. + +Do not forget `--http.addr 0.0.0.0`, if you want to access RPC from other containers +and/or hosts. By default, `geth` binds to the local interface and RPC endpoints are not +accessible from the outside. + +### Programmatically interfacing `geth` nodes + +As a developer, sooner rather than later you'll want to start interacting with `geth` and the +Ethereum network via your own programs and not manually through the console. To aid +this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://eth.wiki/json-rpc/API) +and [`geth` specific APIs](https://geth.ethereum.org/docs/rpc/server)). +These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based +platforms, and named pipes on Windows). + +The IPC interface is enabled by default and exposes all the APIs supported by `geth`, +whereas the HTTP and WS interfaces need to manually be enabled and only expose a +subset of APIs due to security reasons. These can be turned on/off and configured as +you'd expect. + +HTTP based JSON-RPC API options: + + * `--http` Enable the HTTP-RPC server + * `--http.addr` HTTP-RPC server listening interface (default: `localhost`) + * `--http.port` HTTP-RPC server listening port (default: `8545`) + * `--http.api` API's offered over the HTTP-RPC interface (default: `eth,net,web3`) + * `--http.corsdomain` Comma separated list of domains from which to accept cross origin requests (browser enforced) + * `--ws` Enable the WS-RPC server + * `--ws.addr` WS-RPC server listening interface (default: `localhost`) + * `--ws.port` WS-RPC server listening port (default: `8546`) + * `--ws.api` API's offered over the WS-RPC interface (default: `eth,net,web3`) + * `--ws.origins` Origins from which to accept websockets requests + * `--ipcdisable` Disable the IPC-RPC server + * `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,shh,txpool,web3`) + * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it) + +You'll need to use your own programming environments' capabilities (libraries, tools, etc) to +connect via HTTP, WS or IPC to a `geth` node configured with the above flags and you'll +need to speak [JSON-RPC](https://www.jsonrpc.org/specification) on all transports. You +can reuse the same connection for multiple requests! + +**Note: Please understand the security implications of opening up an HTTP/WS based +transport before doing so! Hackers on the internet are actively trying to subvert +Ethereum nodes with exposed APIs! Further, all browser tabs can access locally +running web servers, so malicious web pages could try to subvert locally available +APIs!** + +### Operating a private network + +Maintaining your own private network is more involved as a lot of configurations taken for +granted in the official networks need to be manually set up. + +#### Defining the private genesis state + +First, you'll need to create the genesis state of your networks, which all nodes need to be +aware of and agree upon. This consists of a small JSON file (e.g. call it `genesis.json`): + +```json +{ + "config": { + "chainId": , + "homesteadBlock": 0, + "eip150Block": 0, + "eip155Block": 0, + "eip158Block": 0, + "byzantiumBlock": 0, + "constantinopleBlock": 0, + "petersburgBlock": 0, + "istanbulBlock": 0, + "berlinBlock": 0, + "londonBlock": 0 + }, + "alloc": {}, + "coinbase": "0x0000000000000000000000000000000000000000", + "difficulty": "0x20000", + "extraData": "", + "gasLimit": "0x2fefd8", + "nonce": "0x0000000000000042", + "mixhash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "timestamp": "0x00" +} +``` + +The above fields should be fine for most purposes, although we'd recommend changing +the `nonce` to some random value so you prevent unknown remote nodes from being able +to connect to you. If you'd like to pre-fund some accounts for easier testing, create +the accounts and populate the `alloc` field with their addresses. + +```json +"alloc": { + "0x0000000000000000000000000000000000000001": { + "balance": "111111111" + }, + "0x0000000000000000000000000000000000000002": { + "balance": "222222222" + } +} +``` + +With the genesis state defined in the above JSON file, you'll need to initialize **every** +`geth` node with it prior to starting it up to ensure all blockchain parameters are correctly +set: + +```shell +$ geth init path/to/genesis.json +``` + +#### Creating the rendezvous point + +With all nodes that you want to run initialized to the desired genesis state, you'll need to +start a bootstrap node that others can use to find each other in your network and/or over +the internet. The clean way is to configure and run a dedicated bootnode: + +```shell +$ bootnode --genkey=boot.key +$ bootnode --nodekey=boot.key +``` + +With the bootnode online, it will display an [`enode` URL](https://eth.wiki/en/fundamentals/enode-url-format) +that other nodes can use to connect to it and exchange peer information. Make sure to +replace the displayed IP address information (most probably `[::]`) with your externally +accessible IP to get the actual `enode` URL. + +*Note: You could also use a full-fledged `geth` node as a bootnode, but it's the less +recommended way.* + +#### Starting up your member nodes + +With the bootnode operational and externally reachable (you can try +`telnet ` to ensure it's indeed reachable), start every subsequent `geth` +node pointed to the bootnode for peer discovery via the `--bootnodes` flag. It will +probably also be desirable to keep the data directory of your private network separated, so +do also specify a custom `--datadir` flag. + +```shell +$ geth --datadir=path/to/custom/data/folder --bootnodes= +``` + +*Note: Since your network will be completely cut off from the main and test networks, you'll +also need to configure a miner to process transactions and create new blocks for you.* + +#### Running a private miner + +Mining on the public Ethereum network is a complex task as it's only feasible using GPUs, +requiring an OpenCL or CUDA enabled `ethminer` instance. For information on such a +setup, please consult the [EtherMining subreddit](https://www.reddit.com/r/EtherMining/) +and the [ethminer](https://github.com/ethereum-mining/ethminer) repository. + +In a private network setting, however a single CPU miner instance is more than enough for +practical purposes as it can produce a stable stream of blocks at the correct intervals +without needing heavy resources (consider running on a single thread, no need for multiple +ones either). To start a `geth` instance for mining, run it with all your usual flags, extended +by: + +```shell +$ geth --mine --miner.threads=1 --miner.etherbase=0x0000000000000000000000000000000000000000 +``` + +Which will start mining blocks and transactions on a single CPU thread, crediting all +proceedings to the account specified by `--miner.etherbase`. You can further tune the mining +by changing the default gas limit blocks converge to (`--miner.targetgaslimit`) and the price +transactions are accepted at (`--miner.gasprice`). + +## Contribution + +Thank you for considering to help out with the source code! We welcome contributions +from anyone on the internet, and are grateful for even the smallest of fixes! + +If you'd like to contribute to go-ethereum, please fork, fix, commit and send a pull request +for the maintainers to review and merge into the main code base. If you wish to submit +more complex changes though, please check up with the core devs first on [our Discord Server](https://discord.gg/invite/nthXNEv) +to ensure those changes are in line with the general philosophy of the project and/or get +some early feedback which can make both your efforts much lighter as well as our review +and merge procedures quick and simple. + +Please make sure your contributions adhere to our coding guidelines: + + * Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) + guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)). + * Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) + guidelines. + * Pull requests need to be based on and opened against the `master` branch. + * Commit messages should be prefixed with the package(s) they modify. + * E.g. "eth, rpc: make trace configs optional" + +Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/devguide) +for more details on configuring your environment, managing project dependencies, and +testing procedures. + +## License + +The go-ethereum library (i.e. all code outside of the `cmd` directory) is licensed under the +[GNU Lesser General Public License v3.0](https://www.gnu.org/licenses/lgpl-3.0.en.html), +also included in our repository in the `COPYING.LESSER` file. + +The go-ethereum binaries (i.e. all code inside of the `cmd` directory) is licensed under the +[GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html), also +included in our repository in the `COPYING` file. diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 94a0b16a8dbf..68f8ef8af76b 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -131,6 +131,7 @@ var ( utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, utils.MinerNoVerifyFlag, + utils.MinerMaxMergedBundles, utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV5Flag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 417fba68923d..d3d39e2861d2 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -189,6 +189,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, utils.MinerNoVerifyFlag, + utils.MinerMaxMergedBundles, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7d11b0631a28..030ba4a4c851 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -475,6 +475,11 @@ var ( Usage: "Time interval to recreate the block being mined", Value: ethconfig.Defaults.Miner.Recommit, } + MinerMaxMergedBundles = cli.IntFlag{ + Name: "miner.maxmergedbundles", + Usage: "flashbots - The maximum amount of bundles to merge. The miner will run this many workers in parallel to calculate if the full block is more profitable with these additional bundles.", + Value: 3, + } MinerNoVerifyFlag = cli.BoolFlag{ Name: "miner.noverify", Usage: "Disable remote sealing verification", @@ -1402,6 +1407,8 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { if ctx.GlobalIsSet(LegacyMinerGasTargetFlag.Name) { log.Warn("The generic --miner.gastarget flag is deprecated and will be removed in the future!") } + + cfg.MaxMergedBundles = ctx.GlobalInt(MinerMaxMergedBundles.Name) } func setWhitelist(ctx *cli.Context, cfg *ethconfig.Config) { diff --git a/core/tx_pool.go b/core/tx_pool.go index 3329d736a37f..51312d3e78b6 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -251,11 +251,12 @@ type TxPool struct { locals *accountSet // Set of local transaction to exempt from eviction rules journal *txJournal // Journal of local transaction to back up to disk - pending map[common.Address]*txList // All currently processable transactions - queue map[common.Address]*txList // Queued but non-processable transactions - beats map[common.Address]time.Time // Last heartbeat from each known account - all *txLookup // All transactions to allow lookups - priced *txPricedList // All transactions sorted by price + pending map[common.Address]*txList // All currently processable transactions + queue map[common.Address]*txList // Queued but non-processable transactions + beats map[common.Address]time.Time // Last heartbeat from each known account + mevBundles []types.MevBundle + all *txLookup // All transactions to allow lookups + priced *txPricedList // All transactions sorted by price chainHeadCh chan ChainHeadEvent chainHeadSub event.Subscription @@ -557,6 +558,59 @@ func (pool *TxPool) Pending(enforceTips bool) map[common.Address]types.Transacti return pending } +/// AllMevBundles returns all the MEV Bundles currently in the pool +func (pool *TxPool) AllMevBundles() []types.MevBundle { + return pool.mevBundles +} + +// MevBundles returns a list of bundles valid for the given blockNumber/blockTimestamp +// also prunes bundles that are outdated +func (pool *TxPool) MevBundles(blockNumber *big.Int, blockTimestamp uint64) ([]types.MevBundle, error) { + pool.mu.Lock() + defer pool.mu.Unlock() + + // returned values + var ret []types.MevBundle + // rolled over values + var bundles []types.MevBundle + + for _, bundle := range pool.mevBundles { + // Prune outdated bundles + if (bundle.MaxTimestamp != 0 && blockTimestamp > bundle.MaxTimestamp) || blockNumber.Cmp(bundle.BlockNumber) > 0 { + continue + } + + // Roll over future bundles + if (bundle.MinTimestamp != 0 && blockTimestamp < bundle.MinTimestamp) || blockNumber.Cmp(bundle.BlockNumber) < 0 { + bundles = append(bundles, bundle) + continue + } + + // return the ones which are in time + ret = append(ret, bundle) + // keep the bundles around internally until they need to be pruned + bundles = append(bundles, bundle) + } + + pool.mevBundles = bundles + return ret, nil +} + +// AddMevBundle adds a mev bundle to the pool +func (pool *TxPool) AddMevBundle(txs types.Transactions, blockNumber *big.Int, minTimestamp, maxTimestamp uint64, revertingTxHashes []common.Hash) error { + pool.mu.Lock() + defer pool.mu.Unlock() + + pool.mevBundles = append(pool.mevBundles, types.MevBundle{ + Txs: txs, + BlockNumber: blockNumber, + MinTimestamp: minTimestamp, + MaxTimestamp: maxTimestamp, + RevertingTxHashes: revertingTxHashes, + }) + return nil +} + // Locals retrieves the accounts currently considered local by the pool. func (pool *TxPool) Locals() []common.Address { pool.mu.Lock() diff --git a/core/types/transaction.go b/core/types/transaction.go index 83f1766e67e2..5425249ef0af 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -635,3 +635,11 @@ func copyAddressPtr(a *common.Address) *common.Address { cpy := *a return &cpy } + +type MevBundle struct { + Txs Transactions + BlockNumber *big.Int + MinTimestamp uint64 + MaxTimestamp uint64 + RevertingTxHashes []common.Hash +} diff --git a/eth/api_backend.go b/eth/api_backend.go index 6577ac1e1af4..8454c0afe701 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -239,6 +239,10 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) return b.eth.txPool.AddLocal(signedTx) } +func (b *EthAPIBackend) SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64, revertingTxHashes []common.Hash) error { + return b.eth.txPool.AddMevBundle(txs, big.NewInt(blockNumber.Int64()), minTimestamp, maxTimestamp, revertingTxHashes) +} + func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { pending := b.eth.txPool.Pending(false) var txs types.Transactions diff --git a/infra/Dockerfile.node b/infra/Dockerfile.node new file mode 100644 index 000000000000..db8e99ac937e --- /dev/null +++ b/infra/Dockerfile.node @@ -0,0 +1,23 @@ +# Build Geth in a stock Go builder container +FROM golang:1.15-alpine as builder + +RUN apk add --no-cache make gcc musl-dev linux-headers git + +ADD . /go-ethereum +RUN cd /go-ethereum && make geth + +# Pull Geth into a second stage deploy alpine container +FROM alpine:latest + +ENV PYTHONUNBUFFERED=1 +RUN apk add --update --no-cache groff less python3 curl jq ca-certificates && ln -sf python3 /usr/bin/python +RUN python3 -m ensurepip +RUN pip3 install --no-cache --upgrade pip setuptools awscli + +COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/ + +COPY ./infra/start-mev-geth-node.sh /root/start-mev-geth-node.sh +RUN chmod 755 /root/start-mev-geth-node.sh + +EXPOSE 8545 8546 30303 30303/udp +ENTRYPOINT ["/root/start-mev-geth-node.sh"] diff --git a/infra/Dockerfile.updater b/infra/Dockerfile.updater new file mode 100644 index 000000000000..d3099d19ce1a --- /dev/null +++ b/infra/Dockerfile.updater @@ -0,0 +1,23 @@ +# Build Geth in a stock Go builder container +FROM golang:1.15-alpine as builder + +RUN apk add --no-cache make gcc musl-dev linux-headers git + +ADD . /go-ethereum +RUN cd /go-ethereum && make geth + +# Pull Geth into a second stage deploy alpine container +FROM alpine:latest + +ENV PYTHONUNBUFFERED=1 +RUN apk add --update --no-cache groff less python3 curl jq ca-certificates && ln -sf python3 /usr/bin/python +RUN python3 -m ensurepip +RUN pip3 install --no-cache --upgrade pip setuptools awscli + +COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/ + +COPY ./infra/start-mev-geth-updater.sh /root/start-mev-geth-updater.sh +RUN chmod 755 /root/start-mev-geth-updater.sh + +EXPOSE 8545 8546 30303 30303/udp +ENTRYPOINT ["/root/start-mev-geth-updater.sh"] diff --git a/infra/mev-geth-nodes-arm64.yaml b/infra/mev-geth-nodes-arm64.yaml new file mode 100644 index 000000000000..af76b6aada82 --- /dev/null +++ b/infra/mev-geth-nodes-arm64.yaml @@ -0,0 +1,979 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Description: > + This template creates an automated continuous deployment pipeline to Amazon Elastic Container Service (ECS) + Created by Luke Youngblood, luke@blockscale.net + +Parameters: + +# GitHub Parameters + + GitHubUser: + Type: String + Default: lyoungblood + Description: Your team or username on GitHub. + + NodeGitHubRepo: + Type: String + Default: mev-geth + Description: The repo name of the node service. + + NodeGitHubBranch: + Type: String + Default: master + Description: The branch of the node repo to continuously deploy. + + GitHubToken: + Type: String + NoEcho: true + Description: > + Token for the team or user specified above. (https://github.com/settings/tokens) + +# VPC Parameters + + VPC: + Type: AWS::EC2::VPC::Id + + Subnets: + Type: List + + VpcCIDR: + Type: String + Default: 172.31.0.0/16 + +# ECS Parameters + + InstanceType: + Type: String + Default: m6gd.large + + MemoryLimit: + Type: Number + Default: 6144 + + KeyPair: + Type: AWS::EC2::KeyPair::KeyName + + SpotPrice: + Type: Number + Default: 0.0904 + + ClusterSize: + Type: Number + Default: 5 + + Bandwidth: + Type: Number + Default: 2048 + + BandwidthCeiling: + Type: Number + Default: 4096 + + NodeDesiredCount: + Type: Number + Default: 0 + + NodeTaskName: + Type: String + Default: mev-geth-node + + ECSAMI: + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ecs/optimized-ami/amazon-linux-2/arm64/recommended/image_id + +# SNS Parameters + + SNSSubscriptionEndpoint: + Type: String + Default: https://events.pagerduty.com/integration/44cbdb66f22b4f3caf5dd15741c7eb17/enqueue + + SNSSubscriptionProtocol: + Type: String + Default: HTTPS + +# CloudWatch Alarm Parameters + + CPUAlarmThreshold: + Type: Number + Default: 80 + + MemoryAlarmThreshold: + Type: Number + Default: 80 + +# Mev-Geth Parameters + + Network: + Type: String + Default: mainnet + AllowedValues: + - mainnet + - goerli + + SyncMode: + Type: String + Default: fast + AllowedValues: + - full + - fast + - light + + Connections: + Type: Number + Default: 50 + + RpcPort: + Type: Number + Default: 8545 + + WsPort: + Type: Number + Default: 8546 + + NetPort: + Type: Number + Default: 30303 + +Metadata: + + AWS::CloudFormation::Interface: + ParameterLabels: + GitHubUser: + default: "User" + NodeGitHubRepo: + default: "Node Repo" + NodeGitHubBranch: + default: "Node Branch" + GitHubToken: + default: "Personal Access Token" + VPC: + default: "Choose which VPC the autoscaling group should be deployed to" + Subnets: + default: "Choose which subnets the autoscaling group should be deployed to" + VpcCIDR: + default: "VPC CIDR Block" + InstanceType: + default: "Which instance type should we use to build the ECS cluster?" + MemoryLimit: + default: "How much memory should be reserved for each task. Set to greater than 50% of instance memory capacity." + KeyPair: + default: "Which keypair should be used to allow SSH to the nodes?" + ClusterSize: + default: "How many ECS hosts do you want to initially deploy?" + SpotPrice: + default: "The maximum spot price to pay for instances - this should normally be set to the on demand price." + Bandwidth: + default: "How much bandwidth, in kb/sec., should be allocated to Ethereum peers (upload) per EC2 instance" + BandwidthCeiling: + default: "How much bandwidth, in kb/sec., should be allocated to Ethereum peers as a ceiling (max. upload)" + NodeDesiredCount: + default: "How many ECS Tasks do you want to initially execute?" + NodeTaskName: + default: "The name of the node ECS Task" + ECSAMI: + default: "The ECS AMI ID populated from SSM." + Network: + default: "The Ethereum network you will be connecting to" + SyncMode: + default: "The synchronization mode that Mev-Geth should use (full, fast, or light)" + Connections: + default: "The number of desired connections on the Mev-Geth node" + RpcPort: + default: "The RPC port used for communication with the local Mev-Geth node" + WsPort: + default: "The Websockets port used for communication with the local Mev-Geth node" + NetPort: + default: "The TCP port used for connectivity to other Ethereum peer nodes" + ParameterGroups: + - Label: + default: GitHub Configuration + Parameters: + - NodeGitHubRepo + - NodeGitHubBranch + - GitHubUser + - GitHubToken + - Label: + default: VPC Configuration + Parameters: + - VPC + - Subnets + - VpcCIDR + - Label: + default: ECS Configuration + Parameters: + - InstanceType + - MemoryLimit + - KeyPair + - SpotPrice + - ClusterSize + - Bandwidth + - BandwidthCeiling + - NodeDesiredCount + - NodeTaskName + - ECSAMI + - Label: + default: Mev-Geth Configuration + Parameters: + - Network + - SyncMode + - Connections + - RpcPort + - WsPort + - NetPort + - Label: + default: PagerDuty Endpoint Configuration + Parameters: + - SNSSubscriptionEndpoint + - SNSSubscriptionProtocol + - Label: + default: CloudWatch Alarms Configuration + Parameters: + - CPUAlarmThreshold + - MemoryAlarmThreshold + +# Mappings + +Mappings: + + RegionMap: + us-east-2: + mainnet: mev-geth-updater-fast-chainbucket-17p2xhnhcydlz + goerli: mev-geth-updater-fast-goerli-chainbucket-j6dujg8apbna + #us-west-2: + # mainnet: + # goerli: + +Resources: + +# ECS Resources + + Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Ref AWS::StackName + + SecurityGroup: + Type: "AWS::EC2::SecurityGroup" + Properties: + GroupDescription: !Sub ${AWS::StackName}-sg + VpcId: !Ref VPC + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref RpcPort + ToPort: !Ref RpcPort + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref WsPort + ToPort: !Ref WsPort + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + + ECSAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + VPCZoneIdentifier: !Ref Subnets + LaunchConfigurationName: !Ref ECSLaunchConfiguration + MinSize: !Ref ClusterSize + MaxSize: !Ref ClusterSize + DesiredCapacity: !Ref ClusterSize + Tags: + - Key: Name + Value: !Sub ${AWS::StackName} ECS host + PropagateAtLaunch: true + CreationPolicy: + ResourceSignal: + Timeout: PT15M + UpdatePolicy: + AutoScalingRollingUpdate: + MinInstancesInService: 2 + MaxBatchSize: 1 + PauseTime: PT15M + SuspendProcesses: + - HealthCheck + - ReplaceUnhealthy + - AZRebalance + - AlarmNotification + - ScheduledActions + WaitOnResourceSignals: true + + ECSLaunchConfiguration: + Type: AWS::AutoScaling::LaunchConfiguration + Properties: + ImageId: !Ref ECSAMI + InstanceType: !Ref InstanceType + KeyName: !Ref KeyPair + AssociatePublicIpAddress: True + # Uncomment if you would like to use Spot instances (subject to unexpected termination) + # SpotPrice: !Ref SpotPrice + SecurityGroups: + - !Ref SecurityGroup + IamInstanceProfile: !Ref ECSInstanceProfile + UserData: + "Fn::Base64": !Sub | + #!/bin/bash + yum install -y aws-cfn-bootstrap hibagent rsync awscli + yum update -y + service amazon-ssm-agent restart + + # determine if we have an NVMe SSD attached + find /dev/nvme1 + if [ $? -eq 0 ] + then + mount_point=/var/lib/docker + + # copy existing files from mount point + service docker stop + echo 'DOCKER_STORAGE_OPTIONS="--storage-driver overlay2"' > /etc/sysconfig/docker-storage + mkdir -p /tmp$mount_point + rsync -val $mount_point/ /tmp/$mount_point/ + + # make a new filesystem and mount it + mkfs -t ext4 /dev/nvme1n1 + mkdir -p $mount_point + mount -t ext4 -o noatime /dev/nvme1n1 $mount_point + + # Copy files back to new mount point + rsync -val /tmp/$mount_point/ $mount_point/ + rm -rf /tmp$mount_point + service docker start + + # Make raid appear on reboot + echo >> /etc/fstab + echo "/dev/nvme1n1 $mount_point ext4 noatime 0 0" | tee -a /etc/fstab + fi + + # Set Linux traffic control to limit outbound bandwidth usage of peering + #tc qdisc add dev eth0 root handle 1:0 htb default 1 + #tc class add dev eth0 parent 1:0 classid 1:10 htb rate ${Bandwidth}kbit ceil {BandwidthCeiling}kbit prio 0 + #tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dport 30303 0xffff flowid 1:10 + + /opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + /opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSAutoScalingGroup + /usr/bin/enable-ec2-spot-hibernation + + # Attach an EIP from the pool of available EIPs in scope "vpc" + alloc=`aws ec2 describe-addresses --region ${AWS::Region} --output text | grep -v eni | head -1 | cut -f 2` + instanceid=`curl --silent 169.254.169.254/latest/meta-data/instance-id` + aws ec2 associate-address --region ${AWS::Region} --allocation-id $alloc --instance-id $instanceid + echo "ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION=5m" >> /etc/ecs/ecs.config + + reboot + + Metadata: + AWS::CloudFormation::Init: + config: + packages: + yum: + awslogs: [] + + commands: + 01_add_instance_to_cluster: + command: !Sub echo ECS_CLUSTER=${Cluster} >> /etc/ecs/ecs.config + files: + "/etc/cfn/cfn-hup.conf": + mode: 000400 + owner: root + group: root + content: !Sub | + [main] + stack=${AWS::StackId} + region=${AWS::Region} + + "/etc/cfn/hooks.d/cfn-auto-reloader.conf": + content: !Sub | + [cfn-auto-reloader-hook] + triggers=post.update + path=Resources.ECSLaunchConfiguration.Metadata.AWS::CloudFormation::Init + action=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + + services: + sysvinit: + cfn-hup: + enabled: true + ensureRunning: true + files: + - /etc/cfn/cfn-hup.conf + - /etc/cfn/hooks.d/cfn-auto-reloader.conf + + NodeLoadBalancer: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: !Sub ${AWS::StackName}-node-NLB + Type: network + Scheme: internal + Subnets: !Ref Subnets + Tags: + - Key: Name + Value: !Sub ${AWS::StackName}-node-NLB + + NodeTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + DependsOn: NodeLoadBalancer + Properties: + VpcId: !Ref VPC + Port: !Ref RpcPort + Protocol: TCP + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 120 + + NodeListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: !Ref NodeTargetGroup + LoadBalancerArn: !Ref NodeLoadBalancer + Port: !Ref RpcPort + Protocol: TCP + + NodeWsTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + DependsOn: NodeLoadBalancer + Properties: + VpcId: !Ref VPC + Port: !Ref WsPort + Protocol: TCP + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 120 + + NodeWsListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: !Ref NodeWsTargetGroup + LoadBalancerArn: !Ref NodeLoadBalancer + Port: !Ref WsPort + Protocol: TCP + + # This IAM Role is attached to all of the ECS hosts. It is based on the default role + # published here: + # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/instance_IAM_role.html + # + # You can add other IAM policy statements here to allow access from your ECS hosts + # to other AWS services. + + ECSRole: + Type: AWS::IAM::Role + Properties: + Path: / + RoleName: !Sub ${AWS::StackName}-ECSRole-${AWS::Region} + AssumeRolePolicyDocument: | + { + "Statement": [{ + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + }] + } + Policies: + - PolicyName: ecs-service + PolicyDocument: | + { + "Statement": [{ + "Effect": "Allow", + "Action": [ + "ecs:CreateCluster", + "ecs:DeregisterContainerInstance", + "ecs:DiscoverPollEndpoint", + "ecs:Poll", + "ecs:RegisterContainerInstance", + "ecs:StartTelemetrySession", + "ecs:Submit*", + "logs:CreateLogStream", + "logs:PutLogEvents", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:GetAuthorizationToken", + "ssm:DescribeAssociation", + "ssm:GetDeployablePatchSnapshotForInstance", + "ssm:GetDocument", + "ssm:GetManifest", + "ssm:GetParameters", + "ssm:ListAssociations", + "ssm:ListInstanceAssociations", + "ssm:PutInventory", + "ssm:PutComplianceItems", + "ssm:PutConfigurePackageResult", + "ssm:PutParameter", + "ssm:UpdateAssociationStatus", + "ssm:UpdateInstanceAssociationStatus", + "ssm:UpdateInstanceInformation", + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply", + "cloudwatch:PutMetricData", + "ec2:DescribeInstanceStatus", + "ds:CreateComputer", + "ds:DescribeDirectories", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "firehose:PutRecord", + "firehose:PutRecordBatch", + "ec2:DescribeAddresses", + "ec2:DescribeInstances", + "ec2:AssociateAddress" + ], + "Resource": "*" + }] + } + + ECSInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: / + Roles: + - !Ref ECSRole + + ECSServiceAutoScalingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + Action: + - 'sts:AssumeRole' + Effect: Allow + Principal: + Service: + - application-autoscaling.amazonaws.com + Path: / + Policies: + - PolicyName: ecs-service-autoscaling + PolicyDocument: + Statement: + Effect: Allow + Action: + - application-autoscaling:* + - cloudwatch:DescribeAlarms + - cloudwatch:PutMetricAlarm + - ecs:DescribeServices + - ecs:UpdateService + Resource: "*" + + NodeTaskExecutionRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: !Sub ecs-task-S3-${AWS::StackName} + PolicyDocument: + Version: 2012-10-17 + Statement: + - + Effect: Allow + Action: + - "s3:Get*" + - "s3:List*" + Resource: + - Fn::Join: + - "" + - + - "arn:aws:s3:::" + - !FindInMap + - RegionMap + - !Ref 'AWS::Region' + - !Ref Network + + NodeLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub /ecs/${AWS::StackName}-node + RetentionInDays: 14 + + NodeECSService: + Type: AWS::ECS::Service + DependsOn: NodeListener + Properties: + Cluster: !Ref Cluster + DesiredCount: !Ref NodeDesiredCount + HealthCheckGracePeriodSeconds: 3600 + TaskDefinition: !Ref NodeTaskDefinition + LaunchType: EC2 + DeploymentConfiguration: + MaximumPercent: 150 + MinimumHealthyPercent: 50 + LoadBalancers: + - ContainerName: !Ref NodeTaskName + ContainerPort: !Ref RpcPort + TargetGroupArn: !Ref NodeTargetGroup + - ContainerName: !Ref NodeTaskName + ContainerPort: !Ref WsPort + TargetGroupArn: !Ref NodeWsTargetGroup + + NodeTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Ref NodeTaskName + RequiresCompatibilities: + - EC2 + NetworkMode: host + ExecutionRoleArn: !Ref NodeTaskExecutionRole + ContainerDefinitions: + - Name: !Ref NodeTaskName + Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${NodeRepository} + Essential: true + MemoryReservation: !Ref MemoryLimit + Environment: + - Name: "region" + Value: !Ref AWS::Region + - Name: "network" + Value: !Ref Network + - Name: "syncmode" + Value: !Ref SyncMode + - Name: "connections" + Value: !Ref Connections + - Name: "rpcport" + Value: !Ref RpcPort + - Name: "wsport" + Value: !Ref WsPort + - Name: "netport" + Value: !Ref NetPort + - Name: "chainbucket" + Value: !FindInMap + - RegionMap + - !Ref 'AWS::Region' + - !Ref Network + - Name: "s3key" + Value: node + PortMappings: + - ContainerPort: !Ref RpcPort + - ContainerPort: !Ref WsPort + - ContainerPort: !Ref NetPort + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-region: !Ref AWS::Region + awslogs-group: !Ref NodeLogGroup + awslogs-stream-prefix: !Ref AWS::StackName + #HealthCheck: + # Command: + # - CMD-SHELL + # - '[ `echo "eth.syncing.highestBlock - eth.syncing.currentBlock"|geth attach|head -10|tail -1` -lt 200 ] || exit 1' + # Interval: 300 + # Timeout: 60 + # Retries: 10 + # StartPeriod: 300 + +# CodePipeline Resources + + NodeRepository: + Type: AWS::ECR::Repository + + NodeCodeBuildServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: "*" + Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - ecr:GetAuthorizationToken + - Resource: !Sub arn:aws:s3:::${NodeArtifactBucket}/* + Effect: Allow + Action: + - s3:GetObject + - s3:PutObject + - s3:GetObjectVersion + - Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${NodeRepository} + Effect: Allow + Action: + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - ecr:BatchCheckLayerAvailability + - ecr:PutImage + - ecr:InitiateLayerUpload + - ecr:UploadLayerPart + - ecr:CompleteLayerUpload + + NodeCodePipelineServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codepipeline.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: + - !Sub arn:aws:s3:::${NodeArtifactBucket}/* + Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + - s3:GetObjectVersion + - s3:GetBucketVersioning + - Resource: "*" + Effect: Allow + Action: + - ecs:DescribeServices + - ecs:DescribeTaskDefinition + - ecs:DescribeTasks + - ecs:ListTasks + - ecs:RegisterTaskDefinition + - ecs:UpdateService + - codebuild:StartBuild + - codebuild:BatchGetBuilds + - iam:PassRole + + NodeArtifactBucket: + Type: AWS::S3::Bucket + + NodeCodeBuildProject: + Type: AWS::CodeBuild::Project + Properties: + Artifacts: + Type: CODEPIPELINE + Source: + Type: CODEPIPELINE + BuildSpec: | + version: 0.2 + phases: + install: + runtime-versions: + docker: 19 + pre_build: + commands: + - $(aws ecr get-login --no-include-email) + - TAG="$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | head -c 8)" + - IMAGE_URI="${REPOSITORY_URI}:${TAG}" + - cp infra/Dockerfile.node ./Dockerfile + build: + commands: + - docker build --tag "$IMAGE_URI" . + - docker build --tag "${REPOSITORY_URI}:latest" . + post_build: + commands: + - docker push "$IMAGE_URI" + - docker push "${REPOSITORY_URI}:latest" + - printf '[{"name":"mev-geth-node","imageUri":"%s"}]' "$IMAGE_URI" > images.json + artifacts: + files: images.json + Environment: + ComputeType: BUILD_GENERAL1_LARGE + Image: aws/codebuild/amazonlinux2-aarch64-standard:1.0 + Type: ARM_CONTAINER + PrivilegedMode: true + EnvironmentVariables: + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: REPOSITORY_URI + Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${NodeRepository} + Cache: + Type: S3 + Location: !Sub ${NodeArtifactBucket}/buildcache + Name: !Sub ${AWS::StackName}-node + ServiceRole: !Ref NodeCodeBuildServiceRole + + NodePipeline: + Type: AWS::CodePipeline::Pipeline + Properties: + RoleArn: !GetAtt NodeCodePipelineServiceRole.Arn + ArtifactStore: + Type: S3 + Location: !Ref NodeArtifactBucket + Stages: + - Name: Source + Actions: + - Name: App + ActionTypeId: + Category: Source + Owner: ThirdParty + Version: 1 + Provider: GitHub + Configuration: + Owner: !Ref GitHubUser + Repo: !Ref NodeGitHubRepo + Branch: !Ref NodeGitHubBranch + OAuthToken: !Ref GitHubToken + OutputArtifacts: + - Name: App + RunOrder: 1 + - Name: Build + Actions: + - Name: Build + ActionTypeId: + Category: Build + Owner: AWS + Version: 1 + Provider: CodeBuild + Configuration: + ProjectName: !Ref NodeCodeBuildProject + InputArtifacts: + - Name: App + OutputArtifacts: + - Name: BuildOutput + RunOrder: 1 + - Name: Deploy + Actions: + - Name: Deploy + ActionTypeId: + Category: Deploy + Owner: AWS + Version: 1 + Provider: ECS + Configuration: + ClusterName: !Ref Cluster + ServiceName: !Ref NodeECSService + FileName: images.json + InputArtifacts: + - Name: BuildOutput + RunOrder: 1 + +# SNS Resources + + SNSTopic: + Type: AWS::SNS::Topic + Properties: + DisplayName: String + Subscription: + - + Endpoint: !Ref SNSSubscriptionEndpoint + Protocol: !Ref SNSSubscriptionProtocol + TopicName: !Ref AWS::StackName + +# CloudWatch Resources + + CPUAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub ${AWS::StackName} average CPU utilization greater than threshold. + AlarmDescription: Alarm if CPU utilization is greater than threshold. + Namespace: AWS/ECS + MetricName: CPUUtilization + Dimensions: + - Name: ClusterName + Value: !Ref Cluster + Statistic: Average + Period: '60' + EvaluationPeriods: '3' + Threshold: !Ref CPUAlarmThreshold + ComparisonOperator: GreaterThanThreshold + AlarmActions: + - Ref: SNSTopic + OKActions: + - Ref: SNSTopic + + MemoryAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub ${AWS::StackName} average memory utilization greater than threshold. + AlarmDescription: Alarm if memory utilization is greater than threshold. + Namespace: AWS/ECS + MetricName: MemoryUtilization + Dimensions: + - Name: ClusterName + Value: !Ref Cluster + Statistic: Average + Period: '60' + EvaluationPeriods: '3' + Threshold: !Ref MemoryAlarmThreshold + ComparisonOperator: GreaterThanThreshold + AlarmActions: + - Ref: SNSTopic + OKActions: + - Ref: SNSTopic + + HealthyHostAlarm: + Type: 'AWS::CloudWatch::Alarm' + Properties: + AlarmName: !Sub ${AWS::StackName} alarm no healthy hosts connected to ELB. + AlarmDescription: Alarm if no healthy hosts connected to ELB. + MetricName: HealthyHostCount + Namespace: AWS/NetworkELB + Statistic: Average + Period: '60' + EvaluationPeriods: '3' + Threshold: '1' + ComparisonOperator: LessThanThreshold + Dimensions: + - Name: TargetGroup + Value: !GetAtt NodeTargetGroup.TargetGroupFullName + - Name: LoadBalancer + Value: !GetAtt NodeLoadBalancer.LoadBalancerFullName + AlarmActions: + - Ref: SNSTopic + OKActions: + - Ref: SNSTopic + +Outputs: + ClusterName: + Value: !Ref Cluster + NodeService: + Value: !Ref NodeECSService + NodePipelineUrl: + Value: !Sub https://console.aws.amazon.com/codepipeline/home?region=${AWS::Region}#/view/${NodePipeline} + NodeTargetGroup: + Value: !Ref NodeTargetGroup + NodeServiceUrl: + Description: URL of the load balancer for the node service. + Value: !Sub http://${NodeLoadBalancer.DNSName} diff --git a/infra/mev-geth-nodes-x86-64.yaml b/infra/mev-geth-nodes-x86-64.yaml new file mode 100644 index 000000000000..bf7a196caa52 --- /dev/null +++ b/infra/mev-geth-nodes-x86-64.yaml @@ -0,0 +1,972 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Description: > + This template creates an automated continuous deployment pipeline to Amazon Elastic Container Service (ECS) + Created by Luke Youngblood, luke@blockscale.net + +Parameters: + # GitHub Parameters + + GitHubUser: + Type: String + Default: lyoungblood + Description: Your team or username on GitHub. + + NodeGitHubRepo: + Type: String + Default: mev-geth + Description: The repo name of the node service. + + NodeGitHubBranch: + Type: String + Default: master + Description: The branch of the node repo to continuously deploy. + + GitHubToken: + Type: String + NoEcho: true + Description: > + Token for the team or user specified above. (https://github.com/settings/tokens) + + # VPC Parameters + + VPC: + Type: AWS::EC2::VPC::Id + + Subnets: + Type: List + + VpcCIDR: + Type: String + Default: 172.31.0.0/16 + + # ECS Parameters + + InstanceType: + Type: String + Default: i3en.large + + MemoryLimit: + Type: Number + Default: 6144 + + KeyPair: + Type: AWS::EC2::KeyPair::KeyName + + SpotPrice: + Type: Number + Default: 0.0904 + + ClusterSize: + Type: Number + Default: 5 + + Bandwidth: + Type: Number + Default: 2048 + + BandwidthCeiling: + Type: Number + Default: 4096 + + NodeDesiredCount: + Type: Number + Default: 0 + + NodeTaskName: + Type: String + Default: mev-geth-node + + ECSAMI: + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + + # SNS Parameters + + SNSSubscriptionEndpoint: + Type: String + Default: https://events.pagerduty.com/integration/44cbdb66f22b4f3caf5dd15741c7eb17/enqueue + + SNSSubscriptionProtocol: + Type: String + Default: HTTPS + + # CloudWatch Alarm Parameters + + CPUAlarmThreshold: + Type: Number + Default: 80 + + MemoryAlarmThreshold: + Type: Number + Default: 80 + + # Mev-Geth Parameters + + Network: + Type: String + Default: mainnet + AllowedValues: + - mainnet + - goerli + + SyncMode: + Type: String + Default: fast + AllowedValues: + - full + - fast + - light + + Connections: + Type: Number + Default: 50 + + RpcPort: + Type: Number + Default: 8545 + + WsPort: + Type: Number + Default: 8546 + + NetPort: + Type: Number + Default: 30303 + +Metadata: + AWS::CloudFormation::Interface: + ParameterLabels: + GitHubUser: + default: "User" + NodeGitHubRepo: + default: "Node Repo" + NodeGitHubBranch: + default: "Node Branch" + GitHubToken: + default: "Personal Access Token" + VPC: + default: "Choose which VPC the autoscaling group should be deployed to" + Subnets: + default: "Choose which subnets the autoscaling group should be deployed to" + VpcCIDR: + default: "VPC CIDR Block" + InstanceType: + default: "Which instance type should we use to build the ECS cluster?" + MemoryLimit: + default: "How much memory should be reserved for each task. Set to greater than 50% of instance memory capacity." + KeyPair: + default: "Which keypair should be used to allow SSH to the nodes?" + ClusterSize: + default: "How many ECS hosts do you want to initially deploy?" + SpotPrice: + default: "The maximum spot price to pay for instances - this should normally be set to the on demand price." + Bandwidth: + default: "How much bandwidth, in kb/sec., should be allocated to Ethereum peers (upload) per EC2 instance" + BandwidthCeiling: + default: "How much bandwidth, in kb/sec., should be allocated to Ethereum peers as a ceiling (max. upload)" + NodeDesiredCount: + default: "How many ECS Tasks do you want to initially execute?" + NodeTaskName: + default: "The name of the node ECS Task" + ECSAMI: + default: "The ECS AMI ID populated from SSM." + Network: + default: "The Ethereum network you will be connecting to" + SyncMode: + default: "The synchronization mode that Mev-Geth should use (full, fast, or light)" + Connections: + default: "The number of desired connections on the Mev-Geth node" + RpcPort: + default: "The RPC port used for communication with the local Mev-Geth node" + WsPort: + default: "The Websockets port used for communication with the local Mev-Geth node" + NetPort: + default: "The TCP port used for connectivity to other Ethereum peer nodes" + ParameterGroups: + - Label: + default: GitHub Configuration + Parameters: + - NodeGitHubRepo + - NodeGitHubBranch + - GitHubUser + - GitHubToken + - Label: + default: VPC Configuration + Parameters: + - VPC + - Subnets + - VpcCIDR + - Label: + default: ECS Configuration + Parameters: + - InstanceType + - MemoryLimit + - KeyPair + - SpotPrice + - ClusterSize + - Bandwidth + - BandwidthCeiling + - NodeDesiredCount + - NodeTaskName + - ECSAMI + - Label: + default: Mev-Geth Configuration + Parameters: + - Network + - SyncMode + - Connections + - RpcPort + - WsPort + - NetPort + - Label: + default: PagerDuty Endpoint Configuration + Parameters: + - SNSSubscriptionEndpoint + - SNSSubscriptionProtocol + - Label: + default: CloudWatch Alarms Configuration + Parameters: + - CPUAlarmThreshold + - MemoryAlarmThreshold + +# Mappings + +Mappings: + RegionMap: + us-east-2: + mainnet: mev-geth-updater-fast-chainbucket-17p2xhnhcydlz + goerli: mev-geth-updater-fast-goerli-chainbucket-j6dujg8apbna + #us-west-2: + # mainnet: + # goerli: + +Resources: + # ECS Resources + + Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Ref AWS::StackName + + SecurityGroup: + Type: "AWS::EC2::SecurityGroup" + Properties: + GroupDescription: !Sub ${AWS::StackName}-sg + VpcId: !Ref VPC + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref RpcPort + ToPort: !Ref RpcPort + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref WsPort + ToPort: !Ref WsPort + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + + ECSAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + VPCZoneIdentifier: !Ref Subnets + LaunchConfigurationName: !Ref ECSLaunchConfiguration + MinSize: !Ref ClusterSize + MaxSize: !Ref ClusterSize + DesiredCapacity: !Ref ClusterSize + Tags: + - Key: Name + Value: !Sub ${AWS::StackName} ECS host + PropagateAtLaunch: true + CreationPolicy: + ResourceSignal: + Timeout: PT15M + UpdatePolicy: + AutoScalingRollingUpdate: + MinInstancesInService: 2 + MaxBatchSize: 1 + PauseTime: PT15M + SuspendProcesses: + - HealthCheck + - ReplaceUnhealthy + - AZRebalance + - AlarmNotification + - ScheduledActions + WaitOnResourceSignals: true + + ECSLaunchConfiguration: + Type: AWS::AutoScaling::LaunchConfiguration + Properties: + ImageId: !Ref ECSAMI + InstanceType: !Ref InstanceType + KeyName: !Ref KeyPair + AssociatePublicIpAddress: True + # Uncomment if you would like to use Spot instances (subject to unexpected termination) + # SpotPrice: !Ref SpotPrice + SecurityGroups: + - !Ref SecurityGroup + IamInstanceProfile: !Ref ECSInstanceProfile + UserData: + "Fn::Base64": !Sub | + #!/bin/bash + yum install -y aws-cfn-bootstrap hibagent rsync awscli + yum update -y + service amazon-ssm-agent restart + + # determine if we have an NVMe SSD attached + find /dev/nvme1 + if [ $? -eq 0 ] + then + mount_point=/var/lib/docker + + # copy existing files from mount point + service docker stop + echo 'DOCKER_STORAGE_OPTIONS="--storage-driver overlay2"' > /etc/sysconfig/docker-storage + mkdir -p /tmp$mount_point + rsync -val $mount_point/ /tmp/$mount_point/ + + # make a new filesystem and mount it + mkfs -t ext4 /dev/nvme1n1 + mkdir -p $mount_point + mount -t ext4 -o noatime /dev/nvme1n1 $mount_point + + # Copy files back to new mount point + rsync -val /tmp/$mount_point/ $mount_point/ + rm -rf /tmp$mount_point + service docker start + + # Make raid appear on reboot + echo >> /etc/fstab + echo "/dev/nvme1n1 $mount_point ext4 noatime 0 0" | tee -a /etc/fstab + fi + + # Set Linux traffic control to limit outbound bandwidth usage of peering + #tc qdisc add dev eth0 root handle 1:0 htb default 1 + #tc class add dev eth0 parent 1:0 classid 1:10 htb rate ${Bandwidth}kbit ceil {BandwidthCeiling}kbit prio 0 + #tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dport 30303 0xffff flowid 1:10 + + /opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + /opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSAutoScalingGroup + /usr/bin/enable-ec2-spot-hibernation + + # Attach an EIP from the pool of available EIPs in scope "vpc" + alloc=`aws ec2 describe-addresses --region ${AWS::Region} --output text | grep -v eni | head -1 | cut -f 2` + instanceid=`curl --silent 169.254.169.254/latest/meta-data/instance-id` + aws ec2 associate-address --region ${AWS::Region} --allocation-id $alloc --instance-id $instanceid + echo "ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION=5m" >> /etc/ecs/ecs.config + + reboot + + Metadata: + AWS::CloudFormation::Init: + config: + packages: + yum: + awslogs: [] + + commands: + 01_add_instance_to_cluster: + command: !Sub echo ECS_CLUSTER=${Cluster} >> /etc/ecs/ecs.config + files: + "/etc/cfn/cfn-hup.conf": + mode: 000400 + owner: root + group: root + content: !Sub | + [main] + stack=${AWS::StackId} + region=${AWS::Region} + + "/etc/cfn/hooks.d/cfn-auto-reloader.conf": + content: !Sub | + [cfn-auto-reloader-hook] + triggers=post.update + path=Resources.ECSLaunchConfiguration.Metadata.AWS::CloudFormation::Init + action=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + + services: + sysvinit: + cfn-hup: + enabled: true + ensureRunning: true + files: + - /etc/cfn/cfn-hup.conf + - /etc/cfn/hooks.d/cfn-auto-reloader.conf + + NodeLoadBalancer: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Name: !Sub ${AWS::StackName}-node-NLB + Type: network + Scheme: internal + Subnets: !Ref Subnets + Tags: + - Key: Name + Value: !Sub ${AWS::StackName}-node-NLB + + NodeTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + DependsOn: NodeLoadBalancer + Properties: + VpcId: !Ref VPC + Port: !Ref RpcPort + Protocol: TCP + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 120 + + NodeListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: !Ref NodeTargetGroup + LoadBalancerArn: !Ref NodeLoadBalancer + Port: !Ref RpcPort + Protocol: TCP + + NodeWsTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + DependsOn: NodeLoadBalancer + Properties: + VpcId: !Ref VPC + Port: !Ref WsPort + Protocol: TCP + TargetGroupAttributes: + - Key: deregistration_delay.timeout_seconds + Value: 120 + + NodeWsListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + DefaultActions: + - Type: forward + TargetGroupArn: !Ref NodeWsTargetGroup + LoadBalancerArn: !Ref NodeLoadBalancer + Port: !Ref WsPort + Protocol: TCP + + # This IAM Role is attached to all of the ECS hosts. It is based on the default role + # published here: + # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/instance_IAM_role.html + # + # You can add other IAM policy statements here to allow access from your ECS hosts + # to other AWS services. + + ECSRole: + Type: AWS::IAM::Role + Properties: + Path: / + RoleName: !Sub ${AWS::StackName}-ECSRole-${AWS::Region} + AssumeRolePolicyDocument: | + { + "Statement": [{ + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + }] + } + Policies: + - PolicyName: ecs-service + PolicyDocument: | + { + "Statement": [{ + "Effect": "Allow", + "Action": [ + "ecs:CreateCluster", + "ecs:DeregisterContainerInstance", + "ecs:DiscoverPollEndpoint", + "ecs:Poll", + "ecs:RegisterContainerInstance", + "ecs:StartTelemetrySession", + "ecs:Submit*", + "logs:CreateLogStream", + "logs:PutLogEvents", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:GetAuthorizationToken", + "ssm:DescribeAssociation", + "ssm:GetDeployablePatchSnapshotForInstance", + "ssm:GetDocument", + "ssm:GetManifest", + "ssm:GetParameters", + "ssm:ListAssociations", + "ssm:ListInstanceAssociations", + "ssm:PutInventory", + "ssm:PutComplianceItems", + "ssm:PutConfigurePackageResult", + "ssm:PutParameter", + "ssm:UpdateAssociationStatus", + "ssm:UpdateInstanceAssociationStatus", + "ssm:UpdateInstanceInformation", + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply", + "cloudwatch:PutMetricData", + "ec2:DescribeInstanceStatus", + "ds:CreateComputer", + "ds:DescribeDirectories", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "firehose:PutRecord", + "firehose:PutRecordBatch", + "ec2:DescribeAddresses", + "ec2:DescribeInstances", + "ec2:AssociateAddress" + ], + "Resource": "*" + }] + } + + ECSInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: / + Roles: + - !Ref ECSRole + + ECSServiceAutoScalingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + Action: + - "sts:AssumeRole" + Effect: Allow + Principal: + Service: + - application-autoscaling.amazonaws.com + Path: / + Policies: + - PolicyName: ecs-service-autoscaling + PolicyDocument: + Statement: + Effect: Allow + Action: + - application-autoscaling:* + - cloudwatch:DescribeAlarms + - cloudwatch:PutMetricAlarm + - ecs:DescribeServices + - ecs:UpdateService + Resource: "*" + + NodeTaskExecutionRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: !Sub ecs-task-S3-${AWS::StackName} + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - "s3:Get*" + - "s3:List*" + Resource: + - Fn::Join: + - "" + - - "arn:aws:s3:::" + - !FindInMap + - RegionMap + - !Ref "AWS::Region" + - !Ref Network + + NodeLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub /ecs/${AWS::StackName}-node + RetentionInDays: 14 + + NodeECSService: + Type: AWS::ECS::Service + DependsOn: NodeListener + Properties: + Cluster: !Ref Cluster + DesiredCount: !Ref NodeDesiredCount + HealthCheckGracePeriodSeconds: 3600 + TaskDefinition: !Ref NodeTaskDefinition + LaunchType: EC2 + DeploymentConfiguration: + MaximumPercent: 150 + MinimumHealthyPercent: 50 + LoadBalancers: + - ContainerName: !Ref NodeTaskName + ContainerPort: !Ref RpcPort + TargetGroupArn: !Ref NodeTargetGroup + - ContainerName: !Ref NodeTaskName + ContainerPort: !Ref WsPort + TargetGroupArn: !Ref NodeWsTargetGroup + + NodeTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Ref NodeTaskName + RequiresCompatibilities: + - EC2 + NetworkMode: host + ExecutionRoleArn: !Ref NodeTaskExecutionRole + ContainerDefinitions: + - Name: !Ref NodeTaskName + Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${NodeRepository} + Essential: true + MemoryReservation: !Ref MemoryLimit + Environment: + - Name: "region" + Value: !Ref AWS::Region + - Name: "network" + Value: !Ref Network + - Name: "syncmode" + Value: !Ref SyncMode + - Name: "connections" + Value: !Ref Connections + - Name: "rpcport" + Value: !Ref RpcPort + - Name: "wsport" + Value: !Ref WsPort + - Name: "netport" + Value: !Ref NetPort + - Name: "chainbucket" + Value: !FindInMap + - RegionMap + - !Ref "AWS::Region" + - !Ref Network + - Name: "s3key" + Value: node + PortMappings: + - ContainerPort: !Ref RpcPort + - ContainerPort: !Ref WsPort + - ContainerPort: !Ref NetPort + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-region: !Ref AWS::Region + awslogs-group: !Ref NodeLogGroup + awslogs-stream-prefix: !Ref AWS::StackName + #HealthCheck: + # Command: + # - CMD-SHELL + # - '[ `echo "eth.syncing.highestBlock - eth.syncing.currentBlock"|geth attach|head -10|tail -1` -lt 200 ] || exit 1' + # Interval: 300 + # Timeout: 60 + # Retries: 10 + # StartPeriod: 300 + + # CodePipeline Resources + + NodeRepository: + Type: AWS::ECR::Repository + + NodeCodeBuildServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: "*" + Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - ecr:GetAuthorizationToken + - Resource: !Sub arn:aws:s3:::${NodeArtifactBucket}/* + Effect: Allow + Action: + - s3:GetObject + - s3:PutObject + - s3:GetObjectVersion + - Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${NodeRepository} + Effect: Allow + Action: + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - ecr:BatchCheckLayerAvailability + - ecr:PutImage + - ecr:InitiateLayerUpload + - ecr:UploadLayerPart + - ecr:CompleteLayerUpload + + NodeCodePipelineServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codepipeline.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: + - !Sub arn:aws:s3:::${NodeArtifactBucket}/* + Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + - s3:GetObjectVersion + - s3:GetBucketVersioning + - Resource: "*" + Effect: Allow + Action: + - ecs:DescribeServices + - ecs:DescribeTaskDefinition + - ecs:DescribeTasks + - ecs:ListTasks + - ecs:RegisterTaskDefinition + - ecs:UpdateService + - codebuild:StartBuild + - codebuild:BatchGetBuilds + - iam:PassRole + + NodeArtifactBucket: + Type: AWS::S3::Bucket + + NodeCodeBuildProject: + Type: AWS::CodeBuild::Project + Properties: + Artifacts: + Type: CODEPIPELINE + Source: + Type: CODEPIPELINE + BuildSpec: | + version: 0.2 + phases: + install: + runtime-versions: + docker: 19 + pre_build: + commands: + - $(aws ecr get-login --no-include-email) + - TAG="$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | head -c 8)" + - IMAGE_URI="${REPOSITORY_URI}:${TAG}" + - cp infra/Dockerfile.node ./Dockerfile + build: + commands: + - docker build --tag "$IMAGE_URI" . + - docker build --tag "${REPOSITORY_URI}:latest" . + post_build: + commands: + - docker push "$IMAGE_URI" + - docker push "${REPOSITORY_URI}:latest" + - printf '[{"name":"mev-geth-node","imageUri":"%s"}]' "$IMAGE_URI" > images.json + artifacts: + files: images.json + Environment: + ComputeType: BUILD_GENERAL1_SMALL + Image: aws/codebuild/docker:17.09.0 + Type: LINUX_CONTAINER + PrivilegedMode: true + EnvironmentVariables: + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: REPOSITORY_URI + Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${NodeRepository} + Cache: + Type: S3 + Location: !Sub ${NodeArtifactBucket}/buildcache + Name: !Sub ${AWS::StackName}-node + ServiceRole: !Ref NodeCodeBuildServiceRole + + NodePipeline: + Type: AWS::CodePipeline::Pipeline + Properties: + RoleArn: !GetAtt NodeCodePipelineServiceRole.Arn + ArtifactStore: + Type: S3 + Location: !Ref NodeArtifactBucket + Stages: + - Name: Source + Actions: + - Name: App + ActionTypeId: + Category: Source + Owner: ThirdParty + Version: 1 + Provider: GitHub + Configuration: + Owner: !Ref GitHubUser + Repo: !Ref NodeGitHubRepo + Branch: !Ref NodeGitHubBranch + OAuthToken: !Ref GitHubToken + OutputArtifacts: + - Name: App + RunOrder: 1 + - Name: Build + Actions: + - Name: Build + ActionTypeId: + Category: Build + Owner: AWS + Version: 1 + Provider: CodeBuild + Configuration: + ProjectName: !Ref NodeCodeBuildProject + InputArtifacts: + - Name: App + OutputArtifacts: + - Name: BuildOutput + RunOrder: 1 + - Name: Deploy + Actions: + - Name: Deploy + ActionTypeId: + Category: Deploy + Owner: AWS + Version: 1 + Provider: ECS + Configuration: + ClusterName: !Ref Cluster + ServiceName: !Ref NodeECSService + FileName: images.json + InputArtifacts: + - Name: BuildOutput + RunOrder: 1 + + # SNS Resources + + SNSTopic: + Type: AWS::SNS::Topic + Properties: + DisplayName: String + Subscription: + - Endpoint: !Ref SNSSubscriptionEndpoint + Protocol: !Ref SNSSubscriptionProtocol + TopicName: !Ref AWS::StackName + + # CloudWatch Resources + + CPUAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub ${AWS::StackName} average CPU utilization greater than threshold. + AlarmDescription: Alarm if CPU utilization is greater than threshold. + Namespace: AWS/ECS + MetricName: CPUUtilization + Dimensions: + - Name: ClusterName + Value: !Ref Cluster + Statistic: Average + Period: "60" + EvaluationPeriods: "3" + Threshold: !Ref CPUAlarmThreshold + ComparisonOperator: GreaterThanThreshold + AlarmActions: + - Ref: SNSTopic + OKActions: + - Ref: SNSTopic + + MemoryAlarm: + Type: AWS::CloudWatch::Alarm + Properties: + AlarmName: !Sub ${AWS::StackName} average memory utilization greater than threshold. + AlarmDescription: Alarm if memory utilization is greater than threshold. + Namespace: AWS/ECS + MetricName: MemoryUtilization + Dimensions: + - Name: ClusterName + Value: !Ref Cluster + Statistic: Average + Period: "60" + EvaluationPeriods: "3" + Threshold: !Ref MemoryAlarmThreshold + ComparisonOperator: GreaterThanThreshold + AlarmActions: + - Ref: SNSTopic + OKActions: + - Ref: SNSTopic + + HealthyHostAlarm: + Type: "AWS::CloudWatch::Alarm" + Properties: + AlarmName: !Sub ${AWS::StackName} alarm no healthy hosts connected to ELB. + AlarmDescription: Alarm if no healthy hosts connected to ELB. + MetricName: HealthyHostCount + Namespace: AWS/NetworkELB + Statistic: Average + Period: "60" + EvaluationPeriods: "3" + Threshold: "1" + ComparisonOperator: LessThanThreshold + Dimensions: + - Name: TargetGroup + Value: !GetAtt NodeTargetGroup.TargetGroupFullName + - Name: LoadBalancer + Value: !GetAtt NodeLoadBalancer.LoadBalancerFullName + AlarmActions: + - Ref: SNSTopic + OKActions: + - Ref: SNSTopic + +Outputs: + ClusterName: + Value: !Ref Cluster + NodeService: + Value: !Ref NodeECSService + NodePipelineUrl: + Value: !Sub https://console.aws.amazon.com/codepipeline/home?region=${AWS::Region}#/view/${NodePipeline} + NodeTargetGroup: + Value: !Ref NodeTargetGroup + NodeServiceUrl: + Description: URL of the load balancer for the node service. + Value: !Sub http://${NodeLoadBalancer.DNSName} diff --git a/infra/mev-geth-updater-arm64.yaml b/infra/mev-geth-updater-arm64.yaml new file mode 100644 index 000000000000..ad81ece1b034 --- /dev/null +++ b/infra/mev-geth-updater-arm64.yaml @@ -0,0 +1,749 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Description: > + This template creates an automated continuous deployment pipeline to Amazon Elastic Container Service (ECS) + Created by Luke Youngblood, luke@blockscale.net + +Parameters: + +# GitHub Parameters + + GitHubUser: + Type: String + Default: lyoungblood + Description: Your team or username on GitHub. + + GitHubRepo: + Type: String + Default: mev-geth + Description: The repo name of the baker service. + + GitHubBranch: + Type: String + Default: master + Description: The branch of the repo to continuously deploy. + + GitHubToken: + Type: String + NoEcho: true + Description: > + Token for the team or user specified above. (https://github.com/settings/tokens) + +# VPC Parameters + + VPC: + Type: AWS::EC2::VPC::Id + + Subnets: + Type: List + + VpcCIDR: + Type: String + Default: 172.31.0.0/16 + +# ECS Parameters + + InstanceType: + Type: String + Default: m6gd.large + + KeyPair: + Type: AWS::EC2::KeyPair::KeyName + + ClusterSize: + Type: Number + Default: 1 + + DesiredCount: + Type: Number + Default: 0 + + TaskName: + Type: String + Default: mev-geth-updater + + ECSAMI: + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ecs/optimized-ami/amazon-linux-2/arm64/recommended/image_id + +# Mev-Geth Parameters + + Network: + Type: String + Default: mainnet + AllowedValues: + - mainnet + - goerli + + SyncMode: + Type: String + Default: fast + AllowedValues: + - full + - fast + - light + + Connections: + Type: Number + Default: 50 + + NetPort: + Type: Number + Default: 30303 + +Metadata: + + AWS::CloudFormation::Interface: + ParameterLabels: + GitHubUser: + default: "User" + GitHubRepo: + default: "Mev-Geth GitHub Repository" + GitHubBranch: + default: "Branch in GitHub repository" + GitHubToken: + default: "Personal Access Token" + VPC: + default: "Choose which VPC the autoscaling group should be deployed to" + Subnets: + default: "Choose which subnets the autoscaling group should be deployed to" + VpcCIDR: + default: "VPC CIDR Block" + InstanceType: + default: "Which instance type should we use to build the ECS cluster?" + KeyPair: + default: "Which keypair should be used for access to the ECS cluster?" + ClusterSize: + default: "How many ECS hosts do you want to initially deploy?" + DesiredCount: + default: "How many Updater tasks do you want to initially execute?" + TaskName: + default: "The name of the Updater ECS Task" + ECSAMI: + default: "The ECS AMI ID populated from SSM." + Network: + default: "The network the Mev-Geth node should join" + SyncMode: + default: "The synchronization mode that Mev-Geth should use (full, fast, or light)" + Connections: + default: "The number of connections the Mev-Geth node should be configured with" + NetPort: + default: "The TCP/UDP port used for Mev-Geth connectivity to other Ethereum peer nodes" + ParameterGroups: + - Label: + default: GitHub Configuration + Parameters: + - GitHubRepo + - GitHubBranch + - GitHubUser + - GitHubToken + - Label: + default: VPC Configuration + Parameters: + - VPC + - Subnets + - VpcCIDR + - Label: + default: ECS Configuration + Parameters: + - InstanceType + - KeyPair + - ClusterSize + - DesiredCount + - TaskName + - ECSAMI + - Label: + default: Mev-Geth Configuration + Parameters: + - Network + - SyncMode + - Connections + - NetPort + +Resources: + +# ECS Resources + + ChainBucket: + Type: AWS::S3::Bucket + + ChainBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref ChainBucket + PolicyDocument: + Statement: + - + Action: + - s3:GetObject + - s3:ListBucket + Effect: Allow + Resource: + - Fn::Join: + - "" + - + - "arn:aws:s3:::" + - + Ref: "ChainBucket" + - "/*" + - Fn::Join: + - "" + - + - "arn:aws:s3:::" + - + Ref: "ChainBucket" + Principal: + AWS: "*" + + Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Ref AWS::StackName + + SecurityGroup: + Type: "AWS::EC2::SecurityGroup" + Properties: + GroupDescription: !Sub ${AWS::StackName}-sg + VpcId: !Ref VPC + Tags: + - + Key: Name + Value: !Sub ${AWS::StackName}-sg + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + + ECSAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + VPCZoneIdentifier: !Ref Subnets + LaunchConfigurationName: !Ref ECSLaunchConfiguration + MinSize: !Ref ClusterSize + MaxSize: !Ref ClusterSize + DesiredCapacity: !Ref ClusterSize + Tags: + - Key: Name + Value: !Sub ${AWS::StackName} ECS host + PropagateAtLaunch: true + CreationPolicy: + ResourceSignal: + Timeout: PT15M + UpdatePolicy: + AutoScalingRollingUpdate: + MinInstancesInService: 0 + MaxBatchSize: 1 + PauseTime: PT15M + SuspendProcesses: + - HealthCheck + - ReplaceUnhealthy + - AZRebalance + - AlarmNotification + - ScheduledActions + WaitOnResourceSignals: true + + ECSLaunchConfiguration: + Type: AWS::AutoScaling::LaunchConfiguration + Properties: + ImageId: !Ref ECSAMI + InstanceType: !Ref InstanceType + KeyName: !Ref KeyPair + SecurityGroups: + - !Ref SecurityGroup + IamInstanceProfile: !Ref ECSInstanceProfile + UserData: + "Fn::Base64": !Sub | + #!/bin/bash + yum install -y aws-cfn-bootstrap hibagent rsync awscli + yum update -y + service amazon-ssm-agent restart + + # determine if we have an NVMe SSD attached + find /dev/nvme1 + if [ $? -eq 0 ] + then + mount_point=/var/lib/docker + + # copy existing files from mount point + service docker stop + echo 'DOCKER_STORAGE_OPTIONS="--storage-driver overlay2"' > /etc/sysconfig/docker-storage + mkdir -p /tmp$mount_point + rsync -val $mount_point/ /tmp/$mount_point/ + + # make a new filesystem and mount it + mkfs -t ext4 /dev/nvme1n1 + mkdir -p $mount_point + mount -t ext4 -o noatime /dev/nvme1n1 $mount_point + + # Copy files back to new mount point + rsync -val /tmp/$mount_point/ $mount_point/ + rm -rf /tmp$mount_point + service docker start + + # Make raid appear on reboot + echo >> /etc/fstab + echo "/dev/nvme1n1 $mount_point ext4 noatime 0 0" | tee -a /etc/fstab + fi + + /opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + /opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSAutoScalingGroup + /usr/bin/enable-ec2-spot-hibernation + echo "ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION=5m" >> /etc/ecs/ecs.config + + reboot + + Metadata: + AWS::CloudFormation::Init: + config: + packages: + yum: + awslogs: [] + + commands: + 01_add_instance_to_cluster: + command: !Sub echo ECS_CLUSTER=${Cluster} >> /etc/ecs/ecs.config + files: + "/etc/cfn/cfn-hup.conf": + mode: 000400 + owner: root + group: root + content: !Sub | + [main] + stack=${AWS::StackId} + region=${AWS::Region} + + "/etc/cfn/hooks.d/cfn-auto-reloader.conf": + content: !Sub | + [cfn-auto-reloader-hook] + triggers=post.update + path=Resources.ECSLaunchConfiguration.Metadata.AWS::CloudFormation::Init + action=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + + "/etc/awslogs/awscli.conf": + content: !Sub | + [plugins] + cwlogs = cwlogs + [default] + region = ${AWS::Region} + + services: + sysvinit: + cfn-hup: + enabled: true + ensureRunning: true + files: + - /etc/cfn/cfn-hup.conf + - /etc/cfn/hooks.d/cfn-auto-reloader.conf + + # This IAM Role is attached to all of the ECS hosts. It is based on the default role + # published here: + # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/instance_IAM_role.html + # + # You can add other IAM policy statements here to allow access from your ECS hosts + # to other AWS services. + + ECSRole: + Type: AWS::IAM::Role + Properties: + Path: / + RoleName: !Sub ${AWS::StackName}-ECSRole-${AWS::Region} + AssumeRolePolicyDocument: | + { + "Statement": [{ + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + }] + } + Policies: + - PolicyName: ecs-service + PolicyDocument: | + { + "Statement": [{ + "Effect": "Allow", + "Action": [ + "ecs:CreateCluster", + "ecs:DeregisterContainerInstance", + "ecs:DiscoverPollEndpoint", + "ecs:Poll", + "ecs:RegisterContainerInstance", + "ecs:StartTelemetrySession", + "ecs:Submit*", + "logs:CreateLogStream", + "logs:PutLogEvents", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:GetAuthorizationToken", + "ssm:DescribeAssociation", + "ssm:GetDeployablePatchSnapshotForInstance", + "ssm:GetDocument", + "ssm:GetManifest", + "ssm:GetParameters", + "ssm:ListAssociations", + "ssm:ListInstanceAssociations", + "ssm:PutInventory", + "ssm:PutComplianceItems", + "ssm:PutConfigurePackageResult", + "ssm:PutParameter", + "ssm:UpdateAssociationStatus", + "ssm:UpdateInstanceAssociationStatus", + "ssm:UpdateInstanceInformation", + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply", + "cloudwatch:PutMetricData", + "ec2:DescribeInstanceStatus", + "ds:CreateComputer", + "ds:DescribeDirectories", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + "s3:*" + ], + "Resource": "*" + }] + } + + ECSInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: / + Roles: + - !Ref ECSRole + + ECSServiceAutoScalingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + Action: + - 'sts:AssumeRole' + Effect: Allow + Principal: + Service: + - application-autoscaling.amazonaws.com + Path: / + Policies: + - PolicyName: ecs-service-autoscaling + PolicyDocument: + Statement: + Effect: Allow + Action: + - application-autoscaling:* + - cloudwatch:DescribeAlarms + - cloudwatch:PutMetricAlarm + - ecs:DescribeServices + - ecs:UpdateService + Resource: "*" + + TaskExecutionRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: !Sub ecs-task-S3-${AWS::StackName} + PolicyDocument: + Version: 2012-10-17 + Statement: + - + Effect: Allow + Action: + - "s3:Get*" + - "s3:List*" + - "s3:Put*" + Resource: + - !GetAtt ChainBucket.Arn + - PolicyName: !Sub ecs-task-SSM-${AWS::StackName} + PolicyDocument: + Version: 2012-10-17 + Statement: + - + Effect: Allow + Action: + - "ssm:DescribeParameters" + - "ssm:PutParameter" + - "ssm:GetParameters" + Resource: + - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/${AWS::StackName}/*" + + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub /ecs/${AWS::StackName} + RetentionInDays: 14 + + ECSService: + Type: AWS::ECS::Service + Properties: + Cluster: !Ref Cluster + DesiredCount: !Ref DesiredCount + TaskDefinition: !Ref TaskDefinition + LaunchType: EC2 + DeploymentConfiguration: + MaximumPercent: 100 + MinimumHealthyPercent: 0 + + TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub ${AWS::StackName}-${TaskName} + RequiresCompatibilities: + - EC2 + NetworkMode: host + ExecutionRoleArn: !Ref TaskExecutionRole + ContainerDefinitions: + - Name: !Ref TaskName + Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${Repository} + Essential: true + MemoryReservation: 6144 + Environment: + - Name: "network" + Value: !Ref Network + - Name: "syncmode" + Value: !Ref SyncMode + - Name: "connections" + Value: !Ref Connections + - Name: "netport" + Value: !Ref NetPort + - Name: "region" + Value: !Ref AWS::Region + - Name: "chainbucket" + Value: !Ref ChainBucket + - Name: "s3key" + Value: node + PortMappings: + - ContainerPort: !Ref NetPort + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-region: !Ref AWS::Region + awslogs-group: !Ref LogGroup + awslogs-stream-prefix: !Ref AWS::StackName + +# CodePipeline Resources + + Repository: + Type: AWS::ECR::Repository + + CodeBuildServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: "*" + Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - ecr:GetAuthorizationToken + - Resource: !Sub arn:aws:s3:::${ArtifactBucket}/* + Effect: Allow + Action: + - s3:GetObject + - s3:PutObject + - s3:GetObjectVersion + - Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${Repository} + Effect: Allow + Action: + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - ecr:BatchCheckLayerAvailability + - ecr:PutImage + - ecr:InitiateLayerUpload + - ecr:UploadLayerPart + - ecr:CompleteLayerUpload + + CodePipelineServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codepipeline.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: + - !Sub arn:aws:s3:::${ArtifactBucket}/* + Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + - s3:GetObjectVersion + - s3:GetBucketVersioning + - Resource: "*" + Effect: Allow + Action: + - ecs:DescribeServices + - ecs:DescribeTaskDefinition + - ecs:DescribeTasks + - ecs:ListTasks + - ecs:RegisterTaskDefinition + - ecs:UpdateService + - codebuild:StartBuild + - codebuild:BatchGetBuilds + - iam:PassRole + + ArtifactBucket: + Type: AWS::S3::Bucket + + CodeBuildProject: + Type: AWS::CodeBuild::Project + Properties: + Artifacts: + Type: CODEPIPELINE + Source: + Type: CODEPIPELINE + BuildSpec: | + version: 0.2 + phases: + install: + runtime-versions: + docker: 19 + pre_build: + commands: + - $(aws ecr get-login --no-include-email) + - TAG="$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | head -c 8)" + - IMAGE_URI="${REPOSITORY_URI}:${TAG}" + - cp infra/Dockerfile.updater ./Dockerfile + build: + commands: + - docker build --tag "$IMAGE_URI" . + - docker build --tag "${REPOSITORY_URI}:latest" . + post_build: + commands: + - docker push "$IMAGE_URI" + - docker push "${REPOSITORY_URI}:latest" + - printf '[{"name":"mev-geth-updater","imageUri":"%s"}]' "$IMAGE_URI" > images.json + artifacts: + files: images.json + Environment: + ComputeType: BUILD_GENERAL1_LARGE + Image: aws/codebuild/amazonlinux2-aarch64-standard:1.0 + Type: ARM_CONTAINER + PrivilegedMode: true + EnvironmentVariables: + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: REPOSITORY_URI + Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${Repository} + Name: !Ref AWS::StackName + ServiceRole: !Ref CodeBuildServiceRole + + Pipeline: + Type: AWS::CodePipeline::Pipeline + Properties: + RoleArn: !GetAtt CodePipelineServiceRole.Arn + ArtifactStore: + Type: S3 + Location: !Ref ArtifactBucket + Stages: + - Name: Source + Actions: + - Name: App + ActionTypeId: + Category: Source + Owner: ThirdParty + Version: 1 + Provider: GitHub + Configuration: + Owner: !Ref GitHubUser + Repo: !Ref GitHubRepo + Branch: !Ref GitHubBranch + OAuthToken: !Ref GitHubToken + OutputArtifacts: + - Name: App + RunOrder: 1 + - Name: Build + Actions: + - Name: Build + ActionTypeId: + Category: Build + Owner: AWS + Version: 1 + Provider: CodeBuild + Configuration: + ProjectName: !Ref CodeBuildProject + InputArtifacts: + - Name: App + OutputArtifacts: + - Name: BuildOutput + RunOrder: 1 + - Name: Deploy + Actions: + - Name: Deploy + ActionTypeId: + Category: Deploy + Owner: AWS + Version: 1 + Provider: ECS + Configuration: + ClusterName: !Ref Cluster + ServiceName: !Ref ECSService + FileName: images.json + InputArtifacts: + - Name: BuildOutput + RunOrder: 1 + +Outputs: + + ClusterName: + Value: !Ref Cluster + Service: + Value: !Ref ECSService + PipelineUrl: + Value: !Sub https://console.aws.amazon.com/codepipeline/home?region=${AWS::Region}#/view/${Pipeline} \ No newline at end of file diff --git a/infra/mev-geth-updater-x86-64.yaml b/infra/mev-geth-updater-x86-64.yaml new file mode 100644 index 000000000000..a69d1bb10d18 --- /dev/null +++ b/infra/mev-geth-updater-x86-64.yaml @@ -0,0 +1,737 @@ +--- +AWSTemplateFormatVersion: 2010-09-09 + +Description: > + This template creates an automated continuous deployment pipeline to Amazon Elastic Container Service (ECS) + Created by Luke Youngblood, luke@blockscale.net + +Parameters: + # GitHub Parameters + + GitHubUser: + Type: String + Default: lyoungblood + Description: Your team or username on GitHub. + + GitHubRepo: + Type: String + Default: mev-geth + Description: The repo name of the baker service. + + GitHubBranch: + Type: String + Default: master + Description: The branch of the repo to continuously deploy. + + GitHubToken: + Type: String + NoEcho: true + Description: > + Token for the team or user specified above. (https://github.com/settings/tokens) + + # VPC Parameters + + VPC: + Type: AWS::EC2::VPC::Id + + Subnets: + Type: List + + VpcCIDR: + Type: String + Default: 172.31.0.0/16 + + # ECS Parameters + + InstanceType: + Type: String + Default: i3en.large + + KeyPair: + Type: AWS::EC2::KeyPair::KeyName + + ClusterSize: + Type: Number + Default: 1 + + DesiredCount: + Type: Number + Default: 0 + + TaskName: + Type: String + Default: mev-geth-updater + + ECSAMI: + Type: AWS::SSM::Parameter::Value + Default: /aws/service/ecs/optimized-ami/amazon-linux-2/recommended/image_id + + # Mev-Geth Parameters + + Network: + Type: String + Default: mainnet + AllowedValues: + - mainnet + - goerli + + SyncMode: + Type: String + Default: fast + AllowedValues: + - full + - fast + - light + + Connections: + Type: Number + Default: 50 + + NetPort: + Type: Number + Default: 30303 + +Metadata: + AWS::CloudFormation::Interface: + ParameterLabels: + GitHubUser: + default: "User" + GitHubRepo: + default: "Mev-Geth GitHub Repository" + GitHubBranch: + default: "Branch in GitHub repository" + GitHubToken: + default: "Personal Access Token" + VPC: + default: "Choose which VPC the autoscaling group should be deployed to" + Subnets: + default: "Choose which subnets the autoscaling group should be deployed to" + VpcCIDR: + default: "VPC CIDR Block" + InstanceType: + default: "Which instance type should we use to build the ECS cluster?" + KeyPair: + default: "Which keypair should be used for access to the ECS cluster?" + ClusterSize: + default: "How many ECS hosts do you want to initially deploy?" + DesiredCount: + default: "How many Updater tasks do you want to initially execute?" + TaskName: + default: "The name of the Updater ECS Task" + ECSAMI: + default: "The ECS AMI ID populated from SSM." + Network: + default: "The network the Mev-Geth node should join" + SyncMode: + default: "The synchronization mode that Mev-Geth should use (full, fast, or light)" + Connections: + default: "The number of connections the Mev-Geth node should be configured with" + NetPort: + default: "The TCP/UDP port used for Mev-Geth connectivity to other Ethereum peer nodes" + ParameterGroups: + - Label: + default: GitHub Configuration + Parameters: + - GitHubRepo + - GitHubBranch + - GitHubUser + - GitHubToken + - Label: + default: VPC Configuration + Parameters: + - VPC + - Subnets + - VpcCIDR + - Label: + default: ECS Configuration + Parameters: + - InstanceType + - KeyPair + - ClusterSize + - DesiredCount + - TaskName + - ECSAMI + - Label: + default: Mev-Geth Configuration + Parameters: + - Network + - SyncMode + - Connections + - NetPort + +Resources: + # ECS Resources + + ChainBucket: + Type: AWS::S3::Bucket + + ChainBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref ChainBucket + PolicyDocument: + Statement: + - Action: + - s3:GetObject + - s3:ListBucket + Effect: Allow + Resource: + - Fn::Join: + - "" + - - "arn:aws:s3:::" + - Ref: "ChainBucket" + - "/*" + - Fn::Join: + - "" + - - "arn:aws:s3:::" + - Ref: "ChainBucket" + Principal: + AWS: "*" + + Cluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Ref AWS::StackName + + SecurityGroup: + Type: "AWS::EC2::SecurityGroup" + Properties: + GroupDescription: !Sub ${AWS::StackName}-sg + VpcId: !Ref VPC + Tags: + - Key: Name + Value: !Sub ${AWS::StackName}-sg + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + CidrIp: !Ref VpcCIDR + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: tcp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIp: 0.0.0.0/0 + - IpProtocol: udp + FromPort: !Ref NetPort + ToPort: !Ref NetPort + CidrIpv6: ::/0 + + ECSAutoScalingGroup: + Type: AWS::AutoScaling::AutoScalingGroup + Properties: + VPCZoneIdentifier: !Ref Subnets + LaunchConfigurationName: !Ref ECSLaunchConfiguration + MinSize: !Ref ClusterSize + MaxSize: !Ref ClusterSize + DesiredCapacity: !Ref ClusterSize + Tags: + - Key: Name + Value: !Sub ${AWS::StackName} ECS host + PropagateAtLaunch: true + CreationPolicy: + ResourceSignal: + Timeout: PT15M + UpdatePolicy: + AutoScalingRollingUpdate: + MinInstancesInService: 0 + MaxBatchSize: 1 + PauseTime: PT15M + SuspendProcesses: + - HealthCheck + - ReplaceUnhealthy + - AZRebalance + - AlarmNotification + - ScheduledActions + WaitOnResourceSignals: true + + ECSLaunchConfiguration: + Type: AWS::AutoScaling::LaunchConfiguration + Properties: + ImageId: !Ref ECSAMI + InstanceType: !Ref InstanceType + KeyName: !Ref KeyPair + SecurityGroups: + - !Ref SecurityGroup + IamInstanceProfile: !Ref ECSInstanceProfile + UserData: + "Fn::Base64": !Sub | + #!/bin/bash + yum install -y aws-cfn-bootstrap hibagent rsync awscli + yum update -y + service amazon-ssm-agent restart + + # determine if we have an NVMe SSD attached + find /dev/nvme1 + if [ $? -eq 0 ] + then + mount_point=/var/lib/docker + + # copy existing files from mount point + service docker stop + echo 'DOCKER_STORAGE_OPTIONS="--storage-driver overlay2"' > /etc/sysconfig/docker-storage + mkdir -p /tmp$mount_point + rsync -val $mount_point/ /tmp/$mount_point/ + + # make a new filesystem and mount it + mkfs -t ext4 /dev/nvme1n1 + mkdir -p $mount_point + mount -t ext4 -o noatime /dev/nvme1n1 $mount_point + + # Copy files back to new mount point + rsync -val /tmp/$mount_point/ $mount_point/ + rm -rf /tmp$mount_point + service docker start + + # Make raid appear on reboot + echo >> /etc/fstab + echo "/dev/nvme1n1 $mount_point ext4 noatime 0 0" | tee -a /etc/fstab + fi + + /opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + /opt/aws/bin/cfn-signal -e $? --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSAutoScalingGroup + /usr/bin/enable-ec2-spot-hibernation + echo "ECS_ENGINE_TASK_CLEANUP_WAIT_DURATION=5m" >> /etc/ecs/ecs.config + + reboot + + Metadata: + AWS::CloudFormation::Init: + config: + packages: + yum: + awslogs: [] + + commands: + 01_add_instance_to_cluster: + command: !Sub echo ECS_CLUSTER=${Cluster} >> /etc/ecs/ecs.config + files: + "/etc/cfn/cfn-hup.conf": + mode: 000400 + owner: root + group: root + content: !Sub | + [main] + stack=${AWS::StackId} + region=${AWS::Region} + + "/etc/cfn/hooks.d/cfn-auto-reloader.conf": + content: !Sub | + [cfn-auto-reloader-hook] + triggers=post.update + path=Resources.ECSLaunchConfiguration.Metadata.AWS::CloudFormation::Init + action=/opt/aws/bin/cfn-init -v --region ${AWS::Region} --stack ${AWS::StackName} --resource ECSLaunchConfiguration + + "/etc/awslogs/awscli.conf": + content: !Sub | + [plugins] + cwlogs = cwlogs + [default] + region = ${AWS::Region} + + services: + sysvinit: + cfn-hup: + enabled: true + ensureRunning: true + files: + - /etc/cfn/cfn-hup.conf + - /etc/cfn/hooks.d/cfn-auto-reloader.conf + + # This IAM Role is attached to all of the ECS hosts. It is based on the default role + # published here: + # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/instance_IAM_role.html + # + # You can add other IAM policy statements here to allow access from your ECS hosts + # to other AWS services. + + ECSRole: + Type: AWS::IAM::Role + Properties: + Path: / + RoleName: !Sub ${AWS::StackName}-ECSRole-${AWS::Region} + AssumeRolePolicyDocument: | + { + "Statement": [{ + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + }] + } + Policies: + - PolicyName: ecs-service + PolicyDocument: | + { + "Statement": [{ + "Effect": "Allow", + "Action": [ + "ecs:CreateCluster", + "ecs:DeregisterContainerInstance", + "ecs:DiscoverPollEndpoint", + "ecs:Poll", + "ecs:RegisterContainerInstance", + "ecs:StartTelemetrySession", + "ecs:Submit*", + "logs:CreateLogStream", + "logs:PutLogEvents", + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + "ecr:GetAuthorizationToken", + "ssm:DescribeAssociation", + "ssm:GetDeployablePatchSnapshotForInstance", + "ssm:GetDocument", + "ssm:GetManifest", + "ssm:GetParameters", + "ssm:ListAssociations", + "ssm:ListInstanceAssociations", + "ssm:PutInventory", + "ssm:PutComplianceItems", + "ssm:PutConfigurePackageResult", + "ssm:PutParameter", + "ssm:UpdateAssociationStatus", + "ssm:UpdateInstanceAssociationStatus", + "ssm:UpdateInstanceInformation", + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply", + "cloudwatch:PutMetricData", + "ec2:DescribeInstanceStatus", + "ds:CreateComputer", + "ds:DescribeDirectories", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + "s3:*" + ], + "Resource": "*" + }] + } + + ECSInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: / + Roles: + - !Ref ECSRole + + ECSServiceAutoScalingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + Action: + - "sts:AssumeRole" + Effect: Allow + Principal: + Service: + - application-autoscaling.amazonaws.com + Path: / + Policies: + - PolicyName: ecs-service-autoscaling + PolicyDocument: + Statement: + Effect: Allow + Action: + - application-autoscaling:* + - cloudwatch:DescribeAlarms + - cloudwatch:PutMetricAlarm + - ecs:DescribeServices + - ecs:UpdateService + Resource: "*" + + TaskExecutionRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: !Sub ecs-task-S3-${AWS::StackName} + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - "s3:Get*" + - "s3:List*" + - "s3:Put*" + Resource: + - !GetAtt ChainBucket.Arn + - PolicyName: !Sub ecs-task-SSM-${AWS::StackName} + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - "ssm:DescribeParameters" + - "ssm:PutParameter" + - "ssm:GetParameters" + Resource: + - !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/${AWS::StackName}/*" + + LogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub /ecs/${AWS::StackName} + RetentionInDays: 14 + + ECSService: + Type: AWS::ECS::Service + Properties: + Cluster: !Ref Cluster + DesiredCount: !Ref DesiredCount + TaskDefinition: !Ref TaskDefinition + LaunchType: EC2 + DeploymentConfiguration: + MaximumPercent: 100 + MinimumHealthyPercent: 0 + + TaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub ${AWS::StackName}-${TaskName} + RequiresCompatibilities: + - EC2 + NetworkMode: host + ExecutionRoleArn: !Ref TaskExecutionRole + ContainerDefinitions: + - Name: !Ref TaskName + Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${Repository} + Essential: true + MemoryReservation: 6144 + Environment: + - Name: "network" + Value: !Ref Network + - Name: "syncmode" + Value: !Ref SyncMode + - Name: "connections" + Value: !Ref Connections + - Name: "netport" + Value: !Ref NetPort + - Name: "region" + Value: !Ref AWS::Region + - Name: "chainbucket" + Value: !Ref ChainBucket + - Name: "s3key" + Value: node + PortMappings: + - ContainerPort: !Ref NetPort + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-region: !Ref AWS::Region + awslogs-group: !Ref LogGroup + awslogs-stream-prefix: !Ref AWS::StackName + + # CodePipeline Resources + + Repository: + Type: AWS::ECR::Repository + + CodeBuildServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codebuild.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: "*" + Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + - ecr:GetAuthorizationToken + - Resource: !Sub arn:aws:s3:::${ArtifactBucket}/* + Effect: Allow + Action: + - s3:GetObject + - s3:PutObject + - s3:GetObjectVersion + - Resource: !Sub arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${Repository} + Effect: Allow + Action: + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - ecr:BatchCheckLayerAvailability + - ecr:PutImage + - ecr:InitiateLayerUpload + - ecr:UploadLayerPart + - ecr:CompleteLayerUpload + + CodePipelineServiceRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: codepipeline.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Resource: + - !Sub arn:aws:s3:::${ArtifactBucket}/* + Effect: Allow + Action: + - s3:PutObject + - s3:GetObject + - s3:GetObjectVersion + - s3:GetBucketVersioning + - Resource: "*" + Effect: Allow + Action: + - ecs:DescribeServices + - ecs:DescribeTaskDefinition + - ecs:DescribeTasks + - ecs:ListTasks + - ecs:RegisterTaskDefinition + - ecs:UpdateService + - codebuild:StartBuild + - codebuild:BatchGetBuilds + - iam:PassRole + + ArtifactBucket: + Type: AWS::S3::Bucket + + CodeBuildProject: + Type: AWS::CodeBuild::Project + Properties: + Artifacts: + Type: CODEPIPELINE + Source: + Type: CODEPIPELINE + BuildSpec: | + version: 0.2 + phases: + install: + runtime-versions: + docker: 19 + pre_build: + commands: + - $(aws ecr get-login --no-include-email) + - TAG="$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | head -c 8)" + - IMAGE_URI="${REPOSITORY_URI}:${TAG}" + - cp infra/Dockerfile.updater ./Dockerfile + build: + commands: + - docker build --tag "$IMAGE_URI" . + - docker build --tag "${REPOSITORY_URI}:latest" . + post_build: + commands: + - docker push "$IMAGE_URI" + - docker push "${REPOSITORY_URI}:latest" + - printf '[{"name":"mev-geth-updater","imageUri":"%s"}]' "$IMAGE_URI" > images.json + artifacts: + files: images.json + Environment: + ComputeType: BUILD_GENERAL1_SMALL + Image: aws/codebuild/docker:17.09.0 + Type: LINUX_CONTAINER + PrivilegedMode: true + EnvironmentVariables: + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: REPOSITORY_URI + Value: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${Repository} + Name: !Ref AWS::StackName + ServiceRole: !Ref CodeBuildServiceRole + + Pipeline: + Type: AWS::CodePipeline::Pipeline + Properties: + RoleArn: !GetAtt CodePipelineServiceRole.Arn + ArtifactStore: + Type: S3 + Location: !Ref ArtifactBucket + Stages: + - Name: Source + Actions: + - Name: App + ActionTypeId: + Category: Source + Owner: ThirdParty + Version: 1 + Provider: GitHub + Configuration: + Owner: !Ref GitHubUser + Repo: !Ref GitHubRepo + Branch: !Ref GitHubBranch + OAuthToken: !Ref GitHubToken + OutputArtifacts: + - Name: App + RunOrder: 1 + - Name: Build + Actions: + - Name: Build + ActionTypeId: + Category: Build + Owner: AWS + Version: 1 + Provider: CodeBuild + Configuration: + ProjectName: !Ref CodeBuildProject + InputArtifacts: + - Name: App + OutputArtifacts: + - Name: BuildOutput + RunOrder: 1 + - Name: Deploy + Actions: + - Name: Deploy + ActionTypeId: + Category: Deploy + Owner: AWS + Version: 1 + Provider: ECS + Configuration: + ClusterName: !Ref Cluster + ServiceName: !Ref ECSService + FileName: images.json + InputArtifacts: + - Name: BuildOutput + RunOrder: 1 + +Outputs: + ClusterName: + Value: !Ref Cluster + Service: + Value: !Ref ECSService + PipelineUrl: + Value: !Sub https://console.aws.amazon.com/codepipeline/home?region=${AWS::Region}#/view/${Pipeline} diff --git a/infra/start-mev-geth-node.sh b/infra/start-mev-geth-node.sh new file mode 100755 index 000000000000..05ad50c61003 --- /dev/null +++ b/infra/start-mev-geth-node.sh @@ -0,0 +1,96 @@ +#!/bin/sh -x +# Starts the Mev-Geth node client +# Written by Luke Youngblood, luke@blockscale.net + +# network=mainnet # normally set by environment +# syncmode=fast # normally set by environment +# rpcport=8545 # normally set by environment +# wsport=8546 # normally set by environment +# netport=30303 # normally set by environment + +init_node() { + # You can put any commands you would like to run to initialize the node here. + echo Initializing node... +} + +start_node() { + if [ $network = "goerli" ] + then + geth \ + --port $netport \ + --http \ + --http.addr 0.0.0.0 \ + --http.port $rpcport \ + --http.api eth,net,web3 \ + --http.vhosts '*' \ + --http.corsdomain '*' \ + --graphql \ + --graphql.corsdomain '*' \ + --graphql.vhosts '*' \ + --ws \ + --ws.addr 0.0.0.0 \ + --ws.port $wsport \ + --ws.api eth,net,web3 \ + --ws.origins '*' \ + --syncmode $syncmode \ + --cache 4096 \ + --maxpeers $connections \ + --goerli + if [ $? -ne 0 ] + then + echo "Node failed to start; exiting." + exit 1 + fi + else + geth \ + --port $netport \ + --http \ + --http.addr 0.0.0.0 \ + --http.port $rpcport \ + --http.api eth,net,web3 \ + --http.vhosts '*' \ + --http.corsdomain '*' \ + --graphql \ + --graphql.corsdomain '*' \ + --graphql.vhosts '*' \ + --ws \ + --ws.addr 0.0.0.0 \ + --ws.port $wsport \ + --ws.api eth,net,web3 \ + --ws.origins '*' \ + --syncmode $syncmode \ + --cache 4096 \ + --maxpeers $connections + if [ $? -ne 0 ] + then + echo "Node failed to start; exiting." + exit 1 + fi + fi +} + +s3_sync() { + # Determine data directory + if [ $network = "goerli" ] + then + datadir=/root/.ethereum/goerli/geth/chaindata + else + datadir=/root/.ethereum/geth/chaindata + fi + # If the current1 key exists, node1 is the most current set of blockchain data + echo "A 404 error below is expected and nothing to be concerned with." + aws s3api head-object --request-payer requester --bucket $chainbucket --key current1 + if [ $? -eq 0 ] + then + s3key=node1 + else + s3key=node2 + fi + aws s3 sync --only-show-errors --request-payer requester --region $region s3://$chainbucket/$s3key $datadir +} + +# main + +init_node +s3_sync +start_node diff --git a/infra/start-mev-geth-updater.sh b/infra/start-mev-geth-updater.sh new file mode 100755 index 000000000000..11a6a533aa14 --- /dev/null +++ b/infra/start-mev-geth-updater.sh @@ -0,0 +1,181 @@ +#!/bin/sh -x +# Starts the Mev-Geth updater client +# Written by Luke Youngblood, luke@blockscale.net + +# netport=30303 # normally set by environment + +init_node() { + # Initialization steps can go here + echo Initializing node... + aws configure set default.s3.max_concurrent_requests 64 + aws configure set default.s3.max_queue_size 20000 +} + +start_node() { + if [ $network = "goerli" ] + then + geth \ + --port $netport \ + --syncmode $syncmode \ + --cache 4096 \ + --maxpeers $connections \ + --goerli & + if [ $? -ne 0 ] + then + echo "Node failed to start; exiting." + exit 1 + fi + else + geth \ + --port $netport \ + --syncmode $syncmode \ + --cache 4096 \ + --maxpeers $connections & + if [ $? -ne 0 ] + then + echo "Node failed to start; exiting." + exit 1 + fi + fi +} + +s3_sync_down() { + # Determine data directory + if [ $network = "goerli" ] + then + datadir=/root/.ethereum/goerli/geth/chaindata + else + datadir=/root/.ethereum/geth/chaindata + fi + + # If the current1 object exists, node1 is the key we should download + echo "A 404 error below is expected and nothing to be concerned with." + aws s3api head-object --bucket $chainbucket --key current1 + if [ $? -eq 0 ] + then + echo "current1 key exists; downloading node1" + s3key=node1 + else + echo "current1 key doesn't exist; downloading node2" + s3key=node2 + fi + + aws s3 sync --region $region --only-show-errors s3://$chainbucket/$s3key $datadir + if [ $? -ne 0 ] + then + echo "aws s3 sync command failed; exiting." + exit 2 + fi +} + +kill_node() { + tries=0 + while [ ! -z `ps -ef |grep geth|grep -v geth-updater|grep -v grep|awk '{print $1}'` ] + do + ps -ef |grep geth|grep -v geth-updater|grep -v grep + pid=`ps -ef |grep geth|grep -v geth-updater|grep -v grep|awk '{print $1}'` + kill $pid + sleep 30 + echo "Waiting for the node to shutdown cleanly... try number $tries" + let "tries+=1" + if [ $tries -gt 29 ] + then + echo "Node has not stopped cleanly after $tries, forcibly killing." + ps -ef |grep geth|grep -v geth-updater|grep -v grep + pid=`ps -ef |grep geth|grep -v geth-updater|grep -v grep|awk '{print $1}'` + kill -9 $pid + fi + if [ $tries -gt 30 ] + then + echo "Node has not stopped cleanly after $tries, exiting..." + exit 3 + fi + done +} + +s3_sync_up() { + # Determine data directory + if [ $network = "goerli" ] + then + datadir=/root/.ethereum/goerli/geth/chaindata + else + datadir=/root/.ethereum/geth/chaindata + fi + + # If the current1 object exists, node1 is the folder that clients will download, so we should update node2 + aws s3api head-object --bucket $chainbucket --key current1 + if [ $? -eq 0 ] + then + echo "current1 key exists; updating node2" + s3key=node2 + else + echo "current1 key doesn't exist; updating node1" + s3key=node1 + fi + + aws s3 sync --delete --region $region --only-show-errors --acl public-read $datadir s3://$chainbucket/$s3key + if [ $? -ne 0 ] + then + echo "aws s3 sync upload command failed; exiting." + exit 4 + fi + + if [ "$s3key" = "node2" ] + then + echo "Removing current1 key, as the node2 key was just updated." + aws s3 rm --region $region s3://$chainbucket/current1 + if [ $? -ne 0 ] + then + echo "aws s3 rm command failed; retrying." + sleep 5 + aws s3 rm --region $region s3://$chainbucket/current1 + if [ $? -ne 0 ] + then + echo "aws s3 rm command failed; exiting." + exit 5 + fi + fi + else + echo "Touching current1 key, as the node1 key was just updated." + touch ~/current1 + aws s3 cp --region $region --acl public-read ~/current1 s3://$chainbucket/ + if [ $? -ne 0 ] + then + echo "aws s3 cp command failed; retrying." + sleep 5 + aws s3 cp --region $region --acl public-read ~/current1 s3://$chainbucket/ + if [ $? -ne 0 ] + then + echo "aws s3 cp command failed; exiting." + exit 6 + fi + fi + fi +} + +continuous() { + # This function continuously stops the node every hour + # and syncs the chain data with S3, then restarts the node. + while true + do + echo "Sleeping for 60 minutes at `date`..." + sleep 3600 + echo "Cleanly shutting down the node so we can update S3 with the latest chaindata at `date`..." + kill_node + echo "Syncing chain data to S3 at `date`..." + s3_sync_up + echo "Restarting the node after syncing to S3 at `date`..." + start_node + done +} + +# main + +echo "Initializing the node at `date`..." +init_node +echo "Syncing initial chain data with stored chain data in S3 at `date`..." +s3_sync_down +echo "Starting the node at `date`..." +start_node +echo "Starting the continuous loop at `date`..." +continuous diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 366d5714175d..5599714f7ab3 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -2082,3 +2082,54 @@ func toHexSlice(b [][]byte) []string { } return r } + +// ---------------------------------------------------------------- FlashBots ---------------------------------------------------------------- + +// PrivateTxBundleAPI offers an API for accepting bundled transactions +type PrivateTxBundleAPI struct { + b Backend +} + +// NewPrivateTxBundleAPI creates a new Tx Bundle API instance. +func NewPrivateTxBundleAPI(b Backend) *PrivateTxBundleAPI { + return &PrivateTxBundleAPI{b} +} + +// SendBundleArgs represents the arguments for a call. +type SendBundleArgs struct { + Txs []hexutil.Bytes `json:"txs"` + BlockNumber rpc.BlockNumber `json:"blockNumber"` + MinTimestamp *uint64 `json:"minTimestamp"` + MaxTimestamp *uint64 `json:"maxTimestamp"` + RevertingTxHashes []common.Hash `json:"revertingTxHashes"` +} + +// SendBundle will add the signed transaction to the transaction pool. +// The sender is responsible for signing the transaction and using the correct nonce and ensuring validity +func (s *PrivateTxBundleAPI) SendBundle(ctx context.Context, args SendBundleArgs) error { + var txs types.Transactions + if len(args.Txs) == 0 { + return errors.New("bundle missing txs") + } + if args.BlockNumber == 0 { + return errors.New("bundle missing blockNumber") + } + + for _, encodedTx := range args.Txs { + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(encodedTx); err != nil { + return err + } + txs = append(txs, tx) + } + + var minTimestamp, maxTimestamp uint64 + if args.MinTimestamp != nil { + minTimestamp = *args.MinTimestamp + } + if args.MaxTimestamp != nil { + maxTimestamp = *args.MaxTimestamp + } + + return s.b.SendBundle(ctx, txs, args.BlockNumber, minTimestamp, maxTimestamp, args.RevertingTxHashes) +} diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index bc60fb2a64f6..bcdccf2bd9d6 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -74,6 +74,7 @@ type Backend interface { // Transaction pool API SendTx(ctx context.Context, signedTx *types.Transaction) error + SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64, revertingTxHashes []common.Hash) error GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) GetPoolTransactions() (types.Transactions, error) GetPoolTransaction(txHash common.Hash) *types.Transaction @@ -137,6 +138,11 @@ func GetAPIs(apiBackend Backend) []rpc.API { Version: "1.0", Service: NewPrivateAccountAPI(apiBackend, nonceLock), Public: false, + }, { + Namespace: "eth", + Version: "1.0", + Service: NewPrivateTxBundleAPI(apiBackend), + Public: true, }, } } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 87bf464157ba..7fb98255d0ff 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -581,6 +581,11 @@ web3._extend({ call: 'eth_getLogs', params: 1, }), + new web3._extend.Method({ + name: 'sendBundle', + call: 'eth_sendBundle', + params: 1, + }), ], properties: [ new web3._extend.Property({ diff --git a/les/api_backend.go b/les/api_backend.go index 11a9ca128aab..9bb08c79f6a7 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -198,6 +198,9 @@ func (b *LesApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) func (b *LesApiBackend) RemoveTx(txHash common.Hash) { b.eth.txPool.RemoveTx(txHash) } +func (b *LesApiBackend) SendBundle(ctx context.Context, txs types.Transactions, blockNumber rpc.BlockNumber, minTimestamp uint64, maxTimestamp uint64, revertingTxHashes []common.Hash) error { + return b.eth.txPool.AddMevBundle(txs, big.NewInt(blockNumber.Int64()), minTimestamp, maxTimestamp, revertingTxHashes) +} func (b *LesApiBackend) GetPoolTransactions() (types.Transactions, error) { return b.eth.txPool.GetTransactions() diff --git a/light/txpool.go b/light/txpool.go index a7df4aeec388..f8563f91d3d6 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -550,3 +550,14 @@ func (pool *TxPool) RemoveTx(hash common.Hash) { pool.chainDb.Delete(hash[:]) pool.relay.Discard([]common.Hash{hash}) } + +// MevBundles returns a list of bundles valid for the given blockNumber/blockTimestamp +// also prunes bundles that are outdated +func (pool *TxPool) MevBundles(blockNumber *big.Int, blockTimestamp uint64) ([]types.Transactions, error) { + return nil, nil +} + +// AddMevBundle adds a mev bundle to the pool +func (pool *TxPool) AddMevBundle(txs types.Transactions, blockNumber *big.Int, minTimestamp uint64, maxTimestamp uint64, revertingTxHashes []common.Hash) error { + return nil +} diff --git a/miner/miner.go b/miner/miner.go index 20e12c240e12..923adef9f5d1 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -45,21 +45,22 @@ type Backend interface { // Config is the configuration parameters of mining. type Config struct { - Etherbase common.Address `toml:",omitempty"` // Public address for block mining rewards (default = first account) - Notify []string `toml:",omitempty"` // HTTP URL list to be notified of new work packages (only useful in ethash). - NotifyFull bool `toml:",omitempty"` // Notify with pending block headers instead of work packages - ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner - GasFloor uint64 // Target gas floor for mined blocks. - GasCeil uint64 // Target gas ceiling for mined blocks. - GasPrice *big.Int // Minimum gas price for mining a transaction - Recommit time.Duration // The time interval for miner to re-create mining work. - Noverify bool // Disable remote mining solution verification(only useful in ethash). + Etherbase common.Address `toml:",omitempty"` // Public address for block mining rewards (default = first account) + Notify []string `toml:",omitempty"` // HTTP URL list to be notified of new work packages (only useful in ethash). + NotifyFull bool `toml:",omitempty"` // Notify with pending block headers instead of work packages + ExtraData hexutil.Bytes `toml:",omitempty"` // Block extra data set by the miner + GasFloor uint64 // Target gas floor for mined blocks. + GasCeil uint64 // Target gas ceiling for mined blocks. + GasPrice *big.Int // Minimum gas price for mining a transaction + Recommit time.Duration // The time interval for miner to re-create mining work. + Noverify bool // Disable remote mining solution verification(only useful in ethash). + MaxMergedBundles int } // Miner creates blocks and searches for proof-of-work values. type Miner struct { mux *event.TypeMux - worker *worker + worker *multiWorker coinbase common.Address eth Backend engine consensus.Engine @@ -78,7 +79,7 @@ func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *even exitCh: make(chan struct{}), startCh: make(chan common.Address), stopCh: make(chan struct{}), - worker: newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true), + worker: newMultiWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true), } miner.wg.Add(1) go miner.update() @@ -190,7 +191,7 @@ func (miner *Miner) SetRecommitInterval(interval time.Duration) { // Pending returns the currently pending block and associated state. func (miner *Miner) Pending() (*types.Block, *state.StateDB) { - return miner.worker.pending() + return miner.worker.regularWorker.pending() } // PendingBlock returns the currently pending block. @@ -199,7 +200,7 @@ func (miner *Miner) Pending() (*types.Block, *state.StateDB) { // simultaneously, please use Pending(), as the pending state can // change between multiple method calls func (miner *Miner) PendingBlock() *types.Block { - return miner.worker.pendingBlock() + return miner.worker.regularWorker.pendingBlock() } // PendingBlockAndReceipts returns the currently pending block and corresponding receipts. @@ -238,11 +239,11 @@ func (miner *Miner) DisablePreseal() { // GetSealingBlock retrieves a sealing block based on the given parameters. // The returned block is not sealed but all other fields should be filled. func (miner *Miner) GetSealingBlock(parent common.Hash, timestamp uint64, coinbase common.Address, random common.Hash) (*types.Block, error) { - return miner.worker.getSealingBlock(parent, timestamp, coinbase, random) + return miner.worker.regularWorker.getSealingBlock(parent, timestamp, coinbase, random) } // SubscribePendingLogs starts delivering logs from pending transactions // to the given channel. func (miner *Miner) SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription { - return miner.worker.pendingLogsFeed.Subscribe(ch) + return miner.worker.regularWorker.pendingLogsFeed.Subscribe(ch) } diff --git a/miner/multi_worker.go b/miner/multi_worker.go new file mode 100644 index 000000000000..9a39983c5a43 --- /dev/null +++ b/miner/multi_worker.go @@ -0,0 +1,118 @@ +package miner + +import ( + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" +) + +type multiWorker struct { + workers []*worker + regularWorker *worker +} + +func (w *multiWorker) stop() { + for _, worker := range w.workers { + worker.stop() + } +} + +func (w *multiWorker) start() { + for _, worker := range w.workers { + worker.start() + } +} + +func (w *multiWorker) close() { + for _, worker := range w.workers { + worker.close() + } +} + +func (w *multiWorker) isRunning() bool { + for _, worker := range w.workers { + if worker.isRunning() { + return true + } + } + return false +} + +// pendingBlockAndReceipts returns pending block and corresponding receipts from the `regularWorker` +func (w *multiWorker) pendingBlockAndReceipts() (*types.Block, types.Receipts) { + // return a snapshot to avoid contention on currentMu mutex + return w.regularWorker.pendingBlockAndReceipts() +} + +func (w *multiWorker) setGasCeil(ceil uint64) { + for _, worker := range w.workers { + worker.setGasCeil(ceil) + } +} + +func (w *multiWorker) setExtra(extra []byte) { + for _, worker := range w.workers { + worker.setExtra(extra) + } +} + +func (w *multiWorker) setRecommitInterval(interval time.Duration) { + for _, worker := range w.workers { + worker.setRecommitInterval(interval) + } +} + +func (w *multiWorker) setEtherbase(addr common.Address) { + for _, worker := range w.workers { + worker.setEtherbase(addr) + } +} + +func (w *multiWorker) enablePreseal() { + for _, worker := range w.workers { + worker.enablePreseal() + } +} + +func (w *multiWorker) disablePreseal() { + for _, worker := range w.workers { + worker.disablePreseal() + } +} + +func newMultiWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(header *types.Header) bool, init bool) *multiWorker { + queue := make(chan *task) + + regularWorker := newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, init, &flashbotsData{ + isFlashbots: false, + queue: queue, + }) + + workers := []*worker{regularWorker} + + for i := 1; i <= config.MaxMergedBundles; i++ { + workers = append(workers, + newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, init, &flashbotsData{ + isFlashbots: true, + queue: queue, + maxMergedBundles: i, + })) + } + + log.Info("creating multi worker", "config.MaxMergedBundles", config.MaxMergedBundles, "worker", len(workers)) + return &multiWorker{ + regularWorker: regularWorker, + workers: workers, + } +} + +type flashbotsData struct { + isFlashbots bool + queue chan *task + maxMergedBundles int +} diff --git a/miner/worker.go b/miner/worker.go index c6927a1ca1e8..1052b5291621 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "math/big" + "sort" "sync" "sync/atomic" "time" @@ -39,7 +40,7 @@ import ( const ( // resultQueueSize is the size of channel listening to sealing result. - resultQueueSize = 10 + resultQueueSize = 20 // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. @@ -88,6 +89,7 @@ type environment struct { tcount int // tx count in cycle gasPool *core.GasPool // available gas used to pack transactions coinbase common.Address + profit *big.Int header *types.Header txs []*types.Transaction @@ -104,6 +106,7 @@ func (env *environment) copy() *environment { family: env.family.Clone(), tcount: env.tcount, coinbase: env.coinbase, + profit: new(big.Int).Set(env.profit), header: types.CopyHeader(env.header), receipts: copyReceipts(env.receipts), } @@ -147,6 +150,10 @@ type task struct { state *state.StateDB block *types.Block createdAt time.Time + + profit *big.Int + isFlashbots bool + worker int } const ( @@ -239,6 +246,8 @@ type worker struct { // External functions isLocalBlock func(header *types.Header) bool // Function used to determine whether the specified block is mined by local miner. + flashbots *flashbotsData + // Test hooks newTaskHook func(*task) // Method to call upon receiving a new sealing task. skipSealHook func(*task) bool // Method to decide whether skipping the sealing. @@ -246,7 +255,30 @@ type worker struct { resubmitHook func(time.Duration, time.Duration) // Method to call upon updating resubmitting interval. } -func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(header *types.Header) bool, init bool) *worker { +func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(header *types.Header) bool, init bool, flashbots *flashbotsData) *worker { + exitCh := make(chan struct{}) + taskCh := make(chan *task) + if flashbots.isFlashbots { + // publish to the flashbots queue + taskCh = flashbots.queue + } else { + // read from the flashbots queue + go func() { + for { + select { + case flashbotsTask := <-flashbots.queue: + select { + case taskCh <- flashbotsTask: + case <-exitCh: + return + } + case <-exitCh: + return + } + } + }() + } + worker := &worker{ config: config, chainConfig: chainConfig, @@ -264,12 +296,13 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), newWorkCh: make(chan *newWorkReq), getWorkCh: make(chan *getWorkReq), - taskCh: make(chan *task), + taskCh: taskCh, resultCh: make(chan *types.Block, resultQueueSize), - exitCh: make(chan struct{}), + exitCh: exitCh, startCh: make(chan struct{}, 1), resubmitIntervalCh: make(chan time.Duration), resubmitAdjustCh: make(chan *intervalAdjust, resubmitAdjustChanSize), + flashbots: flashbots, } // Subscribe NewTxsEvent for tx pool worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) @@ -284,11 +317,15 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus recommit = minRecommitInterval } - worker.wg.Add(4) + worker.wg.Add(2) go worker.mainLoop() go worker.newWorkLoop(recommit) - go worker.resultLoop() - go worker.taskLoop() + if !flashbots.isFlashbots { + // only mine if not flashbots + worker.wg.Add(2) + go worker.resultLoop() + go worker.taskLoop() + } // Submit first work to initialize pending state. if init { @@ -629,6 +666,9 @@ func (w *worker) taskLoop() { var ( stopCh chan struct{} prev common.Hash + + prevParentHash common.Hash + prevProfit *big.Int ) // interrupt aborts the in-flight sealing task. @@ -649,10 +689,20 @@ func (w *worker) taskLoop() { if sealHash == prev { continue } + + taskParentHash := task.block.Header().ParentHash + // reject new tasks which don't profit + if taskParentHash == prevParentHash && + prevProfit != nil && task.profit.Cmp(prevProfit) < 0 { + continue + } + prevParentHash = taskParentHash + prevProfit = task.profit + // Interrupt previous sealing operation interrupt() stopCh, prev = make(chan struct{}), sealHash - + log.Info("Proposed miner block", "blockNumber", task.block.Number(), "profit", ethIntToFloat(prevProfit), "isFlashbots", task.isFlashbots, "sealhash", sealHash, "parentHash", prevParentHash, "worker", task.worker) if w.skipSealHook != nil && w.skipSealHook(task) { continue } @@ -775,6 +825,7 @@ func (w *worker) makeEnv(parent *types.Block, header *types.Header, coinbase com family: mapset.NewSet(), header: header, uncles: make(map[common.Hash]*types.Header), + profit: new(big.Int), } // when 08 is processed ancestors contain 07 (quick block) for _, ancestor := range w.chain.GetBlocksFromHash(parent.Hash(), 7) { @@ -786,6 +837,7 @@ func (w *worker) makeEnv(parent *types.Block, header *types.Header, coinbase com } // Keep track of transactions which return errors so they can be removed env.tcount = 0 + env.gasPool = new(core.GasPool).AddGas(header.GasLimit) return env, nil } @@ -830,6 +882,11 @@ func (w *worker) updateSnapshot(env *environment) { func (w *worker) commitTransaction(env *environment, tx *types.Transaction) ([]*types.Log, error) { snap := env.state.Snapshot() + gasPrice, err := tx.EffectiveGasTip(env.header.BaseFee) + if err != nil { + return nil, err + } + receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &env.coinbase, env.gasPool, env.state, env.header, tx, &env.header.GasUsed, *w.chain.GetVMConfig()) if err != nil { env.state.RevertToSnapshot(snap) @@ -838,9 +895,121 @@ func (w *worker) commitTransaction(env *environment, tx *types.Transaction) ([]* env.txs = append(env.txs, tx) env.receipts = append(env.receipts, receipt) + gasUsed := new(big.Int).SetUint64(receipt.GasUsed) + env.profit.Add(env.profit, gasUsed.Mul(gasUsed, gasPrice)) + return receipt.Logs, nil } +func (w *worker) commitBundle(env *environment, txs types.Transactions, interrupt *int32) bool { + gasLimit := env.header.GasLimit + if env.gasPool == nil { + env.gasPool = new(core.GasPool).AddGas(gasLimit) + } + + var coalescedLogs []*types.Log + + for _, tx := range txs { + // In the following three cases, we will interrupt the execution of the transaction. + // (1) new head block event arrival, the interrupt signal is 1 + // (2) worker start or restart, the interrupt signal is 1 + // (3) worker recreate the sealing block with any newly arrived transactions, the interrupt signal is 2. + // For the first two cases, the semi-finished work will be discarded. + // For the third case, the semi-finished work will be submitted to the consensus engine. + if interrupt != nil && atomic.LoadInt32(interrupt) != commitInterruptNone { + // Notify resubmit loop to increase resubmitting interval due to too frequent commits. + if atomic.LoadInt32(interrupt) == commitInterruptResubmit { + ratio := float64(gasLimit-env.gasPool.Gas()) / float64(gasLimit) + if ratio < 0.1 { + ratio = 0.1 + } + w.resubmitAdjustCh <- &intervalAdjust{ + ratio: ratio, + inc: true, + } + } + return atomic.LoadInt32(interrupt) == commitInterruptNewHead + } + // If we don't have enough gas for any further transactions then we're done + if env.gasPool.Gas() < params.TxGas { + log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", params.TxGas) + break + } + + // Error may be ignored here. The error has already been checked + // during transaction acceptance is the transaction pool. + // + // We use the eip155 signer regardless of the current hf. + from, _ := types.Sender(env.signer, tx) + // Check whether the tx is replay protected. If we're not in the EIP155 hf + // phase, start ignoring the sender until we do. + if tx.Protected() && !w.chainConfig.IsEIP155(env.header.Number) { + log.Trace("Ignoring reply protected transaction", "hash", tx.Hash(), "eip155", w.chainConfig.EIP155Block) + + return true + } + // Start executing the transaction + env.state.Prepare(tx.Hash(), env.tcount) + + logs, err := w.commitTransaction(env, tx) + switch { + case errors.Is(err, core.ErrGasLimitReached): + // Pop the current out-of-gas transaction without shifting in the next from the account + log.Trace("Gas limit exceeded for current block", "sender", from) + return true + + case errors.Is(err, core.ErrNonceTooLow): + // New head notification data race between the transaction pool and miner, shift + log.Trace("Skipping transaction with low nonce", "sender", from, "nonce", tx.Nonce()) + return true + + case errors.Is(err, core.ErrNonceTooHigh): + // Reorg notification data race between the transaction pool and miner, skip account = + log.Trace("Skipping account with hight nonce", "sender", from, "nonce", tx.Nonce()) + return true + + case errors.Is(err, nil): + // Everything ok, collect the logs and shift in the next transaction from the same account + coalescedLogs = append(coalescedLogs, logs...) + env.tcount++ + continue + + case errors.Is(err, core.ErrTxTypeNotSupported): + // Pop the unsupported transaction without shifting in the next from the account + log.Trace("Skipping unsupported transaction type", "sender", from, "type", tx.Type()) + return true + + default: + // Strange error, discard the transaction and get the next in line (note, the + // nonce-too-high clause will prevent us from executing in vain). + log.Debug("Transaction failed, account skipped", "hash", tx.Hash(), "err", err) + return true + } + } + + if !w.isRunning() && len(coalescedLogs) > 0 { + // We don't push the pendingLogsEvent while we are sealing. The reason is that + // when we are sealing, the worker will regenerate a sealing block every 3 seconds. + // In order to avoid pushing the repeated pendingLog, we disable the pending log pushing. + + // make a copy, the state caches the logs and these logs get "upgraded" from pending to mined + // logs by filling in the block hash when the block was mined by the local miner. This can + // cause a race condition if a log was "upgraded" before the PendingLogsEvent is processed. + cpy := make([]*types.Log, len(coalescedLogs)) + for i, l := range coalescedLogs { + cpy[i] = new(types.Log) + *cpy[i] = *l + } + w.pendingLogsFeed.Send(cpy) + } + // Notify resubmit loop to decrease resubmitting interval if current interval is larger + // than the user-specified one. + if interrupt != nil { + w.resubmitAdjustCh <- &intervalAdjust{inc: false} + } + return false +} + func (w *worker) commitTransactions(env *environment, txs *types.TransactionsByPriceAndNonce, interrupt *int32) bool { gasLimit := env.header.GasLimit if env.gasPool == nil { @@ -1061,6 +1230,27 @@ func (w *worker) fillTransactions(interrupt *int32, env *environment) { localTxs[account] = txs } } + if w.flashbots.isFlashbots { + bundles, err := w.eth.TxPool().MevBundles(env.header.Number, env.header.Time) + if err != nil { + log.Error("Failed to fetch pending transactions", "err", err) + return + } + + bundleTxs, bundle, numBundles, err := w.generateFlashbotsBundle(env, bundles, pending) + if err != nil { + log.Error("Failed to generate flashbots bundle", "err", err) + return + } + log.Info("Flashbots bundle", "ethToCoinbase", ethIntToFloat(bundle.totalEth), "gasUsed", bundle.totalGasUsed, "bundleScore", bundle.mevGasPrice, "bundleLength", len(bundleTxs), "numBundles", numBundles, "worker", w.flashbots.maxMergedBundles) + if len(bundleTxs) == 0 { + return + } + if w.commitBundle(env, bundleTxs, interrupt) { + return + } + env.profit.Add(env.profit, bundle.totalEth) + } if len(localTxs) > 0 { txs := types.NewTransactionsByPriceAndNonce(env.signer, localTxs, env.header.BaseFee) if w.commitTransactions(env, txs, interrupt) { @@ -1144,13 +1334,12 @@ func (w *worker) commit(env *environment, interval func(), update bool, start ti // If we're post merge, just ignore if !w.isTTDReached(block.Header()) { select { - case w.taskCh <- &task{receipts: env.receipts, state: env.state, block: block, createdAt: time.Now()}: + case w.taskCh <- &task{receipts: env.receipts, state: env.state, block: block, createdAt: time.Now(), profit: env.profit, isFlashbots: w.flashbots.isFlashbots, worker: w.flashbots.maxMergedBundles}: w.unconfirmed.Shift(block.NumberU64() - 1) log.Info("Commit new sealing work", "number", block.Number(), "sealhash", w.engine.SealHash(block.Header()), - "uncles", len(env.uncles), "txs", env.tcount, - "gas", block.GasUsed(), "fees", totalFees(block, env.receipts), - "elapsed", common.PrettyDuration(time.Since(start))) - + "uncles", len(env.uncles), "txs", env.tcount, "gas", block.GasUsed(), "fees", totalFees(block, env.receipts), + "profit", ethIntToFloat(env.profit), "elapsed", common.PrettyDuration(time.Since(start)), + "isFlashbots", w.flashbots.isFlashbots, "worker", w.flashbots.maxMergedBundles) case <-w.exitCh: log.Info("Worker has exited") } @@ -1195,6 +1384,195 @@ func (w *worker) isTTDReached(header *types.Header) bool { return td != nil && ttd != nil && td.Cmp(ttd) >= 0 } +type simulatedBundle struct { + mevGasPrice *big.Int + totalEth *big.Int + ethSentToCoinbase *big.Int + totalGasUsed uint64 + originalBundle types.MevBundle +} + +func (w *worker) generateFlashbotsBundle(env *environment, bundles []types.MevBundle, pendingTxs map[common.Address]types.Transactions) (types.Transactions, simulatedBundle, int, error) { + simulatedBundles, err := w.simulateBundles(env, bundles, pendingTxs) + if err != nil { + return nil, simulatedBundle{}, 0, err + } + + sort.SliceStable(simulatedBundles, func(i, j int) bool { + return simulatedBundles[j].mevGasPrice.Cmp(simulatedBundles[i].mevGasPrice) < 0 + }) + + return w.mergeBundles(env, simulatedBundles, pendingTxs) +} + +func (w *worker) mergeBundles(env *environment, bundles []simulatedBundle, pendingTxs map[common.Address]types.Transactions) (types.Transactions, simulatedBundle, int, error) { + finalBundle := types.Transactions{} + + currentState := env.state.Copy() + gasPool := new(core.GasPool).AddGas(env.header.GasLimit) + + var prevState *state.StateDB + var prevGasPool *core.GasPool + + mergedBundle := simulatedBundle{ + totalEth: new(big.Int), + ethSentToCoinbase: new(big.Int), + } + + count := 0 + for _, bundle := range bundles { + prevState = currentState.Copy() + prevGasPool = new(core.GasPool).AddGas(gasPool.Gas()) + + // the floor gas price is 99/100 what was simulated at the top of the block + floorGasPrice := new(big.Int).Mul(bundle.mevGasPrice, big.NewInt(99)) + floorGasPrice = floorGasPrice.Div(floorGasPrice, big.NewInt(100)) + + simmed, err := w.computeBundleGas(env, bundle.originalBundle, currentState, gasPool, pendingTxs, len(finalBundle)) + if err != nil || simmed.mevGasPrice.Cmp(floorGasPrice) <= 0 { + currentState = prevState + gasPool = prevGasPool + continue + } + + log.Info("Included bundle", "ethToCoinbase", ethIntToFloat(simmed.totalEth), "gasUsed", simmed.totalGasUsed, "bundleScore", simmed.mevGasPrice, "bundleLength", len(simmed.originalBundle.Txs), "worker", w.flashbots.maxMergedBundles) + + finalBundle = append(finalBundle, bundle.originalBundle.Txs...) + mergedBundle.totalEth.Add(mergedBundle.totalEth, simmed.totalEth) + mergedBundle.ethSentToCoinbase.Add(mergedBundle.ethSentToCoinbase, simmed.ethSentToCoinbase) + mergedBundle.totalGasUsed += simmed.totalGasUsed + count++ + + if count >= w.flashbots.maxMergedBundles { + break + } + } + + if len(finalBundle) == 0 || count != w.flashbots.maxMergedBundles { + return nil, simulatedBundle{}, count, nil + } + + return finalBundle, simulatedBundle{ + mevGasPrice: new(big.Int).Div(mergedBundle.totalEth, new(big.Int).SetUint64(mergedBundle.totalGasUsed)), + totalEth: mergedBundle.totalEth, + ethSentToCoinbase: mergedBundle.ethSentToCoinbase, + totalGasUsed: mergedBundle.totalGasUsed, + }, count, nil +} + +func (w *worker) simulateBundles(env *environment, bundles []types.MevBundle, pendingTxs map[common.Address]types.Transactions) ([]simulatedBundle, error) { + simulatedBundles := []simulatedBundle{} + + for _, bundle := range bundles { + state := env.state.Copy() + gasPool := new(core.GasPool).AddGas(env.header.GasLimit) + if len(bundle.Txs) == 0 { + continue + } + simmed, err := w.computeBundleGas(env, bundle, state, gasPool, pendingTxs, 0) + + if err != nil { + log.Debug("Error computing gas for a bundle", "error", err) + continue + } + simulatedBundles = append(simulatedBundles, simmed) + } + + return simulatedBundles, nil +} + +func containsHash(arr []common.Hash, match common.Hash) bool { + for _, elem := range arr { + if elem == match { + return true + } + } + return false +} + +// Compute the adjusted gas price for a whole bundle +// Done by calculating all gas spent, adding transfers to the coinbase, and then dividing by gas used +func (w *worker) computeBundleGas(env *environment, bundle types.MevBundle, state *state.StateDB, gasPool *core.GasPool, pendingTxs map[common.Address]types.Transactions, currentTxCount int) (simulatedBundle, error) { + var totalGasUsed uint64 = 0 + var tempGasUsed uint64 + gasFees := new(big.Int) + + ethSentToCoinbase := new(big.Int) + + for i, tx := range bundle.Txs { + if env.header.BaseFee != nil && tx.Type() == 2 { + // Sanity check for extremely large numbers + if tx.GasFeeCap().BitLen() > 256 { + return simulatedBundle{}, core.ErrFeeCapVeryHigh + } + if tx.GasTipCap().BitLen() > 256 { + return simulatedBundle{}, core.ErrTipVeryHigh + } + // Ensure gasFeeCap is greater than or equal to gasTipCap. + if tx.GasFeeCapIntCmp(tx.GasTipCap()) < 0 { + return simulatedBundle{}, core.ErrTipAboveFeeCap + } + } + + state.Prepare(tx.Hash(), i+currentTxCount) + coinbaseBalanceBefore := state.GetBalance(env.coinbase) + + receipt, err := core.ApplyTransaction(w.chainConfig, w.chain, &env.coinbase, gasPool, state, env.header, tx, &tempGasUsed, *w.chain.GetVMConfig()) + if err != nil { + return simulatedBundle{}, err + } + if receipt.Status == types.ReceiptStatusFailed && !containsHash(bundle.RevertingTxHashes, receipt.TxHash) { + return simulatedBundle{}, errors.New("failed tx") + } + + totalGasUsed += receipt.GasUsed + + from, err := types.Sender(env.signer, tx) + if err != nil { + return simulatedBundle{}, err + } + + txInPendingPool := false + if accountTxs, ok := pendingTxs[from]; ok { + // check if tx is in pending pool + txNonce := tx.Nonce() + + for _, accountTx := range accountTxs { + if accountTx.Nonce() == txNonce { + txInPendingPool = true + break + } + } + } + + gasUsed := new(big.Int).SetUint64(receipt.GasUsed) + gasPrice, err := tx.EffectiveGasTip(env.header.BaseFee) + if err != nil { + return simulatedBundle{}, err + } + gasFeesTx := gasUsed.Mul(gasUsed, gasPrice) + coinbaseBalanceAfter := state.GetBalance(env.coinbase) + coinbaseDelta := big.NewInt(0).Sub(coinbaseBalanceAfter, coinbaseBalanceBefore) + coinbaseDelta.Sub(coinbaseDelta, gasFeesTx) + ethSentToCoinbase.Add(ethSentToCoinbase, coinbaseDelta) + + if !txInPendingPool { + // If tx is not in pending pool, count the gas fees + gasFees.Add(gasFees, gasFeesTx) + } + } + + totalEth := new(big.Int).Add(ethSentToCoinbase, gasFees) + + return simulatedBundle{ + mevGasPrice: new(big.Int).Div(totalEth, new(big.Int).SetUint64(totalGasUsed)), + totalEth: totalEth, + ethSentToCoinbase: ethSentToCoinbase, + totalGasUsed: totalGasUsed, + originalBundle: bundle, + }, nil +} + // copyReceipts makes a deep copy of the given receipts. func copyReceipts(receipts []*types.Receipt) []*types.Receipt { result := make([]*types.Receipt, len(receipts)) @@ -1213,6 +1591,14 @@ func (w *worker) postSideBlock(event core.ChainSideEvent) { } } +// ethIntToFloat is for formatting a big.Int in wei to eth +func ethIntToFloat(eth *big.Int) *big.Float { + if eth == nil { + return big.NewFloat(0) + } + return new(big.Float).Quo(new(big.Float).SetInt(eth), new(big.Float).SetInt(big.NewInt(params.Ether))) +} + // totalFees computes total consumed miner fees in ETH. Block transactions and receipts have to have the same order. func totalFees(block *types.Block, receipts []*types.Receipt) *big.Float { feesWei := new(big.Int) @@ -1220,5 +1606,5 @@ func totalFees(block *types.Block, receipts []*types.Receipt) *big.Float { minerFee, _ := tx.EffectiveGasTip(block.BaseFee()) feesWei.Add(feesWei, new(big.Int).Mul(new(big.Int).SetUint64(receipts[i].GasUsed), minerFee)) } - return new(big.Float).Quo(new(big.Float).SetInt(feesWei), new(big.Float).SetInt(big.NewInt(params.Ether))) + return ethIntToFloat(feesWei) } diff --git a/miner/worker_test.go b/miner/worker_test.go index dd029433b8bf..2e06ace9fc92 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -202,7 +202,10 @@ func (b *testWorkerBackend) newRandomTx(creation bool) *types.Transaction { func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*worker, *testWorkerBackend) { backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks) backend.txPool.AddLocals(pendingTxs) - w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false) + w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false, &flashbotsData{ + isFlashbots: false, + queue: nil, + }) w.setEtherbase(testBankAddress) return w, backend }