diff --git a/Dockerfile b/Dockerfile index 81553f9cbe..d556006fe8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM node:18.10.0 # Set labels -LABEL application="gateway-v2" +LABEL application="gateway" LABEL branch=${BRANCH} LABEL commit=${COMMIT} LABEL date=${BUILD_DATE} @@ -23,6 +23,14 @@ RUN apt-get update && \ # app directory WORKDIR /usr/src/app +# copy pwd file to container +COPY . . + +# create sym links +RUN ln -s /conf /usr/src/app/conf && \ + ln -s /logs /usr/src/app/logs && \ + ln -s /certs /usr/src/app/certs + # create app writable directory for db files RUN mkdir /var/lib/gateway RUN chown -R hummingbot /var/lib/gateway diff --git a/README.md b/README.md index c58412e4c3..1b82b3b1e1 100644 --- a/README.md +++ b/README.md @@ -2,93 +2,71 @@ # Hummingbot Gateway ---- - Hummingbot Gateway is a REST API that exposes connections to various blockchains (wallet, node & chain interaction) and decentralized exchanges (pricing, trading & liquidity provision). It is written in Typescript and takes advantage of existing blockchain and DEX SDKs. The advantage of using gateway is it provideds a programming language agnostic approach to interacting with blockchains and DEXs. -Gateway may be used alongside the main Hummingbot client to enable trading on DEXs, or as a standalone module by external developers. - -## Connectors - -This is a list of DEX connections currently supported by Gateway. +Gateway may be used alongside the main [Hummingbot client](https://github.com/hummingbot/hummingbot) to enable trading on DEXs, or as a standalone module by external developers. -| Connector | Blockchain | Trading Interface | -| ----------- | ------------------- | ----------------- | -| UniswapV2 | Ethereum | AMM | -| Sushiswap | Ethereum | AMM | -| UniswapV3 | Ethereum | EVM_Range_AMM | -| Pangolin | Avalanche | AMM | -| PancakeSwap | Binance Smart Chain | AMM | -| Traderjoe | Avalanche | AMM | -| Quickswap | Polygon | AMM | -| Perp | Ethereum | EVM_Perpetual | -| Mad Meerkat | Cronos | AMM | -| VVS | Cronos | AMM | - - -## Contributing +## Install and run locally -There are a number of ways to contribute to gateway. +```bash +# Install dependencies +yarn -- Add a new blockchain. +# Complile Typescript into JS +$ yarn build -- Add a new connector/DEX. +# Generate the config files and edit as needed, especially server.yml +$ setup/generate_conf.sh -- Fix a bug. +# Start the server using the passphrase you used to generate the certs in Hummingbot +$ yarn start --passphrase= +``` -- File an issue at [hummingbot issues](https://github.com/hummingbot/hummingbot/issues) +## Documentation -- Make a PR for anything useful on [hummingbot](https://github.com/hummingbot/hummingbot/). +See the [official Gateway docs](https://docs.hummingbot.org/gateway/). -- Vote on a snapshot proposal: [Hummingbot PRP](https://snapshot.org/#/hbot-prp.eth), [Hummingbot Foundation - HGP](https://snapshot.org/#/hbot.eth), [Hummingbot Improvement Proposals](https://snapshot.org/#/hbot-ip.eth). +The API is documented using [Swagger](./docs/swagger). When Gateway is started, it also generates Swagger API docs at: https://localhost:8080 -## Configuration -Before running gateway, you need to setup some configs. You can start by copying all of the yml files from [src/templates](./src/templates) to [conf](./conf). The format of this files are dictated by [src/services/config-manager-v2.ts](./src/services/config-manager-v2.ts) and the corresponding schema files in [src/services/schema](./src/services/schema) . +## Contributing -### Useful configuration options +There are a number of ways to contribute to gateway. -- If you want to turn off `https`, set `unsafeDevModeWithHTTP` to `true` in [conf/server.yml](./conf/server.yml). +- File an issue at [hummingbot issues](https://github.com/hummingbot/gateway/issues) -- If you gateway to log to standard out, set `logToStdOut` to `true` in [conf/logging.yml](./conf/logging.yml). +- Make a [pull request](https://github.com/hummingbot/gateway/) -- Edit the path to the SSL files in [conf/ssl.yml](./conf/ssl.yml). This can be generated by hummingbot with `gateway generate-certs`. +- Edit the [docs](https://github.com/hummingbot/hummingbot-site/) -The hummingbot client is also able to edit this config files. +- Vote on a [Snapshot proposal](https://snapshot.org/#/hbot.eth) -## Install and run locally -Compile the Typescript code with `npm` or `yarn` . +### Configuration -```bash -yarn -yarn build -yarn dev --passphrase= -# the passphrase can be anything if unsafeDevModeWithHTTP is true -``` +- Edit `certs_path` in [conf/server.yml](./conf/server.yml) and enter the absolute path to the folder where Hummingbot stored the certificates it created with `gateway generate-certs`. You can also edit this config inside the Hummingbot client by running the command: `gateway config server.certs_path`. -## Documentation +- If you want to turn off `https`, set `unsafeDevModeWithHTTP` to `true` in [conf/server.yml](./conf/server.yml). -The API is documented using swagger: [gateway swagger docs](./docs/swagger). You can run the gateway swagger UI by starting gateway and visiting [localhost:8080](localhost:8080). +- If you want Gateway to log to standard out, set `logToStdOut` to `true` in [conf/server.yml](./conf/server.yml). -You can follow details about [trading interfaces](https://hummingbot.notion.site/Gateway-v2-Trading-Interfaces-482e2684d48c450ebcfff5401ba806aa) +- The format of configuration files are dictated by [src/services/config-manager-v2.ts](./src/services/config-manager-v2.ts) and the corresponding schema files in [src/services/schema](./src/services/schema). -Also, we maintain docs on the official website about gateway [here](https://hummingbot.org/protocols/gateway/). -### Files to read +### Architecture -Here are some files we recommend you look at in order to get familiar with the gateway code base. +Here are some files we recommend you look at in order to get familiar with the Gateway codebase: -- [src/services/ethereum-base.ts](./src/services/ethereum-base.ts) is a base class for EVM chains. +- [src/services/ethereum-base.ts](./src/services/ethereum-base.ts): base class for EVM chains. -- [src/connectors/uniswap/uniswap.ts](./src/connectors/uniswap/uniswap.ts) has functionality for interacting with Uniswap V2. +- [src/connectors/uniswap/uniswap.ts](./src/connectors/uniswap/uniswap.ts): functionality for interacting with Uniswap. -- [src/services/validator.ts](./src/services/validator.ts) defines functions for validating request payloads. +- [src/services/validator.ts](./src/services/validator.ts): defines functions for validating request payloads. ### Testing -When making a PR, there are unit test coverage requirements. Look at our [github workflow](../.github/workflows/workflow.yml) for the exact definition. There are some more +For a pull request merged into the codebase, it has to pass unit test coverage requirements. Take a look at [Workflow](../.github/workflows/workflow.yml) for more details. #### Unit tests diff --git a/src/chains/avalanche/avalanche.ts b/src/chains/avalanche/avalanche.ts index c903fff874..fdd5445ebd 100644 --- a/src/chains/avalanche/avalanche.ts +++ b/src/chains/avalanche/avalanche.ts @@ -27,8 +27,8 @@ export class Avalanche extends EthereumBase implements Ethereumish { config.network.tokenListType, config.manualGasPrice, config.gasLimitTransaction, - ConfigManagerV2.getInstance().get('database.nonceDbPath'), - ConfigManagerV2.getInstance().get('database.transactionDbPath') + ConfigManagerV2.getInstance().get('server.nonceDbPath'), + ConfigManagerV2.getInstance().get('server.transactionDbPath') ); this._chain = config.network.name; this._nativeTokenSymbol = config.nativeCurrencySymbol; diff --git a/src/chains/binance-smart-chain/binance-smart-chain.ts b/src/chains/binance-smart-chain/binance-smart-chain.ts index 117a78f6c7..2404759558 100644 --- a/src/chains/binance-smart-chain/binance-smart-chain.ts +++ b/src/chains/binance-smart-chain/binance-smart-chain.ts @@ -26,8 +26,8 @@ export class BinanceSmartChain extends EthereumBase implements Ethereumish { config.network.tokenListType, config.manualGasPrice, config.gasLimitTransaction, - ConfigManagerV2.getInstance().get('database.nonceDbPath'), - ConfigManagerV2.getInstance().get('database.transactionDbPath') + ConfigManagerV2.getInstance().get('server.nonceDbPath'), + ConfigManagerV2.getInstance().get('server.transactionDbPath') ); this._chain = config.network.name; this._nativeTokenSymbol = config.nativeCurrencySymbol; diff --git a/src/chains/cronos/cronos.ts b/src/chains/cronos/cronos.ts index 9b2ae7eee9..e84f5dea00 100755 --- a/src/chains/cronos/cronos.ts +++ b/src/chains/cronos/cronos.ts @@ -26,8 +26,8 @@ export class Cronos extends EthereumBase implements Ethereumish { config.network.tokenListType, config.manualGasPrice, config.gasLimitTransaction, - ConfigManagerV2.getInstance().get('database.nonceDbPath'), - ConfigManagerV2.getInstance().get('database.transactionDbPath') + ConfigManagerV2.getInstance().get('server.nonceDbPath'), + ConfigManagerV2.getInstance().get('server.transactionDbPath') ); this._chain = config.network.name; this._nativeTokenSymbol = config.nativeCurrencySymbol; diff --git a/src/chains/ethereum/ethereum.ts b/src/chains/ethereum/ethereum.ts index a0e9903151..0eb0c7f196 100644 --- a/src/chains/ethereum/ethereum.ts +++ b/src/chains/ethereum/ethereum.ts @@ -32,8 +32,8 @@ export class Ethereum extends EthereumBase implements Ethereumish { config.network.tokenListType, config.manualGasPrice, config.gasLimitTransaction, - ConfigManagerV2.getInstance().get('database.nonceDbPath'), - ConfigManagerV2.getInstance().get('database.transactionDbPath') + ConfigManagerV2.getInstance().get('server.nonceDbPath'), + ConfigManagerV2.getInstance().get('server.transactionDbPath') ); this._chain = network; this._nativeTokenSymbol = config.nativeCurrencySymbol; diff --git a/src/chains/harmony/harmony.ts b/src/chains/harmony/harmony.ts index df331d5dc1..d5ce824af3 100644 --- a/src/chains/harmony/harmony.ts +++ b/src/chains/harmony/harmony.ts @@ -27,8 +27,8 @@ export class Harmony extends EthereumBase implements Ethereumish { config.network.tokenListType, config.manualGasPrice, config.gasLimitTransaction, - ConfigManagerV2.getInstance().get('database.nonceDbPath'), - ConfigManagerV2.getInstance().get('database.transactionDbPath') + ConfigManagerV2.getInstance().get('server.nonceDbPath'), + ConfigManagerV2.getInstance().get('server.transactionDbPath') ); this._chain = network; this._nativeTokenSymbol = config.nativeCurrencySymbol; diff --git a/src/chains/near/near.ts b/src/chains/near/near.ts index bcb298492a..da6bc37fad 100644 --- a/src/chains/near/near.ts +++ b/src/chains/near/near.ts @@ -23,7 +23,7 @@ export class Near extends NearBase { config.network.tokenListType, config.manualGasPrice, config.gasLimitTransaction, - ConfigManagerV2.getInstance().get('database.transactionDbPath') + ConfigManagerV2.getInstance().get('server.transactionDbPath') ); this._chain = config.network.name; this._nativeTokenSymbol = config.nativeCurrencySymbol; diff --git a/src/chains/polygon/polygon.ts b/src/chains/polygon/polygon.ts index c971b79196..137fd27a32 100644 --- a/src/chains/polygon/polygon.ts +++ b/src/chains/polygon/polygon.ts @@ -25,8 +25,8 @@ export class Polygon extends EthereumBase implements Ethereumish { config.network.tokenListType, config.manualGasPrice, config.gasLimitTransaction, - ConfigManagerV2.getInstance().get('database.nonceDbPath'), - ConfigManagerV2.getInstance().get('database.transactionDbPath') + ConfigManagerV2.getInstance().get('server.nonceDbPath'), + ConfigManagerV2.getInstance().get('server.transactionDbPath') ); this._chain = config.network.name; this._nativeTokenSymbol = config.nativeCurrencySymbol; diff --git a/src/https.ts b/src/https.ts index 0175d47e7e..11f61b6fa2 100644 --- a/src/https.ts +++ b/src/https.ts @@ -6,22 +6,24 @@ import { ConfigManagerV2 } from './services/config-manager-v2'; export const addHttps = (app: Application) => { const serverKey = fs.readFileSync( - ConfigManagerV2.getInstance().get('ssl.keyPath'), - { - encoding: 'utf-8', - } + addSlashToPath( + ConfigManagerV2.getInstance().get('server.certificatePath') + ) + 'server_key.pem', + { encoding: 'utf-8' } ); + const serverCert = fs.readFileSync( - ConfigManagerV2.getInstance().get('ssl.certificatePath'), - { - encoding: 'utf-8', - } + addSlashToPath( + ConfigManagerV2.getInstance().get('server.certificatePath') + ) + 'server_cert.pem', + { encoding: 'utf-8' } ); + const caCert = fs.readFileSync( - ConfigManagerV2.getInstance().get('ssl.caCertificatePath'), - { - encoding: 'utf-8', - } + addSlashToPath( + ConfigManagerV2.getInstance().get('server.certificatePath') + ) + 'ca_cert.pem', + { encoding: 'utf-8' } ); return https.createServer( @@ -39,3 +41,10 @@ export const addHttps = (app: Application) => { app ); }; + +const addSlashToPath = (path: string) => { + if (!path.endsWith('/')) { + path += '/'; + } + return path; +}; diff --git a/src/services/logger.ts b/src/services/logger.ts index 90c3a304a8..8c77abaaf2 100644 --- a/src/services/logger.ts +++ b/src/services/logger.ts @@ -46,7 +46,7 @@ const sdtoutFormat = winston.format.combine( ); const getLogPath = () => { - let logPath = ConfigManagerV2.getInstance().get('logging.logPath'); + let logPath = ConfigManagerV2.getInstance().get('server.logPath'); logPath = [appRoot.path, 'logs'].join('/'); return logPath; }; @@ -77,13 +77,13 @@ const reportingProxy = new TelemetryTransport({ }); export const updateLoggerToStdout = () => { - ConfigManagerV2.getInstance().get('logging.logToStdOut') === true + ConfigManagerV2.getInstance().get('server.logToStdOut') === true ? logger.add(toStdout) : logger.remove(toStdout); }; export const telemetry = () => { - ConfigManagerV2.getInstance().get('telemetry.enabled') === true + ConfigManagerV2.getInstance().get('server.telemetry_enabled') === true ? logger.add(reportingProxy) : logger.remove(reportingProxy); }; diff --git a/src/services/schema/database-schema.json b/src/services/schema/database-schema.json deleted file mode 100644 index 68eb48487d..0000000000 --- a/src/services/schema/database-schema.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "nonceDbPath": { "type": "string" }, - "transactionDbPath": { "type": "string" } - }, - "additionalProperties": false, - "required": ["nonceDbPath", "transactionDbPath"] -} diff --git a/src/services/schema/ethereum-gas-station-schema.json b/src/services/schema/ethereum-gas-station-schema.json deleted file mode 100644 index 8c35c86066..0000000000 --- a/src/services/schema/ethereum-gas-station-schema.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "enabled": { "type": "boolean" }, - "gasStationURL": { "type": "string" }, - "APIKey": { "type": ["string", "null"] }, - "gasLevel": { "enum": ["fast", "fastest", "safeLow", "average"] } - }, - "additionalProperties": false, - "required": ["enabled"] -} diff --git a/src/services/schema/server-schema.json b/src/services/schema/server-schema.json index 95d17686e5..0df687b62c 100644 --- a/src/services/schema/server-schema.json +++ b/src/services/schema/server-schema.json @@ -2,12 +2,26 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "certificatePath": { "type": "string" }, + "logPath": { "type": "string" }, "port": { "type": "integer" }, "ipWhitelist": { "type": "array" }, - "unsafeDevModeWithHTTP": { "type": "boolean" }, "GMTOffset": { "type": "number" }, - "id": { "type": "string" } + "id": { "type": "string" }, + "logToStdOut": { "type": "boolean" }, + "unsafeDevModeWithHTTP": { "type": "boolean" }, + "telemetry_enabled": { "type": "boolean" }, + "nonceDbPath": { "type": "string" }, + "transactionDbPath": { "type": "string" } }, "additionalProperties": false, - "required": ["port", "unsafeDevModeWithHTTP"] + "required": [ + "port", + "unsafeDevModeWithHTTP", + "certificatePath", + "telemetry_enabled", + "logPath", + "nonceDbPath", + "transactionDbPath" + ] } diff --git a/src/services/schema/ssl-schema.json b/src/services/schema/ssl-schema.json deleted file mode 100644 index 6465fb7914..0000000000 --- a/src/services/schema/ssl-schema.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "caCertificatePath": { "type": "string" }, - "certificatePath": { "type": "string" }, - "keyPath": { "type": "string" } - }, - "additionalProperties": false, - "required": ["caCertificatePath", "certificatePath", "keyPath"] -} diff --git a/src/services/schema/telemetry-schema.json b/src/services/schema/telemetry-schema.json deleted file mode 100644 index cdb8612791..0000000000 --- a/src/services/schema/telemetry-schema.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "enabled": { "type": "boolean" } - }, - "required": ["enabled"] -} diff --git a/src/templates/database.yml b/src/templates/database.yml deleted file mode 100644 index ccbd1db6af..0000000000 --- a/src/templates/database.yml +++ /dev/null @@ -1,2 +0,0 @@ -nonceDbPath: 'nonce.level' -transactionDbPath: 'transaction.level' diff --git a/src/templates/logging.yml b/src/templates/logging.yml deleted file mode 100644 index c5ffe44046..0000000000 --- a/src/templates/logging.yml +++ /dev/null @@ -1,6 +0,0 @@ -# The directory path where logs will be stored. -logPath: './logs' - -# If true, logs will be stored in logPath and printed to stdout. If false, they -# will only be stored in logPath and not printed to stdout. -logToStdOut: true diff --git a/src/templates/root.yml b/src/templates/root.yml index 74976f195f..706812c05e 100644 --- a/src/templates/root.yml +++ b/src/templates/root.yml @@ -1,5 +1,13 @@ version: 1 configurations: + $namespace server: + configurationPath: server.yml + schemaPath: server-schema.json + + $namespace ethereum: + configurationPath: ethereum.yml + schemaPath: ethereum-schema.json + $namespace harmony: configurationPath: harmony.yml schemaPath: harmony-schema.json @@ -8,30 +16,10 @@ configurations: configurationPath: avalanche.yml schemaPath: ethereum-schema.json - $namespace database: - configurationPath: database.yml - schemaPath: database-schema.json - - $namespace ethereum: - configurationPath: ethereum.yml - schemaPath: ethereum-schema.json - $namespace polygon: configurationPath: polygon.yml schemaPath: ethereum-schema.json - $namespace ethereumGasStation: - configurationPath: ethereum-gas-station.yml - schemaPath: ethereum-gas-station-schema.json - - $namespace logging: - configurationPath: logging.yml - schemaPath: logging-schema.json - - $namespace ssl: - configurationPath: ssl.yml - schemaPath: ssl-schema.json - $namespace defira: configurationPath: defira.yml schemaPath: defira-schema.json @@ -60,14 +48,6 @@ configurations: configurationPath: traderjoe.yml schemaPath: traderjoe-schema.json - $namespace server: - configurationPath: server.yml - schemaPath: server-schema.json - - $namespace telemetry: - configurationPath: telemetry.yml - schemaPath: telemetry-schema.json - $namespace uniswap: configurationPath: uniswap.yml schemaPath: uniswap-schema.json diff --git a/src/templates/server.yml b/src/templates/server.yml index 93d4137ebb..7a331a66bd 100644 --- a/src/templates/server.yml +++ b/src/templates/server.yml @@ -1,10 +1,33 @@ -# The port to expose the gateway server on. +# Path to folder where Hummingbot generates self-signed certificates +certificatePath: /certs/ + +# Path to folder where logs will be stored. +logPath: './logs' + +# Port to expose the gateway server on port: 15888 -# The IPs allowed to access gateway. localhost is allowed by default. +# IPs allowed to access gateway. localhost is allowed by default. ipWhitelist: [] +# GMT Offset +GMTOffset: +0800 + +# If true, logs will be stored in logPath and printed to stdout. If false, they +# will only be stored in logPath and not printed to stdout. +logToStdOut: true + +# Runs Gateway in unsafe dev mode with HTTP if true unsafeDevModeWithHTTP: false +# Collects data about API usage if true +telemetry_enabled: false + +# Nonce database +nonceDbPath: 'nonce.level' + +# Transaction database +transactionDbPath: 'transaction.level' + + -GMTOffset: +0800 diff --git a/src/templates/ssl.yml b/src/templates/ssl.yml deleted file mode 100644 index e5bfacf950..0000000000 --- a/src/templates/ssl.yml +++ /dev/null @@ -1,8 +0,0 @@ -# need more details... -caCertificatePath: /usr/src/app/certs/ca_cert.pem - -# need more details... -certificatePath: /usr/src/app/certs/server_cert.pem - -# need more details... -keyPath: /usr/src/app/certs/server_key.pem diff --git a/src/templates/telemetry.yml b/src/templates/telemetry.yml deleted file mode 100644 index d8e96575f4..0000000000 --- a/src/templates/telemetry.yml +++ /dev/null @@ -1,2 +0,0 @@ -# If true it will collect data about API usage. -enabled: false diff --git a/test/services/logger.test.ts b/test/services/logger.test.ts index 30228ebe01..37de22f17f 100644 --- a/test/services/logger.test.ts +++ b/test/services/logger.test.ts @@ -8,12 +8,12 @@ import { describe('Test logger', () => { it('updateLoggerToStdout works', (done) => { - ConfigManagerV2.getInstance().set('logging.logToStdOut', true); + ConfigManagerV2.getInstance().set('server.logToStdOut', true); updateLoggerToStdout(); const ofTypeConsole = (element: any) => element instanceof winston.transports.Console; expect(logger.transports.some(ofTypeConsole)).toEqual(true); - ConfigManagerV2.getInstance().set('logging.logToStdOut', false); + ConfigManagerV2.getInstance().set('server.logToStdOut', false); updateLoggerToStdout(); // Not sure why the below test doesn't on Github but passes on local // expect(logger.transports.some(ofTypeConsole)).toEqual(false);