diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ad92300d..d05856c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,10 +42,41 @@ jobs: run: pnpm run prettier:check && pnpm run eslint:check - name: Test run: pnpm run test + e2e: + runs-on: ubuntu-latest + name: E2e tests + steps: + - name: Clone repo + uses: actions/checkout@v4 + - name: Install pnpm + uses: pnpm/action-setup@v2 + with: + version: 8.8.0 + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: '18.x' + cache: 'pnpm' + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Build + run: pnpm run build + - name: Build Docker images + run: pnpm run docker:build + - name: Start services + # Start the e2e services in the background and wait a small amount of time for them to start. + run: | + pnpm run --recursive --filter e2e start:data-provider-api & + pnpm run --recursive --filter e2e start:ci:signed-api & + pnpm run --recursive --filter e2e start:ci:pusher & + sleep 5 + - name: Run e2e tests + run: pnpm run --recursive --filter e2e test:e2e required-checks-passed: name: All required checks passed runs-on: ubuntu-latest - needs: [lint-build-test] + needs: [documentation, lint-build-test, e2e] steps: - run: exit 0 diff --git a/.gitignore b/.gitignore index e2bd78a1..32067655 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,9 @@ pusher.json secrets.env signed-api.json .DS_Store + +# Do not ignore e2e config files. These don't need to be listed on Dockerignore because e2e package is not dockerized. +!packages/e2e/**/pusher.json +!packages/e2e/**/secrets.env +!packages/e2e/**/.env +!packages/e2e/**/signed-api.json diff --git a/README.md b/README.md index fe788203..bd73f446 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ A monorepo for managing signed data. Consists of: - [common](./packages/common/README.md) - An internal-only package with common types and utilities used by other packages. - [pusher](./packages/pusher/README.md) - A service for pushing data provider signed data. +- [e2e](./packages/e2e/README.md) - End to end test utilizing Mock API, pusher and signed API. ## Getting started diff --git a/packages/api/package.json b/packages/api/package.json index 459d8c09..0f90d73c 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -27,7 +27,7 @@ "nodemon": "^3.0.1" }, "dependencies": { - "@api3/promise-utils": "0.4.0", + "@api3/promise-utils": "^0.4.0", "@aws-sdk/client-s3": "^3.421.0", "dotenv": "^16.3.1", "ethers": "^5.7.2", diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 721a964b..d06dc642 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -14,7 +14,7 @@ export const startServer = (config: Config) => { const result = await batchInsertData(req.body); res.status(result.statusCode).header(result.headers).send(result.body); - logger.info('Responded to request "POST /"', result); + logger.debug('Responded to request "POST /"', result); }); app.get('/', async (_req, res) => { @@ -23,7 +23,7 @@ export const startServer = (config: Config) => { const result = await listAirnodeAddresses(); res.status(result.statusCode).header(result.headers).send(result.body); - logger.info('Responded to request "GET /"', result); + logger.debug('Responded to request "GET /"', result); }); for (const endpoint of config.endpoints) { @@ -36,7 +36,7 @@ export const startServer = (config: Config) => { const result = await getData(req.params.airnodeAddress, delaySeconds); res.status(result.statusCode).header(result.headers).send(result.body); - logger.info('Responded to request "GET /:airnode"', result); + logger.debug('Responded to request "GET /:airnode"', result); }); } diff --git a/packages/common/README.md b/packages/common/README.md index 877cc1a4..c7b451fc 100644 --- a/packages/common/README.md +++ b/packages/common/README.md @@ -1,7 +1,9 @@ # common -Utilities commonly used by other packages. Each common utility lives in its own folder together with its documentation. -The implementation is re-exported by main entry of this package. The package consists of: +> Utilities commonly used by other packages. + +Each common utility lives in its own folder together with its documentation. The implementation is re-exported by main +entry of this package. The package consists of: - [logger](./src/logger) - Backend-only logger for Node.js packages based on Winston logger. diff --git a/packages/common/src/logger/index.ts b/packages/common/src/logger/index.ts index 4e885089..0e32a0a5 100644 --- a/packages/common/src/logger/index.ts +++ b/packages/common/src/logger/index.ts @@ -87,15 +87,15 @@ export interface Logger { // That's causing an override of fields `name` and `message` if they are present. const wrapper = (logger: Logger): Logger => { return { - debug: (message, context) => logger.debug(message, { context }), - info: (message, context) => logger.info(message, { context }), - warn: (message, context) => logger.warn(message, { context }), + debug: (message, context) => logger.debug(message, context ? { context } : undefined), + info: (message, context) => logger.info(message, context ? { context } : undefined), + warn: (message, context) => logger.warn(message, context ? { context } : undefined), // We need to handle both overloads of the `error` function error: (message, errorOrContext, context) => { if (errorOrContext instanceof Error) { - logger.error(message, errorOrContext, { context }); + logger.error(message, errorOrContext, context ? { context } : undefined); } else { - logger.error(message, { context: errorOrContext }); + logger.error(message, errorOrContext ? { context: errorOrContext } : undefined); } }, child: (options) => wrapper(logger.child(options)), diff --git a/packages/e2e/.eslintignore b/packages/e2e/.eslintignore new file mode 100644 index 00000000..e2bd78a1 --- /dev/null +++ b/packages/e2e/.eslintignore @@ -0,0 +1,15 @@ +# NOTE: Keep in sync with .dockerignore +.build +.env +.idea +.log +.tsbuildinfo +.vscode +build +dist +node_modules +coverage +pusher.json +secrets.env +signed-api.json +.DS_Store diff --git a/packages/e2e/.prettierignore b/packages/e2e/.prettierignore new file mode 100644 index 00000000..e2bd78a1 --- /dev/null +++ b/packages/e2e/.prettierignore @@ -0,0 +1,15 @@ +# NOTE: Keep in sync with .dockerignore +.build +.env +.idea +.log +.tsbuildinfo +.vscode +build +dist +node_modules +coverage +pusher.json +secrets.env +signed-api.json +.DS_Store diff --git a/packages/e2e/README.md b/packages/e2e/README.md new file mode 100644 index 00000000..7353cf3b --- /dev/null +++ b/packages/e2e/README.md @@ -0,0 +1,26 @@ +# e2e + +> End to end test utilizing Mock API, pusher and signed API. + +## Getting started + +1. If you are using Docker Desktop, you need to change the URL in `pusher/secrets.env` from `localhost` to + `host.docker.internal`, because pusher is running inside a Docker container. +2. Build the latest Docker images. Run `pnpm run docker:build` from the monorepo root. The e2e flow uses the docker + images. +3. This module contains services (or configurations) that are integrated together. Specifically: + + - `pusher` - Contains the configuration for the pusher service. + - `signed-api` - Contains the configuration for the signed API service. + - `data-provider-api.ts` - Contains the configuration for the data provider API service (mocked express server). + - `user.ts` - Contains the configuration for the user service (infinite fetch from signed API). + + You are free to modify the configurations to test different scenarios. + +4. There are `start:` scripts to start the services. It is recommended to start each service in a separate + terminal and in this order: + + 1. `pnpm run start:data-provider-api` + 2. `pnpm run start:signed-api` + 3. `pnpm run start:pusher` + 4. `pnpm run start:user` diff --git a/packages/e2e/jest.config.js b/packages/e2e/jest.config.js new file mode 100644 index 00000000..97126edb --- /dev/null +++ b/packages/e2e/jest.config.js @@ -0,0 +1,5 @@ +const config = require('../../jest.config'); + +module.exports = { + ...config, +}; diff --git a/packages/e2e/package.json b/packages/e2e/package.json new file mode 100644 index 00000000..a2555b3f --- /dev/null +++ b/packages/e2e/package.json @@ -0,0 +1,38 @@ +{ + "name": "e2e", + "version": "1.0.0", + "engines": { + "node": "^18.14.0", + "pnpm": "^8.8.0" + }, + "scripts": { + "build": "tsc --project tsconfig.build.json", + "clean": "rm -rf coverage dist", + "eslint:check": "eslint . --ext .js,.ts --max-warnings 0", + "eslint:fix": "eslint . --ext .js,.ts --fix", + "prettier:check": "prettier --check \"./**/*.{js,ts,md,yml,json}\"", + "prettier:fix": "prettier --write \"./**/*.{js,ts,md,yml,json}\"", + "start:data-provider-api": "ts-node src/data-provider-api.ts", + "start:pusher": "docker run -it --init --volume $(pwd)/src/pusher:/app/config --network host --env-file ./src/pusher/.env --rm --memory=256m pusher:latest", + "start:signed-api": "docker run --publish 8090:8090 -it --init --volume $(pwd)/src/signed-api:/app/config --env-file ./src/signed-api/.env --rm --memory=256m api:latest", + "start:ci:pusher": "docker run --init --volume $(pwd)/src/pusher:/app/config --network host --env-file ./src/pusher/.env --rm --memory=256m pusher:latest", + "start:ci:signed-api": "docker run --publish 8090:8090 --init --volume $(pwd)/src/signed-api:/app/config --env-file ./src/signed-api/.env --rm --memory=256m api:latest", + "start:user": "ts-node src/user.ts", + "test:e2e": "jest", + "tsc": "tsc --project ." + }, + "license": "MIT", + "dependencies": { + "@api3/promise-utils": "^0.4.0", + "axios": "^1.5.1", + "ethers": "^5.7.2", + "express": "^4.18.2", + "lodash": "^4.17.21", + "signed-api/common": "workspace:common@*", + "zod": "^3.22.2" + }, + "devDependencies": { + "@types/express": "^4.17.18", + "@types/lodash": "^4.14.199" + } +} diff --git a/packages/e2e/src/data-provider-api.ts b/packages/e2e/src/data-provider-api.ts new file mode 100644 index 00000000..c91a699e --- /dev/null +++ b/packages/e2e/src/data-provider-api.ts @@ -0,0 +1,52 @@ +import express from 'express'; +import { logger } from './logger'; + +const app = express(); +const PORT = 9876 || process.env.PORT; + +interface Asset { + value: number; + // Everytime the API is queried, the value will be updated by a random percentage. + deltaPercent: number; + name: string; +} + +const assets: Asset[] = [ + { + value: 1000, + deltaPercent: 10, + name: 'MOCK-ETH/USD', + }, + { + value: 5000, + deltaPercent: 2, + name: 'MOCK-BTC/USD', + }, + { + value: 750, + deltaPercent: 80, + name: 'MOCK-ABC/DEF', + }, + { + value: 50000, + deltaPercent: 20, + name: 'MOCK-HJK/KOP', + }, +]; + +app.get('/', (_req, res) => { + logger.debug('Request GET /'); + + for (const asset of assets) { + asset.value = parseFloat((asset.value * (1 + ((Math.random() - 0.5) * asset.deltaPercent) / 100)).toFixed(5)); + } + + const response = Object.fromEntries(assets.map((asset) => [asset.name, asset.value])); + logger.debug('Response GET /', response); + + res.json(response); +}); + +app.listen(PORT, () => { + logger.info(`Server is running on http://localhost:${PORT}`); +}); diff --git a/packages/e2e/src/logger.ts b/packages/e2e/src/logger.ts new file mode 100644 index 00000000..f8d8d223 --- /dev/null +++ b/packages/e2e/src/logger.ts @@ -0,0 +1,8 @@ +import { createLogger } from 'signed-api/common'; + +export const logger = createLogger({ + colorize: true, + enabled: true, + minLevel: 'debug', + format: 'pretty', +}); diff --git a/packages/e2e/src/pusher/.env b/packages/e2e/src/pusher/.env new file mode 100644 index 00000000..2cfd0a3f --- /dev/null +++ b/packages/e2e/src/pusher/.env @@ -0,0 +1,4 @@ +LOGGER_ENABLED=true +LOG_COLORIZE=true +LOG_FORMAT=pretty +LOG_LEVEL=debug diff --git a/packages/e2e/src/pusher/pusher.json b/packages/e2e/src/pusher/pusher.json new file mode 100644 index 00000000..20ae23bb --- /dev/null +++ b/packages/e2e/src/pusher/pusher.json @@ -0,0 +1,111 @@ +{ + "airnodeWalletMnemonic": "diamond result history offer forest diagram crop armed stumble orchard stage glance", + "rateLimiting": { "Mock API": { "maxConcurrency": 25, "minTime": 0 } }, + "templates": { + "0xd1c9c79ccb6e24f77c14456b9b037ded9bfd0709468297e4e1e1bedbfe1bbf1a": { + "endpointId": "0xa02c7e24d1d73f429927eedb78185a7d7d7c82d410acc3914cf6213aa29fea3f", + "parameters": [{ "type": "string32", "name": "name", "value": "MOCK-ETH/USD" }] + }, + "0x4edc7440a34c8420ff592e4b2455785a9218c388a3112a2a92beedcb3e8feeba": { + "endpointId": "0xa02c7e24d1d73f429927eedb78185a7d7d7c82d410acc3914cf6213aa29fea3f", + "parameters": [{ "type": "string32", "name": "name", "value": "MOCK-BTC/USD" }] + }, + "0xcc72aa1d058c5db90c4595a6fcfb320dce997b978dcb6d1f176a121c05d47eeb": { + "endpointId": "0xa02c7e24d1d73f429927eedb78185a7d7d7c82d410acc3914cf6213aa29fea3f", + "parameters": [{ "type": "string32", "name": "name", "value": "MOCK-ABC/DEF" }] + }, + "0xcd2516b29a7634861a010155a3666b1be9af9cdb2d442e8f1ac1607c42862f9e": { + "endpointId": "0xa02c7e24d1d73f429927eedb78185a7d7d7c82d410acc3914cf6213aa29fea3f", + "parameters": [{ "type": "string32", "name": "name", "value": "MOCK-HJK/KOP" }] + } + }, + "endpoints": { + "0xa02c7e24d1d73f429927eedb78185a7d7d7c82d410acc3914cf6213aa29fea3f": { + "endpointName": "mock-feed", + "oisTitle": "Mock API" + } + }, + "triggers": { + "signedApiUpdates": [ + { + "signedApiName": "localhost", + "templateIds": [ + "0xd1c9c79ccb6e24f77c14456b9b037ded9bfd0709468297e4e1e1bedbfe1bbf1a", + "0x4edc7440a34c8420ff592e4b2455785a9218c388a3112a2a92beedcb3e8feeba" + ], + "fetchInterval": 5, + "updateDelay": 0 + }, + { + "signedApiName": "localhost", + "templateIds": [ + "0x4edc7440a34c8420ff592e4b2455785a9218c388a3112a2a92beedcb3e8feeba", + "0xcc72aa1d058c5db90c4595a6fcfb320dce997b978dcb6d1f176a121c05d47eeb", + "0xcd2516b29a7634861a010155a3666b1be9af9cdb2d442e8f1ac1607c42862f9e" + ], + "fetchInterval": 3, + "updateDelay": 5 + }, + { + "signedApiName": "localhost", + "templateIds": [ + "0xd1c9c79ccb6e24f77c14456b9b037ded9bfd0709468297e4e1e1bedbfe1bbf1a", + "0xcc72aa1d058c5db90c4595a6fcfb320dce997b978dcb6d1f176a121c05d47eeb", + "0xcd2516b29a7634861a010155a3666b1be9af9cdb2d442e8f1ac1607c42862f9e" + ], + "fetchInterval": 1, + "updateDelay": 0 + } + ] + }, + "signedApis": [ + { + "name": "localhost", + "url": "http://${HOST_IP}:8090" + } + ], + "ois": [ + { + "oisFormat": "2.2.0", + "title": "Mock API", + "version": "0.2.0", + "apiSpecifications": { + "components": { + "securitySchemes": {} + }, + "paths": { + "/": { "get": { "parameters": [] } } + }, + "servers": [{ "url": "http://${HOST_IP}:9876" }], + "security": {} + }, + "endpoints": [ + { + "fixedOperationParameters": [], + "name": "mock-feed", + "operation": { "method": "get", "path": "/" }, + "parameters": [{ "name": "name" }], + "reservedParameters": [ + { "name": "_type", "fixed": "int256" }, + { "name": "_times", "fixed": "1000000000000000000" } + ], + "preProcessingSpecifications": [ + { + "environment": "Node", + "value": "const output = {};", + "timeoutMs": 5000 + } + ], + "postProcessingSpecifications": [ + { + "environment": "Node", + "value": "const output = input[endpointParameters.name];", + "timeoutMs": 5000 + } + ] + } + ] + } + ], + "apiCredentials": [] +} diff --git a/packages/e2e/src/pusher/secrets.env b/packages/e2e/src/pusher/secrets.env new file mode 100644 index 00000000..0896b397 --- /dev/null +++ b/packages/e2e/src/pusher/secrets.env @@ -0,0 +1,4 @@ +# Secrets must NOT be quoted. +# +# Use "host.docker.internal" if using Docker for Desktop. +HOST_IP=localhost diff --git a/packages/e2e/src/signed-api/.env b/packages/e2e/src/signed-api/.env new file mode 100644 index 00000000..ab42d4da --- /dev/null +++ b/packages/e2e/src/signed-api/.env @@ -0,0 +1,16 @@ +LOGGER_ENABLED=true +LOG_COLORIZE=true +LOG_FORMAT=pretty +LOG_LEVEL=debug + +# Available options +# local (default) - loads config/signed-api.json from the filesystem +# aws-s3 - loads the config file from AWS S3 +CONFIG_SOURCE=local + +# Set these variables if you would like to source your config from AWS S3 +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_REGION= +AWS_S3_BUCKET_NAME= +AWS_S3_BUCKET_PATH= diff --git a/packages/e2e/src/signed-api/signed-api.json b/packages/e2e/src/signed-api/signed-api.json new file mode 100644 index 00000000..9d36ac72 --- /dev/null +++ b/packages/e2e/src/signed-api/signed-api.json @@ -0,0 +1,17 @@ +{ + "endpoints": [ + { + "urlPath": "/real-time", + "delaySeconds": 0 + }, + { + "urlPath": "/delayed", + "delaySeconds": 10 + } + ], + "maxBatchSize": 10, + "port": 8090, + "cache": { + "maxAgeSeconds": 0 + } +} diff --git a/packages/e2e/src/user.test.ts b/packages/e2e/src/user.test.ts new file mode 100644 index 00000000..822cd42f --- /dev/null +++ b/packages/e2e/src/user.test.ts @@ -0,0 +1,28 @@ +import axios from 'axios'; +import { airnode, formatData } from './utils'; + +it('respects the delay', async () => { + const start = Date.now(); + let [realCount, delayedCount] = [0, 0]; + + while (Date.now() - start < 15_000) { + const realTimeResponse = await axios.get(`http://localhost:8090/real-time/${airnode}`); + const realTimeData = formatData(realTimeResponse.data); + const delayedResponse = await axios.get(`http://localhost:8090/delayed/${airnode}`); + const delayedData = formatData(delayedResponse.data); + + for (const data of realTimeData) { + expect(data.delay).toBeGreaterThan(0); + realCount++; + } + for (const data of delayedData) { + expect(data.delay).toBeGreaterThan(10_000); + delayedCount++; + } + + await new Promise((resolve) => setTimeout(resolve, 300)); + } + + expect(realCount).toBeGreaterThan(0); + expect(delayedCount).toBeGreaterThan(0); +}, 20_000); diff --git a/packages/e2e/src/user.ts b/packages/e2e/src/user.ts new file mode 100644 index 00000000..5c62f3cb --- /dev/null +++ b/packages/e2e/src/user.ts @@ -0,0 +1,20 @@ +import axios from 'axios'; +import { logger } from './logger'; +import { airnode, formatData } from './utils'; + +async function main() { + // eslint-disable-next-line no-constant-condition + while (true) { + logger.debug('Making requests'); + + const realTimeResponse = await axios.get(`http://localhost:8090/real-time/${airnode}`); + logger.debug('Response GET /real-time', formatData(realTimeResponse.data)); + + const delayedResponse = await axios.get(`http://localhost:8090/delayed/${airnode}`); + logger.debug('Response GET /delayed', formatData(delayedResponse.data)); + + await new Promise((resolve) => setTimeout(resolve, 1000)); + } +} + +main(); diff --git a/packages/e2e/src/utils.ts b/packages/e2e/src/utils.ts new file mode 100644 index 00000000..77d878c8 --- /dev/null +++ b/packages/e2e/src/utils.ts @@ -0,0 +1,21 @@ +import { ethers } from 'ethers'; +import { goSync } from '@api3/promise-utils'; + +export const formatData = (networkResponse: any) => { + const goFormat = goSync(() => + Object.values(networkResponse.data).map((d: any) => { + return { + templateId: d.templateId, + delay: Date.now() - d.timestamp * 1000, + value: ethers.utils.defaultAbiCoder.decode(['int256'], d.encodedValue).toString(), + }; + }) + ); + + if (!goFormat.success) return networkResponse; + return goFormat.data; +}; + +export const airnode = ethers.Wallet.fromMnemonic( + 'diamond result history offer forest diagram crop armed stumble orchard stage glance' +).address; diff --git a/packages/e2e/tsconfig.build.json b/packages/e2e/tsconfig.build.json new file mode 100644 index 00000000..daf9b72f --- /dev/null +++ b/packages/e2e/tsconfig.build.json @@ -0,0 +1,13 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "outDir": "dist", + "baseUrl": "./", + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": ["./src/**/*"], + "exclude": ["./src/**/*.test.ts"] +} diff --git a/packages/e2e/tsconfig.json b/packages/e2e/tsconfig.json new file mode 100644 index 00000000..9926f476 --- /dev/null +++ b/packages/e2e/tsconfig.json @@ -0,0 +1,25 @@ +{ + "compilerOptions": { + "noEmit": true, + "lib": ["esnext"], + "module": "commonjs", + "esModuleInterop": true, + "moduleResolution": "node", + "strict": true, + "target": "esnext", + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + "exactOptionalPropertyTypes": true, + "noUncheckedIndexedAccess": true, + "noImplicitReturns": true, + + // Disabled because they are covered by Eslint rules + "noUnusedLocals": false, + "noUnusedParameters": false, + + // Disabled because prefer the property syntax + "noPropertyAccessFromIndexSignature": false + }, + "exclude": ["dist/**/*", "node_modules/**/*"], + "include": ["./src/**/*", "./test/**/*", "jest.config.js"] +} diff --git a/packages/pusher/README.md b/packages/pusher/README.md index ae2bbc97..6305d1e0 100644 --- a/packages/pusher/README.md +++ b/packages/pusher/README.md @@ -352,6 +352,10 @@ Pusher is also dockerized. To run the dockerized pusher you need to: [this](https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md#handling-kernel-signals) for details. 3. Specify the `--env-file` with the path to the `.env` file containing the [ENV configuration](#environment-variables). 4. Optionally, pass the `--rm` flag to remove the container after it is stopped. +5. If running on Linux, you need to use the `--network host` to access the host network. This has no effect for Docker + Desktop. +6. Lastly, if you are using Docker Desktop and you want to access the host machine, you need to change the host URL from + `localhost` to `host.docker.internal` in the configuration files. For example: diff --git a/packages/pusher/package.json b/packages/pusher/package.json index 57dabcba..83436585 100644 --- a/packages/pusher/package.json +++ b/packages/pusher/package.json @@ -10,7 +10,7 @@ "clean": "rm -rf coverage dist", "dev": "nodemon --ext ts,js,json,env --exec \"pnpm ts-node src/index.ts\"", "docker:build": "docker build --target pusher --tag pusher:latest ../../", - "docker:start": "docker run -it --init --volume $(pwd)/config:/app/config --env-file .env --rm pusher:latest", + "docker:start": "docker run -it --init --volume $(pwd)/config:/app/config --network host --env-file .env --rm pusher:latest", "eslint:check": "eslint . --ext .js,.ts --max-warnings 0", "eslint:fix": "eslint . --ext .js,.ts --fix", "prettier:check": "prettier --check \"./**/*.{js,ts,md,yml,json}\"", diff --git a/packages/pusher/src/api-requests/data-provider.ts b/packages/pusher/src/api-requests/data-provider.ts index adac40aa..48cab4de 100644 --- a/packages/pusher/src/api-requests/data-provider.ts +++ b/packages/pusher/src/api-requests/data-provider.ts @@ -9,7 +9,7 @@ import { TemplateResponse } from '../sign-template-data'; export const callApi = async (payload: node.ApiCallPayload) => { logger.debug('Preprocessing API call payload', pick(payload.aggregatedApiCall, ['endpointName', 'oisTitle'])); const processedPayload = await preProcessApiSpecifications(payload); - logger.debug('Performing API call', { processedPayload: processedPayload }); + logger.debug('Performing API call', pick(processedPayload.aggregatedApiCall, ['endpointName', 'oisTitle'])); return node.api.performApiCall(processedPayload); }; @@ -50,8 +50,11 @@ export const makeTemplateRequests = async (signedApiUpdate: SignedApiUpdate): Pr : callApi(operationPayload)); if (node.api.isPerformApiCallFailure(apiCallResponse)) { - const message = `Failed to make API call for the endpoint [${endpoint.oisTitle}] ${endpoint.endpointName}.`; - logger.warn(message, { operationTemplateId }); + logger.warn(`Failed to make API call`, { + operationTemplateId, + ...pick(endpoint, ['endpointName', 'oisTitle']), + errorMessage: apiCallResponse.errorMessage, + }); return []; } @@ -78,9 +81,11 @@ export const makeTemplateRequests = async (signedApiUpdate: SignedApiUpdate): Pr logger.debug('Processing successful API call', { templateId, operationTemplateId }); const [_, response] = await node.api.processSuccessfulApiCall(payload, apiCallResponse); - if (!response.success) { + // We need to check both success and error property because Airnode returns success. See: + // https://github.com/api3dao/airnode/blob/c0ad71f1cbf40235f2ee765000b430f7f0f026a5/packages/airnode-node/src/api/index.ts#L243 + if (!response.success || (response as any).errorMessage) { const message = `Failed to post process successful API call`; - logger.warn(message, { templateId, operationTemplateId, errorMessage: response.errorMessage }); + logger.warn(message, { templateId, operationTemplateId, errorMessage: (response as any).errorMessage }); return null; } return [templateId, response]; diff --git a/packages/pusher/src/api-requests/signed-api.test.ts b/packages/pusher/src/api-requests/signed-api.test.ts index 9668c5f0..492d1e01 100644 --- a/packages/pusher/src/api-requests/signed-api.test.ts +++ b/packages/pusher/src/api-requests/signed-api.test.ts @@ -74,7 +74,8 @@ describe(postSignedApiData.name, () => { expect(response).toEqual({ success: false }); expect(logger.warn).toHaveBeenCalledWith('Failed to make update signed API request.', { - error: new Error('simulated-network-error'), + errorMessage: 'simulated-network-error', + axiosResponse: {}, signedApiName: 'localhost', updateDelay: 5, }); diff --git a/packages/pusher/src/api-requests/signed-api.ts b/packages/pusher/src/api-requests/signed-api.ts index 7605a013..673958a7 100644 --- a/packages/pusher/src/api-requests/signed-api.ts +++ b/packages/pusher/src/api-requests/signed-api.ts @@ -1,6 +1,6 @@ import { go } from '@api3/promise-utils'; import axios, { AxiosError } from 'axios'; -import { isEmpty, isNil } from 'lodash'; +import { isEmpty, isNil, pick } from 'lodash'; import { ethers } from 'ethers'; import { deriveBeaconId } from '@api3/airnode-node'; import { logger } from '../logger'; @@ -50,7 +50,11 @@ export const postSignedApiData = async (group: SignedApiNameUpdateDelayGroup) => logger.warn( `Failed to make update signed API request.`, // See: https://axios-http.com/docs/handling_errors - { ...logContext, error: goAxiosRequest.error.response ?? goAxiosRequest.error } + { + ...logContext, + axiosResponse: pick(goAxiosRequest.error.response, ['data', 'status', 'headers']), + errorMessage: goAxiosRequest.error.message, + } ); return { success: false }; } diff --git a/packages/pusher/src/update-signed-api.ts b/packages/pusher/src/update-signed-api.ts index e737ba51..8a94cd04 100644 --- a/packages/pusher/src/update-signed-api.ts +++ b/packages/pusher/src/update-signed-api.ts @@ -1,4 +1,4 @@ -import { get, isEmpty } from 'lodash'; +import { get, isEmpty, uniq } from 'lodash'; import { logger } from './logger'; import { getState } from './state'; import { sleep } from './utils'; @@ -19,22 +19,20 @@ export const initiateUpdatingSignedApi = async () => { logger.debug('Initiating updating signed API'); const { config } = getState(); - const signedApiUpdateDelayTemplateIdsMap = config.triggers.signedApiUpdates.reduce( - (acc: SignedApiUpdateDelayTemplateIdsMap, signedApiUpdate) => { - if (isEmpty(signedApiUpdate.templateIds)) return acc; - return { - ...acc, - [signedApiUpdate.signedApiName]: { - ...acc[signedApiUpdate.signedApiName], - [signedApiUpdate.updateDelay]: [ - ...get(acc, [signedApiUpdate.signedApiName, signedApiUpdate.updateDelay], []), - ...signedApiUpdate.templateIds, - ], - }, - }; - }, - {} - ); + const signedApiUpdateDelayTemplateIdsMap = config.triggers.signedApiUpdates.reduce((acc, signedApiUpdate) => { + if (isEmpty(signedApiUpdate.templateIds)) return acc; + + return { + ...acc, + [signedApiUpdate.signedApiName]: { + ...acc[signedApiUpdate.signedApiName], + [signedApiUpdate.updateDelay]: uniq([ + ...get(acc, [signedApiUpdate.signedApiName, signedApiUpdate.updateDelay], []), + ...signedApiUpdate.templateIds, + ]), + }, + }; + }, {} as SignedApiUpdateDelayTemplateIdsMap); const signedApiUpdateDelayGroups: SignedApiNameUpdateDelayGroup[] = Object.entries( signedApiUpdateDelayTemplateIdsMap diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9c6d67da..a5b6ac97 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -54,7 +54,7 @@ importers: packages/api: dependencies: '@api3/promise-utils': - specifier: 0.4.0 + specifier: ^0.4.0 version: 0.4.0 '@aws-sdk/client-s3': specifier: ^3.421.0 @@ -100,6 +100,37 @@ importers: specifier: ^3.22.2 version: 3.22.2 + packages/e2e: + dependencies: + '@api3/promise-utils': + specifier: ^0.4.0 + version: 0.4.0 + axios: + specifier: ^1.5.1 + version: 1.5.1 + ethers: + specifier: ^5.7.2 + version: 5.7.2 + express: + specifier: ^4.18.2 + version: 4.18.2 + lodash: + specifier: ^4.17.21 + version: 4.17.21 + signed-api/common: + specifier: workspace:common@* + version: link:../common + zod: + specifier: ^3.22.2 + version: 3.22.2 + devDependencies: + '@types/express': + specifier: ^4.17.18 + version: 4.17.18 + '@types/lodash': + specifier: ^4.14.199 + version: 4.14.199 + packages/pusher: dependencies: '@api3/airnode-abi':