From 43f4653a46256f6e62a7733ee2c19e17c65629c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Thu, 21 Sep 2023 15:04:42 +0200 Subject: [PATCH 01/12] Save all beacon signed data, not just the freshest --- packages/api/README.md | 13 ------ packages/api/src/handlers.ts | 72 ++++------------------------- packages/api/src/in-memory-cache.ts | 8 ++-- packages/api/src/server.ts | 13 +----- 4 files changed, 16 insertions(+), 90 deletions(-) diff --git a/packages/api/README.md b/packages/api/README.md index 16343ad6..f655ec81 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -15,7 +15,6 @@ TODO: Write example how to deploy on AWS (and maybe other cloud providers as wel The API provides the following endpoints: -- `PUT /`: Upsert single signed data. - `POST /`: Upsert batch of signed data. - `GET /{airnode}`: Retrieve signed data for the airnode. - `GET /`: Retrieve list of all available airnode address. @@ -46,18 +45,6 @@ PORT=5123 pnpm run docker:start Here are some examples of how to use the API with `curl`. Note, the port may differ based on the `.env` value. ```bash -# Upsert signed data (HTTP PUT) -curl --location --request PUT 'http://localhost:8090' \ ---header 'Content-Type: application/json' \ ---data '{ - "airnode": "0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4", - "beaconId": "0x70601427c8ff03560563917eed9837651ad9d6eb3414e46e8f96302c6f0aefcd", - "templateId": "0x8f255387c5fdb03117d82372b8fa5c7813881fd9a8202b7cc373f1a5868496b2", - "timestamp": "1694644051", - "encodedValue": "0x000000000000000000000000000000000000000000000002eb268c108b0b1da0", - "signature": "0x8e540abb31f6ef161153c508b9cc3909dcc3cf6596deff88ed4f9f2226fa28c61b8c23078373f64a7125035d1f70fd3befa6dfc48a31e7e15cc23133331ed9221b" - }' - # Upsert batch of signed data (HTTP POST) curl --location 'http://localhost:8090' \ --header 'Content-Type: application/json' \ diff --git a/packages/api/src/handlers.ts b/packages/api/src/handlers.ts index 2f2e0223..4e54fae1 100644 --- a/packages/api/src/handlers.ts +++ b/packages/api/src/handlers.ts @@ -1,67 +1,10 @@ -import { isEmpty, isNil, omit, size } from 'lodash'; +import { isEmpty, isNil, last, omit, size } from 'lodash'; import { go, goSync } from '@api3/promise-utils'; -import { - ApiRequest, - ApiResponse, - PromiseError, - batchSignedDataSchema, - evmAddressSchema, - signedDataSchema, -} from './types'; +import { ApiRequest, ApiResponse, PromiseError, batchSignedDataSchema, evmAddressSchema } from './types'; import { deriveBeaconId, recoverSignerAddress } from './evm'; import { generateErrorResponse, isBatchUnique } from './utils'; import { CACHE_HEADERS, COMMON_HEADERS, MAX_BATCH_SIZE } from './constants'; -import { getAll, getAllBy, getBy, put, putAll } from './in-memory-cache'; - -export const upsertData = async (request: ApiRequest): Promise => { - if (isNil(request.body)) return generateErrorResponse(400, 'Invalid request, http body is missing'); - - const goJsonParseBody = goSync(() => JSON.parse(request.body)); - if (!goJsonParseBody.success) return generateErrorResponse(400, 'Invalid request, body must be in JSON'); - - const goValidateSchema = await go(() => signedDataSchema.parseAsync(goJsonParseBody.data)); - if (!goValidateSchema.success) - return generateErrorResponse( - 400, - 'Invalid request, body must fit schema for signed data', - goValidateSchema.error.message - ); - - const signedData = goValidateSchema.data; - - const goRecoverSigner = goSync(() => recoverSignerAddress(signedData)); - if (!goRecoverSigner.success) - return generateErrorResponse(400, 'Unable to recover signer address', goRecoverSigner.error.message); - - if (signedData.airnode !== goRecoverSigner.data) return generateErrorResponse(400, 'Signature is invalid'); - - const goDeriveBeaconId = goSync(() => deriveBeaconId(signedData.airnode, signedData.templateId)); - if (!goDeriveBeaconId.success) - return generateErrorResponse( - 400, - 'Unable to derive beaconId by given airnode and templateId', - goDeriveBeaconId.error.message - ); - - if (signedData.beaconId !== goDeriveBeaconId.data) return generateErrorResponse(400, 'beaconId is invalid'); - - const goReadDb = await go(() => getBy(signedData.airnode, signedData.templateId)); - if (!goReadDb.success) - return generateErrorResponse( - 500, - 'Unable to get signed data from database to validate timestamp', - goReadDb.error.message - ); - - if (!isNil(goReadDb.data) && parseInt(signedData.timestamp) <= parseInt(goReadDb.data.timestamp)) - return generateErrorResponse(400, "Request isn't updating the timestamp"); - - const goWriteDb = await go(() => put(signedData)); - if (!goWriteDb.success) - return generateErrorResponse(500, 'Unable to send signed data to database', goWriteDb.error.message); - - return { statusCode: 201, headers: COMMON_HEADERS, body: JSON.stringify({ count: 1 }) }; -}; +import { getAll, getAllBy, getBy, putAll } from './in-memory-cache'; export const batchUpsertData = async (request: ApiRequest): Promise => { if (isNil(request.body)) return generateErrorResponse(400, 'Invalid request, http body is missing'); @@ -137,8 +80,10 @@ export const batchUpsertData = async (request: ApiRequest): Promise signedData ) ); + if (isNil(goReadDb.data)) return; - if (!isNil(goReadDb.data) && parseInt(signedData.timestamp) <= parseInt(goReadDb.data.timestamp)) + const lastDbSignedData = goReadDb.data[goReadDb.data.length - 1]!; + if (parseInt(signedData.timestamp) <= parseInt(lastDbSignedData.timestamp)) return Promise.reject(generateErrorResponse(400, "Request isn't updating the timestamp", undefined, signedData)); }); @@ -166,7 +111,10 @@ export const getData = async (request: ApiRequest): Promise => { return generateErrorResponse(500, 'Unable to get signed data from database', goReadDb.error.message); // Transform array of signed data to be in form {[beaconId]: SignedData} - const data = goReadDb.data.reduce((acc, Item) => ({ ...acc, [Item.beaconId]: omit(Item, 'beaconId') }), {}); + const data = goReadDb.data.reduce((acc, signedDataArray) => { + const signedData = last(signedDataArray)!; // Guaranteed to exist because we never remove data. TODO: This might not be true after we do removals. + return { ...acc, [signedData.beaconId]: omit(signedData, 'beaconId') }; + }, {}); return { statusCode: 200, diff --git a/packages/api/src/in-memory-cache.ts b/packages/api/src/in-memory-cache.ts index deaac084..ad2dd15c 100644 --- a/packages/api/src/in-memory-cache.ts +++ b/packages/api/src/in-memory-cache.ts @@ -1,6 +1,6 @@ import { SignedData } from './types'; -const signedDataCache: Record> = {}; +const signedDataCache: Record> = {}; // The API is deliberately asynchronous to mimic a database call. export const getBy = async (airnodeId: string, templateId: string) => { @@ -18,8 +18,10 @@ export const getAll = async () => signedDataCache; // The API is deliberately asynchronous to mimic a database call. export const put = async (signedData: SignedData) => { - signedDataCache[signedData.airnode] = signedDataCache[signedData.airnode] ?? {}; - signedDataCache[signedData.airnode]![signedData.templateId] = signedData; + const { airnode, templateId } = signedData; + signedDataCache[airnode] ??= {}; + signedDataCache[airnode]![templateId] ??= []; + signedDataCache[airnode]![templateId]!.push(signedData); }; // The API is deliberately asynchronous to mimic a database call. diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index 4a7b14c8..becd250c 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -1,6 +1,6 @@ import * as dotenv from 'dotenv'; import express from 'express'; -import { getData, listAirnodeAddresses, batchUpsertData, upsertData } from './handlers'; +import { getData, listAirnodeAddresses, batchUpsertData } from './handlers'; export const startServer = () => { dotenv.config(); @@ -11,17 +11,6 @@ export const startServer = () => { app.use(express.json()); - app.put('/', async (req, res) => { - // eslint-disable-next-line no-console - console.log('Received request "PUT /"', req.body, req.params, req.query); - - const result = await upsertData({ - body: JSON.stringify(req.body), - queryParams: {}, - }); - res.status(result.statusCode).header(result.headers).send(result.body); - }); - app.post('/', async (req, res) => { // eslint-disable-next-line no-console console.log('Received request "POST /"', req.body, req.params, req.query); From 3e4a012c04cd2289718909e318ee8b90f5a1890f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 24 Sep 2023 10:58:12 +0200 Subject: [PATCH 02/12] Implement delayed endpoint --- .dockerignore | 1 + .eslintrc.js | 3 + .gitignore | 1 + packages/.DS_Store | Bin 0 -> 6148 bytes packages/api/.env.example | 2 - packages/api/README.md | 20 ++- packages/api/config/signed-api.example.json | 14 ++ packages/api/package.json | 3 +- packages/api/src/cache.ts | 18 +++ packages/api/src/constants.ts | 2 - packages/api/src/evm.ts | 2 +- packages/api/src/handlers.test.ts | 156 ++++++++++++++++++++ packages/api/src/handlers.ts | 131 +++++++--------- packages/api/src/in-memory-cache.ts | 45 +++++- packages/api/src/index.ts | 1 - packages/api/src/schema.test.ts | 31 ++++ packages/api/src/schema.ts | 42 ++++++ packages/api/src/server.ts | 45 +++--- packages/api/src/types.ts | 31 ---- packages/api/src/utils.ts | 13 +- packages/api/test/utils.ts | 45 ++++++ pnpm-lock.yaml | 3 - 22 files changed, 449 insertions(+), 160 deletions(-) create mode 100644 packages/.DS_Store delete mode 100644 packages/api/.env.example create mode 100644 packages/api/config/signed-api.example.json create mode 100644 packages/api/src/cache.ts create mode 100644 packages/api/src/handlers.test.ts create mode 100644 packages/api/src/schema.test.ts create mode 100644 packages/api/src/schema.ts create mode 100644 packages/api/test/utils.ts diff --git a/.dockerignore b/.dockerignore index 64743767..4d043a3c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,3 +17,4 @@ **/coverage **/pusher.json **/secrets.env +**/signed-api.json diff --git a/.eslintrc.js b/.eslintrc.js index b03433f5..c3862998 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -65,5 +65,8 @@ module.exports = { 'no-useless-escape': 'off', semi: 'error', eqeqeq: ['error', 'smart'], + + // Jest + 'jest/valid-title': 'off', // Prevents using ".name" as a test name }, }; diff --git a/.gitignore b/.gitignore index 5d15bbf9..aeff9f7c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ node_modules coverage pusher.json secrets.env +signed-api.json diff --git a/packages/.DS_Store b/packages/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..941feefd24bf5da2c753d1819aa501371792b4ad GIT binary patch literal 6148 zcmeH~O=`nH427SXECStlndNMHfZkvT$q90S1_~h%(o$&GbM!v_Y~prZOnCz7jWiav zzlFyFV7t%PXJ7)bp}XSA!_18N3Mahd@^$>UUEgkJR=f&4M9hqp3A6p$mWY4|h=2%) zfCwyzK%U~*JTB;&^e7@A0?Q!a--kwb?WH3%J{=4(0#Mg1hjAUV1hsjA+Dk_&D>SR= z!K&3_4DoujQ(IlvOGj$9!)o}jy0iHdL$hp$H6}FcAqpZO0y6@u%qKtp5A@&W|5=Mt z5fFiYM!?ql<9^4Nsv{d9s=gj{YFy56_Y=UxkK!#ojQhk literal 0 HcmV?d00001 diff --git a/packages/api/.env.example b/packages/api/.env.example deleted file mode 100644 index 4f822655..00000000 --- a/packages/api/.env.example +++ /dev/null @@ -1,2 +0,0 @@ -MAX_BATCH_SIZE=10 -PORT=8090 diff --git a/packages/api/README.md b/packages/api/README.md index f655ec81..ddca1824 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -4,8 +4,9 @@ A service for storing and accessing signed data. It provides endpoints to handle ## Local development -1. `cp .env.example .env` - To copy `.env` from the `example.env` file. Optionally change the defaults. -2. `pnpm run dev` - To start the API server. The port number can be configured in the `.env` file. +1. `cp config/signed-api.example.json config/signed-api.json` - To create a config file from the example one. Optionally + change the defaults. +2. `pnpm run dev` - To start the API server. The port number can be configured in the configuration file. ## Deployment @@ -15,9 +16,14 @@ TODO: Write example how to deploy on AWS (and maybe other cloud providers as wel The API provides the following endpoints: -- `POST /`: Upsert batch of signed data. -- `GET /{airnode}`: Retrieve signed data for the airnode. -- `GET /`: Retrieve list of all available airnode address. +- `POST /`: Insert a batch of signed data. + - The batch is validated for consistency and data integrity errors. If there is any issue during this step, the whole + batch is rejected. Otherwise the batch is accepted. +- `GET /{endpoint-name}/{airnode}`: Retrieve signed data for the Airnode respecting the endpoint configuration. + - Only returns the freshest signed data available for the given Airnode, respecting the configured endpoint delay. +- `GET /`: Retrieve list of all available Airnode address. + - Returns all Airnode addresses for which there is signed data. It is possible that this data cannot be shown by the + delayed endpoints (in case the data is too fresh and there is not an older alternative). ## Local development @@ -42,7 +48,7 @@ PORT=5123 pnpm run docker:start ### Examples -Here are some examples of how to use the API with `curl`. Note, the port may differ based on the `.env` value. +Here are some examples of how to use the API with `curl`. Note, the port may differ based on the configuration. ```bash # Upsert batch of signed data (HTTP POST) @@ -65,7 +71,7 @@ curl --location 'http://localhost:8090' \ }]' # Get data for the airnode address (HTTP GET) -curl --location 'http://localhost:8090/0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4' \ +curl --location 'http://localhost:8090/real-time/0xc52EeA00154B4fF1EbbF8Ba39FDe37F1AC3B9Fd4' \ --header 'Content-Type: application/json' # List available airnode addresses (HTTP GET) diff --git a/packages/api/config/signed-api.example.json b/packages/api/config/signed-api.example.json new file mode 100644 index 00000000..ed999f0c --- /dev/null +++ b/packages/api/config/signed-api.example.json @@ -0,0 +1,14 @@ +{ + "endpoints": [ + { + "urlPath": "/real-time", + "delaySeconds": 0 + }, + { + "urlPath": "/delayed", + "delaySeconds": 15 + } + ], + "maxBatchSize": 10, + "port": 8090 +} diff --git a/packages/api/package.json b/packages/api/package.json index 1b13db4b..ff28ad38 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -7,6 +7,7 @@ }, "main": "index.js", "scripts": { + "print-info": "which pnpm", "build": "tsc --project .", "clean": "rm -rf coverage dist", "dev": "pnpm ts-node src/index.ts", @@ -18,7 +19,7 @@ "eslint:fix": "eslint . --ext .js,.ts --fix", "prettier:check": "prettier --check \"./**/*.{js,ts,md,json}\"", "prettier:fix": "prettier --write \"./**/*.{js,ts,md,json}\"", - "test": "jest --passWithNoTests" + "test": "jest" }, "license": "MIT", "devDependencies": { diff --git a/packages/api/src/cache.ts b/packages/api/src/cache.ts new file mode 100644 index 00000000..07f9e255 --- /dev/null +++ b/packages/api/src/cache.ts @@ -0,0 +1,18 @@ +import { SignedData } from './schema'; + +type SignedDataCache = Record< + string, // Airnode ID. + Record< + string, // Template ID. + SignedData[] // Signed data is ordered by timestamp (oldest first). + > +>; + +let signedDataCache: SignedDataCache = {}; + +// Making this a getter function makes it easier to mock the cache in storage. +export const getCache = () => signedDataCache; + +export const setCache = (cache: SignedDataCache) => { + signedDataCache = cache; +}; diff --git a/packages/api/src/constants.ts b/packages/api/src/constants.ts index b19234c6..5be95b96 100644 --- a/packages/api/src/constants.ts +++ b/packages/api/src/constants.ts @@ -1,5 +1,3 @@ -export const MAX_BATCH_SIZE = parseInt(process.env.MAX_BATCH_SIZE as string); - export const COMMON_HEADERS = { 'content-type': 'application/json', 'access-control-allow-origin': '*', diff --git a/packages/api/src/evm.ts b/packages/api/src/evm.ts index 89e98142..6145cf89 100644 --- a/packages/api/src/evm.ts +++ b/packages/api/src/evm.ts @@ -1,5 +1,5 @@ import { ethers } from 'ethers'; -import { SignedData } from './types'; +import { SignedData } from './schema'; export const decodeData = (data: string) => ethers.utils.defaultAbiCoder.decode(['int256'], data); diff --git a/packages/api/src/handlers.test.ts b/packages/api/src/handlers.test.ts new file mode 100644 index 00000000..2819d241 --- /dev/null +++ b/packages/api/src/handlers.test.ts @@ -0,0 +1,156 @@ +import { omit } from 'lodash'; +import * as cacheModule from './cache'; +import { batchInsertData, getData, listAirnodeAddresses } from './handlers'; +import { createSignedData, generateRandomWallet } from '../test/utils'; + +afterEach(() => { + cacheModule.setCache({}); +}); + +describe(batchInsertData.name, () => { + it('drops the batch if it is invalid', async () => { + const invalidData = await createSignedData({ signature: '0xInvalid' }); + const batchData = [await createSignedData(), invalidData]; + + const result = await batchInsertData(batchData); + + expect(result).toEqual({ + body: JSON.stringify({ + message: 'Unable to recover signer address', + detail: + 'signature missing v and recoveryParam (argument="signature", value="0xInvalid", code=INVALID_ARGUMENT, version=bytes/5.7.0)', + extra: invalidData, + }), + headers: { + 'access-control-allow-methods': '*', + 'access-control-allow-origin': '*', + 'content-type': 'application/json', + }, + statusCode: 400, + }); + expect(cacheModule.getCache()).toEqual({}); + }); + + it('inserts the batch if data is valid', async () => { + const batchData = [await createSignedData(), await createSignedData()]; + + const result = await batchInsertData(batchData); + + expect(result).toEqual({ + body: JSON.stringify({ count: 2 }), + headers: { + 'access-control-allow-methods': '*', + 'access-control-allow-origin': '*', + 'content-type': 'application/json', + }, + statusCode: 201, + }); + expect(cacheModule.getCache()).toEqual({ + [batchData[0]!.airnode]: { + [batchData[0]!.templateId]: [batchData[0]], + }, + [batchData[1]!.airnode]: { + [batchData[1]!.templateId]: [batchData[1]], + }, + }); + }); +}); + +describe(getData.name, () => { + it('drops the request if the airnode address is invalid', async () => { + const batchData = [await createSignedData(), await createSignedData()]; + await batchInsertData(batchData); + + const result = await getData('0xInvalid', 0); + + expect(result).toEqual({ + body: JSON.stringify({ message: 'Invalid request, path parameter must be an EVM address' }), + headers: { + 'access-control-allow-methods': '*', + 'access-control-allow-origin': '*', + 'content-type': 'application/json', + }, + statusCode: 400, + }); + }); + + it('returns the live data', async () => { + const airnodeWallet = generateRandomWallet(); + const batchData = [await createSignedData({ airnodeWallet }), await createSignedData({ airnodeWallet })]; + await batchInsertData(batchData); + + const result = await getData(airnodeWallet.address, 0); + + expect(result).toEqual({ + body: JSON.stringify({ + count: 2, + data: { + [batchData[0]!.beaconId]: omit(batchData[0], 'beaconId'), + [batchData[1]!.beaconId]: omit(batchData[1], 'beaconId'), + }, + }), + headers: { + 'access-control-allow-methods': '*', + 'access-control-allow-origin': '*', + 'cache-control': 'no-store', + 'cdn-cache-control': 'max-age=10', + 'content-type': 'application/json', + }, + statusCode: 200, + }); + }); + + it('returns the delayed data', async () => { + const airnodeWallet = generateRandomWallet(); + const delayTimestamp = (Math.floor(Date.now() / 1000) - 60).toString(); // Delayed by 60 seconds + const batchData = [ + await createSignedData({ airnodeWallet, timestamp: delayTimestamp }), + await createSignedData({ airnodeWallet }), + ]; + await batchInsertData(batchData); + + const result = await getData(airnodeWallet.address, 30); + + expect(result).toEqual({ + body: JSON.stringify({ + count: 1, + data: { + [batchData[0]!.beaconId]: omit(batchData[0], 'beaconId'), + }, + }), + headers: { + 'access-control-allow-methods': '*', + 'access-control-allow-origin': '*', + 'cache-control': 'no-store', + 'cdn-cache-control': 'max-age=10', + 'content-type': 'application/json', + }, + statusCode: 200, + }); + }); +}); + +describe(listAirnodeAddresses.name, () => { + it('returns the list of airnode addresses', async () => { + const airnodeWallet = generateRandomWallet(); + const batchData = [await createSignedData({ airnodeWallet }), await createSignedData({ airnodeWallet })]; + await batchInsertData(batchData); + + const result = await listAirnodeAddresses(); + + expect(result).toEqual({ + body: JSON.stringify({ + count: 1, + 'available-airnodes': [airnodeWallet.address], + }), + headers: { + 'access-control-allow-methods': '*', + 'access-control-allow-origin': '*', + 'cache-control': 'no-store', + 'cdn-cache-control': 'max-age=300', + 'content-type': 'application/json', + }, + statusCode: 200, + }); + }); +}); diff --git a/packages/api/src/handlers.ts b/packages/api/src/handlers.ts index 4e54fae1..ce6d714c 100644 --- a/packages/api/src/handlers.ts +++ b/packages/api/src/handlers.ts @@ -1,18 +1,19 @@ -import { isEmpty, isNil, last, omit, size } from 'lodash'; import { go, goSync } from '@api3/promise-utils'; -import { ApiRequest, ApiResponse, PromiseError, batchSignedDataSchema, evmAddressSchema } from './types'; +import { isEmpty, isNil, omit, size } from 'lodash'; +import { CACHE_HEADERS, COMMON_HEADERS } from './constants'; import { deriveBeaconId, recoverSignerAddress } from './evm'; -import { generateErrorResponse, isBatchUnique } from './utils'; -import { CACHE_HEADERS, COMMON_HEADERS, MAX_BATCH_SIZE } from './constants'; -import { getAll, getAllBy, getBy, putAll } from './in-memory-cache'; - -export const batchUpsertData = async (request: ApiRequest): Promise => { - if (isNil(request.body)) return generateErrorResponse(400, 'Invalid request, http body is missing'); - - const goJsonParseBody = goSync(() => JSON.parse(request.body)); - if (!goJsonParseBody.success) return generateErrorResponse(400, 'Invalid request, body must be in JSON'); - - const goValidateSchema = await go(() => batchSignedDataSchema.parseAsync(goJsonParseBody.data)); +import { getAll, getAllAirnodeAddresses, putAll } from './in-memory-cache'; +import { ApiResponse } from './types'; +import { generateErrorResponse, getConfig, isBatchUnique } from './utils'; +import { batchSignedDataSchema, evmAddressSchema } from './schema'; + +// Accepts a batch of signed data that is first validated for consistency and data integrity errors. If there is any +// issue during this step, the whole batch is rejected. +// +// Otherwise, each data is inserted to the storage even though they might already be more fresh data. This might be +// important for the delayed endpoint which may not be allowed to return the fresh data yet. +export const batchInsertData = async (requestBody: unknown): Promise => { + const goValidateSchema = await go(() => batchSignedDataSchema.parseAsync(requestBody)); if (!goValidateSchema.success) return generateErrorResponse( 400, @@ -20,77 +21,45 @@ export const batchUpsertData = async (request: ApiRequest): Promise goValidateSchema.error.message ); + // Ensure there is at least one signed data to push const batchSignedData = goValidateSchema.data; - - /* - The following validations behave similarly to Promise.all. - If any of the validations fail, the entire batch will be dropped. - This approach ensures consistent processing of the batch, - preventing partial or inconsistent results. - */ - - // Phase 1: Check whether batch is empty if (isEmpty(batchSignedData)) return generateErrorResponse(400, 'No signed data to push'); - // Phase 2: Check whether the size of batch exceeds a maximum batch size - if (size(batchSignedData) > MAX_BATCH_SIZE) - return generateErrorResponse(400, `Maximum batch size (${MAX_BATCH_SIZE}) exceeded`); + // Check whether the size of batch exceeds a maximum batch size + const { maxBatchSize } = getConfig(); + if (size(batchSignedData) > maxBatchSize) + return generateErrorResponse(400, `Maximum batch size (${maxBatchSize}) exceeded`); - // Phase 3: Check whether any duplications exist + // Check whether any duplications exist if (!isBatchUnique(batchSignedData)) return generateErrorResponse(400, 'No duplications are allowed'); - // Phase 4: Check validations that can be done without using http request, returns fail response in first error - const phase4Promises = batchSignedData.map(async (signedData) => { + // Check validations that can be done without using http request, returns fail response in first error + const signedDataValidationResults = batchSignedData.map((signedData) => { const goRecoverSigner = goSync(() => recoverSignerAddress(signedData)); if (!goRecoverSigner.success) - return Promise.reject( - generateErrorResponse(400, 'Unable to recover signer address', goRecoverSigner.error.message, signedData) - ); + return generateErrorResponse(400, 'Unable to recover signer address', goRecoverSigner.error.message, signedData); if (signedData.airnode !== goRecoverSigner.data) - return Promise.reject(generateErrorResponse(400, 'Signature is invalid', undefined, signedData)); + return generateErrorResponse(400, 'Signature is invalid', undefined, signedData); const goDeriveBeaconId = goSync(() => deriveBeaconId(signedData.airnode, signedData.templateId)); if (!goDeriveBeaconId.success) - return Promise.reject( - generateErrorResponse( - 400, - 'Unable to derive beaconId by given airnode and templateId', - goDeriveBeaconId.error.message, - signedData - ) + return generateErrorResponse( + 400, + 'Unable to derive beaconId by given airnode and templateId', + goDeriveBeaconId.error.message, + signedData ); if (signedData.beaconId !== goDeriveBeaconId.data) - return Promise.reject(generateErrorResponse(400, 'beaconId is invalid', undefined, signedData)); - }); - - const goPhase4Results = await go>(() => Promise.all(phase4Promises)); - if (!goPhase4Results.success) return goPhase4Results.error.reason; - - // Phase 5: Get current signed data to compare timestamp, returns fail response in first error - const phase5Promises = batchSignedData.map(async (signedData) => { - const goReadDb = await go(() => getBy(signedData.airnode, signedData.templateId)); - if (!goReadDb.success) - return Promise.reject( - generateErrorResponse( - 500, - 'Unable to get signed data from database to validate timestamp', - goReadDb.error.message, - signedData - ) - ); - if (isNil(goReadDb.data)) return; + return generateErrorResponse(400, 'beaconId is invalid', undefined, signedData); - const lastDbSignedData = goReadDb.data[goReadDb.data.length - 1]!; - if (parseInt(signedData.timestamp) <= parseInt(lastDbSignedData.timestamp)) - return Promise.reject(generateErrorResponse(400, "Request isn't updating the timestamp", undefined, signedData)); + return null; }); + const firstError = signedDataValidationResults.find(Boolean); + if (firstError) return firstError; - const goPhase5Results = await go>(() => Promise.all(phase5Promises)); - if (!goPhase5Results.success) return goPhase5Results.error.reason; - - // Phase 6: Write batch of validated data to the database + // Write batch of validated data to the database const goBatchWriteDb = await go(() => putAll(batchSignedData)); if (!goBatchWriteDb.success) return generateErrorResponse(500, 'Unable to send batch of signed data to database', goBatchWriteDb.error.message); @@ -98,21 +67,24 @@ export const batchUpsertData = async (request: ApiRequest): Promise return { statusCode: 201, headers: COMMON_HEADERS, body: JSON.stringify({ count: batchSignedData.length }) }; }; -export const getData = async (request: ApiRequest): Promise => { - const airnode = request.queryParams.airnode; - if (isNil(airnode)) return generateErrorResponse(400, 'Invalid request, path parameter airnode address is missing'); +// Returns the most fresh signed data for each templateId for the given airnode address. The API can be delayed, which +// filter out all signed data that happend in the specified "delaySeconds" parameter (essentially, such signed data is +// treated as non-existant). +export const getData = async (airnodeAddress: string, delaySeconds: number): Promise => { + if (isNil(airnodeAddress)) + // TODO: Change error messages, avoid "path parameter" + return generateErrorResponse(400, 'Invalid request, path parameter airnode address is missing'); - const goValidateSchema = await go(() => evmAddressSchema.parseAsync(airnode)); + const goValidateSchema = await go(() => evmAddressSchema.parseAsync(airnodeAddress)); if (!goValidateSchema.success) return generateErrorResponse(400, 'Invalid request, path parameter must be an EVM address'); - const goReadDb = await go(() => getAllBy(airnode)); + const ignoreAfterTimestamp = Math.floor(Date.now() / 1000 - delaySeconds); + const goReadDb = await go(() => getAll(airnodeAddress, ignoreAfterTimestamp)); if (!goReadDb.success) return generateErrorResponse(500, 'Unable to get signed data from database', goReadDb.error.message); - // Transform array of signed data to be in form {[beaconId]: SignedData} - const data = goReadDb.data.reduce((acc, signedDataArray) => { - const signedData = last(signedDataArray)!; // Guaranteed to exist because we never remove data. TODO: This might not be true after we do removals. + const data = goReadDb.data.reduce((acc, signedData) => { return { ...acc, [signedData.beaconId]: omit(signedData, 'beaconId') }; }, {}); @@ -123,14 +95,17 @@ export const getData = async (request: ApiRequest): Promise => { }; }; -export const listAirnodeAddresses = async (_request: ApiRequest): Promise => { - const goScanDb = await go(() => getAll()); - if (!goScanDb.success) return generateErrorResponse(500, 'Unable to scan database', goScanDb.error.message); - - const airnodeAddresses = Object.keys(goScanDb.data); +// Returns all airnode addresses for which there is data. Note, that the delayed endpoint may not be allowed to show +// it. +export const listAirnodeAddresses = async (): Promise => { + const goAirnodeAddresses = await go(() => getAllAirnodeAddresses()); + if (!goAirnodeAddresses.success) + return generateErrorResponse(500, 'Unable to scan database', goAirnodeAddresses.error.message); + const airnodeAddresses = goAirnodeAddresses.data; return { statusCode: 200, + // TODO: The cache headers should be configurable via config file, esp. the max-age property. headers: { ...COMMON_HEADERS, ...CACHE_HEADERS, 'cdn-cache-control': 'max-age=300' }, body: JSON.stringify({ count: airnodeAddresses.length, 'available-airnodes': airnodeAddresses }), }; diff --git a/packages/api/src/in-memory-cache.ts b/packages/api/src/in-memory-cache.ts index ad2dd15c..c9afd811 100644 --- a/packages/api/src/in-memory-cache.ts +++ b/packages/api/src/in-memory-cache.ts @@ -1,29 +1,58 @@ -import { SignedData } from './types'; +import { last } from 'lodash'; +import { isIgnored } from './utils'; +import { SignedData } from './schema'; +import { getCache } from './cache'; -const signedDataCache: Record> = {}; +// TODO: Tests + +const ignoreTooFreshData = (signedDatas: SignedData[], ignoreAfterTimestamp: number) => + signedDatas.filter((data) => !isIgnored(data, ignoreAfterTimestamp)); // The API is deliberately asynchronous to mimic a database call. -export const getBy = async (airnodeId: string, templateId: string) => { +export const get = async (airnodeId: string, templateId: string, ignoreAfterTimestamp: number) => { + const signedDataCache = getCache(); if (!signedDataCache[airnodeId]) return null; - return signedDataCache[airnodeId]![templateId] ?? null; + const signedDatas = signedDataCache[airnodeId]![templateId]; + if (!signedDatas) return null; + + return last(ignoreTooFreshData(signedDatas, ignoreAfterTimestamp)) ?? null; }; // The API is deliberately asynchronous to mimic a database call. -export const getAllBy = async (airnodeId: string) => { - return Object.values(signedDataCache[airnodeId] ?? {}); +export const getAll = async (airnodeId: string, ignoreAfterTimestamp: number) => { + const signedDataCache = getCache(); + const signedDataByTemplateId = signedDataCache[airnodeId] ?? {}; + const freshestSignedData: SignedData[] = []; + for (const templateId of Object.keys(signedDataByTemplateId)) { + const freshest = await get(airnodeId, templateId, ignoreAfterTimestamp); + if (freshest) freshestSignedData.push(freshest); + } + + return freshestSignedData; }; // The API is deliberately asynchronous to mimic a database call. -export const getAll = async () => signedDataCache; +// +// The Airnode addresses are returned independently of how old the data is. This means that an API can get all Airnode +// addresses and then use a delayed endpoint to get data from each, but fail to get data from some of them. +export const getAllAirnodeAddresses = async () => Object.keys(getCache()); // The API is deliberately asynchronous to mimic a database call. export const put = async (signedData: SignedData) => { + const signedDataCache = getCache(); const { airnode, templateId } = signedData; signedDataCache[airnode] ??= {}; signedDataCache[airnode]![templateId] ??= []; - signedDataCache[airnode]![templateId]!.push(signedData); + + // We need to insert the signed data in the correct position in the array based on the timestamp. It would be more + // efficient to use a priority queue, but the proper solution is not to store the data in memory. + const signedDatas = signedDataCache[airnode]![templateId]!; + const index = signedDatas.findIndex((data) => parseInt(data.timestamp) > parseInt(signedData.timestamp)); + if (index < 0) signedDatas.push(signedData); + else signedDatas.splice(index, 0, signedData); }; +// TODO: Tests that it inserts in correct order. // The API is deliberately asynchronous to mimic a database call. export const putAll = async (signedDataArray: SignedData[]) => { for (const signedData of signedDataArray) await put(signedData); diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 7b0973b6..1fe84682 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -4,7 +4,6 @@ // You can check how this works by following the demo from https://github.com/evanw/node-source-map-support#demos. Just // create a test script with/without the source map support, build the project and run the built script using node. import 'source-map-support/register'; - import { startServer } from './server'; startServer(); diff --git a/packages/api/src/schema.test.ts b/packages/api/src/schema.test.ts new file mode 100644 index 00000000..8881fbca --- /dev/null +++ b/packages/api/src/schema.test.ts @@ -0,0 +1,31 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { ZodError } from 'zod'; +import { configSchema, endpointSchema } from './schema'; + +describe('endpointSchema', () => { + it('validates urlPath', () => { + const expectedError = new ZodError([ + { + validation: 'regex', + code: 'invalid_string', + message: 'Must start with a slash and contain only alphanumeric characters and dashes', + path: ['urlPath'], + }, + ]); + expect(() => endpointSchema.parse({ urlPath: '', delaySeconds: 0 })).toThrow(expectedError); + expect(() => endpointSchema.parse({ urlPath: '/', delaySeconds: 0 })).toThrow(expectedError); + expect(() => endpointSchema.parse({ urlPath: 'url-path', delaySeconds: 0 })).toThrow(expectedError); + expect(() => endpointSchema.parse({ urlPath: 'url-path', delaySeconds: 0 })).toThrow(expectedError); + + expect(() => endpointSchema.parse({ urlPath: '/url-path', delaySeconds: 0 })).not.toThrow(); + }); +}); + +describe('configSchema', () => { + it('validates example config', () => { + const config = JSON.parse(readFileSync(join(__dirname, '../config/signed-api.example.json'), 'utf8')); + + expect(() => configSchema.parse(config)).not.toThrow(); + }); +}); diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts new file mode 100644 index 00000000..ea271eab --- /dev/null +++ b/packages/api/src/schema.ts @@ -0,0 +1,42 @@ +import { z } from 'zod'; + +export const endpointSchema = z + .object({ + urlPath: z + .string() + .regex(/^\/[a-zA-Z0-9\-]+$/, 'Must start with a slash and contain only alphanumeric characters and dashes'), + delaySeconds: z.number().nonnegative().int(), + }) + .strict(); + +export type Endpoint = z.infer; + +export const configSchema = z + .object({ + endpoints: z.array(endpointSchema), + maxBatchSize: z.number().nonnegative().int(), + port: z.number().nonnegative().int(), + }) + .strict(); + +export type Config = z.infer; + +// TODO: add tests for lines below, better error messages +export const evmAddressSchema = z.string().regex(/^0x[a-fA-F0-9]{40}$/); + +export const evmIdSchema = z.string().regex(/^0x[a-fA-F0-9]{64}$/); + +export const signedDataSchema = z.object({ + airnode: evmAddressSchema, + templateId: evmIdSchema, + beaconId: evmIdSchema, + timestamp: z.string(), + encodedValue: z.string(), + signature: z.string(), +}); + +export type SignedData = z.infer; + +export const batchSignedDataSchema = z.array(signedDataSchema); + +export type BatchSignedData = z.infer; diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index becd250c..dd53285c 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -1,12 +1,9 @@ -import * as dotenv from 'dotenv'; import express from 'express'; -import { getData, listAirnodeAddresses, batchUpsertData } from './handlers'; +import { getData, listAirnodeAddresses, batchInsertData } from './handlers'; +import { getConfig } from './utils'; export const startServer = () => { - dotenv.config(); - const { PORT } = process.env; - - const port = PORT; + const config = getConfig(); const app = express(); app.use(express.json()); @@ -15,37 +12,35 @@ export const startServer = () => { // eslint-disable-next-line no-console console.log('Received request "POST /"', req.body, req.params, req.query); - const result = await batchUpsertData({ - body: JSON.stringify(req.body), - queryParams: {}, - }); + // TODO: Test if this ensures the data is json + const result = await batchInsertData(req.body); res.status(result.statusCode).header(result.headers).send(result.body); }); - app.get('/:airnode', async (req, res) => { + app.get('/', async (_req, res) => { // eslint-disable-next-line no-console - console.log('Received request "GET /:airnode"', req.body, req.params, req.query); + console.log('Received request "GET /"'); - const result = await getData({ - body: '', - queryParams: { airnode: req.params.airnode }, - }); + const result = await listAirnodeAddresses(); res.status(result.statusCode).header(result.headers).send(result.body); }); - app.get('/', async (_req, res) => { + for (const endpoint of config.endpoints) { // eslint-disable-next-line no-console - console.log('Received request "GET /"'); + console.log('Registering endpoint', endpoint); + const { urlPath, delaySeconds } = endpoint; + + app.get(`${urlPath}/:airnodeAddress`, async (req, res) => { + // eslint-disable-next-line no-console + console.log('Received request "GET /:airnode"', req.body, req.params, req.query); - const result = await listAirnodeAddresses({ - body: '', - queryParams: {}, + const result = await getData(req.params.airnodeAddress, delaySeconds); + res.status(result.statusCode).header(result.headers).send(result.body); }); - res.status(result.statusCode).header(result.headers).send(result.body); - }); + } - app.listen(port, () => { + app.listen(config.port, () => { // eslint-disable-next-line no-console - console.log(`Server listening at http://localhost:${port}`); + console.log(`Server listening at http://localhost:${config.port}`); }); }; diff --git a/packages/api/src/types.ts b/packages/api/src/types.ts index 9bedb2fb..32113b7b 100644 --- a/packages/api/src/types.ts +++ b/packages/api/src/types.ts @@ -1,36 +1,5 @@ -import { z } from 'zod'; - -/** - * Common EVM Data Schema - */ -export const evmAddressSchema = z.string().regex(/^0x[a-fA-F0-9]{40}$/); -export const evmIdSchema = z.string().regex(/^0x[a-fA-F0-9]{64}$/); - -export const signedDataSchema = z.object({ - airnode: evmAddressSchema, - templateId: evmIdSchema, - beaconId: evmIdSchema, - timestamp: z.string(), - encodedValue: z.string(), - signature: z.string(), -}); - -export const batchSignedDataSchema = z.array(signedDataSchema); - -export type SignedData = z.infer; -export type BatchSignedData = z.infer; - -export interface PromiseError extends Error { - reason: T; -} - export interface ApiResponse { statusCode: number; headers: Record; body: string; } - -export interface ApiRequest { - body: string; - queryParams: Record; -} diff --git a/packages/api/src/utils.ts b/packages/api/src/utils.ts index ee9b41c2..e7511a2e 100644 --- a/packages/api/src/utils.ts +++ b/packages/api/src/utils.ts @@ -1,9 +1,16 @@ -import { ApiResponse, BatchSignedData } from './types'; +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { ApiResponse } from './types'; import { COMMON_HEADERS } from './constants'; +import { BatchSignedData, SignedData, configSchema } from './schema'; export const isBatchUnique = (batchSignedData: BatchSignedData) => batchSignedData.length === new Set(batchSignedData.map(({ airnode, templateId }) => airnode.concat(templateId))).size; +export const isIgnored = (signedData: SignedData, ignoreAfterTimestamp: number) => { + return parseInt(signedData.timestamp) > ignoreAfterTimestamp; +}; + export const generateErrorResponse = ( statusCode: number, message: string, @@ -12,3 +19,7 @@ export const generateErrorResponse = ( ): ApiResponse => { return { statusCode, headers: COMMON_HEADERS, body: JSON.stringify({ message, detail, extra }) }; }; + +// TODO: Ensure this works in a docker (need to mount the config folder). +export const getConfig = () => + configSchema.parse(JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8'))); diff --git a/packages/api/test/utils.ts b/packages/api/test/utils.ts new file mode 100644 index 00000000..7a5907e4 --- /dev/null +++ b/packages/api/test/utils.ts @@ -0,0 +1,45 @@ +import { ethers } from 'ethers'; +import { SignedData } from '../src/schema'; +import { deriveBeaconId } from '../src/evm'; + +export const deriveTemplateId = (endpointId: string, encodedParameters: string) => + ethers.utils.keccak256(ethers.utils.solidityPack(['bytes32', 'bytes'], [endpointId, encodedParameters])); + +export const generateRandomBytes = (len: number) => ethers.utils.hexlify(ethers.utils.randomBytes(len)); + +export const generateRandomWallet = () => ethers.Wallet.createRandom(); + +export const generateRandomEvmAddress = () => generateRandomWallet().address; + +export const generateDataSignature = (wallet: ethers.Wallet, templateId: string, timestamp: string, data: string) => { + return wallet.signMessage( + ethers.utils.arrayify( + ethers.utils.keccak256( + ethers.utils.solidityPack(['bytes32', 'uint256', 'bytes'], [templateId, timestamp, data || '0x']) + ) + ) + ); +}; + +export const createSignedData = async ( + overrides?: Partial & { airnodeWallet: ethers.Wallet }> +) => { + const airnodeWallet = overrides?.airnodeWallet ?? ethers.Wallet.createRandom(); + + const airnode = airnodeWallet.address; + const templateId = overrides?.templateId ?? generateRandomBytes(32); + const beaconId = overrides?.beaconId ?? deriveBeaconId(airnode, templateId); + const timestamp = overrides?.timestamp ?? Math.floor(Date.now() / 1000).toString(); + const encodedValue = overrides?.encodedValue ?? '0x00000000000000000000000000000000000000000000005718e3a22ce01f7a40'; + const signature = + overrides?.signature ?? (await generateDataSignature(airnodeWallet, templateId, timestamp, encodedValue)); + + return { + airnode, + templateId, + beaconId, + timestamp, + encodedValue, + signature, + }; +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 93b5784b..14a7ac53 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,9 +56,6 @@ importers: '@api3/promise-utils': specifier: 0.4.0 version: 0.4.0 - dotenv: - specifier: ^16.3.1 - version: 16.3.1 ethers: specifier: ^5.7.2 version: 5.7.2 From 9a0057903667b92ca3485dc26c6256656f1cb6ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 24 Sep 2023 11:00:28 +0200 Subject: [PATCH 03/12] Remove .DS_Store --- .dockerignore | 1 + .gitignore | 1 + packages/.DS_Store | Bin 6148 -> 0 bytes 3 files changed, 2 insertions(+) delete mode 100644 packages/.DS_Store diff --git a/.dockerignore b/.dockerignore index 4d043a3c..cd7710a8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,4 @@ **/pusher.json **/secrets.env **/signed-api.json +**/.DS_Store diff --git a/.gitignore b/.gitignore index aeff9f7c..e2bd78a1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ coverage pusher.json secrets.env signed-api.json +.DS_Store diff --git a/packages/.DS_Store b/packages/.DS_Store deleted file mode 100644 index 941feefd24bf5da2c753d1819aa501371792b4ad..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~O=`nH427SXECStlndNMHfZkvT$q90S1_~h%(o$&GbM!v_Y~prZOnCz7jWiav zzlFyFV7t%PXJ7)bp}XSA!_18N3Mahd@^$>UUEgkJR=f&4M9hqp3A6p$mWY4|h=2%) zfCwyzK%U~*JTB;&^e7@A0?Q!a--kwb?WH3%J{=4(0#Mg1hjAUV1hsjA+Dk_&D>SR= z!K&3_4DoujQ(IlvOGj$9!)o}jy0iHdL$hp$H6}FcAqpZO0y6@u%qKtp5A@&W|5=Mt z5fFiYM!?ql<9^4Nsv{d9s=gj{YFy56_Y=UxkK!#ojQhk From 45d11342055b17d2a5f60e513fb856b5c2f3ccf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 24 Sep 2023 11:05:37 +0200 Subject: [PATCH 04/12] Improve error messages --- packages/api/package.json | 1 - packages/api/src/handlers.test.ts | 2 +- packages/api/src/handlers.ts | 6 ++---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index ff28ad38..c0df7f4b 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -7,7 +7,6 @@ }, "main": "index.js", "scripts": { - "print-info": "which pnpm", "build": "tsc --project .", "clean": "rm -rf coverage dist", "dev": "pnpm ts-node src/index.ts", diff --git a/packages/api/src/handlers.test.ts b/packages/api/src/handlers.test.ts index 2819d241..446a58cf 100644 --- a/packages/api/src/handlers.test.ts +++ b/packages/api/src/handlers.test.ts @@ -64,7 +64,7 @@ describe(getData.name, () => { const result = await getData('0xInvalid', 0); expect(result).toEqual({ - body: JSON.stringify({ message: 'Invalid request, path parameter must be an EVM address' }), + body: JSON.stringify({ message: 'Invalid request, airnode address must be an EVM address' }), headers: { 'access-control-allow-methods': '*', 'access-control-allow-origin': '*', diff --git a/packages/api/src/handlers.ts b/packages/api/src/handlers.ts index ce6d714c..d93d6637 100644 --- a/packages/api/src/handlers.ts +++ b/packages/api/src/handlers.ts @@ -71,13 +71,11 @@ export const batchInsertData = async (requestBody: unknown): Promise => { - if (isNil(airnodeAddress)) - // TODO: Change error messages, avoid "path parameter" - return generateErrorResponse(400, 'Invalid request, path parameter airnode address is missing'); + if (isNil(airnodeAddress)) return generateErrorResponse(400, 'Invalid request, airnode address is missing'); const goValidateSchema = await go(() => evmAddressSchema.parseAsync(airnodeAddress)); if (!goValidateSchema.success) - return generateErrorResponse(400, 'Invalid request, path parameter must be an EVM address'); + return generateErrorResponse(400, 'Invalid request, airnode address must be an EVM address'); const ignoreAfterTimestamp = Math.floor(Date.now() / 1000 - delaySeconds); const goReadDb = await go(() => getAll(airnodeAddress, ignoreAfterTimestamp)); From 64e044531187244ff0c36aa1217f7726b59ca2ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 24 Sep 2023 11:07:50 +0200 Subject: [PATCH 05/12] Use cache value from config --- packages/api/config/signed-api.example.json | 5 ++++- packages/api/src/handlers.ts | 3 +-- packages/api/src/schema.ts | 3 +++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/api/config/signed-api.example.json b/packages/api/config/signed-api.example.json index ed999f0c..8c7d8394 100644 --- a/packages/api/config/signed-api.example.json +++ b/packages/api/config/signed-api.example.json @@ -10,5 +10,8 @@ } ], "maxBatchSize": 10, - "port": 8090 + "port": 8090, + "cache": { + "maxAgeSeconds": 300 + } } diff --git a/packages/api/src/handlers.ts b/packages/api/src/handlers.ts index d93d6637..f3bf662f 100644 --- a/packages/api/src/handlers.ts +++ b/packages/api/src/handlers.ts @@ -103,8 +103,7 @@ export const listAirnodeAddresses = async (): Promise => { return { statusCode: 200, - // TODO: The cache headers should be configurable via config file, esp. the max-age property. - headers: { ...COMMON_HEADERS, ...CACHE_HEADERS, 'cdn-cache-control': 'max-age=300' }, + headers: { ...COMMON_HEADERS, ...CACHE_HEADERS, 'cdn-cache-control': `max-age=${getConfig().cache.maxAgeSeconds}` }, body: JSON.stringify({ count: airnodeAddresses.length, 'available-airnodes': airnodeAddresses }), }; }; diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index ea271eab..e9fe3f8b 100644 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -16,6 +16,9 @@ export const configSchema = z endpoints: z.array(endpointSchema), maxBatchSize: z.number().nonnegative().int(), port: z.number().nonnegative().int(), + cache: z.object({ + maxAgeSeconds: z.number().nonnegative().int(), + }), }) .strict(); From f9b0c40d1290e2cf497238df02f4f657aa06df93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 24 Sep 2023 13:23:54 +0200 Subject: [PATCH 06/12] Implement in-memory-api tests --- jest.config.js | 2 +- packages/api/src/in-memory-cache.test.ts | 211 +++++++++++++++++++++++ packages/api/src/in-memory-cache.ts | 5 +- 3 files changed, 213 insertions(+), 5 deletions(-) create mode 100644 packages/api/src/in-memory-cache.test.ts diff --git a/jest.config.js b/jest.config.js index 5a8a0b4f..3d932fd0 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,7 +3,7 @@ * https://jestjs.io/docs/configuration */ module.exports = { - clearMocks: true, + restoreMocks: true, collectCoverage: true, coverageDirectory: 'coverage', coverageProvider: 'v8', diff --git a/packages/api/src/in-memory-cache.test.ts b/packages/api/src/in-memory-cache.test.ts new file mode 100644 index 00000000..b08a6ad0 --- /dev/null +++ b/packages/api/src/in-memory-cache.test.ts @@ -0,0 +1,211 @@ +import { groupBy } from 'lodash'; +import { get, getAll, getAllAirnodeAddresses, ignoreTooFreshData, put, putAll } from './in-memory-cache'; +import * as cacheModule from './cache'; +import { createSignedData, generateRandomWallet } from '../test/utils'; + +describe(ignoreTooFreshData.name, () => { + const createData = async () => [ + await createSignedData({ timestamp: '100' }), + await createSignedData({ timestamp: '199' }), + await createSignedData({ timestamp: '200' }), + await createSignedData({ timestamp: '201' }), + await createSignedData({ timestamp: '300' }), + ]; + + it('ignores all values with higher timestamp', async () => { + const data = await createData(); + + const result = ignoreTooFreshData(data, 200); + + expect(result).toEqual(data.slice(0, 3)); + }); + + it('returns all data when compared with infinity', async () => { + const data = await createData(); + + const result = ignoreTooFreshData(data, Infinity); + + expect(result).toEqual(data); + }); +}); + +describe(get.name, () => { + const mockCacheData = async () => { + const airnodeWallet = generateRandomWallet(); + const data = await createSignedData({ airnodeWallet, timestamp: '100' }); + const allData = [ + data, + await createSignedData({ airnodeWallet, templateId: data.templateId, timestamp: '101' }), + await createSignedData({ airnodeWallet, templateId: data.templateId, timestamp: '103' }), + ]; + jest.spyOn(cacheModule, 'getCache').mockReturnValueOnce({ + [data.airnode]: { [data.templateId]: allData }, + }); + + return allData; + }; + + it('returns null if there is no data', async () => { + const result = await get('non-existent-airnode', 'non-existent-template', 0); + + expect(result).toEqual(null); + }); + + it('returns null if data is too fresh', async () => { + const [data] = await mockCacheData(); + + const result = await get(data!.airnode, data!.templateId, 50); + + expect(result).toEqual(null); + }); + + it('returns the freshest non-ignored data', async () => { + const allData = await mockCacheData(); + const data = allData[0]!; + + const result = await get(data!.airnode, data!.templateId, 101); + + expect(result).toEqual(allData[1]); + expect(allData[1]!.timestamp).toEqual('101'); + }); + + it('returns the freshest data available to be returned', async () => { + const allData = await mockCacheData(); + const data = allData[0]!; + + const result = await get(data.airnode, data.templateId, Infinity); + + expect(result).toEqual(allData[2]); + expect(allData[2]!.timestamp).toEqual('103'); + }); +}); + +describe(getAll.name, () => { + const mockCacheData = async () => { + const airnodeWallet = generateRandomWallet(); + const data = await createSignedData({ airnodeWallet: airnodeWallet, timestamp: '100' }); + const allData = [ + data, + await createSignedData({ airnodeWallet: airnodeWallet, templateId: data.templateId, timestamp: '105' }), + await createSignedData({ airnodeWallet: airnodeWallet, timestamp: '300' }), + await createSignedData({ airnodeWallet: airnodeWallet, timestamp: '400' }), + ]; + + // Ned to use mockReturnValue instead of mockReturnValueOnce because the getCache call is used multiple times + // internally depending on the number of data inserted. + jest.spyOn(cacheModule, 'getCache').mockReturnValue({ + [data.airnode]: groupBy(allData, 'templateId'), + }); + + return allData; + }; + + it('returns freshest data for the given airnode', async () => { + const allData = await mockCacheData(); + + const result = await getAll(allData[0]!.airnode, Infinity); + + // The first data is overridden by the fresher (second) data. + expect(result).toEqual([allData[1], allData[2], allData[3]]); + }); + + it('returns freshest data for the given airnode respecting delay', async () => { + const allData = await mockCacheData(); + + const result = await getAll(allData[0]!.airnode, 100); + + expect(result).toEqual([allData[0]]); + }); +}); + +describe(getAllAirnodeAddresses.name, () => { + const mockCacheData = async () => { + const airnodeWallet1 = generateRandomWallet(); + const airnodeWallet2 = generateRandomWallet(); + const data1 = await createSignedData({ airnodeWallet: airnodeWallet1, timestamp: '100' }); + const data2 = await createSignedData({ airnodeWallet: airnodeWallet2, timestamp: '200' }); + const allData1 = [ + data1, + await createSignedData({ airnodeWallet: airnodeWallet1, templateId: data1.templateId, timestamp: '105' }), + ]; + const allData2 = [ + data2, + await createSignedData({ airnodeWallet: airnodeWallet2, templateId: data2.templateId, timestamp: '205' }), + ]; + const cache = { + [data1.airnode]: { [data1.templateId]: allData1 }, + [data2.airnode]: { [data2.templateId]: allData2 }, + }; + jest.spyOn(cacheModule, 'getCache').mockReturnValueOnce(cache); + + return cache; + }; + + it('returns all airnode addresses', async () => { + const cache = await mockCacheData(); + + const result = await getAllAirnodeAddresses(); + + expect(result).toEqual(Object.keys(cache)); + }); +}); + +describe(put.name, () => { + afterEach(() => { + cacheModule.setCache({}); + }); + + it('inserts the data in the correct position', async () => { + const airnodeWallet = generateRandomWallet(); + const data = await createSignedData({ airnodeWallet: airnodeWallet, timestamp: '100' }); + const allData = [ + data, + await createSignedData({ airnodeWallet: airnodeWallet, templateId: data.templateId, timestamp: '105' }), + await createSignedData({ airnodeWallet: airnodeWallet, templateId: data.templateId, timestamp: '110' }), + ]; + // We can't mock because the implementation mutates the cache value directly. + cacheModule.setCache({ + [data.airnode]: groupBy(allData, 'templateId'), + }); + const newData = await createSignedData({ + airnodeWallet, + templateId: data!.templateId, + timestamp: '103', + }); + + await put(newData); + + const cache = cacheModule.getCache(); + expect(cache[data!.airnode]![data!.templateId]).toEqual([allData[0], newData, allData[1], allData[2]]); + }); +}); + +describe(putAll.name, () => { + it('inserts the data in the correct positions', async () => { + const airnodeWallet = generateRandomWallet(); + const data = await createSignedData({ airnodeWallet: airnodeWallet, timestamp: '100' }); + const allData = [ + data, + await createSignedData({ airnodeWallet: airnodeWallet, templateId: data.templateId, timestamp: '105' }), + await createSignedData({ airnodeWallet: airnodeWallet, templateId: data.templateId, timestamp: '110' }), + ]; + // We can't mock because the implementation mutates the cache value directly. + cacheModule.setCache({ + [data.airnode]: groupBy(allData, 'templateId'), + }); + const newDataBatch = [ + await createSignedData({ + airnodeWallet, + templateId: data!.templateId, + timestamp: '103', + }), + await createSignedData(), + ]; + + await putAll(newDataBatch); + + const cache = cacheModule.getCache(); + expect(cache[data!.airnode]![data!.templateId]).toEqual([allData[0], newDataBatch[0], allData[1], allData[2]]); + expect(cache[newDataBatch[1]!.airnode]![newDataBatch[1]!.templateId]).toEqual([newDataBatch[1]]); + }); +}); diff --git a/packages/api/src/in-memory-cache.ts b/packages/api/src/in-memory-cache.ts index c9afd811..086e5741 100644 --- a/packages/api/src/in-memory-cache.ts +++ b/packages/api/src/in-memory-cache.ts @@ -3,9 +3,7 @@ import { isIgnored } from './utils'; import { SignedData } from './schema'; import { getCache } from './cache'; -// TODO: Tests - -const ignoreTooFreshData = (signedDatas: SignedData[], ignoreAfterTimestamp: number) => +export const ignoreTooFreshData = (signedDatas: SignedData[], ignoreAfterTimestamp: number) => signedDatas.filter((data) => !isIgnored(data, ignoreAfterTimestamp)); // The API is deliberately asynchronous to mimic a database call. @@ -52,7 +50,6 @@ export const put = async (signedData: SignedData) => { else signedDatas.splice(index, 0, signedData); }; -// TODO: Tests that it inserts in correct order. // The API is deliberately asynchronous to mimic a database call. export const putAll = async (signedDataArray: SignedData[]) => { for (const signedData of signedDataArray) await put(signedData); From 16a7a392160d4e10239564d5d29810d7824307f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 24 Sep 2023 13:25:33 +0200 Subject: [PATCH 07/12] Add better error messages --- packages/api/src/schema.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index e9fe3f8b..0d3448a7 100644 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -24,10 +24,9 @@ export const configSchema = z export type Config = z.infer; -// TODO: add tests for lines below, better error messages -export const evmAddressSchema = z.string().regex(/^0x[a-fA-F0-9]{40}$/); +export const evmAddressSchema = z.string().regex(/^0x[a-fA-F0-9]{40}$/, 'Must be a valid EVM address'); -export const evmIdSchema = z.string().regex(/^0x[a-fA-F0-9]{64}$/); +export const evmIdSchema = z.string().regex(/^0x[a-fA-F0-9]{64}$/, 'Must be a valid EVM hash'); export const signedDataSchema = z.object({ airnode: evmAddressSchema, From 349eacaf588b762e4c92025623d4ee716aaa3029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 24 Sep 2023 13:27:13 +0200 Subject: [PATCH 08/12] Verify that express json body parser works --- packages/api/src/server.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/src/server.ts b/packages/api/src/server.ts index dd53285c..6a295e1e 100644 --- a/packages/api/src/server.ts +++ b/packages/api/src/server.ts @@ -12,7 +12,6 @@ export const startServer = () => { // eslint-disable-next-line no-console console.log('Received request "POST /"', req.body, req.params, req.query); - // TODO: Test if this ensures the data is json const result = await batchInsertData(req.body); res.status(result.statusCode).header(result.headers).send(result.body); }); From 2ad4b6f4d9e9a686c1ad7a0f052d15964c11443e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 24 Sep 2023 13:52:38 +0200 Subject: [PATCH 09/12] Fix docker compose, edit documentation --- packages/api/README.md | 13 +++++++++---- packages/api/docker/docker-compose.yml | 15 +++++++++++++-- packages/api/src/utils.ts | 1 - 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/packages/api/README.md b/packages/api/README.md index ddca1824..54cad75d 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -35,14 +35,19 @@ pnpm run dev ## Docker -The API is also dockerized. In order to run the API from a docker, run: +The API is also dockerized. Docker needs to publish the port of the server (running inside the docker) to the port on +the host machine. By default, it expects the server is running on port `8090` and publishes it to the `8090` on the +host. To change this, you need to modify both `signed-api.json` configuration and use `PORT` environment variable with +the same value. + +In order to run the API from a docker, run: ```bash -# starts the API on port 4000 +# Starts the API on port 8090 pnpm run docker:start -# or in a detached mode +# Or in a detached mode pnpm run docker:detach:start -# optionally specify port +# Optionally specify port (also make sure the same port is specified inside `signed-api.json`) PORT=5123 pnpm run docker:start ``` diff --git a/packages/api/docker/docker-compose.yml b/packages/api/docker/docker-compose.yml index 9c606eae..a902f6ba 100644 --- a/packages/api/docker/docker-compose.yml +++ b/packages/api/docker/docker-compose.yml @@ -6,7 +6,18 @@ services: context: ../../../ dockerfile: ./packages/api/docker/Dockerfile ports: - - '${PORT:-4000}:${PORT:-4000}' + - '${PORT-8090}:${PORT:-8090}' environment: - NODE_ENV=production - - PORT=${PORT:-4000} + volumes: + # Mount the config file path to the container /dist/config folder where the API expects it to be. + # + # Environment variables specified in the docker-compose.yml file are applied to all operations that affect the + # service defined in the compose file, including the build process ("docker compose build"). Docker Compose will + # expect that variable to be defined in the environment where it's being run, and it will throw an error if it is + # not. For this reason we provide a default value. + # + # Docker Compose doesn't allow setting default values based on the current shell's environment, so we can't access + # current working directory and default to "$(pwd)/config". For this reason we default to the config folder + # relative to the docker-compose.yml file. + - ${CONFIG_PATH:-../config}:/usr/src/app/packages/api/dist/config diff --git a/packages/api/src/utils.ts b/packages/api/src/utils.ts index e7511a2e..36c351f5 100644 --- a/packages/api/src/utils.ts +++ b/packages/api/src/utils.ts @@ -20,6 +20,5 @@ export const generateErrorResponse = ( return { statusCode, headers: COMMON_HEADERS, body: JSON.stringify({ message, detail, extra }) }; }; -// TODO: Ensure this works in a docker (need to mount the config folder). export const getConfig = () => configSchema.parse(JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8'))); From 2c4e32f9d713fa8abda84f682f441816df1b9881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 24 Sep 2023 13:55:17 +0200 Subject: [PATCH 10/12] Remove unused dependency --- packages/api/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/package.json b/packages/api/package.json index c0df7f4b..2d5d7366 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -28,7 +28,6 @@ }, "dependencies": { "@api3/promise-utils": "0.4.0", - "dotenv": "^16.3.1", "ethers": "^5.7.2", "express": "^4.18.2", "lodash": "^4.17.21", From b4246382e5e496d9a8b2e8c5933467ec20f3ec39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 24 Sep 2023 19:12:04 +0200 Subject: [PATCH 11/12] Make each endpoint unique, document configuration --- packages/api/README.md | 26 ++++++++++++++++++++++++++ packages/api/src/schema.test.ts | 21 ++++++++++++++++++++- packages/api/src/schema.ts | 10 +++++++++- 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/packages/api/README.md b/packages/api/README.md index 54cad75d..c9fe5d9d 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -12,6 +12,32 @@ A service for storing and accessing signed data. It provides endpoints to handle TODO: Write example how to deploy on AWS (and maybe other cloud providers as well). +## Configuration + +The API is configured via `signed-api.json`. You can use this file to specify the port of the server, configure cache +header longevity or maximum batch size the API accepts. + +### Configuring endpoints + +The API needs to be configured with endpoints to be served. This is done via the `endpoints` section. For example: + +```json + "endpoints": [ + { + "urlPath": "/real-time", + "delaySeconds": 0 + }, + { + "urlPath": "/delayed", + "delaySeconds": 15 + } + ], +``` + +defines two endpoints. The `/real-time` serves the non-delayed data, the latter (`/delayed`) ignores all signed data +that has bee pushed in the last 15 seconds (configured by `delaySeconds` parameter). You can define multiple endpoints +as long as the `urlPath` is unique. + ## Usage The API provides the following endpoints: diff --git a/packages/api/src/schema.test.ts b/packages/api/src/schema.test.ts index 8881fbca..d073f206 100644 --- a/packages/api/src/schema.test.ts +++ b/packages/api/src/schema.test.ts @@ -1,7 +1,7 @@ import { readFileSync } from 'fs'; import { join } from 'path'; import { ZodError } from 'zod'; -import { configSchema, endpointSchema } from './schema'; +import { configSchema, endpointSchema, endpointsSchema } from './schema'; describe('endpointSchema', () => { it('validates urlPath', () => { @@ -22,6 +22,25 @@ describe('endpointSchema', () => { }); }); +describe('endpointsSchema', () => { + it('ensures each urlPath is unique', () => { + expect(() => + endpointsSchema.parse([ + { urlPath: '/url-path', delaySeconds: 0 }, + { urlPath: '/url-path', delaySeconds: 0 }, + ]) + ).toThrow( + new ZodError([ + { + code: 'custom', + message: 'Each "urlPath" of an endpoint must be unique', + path: [], + }, + ]) + ); + }); +}); + describe('configSchema', () => { it('validates example config', () => { const config = JSON.parse(readFileSync(join(__dirname, '../config/signed-api.example.json'), 'utf8')); diff --git a/packages/api/src/schema.ts b/packages/api/src/schema.ts index 0d3448a7..6a52b2f1 100644 --- a/packages/api/src/schema.ts +++ b/packages/api/src/schema.ts @@ -1,3 +1,4 @@ +import { uniqBy } from 'lodash'; import { z } from 'zod'; export const endpointSchema = z @@ -11,9 +12,16 @@ export const endpointSchema = z export type Endpoint = z.infer; +export const endpointsSchema = z + .array(endpointSchema) + .refine( + (endpoints) => uniqBy(endpoints, 'urlPath').length === endpoints.length, + 'Each "urlPath" of an endpoint must be unique' + ); + export const configSchema = z .object({ - endpoints: z.array(endpointSchema), + endpoints: endpointsSchema, maxBatchSize: z.number().nonnegative().int(), port: z.number().nonnegative().int(), cache: z.object({ From e1c1560839806fcd28921d8ffe83c58b2834f693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emanuel=20Tesa=C5=99?= Date: Sun, 24 Sep 2023 19:18:59 +0200 Subject: [PATCH 12/12] Fix tests --- packages/api/src/handlers.test.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/api/src/handlers.test.ts b/packages/api/src/handlers.test.ts index 446a58cf..50eb3c9d 100644 --- a/packages/api/src/handlers.test.ts +++ b/packages/api/src/handlers.test.ts @@ -1,5 +1,8 @@ +import { readFileSync } from 'fs'; +import { join } from 'path'; import { omit } from 'lodash'; import * as cacheModule from './cache'; +import * as utilsModule from './utils'; import { batchInsertData, getData, listAirnodeAddresses } from './handlers'; import { createSignedData, generateRandomWallet } from '../test/utils'; @@ -7,6 +10,12 @@ afterEach(() => { cacheModule.setCache({}); }); +beforeEach(() => { + jest + .spyOn(utilsModule, 'getConfig') + .mockImplementation(() => JSON.parse(readFileSync(join(__dirname, '../config/signed-api.example.json'), 'utf8'))); +}); + describe(batchInsertData.name, () => { it('drops the batch if it is invalid', async () => { const invalidData = await createSignedData({ signature: '0xInvalid' });