Skip to content

Commit

Permalink
Fix lint issues in api
Browse files Browse the repository at this point in the history
  • Loading branch information
Siegrift committed Oct 4, 2023
1 parent 7e3f403 commit b97456d
Show file tree
Hide file tree
Showing 18 changed files with 113 additions and 93 deletions.
11 changes: 1 addition & 10 deletions .eslintrc.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,6 @@ module.exports = {
project: ['./tsconfig.json', './packages/*/tsconfig.json'],
},
rules: {
// TODO: Remove this once the eslint configuration is settled.
'multiline-comment-style': 'off',
'unicorn/no-array-callback-reference': 'off', // We prefer point free notation using "functional/prefer-tacit" rule.
'@typescript-eslint/unbound-method': 'off', // Reports issues for common patterns in tests (e.g. "expect(logger.warn)..."). Often the issue yields false positives.
'jest/no-hooks': [
'error', // Prefer using setup functions instead of beforeXXX hooks. AfterXyz are sometimes necessary (e.g. to reset Jest timers).
{
allow: ['afterEach', 'afterAll'],
},
],
'unicorn/consistent-function-scoping': 'off', // Disabling due to the rule's constraints conflicting with established patterns, especially in test suites where local helper or mocking functions are prevalent and do not necessitate exports.
},
};
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
"@types/node": "^20.8.0",
"@typescript-eslint/eslint-plugin": "^6.7.3",
"@typescript-eslint/parser": "^6.7.3",
"commons": "^1.0.2",
"commons": "^1.0.3",
"eslint": "^8.50.0",
"eslint-plugin-import": "^2.28.1",
"eslint-plugin-jest": "^27.4.2",
Expand Down
1 change: 1 addition & 0 deletions packages/api/jest.config.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
// eslint-disable-next-line @typescript-eslint/no-var-requires
const config = require('../../jest.config');

module.exports = {
Expand Down
2 changes: 1 addition & 1 deletion packages/api/src/cache.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { SignedData } from './schema';
import type { SignedData } from './schema';

type SignedDataCache = Record<
string, // Airnode ID.
Expand Down
25 changes: 14 additions & 11 deletions packages/api/src/config.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
import { readFileSync } from 'fs';
import { join } from 'path';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';

import { go } from '@api3/promise-utils';
import { S3 } from '@aws-sdk/client-s3';
import { logger } from './logger';
import { Config, configSchema } from './schema';

import { loadEnv } from './env';
import { logger } from './logger';
import { type Config, configSchema } from './schema';

let config: Config | undefined;

Expand All @@ -23,13 +25,14 @@ export const fetchAndCacheConfig = async (): Promise<Config> => {
const fetchConfig = async (): Promise<any> => {
const env = loadEnv();
const source = env.CONFIG_SOURCE;
if (!source || source === 'local') {
return JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8'));
}
if (source === 'aws-s3') {
return await fetchConfigFromS3();
switch (source) {
case 'local': {
return JSON.parse(readFileSync(join(__dirname, '../config/signed-api.json'), 'utf8'));
}
case 'aws-s3': {
return fetchConfigFromS3();
}
}
throw new Error(`Unable to load config CONFIG_SOURCE:${source}`);
};

const fetchConfigFromS3 = async (): Promise<any> => {
Expand All @@ -43,7 +46,7 @@ const fetchConfigFromS3 = async (): Promise<any> => {
};

logger.info(`Fetching config from AWS S3 region:${region}...`);
const res = await go(() => s3.getObject(params), { retries: 1 });
const res = await go(async () => s3.getObject(params), { retries: 1 });
if (!res.success) {
logger.error('Error fetching config from AWS S3:', res.error);
throw res.error;
Expand Down
6 changes: 4 additions & 2 deletions packages/api/src/env.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { join } from 'path';
import { join } from 'node:path';

import dotenv from 'dotenv';
import { EnvConfig, envConfigSchema } from './schema';

import { type EnvConfig, envConfigSchema } from './schema';

let env: EnvConfig | undefined;

Expand Down
3 changes: 2 additions & 1 deletion packages/api/src/evm.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { ethers } from 'ethers';
import { SignedData } from './schema';

import type { SignedData } from './schema';

export const decodeData = (data: string) => ethers.utils.defaultAbiCoder.decode(['int256'], data);

Expand Down
9 changes: 5 additions & 4 deletions packages/api/src/handlers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,17 @@ import * as cacheModule from './cache';
import * as configModule from './config';
import { batchInsertData, getData, listAirnodeAddresses } from './handlers';

afterEach(() => {
cacheModule.setCache({});
});

// eslint-disable-next-line jest/no-hooks
beforeEach(() => {
jest
.spyOn(configModule, 'getConfig')
.mockImplementation(() => JSON.parse(readFileSync(join(__dirname, '../config/signed-api.example.json'), 'utf8')));
});

afterEach(() => {
cacheModule.setCache({});
});

describe(batchInsertData.name, () => {
it('drops the batch if it is invalid', async () => {
const invalidData = await createSignedData({ signature: '0xInvalid' });
Expand Down
78 changes: 45 additions & 33 deletions packages/api/src/handlers.ts
Original file line number Diff line number Diff line change
@@ -1,76 +1,85 @@
import { go, goSync } from '@api3/promise-utils';
import { isEmpty, isNil, omit, size } from 'lodash';

import { getConfig } from './config';
import { CACHE_HEADERS, COMMON_HEADERS } from './constants';
import { deriveBeaconId, recoverSignerAddress } from './evm';
import { getAll, getAllAirnodeAddresses, prune, putAll } from './in-memory-cache';
import { ApiResponse } from './types';
import { generateErrorResponse, isBatchUnique } from './utils';
import { batchSignedDataSchema, evmAddressSchema } from './schema';
import { getConfig } from './config';
import type { ApiResponse } from './types';
import { generateErrorResponse, isBatchUnique } from './utils';

// Accepts a batch of signed data that is first validated for consistency and data integrity errors. If there is any
// issue during this step, the whole batch is rejected.
//
// Otherwise, each data is inserted to the storage even though they might already be more fresh data. This might be
// important for the delayed endpoint which may not be allowed to return the fresh data yet.
export const batchInsertData = async (requestBody: unknown): Promise<ApiResponse> => {
const goValidateSchema = await go(() => batchSignedDataSchema.parseAsync(requestBody));
if (!goValidateSchema.success)
return generateErrorResponse(
const goValidateSchema = await go(async () => batchSignedDataSchema.parseAsync(requestBody));
if (!goValidateSchema.success) {
return generateErrorResponse(
400,
'Invalid request, body must fit schema for batch of signed data',
goValidateSchema.error.message
);
);
}

// Ensure there is at least one signed data to push
const batchSignedData = goValidateSchema.data;
if (isEmpty(batchSignedData)) return generateErrorResponse(400, 'No signed data to push');

// Check whether the size of batch exceeds a maximum batch size
const { maxBatchSize, endpoints } = getConfig();
if (size(batchSignedData) > maxBatchSize)
return generateErrorResponse(400, `Maximum batch size (${maxBatchSize}) exceeded`);
if (size(batchSignedData) > maxBatchSize) {
return generateErrorResponse(400, `Maximum batch size (${maxBatchSize}) exceeded`);
}

// Check whether any duplications exist
if (!isBatchUnique(batchSignedData)) return generateErrorResponse(400, 'No duplications are allowed');

// Check validations that can be done without using http request, returns fail response in first error
const signedDataValidationResults = batchSignedData.map((signedData) => {
const goRecoverSigner = goSync(() => recoverSignerAddress(signedData));
if (!goRecoverSigner.success)
return generateErrorResponse(400, 'Unable to recover signer address', goRecoverSigner.error.message, signedData);
if (!goRecoverSigner.success) {
return generateErrorResponse(400, 'Unable to recover signer address', goRecoverSigner.error.message, signedData);
}

if (signedData.airnode !== goRecoverSigner.data)
return generateErrorResponse(400, 'Signature is invalid', undefined, signedData);
if (signedData.airnode !== goRecoverSigner.data) {
return generateErrorResponse(400, 'Signature is invalid', undefined, signedData);
}

const goDeriveBeaconId = goSync(() => deriveBeaconId(signedData.airnode, signedData.templateId));
if (!goDeriveBeaconId.success)
return generateErrorResponse(
if (!goDeriveBeaconId.success) {
return generateErrorResponse(
400,
'Unable to derive beaconId by given airnode and templateId',
goDeriveBeaconId.error.message,
signedData
);
);
}

if (signedData.beaconId !== goDeriveBeaconId.data)
return generateErrorResponse(400, 'beaconId is invalid', undefined, signedData);
if (signedData.beaconId !== goDeriveBeaconId.data) {
return generateErrorResponse(400, 'beaconId is invalid', undefined, signedData);
}

return null;
});
const firstError = signedDataValidationResults.find(Boolean);
if (firstError) return firstError;

// Write batch of validated data to the database
const goBatchWriteDb = await go(() => putAll(batchSignedData));
if (!goBatchWriteDb.success)
return generateErrorResponse(500, 'Unable to send batch of signed data to database', goBatchWriteDb.error.message);
const goBatchWriteDb = await go(async () => putAll(batchSignedData));
if (!goBatchWriteDb.success) {
return generateErrorResponse(500, 'Unable to send batch of signed data to database', goBatchWriteDb.error.message);
}

// Prune the cache with the data that is too old (no endpoint will ever return it)
const maxDelay = endpoints.reduce((acc, endpoint) => Math.max(acc, endpoint.delaySeconds), 0);
const maxIgnoreAfterTimestamp = Math.floor(Date.now() / 1000 - maxDelay);
const goPruneCache = await go(() => prune(batchSignedData, maxIgnoreAfterTimestamp));
if (!goPruneCache.success)
return generateErrorResponse(500, 'Unable to remove outdated cache data', goPruneCache.error.message);
const goPruneCache = await go(async () => prune(batchSignedData, maxIgnoreAfterTimestamp));
if (!goPruneCache.success) {
return generateErrorResponse(500, 'Unable to remove outdated cache data', goPruneCache.error.message);
}

return { statusCode: 201, headers: COMMON_HEADERS, body: JSON.stringify({ count: batchSignedData.length }) };
};
Expand All @@ -81,14 +90,16 @@ export const batchInsertData = async (requestBody: unknown): Promise<ApiResponse
export const getData = async (airnodeAddress: string, delaySeconds: number): Promise<ApiResponse> => {
if (isNil(airnodeAddress)) return generateErrorResponse(400, 'Invalid request, airnode address is missing');

const goValidateSchema = await go(() => evmAddressSchema.parseAsync(airnodeAddress));
if (!goValidateSchema.success)
return generateErrorResponse(400, 'Invalid request, airnode address must be an EVM address');
const goValidateSchema = await go(async () => evmAddressSchema.parseAsync(airnodeAddress));
if (!goValidateSchema.success) {
return generateErrorResponse(400, 'Invalid request, airnode address must be an EVM address');
}

const ignoreAfterTimestamp = Math.floor(Date.now() / 1000 - delaySeconds);
const goReadDb = await go(() => getAll(airnodeAddress, ignoreAfterTimestamp));
if (!goReadDb.success)
return generateErrorResponse(500, 'Unable to get signed data from database', goReadDb.error.message);
const goReadDb = await go(async () => getAll(airnodeAddress, ignoreAfterTimestamp));
if (!goReadDb.success) {
return generateErrorResponse(500, 'Unable to get signed data from database', goReadDb.error.message);
}

const data = goReadDb.data.reduce((acc, signedData) => {
return { ...acc, [signedData.beaconId]: omit(signedData, 'beaconId') };
Expand All @@ -104,9 +115,10 @@ export const getData = async (airnodeAddress: string, delaySeconds: number): Pro
// Returns all airnode addresses for which there is data. Note, that the delayed endpoint may not be allowed to show
// it.
export const listAirnodeAddresses = async (): Promise<ApiResponse> => {
const goAirnodeAddresses = await go(() => getAllAirnodeAddresses());
if (!goAirnodeAddresses.success)
return generateErrorResponse(500, 'Unable to scan database', goAirnodeAddresses.error.message);
const goAirnodeAddresses = await go(async () => getAllAirnodeAddresses());
if (!goAirnodeAddresses.success) {
return generateErrorResponse(500, 'Unable to scan database', goAirnodeAddresses.error.message);
}
const airnodeAddresses = goAirnodeAddresses.data;

return {
Expand Down
13 changes: 9 additions & 4 deletions packages/api/src/in-memory-cache.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
import { last, uniqBy } from 'lodash';
import { isIgnored } from './utils';
import { SignedData } from './schema';

import { getCache } from './cache';
import { logger } from './logger';
import type { SignedData } from './schema';
import { isIgnored } from './utils';

export const ignoreTooFreshData = (signedDatas: SignedData[], ignoreAfterTimestamp: number) =>
signedDatas.filter((data) => !isIgnored(data, ignoreAfterTimestamp));

// The API is deliberately asynchronous to mimic a database call.
// eslint-disable-next-line @typescript-eslint/require-await
export const get = async (airnodeId: string, templateId: string, ignoreAfterTimestamp: number) => {
logger.debug('Getting signed data', { airnodeId, templateId, ignoreAfterTimestamp });

Expand Down Expand Up @@ -38,25 +40,27 @@ export const getAll = async (airnodeId: string, ignoreAfterTimestamp: number) =>
//
// The Airnode addresses are returned independently of how old the data is. This means that an API can get all Airnode
// addresses and then use a delayed endpoint to get data from each, but fail to get data from some of them.
// eslint-disable-next-line @typescript-eslint/require-await
export const getAllAirnodeAddresses = async () => {
logger.debug('Getting all Airnode addresses');

return Object.keys(getCache());
};

// The API is deliberately asynchronous to mimic a database call.
// eslint-disable-next-line @typescript-eslint/require-await
export const put = async (signedData: SignedData) => {
logger.debug('Putting signed data', { signedData });

const signedDataCache = getCache();
const { airnode, templateId } = signedData;
const { airnode, templateId, timestamp } = signedData;
signedDataCache[airnode] ??= {};
signedDataCache[airnode]![templateId] ??= [];

// We need to insert the signed data in the correct position in the array based on the timestamp. It would be more
// efficient to use a priority queue, but the proper solution is not to store the data in memory.
const signedDatas = signedDataCache[airnode]![templateId]!;
const index = signedDatas.findIndex((data) => parseInt(data.timestamp) > parseInt(signedData.timestamp));
const index = signedDatas.findIndex((data) => Number.parseInt(data.timestamp, 10) > Number.parseInt(timestamp, 10));
if (index < 0) signedDatas.push(signedData);
else signedDatas.splice(index, 0, signedData);
};
Expand All @@ -73,6 +77,7 @@ export const putAll = async (signedDataArray: SignedData[]) => {
// Removes all signed data that is no longer needed to be kept in memory (because it is too old and there exist a newer
// signed data for each endpoint). The function is intended to be called after each insertion of new signed data for
// performance reasons, because it only looks to prune the data that for beacons that have been just inserted.
// eslint-disable-next-line @typescript-eslint/require-await
export const prune = async (signedDataArray: SignedData[], maxIgnoreAfterTimestamp: number) => {
const beaconsToPrune = uniqBy(signedDataArray, 'beaconId');
logger.debug('Pruning signed data', { maxIgnoreAfterTimestamp });
Expand Down
10 changes: 5 additions & 5 deletions packages/api/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import { startServer } from './server';
import { logger } from './logger';
import { fetchAndCacheConfig } from './config';
import { logger } from './logger';
import { startServer } from './server';

async function main() {
const main = async () => {
const config = await fetchAndCacheConfig();
logger.info('Using configuration', config);

startServer(config);
}
};

main();
void main();
1 change: 1 addition & 0 deletions packages/api/src/logger.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { createLogger, logConfigSchema } from 'signed-api/common';

import { loadEnv } from './env';

// We need to load the environment variables before we can use the logger. Because we want the logger to always be
Expand Down
10 changes: 6 additions & 4 deletions packages/api/src/schema.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import { readFileSync } from 'fs';
import { join } from 'path';
import { ZodError } from 'zod';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';

import dotenv from 'dotenv';
import { ZodError } from 'zod';

import { configSchema, endpointSchema, endpointsSchema, envBooleanSchema, envConfigSchema } from './schema';

describe('endpointSchema', () => {
Expand Down Expand Up @@ -96,7 +98,7 @@ describe('env config schema', () => {
expect(() => envConfigSchema.parse(env)).not.toThrow();
});

it('AWS_REGION is set when CONFIG_SOURCE is aws-s3', () => {
it('aWS_REGION is set when CONFIG_SOURCE is aws-s3', () => {
const env = {
CONFIG_SOURCE: 'aws-s3',
};
Expand Down
Loading

0 comments on commit b97456d

Please sign in to comment.