From b4f029a7afc8cbb07956dd576260e2a5745537f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pr=C3=A9vost?= <998369+prevostc@users.noreply.github.com> Date: Thu, 17 Oct 2024 01:28:19 +0200 Subject: [PATCH] Paginate harvests --- .vscode/settings.json | 3 + src/queries/VaultsHarvests.graphql | 28 +++++++--- src/queries/VaultsHarvestsFiltered.graphql | 27 ++++++--- src/routes/v1/vaults.ts | 64 ++++++++++++++++++++-- src/utils/sdk.ts | 4 +- 5 files changed, 100 insertions(+), 26 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 059b2c1..311303a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,5 +5,8 @@ }, "[typescript]": { "editor.defaultFormatter": "biomejs.biome" + }, + "[graphql]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" } } diff --git a/src/queries/VaultsHarvests.graphql b/src/queries/VaultsHarvests.graphql index 76c8525..563e5d7 100644 --- a/src/queries/VaultsHarvests.graphql +++ b/src/queries/VaultsHarvests.graphql @@ -1,5 +1,15 @@ -query VaultsHarvests($since: BigInt!, $first: Int = 1000, $skip: Int = 0) { - clms(first: $first, skip: $skip, where: { lifecycle_not: INITIALIZING }) { +query VaultsHarvests( + $since: BigInt! + $vaultsFirst: Int = 1000 + $vaultsSkip: Int = 0 + $harvestsFirst: Int = 1000 + $harvestsSkip: Int = 0 +) { + clms( + first: $vaultsFirst + skip: $vaultsSkip + where: { lifecycle_not: INITIALIZING } + ) { vaultAddress: id underlyingToken0 { decimals @@ -13,18 +23,17 @@ query VaultsHarvests($since: BigInt!, $first: Int = 1000, $skip: Int = 0) { harvests( orderBy: timestamp orderDirection: desc - first: 1000 + first: $harvestsFirst + skip: $harvestsSkip where: { timestamp_gte: $since } ) { ...ClmHarvestData } } classics( - first: $first - skip: $skip - where: { - lifecycle_not: INITIALIZING - } + first: $vaultsFirst + skip: $vaultsSkip + where: { lifecycle_not: INITIALIZING } ) { vaultAddress: id underlyingToken { @@ -36,7 +45,8 @@ query VaultsHarvests($since: BigInt!, $first: Int = 1000, $skip: Int = 0) { harvests( orderBy: timestamp orderDirection: desc - first: 1000 + first: $harvestsFirst + skip: $harvestsSkip where: { timestamp_gte: $since } ) { ...ClassicHarvestData diff --git a/src/queries/VaultsHarvestsFiltered.graphql b/src/queries/VaultsHarvestsFiltered.graphql index 4350676..cd1dd64 100644 --- a/src/queries/VaultsHarvestsFiltered.graphql +++ b/src/queries/VaultsHarvestsFiltered.graphql @@ -1,10 +1,16 @@ query VaultsHarvestsFiltered( $since: BigInt! - $first: Int = 1000 - $skip: Int = 0 $vaults: [Bytes!]! + $vaultsFirst: Int = 1000 + $vaultsSkip: Int = 0 + $harvestsFirst: Int = 1000 + $harvestsSkip: Int = 0 ) { - clms(first: $first, skip: $skip, where: { lifecycle_not: INITIALIZING, id_in: $vaults }) { + clms( + first: $vaultsFirst + skip: $vaultsSkip + where: { lifecycle_not: INITIALIZING, id_in: $vaults } + ) { vaultAddress: id underlyingToken0 { decimals @@ -18,16 +24,18 @@ query VaultsHarvestsFiltered( harvests( orderBy: timestamp orderDirection: desc - first: 1000 + first: $harvestsFirst + skip: $harvestsSkip where: { timestamp_gte: $since } ) { ...ClmHarvestData } } - classics(first: $first, skip: $skip, where: { - lifecycle_not: INITIALIZING, - id_in: $vaults, - }) { + classics( + first: $vaultsFirst + skip: $vaultsSkip + where: { lifecycle_not: INITIALIZING, id_in: $vaults } + ) { vaultAddress: id underlyingToken { decimals @@ -38,7 +46,8 @@ query VaultsHarvestsFiltered( harvests( orderBy: timestamp orderDirection: desc - first: 1000 + first: $harvestsFirst + skip: $harvestsSkip where: { timestamp_gte: $since } ) { ...ClassicHarvestData diff --git a/src/routes/v1/vaults.ts b/src/routes/v1/vaults.ts index 79c2a3b..6d4040a 100644 --- a/src/routes/v1/vaults.ts +++ b/src/routes/v1/vaults.ts @@ -1,6 +1,6 @@ import { type Static, Type } from '@sinclair/typebox'; import type { FastifyInstance, FastifyPluginOptions, FastifySchema } from 'fastify'; -import { max, sortedUniq } from 'lodash'; +import { groupBy, max, sortedUniq, values } from 'lodash'; import { type ChainId, chainIdSchema } from '../../config/chains'; import type { VaultsQuery } from '../../queries/codegen/sdk'; import { addressSchema } from '../../schema/address'; @@ -217,14 +217,66 @@ const getManyVaultsHarvests = async ( ): Promise => { const res = await Promise.all( getSdksForChain(chain).map(sdk => - vaults.length - ? sdk.VaultsHarvestsFiltered({ since: since.toString(), vaults }) - : sdk.VaultsHarvests({ since: since.toString() }) + paginate({ + fetchPage: ({ skip: vaultsSkip, first: vaultsFirst }) => + paginate({ + fetchPage: ({ skip: harvestsSkip, first: harvestsFirst }) => + vaults.length + ? sdk.VaultsHarvestsFiltered({ + since: since.toString(), + vaults, + vaultsSkip, + vaultsFirst, + harvestsSkip, + harvestsFirst, + }) + : sdk.VaultsHarvests({ + since: since.toString(), + vaultsSkip, + vaultsFirst, + harvestsSkip, + harvestsFirst, + }), + count: res => + max([ + ...res.data.clms.flatMap(clmPage => clmPage.harvests.length), + ...res.data.classics.flatMap(classicPage => classicPage.harvests.length), + ]) || 0, + }), + count: res => + max([ + ...res.flatMap(page => page.data.clms.length), + ...res.flatMap(page => page.data.classics.length), + ]) || 0, + }) ) ); - const rawClms = res.flatMap(chainRes => chainRes.data.clms); - const rawClassics = res.flatMap(chainRes => chainRes.data.classics); + const rawClms = values( + groupBy( + res.flatMap(chainRes => + chainRes.flatMap(vaultsPage => vaultsPage.flatMap(harvestsPage => harvestsPage.data.clms)) + ), + v => v.vaultAddress + ) + ).map(vaults => ({ + ...vaults[0], + harvests: vaults.flatMap(vault => vault.harvests), + })); + + const rawClassics = values( + groupBy( + res.flatMap(chainRes => + chainRes.flatMap(vaultsPage => + vaultsPage.flatMap(harvestsPage => harvestsPage.data.classics) + ) + ), + v => v.vaultAddress + ) + ).map(vaults => ({ + ...vaults[0], + harvests: vaults.flatMap(vault => vault.harvests), + })); const vaultsWithHarvests: ManyVaultsHarvests = []; rawClms.forEach(vault => { diff --git a/src/utils/sdk.ts b/src/utils/sdk.ts index ef1c49b..5fe203e 100644 --- a/src/utils/sdk.ts +++ b/src/utils/sdk.ts @@ -112,10 +112,10 @@ export async function paginate({ fetchAtMost = 10000, }: { fetchPage: (params: { skip: number; first: number }) => Promise; - count: (res: R) => number | number[]; + count: (res: NoInfer) => number | number[]; pageSize?: number; fetchAtMost?: number; -}): Promise { +}): Promise[]> { const results: R[] = []; let skip = 0; let fetched = 0;