Paginate harvests

.vscode/settings.json

@@ -5,5 +5,8 @@
   },
   "[typescript]": {
     "editor.defaultFormatter": "biomejs.biome"
+  },
+  "[graphql]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
   }
 }

src/queries/VaultsHarvests.graphql

@@ -1,5 +1,15 @@
-query VaultsHarvests($since: BigInt!, $first: Int = 1000, $skip: Int = 0) {
-  clms(first: $first, skip: $skip, where: { lifecycle_not: INITIALIZING }) {
+query VaultsHarvests(
+  $since: BigInt!
+  $vaultsFirst: Int = 1000
+  $vaultsSkip: Int = 0
+  $harvestsFirst: Int = 1000
+  $harvestsSkip: Int = 0
+) {
+  clms(
+    first: $vaultsFirst
+    skip: $vaultsSkip
+    where: { lifecycle_not: INITIALIZING }
+  ) {
     vaultAddress: id
     underlyingToken0 {
       decimals
@@ -13,18 +23,17 @@ query VaultsHarvests($since: BigInt!, $first: Int = 1000, $skip: Int = 0) {
     harvests(
       orderBy: timestamp
       orderDirection: desc
-      first: 1000
+      first: $harvestsFirst
+      skip: $harvestsSkip
       where: { timestamp_gte: $since }
     ) {
       ...ClmHarvestData
     }
   }
   classics(
-    first: $first
-    skip: $skip
-    where: {
-      lifecycle_not: INITIALIZING
-    }
+    first: $vaultsFirst
+    skip: $vaultsSkip
+    where: { lifecycle_not: INITIALIZING }
   ) {
     vaultAddress: id
     underlyingToken {
@@ -36,7 +45,8 @@ query VaultsHarvests($since: BigInt!, $first: Int = 1000, $skip: Int = 0) {
     harvests(
       orderBy: timestamp
       orderDirection: desc
-      first: 1000
+      first: $harvestsFirst
+      skip: $harvestsSkip
       where: { timestamp_gte: $since }
     ) {
       ...ClassicHarvestData

src/queries/VaultsHarvestsFiltered.graphql

@@ -1,10 +1,16 @@
 query VaultsHarvestsFiltered(
   $since: BigInt!
-  $first: Int = 1000
-  $skip: Int = 0
   $vaults: [Bytes!]!
+  $vaultsFirst: Int = 1000
+  $vaultsSkip: Int = 0
+  $harvestsFirst: Int = 1000
+  $harvestsSkip: Int = 0
 ) {
-  clms(first: $first, skip: $skip, where: { lifecycle_not: INITIALIZING, id_in: $vaults }) {
+  clms(
+    first: $vaultsFirst
+    skip: $vaultsSkip
+    where: { lifecycle_not: INITIALIZING, id_in: $vaults }
+  ) {
     vaultAddress: id
     underlyingToken0 {
       decimals
@@ -18,16 +24,18 @@ query VaultsHarvestsFiltered(
     harvests(
       orderBy: timestamp
       orderDirection: desc
-      first: 1000
+      first: $harvestsFirst
+      skip: $harvestsSkip
       where: { timestamp_gte: $since }
     ) {
       ...ClmHarvestData
     }
   }
-  classics(first: $first, skip: $skip, where: { 
-    lifecycle_not: INITIALIZING, 
-    id_in: $vaults,
-  }) {
+  classics(
+    first: $vaultsFirst
+    skip: $vaultsSkip
+    where: { lifecycle_not: INITIALIZING, id_in: $vaults }
+  ) {
     vaultAddress: id
     underlyingToken {
       decimals
@@ -38,7 +46,8 @@ query VaultsHarvestsFiltered(
     harvests(
       orderBy: timestamp
       orderDirection: desc
-      first: 1000
+      first: $harvestsFirst
+      skip: $harvestsSkip
       where: { timestamp_gte: $since }
     ) {
       ...ClassicHarvestData

src/routes/v1/vaults.ts

@@ -1,6 +1,6 @@
 import { type Static, Type } from '@sinclair/typebox';
 import type { FastifyInstance, FastifyPluginOptions, FastifySchema } from 'fastify';
-import { max, sortedUniq } from 'lodash';
+import { groupBy, max, sortedUniq, values } from 'lodash';
 import { type ChainId, chainIdSchema } from '../../config/chains';
 import type { VaultsQuery } from '../../queries/codegen/sdk';
 import { addressSchema } from '../../schema/address';
@@ -217,14 +217,66 @@ const getManyVaultsHarvests = async (
 ): Promise<ManyVaultsHarvests> => {
   const res = await Promise.all(
     getSdksForChain(chain).map(sdk =>
-      vaults.length
-        ? sdk.VaultsHarvestsFiltered({ since: since.toString(), vaults })
-        : sdk.VaultsHarvests({ since: since.toString() })
+      paginate({
+        fetchPage: ({ skip: vaultsSkip, first: vaultsFirst }) =>
+          paginate({
+            fetchPage: ({ skip: harvestsSkip, first: harvestsFirst }) =>
+              vaults.length
+                ? sdk.VaultsHarvestsFiltered({
+                    since: since.toString(),
+                    vaults,
+                    vaultsSkip,
+                    vaultsFirst,
+                    harvestsSkip,
+                    harvestsFirst,
+                  })
+                : sdk.VaultsHarvests({
+                    since: since.toString(),
+                    vaultsSkip,
+                    vaultsFirst,
+                    harvestsSkip,
+                    harvestsFirst,
+                  }),
+            count: res =>
+              max([
+                ...res.data.clms.flatMap(clmPage => clmPage.harvests.length),
+                ...res.data.classics.flatMap(classicPage => classicPage.harvests.length),
+              ]) || 0,
+          }),
+        count: res =>
+          max([
+            ...res.flatMap(page => page.data.clms.length),
+            ...res.flatMap(page => page.data.classics.length),
+          ]) || 0,
+      })
     )
   );
 
-  const rawClms = res.flatMap(chainRes => chainRes.data.clms);
-  const rawClassics = res.flatMap(chainRes => chainRes.data.classics);
+  const rawClms = values(
+    groupBy(
+      res.flatMap(chainRes =>
+        chainRes.flatMap(vaultsPage => vaultsPage.flatMap(harvestsPage => harvestsPage.data.clms))
+      ),
+      v => v.vaultAddress
+    )
+  ).map(vaults => ({
+    ...vaults[0],
+    harvests: vaults.flatMap(vault => vault.harvests),
+  }));
+
+  const rawClassics = values(
+    groupBy(
+      res.flatMap(chainRes =>
+        chainRes.flatMap(vaultsPage =>
+          vaultsPage.flatMap(harvestsPage => harvestsPage.data.classics)
+        )
+      ),
+      v => v.vaultAddress
+    )
+  ).map(vaults => ({
+    ...vaults[0],
+    harvests: vaults.flatMap(vault => vault.harvests),
+  }));
   const vaultsWithHarvests: ManyVaultsHarvests = [];
 
   rawClms.forEach(vault => {

src/utils/sdk.ts

@@ -112,10 +112,10 @@ export async function paginate<R>({
   fetchAtMost = 10000,
 }: {
   fetchPage: (params: { skip: number; first: number }) => Promise<R>;
-  count: (res: R) => number | number[];
+  count: (res: NoInfer<R>) => number | number[];
   pageSize?: number;
   fetchAtMost?: number;
-}): Promise<R[]> {
+}): Promise<NoInfer<R>[]> {
   const results: R[] = [];
   let skip = 0;
   let fetched = 0;