chore: new merkl tree + new overDistributed check (#29)

* fix

* fix: tests

src/bot/runner.ts

@@ -102,8 +102,14 @@ export const checkHolderValidity: Step = async ({ onChainProvider }, report) =>
     const { startTree, endTree } = report;
     holdersReport = await validateHolders(onChainProvider, startTree, endTree);
     const negativeDiffs = holdersReport.negativeDiffs;
-
-    if (negativeDiffs.length > 0) throw negativeDiffs.join('\n');
+    const overDistributed = holdersReport.overDistributed;
+
+    if (negativeDiffs.length > 0) {
+      return Result.Error({ code: BotError.NegativeDiff, reason: negativeDiffs.join('\n'), report: { ...report, holdersReport } });
+    }
+    if (overDistributed.length > 0) {
+      return Result.Error({ code: BotError.OverDistributed, reason: overDistributed.join('\n'), report: { ...report, holdersReport } });
+    }
 
     return Result.Success({ ...report, holdersReport });
   } catch (reason) {

src/bot/validity.ts

@@ -1,6 +1,7 @@
 import { AggregatedRewardsType, Int256 } from '@angleprotocol/sdk';
 import { BigNumber } from 'ethers';
 
+import { HOUR } from '../constants';
 import { round } from '../helpers';
 import OnChainProvider from '../providers/on-chain/OnChainProvider';
 import { DistributionChanges, HolderClaims, HolderDetail, HoldersReport, UnclaimedRewards } from '../types/holders';
@@ -32,14 +33,18 @@ export async function validateHolders(
   endTree: AggregatedRewardsType
 ): Promise<HoldersReport> {
   const holders = gatherHolders(startTree, endTree);
-  const activeDistributions = await onChainProvider.fetchActiveDistributions();
+  const activeDistributions = await onChainProvider.fetchActiveDistributionsBetween(
+    startTree.lastUpdateEpoch * HOUR,
+    endTree.lastUpdateEpoch * HOUR
+  );
 
   const poolName = {};
 
   const details: HolderDetail[] = [];
   const changePerDistrib: DistributionChanges = {};
   const unclaimed: UnclaimedRewards = {};
   const negativeDiffs: string[] = [];
+  const overDistributed: string[] = [];
 
   for (const holder of holders) {
     unclaimed[holder] = {};
@@ -110,15 +115,30 @@ export async function validateHolders(
     l.percent = (l?.diff / changePerDistrib[l?.distribution]?.diff) * 100;
   }
 
-  return { details, changePerDistrib, unclaimed, negativeDiffs };
+  for (const k of Object.keys(changePerDistrib)) {
+    const solidityDist = activeDistributions?.find((d) => d.base.rewardId === k);
+
+    // Either the distributed amount is less than what would be distributed since the distrib start and there is no dis in the start tree
+    // Either it's less than what would be distributed since the startTree update
+    if (
+      (!!startTree.rewards[k]?.lastUpdateEpoch &&
+        changePerDistrib[k].epoch > endTree.rewards[k].lastUpdateEpoch - startTree.rewards[k].lastUpdateEpoch) ||
+      (!startTree.rewards[k]?.lastUpdateEpoch &&
+        changePerDistrib[k].epoch > endTree.rewards[k].lastUpdateEpoch - solidityDist.base.epochStart / HOUR)
+    ) {
+      overDistributed.push(k);
+    }
+  }
+
+  return { details, changePerDistrib, unclaimed, negativeDiffs, overDistributed };
 }
 
 export async function validateClaims(onChainProvider: OnChainProvider, holdersReport: HoldersReport): Promise<HoldersReport> {
   const { details, unclaimed } = holdersReport;
   const alreadyClaimed: HolderClaims = await onChainProvider.fetchClaimed(details);
 
   const overclaimed: string[] = [];
-  
+
   // Sort details by distribution and format numbers
   const expandedDetails = await Promise.all(
     details
@@ -128,7 +148,7 @@ export async function validateClaims(onChainProvider: OnChainProvider, holdersRe
       .map(async (d) => {
         const alreadyClaimedValue = round(Int256.from(alreadyClaimed[d.holder][d.tokenAddress], d.decimals).toNumber(), 2);
         const totalCumulated = round(unclaimed[d.holder][d.symbol].toNumber(), 2);
-        
+
         if (totalCumulated < alreadyClaimedValue) {
           overclaimed.push(`${d.holder}: ${alreadyClaimedValue} / ${totalCumulated} ${d.symbol}`);
         }

src/helpers/index.ts

@@ -98,7 +98,7 @@ export const buildMerklTree = (
   /**
    * 3 - Build the tree
    */
-  const elements = [];
+  const leaves = [];
   for (const u of users) {
     for (const t of tokens) {
       let sum = BigNumber.from(0);
@@ -108,14 +108,15 @@ export const buildMerklTree = (
           sum = sum?.add(distribution?.holders[u]?.amount.toString() ?? 0);
         }
       }
-      const hash = ethers.utils.keccak256(
-        ethers.utils.defaultAbiCoder.encode(['address', 'address', 'uint256'], [utils.getAddress(u), t, sum])
-      );
-
-      elements.push(hash);
+      if (!!sum && sum.gt(0)) {
+        const hash = ethers.utils.keccak256(
+          ethers.utils.defaultAbiCoder.encode(['address', 'address', 'uint256'], [utils.getAddress(u), t, sum])
+        );
+        leaves.push(hash);
+      }
     }
   }
-  const tree = new MerkleTree(elements, keccak256, { hashLeaves: false, sortPairs: true, sortLeaves: false });
+  const tree = new MerkleTree(leaves, keccak256, { hashLeaves: false, sortPairs: true, sortLeaves: false });
 
   return {
     tokens,

src/providers/on-chain/OnChainProvider.ts

@@ -28,6 +28,7 @@ export default abstract class OnChainProvider extends ExponentialBackoffProvider
   protected abstract onChainParams: () => Promise<OnChainParams>;
   protected abstract timestampAt: (blockNumber: number) => Promise<number>;
   protected abstract activeDistributions: () => Promise<ExtensiveDistributionParametersStructOutput[]>;
+  protected abstract activeDistributionsBetween: (start: number, end: number) => Promise<ExtensiveDistributionParametersStructOutput[]>;
   protected abstract poolName: (pool: string, amm: AMMType) => Promise<string>;
   protected abstract claimed: (holderDetails: HolderDetail[]) => Promise<HolderClaims>;
   protected abstract approve: (
@@ -73,6 +74,10 @@ export default abstract class OnChainProvider extends ExponentialBackoffProvider
     return this.retryWithExponentialBackoff(this.activeDistributions, this.fetchParams);
   }
 
+  async fetchActiveDistributionsBetween(start: number, end: number): Promise<ExtensiveDistributionParametersStructOutput[]> {
+    return this.retryWithExponentialBackoff(this.activeDistributionsBetween, this.fetchParams, start, end);
+  }
+
   async fetchOnChainParams(): Promise<OnChainParams> {
     return this.retryWithExponentialBackoff(this.onChainParams, this.fetchParams);
   }

src/providers/on-chain/RpcProvider.ts

@@ -67,6 +67,12 @@ export default class RpcProvider extends OnChainProvider {
     return instance.getActiveDistributions({ blockTag: this.blockNumber });
   };
 
+  override activeDistributionsBetween = async (start: number, end: number) => {
+    const instance = DistributionCreator__factory.connect(this.distributorCreator, this.provider);
+
+    return instance.getDistributionsBetweenEpochs(start, end, { blockTag: this.blockNumber });
+  };
+
   override poolName = async (pool: string, amm: AMMType) => {
     const multicall = Multicall__factory.connect('0xcA11bde05977b3631167028862bE2a173976CA11', this.provider);
     const poolInterface = PoolInterface(AMMAlgorithmMapping[amm]);

src/types/bot.ts

@@ -51,6 +51,7 @@ export enum BotError {
   TreeFetch,
   TreeRoot,
   NegativeDiff,
+  OverDistributed,
   AlreadyClaimed,
   KeeperCreate,
   KeeperApprove,

src/types/holders.ts

@@ -33,4 +33,5 @@ export type HoldersReport = {
   unclaimed: UnclaimedRewards;
   negativeDiffs: string[];
   overclaimed?: string[];
+  overDistributed?: string[];
 };

tests/helpers/ManualChainProvider.ts

@@ -7,15 +7,18 @@ import { HolderClaims } from '../../src/types/holders';
 export default class ManualChainProvider extends OnChainProvider {
   claimedCall: () => HolderClaims;
   activeDistributionCall: () => ExtensiveDistributionParametersStructOutput[];
+  activeDistributionsBetweenCall: (start: number, end: number) => ExtensiveDistributionParametersStructOutput[];
   poolNameCall: () => string;
 
   constructor(
     activeDistributionCall: () => ExtensiveDistributionParametersStructOutput[],
+    activeDistributionsBetweenCall: (start: number, end: number) => ExtensiveDistributionParametersStructOutput[],
     claimedCall: () => HolderClaims,
     poolNameCall: () => string
   ) {
     super({ retries: 1, delay: 1, multiplier: 1 });
     this.activeDistributionCall = activeDistributionCall;
+    this.activeDistributionsBetweenCall = activeDistributionsBetweenCall;
     this.claimedCall = claimedCall;
     this.poolNameCall = poolNameCall;
   }
@@ -24,6 +27,10 @@ export default class ManualChainProvider extends OnChainProvider {
     return this?.activeDistributionCall();
   };
 
+  override activeDistributionsBetween = async (start: number, end: number) => {
+    return this?.activeDistributionsBetweenCall(start, end);
+  };
+
   override claimed = async () => {
     return this?.claimedCall();
   };

tests/history.test.ts

@@ -27,10 +27,10 @@ const tryAtBlock = async ({ chainId, blockNumber, errorCode }: ProblematicBlock)
 describe('Known cases of past disputes', async function () {
   it('Should output same errors cases from Merkl history', async function () {
     const problematicBlocks: ProblematicBlock[] = [
-      { chainId: ChainId.MAINNET, blockNumber: 17812800, errorCode: BotError.NegativeDiff },
-      { chainId: ChainId.MAINNET, blockNumber: 18013500, errorCode: BotError.AlreadyClaimed }, // Aug 28 - Start of already claimed problem
-      { chainId: ChainId.MAINNET, blockNumber: 18052100, errorCode: BotError.AlreadyClaimed }, // Sep 2 - Still spreading incorrect claims...
-      { chainId: ChainId.MAINNET, blockNumber: 18059300, errorCode: undefined }, // Sep 4 - Rewards spread enough to cover anomaly
+      { chainId: ChainId.MAINNET, blockNumber: 17812800, errorCode: BotError.TreeRoot },
+      { chainId: ChainId.MAINNET, blockNumber: 18013500, errorCode: BotError.TreeRoot }, // Aug 28 - Start of already claimed problem
+      { chainId: ChainId.MAINNET, blockNumber: 18052100, errorCode: BotError.TreeRoot }, // Sep 2 - Still spreading incorrect claims...
+      { chainId: ChainId.MAINNET, blockNumber: 18059300, errorCode: BotError.TreeRoot }, // Sep 4 - Rewards spread enough to cover anomaly
     ];
 
     await Promise.all(problematicBlocks.map(tryAtBlock));

tests/holderDiffs.test.ts

@@ -22,6 +22,7 @@ describe('Errors in the differences between two trees', async function () {
       logger: new ConsoleLogger(),
       onChainProvider: new ManualChainProvider(
         createActiveDistribution,
+        (start: number, end: number) => createActiveDistribution(),
         () => createClaims('0'),
         () => 'PESOS-STERLING'
       ),
@@ -46,13 +47,12 @@ describe('Errors in the differences between two trees', async function () {
       logger: new ConsoleLogger(),
       onChainProvider: new ManualChainProvider(
         createActiveDistribution,
+        (start: number, end: number) => createActiveDistribution(),
         () => createClaims('1000'),
         () => 'PESOS-STERLING'
       ),
       merkleRootsProvider: new ManualMerkleRootsProvider(),
     };
-
-
     const report = await checkHolderValidity(testContext, testReport);
 
     expect(report.err).to.equal(false);

tests/overclaims.test.ts

@@ -23,6 +23,7 @@ describe('Overclaim detections', async function () {
       logger: new ConsoleLogger(),
       onChainProvider: new ManualChainProvider(
         createActiveDistribution,
+        (start: number, end: number) => createActiveDistribution(),
         () => createClaims('1001000000000000000000'),
         () => 'PESOS-STERLING'
       ),
@@ -48,6 +49,7 @@ describe('Overclaim detections', async function () {
       logger: new ConsoleLogger(),
       onChainProvider: new ManualChainProvider(
         createActiveDistribution,
+        (start: number, end: number) => createActiveDistribution(),
         () => createClaims('1000000000000000000002'),
         () => 'PESOS-STERLING'
       ),