chore: update gitcoin-passport-holders data endpoint

sismo-core · Sep 13, 2023 · c3b0c1b · c3b0c1b
1 parent 7f389bb
commit c3b0c1b
Showing 1 changed file with 54 additions and 33 deletions.
diff --git a/group-generators/generators/gitcoin-passport-holders/index.ts b/group-generators/generators/gitcoin-passport-holders/index.ts
@@ -1,5 +1,4 @@
-import { dataProviders } from "@group-generators/helpers/data-providers";
-import { ApiConfig } from "@group-generators/helpers/data-providers/rest-api";
+import axios from "axios";
 import { Tags, ValueType, GroupWithData, FetchedData, AccountSource } from "topics/group";
 import { GenerationContext, GenerationFrequency, GroupGenerator } from "topics/group-generator";
 
@@ -8,45 +7,67 @@ const generator: GroupGenerator = {
 
   generate: async (context: GenerationContext): Promise<GroupWithData[]> => {
     type Passport = {
-      address: string;
-      score: string;
-      status: string;
-      last_score_timestamp: string;
-      evidence: {
-        type: string;
-        success: boolean;
-        rawScore: string;
-        threshold: string;
-      };
-      error: any;
+      passport: { address: string };
+      evidence: { rawScore: string };
     };
 
     const evmAddressRegEx = new RegExp("^0x[a-fA-F0-9]{40}$");
 
-    const restProvider = new dataProviders.RestProvider();
     const gitcoinPassportHolders: FetchedData = {};
-    const url = "https://indexer-grants-stack.gitcoin.co/data/passport_scores.json";
+    const url = "https://public.scorer.gitcoin.co/passport_scores/registry_score.jsonl";
 
-    const apiConfig: ApiConfig = {
-      url: url,
-      method: "GET",
-      headers: {
-        accept: "application/json",
-      },
-    };
+    // Create a promise to wrap the streaming logic
+    const streamData = new Promise<void>((resolve, reject) => {
+      axios
+        .get(url, {
+          responseType: "stream",
+        })
+        .then((response) => {
+          // Buffer to store the streamed data
+          let textBuffer = "";
+
+          // Listen to the 'data' event on the response stream
+          response.data.on("data", (chunk: Buffer) => {
+            // save the data chunk to the buffer
+            textBuffer += chunk.toString();
+
+            let newlineIndex;
+            while ((newlineIndex = textBuffer.indexOf("\n")) >= 0) {
+              // extract the line from the buffer
+              const line = textBuffer.slice(0, newlineIndex).trim();
+              // remove the line from the buffer
+              textBuffer = textBuffer.slice(newlineIndex + 1);
+
+              if (line) {
+                const { passport, evidence } = JSON.parse(line) as Passport;
+                if (evmAddressRegEx.test(passport.address)) {
+                  if (!evidence?.rawScore) {
+                    continue;
+                  }
+                  gitcoinPassportHolders[passport.address] = Math.floor(
+                    Number(evidence.rawScore)
+                  ).toString();
+                }
+              }
+            }
+          });
+
+          response.data.on("end", () => {
+            resolve();
+          });
 
-    const passports: Passport[] = (await restProvider.fetchData(apiConfig)) as any as Passport[];
+          response.data.on("error", (err: any) => {
+            console.log("Error during streaming.");
+            reject(err);
+          });
+        })
+        .catch((err) => {
+          console.log("Failed to fetch data while streaming.");
+          reject(err);
+        });
+    });
 
-    for (const passport of passports) {
-      if (evmAddressRegEx.test(passport.address)) {
-        if (!passport?.evidence?.rawScore) {
-          continue;
-        }
-        gitcoinPassportHolders[passport.address] = Math.floor(
-          Number(passport.evidence.rawScore)
-        ).toString();
-      }
-    }
+    await streamData;
 
     return [
       {