feat(qf-funding-score): to calculate score for each project from past…

… donations

for thematters/developer-resource#350 & #103

1. add trust points details for each sender, in tsv format good for export/import spreadsheet for human review;
2. saved to s3 bucket `s3://matters-billboard/pre-rounds/` for automation later;
3. send qf notifications emails;

bin/qf-calculate.ts

@@ -0,0 +1,44 @@
+#!/usr/bin/env -S node --trace-warnings --loader ts-node/esm
+
+import {
+  calculateQFScore,
+  checkDropEventsAndNotifs,
+} from '../lib/qf-calculate.js'
+
+async function main() {
+  const args = process.argv.slice(2)
+  let mode: 'checkDropEventsAndNotifs' | 'calculateQFScore' = 'calculateQFScore'
+
+  switch (args?.[0]) {
+    case '--checkDropEventsAndNotifs':
+    case '--calculateQFScore':
+      mode = args?.[0].substring(2) as any
+      args.shift()
+      break
+  }
+  if (mode === 'checkDropEventsAndNotifs') {
+    return checkDropEventsAndNotifs()
+  }
+
+  const amountTotal = BigInt(+(args?.[0] || 10_000))
+
+  // let fromTime: string | undefined, toTime: string | undefined;
+  // let [fromTime, toTime] = args.length >= 0 ? [args?.[1] || '2023-10-01', args?.[2] || '2024-12-31T23:59:59.999Z'] : [undefined, undefined]
+  // const [fromTime,toTime] = [Date.parse(from), Date.parse(to)];
+  const fromBlock = BigInt(
+    args?.[1] || 117_741_511n // the Round1 launch time "2024年3月22日 星期五 中午12:30 [台北標準時間]"
+  )
+  const toBlock = BigInt(
+    args?.[2] || 1_118_000_000n // a big block number in long future
+  )
+
+  const res = await calculateQFScore({
+    fromBlock,
+    toBlock,
+    amountTotal,
+    write_gist: true, // for server side running output;
+  })
+  console.log(new Date(), 'res:', res)
+}
+
+main().catch((err) => console.error(new Date(), 'ERROR:', err))

handlers/qf-calculate.ts

@@ -0,0 +1,200 @@
+import { Context, APIGatewayProxyResult, APIGatewayEvent } from 'aws-lambda'
+
+import * as d3 from 'd3-array'
+import {
+  calculateQFScore,
+  sendQfNotifications, //  sendQfNotificationNEmails,
+  MattersBillboardS3Bucket,
+  isProd,
+  s3FilePathPrefix,
+} from '../lib/qf-calculate.js'
+import { s3GetFile } from '../lib/utils/aws.js'
+
+const ACCESS_TOKEN = `${process.env.ACCESS_TOKEN}`
+
+interface InputBodyParameters {
+  fromTime?: string
+  toTime?: string
+  fromBlock: string
+  toBlock: string
+  amountTotal?: string
+  finalize?: boolean
+}
+
+export const handler = async (
+  event: APIGatewayEvent & {
+    forceRun?: boolean
+  },
+  context: Context
+): Promise<APIGatewayProxyResult> => {
+  console.log(`Event: ${JSON.stringify(event, null, 2)}`)
+  console.log(`Context: ${JSON.stringify(context, null, 2)}`)
+  // const forceRun = !!("forceRun" in ((event?.queryStringParameters as any) || {}));
+
+  const { method, path } = ((event?.requestContext as any)?.http as any) || {}
+  const queryStringParameters = (event?.queryStringParameters as any) || {}
+  const {
+    accept,
+    origin,
+    authorization,
+  }: { accept?: string; origin?: string; authorization?: string } =
+    event?.headers || {}
+
+  const accessToken = authorization?.split(/\s+/)?.[1]
+  if (accessToken !== ACCESS_TOKEN) {
+    return {
+      statusCode: 403,
+      body: JSON.stringify({ message: 'invalid access token' }),
+    }
+  }
+
+  if (
+    !(
+      event?.forceRun ||
+      (path === '/qf-calculator' && accept?.startsWith('application/json')) ||
+      (path === '/send-notifications' && accept) ||
+      (path === '/get-rounds' && accept)
+    )
+  ) {
+    return {
+      statusCode: 400,
+      // contentType: 'text/plain',
+      headers: { 'content-type': 'text/plain' },
+      // JSON.stringify({ error:
+      body: 'input error, call with POST /qf-calculator with accept: application/json',
+      // }),
+    }
+  }
+
+  if (path === '/get-rounds') {
+    const { key } = queryStringParameters
+    const res = await s3GetFile({
+      bucket: MattersBillboardS3Bucket, // 'matters-billboard',
+      key, // : key?.endsWith('.json') ? key : `pre-rounds/rounds.json`,
+    }) // .then((res) => console.log(new Date(), `s3 get pre-rounds:`, res));
+    console.log(new Date(), `s3 get rounds:`, res.ContentLength)
+
+    if (!res.Body) {
+      return {
+        statusCode: 404,
+        headers: { 'content-type': 'text/plain' },
+        body: 'file not found',
+      }
+    }
+    const body = await res.Body.transformToString()
+    return {
+      statusCode: 200,
+      headers: { 'content-type': 'application/json; charset=utf-8' },
+      body: body,
+    }
+  } else if (method === 'POST' && path === '/send-notifications' && accept) {
+    // get distrib.json and send notifications;
+    let { key = 'latest', roundEnd } = queryStringParameters
+    // let key = (queryStringParameters?.key || 'latest') as string
+    // get latest round path to distrib.json
+    try {
+      if (key === 'latest') {
+        const res = await s3GetFile({
+          bucket: MattersBillboardS3Bucket, // 'matters-billboard',
+          key: `${s3FilePathPrefix}/rounds.json`, // : key?.endsWith('.json') ? key : `pre-rounds/rounds.json`,
+        }) // .then((res) => console.log(new Date(), `s3 get pre-rounds:`, res));
+        if (res.Body && res.ContentLength! > 0) {
+          const existingRounds = JSON.parse(
+            await res.Body.transformToString()
+          ) as any[]
+          const latestRound = existingRounds
+            .filter(({ draft }) => !draft) // skip if not finalized yet;
+            .sort((a, b) => d3.descending(a.toTime, b.toTime))?.[0]
+          if (latestRound?.dirpath) {
+            key = `${s3FilePathPrefix}/${latestRound.dirpath}/distrib.json`
+            roundEnd = latestRound?.toTime
+          }
+        }
+      }
+    } catch (err) {
+      console.error(new Date(), `ERROR in retrieving latest round:`, err) // continue try the given key path
+    }
+
+    const res1 = await s3GetFile({
+      bucket: MattersBillboardS3Bucket, // 'matters-billboard',
+      key, // : key?.endsWith('.json') ? key :
+    })
+    console.log(
+      new Date(),
+      `s3 get distribs for notifying:`,
+      res1.ContentLength,
+      { key }
+    )
+
+    if (!res1.Body) {
+      return {
+        statusCode: 404,
+        headers: { 'content-type': 'text/plain' },
+        body: 'file not found',
+      }
+    }
+
+    try {
+      const distribs = JSON.parse(await res1.Body.transformToString())
+      const sent = await sendQfNotifications(
+        distribs,
+        roundEnd,
+        queryStringParameters?.doNotify === 'true'
+      )
+      return {
+        statusCode: 200,
+        body: `sent qf notifications to ${sent?.length ?? 0} authors`,
+      }
+    } catch (err) {
+      console.error(new Date(), `got ERROR in send qf notif:`, err)
+      return {
+        statusCode: 400,
+        body: '',
+      }
+    }
+  } else if (
+    method === 'POST' &&
+    path === '/qf-calculator' &&
+    accept?.startsWith('application/json')
+  ) {
+    const { fromTime, toTime, fromBlock, toBlock, amountTotal, finalize } = (
+      event?.forceRun ? event : queryStringParameters
+    ) as InputBodyParameters
+
+    const { root, gist_url } =
+      (await calculateQFScore({
+        // fromTime, toTime,
+        fromBlock: BigInt(fromBlock),
+        toBlock: BigInt(toBlock),
+        amountTotal: BigInt(+(amountTotal || 10_000)),
+        finalize: !!finalize,
+      })) || {}
+    if (!root) {
+      return {
+        statusCode: 400,
+        body: JSON.stringify({
+          message: 'bad parameters, no tree root, check logs for details.',
+        }),
+      }
+    }
+
+    return {
+      statusCode: 200,
+      headers: { 'content-type': 'application/json; charset=utf-8' },
+      body: JSON.stringify({
+        message: 'done.',
+        root, // tree
+        gist_url,
+      }),
+    }
+  }
+
+  return {
+    statusCode: 400,
+    // contentType: 'text/plain',
+    headers: { 'content-type': 'text/plain; charset=utf-8' },
+    // JSON.stringify({ error:
+    body: 'input error, call with POST /qf-calculator with accept: application/json',
+    // }),
+  }
+}

lib/bigint-math.ts

@@ -0,0 +1,77 @@
+// https://golb.hplar.ch/2018/09/javascript-bigint.html
+export class BigIntMath {
+  static max(...values: Array<bigint | number>) {
+    if (values.length === 0) {
+      return null
+    }
+
+    if (values.length === 1) {
+      return values[0]
+    }
+
+    let max = values[0]
+    for (let i = 1; i < values.length; i++) {
+      if (values[i] > max) {
+        max = values[i]
+      }
+    }
+    return max
+  }
+
+  static min(...values: Array<bigint | number>) {
+    if (values.length === 0) {
+      return null
+    }
+
+    if (values.length === 1) {
+      return values[0]
+    }
+
+    let min = values[0]
+    for (let i = 1; i < values.length; i++) {
+      if (values[i] < min) {
+        min = values[i]
+      }
+    }
+    return min
+  }
+
+  static sign(value: bigint | number) {
+    if (value > 0n) {
+      return 1n
+    }
+    if (value < 0n) {
+      return -1n
+    }
+    return 0n
+  }
+
+  static abs(value: bigint | number) {
+    if (this.sign(value) === -1n) {
+      return -value
+    }
+    return value
+  }
+
+  // https://stackoverflow.com/questions/53683995/javascript-big-integer-square-root/58863398#58863398
+  static rootNth(value: bigint, k: bigint = 2n) {
+    if (value < 0n) {
+      throw 'negative number is not supported'
+    }
+
+    let o = 0n
+    let x = value
+    let limit = 100
+
+    while (x ** k !== k && x !== o && --limit) {
+      o = x
+      x = ((k - 1n) * x + value / x ** (k - 1n)) / k
+    }
+
+    return x
+  }
+
+  static sqrt(value: bigint) {
+    return BigIntMath.rootNth(value)
+  }
+}