feat(ipns-with-gw3): add ./bin/refresh-ipns-gw3.ts & lambda handler

bin/refresh-ipns-gw3.ts

@@ -0,0 +1,127 @@
+#!/usr/bin/env -S node --trace-warnings --loader ts-node/esm
+
+import path from "node:path";
+import shuffle from "lodash/shuffle.js";
+import {
+  // HomepageArticleDigest,
+  HomepageContext,
+  makeHomepage,
+} from "@matters/ipns-site-generator";
+import slugify from "@matters/slugify";
+
+import {
+  gw3Client,
+  refreshPinLatest,
+  refreshIPNSFeed,
+} from "../lib/refresh-ipns-gw3.js";
+import { AuthorFeed } from "../lib/author-feed-ipns.js";
+import { ipfsPool } from "../lib/ipfs-servers.js";
+import { dbApi, Item } from "../lib/db.js";
+
+async function main() {
+  const args = process.argv.slice(2);
+  let mode: "publishIPNS" | "publishIPFS" | "uploadPinning" = "publishIPNS";
+
+  switch (args?.[0]) {
+    case "--publishIPNS":
+    case "--publishIPFS":
+    case "--uploadPinning":
+      mode = args?.[0].substring(2) as any;
+      args.shift();
+      break;
+  }
+
+  if (mode === "publishIPFS") {
+    const limit = parseInt(args?.[0] ?? "10");
+    const offset = parseInt(args?.[1] ?? "0");
+
+    const articles = await dbApi.listRecentArticlesToPublish({
+      take: limit,
+      skip: offset,
+    });
+    const drafts = await dbApi.listDrafts({
+      ids: articles.map((item: Item) => item.draftId as string),
+      take: limit,
+    });
+    console.log(
+      new Date(),
+      `found ${articles.length} articles /${drafts.length} drafts not published:`,
+      articles
+    );
+
+    const [author] = await dbApi.getAuthor(articles?.[0]?.userName as string);
+    console.log(new Date(), "get author:", author);
+    if (!author) {
+      console.error(new Date(), "no such user.");
+      return;
+    }
+
+    const [ipnsKeyRec] = await dbApi.getUserIPNSKey(author.id);
+    console.log(new Date(), "get user ipns:", ipnsKeyRec);
+
+    const feed = new AuthorFeed(
+      author,
+      ipnsKeyRec?.ipnsKey,
+      drafts.slice(0, 1),
+      articles.slice(0, 1)
+    );
+
+    // console.log(new Date(), "get author feed:", feed);
+    await feed.loadData();
+
+    console.log(
+      new Date(),
+      `found ${articles.length} articles to publish:`,
+      articles
+    );
+    const res = await feed.publishToIPFS(drafts[0]);
+    console.log(new Date(), `from published IPFS:`, res);
+    return;
+  } else if (mode === "uploadPinning") {
+    const limit = parseInt(args?.[0] ?? "100");
+    const offset = parseInt(args?.[1] ?? "0");
+    await refreshPinLatest({ limit, offset });
+    return;
+  }
+
+  let forceReplace = false;
+  let useMattersIPNS = false;
+
+  // publish IPNS mode
+  console.log(new Date(), "running with:", args);
+  switch (args?.[0]) {
+    case "--forceReplace":
+      forceReplace = true;
+      args.shift();
+      break;
+    case "--useMattersIPNS":
+      useMattersIPNS = true;
+      args.shift();
+      break;
+  }
+
+  // await testPinning();
+
+  const userName = args?.[0] || "hi176";
+  const limit = parseInt(args?.[1] || "50");
+
+  let res = await refreshIPNSFeed(userName, {
+    limit,
+    forceReplace,
+    useMattersIPNS,
+  });
+  console.log(new Date(), `refreshIPNSFeed res:`, res);
+  if (res && res.missing <= res.limit / 10) return;
+
+  if (limit > 10 && !((res?.missingInLast50 ?? res?.missing) === 0)) {
+    // try again with limit: 10
+    res = await refreshIPNSFeed(userName, {
+      limit: 10,
+      forceReplace,
+      useMattersIPNS,
+    });
+    console.log(new Date(), `try again refreshIPNSFeed res:`, res);
+  }
+}
+
+main().catch((err) => console.error(new Date(), "ERROR:", err));

handlers/refresh-ipns-gw3.ts

@@ -0,0 +1,109 @@
+import { Context, APIGatewayProxyResult, APIGatewayEvent } from "aws-lambda";
+
+import { refreshIPNSFeed } from "../lib/refresh-ipns-gw3.js";
+import { dbApi, Item } from "../lib/db.js";
+
+export const handler = async (
+  event: APIGatewayEvent & {
+    userName?: string | string[];
+    limit?: number;
+    batchCount?: number;
+    forceReplace?: boolean;
+  },
+  context: Context
+): Promise<APIGatewayProxyResult> => {
+  console.log(`Event: ${JSON.stringify(event, null, 2)}`);
+  console.log(`Context: ${JSON.stringify(context, null, 2)}`);
+
+  // let userName = event.userName as string;
+  let names: string[] = [];
+  if (Array.isArray(event?.userName)) {
+    names = Array.from(event?.userName);
+  } else if (event?.userName) {
+    names = [event?.userName];
+  } else {
+    const authors = await dbApi.listRecentAuthors({
+      limit: event?.batchCount ?? 15,
+    });
+    // if (authors?.[0]?.userName) userName = authors?.[0]?.userName;
+    names = authors.map(({ userName }) => userName).filter(Boolean);
+    console.log(
+      new Date(),
+      `got latest author '${names}' fromrecent authors:`,
+      authors
+    );
+  }
+
+  const data: any[] = [];
+
+  const promises = new Map(
+    names
+      .splice(0, 3) // start with CONCURRENCY=3
+      .map((userName: string) => [
+        userName,
+        processUser(userName, {
+          limit: event?.limit ?? 50,
+          forceReplace: event?.forceReplace ?? false,
+        }),
+      ])
+  );
+
+  while (promises.size > 0) {
+    const item = await Promise.race(promises.values());
+    data.push(item);
+    promises.delete(item.userName);
+
+    if (names.length > 0) {
+      const userName = names.shift() as string;
+      promises.set(
+        userName,
+        processUser(userName, {
+          limit: event?.limit ?? 50,
+          forceReplace: event?.forceReplace ?? false,
+        })
+      );
+    }
+  }
+
+  return {
+    statusCode: 200,
+    body: JSON.stringify({
+      message: "done.",
+      data,
+    }),
+  };
+};
+
+async function processUser(
+  userName: string,
+  { limit = 50, forceReplace = false } = {}
+) {
+  // let limit = event.limit ?? 50; // default try 50 articles
+  let data = await refreshIPNSFeed(userName, { limit, forceReplace });
+  while (!(data?.missing === 0)) {
+    console.log(new Date(), `for ${userName} not up-to-date, got data:`, data);
+    if (limit >= 150) limit = 50;
+    else if (limit >= 50) limit = 30;
+    else if (limit > 10) limit = 10;
+    else if (limit > 1) limit = Math.ceil(limit / 2); // try 10, 5, 3, 2, 1
+
+    data = await refreshIPNSFeed(userName, {
+      limit, // if no success, try again with latest 10 only
+      forceReplace,
+    });
+    if (limit === 1) break;
+  }
+  if (!(data?.missing === 0)) {
+    console.log(
+      new Date(),
+      `for ${userName} still not up-to-date, try last 1 time with limit:10 entries, from data:`,
+      data
+    );
+    data = await refreshIPNSFeed(userName, {
+      limit: 10,
+      forceReplace,
+    });
+  }
+
+  return data ?? { userName };
+}