feat(ipns): refresh only given authors' IPNS

.husky/pre-commit

@@ -1,4 +1,3 @@
 npm run format:check
 npm run lint
 npm run build
-npm test

handlers/refresh-ipns-gw3.ts

@@ -7,8 +7,6 @@ export const handler = async (
   event: APIGatewayEvent & {
     userName?: string | string[]
     limit?: number
-    offset?: number
-    batchCount?: number
     concurrency?: number
     forceReplace?: boolean
     useMattersIPNS?: boolean
@@ -18,39 +16,19 @@ export const handler = async (
   console.log(`Event: ${JSON.stringify(event, null, 2)}`)
   console.log(`Context: ${JSON.stringify(context, null, 2)}`)
 
-  const {
-    limit = 50,
-    batchCount = 15,
-    concurrency = 5,
-    forceReplace,
-    useMattersIPNS,
-  } = event
-  // let userName = event.userName as string;
+  const { limit = 50, concurrency = 5, forceReplace, useMattersIPNS } = event
   let names: string[] = []
   if (Array.isArray(event?.userName)) {
     names = Array.from(event?.userName)
   } else if (typeof event?.userName === 'string') {
     names = event.userName.trim().split(/\s+/).filter(Boolean)
-  } else {
-    const authors = await dbApi.listRecentAuthors({
-      limit: batchCount,
-      offset: event?.offset,
-    })
-    // if (authors?.[0]?.userName) userName = authors?.[0]?.userName;
-    names = authors.map(({ userName }) => userName).filter(Boolean)
-    console.log(
-      new Date(),
-      `got latest recent ${authors.length} authors:`,
-      names,
-      authors
-    )
   }
 
   const data: any[] = []
 
   const promises = new Map(
     names
-      .splice(0, concurrency) // start with CONCURRENCY=3
+      .splice(0, concurrency)
       .map((userName: string) => [
         userName,
         processUser(userName, { limit, forceReplace, useMattersIPNS }),
@@ -71,14 +49,11 @@ export const handler = async (
     }
   }
 
-  await purgeIPNS()
+  // await purgeIPNS()
 
   return {
     statusCode: 200,
-    body: JSON.stringify({
-      message: 'done.',
-      data,
-    }),
+    body: JSON.stringify({ message: 'done.', data }),
   }
 }
 
@@ -87,37 +62,22 @@ async function processUser(
   {
     limit = 50,
     forceReplace = false,
-    useMattersIPNS,
+    useMattersIPNS = true,
   }: {
     limit?: number
     forceReplace?: boolean
     useMattersIPNS?: boolean
   } = {}
 ) {
-  // let limit = event.limit ?? 50; // default try 50 articles
   let data = await refreshIPNSFeed(userName, {
     limit,
     forceReplace,
     useMattersIPNS,
   })
-  while (!(+(data?.missingInLast50 ?? data?.missing) === 0)) {
-    console.log(new Date(), `for ${userName} not up-to-date, got data:`, data)
-    if (limit >= 150) limit = 50
-    else if (limit >= 50) limit = 30
-    else if (limit > 10) limit = 10
-    else if (limit > 1) limit = Math.ceil(limit / 2) // try 10, 5, 3, 2, 1
-
-    data = await refreshIPNSFeed(userName, {
-      limit, // if no success, try again with latest 10 only
-      forceReplace,
-      useMattersIPNS,
-    })
-    if (limit === 1) break
-  }
   if (limit > 10 && !(+(data?.missingInLast50 ?? data?.missing) === 0)) {
     console.log(
       new Date(),
-      `for ${userName} still not up-to-date, try last 1 time with limit:10 entries, from data:`,
+      `for ${userName} not up-to-date, try 1 time with limit:10 entries, from data:`,
       data
     )
     data = await refreshIPNSFeed(userName, {

lib/author-feed-ipns.ts

@@ -62,62 +62,6 @@ export class AuthorFeed {
     console.log(new Date(), 'loadData got userImg:', this.userImg)
   }
 
-  // move from articleService.ts
-  generate() {
-    const { userName, displayName, description } = this.author
-
-    const context = {
-      meta: {
-        title: `${displayName} (${userName}) - Matters`,
-        description,
-        authorName: displayName,
-        image: this.userImg || undefined,
-        siteDomain,
-      },
-      byline: {
-        author: {
-          name: `${displayName} (${userName})`,
-          displayName,
-          userName,
-          uri: `https://${siteDomain}/@${userName}`,
-          ipnsKey: this.ipnsKey,
-          webfDomain: this.webfHost,
-        },
-        website: {
-          name: 'Matters',
-          uri: `https://${siteDomain}`,
-        },
-      },
-      rss: this.ipnsKey
-        ? {
-            // ipnsKey: this.ipnsKey,
-            xml: './rss.xml',
-            json: './feed.json',
-          }
-        : undefined,
-      articles: this.articles
-        .map((arti) => {
-          return {
-            id: arti.id,
-            author: {
-              userName,
-              displayName,
-            },
-            title: arti.title,
-            summary: arti.summary,
-            date: arti.createdAt,
-            content: arti.content,
-            // tags: draft.tags || [],
-            uri: `./${arti.id}-${slugify(arti.title)}/`,
-            sourceUri: `https://${siteDomain}/a/${arti.shortHash}`,
-          }
-        })
-        .filter(Boolean) as any[],
-    } as HomepageContext
-
-    return makeHomepage(context)
-  }
-
   // dup from articleService.ts
   async publishToIPFS(article: ArticleToPublish) {
     // prepare metadata
@@ -299,12 +243,6 @@ export class AuthorFeed {
     } as HomepageContext
 
     return makeHomepageBundles(context)
-    /* const { html, xml, json } = await makeHomepage(context);
-    return [
-      { path: "index.html", content: html },
-      { path: "rss.xml", content: xml },
-      { path: "feed.json", content: json },
-    ] as { path: string; content: string }[]; */
   }
 
   async activityPubBundles() {

lib/db.ts

@@ -282,40 +282,6 @@ ORDER BY id DESC
 LIMIT ${take} OFFSET ${skip};`
   }
 
-  listRecentAuthors({
-    limit = 100,
-    offset = 0,
-    since = '2022-01-01',
-  }: { limit?: number; offset?: number; since?: string | Date } = {}) {
-    return sqlRO`-- check latest articles' author ipns_key
-SELECT u2.user_name, u2.display_name, GREATEST(ul.last_at ::date, u2.last_seen ::date) AS last_seen,
-  count_articles, ipns_key, last_data_hash AS top_dir_data_hash, last_published, a.*,
-  concat('https://matters.town/@', u2.user_name, '/', a.id, '-', a.slug) AS last_article_url,
-  priv_key_pem, priv_key_name
-FROM (
-  SELECT DISTINCT ON (author_id) author_id, id, title, slug, summary, data_hash AS last_article_data_hash, media_hash, created_at AS last_article_published
-  FROM article
-  WHERE state IN ('active')
-    AND author_id NOT IN (SELECT user_id FROM user_restriction) -- skip restricted authors
-  ORDER BY author_id, id DESC
-) a
-LEFT JOIN mat_views.users_lasts ul ON author_id=ul.id
-LEFT JOIN public.user u2 ON author_id=u2.id
-LEFT JOIN user_ipns_keys k ON author_id=k.user_id
-LEFT JOIN (
-  SELECT author_id, COUNT(*)::int AS count_articles
-  FROM article
-  WHERE state NOT IN ('archived')
-  GROUP BY 1
-) ta USING (author_id)
--- WHERE -- WHERE user_name IN ('Brianliu', '...', 'oldcat')
-WHERE u2.state NOT IN ('archived', 'banned')
-  AND a.last_article_published >= ${since}
-  AND (last_published IS NULL OR last_published < a.last_article_published)
-ORDER BY id DESC
-LIMIT ${limit} OFFSET ${offset} `
-  }
-
   listRecentIPNSAuthors({
     skip = 10000,
     range = '1 year',
@@ -325,20 +291,20 @@ LIMIT ${limit} OFFSET ${offset} `
 SELECT user_name, display_name, author.state AS author_state, author.last_seen, eth_address,
   ipns_key, last_data_hash,
   (author.state != 'active' OR author_id IN (SELECT user_id FROM user_restriction)) AS is_restricted,
-  article.*, k.stats, GREATEST(author.last_seen, article.created_at) AS last_at
+  article.*, avn.title, k.stats, GREATEST(author.last_seen, article.created_at) AS last_at
 FROM (
   SELECT DISTINCT ON (author_id) author_id ::int,
-    article.id ::int, title, state AS article_state, article.created_at
+    article.id ::int, state AS article_state, article.created_at
   FROM public.article
   ORDER BY author_id, id DESC
 ) article
+LEFT JOIN public.article_version_newest avn ON avn.article_id = article.id
 LEFT JOIN public.user author ON author_id=author.id
 LEFT JOIN public.user_ipns_keys k ON user_id=author.id
 WHERE (stats->'isPurged')::bool IS NOT true
-  -- AND author.last_seen >= CURRENT_DATE - $ {range}::interval
   ${Array.isArray(userIds) ? sqlRO`AND user_id=ANY(${userIds})` : sqlRO``}
   AND article.created_at >= CURRENT_DATE - ${range}::interval
-ORDER BY last_at DESC NULLS LAST -- LIMIT 13000`
+ORDER BY last_at DESC NULLS LAST`
   }
 
   listRecentUsers({
@@ -515,7 +481,12 @@ RETURNING * ;`
   }
 
   queryArticlesByUuid(uuids: string[]) {
-    return sqlRO` SELECT id, title, slug, data_hash, media_hash, created_at FROM article WHERE uuid =ANY(${uuids}) `
+    return sqlRO` 
+      SELECT a.id, avn.title, avn.data_hash, avn.media_hash, a.created_at 
+      FROM article a
+      LEFT JOIN article_version_newest avn ON a.id = avn.article_id 
+      WHERE a.uuid =ANY(${uuids}) 
+    `
   }
 
   async checkVersion() {

lib/refresh-ipns-gw3.ts

@@ -861,9 +861,8 @@ export async function refreshIPNSFeed(
   } = {}
 ) {
   const [author] = await dbApi.getAuthor(userName)
-  console.log(new Date(), 'get author:', author)
   if (!author || !ALLOWED_USER_STATES.has(author?.state)) {
-    console.log(new Date(), `no such user:`, author)
+    console.log(new Date(), `no such user or inactive state:`, userName)
     return
   }
   const [ipnsKeyRec] = await dbApi.getUserIPNSKey(author.id)
@@ -874,15 +873,8 @@ export async function refreshIPNSFeed(
       new Date(),
       `skip no ipnsKeyRec: for author: '${author.displayName} (@${author.userName})'`
     )
-    // return;
   }
-  console.log(
-    new Date(),
-    `input useMattersIPNS:`,
-    typeof useMattersIPNS,
-    useMattersIPNS,
-    ipnsKeyRec?.stats?.useMattersIPNS
-  )
+
   useMattersIPNS = useMattersIPNS ?? ipnsKeyRec?.stats?.useMattersIPNS ?? false // remember the settings from last time
   console.log(
     new Date(),
@@ -902,7 +894,7 @@ export async function refreshIPNSFeed(
   console.log(
     new Date(),
     `get ${articles.length} articles for author: '${author.displayName} (@${author.userName})', last_article:`,
-    lastArti
+    lastArti.id
   )
   if (articles.length <= 10) {
     forceReplace = true
@@ -929,10 +921,7 @@ export async function refreshIPNSFeed(
     webfHost,
     articles,
   })
-  // console.log(new Date(), "get author feed:", feed);
   await feed.loadData()
-  // const { html, xml, json } = feed.generate();
-  // console.log(new Date(), "get author feed generated:", { html, xml, json });
 
   const kname = `matters-town-homepages-topdir-for-${author.userName}-${author.uuid}`
 
@@ -1008,9 +997,6 @@ export async function refreshIPNSFeed(
   // TO MOVE TO library
   // const extraBundles = await feed.activityPubBundles(); // { author, // userName: author.userName, ipnsKey: keyPair.ipnsKey, });
   const contents = [
-    // { path: `${directoryName}/index.html`, content: html, },
-    // { path: `${directoryName}/rss.xml`, content: xml, },
-    // { path: `${directoryName}/feed.json`, content: json, },
     ...(await feed.feedBundles()),
     ...(await feed.activityPubBundles()),
   ].map(({ path, content }) => ({