From 609c36d8cb67a4f174e9105c75949e6117e23431 Mon Sep 17 00:00:00 2001 From: 49659410+tx0c <> Date: Wed, 30 Aug 2023 14:09:32 +0000 Subject: [PATCH] feat(ipns-with-gw3): add ./bin/refresh-ipns-gw3.ts & lambda handler --- bin/refresh-ipns-gw3.ts | 127 ++++++ handlers/refresh-ipns-gw3.ts | 109 +++++ lib/author-feed-ipns.ts | 251 +++++++++++ lib/db.ts | 151 ++++++- lib/ipfs-servers.ts | 64 ++- lib/refresh-ipns-gw3.ts | 803 +++++++++++++++++++++++++++++++++ package-lock.json | 845 ++++++++++++++++++++++++++++++++++- package.json | 8 +- 8 files changed, 2314 insertions(+), 44 deletions(-) create mode 100644 bin/refresh-ipns-gw3.ts create mode 100644 handlers/refresh-ipns-gw3.ts create mode 100644 lib/author-feed-ipns.ts create mode 100644 lib/refresh-ipns-gw3.ts diff --git a/bin/refresh-ipns-gw3.ts b/bin/refresh-ipns-gw3.ts new file mode 100644 index 0000000..31862ff --- /dev/null +++ b/bin/refresh-ipns-gw3.ts @@ -0,0 +1,127 @@ +#!/usr/bin/env -S node --trace-warnings --loader ts-node/esm + +import path from "node:path"; +import shuffle from "lodash/shuffle.js"; +import { + // HomepageArticleDigest, + HomepageContext, + makeHomepage, +} from "@matters/ipns-site-generator"; +import slugify from "@matters/slugify"; + +import { + gw3Client, + refreshPinLatest, + refreshIPNSFeed, +} from "../lib/refresh-ipns-gw3.js"; +import { AuthorFeed } from "../lib/author-feed-ipns.js"; +import { ipfsPool } from "../lib/ipfs-servers.js"; +import { dbApi, Item } from "../lib/db.js"; + +async function main() { + const args = process.argv.slice(2); + let mode: "publishIPNS" | "publishIPFS" | "uploadPinning" = "publishIPNS"; + + switch (args?.[0]) { + case "--publishIPNS": + case "--publishIPFS": + case "--uploadPinning": + mode = args?.[0].substring(2) as any; + args.shift(); + break; + } + + if (mode === "publishIPFS") { + const limit = parseInt(args?.[0] ?? "10"); + const offset = parseInt(args?.[1] ?? "0"); + + const articles = await dbApi.listRecentArticlesToPublish({ + take: limit, + skip: offset, + }); + const drafts = await dbApi.listDrafts({ + ids: articles.map((item: Item) => item.draftId as string), + take: limit, + }); + console.log( + new Date(), + `found ${articles.length} articles /${drafts.length} drafts not published:`, + articles + ); + + const [author] = await dbApi.getAuthor(articles?.[0]?.userName as string); + console.log(new Date(), "get author:", author); + if (!author) { + console.error(new Date(), "no such user."); + return; + } + + const [ipnsKeyRec] = await dbApi.getUserIPNSKey(author.id); + console.log(new Date(), "get user ipns:", ipnsKeyRec); + + const feed = new AuthorFeed( + author, + ipnsKeyRec?.ipnsKey, + drafts.slice(0, 1), + articles.slice(0, 1) + ); + + // console.log(new Date(), "get author feed:", feed); + await feed.loadData(); + + console.log( + new Date(), + `found ${articles.length} articles to publish:`, + articles + ); + const res = await feed.publishToIPFS(drafts[0]); + console.log(new Date(), `from published IPFS:`, res); + return; + } else if (mode === "uploadPinning") { + const limit = parseInt(args?.[0] ?? "100"); + const offset = parseInt(args?.[1] ?? "0"); + await refreshPinLatest({ limit, offset }); + return; + } + + let forceReplace = false; + let useMattersIPNS = false; + + // publish IPNS mode + console.log(new Date(), "running with:", args); + switch (args?.[0]) { + case "--forceReplace": + forceReplace = true; + args.shift(); + break; + case "--useMattersIPNS": + useMattersIPNS = true; + args.shift(); + break; + } + + // await testPinning(); + + const userName = args?.[0] || "hi176"; + const limit = parseInt(args?.[1] || "50"); + + let res = await refreshIPNSFeed(userName, { + limit, + forceReplace, + useMattersIPNS, + }); + console.log(new Date(), `refreshIPNSFeed res:`, res); + if (res && res.missing <= res.limit / 10) return; + + if (limit > 10 && !((res?.missingInLast50 ?? res?.missing) === 0)) { + // try again with limit: 10 + res = await refreshIPNSFeed(userName, { + limit: 10, + forceReplace, + useMattersIPNS, + }); + console.log(new Date(), `try again refreshIPNSFeed res:`, res); + } +} + +main().catch((err) => console.error(new Date(), "ERROR:", err)); diff --git a/handlers/refresh-ipns-gw3.ts b/handlers/refresh-ipns-gw3.ts new file mode 100644 index 0000000..082366f --- /dev/null +++ b/handlers/refresh-ipns-gw3.ts @@ -0,0 +1,109 @@ +import { Context, APIGatewayProxyResult, APIGatewayEvent } from "aws-lambda"; + +import { refreshIPNSFeed } from "../lib/refresh-ipns-gw3.js"; +import { dbApi, Item } from "../lib/db.js"; + +export const handler = async ( + event: APIGatewayEvent & { + userName?: string | string[]; + limit?: number; + batchCount?: number; + forceReplace?: boolean; + }, + context: Context +): Promise => { + console.log(`Event: ${JSON.stringify(event, null, 2)}`); + console.log(`Context: ${JSON.stringify(context, null, 2)}`); + + // let userName = event.userName as string; + let names: string[] = []; + if (Array.isArray(event?.userName)) { + names = Array.from(event?.userName); + } else if (event?.userName) { + names = [event?.userName]; + } else { + const authors = await dbApi.listRecentAuthors({ + limit: event?.batchCount ?? 15, + }); + // if (authors?.[0]?.userName) userName = authors?.[0]?.userName; + names = authors.map(({ userName }) => userName).filter(Boolean); + console.log( + new Date(), + `got latest author '${names}' fromrecent authors:`, + authors + ); + } + + const data: any[] = []; + + const promises = new Map( + names + .splice(0, 3) // start with CONCURRENCY=3 + .map((userName: string) => [ + userName, + processUser(userName, { + limit: event?.limit ?? 50, + forceReplace: event?.forceReplace ?? false, + }), + ]) + ); + + while (promises.size > 0) { + const item = await Promise.race(promises.values()); + data.push(item); + promises.delete(item.userName); + + if (names.length > 0) { + const userName = names.shift() as string; + promises.set( + userName, + processUser(userName, { + limit: event?.limit ?? 50, + forceReplace: event?.forceReplace ?? false, + }) + ); + } + } + + return { + statusCode: 200, + body: JSON.stringify({ + message: "done.", + data, + }), + }; +}; + +async function processUser( + userName: string, + { limit = 50, forceReplace = false } = {} +) { + // let limit = event.limit ?? 50; // default try 50 articles + let data = await refreshIPNSFeed(userName, { limit, forceReplace }); + while (!(data?.missing === 0)) { + console.log(new Date(), `for ${userName} not up-to-date, got data:`, data); + if (limit >= 150) limit = 50; + else if (limit >= 50) limit = 30; + else if (limit > 10) limit = 10; + else if (limit > 1) limit = Math.ceil(limit / 2); // try 10, 5, 3, 2, 1 + + data = await refreshIPNSFeed(userName, { + limit, // if no success, try again with latest 10 only + forceReplace, + }); + if (limit === 1) break; + } + if (!(data?.missing === 0)) { + console.log( + new Date(), + `for ${userName} still not up-to-date, try last 1 time with limit:10 entries, from data:`, + data + ); + data = await refreshIPNSFeed(userName, { + limit: 10, + forceReplace, + }); + } + + return data ?? { userName }; +} diff --git a/lib/author-feed-ipns.ts b/lib/author-feed-ipns.ts new file mode 100644 index 0000000..f2771fc --- /dev/null +++ b/lib/author-feed-ipns.ts @@ -0,0 +1,251 @@ +import { + // HomepageArticleDigest, + ArticlePageContext, + HomepageContext, + makeArticlePage, + makeHomepage, +} from "@matters/ipns-site-generator"; +import slugify from "@matters/slugify"; + +import { ipfsPool } from "../lib/ipfs-servers.js"; +import { dbApi, Item } from "../lib/db.js"; + +export const ARTICLE_ACCESS_TYPE = { + public: "public", + paywall: "paywall", +} as const; + +const siteDomain = process.env.MATTERS_SITE_DOMAIN || "matters.town"; + +export class AuthorFeed { + author: Item; + ipnsKey: string; // the ipns key + + // internal use + publishedDrafts: Item[]; + userImg?: string | null; + articles?: Map; + + // articleService: InstanceType + // draftService: InstanceType + // tagService: InstanceType + // systemService: InstanceType + + constructor( + author: Item, + ipnsKey: string, + drafts: Item[], + articles?: Item[] + ) { + this.author = author; + this.ipnsKey = ipnsKey; + + // this.articleService = new ArticleService() + // this.draftService = new DraftService() + // this.systemService = new SystemService() + + this.publishedDrafts = drafts; + if (articles) { + this.articles = new Map(articles.map((arti) => [arti.id, arti])); + } + } + + async loadData() { + this.userImg = await dbApi.findAssetUrl( + this.author.avatar, + "w=240,h=240,fit=crop" + ); // this.author.avatar || null; // && (await this.systemService.findAssetUrl(this.author.avatar)) + console.log(new Date(), "loadData got userImg:", this.userImg); + } + + // mov from articleService.ts + generate() { + const { userName, displayName, description } = this.author; + + const context = { + meta: { + title: `${displayName} (${userName}) - Matters`, + description, + authorName: displayName, + image: this.userImg || undefined, + siteDomain, + }, + byline: { + author: { + name: `${displayName} (${userName})`, + uri: `https://${siteDomain}/@${userName}`, + }, + website: { + name: "Matters", + uri: `https://${siteDomain}`, + }, + }, + rss: this.ipnsKey + ? { + ipnsKey: this.ipnsKey, + xml: "./rss.xml", + json: "./feed.json", + } + : undefined, + articles: this.publishedDrafts + // .sort((a, b) => +b.articleId - +a.articleId) + .map((draft) => { + const arti = this.articles?.get(draft.articleId); + if (!arti) return; + + return { + id: draft.articleId ?? arti.id, + author: { + userName, + displayName, + }, + title: draft.title, + summary: draft.summary, + // date: draft.updatedAt, + date: arti.createdAt, // || draft.updatedAt, + content: draft.content, + // tags: draft.tags || [], + uri: `./${draft.articleId ?? arti.id}-${ + arti.slug ?? slugify(arti.title) + }/`, + sourceUri: `https://${siteDomain}/@${userName}/${ + draft.articleId ?? arti.id + }-${arti.slug ?? slugify(arti.title)}/`, + }; + }) + .filter(Boolean) as any[], + } as HomepageContext; + + return makeHomepage(context); + } + + // dup from articleService.ts + async publishToIPFS(draft: Item) { + // prepare metadata + const { + title, + content, + summary, + cover, + tags, + circleId, + access, + authorId, + articleId, + updatedAt: publishedAt, + } = draft; + + const { + userName, + displayName, + description, // paymentPointer + } = this.author; + const articleCoverImg = await dbApi.findAssetUrl(cover); + + const context: ArticlePageContext = { + encrypted: false, + meta: { + title: `${title} - ${displayName} (${userName})`, + description: summary, + authorName: displayName, + image: articleCoverImg || undefined, + siteDomain, + }, + byline: { + date: publishedAt, + author: { + name: `${displayName} (${userName})`, + uri: `https://${siteDomain}/@${userName}`, + }, + website: { + name: "Matters", + uri: `https://${siteDomain}`, + }, + }, + rss: this.ipnsKey + ? { + ipnsKey: this.ipnsKey, + xml: "../rss.xml", + json: "../feed.json", + } + : undefined, + article: { + id: articleId, + author: { + userName, + displayName, + }, + title, + summary, + date: publishedAt, + content, + tags: tags?.map((t: string) => t.trim()).filter(Boolean) || [], + }, + }; + + // paywalled content + if (circleId && access === ARTICLE_ACCESS_TYPE.paywall) { + context.encrypted = true; + console.error( + new Date(), + `TODO: support ARTICLE_ACCESS_TYPE.paywall`, + draft + ); + return; + } + + // payment pointer + // if (paymentPointer) { context.paymentPointer = paymentPointer } + + // make bundle and add content to ipfs + const directoryName = "article"; + const { bundle, key } = await makeArticlePage(context); + + let ipfs = ipfsPool.client; + let retries = 0; + + do { + try { + const results = []; + for await (const result of ipfs.addAll( + bundle + .map((file) => + file + ? { ...file, path: `${directoryName}/${file.path}` } + : undefined + ) + .filter(Boolean) as any + )) { + results.push(result); + } + + // filter out the hash for the bundle + let entry = results.filter( + ({ path }: { path: string }) => path === directoryName + ); + + // FIXME: fix missing bundle path and remove fallback logic + // fallback to index file when no bundle path is matched + if (entry.length === 0) { + entry = results.filter(({ path }: { path: string }) => + path.endsWith("index.html") + ); + } + + const contentHash = entry[0].cid.toString(); + const mediaHash = entry[0].cid.toV1().toString(); // cid.toV1().toString() // cid.toBaseEncodedString() + return { contentHash, mediaHash, key }; + } catch (err) { + // if the active IPFS client throws exception, try a few more times on Secondary + console.error( + `publishToIPFS failed, retries ${++retries} time, ERROR:`, + err + ); + ipfs = ipfsPool.backupClient; + } + } while (ipfs && retries <= ipfsPool.size); // break the retry if there's no backup + + // re-fill dataHash & mediaHash later in IPNS-listener + console.error(`failed publishToIPFS after ${retries} retries.`); + } +} diff --git a/lib/db.ts b/lib/db.ts index 3b277a1..9d69f2f 100644 --- a/lib/db.ts +++ b/lib/db.ts @@ -2,6 +2,9 @@ import { getKnexClient, getPostgresJsClient } from "./utils/db.js"; // main connection client +const CLOUDFLARE_IMAGE_ENDPOINT = process.env.CLOUDFLARE_IMAGE_ENDPOINT || ""; +const MATTERS_AWS_S3_ENDPOINT = process.env.MATTERS_AWS_S3_ENDPOINT || ""; + const isTest = process.env.MATTERS_ENV === "test"; const dbHost = process.env.MATTERS_PG_HOST || ""; const dbUser = process.env.MATTERS_PG_USER || ""; @@ -48,8 +51,29 @@ export interface Article { state: string; authorState: string; lastReadAt?: Date; + userName?: string; + dataHash?: string; +} + +export interface Item { + [key: string]: any; + id: string; } +export const IMAGE_ASSET_TYPE = { + avatar: "avatar", + cover: "cover", + embed: "embed", + profileCover: "profileCover", + oauthClientAvatar: "oauthClientAvatar", + tagCover: "tagCover", + circleAvatar: "circleAvatar", + circleCover: "circleCover", + collectionCover: "collectionCover", + announcementCover: "announcementCover", + topicCover: "topicCover", +} as const; + export class DbApi { listArticles({ articleIds, @@ -80,7 +104,7 @@ export class DbApi { SELECT * -- a.*, num_views, extract(epoch from last_read_at) AS last_read_timestamp FROM ( SELECT -- draft.id, - a.id, a.title, a.summary, -- a.slug, a.draft_id, a.summary, + a.id, a.title, a.summary, a.slug, a.draft_id, a.summary, a.data_hash, user_name, draft.content, draft.author_id, a.created_at, a.state, author.state AS author_state, draft.publish_state FROM article a JOIN draft ON draft_id=draft.id -- AND article_id=article.id LEFT JOIN public.user author ON author.id=draft.author_id @@ -106,22 +130,76 @@ ORDER BY ${orderBy === "lastRead" ? sql`last_read_at DESC NULLS LAST,` : sql``} LIMIT ${take} OFFSET ${skip} ; `; } + listRecentArticles({ take = 100, skip = 0, state = ["active"] } = {}) { + return sqlRO<[Item]>`-- list recent articles +SELECT article.id, slug, title, summary, data_hash, user_name +FROM public.article +LEFT JOIN public.user author ON author_id=author.id +WHERE article.state IN ('active') AND author.state NOT IN ('archived', 'bannded') +ORDER BY article.id DESC +LIMIT ${take} OFFSET ${skip} ;`; + } + listRecentArticlesToPublish({ + take = 100, + skip = 0, + // state = ["active"], + } = {}) { + return sqlRO<[Item]>`-- list recent articles +SELECT article.id, article.slug, article.title, article.summary, article.data_hash, user_name, draft.content, draft.tags, draft_id, article_id +FROM public.article +LEFT JOIN public.draft ON article_id=article.id +LEFT JOIN public.user author ON article.author_id=author.id +WHERE article.state IN ('active') AND author.state NOT IN ('archived', 'banned') + AND publish_state IN ('published') + AND article.data_hash IS NULL +ORDER BY article.id DESC +LIMIT ${take} OFFSET ${skip} ;`; + } + listAuthorArticles({ + authorId, + take = 50, + skip = 0, + state = ["active"], + }: { + authorId: string | number; + take?: number; + skip?: number; + state?: string[]; + }) { + return sqlRO< + [Item] + >`SELECT * FROM public.article WHERE author_id=${authorId} AND state =ANY(${state}) ORDER BY id DESC LIMIT ${take} OFFSET ${skip};`; + } + listDrafts({ + ids, + take = 50, + skip = 0, + }: { + ids: string[]; + take?: number; + skip?: number; + }) { + return sqlRO< + [Item] + >`SELECT * FROM public.draft WHERE id=ANY(${ids}) AND article_id IS NOT NULL AND publish_state IN ('published') ORDER BY article_id DESC LIMIT ${take} OFFSET ${skip};`; + } + listRecentAuthors({ - limit = 5000, + limit = 100, since = "2022-01-01", }: { limit?: number; since?: string | Date } = {}) { return sqlRO`-- check latest articles' author ipns_key -SELECT u2.user_name, u2.display_name, COALESCE(u.last_at ::date, CURRENT_DATE) AS last_seen, - count_articles, ipns_key, last_data_hash AS top_dir_data_hash, last_published AS last_refreshed, t.*, - concat('https://matters.town/@', u2.user_name, '/', t.id, '-', t.slug) AS last_article_url, +SELECT u2.user_name, u2.display_name, GREATEST(ul.last_at ::date, u2.last_seen ::date) AS last_seen, + count_articles, ipns_key, last_data_hash AS top_dir_data_hash, last_published, a.*, + concat('https://matters.town/@', u2.user_name, '/', a.id, '-', a.slug) AS last_article_url, priv_key_pem, priv_key_name FROM ( - SELECT DISTINCT ON (author_id) author_id, id, title, slug, data_hash AS last_article_data_hash, media_hash, created_at AS last_article_published + SELECT DISTINCT ON (author_id) author_id, id, title, slug, summary, data_hash AS last_article_data_hash, media_hash, created_at AS last_article_published FROM article - WHERE state NOT IN ('archived', 'banned') + WHERE state IN ('active') ORDER BY author_id, id DESC -) t -LEFT JOIN mat_views.users_lasts u ON author_id=u.id +) a +LEFT JOIN mat_views.users_lasts ul ON author_id=ul.id LEFT JOIN public.user u2 ON author_id=u2.id LEFT JOIN user_ipns_keys k ON author_id=k.user_id LEFT JOIN ( @@ -130,12 +208,12 @@ LEFT JOIN ( WHERE state NOT IN ('archived') GROUP BY 1 ) ta USING (author_id) --- WHERE u.state NOT IN ('archived') --- WHERE user_name IN ('Brianliu', '...', 'oldcat') -WHERE t.last_article_published >= ${since} +-- WHERE -- WHERE user_name IN ('Brianliu', '...', 'oldcat') +WHERE u2.state NOT IN ('archived', 'banned') + AND a.last_article_published >= ${since} + AND (last_published IS NULL OR last_published < a.last_article_published) ORDER BY id DESC --- OFFSET floor(RANDOM() * 100 + 1)::int --- LIMIT 5000 `; +LIMIT ${limit} `; } listRecentUsers({ @@ -227,12 +305,55 @@ ORDER BY ${ LIMIT ${take} OFFSET ${skip} ; `; } + getAuthor(userName: string) { + return sqlRO< + [Item?] + >`SELECT * FROM public.user WHERE user_name=${userName}`; + } + getUserIPNSKey(userId: string | number) { + return sqlRO< + [Item?] + >`SELECT * FROM public.user_ipns_keys WHERE user_id=${userId}`; + } + async findAssetUrl(id: string | number, cf_variant = "public") { + const [asset] = await sqlRO< + [Item?] + >`SELECT * FROM public.asset WHERE id=${id}`; + if (!asset) return null; + + // genAssetUrl + const isImageType = Object.values(IMAGE_ASSET_TYPE).includes( + asset.type as any + ); + return isImageType + ? `${CLOUDFLARE_IMAGE_ENDPOINT}/${asset.path}/${cf_variant}` // this.cfsvc.genUrl(asset.path) + : `${MATTERS_AWS_S3_ENDPOINT}/${asset.path}`; + } + updateUserIPNSKey( + userId: string | number, + stats: { + lastDataHash: string; + lastPublished?: string | Date; + [key: string]: any; + }, + removeKeys: string[] = [] + ) { + const { lastDataHash, lastPublished, ...rest } = stats; + return sql< + [Item?] + >`UPDATE public.user_ipns_keys SET last_data_hash=${lastDataHash}, stats=(COALESCE(stats, '{}' ::jsonb) - ${removeKeys} ::text[]) || ${ + rest as any + } ::jsonb, updated_at=CURRENT_TIMESTAMP, last_published=COALESCE(${ + lastPublished || null + }, CURRENT_TIMESTAMP) WHERE user_id=${userId} RETURNING * ;`; + } + queryArticlesByUuid(uuids: string[]) { return sqlRO` SELECT id, title, slug, data_hash, media_hash, created_at FROM article WHERE uuid =ANY(${uuids}) `; } async checkVersion() { - const [{ version, now }] = await sql` SELECT VERSION(), NOW() `; + const [{ version, now }] = await sqlRO` SELECT VERSION(), NOW() `; console.log("pgres:", { version, now }); } } diff --git a/lib/ipfs-servers.ts b/lib/ipfs-servers.ts index 58c79f5..5feffe6 100644 --- a/lib/ipfs-servers.ts +++ b/lib/ipfs-servers.ts @@ -1,6 +1,12 @@ +import { generateKeyPair } from "crypto"; + import { Readable } from "node:stream"; // import FormData from "form-data"; import { create } from "ipfs-http-client"; +// import { Readable } from 'stream' +import { promisify } from "util"; + +const generateKeyPairPromisified = promisify(generateKeyPair); export class IPFSServerPool { #serverUrls; @@ -15,11 +21,23 @@ export class IPFSServerPool { get size() { return this.#servers.length; } + get client() { + // const idx = active ? 0 : Math.floor(1 + Math.random() * (this.size - 1)) + // return this.clients[0]; + return this.#servers[0]; + } + get backupClient() { + const idx = Math.floor(1 + Math.random() * (this.size - 1)); + return this.#servers[idx]; + } // TODO: manage multiple servers in round-robin get() { - // 97 is a large prime number - const randIdx = Math.floor(Math.random() * 97) % this.#servers.length; + // just a large prime number + // const randIdx = Math.floor(Math.random() * 97) % this.#servers.length; + const randIdx = Math.floor( + Math.pow(Math.random(), 2) * this.#servers.length + ); // leaning toward left // console.log('return randIdx:', { randIdx }); return this.#servers[randIdx]; } @@ -27,27 +45,49 @@ export class IPFSServerPool { // TODO some thing before release a node // release() {} - async importKey({ name, pem }: { name: string; pem: string }) { - const randIdx = Math.floor(Math.random() * 97) % this.#servers.length; - const ipfsServerUrl = this.#serverUrls[randIdx]; - const url = new URL(`${ipfsServerUrl}/api/v0/key/import`); - url.searchParams.set("arg", name); - url.searchParams.set("format", "pem-pkcs8-cleartext"); + // same as `openssl genpkey -algorithm ED25519` + genKey = async () => generateKeyPairPromisified("ed25519"); // { + + async importKey({ + name, + pem, + lastIdx, + }: { + name: string; + pem: string; + lastIdx?: number; + }) { + const idx = + lastIdx != null + ? (lastIdx + 1) % this.#servers.length + : Math.floor(Math.pow(Math.random(), 2) * this.#servers.length); // leaning toward left + const ipfsServerUrl = this.#serverUrls[idx]; + const u = new URL(`${ipfsServerUrl}/api/v0/key/import`); + u.searchParams.set("arg", name); + u.searchParams.set("format", "pem-pkcs8-cleartext"); const formData = new FormData(); formData.append( "file", new Blob([pem]), // Readable.from([pem]), "keyfile" ); - const imported = await fetch(url, { + const imported = await fetch(u, { method: "POST", body: formData, }).then(async (res) => { if (res.ok) return res.json(); - // console.log( "non json in import key:", res.ok, res.status, res.statusText, res.headers, await res.text()); + console.log( + new Date(), + "non json in import key:", + res.ok, + res.status, + res.statusText, + res.headers, + await res.text() + ); }); - // console.log(`imported to server:"${ipfsServerUrl}":`, imported); - return this.#servers[randIdx]; + console.log(new Date(), `imported to server:"${ipfsServerUrl}":`, imported); + return { imported, client: this.#servers[idx], lastIdx: idx }; } } diff --git a/lib/refresh-ipns-gw3.ts b/lib/refresh-ipns-gw3.ts new file mode 100644 index 0000000..5359928 --- /dev/null +++ b/lib/refresh-ipns-gw3.ts @@ -0,0 +1,803 @@ +import path from "node:path"; +import shuffle from "lodash/shuffle.js"; + +import { AuthorFeed } from "../lib/author-feed-ipns.js"; +import { ipfsPool } from "../lib/ipfs-servers.js"; +import { dbApi, Item } from "../lib/db.js"; + +const GW3_API_BASE_URL = "https://gw3.io"; +const GW3_ACCOUNT_API_BASE_URL = "https://account.gw3.io"; + +// import { Client } from 'gw3-sdk'; +const gw3AccessKey = process.env.GW3_ACCESS_KEY || ""; +const gw3AccessSecret = process.env.GW3_ACCESS_SECRET || ""; +const MATTERS_SITE_DOMAIN_PREFIX = + process.env.MATTERS_SITE_DOMAIN_PREFIX || "matters.town"; + +// export const gw3Client = new Client(gw3AccessKey, gw3AccessSecret); + +function getTs() { + return Math.floor(Date.now() / 1000).toString(); +} + +class GW3Client { + #gw3AccessKey: string; + #gw3AccessSecret: string; + #config: { baseURL: string }; + #authHeaders: { + "X-Access-Key": string; + "X-Access-Secret": string; + }; + + constructor(key: string, secret: string) { + this.#gw3AccessKey = key; + this.#gw3AccessSecret = secret; + this.#authHeaders = { + "X-Access-Key": this.#gw3AccessKey, + "X-Access-Secret": this.#gw3AccessSecret, + }; + this.#config = { baseURL: "https://gw3.io" }; + } + #makeAuthHeaders() { + return { + "X-Access-Key": this.#gw3AccessKey, + "X-Access-Secret": this.#gw3AccessSecret, + }; + } + + async addPin(cid: string, name?: string) { + const u = new URL( + `${this.#config.baseURL}/api/v0/pin/add?arg=${cid}&ts=${getTs()}` + ); + // let pinUrl = `/api/v0/pin/add?arg=${cid}&ts=${getTs()}`; + if (name) { + // pinUrl += `&name=${name}`; + u.searchParams.set("name", name); + } + + const res = await fetch(u, { + method: "POST", + headers: this.#authHeaders, // this.#makeAuthHeaders(), + }); + // console.log(new Date(), "addPin res:", res.ok, res.status, res.headers); + + return res.json(); + } + + async rmPin(cid: string) { + const u = new URL( + `${this.#config.baseURL}/api/v0/pin/rm?arg=${cid}&ts=${getTs()}` + ); + const res = await fetch(u, { + method: "POST", + headers: this.#authHeaders, // this.#makeAuthHeaders(), + }); + // console.log(new Date(), "rmPin res:", res.ok, res.status, res.headers); + + return res.json(); + } + + async getPin(cid: string) { + const u = `${GW3_ACCOUNT_API_BASE_URL}/api/v0/pin/${cid}?ts=${getTs()}`; + const res = await fetch(u, { + method: "GET", + headers: this.#authHeaders, // this.#makeAuthHeaders(), + }); + // console.log(new Date(), "getPin res:", res.ok, res.status, res.headers); + + return res.json(); + } + + async addPinWait(cid: string, name?: string, wait = 60e3) { + await this.addPin(cid, name); + do { + const res = await this.getPin(cid); + console.log(new Date(), "check addPin wait:", res?.data); + if (res.code === 200 && res.data?.status !== "pinning") { + return res.data; + } + const r = 1000 * (2 + 10 * Math.random()); + await delay(r); + + wait -= r; + } while (wait > 0); + } + + async getDAG(cid: string) { + const u = `${GW3_API_BASE_URL}/api/v0/dag/get?ts=${getTs()}&arg=${cid}`; + const res = await fetch(u, { + method: "GET", + headers: this.#authHeaders, // this.#makeAuthHeaders(), + }); + console.log( + new Date(), + "getDAG res:", + res.ok, + res.status, + res.statusText, + res.headers + ); + + return res.json(); + } + + async importFolder(cid: string) { + const res = await fetch( + `https://gw3.io/api/v0/folder/${cid}?ts=${getTs()}`, + { + method: "PUT", + headers: this.#authHeaders, // this.#makeAuthHeaders(), + } + ); + console.log( + new Date(), + "folder import res:", + res.ok, + res.status, + res.statusText, + res.headers + ); + + return res.json(); + } + + async callFolderOperation( + cid: string, + { + add, + remove, + pin_new, + unpin_old = true, + }: { + add?: [string, string][]; + remove?: string[]; + pin_new?: boolean; + unpin_old?: boolean; + } = {} + ) { + const reqBody = { cid, add, remove, pin_new, unpin_old }; + const res = await fetch( + `https://gw3.io/api/v0/folder/operation?ts=${getTs()}`, + { + method: "POST", + headers: this.#authHeaders, // this.#makeAuthHeaders(), + body: JSON.stringify(reqBody), + } + ); + console.log( + new Date(), + "folder operation res:", + res.ok, + res.status, + res.statusText, + res.headers, + reqBody + ); + return res.json(); + } + + async updateIPNSName({ ipnsKey, cid }: { ipnsKey: string; cid: string }) { + const res = await fetch( + `https://gw3.io/api/v0/name/publish?key=${ipnsKey}&arg=${cid}&ts=${getTs()}`, + { + method: "POST", + headers: this.#authHeaders, // this.#makeAuthHeaders(), + } + ); + + console.log( + new Date(), + `ipns name update res:`, + res.ok, + res.status, + res.statusText, + res.headers, + { ipnsKey, cid } + // reqBody + ); + return res.json(); + } + + async importIPNSName({ + ipnsKey, + cid, + alias, + pem, + seq = 10000, + }: { + ipnsKey: string; + cid: string; + pem: string; + alias?: string; + seq?: number; + }) { + const reqBody = { + name: ipnsKey, + value: cid, + secret_key: pem, + format: "pem-pkcs8-cleartext", + alias, + seq, + }; + const res = await fetch(`https://gw3.io/api/v0/name/import?ts=${getTs()}`, { + method: "POST", + headers: this.#authHeaders, // this.#makeAuthHeaders(), + body: JSON.stringify(reqBody), + }); + console.log( + new Date(), + "ipns name import res:", + res.ok, + res.status, + res.statusText, + res.headers, + { ipnsKey, cid, alias } + // reqBody + ); + return res.json(); + } + + async getIpns(kid: string) { + const u = `${GW3_ACCOUNT_API_BASE_URL}/api/v0/ipns/${kid}?ts=${getTs()}`; + const res = await fetch(u, { + method: "GET", + headers: this.#authHeaders, // this.#makeAuthHeaders(), + }); + // console.log(new Date(), "getPin res:", res.ok, res.status, res.headers); + + return res.json(); + } +} + +export const gw3Client = new GW3Client(gw3AccessKey, gw3AccessSecret); + +export async function refreshPinLatest({ limit = 100, offset = 0 } = {}) { + const articles = await dbApi.listRecentArticles({ + take: limit, + skip: offset, + }); + console.log( + new Date(), + `got ${articles.length} latest articles`, + articles.map( + ({ id, slug, userName }) => + `${MATTERS_SITE_DOMAIN_PREFIX}/@${userName}/${id}-${slug}` + ) + ); + const resAdd = await Promise.all( + articles.map( + async ({ id, slug, userName, dataHash }) => + dataHash && { + userName, + dataHash, + ...(await gw3Client.addPin( + dataHash, + `${MATTERS_SITE_DOMAIN_PREFIX}/@${userName}/${id}-${slug}` + )), + } + ) + ); + console.log( + new Date(), + `added ${articles.length} pins:`, + resAdd.filter((r) => r?.code !== 200) + ); + await delay(1000 * (2 + 10 * Math.random())); + + const res = await Promise.all( + articles.map( + async ({ id, slug, userName, dataHash }) => + dataHash && { + userName, + dataHash, + ...(await gw3Client.getPin(dataHash)), + } + ) + ); + const rows = res.map((r) => (r?.code === 200 ? r.data : r)); + const stats = rows.reduce((acc, item) => { + const status = item?.status ?? "no-result"; + + return Object.assign(acc, { [status]: (acc[status] ?? 0) + 1 }); + }, {}); + console.log( + new Date(), + `tried pinning ${articles.length} cids:`, + stats, + rows?.slice(0, 10) + ); +} + +const ALLOWED_USER_STATES = new Set(["active", "onboarding", "frozen"]); + +export async function refreshIPNSFeed( + userName: string, + { + limit = 50, + incremental = true, + forceReplace = false, + useMattersIPNS = false, + } = {} +) { + const [author] = await dbApi.getAuthor(userName); + console.log(new Date(), "get author:", author); + if (!author || !ALLOWED_USER_STATES.has(author?.state)) { + console.log(new Date(), `no such user:`, author); + return; + } + const [ipnsKeyRec] = await dbApi.getUserIPNSKey(author.id); + console.log(new Date(), "get user ipns:", ipnsKeyRec); + if (!ipnsKeyRec) { + // TODO: generate ipnsKeyRec here + console.error( + new Date(), + `skip no ipnsKeyRec: for author: '${author.displayName} (@${author.userName})'` + ); + return; + } + + const articles = await dbApi.listAuthorArticles({ + authorId: author.id, + take: limit, + }); + const drafts = await dbApi.listDrafts({ + ids: articles.map((item: Item) => item.draftId as string), + take: limit, + }); + + const lastArti = articles[0]; + + console.log( + new Date(), + `get ${articles.length} articles /${drafts.length} drafts for author: '${author.displayName} (@${author.userName})', last_article:`, + lastArti + ); + if (articles.length <= 10) { + forceReplace = true; + console.log( + new Date(), + `author '${author.displayName} (@${author.userName})' has ${articles.length} (<=10) articles, set all replace mode.` + ); + } + + const feed = new AuthorFeed(author, ipnsKeyRec?.ipnsKey, drafts, articles); + // console.log(new Date(), "get author feed:", feed); + await feed.loadData(); + const { html, xml, json } = feed.generate(); + // console.log(new Date(), "get author feed generated:", { html, xml, json }); + + const kname = `matters-town-homepages-topdir-for-${author.userName}-${author.uuid}`; + + const key = await ipfsPool.genKey(); + // const pem = key.privateKey.export({ format: "pem", type: "pkcs8" }) as string; + let imported, + ipfs = ipfsPool.get(); + const keyPair = { + ipnsKey: "", + pem: key.privateKey.export({ format: "pem", type: "pkcs8" }) as string, + }; + // let failures = 0; + for (let failures = 0; failures <= ipfsPool.size; failures++) { + try { + console.log( + new Date(), + "importing key:", + key // pem + ); + ({ imported, client: ipfs } = await ipfsPool.importKey({ + name: kname, + pem: keyPair.pem, + })); + if (imported) { + console.log(new Date(), "new generated key:", imported); + keyPair.ipnsKey = imported.Id; + // keyPair.pem = key.privateKey.export({ format: "pem", type: "pkcs8" }) as string; + break; + } + console.log( + new Date(), + `got nothing from imported, probably has an existing name:`, + kname + ); + await ipfs.key.rm(kname); + } catch (err) { + console.error(new Date(), "get ipfs import ERROR:", err); + } + } + if (!imported) { + console.log(new Date(), "no keys generated", ipfsPool.size); + return; + } + + // console.log(new Date, `processing ${}'s last article:`, ); + + const directoryName = `${kname}-with-${lastArti?.id || ""}-${ + lastArti?.slug || "" + }@${new Date().toISOString().substring(0, 13)}`; + + const contents = [ + { + path: `${directoryName}/index.html`, + content: html, + }, + { + path: `${directoryName}/rss.xml`, + content: xml, + }, + { + path: `${directoryName}/feed.json`, + content: json, + }, + ]; + + const addEntries: [string, string][] = []; + const promises: Promise[] = []; + + const results = []; + let topDirHash = ""; + for await (const result of ipfs.addAll(contents)) { + results.push(result); + if (result.path === directoryName) { + topDirHash = result.cid?.toString(); + promises.push(gw3Client.addPin(result.cid?.toString(), result.path)); + } else if (result.path.startsWith(directoryName) && !forceReplace) { + // replace mode will start with this new topDirHash, no need to keep pinning files inside + // get all paths below, 'index.html' 'feed.json' 'rss.xml' + // if (result.cid) + addEntries.push([path.basename(result.path), result.cid?.toString()]); + promises.push(gw3Client.addPin(result.cid?.toString(), result.path)); + } + } + await Promise.all(promises); + promises.splice(0, promises.length); + + console.log( + new Date(), + `ipfs.addAll got ${results.length} results:`, + results, + addEntries + ); + + // const res = await Promise.all( results .map(({ path, cid }) => gw3Client.addPin(cid?.toString(), path))); + // console.log(new Date(), "ipfs.addAll results pinning:", res); + + // TODO: pool wait'ing all becomes pinned + + const lastDataHash = ipnsKeyRec?.lastDataHash as string; + + let lastCid = (forceReplace ? topDirHash : lastDataHash) ?? topDirHash; // fallback to topDirHash if non existed before + + const lastCidData = await gw3Client.addPinWait( + lastCid, + `matters.town/@${author.userName}-lastest-top-dir-hash` + ); + if (lastCidData?.status !== "pinned") { + console.log( + new Date(), + `lastCid top-dir-hash: "${lastCid}" not pinned yet:`, + lastCidData + ); + return; + } + + let dagLinks = await gw3Client.getDAG(lastCid); + console.log( + new Date(), + `get prior running last cid ${lastCid} dag ${+dagLinks?.Links + ?.length} links:`, + dagLinks?.Links + ); + let existingLinks = new Map( + dagLinks?.Links?.map((e: any) => [e.Name, e]) + ); + let existingCids = new Set( + dagLinks?.Links?.map((e: any) => e.Hash["/"])?.filter(Boolean) + ); + + articles // .slice(0, limit) + .forEach((arti) => { + if (arti.dataHash && !existingCids.has(arti.dataHash)) { + addEntries.push([`${arti.id}-${arti.slug}`, arti.dataHash]); + + promises.push( + gw3Client + .addPin( + arti.dataHash, + `matters.town/@${author.userName}/${arti.id}-${arti.slug}` + ) + .then((resData) => { + if (resData?.code !== 200) { + console.error( + new Date(), + `failed add pin ${arti.dataHash}:`, + resData + ); + } + }) + ); + } + }); + console.log(new Date(), `wait adding non-existed ${promises.length} cids.`); + if (promises.length > 0) { + await Promise.all(promises); + promises.splice(0, promises.length); + + await delay(1000 * (2 + 10 * Math.random())); + } + + const res = await Promise.all( + addEntries.map(([, cid]) => gw3Client.getPin(cid)) + ); + console.log( + new Date(), + `get all 3 index + lastest articles ${addEntries.length} pinning:`, + // res, + res.map((r) => (r.code === 200 ? r.data : r)) + ); + + const waitCids = new Set([lastCid]); + addEntries // .slice(0, 10) + .forEach(([, cid]) => { + if (cid && !existingCids.has(cid)) waitCids.add(cid); + }); + + for (let i = 0; i < 10 && waitCids.size > 0; i++) { + console.log( + new Date(), + `wait on ${waitCids.size} cids to settle:`, + waitCids + ); + const res = await Promise.all( + Array.from(waitCids, (cid) => cid && gw3Client.getPin(cid)) + ); + const rows = res.map((r) => (r.code === 200 ? r.data : r)); + console.log(new Date(), `pinning status:`, rows); + rows.forEach((r) => { + if (r.status !== "pinning") waitCids.delete(r.cid); + }); + if (waitCids.size === 0) { + console.log(new Date(), "all settled (pinned or failure)."); + break; + } + await delay(1000 * (5 + 20 * Math.random())); + } + + console.log( + new Date(), + `after poll waiting, ${waitCids.size} not pinned, will add rest ${addEntries.length} entries:`, + addEntries + ); + + const toAddEntries = forceReplace + ? articles + .filter(({ dataHash }) => dataHash && !waitCids.has(dataHash)) + .map( + (arti) => + [`${arti.id}-${arti.slug}`, arti.dataHash] as [string, string] + ) + : addEntries.filter(([name, cid]) => { + if (existingLinks.get(name)?.Hash?.["/"] === cid) return false; // already there, no need to re-attach + if (waitCids.has(cid)) return false; // can no way add it if not pinned + return true; + }); + console.log( + new Date(), + `missing ${toAddEntries.length} (from ${addEntries.length}) entries:`, + toAddEntries + ); + + console.log( + new Date(), + `import ${toAddEntries.length} entries into folder:`, + lastCid + ); + while (toAddEntries.length > 0) { + // await gw3Client.addPin(lastCid, `matters-town-homepages/for-${author.userName}-latest-top-dir-hash`); + await gw3Client.importFolder(lastCid); + // after PUT folder, better to wait 1sec + // otherwise, often got { code: 400, msg: 'PIN resource is locked for processing by the system' } + // await delay(1000 * (1 + 10 * Math.random())); + + const resFolderOps = await gw3Client.callFolderOperation(lastCid, { + add: toAddEntries.splice(0, 50), // gw3 API changed the limit to 50 since 8/28 + pin_new: true, + unpin_old: true, + }); + console.log( + new Date(), + "folder/operation res:", + toAddEntries.length, + resFolderOps + ); + if (!resFolderOps?.data?.cid) { + console.error( + new Date(), + `folder/operation failed, stopped at ${lastCid}: with ${toAddEntries.length} remaining addEntries.` + ); + break; + } + const prior = lastCid; + lastCid = resFolderOps.data.cid; + console.log( + new Date(), + "folder/operation after attached new cid:", + lastCid + ); + + if (incremental) { + console.log( + new Date(), + `once folder/operation only for incremental mode, still has ${toAddEntries.length} missing.` + ); + break; + } + + // toAddEntries.splice(0, 10); + shuffle(toAddEntries); + // await gw3Client.addPin( lastCid, `matters-town-homepages/for-${author.userName}-latest`); + // TODO: wait it becomes 'pinned' + + // if (prior !== lastCid) gw3Client.rmPin(prior); // no need to wait, let it run async with best effort + } + + if (ipnsKeyRec?.stats?.testGw3IPNSKey && ipnsKeyRec?.stats?.pem) { + keyPair.ipnsKey = ipnsKeyRec.stats.testGw3IPNSKey; + keyPair.pem = ipnsKeyRec.stats.pem; + } + if (useMattersIPNS && ipnsKeyRec?.privKeyPem) { + console.log( + new Date(), + `use matters pre-generated keypair:`, + useMattersIPNS + ); + keyPair.ipnsKey = ipnsKeyRec.ipnsKey; + keyPair.pem = ipnsKeyRec.privKeyPem; + } + + const testGw3IPNSKey = keyPair.ipnsKey; // imported.Id; + const resGetIpns1 = await gw3Client.getIpns(testGw3IPNSKey); + // console.log(new Date(), "ipns name get:", resGetIpns2); + console.log(new Date(), `ipns name get ${testGw3IPNSKey}:`, resGetIpns1); + + let resIPNS; + + if (resGetIpns1?.code === 200) { + // existed: do update + if (resGetIpns1?.data.value === lastCid) { + console.log(new Date(), `lastCid remained the same, no need to update:`, { + testGw3IPNSKey, + lastCid, + }); + return ipnsKeyRec?.stats; + } + + resIPNS = await gw3Client.updateIPNSName({ + ipnsKey: testGw3IPNSKey, + cid: lastCid, + }); + console.log(new Date(), `updated ipns:`, resIPNS); + } else { + // not existed: do import + + const importArgs = { + ipnsKey: testGw3IPNSKey, + cid: lastCid, + pem: keyPair.pem, + alias: `matters.town/@${author.userName}`, + }; + resIPNS = await gw3Client.importIPNSName(importArgs); + // console.log(new Date(), "ipns name (import) publish:", resImport, { lastDataHash, lastCid, }); + } + if (resIPNS?.code !== 200) { + console.error(new Date(), `failed ipns update:`, resIPNS, { + lastDataHash, + lastCid, + }); + return; + } + + const resGetIpns2 = await gw3Client.getIpns(testGw3IPNSKey); + // console.log(new Date(), "ipns name get:", resGetIpns2); + console.log(new Date(), `ipns name get ${testGw3IPNSKey}:`, resGetIpns2); + + dagLinks = await gw3Client.getDAG(lastCid); // retrieve again + console.log( + new Date(), + `get current running last cid ${lastCid} dag ${+dagLinks?.Links + ?.length} links:`, + dagLinks?.Links + ); + existingLinks = new Map( + dagLinks?.Links?.map((e: any) => [e.Name, e]) + ); + existingCids = new Set( + dagLinks?.Links?.map((e: any) => e.Hash["/"])?.filter(Boolean) + ); + + // add all sub-links to unpin + const toUnpinCids = new Set(existingCids); + + const missingEntries = articles.filter( + ({ id, slug, dataHash }) => + // !(existingLinks.get(name)?.Hash["/"] === cid) && waitCids.has(cid) + !existingLinks.has(`${id}-${slug}`) + ); + const missingEntriesInLast50 = articles.slice(0, 50).filter( + ({ id, slug, dataHash }) => + // !(existingLinks.get(name)?.Hash["/"] === cid) && waitCids.has(cid) + !existingLinks.has(`${id}-${slug}`) + ); + if (missingEntries.length > 0) { + console.log( + new Date(), + `still has ${missingEntries.length} entries missing:`, + missingEntries.map(({ id, slug, dataHash }) => ({ id, slug, dataHash })) + ); + } + + const statsData = { + userName: author.userName, + limit: Math.max(articles.length, drafts.length), + missing: missingEntries.length, + missingInLast50: + missingEntries.length === 0 ? undefined : missingEntriesInLast50.length, + testGw3IPNSKey, + pem: keyPair.pem, + lastDataHash: lastCid, + // lastPublished: ipnsKeyRecUpdated?.lastPublished, + lastPublished: resGetIpns2?.data?.publish_at + ? new Date(resGetIpns2.data.publish_at * 1000) + : undefined, + retriesAfterMissing: + missingEntries.length > 0 + ? (ipnsKeyRec?.stats?.retriesAfterMissing ?? 0) + 1 + : undefined, + useMattersIPNS: useMattersIPNS || undefined, + }; + let ipnsKeyRecUpdated: Item = ipnsKeyRec!; + if (lastCid !== lastDataHash) { + const [ret] = await dbApi.updateUserIPNSKey( + author.id, + statsData, + missingEntries.length > 0 ? undefined : ["retriesAfterMissing"] + ); + console.log(new Date(), "ipns name updated:", ret); + if (ret) { + ipnsKeyRecUpdated = ret; + // statsData.lastPublished = ret.lastPublished; + } + } + + console.log( + new Date(), + `updated for author: '${author.displayName} (@${author.userName})': get ${articles.length} articles /${drafts.length} drafts`, + missingEntriesInLast50.map(({ id, slug, dataHash }) => ({ + id, + slug, + dataHash, + })), + statsData + ); + + const newTopDagLinks = await gw3Client.getDAG(lastCid); + newTopDagLinks?.Links?.forEach((e: any) => toUnpinCids.add(e?.Hash["/"])); + + toUnpinCids.delete(lastCid); + toUnpinCids.delete(ipnsKeyRecUpdated?.lastDataHash); + if (toUnpinCids.size > 0) { + console.log( + new Date(), + `try unpin all ${toUnpinCids.size} unneeded pins:` + // toUnpinCids + ); + toUnpinCids.forEach((cid) => { + gw3Client.rmPin(cid); + }); + } + + return statsData; +} + +function delay(timeout: number) { + return new Promise((fulfilled) => setTimeout(fulfilled, timeout)); +} diff --git a/package-lock.json b/package-lock.json index 7458112..ab0f8ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,27 +1,31 @@ { "name": "lambda-handlers-image", - "version": "0.6.1", + "version": "0.6.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "lambda-handlers-image", - "version": "0.6.1", + "version": "0.6.3", "dependencies": { "@aws-sdk/client-s3": "^3.267.0", "@aws-sdk/client-sqs": "^3.266.0", "@matters/apollo-response-cache": "^1.4.0-rc.0", - "@matters/ipns-site-generator": "^0.1.0", + "@matters/ipns-site-generator": "^0.1.3", + "@matters/slugify": "^0.7.3", "@sendgrid/helpers": "^7.7.0", "@sendgrid/mail": "^7.7.0", "@slack/web-api": "^6.8.1", "axios": "^1.3.2", "cheerio": "^1.0.0-rc.12", "debug": "^4.3.4", + "gw3-sdk": "^0.3.0", "ioredis": "^5.2.4", "ipfs-http-client": "^59.0.0", "js-base64": "^3.7.4", "knex": "^2.3.0", + "lodash": "^4.17.21", + "lodash.shuffle": "^4.2.0", "lodash.uniqby": "^4.7.0", "meilisearch": "^0.30.0", "objection": "^3.0.1", @@ -4411,9 +4415,9 @@ } }, "node_modules/@matters/ipns-site-generator": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@matters/ipns-site-generator/-/ipns-site-generator-0.1.0.tgz", - "integrity": "sha512-pKyXbtxTMrGb63qzWgB2+UdRVYjxg67v1E4/nKMSBYC52K0YJP4hYQmjgaoHYGjOjezVQG6Uf2X5OtaANQieKA==", + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@matters/ipns-site-generator/-/ipns-site-generator-0.1.3.tgz", + "integrity": "sha512-SfgfMTYhQGJr75LhpC9K9xxd+WcyJQPKSKXS39Qctot7zLZjbmySHm7gHlLMlcgjNVENLXFS+xYHHwWIJvJzGA==", "dependencies": { "@peculiar/webcrypto": "^1.1.6", "cheerio": "^1.0.0-rc.9", @@ -4422,6 +4426,26 @@ "nunjucks": "^3.2.3" } }, + "node_modules/@matters/slugify": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/@matters/slugify/-/slugify-0.7.3.tgz", + "integrity": "sha512-u0+rl/R1WbYtGPAFrXt9kDh8CBRd21gz7kJyM87sOMUfrXwY9oj1PAoqjtzHTyNzqSlDGetvtR1cIOntOsKgZw==", + "dependencies": { + "escape-string-regexp": "^1.0.5", + "lodash.deburr": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@matters/slugify/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/@multiformats/multiaddr": { "version": "11.3.0", "resolved": "https://registry.npmjs.org/@multiformats/multiaddr/-/multiaddr-11.3.0.tgz", @@ -5510,6 +5534,22 @@ "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, + "node_modules/asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "dependencies": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + } + }, + "node_modules/asn1.js/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, "node_modules/asn1js": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/asn1js/-/asn1js-3.0.5.tgz", @@ -5545,9 +5585,9 @@ } }, "node_modules/axios": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.3.2.tgz", - "integrity": "sha512-1M3O703bYqYuPhbHeya5bnhpYVsDDRyQSabNja04mZtboLNSuZ4YrltestrLXfHgmzua4TpUqRiVKbiQuo2epw==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz", + "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==", "dependencies": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", @@ -5695,6 +5735,11 @@ "npm": ">=7.0.0" } }, + "node_modules/bn.js": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz", + "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==" + }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -5726,6 +5771,11 @@ "node": ">=8" } }, + "node_modules/brorand": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==" + }, "node_modules/browser-readablestream-to-it": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/browser-readablestream-to-it/-/browser-readablestream-to-it-2.0.0.tgz", @@ -5735,6 +5785,65 @@ "npm": ">=7.0.0" } }, + "node_modules/browserify-aes": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", + "dependencies": { + "buffer-xor": "^1.0.3", + "cipher-base": "^1.0.0", + "create-hash": "^1.1.0", + "evp_bytestokey": "^1.0.3", + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/browserify-cipher": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz", + "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==", + "dependencies": { + "browserify-aes": "^1.0.4", + "browserify-des": "^1.0.0", + "evp_bytestokey": "^1.0.0" + } + }, + "node_modules/browserify-des": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz", + "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==", + "dependencies": { + "cipher-base": "^1.0.1", + "des.js": "^1.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "node_modules/browserify-rsa": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz", + "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==", + "dependencies": { + "bn.js": "^5.0.0", + "randombytes": "^2.0.1" + } + }, + "node_modules/browserify-sign": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz", + "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==", + "dependencies": { + "bn.js": "^5.1.1", + "browserify-rsa": "^4.0.1", + "create-hash": "^1.2.0", + "create-hmac": "^1.1.7", + "elliptic": "^6.5.3", + "inherits": "^2.0.4", + "parse-asn1": "^5.1.5", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + } + }, "node_modules/browserslist": { "version": "4.21.4", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.4.tgz", @@ -5821,6 +5930,11 @@ "node": ">=4" } }, + "node_modules/buffer-xor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", + "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==" + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -5981,6 +6095,15 @@ "node": ">=8" } }, + "node_modules/cipher-base": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", + "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==", + "dependencies": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/cjs-module-lexer": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.2.tgz", @@ -6105,6 +6228,45 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, + "node_modules/create-ecdh": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", + "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==", + "dependencies": { + "bn.js": "^4.1.0", + "elliptic": "^6.5.3" + } + }, + "node_modules/create-ecdh/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, + "node_modules/create-hash": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", + "dependencies": { + "cipher-base": "^1.0.1", + "inherits": "^2.0.1", + "md5.js": "^1.3.4", + "ripemd160": "^2.0.1", + "sha.js": "^2.4.0" + } + }, + "node_modules/create-hmac": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", + "dependencies": { + "cipher-base": "^1.0.3", + "create-hash": "^1.1.0", + "inherits": "^2.0.1", + "ripemd160": "^2.0.0", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, "node_modules/create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -6152,6 +6314,32 @@ "node": ">= 8" } }, + "node_modules/crypto-browserify": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", + "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==", + "dependencies": { + "browserify-cipher": "^1.0.0", + "browserify-sign": "^4.0.0", + "create-ecdh": "^4.0.0", + "create-hash": "^1.1.0", + "create-hmac": "^1.1.0", + "diffie-hellman": "^5.0.0", + "inherits": "^2.0.1", + "pbkdf2": "^3.0.3", + "public-encrypt": "^4.0.0", + "randombytes": "^2.0.0", + "randomfill": "^1.0.3" + }, + "engines": { + "node": "*" + } + }, + "node_modules/crypto-js": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.1.1.tgz", + "integrity": "sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==" + }, "node_modules/css-select": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", @@ -6272,6 +6460,15 @@ "node": ">=0.10" } }, + "node_modules/des.js": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.1.0.tgz", + "integrity": "sha512-r17GxjhUCjSRy8aiJpr8/UadFIzMzJGexI3Nmz4ADi9LYSFx4gTBp80+NaX/YsXWWLhpZ7v/v/ubEc/bCNfKwg==", + "dependencies": { + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0" + } + }, "node_modules/detect-libc": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", @@ -6308,6 +6505,21 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/diffie-hellman": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", + "dependencies": { + "bn.js": "^4.1.0", + "miller-rabin": "^4.0.0", + "randombytes": "^2.0.0" + } + }, + "node_modules/diffie-hellman/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -6424,6 +6636,25 @@ "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", "dev": true }, + "node_modules/elliptic": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", + "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", + "dependencies": { + "bn.js": "^4.11.9", + "brorand": "^1.1.0", + "hash.js": "^1.0.0", + "hmac-drbg": "^1.0.1", + "inherits": "^2.0.4", + "minimalistic-assert": "^1.0.1", + "minimalistic-crypto-utils": "^1.0.1" + } + }, + "node_modules/elliptic/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, "node_modules/emittery": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", @@ -6804,6 +7035,15 @@ "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz", "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==" }, + "node_modules/evp_bytestokey": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz", + "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==", + "dependencies": { + "md5.js": "^1.3.4", + "safe-buffer": "^5.1.1" + } + }, "node_modules/execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -7315,6 +7555,17 @@ "node": ">= 10.x" } }, + "node_modules/gw3-sdk": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/gw3-sdk/-/gw3-sdk-0.3.0.tgz", + "integrity": "sha512-WYvCVev/D6nrQyBbi+Chqk2ZjIuycxanwSHFQbXwm4F7geoGtgXEF07xxpfNCb8D4mYCG0BYi0Z5HLIdmhP9bw==", + "dependencies": { + "axios": "^1.4.0", + "crypto-browserify": "^3.12.0", + "crypto-js": "^4.1.1", + "safe-buffer": "^5.2.1" + } + }, "node_modules/has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -7395,6 +7646,38 @@ "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==" }, + "node_modules/hash-base": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.0.tgz", + "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==", + "dependencies": { + "inherits": "^2.0.4", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/hash.js": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "dependencies": { + "inherits": "^2.0.3", + "minimalistic-assert": "^1.0.1" + } + }, + "node_modules/hmac-drbg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg==", + "dependencies": { + "hash.js": "^1.0.3", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.1" + } + }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -8988,6 +9271,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.deburr": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/lodash.deburr/-/lodash.deburr-4.1.0.tgz", + "integrity": "sha512-m/M1U1f3ddMCs6Hq2tAsYThTBDaAKFDX3dwDo97GEYzamXi9SqUpjWi/Rrj/gf3X2n8ktwgZrlP1z6E3v/IExQ==" + }, "node_modules/lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -9010,6 +9298,11 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, + "node_modules/lodash.shuffle": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.shuffle/-/lodash.shuffle-4.2.0.tgz", + "integrity": "sha512-V/rTAABKLFjoecTZjKSv+A1ZomG8hZg8hlgeG6wwQVD9AGv+10zqqSf6mFq2tVA703Zd5R0YhSuSlXA+E/Ei+Q==" + }, "node_modules/lodash.uniqby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz", @@ -9079,6 +9372,16 @@ "tmpl": "1.0.5" } }, + "node_modules/md5.js": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", + "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==", + "dependencies": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, "node_modules/meilisearch": { "version": "0.30.0", "resolved": "https://registry.npmjs.org/meilisearch/-/meilisearch-0.30.0.tgz", @@ -9126,6 +9429,23 @@ "node": ">=8.6" } }, + "node_modules/miller-rabin": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz", + "integrity": "sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==", + "dependencies": { + "bn.js": "^4.0.0", + "brorand": "^1.0.1" + }, + "bin": { + "miller-rabin": "bin/miller-rabin" + } + }, + "node_modules/miller-rabin/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -9154,6 +9474,16 @@ "node": ">=6" } }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, + "node_modules/minimalistic-crypto-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==" + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -10000,6 +10330,18 @@ "node": ">=6" } }, + "node_modules/parse-asn1": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", + "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==", + "dependencies": { + "asn1.js": "^5.2.0", + "browserify-aes": "^1.0.0", + "evp_bytestokey": "^1.0.0", + "pbkdf2": "^3.0.3", + "safe-buffer": "^5.1.1" + } + }, "node_modules/parse-duration": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-1.0.2.tgz", @@ -10096,6 +10438,21 @@ "node": ">=8" } }, + "node_modules/pbkdf2": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz", + "integrity": "sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==", + "dependencies": { + "create-hash": "^1.1.2", + "create-hmac": "^1.1.4", + "ripemd160": "^2.0.1", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + }, + "engines": { + "node": ">=0.12" + } + }, "node_modules/pg": { "version": "8.9.0", "resolved": "https://registry.npmjs.org/pg/-/pg-8.9.0.tgz", @@ -10406,6 +10763,24 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "node_modules/public-encrypt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", + "integrity": "sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==", + "dependencies": { + "bn.js": "^4.1.0", + "browserify-rsa": "^4.0.0", + "create-hash": "^1.1.0", + "parse-asn1": "^5.0.0", + "randombytes": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "node_modules/public-encrypt/node_modules/bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + }, "node_modules/punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", @@ -10469,6 +10844,23 @@ } ] }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/randomfill": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/randomfill/-/randomfill-1.0.4.tgz", + "integrity": "sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==", + "dependencies": { + "randombytes": "^2.0.5", + "safe-buffer": "^5.1.0" + } + }, "node_modules/rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -10694,6 +11086,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/ripemd160": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", + "integrity": "sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==", + "dependencies": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -10779,6 +11180,18 @@ "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" }, + "node_modules/sha.js": { + "version": "2.4.11", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", + "dependencies": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + }, + "bin": { + "sha.js": "bin.js" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -15446,9 +15859,9 @@ } }, "@matters/ipns-site-generator": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@matters/ipns-site-generator/-/ipns-site-generator-0.1.0.tgz", - "integrity": "sha512-pKyXbtxTMrGb63qzWgB2+UdRVYjxg67v1E4/nKMSBYC52K0YJP4hYQmjgaoHYGjOjezVQG6Uf2X5OtaANQieKA==", + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@matters/ipns-site-generator/-/ipns-site-generator-0.1.3.tgz", + "integrity": "sha512-SfgfMTYhQGJr75LhpC9K9xxd+WcyJQPKSKXS39Qctot7zLZjbmySHm7gHlLMlcgjNVENLXFS+xYHHwWIJvJzGA==", "requires": { "@peculiar/webcrypto": "^1.1.6", "cheerio": "^1.0.0-rc.9", @@ -15457,6 +15870,22 @@ "nunjucks": "^3.2.3" } }, + "@matters/slugify": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/@matters/slugify/-/slugify-0.7.3.tgz", + "integrity": "sha512-u0+rl/R1WbYtGPAFrXt9kDh8CBRd21gz7kJyM87sOMUfrXwY9oj1PAoqjtzHTyNzqSlDGetvtR1cIOntOsKgZw==", + "requires": { + "escape-string-regexp": "^1.0.5", + "lodash.deburr": "^4.1.0" + }, + "dependencies": { + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" + } + } + }, "@multiformats/multiaddr": { "version": "11.3.0", "resolved": "https://registry.npmjs.org/@multiformats/multiaddr/-/multiaddr-11.3.0.tgz", @@ -16310,6 +16739,24 @@ "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, + "asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "requires": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, "asn1js": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/asn1js/-/asn1js-3.0.5.tgz", @@ -16338,9 +16785,9 @@ "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==" }, "axios": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.3.2.tgz", - "integrity": "sha512-1M3O703bYqYuPhbHeya5bnhpYVsDDRyQSabNja04mZtboLNSuZ4YrltestrLXfHgmzua4TpUqRiVKbiQuo2epw==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz", + "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==", "requires": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", @@ -16445,6 +16892,11 @@ "browser-readablestream-to-it": "^2.0.0" } }, + "bn.js": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.1.tgz", + "integrity": "sha512-eXRvHzWyYPBuB4NBy0cmYQjGitUrtqwbvlzP3G6VFnNRbsZQIxQ10PbKKHt8gZ/HW/D/747aDl+QkDqg3KQLMQ==" + }, "boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -16473,11 +16925,75 @@ "fill-range": "^7.0.1" } }, + "brorand": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==" + }, "browser-readablestream-to-it": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/browser-readablestream-to-it/-/browser-readablestream-to-it-2.0.0.tgz", "integrity": "sha512-x7L6NN0FF0LchYKA7D5x2/oJ+n6Y8A0gFaazIxH2AkHr+fjFJvsDUYLLQKAfIkpKiLjQEkbjF0DBw7HRT1ylNA==" }, + "browserify-aes": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", + "requires": { + "buffer-xor": "^1.0.3", + "cipher-base": "^1.0.0", + "create-hash": "^1.1.0", + "evp_bytestokey": "^1.0.3", + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "browserify-cipher": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz", + "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==", + "requires": { + "browserify-aes": "^1.0.4", + "browserify-des": "^1.0.0", + "evp_bytestokey": "^1.0.0" + } + }, + "browserify-des": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz", + "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==", + "requires": { + "cipher-base": "^1.0.1", + "des.js": "^1.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "browserify-rsa": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz", + "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==", + "requires": { + "bn.js": "^5.0.0", + "randombytes": "^2.0.1" + } + }, + "browserify-sign": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz", + "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==", + "requires": { + "bn.js": "^5.1.1", + "browserify-rsa": "^4.0.1", + "create-hash": "^1.2.0", + "create-hmac": "^1.1.7", + "elliptic": "^6.5.3", + "inherits": "^2.0.4", + "parse-asn1": "^5.1.5", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + } + }, "browserslist": { "version": "4.21.4", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.4.tgz", @@ -16528,6 +17044,11 @@ "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" }, + "buffer-xor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", + "integrity": "sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==" + }, "busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -16633,6 +17154,15 @@ "integrity": "sha512-4jYS4MOAaCIStSRwiuxc4B8MYhIe676yO1sYGzARnjXkWpmzZMMYxY6zu8WYWDhSuth5zhrQ1rhNSibyyvv4/w==", "dev": true }, + "cipher-base": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", + "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==", + "requires": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "cjs-module-lexer": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.2.2.tgz", @@ -16732,6 +17262,47 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, + "create-ecdh": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", + "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==", + "requires": { + "bn.js": "^4.1.0", + "elliptic": "^6.5.3" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, + "create-hash": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", + "requires": { + "cipher-base": "^1.0.1", + "inherits": "^2.0.1", + "md5.js": "^1.3.4", + "ripemd160": "^2.0.1", + "sha.js": "^2.4.0" + } + }, + "create-hmac": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", + "requires": { + "cipher-base": "^1.0.3", + "create-hash": "^1.1.0", + "inherits": "^2.0.1", + "ripemd160": "^2.0.0", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, "create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -16767,6 +17338,29 @@ "which": "^2.0.1" } }, + "crypto-browserify": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", + "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==", + "requires": { + "browserify-cipher": "^1.0.0", + "browserify-sign": "^4.0.0", + "create-ecdh": "^4.0.0", + "create-hash": "^1.1.0", + "create-hmac": "^1.1.0", + "diffie-hellman": "^5.0.0", + "inherits": "^2.0.1", + "pbkdf2": "^3.0.3", + "public-encrypt": "^4.0.0", + "randombytes": "^2.0.0", + "randomfill": "^1.0.3" + } + }, + "crypto-js": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.1.1.tgz", + "integrity": "sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==" + }, "css-select": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", @@ -16852,6 +17446,15 @@ "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==" }, + "des.js": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.1.0.tgz", + "integrity": "sha512-r17GxjhUCjSRy8aiJpr8/UadFIzMzJGexI3Nmz4ADi9LYSFx4gTBp80+NaX/YsXWWLhpZ7v/v/ubEc/bCNfKwg==", + "requires": { + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0" + } + }, "detect-libc": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz", @@ -16876,6 +17479,23 @@ "integrity": "sha512-hlM3QR272NXCi4pq+N4Kok4kOp6EsgOM3ZSpJI7Da3UAs+Ttsi8MRmB6trM/lhyzUxGfOgnpkHtgqm5Q/CTcfQ==", "dev": true }, + "diffie-hellman": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", + "requires": { + "bn.js": "^4.1.0", + "miller-rabin": "^4.0.0", + "randombytes": "^2.0.0" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, "dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -16958,6 +17578,27 @@ "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", "dev": true }, + "elliptic": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", + "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", + "requires": { + "bn.js": "^4.11.9", + "brorand": "^1.1.0", + "hash.js": "^1.0.0", + "hmac-drbg": "^1.0.1", + "inherits": "^2.0.4", + "minimalistic-assert": "^1.0.1", + "minimalistic-crypto-utils": "^1.0.1" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, "emittery": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", @@ -17243,6 +17884,15 @@ "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz", "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==" }, + "evp_bytestokey": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz", + "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==", + "requires": { + "md5.js": "^1.3.4", + "safe-buffer": "^5.1.1" + } + }, "execa": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", @@ -17619,6 +18269,17 @@ "integrity": "sha512-5gghUc24tP9HRznNpV2+FIoq3xKkj5dTQqf4v0CpdPbFVwFkWoxOM+o+2OC9ZSvjEMTjfmG9QT+gcvggTwW1zw==", "peer": true }, + "gw3-sdk": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/gw3-sdk/-/gw3-sdk-0.3.0.tgz", + "integrity": "sha512-WYvCVev/D6nrQyBbi+Chqk2ZjIuycxanwSHFQbXwm4F7geoGtgXEF07xxpfNCb8D4mYCG0BYi0Z5HLIdmhP9bw==", + "requires": { + "axios": "^1.4.0", + "crypto-browserify": "^3.12.0", + "crypto-js": "^4.1.1", + "safe-buffer": "^5.2.1" + } + }, "has": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", @@ -17669,6 +18330,35 @@ "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==" }, + "hash-base": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.0.tgz", + "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + } + }, + "hash.js": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "requires": { + "inherits": "^2.0.3", + "minimalistic-assert": "^1.0.1" + } + }, + "hmac-drbg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha512-Tti3gMqLdZfhOQY1Mzf/AanLiqh1WTiJgEj26ZuYQ9fbkLomzGchCws4FyrSd4VkpBfiNhaE1On+lOz894jvXg==", + "requires": { + "hash.js": "^1.0.3", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.1" + } + }, "html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -18826,6 +19516,11 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "lodash.deburr": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/lodash.deburr/-/lodash.deburr-4.1.0.tgz", + "integrity": "sha512-m/M1U1f3ddMCs6Hq2tAsYThTBDaAKFDX3dwDo97GEYzamXi9SqUpjWi/Rrj/gf3X2n8ktwgZrlP1z6E3v/IExQ==" + }, "lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", @@ -18848,6 +19543,11 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, + "lodash.shuffle": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.shuffle/-/lodash.shuffle-4.2.0.tgz", + "integrity": "sha512-V/rTAABKLFjoecTZjKSv+A1ZomG8hZg8hlgeG6wwQVD9AGv+10zqqSf6mFq2tVA703Zd5R0YhSuSlXA+E/Ei+Q==" + }, "lodash.uniqby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz", @@ -18907,6 +19607,16 @@ "tmpl": "1.0.5" } }, + "md5.js": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", + "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==", + "requires": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, "meilisearch": { "version": "0.30.0", "resolved": "https://registry.npmjs.org/meilisearch/-/meilisearch-0.30.0.tgz", @@ -18945,6 +19655,22 @@ "picomatch": "^2.3.1" } }, + "miller-rabin": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz", + "integrity": "sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==", + "requires": { + "bn.js": "^4.0.0", + "brorand": "^1.0.1" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, "mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -18964,6 +19690,16 @@ "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", "dev": true }, + "minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + }, + "minimalistic-crypto-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==" + }, "minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -19621,6 +20357,18 @@ "callsites": "^3.0.0" } }, + "parse-asn1": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", + "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==", + "requires": { + "asn1.js": "^5.2.0", + "browserify-aes": "^1.0.0", + "evp_bytestokey": "^1.0.0", + "pbkdf2": "^3.0.3", + "safe-buffer": "^5.1.1" + } + }, "parse-duration": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/parse-duration/-/parse-duration-1.0.2.tgz", @@ -19693,6 +20441,18 @@ "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", "dev": true }, + "pbkdf2": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz", + "integrity": "sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==", + "requires": { + "create-hash": "^1.1.2", + "create-hmac": "^1.1.4", + "ripemd160": "^2.0.1", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, "pg": { "version": "8.9.0", "resolved": "https://registry.npmjs.org/pg/-/pg-8.9.0.tgz", @@ -19919,6 +20679,26 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "public-encrypt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", + "integrity": "sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==", + "requires": { + "bn.js": "^4.1.0", + "browserify-rsa": "^4.0.0", + "create-hash": "^1.1.0", + "parse-asn1": "^5.0.0", + "randombytes": "^2.0.1", + "safe-buffer": "^5.1.2" + }, + "dependencies": { + "bn.js": { + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", + "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + } + } + }, "punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", @@ -19958,6 +20738,23 @@ "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "dev": true }, + "randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "requires": { + "safe-buffer": "^5.1.0" + } + }, + "randomfill": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/randomfill/-/randomfill-1.0.4.tgz", + "integrity": "sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==", + "requires": { + "randombytes": "^2.0.5", + "safe-buffer": "^5.1.0" + } + }, "rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -20120,6 +20917,15 @@ "glob": "^7.1.3" } }, + "ripemd160": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", + "integrity": "sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==", + "requires": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1" + } + }, "run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -20168,6 +20974,15 @@ "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" }, + "sha.js": { + "version": "2.4.11", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", + "requires": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", diff --git a/package.json b/package.json index 97253c9..10fff35 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lambda-handlers-image", - "version": "0.6.2", + "version": "0.6.3", "private": true, "type": "module", "scripts": { @@ -32,17 +32,21 @@ "@aws-sdk/client-s3": "^3.267.0", "@aws-sdk/client-sqs": "^3.266.0", "@matters/apollo-response-cache": "^1.4.0-rc.0", - "@matters/ipns-site-generator": "^0.1.0", + "@matters/ipns-site-generator": "^0.1.3", + "@matters/slugify": "^0.7.3", "@sendgrid/helpers": "^7.7.0", "@sendgrid/mail": "^7.7.0", "@slack/web-api": "^6.8.1", "axios": "^1.3.2", "cheerio": "^1.0.0-rc.12", "debug": "^4.3.4", + "gw3-sdk": "^0.3.0", "ioredis": "^5.2.4", "ipfs-http-client": "^59.0.0", "js-base64": "^3.7.4", "knex": "^2.3.0", + "lodash": "^4.17.21", + "lodash.shuffle": "^4.2.0", "lodash.uniqby": "^4.7.0", "meilisearch": "^0.30.0", "objection": "^3.0.1",