From e3bafb26dd069669e229f20a024718800fedf985 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Mon, 13 Nov 2023 22:33:27 +0800 Subject: [PATCH] Fix internal links and images replace. --- feishu-pages/package.json | 4 +- feishu-pages/src/doc.ts | 13 +-- feishu-pages/src/feishu.ts | 18 ++-- feishu-pages/src/index.ts | 21 +++-- feishu-pages/src/utils.ts | 54 +++++++---- feishu-pages/tests/doc.test.ts | 8 +- feishu-pages/tests/utils.test.ts | 148 ++++++++++++++++++++----------- 7 files changed, 172 insertions(+), 94 deletions(-) diff --git a/feishu-pages/package.json b/feishu-pages/package.json index 71a7e71..14a7bde 100644 --- a/feishu-pages/package.json +++ b/feishu-pages/package.json @@ -24,7 +24,7 @@ "@types/node": "^20.5.7", "axios": "^1.5.0", "dotenv": "^16.3.1", - "feishu-docx": "0.5.2", + "feishu-docx": "0.5.4", "mime-types": "^2.1.35", "typescript": "^5.2.2" }, @@ -33,4 +33,4 @@ "jest": "^29.6.4", "ts-jest": "^29.1.1" } -} \ No newline at end of file +} diff --git a/feishu-pages/src/doc.ts b/feishu-pages/src/doc.ts index 9a1d4f8..5437918 100644 --- a/feishu-pages/src/doc.ts +++ b/feishu-pages/src/doc.ts @@ -82,16 +82,19 @@ export const generateFileMeta = ( sidebar_position: position, }; - // Replace double quote to avoid YAML parse error - meta.title = meta.title.replace(/"/g, '\\"'); - let output = `---\n`; for (const key in meta) { - const val = meta[key]; + let val = meta[key]; if (val === null || val === undefined) { continue; } - output += `${key}: "${val}"\n`; + + // Replace double quote to avoid YAML parse error + if (typeof val === 'string') { + val = val.replace(/"/g, '\\"'); + val = `"${val}"`; + } + output += `${key}: ${val}\n`; } output += `---\n`; diff --git a/feishu-pages/src/feishu.ts b/feishu-pages/src/feishu.ts index 5acdced..06f5e19 100644 --- a/feishu-pages/src/feishu.ts +++ b/feishu-pages/src/feishu.ts @@ -218,7 +218,9 @@ export const feishuDownload = async (fileToken: string, localPath: string) => { fs.mkdirSync(CACHE_DIR, { recursive: true }); let res: any = {}; + let hasCache = false; if (fs.existsSync(cacheFilePath) && fs.existsSync(cacheFileMetaPath)) { + hasCache = true; res.data = fs.readFileSync(cacheFilePath); res.headers = JSON.parse(fs.readFileSync(cacheFileMetaPath, 'utf-8')); console.info(' -> Cache hit:', fileToken); @@ -261,18 +263,22 @@ export const feishuDownload = async (fileToken: string, localPath: string) => { if (res.data) { let extension = mime.extension(res.headers['content-type']); - console.info( - ' =>', - res.headers['content-type'], - humanizeFileSize(res.data.length) - ); + if (!hasCache) { + console.info( + ' =>', + res.headers['content-type'], + humanizeFileSize(res.data.length) + ); + } if (extension) { localPath = localPath + '.' + extension; } const dir = path.dirname(localPath); fs.mkdirSync(dir, { recursive: true }); - console.info(' -> Writing file:', localPath); + if (!hasCache) { + console.info(' -> Writing file:', localPath); + } fs.writeFileSync(localPath, res.data); } diff --git a/feishu-pages/src/index.ts b/feishu-pages/src/index.ts index 4acf67d..50d53ca 100644 --- a/feishu-pages/src/index.ts +++ b/feishu-pages/src/index.ts @@ -13,7 +13,7 @@ import { fetchTenantAccessToken, } from './feishu'; import { FileDoc, generateSummary, prepareDocSlugs } from './summary'; -import { cleanupDocsForJSON, humanizeFileSize } from './utils'; +import { cleanupDocsForJSON, humanizeFileSize, replaceLinks } from './utils'; import { fetchAllDocs } from './wiki'; // App entry @@ -88,12 +88,15 @@ const fetchDocAndWriteFile = async ( let { content, fileTokens } = doc; - // TODO: Replace link's node_token into slug + // Replace node_token to slug for (const node_token in slugMap) { - // Replace Markdown link and HTML link - const re = new RegExp(`${node_token}`, 'gm'); - const newLink = `${BASE_URL}${slugMap[node_token]}`; - content = content.replace(re, newLink); + if (slugMap[node_token]) { + content = replaceLinks( + content, + node_token, + `${BASE_URL}${slugMap[node_token]}` + ); + } } const metaInfo = generateFileMeta(doc, doc.slug, doc.position); @@ -141,8 +144,10 @@ const downloadFiles = async ( const extension = path.extname(filePath); - const re = new RegExp(`${fileToken}`, 'gm'); - content = content.replace(re, './assets/' + fileToken + extension); + let assetURL = `./assets/${fileToken}${extension}`; + + // Replase Markdown image + content = replaceLinks(content, fileToken, assetURL); } return content; diff --git a/feishu-pages/src/utils.ts b/feishu-pages/src/utils.ts index 34c5046..0af9a6f 100644 --- a/feishu-pages/src/utils.ts +++ b/feishu-pages/src/utils.ts @@ -1,4 +1,4 @@ -import { FileDoc } from "./summary"; +import { FileDoc } from './summary'; export const normalizeSlug = (slug: string | number) => { // force convert slug into string @@ -37,19 +37,19 @@ export const humanizeFileSize = (bytes, dp = 1) => { }; const allowKeys = [ - "depth", - "title", - "slug", - "filename", - "node_token", - "parent_node_token", - "children", - "obj_create_time", - "obj_edit_time", - "obj_token", - "has_child", - "meta", - "position" + 'depth', + 'title', + 'slug', + 'filename', + 'node_token', + 'parent_node_token', + 'children', + 'obj_create_time', + 'obj_edit_time', + 'obj_token', + 'has_child', + 'meta', + 'position', ]; export function cleanupDocsForJSON(docs: FileDoc[]) { @@ -58,7 +58,7 @@ export function cleanupDocsForJSON(docs: FileDoc[]) { for (let idx = 0; idx < docs.length; idx++) { const doc = docs[idx]; - Object.keys(doc).forEach(key => { + Object.keys(doc).forEach((key) => { if (!allowKeys.includes(key)) { delete doc[key]; } @@ -78,3 +78,27 @@ export function cleanupDocsForJSON(docs: FileDoc[]) { docs.splice(nodesToDelete[i], 1); } } + +export function replaceLinks( + content: string, + node_token: string, + newLink?: string +): string { + if (!newLink) { + return content; + } + + /* + match all + + 1 - ]( or src=" or href=" + 2 - https://ywh1bkansf.feishu.cn/wiki/aabbdd + 3 - node_token + 4 - ) or " + */ + const re = new RegExp( + `(]\\(|src="|href=")(http[s]?:\\\/\\\/[\\w]+\\.(feishu\\.cn|larksuite\.com)\\\/.*)?(${node_token}.*)(\\)|")`, + 'gm' + ); + return content.replace(re, `$1${newLink}$5`); +} diff --git a/feishu-pages/tests/doc.test.ts b/feishu-pages/tests/doc.test.ts index 7a37a5a..d30e3d7 100644 --- a/feishu-pages/tests/doc.test.ts +++ b/feishu-pages/tests/doc.test.ts @@ -5,15 +5,15 @@ import { generateFileMeta } from '../src/doc'; describe('Doc', () => { test('generateFileMeta', () => { let doc: any = { - title: 'hello world', + title: 'Docs: "hello world"', }; const urlPath = '/hello/world'; const position = 1; let expected = `--- -title: hello world -slug: /hello/world +title: "Docs: \\"hello world\\"" +slug: "/hello/world" sidebar_position: 1 --- `; @@ -24,7 +24,7 @@ sidebar_position: 1 title: null, }; expected = `--- -slug: /hello/world +slug: "/hello/world" sidebar_position: 1 --- `; diff --git a/feishu-pages/tests/utils.test.ts b/feishu-pages/tests/utils.test.ts index 5bfbc91..d9f4d54 100644 --- a/feishu-pages/tests/utils.test.ts +++ b/feishu-pages/tests/utils.test.ts @@ -1,83 +1,84 @@ -import { describe, test, expect } from "@jest/globals"; -import assert from "node:assert"; +import { describe, expect, test } from '@jest/globals'; +import assert from 'node:assert'; import { + cleanupDocsForJSON, humanizeFileSize, normalizeSlug, - cleanupDocsForJSON -} from "../src/utils"; + replaceLinks, +} from '../src/utils'; -describe("Utils", () => { - test("normalizeSlug", () => { - assert.equal(normalizeSlug("foo-bar"), "foo-bar"); - assert.equal(normalizeSlug("wikcnfoo-bar"), "foo-bar"); - assert.equal(normalizeSlug("wikenfoo-bar"), "foo-bar"); +describe('Utils', () => { + test('normalizeSlug', () => { + assert.equal(normalizeSlug('foo-bar'), 'foo-bar'); + assert.equal(normalizeSlug('wikcnfoo-bar'), 'foo-bar'); + assert.equal(normalizeSlug('wikenfoo-bar'), 'foo-bar'); }); - test("humanizeFileSize", () => { - assert.equal(humanizeFileSize(1209482), "1.2 MB"); - assert.equal(humanizeFileSize(810473), "791.5 kB"); - assert.equal(humanizeFileSize(4621), "4.5 kB"); + test('humanizeFileSize', () => { + assert.equal(humanizeFileSize(1209482), '1.2 MB'); + assert.equal(humanizeFileSize(810473), '791.5 kB'); + assert.equal(humanizeFileSize(4621), '4.5 kB'); }); - test("cleanupDocsForJSON", () => { + test('cleanupDocsForJSON', () => { const rawDocs: any = [ { - title: "title1", + title: 'title1', meta: { - slug: "app1" + slug: 'app1', }, children: [ { - title: "title1-1", + title: 'title1-1', meta: { - slug: "app1-1", - hide: true - } + slug: 'app1-1', + hide: true, + }, }, { - title: "title1-2", + title: 'title1-2', meta: { - slug: "app1-2" + slug: 'app1-2', }, children: [ { - title: "title1-2-1", + title: 'title1-2-1', meta: { - slug: "app1-2-1" - } + slug: 'app1-2-1', + }, }, { - title: "title1-2-2", + title: 'title1-2-2', meta: { - slug: "app1-2-2", - hide: true - } - } - ] - } - ] + slug: 'app1-2-2', + hide: true, + }, + }, + ], + }, + ], }, { - title: "title2", + title: 'title2', meta: { - slug: "app2", - hide: true + slug: 'app2', + hide: true, }, children: [ { - title: "title2-1", + title: 'title2-1', meta: { - slug: "app2-1" - } + slug: 'app2-1', + }, }, { - title: "title2-2", + title: 'title2-2', meta: { - slug: "app2-2" - } - } - ] - } + slug: 'app2-2', + }, + }, + ], + }, ]; cleanupDocsForJSON(rawDocs); @@ -85,16 +86,55 @@ describe("Utils", () => { { children: [ { - children: [ - { meta: { slug: "app1-2-1" }, title: "title1-2-1" }, - ], - meta: { slug: "app1-2" }, - title: "title1-2" - } + children: [{ meta: { slug: 'app1-2-1' }, title: 'title1-2-1' }], + meta: { slug: 'app1-2' }, + title: 'title1-2', + }, ], - meta: { slug: "app1" }, - title: "title1" - } + meta: { slug: 'app1' }, + title: 'title1', + }, ]); }); + + test('replaceLinks', () => { + let raw = ` + ](https://ywh1bkansf.feishu.cn/wiki/aabbdd) + ](https://ywh1bkansf.feishu.cn/foo/aabbdd) + ](https://ywh1bkansf.feishu.cn/F8OMwrI3TisTPokQAJHcMG2knBh) + ](https://ywh1bkansf.feishu.cn/wiki/F8OMwrI3TisTPokQAJHcMG2knBh) + href="https://ywh1bkansf.feishu.cn/wiki/F8OMwrI3TisTPokQAJHcMG2knBh" + href="https://ywh1bkansf.larksuite.com/wiki/F8OMwrI3TisTPokQAJHcMG2knBh" + src="https://ywh1bkansf.feishu.cn/wiki/F8OMwrI3TisTPokQAJHcMG2knBh" + ](F8OMwrI3TisTPokQAJHcMG2knBh) + ](flasdjkgajklsdgjklajklsdgjklal) + href="F8OMwrI3TisTPokQAJHcMG2knBh" + href="./F8OMwrI3TisTPokQAJHcMG2knBh" + src="F8OMwrI3TisTPokQAJHcMG2knBh" + href="/F8OMwrI3TisTPokQAJHcMG2knBh/F8OMwrI3TisTPokQAJHcMG2knBh" + href="https://ywh1bkansf.feishu.cn/wiki/F8OMwrI3TisTPokQAJHcMG2knBh?foo=bar#hash1" + `; + + let expected = ` + ](https://ywh1bkansf.feishu.cn/wiki/aabbdd) + ](https://ywh1bkansf.feishu.cn/foo/aabbdd) + ](/new-link) + ](/new-link) + href="/new-link" + href="/new-link" + src="/new-link" + ](/new-link) + ](flasdjkgajklsdgjklajklsdgjklal) + href="/new-link" + href="./F8OMwrI3TisTPokQAJHcMG2knBh" + src="/new-link" + href="/F8OMwrI3TisTPokQAJHcMG2knBh/F8OMwrI3TisTPokQAJHcMG2knBh" + href="/new-link" + `; + + assert.equal( + replaceLinks(raw, 'F8OMwrI3TisTPokQAJHcMG2knBh', '/new-link'), + expected + ); + }); });