From 8973c7fb9617bcbcd65c3424e889d87d2c29d020 Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Thu, 7 Sep 2023 11:20:00 +1200 Subject: [PATCH 1/4] Update NodeJS API config to match new `output*` settings --- pagefind/src/service/mod.rs | 6 +++--- pagefind/src/service/requests.rs | 2 +- pagefind/src/service/responses.rs | 2 +- wrappers/node/README.md | 6 +++--- wrappers/node/lib/index.js | 4 ++-- wrappers/node/types/index.d.ts | 4 ++-- wrappers/node/types/internal.d.ts | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pagefind/src/service/mod.rs b/pagefind/src/service/mod.rs index 45dee788..459765fd 100644 --- a/pagefind/src/service/mod.rs +++ b/pagefind/src/service/mod.rs @@ -237,13 +237,13 @@ pub async fn run_service() { } RequestAction::WriteFiles { index_id, - bundle_path, + output_path, } => { if let Some(index) = get_index(&mut indexes, index_id, err) { index.build_indexes().await; - let bundle_path = index.write_files(bundle_path.map(Into::into)).await; + let resolved_output_path = index.write_files(output_path.map(Into::into)).await; send(ResponseAction::WriteFiles { - bundle_path: bundle_path.to_string_lossy().into(), + output_path: resolved_output_path.to_string_lossy().into(), }); } } diff --git a/pagefind/src/service/requests.rs b/pagefind/src/service/requests.rs index 7d1aa063..36b85f71 100644 --- a/pagefind/src/service/requests.rs +++ b/pagefind/src/service/requests.rs @@ -39,7 +39,7 @@ pub(super) enum RequestAction { }, WriteFiles { index_id: u32, - bundle_path: Option, + output_path: Option, }, GetFiles { index_id: u32, diff --git a/pagefind/src/service/responses.rs b/pagefind/src/service/responses.rs index 973699de..14e0875a 100644 --- a/pagefind/src/service/responses.rs +++ b/pagefind/src/service/responses.rs @@ -27,7 +27,7 @@ pub(super) enum ResponseAction { }, BuildIndex {}, WriteFiles { - bundle_path: String, + output_path: String, }, GetFiles { files: Vec, diff --git a/wrappers/node/README.md b/wrappers/node/README.md index 5494045b..dddb07a9 100644 --- a/wrappers/node/README.md +++ b/wrappers/node/README.md @@ -47,7 +47,7 @@ await index.getFiles(); // Write the index to disk await index.writeFiles({ - bundlePath: "./public/pagefind" + outputPath: "./public/pagefind" }); ``` @@ -173,11 +173,11 @@ Writes the index files to disk, as they would be written when running the standa ```js const { errors } = await index.writeFiles({ - bundlePath: "./public/pagefind" + outputPath: "./public/pagefind" }); ``` -The `bundlePath` option should contain the path to the desired Pagefind bundle directory. If relative, is relative to the current working directory of your Node process. +The `outputPath` option should contain the path to the desired Pagefind bundle directory. If relative, is relative to the current working directory of your Node process. A response with an `errors` array containing error messages indicates that Pagefind failed to action this request. diff --git a/wrappers/node/lib/index.js b/wrappers/node/lib/index.js index acb6ffb7..f2115aba 100644 --- a/wrappers/node/lib/index.js +++ b/wrappers/node/lib/index.js @@ -212,7 +212,7 @@ const writeFiles = (indexId, options) => new Promise((resolve, reject) => { { type: action, index_id: indexId, - bundle_path: options?.bundlePath + output_path: options?.outputPath }, (response) => { /** @type {function(InternalResponsePayload): Omit?} */ const successCallback = (success) => { @@ -222,7 +222,7 @@ const writeFiles = (indexId, options) => new Promise((resolve, reject) => { } return { - bundlePath: success.bundle_path + outputPath: success.output_path } }; handleApiResponse(resolve, reject, response, successCallback); diff --git a/wrappers/node/types/index.d.ts b/wrappers/node/types/index.d.ts index 3aabd5dd..e07f259a 100644 --- a/wrappers/node/types/index.d.ts +++ b/wrappers/node/types/index.d.ts @@ -162,13 +162,13 @@ export interface WriteOptions { * If relative, is relative to the cwd. * @example "./public/pagefind" */ - bundlePath: string + outputPath: string } export interface WriteFilesResponse { errors: string[], - bundlePath: string + outputPath: string } /** diff --git a/wrappers/node/types/internal.d.ts b/wrappers/node/types/internal.d.ts index cce440b5..408bf376 100644 --- a/wrappers/node/types/internal.d.ts +++ b/wrappers/node/types/internal.d.ts @@ -62,7 +62,7 @@ export interface InternalAddDirRequest { export interface InternalWriteFilesRequest { type: 'WriteFiles', index_id: number, - bundle_path?: string + output_path?: string } export interface InternalGetFilesRequest { @@ -124,7 +124,7 @@ export interface InternalIndexedDirResponse { export interface InternalWriteFilesResponse { type: 'WriteFiles', - bundle_path: string, + output_path: string, } export interface InternalGetFilesResponse { From ba071ca2b50e3892100e532f2673a9a860bd4f1c Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Thu, 7 Sep 2023 12:13:36 +1200 Subject: [PATCH 2/4] Update NodeJS API URL & Path handling --- pagefind/features/node_api/node_base.feature | 714 ++++++++++--------- pagefind/src/fossick/mod.rs | 11 +- pagefind/src/lib.rs | 11 +- 3 files changed, 376 insertions(+), 360 deletions(-) diff --git a/pagefind/features/node_api/node_base.feature b/pagefind/features/node_api/node_base.feature index 0c275b2d..9fb0b38e 100644 --- a/pagefind/features/node_api/node_base.feature +++ b/pagefind/features/node_api/node_base.feature @@ -1,356 +1,360 @@ Feature: Node API Base Tests -# Background: -# Given I have a "public/index.html" file with the body: -# """ -#

Nothing

-# """ -# Given I have a "public/package.json" file with the content: -# """ -# { -# "name": "test", -# "type": "module", -# "version": "1.0.0", -# "main": "index.js", -# "dependencies": { -# "pagefind": "file:{{humane_cwd}}/../wrappers/node" -# } -# } -# """ - -# @platform-unix -# Scenario: Build a synthetic index to disk via the api -# Given I have a "public/index.js" file with the content: -# """ -# import * as pagefind from "pagefind"; - -# const run = async () => { -# const { index } = await pagefind.createIndex(); -# await index.addHTMLFile({path: "dogs/index.html", content: "

Testing, testing

"}); -# await index.writeFiles(); -# console.log(`Successfully wrote files`); -# } - -# run(); -# """ -# When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" -# Then I should see "Successfully wrote files" in stdout -# Then I should see the file "public/pagefind/pagefind.js" -# When I serve the "public" directory -# When I load "/" -# When I evaluate: -# """ -# async function() { -# let pagefind = await import("/pagefind/pagefind.js"); - -# let search = await pagefind.search("testing"); - -# let data = await search.results[0].data(); -# document.querySelector('[data-url]').innerText = data.url; -# } -# """ -# Then There should be no logs -# Then The selector "[data-url]" should contain "/dogs/" - -# @platform-unix -# Scenario: Build a synthetic index to memory via the api -# Given I have a "public/index.js" file with the content: -# """ -# import * as pagefind from "pagefind"; - -# const run = async () => { -# const { index } = await pagefind.createIndex(); -# await index.addHTMLFile({path: "dogs/index.html", content: "

Testing, testing

"}); -# const { files } = await index.getFiles(); - -# const jsFile = files.filter(file => file.path.includes("pagefind.js"))[0]; -# console.log(jsFile.content.toString()); - -# const fragments = files.filter(file => file.path.includes("fragment")); -# console.log(`${fragments.length} fragment(s)`); -# } - -# run(); -# """ -# When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" -# Then I should see "pagefind_version=" in stdout -# Then I should see "1 fragment(s)" in stdout -# Then I should not see the file "public/pagefind/pagefind.js" - -# @platform-unix -# Scenario: Build a true index to disk via the api -# Given I have a "public/custom_files/real/index.html" file with the body: -# """ -#

A testing file that exists on disk

-# """ -# Given I have a "public/index.js" file with the content: -# """ -# import * as pagefind from "pagefind"; - -# const run = async () => { -# const { index } = await pagefind.createIndex(); -# await index.addDirectory({path: "custom_files"}); -# await index.writeFiles(); -# console.log(`Successfully wrote files`); -# } - -# run(); -# """ -# When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" -# Then I should see "Successfully wrote files" in stdout -# Then I should see the file "public/pagefind/pagefind.js" -# When I serve the "public" directory -# When I load "/" -# When I evaluate: -# """ -# async function() { -# let pagefind = await import("/pagefind/pagefind.js"); - -# let search = await pagefind.search("testing"); - -# let data = await search.results[0].data(); -# document.querySelector('[data-url]').innerText = data.url; -# } -# """ -# Then There should be no logs -# Then The selector "[data-url]" should contain "/real/" - -# @platform-unix -# Scenario: Build a blended index to memory via the api -# Given I have a "public/custom_files/real/index.html" file with the body: -# """ -#

A testing file that exists on disk

-# """ -# Given I have a "public/index.js" file with the content: -# """ -# import * as pagefind from "pagefind"; -# import fs from "fs"; -# import path from "path"; - -# const run = async () => { -# const { index } = await pagefind.createIndex(); -# await index.addDirectory({ path: "custom_files" }); -# await index.addCustomRecord({ -# url: "/synth/", -# content: "A testing file that doesn't exist.", -# language: "en" -# }); -# const { files } = await index.getFiles(); - -# for (const file of files) { -# const dir = path.dirname(file.path); -# if (!fs.existsSync(dir)){ -# fs.mkdirSync(dir, { recursive: true }); -# } - -# fs.writeFileSync(file.path, file.content); -# } -# console.log("Donezo!"); -# } - -# run(); -# """ -# When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" -# Then I should see "Donezo!" in stdout -# Then I should see the file "public/pagefind/pagefind.js" -# When I serve the "public" directory -# When I load "/" -# When I evaluate: -# """ -# async function() { -# let pagefind = await import("/pagefind/pagefind.js"); - -# let search = await pagefind.search("testing"); - -# let pages = await Promise.all(search.results.map(r => r.data())); -# document.querySelector('[data-url]').innerText = pages.map(p => p.url).sort().join(", "); -# } -# """ -# Then There should be no logs -# Then The selector "[data-url]" should contain "/real/, /synth/" - -# @platform-unix -# Scenario: Build an index to a custom disk location via the api -# Given I have a "output/index.html" file with the body: -# """ -#

Nothing

-# """ -# Given I have a "public/index.js" file with the content: -# """ -# import * as pagefind from "pagefind"; - -# const run = async () => { -# const { index } = await pagefind.createIndex(); -# await index.addHTMLFile({path: "dogs/index.html", content: "

Testing, testing

"}); -# await index.writeFiles({ bundlePath: "../output/pagefind" }); -# console.log(`Successfully wrote files`); -# } - -# run(); -# """ -# When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" -# Then I should see "Successfully wrote files" in stdout -# Then I should see the file "output/pagefind/pagefind.js" -# When I serve the "output" directory -# When I load "/" -# When I evaluate: -# """ -# async function() { -# let pagefind = await import("/pagefind/pagefind.js"); - -# let search = await pagefind.search("testing"); - -# let data = await search.results[0].data(); -# document.querySelector('[data-url]').innerText = data.url; -# } -# """ -# Then There should be no logs -# Then The selector "[data-url]" should contain "/dogs/" - -# @platform-unix -# Scenario: An index is not consumed on write -# Given I have a "output/index.html" file with the body: -# """ -#

Nothing

-# """ -# Given I have a "public/index.js" file with the content: -# """ -# import * as pagefind from "pagefind"; - -# const run = async () => { -# const { index } = await pagefind.createIndex(); -# await index.addHTMLFile({path: "dogs/index.html", content: "

Testing, testing

"}); -# await index.writeFiles({ bundlePath: "../output/pagefind" }); - -# await index.addHTMLFile({path: "rabbits/index.html", content: "

Testing, testing

"}); -# const { files } = await index.getFiles(); - -# const fragments = files.filter(file => file.path.includes("fragment")); -# console.log(`${fragments.length} fragment(s)`); - -# await index.addHTMLFile({path: "cats/index.html", content: "

Testing, testing

"}); -# await index.writeFiles({ bundlePath: "./pagefind" }); - -# console.log(`Successfully wrote files`); -# } - -# run(); -# """ -# When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" -# Then I should see "Successfully wrote files" in stdout -# Then I should see "2 fragment(s)" in stdout -# Then I should see the file "output/pagefind/pagefind.js" -# When I serve the "output" directory -# When I load "/" -# When I evaluate: -# """ -# async function() { -# let pagefind = await import("/pagefind/pagefind.js"); - -# let search = await pagefind.search("testing"); - -# let pages = await Promise.all(search.results.map(r => r.data())); -# document.querySelector('[data-url]').innerText = pages.map(p => p.url).sort().join(", "); -# } -# """ -# Then There should be no logs -# Then The selector "[data-url]" should contain "/dogs/" -# When I serve the "public" directory -# When I load "/" -# When I evaluate: -# """ -# async function() { -# let pagefind = await import("/pagefind/pagefind.js"); - -# let search = await pagefind.search("testing"); - -# let pages = await Promise.all(search.results.map(r => r.data())); -# document.querySelector('[data-url]').innerText = pages.map(p => p.url).sort().join(", "); -# } -# """ -# Then There should be no logs -# Then The selector "[data-url]" should contain "/cats/, /dogs/, /rabbits/" - -# @platform-unix -# Scenario: Pagefind service config -# Given I have a "public/index.js" file with the content: -# """ -# import * as pagefind from "pagefind"; - -# const run = async () => { -# const { index } = await pagefind.createIndex({ -# rootSelector: "h1", -# excludeSelectors: ["span"], -# keepIndexUrl: true, -# }); -# await index.addHTMLFile({path: "dogs/index.html", content: "

Testing, testing

"}); -# await index.writeFiles(); -# console.log(`Successfully wrote files`); -# } - -# run(); -# """ -# When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" -# Then I should see "Successfully wrote files" in stdout -# Then I should see the file "public/pagefind/pagefind.js" -# When I serve the "public" directory -# When I load "/" -# When I evaluate: -# """ -# async function() { -# let pagefind = await import("/pagefind/pagefind.js"); - -# let search = await pagefind.search("testing"); - -# let data = await search.results[0].data(); -# document.querySelector('[data-url]').innerText = `${data.url} • ${data.content}`; -# } -# """ -# Then There should be no logs -# Then The selector "[data-url]" should contain "/dogs/index.html • Testing," - -# @platform-unix -# Scenario: Pagefind error handling -# Given I have a "public/index.js" file with the content: -# """ -# import * as pagefind from "pagefind"; - -# const bad = async () => { -# const { index } = await pagefind.createIndex(); -# await index.deleteIndex(); -# const { errors, files } = await index.getFiles(); -# console.log(JSON.stringify(errors)); - -# try { -# const response = await pagefind.createIndex({ -# rootSelector: 5 -# }); -# } catch(e) { -# console.log(e.toString()); -# } -# } -# bad(); -# """ -# When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" -# Then I should see "invalid type: integer `5`" in stdout -# Then I should see "Index has been deleted from the Pagefind service and no longer exists" in stdout - -# @platform-unix -# Scenario: Pagefind empty index returns assets -# Given I have a "public/index.js" file with the content: -# """ -# import * as pagefind from "pagefind"; - -# const run = async () => { -# const { index } = await pagefind.createIndex(); -# const { errors, files } = await index.getFiles(); -# console.log(files.map(f => f.path).join(', ')); -# } -# run(); -# """ -# When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" -# Then I should see "_pagefind/pagefind.js" in stdout -# Then I should see "_pagefind/pagefind-ui.js" in stdout -# Then I should see "_pagefind/pagefind-ui.css" in stdout -# Then I should see "_pagefind/pagefind-modular-ui.js" in stdout -# Then I should see "_pagefind/pagefind-modular-ui.css" in stdout -# Then I should see "_pagefind/wasm.unknown.pagefind" in stdout + Background: + Given I have a "public/index.html" file with the body: + """ +

Nothing

+ """ + Given I have a "public/package.json" file with the content: + """ + { + "name": "test", + "type": "module", + "version": "1.0.0", + "main": "index.js", + "dependencies": { + "pagefind": "file:{{humane_cwd}}/../wrappers/node" + } + } + """ + + @platform-unix + Scenario: Build a synthetic index to disk via the api + Given I have a "public/index.js" file with the content: + """ + import * as pagefind from "pagefind"; + + const run = async () => { + const { index } = await pagefind.createIndex(); + await index.addHTMLFile({path: "dogs/index.html", content: "

Testing, testing

"}); + await index.writeFiles(); + console.log(`Successfully wrote files`); + } + + run(); + """ + When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + Then I should see "Successfully wrote files" in stdout + Then I should see the file "public/pagefind/pagefind.js" + When I serve the "public" directory + When I load "/" + When I evaluate: + """ + async function() { + let pagefind = await import("/pagefind/pagefind.js"); + + let search = await pagefind.search("testing"); + + let data = await search.results[0].data(); + document.querySelector('[data-url]').innerText = data.url; + } + """ + Then There should be no logs + Then The selector "[data-url]" should contain "/dogs/" + + @platform-unix + Scenario: Build a synthetic index to memory via the api + Given I have a "public/index.js" file with the content: + """ + import * as pagefind from "pagefind"; + + const run = async () => { + const { index } = await pagefind.createIndex(); + await index.addHTMLFile({path: "dogs/index.html", content: "

Testing, testing

"}); + const { files } = await index.getFiles(); + + const jsFile = files.filter(file => file.path.includes("pagefind.js"))[0]; + console.log(jsFile.content.toString()); + + console.log(`JS is at ${jsFile.path}`); + + const fragments = files.filter(file => file.path.includes("fragment")); + console.log(`${fragments.length} fragment(s)`); + } + + run(); + """ + When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + Then I should see "pagefind_version=" in stdout + Then I should see "JS is at pagefind.js" in stdout + Then I should see "1 fragment(s)" in stdout + Then I should not see the file "public/pagefind/pagefind.js" + + @platform-unix + Scenario: Build a true index to disk via the api + Given I have a "public/custom_files/real/index.html" file with the body: + """ +

A testing file that exists on disk

+ """ + Given I have a "public/index.js" file with the content: + """ + import * as pagefind from "pagefind"; + + const run = async () => { + const { index } = await pagefind.createIndex(); + await index.addDirectory({path: "custom_files"}); + await index.writeFiles(); + console.log(`Successfully wrote files`); + } + + run(); + """ + When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + Then I should see "Successfully wrote files" in stdout + Then I should see the file "public/pagefind/pagefind.js" + When I serve the "public" directory + When I load "/" + When I evaluate: + """ + async function() { + let pagefind = await import("/pagefind/pagefind.js"); + + let search = await pagefind.search("testing"); + + let data = await search.results[0].data(); + document.querySelector('[data-url]').innerText = data.url; + } + """ + Then There should be no logs + Then The selector "[data-url]" should contain "/real/" + + @platform-unix + Scenario: Build a blended index to memory via the api + Given I have a "public/custom_files/real/index.html" file with the body: + """ +

A testing file that exists on disk

+ """ + Given I have a "public/index.js" file with the content: + """ + import * as pagefind from "pagefind"; + import fs from "fs"; + import path from "path"; + + const run = async () => { + const { index } = await pagefind.createIndex(); + await index.addDirectory({ path: "custom_files" }); + await index.addCustomRecord({ + url: "/synth/", + content: "A testing file that doesn't exist.", + language: "en" + }); + const { files } = await index.getFiles(); + + for (const file of files) { + const output_path = path.join("pagefind", file.path); + const dir = path.dirname(output_path); + if (!fs.existsSync(dir)){ + fs.mkdirSync(dir, { recursive: true }); + } + + fs.writeFileSync(output_path, file.content); + } + console.log("Donezo!"); + } + + run(); + """ + When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + Then I should see "Donezo!" in stdout + Then I should see the file "public/pagefind/pagefind.js" + When I serve the "public" directory + When I load "/" + When I evaluate: + """ + async function() { + let pagefind = await import("/pagefind/pagefind.js"); + + let search = await pagefind.search("testing"); + + let pages = await Promise.all(search.results.map(r => r.data())); + document.querySelector('[data-url]').innerText = pages.map(p => p.url).sort().join(", "); + } + """ + Then There should be no logs + Then The selector "[data-url]" should contain "/real/, /synth/" + + @platform-unix + Scenario: Build an index to a custom disk location via the api + Given I have a "output/index.html" file with the body: + """ +

Nothing

+ """ + Given I have a "public/index.js" file with the content: + """ + import * as pagefind from "pagefind"; + + const run = async () => { + const { index } = await pagefind.createIndex(); + await index.addHTMLFile({path: "dogs/index.html", content: "

Testing, testing

"}); + await index.writeFiles({ outputPath: "../output/pagefind" }); + console.log(`Successfully wrote files`); + } + + run(); + """ + When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + Then I should see "Successfully wrote files" in stdout + Then I should see the file "output/pagefind/pagefind.js" + When I serve the "output" directory + When I load "/" + When I evaluate: + """ + async function() { + let pagefind = await import("/pagefind/pagefind.js"); + + let search = await pagefind.search("testing"); + + let data = await search.results[0].data(); + document.querySelector('[data-url]').innerText = data.url; + } + """ + Then There should be no logs + Then The selector "[data-url]" should contain "/dogs/" + + @platform-unix + Scenario: An index is not consumed on write + Given I have a "output/index.html" file with the body: + """ +

Nothing

+ """ + Given I have a "public/index.js" file with the content: + """ + import * as pagefind from "pagefind"; + + const run = async () => { + const { index } = await pagefind.createIndex(); + await index.addHTMLFile({path: "dogs/index.html", content: "

Testing, testing

"}); + await index.writeFiles({ outputPath: "../output/pagefind" }); + + await index.addHTMLFile({path: "rabbits/index.html", content: "

Testing, testing

"}); + const { files } = await index.getFiles(); + + const fragments = files.filter(file => file.path.includes("fragment")); + console.log(`${fragments.length} fragment(s)`); + + await index.addHTMLFile({path: "cats/index.html", content: "

Testing, testing

"}); + await index.writeFiles({ outputPath: "./pagefind" }); + + console.log(`Successfully wrote files`); + } + + run(); + """ + When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + Then I should see "Successfully wrote files" in stdout + Then I should see "2 fragment(s)" in stdout + Then I should see the file "output/pagefind/pagefind.js" + When I serve the "output" directory + When I load "/" + When I evaluate: + """ + async function() { + let pagefind = await import("/pagefind/pagefind.js"); + + let search = await pagefind.search("testing"); + + let pages = await Promise.all(search.results.map(r => r.data())); + document.querySelector('[data-url]').innerText = pages.map(p => p.url).sort().join(", "); + } + """ + Then There should be no logs + Then The selector "[data-url]" should contain "/dogs/" + When I serve the "public" directory + When I load "/" + When I evaluate: + """ + async function() { + let pagefind = await import("/pagefind/pagefind.js"); + + let search = await pagefind.search("testing"); + + let pages = await Promise.all(search.results.map(r => r.data())); + document.querySelector('[data-url]').innerText = pages.map(p => p.url).sort().join(", "); + } + """ + Then There should be no logs + Then The selector "[data-url]" should contain "/cats/, /dogs/, /rabbits/" + + @platform-unix + Scenario: Pagefind service config + Given I have a "public/index.js" file with the content: + """ + import * as pagefind from "pagefind"; + + const run = async () => { + const { index } = await pagefind.createIndex({ + rootSelector: "h1", + excludeSelectors: ["span"], + keepIndexUrl: true, + }); + await index.addHTMLFile({path: "dogs/index.html", content: "

Testing, testing

"}); + await index.writeFiles(); + console.log(`Successfully wrote files`); + } + + run(); + """ + When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + Then I should see "Successfully wrote files" in stdout + Then I should see the file "public/pagefind/pagefind.js" + When I serve the "public" directory + When I load "/" + When I evaluate: + """ + async function() { + let pagefind = await import("/pagefind/pagefind.js"); + + let search = await pagefind.search("testing"); + + let data = await search.results[0].data(); + document.querySelector('[data-url]').innerText = `${data.url} • ${data.content}`; + } + """ + Then There should be no logs + Then The selector "[data-url]" should contain "/dogs/index.html • Testing," + + @platform-unix + Scenario: Pagefind error handling + Given I have a "public/index.js" file with the content: + """ + import * as pagefind from "pagefind"; + + const bad = async () => { + const { index } = await pagefind.createIndex(); + await index.deleteIndex(); + const { errors, files } = await index.getFiles(); + console.log(JSON.stringify(errors)); + + try { + const response = await pagefind.createIndex({ + rootSelector: 5 + }); + } catch(e) { + console.log(e.toString()); + } + } + bad(); + """ + When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + Then I should see "invalid type: integer `5`" in stdout + Then I should see "Index has been deleted from the Pagefind service and no longer exists" in stdout + + @platform-unix + Scenario: Pagefind empty index returns assets + Given I have a "public/index.js" file with the content: + """ + import * as pagefind from "pagefind"; + + const run = async () => { + const { index } = await pagefind.createIndex(); + const { errors, files } = await index.getFiles(); + console.log(files.map(f => f.path).join(', ')); + } + run(); + """ + When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + Then I should see "pagefind.js" in stdout + Then I should see "pagefind-ui.js" in stdout + Then I should see "pagefind-ui.css" in stdout + Then I should see "pagefind-modular-ui.js" in stdout + Then I should see "pagefind-modular-ui.css" in stdout + Then I should see "wasm.unknown.pagefind" in stdout diff --git a/pagefind/src/fossick/mod.rs b/pagefind/src/fossick/mod.rs index 9d35ea00..7b81752e 100644 --- a/pagefind/src/fossick/mod.rs +++ b/pagefind/src/fossick/mod.rs @@ -422,11 +422,14 @@ impl Fossicker { fn build_url(page_url: &Path, relative_to: Option<&Path>, options: &SearchOptions) -> String { let prefix = relative_to.unwrap_or(&options.site_source); - let trimmed = page_url.strip_prefix(prefix); - let Ok(url) = trimmed else { + + let url = if let Ok(trimmed) = page_url.strip_prefix(prefix) { + trimmed + } else if page_url.is_relative() { + page_url + } else { options.logger.error(format!( - "File was found that does not start with the source directory: {}\nSource: {:?}\nFile: {:?}", - trimmed.err().unwrap(), + "Absolute file was found that does not start with the source directory. Source: {:?}\nFile: {:?}", prefix, page_url )); diff --git a/pagefind/src/lib.rs b/pagefind/src/lib.rs index e511fd0b..5acafe09 100644 --- a/pagefind/src/lib.rs +++ b/pagefind/src/lib.rs @@ -58,7 +58,7 @@ impl SearchState { } pub async fn fossick_many(&mut self, dir: PathBuf, glob: String) -> Result { - let files = self.walk_for_files(dir, glob).await; + let files = self.walk_for_files(dir.clone(), glob).await; let log = &self.options.logger; log.info(format!( @@ -322,6 +322,15 @@ impl SearchState { .into_iter(), ); + // SyntheticFiles should only return the relative path to the file + // _within_ the bundle directory — placing them in a final location + // is left to the API consumer. + for file in files.iter_mut() { + if let Ok(relative_path) = file.filename.strip_prefix(outdir) { + file.filename = relative_path.to_path_buf(); + } + } + files } From 033af6dcf65701a407b7de79aece38daa332b39f Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Thu, 7 Sep 2023 14:06:18 +1200 Subject: [PATCH 3/4] Change return type of `index.getFiles()` to a Uint8Array --- wrappers/node/README.md | 6 +++--- wrappers/node/lib/encoding.js | 27 +++++++++++++++++++++++++++ wrappers/node/lib/index.js | 3 ++- wrappers/node/types/index.d.ts | 2 +- 4 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 wrappers/node/lib/encoding.js diff --git a/wrappers/node/README.md b/wrappers/node/README.md index dddb07a9..9ebd06ce 100644 --- a/wrappers/node/README.md +++ b/wrappers/node/README.md @@ -152,20 +152,20 @@ If successful, the `file` object is returned containing some metadata about the ### index.getFiles -Get buffers of all files in the Pagefind index. Useful for integrating a Pagefind index into the development mode of a static site generator and hosting these files yourself. +Get raw data of all files in the Pagefind index. Useful for integrating a Pagefind index into the development mode of a static site generator and hosting these files yourself. ```js const { errors, files } = await index.getFiles(); for (const file of files) { console.log(file.path); - // do something with file.content + // do something with the file.content Uint8Array } ``` A response with an `errors` array containing error messages indicates that Pagefind failed to action this request. -If successful, `files` will be an array containing file objects. Each object contains a `path` key, which is the URL this file should be served at, and a `content` key containing the raw Buffer of this file. +If successful, `files` will be an array containing file objects. Each object contains a `path` key, which is the URL this file should be served at, and a `content` key containing the raw data as a Uint8Array. ### index.writeFiles diff --git a/wrappers/node/lib/encoding.js b/wrappers/node/lib/encoding.js new file mode 100644 index 00000000..e2a70a71 --- /dev/null +++ b/wrappers/node/lib/encoding.js @@ -0,0 +1,27 @@ +/** + * + * @param {string} b64 + * @returns {Uint8Array} + */ +export const decode = (b64) => { + let binString; + if (typeof Buffer !== "undefined" && typeof Buffer.from === "function") { + return Buffer.from(b64, "base64"); + } else if ( + typeof window !== "undefined" && + typeof window.atob === "function" + ) { + binString = window.atob(b64); + } else if (typeof atob === "function") { + binString = atob(b64); + } else { + throw new Error("Unable to decode base64 data"); + } + + const size = binString.length; + const bytes = new Uint8Array(size); + for (let i = 0; i < size; i++) { + bytes[i] = binString.charCodeAt(i); + } + return bytes; +}; diff --git a/wrappers/node/lib/index.js b/wrappers/node/lib/index.js index f2115aba..c9465101 100644 --- a/wrappers/node/lib/index.js +++ b/wrappers/node/lib/index.js @@ -1,4 +1,5 @@ import { PagefindService } from "./service.js"; +import { decode } from "./encoding.js"; /** * @typedef {import('pagefindInternal').InternalResponseCallback} InternalResponseCallback @@ -254,7 +255,7 @@ const getFiles = (indexId) => new Promise((resolve, reject) => { files: success.files.map(file => { return { path: file.path, - content: Buffer.from(file.content, 'base64') + content: decode(file.content) } }) } diff --git a/wrappers/node/types/index.d.ts b/wrappers/node/types/index.d.ts index e07f259a..28b08a19 100644 --- a/wrappers/node/types/index.d.ts +++ b/wrappers/node/types/index.d.ts @@ -183,7 +183,7 @@ export interface GetFilesResponse { export interface IndexFile { path: string, - content: Buffer + content: Uint8Array } /** From a19d4360ccbe5e50ee452f3c42d25320fd7c840c Mon Sep 17 00:00:00 2001 From: Liam Bigelow <40188355+bglw@users.noreply.github.com> Date: Thu, 7 Sep 2023 14:21:46 +1200 Subject: [PATCH 4/4] Use the TEST_BINARY path in Node API tests to fix release pipelines --- pagefind/features/node_api/node_base.feature | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pagefind/features/node_api/node_base.feature b/pagefind/features/node_api/node_base.feature index 9fb0b38e..3b0e9ad3 100644 --- a/pagefind/features/node_api/node_base.feature +++ b/pagefind/features/node_api/node_base.feature @@ -32,7 +32,7 @@ Feature: Node API Base Tests run(); """ - When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + When I run "cd public && npm i && PAGEFIND_BINARY_PATH={{humane_cwd}}/$TEST_BINARY node index.js" Then I should see "Successfully wrote files" in stdout Then I should see the file "public/pagefind/pagefind.js" When I serve the "public" directory @@ -73,7 +73,7 @@ Feature: Node API Base Tests run(); """ - When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + When I run "cd public && npm i && PAGEFIND_BINARY_PATH={{humane_cwd}}/$TEST_BINARY node index.js" Then I should see "pagefind_version=" in stdout Then I should see "JS is at pagefind.js" in stdout Then I should see "1 fragment(s)" in stdout @@ -98,7 +98,7 @@ Feature: Node API Base Tests run(); """ - When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + When I run "cd public && npm i && PAGEFIND_BINARY_PATH={{humane_cwd}}/$TEST_BINARY node index.js" Then I should see "Successfully wrote files" in stdout Then I should see the file "public/pagefind/pagefind.js" When I serve the "public" directory @@ -153,7 +153,7 @@ Feature: Node API Base Tests run(); """ - When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + When I run "cd public && npm i && PAGEFIND_BINARY_PATH={{humane_cwd}}/$TEST_BINARY node index.js" Then I should see "Donezo!" in stdout Then I should see the file "public/pagefind/pagefind.js" When I serve the "public" directory @@ -191,7 +191,7 @@ Feature: Node API Base Tests run(); """ - When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + When I run "cd public && npm i && PAGEFIND_BINARY_PATH={{humane_cwd}}/$TEST_BINARY node index.js" Then I should see "Successfully wrote files" in stdout Then I should see the file "output/pagefind/pagefind.js" When I serve the "output" directory @@ -239,7 +239,7 @@ Feature: Node API Base Tests run(); """ - When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + When I run "cd public && npm i && PAGEFIND_BINARY_PATH={{humane_cwd}}/$TEST_BINARY node index.js" Then I should see "Successfully wrote files" in stdout Then I should see "2 fragment(s)" in stdout Then I should see the file "output/pagefind/pagefind.js" @@ -293,7 +293,7 @@ Feature: Node API Base Tests run(); """ - When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + When I run "cd public && npm i && PAGEFIND_BINARY_PATH={{humane_cwd}}/$TEST_BINARY node index.js" Then I should see "Successfully wrote files" in stdout Then I should see the file "public/pagefind/pagefind.js" When I serve the "public" directory @@ -334,7 +334,7 @@ Feature: Node API Base Tests } bad(); """ - When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + When I run "cd public && npm i && PAGEFIND_BINARY_PATH={{humane_cwd}}/$TEST_BINARY node index.js" Then I should see "invalid type: integer `5`" in stdout Then I should see "Index has been deleted from the Pagefind service and no longer exists" in stdout @@ -351,7 +351,7 @@ Feature: Node API Base Tests } run(); """ - When I run "cd public && npm i && PAGEFIND_BINARY_PATH='{{humane_cwd}}/../target/release/pagefind' node index.js" + When I run "cd public && npm i && PAGEFIND_BINARY_PATH={{humane_cwd}}/$TEST_BINARY node index.js" Then I should see "pagefind.js" in stdout Then I should see "pagefind-ui.js" in stdout Then I should see "pagefind-ui.css" in stdout