diff --git a/config.ts b/config.ts index 9775de5..ab325f6 100644 --- a/config.ts +++ b/config.ts @@ -32,6 +32,8 @@ export type ConfigType = { ffprobe_path: string; redirect_to_flutter: boolean; download_timeout_check_interval: number; + /** EH metadata cache time in hours */ + eh_metadata_cache_time: number; }; export enum ThumbnailMethod { @@ -194,6 +196,10 @@ export class Config { get download_timeout_check_interval() { return this._return_number("download_timeout_check_interval") || 10; } + /** EH metadata cache time in hours */ + get eh_metadata_cache_time() { + return this._return_number("eh_metadata_cache_time") || 168; + } to_json(): ConfigType { return { cookies: typeof this.cookies === "string", @@ -226,6 +232,7 @@ export class Config { redirect_to_flutter: this.redirect_to_flutter, download_timeout_check_interval: this.download_timeout_check_interval, + eh_metadata_cache_time: this.eh_metadata_cache_time, }; } } diff --git a/db.ts b/db.ts index 9eb025f..827d1c3 100644 --- a/db.ts +++ b/db.ts @@ -12,6 +12,7 @@ import { Status } from "sqlite/src/constants.ts"; import { parse_bool, sleep, sure_dir_sync, try_remove_sync } from "./utils.ts"; import { Task, TaskType } from "./task.ts"; import { generate as randomstring } from "randomstring"; +import type { GalleryMetadataSingle } from "./page/GalleryMetadata.ts"; type SqliteMaster = { type: string; @@ -187,6 +188,7 @@ const ALL_TABLES = [ "filemeta", "user", "token", + "ehmeta", ]; const VERSION_TABLE = `CREATE TABLE version ( id TEXT, @@ -274,6 +276,12 @@ const TOKEN_TABLE = `CREATE TABLE token ( client_version TEXT, client_platform TEXT );`; +const EHMETA_TABLE = `CREATE TABLE ehmeta ( + gid INT, + data TEXT, + cached_time TEXT, + PRIMARY KEY (gid) +);`; function escape_fields(fields: string, namespace: string) { const fs = fields.split(","); @@ -497,6 +505,9 @@ export class EhDb { if (!this.#exist_table.has("token")) { this.db.execute(TOKEN_TABLE); } + if (!this.#exist_table.has("ehmeta")) { + this.db.execute(EHMETA_TABLE); + } this.#updateExistsTable(); } #read_version() { @@ -538,6 +549,12 @@ export class EhDb { ]); }); } + add_ehmeta(data: GalleryMetadataSingle) { + this.db.query( + "INSERT OR REPLACE INTO ehmeta VALUES (?, ?, ?);", + [data.gid, JSON.stringify(data), new Date()], + ); + } add_gmeta(gmeta: GMeta) { this.db.queryEntries( "INSERT OR REPLACE INTO gmeta VALUES (:gid, :token, :title, :title_jpn, :category, :uploader, :posted, :filecount, :filesize, :expunged, :rating, :parent_gid, :parent_key, :first_gid, :first_key);", @@ -949,6 +966,13 @@ export class EhDb { if (!this.#dblock) return; eval(`Deno.funlockSync(${this.#dblock.rid});`); } + get_ehmeta(gid: number) { + const d = this.db.query<[string]>( + "SELECT data FROM ehmeta WHERE gid = ?;", + [gid], + ); + return d.length ? JSON.parse(d[0][0]) as GalleryMetadataSingle : null; + } get_extended_pmeta(gid: number) { return this.convert_extended_pmeta( this.db.queryEntries( @@ -1244,6 +1268,11 @@ export class EhDb { optimize() { this.db.execute("VACUUM;"); } + remove_expired_ehmeta(cache_time: number) { + const date = new Date(); + date.setTime(date.getTime() - cache_time * 3600_000); + this.db.query("DELETE FROM ehmeta WHERE cached_time < ?;", [date]); + } remove_expired_token() { this.db.query("DELETE FROM token WHERE expired < ?;", [new Date()]); } diff --git a/fresh.gen.ts b/fresh.gen.ts index fbc49b8..c41e87f 100644 --- a/fresh.gen.ts +++ b/fresh.gen.ts @@ -7,6 +7,7 @@ import * as $api_middleware from "./routes/api/_middleware.ts"; import * as $api_config from "./routes/api/config.ts"; import * as $api_deploy_id from "./routes/api/deploy_id.ts"; import * as $api_eh_image_limit from "./routes/api/eh/image_limit.ts"; +import * as $api_eh_metadata from "./routes/api/eh/metadata.ts"; import * as $api_exit from "./routes/api/exit.ts"; import * as $api_export_gallery_zip_gid_ from "./routes/api/export/gallery/zip/[gid].ts"; import * as $api_file_id_ from "./routes/api/file/[id].ts"; @@ -45,6 +46,7 @@ const manifest = { "./routes/api/config.ts": $api_config, "./routes/api/deploy_id.ts": $api_deploy_id, "./routes/api/eh/image_limit.ts": $api_eh_image_limit, + "./routes/api/eh/metadata.ts": $api_eh_metadata, "./routes/api/exit.ts": $api_exit, "./routes/api/export/gallery/zip/[gid].ts": $api_export_gallery_zip_gid_, diff --git a/routes/api/eh/metadata.ts b/routes/api/eh/metadata.ts new file mode 100644 index 0000000..d6bec90 --- /dev/null +++ b/routes/api/eh/metadata.ts @@ -0,0 +1,86 @@ +import { Handlers } from "$fresh/server.ts"; +import type { GID } from "../../../client.ts"; +import { User, UserPermission } from "../../../db.ts"; +import { get_task_manager } from "../../../server.ts"; +import { EHMetaInfo } from "../../../server/eh.ts"; +import { parse_int } from "../../../server/parse_form.ts"; +import { + gen_data, + gen_error, + return_data, + return_error, +} from "../../../server/utils.ts"; + +export const handler: Handlers = { + async GET(req, ctx) { + const user = ctx.state.user; + if ( + user && !user.is_admin && + !(user.permissions & UserPermission.ManageTasks) + ) { + return return_error(403, "Permission denied."); + } + const m = get_task_manager(); + const url = new URL(req.url); + const gids: Array = []; + for (const gid of url.searchParams.getAll("gid")) { + const i = await parse_int(gid, null); + if (i === null) { + return return_error(1, `Invalid gid: ${gid}`); + } + gids.push(i); + } + for (const gid of url.searchParams.getAll("gid[]")) { + const i = await parse_int(gid, null); + if (i === null) { + return return_error(1, `Invalid gid: ${gid}`); + } + gids.push(i); + } + const tokens = url.searchParams.getAll("token").concat( + url.searchParams.getAll("token[]"), + ); + if (gids.length === 0 && tokens.length === 0) { + return return_error(2, "No gids and tokens provided."); + } + if (gids.length !== tokens.length) { + return return_error(3, "Length of gids and tokens do not match."); + } + const data: EHMetaInfo = {}; + const needed: GID[] = []; + for (let i = 0; i < gids.length; i++) { + const gid = gids[i]; + const token = tokens[i]; + const cache = m.db.get_ehmeta(gid); + if (cache && cache.gid === gid && cache.token === token) { + data[gid] = gen_data(cache); + } else if (cache && cache.gid === gid) { + data[gid] = gen_error(1, "Token not matched."); + } else { + needed.push([gid, token]); + } + } + while (needed.length > 0) { + const query = needed.splice(0, 25); + try { + const metas = await m.client.fetchGalleryMetadataByAPI( + ...query, + ); + for (const [k, v] of metas.map) { + if (typeof v === "string") { + data[k] = gen_error(2, v); + } else { + data[k] = gen_data(v); + m.db.add_ehmeta(v); + } + } + } catch (e) { + return return_error( + 4, + `Failed to fetch metadata: ${e.message}`, + ); + } + } + return return_data(data); + }, +}; diff --git a/server.ts b/server.ts index d364b08..7992bb1 100644 --- a/server.ts +++ b/server.ts @@ -38,6 +38,12 @@ export async function startServer(path: string) { setInterval(() => { task_manager?.db.remove_expired_token(); }, 86_400_000); + setInterval(() => { + if (!task_manager) return; + task_manager.db.remove_expired_ehmeta( + task_manager.cfg.eh_metadata_cache_time, + ); + }, 3600_000); return start(manifest, { signal: task_manager.aborts, plugins: [twindPlugin(twindConfig)], diff --git a/server/eh.ts b/server/eh.ts index b5f41a4..0f34e14 100644 --- a/server/eh.ts +++ b/server/eh.ts @@ -1,4 +1,9 @@ +import type { JSONResult } from "./utils.ts"; +import type { GalleryMetadataSingle } from "../page/GalleryMetadata.ts"; + export type EHImageLimit = { current: number; max: number; }; + +export type EHMetaInfo = Record>;