From dc227b2d0010dd66c7564ef5d88cc3b006cd07a0 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sat, 14 Sep 2024 15:21:41 +1200 Subject: [PATCH 01/18] Create a simple CONTRIBUTING.md --- CONTRIBUTING.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a6eef48 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,18 @@ +### Coding conventions +Since the provider comes in two parts(the **Provider**(coded in typescript) and the **Provider plugin**(coded in python)), we have different code formatting standards for them. +Please format your code by running this script below before you push a commit to your pull request: +```shell +# Make sure you have ruff, autopep8 and prettier installed already +ruff check --fix plugin/ +autopep8 --in-place plugin/ + +cd server +npx prettier --check --write 'src/**/*.{js,ts}' +cd .. +``` + +#### **Provider**(typescript): + + +#### **Provider plugin**(python): +We follow the [yt-dlp coding conventions](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#yt-dlp-coding-conventions). From 18dd0c3d2cfe2df5f07b3221f8f268c3663d11d0 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sat, 14 Sep 2024 15:22:33 +1200 Subject: [PATCH 02/18] add a newline --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6eef48..ebe3f76 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,7 @@ ### Coding conventions Since the provider comes in two parts(the **Provider**(coded in typescript) and the **Provider plugin**(coded in python)), we have different code formatting standards for them. Please format your code by running this script below before you push a commit to your pull request: + ```shell # Make sure you have ruff, autopep8 and prettier installed already ruff check --fix plugin/ From 86e4828e1534ed7ff568ad27d3d21764ca16f17e Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sat, 14 Sep 2024 09:20:50 +0000 Subject: [PATCH 03/18] [server/generate_once] Cache sessionData to file - [server/generate_once] Better logging - [server/generate_once] Parse date with new Date() - [server/session_manager] add exporting interface YoutubeSessionDataCaches - [server/session_manager] add private property TOKEN_TTL_HOURS to SessionManager - [server/session_manager] add public method cleanupCaches, getYoutubeSessionDataCaches(boolean), setYoutubeSessionDataCaches(YoutubeSessionDataCaches) - [server/session_manager+generate_once] automatically cleanup caches - [server/session_manager] rename private property youtubeSessionData to youtubeSessionDataCaches --- .gitignore | 1 + server/src/generate_once.ts | 63 ++++++++++++++++++++++---------- server/src/session_manager.ts | 69 ++++++++++++++++++++++++----------- 3 files changed, 92 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 69d8141..950c2d5 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ plugin/**.pyo # yt-dlp source code **/tmp/ +**/cache.json diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index 38735fa..b0477eb 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -1,6 +1,9 @@ -import { SessionManager } from "./session_manager"; +import { SessionManager, YoutubeSessionDataCaches } from "./session_manager"; import { Command } from "@commander-js/extra-typings"; +import * as fs from "fs"; +import * as path from "path"; +const CACHE_PATH = path.resolve(process.cwd(), "cache.json"); const program = new Command() .option("-v, --visitor-data ") .option("-d, --data-sync-id ") @@ -14,37 +17,59 @@ const options = program.opts(); const visitorData = options.visitorData; const verbose = options.verbose || false; let visitorIdentifier: string; - const sessionManager = new SessionManager(verbose); + let cache: YoutubeSessionDataCaches; + try { + const parsedCaches: YoutubeSessionDataCaches = JSON.parse( + fs.readFileSync(CACHE_PATH, "utf8"), + ); + for (const visitIdentifier in parsedCaches) { + if ( + parsedCaches[visitIdentifier] && + typeof parsedCaches[visitIdentifier].generatedAt === "string" + ) { + const parsedDate = parsedCaches[visitIdentifier].generatedAt; + parsedCaches[visitIdentifier].generatedAt = new Date( + parsedDate, + ); + } + } + cache = parsedCaches; + // eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch (_error) { + cache = {}; + } + + const sessionManager = new SessionManager(verbose, cache); + function log(msg: string) { + sessionManager.log(msg); + } if (dataSyncId) { - if (verbose) { - console.log(`Received request for data sync ID: '${dataSyncId}'`); - } + log(`Received request for data sync ID: '${dataSyncId}'`); visitorIdentifier = dataSyncId; } else if (visitorData) { - if (verbose) { - console.log(`Received request for visitor data: '${visitorData}'`); - } + log(`Received request for visitor data: '${visitorData}'`); visitorIdentifier = visitorData; } else { - if (verbose) { - console.log( - `Received request for visitor data, grabbing from Innertube`, - ); - } + log(`Received request for visitor data, grabbing from Innertube`); const generatedVisitorData = await sessionManager.generateVisitorData(); if (!generatedVisitorData) { + // Should we remove this console.error? There's one already in sessionManager.generateVisitorData(). console.error("Error generating visitor data"); process.exit(1); } - - if (verbose) { - console.log(`Generated visitor data: ${generatedVisitorData}`); - } - + log(`Generated visitor data: ${generatedVisitorData}`); visitorIdentifier = generatedVisitorData; } const sessionData = await sessionManager.generatePoToken(visitorIdentifier); - console.log(JSON.stringify(sessionData)); + try { + fs.writeFileSync( + CACHE_PATH, + JSON.stringify(sessionManager.getYoutubeSessionDataCaches(true)), + "utf8", + ); + } finally { + console.log(JSON.stringify(sessionData)); + } })(); diff --git a/server/src/session_manager.ts b/server/src/session_manager.ts index 961b9b4..01acbda 100644 --- a/server/src/session_manager.ts +++ b/server/src/session_manager.ts @@ -8,27 +8,61 @@ interface YoutubeSessionData { generatedAt: Date; } +export interface YoutubeSessionDataCaches { + [visitIdentifier: string]: YoutubeSessionData; +} + export class SessionManager { shouldLog: boolean; - private youtubeSessionData: { - [visitIdentifier: string]: YoutubeSessionData; - } = {}; + private youtubeSessionDataCaches: YoutubeSessionDataCaches = {}; + private TOKEN_TTL_HOURS: number; - constructor(shouldLog = true) { + constructor( + shouldLog = true, + youtubeSessionDataCaches: YoutubeSessionDataCaches = {}, + ) { this.shouldLog = shouldLog; + this.setYoutubeSessionDataCaches(youtubeSessionDataCaches); + this.TOKEN_TTL_HOURS = process.env.TOKEN_TTL + ? parseInt(process.env.TOKEN_TTL) + : 6; } invalidateCaches() { - this.youtubeSessionData = {}; + this.setYoutubeSessionDataCaches(); } - log(msg: string) { - if (this.shouldLog) { - console.log(msg); + cleanupCaches() { + for (const visitIdentifier in this.youtubeSessionDataCaches) { + const sessionData = this.youtubeSessionDataCaches[visitIdentifier]; + if ( + sessionData && + sessionData.generatedAt < + new Date( + new Date().getTime() - + this.TOKEN_TTL_HOURS * 60 * 60 * 1000, + ) + ) + delete this.youtubeSessionDataCaches[visitIdentifier]; } } + getYoutubeSessionDataCaches(cleanup = false) { + if (cleanup) this.cleanupCaches(); + return this.youtubeSessionDataCaches; + } + + setYoutubeSessionDataCaches( + youtubeSessionData: YoutubeSessionDataCaches = {}, + ) { + this.youtubeSessionDataCaches = youtubeSessionData || {}; + } + + log(msg: string) { + if (this.shouldLog) console.log(msg); + } + async generateVisitorData(): Promise { const innertube = await Innertube.create({ retrieve_player: false }); const visitorData = innertube.session.context.client.visitorData; @@ -44,18 +78,9 @@ export class SessionManager { async generatePoToken( visitIdentifier: string, ): Promise { - const TOKEN_TTL_HOURS = process.env.TOKEN_TTL - ? parseInt(process.env.TOKEN_TTL) - : 6; - - const sessionData = this.youtubeSessionData[visitIdentifier]; - if ( - sessionData && - sessionData.generatedAt > - new Date( - new Date().getTime() - TOKEN_TTL_HOURS * 60 * 60 * 1000, - ) - ) { + this.cleanupCaches(); + const sessionData = this.youtubeSessionDataCaches[visitIdentifier]; + if (sessionData) { this.log( `POT for ${visitIdentifier} still fresh, returning cached token`, ); @@ -104,12 +129,12 @@ export class SessionManager { throw new Error("po_token unexpected undefined"); } - this.youtubeSessionData[visitIdentifier] = { + this.youtubeSessionDataCaches[visitIdentifier] = { visitIdentifier: visitIdentifier, poToken: poToken, generatedAt: new Date(), }; - return this.youtubeSessionData[visitIdentifier]; + return this.youtubeSessionDataCaches[visitIdentifier]; } } From 23539dcc19381927dac397e9e7a0d6c943acbeda Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sat, 14 Sep 2024 09:24:00 +0000 Subject: [PATCH 04/18] Revert "Create a simple CONTRIBUTING.md" This reverts commit dc227b2d0010dd66c7564ef5d88cc3b006cd07a0. modified: CONTRIBUTING.md --- CONTRIBUTING.md | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ebe3f76..e69de29 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,19 +0,0 @@ -### Coding conventions -Since the provider comes in two parts(the **Provider**(coded in typescript) and the **Provider plugin**(coded in python)), we have different code formatting standards for them. -Please format your code by running this script below before you push a commit to your pull request: - -```shell -# Make sure you have ruff, autopep8 and prettier installed already -ruff check --fix plugin/ -autopep8 --in-place plugin/ - -cd server -npx prettier --check --write 'src/**/*.{js,ts}' -cd .. -``` - -#### **Provider**(typescript): - - -#### **Provider plugin**(python): -We follow the [yt-dlp coding conventions](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#yt-dlp-coding-conventions). From 219a5a809fd78f5d31b2ca58d5361cf961fb4222 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sat, 14 Sep 2024 09:25:46 +0000 Subject: [PATCH 05/18] revert --- CONTRIBUTING.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index e69de29..0000000 From 9d5dae59386753f2a3730a539f7e7f0f2e239895 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:02:26 +0000 Subject: [PATCH 06/18] Apply suggestions from code review fix `Conversion of type 'Date' to type 'string' may be a mistake because neither type sufficiently overlaps with the other. If this was intentional, convert the expression to 'unknown' first.ts(2352)` Co-authored-by: Brian Le --- server/src/generate_once.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index b0477eb..8194662 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -23,11 +23,9 @@ const options = program.opts(); fs.readFileSync(CACHE_PATH, "utf8"), ); for (const visitIdentifier in parsedCaches) { - if ( - parsedCaches[visitIdentifier] && - typeof parsedCaches[visitIdentifier].generatedAt === "string" - ) { - const parsedDate = parsedCaches[visitIdentifier].generatedAt; + if (parsedCaches[visitIdentifier]) { + const parsedDate = parsedCaches[visitIdentifier] + .generatedAt as string; parsedCaches[visitIdentifier].generatedAt = new Date( parsedDate, ); @@ -53,11 +51,7 @@ const options = program.opts(); } else { log(`Received request for visitor data, grabbing from Innertube`); const generatedVisitorData = await sessionManager.generateVisitorData(); - if (!generatedVisitorData) { - // Should we remove this console.error? There's one already in sessionManager.generateVisitorData(). - console.error("Error generating visitor data"); - process.exit(1); - } + if (!generatedVisitorData) process.exit(1); log(`Generated visitor data: ${generatedVisitorData}`); visitorIdentifier = generatedVisitorData; } From 2cba942bc1b958d682f9d6c48f20459449ed6c3d Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:03:04 +0000 Subject: [PATCH 07/18] fix: Conversion of type 'Date' to type 'string' may be a mistake because neither type sufficiently overlaps with the other. If this was intentional, convert the expression to 'unknown' first.ts(2352) --- server/src/generate_once.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index 8194662..bbda72e 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -25,7 +25,7 @@ const options = program.opts(); for (const visitIdentifier in parsedCaches) { if (parsedCaches[visitIdentifier]) { const parsedDate = parsedCaches[visitIdentifier] - .generatedAt as string; + .generatedAt as unknown as string; parsedCaches[visitIdentifier].generatedAt = new Date( parsedDate, ); From 9ac4a397687d17d0affe8df3520b419fcd96b610 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sat, 14 Sep 2024 11:05:08 +0000 Subject: [PATCH 08/18] use __dirname/cache.json as cache path Co-authored-by: Brian Le --- server/src/generate_once.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index bbda72e..226780a 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -3,7 +3,6 @@ import { Command } from "@commander-js/extra-typings"; import * as fs from "fs"; import * as path from "path"; -const CACHE_PATH = path.resolve(process.cwd(), "cache.json"); const program = new Command() .option("-v, --visitor-data ") .option("-d, --data-sync-id ") @@ -13,6 +12,7 @@ program.parse(); const options = program.opts(); (async () => { + const CACHE_PATH = path.resolve(__dirname, "cache.json"); const dataSyncId = options.dataSyncId; const visitorData = options.visitorData; const verbose = options.verbose || false; From 4221693343ca9807bdc8d4d815d643fb57d2c504 Mon Sep 17 00:00:00 2001 From: Brian Le Date: Sat, 14 Sep 2024 12:41:21 -0700 Subject: [PATCH 09/18] Refactor cache parsing --- server/src/generate_once.ts | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index 226780a..ecfcf00 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -3,6 +3,7 @@ import { Command } from "@commander-js/extra-typings"; import * as fs from "fs"; import * as path from "path"; +const CACHE_PATH = path.resolve(__dirname, "cache.json"); const program = new Command() .option("-v, --visitor-data ") .option("-d, --data-sync-id ") @@ -12,29 +13,31 @@ program.parse(); const options = program.opts(); (async () => { - const CACHE_PATH = path.resolve(__dirname, "cache.json"); const dataSyncId = options.dataSyncId; const visitorData = options.visitorData; const verbose = options.verbose || false; let visitorIdentifier: string; - let cache: YoutubeSessionDataCaches; - try { - const parsedCaches: YoutubeSessionDataCaches = JSON.parse( - fs.readFileSync(CACHE_PATH, "utf8"), - ); - for (const visitIdentifier in parsedCaches) { - if (parsedCaches[visitIdentifier]) { - const parsedDate = parsedCaches[visitIdentifier] - .generatedAt as unknown as string; - parsedCaches[visitIdentifier].generatedAt = new Date( - parsedDate, - ); + let cache: YoutubeSessionDataCaches = {}; + if (fs.existsSync(CACHE_PATH)) { + try { + const parsedCaches = JSON.parse( + fs.readFileSync(CACHE_PATH, "utf8"), + ); + for (const visitIdentifier in parsedCaches) { + const parsedCache = parsedCaches[visitIdentifier]; + if (parsedCache) { + cache[visitIdentifier] = { + poToken: parsedCache.poToken, + generatedAt: new Date(parsedCache.generatedAt), + visitIdentifier, + }; + } } + cache = parsedCaches; + } catch (e) { + log(`Error parsing cache. e = ${e}`); + cache = {}; } - cache = parsedCaches; - // eslint-disable-next-line @typescript-eslint/no-unused-vars - } catch (_error) { - cache = {}; } const sessionManager = new SessionManager(verbose, cache); From db3d03242f6c9d8c30a820610799bae605a5db5a Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sun, 15 Sep 2024 11:24:50 +1200 Subject: [PATCH 10/18] Update server/src/generate_once.ts --- server/src/generate_once.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index ecfcf00..4f83617 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -3,7 +3,7 @@ import { Command } from "@commander-js/extra-typings"; import * as fs from "fs"; import * as path from "path"; -const CACHE_PATH = path.resolve(__dirname, "cache.json"); +const CACHE_PATH = path.resolve(__dirname, "..", "cache.json"); const program = new Command() .option("-v, --visitor-data ") .option("-d, --data-sync-id ") From 9f4e67f1e1d62b66e84c7fd97e2ba0af1d88ca96 Mon Sep 17 00:00:00 2001 From: Brian Le Date: Sat, 14 Sep 2024 19:05:42 -0700 Subject: [PATCH 11/18] Update server/src/generate_once.ts Co-authored-by: N/Ame <173015200+grqz@users.noreply.github.com> --- server/src/generate_once.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index 4f83617..5d437f6 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -36,7 +36,6 @@ const options = program.opts(); cache = parsedCaches; } catch (e) { log(`Error parsing cache. e = ${e}`); - cache = {}; } } From 9df80de9f00b30b517419b98fbcdb10e593a322a Mon Sep 17 00:00:00 2001 From: Brian Le Date: Sat, 14 Sep 2024 19:06:38 -0700 Subject: [PATCH 12/18] Update server/src/generate_once.ts Co-authored-by: N/Ame <173015200+grqz@users.noreply.github.com> --- server/src/generate_once.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index 5d437f6..d6b9f24 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -33,7 +33,6 @@ const options = program.opts(); }; } } - cache = parsedCaches; } catch (e) { log(`Error parsing cache. e = ${e}`); } From 4a2d0ec8ad4e494b78daa8b0793d82195753f82f Mon Sep 17 00:00:00 2001 From: Brian Le Date: Sat, 14 Sep 2024 19:08:10 -0700 Subject: [PATCH 13/18] visitorIdentifier => visitIdentifier --- server/src/generate_once.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index ecfcf00..f093c9b 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -16,7 +16,7 @@ const options = program.opts(); const dataSyncId = options.dataSyncId; const visitorData = options.visitorData; const verbose = options.verbose || false; - let visitorIdentifier: string; + let visitIdentifier: string; let cache: YoutubeSessionDataCaches = {}; if (fs.existsSync(CACHE_PATH)) { try { @@ -33,7 +33,6 @@ const options = program.opts(); }; } } - cache = parsedCaches; } catch (e) { log(`Error parsing cache. e = ${e}`); cache = {}; @@ -47,19 +46,19 @@ const options = program.opts(); if (dataSyncId) { log(`Received request for data sync ID: '${dataSyncId}'`); - visitorIdentifier = dataSyncId; + visitIdentifier = dataSyncId; } else if (visitorData) { log(`Received request for visitor data: '${visitorData}'`); - visitorIdentifier = visitorData; + visitIdentifier = visitorData; } else { log(`Received request for visitor data, grabbing from Innertube`); const generatedVisitorData = await sessionManager.generateVisitorData(); if (!generatedVisitorData) process.exit(1); log(`Generated visitor data: ${generatedVisitorData}`); - visitorIdentifier = generatedVisitorData; + visitIdentifier = generatedVisitorData; } - const sessionData = await sessionManager.generatePoToken(visitorIdentifier); + const sessionData = await sessionManager.generatePoToken(visitIdentifier); try { fs.writeFileSync( CACHE_PATH, From bdd684938d76d16f8dac3293d01f266d0beb5e5a Mon Sep 17 00:00:00 2001 From: Brian Le Date: Sat, 14 Sep 2024 19:10:46 -0700 Subject: [PATCH 14/18] linting --- server/src/generate_once.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index 5d42318..81341a4 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -17,7 +17,7 @@ const options = program.opts(); const visitorData = options.visitorData; const verbose = options.verbose || false; let visitIdentifier: string; - let cache: YoutubeSessionDataCaches = {}; + const cache: YoutubeSessionDataCaches = {}; if (fs.existsSync(CACHE_PATH)) { try { const parsedCaches = JSON.parse( From e5706a1313884f6d24533b2df4735e2a8c50beff Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sun, 15 Sep 2024 02:31:28 +0000 Subject: [PATCH 15/18] catch and log error when writing cache - use independent `log` instead of using `sessioinManager.log` --- server/src/generate_once.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index 81341a4..5706db0 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -16,6 +16,9 @@ const options = program.opts(); const dataSyncId = options.dataSyncId; const visitorData = options.visitorData; const verbose = options.verbose || false; + function log(msg: string) { + if (verbose) console.log(msg); + } let visitIdentifier: string; const cache: YoutubeSessionDataCaches = {}; if (fs.existsSync(CACHE_PATH)) { @@ -39,9 +42,6 @@ const options = program.opts(); } const sessionManager = new SessionManager(verbose, cache); - function log(msg: string) { - sessionManager.log(msg); - } if (dataSyncId) { log(`Received request for data sync ID: '${dataSyncId}'`); @@ -64,6 +64,8 @@ const options = program.opts(); JSON.stringify(sessionManager.getYoutubeSessionDataCaches(true)), "utf8", ); + } catch (e) { + log(`Error writing cache. e = ${e}`); } finally { console.log(JSON.stringify(sessionData)); } From 62fc292ca7513d2d5f28cac0329993a581a6011c Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sun, 15 Sep 2024 02:40:06 +0000 Subject: [PATCH 16/18] use console.warn always log the script's stderr output --- .../yt_dlp_plugins/extractor/getpot_bgutil_script.py | 4 ++-- server/src/generate_once.ts | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index c7ccf66..ab0599f 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -70,10 +70,10 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= raise RequestError( f'_get_pot_via_script failed: Unable to run script (caused by {e!s})') from e - self._logger.debug(f'stdout = {stdout}') + self._logger.debug(f'stdout:\n{stdout}\nstderr:\n{stderr}') if returncode: raise RequestError( - f'_get_pot_via_script failed with returncode {returncode}:\n{stderr.strip()}') + f'_get_pot_via_script failed with returncode {returncode}') # The JSON response is always the last line script_data_resp = stdout.splitlines()[-1] diff --git a/server/src/generate_once.ts b/server/src/generate_once.ts index 5706db0..a90e6d0 100644 --- a/server/src/generate_once.ts +++ b/server/src/generate_once.ts @@ -16,9 +16,6 @@ const options = program.opts(); const dataSyncId = options.dataSyncId; const visitorData = options.visitorData; const verbose = options.verbose || false; - function log(msg: string) { - if (verbose) console.log(msg); - } let visitIdentifier: string; const cache: YoutubeSessionDataCaches = {}; if (fs.existsSync(CACHE_PATH)) { @@ -37,11 +34,14 @@ const options = program.opts(); } } } catch (e) { - log(`Error parsing cache. e = ${e}`); + console.warn(`Error parsing cache. e = ${e}`); } } const sessionManager = new SessionManager(verbose, cache); + function log(msg: string) { + if (verbose) console.log(msg); + } if (dataSyncId) { log(`Received request for data sync ID: '${dataSyncId}'`); @@ -65,7 +65,7 @@ const options = program.opts(); "utf8", ); } catch (e) { - log(`Error writing cache. e = ${e}`); + console.warn(`Error writing cache. e = ${e}`); } finally { console.log(JSON.stringify(sessionData)); } From 649f65129d1c0e47940e0f8054c288d068292d6e Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sun, 15 Sep 2024 02:58:51 +0000 Subject: [PATCH 17/18] remove trailing newlines --- .../extractor/getpot_bgutil_script.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index ab0599f..56fd57a 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -70,20 +70,26 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= raise RequestError( f'_get_pot_via_script failed: Unable to run script (caused by {e!s})') from e - self._logger.debug(f'stdout:\n{stdout}\nstderr:\n{stderr}') + msg = f'stdout:\n{stdout.strip()}' + if stderr.strip(): # Empty strings are falsy + msg += f'\nstderr:\n{stderr.strip()}' + self._logger.debug(msg) if returncode: - raise RequestError( - f'_get_pot_via_script failed with returncode {returncode}') + raise RequestError(f'_get_pot_via_script failed with returncode {returncode}') - # The JSON response is always the last line - script_data_resp = stdout.splitlines()[-1] - self._logger.debug( - f'_get_pot_via_script response = {script_data_resp}') try: - return json.loads(script_data_resp)['poToken'] - except (json.JSONDecodeError, TypeError, KeyError) as e: + # The JSON response is always the last line + script_data_resp = json.loads(stdout.splitlines()[-1]) + except json.JSONDecodeError as e: raise RequestError( f'Error parsing JSON response from _get_pot_via_script (caused by {e!s})') from e + else: + self._logger.debug( + f'_get_pot_via_script response = {script_data_resp}') + if potoken := script_data_resp.get('poToken'): + return potoken + else: + raise RequestError('The script did not respond with a po_token') @register_preference(BgUtilScriptPotProviderRH) From c3831cf85be45ff29703d1f3f48fd6c4432e287e Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Sun, 15 Sep 2024 03:04:11 +0000 Subject: [PATCH 18/18] [main] refactor: rename visitorIdentifier -> visitIdentifier --- server/src/main.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main.ts b/server/src/main.ts index a23ee01..6ba92e8 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -26,15 +26,15 @@ httpServer.post("/get_pot", async (request, response) => { const visitorData = request.body.visitor_data as string; const dataSyncId = request.body.data_sync_id as string; - let visitorIdentifier: string; + let visitIdentifier: string; // prioritize data sync id for authenticated requests, if passed if (dataSyncId) { console.log(`Received request for data sync ID: '${dataSyncId}'`); - visitorIdentifier = dataSyncId; + visitIdentifier = dataSyncId; } else if (visitorData) { console.log(`Received request for visitor data: '${visitorData}'`); - visitorIdentifier = visitorData; + visitIdentifier = visitorData; } else { console.log( `Received request for visitor data, grabbing from Innertube`, @@ -48,10 +48,10 @@ httpServer.post("/get_pot", async (request, response) => { } console.log(`Generated visitor data: ${generatedVisitorData}`); - visitorIdentifier = generatedVisitorData; + visitIdentifier = generatedVisitorData; } - const sessionData = await sessionManager.generatePoToken(visitorIdentifier); + const sessionData = await sessionManager.generatePoToken(visitIdentifier); response.send({ po_token: sessionData.poToken, visit_identifier: sessionData.visitIdentifier,