[server/generate_once] write sessionData caches to file

.gitignore

@@ -25,3 +25,4 @@ plugin/**.pyo
 # yt-dlp source code
 
 **/tmp/
+**/cache.json

plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py

@@ -70,10 +70,10 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=
             raise RequestError(
                 f'_get_pot_via_script failed: Unable to run script (caused by {e!s})') from e
 
-        self._logger.debug(f'stdout = {stdout}')
+        self._logger.debug(f'stdout:\n{stdout}\nstderr:\n{stderr}')
         if returncode:
             raise RequestError(
-                f'_get_pot_via_script failed with returncode {returncode}:\n{stderr.strip()}')
+                f'_get_pot_via_script failed with returncode {returncode}')
 
         # The JSON response is always the last line
         script_data_resp = stdout.splitlines()[-1]

server/src/generate_once.ts

@@ -1,6 +1,9 @@
-import { SessionManager } from "./session_manager";
+import { SessionManager, YoutubeSessionDataCaches } from "./session_manager";
 import { Command } from "@commander-js/extra-typings";
+import * as fs from "fs";
+import * as path from "path";
 
+const CACHE_PATH = path.resolve(__dirname, "..", "cache.json");
 const program = new Command()
     .option("-v, --visitor-data <visitordata>")
     .option("-d, --data-sync-id <data-sync-id>")
@@ -13,38 +16,57 @@ const options = program.opts();
     const dataSyncId = options.dataSyncId;
     const visitorData = options.visitorData;
     const verbose = options.verbose || false;
-    let visitorIdentifier: string;
-    const sessionManager = new SessionManager(verbose);
+    let visitIdentifier: string;
+    const cache: YoutubeSessionDataCaches = {};
+    if (fs.existsSync(CACHE_PATH)) {
+        try {
+            const parsedCaches = JSON.parse(
+                fs.readFileSync(CACHE_PATH, "utf8"),
+            );
+            for (const visitIdentifier in parsedCaches) {
+                const parsedCache = parsedCaches[visitIdentifier];
+                if (parsedCache) {
+                    cache[visitIdentifier] = {
+                        poToken: parsedCache.poToken,
+                        generatedAt: new Date(parsedCache.generatedAt),
+                        visitIdentifier,
+                    };
+                }
+            }
+        } catch (e) {
+            console.warn(`Error parsing cache. e = ${e}`);
+        }
+    }
+
+    const sessionManager = new SessionManager(verbose, cache);
+    function log(msg: string) {
+        if (verbose) console.log(msg);
+    }
 
     if (dataSyncId) {
-        if (verbose) {
-            console.log(`Received request for data sync ID: '${dataSyncId}'`);
-        }
-        visitorIdentifier = dataSyncId;
+        log(`Received request for data sync ID: '${dataSyncId}'`);
+        visitIdentifier = dataSyncId;
     } else if (visitorData) {
-        if (verbose) {
-            console.log(`Received request for visitor data: '${visitorData}'`);
-        }
-        visitorIdentifier = visitorData;
+        log(`Received request for visitor data: '${visitorData}'`);
+        visitIdentifier = visitorData;
     } else {
-        if (verbose) {
-            console.log(
-                `Received request for visitor data, grabbing from Innertube`,
-            );
-        }
+        log(`Received request for visitor data, grabbing from Innertube`);
         const generatedVisitorData = await sessionManager.generateVisitorData();
-        if (!generatedVisitorData) {
-            console.error("Error generating visitor data");
-            process.exit(1);
-        }
-
-        if (verbose) {
-            console.log(`Generated visitor data: ${generatedVisitorData}`);
-        }
-
-        visitorIdentifier = generatedVisitorData;
+        if (!generatedVisitorData) process.exit(1);
+        log(`Generated visitor data: ${generatedVisitorData}`);
+        visitIdentifier = generatedVisitorData;
     }
 
-    const sessionData = await sessionManager.generatePoToken(visitorIdentifier);
-    console.log(JSON.stringify(sessionData));
+    const sessionData = await sessionManager.generatePoToken(visitIdentifier);
+    try {
+        fs.writeFileSync(
+            CACHE_PATH,
+            JSON.stringify(sessionManager.getYoutubeSessionDataCaches(true)),
+            "utf8",
+        );
+    } catch (e) {
+        console.warn(`Error writing cache. e = ${e}`);
+    } finally {
+        console.log(JSON.stringify(sessionData));
+    }
 })();

server/src/session_manager.ts

@@ -8,27 +8,61 @@ interface YoutubeSessionData {
     generatedAt: Date;
 }
 
+export interface YoutubeSessionDataCaches {
+    [visitIdentifier: string]: YoutubeSessionData;
+}
+
 export class SessionManager {
     shouldLog: boolean;
 
-    private youtubeSessionData: {
-        [visitIdentifier: string]: YoutubeSessionData;
-    } = {};
+    private youtubeSessionDataCaches: YoutubeSessionDataCaches = {};
+    private TOKEN_TTL_HOURS: number;
 
-    constructor(shouldLog = true) {
+    constructor(
+        shouldLog = true,
+        youtubeSessionDataCaches: YoutubeSessionDataCaches = {},
+    ) {
         this.shouldLog = shouldLog;
+        this.setYoutubeSessionDataCaches(youtubeSessionDataCaches);
+        this.TOKEN_TTL_HOURS = process.env.TOKEN_TTL
+            ? parseInt(process.env.TOKEN_TTL)
+            : 6;
     }
 
     invalidateCaches() {
-        this.youtubeSessionData = {};
+        this.setYoutubeSessionDataCaches();
     }
 
-    log(msg: string) {
-        if (this.shouldLog) {
-            console.log(msg);
+    cleanupCaches() {
+        for (const visitIdentifier in this.youtubeSessionDataCaches) {
+            const sessionData = this.youtubeSessionDataCaches[visitIdentifier];
+            if (
+                sessionData &&
+                sessionData.generatedAt <
+                    new Date(
+                        new Date().getTime() -
+                            this.TOKEN_TTL_HOURS * 60 * 60 * 1000,
+                    )
+            )
+                delete this.youtubeSessionDataCaches[visitIdentifier];
         }
     }
 
+    getYoutubeSessionDataCaches(cleanup = false) {
+        if (cleanup) this.cleanupCaches();
+        return this.youtubeSessionDataCaches;
+    }
+
+    setYoutubeSessionDataCaches(
+        youtubeSessionData: YoutubeSessionDataCaches = {},
+    ) {
+        this.youtubeSessionDataCaches = youtubeSessionData || {};
+    }
+
+    log(msg: string) {
+        if (this.shouldLog) console.log(msg);
+    }
+
     async generateVisitorData(): Promise<string | null> {
         const innertube = await Innertube.create({ retrieve_player: false });
         const visitorData = innertube.session.context.client.visitorData;
@@ -44,18 +78,9 @@ export class SessionManager {
     async generatePoToken(
         visitIdentifier: string,
     ): Promise<YoutubeSessionData> {
-        const TOKEN_TTL_HOURS = process.env.TOKEN_TTL
-            ? parseInt(process.env.TOKEN_TTL)
-            : 6;
-
-        const sessionData = this.youtubeSessionData[visitIdentifier];
-        if (
-            sessionData &&
-            sessionData.generatedAt >
-                new Date(
-                    new Date().getTime() - TOKEN_TTL_HOURS * 60 * 60 * 1000,
-                )
-        ) {
+        this.cleanupCaches();
+        const sessionData = this.youtubeSessionDataCaches[visitIdentifier];
+        if (sessionData) {
             this.log(
                 `POT for ${visitIdentifier} still fresh, returning cached token`,
             );
@@ -104,12 +129,12 @@ export class SessionManager {
             throw new Error("po_token unexpected undefined");
         }
 
-        this.youtubeSessionData[visitIdentifier] = {
+        this.youtubeSessionDataCaches[visitIdentifier] = {
             visitIdentifier: visitIdentifier,
             poToken: poToken,
             generatedAt: new Date(),
         };
 
-        return this.youtubeSessionData[visitIdentifier];
+        return this.youtubeSessionDataCaches[visitIdentifier];
     }
 }