Skip to content

Commit

Permalink
[server/generate_once] write sessionData caches to file (#29)
Browse files Browse the repository at this point in the history
* Create a simple CONTRIBUTING.md

* add a newline

* [server/generate_once] Cache sessionData to file
- [server/generate_once] Better logging
- [server/generate_once] Parse date with new Date()
- [server/session_manager] add exporting interface YoutubeSessionDataCaches
- [server/session_manager] add private property TOKEN_TTL_HOURS to SessionManager
- [server/session_manager] add public method cleanupCaches, getYoutubeSessionDataCaches(boolean), setYoutubeSessionDataCaches(YoutubeSessionDataCaches)
- [server/session_manager+generate_once] automatically cleanup caches
- [server/session_manager] rename private property youtubeSessionData to youtubeSessionDataCaches

* Revert "Create a simple CONTRIBUTING.md"

This reverts commit dc227b2.

	modified:   CONTRIBUTING.md

* revert

* Apply suggestions from code review
fix `Conversion of type 'Date' to type 'string' may be a mistake because neither type sufficiently overlaps with the other. If this was intentional, convert the expression to 'unknown' first.ts(2352)`

Co-authored-by: Brian Le <[email protected]>

* fix:
Conversion of type 'Date' to type 'string' may be a mistake because neither type sufficiently overlaps with the other. If this was intentional, convert the expression to 'unknown' first.ts(2352)

* use __dirname/cache.json as cache path

Co-authored-by: Brian Le <[email protected]>

* Refactor cache parsing

* Update server/src/generate_once.ts

* Update server/src/generate_once.ts

Co-authored-by: N/Ame <[email protected]>

* Update server/src/generate_once.ts

Co-authored-by: N/Ame <[email protected]>

* visitorIdentifier => visitIdentifier

* linting

* catch and log  error when writing cache
- use independent `log` instead of using `sessioinManager.log`

* use console.warn
always log the script's stderr output

* remove trailing newlines

* [main] refactor: rename visitorIdentifier -> visitIdentifier

---------

Co-authored-by: Brian Le <[email protected]>
  • Loading branch information
grqz and Brainicism authored Sep 15, 2024
1 parent 214f524 commit 5bc73a9
Show file tree
Hide file tree
Showing 5 changed files with 118 additions and 64 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,4 @@ plugin/**.pyo
# yt-dlp source code

**/tmp/
**/cache.json
24 changes: 15 additions & 9 deletions plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,20 +70,26 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=
raise RequestError(
f'_get_pot_via_script failed: Unable to run script (caused by {e!s})') from e

self._logger.debug(f'stdout = {stdout}')
msg = f'stdout:\n{stdout.strip()}'
if stderr.strip(): # Empty strings are falsy
msg += f'\nstderr:\n{stderr.strip()}'
self._logger.debug(msg)
if returncode:
raise RequestError(
f'_get_pot_via_script failed with returncode {returncode}:\n{stderr.strip()}')
raise RequestError(f'_get_pot_via_script failed with returncode {returncode}')

# The JSON response is always the last line
script_data_resp = stdout.splitlines()[-1]
self._logger.debug(
f'_get_pot_via_script response = {script_data_resp}')
try:
return json.loads(script_data_resp)['poToken']
except (json.JSONDecodeError, TypeError, KeyError) as e:
# The JSON response is always the last line
script_data_resp = json.loads(stdout.splitlines()[-1])
except json.JSONDecodeError as e:
raise RequestError(
f'Error parsing JSON response from _get_pot_via_script (caused by {e!s})') from e
else:
self._logger.debug(
f'_get_pot_via_script response = {script_data_resp}')
if potoken := script_data_resp.get('poToken'):
return potoken
else:
raise RequestError('The script did not respond with a po_token')


@register_preference(BgUtilScriptPotProviderRH)
Expand Down
78 changes: 50 additions & 28 deletions server/src/generate_once.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
import { SessionManager } from "./session_manager";
import { SessionManager, YoutubeSessionDataCaches } from "./session_manager";
import { Command } from "@commander-js/extra-typings";
import * as fs from "fs";
import * as path from "path";

const CACHE_PATH = path.resolve(__dirname, "..", "cache.json");
const program = new Command()
.option("-v, --visitor-data <visitordata>")
.option("-d, --data-sync-id <data-sync-id>")
Expand All @@ -13,38 +16,57 @@ const options = program.opts();
const dataSyncId = options.dataSyncId;
const visitorData = options.visitorData;
const verbose = options.verbose || false;
let visitorIdentifier: string;
const sessionManager = new SessionManager(verbose);
let visitIdentifier: string;
const cache: YoutubeSessionDataCaches = {};
if (fs.existsSync(CACHE_PATH)) {
try {
const parsedCaches = JSON.parse(
fs.readFileSync(CACHE_PATH, "utf8"),
);
for (const visitIdentifier in parsedCaches) {
const parsedCache = parsedCaches[visitIdentifier];
if (parsedCache) {
cache[visitIdentifier] = {
poToken: parsedCache.poToken,
generatedAt: new Date(parsedCache.generatedAt),
visitIdentifier,
};
}
}
} catch (e) {
console.warn(`Error parsing cache. e = ${e}`);
}
}

const sessionManager = new SessionManager(verbose, cache);
function log(msg: string) {
if (verbose) console.log(msg);
}

if (dataSyncId) {
if (verbose) {
console.log(`Received request for data sync ID: '${dataSyncId}'`);
}
visitorIdentifier = dataSyncId;
log(`Received request for data sync ID: '${dataSyncId}'`);
visitIdentifier = dataSyncId;
} else if (visitorData) {
if (verbose) {
console.log(`Received request for visitor data: '${visitorData}'`);
}
visitorIdentifier = visitorData;
log(`Received request for visitor data: '${visitorData}'`);
visitIdentifier = visitorData;
} else {
if (verbose) {
console.log(
`Received request for visitor data, grabbing from Innertube`,
);
}
log(`Received request for visitor data, grabbing from Innertube`);
const generatedVisitorData = await sessionManager.generateVisitorData();
if (!generatedVisitorData) {
console.error("Error generating visitor data");
process.exit(1);
}

if (verbose) {
console.log(`Generated visitor data: ${generatedVisitorData}`);
}

visitorIdentifier = generatedVisitorData;
if (!generatedVisitorData) process.exit(1);
log(`Generated visitor data: ${generatedVisitorData}`);
visitIdentifier = generatedVisitorData;
}

const sessionData = await sessionManager.generatePoToken(visitorIdentifier);
console.log(JSON.stringify(sessionData));
const sessionData = await sessionManager.generatePoToken(visitIdentifier);
try {
fs.writeFileSync(
CACHE_PATH,
JSON.stringify(sessionManager.getYoutubeSessionDataCaches(true)),
"utf8",
);
} catch (e) {
console.warn(`Error writing cache. e = ${e}`);
} finally {
console.log(JSON.stringify(sessionData));
}
})();
10 changes: 5 additions & 5 deletions server/src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ httpServer.post("/get_pot", async (request, response) => {
const visitorData = request.body.visitor_data as string;
const dataSyncId = request.body.data_sync_id as string;

let visitorIdentifier: string;
let visitIdentifier: string;

// prioritize data sync id for authenticated requests, if passed
if (dataSyncId) {
console.log(`Received request for data sync ID: '${dataSyncId}'`);
visitorIdentifier = dataSyncId;
visitIdentifier = dataSyncId;
} else if (visitorData) {
console.log(`Received request for visitor data: '${visitorData}'`);
visitorIdentifier = visitorData;
visitIdentifier = visitorData;
} else {
console.log(
`Received request for visitor data, grabbing from Innertube`,
Expand All @@ -48,10 +48,10 @@ httpServer.post("/get_pot", async (request, response) => {
}

console.log(`Generated visitor data: ${generatedVisitorData}`);
visitorIdentifier = generatedVisitorData;
visitIdentifier = generatedVisitorData;
}

const sessionData = await sessionManager.generatePoToken(visitorIdentifier);
const sessionData = await sessionManager.generatePoToken(visitIdentifier);
response.send({
po_token: sessionData.poToken,
visit_identifier: sessionData.visitIdentifier,
Expand Down
69 changes: 47 additions & 22 deletions server/src/session_manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,27 +8,61 @@ interface YoutubeSessionData {
generatedAt: Date;
}

export interface YoutubeSessionDataCaches {
[visitIdentifier: string]: YoutubeSessionData;
}

export class SessionManager {
shouldLog: boolean;

private youtubeSessionData: {
[visitIdentifier: string]: YoutubeSessionData;
} = {};
private youtubeSessionDataCaches: YoutubeSessionDataCaches = {};
private TOKEN_TTL_HOURS: number;

constructor(shouldLog = true) {
constructor(
shouldLog = true,
youtubeSessionDataCaches: YoutubeSessionDataCaches = {},
) {
this.shouldLog = shouldLog;
this.setYoutubeSessionDataCaches(youtubeSessionDataCaches);
this.TOKEN_TTL_HOURS = process.env.TOKEN_TTL
? parseInt(process.env.TOKEN_TTL)
: 6;
}

invalidateCaches() {
this.youtubeSessionData = {};
this.setYoutubeSessionDataCaches();
}

log(msg: string) {
if (this.shouldLog) {
console.log(msg);
cleanupCaches() {
for (const visitIdentifier in this.youtubeSessionDataCaches) {
const sessionData = this.youtubeSessionDataCaches[visitIdentifier];
if (
sessionData &&
sessionData.generatedAt <
new Date(
new Date().getTime() -
this.TOKEN_TTL_HOURS * 60 * 60 * 1000,
)
)
delete this.youtubeSessionDataCaches[visitIdentifier];
}
}

getYoutubeSessionDataCaches(cleanup = false) {
if (cleanup) this.cleanupCaches();
return this.youtubeSessionDataCaches;
}

setYoutubeSessionDataCaches(
youtubeSessionData: YoutubeSessionDataCaches = {},
) {
this.youtubeSessionDataCaches = youtubeSessionData || {};
}

log(msg: string) {
if (this.shouldLog) console.log(msg);
}

async generateVisitorData(): Promise<string | null> {
const innertube = await Innertube.create({ retrieve_player: false });
const visitorData = innertube.session.context.client.visitorData;
Expand All @@ -44,18 +78,9 @@ export class SessionManager {
async generatePoToken(
visitIdentifier: string,
): Promise<YoutubeSessionData> {
const TOKEN_TTL_HOURS = process.env.TOKEN_TTL
? parseInt(process.env.TOKEN_TTL)
: 6;

const sessionData = this.youtubeSessionData[visitIdentifier];
if (
sessionData &&
sessionData.generatedAt >
new Date(
new Date().getTime() - TOKEN_TTL_HOURS * 60 * 60 * 1000,
)
) {
this.cleanupCaches();
const sessionData = this.youtubeSessionDataCaches[visitIdentifier];
if (sessionData) {
this.log(
`POT for ${visitIdentifier} still fresh, returning cached token`,
);
Expand Down Expand Up @@ -104,12 +129,12 @@ export class SessionManager {
throw new Error("po_token unexpected undefined");
}

this.youtubeSessionData[visitIdentifier] = {
this.youtubeSessionDataCaches[visitIdentifier] = {
visitIdentifier: visitIdentifier,
poToken: poToken,
generatedAt: new Date(),
};

return this.youtubeSessionData[visitIdentifier];
return this.youtubeSessionDataCaches[visitIdentifier];
}
}

0 comments on commit 5bc73a9

Please sign in to comment.