From f74dd4d0a97bdac48282c01b3ce3e4d8a8b3bb61 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Wed, 11 Sep 2024 08:29:49 +0000 Subject: [PATCH 01/14] [plugin] split RH, add preferences --- .../yt_dlp_plugins/extractor/getpot_bgutil.py | 118 ++++++++++-------- 1 file changed, 68 insertions(+), 50 deletions(-) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil.py index 2ede751..1a58ece 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil.py @@ -7,62 +7,25 @@ from yt_dlp.networking.common import Request from yt_dlp.networking.exceptions import RequestError, UnsupportedRequest from yt_dlp.utils import Popen -from yt_dlp_plugins.extractor.getpot import GetPOTProvider, register_provider +from yt_dlp_plugins.extractor.getpot import GetPOTProvider, register_provider, register_preference + + +__version__ = '0.4.0' @register_provider -class BgUtilPotProviderRH(GetPOTProvider): - _PROVIDER_NAME = 'BgUtilPot' +class BgUtilScriptPotProviderRH(GetPOTProvider): + _PROVIDER_NAME = 'BgUtilScriptPot' _SUPPORTED_CLIENTS = ('web_creator', 'web', 'web_embedded', 'web_music') + VERSION = __version__ def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs): + script_path = ydl.get_info_extractor('Youtube')._configuration_arg( + 'getpot_bgutil_script', [None], casesense=True)[0] + # TODO: Add default script path if not data_sync_id and not visitor_data: raise UnsupportedRequest( 'One of [data_sync_id, visitor_data] must be passed') - - def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs) -> str: - generate_pot_script_path = ydl.get_info_extractor('Youtube')._configuration_arg( - 'getpot_bgutil_script', [None], casesense=True)[0] - http_base_url = ydl.get_info_extractor('Youtube')._configuration_arg( - 'getpot_bgutil_baseurl', [None], casesense=True)[0] - if generate_pot_script_path: - self._logger.info( - f'Generating POT via script: {generate_pot_script_path}') - po_token = self._get_pot_via_script( - generate_pot_script_path, visitor_data, data_sync_id) - else: - self._logger.info('Generating POT via HTTP server') - po_token = self._get_pot_via_http( - ydl, client, visitor_data, data_sync_id, http_base_url) - - return po_token - - def _get_pot_via_http(self, ydl, client, visitor_data, data_sync_id, base_url): - if base_url is None: - base_url = 'http://127.0.0.1:4416' - try: - response = ydl.urlopen(Request(f'{base_url}/get_pot', data=json.dumps({ - 'client': client, - 'visitor_data': visitor_data, - 'data_sync_id': data_sync_id, - }).encode(), headers={'Content-Type': 'application/json'})) - except Exception as e: - raise RequestError(f'Error reaching POST /get_pot: {str(e)}') - - try: - response_json = json.load(response) - except Exception as e: - raise RequestError( - f'Error parsing response JSON(caused by {str(e)}). response = {response.read().decode()}') - - if error_msg := response_json.get('error'): - raise RequestError(error_msg) - if 'po_token' not in response_json: - raise RequestError('Server did not respond with a po_token') - - return response_json['po_token'] - - def _get_pot_via_script(self, script_path, visitor_data, data_sync_id): if not os.path.isfile(script_path): raise RequestError(f"Script path doesn't exist: {script_path}") if os.path.basename(script_path) != 'generate_once.js': @@ -70,8 +33,13 @@ def _get_pot_via_script(self, script_path, visitor_data, data_sync_id): 'Incorrect script passed to extractor args. Path to generate_once.js required') if shutil.which('node') is None: raise RequestError('node is not in PATH') + self.script_path = script_path + + def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs) -> str: + self._logger.info( + f'Generating POT via script: {self.script_path}') - command_args = ['node', script_path] + command_args = ['node', self.script_path] if data_sync_id: command_args.extend(['-d', data_sync_id]) elif visitor_data: @@ -79,6 +47,7 @@ def _get_pot_via_script(self, script_path, visitor_data, data_sync_id): else: raise RequestError( 'Unexpected missing visitorData and dataSyncId in _get_pot_via_script') + # After invoking _validate_get_pot, do we still need this? self._logger.debug( f'Executing command to get POT via script: {" ".join(command_args)}') @@ -87,7 +56,7 @@ def _get_pot_via_script(self, script_path, visitor_data, data_sync_id): command_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) except Exception as e: raise RequestError( - f'_get_pot_via_script failed: Unable to run script(caused by {str(e)})') + f'_get_pot_via_script failed: Unable to run script (caused by {str(e)})') self._logger.debug(f'stdout = {stdout}') if returncode: @@ -102,4 +71,53 @@ def _get_pot_via_script(self, script_path, visitor_data, data_sync_id): return json.loads(script_data_resp)['poToken'] except (json.JSONDecodeError, TypeError, KeyError) as e: raise RequestError( - f'Error parsing JSON response from _get_pot_via_script(caused by {str(e)})') + f'Error parsing JSON response from _get_pot_via_script (caused by {str(e)})') + +@register_preference(BgUtilScriptPotProviderRH) +def bgutil_script_getpot_peference(rh, request): + return 100 + +@register_provider +class BgUtilHTTPPotProviderRH(GetPOTProvider): + _PROVIDER_NAME = 'BgUtilHTTPPot' + _SUPPORTED_CLIENTS = ('web_creator', 'web', 'web_embedded', 'web_music') + VERSION = __version__ + + def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs): + base_url = ydl.get_info_extractor('Youtube')._configuration_arg( + 'getpot_bgutil_baseurl', ['http://127.0.0.1:4416'], casesense=True)[0] + if not data_sync_id and not visitor_data: + raise UnsupportedRequest( + 'One of [data_sync_id, visitor_data] must be passed') + # TODO: Ping the server + self.base_url = base_url + + def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs) -> str: + self._logger.info('Generating POT via HTTP server') + + try: + response = ydl.urlopen(Request(f'{self.base_url}/get_pot', data=json.dumps({ + 'client': client, + 'visitor_data': visitor_data, + 'data_sync_id': data_sync_id, + }).encode(), headers={'Content-Type': 'application/json'})) + except Exception as e: + raise RequestError(f'Error reaching POST /get_pot: {str(e)}') + + try: + response_json = json.load(response) + except Exception as e: + raise RequestError( + f'Error parsing response JSON (caused by {str(e)}). response = {response.read().decode()}') + + if error_msg := response_json.get('error'): + raise RequestError(error_msg) + if 'po_token' not in response_json: + raise RequestError('Server did not respond with a po_token') + # Should we return None here? + + return response_json['po_token'] + +@register_preference(BgUtilHTTPPotProviderRH) +def bgutil_HTTP_getpot_peference(rh, request): + return 0 From 1d7216bc3f0dc4e543014d85cb3475cc913243b7 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Wed, 11 Sep 2024 08:42:40 +0000 Subject: [PATCH 02/14] [plugin] split files for the two RHs --- .../extractor/getpot_bgutil_http.py | 56 +++++++++++++++++++ ...tpot_bgutil.py => getpot_bgutil_script.py} | 46 --------------- 2 files changed, 56 insertions(+), 46 deletions(-) create mode 100644 plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py rename plugin/yt_dlp_plugins/extractor/{getpot_bgutil.py => getpot_bgutil_script.py} (62%) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py new file mode 100644 index 0000000..119ef9d --- /dev/null +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py @@ -0,0 +1,56 @@ +import json +from yt_dlp import YoutubeDL + +from yt_dlp.networking.common import Request +from yt_dlp.networking.exceptions import RequestError, UnsupportedRequest +from yt_dlp.utils import Popen +from yt_dlp_plugins.extractor.getpot import GetPOTProvider, register_provider, register_preference + + +__version__ = '0.4.0' + + +@register_provider +class BgUtilHTTPPotProviderRH(GetPOTProvider): + _PROVIDER_NAME = 'BgUtilHTTPPot' + _SUPPORTED_CLIENTS = ('web_creator', 'web', 'web_embedded', 'web_music') + VERSION = __version__ + + def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs): + base_url = ydl.get_info_extractor('Youtube')._configuration_arg( + 'getpot_bgutil_baseurl', ['http://127.0.0.1:4416'], casesense=True)[0] + if not data_sync_id and not visitor_data: + raise UnsupportedRequest( + 'One of [data_sync_id, visitor_data] must be passed') + # TODO: Ping the server + self.base_url = base_url + + def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs) -> str: + self._logger.info('Generating POT via HTTP server') + + try: + response = ydl.urlopen(Request(f'{self.base_url}/get_pot', data=json.dumps({ + 'client': client, + 'visitor_data': visitor_data, + 'data_sync_id': data_sync_id, + }).encode(), headers={'Content-Type': 'application/json'})) + except Exception as e: + raise RequestError(f'Error reaching POST /get_pot: {str(e)}') + + try: + response_json = json.load(response) + except Exception as e: + raise RequestError( + f'Error parsing response JSON (caused by {str(e)}). response = {response.read().decode()}') + + if error_msg := response_json.get('error'): + raise RequestError(error_msg) + if 'po_token' not in response_json: + raise RequestError('Server did not respond with a po_token') + # Should we return None here? + + return response_json['po_token'] + +@register_preference(BgUtilHTTPPotProviderRH) +def bgutil_HTTP_getpot_peference(rh, request): + return 0 diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py similarity index 62% rename from plugin/yt_dlp_plugins/extractor/getpot_bgutil.py rename to plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index 1a58ece..acc08ab 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -4,7 +4,6 @@ import shutil from yt_dlp import YoutubeDL -from yt_dlp.networking.common import Request from yt_dlp.networking.exceptions import RequestError, UnsupportedRequest from yt_dlp.utils import Popen from yt_dlp_plugins.extractor.getpot import GetPOTProvider, register_provider, register_preference @@ -76,48 +75,3 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= @register_preference(BgUtilScriptPotProviderRH) def bgutil_script_getpot_peference(rh, request): return 100 - -@register_provider -class BgUtilHTTPPotProviderRH(GetPOTProvider): - _PROVIDER_NAME = 'BgUtilHTTPPot' - _SUPPORTED_CLIENTS = ('web_creator', 'web', 'web_embedded', 'web_music') - VERSION = __version__ - - def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs): - base_url = ydl.get_info_extractor('Youtube')._configuration_arg( - 'getpot_bgutil_baseurl', ['http://127.0.0.1:4416'], casesense=True)[0] - if not data_sync_id and not visitor_data: - raise UnsupportedRequest( - 'One of [data_sync_id, visitor_data] must be passed') - # TODO: Ping the server - self.base_url = base_url - - def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs) -> str: - self._logger.info('Generating POT via HTTP server') - - try: - response = ydl.urlopen(Request(f'{self.base_url}/get_pot', data=json.dumps({ - 'client': client, - 'visitor_data': visitor_data, - 'data_sync_id': data_sync_id, - }).encode(), headers={'Content-Type': 'application/json'})) - except Exception as e: - raise RequestError(f'Error reaching POST /get_pot: {str(e)}') - - try: - response_json = json.load(response) - except Exception as e: - raise RequestError( - f'Error parsing response JSON (caused by {str(e)}). response = {response.read().decode()}') - - if error_msg := response_json.get('error'): - raise RequestError(error_msg) - if 'po_token' not in response_json: - raise RequestError('Server did not respond with a po_token') - # Should we return None here? - - return response_json['po_token'] - -@register_preference(BgUtilHTTPPotProviderRH) -def bgutil_HTTP_getpot_peference(rh, request): - return 0 From 1b5c5c23d0dfae8cd143695e45d32c5a774a4519 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:26:23 +0000 Subject: [PATCH 03/14] dynamic version --- plugin/pyproject.toml | 5 ++++- plugin/yt_dlp_plugins/extractor/getpot_bgutil.py | 1 + plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py | 4 +--- plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py | 4 +--- 4 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 plugin/yt_dlp_plugins/extractor/getpot_bgutil.py diff --git a/plugin/pyproject.toml b/plugin/pyproject.toml index fcf1834..ead9bc9 100644 --- a/plugin/pyproject.toml +++ b/plugin/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "bgutil-ytdlp-pot-provider" -version = "0.4.0" +dynamic = ["version"] readme = {file = "README.md", content-type = "text/markdown"} classifiers = [ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)" @@ -27,3 +27,6 @@ packages = ["yt_dlp_plugins"] [tool.hatch.metadata] allow-direct-references = true + +[tool.hatch.version] +path = "yt_dlp_plugins/extractor/getpot_bgutil.py" diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil.py new file mode 100644 index 0000000..abeeedb --- /dev/null +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil.py @@ -0,0 +1 @@ +__version__ = '0.4.0' diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py index 119ef9d..53846cb 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py @@ -5,9 +5,7 @@ from yt_dlp.networking.exceptions import RequestError, UnsupportedRequest from yt_dlp.utils import Popen from yt_dlp_plugins.extractor.getpot import GetPOTProvider, register_provider, register_preference - - -__version__ = '0.4.0' +from yt_dlp_plugins.extractor.getpot_bgutil import __version__ @register_provider diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index acc08ab..e587b56 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -7,9 +7,7 @@ from yt_dlp.networking.exceptions import RequestError, UnsupportedRequest from yt_dlp.utils import Popen from yt_dlp_plugins.extractor.getpot import GetPOTProvider, register_provider, register_preference - - -__version__ = '0.4.0' +from yt_dlp_plugins.extractor.getpot_bgutil import __version__ @register_provider From f751c7fd3f8127723f04a2e21964d41c1d215682 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Wed, 11 Sep 2024 21:30:23 +1200 Subject: [PATCH 04/14] fix typo in function names `preference` --- plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py | 2 +- plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py index 53846cb..adb0ef4 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py @@ -50,5 +50,5 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= return response_json['po_token'] @register_preference(BgUtilHTTPPotProviderRH) -def bgutil_HTTP_getpot_peference(rh, request): +def bgutil_HTTP_getpot_preference(rh, request): return 0 diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index e587b56..6b0c76b 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -71,5 +71,5 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= f'Error parsing JSON response from _get_pot_via_script (caused by {str(e)})') @register_preference(BgUtilScriptPotProviderRH) -def bgutil_script_getpot_peference(rh, request): +def bgutil_script_getpot_preference(rh, request): return 100 From 532e6102b21faf562bde05fb49f955eb41c694c0 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Wed, 11 Sep 2024 21:32:12 +1200 Subject: [PATCH 05/14] remove annotation --- plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index 6b0c76b..6ee1b4d 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -44,7 +44,6 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= else: raise RequestError( 'Unexpected missing visitorData and dataSyncId in _get_pot_via_script') - # After invoking _validate_get_pot, do we still need this? self._logger.debug( f'Executing command to get POT via script: {" ".join(command_args)}') From ea7f8c9d8a8125ca58881018b704a551b4ffca00 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Wed, 11 Sep 2024 09:39:06 +0000 Subject: [PATCH 06/14] add `from` to error handling, code formatting --- plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py | 7 ++++--- plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py index adb0ef4..1a1f01f 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py @@ -3,7 +3,6 @@ from yt_dlp.networking.common import Request from yt_dlp.networking.exceptions import RequestError, UnsupportedRequest -from yt_dlp.utils import Popen from yt_dlp_plugins.extractor.getpot import GetPOTProvider, register_provider, register_preference from yt_dlp_plugins.extractor.getpot_bgutil import __version__ @@ -33,13 +32,14 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= 'data_sync_id': data_sync_id, }).encode(), headers={'Content-Type': 'application/json'})) except Exception as e: - raise RequestError(f'Error reaching POST /get_pot: {str(e)}') + raise RequestError( + f'Error reaching POST /get_pot: {str(e)}') from e try: response_json = json.load(response) except Exception as e: raise RequestError( - f'Error parsing response JSON (caused by {str(e)}). response = {response.read().decode()}') + f'Error parsing response JSON (caused by {str(e)}). response = {response.read().decode()}') from e if error_msg := response_json.get('error'): raise RequestError(error_msg) @@ -49,6 +49,7 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= return response_json['po_token'] + @register_preference(BgUtilHTTPPotProviderRH) def bgutil_HTTP_getpot_preference(rh, request): return 0 diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index 6ee1b4d..e44615e 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -52,7 +52,7 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= command_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) except Exception as e: raise RequestError( - f'_get_pot_via_script failed: Unable to run script (caused by {str(e)})') + f'_get_pot_via_script failed: Unable to run script (caused by {str(e)})') from e self._logger.debug(f'stdout = {stdout}') if returncode: @@ -67,7 +67,8 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= return json.loads(script_data_resp)['poToken'] except (json.JSONDecodeError, TypeError, KeyError) as e: raise RequestError( - f'Error parsing JSON response from _get_pot_via_script (caused by {str(e)})') + f'Error parsing JSON response from _get_pot_via_script (caused by {str(e)})') from e + @register_preference(BgUtilScriptPotProviderRH) def bgutil_script_getpot_preference(rh, request): From ccb9bc6e70f227ae6dd7748c756743575fb5af1d Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Wed, 11 Sep 2024 21:40:53 +1200 Subject: [PATCH 07/14] remove annotation --- plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py index 1a1f01f..da00e89 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py @@ -45,7 +45,6 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= raise RequestError(error_msg) if 'po_token' not in response_json: raise RequestError('Server did not respond with a po_token') - # Should we return None here? return response_json['po_token'] From 4b5d6ef13607e34519e05584a553685d65b5e938 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Wed, 11 Sep 2024 10:01:04 +0000 Subject: [PATCH 08/14] add default script path --- .../yt_dlp_plugins/extractor/getpot_bgutil_script.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index e44615e..a478ec6 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -5,7 +5,7 @@ from yt_dlp import YoutubeDL from yt_dlp.networking.exceptions import RequestError, UnsupportedRequest -from yt_dlp.utils import Popen +from yt_dlp.utils import Popen, classproperty from yt_dlp_plugins.extractor.getpot import GetPOTProvider, register_provider, register_preference from yt_dlp_plugins.extractor.getpot_bgutil import __version__ @@ -16,10 +16,15 @@ class BgUtilScriptPotProviderRH(GetPOTProvider): _SUPPORTED_CLIENTS = ('web_creator', 'web', 'web_embedded', 'web_music') VERSION = __version__ + @classproperty(cache=True) + def _default_script_path(self): + home = os.path.expanduser("~") + return os.path.join( + home, 'bgutil-ytdlp-pot-provider', 'server', 'build', 'generate_once.js') + def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs): script_path = ydl.get_info_extractor('Youtube')._configuration_arg( - 'getpot_bgutil_script', [None], casesense=True)[0] - # TODO: Add default script path + 'getpot_bgutil_script', [self._default_script_path], casesense=True)[0] if not data_sync_id and not visitor_data: raise UnsupportedRequest( 'One of [data_sync_id, visitor_data] must be passed') From 8470d0b715b8bb356dd4e6d07b4d130540a18848 Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Wed, 11 Sep 2024 10:10:12 +0000 Subject: [PATCH 09/14] use UnsupportedRequest in _validate_get_pot --- plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index a478ec6..02d41d2 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -29,12 +29,13 @@ def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data raise UnsupportedRequest( 'One of [data_sync_id, visitor_data] must be passed') if not os.path.isfile(script_path): - raise RequestError(f"Script path doesn't exist: {script_path}") + raise UnsupportedRequest( + f"Script path doesn't exist: {script_path}") if os.path.basename(script_path) != 'generate_once.js': - raise RequestError( + raise UnsupportedRequest( 'Incorrect script passed to extractor args. Path to generate_once.js required') if shutil.which('node') is None: - raise RequestError('node is not in PATH') + raise UnsupportedRequest('node is not in PATH') self.script_path = script_path def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs) -> str: From aaa02d5c14027cc9d9957b298718f37ff7a894eb Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Wed, 11 Sep 2024 10:24:03 +0000 Subject: [PATCH 10/14] update docs --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2e1576d..d4073b1 100644 --- a/README.md +++ b/README.md @@ -60,12 +60,12 @@ node build/main.js **Endpoints** - **POST /get_pot**: Accepts a `visitor_data` (unauthenticated), `data_sync_id` (authenticated) or an empty body in the request body. If no identifier is passed, a new unauthenticated `visitor_data` will be generated. Returns `po_token` and the associated identifier `visit_identifier`. -- **POST /invalidate_caches**: Resets the PO token cache, forcing new tokens to be generated on next fetch +- **POST /invalidate_caches**: Resets the PO token cache, forcing new tokens to be generated on next fetch. #### (b) Generation Script Option -The generation script needs to be transpiled to Javascript before it can be used by the plugin. +1. Transpile the generation script to Javascript: ```shell cd server/ @@ -73,7 +73,9 @@ yarn install --frozen-lockfile npx tsc ``` -Make sure `node` is available in your `PATH`. +2. Move the `server` directory with transpiled Javascript to `~/bgutil-ytdlp-pot-provider/` if you're using Linux or MacOS. +Use `%USERPROFILE%\bgutil-ytdlp-pot-provider\` instead if you're using Windows. +3. Make sure `node` is available in your `PATH`. ### 2. Install the plugin @@ -108,7 +110,8 @@ If changing the port or IP used for the provider server, pass it to yt-dlp via ` --- -If using option (b) script for the provider, you need to pass the extractor argument `getpot_bgutil_script` to `youtube` for each yt-dlp call. The argument should include the path to the transpiled generation script (`server/build/generate_once.js`). +If using option (b) script for the provider. +When using option (b) script for the provider, if you want to use a custom path to the transpiled generation script (`server/build/generate_once.js`), pass it as the extractor argument `getpot_bgutil_script` to `youtube` for each yt-dlp call. ```shell --extractor-args "youtube:getpot_bgutil_script=$WORKSPACE/bgutil-ytdlp-pot-provider/server/build/generate_once.js" From 429ae5fa1a2739f3bebc271e3da84a638063986e Mon Sep 17 00:00:00 2001 From: N/Ame <173015200+grqz@users.noreply.github.com> Date: Wed, 11 Sep 2024 23:12:46 +1200 Subject: [PATCH 11/14] code formatting --- plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py | 2 +- plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py index da00e89..0ac95f1 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py @@ -33,7 +33,7 @@ def _get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id= }).encode(), headers={'Content-Type': 'application/json'})) except Exception as e: raise RequestError( - f'Error reaching POST /get_pot: {str(e)}') from e + f'Error reaching POST /get_pot (caused by {str(e)})') from e try: response_json = json.load(response) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index 02d41d2..0707c49 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -18,7 +18,7 @@ class BgUtilScriptPotProviderRH(GetPOTProvider): @classproperty(cache=True) def _default_script_path(self): - home = os.path.expanduser("~") + home = os.path.expanduser('~') return os.path.join( home, 'bgutil-ytdlp-pot-provider', 'server', 'build', 'generate_once.js') From a4eeaafadaba182585d1db2669b1eb653af5aef4 Mon Sep 17 00:00:00 2001 From: Brian Le Date: Wed, 11 Sep 2024 14:16:28 -0700 Subject: [PATCH 12/14] Support all web clients (#21) --- plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py | 2 +- plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py index 0ac95f1..c1a82ac 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_http.py @@ -10,7 +10,7 @@ @register_provider class BgUtilHTTPPotProviderRH(GetPOTProvider): _PROVIDER_NAME = 'BgUtilHTTPPot' - _SUPPORTED_CLIENTS = ('web_creator', 'web', 'web_embedded', 'web_music') + _SUPPORTED_CLIENTS = ('web', 'web_safari', 'web_embedded', 'web_music', 'web_creator') VERSION = __version__ def _validate_get_pot(self, client: str, ydl: YoutubeDL, visitor_data=None, data_sync_id=None, player_url=None, **kwargs): diff --git a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py index 0707c49..383b23d 100644 --- a/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py +++ b/plugin/yt_dlp_plugins/extractor/getpot_bgutil_script.py @@ -13,7 +13,7 @@ @register_provider class BgUtilScriptPotProviderRH(GetPOTProvider): _PROVIDER_NAME = 'BgUtilScriptPot' - _SUPPORTED_CLIENTS = ('web_creator', 'web', 'web_embedded', 'web_music') + _SUPPORTED_CLIENTS = ('web', 'web_safari', 'web_embedded', 'web_music', 'web_creator') VERSION = __version__ @classproperty(cache=True) From 13e620380f3f5b6ffc7b9c5a2b750eae00858ea3 Mon Sep 17 00:00:00 2001 From: Brian Le Date: Wed, 11 Sep 2024 14:52:23 -0700 Subject: [PATCH 13/14] Update README with home directory instructions --- README.md | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d4073b1..432982b 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,8 @@ docker run --name bgutil-provider -d -p 4416:4416 brainicism/bgutil-ytdlp-pot-pr **Native:** ```shell -cd server/ +# download and extract latest source code from https://github.com/Brainicism/bgutil-ytdlp-pot-provider/releases +cd bgutils-ytdlp-pot-provider/server/ yarn install --frozen-lockfile npx tsc node build/main.js @@ -65,17 +66,19 @@ node build/main.js #### (b) Generation Script Option -1. Transpile the generation script to Javascript: +1. Download the latest source code (.zip/.tar.gz) from https://github.com/Brainicism/bgutil-ytdlp-pot-provider/releases. Extract the contents into your home directory (`~/` on Unix-based systems, `%USERPROFILE%` for Windows) if you want to use this method without needing to specify `getpot_bgutil_script` extractor argument on each yt-dlp invocation. + +2. Rename the folder to `bgutils-ytdlp-pot-provider` + +3. Transpile the generation script to Javascript: ```shell -cd server/ +cd bgutils-ytdlp-pot-provider/server/ yarn install --frozen-lockfile npx tsc ``` -2. Move the `server` directory with transpiled Javascript to `~/bgutil-ytdlp-pot-provider/` if you're using Linux or MacOS. -Use `%USERPROFILE%\bgutil-ytdlp-pot-provider\` instead if you're using Windows. -3. Make sure `node` is available in your `PATH`. +4. Make sure `node` is available in your `PATH`. ### 2. Install the plugin @@ -94,7 +97,7 @@ This will automatically install [coletdjnz's POT plugin framework](https://githu ## Usage -If using option (a) HTTP Server for the provider, use yt-dlp like normal 🙂. +If using option (a) HTTP Server for the provider, and the default IP/port number, you can use yt-dlp like normal 🙂. If you want to change the port number used by the provider server, use the `--port` option. @@ -110,9 +113,14 @@ If changing the port or IP used for the provider server, pass it to yt-dlp via ` --- -If using option (b) script for the provider. -When using option (b) script for the provider, if you want to use a custom path to the transpiled generation script (`server/build/generate_once.js`), pass it as the extractor argument `getpot_bgutil_script` to `youtube` for each yt-dlp call. +If using option (b) script for the provider, with the default script location in your home directory (i.e: `~/bgutil-ytdlp-pot-provider`), you can also use yt-dlp like normal. + +If you installed the script in a different location, pass it as the extractor argument `getpot_bgutil_script` to `youtube` for each yt-dlp call. ```shell --extractor-args "youtube:getpot_bgutil_script=$WORKSPACE/bgutil-ytdlp-pot-provider/server/build/generate_once.js" ``` + +--- + +If both methods are available for use, the option (b) script will be prioritized. From 7f51a34e61792e34de096a1f12a69e24172f4c7d Mon Sep 17 00:00:00 2001 From: Brian Le Date: Wed, 11 Sep 2024 17:19:03 -0700 Subject: [PATCH 14/14] Replace extract instructions with git clone --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 432982b..ba5e6a6 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,8 @@ docker run --name bgutil-provider -d -p 4416:4416 brainicism/bgutil-ytdlp-pot-pr **Native:** ```shell -# download and extract latest source code from https://github.com/Brainicism/bgutil-ytdlp-pot-provider/releases +# replace 0.4.0 with the latest version, or a matching plugin +git clone --single-branch --branch 0.4.0 https://github.com/Brainicism/bgutil-ytdlp-pot-provider.git cd bgutils-ytdlp-pot-provider/server/ yarn install --frozen-lockfile npx tsc @@ -66,19 +67,19 @@ node build/main.js #### (b) Generation Script Option -1. Download the latest source code (.zip/.tar.gz) from https://github.com/Brainicism/bgutil-ytdlp-pot-provider/releases. Extract the contents into your home directory (`~/` on Unix-based systems, `%USERPROFILE%` for Windows) if you want to use this method without needing to specify `getpot_bgutil_script` extractor argument on each yt-dlp invocation. - -2. Rename the folder to `bgutils-ytdlp-pot-provider` - -3. Transpile the generation script to Javascript: +1. Transpile the generation script to Javascript: ```shell +# Clone/extract the contents into your home directory (`~/` on Unix-based systems, `%USERPROFILE%` for Windows) if you want to use this method without needing to specify `getpot_bgutil_script` extractor argument on each yt-dlp invocation. +cd ~ +# replace 0.4.0 with the latest version, or a matching plugin +git clone --single-branch --branch 0.4.0 https://github.com/Brainicism/bgutil-ytdlp-pot-provider.git cd bgutils-ytdlp-pot-provider/server/ yarn install --frozen-lockfile npx tsc ``` -4. Make sure `node` is available in your `PATH`. +2. Make sure `node` is available in your `PATH`. ### 2. Install the plugin