From 473241c911f80919c6e6ede4ed40aea55f88e12d Mon Sep 17 00:00:00 2001 From: TechShreyash <82265247+TechShreyash@users.noreply.github.com> Date: Thu, 4 Jul 2024 11:06:15 +0530 Subject: [PATCH] Fixed Single Threaded Download, Removed timeout argument --- DOCS.md | 3 +- README.md | 2 +- ...eout_and_max_retries.py => max_retries.py} | 6 +- setup.py | 2 +- techzdl/__init__.py | 90 +++++++++---------- 5 files changed, 48 insertions(+), 55 deletions(-) rename demos/{timeout_and_max_retries.py => max_retries.py} (70%) diff --git a/DOCS.md b/DOCS.md index ad20b21..aa4da8f 100644 --- a/DOCS.md +++ b/DOCS.md @@ -1,4 +1,4 @@ -# TechZDL v1.2.4 Documentation +# TechZDL v1.2.5 Documentation ## Installation @@ -66,7 +66,6 @@ Here is a list of arguments you can pass to the `TechZDL` class to modify your d - `chunk_size` `(int)`: Size of each download chunk in bytes. Defaults to 5 MB. - `single_threaded` `(bool)`: Force single-threaded download. Defaults to False. - `max_retries` `(int)`: Maximum retries for each chunk/file download. Defaults to 3. -- `timeout` `(int)`: Timeout for each request in seconds. Defaults to 60. ### Attributes diff --git a/README.md b/README.md index 05dfb9b..57bb43c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# TechZDL v1.2.4 +# TechZDL v1.2.5 TechZDL is a powerful file downloader package for Python that supports multi-threaded downloads, dynamic worker adjustments based on network speed, custom headers, and more. diff --git a/demos/timeout_and_max_retries.py b/demos/max_retries.py similarity index 70% rename from demos/timeout_and_max_retries.py rename to demos/max_retries.py index 60c11f6..f222a34 100644 --- a/demos/timeout_and_max_retries.py +++ b/demos/max_retries.py @@ -1,16 +1,14 @@ -# This script demonstrates how to configure the downloader to handle timeouts and retries. -# The 'timeout' parameter sets the maximum time (in seconds) to wait for a server response. +# This script demonstrates how to configure the downloader to handle retries. # The 'max_retries' parameter sets the maximum number of retry attempts for each chunk or file download. # These settings are useful for handling unreliable network conditions or server issues. import asyncio -from techzdl.api import TechZDL +from techzdl import TechZDL async def main(): downloader = TechZDL( url="https://link.testfile.org/bNYZFw", # URL of the file to download - timeout=30, # Timeout in seconds for each request (default: 60 seconds) max_retries=5, # Maximum number of retries for each chunk/file download (default: 3) ) await downloader.start() diff --git a/setup.py b/setup.py index 3012ab6..fccc05a 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="techzdl", - version="1.2.4", + version="1.2.5", author="TechShreyash", author_email="techshreyash123@gmail.com", description="A simple yet powerfull file downloader package for python", diff --git a/techzdl/__init__.py b/techzdl/__init__.py index 1fe7be5..07b9e01 100644 --- a/techzdl/__init__.py +++ b/techzdl/__init__.py @@ -1,5 +1,5 @@ # Name: techzdl -# Version: 1.2.4 +# Version: 1.2.5 # Summary: A simple yet powerfull file downloader package for python # Home-page: https://github.com/TechShreyash/techzdl # Author: TechShreyash @@ -43,7 +43,6 @@ def __init__( chunk_size: int = 5 * 1024 * 1024, single_threaded: bool = False, max_retries: int = 3, - timeout: int = 60, ) -> None: """ Initialize the TechZDL object. @@ -64,7 +63,6 @@ def __init__( - `chunk_size` `(int, optional)`: Size of each download chunk in bytes. Defaults to 5 MB. - `single_threaded` `(bool, optional)`: Force single-threaded download. Defaults to False. - `max_retries` `(int, optional)`: Maximum retries for each chunk/file download. Defaults to 3. - - `timeout` `(int, optional)`: Timeout for each request in seconds. Defaults to 60. #### Examples: ```python @@ -105,7 +103,6 @@ async def main(): self.curl_cffi_required = False self.max_retries = max_retries self.session = None - self.timeout = timeout self.is_running = False self.downloader_tasks = [] self.temp_file_path = None @@ -185,9 +182,7 @@ async def get_file_info(self) -> dict: for i in range(self.max_retries): try: - session = aiohttp.ClientSession( - timeout=aiohttp.ClientTimeout(total=self.timeout) - ) + session = aiohttp.ClientSession() self._log(f"Fetching file info from {self.url}") response = None @@ -213,7 +208,7 @@ async def get_file_info(self) -> dict: ) await session.close() - session = AsyncSession(timeout=self.timeout) + session = AsyncSession() response = None try: @@ -451,41 +446,49 @@ async def _dynamic_worker_updater(self, semaphore: AdjustableSemaphore) -> None: prev_downloaded = self.size_done prev_speed = speed + async def _single_threaded_download_child(self) -> None: + response = None + if self.curl_cffi_required: + try: + response = await self.session.get( + url=self.url, headers=self.custom_headers, stream=True + ) + async with aiofiles.open(self.output_path, "wb") as output_file: + async for chunk in response.aiter_content(): + await output_file.write(chunk) + self.size_done += len(chunk) + except Exception as e: + raise e + finally: + if response: + response.close() + else: + try: + response = await self.session.get(self.url, headers=self.custom_headers) + async with aiofiles.open(self.output_path, "wb") as output_file: + while chunk := await response.content.read(self.chunk_size): + await output_file.write(chunk) + self.size_done += len(chunk) + except Exception as e: + raise e + finally: + if response: + response.close() + async def _single_threaded_download(self) -> None: """ Perform a single-threaded download of the file. """ for i in range(self.max_retries): + self.size_done = 0 # Reset size_done if retrying + try: - response = None - if self.curl_cffi_required: - try: - response = await self.session.get( - url=self.url, headers=self.custom_headers, stream=True - ) - async with aiofiles.open(self.output_path, "wb") as output_file: - async for chunk in response.aiter_content(): - await output_file.write(chunk) - self.size_done += len(chunk) - except Exception as e: - raise e - finally: - if response: - response.close() - else: - try: - response = await self.session.get( - self.url, headers=self.custom_headers - ) - async with aiofiles.open(self.output_path, "wb") as output_file: - while chunk := await response.content.read(self.chunk_size): - await output_file.write(chunk) - self.size_done += len(chunk) - except Exception as e: - raise e - finally: - if response: - response.close() + await self._task_runner( + [ + self._single_threaded_download_child(), + self._show_progress("Downloading"), + ] + ) break except Exception as e: self._log(f"Error downloading file: {e}", level="error") @@ -557,9 +560,7 @@ async def _download_manager(self) -> Path: try: if self.session: await self.session.close() - self.session = aiohttp.ClientSession( - timeout=aiohttp.ClientTimeout(total=self.timeout) - ) + self.session = aiohttp.ClientSession() self._log(f"Fetching file info from {self.url}") response = None @@ -590,7 +591,7 @@ async def _download_manager(self) -> Path: ) await self.session.close() - self.session = AsyncSession(timeout=self.timeout) + self.session = AsyncSession() self.curl_cffi_required = True response = None @@ -642,12 +643,7 @@ async def _download_manager(self) -> Path: self._log("Starting single-threaded download") self._log(f"Downloading {self.filename}") - await self._task_runner( - [ - self._single_threaded_download(), - self._show_progress("Downloading"), - ] - ) + await self._single_threaded_download() else: self._log( "Server supports range requests. Starting multi-threaded download"