Skip to content

Commit

Permalink
Fixed Single Threaded Download, Removed timeout argument
Browse files Browse the repository at this point in the history
  • Loading branch information
TechShreyash committed Jul 4, 2024
1 parent ff8d2db commit 473241c
Show file tree
Hide file tree
Showing 5 changed files with 48 additions and 55 deletions.
3 changes: 1 addition & 2 deletions DOCS.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# TechZDL v1.2.4 Documentation
# TechZDL v1.2.5 Documentation

## Installation

Expand Down Expand Up @@ -66,7 +66,6 @@ Here is a list of arguments you can pass to the `TechZDL` class to modify your d
- `chunk_size` `(int)`: Size of each download chunk in bytes. Defaults to 5 MB.
- `single_threaded` `(bool)`: Force single-threaded download. Defaults to False.
- `max_retries` `(int)`: Maximum retries for each chunk/file download. Defaults to 3.
- `timeout` `(int)`: Timeout for each request in seconds. Defaults to 60.

### Attributes

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# TechZDL v1.2.4
# TechZDL v1.2.5

TechZDL is a powerful file downloader package for Python that supports multi-threaded downloads, dynamic worker adjustments based on network speed, custom headers, and more.

Expand Down
6 changes: 2 additions & 4 deletions demos/timeout_and_max_retries.py → demos/max_retries.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,14 @@
# This script demonstrates how to configure the downloader to handle timeouts and retries.
# The 'timeout' parameter sets the maximum time (in seconds) to wait for a server response.
# This script demonstrates how to configure the downloader to handle retries.
# The 'max_retries' parameter sets the maximum number of retry attempts for each chunk or file download.
# These settings are useful for handling unreliable network conditions or server issues.

import asyncio
from techzdl.api import TechZDL
from techzdl import TechZDL


async def main():
downloader = TechZDL(
url="https://link.testfile.org/bNYZFw", # URL of the file to download
timeout=30, # Timeout in seconds for each request (default: 60 seconds)
max_retries=5, # Maximum number of retries for each chunk/file download (default: 3)
)
await downloader.start()
Expand Down
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

setup(
name="techzdl",
version="1.2.4",
version="1.2.5",
author="TechShreyash",
author_email="[email protected]",
description="A simple yet powerfull file downloader package for python",
Expand Down
90 changes: 43 additions & 47 deletions techzdl/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Name: techzdl
# Version: 1.2.4
# Version: 1.2.5
# Summary: A simple yet powerfull file downloader package for python
# Home-page: https://github.com/TechShreyash/techzdl
# Author: TechShreyash
Expand Down Expand Up @@ -43,7 +43,6 @@ def __init__(
chunk_size: int = 5 * 1024 * 1024,
single_threaded: bool = False,
max_retries: int = 3,
timeout: int = 60,
) -> None:
"""
Initialize the TechZDL object.
Expand All @@ -64,7 +63,6 @@ def __init__(
- `chunk_size` `(int, optional)`: Size of each download chunk in bytes. Defaults to 5 MB.
- `single_threaded` `(bool, optional)`: Force single-threaded download. Defaults to False.
- `max_retries` `(int, optional)`: Maximum retries for each chunk/file download. Defaults to 3.
- `timeout` `(int, optional)`: Timeout for each request in seconds. Defaults to 60.
#### Examples:
```python
Expand Down Expand Up @@ -105,7 +103,6 @@ async def main():
self.curl_cffi_required = False
self.max_retries = max_retries
self.session = None
self.timeout = timeout
self.is_running = False
self.downloader_tasks = []
self.temp_file_path = None
Expand Down Expand Up @@ -185,9 +182,7 @@ async def get_file_info(self) -> dict:
for i in range(self.max_retries):
try:

session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=self.timeout)
)
session = aiohttp.ClientSession()

self._log(f"Fetching file info from {self.url}")
response = None
Expand All @@ -213,7 +208,7 @@ async def get_file_info(self) -> dict:
)
await session.close()

session = AsyncSession(timeout=self.timeout)
session = AsyncSession()

response = None
try:
Expand Down Expand Up @@ -451,41 +446,49 @@ async def _dynamic_worker_updater(self, semaphore: AdjustableSemaphore) -> None:
prev_downloaded = self.size_done
prev_speed = speed

async def _single_threaded_download_child(self) -> None:
response = None
if self.curl_cffi_required:
try:
response = await self.session.get(
url=self.url, headers=self.custom_headers, stream=True
)
async with aiofiles.open(self.output_path, "wb") as output_file:
async for chunk in response.aiter_content():
await output_file.write(chunk)
self.size_done += len(chunk)
except Exception as e:
raise e
finally:
if response:
response.close()
else:
try:
response = await self.session.get(self.url, headers=self.custom_headers)
async with aiofiles.open(self.output_path, "wb") as output_file:
while chunk := await response.content.read(self.chunk_size):
await output_file.write(chunk)
self.size_done += len(chunk)
except Exception as e:
raise e
finally:
if response:
response.close()

async def _single_threaded_download(self) -> None:
"""
Perform a single-threaded download of the file.
"""
for i in range(self.max_retries):
self.size_done = 0 # Reset size_done if retrying

try:
response = None
if self.curl_cffi_required:
try:
response = await self.session.get(
url=self.url, headers=self.custom_headers, stream=True
)
async with aiofiles.open(self.output_path, "wb") as output_file:
async for chunk in response.aiter_content():
await output_file.write(chunk)
self.size_done += len(chunk)
except Exception as e:
raise e
finally:
if response:
response.close()
else:
try:
response = await self.session.get(
self.url, headers=self.custom_headers
)
async with aiofiles.open(self.output_path, "wb") as output_file:
while chunk := await response.content.read(self.chunk_size):
await output_file.write(chunk)
self.size_done += len(chunk)
except Exception as e:
raise e
finally:
if response:
response.close()
await self._task_runner(
[
self._single_threaded_download_child(),
self._show_progress("Downloading"),
]
)
break
except Exception as e:
self._log(f"Error downloading file: {e}", level="error")
Expand Down Expand Up @@ -557,9 +560,7 @@ async def _download_manager(self) -> Path:
try:
if self.session:
await self.session.close()
self.session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=self.timeout)
)
self.session = aiohttp.ClientSession()

self._log(f"Fetching file info from {self.url}")
response = None
Expand Down Expand Up @@ -590,7 +591,7 @@ async def _download_manager(self) -> Path:
)
await self.session.close()

self.session = AsyncSession(timeout=self.timeout)
self.session = AsyncSession()
self.curl_cffi_required = True

response = None
Expand Down Expand Up @@ -642,12 +643,7 @@ async def _download_manager(self) -> Path:
self._log("Starting single-threaded download")
self._log(f"Downloading {self.filename}")

await self._task_runner(
[
self._single_threaded_download(),
self._show_progress("Downloading"),
]
)
await self._single_threaded_download()
else:
self._log(
"Server supports range requests. Starting multi-threaded download"
Expand Down

0 comments on commit 473241c

Please sign in to comment.