diff --git a/src/koheesio/steps/download_file.py b/src/koheesio/steps/download_file.py index d4cb01b1..81ba72b9 100644 --- a/src/koheesio/steps/download_file.py +++ b/src/koheesio/steps/download_file.py @@ -1,3 +1,5 @@ +# TODO: add module description + from __future__ import annotations from typing import Any, Optional, Type @@ -50,10 +52,10 @@ def from_string(cls, mode: str) -> Type[FileWriteMode[Any]]: def write_mode(self) -> str: """Return the write mode for the given SFTPWriteMode.""" if self in {FileWriteMode.OVERWRITE, FileWriteMode.BACKUP, FileWriteMode.EXCLUSIVE}: - # Overwrite, Backup, and Exclusive modes set the file to be written from the beginning + # OVERWRITE, BACKUP, and EXCLUSIVE modes set the file to be written from the beginning return "wb" if self == FileWriteMode.APPEND: - # Append mode sets the file to be written from the end + # APPEND mode sets the file to be written from the end return "ab" @@ -126,7 +128,7 @@ def handle_file_write_modes(self, _filepath: Path, _filename: str) -> Optional[s def execute(self) -> Output: """ - Executes the file download process, handling different write modes and saving the file to the specified path. + Executes the file download process, handling different write modes, and saving the file to the specified path. """ _filename = Path(self.url).name _filepath = self.download_path / _filename @@ -135,14 +137,14 @@ def execute(self) -> Output: if (mode := self.handle_file_write_modes(_filepath, _filename)) is None: return self.output - # create the download path if it does not exist + # Create the download path if it does not exist self.output.download_file_path = _filepath self.output.download_file_path.touch(exist_ok=True) - # download the file content and write the downloaded content to the file + # Download the file content and write the downloaded content to the file with self.request() as response, self.output.download_file_path.open(mode=mode) as f: for chunk in response.iter_content(chunk_size=self.chunk_size): self.log.debug(f"Downloading chunk of size {len(chunk)}") - self.log.debug(f"Writing to file {self.output.download_file_path}") self.log.debug(f"Downloaded {f.tell()} bytes") + self.log.debug(f"Writing to file {self.output.download_file_path}") f.write(chunk) diff --git a/src/koheesio/steps/http.py b/src/koheesio/steps/http.py index 32309629..dc8de434 100644 --- a/src/koheesio/steps/http.py +++ b/src/koheesio/steps/http.py @@ -201,7 +201,7 @@ def set_outputs(self, response: requests.Response) -> None: self.output.response_json = response.json() except json.decoder.JSONDecodeError as e: - self.log.info(f"An error occurred while processing the JSON payload. Error message:\n{e.msg}") + self.log.error(f"An error occurred while processing the JSON payload. Error message:\n{e.msg}") def get_options(self) -> dict: """options to be passed to requests.request()""" @@ -298,7 +298,7 @@ def execute(self) -> None: The last exception that was caught if `self.request()` fails after `self.max_retries` attempts. """ with self.request() as response: - self.log.info(f"HTTP request to {self.url} was successful with status code {response.status_code}") + self.log.info(f"HTTP request to {self.url}, status code {response.status_code}") class HttpGetStep(HttpStep): diff --git a/tests/steps/test_download_file.py b/tests/steps/test_download_file.py index 6d6269e7..01b03a61 100644 --- a/tests/steps/test_download_file.py +++ b/tests/steps/test_download_file.py @@ -90,25 +90,32 @@ def test_download_file_step_ignore_mode(self, download_path, downloaded_file, ca # Arrange downloaded_file.write_bytes(b"foo") - with caplog.at_level("INFO"), Mocker() as mocker: - mocker.get(URL, content=b"bar") - - # FIXME: logging is not working in the unit tests - - # Act + # Act and Assert -- dry run + with caplog.at_level(logging.INFO): step = DownloadFileStep(url=URL, download_path=download_path, mode="ignore") step.log.setLevel("INFO") step.execute() - print(f"2 {caplog.record_tuples = }") - - # Assert - print(f"5 {caplog.text = }") assert "Ignoring testfile.txt based on IGNORE mode." in caplog.text - print(f"3 {caplog.text = }") - assert downloaded_file.exists() - print(f"4 {caplog.text = }") - assert downloaded_file.read_bytes() == b"foo" + # with caplog.at_level("INFO"), Mocker() as mocker: + # mocker.get(URL, content=b"bar") + # + # # FIXME: logging is not working in the unit tests + # + # # Act + # step = DownloadFileStep(url=URL, download_path=download_path, mode="ignore") + # step.log.setLevel("INFO") + # step.execute() + # print(f"2 {caplog.record_tuples = }") + # + # # Assert + # print(f"5 {caplog.text = }") + # assert "Ignoring testfile.txt based on IGNORE mode." in caplog.text + # + # print(f"3 {caplog.text = }") + # assert downloaded_file.exists() + # print(f"4 {caplog.text = }") + # assert downloaded_file.read_bytes() == b"foo" def test_download_file_step_exclusive_mode(self, download_path, downloaded_file): """In EXCLUSIVE mode, an error should be raised if the file exists"""