Skip to content

Commit

Permalink
file-based: fix mypy and styles
Browse files Browse the repository at this point in the history
  • Loading branch information
aldogonzalez8 committed Jan 7, 2025
1 parent 8ee5ac3 commit 1fd48aa
Show file tree
Hide file tree
Showing 4 changed files with 12 additions and 5 deletions.
4 changes: 3 additions & 1 deletion airbyte_cdk/sources/file_based/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,9 @@ class FileSizeLimitError(CustomFileBasedException):
pass


def format_duplicate_files_error_message(stream_name: str, duplicated_files_names: List):
def format_duplicate_files_error_message(
stream_name: str, duplicated_files_names: List[dict[str, List[str]]]
) -> str:
duplicated_files_messages = []
for duplicated_file in duplicated_files_names:
for duplicated_file_name, file_paths in duplicated_file.items():
Expand Down
8 changes: 5 additions & 3 deletions airbyte_cdk/sources/file_based/file_based_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -390,10 +390,12 @@ def _use_file_transfer(parsed_config: AbstractFileBasedSpec) -> bool:
return use_file_transfer

@staticmethod
def _preserve_subdirectories_directories(parsed_config: AbstractFileBasedSpec):
def _preserve_subdirectories_directories(parsed_config: AbstractFileBasedSpec) -> bool:
# fall back to preserve subdirectories if config is not present or incomplete
if hasattr(parsed_config, "delivery_options") and hasattr(
parsed_config.delivery_options, "preserve_subdirectories_directories"
if (
hasattr(parsed_config, "delivery_options")
and parsed_config.delivery_options is not None
and hasattr(parsed_config.delivery_options, "preserve_subdirectories_directories")
):
return parsed_config.delivery_options.preserve_subdirectories_directories
return True
1 change: 1 addition & 0 deletions airbyte_cdk/sources/file_based/file_based_stream_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ def preserve_subdirectories_directories(self) -> bool:
if (
self.config
and hasattr(self.config, "delivery_options")
and self.config.delivery_options is not None
and hasattr(self.config.delivery_options, "preserve_subdirectories_directories")
):
return self.config.delivery_options.preserve_subdirectories_directories
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,9 @@ def _filter_schema_invalid_properties(
else:
return super()._filter_schema_invalid_properties(configured_catalog_json_schema)

def _duplicated_files_names(self, slices: List) -> list[dict]:
def _duplicated_files_names(
self, slices: List[dict[str, List[RemoteFile]]]
) -> List[dict[str, List[str]]]:
seen_file_names = set()
duplicates_file_names = set()
file_paths = defaultdict(list)
Expand Down

0 comments on commit 1fd48aa

Please sign in to comment.