From edf68169225049bc0b26e5934909e7ad3e6efe50 Mon Sep 17 00:00:00 2001 From: Aldo Gonzalez Date: Tue, 7 Jan 2025 17:28:48 -0800 Subject: [PATCH] file-based: move toggle to root --- .../config/abstract_file_based_spec.py | 24 ++++------ .../sources/file_based/file_based_source.py | 7 ++- .../file_based/file_based_stream_reader.py | 7 ++- .../file_based/scenarios/csv_scenarios.py | 46 ++++--------------- 4 files changed, 25 insertions(+), 59 deletions(-) diff --git a/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py b/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py index 49e1618b..fda17715 100644 --- a/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +++ b/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py @@ -14,13 +14,6 @@ from airbyte_cdk.sources.utils import schema_helpers -class DeliveryOptions(BaseModel): - preserve_subdirectories_directories: bool = Field( - True, - description="Flag indicating we should preserve subdirectories directories", - ) - - class DeliverRecords(BaseModel): class Config(OneOfOptionConfig): title = "Replicate Records" @@ -37,10 +30,11 @@ class Config(OneOfOptionConfig): discriminator = "delivery_type" delivery_type: Literal["use_file_transfer"] = Field("use_file_transfer", const=True) - delivery_options: Optional[DeliveryOptions] = Field( - title="Delivery Options", - type="object", - order=2, + + preserve_subdirectories_directories: bool = Field( + title="Preserve Subdirectories Directories", + description="Flag indicating we should preserve subdirectories directories", + default=True, ) @@ -77,10 +71,10 @@ class AbstractFileBasedSpec(BaseModel): airbyte_hidden=True, ) - delivery_options: Optional[DeliveryOptions] = Field( - title="Delivery Options", - type="object", - order=8, + preserve_subdirectories_directories: bool = Field( + title="Preserve Subdirectories Directories", + description="Flag indicating we should preserve subdirectories directories", + default=True, ) @classmethod diff --git a/airbyte_cdk/sources/file_based/file_based_source.py b/airbyte_cdk/sources/file_based/file_based_source.py index e63d1b87..400ed421 100644 --- a/airbyte_cdk/sources/file_based/file_based_source.py +++ b/airbyte_cdk/sources/file_based/file_based_source.py @@ -393,9 +393,8 @@ def _use_file_transfer(parsed_config: AbstractFileBasedSpec) -> bool: def _preserve_subdirectories_directories(parsed_config: AbstractFileBasedSpec) -> bool: # fall back to preserve subdirectories if config is not present or incomplete if ( - hasattr(parsed_config, "delivery_options") - and parsed_config.delivery_options is not None - and hasattr(parsed_config.delivery_options, "preserve_subdirectories_directories") + hasattr(parsed_config, "preserve_subdirectories_directories") + and parsed_config.preserve_subdirectories_directories is not None ): - return parsed_config.delivery_options.preserve_subdirectories_directories + return parsed_config.preserve_subdirectories_directories return True diff --git a/airbyte_cdk/sources/file_based/file_based_stream_reader.py b/airbyte_cdk/sources/file_based/file_based_stream_reader.py index babac114..45354f49 100644 --- a/airbyte_cdk/sources/file_based/file_based_stream_reader.py +++ b/airbyte_cdk/sources/file_based/file_based_stream_reader.py @@ -139,11 +139,10 @@ def preserve_subdirectories_directories(self) -> bool: # fall back to preserve subdirectories if config is not present or incomplete if ( self.config - and hasattr(self.config, "delivery_options") - and self.config.delivery_options is not None - and hasattr(self.config.delivery_options, "preserve_subdirectories_directories") + and hasattr(self.config, "preserve_subdirectories_directories") + and self.config.preserve_subdirectories_directories is not None ): - return self.config.delivery_options.preserve_subdirectories_directories + return self.config.preserve_subdirectories_directories return True @abstractmethod diff --git a/unit_tests/sources/file_based/scenarios/csv_scenarios.py b/unit_tests/sources/file_based/scenarios/csv_scenarios.py index ad1aa6b8..906d115e 100644 --- a/unit_tests/sources/file_based/scenarios/csv_scenarios.py +++ b/unit_tests/sources/file_based/scenarios/csv_scenarios.py @@ -517,25 +517,6 @@ "title": "Copy Raw Files", "type": "object", "properties": { - "delivery_options": { - "allOf": [ - { - "properties": { - "preserve_subdirectories_directories": { - "default": True, - "description": "Flag indicating we should preserve subdirectories directories", - "title": "Preserve Subdirectories Directories", - "type": "boolean", - } - }, - "title": "DeliveryOptions", - "type": "object", - } - ], - "order": 2, - "title": "Delivery Options", - "type": "object", - }, "delivery_type": { "title": "Delivery Type", "default": "use_file_transfer", @@ -543,30 +524,23 @@ "enum": ["use_file_transfer"], "type": "string", }, - }, - "description": "Copy raw files without parsing their contents. Bits are copied into the destination exactly as they appeared in the source. Recommended for use with unstructured text data, non-text and compressed files.", - "required": ["delivery_type"], - }, - ], - }, - "delivery_options": { - "allOf": [ - { - "properties": { "preserve_subdirectories_directories": { "default": True, "description": "Flag indicating we should preserve subdirectories directories", "title": "Preserve Subdirectories Directories", "type": "boolean", - } + }, }, - "title": "DeliveryOptions", - "type": "object", - } + "description": "Copy raw files without parsing their contents. Bits are copied into the destination exactly as they appeared in the source. Recommended for use with unstructured text data, non-text and compressed files.", + "required": ["delivery_type"], + }, ], - "order": 8, - "title": "Delivery Options", - "type": "object", + }, + "preserve_subdirectories_directories": { + "default": True, + "description": "Flag indicating we should preserve subdirectories directories", + "title": "Preserve Subdirectories Directories", + "type": "boolean", }, }, "required": ["streams"],