From 05e4f74e88388a1316703045b425ee1c0e8637d3 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi Date: Tue, 3 Dec 2024 18:20:05 +0100 Subject: [PATCH] Add default value for schema pointer --- .../declarative/declarative_component_schema.yaml | 1 - .../declarative/models/declarative_component_schema.py | 4 ++-- .../declarative/parsers/model_to_component_factory.py | 6 +++--- .../declarative/schema/dynamic_schema_loader.py | 10 ++++++---- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index b2e1a66c1..4cea78465 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -1701,7 +1701,6 @@ definitions: description: (This component is experimental. Use at your own risk.) Identifies schema details for dynamic schema extraction and processing. type: object required: - - schema_pointer - key_pointer properties: type: diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 4e3f26ca9..7a9bd9cda 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -657,8 +657,8 @@ class TypesMap(BaseModel): class SchemaTypeIdentifier(BaseModel): type: Optional[Literal["SchemaTypeIdentifier"]] = None - schema_pointer: List[str] = Field( - ..., + schema_pointer: Optional[List[str]] = Field( + [], description="List of nested fields defining the schema field path to extract. Defaults to [].", title="Schema Path", ) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index f92c4d416..f8f62eeb5 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -1593,9 +1593,9 @@ def create_schema_type_identifier( for types_pair in model.types_mapping ] ) - model_schema_pointer: List[Union[InterpolatedString, str]] = [ - x for x in model.schema_pointer - ] + model_schema_pointer: List[Union[InterpolatedString, str]] = ( + [x for x in model.schema_pointer] if model.schema_pointer else [] + ) model_key_pointer: List[Union[InterpolatedString, str]] = [x for x in model.key_pointer] model_type_pointer: Optional[List[Union[InterpolatedString, str]]] = ( [x for x in model.type_pointer] if model.type_pointer else None diff --git a/airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py b/airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py index b91a2558e..4b6e9e864 100644 --- a/airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py +++ b/airbyte_cdk/sources/declarative/schema/dynamic_schema_loader.py @@ -61,14 +61,16 @@ class SchemaTypeIdentifier: Identifies schema details for dynamic schema extraction and processing. """ - schema_pointer: List[Union[InterpolatedString, str]] key_pointer: List[Union[InterpolatedString, str]] parameters: InitVar[Mapping[str, Any]] type_pointer: Optional[List[Union[InterpolatedString, str]]] = None types_map: Optional[List[TypesMap]] = None + schema_pointer: Optional[List[Union[InterpolatedString, str]]] = None def __post_init__(self, parameters: Mapping[str, Any]) -> None: - self.schema_pointer = self._update_pointer(self.schema_pointer, parameters) # type: ignore[assignment] # This is reqired field in model + self.schema_pointer = ( + self._update_pointer(self.schema_pointer, parameters) if self.schema_pointer else [] + ) # type: ignore[assignment] # This is reqired field in model self.key_pointer = self._update_pointer(self.key_pointer, parameters) # type: ignore[assignment] # This is reqired field in model self.type_pointer = ( self._update_pointer(self.type_pointer, parameters) if self.type_pointer else None @@ -199,14 +201,14 @@ def _get_airbyte_type(field_type: str) -> Mapping[str, Any]: def _extract_data( self, body: Mapping[str, Any], - extraction_path: List[Union[InterpolatedString, str]], + extraction_path: Optional[List[Union[InterpolatedString, str]]] = None, default: Any = None, ) -> Any: """ Extracts data from the body based on the provided extraction path. """ - if len(extraction_path) == 0: + if not extraction_path: return body path = [