Skip to content

Commit

Permalink
replace convert_config_to_declarative_stream_args with default values…
Browse files Browse the repository at this point in the history
… from the interpolation
  • Loading branch information
brianjlai committed Nov 18, 2024
1 parent 7d2baea commit 485380a
Show file tree
Hide file tree
Showing 4 changed files with 8 additions and 217 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"todo-stream-name": {
"todo-field-name": "todo-abnormal-value"
}
}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ definitions:
datetime_format: "%Y-%m-%dT%H:%M:%SZ"
cursor_field: "{{ parameters.get('cursor_field', 'updated_at') }}"
start_datetime:
datetime: "{{ config.get('start_date') }}"
datetime: "{{ config.get('start_date') or day_delta(-730, '%Y-%m-%dT%H:%M:%SZ') }}"

cursor_incremental_sync:
type: DatetimeBasedCursor
Expand All @@ -98,7 +98,7 @@ definitions:
datetime_format: "%s"
cursor_field: "{{ parameters.get('cursor_field', 'updated_at') }}"
start_datetime:
datetime: "{{ timestamp(config.get('start_date')) | int or day_delta(-730, '%Y-%m-%dT%H:%M:%SZ') }}"
datetime: "{{ timestamp(config.get('start_date')) | int if config.get('start_date') else day_delta(-730, '%s') }}"
start_time_option:
inject_into: request_parameter
field_name: "{{ parameters['cursor_filter'] }}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,7 @@ def __init__(self, email: str, password: str):

class SourceZendeskSupport(YamlDeclarativeSource):
def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], state: TState, **kwargs):
# Before 2024-11-11, the config was being modified in `streams`. We can't do that anymore because `ConcurrentDeclarativeSource` use
# the config to make concurrent components. Hence, the "main" config needs to be the declarative one and if Python sources need
# something a bit different, it needs to deduce it from the declarative config
declarative_config = SourceZendeskSupport.convert_config_to_declarative_stream_args(config) if config else None
super().__init__(catalog=catalog, config=declarative_config, state=state, **{"path_to_yaml": "manifest.yaml"})
super().__init__(catalog=catalog, config=config, state=state, **{"path_to_yaml": "manifest.yaml"})

@classmethod
def get_default_start_date(cls) -> str:
Expand Down Expand Up @@ -112,18 +108,6 @@ def convert_config2stream_args(cls, config: Mapping[str, Any]) -> Mapping[str, A
"ignore_pagination": config.get("ignore_pagination", False),
}

@classmethod
def convert_config_to_declarative_stream_args(cls, config: Mapping[str, Any]) -> Mapping[str, Any]:
"""
Sets defaults to `start_date` and `ignore_pagination`.
"""
return {
"subdomain": config["subdomain"],
"start_date": config.get("start_date", cls.get_default_start_date()),
"credentials": config.get("credentials"),
"ignore_pagination": config.get("ignore_pagination", False),
}

def get_nested_streams(self, config: Mapping[str, Any]) -> List[Stream]:
"""Returns relevant a list of available streams
:param config: A Mapping of the user input configuration as defined in the connector spec.
Expand Down

0 comments on commit 485380a

Please sign in to comment.