From 47a6f21e46b290b9fd0ecbb63664536a8a86e3f0 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Fri, 24 May 2024 19:50:21 +0300 Subject: [PATCH] Fix minor errors --- .../incremental/per_partition_cursor.py | 4 +- .../source_zendesk_support/manifest.yaml | 75 ++++++------------- 2 files changed, 25 insertions(+), 54 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py index 9a5aa7fa67dd..55d7917fac35 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py @@ -6,7 +6,7 @@ from typing import Any, Callable, Iterable, Mapping, MutableMapping, Optional, Union from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor -from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter +from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.types import Record, StreamSlice, StreamState @@ -62,7 +62,7 @@ class PerPartitionCursor(DeclarativeCursor): _KEY = 0 _VALUE = 1 - def __init__(self, cursor_factory: CursorFactory, partition_router: PartitionRouter): + def __init__(self, cursor_factory: CursorFactory, partition_router: StreamSlicer): self._cursor_factory = cursor_factory self._partition_router = partition_router self._cursor_per_partition: MutableMapping[str, DeclarativeCursor] = {} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/manifest.yaml index 641204ba231e..19e0283bdba2 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/manifest.yaml +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/manifest.yaml @@ -1,4 +1,4 @@ -version: 0.77.2 +version: 0.51.42 type: DeclarativeSource @@ -593,18 +593,7 @@ definitions: request_parameters: sort_by: "updated_at" paginator: - type: DefaultPaginator - page_size_option: - type: RequestOption - field_name: "page[size]" - inject_into: request_parameter - pagination_strategy: - type: CursorPagination - page_size: 4 - cursor_value: "{{ response['links']['next'] }}" - stop_condition: "{{ config.get('ignore_pagination') or not response['meta']['has_more'] }}" - page_token_option: - type: RequestPath + $ref: "#/definitions/links_next_paginator" $parameters: name: "posts" path: "community/posts" @@ -614,33 +603,34 @@ definitions: # Incremental substreams article_comments_stream: - $ref: "#/definitions/base_incremental_stream" + $ref: "#/definitions/semi_incremental_stream" retriever: - $ref: "#/definitions/base_stream/retriever" + $ref: "#/definitions/semi_incremental_stream/retriever" partition_router: type: SubstreamPartitionRouter parent_stream_configs: - stream: "#/definitions/articles_stream" parent_key: id partition_field: id + incremental_dependency: true $parameters: name: "article_comments" path: "help_center/articles/{{ stream_slice.id }}/comments" data_path: "comments" cursor_field: "updated_at" - cursor_filter: "start_time" primary_key: "id" article_comment_votes_stream: - $ref: "#/definitions/base_incremental_stream" + $ref: "#/definitions/semi_incremental_stream" retriever: - $ref: "#/definitions/base_stream/retriever" + $ref: "#/definitions/semi_incremental_stream/retriever" partition_router: type: SubstreamPartitionRouter parent_stream_configs: - stream: "#/definitions/article_comments_stream" parent_key: id partition_field: id + incremental_dependency: true $parameters: name: "article_comment_votes" path: "help_center/articles/{{ stream_slice.parent_slice.id }}/comments/{{ stream_slice.id }}/votes" @@ -652,34 +642,35 @@ definitions: primary_key: "id" article_votes_stream: - $ref: "#/definitions/base_incremental_stream" + $ref: "#/definitions/semi_incremental_stream" retriever: - $ref: "#/definitions/base_stream/retriever" + $ref: "#/definitions/semi_incremental_stream/retriever" partition_router: type: SubstreamPartitionRouter parent_stream_configs: - stream: "#/definitions/articles_stream" parent_key: id partition_field: id + incremental_dependency: true $parameters: name: "article_votes" path: "help_center/articles/{{ stream_slice.id }}/votes" data_path: "votes" schema_name: "votes" cursor_field: "updated_at" - cursor_filter: "start_time" primary_key: "id" post_comments_stream: - $ref: "#/definitions/base_incremental_stream" + $ref: "#/definitions/semi_incremental_stream" retriever: - $ref: "#/definitions/base_stream/retriever" + $ref: "#/definitions/semi_incremental_stream/retriever" partition_router: type: SubstreamPartitionRouter parent_stream_configs: - stream: "#/definitions/posts_stream" parent_key: id partition_field: id + incremental_dependency: true $parameters: name: "post_comments" path: "community/posts/{{ stream_slice.id }}/comments" @@ -689,15 +680,16 @@ definitions: primary_key: "id" post_comment_votes_stream: - $ref: "#/definitions/base_incremental_stream" + $ref: "#/definitions/semi_incremental_stream" retriever: - $ref: "#/definitions/base_stream/retriever" + $ref: "#/definitions/semi_incremental_stream/retriever" partition_router: type: SubstreamPartitionRouter parent_stream_configs: - stream: "#/definitions/post_comments_stream" parent_key: id partition_field: id + incremental_dependency: true $parameters: name: "post_comment_votes" path: "community/posts/{{ stream_slice.parent_slice.id }}/comments/{{ stream_slice.id }}/votes" @@ -708,15 +700,16 @@ definitions: primary_key: "id" post_votes_stream: - $ref: "#/definitions/base_incremental_stream" + $ref: "#/definitions/semi_incremental_stream" retriever: - $ref: "#/definitions/base_stream/retriever" + $ref: "#/definitions/semi_incremental_stream/retriever" partition_router: type: SubstreamPartitionRouter parent_stream_configs: - stream: "#/definitions/posts_stream" parent_key: id partition_field: id + incremental_dependency: true $parameters: name: "post_votes" path: "community/posts/{{ stream_slice.id }}/votes" @@ -732,18 +725,7 @@ definitions: $ref: "#/definitions/retriever" ignore_stream_slicer_parameters_on_paginated_requests: true paginator: - type: DefaultPaginator - pagination_strategy: - type: CursorPagination - cursor_value: '{{ response.get("after_url") }}' - stop_condition: '{{ response.get("end_of_stream") }}' - page_size: 100 - page_size_option: - type: RequestOption - field_name: "per_page" - inject_into: request_parameter - page_token_option: - type: RequestPath + $ref: "#/definitions/after_url_paginator" $parameters: name: "tickets" path: "incremental/tickets/cursor.json" @@ -761,19 +743,9 @@ definitions: - stream: "#/definitions/tickets_stream" parent_key: id partition_field: id + incremental_dependency: true paginator: - type: DefaultPaginator - page_size_option: - type: RequestOption - field_name: "page[size]" - inject_into: request_parameter - pagination_strategy: - type: CursorPagination - page_size: 100 - cursor_value: "{{ response['links']['next'] }}" - stop_condition: "{{ not response['meta']['has_more'] }}" - page_token_option: - type: RequestPath + $ref: "#/definitions/links_next_paginator" $parameters: name: "ticket_metrics" path: "tickets/{{ stream_slice.id }}/metrics" @@ -807,7 +779,6 @@ streams: - $ref: "#/definitions/ticket_metric_events_stream" - $ref: "#/definitions/ticket_skips_stream" - $ref: "#/definitions/users_stream" - # Incremental substreams - $ref: "#/definitions/articles_stream" - $ref: "#/definitions/article_comments_stream"