diff --git a/tap_fleetio/client.py b/tap_fleetio/client.py index f4bd65ba..2e5dcbc5 100644 --- a/tap_fleetio/client.py +++ b/tap_fleetio/client.py @@ -7,12 +7,30 @@ import requests from singer_sdk.helpers.jsonpath import extract_jsonpath -from singer_sdk.pagination import BaseAPIPaginator # noqa: TCH002 +from singer_sdk.pagination import BasePageNumberPaginator # noqa: TCH002 from singer_sdk.streams import RESTStream _Auth = Callable[[requests.PreparedRequest], requests.PreparedRequest] SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") +class fleetioPagination(BasePageNumberPaginator): + def has_more(self, response) -> bool: + data = response.headers + if (data.get('X-Pagination-Current-Page') == data.get('X-Pagination-Total-Pages') ): + has_more = False + else: + has_more = True + return has_more + + def get_next(self, response): + data = response.headers + current_page = data.get('X-Pagination-Current-Page') + max_page = data.get('X-Pagination-Total-Pages') + next_page = None + if (current_page < max_page): + next_page = int(current_page) + 1 + + return next_page class fleetioStream(RESTStream): """fleetio stream class.""" @@ -24,9 +42,6 @@ def url_base(self) -> str: records_jsonpath = "$[*]" # Or override `parse_response`. - # Set this value or override `get_new_paginator`. - next_page_token_jsonpath = "$.next_page" # noqa: S105 - @property def http_headers(self) -> dict: """Return the http headers needed. @@ -39,9 +54,10 @@ def http_headers(self) -> dict: headers["User-Agent"] = self.config.get("user_agent") headers["Authorization"] = f"Token {self.config.get('api_token')}" headers["Account-Token"] = self.config.get('account_token') + headers["request_source"] = "fleetio_singer_tap" return headers - def get_new_paginator(self) -> BaseAPIPaginator: + def get_new_paginator(self): """Create a new pagination helper instance. If the source API can make use of the `next_page_token_jsonpath` @@ -54,7 +70,8 @@ def get_new_paginator(self) -> BaseAPIPaginator: Returns: A pagination helper instance. """ - return super().get_new_paginator() + + return fleetioPagination(1) def get_url_params( self, diff --git a/tap_fleetio/schemas/service_entries.json b/tap_fleetio/schemas/service_entries.json index f5984037..5b59d6ee 100644 --- a/tap_fleetio/schemas/service_entries.json +++ b/tap_fleetio/schemas/service_entries.json @@ -215,7 +215,7 @@ "items": { "properties": { "account_membership_id": { - "type": "number" + "type": ["null", "number"] }, "attachment_permissions": { "type": "object" @@ -499,7 +499,7 @@ "type": "string" }, "username": { - "type": "string" + "type": ["null", "string"] }, "vehicles_record_set": { "properties": { @@ -513,7 +513,7 @@ "type": "object" } }, - "type": "object" + "type": ["null", "object"] }, "vehicle_operator": { "type": "boolean" @@ -1319,16 +1319,16 @@ "type": "number" }, "vehicle_image_url": { - "type": "string" + "type": ["null", "string"] }, "vehicle_image_url_large": { - "type": "string" + "type": ["null", "string"] }, "vehicle_image_url_medium": { - "type": "string" + "type": ["null", "string"] }, "vehicle_image_url_small": { - "type": "string" + "type": ["null", "string"] }, "vehicle_name": { "type": "string" @@ -1474,7 +1474,7 @@ "items": { "properties": { "account_membership_id": { - "type": "number" + "type": ["null", "number"] }, "attachment_permissions": { "type": "object" @@ -1623,7 +1623,7 @@ "type": "number" }, "name": { - "type": "string" + "type": ["null", "string"] } }, "type": "object" @@ -1635,7 +1635,7 @@ "type": "string" }, "username": { - "type": "string" + "type": ["null", "string"] }, "vehicles_record_set": { "properties": { @@ -1649,7 +1649,7 @@ "type": "object" } }, - "type": "object" + "type": ["null", "object"] }, "vehicle_operator": { "type": "boolean" @@ -2219,7 +2219,7 @@ "type": "string" }, "username": { - "type": "string" + "type": ["null", "string"] }, "vehicles_record_set": { "properties": { diff --git a/tap_fleetio/schemas/submitted_inspection_forms.json b/tap_fleetio/schemas/submitted_inspection_forms.json index 45fa49ed..34ee7add 100644 --- a/tap_fleetio/schemas/submitted_inspection_forms.json +++ b/tap_fleetio/schemas/submitted_inspection_forms.json @@ -204,13 +204,13 @@ "result": { "properties": { "file_url": { - "type": "string" + "type": ["null", "string"] }, "full_file_url": { - "type": "string" + "type": ["null", "string"] }, "full_signature_file_url": { - "type": "string" + "type": ["null", "string"] }, "label": { "type": [