Merge pull request #2 from fleetio/fleetio-tap-dev

Fleetio tap dev
fleetio · Jul 19, 2023 · 714b7e5 · 714b7e5
2 parents 26fe5ff + eeb5321
commit 714b7e5
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 21 deletions.
diff --git a/tap_fleetio/client.py b/tap_fleetio/client.py
@@ -7,12 +7,30 @@
 
 import requests
 from singer_sdk.helpers.jsonpath import extract_jsonpath
-from singer_sdk.pagination import BaseAPIPaginator  # noqa: TCH002
+from singer_sdk.pagination import BasePageNumberPaginator  # noqa: TCH002
 from singer_sdk.streams import RESTStream
 
 _Auth = Callable[[requests.PreparedRequest], requests.PreparedRequest]
 SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
 
+class fleetioPagination(BasePageNumberPaginator):
+    def has_more(self, response) -> bool:
+        data = response.headers
+        if (data.get('X-Pagination-Current-Page') == data.get('X-Pagination-Total-Pages') ):
+            has_more = False
+        else:
+            has_more = True
+        return has_more
+
+    def get_next(self, response):
+        data = response.headers
+        current_page = data.get('X-Pagination-Current-Page')
+        max_page = data.get('X-Pagination-Total-Pages')
+        next_page = None
+        if (current_page < max_page):
+            next_page = int(current_page) + 1
+
+        return next_page
 
 class fleetioStream(RESTStream):
     """fleetio stream class."""
@@ -24,9 +42,6 @@ def url_base(self) -> str:
 
     records_jsonpath = "$[*]"  # Or override `parse_response`.
 
-    # Set this value or override `get_new_paginator`.
-    next_page_token_jsonpath = "$.next_page"  # noqa: S105
-
     @property
     def http_headers(self) -> dict:
         """Return the http headers needed.
@@ -39,9 +54,10 @@ def http_headers(self) -> dict:
             headers["User-Agent"] = self.config.get("user_agent")
         headers["Authorization"] = f"Token {self.config.get('api_token')}"
         headers["Account-Token"] = self.config.get('account_token')
+        headers["request_source"] = "fleetio_singer_tap"
         return headers
 
-    def get_new_paginator(self) -> BaseAPIPaginator:
+    def get_new_paginator(self):
         """Create a new pagination helper instance.
 
         If the source API can make use of the `next_page_token_jsonpath`
@@ -54,7 +70,8 @@ def get_new_paginator(self) -> BaseAPIPaginator:
         Returns:
             A pagination helper instance.
         """
-        return super().get_new_paginator()
+
+        return fleetioPagination(1)
 
     def get_url_params(
         self,

diff --git a/tap_fleetio/schemas/service_entries.json b/tap_fleetio/schemas/service_entries.json
@@ -215,7 +215,7 @@
                 "items": {
                   "properties": {
                     "account_membership_id": {
-                      "type": "number"
+                      "type": ["null", "number"]
                     },
                     "attachment_permissions": {
                       "type": "object"
@@ -499,7 +499,7 @@
                           "type": "string"
                         },
                         "username": {
-                          "type": "string"
+                          "type": ["null", "string"]
                         },
                         "vehicles_record_set": {
                           "properties": {
@@ -513,7 +513,7 @@
                           "type": "object"
                         }
                       },
-                      "type": "object"
+                      "type": ["null", "object"]
                     },
                     "vehicle_operator": {
                       "type": "boolean"
@@ -1319,16 +1319,16 @@
                 "type": "number"
               },
               "vehicle_image_url": {
-                "type": "string"
+                "type": ["null", "string"]
               },
               "vehicle_image_url_large": {
-                "type": "string"
+                "type": ["null", "string"]
               },
               "vehicle_image_url_medium": {
-                "type": "string"
+                "type": ["null", "string"]
               },
               "vehicle_image_url_small": {
-                "type": "string"
+                "type": ["null", "string"]
               },
               "vehicle_name": {
                 "type": "string"
@@ -1474,7 +1474,7 @@
                       "items": {
                         "properties": {
                           "account_membership_id": {
-                            "type": "number"
+                            "type": ["null", "number"]
                           },
                           "attachment_permissions": {
                             "type": "object"
@@ -1623,7 +1623,7 @@
                                     "type": "number"
                                   },
                                   "name": {
-                                    "type": "string"
+                                    "type": ["null", "string"]
                                   }
                                 },
                                 "type": "object"
@@ -1635,7 +1635,7 @@
                                 "type": "string"
                               },
                               "username": {
-                                "type": "string"
+                                "type": ["null", "string"]
                               },
                               "vehicles_record_set": {
                                 "properties": {
@@ -1649,7 +1649,7 @@
                                 "type": "object"
                               }
                             },
-                            "type": "object"
+                            "type": ["null", "object"]
                           },
                           "vehicle_operator": {
                             "type": "boolean"
@@ -2219,7 +2219,7 @@
                               "type": "string"
                             },
                             "username": {
-                              "type": "string"
+                              "type": ["null", "string"]
                             },
                             "vehicles_record_set": {
                               "properties": {

diff --git a/tap_fleetio/schemas/submitted_inspection_forms.json b/tap_fleetio/schemas/submitted_inspection_forms.json
@@ -204,13 +204,13 @@
               "result": {
                 "properties": {
                   "file_url": {
-                    "type": "string"
+                    "type": ["null", "string"]
                   },
                   "full_file_url": {
-                    "type": "string"
+                    "type": ["null", "string"]
                   },
                   "full_signature_file_url": {
-                    "type": "string"
+                    "type": ["null", "string"]
                   },
                   "label": {
                     "type": [