diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c6f0db7..681c238a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Changelog +## 2.1.0 + * Adds new streams `talk_phone_numbers` and `ticket_metric_events` [#111](https://github.com/singer-io/tap-zendesk/pull/111) ## 2.0.1 * Adds backoff/retry for `ProtocolError` and `ChunkedEncodingError` [#131](https://github.com/singer-io/tap-zendesk/pull/131) ## 2.0.0 diff --git a/setup.py b/setup.py index 9d2a9c75..18c3f2d8 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup setup(name='tap-zendesk', - version='2.0.1', + version='2.1.0', description='Singer.io tap for extracting data from the Zendesk API', author='Stitch', url='https://singer.io', diff --git a/tap_zendesk/schemas/talk_phone_numbers.json b/tap_zendesk/schemas/talk_phone_numbers.json new file mode 100644 index 00000000..5f2952ce --- /dev/null +++ b/tap_zendesk/schemas/talk_phone_numbers.json @@ -0,0 +1,207 @@ +{ + "type": [ + "null", + "object" + ], + "properties": { + "capabilities": { + "type": [ + "null", + "object" + ], + "properties": { + "sms": { + "type": [ + "null", + "boolean" + ] + }, + "mms": { + "type": [ + "null", + "boolean" + ] + }, + "voice": { + "type": [ + "null", + "boolean" + ] + } + } + }, + "categorised_greetings": { + "properties": { }, + "type": [ + "null", + "object" + ] + }, + "categorised_greetings_with_sub_settings": { + "properties": { }, + "type": [ + "null", + "object" + ] + }, + "country_code": { + "type": [ + "null", + "string" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "default_greeting_ids": { + "type": ["null", "array"], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "default_group_id": { + "type": [ + "null", + "integer" + ] + }, + "display_number": { + "type": [ + "null", + "string" + ] + }, + "external": { + "type": [ + "null", + "boolean" + ] + }, + "greeting_ids": { + "type": ["null", "array"], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "group_ids": { + "type": ["null", "array"], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "id": { + "type": [ + "integer" + ] + }, + "ivr_id": { + "type": [ + "null", + "integer" + ] + }, + "line_type": { + "type": [ + "null", + "string" + ] + }, + "location": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "nickname": { + "type": [ + "null", + "string" + ] + }, + "number": { + "type": [ + "null", + "string" + ] + }, + "outbound_enabled": { + "type": [ + "null", + "boolean" + ] + }, + "priority": { + "type": [ + "null", + "integer" + ] + }, + "recorded": { + "type": [ + "null", + "boolean" + ] + }, + "schedule_id": { + "type": [ + "null", + "integer" + ] + }, + "sms_enabled": { + "type": [ + "null", + "boolean" + ] + }, + "sms_group_id": { + "type": [ + "null", + "integer" + ] + }, + "token": { + "type": [ + "null", + "string" + ] + }, + "toll_free": { + "type": [ + "null", + "boolean" + ] + }, + "transcription": { + "type": [ + "null", + "boolean" + ] + }, + "voice_enabled": { + "type": [ + "null", + "boolean" + ] + } + } +} \ No newline at end of file diff --git a/tap_zendesk/schemas/ticket_metric_events.json b/tap_zendesk/schemas/ticket_metric_events.json new file mode 100644 index 00000000..cda82ac2 --- /dev/null +++ b/tap_zendesk/schemas/ticket_metric_events.json @@ -0,0 +1,116 @@ +{ + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "integer" + ] + }, + "instance_id": { + "type": [ + "null", + "integer" + ] + }, + "metric": { + "type": [ + "null", + "string" + ] + }, + "ticket_id": { + "type": [ + "null", + "integer" + ] + }, + "time": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "sla": { + "type": [ + "null", + "object" + ], + "properties": { + "target": { + "type": [ + "null", + "integer" + ] + }, + "business_hours": { + "type": [ + "null", + "boolean" + ] + }, + "policy": { + "type": [ + "null", + "object" + ], + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "status": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "integer" + ] + }, + "business": { + "type": [ + "null", + "integer" + ] + } + } + }, + "deleted": { + "type": [ + "null", + "boolean" + ] + } + } +} diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index 4dc89ab4..5c7ea7cd 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -406,6 +406,34 @@ def check_access(self): #Skip 404 ZendeskNotFoundError error as goal is just to check whether TicketComments have read permission or not pass +class TicketMetricEvents(Stream): + name = "ticket_metric_events" + replication_method = "INCREMENTAL" + replication_key = "time" + count = 0 + + def sync(self, state): + bookmark = self.get_bookmark(state) + start = bookmark - datetime.timedelta(seconds=1) + + epoch_start = int(start.strftime('%s')) + parsed_start = singer.strftime(start, "%Y-%m-%dT%H:%M:%SZ") + ticket_metric_events = self.client.tickets.metrics_incremental(start_time=epoch_start) + for event in ticket_metric_events: + self.count += 1 + if bookmark < utils.strptime_with_tz(event.time): + self.update_bookmark(state, event.time) + if parsed_start <= event.time: + yield (self.stream, event) + + def check_access(self): + try: + epoch_start = int(utils.now().strftime('%s')) + self.client.tickets.metrics_incremental(start_time=epoch_start) + except http.ZendeskNotFoundError: + #Skip 404 ZendeskNotFoundError error as goal is just to check whether TicketComments have read permission or not + pass + class TicketComments(Stream): name = "ticket_comments" replication_method = "INCREMENTAL" @@ -440,6 +468,21 @@ def check_access(self): #Skip 404 ZendeskNotFoundError error as goal is to just check to whether TicketComments have read permission or not pass +class TalkPhoneNumbers(Stream): + name = 'talk_phone_numbers' + replication_method = "FULL_TABLE" + + def sync(self, state): # pylint: disable=unused-argument + for phone_number in self.client.talk.phone_numbers(): + yield (self.stream, phone_number) + + def check_access(self): + try: + self.client.talk.phone_numbers() + except http.ZendeskNotFoundError: + #Skip 404 ZendeskNotFoundError error as goal is to just check to whether TicketComments have read permission or not + pass + class SatisfactionRatings(CursorBasedStream): name = "satisfaction_ratings" replication_method = "INCREMENTAL" @@ -608,5 +651,7 @@ def check_access(self): "satisfaction_ratings": SatisfactionRatings, "tags": Tags, "ticket_metrics": TicketMetrics, + "ticket_metric_events": TicketMetricEvents, "sla_policies": SLAPolicies, + "talk_phone_numbers": TalkPhoneNumbers, } diff --git a/test/base.py b/test/base.py index 807e2ad6..2a9f2b21 100644 --- a/test/base.py +++ b/test/base.py @@ -146,6 +146,12 @@ def expected_metadata(self): self.REPLICATION_METHOD: self.FULL_TABLE, self.OBEYS_START_DATE: False }, + "ticket_metric_events": { + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.INCREMENTAL, + self.REPLICATION_KEYS: {"time"}, + self.OBEYS_START_DATE: True + }, "tickets": { self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.INCREMENTAL, @@ -165,6 +171,11 @@ def expected_metadata(self): self.PRIMARY_KEYS: {"id"}, self.REPLICATION_METHOD: self.FULL_TABLE, self.OBEYS_START_DATE: False + }, + "talk_phone_numbers": { + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.FULL_TABLE, + self.OBEYS_START_DATE: False } } diff --git a/test/test_all_fields.py b/test/test_all_fields.py index cb82bb1f..dc1c7b28 100644 --- a/test/test_all_fields.py +++ b/test/test_all_fields.py @@ -18,7 +18,7 @@ def test_run(self): # Streams to verify all fields tests - expected_streams = self.expected_check_streams() + expected_streams = self.expected_check_streams() - {"talk_phone_numbers"} expected_automatic_fields = self.expected_automatic_fields() conn_id = connections.ensure_connection(self) @@ -78,6 +78,8 @@ def test_run(self): expected_all_keys = expected_all_keys - {'chat_only'} elif stream == "ticket_metrics": expected_all_keys = expected_all_keys - {'status', 'instance_id', 'metric', 'type', 'time'} + elif stream == "talk_phone_numbers": + expected_all_keys = expected_all_keys - {'token'} # verify all fields for each stream are replicated self.assertSetEqual(expected_all_keys, actual_all_keys) diff --git a/test/test_automatic_fields.py b/test/test_automatic_fields.py index f9d38e52..ac0670bb 100644 --- a/test/test_automatic_fields.py +++ b/test/test_automatic_fields.py @@ -21,7 +21,7 @@ def test_run(self): Verify that all replicated records have unique primary key values. """ - streams_to_test = self.expected_check_streams() + streams_to_test = self.expected_check_streams() - {"talk_phone_numbers"} conn_id = connections.ensure_connection(self) @@ -64,7 +64,7 @@ def test_run(self): self.assertSetEqual(expected_keys, actual_keys) # Verify that all replicated records have unique primary key values. - if stream == 'organizations': # BUG_TDL-19428 + if stream in {'organizations', 'ticket_metric_events'}: # BUG_TDL-19428 continue # skipping self.assertEqual(len(primary_keys_list), len(unique_primary_keys_list), diff --git a/test/test_pagination.py b/test/test_pagination.py index 97507c6b..743a05e6 100644 --- a/test/test_pagination.py +++ b/test/test_pagination.py @@ -27,6 +27,7 @@ def test_run(self): expected_streams = expected_streams - { "satisfaction_ratings", # skip as only end user of tickets can create data "tags", # Test Stability Issue: TDL-17980 + "talk_phone_numbers" } conn_id = connections.ensure_connection(self) diff --git a/test/test_standard_bookmark.py b/test/test_standard_bookmark.py index 149e983d..c4b3394f 100644 --- a/test/test_standard_bookmark.py +++ b/test/test_standard_bookmark.py @@ -186,7 +186,7 @@ def test_run(self): # Verify at least 1 record was replicated in the second sync # 'tags' stream (FULL_TABLE) data appears to have aged out 11/18/2022. Since we do not have CRUD # we will allow this stream to pass with a warning about decreased coverage - if stream == 'tags' and second_sync_count == 0 and first_sync_count == 0: + if stream in {'tags', 'talk_phone_numbers'} and second_sync_count == 0 and first_sync_count == 0: print(f"FULL_TABLE stream 'tags' replicated 0 records, stream not fully tested") continue self.assertGreater( diff --git a/test/unittests/test_discovery_mode.py b/test/unittests/test_discovery_mode.py index 952d228c..6e0b98fd 100644 --- a/test/unittests/test_discovery_mode.py +++ b/test/unittests/test_discovery_mode.py @@ -27,6 +27,8 @@ class TestDiscovery(unittest.TestCase): Test that we can call api for each stream in discovey mode and handle forbidden error. ''' @patch("tap_zendesk.discover.LOGGER.warning") + @patch('tap_zendesk.streams.TalkPhoneNumbers.check_access') + @patch('tap_zendesk.streams.TicketMetricEvents.check_access') @patch('tap_zendesk.streams.Organizations.check_access',side_effect=zenpy.lib.exception.APIException(ACCSESS_TOKEN_ERROR)) @patch('tap_zendesk.streams.Users.check_access',side_effect=zenpy.lib.exception.APIException(ACCSESS_TOKEN_ERROR)) @patch('tap_zendesk.streams.TicketForms.check_access',side_effect=zenpy.lib.exception.APIException(ACCSESS_TOKEN_ERROR)) @@ -50,7 +52,7 @@ class TestDiscovery(unittest.TestCase): ]) def test_discovery_handles_403__raise_tap_zendesk_forbidden_error(self, mock_get, mock_resolve_schema_references, mock_load_metadata, mock_load_schema,mock_load_shared_schema_refs, mocked_sla_policies, - mocked_ticket_forms, mock_users, mock_organizations, mock_logger): + mocked_ticket_forms, mock_users, mock_organizations, mocked_ticket_metric_events, mocked_talk_phone_numbers, mock_logger): ''' Test that we handle forbidden error for child streams. discover_streams calls check_access for each stream to check the read perission. discover_streams call many other methods including load_shared_schema_refs, load_metadata, @@ -70,6 +72,8 @@ def test_discovery_handles_403__raise_tap_zendesk_forbidden_error(self, mock_get "permission.") @patch("tap_zendesk.discover.LOGGER.warning") + @patch('tap_zendesk.streams.TalkPhoneNumbers.check_access') + @patch('tap_zendesk.streams.TicketMetricEvents.check_access') @patch('tap_zendesk.streams.Organizations.check_access',side_effect=zenpy.lib.exception.APIException(ACCSESS_TOKEN_ERROR)) @patch('tap_zendesk.streams.Users.check_access',side_effect=zenpy.lib.exception.APIException(ACCSESS_TOKEN_ERROR)) @patch('tap_zendesk.streams.TicketForms.check_access',side_effect=zenpy.lib.exception.APIException(ACCSESS_TOKEN_ERROR)) @@ -93,7 +97,7 @@ def test_discovery_handles_403__raise_tap_zendesk_forbidden_error(self, mock_get ]) def test_discovery_handles_403_raise_zenpy_forbidden_error_for_access_token(self, mock_get, mock_resolve_schema_references, mock_load_metadata, mock_load_schema,mock_load_shared_schema_refs, mocked_sla_policies, mocked_ticket_forms, - mock_users, mock_organizations, mock_logger): + mock_users, mock_organizations, mocked_ticket_metric_events, mocked_talk_phone_numbers, mock_logger): ''' Test that we handle forbidden error received from last failed request which we called from zenpy module and log proper warning message. discover_streams calls check_access for each stream to check the @@ -114,6 +118,8 @@ def test_discovery_handles_403_raise_zenpy_forbidden_error_for_access_token(self "lack of required permission.") @patch("tap_zendesk.discover.LOGGER.warning") + @patch('tap_zendesk.streams.TalkPhoneNumbers.check_access') + @patch('tap_zendesk.streams.TicketMetricEvents.check_access') @patch('tap_zendesk.streams.Organizations.check_access',side_effect=zenpy.lib.exception.APIException(API_TOKEN_ERROR)) @patch('tap_zendesk.streams.Users.check_access',side_effect=zenpy.lib.exception.APIException(API_TOKEN_ERROR)) @patch('tap_zendesk.streams.TicketForms.check_access',side_effect=zenpy.lib.exception.APIException(API_TOKEN_ERROR)) @@ -137,7 +143,7 @@ def test_discovery_handles_403_raise_zenpy_forbidden_error_for_access_token(self ]) def test_discovery_handles_403_raise_zenpy_forbidden_error_for_api_token(self, mock_get, mock_resolve_schema_references, mock_load_metadata, mock_load_schema,mock_load_shared_schema_refs, mocked_sla_policies, - mocked_ticket_forms, mock_users, mock_organizations, mock_logger): + mocked_ticket_forms, mock_users, mock_organizations, mocked_ticket_metric_events, mocked_talk_phone_numbers, mock_logger): ''' Test that we handle forbidden error received from last failed request which we called from zenpy module and log proper warning message. discover_streams calls check_access for each stream to check the @@ -156,6 +162,8 @@ def test_discovery_handles_403_raise_zenpy_forbidden_error_for_api_token(self, m "tickets, groups, users, organizations, ticket_fields, ticket_forms, group_memberships, macros, satisfaction_ratings, "\ "tags. The data for these streams would not be collected due to lack of required permission.") + @patch('tap_zendesk.streams.TalkPhoneNumbers.check_access') + @patch('tap_zendesk.streams.TicketMetricEvents.check_access') @patch('tap_zendesk.streams.Organizations.check_access',side_effect=zenpy.lib.exception.APIException(ACCSESS_TOKEN_ERROR)) @patch('tap_zendesk.streams.Users.check_access',side_effect=zenpy.lib.exception.APIException(ACCSESS_TOKEN_ERROR)) @patch('tap_zendesk.streams.TicketForms.check_access',side_effect=zenpy.lib.exception.APIException(ACCSESS_TOKEN_ERROR)) @@ -176,7 +184,7 @@ def test_discovery_handles_403_raise_zenpy_forbidden_error_for_api_token(self, m ]) def test_discovery_handles_except_403_error_requests_module(self, mock_get, mock_resolve_schema_references, mock_load_metadata, mock_load_schema,mock_load_shared_schema_refs, mocked_sla_policies, - mocked_ticket_forms, mock_users, mock_organizations): + mocked_ticket_forms, mock_users, mock_organizations, mocked_ticket_metric_events, mocked_talk_phone_numbers): ''' Test that function raises error directly if error code is other than 403. discover_streams calls check_access for each stream to check the read perission. discover_streams call many other methods including load_shared_schema_refs, load_metadata, @@ -195,6 +203,8 @@ def test_discovery_handles_except_403_error_requests_module(self, mock_get, mock self.assertEqual(expected_call_count, actual_call_count) + @patch('tap_zendesk.streams.TalkPhoneNumbers.check_access') + @patch('tap_zendesk.streams.TicketMetricEvents.check_access') @patch('tap_zendesk.streams.Organizations.check_access',side_effect=zenpy.lib.exception.APIException(AUTH_ERROR)) @patch('tap_zendesk.streams.Users.check_access',side_effect=zenpy.lib.exception.APIException(AUTH_ERROR)) @patch('tap_zendesk.streams.TicketForms.check_access',side_effect=zenpy.lib.exception.APIException(AUTH_ERROR)) @@ -215,7 +225,7 @@ def test_discovery_handles_except_403_error_requests_module(self, mock_get, mock ]) def test_discovery_handles_except_403_error_zenpy_module(self, mock_get, mock_resolve_schema_references, mock_load_metadata, mock_load_schema,mock_load_shared_schema_refs, mocked_sla_policies, - mocked_ticket_forms, mock_users, mock_organizations): + mocked_ticket_forms, mock_users, mock_organizations, mocked_ticket_metric_events, mocked_talk_phone_numbers): ''' Test that discovery mode raise error direclty if it is rather than 403 for request zenpy module. discover_streams call many other methods including load_shared_schema_refs, load_metadata, load_schema, resolve_schema_references @@ -234,6 +244,8 @@ def test_discovery_handles_except_403_error_zenpy_module(self, mock_get, mock_re self.assertEqual(expected_call_count, actual_call_count) + @patch('tap_zendesk.streams.TalkPhoneNumbers.check_access') + @patch('tap_zendesk.streams.TicketMetricEvents.check_access') @patch('tap_zendesk.streams.Organizations.check_access',side_effect=[mocked_get(status_code=200, json={"key1": "val1"})]) @patch('tap_zendesk.streams.Users.check_access',side_effect=[mocked_get(status_code=200, json={"key1": "val1"})]) @patch('tap_zendesk.streams.TicketForms.check_access',side_effect=[mocked_get(status_code=200, json={"key1": "val1"})]) @@ -257,7 +269,7 @@ def test_discovery_handles_except_403_error_zenpy_module(self, mock_get, mock_re ]) def test_discovery_handles_200_response(self, mock_get, mock_resolve_schema_references, mock_load_metadata, mock_load_schema,mock_load_shared_schema_refs, mocked_sla_policies, - mocked_ticket_forms, mock_users, mock_organizations): + mocked_ticket_forms, mock_users, mock_organizations, mocked_ticket_metric_events, mocked_talk_phone_numbers): ''' Test that discovery mode does not raise any error in case of all streams have read permission ''' @@ -268,6 +280,8 @@ def test_discovery_handles_200_response(self, mock_get, mock_resolve_schema_refe self.assertEqual(expected_call_count, actual_call_count) @patch("tap_zendesk.discover.LOGGER.warning") + @patch('tap_zendesk.streams.TalkPhoneNumbers.check_access') + @patch('tap_zendesk.streams.TicketMetricEvents.check_access') @patch('tap_zendesk.streams.Organizations.check_access',side_effect=zenpy.lib.exception.APIException(API_TOKEN_ERROR)) @patch('tap_zendesk.streams.Users.check_access',side_effect=zenpy.lib.exception.APIException(API_TOKEN_ERROR)) @patch('tap_zendesk.streams.TicketForms.check_access',side_effect=zenpy.lib.exception.APIException(API_TOKEN_ERROR)) @@ -291,7 +305,7 @@ def test_discovery_handles_200_response(self, mock_get, mock_resolve_schema_refe ]) def test_discovery_handles_403_for_all_streams_api_token(self, mock_get, mock_resolve_schema_references, mock_load_metadata, mock_load_schema,mock_load_shared_schema_refs, mocked_sla_policies, - mocked_ticket_forms, mock_users, mock_organizations, mock_logger): + mocked_ticket_forms, mock_users, mock_organizations, mocked_ticket_metric_events, mocked_talk_phone_numbers, mock_logger): ''' Test that we handle forbidden error received from all streams and raise the ZendeskForbiddenError with proper error message. discover_streams calls check_access for each stream to check the