From 98298f11df4f675dac528f0aaf26c1dd1fd77f67 Mon Sep 17 00:00:00 2001 From: Aishwarya Gupta Date: Thu, 15 Jul 2021 13:20:00 +0100 Subject: [PATCH 01/10] sideloading enable test --- setup.py | 6 ++++-- tap_zendesk/streams.py | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index ec32964..c855738 100644 --- a/setup.py +++ b/setup.py @@ -2,8 +2,10 @@ from setuptools import setup -setup(name='twilio-tap-zendesk', - version='1.0.1', +setup( + # name='twilio-tap-zendesk', + name='twiliointernal_tap-zendesk-des-1610', + version='0.0.1', description='Singer.io tap for extracting data from the Zendesk API', author='Twilio', url='https://github.com/twilio-labs/twilio-tap-zendesk', diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index 2c8fdf6..791d36e 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -28,6 +28,9 @@ DEFAULT_SEARCH_WINDOW_SIZE = (60 * 60 * 24) * 30 # defined in seconds, default to a month (30 days) +def get_sideload_objects(stream): + return metadata.to_map(stream.metadata).get((), {}).get('sideload-objects') + def get_abs_path(path): return os.path.join(os.path.dirname(os.path.realpath(__file__)), path) @@ -250,7 +253,8 @@ def _empty_buffer(self): def sync(self, state): bookmark = self.get_bookmark(state) - tickets = self.client.tickets.incremental(start_time=bookmark) + sideload_objects = get_sideload_objects(self.stream) + tickets = self.client.tickets.incremental(start_time=bookmark, include=sideload_objects) audits_stream = TicketAudits(self.client) metrics_stream = TicketMetrics(self.client) From aa489281e071aed811657289cd1e96deb20f5ab3 Mon Sep 17 00:00:00 2001 From: Aishwarya Gupta Date: Thu, 15 Jul 2021 13:55:21 +0100 Subject: [PATCH 02/10] DES-1610 --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c855738..c8f5f50 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,7 @@ from setuptools import setup setup( - # name='twilio-tap-zendesk', - name='twiliointernal_tap-zendesk-des-1610', + name='twilio-tap-zendesk-guptaa3', version='0.0.1', description='Singer.io tap for extracting data from the Zendesk API', author='Twilio', From 929907adffe9d0b43b27c9f9f34ee25d826c98bc Mon Sep 17 00:00:00 2001 From: Aishwarya Gupta Date: Thu, 15 Jul 2021 19:41:58 +0100 Subject: [PATCH 03/10] DES-1610 --- tap_zendesk/streams.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index 791d36e..48fce98 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -29,6 +29,7 @@ DEFAULT_SEARCH_WINDOW_SIZE = (60 * 60 * 24) * 30 # defined in seconds, default to a month (30 days) def get_sideload_objects(stream): + LOGGER.info("inside sideload values === " + str(metadata.to_map(stream.metadata).get((), {}).get('sideload-objects'))) return metadata.to_map(stream.metadata).get((), {}).get('sideload-objects') def get_abs_path(path): @@ -255,7 +256,7 @@ def sync(self, state): bookmark = self.get_bookmark(state) sideload_objects = get_sideload_objects(self.stream) tickets = self.client.tickets.incremental(start_time=bookmark, include=sideload_objects) - + LOGGER.info("one record -----> " + str(tickets[0].to_dict())) audits_stream = TicketAudits(self.client) metrics_stream = TicketMetrics(self.client) comments_stream = TicketComments(self.client) From 65f74393ab99c9bd8c187df50d2af814db810bac Mon Sep 17 00:00:00 2001 From: Aishwarya Gupta Date: Thu, 15 Jul 2021 19:48:58 +0100 Subject: [PATCH 04/10] DES-1610 --- tap_zendesk/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index 48fce98..ad70d51 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -256,7 +256,6 @@ def sync(self, state): bookmark = self.get_bookmark(state) sideload_objects = get_sideload_objects(self.stream) tickets = self.client.tickets.incremental(start_time=bookmark, include=sideload_objects) - LOGGER.info("one record -----> " + str(tickets[0].to_dict())) audits_stream = TicketAudits(self.client) metrics_stream = TicketMetrics(self.client) comments_stream = TicketComments(self.client) @@ -278,6 +277,7 @@ def emit_sub_stream_metrics(sub_stream): self.update_bookmark(state, utils.strftime(generated_timestamp_dt)) ticket_dict = ticket.to_dict() + LOGGER.info("keys ====" + str(ticket_dict.keys)) ticket_dict.pop('fields') # NB: Fields is a duplicate of custom_fields, remove before emitting should_yield = self._buffer_record((self.stream, ticket_dict)) From 08672730f5490a89803fb9188f5504c012313607 Mon Sep 17 00:00:00 2001 From: Aishwarya Gupta Date: Thu, 15 Jul 2021 19:51:16 +0100 Subject: [PATCH 05/10] DES-1610 --- tap_zendesk/streams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index ad70d51..258a388 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -277,7 +277,7 @@ def emit_sub_stream_metrics(sub_stream): self.update_bookmark(state, utils.strftime(generated_timestamp_dt)) ticket_dict = ticket.to_dict() - LOGGER.info("keys ====" + str(ticket_dict.keys)) + LOGGER.info("keys ====" + str(ticket_dict.keys())) ticket_dict.pop('fields') # NB: Fields is a duplicate of custom_fields, remove before emitting should_yield = self._buffer_record((self.stream, ticket_dict)) From c91543646aa6de2638f3558e5188f1f2a9bd54ec Mon Sep 17 00:00:00 2001 From: Aishwarya Gupta Date: Mon, 19 Jul 2021 17:01:45 +0100 Subject: [PATCH 06/10] DES-1610 --- Makefile | 6 ++++++ tap_zendesk/streams.py | 2 -- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 97bc398..33407e2 100644 --- a/Makefile +++ b/Makefile @@ -4,3 +4,9 @@ test: pylint tap_zendesk -d missing-docstring,invalid-name,line-too-long,too-many-locals,too-few-public-methods,fixme,stop-iteration-return,too-many-branches,useless-import-alias,no-else-return,logging-not-lazy nosetests test/unittests + + +setup-environment: + rm -rf env || true + python3 -m venv env/tap-zendesk + source env/tap-zendesk/bin/activate && pip3 install . \ No newline at end of file diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index 258a388..c4aaa1a 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -29,7 +29,6 @@ DEFAULT_SEARCH_WINDOW_SIZE = (60 * 60 * 24) * 30 # defined in seconds, default to a month (30 days) def get_sideload_objects(stream): - LOGGER.info("inside sideload values === " + str(metadata.to_map(stream.metadata).get((), {}).get('sideload-objects'))) return metadata.to_map(stream.metadata).get((), {}).get('sideload-objects') def get_abs_path(path): @@ -277,7 +276,6 @@ def emit_sub_stream_metrics(sub_stream): self.update_bookmark(state, utils.strftime(generated_timestamp_dt)) ticket_dict = ticket.to_dict() - LOGGER.info("keys ====" + str(ticket_dict.keys())) ticket_dict.pop('fields') # NB: Fields is a duplicate of custom_fields, remove before emitting should_yield = self._buffer_record((self.stream, ticket_dict)) From d6f79f9a4c2321dffaf6e5b7739b7ab39497ac74 Mon Sep 17 00:00:00 2001 From: Aishwarya Gupta Date: Mon, 19 Jul 2021 17:03:25 +0100 Subject: [PATCH 07/10] DES-1610 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c8f5f50..9e585f2 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,8 @@ from setuptools import setup setup( - name='twilio-tap-zendesk-guptaa3', - version='0.0.1', + name='twilio-tap-zendesk', + version='1.0.2', description='Singer.io tap for extracting data from the Zendesk API', author='Twilio', url='https://github.com/twilio-labs/twilio-tap-zendesk', From c56b13790d90b89846b28e6170669c588d297e6f Mon Sep 17 00:00:00 2001 From: Aishwarya Gupta Date: Thu, 22 Jul 2021 11:40:18 +0100 Subject: [PATCH 08/10] sideloading ticket objects --- MANIFEST.in | 1 + tap_zendesk/__init__.py | 60 ++++- .../sideload_schemas/comment_count.json | 15 ++ .../schemas/sideload_schemas/dates.json | 65 +++++ .../schemas/sideload_schemas/metric_sets.json | 240 ++++++++++++++++++ .../schemas/sideload_schemas/slas.json | 51 ++++ 6 files changed, 418 insertions(+), 14 deletions(-) create mode 100644 tap_zendesk/schemas/sideload_schemas/comment_count.json create mode 100644 tap_zendesk/schemas/sideload_schemas/dates.json create mode 100644 tap_zendesk/schemas/sideload_schemas/metric_sets.json create mode 100644 tap_zendesk/schemas/sideload_schemas/slas.json diff --git a/MANIFEST.in b/MANIFEST.in index d4ec702..fa777ff 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include LICENSE include tap_zendesk/schemas/*.json include tap_zendesk/schemas/shared/*.json +include tap_zendesk/schemas/sideload_schemas/*.json diff --git a/tap_zendesk/__init__.py b/tap_zendesk/__init__.py index 2fb3359..39a901e 100755 --- a/tap_zendesk/__init__.py +++ b/tap_zendesk/__init__.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 import json -import sys +import sys, os from zenpy import Zenpy import requests from requests import Session from requests.adapters import HTTPAdapter import singer -from singer import metadata, metrics as singer_metrics +from singer import metadata, Schema, metrics as singer_metrics from tap_zendesk import metrics as zendesk_metrics from tap_zendesk.discover import discover_streams from tap_zendesk.streams import STREAMS @@ -34,11 +34,15 @@ # patch Session.request to record HTTP request metrics request = Session.request + def request_metrics_patch(self, method, url, **kwargs): with singer_metrics.http_request_timer(None): return request(self, method, url, **kwargs) + Session.request = request_metrics_patch + + # end patch def do_discover(client): @@ -47,9 +51,11 @@ def do_discover(client): json.dump(catalog, sys.stdout, indent=2) LOGGER.info("Finished discover") + def stream_is_selected(mdata): return mdata.get((), {}).get('selected', False) + def get_selected_streams(catalog): selected_stream_names = [] for stream in catalog.streams: @@ -63,15 +69,38 @@ def get_selected_streams(catalog): 'tickets': ['ticket_audits', 'ticket_metrics', 'ticket_comments'] } +# only side loading objects that are returned as a child object and not a separate array +SIDELOAD_OBJECTS = { + 'tickets': ['metric_sets', 'dates', 'comment_count', 'slas'] +} + + def get_sub_stream_names(): sub_stream_names = [] for parent_stream in SUB_STREAMS: sub_stream_names.extend(SUB_STREAMS[parent_stream]) return sub_stream_names + +def get_abs_path(path): + return os.path.join(os.path.dirname(os.path.realpath(__file__)), path) + + +def get_side_load_schemas(sideload_objects, stream): + stream_schema = stream.schema.to_dict() + for sideload_object in sideload_objects: + if sideload_object in SIDELOAD_OBJECTS[stream.tap_stream_id]: + schema_file = "schemas/sideload_schemas/{}.json".format(sideload_object) + with open(get_abs_path(schema_file)) as f: + schema = json.load(f) + stream_schema['properties'][list(schema['properties'].keys())[0]] = list(schema['properties'].values())[0] + return stream_schema + + class DependencyException(Exception): pass + def validate_dependencies(selected_stream_ids): errs = [] msg_tmpl = ("Unable to extract {0} data. " @@ -85,13 +114,14 @@ def validate_dependencies(selected_stream_ids): if errs: raise DependencyException(" ".join(errs)) + def populate_class_schemas(catalog, selected_stream_names): for stream in catalog.streams: if stream.tap_stream_id in selected_stream_names: STREAMS[stream.tap_stream_id].stream = stream -def do_sync(client, catalog, state, config): +def do_sync(client, catalog, state, config): selected_stream_names = get_selected_streams(catalog) validate_dependencies(selected_stream_names) populate_class_schemas(catalog, selected_stream_names) @@ -104,18 +134,12 @@ def do_sync(client, catalog, state, config): LOGGER.info("%s: Skipping - not selected", stream_name) continue - # if starting_stream: - # if starting_stream == stream_name: - # LOGGER.info("%s: Resuming", stream_name) - # starting_stream = None - # else: - # LOGGER.info("%s: Skipping - already synced", stream_name) - # continue - # else: - # LOGGER.info("%s: Starting", stream_name) - - key_properties = metadata.get(mdata, (), 'table-key-properties') + sideload_objects = metadata.get(mdata, (), 'sideload-objects') + if sideload_objects: + stream_schema = get_side_load_schemas(sideload_objects, stream) + stream.schema = Schema.from_dict(stream_schema) + singer.write_schema(stream_name, stream.schema.to_dict(), key_properties) sub_stream_names = SUB_STREAMS.get(stream_name) @@ -126,6 +150,10 @@ def do_sync(client, catalog, state, config): sub_stream = STREAMS[sub_stream_name].stream sub_mdata = metadata.to_map(sub_stream.metadata) sub_key_properties = metadata.get(sub_mdata, (), 'table-key-properties') + sideload_objects = metadata.get(mdata, (), 'sideload-objects') + if sideload_objects: + sub_stream_schema = get_side_load_schemas(sideload_objects, sub_stream) + sub_stream.schema = Schema.from_dict(sub_stream_schema) singer.write_schema(sub_stream.tap_stream_id, sub_stream.schema.to_dict(), sub_key_properties) # parent stream will sync sub stream @@ -143,6 +171,7 @@ def do_sync(client, catalog, state, config): LOGGER.info("Finished sync") zendesk_metrics.log_aggregate_rates() + def oauth_auth(args): if not set(OAUTH_CONFIG_KEYS).issubset(args.config.keys()): LOGGER.debug("OAuth authentication unavailable.") @@ -154,6 +183,7 @@ def oauth_auth(args): "oauth_token": args.config['access_token'], } + def api_token_auth(args): if not set(API_TOKEN_CONFIG_KEYS).issubset(args.config.keys()): LOGGER.debug("API Token authentication unavailable.") @@ -166,6 +196,7 @@ def api_token_auth(args): "token": args.config['api_token'] } + def get_session(config): """ Add partner information to requests Session object if specified in the config. """ if not all(k in config for k in ["marketplace_name", @@ -181,6 +212,7 @@ def get_session(config): session.headers["X-Zendesk-Marketplace-App-Id"] = str(config.get("marketplace_app_id", "")) return session + @singer.utils.handle_top_exception(LOGGER) def main(): parsed_args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS) diff --git a/tap_zendesk/schemas/sideload_schemas/comment_count.json b/tap_zendesk/schemas/sideload_schemas/comment_count.json new file mode 100644 index 0000000..d521042 --- /dev/null +++ b/tap_zendesk/schemas/sideload_schemas/comment_count.json @@ -0,0 +1,15 @@ +{ + "type": [ + "null", + "object" + ], + "properties": { + "comment_count": { + "type": [ + "null", + "integer" + ] + } + }, + "additionalProperties": true +} \ No newline at end of file diff --git a/tap_zendesk/schemas/sideload_schemas/dates.json b/tap_zendesk/schemas/sideload_schemas/dates.json new file mode 100644 index 0000000..2994c6d --- /dev/null +++ b/tap_zendesk/schemas/sideload_schemas/dates.json @@ -0,0 +1,65 @@ +{ + "type": [ + "null", + "object" + ], + "properties": { + "dates": { + "type": [ + "null", + "object" + ], + "properties": { + "solved_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "status_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "assignee_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "assigned_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "requester_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "latest_comment_added_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "initially_assigned_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } + } + } +} diff --git a/tap_zendesk/schemas/sideload_schemas/metric_sets.json b/tap_zendesk/schemas/sideload_schemas/metric_sets.json new file mode 100644 index 0000000..e499b59 --- /dev/null +++ b/tap_zendesk/schemas/sideload_schemas/metric_sets.json @@ -0,0 +1,240 @@ +{ + "type": [ + "null", + "object" + ], + "properties": { + "metric_set": { + "type": [ + "null", + "object" + ], + "properties": { + "url": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "ticket_id": { + "type": [ + "null", + "integer" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ] + }, + "updated_at": { + "type": [ + "null", + "string" + ] + }, + "group_stations": { + "type": [ + "null", + "integer" + ] + }, + "assignee_stations": { + "type": [ + "null", + "integer" + ] + }, + "reopens": { + "type": [ + "null", + "integer" + ] + }, + "replies": { + "type": [ + "null", + "integer" + ] + }, + "assignee_updated_at": { + "type": [ + "null", + "string" + ] + }, + "requester_updated_at": { + "type": [ + "null", + "string" + ] + }, + "status_updated_at": { + "type": [ + "null", + "string" + ] + }, + "initially_assigned_at": { + "type": [ + "null", + "string" + ] + }, + "assigned_at": { + "type": [ + "null", + "string" + ] + }, + "solved_at": { + "type": [ + "null", + "string" + ] + }, + "latest_comment_added_at": { + "type": [ + "null", + "string" + ] + }, + "reply_time_in_minutes": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "string" + ] + }, + "business": { + "type": [ + "null", + "string" + ] + } + }, + "additionalProperties": true + }, + "first_resolution_time_in_minutes": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "string" + ] + }, + "business": { + "type": [ + "null", + "string" + ] + } + }, + "additionalProperties": true + }, + "full_resolution_time_in_minutes": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "string" + ] + }, + "business": { + "type": [ + "null", + "string" + ] + } + }, + "additionalProperties": true + }, + "agent_wait_time_in_minutes": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "string" + ] + }, + "business": { + "type": [ + "null", + "string" + ] + } + }, + "additionalProperties": true + }, + "requester_wait_time_in_minutes": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "string" + ] + }, + "business": { + "type": [ + "null", + "string" + ] + } + }, + "additionalProperties": true + }, + "on_hold_time_in_minutes": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "integer" + ] + }, + "business": { + "type": [ + "null", + "integer" + ] + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} \ No newline at end of file diff --git a/tap_zendesk/schemas/sideload_schemas/slas.json b/tap_zendesk/schemas/sideload_schemas/slas.json new file mode 100644 index 0000000..a8d7c11 --- /dev/null +++ b/tap_zendesk/schemas/sideload_schemas/slas.json @@ -0,0 +1,51 @@ +{ + "type": [ + "null", + "object" + ], + "properties": { + "slas": { + "type": [ + "null", + "object" + ], + "properties": { + "policy_metrics": { + "type": "array", + "items":{ + "type": [ + "null", + "object" + ], + "properties": { + "breach_at": { + "type": [ + "null", + "string" + ] + }, + "stage": { + "type": [ + "null", + "string" + ] + }, + "metric": { + "type": [ + "null", + "string" + ] + }, + "hours": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + } +} \ No newline at end of file From a9c4b05b4e40313aeca89bcfc3f2748791d3c73e Mon Sep 17 00:00:00 2001 From: Aishwarya Gupta Date: Thu, 22 Jul 2021 13:08:43 +0100 Subject: [PATCH 09/10] removing extra lines --- tap_zendesk/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tap_zendesk/__init__.py b/tap_zendesk/__init__.py index 39a901e..b72ff88 100755 --- a/tap_zendesk/__init__.py +++ b/tap_zendesk/__init__.py @@ -39,10 +39,7 @@ def request_metrics_patch(self, method, url, **kwargs): with singer_metrics.http_request_timer(None): return request(self, method, url, **kwargs) - Session.request = request_metrics_patch - - # end patch def do_discover(client): From 50b2d14363c0f8e789e69bcec6f4aa9db276e783 Mon Sep 17 00:00:00 2001 From: Aishwarya Gupta Date: Thu, 22 Jul 2021 13:13:44 +0100 Subject: [PATCH 10/10] adding function docstring for new methods --- tap_zendesk/__init__.py | 1 + tap_zendesk/streams.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tap_zendesk/__init__.py b/tap_zendesk/__init__.py index b72ff88..434bc82 100755 --- a/tap_zendesk/__init__.py +++ b/tap_zendesk/__init__.py @@ -84,6 +84,7 @@ def get_abs_path(path): def get_side_load_schemas(sideload_objects, stream): + """Returns the updated schema after adding side load objects to schema dict""" stream_schema = stream.schema.to_dict() for sideload_object in sideload_objects: if sideload_object in SIDELOAD_OBJECTS[stream.tap_stream_id]: diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index c4aaa1a..847c523 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -29,6 +29,7 @@ DEFAULT_SEARCH_WINDOW_SIZE = (60 * 60 * 24) * 30 # defined in seconds, default to a month (30 days) def get_sideload_objects(stream): + """Returns the value of sideload-objects from metadata, returns None if no values are present""" return metadata.to_map(stream.metadata).get((), {}).get('sideload-objects') def get_abs_path(path):