diff --git a/MANIFEST.in b/MANIFEST.in index d4ec702..fa777ff 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include LICENSE include tap_zendesk/schemas/*.json include tap_zendesk/schemas/shared/*.json +include tap_zendesk/schemas/sideload_schemas/*.json diff --git a/Makefile b/Makefile index 97bc398..33407e2 100644 --- a/Makefile +++ b/Makefile @@ -4,3 +4,9 @@ test: pylint tap_zendesk -d missing-docstring,invalid-name,line-too-long,too-many-locals,too-few-public-methods,fixme,stop-iteration-return,too-many-branches,useless-import-alias,no-else-return,logging-not-lazy nosetests test/unittests + + +setup-environment: + rm -rf env || true + python3 -m venv env/tap-zendesk + source env/tap-zendesk/bin/activate && pip3 install . \ No newline at end of file diff --git a/setup.py b/setup.py index ec32964..9e585f2 100644 --- a/setup.py +++ b/setup.py @@ -2,8 +2,9 @@ from setuptools import setup -setup(name='twilio-tap-zendesk', - version='1.0.1', +setup( + name='twilio-tap-zendesk', + version='1.0.2', description='Singer.io tap for extracting data from the Zendesk API', author='Twilio', url='https://github.com/twilio-labs/twilio-tap-zendesk', diff --git a/tap_zendesk/__init__.py b/tap_zendesk/__init__.py index 2fb3359..434bc82 100755 --- a/tap_zendesk/__init__.py +++ b/tap_zendesk/__init__.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 import json -import sys +import sys, os from zenpy import Zenpy import requests from requests import Session from requests.adapters import HTTPAdapter import singer -from singer import metadata, metrics as singer_metrics +from singer import metadata, Schema, metrics as singer_metrics from tap_zendesk import metrics as zendesk_metrics from tap_zendesk.discover import discover_streams from tap_zendesk.streams import STREAMS @@ -34,6 +34,7 @@ # patch Session.request to record HTTP request metrics request = Session.request + def request_metrics_patch(self, method, url, **kwargs): with singer_metrics.http_request_timer(None): return request(self, method, url, **kwargs) @@ -47,9 +48,11 @@ def do_discover(client): json.dump(catalog, sys.stdout, indent=2) LOGGER.info("Finished discover") + def stream_is_selected(mdata): return mdata.get((), {}).get('selected', False) + def get_selected_streams(catalog): selected_stream_names = [] for stream in catalog.streams: @@ -63,15 +66,39 @@ def get_selected_streams(catalog): 'tickets': ['ticket_audits', 'ticket_metrics', 'ticket_comments'] } +# only side loading objects that are returned as a child object and not a separate array +SIDELOAD_OBJECTS = { + 'tickets': ['metric_sets', 'dates', 'comment_count', 'slas'] +} + + def get_sub_stream_names(): sub_stream_names = [] for parent_stream in SUB_STREAMS: sub_stream_names.extend(SUB_STREAMS[parent_stream]) return sub_stream_names + +def get_abs_path(path): + return os.path.join(os.path.dirname(os.path.realpath(__file__)), path) + + +def get_side_load_schemas(sideload_objects, stream): + """Returns the updated schema after adding side load objects to schema dict""" + stream_schema = stream.schema.to_dict() + for sideload_object in sideload_objects: + if sideload_object in SIDELOAD_OBJECTS[stream.tap_stream_id]: + schema_file = "schemas/sideload_schemas/{}.json".format(sideload_object) + with open(get_abs_path(schema_file)) as f: + schema = json.load(f) + stream_schema['properties'][list(schema['properties'].keys())[0]] = list(schema['properties'].values())[0] + return stream_schema + + class DependencyException(Exception): pass + def validate_dependencies(selected_stream_ids): errs = [] msg_tmpl = ("Unable to extract {0} data. " @@ -85,13 +112,14 @@ def validate_dependencies(selected_stream_ids): if errs: raise DependencyException(" ".join(errs)) + def populate_class_schemas(catalog, selected_stream_names): for stream in catalog.streams: if stream.tap_stream_id in selected_stream_names: STREAMS[stream.tap_stream_id].stream = stream -def do_sync(client, catalog, state, config): +def do_sync(client, catalog, state, config): selected_stream_names = get_selected_streams(catalog) validate_dependencies(selected_stream_names) populate_class_schemas(catalog, selected_stream_names) @@ -104,18 +132,12 @@ def do_sync(client, catalog, state, config): LOGGER.info("%s: Skipping - not selected", stream_name) continue - # if starting_stream: - # if starting_stream == stream_name: - # LOGGER.info("%s: Resuming", stream_name) - # starting_stream = None - # else: - # LOGGER.info("%s: Skipping - already synced", stream_name) - # continue - # else: - # LOGGER.info("%s: Starting", stream_name) - - key_properties = metadata.get(mdata, (), 'table-key-properties') + sideload_objects = metadata.get(mdata, (), 'sideload-objects') + if sideload_objects: + stream_schema = get_side_load_schemas(sideload_objects, stream) + stream.schema = Schema.from_dict(stream_schema) + singer.write_schema(stream_name, stream.schema.to_dict(), key_properties) sub_stream_names = SUB_STREAMS.get(stream_name) @@ -126,6 +148,10 @@ def do_sync(client, catalog, state, config): sub_stream = STREAMS[sub_stream_name].stream sub_mdata = metadata.to_map(sub_stream.metadata) sub_key_properties = metadata.get(sub_mdata, (), 'table-key-properties') + sideload_objects = metadata.get(mdata, (), 'sideload-objects') + if sideload_objects: + sub_stream_schema = get_side_load_schemas(sideload_objects, sub_stream) + sub_stream.schema = Schema.from_dict(sub_stream_schema) singer.write_schema(sub_stream.tap_stream_id, sub_stream.schema.to_dict(), sub_key_properties) # parent stream will sync sub stream @@ -143,6 +169,7 @@ def do_sync(client, catalog, state, config): LOGGER.info("Finished sync") zendesk_metrics.log_aggregate_rates() + def oauth_auth(args): if not set(OAUTH_CONFIG_KEYS).issubset(args.config.keys()): LOGGER.debug("OAuth authentication unavailable.") @@ -154,6 +181,7 @@ def oauth_auth(args): "oauth_token": args.config['access_token'], } + def api_token_auth(args): if not set(API_TOKEN_CONFIG_KEYS).issubset(args.config.keys()): LOGGER.debug("API Token authentication unavailable.") @@ -166,6 +194,7 @@ def api_token_auth(args): "token": args.config['api_token'] } + def get_session(config): """ Add partner information to requests Session object if specified in the config. """ if not all(k in config for k in ["marketplace_name", @@ -181,6 +210,7 @@ def get_session(config): session.headers["X-Zendesk-Marketplace-App-Id"] = str(config.get("marketplace_app_id", "")) return session + @singer.utils.handle_top_exception(LOGGER) def main(): parsed_args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS) diff --git a/tap_zendesk/schemas/sideload_schemas/comment_count.json b/tap_zendesk/schemas/sideload_schemas/comment_count.json new file mode 100644 index 0000000..d521042 --- /dev/null +++ b/tap_zendesk/schemas/sideload_schemas/comment_count.json @@ -0,0 +1,15 @@ +{ + "type": [ + "null", + "object" + ], + "properties": { + "comment_count": { + "type": [ + "null", + "integer" + ] + } + }, + "additionalProperties": true +} \ No newline at end of file diff --git a/tap_zendesk/schemas/sideload_schemas/dates.json b/tap_zendesk/schemas/sideload_schemas/dates.json new file mode 100644 index 0000000..2994c6d --- /dev/null +++ b/tap_zendesk/schemas/sideload_schemas/dates.json @@ -0,0 +1,65 @@ +{ + "type": [ + "null", + "object" + ], + "properties": { + "dates": { + "type": [ + "null", + "object" + ], + "properties": { + "solved_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "status_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "assignee_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "assigned_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "requester_updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "latest_comment_added_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "initially_assigned_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } + } + } +} diff --git a/tap_zendesk/schemas/sideload_schemas/metric_sets.json b/tap_zendesk/schemas/sideload_schemas/metric_sets.json new file mode 100644 index 0000000..e499b59 --- /dev/null +++ b/tap_zendesk/schemas/sideload_schemas/metric_sets.json @@ -0,0 +1,240 @@ +{ + "type": [ + "null", + "object" + ], + "properties": { + "metric_set": { + "type": [ + "null", + "object" + ], + "properties": { + "url": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "ticket_id": { + "type": [ + "null", + "integer" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ] + }, + "updated_at": { + "type": [ + "null", + "string" + ] + }, + "group_stations": { + "type": [ + "null", + "integer" + ] + }, + "assignee_stations": { + "type": [ + "null", + "integer" + ] + }, + "reopens": { + "type": [ + "null", + "integer" + ] + }, + "replies": { + "type": [ + "null", + "integer" + ] + }, + "assignee_updated_at": { + "type": [ + "null", + "string" + ] + }, + "requester_updated_at": { + "type": [ + "null", + "string" + ] + }, + "status_updated_at": { + "type": [ + "null", + "string" + ] + }, + "initially_assigned_at": { + "type": [ + "null", + "string" + ] + }, + "assigned_at": { + "type": [ + "null", + "string" + ] + }, + "solved_at": { + "type": [ + "null", + "string" + ] + }, + "latest_comment_added_at": { + "type": [ + "null", + "string" + ] + }, + "reply_time_in_minutes": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "string" + ] + }, + "business": { + "type": [ + "null", + "string" + ] + } + }, + "additionalProperties": true + }, + "first_resolution_time_in_minutes": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "string" + ] + }, + "business": { + "type": [ + "null", + "string" + ] + } + }, + "additionalProperties": true + }, + "full_resolution_time_in_minutes": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "string" + ] + }, + "business": { + "type": [ + "null", + "string" + ] + } + }, + "additionalProperties": true + }, + "agent_wait_time_in_minutes": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "string" + ] + }, + "business": { + "type": [ + "null", + "string" + ] + } + }, + "additionalProperties": true + }, + "requester_wait_time_in_minutes": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "string" + ] + }, + "business": { + "type": [ + "null", + "string" + ] + } + }, + "additionalProperties": true + }, + "on_hold_time_in_minutes": { + "type": [ + "null", + "object" + ], + "properties": { + "calendar": { + "type": [ + "null", + "integer" + ] + }, + "business": { + "type": [ + "null", + "integer" + ] + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} \ No newline at end of file diff --git a/tap_zendesk/schemas/sideload_schemas/slas.json b/tap_zendesk/schemas/sideload_schemas/slas.json new file mode 100644 index 0000000..a8d7c11 --- /dev/null +++ b/tap_zendesk/schemas/sideload_schemas/slas.json @@ -0,0 +1,51 @@ +{ + "type": [ + "null", + "object" + ], + "properties": { + "slas": { + "type": [ + "null", + "object" + ], + "properties": { + "policy_metrics": { + "type": "array", + "items":{ + "type": [ + "null", + "object" + ], + "properties": { + "breach_at": { + "type": [ + "null", + "string" + ] + }, + "stage": { + "type": [ + "null", + "string" + ] + }, + "metric": { + "type": [ + "null", + "string" + ] + }, + "hours": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tap_zendesk/streams.py b/tap_zendesk/streams.py index 2c8fdf6..847c523 100644 --- a/tap_zendesk/streams.py +++ b/tap_zendesk/streams.py @@ -28,6 +28,10 @@ DEFAULT_SEARCH_WINDOW_SIZE = (60 * 60 * 24) * 30 # defined in seconds, default to a month (30 days) +def get_sideload_objects(stream): + """Returns the value of sideload-objects from metadata, returns None if no values are present""" + return metadata.to_map(stream.metadata).get((), {}).get('sideload-objects') + def get_abs_path(path): return os.path.join(os.path.dirname(os.path.realpath(__file__)), path) @@ -250,8 +254,8 @@ def _empty_buffer(self): def sync(self, state): bookmark = self.get_bookmark(state) - tickets = self.client.tickets.incremental(start_time=bookmark) - + sideload_objects = get_sideload_objects(self.stream) + tickets = self.client.tickets.incremental(start_time=bookmark, include=sideload_objects) audits_stream = TicketAudits(self.client) metrics_stream = TicketMetrics(self.client) comments_stream = TicketComments(self.client)