From a83a772221d81470ee151174eef1728dbbc547c9 Mon Sep 17 00:00:00 2001 From: Brendan <125093480+brenhogan@users.noreply.github.com> Date: Tue, 17 Sep 2024 01:19:19 -0400 Subject: [PATCH] =?UTF-8?q?Revert=20"HGI-6435=20/=20Add=20streams=20for=20?= =?UTF-8?q?sessions=20and=20breakdown=20analytics=20reports=20(=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 3f14def34688fce9028dffaf8dac109bda2cd89a. --- tap_hubspot_beta/client_base.py | 2 +- tap_hubspot_beta/client_v1.py | 2 +- tap_hubspot_beta/client_v3.py | 2 +- tap_hubspot_beta/streams.py | 176 +------------------------------- tap_hubspot_beta/tap.py | 14 --- 5 files changed, 5 insertions(+), 191 deletions(-) diff --git a/tap_hubspot_beta/client_base.py b/tap_hubspot_beta/client_base.py index b743773..a0c0dc7 100644 --- a/tap_hubspot_beta/client_base.py +++ b/tap_hubspot_beta/client_base.py @@ -28,7 +28,7 @@ class hubspotStream(RESTStream): url_base = "https://api.hubapi.com/" base_properties = [] - additional_params = {} + additional_prarams = {} properties_url = None page_size = 100 diff --git a/tap_hubspot_beta/client_v1.py b/tap_hubspot_beta/client_v1.py index 6c5d27f..b579c46 100644 --- a/tap_hubspot_beta/client_v1.py +++ b/tap_hubspot_beta/client_v1.py @@ -50,7 +50,7 @@ def get_url_params( params["count"] = self.page_size if next_page_token: params.update(next_page_token) - params.update(self.additional_params) + params.update(self.additional_prarams) params["property"] = self.selected_properties return params diff --git a/tap_hubspot_beta/client_v3.py b/tap_hubspot_beta/client_v3.py index 820600c..98e4f60 100644 --- a/tap_hubspot_beta/client_v3.py +++ b/tap_hubspot_beta/client_v3.py @@ -237,7 +237,7 @@ def get_url_params( """Return a dictionary of values to be used in URL parameterization.""" params: dict = {} params["limit"] = self.page_size - params.update(self.additional_params) + params.update(self.additional_prarams) if self.properties_url: params["properties"] = ",".join(self.selected_properties) if next_page_token: diff --git a/tap_hubspot_beta/streams.py b/tap_hubspot_beta/streams.py index 55411e0..02a11f2 100644 --- a/tap_hubspot_beta/streams.py +++ b/tap_hubspot_beta/streams.py @@ -18,6 +18,7 @@ from tap_hubspot_beta.client_v1 import hubspotV1Stream from tap_hubspot_beta.client_v3 import hubspotV3SearchStream, hubspotV3Stream, hubspotV3SingleSearchStream, AssociationsV3ParentStream from tap_hubspot_beta.client_v4 import hubspotV4Stream +import time import pytz from singer_sdk.helpers._state import log_sort_error from pendulum import parse @@ -33,7 +34,6 @@ th.Property("associationTypes", th.CustomType({"type": ["array", "object"]})), ).to_dict() - class AccountStream(hubspotV1Stream): """Account Stream""" @@ -161,7 +161,7 @@ class ContactsStream(hubspotV1Stream): records_jsonpath = "$.contacts[*]" primary_keys = ["vid"] replication_key = None - additional_params = dict(showListMemberships=True) + additional_prarams = dict(showListMemberships=True) properties_url = "properties/v1/contacts/properties" base_properties = [ @@ -1743,178 +1743,6 @@ class AssociationTasksDealsStream(AssociationTasksStream): name = "associations_tasks_deals" path = "crm/v4/associations/tasks/deals/batch/read" - -breakdown_properties_list = [ - th.Property("breakdown", th.StringType), - th.Property("others", th.IntegerType), - th.Property("otherCampaigns", th.IntegerType), - th.Property("mobile", th.IntegerType), - th.Property("desktop", th.IntegerType), - th.Property("organicSearch", th.IntegerType), - th.Property("paidSearch", th.IntegerType), - th.Property("paidSocial", th.IntegerType), - th.Property("socialMedia", th.IntegerType), - th.Property("directTraffic", th.IntegerType), - th.Property("referrals", th.IntegerType), - th.Property("date", th.DateType), - th.Property("rawViews", th.IntegerType), - th.Property("visits", th.IntegerType), - th.Property("visitors", th.IntegerType), - th.Property("leads", th.IntegerType), - th.Property("contacts", th.IntegerType), - th.Property("subscribers", th.IntegerType), - th.Property("opportunities", th.IntegerType), - th.Property("customers", th.IntegerType), - th.Property("pageviewsPerSession", th.NumberType), - th.Property("bounceRate", th.NumberType), - th.Property("timePerSession", th.NumberType), - th.Property("newVisitorSessionRate", th.NumberType), - th.Property("sessionToContactRate", th.NumberType), - th.Property("contactToCustomerRate", th.NumberType), -] - -class SessionAnalyticsReportsBaseStream(hubspotV3Stream): - - schema = th.PropertiesList( - *breakdown_properties_list - ).to_dict() - - def parse_response(self, response: requests.Response): - res_json = response.json() - for date_str, list_data_obj in res_json.items(): - for data_obj in list_data_obj: - data_obj["date"] = date_str - yield data_obj - - -class SessionAnalyticsDailyReportsStream(SessionAnalyticsReportsBaseStream): - name = "session_analytics_daily_reports" - path = "analytics/v2/reports/sessions/daily" - - -class SessionAnalyticsWeeklyReportsStream(SessionAnalyticsReportsBaseStream): - name = "session_analytics_weekly_reports" - path = "analytics/v2/reports/sessions/weekly" - - -class SessionAnalyticsMonthlyReportsStream(SessionAnalyticsReportsBaseStream): - name = "session_analytics_monthly_reports" - path = "analytics/v2/reports/sessions/monthly" - - -class SessionAnalyticsTotalReportStream(SessionAnalyticsReportsBaseStream): - name = "session_analytics_total_report" - path = "analytics/v2/reports/sessions/total" - __offset = 0 - - def parse_response(self, response: requests.Response) -> Iterable[Dict]: - yield from (row for row in response.json().get('breakdowns', [{}])) - - @property - def additional_params(self): - return dict(offset=self.__offset) - - def get_next_page_token(self, response: requests.Response, previous_token: Any | None) -> Any | None: - offset = response.json().get('offset', 0) - if not offset: - return None - - total = response.json().get('total') - if not total or offset >= total: - return None - - if not self.page_size: - self.page_size = offset - - self.__offset += self.page_size - return self.additional_params - - -class BreakdownsAnalyticsReportsBaseStream(hubspotV3Stream): - page_size = None - __offset = 0 - __current_d1 = None - - @property - def d1_options(self): - raise NotImplementedError #['regions', 'us', 'organic', 'referrals', 'social', 'test'] - - schema = th.PropertiesList( - *breakdown_properties_list, - th.Property("d1", th.StringType) - ).to_dict() - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - d1_options = self._config.get(f"d1_options_{self.name}") - if d1_options and isinstance(d1_options, list): - self.__d1_options = d1_options - elif d1_options and isinstance(d1_options, str): - self.__d1_options = [d1_options] - else: - self.__d1_options = self.d1_options - - @property - def d1(self): - if not self.__current_d1: - self.update_d1() - return self.__current_d1 - - @property - def additional_params(self): - return dict(offset=self.__offset, d1=self.d1) - - def update_d1(self): - self.__current_d1 = self.__d1_options.pop() - - def reset_offset(self): - self.__offset = 0 - - def parse_response(self, response: requests.Response) -> Iterable[Dict]: - for row in response.json().get('breakdowns', [{}]): - row["d1"] = self.d1 - yield row - - def next_token(self): - if len(self.__d1_options) == 0: - return None - - self.update_d1() - self.reset_offset() - return self.additional_params - - def get_next_page_token(self, response: requests.Response, previous_token: Any | None) -> Any | None: - offset = response.json().get('offset', 0) - if not offset: - return self.next_token() - - total = response.json().get('total') - if not total or offset >= total: - return self.next_token() - - if not self.page_size: - self.page_size = offset - - self.__offset += self.page_size - return self.additional_params - - -class BreakdownsAnalyticsReportsSourcesStream(BreakdownsAnalyticsReportsBaseStream): - name = "analytics_reports_sources" - path = "analytics/v2/reports/sources/total" - d1_options = ['organic', 'referrals', 'social'] - - -class BreakdownsAnalyticsReportsGeolocationStream(BreakdownsAnalyticsReportsBaseStream): - name = "analytics_reports_geolocation" - path = "analytics/v2/reports/geolocation/total" - d1_options = ['regions', 'us', 'organic', 'referrals', 'social', 'test'] - - -class BreakdownsAnalyticsReportsUtmCampaignsStream(BreakdownsAnalyticsReportsBaseStream): - name = "analytics_reports_utm_campaigns" - path = "analytics/v2/reports/utm-campaigns/total" - d1_options = ['regions', 'us', 'organic', 'referrals', 'social', 'test'] class FormsSummaryMonthlyStream(hubspotV1Stream): """Association Base Stream""" #https://legacydocs.hubspot.com/docs/methods/analytics/get-analytics-data-by-object diff --git a/tap_hubspot_beta/tap.py b/tap_hubspot_beta/tap.py index 6430bf0..051f0a0 100644 --- a/tap_hubspot_beta/tap.py +++ b/tap_hubspot_beta/tap.py @@ -69,13 +69,6 @@ AssociationTasksCompaniesStream, AssociationTasksContactsStream, AssociationTasksDealsStream, - SessionAnalyticsDailyReportsStream, - SessionAnalyticsWeeklyReportsStream, - SessionAnalyticsMonthlyReportsStream, - SessionAnalyticsTotalReportStream, - BreakdownsAnalyticsReportsSourcesStream, - BreakdownsAnalyticsReportsGeolocationStream, - BreakdownsAnalyticsReportsUtmCampaignsStream, FormsSummaryMonthlyStream, TeamsStream, MeetingsAssociationStream, @@ -159,13 +152,6 @@ AssociationTasksCompaniesStream, AssociationTasksContactsStream, AssociationTasksDealsStream, - SessionAnalyticsDailyReportsStream, - SessionAnalyticsWeeklyReportsStream, - SessionAnalyticsMonthlyReportsStream, - SessionAnalyticsTotalReportStream, - BreakdownsAnalyticsReportsSourcesStream, - BreakdownsAnalyticsReportsGeolocationStream, - BreakdownsAnalyticsReportsUtmCampaignsStream, FormsSummaryMonthlyStream, TeamsStream, MeetingsAssociationStream,