General fixes due to Unit tests

megalista_dataflow/uploaders/campaign_manager/utils.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from config.utils import Utils as BaseUtils
+from uploaders.utils import Utils as BaseUtils
 
 class Utils(BaseUtils):
     pass

megalista_dataflow/uploaders/display_video/utils.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from config.utils import Utils as BaseUtils
+from uploaders.utils import Utils as BaseUtils
 
 class Utils(BaseUtils):
     pass

megalista_dataflow/uploaders/google_ads/conversions/google_ads_enhanced_conversions_leads_uploader.py

@@ -53,15 +53,15 @@ def _get_customer_id(self, account_config: AccountConfig, destination: Destinati
             If the customer_id is present on the destination, returns it, otherwise defaults to the account_config info.
         """
         if len(destination.destination_metadata) >= 2 and len(destination.destination_metadata[1]) > 0:
-            return Utils.clean_ads_customer_id(destination.destination_metadata[1])
-        return Utils.clean_ads_customer_id(account_config.google_ads_account_id)
+            return Utils.filter_text_only_numbers(destination.destination_metadata[1])
+        return Utils.filter_text_only_numbers(account_config.google_ads_account_id)
 
     def _get_login_customer_id(self, account_config: AccountConfig, destination: Destination) -> str:
         """
             If the customer_id in account_config is a mcc, then login with the mcc account id, otherwise use the customer id.
         """
         if account_config._mcc:
-            return Utils.clean_ads_customer_id(account_config.google_ads_account_id)
+            return Utils.filter_text_only_numbers(account_config.google_ads_account_id)
 
         return self._get_customer_id(account_config, destination)
 

megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_calls_uploader.py

@@ -60,8 +60,8 @@ def _get_customer_id(self, account_config:AccountConfig, destination:Destination
       If the customer_id is present on the destination, returns it, otherwise defaults to the account_config info.
     """
     if len(destination.destination_metadata) >= 2 and len(destination.destination_metadata[1]) > 0:
-      return Utils.clean_ads_customer_id(destination.destination_metadata[1])
-    return Utils.clean_ads_customer_id(account_config.google_ads_account_id)
+      return Utils.filter_text_only_numbers(destination.destination_metadata[1])
+    return Utils.filter_text_only_numbers(account_config.google_ads_account_id)
 
 
   @staticmethod

megalista_dataflow/uploaders/google_ads/conversions/google_ads_offline_conversions_uploader.py

@@ -53,15 +53,15 @@ def _get_customer_id(self, account_config:AccountConfig, destination:Destination
       If the customer_id is present on the destination, returns it, otherwise defaults to the account_config info.
     """
     if len(destination.destination_metadata) >= 2 and len(destination.destination_metadata[1]) > 0:
-      return Utils.clean_ads_customer_id(destination.destination_metadata[1])
-    return Utils.clean_ads_customer_id(account_config.google_ads_account_id)
+      return Utils.filter_text_only_numbers(destination.destination_metadata[1])
+    return Utils.filter_text_only_numbers(account_config.google_ads_account_id)
 
   def _get_login_customer_id(self, account_config: AccountConfig, destination: Destination) -> str:
     """
       If the customer_id in account_config is a mcc, then login with the mcc account id, otherwise use the customer id.
     """
     if account_config._mcc:
-        return Utils.clean_ads_customer_id(account_config.google_ads_account_id)
+        return Utils.filter_text_only_numbers(account_config.google_ads_account_id)
 
     return self._get_customer_id(account_config, destination)
 

megalista_dataflow/uploaders/google_ads/conversions/google_ads_ssd_uploader.py

@@ -17,7 +17,7 @@
 
 from error.error_handling import ErrorHandler
 from models.execution import AccountConfig, Batch, Destination, Execution
-from uploaders.utils import Utils
+from uploaders.google_ads.utils import Utils
 from uploaders.google_ads import ADS_API_VERSION
 from uploaders.uploaders import MegalistaUploader
 
@@ -130,15 +130,15 @@ def _get_customer_id(self, account_config: AccountConfig, destination: Destinati
           If the customer_id is present on the destination, returns it, otherwise defaults to the account_config info.
         """
         if len(destination.destination_metadata) >= 5 and len(destination.destination_metadata[4]) > 0:
-            return Utils.clean_ads_customer_id(destination.destination_metadata[4])
-        return Utils.clean_ads_customer_id(account_config.google_ads_account_id)
+            return Utils.filter_text_only_numbers(destination.destination_metadata[4])
+        return Utils.filter_text_only_numbers(account_config.google_ads_account_id)
 
     def _get_login_customer_id(self, account_config: AccountConfig, destination: Destination) -> str:
         """
           If the customer_id in account_config is a mcc, then login with the mcc account id, otherwise use the customer id.
         """
         if account_config._mcc:
-            return Utils.clean_ads_customer_id(account_config.google_ads_account_id)
+            return Utils.filter_text_only_numbers(account_config.google_ads_account_id)
 
         return self._get_customer_id(account_config, destination)
 

megalista_dataflow/uploaders/google_ads/customer_match/abstract_uploader.py

@@ -145,15 +145,15 @@ def _get_customer_id(self, account_config: AccountConfig, destination: Destinati
           If the customer_id is present on the destination, returns it, otherwise defaults to the account_config info.
         """
         if len(destination.destination_metadata) >= 5 and len(destination.destination_metadata[4]) > 0:
-            return Utils.clean_ads_customer_id(destination.destination_metadata[4])
-        return Utils.clean_ads_customer_id(account_config.google_ads_account_id)
+            return Utils.filter_text_only_numbers(destination.destination_metadata[4])
+        return Utils.filter_text_only_numbers(account_config.google_ads_account_id)
 
     def _get_login_customer_id(self, account_config: AccountConfig, destination: Destination) -> str:
         """
           If the customer_id in account_config is a mcc, then login with the mcc account id, otherwise use the customer id.
         """
         if account_config._mcc:
-            return Utils.clean_ads_customer_id(account_config.google_ads_account_id)
+            return Utils.filter_text_only_numbers(account_config.google_ads_account_id)
 
         return self._get_customer_id(account_config, destination)
 

megalista_dataflow/uploaders/google_ads/utils.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from google.ads.googleads.client import GoogleAdsClient
-from google.ads.googleads import oauth2
 import logging
 from typing import Optional
 
@@ -23,6 +21,9 @@ class Utils(BaseUtils):
 
     @staticmethod
     def get_ads_client(oauth_credentials, developer_token, customer_id):
+        from google.ads.googleads.client import GoogleAdsClient
+        from google.ads.googleads import oauth2
+
         oauth2_client = oauth2.get_installed_app_credentials(
             oauth_credentials.get_client_id(), oauth_credentials.get_client_secret(),
             oauth_credentials.get_refresh_token())
@@ -34,7 +35,7 @@ def get_ads_client(oauth_credentials, developer_token, customer_id):
     @staticmethod
     def get_ads_service(service_name, version, oauth_credentials, developer_token,
                         customer_id):
-        return get_ads_client(oauth_credentials, developer_token, customer_id).get_service(service_name, version=version)
+        return Utils.get_ads_client(oauth_credentials, developer_token, customer_id).get_service(service_name, version=version)
 
     @staticmethod
     def print_partial_error_messages(logger_name, action, response) -> Optional[str]:

megalista_dataflow/uploaders/google_analytics/utils.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from config.utils import Utils as BaseUtils
+from uploaders.utils import Utils as BaseUtils
 
 class Utils(BaseUtils):
     pass

megalista_dataflow/uploaders/utils.py

@@ -21,130 +21,9 @@
 from typing import Optional
 from models.execution import Batch
 from uploaders.uploaders import MegalistaUploader
-
-MAX_RETRIES = 3
-
-timezone = pytz.timezone('America/Sao_Paulo')
-
-from config.utils import Utils as BaseUtils
-
-
-def get_ads_client(oauth_credentials, developer_token, customer_id):
-    from google.ads.googleads.client import GoogleAdsClient
-    from google.ads.googleads import oauth2
-
-    oauth2_client = oauth2.get_installed_app_credentials(
-        oauth_credentials.get_client_id(), oauth_credentials.get_client_secret(),
-        oauth_credentials.get_refresh_token())
-
-    return GoogleAdsClient(
-        oauth2_client, developer_token,
-        login_customer_id=customer_id)
-
-
-def get_ads_service(service_name, version, oauth_credentials, developer_token,
-                    customer_id):
-    return get_ads_client(oauth_credentials, developer_token, customer_id).get_service(service_name, version=version)
-
-
-def format_date(date):
-    if isinstance(date, datetime.datetime):
-        pdate = date
-    else:
-        pdate = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f')
-
-    pdate = timezone.localize(pdate)
-    str_timezone = pdate.strftime("%z")
-    return f'{datetime.datetime.strftime(pdate, "%Y-%m-%d %H:%M:%S")}{str_timezone[-5:-2]}:{str_timezone[-2:]}'
-
-def get_timestamp_micros(date):
-    if isinstance(date, datetime.datetime):
-        pdate = date
-    else:
-        pdate = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f')
-
-    return math.floor(pdate.timestamp() * 10e5)
 
+from config.utils import Utils as BaseUtils
 
-def safe_process(logger):
-    def deco(func):
-        def inner(*args, **kwargs):
-            self_ = args[0]
-            batch = args[1]
-            if not batch:
-                logger.warning('Skipping upload, received no elements.')
-                return
-            logger.info(f'Uploading {len(batch.elements)} rows...')
-            try:
-                return func(*args, **kwargs)
-            except BaseException as e:
-                self_._add_error(batch.execution, f'Error uploading data: {e}')
-                logger.error(f'Error uploading data for :{batch.elements}')
-                logger.error(e, exc_info=True)
-                logger.exception('Error uploading data.')
-
-        return inner
-
-    return deco
-
-
-def safe_call_api(function, logger, *args, **kwargs):
-    current_retry = 1
-    _do_safe_call_api(function, logger, current_retry, *args, **kwargs)
-
-
-def _do_safe_call_api(function, logger, current_retry, *args, **kwargs):
-    try:
-        return function(*args, *kwargs)
-    except Exception as e:
-        if current_retry < MAX_RETRIES:
-            logger.exception(
-                f'Fail number {current_retry}. Stack track follows. Trying again.')
-            current_retry += 1
-            return _do_safe_call_api(function, logger, current_retry, *args, **kwargs)
-
-
-def convert_datetime_tz(dt, origin_tz, destination_tz):
-    datetime_obj = pytz.timezone(origin_tz).localize(dt)
-    return datetime_obj.astimezone(pytz.timezone(destination_tz))
-
-
-def print_partial_error_messages(logger_name, action, response) -> Optional[str]:
-    """
-    Print partials errors received in the response.
-    @param logger_name: logger name to be used
-    @param action: action to be part of the message
-    @param response: response body of the API call
-    @return: the error_message returned, if there was one, None otherwise.
-    """
-    error_message = None
-
-    partial_failure = getattr(response, 'partial_failure_error', None)
-    if partial_failure is not None and partial_failure.message != '':
-        error_message = f'Error on {action}: {partial_failure.message}.'
-        logging.getLogger(logger_name).error(error_message)
-    results = getattr(response, 'results', [])
-    for result in results:
-        gclid = getattr(result, 'gclid', None)
-        caller_id = getattr(result, 'caller_id', None)
-        if gclid is not None:
-            message = f'gclid {result.gclid} uploaded.'
-        elif caller_id is not None:
-            message = f'caller_id {result.caller_id} uploaded.'
-        else:
-            message = f'item {result} uploaded.'
-
-        logging.getLogger(logger_name).debug(message)
-
-    return error_message
-
-def clean_ads_customer_id(customer_id: str) -> str:
-    return re.sub(r'[^0-9]', '', customer_id)
-
-def google_analytics_account_id(analytics_account_id: str) -> str:
-    return re.sub(r'[^0-9]', '', analytics_account_id)
-
-
 class Utils(BaseUtils):
     _TIMEZONE = pytz.timezone('America/Sao_Paulo')
     _MAX_RETRIES = 3
@@ -156,7 +35,7 @@ def format_date(date):
         else:
             pdate = datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S.%f')
 
-        pdate = _TIMEZONE.localize(pdate)
+        pdate = Utils._TIMEZONE.localize(pdate)
         str_timezone = pdate.strftime("%z")
         return f'{datetime.datetime.strftime(pdate, "%Y-%m-%d %H:%M:%S")}{str_timezone[-5:-2]}:{str_timezone[-2:]}'