diff --git a/docs/index.rst b/docs/index.rst index 093849d15a..aed7ccca3a 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -205,6 +205,7 @@ Indices and tables google hustle mailchimp + mobilecommons mobilize_america nation_builder newmode diff --git a/docs/mobilecommons.rst b/docs/mobilecommons.rst new file mode 100644 index 0000000000..ebdc4e1236 --- /dev/null +++ b/docs/mobilecommons.rst @@ -0,0 +1,49 @@ +MobileCommons +========== + +******** +Overview +******** + +`MobileCommons `_ is a broadcast text messaging tool that helps orgranizations +mobilize supporters and fundraise by building opt-ed in audiences. You can read more about the product +`here `_. + +*********** +Quick Start +*********** + +To instantiate a class you must pass the username and password of a MobileCommons account as an argument +or store the username and password into environmental variables called ``MOBILECOMMONS_USERNAME`` and +``MOBILECOMMONS_PASSWORD``, respectively. If you MobileCommons account has access to various MobileCommons +companies (i.e. organizations), you'll need to specify which MobileCommons company you'd like to interact +with by specifying the Company ID in the ``company_id`` parameter. To find the Company ID, navigate to the +`Company and Users page `_. + +.. code-block:: python + from parsons import MobileCommons + + # Pass credentials via environmental variables for account has access to only one MobileCommons company + mc = MobileCommons() + + # Pass credentials via environmental variables for account has access to multiple MobileCommons companies + mc = MobileCommons(company_id='EXAMPLE78363BOCA483954419EB70986A68888') + + # Pass credentials via argument for account has access to only one MobileCommons company + mc = MobileCommons(username='octavia.b@scifi.net', password='badpassword123') + +Then you can call various endpoints: + +.. code-block:: python + # Return all MobileCommons subscribers in a table + subscribers = get_campaign_subscribers(campaign_id=1234567) + + # Create a new profile, return profile_id + new_profile=create_profile(phone=3073991987, first_name='Jane', last_name='Fonda') + + +*** +API +*** +.. autoclass :: parsons.MobileCommons + :inherited-members: \ No newline at end of file diff --git a/parsons/__init__.py b/parsons/__init__.py index 132d90db75..b8793ec649 100644 --- a/parsons/__init__.py +++ b/parsons/__init__.py @@ -65,6 +65,7 @@ ("parsons.google.google_sheets", "GoogleSheets"), ("parsons.hustle.hustle", "Hustle"), ("parsons.mailchimp.mailchimp", "Mailchimp"), + ("parsons.mobilecommons.mobilecommons", "MobileCommons"), ("parsons.mobilize_america.ma", "MobilizeAmerica"), ("parsons.nation_builder.nation_builder", "NationBuilder"), ("parsons.newmode.newmode", "Newmode"), diff --git a/parsons/mobilecommons/__init__.py b/parsons/mobilecommons/__init__.py new file mode 100644 index 0000000000..377c3e6319 --- /dev/null +++ b/parsons/mobilecommons/__init__.py @@ -0,0 +1,3 @@ +from parsons.mobilecommons.mobilecommons import MobileCommons + +__all__ = ["MobileCommons"] diff --git a/parsons/mobilecommons/mobilecommons.py b/parsons/mobilecommons/mobilecommons.py new file mode 100644 index 0000000000..41acf31386 --- /dev/null +++ b/parsons/mobilecommons/mobilecommons.py @@ -0,0 +1,428 @@ +from parsons.utilities import check_env +from parsons.utilities.api_connector import APIConnector +from parsons.utilities.datetime import parse_date +from parsons import Table +from bs4 import BeautifulSoup +from requests import HTTPError +import xmltodict +import logging + +logger = logging.getLogger(__name__) + +MC_URI = "https://secure.mcommons.com/api/" +DATE_FMT = "%Y-%m-%d" + + +def _format_date(user_entered_date): + if user_entered_date: + formatted_date = parse_date(user_entered_date).strftime(DATE_FMT) + else: + formatted_date = None + return formatted_date + + +class MobileCommons: + """ + Instantiate the MobileCommons class. + + `Args:` + username: str + A valid email address connected to a MobileCommons account. Not required if + ``MOBILECOMMONS_USERNAME`` env variable is set. + password: str + Password associated with Zoom account. Not required if ``MOBILECOMMONS_PASSWORD`` + env variable set. + company_id: str + The company id of the MobileCommons organization to connect to. Not required if + username and password are for an account associated with only one MobileCommons + organization. + """ + + def __init__(self, username=None, password=None, company_id=None): + self.username = check_env.check("MOBILECOMMONS_USERNAME", username) + self.password = check_env.check("MOBILECOMMONS_PASSWORD", password) + self.default_params = {"company": company_id} if company_id else {} + self.client = APIConnector(uri=MC_URI, auth=(self.username, self.password)) + + def _mc_get_request( + self, + endpoint, + first_data_key, + second_data_key, + params, + elements_to_unpack=None, + limit=None, + ): + """ + A function for GET requests that handles MobileCommons xml responses and pagination + + `Args:` + endpoint: str + The endpoint, which will be appended to the base URL for each request + first_data_key: str + The first key used to extract the desired data from the response dictionary derived + from the xml response. E.g. 'broadcasts' + second_data_key: str + The second key used to extract the desired data from the response dictionary derived + from the xml response. The value of this key is a list of values. E.g. 'broadcast' + params: dict + Parameters to be passed into GET request + elements_to_unpack: list + A list of elements that contain dictionaries to be unpacked into new columns in the + final table + limit: int + The maximum number of rows to return + `Returns:` + Parsons table with requested data + """ + + # Create a table to compile results from different pages in + final_table = Table() + # Max page_limit is 1000 for MC + page_limit = min((limit or 1000), 1000) + + # Set get request params + params = {"limit": page_limit, **self.default_params, **params} + + logger.info( + f"Working on fetching first {page_limit} rows. This can take a long time." + ) + + # Make get call and parse XML into list of dicts + page = 1 + response_dict = self._parse_get_request(endpoint=endpoint, params=params) + + # If there's only one row, then it is returned as a dict, otherwise as a list + data = response_dict["response"][first_data_key][second_data_key] + if isinstance(data, dict): + data = [data] + + # Convert to parsons table + response_table = Table(data) + + # empty_page used for pagination below + if response_table.num_rows > 0: + empty_page = False + else: + raise ValueError("There are no records for specified resource") + + # Unpack any specified elements + if elements_to_unpack: + for col in elements_to_unpack: + response_table.unpack_dict(col) + + # Append to final table + final_table.concat(response_table) + final_table.materialize() + + # Now must paginate to get more records + # MC GET responses sometimes includes a page_count parameter to indicate how many pages + # are available. Other times, the response includes a 'num' parameter that, when + # you reach an empty page, equals 0. In the first scenario we know to stop paginating when + # we reach the last page. In the second, we know to stop paginating when we encounter + # and empty page + + # First scenario + try: + avail_pages = int(response_dict["response"][first_data_key]["page_count"]) + total_records = avail_pages * page_limit + page_indicator = "page_count" + + # Second scenario + except KeyError: + response_dict["response"][first_data_key]["num"] + page_indicator = "num" + # If page_count is not available, we cannot calculate total_records and will paginate + # until we hit user defined limit or an empty page + total_records = float("inf") + + # Go fetch other pages of data + while final_table.num_rows < (limit or total_records) and not empty_page: + page += 1 + page_params = {"page": str(page), **params} + logger.info( + f"Fetching rows {(page - 1) * page_limit + 1} - {(page) * page_limit} " + f"of {limit}" + ) + # Send get request + response_dict = self._parse_get_request( + endpoint=endpoint, params=page_params + ) + # Check to see if page was empty if num parameter is available + if page_indicator == "num": + empty_page = int(response_dict["response"][first_data_key]["num"]) > 0 + + if not empty_page: + # Extract data + response_table = Table( + response_dict["response"][first_data_key][second_data_key] + ) + # Append to final table + final_table.concat(response_table) + final_table.materialize() + + return Table(final_table[:limit]) + + def _check_response_status_code(self, response): + """ + A helper function that checks the status code of a response and raises an error if the + response code is not 200 + + `Args:` + response: requests package response object + """ + if response.status_code != 200: + error = f"Response Code {str(response.status_code)}" + error_html = BeautifulSoup(response.text, features="html.parser") + error += "\n" + error_html.h4.next + error += "\n" + error_html.p.next + raise HTTPError(error) + + def _parse_get_request(self, endpoint, params): + """ + A helper function that sends a get request to MobileCommons and then parses XML responses in + order to return the response as a dictionary + + `Args:` + endpoint: str + The endpoint, which will be appended to the base URL for each request + params: dict + Parameters to be passed into GET request + `Returns:` + xml response parsed into list or dictionary + """ + response = self.client.request(endpoint, "GET", params=params) + + # If there's an error with initial response, raise error + self._check_response_status_code(response) + + # If good response, compile data into final_table + # Parse xml to nested dictionary and load to parsons table + response_dict = xmltodict.parse( + response.text, attr_prefix="", cdata_key="", dict_constructor=dict + ) + return response_dict + + def _mc_post_request(self, endpoint, params): + """ + A function for POST requests that handles MobileCommons xml responses + + `Args:` + endpoint: str + The endpoint, which will be appended to the base URL for each request + params: dict + Parameters to be passed into GET request + `Returns:` + xml response parsed into list or dictionary + """ + + response = self.client.request(endpoint, "POST", params=params) + + response_dict = xmltodict.parse( + response.text, attr_prefix="", cdata_key="", dict_constructor=dict + ) + if response_dict["response"]["success"] == "true": + return response_dict["response"] + else: + raise HTTPError(response_dict["response"]["error"]) + + def get_broadcasts( + self, first_date=None, last_date=None, status=None, campaign_id=None, limit=None + ): + """ + A function for get broadcasts + + `Args:` + first_date: str + The date of the earliest possible broadcast you'd like returned. All common date + format should work (e.g. mm/dd/yy or yyyy-mm-dd) + last_date: str + The date of the latest possible broadcast you'd like returned. All common date + format should work (e.g. mm/dd/yy or yyyy-mm-dd) + status: str + 'draft', 'scheduled', or 'generated' + campaign_id: int + Specify to return broadcasts from a specific campaign + limit: int + Max rows you want returned + + `Returns:` + Parsons table with requested broadcasts + """ + + params = { + "start_time": _format_date(first_date), + "end_time": _format_date(last_date), + "campaign_id": campaign_id, + "status": status, + **self.default_params, + } + + return self._mc_get_request( + endpoint="broadcasts", + first_data_key="broadcasts", + second_data_key="broadcast", + params=params, + elements_to_unpack=["campaign"], + limit=limit, + ) + + def get_campaign_subscribers( + self, + campaign_id: int, + first_date: str = None, + last_date: str = None, + opt_in_path_id: int = None, + limit: int = None, + ): + """ + A function for getting subscribers of a specified campaign + + `Args:` + campaign_id: int + The campaign for which you'd like to get subscribers. You can get this from the url + of the campaign's page after select a campaign at + https://secure.mcommons.com/campaigns + first_date: str + The date of the earliest possible subscription returned. All common date + format should work (e.g. mm/dd/yy or yyyy-mm-dd) + last_date: str + The date of the latest possible subscription you'd like returned. All common date + format should work (e.g. mm/dd/yy or yyyy-mm-dd) + opt_in_path_id: int + Optional parameter to narrow results to on particular opt-in path. You can get this + from the url of the opt in paths page https://secure.mcommons.com/opt_in_paths + limit: int + Max rows you want returned + + `Returns:` + Parsons table with requested broadcasts + """ + + params = { + "campaign_id": campaign_id, + "from": _format_date(first_date), + "to": _format_date(last_date), + "opt_in_path_id": opt_in_path_id, + **self.default_params, + } + + return self._mc_get_request( + endpoint="campaign_subscribers", + first_data_key="subscriptions", + second_data_key="sub", + params=params, + limit=limit, + ) + + def get_profiles( + self, + phones: list = None, + first_date: str = None, + last_date: str = None, + include_custom_columns: bool = False, + include_subscriptions: bool = False, + limit: int = None, + ): + """ + A function for getting profiles, which are MobileCommons people records + + `Args:` + phones: list + A list of phone numbers including country codes for which you want profiles returned + MobileCommons claims to recognize most formats. + first_date: str + The date of the earliest possible subscription returned. All common date + format should work (e.g. mm/dd/yy or yyyy-mm-dd). + last_date: str + The date of the latest possible subscription you'd like returned. All common date + format should work (e.g. mm/dd/yy or yyyy-mm-dd). + include_custom_columns: bool + Optional parameter to that, if set to True, will return custom column values for + profiles as a list of dictionaries contained within a column. + include_subscriptions: bool + Optional parameter to that, if set to True, will return a list of campaigns a + given profile is subscribed to in a single column + limit: int + Max rows you want returned + + `Returns:` + Parsons table with requested broadcasts + """ + + custom_cols = "true" if include_custom_columns else "false" + subscriptions = "true" if include_subscriptions else "false" + + params = { + "phone_number": phones, + "from": _format_date(first_date), + "to": _format_date(last_date), + "include_custom_columns": custom_cols, + "include_subscriptions": subscriptions, + **self.default_params, + } + + return self._mc_get_request( + endpoint="profiles", + first_data_key="profiles", + second_data_key="profile", + elements_to_unpack=["source", "address"], + params=params, + limit=limit, + ) + + def create_profile( + self, + phone, + first_name=None, + last_name=None, + zip=None, + addressline1=None, + addressline2=None, + city=None, + state=None, + opt_in_path_id=None, + ): + """ + A function for creating a single MobileCommons profile + + `Args:` + phone: str + Phone number to assign profile + first_name: str + Profile first name + last_name: str + Profile last name + zip: str + Profile 5-digit postal code + addressline1: str + Profile address line 1 + addressline2: str + Profile address line 2 + city: str + Profile city + state: str + Profile state + opt_in_path_id: str + ID of the opt-in path to send new profile through. This will determine the welcome + text they receive. + + `Returns:` + ID of created profile + """ + + params = { + "phone_number": phone, + "first_name": first_name, + "last_name": last_name, + "postal_code": zip, + "street1": addressline1, + "street2": addressline2, + "city": city, + "state": state, + "opt_in_path_id": opt_in_path_id, + **self.default_params, + } + + response = self._mc_post_request("profile_update", params=params) + return response["profile"]["id"] diff --git a/requirements.txt b/requirements.txt index d1f416949b..2c171c56a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,6 +38,7 @@ surveygizmo==1.2.3 PyJWT==2.4.0 # Otherwise `import jwt` would refer to python-jwt package SQLAlchemy==1.3.23 requests_oauthlib==1.3.0 +bs4==0.0.1 # Testing Requirements requests-mock==1.5.2 diff --git a/test/test_mobilecommons/mobilecommons_responses.py b/test/test_mobilecommons/mobilecommons_responses.py new file mode 100644 index 0000000000..f9dd87c69e --- /dev/null +++ b/test/test_mobilecommons/mobilecommons_responses.py @@ -0,0 +1,250 @@ +class get_profiles_response: + status_code = 200 + text = """ + + + + James + Holden + 13073997994 + james.holden.boo@gmail.com + Undeliverable + 2022-06-29 17:28:24 UTC + 2023-03-14 21:46:58 UTC + 2022-09-16 01:07:50 UTC + Opt-out from API + +
+ 2430 Douglas Dr + + Salt Lake City + UT + + US +
+ + 41.692249 + -112.854408 + rooftop + Salt Lake City + UT + 84106 + US + + + UT-2 + UT-13 + UT-32 + No + + + + + + + + Cormac + + + + Martinez del Rio + + + + + + + + + + + + + + + + +""" + + +class get_broadcasts_response: + status_code = 200 + text = """ + + + + Test Round 2 + Test :) http://lil.ms/m9c2 + + TEST DH DD + + 2023-06-23 18:45:00 UTC + true + false + false + false + 2 + 2 + 0 + + + + + + 808sandheartbreaks@gmail.com + + + + Test 6/6 DD DH + Hey {{first_name}}!! On a scale of 1-5, how is your day going? +Reply STOP to quit. Msg&DataRatesMayApply + + TEST DH DD + + 2023-06-06 20:08:56 UTC + true + false + false + false + 2 + 2 + 0 + + + + + + how.to.eat.water.with.a.fork@gmail.com + + + + Broadcast 04/10/2023 01:34PM test + hi name, what's your email? Recurring Msgs. Reply STOP to quit, HELP for info. + Msg&DataRatesMayApply + + Dog King 2024 + + + false + false + false + false + + 0 + 0 + + + + + + + + + + + +""" + + +class post_profile_response: + status_code = 200 + text = """ + + + Hardcoremac + Del Sangre + 13073997990 + hardcore.smack@gmail.com + Undeliverable + 2022-06-29 17:28:24 UTC + 2023-03-14 21:46:58 UTC + 2022-09-16 01:07:50 UTC + Opt-out from API + +
+ 2430 Douglas Dr + + Salt Lake City + UT + + US +
+ + 41.692249 + -112.854408 + rooftop + Salt Lake City + UT + 84106 + US + + + UT-2 + UT-13 + UT-32 + No + + + + + + + + Cormac + + + + Martinez del Rio + + + + + + + + + + + + + + + +""" diff --git a/test/test_mobilecommons/test_mobilecommons.py b/test/test_mobilecommons/test_mobilecommons.py new file mode 100644 index 0000000000..dd80d8a562 --- /dev/null +++ b/test/test_mobilecommons/test_mobilecommons.py @@ -0,0 +1,127 @@ +import unittest +import requests_mock +from parsons.mobilecommons import MobileCommons +from parsons.etl import Table +from mobilecommons_responses import ( + get_profiles_response, + get_broadcasts_response, + post_profile_response, +) + + +MOBILECOMMONS_USERNAME = "MOBILECOMMONS_USERNAME" +MOBILECOMMONS_PASSWORD = "MOBILECOMMONS_PASSWORD" +DEFAULT_GET_PARAMS = {"page": 1, "limit": 1000} +DEFAULT_GET_ENDPOINT = "broadcasts" +DEFAULT_FIRST_KEY = "broadcasts" +DEFAULT_SECOND_KEY = "broadcast" +DEFAULT_POST_PARAMS = {"phone": 13073997994} +DEFAULT_POST_ENDPOINT = "profile_update" + + +class TestMobileCommons(unittest.TestCase): + @requests_mock.Mocker() + def setUp(self, m): + self.base_uri = "https://secure.mcommons.com/api/" + + self.mc = MobileCommons( + username=MOBILECOMMONS_USERNAME, password=MOBILECOMMONS_PASSWORD + ) + + @requests_mock.Mocker() + def test_parse_get_request(self, m): + m.get( + self.base_uri + DEFAULT_GET_ENDPOINT, + status_code=get_profiles_response.status_code, + text=get_profiles_response.text, + ) + parsed_get_request_text = self.mc._parse_get_request( + endpoint=DEFAULT_GET_ENDPOINT, params=DEFAULT_GET_PARAMS + ) + self.assertIsInstance(parsed_get_request_text, dict) + + @requests_mock.Mocker() + def test_mc_get_request(self, m): + m.get( + self.base_uri + DEFAULT_GET_ENDPOINT, + status_code=get_broadcasts_response.status_code, + text=get_broadcasts_response.text, + ) + parsed_get_response_text = self.mc._mc_get_request( + params=DEFAULT_GET_PARAMS, + endpoint=DEFAULT_GET_ENDPOINT, + first_data_key=DEFAULT_FIRST_KEY, + second_data_key=DEFAULT_SECOND_KEY, + ) + self.assertIsInstance( + parsed_get_response_text, + Table, + "MobileCommons.mc_get_request does output parsons table", + ) + + @requests_mock.Mocker() + def test_get_profiles(self, m): + m.get( + self.base_uri + "profiles", + status_code=get_profiles_response.status_code, + text=get_profiles_response.text, + ) + profiles = self.mc.get_profiles(limit=1000) + self.assertIsInstance( + profiles, + Table, + "MobileCommons.get_profiles method did not return a parsons Table", + ) + self.assertEqual( + profiles[0]["first_name"], + "James", + "MobileCommons.get_profiles method not returning a table structured" + "as expected", + ) + + @requests_mock.Mocker() + def test_get_broadcasts(self, m): + m.get( + self.base_uri + "broadcasts", + status_code=get_broadcasts_response.status_code, + text=get_broadcasts_response.text, + ) + broadcasts = self.mc.get_broadcasts(limit=1000) + self.assertIsInstance( + broadcasts, + Table, + "MobileCommons.get_broadcasts method did not return a parsons Table", + ) + self.assertEqual( + broadcasts[0]["id"], + "2543129", + "MobileCommons.get_broadcasts method not returning a table structured" + "as expected", + ) + + @requests_mock.Mocker() + def test_mc_post_request(self, m): + m.post(self.base_uri + "profile_update", text=post_profile_response.text) + response_dict = self.mc._mc_post_request( + endpoint=DEFAULT_POST_ENDPOINT, params=DEFAULT_POST_PARAMS + ) + self.assertIsInstance( + response_dict, + dict, + "MobileCommons.mc_post_request output not expected type dictionary", + ) + self.assertEqual( + response_dict["profile"]["id"], + "602169563", + "MobileCommons.mc_post_request output value not expected", + ) + + @requests_mock.Mocker() + def test_create_profile(self, m): + m.post(self.base_uri + "profile_update", text=post_profile_response.text) + profile_id = self.mc.create_profile(phone=13073997994) + self.assertEqual( + profile_id, + "602169563", + "MobileCommons.create_profile does not return expected profile_id", + )