From 217347887b0a0f49e0e77ae00bed4e4e0c52e51a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 03:03:55 -1000 Subject: [PATCH 1/4] Bump urllib3 from 1.26.5 to 1.26.17 (#901) Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.5 to 1.26.17. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.26.5...1.26.17) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 85ec07391..d1f416949 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ airtable-python-wrapper==0.13.0 google-cloud-storage==2.2.0 google-cloud-bigquery==3.4.0 docutils<0.18,>=0.14 -urllib3==1.26.5 +urllib3==1.26.17 simplejson==3.16.0 twilio==8.2.1 simple-salesforce==1.11.6 From 5b627fbde0b1438b9afba1d7d4e998923cfe4fd2 Mon Sep 17 00:00:00 2001 From: Cormac Martinez del Rio <66973815+cmdelrio@users.noreply.github.com> Date: Wed, 25 Oct 2023 11:39:29 -0600 Subject: [PATCH 2/4] Add MobileCommons Connector (#896) * mobilecommons class * Update __init__.py * get broadcasts * fix get broadcast request * Add mc_get_request method * Add annotation * Incorporate Daniel's suggestions and finish up get_broadcasts * A few more methods Need to figure out page_count issue * small fix * Remove page_count, use page record num instead * Add in page_count again Not all get responses include num param, but do include page_count. wft * Fix logging numbers * Add create_profile * Fix error message for post request * Start tests * Add some tests * Continue testing * Update test_mobilecommons.py * functionalize status_code check * break out parse_get_request function * fix test data * fix documentation typo * Add several tests * Update mobilecommons.py * Fix limit and pagination logic * debug unit testing * better commenting and logic * Documentation * Add MC to init file * Revert "Merge branch 'main' into cormac-mobilecommons-connector" This reverts commit cad250f44d94084e2c1a65b0662117c507b7e7cd, reversing changes made to 493e117fb4921994d4efc534e0a5c195c6170a60. * Revert "Add MC to init file" This reverts commit 493e117fb4921994d4efc534e0a5c195c6170a60. * Revert "Revert "Add MC to init file"" This reverts commit 8f87ec20115cc0ba6eff77fcf7d89fcbe6696504. * Revert "Revert "Merge branch 'main' into cormac-mobilecommons-connector"" This reverts commit 819005272152a6bebe0529fe0606f01102ee482c. * Fix init destruction * fix init yet again * Update testing docs with underscores * Lint * Lint tests * break up long responses * Fix more linting issues * Hopefully last linting issue * DGJKSNCHIVBN * Documentation fixes * Remove note to self * date format * remove random notes * Update test_mobilecommons.py --------- Co-authored-by: sharinetmc <128429303+sharinetmc@users.noreply.github.com> --- docs/index.rst | 1 + docs/mobilecommons.rst | 49 ++ parsons/__init__.py | 1 + parsons/mobilecommons/__init__.py | 3 + parsons/mobilecommons/mobilecommons.py | 428 ++++++++++++++++++ requirements.txt | 1 + .../mobilecommons_responses.py | 250 ++++++++++ test/test_mobilecommons/test_mobilecommons.py | 127 ++++++ 8 files changed, 860 insertions(+) create mode 100644 docs/mobilecommons.rst create mode 100644 parsons/mobilecommons/__init__.py create mode 100644 parsons/mobilecommons/mobilecommons.py create mode 100644 test/test_mobilecommons/mobilecommons_responses.py create mode 100644 test/test_mobilecommons/test_mobilecommons.py diff --git a/docs/index.rst b/docs/index.rst index 093849d15..aed7ccca3 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -205,6 +205,7 @@ Indices and tables google hustle mailchimp + mobilecommons mobilize_america nation_builder newmode diff --git a/docs/mobilecommons.rst b/docs/mobilecommons.rst new file mode 100644 index 000000000..ebdc4e123 --- /dev/null +++ b/docs/mobilecommons.rst @@ -0,0 +1,49 @@ +MobileCommons +========== + +******** +Overview +******** + +`MobileCommons `_ is a broadcast text messaging tool that helps orgranizations +mobilize supporters and fundraise by building opt-ed in audiences. You can read more about the product +`here `_. + +*********** +Quick Start +*********** + +To instantiate a class you must pass the username and password of a MobileCommons account as an argument +or store the username and password into environmental variables called ``MOBILECOMMONS_USERNAME`` and +``MOBILECOMMONS_PASSWORD``, respectively. If you MobileCommons account has access to various MobileCommons +companies (i.e. organizations), you'll need to specify which MobileCommons company you'd like to interact +with by specifying the Company ID in the ``company_id`` parameter. To find the Company ID, navigate to the +`Company and Users page `_. + +.. code-block:: python + from parsons import MobileCommons + + # Pass credentials via environmental variables for account has access to only one MobileCommons company + mc = MobileCommons() + + # Pass credentials via environmental variables for account has access to multiple MobileCommons companies + mc = MobileCommons(company_id='EXAMPLE78363BOCA483954419EB70986A68888') + + # Pass credentials via argument for account has access to only one MobileCommons company + mc = MobileCommons(username='octavia.b@scifi.net', password='badpassword123') + +Then you can call various endpoints: + +.. code-block:: python + # Return all MobileCommons subscribers in a table + subscribers = get_campaign_subscribers(campaign_id=1234567) + + # Create a new profile, return profile_id + new_profile=create_profile(phone=3073991987, first_name='Jane', last_name='Fonda') + + +*** +API +*** +.. autoclass :: parsons.MobileCommons + :inherited-members: \ No newline at end of file diff --git a/parsons/__init__.py b/parsons/__init__.py index 132d90db7..b8793ec64 100644 --- a/parsons/__init__.py +++ b/parsons/__init__.py @@ -65,6 +65,7 @@ ("parsons.google.google_sheets", "GoogleSheets"), ("parsons.hustle.hustle", "Hustle"), ("parsons.mailchimp.mailchimp", "Mailchimp"), + ("parsons.mobilecommons.mobilecommons", "MobileCommons"), ("parsons.mobilize_america.ma", "MobilizeAmerica"), ("parsons.nation_builder.nation_builder", "NationBuilder"), ("parsons.newmode.newmode", "Newmode"), diff --git a/parsons/mobilecommons/__init__.py b/parsons/mobilecommons/__init__.py new file mode 100644 index 000000000..377c3e631 --- /dev/null +++ b/parsons/mobilecommons/__init__.py @@ -0,0 +1,3 @@ +from parsons.mobilecommons.mobilecommons import MobileCommons + +__all__ = ["MobileCommons"] diff --git a/parsons/mobilecommons/mobilecommons.py b/parsons/mobilecommons/mobilecommons.py new file mode 100644 index 000000000..41acf3138 --- /dev/null +++ b/parsons/mobilecommons/mobilecommons.py @@ -0,0 +1,428 @@ +from parsons.utilities import check_env +from parsons.utilities.api_connector import APIConnector +from parsons.utilities.datetime import parse_date +from parsons import Table +from bs4 import BeautifulSoup +from requests import HTTPError +import xmltodict +import logging + +logger = logging.getLogger(__name__) + +MC_URI = "https://secure.mcommons.com/api/" +DATE_FMT = "%Y-%m-%d" + + +def _format_date(user_entered_date): + if user_entered_date: + formatted_date = parse_date(user_entered_date).strftime(DATE_FMT) + else: + formatted_date = None + return formatted_date + + +class MobileCommons: + """ + Instantiate the MobileCommons class. + + `Args:` + username: str + A valid email address connected to a MobileCommons account. Not required if + ``MOBILECOMMONS_USERNAME`` env variable is set. + password: str + Password associated with Zoom account. Not required if ``MOBILECOMMONS_PASSWORD`` + env variable set. + company_id: str + The company id of the MobileCommons organization to connect to. Not required if + username and password are for an account associated with only one MobileCommons + organization. + """ + + def __init__(self, username=None, password=None, company_id=None): + self.username = check_env.check("MOBILECOMMONS_USERNAME", username) + self.password = check_env.check("MOBILECOMMONS_PASSWORD", password) + self.default_params = {"company": company_id} if company_id else {} + self.client = APIConnector(uri=MC_URI, auth=(self.username, self.password)) + + def _mc_get_request( + self, + endpoint, + first_data_key, + second_data_key, + params, + elements_to_unpack=None, + limit=None, + ): + """ + A function for GET requests that handles MobileCommons xml responses and pagination + + `Args:` + endpoint: str + The endpoint, which will be appended to the base URL for each request + first_data_key: str + The first key used to extract the desired data from the response dictionary derived + from the xml response. E.g. 'broadcasts' + second_data_key: str + The second key used to extract the desired data from the response dictionary derived + from the xml response. The value of this key is a list of values. E.g. 'broadcast' + params: dict + Parameters to be passed into GET request + elements_to_unpack: list + A list of elements that contain dictionaries to be unpacked into new columns in the + final table + limit: int + The maximum number of rows to return + `Returns:` + Parsons table with requested data + """ + + # Create a table to compile results from different pages in + final_table = Table() + # Max page_limit is 1000 for MC + page_limit = min((limit or 1000), 1000) + + # Set get request params + params = {"limit": page_limit, **self.default_params, **params} + + logger.info( + f"Working on fetching first {page_limit} rows. This can take a long time." + ) + + # Make get call and parse XML into list of dicts + page = 1 + response_dict = self._parse_get_request(endpoint=endpoint, params=params) + + # If there's only one row, then it is returned as a dict, otherwise as a list + data = response_dict["response"][first_data_key][second_data_key] + if isinstance(data, dict): + data = [data] + + # Convert to parsons table + response_table = Table(data) + + # empty_page used for pagination below + if response_table.num_rows > 0: + empty_page = False + else: + raise ValueError("There are no records for specified resource") + + # Unpack any specified elements + if elements_to_unpack: + for col in elements_to_unpack: + response_table.unpack_dict(col) + + # Append to final table + final_table.concat(response_table) + final_table.materialize() + + # Now must paginate to get more records + # MC GET responses sometimes includes a page_count parameter to indicate how many pages + # are available. Other times, the response includes a 'num' parameter that, when + # you reach an empty page, equals 0. In the first scenario we know to stop paginating when + # we reach the last page. In the second, we know to stop paginating when we encounter + # and empty page + + # First scenario + try: + avail_pages = int(response_dict["response"][first_data_key]["page_count"]) + total_records = avail_pages * page_limit + page_indicator = "page_count" + + # Second scenario + except KeyError: + response_dict["response"][first_data_key]["num"] + page_indicator = "num" + # If page_count is not available, we cannot calculate total_records and will paginate + # until we hit user defined limit or an empty page + total_records = float("inf") + + # Go fetch other pages of data + while final_table.num_rows < (limit or total_records) and not empty_page: + page += 1 + page_params = {"page": str(page), **params} + logger.info( + f"Fetching rows {(page - 1) * page_limit + 1} - {(page) * page_limit} " + f"of {limit}" + ) + # Send get request + response_dict = self._parse_get_request( + endpoint=endpoint, params=page_params + ) + # Check to see if page was empty if num parameter is available + if page_indicator == "num": + empty_page = int(response_dict["response"][first_data_key]["num"]) > 0 + + if not empty_page: + # Extract data + response_table = Table( + response_dict["response"][first_data_key][second_data_key] + ) + # Append to final table + final_table.concat(response_table) + final_table.materialize() + + return Table(final_table[:limit]) + + def _check_response_status_code(self, response): + """ + A helper function that checks the status code of a response and raises an error if the + response code is not 200 + + `Args:` + response: requests package response object + """ + if response.status_code != 200: + error = f"Response Code {str(response.status_code)}" + error_html = BeautifulSoup(response.text, features="html.parser") + error += "\n" + error_html.h4.next + error += "\n" + error_html.p.next + raise HTTPError(error) + + def _parse_get_request(self, endpoint, params): + """ + A helper function that sends a get request to MobileCommons and then parses XML responses in + order to return the response as a dictionary + + `Args:` + endpoint: str + The endpoint, which will be appended to the base URL for each request + params: dict + Parameters to be passed into GET request + `Returns:` + xml response parsed into list or dictionary + """ + response = self.client.request(endpoint, "GET", params=params) + + # If there's an error with initial response, raise error + self._check_response_status_code(response) + + # If good response, compile data into final_table + # Parse xml to nested dictionary and load to parsons table + response_dict = xmltodict.parse( + response.text, attr_prefix="", cdata_key="", dict_constructor=dict + ) + return response_dict + + def _mc_post_request(self, endpoint, params): + """ + A function for POST requests that handles MobileCommons xml responses + + `Args:` + endpoint: str + The endpoint, which will be appended to the base URL for each request + params: dict + Parameters to be passed into GET request + `Returns:` + xml response parsed into list or dictionary + """ + + response = self.client.request(endpoint, "POST", params=params) + + response_dict = xmltodict.parse( + response.text, attr_prefix="", cdata_key="", dict_constructor=dict + ) + if response_dict["response"]["success"] == "true": + return response_dict["response"] + else: + raise HTTPError(response_dict["response"]["error"]) + + def get_broadcasts( + self, first_date=None, last_date=None, status=None, campaign_id=None, limit=None + ): + """ + A function for get broadcasts + + `Args:` + first_date: str + The date of the earliest possible broadcast you'd like returned. All common date + format should work (e.g. mm/dd/yy or yyyy-mm-dd) + last_date: str + The date of the latest possible broadcast you'd like returned. All common date + format should work (e.g. mm/dd/yy or yyyy-mm-dd) + status: str + 'draft', 'scheduled', or 'generated' + campaign_id: int + Specify to return broadcasts from a specific campaign + limit: int + Max rows you want returned + + `Returns:` + Parsons table with requested broadcasts + """ + + params = { + "start_time": _format_date(first_date), + "end_time": _format_date(last_date), + "campaign_id": campaign_id, + "status": status, + **self.default_params, + } + + return self._mc_get_request( + endpoint="broadcasts", + first_data_key="broadcasts", + second_data_key="broadcast", + params=params, + elements_to_unpack=["campaign"], + limit=limit, + ) + + def get_campaign_subscribers( + self, + campaign_id: int, + first_date: str = None, + last_date: str = None, + opt_in_path_id: int = None, + limit: int = None, + ): + """ + A function for getting subscribers of a specified campaign + + `Args:` + campaign_id: int + The campaign for which you'd like to get subscribers. You can get this from the url + of the campaign's page after select a campaign at + https://secure.mcommons.com/campaigns + first_date: str + The date of the earliest possible subscription returned. All common date + format should work (e.g. mm/dd/yy or yyyy-mm-dd) + last_date: str + The date of the latest possible subscription you'd like returned. All common date + format should work (e.g. mm/dd/yy or yyyy-mm-dd) + opt_in_path_id: int + Optional parameter to narrow results to on particular opt-in path. You can get this + from the url of the opt in paths page https://secure.mcommons.com/opt_in_paths + limit: int + Max rows you want returned + + `Returns:` + Parsons table with requested broadcasts + """ + + params = { + "campaign_id": campaign_id, + "from": _format_date(first_date), + "to": _format_date(last_date), + "opt_in_path_id": opt_in_path_id, + **self.default_params, + } + + return self._mc_get_request( + endpoint="campaign_subscribers", + first_data_key="subscriptions", + second_data_key="sub", + params=params, + limit=limit, + ) + + def get_profiles( + self, + phones: list = None, + first_date: str = None, + last_date: str = None, + include_custom_columns: bool = False, + include_subscriptions: bool = False, + limit: int = None, + ): + """ + A function for getting profiles, which are MobileCommons people records + + `Args:` + phones: list + A list of phone numbers including country codes for which you want profiles returned + MobileCommons claims to recognize most formats. + first_date: str + The date of the earliest possible subscription returned. All common date + format should work (e.g. mm/dd/yy or yyyy-mm-dd). + last_date: str + The date of the latest possible subscription you'd like returned. All common date + format should work (e.g. mm/dd/yy or yyyy-mm-dd). + include_custom_columns: bool + Optional parameter to that, if set to True, will return custom column values for + profiles as a list of dictionaries contained within a column. + include_subscriptions: bool + Optional parameter to that, if set to True, will return a list of campaigns a + given profile is subscribed to in a single column + limit: int + Max rows you want returned + + `Returns:` + Parsons table with requested broadcasts + """ + + custom_cols = "true" if include_custom_columns else "false" + subscriptions = "true" if include_subscriptions else "false" + + params = { + "phone_number": phones, + "from": _format_date(first_date), + "to": _format_date(last_date), + "include_custom_columns": custom_cols, + "include_subscriptions": subscriptions, + **self.default_params, + } + + return self._mc_get_request( + endpoint="profiles", + first_data_key="profiles", + second_data_key="profile", + elements_to_unpack=["source", "address"], + params=params, + limit=limit, + ) + + def create_profile( + self, + phone, + first_name=None, + last_name=None, + zip=None, + addressline1=None, + addressline2=None, + city=None, + state=None, + opt_in_path_id=None, + ): + """ + A function for creating a single MobileCommons profile + + `Args:` + phone: str + Phone number to assign profile + first_name: str + Profile first name + last_name: str + Profile last name + zip: str + Profile 5-digit postal code + addressline1: str + Profile address line 1 + addressline2: str + Profile address line 2 + city: str + Profile city + state: str + Profile state + opt_in_path_id: str + ID of the opt-in path to send new profile through. This will determine the welcome + text they receive. + + `Returns:` + ID of created profile + """ + + params = { + "phone_number": phone, + "first_name": first_name, + "last_name": last_name, + "postal_code": zip, + "street1": addressline1, + "street2": addressline2, + "city": city, + "state": state, + "opt_in_path_id": opt_in_path_id, + **self.default_params, + } + + response = self._mc_post_request("profile_update", params=params) + return response["profile"]["id"] diff --git a/requirements.txt b/requirements.txt index d1f416949..2c171c56a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -38,6 +38,7 @@ surveygizmo==1.2.3 PyJWT==2.4.0 # Otherwise `import jwt` would refer to python-jwt package SQLAlchemy==1.3.23 requests_oauthlib==1.3.0 +bs4==0.0.1 # Testing Requirements requests-mock==1.5.2 diff --git a/test/test_mobilecommons/mobilecommons_responses.py b/test/test_mobilecommons/mobilecommons_responses.py new file mode 100644 index 000000000..f9dd87c69 --- /dev/null +++ b/test/test_mobilecommons/mobilecommons_responses.py @@ -0,0 +1,250 @@ +class get_profiles_response: + status_code = 200 + text = """ + + + + James + Holden + 13073997994 + james.holden.boo@gmail.com + Undeliverable + 2022-06-29 17:28:24 UTC + 2023-03-14 21:46:58 UTC + 2022-09-16 01:07:50 UTC + Opt-out from API + +
+ 2430 Douglas Dr + + Salt Lake City + UT + + US +
+ + 41.692249 + -112.854408 + rooftop + Salt Lake City + UT + 84106 + US + + + UT-2 + UT-13 + UT-32 + No + + + + + + + + Cormac + + + + Martinez del Rio + + + + + + + + + + + + + + + + +""" + + +class get_broadcasts_response: + status_code = 200 + text = """ + + + + Test Round 2 + Test :) http://lil.ms/m9c2 + + TEST DH DD + + 2023-06-23 18:45:00 UTC + true + false + false + false + 2 + 2 + 0 + + + + + + 808sandheartbreaks@gmail.com + + + + Test 6/6 DD DH + Hey {{first_name}}!! On a scale of 1-5, how is your day going? +Reply STOP to quit. Msg&DataRatesMayApply + + TEST DH DD + + 2023-06-06 20:08:56 UTC + true + false + false + false + 2 + 2 + 0 + + + + + + how.to.eat.water.with.a.fork@gmail.com + + + + Broadcast 04/10/2023 01:34PM test + hi name, what's your email? Recurring Msgs. Reply STOP to quit, HELP for info. + Msg&DataRatesMayApply + + Dog King 2024 + + + false + false + false + false + + 0 + 0 + + + + + + + + + + + +""" + + +class post_profile_response: + status_code = 200 + text = """ + + + Hardcoremac + Del Sangre + 13073997990 + hardcore.smack@gmail.com + Undeliverable + 2022-06-29 17:28:24 UTC + 2023-03-14 21:46:58 UTC + 2022-09-16 01:07:50 UTC + Opt-out from API + +
+ 2430 Douglas Dr + + Salt Lake City + UT + + US +
+ + 41.692249 + -112.854408 + rooftop + Salt Lake City + UT + 84106 + US + + + UT-2 + UT-13 + UT-32 + No + + + + + + + + Cormac + + + + Martinez del Rio + + + + + + + + + + + + + + + +""" diff --git a/test/test_mobilecommons/test_mobilecommons.py b/test/test_mobilecommons/test_mobilecommons.py new file mode 100644 index 000000000..dd80d8a56 --- /dev/null +++ b/test/test_mobilecommons/test_mobilecommons.py @@ -0,0 +1,127 @@ +import unittest +import requests_mock +from parsons.mobilecommons import MobileCommons +from parsons.etl import Table +from mobilecommons_responses import ( + get_profiles_response, + get_broadcasts_response, + post_profile_response, +) + + +MOBILECOMMONS_USERNAME = "MOBILECOMMONS_USERNAME" +MOBILECOMMONS_PASSWORD = "MOBILECOMMONS_PASSWORD" +DEFAULT_GET_PARAMS = {"page": 1, "limit": 1000} +DEFAULT_GET_ENDPOINT = "broadcasts" +DEFAULT_FIRST_KEY = "broadcasts" +DEFAULT_SECOND_KEY = "broadcast" +DEFAULT_POST_PARAMS = {"phone": 13073997994} +DEFAULT_POST_ENDPOINT = "profile_update" + + +class TestMobileCommons(unittest.TestCase): + @requests_mock.Mocker() + def setUp(self, m): + self.base_uri = "https://secure.mcommons.com/api/" + + self.mc = MobileCommons( + username=MOBILECOMMONS_USERNAME, password=MOBILECOMMONS_PASSWORD + ) + + @requests_mock.Mocker() + def test_parse_get_request(self, m): + m.get( + self.base_uri + DEFAULT_GET_ENDPOINT, + status_code=get_profiles_response.status_code, + text=get_profiles_response.text, + ) + parsed_get_request_text = self.mc._parse_get_request( + endpoint=DEFAULT_GET_ENDPOINT, params=DEFAULT_GET_PARAMS + ) + self.assertIsInstance(parsed_get_request_text, dict) + + @requests_mock.Mocker() + def test_mc_get_request(self, m): + m.get( + self.base_uri + DEFAULT_GET_ENDPOINT, + status_code=get_broadcasts_response.status_code, + text=get_broadcasts_response.text, + ) + parsed_get_response_text = self.mc._mc_get_request( + params=DEFAULT_GET_PARAMS, + endpoint=DEFAULT_GET_ENDPOINT, + first_data_key=DEFAULT_FIRST_KEY, + second_data_key=DEFAULT_SECOND_KEY, + ) + self.assertIsInstance( + parsed_get_response_text, + Table, + "MobileCommons.mc_get_request does output parsons table", + ) + + @requests_mock.Mocker() + def test_get_profiles(self, m): + m.get( + self.base_uri + "profiles", + status_code=get_profiles_response.status_code, + text=get_profiles_response.text, + ) + profiles = self.mc.get_profiles(limit=1000) + self.assertIsInstance( + profiles, + Table, + "MobileCommons.get_profiles method did not return a parsons Table", + ) + self.assertEqual( + profiles[0]["first_name"], + "James", + "MobileCommons.get_profiles method not returning a table structured" + "as expected", + ) + + @requests_mock.Mocker() + def test_get_broadcasts(self, m): + m.get( + self.base_uri + "broadcasts", + status_code=get_broadcasts_response.status_code, + text=get_broadcasts_response.text, + ) + broadcasts = self.mc.get_broadcasts(limit=1000) + self.assertIsInstance( + broadcasts, + Table, + "MobileCommons.get_broadcasts method did not return a parsons Table", + ) + self.assertEqual( + broadcasts[0]["id"], + "2543129", + "MobileCommons.get_broadcasts method not returning a table structured" + "as expected", + ) + + @requests_mock.Mocker() + def test_mc_post_request(self, m): + m.post(self.base_uri + "profile_update", text=post_profile_response.text) + response_dict = self.mc._mc_post_request( + endpoint=DEFAULT_POST_ENDPOINT, params=DEFAULT_POST_PARAMS + ) + self.assertIsInstance( + response_dict, + dict, + "MobileCommons.mc_post_request output not expected type dictionary", + ) + self.assertEqual( + response_dict["profile"]["id"], + "602169563", + "MobileCommons.mc_post_request output value not expected", + ) + + @requests_mock.Mocker() + def test_create_profile(self, m): + m.post(self.base_uri + "profile_update", text=post_profile_response.text) + profile_id = self.mc.create_profile(phone=13073997994) + self.assertEqual( + profile_id, + "602169563", + "MobileCommons.create_profile does not return expected profile_id", + ) From 1e799262a140f3869f3fcd2c5936a7575417ae31 Mon Sep 17 00:00:00 2001 From: Angela Gloyna Date: Wed, 25 Oct 2023 13:02:21 -0500 Subject: [PATCH 3/4] #741 : Deprecate Slack chat.postMessage `as_user` argument and allow for new authorship arguments (#891) * remove the argument and add a warning that the usage is deprecated * remove usage of as_user from sample code * add in the user customization arguments in lieu of the deprecated as_user argument * add comment regarding the permissions required to use these arguments * use kwargs * surface the whole response * allow usage of the deprecated argument but surface the failed response better * add to retry * delete test file * fix linting * formatting to fix tests * fix if style * add warning for using thread_ts * move the documentation to the optional arguments --- parsons/notifications/slack.py | 47 +++++++++++++------ .../civis_job_status_slack_alert.py | 2 +- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/parsons/notifications/slack.py b/parsons/notifications/slack.py index 5b36aead4..ced3c20fa 100644 --- a/parsons/notifications/slack.py +++ b/parsons/notifications/slack.py @@ -1,5 +1,6 @@ import os import time +import warnings from parsons.etl.table import Table from parsons.utilities.check_env import check @@ -12,9 +13,7 @@ class Slack(object): def __init__(self, api_key=None): - if api_key is None: - try: self.api_key = os.environ["SLACK_API_TOKEN"] @@ -25,7 +24,6 @@ def __init__(self, api_key=None): ) else: - self.api_key = api_key self.client = SlackClient(self.api_key) @@ -130,7 +128,7 @@ def message(cls, channel, text, webhook=None, parent_message_id=None): payload["thread_ts"] = parent_message_id return requests.post(webhook, json=payload) - def message_channel(self, channel, text, as_user=False, parent_message_id=None): + def message_channel(self, channel, text, parent_message_id=None, **kwargs): """ Send a message to a Slack channel @@ -140,35 +138,56 @@ def message_channel(self, channel, text, as_user=False, parent_message_id=None): an `im` (aka 1-1 message). text: str Text of the message to send. - as_user: str - Pass true to post the message as the authenticated user, - instead of as a bot. Defaults to false. See - https://api.slack.com/methods/chat.postMessage#authorship for - more information about Slack authorship. parent_message_id: str The `ts` value of the parent message. If used, this will thread the message. + **kwargs: kwargs + as_user: str + This is a deprecated argument. Use optional username, icon_url, and icon_emoji + args to customize the attributes of the user posting the message. + See https://api.slack.com/methods/chat.postMessage#legacy_authorship for + more information about legacy authorship + Additional arguments for chat.postMessage API call. See documentation + ` for more info. + + `Returns:` `dict`: A response json """ + + if "as_user" in kwargs: + warnings.warn( + "as_user is a deprecated argument on message_channel().", + DeprecationWarning, + stacklevel=2, + ) + if "thread_ts" in kwargs: + warnings.warn( + "thread_ts argument on message_channel() will be ignored. Use parent_message_id.", + Warning, + stacklevel=2, + ) + kwargs.pop("thread_ts", None) + resp = self.client.api_call( "chat.postMessage", channel=channel, text=text, - as_user=as_user, thread_ts=parent_message_id, + **kwargs, ) if not resp["ok"]: - if resp["error"] == "ratelimited": time.sleep(int(resp["headers"]["Retry-After"])) resp = self.client.api_call( - "chat.postMessage", channel=channel, text=text, as_user=as_user + "chat.postMessage", channel=channel, text=text, **kwargs ) - raise SlackClientError(resp["error"]) + resp.pop("headers", None) + + raise SlackClientError(resp) return resp @@ -221,7 +240,6 @@ def upload_file( ) if not resp["ok"]: - if resp["error"] == "ratelimited": time.sleep(int(resp["headers"]["Retry-After"])) @@ -250,7 +268,6 @@ def _paginate_request(self, endpoint, collection, **kwargs): resp = self.client.api_call(endpoint, cursor=cursor, limit=LIMIT, **kwargs) if not resp["ok"]: - if resp["error"] == "ratelimited": time.sleep(int(resp["headers"]["Retry-After"])) continue diff --git a/useful_resources/sample_code/civis_job_status_slack_alert.py b/useful_resources/sample_code/civis_job_status_slack_alert.py index 16504d838..6c0885a49 100644 --- a/useful_resources/sample_code/civis_job_status_slack_alert.py +++ b/useful_resources/sample_code/civis_job_status_slack_alert.py @@ -140,7 +140,7 @@ def main(): message = f"*{project_name} Status*\n{line_items}" logger.info(f"Posting message to Slack channel {SLACK_CHANNEL}") # Post message - slack.message_channel(SLACK_CHANNEL, message, as_user=True) + slack.message_channel(SLACK_CHANNEL, message) logger.info("Slack message posted") From 2f15a5389a10201f62ab91e21408a0df8dc92eac Mon Sep 17 00:00:00 2001 From: Angela Gloyna Date: Wed, 25 Oct 2023 13:05:54 -0500 Subject: [PATCH 4/4] #816 Airtable.get_records() fields argument can be either str or list (#892) * all fields to be a str object * remove newline --- parsons/airtable/airtable.py | 2 ++ test/test_airtable/test_airtable.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/parsons/airtable/airtable.py b/parsons/airtable/airtable.py index 3465d4cd2..b9fa866d8 100644 --- a/parsons/airtable/airtable.py +++ b/parsons/airtable/airtable.py @@ -94,6 +94,8 @@ def get_records( See :ref:`parsons-table` for output options. """ + if isinstance(fields, str): + fields = [fields] # Raises an error if sort is None type. Thus, only adding if populated. kwargs = { "fields": fields, diff --git a/test/test_airtable/test_airtable.py b/test/test_airtable/test_airtable.py index e85db472b..ff326c43b 100644 --- a/test/test_airtable/test_airtable.py +++ b/test/test_airtable/test_airtable.py @@ -99,6 +99,16 @@ def test_get_records_with_explicit_headers(self, m): assert airtable_res.columns == ["id", "createdTime", "Name", "SecondColumn"] + @requests_mock.Mocker() + def test_get_records_with_single_field(self, m): + m.get(self.base_uri, json=records_response_with_more_columns) + + fields = "Name" + + airtable_res = self.at.get_records(fields, sample_size=1) + + assert airtable_res.columns == ["id", "createdTime", "Name"] + @requests_mock.Mocker() def test_insert_record(self, m):