From ff313b84d37e49002fcdac2cc14e3b68fb0e88ad Mon Sep 17 00:00:00 2001
From: Kasia Hinkson <52927664+KasiaHinkson@users.noreply.github.com>
Date: Mon, 12 Aug 2024 11:05:14 -0500
Subject: [PATCH] Kasiah/community dot com connector (#1112)
* build out community.com connector
* fix url
* add to init
* get not post
* small edits to fix connector
* Update community.py
* change some names and add test
* fix param name
* fix url
* wrong url again
* maybe this will magically work
* add a log for test
* another log
* another log
* trying again
* will this pass
* more stuff
* change test response
* content not json
* another log
* more logs
* different test values
* shorten the test string
* values don't matter
* cleanup
* docs
* fix link
* remove commented code
* don't need these
* remove another thing
* linting
* incorporate sharine's fix with an extra try except
* update links
* links
---
parsons/__init__.py | 1 +
parsons/community/__init__.py | 3 +
parsons/community/community.py | 101 +++++++++++++++++++++++++
parsons/google/google_cloud_storage.py | 5 +-
parsons/utilities/api_connector.py | 11 ++-
test/test_community/test_community.py | 40 ++++++++++
6 files changed, 157 insertions(+), 4 deletions(-)
create mode 100644 parsons/community/__init__.py
create mode 100644 parsons/community/community.py
create mode 100644 test/test_community/test_community.py
diff --git a/parsons/__init__.py b/parsons/__init__.py
index ebc743e37f..1219713152 100644
--- a/parsons/__init__.py
+++ b/parsons/__init__.py
@@ -45,6 +45,7 @@
("parsons.catalist.catalist", "CatalistMatch"),
("parsons.census.census", "Census"),
("parsons.civis.civisclient", "CivisClient"),
+ ("parsons.community.community", "Community"),
("parsons.controlshift.controlshift", "Controlshift"),
("parsons.copper.copper", "Copper"),
("parsons.crowdtangle.crowdtangle", "CrowdTangle"),
diff --git a/parsons/community/__init__.py b/parsons/community/__init__.py
new file mode 100644
index 0000000000..c9e7defc6e
--- /dev/null
+++ b/parsons/community/__init__.py
@@ -0,0 +1,3 @@
+from parsons.community.community import Community
+
+__all__ = ["Community"]
diff --git a/parsons/community/community.py b/parsons/community/community.py
new file mode 100644
index 0000000000..4fd68cc982
--- /dev/null
+++ b/parsons/community/community.py
@@ -0,0 +1,101 @@
+import logging
+from parsons.utilities.api_connector import APIConnector
+from parsons.utilities import check_env
+from parsons.etl import Table
+
+logger = logging.getLogger(__name__)
+
+COMMUNITY_API_ENDPOINT = "https://dl.community.com/download/v1/files/"
+
+
+class Community(object):
+ """
+ Instantiate class.
+
+ `Args:`
+ community_client_id: str
+ The Community provided Client ID. Not required if ``COMMUNITY_CLIENT_ID`` env
+ variable set.
+ community_access_token: str
+ The Community provided access token. Not required if ``COMMUNITY_ACCESS_TOKEN`` env
+ variable set.
+ community_uri: str
+ The URI to access the API. Not required, default is
+ ``_. You can set an ``COMMUNITY_URL`` env
+ variable or use this URI parameter if a different endpoint is necessary.
+
+ `API Documentation `_
+ """
+
+ def __init__(self, community_client_id=None, community_access_token=None, community_url=None):
+ self.community_client_id = check_env.check("community_client_id", community_client_id)
+ self.community_access_token = check_env.check(
+ "community_access_token", community_access_token
+ )
+ self.uri = (
+ check_env.check("COMMUNITY_URL", community_url, optional=True)
+ or f"{COMMUNITY_API_ENDPOINT}/{community_client_id}/"
+ )
+ self.headers = {
+ "Authorization": f"Bearer {self.community_access_token}",
+ }
+ self.client = APIConnector(
+ self.uri,
+ headers=self.headers,
+ )
+
+ def get_request(self, filename):
+ """
+ GET request to Community.com API to get the CSV data.
+
+ `Args:`
+ filename: str
+ Data filename you are requesting.
+ Options:
+ 'campaigns': Campaign Performance data
+ 'outbound_message_type_usage`: Message Segment Usage data
+ 'campaign_links': Campaign Link Performance data
+ 'members': Member Details data
+ 'member_state_changes': Member Subscription Status data
+ 'custom_member_data': Custom Member Data
+ 'communities': Communities data
+ 'member_communities': Member Communities data
+
+ `Returns:`
+ Response of GET request; a successful response returns the CSV formatted data
+ """
+
+ logger.info(f"Requesting {filename}")
+ url = (
+ f"{filename}.csv.gz"
+ if filename != "outbound_message_type_usage"
+ else f"{filename}.csv.gz/segment-based-subscription"
+ )
+ response = self.client.get_request(url=url, return_format="content")
+ return response
+
+ def get_data_export(self, filename):
+ """
+ Get specified data from Community.com API as Parsons table.
+
+ `Args:`
+ filename: str
+ Data filename you are requesting.
+ Options:
+ 'campaigns': Campaign Performance data
+ 'outbound_message_type_usage`: Message Segment Usage data
+ 'campaign_links': Campaign Link Performance data
+ 'members': Member Details data
+ 'member_state_changes': Member Subscription Status data
+ 'custom_member_data': Custom Member Data
+ 'communities': Communities data
+ 'member_communities': Member Communities data
+
+ `Returns:`
+ Contents of the generated contribution CSV as a Parsons table.
+ """
+
+ get_request_response = self.get_request(filename=filename)
+ response_string = get_request_response.decode("utf-8")
+ table = Table.from_csv_string(response_string)
+ return table
diff --git a/parsons/google/google_cloud_storage.py b/parsons/google/google_cloud_storage.py
index 662ac58cad..e92d47572e 100644
--- a/parsons/google/google_cloud_storage.py
+++ b/parsons/google/google_cloud_storage.py
@@ -323,7 +323,10 @@ def upload_table(self, table, bucket_name, blob_name, data_type="csv", default_a
# CSVView. Once any transformations are made, the Table.table
# becomes a different petl class
if isinstance(table.table, petl.io.csv_py3.CSVView):
- local_file = table.table.source.filename
+ try:
+ local_file = table.table.source.filename
+ except AttributeError:
+ local_file = table.to_csv()
else:
local_file = table.to_csv()
content_type = "text/csv"
diff --git a/parsons/utilities/api_connector.py b/parsons/utilities/api_connector.py
index dfced6487b..39564cc20b 100644
--- a/parsons/utilities/api_connector.py
+++ b/parsons/utilities/api_connector.py
@@ -81,7 +81,7 @@ def request(self, url, req_type, json=None, data=None, params=None):
params=params,
)
- def get_request(self, url, params=None):
+ def get_request(self, url, params=None, return_format="json"):
"""
Make a GET request.
@@ -96,9 +96,14 @@ def get_request(self, url, params=None):
r = self.request(url, "GET", params=params)
self.validate_response(r)
- logger.debug(r.json())
- return r.json()
+ if return_format == "json":
+ logger.debug(r.json())
+ return r.json()
+ elif return_format == "content":
+ return r.content
+ else:
+ raise RuntimeError(f"{return_format} is not a valid format, change to json or content")
def post_request(
self, url, params=None, data=None, json=None, success_codes=[200, 201, 202, 204]
diff --git a/test/test_community/test_community.py b/test/test_community/test_community.py
new file mode 100644
index 0000000000..71f2900941
--- /dev/null
+++ b/test/test_community/test_community.py
@@ -0,0 +1,40 @@
+import unittest
+import requests_mock
+from parsons import Community
+
+
+TEST_CLIENT_ID = "someuuid"
+TEST_CLIENT_TOKEN = "somesecret"
+
+TEST_FILENAME = "campaigns"
+TEST_URI = f"https://faketestingurl.com/{TEST_CLIENT_ID}"
+TEST_FULL_URL = f"{TEST_URI}/{TEST_FILENAME}.csv.gz"
+
+TEST_GET_RESPONSE_CSV_STRING = b'"CAMPAIGN_ID","LEADER_ID"\n"0288","6e83b"\n'
+
+TEST_EXPECTED_COLUMNS = [
+ "CAMPAIGN_ID",
+ "LEADER_ID",
+]
+
+
+class TestCommunity(unittest.TestCase):
+ @requests_mock.Mocker()
+ def setUp(self, m):
+ self.com = Community(TEST_CLIENT_ID, TEST_CLIENT_TOKEN, TEST_URI)
+
+ @requests_mock.Mocker()
+ def test_successful_get_request(self, m):
+ m.get(TEST_FULL_URL, content=TEST_GET_RESPONSE_CSV_STRING)
+
+ assert self.com.get_request(filename=TEST_FILENAME) == TEST_GET_RESPONSE_CSV_STRING
+
+ # test get resource
+ @requests_mock.Mocker()
+ def test_successful_get_data_export(self, m):
+ m.get(TEST_FULL_URL, content=TEST_GET_RESPONSE_CSV_STRING)
+
+ table = self.com.get_data_export(
+ TEST_FILENAME,
+ )
+ assert TEST_EXPECTED_COLUMNS == table.columns