diff --git a/hq_superset/api.py b/hq_superset/api.py index e5e5254..8f7b73b 100644 --- a/hq_superset/api.py +++ b/hq_superset/api.py @@ -11,9 +11,9 @@ json_success, ) -from .models import DataSetChange -from .oauth2_server import authorization, require_oauth -from .tasks import process_dataset_change +from hq_superset.models import DataSetChange +from hq_superset.oauth2_server import authorization, require_oauth +from hq_superset.tasks import process_dataset_change class OAuth(BaseApi): diff --git a/hq_superset/const.py b/hq_superset/const.py index e94d85d..54fd77b 100644 --- a/hq_superset/const.py +++ b/hq_superset/const.py @@ -48,3 +48,8 @@ "Datasets": [MENU_ACCESS_PERMISSION], "ExploreFormDataRestApi": [CAN_READ_PERMISSION] } + +DOMAIN_PREFIX = "hqdomain_" +SESSION_USER_DOMAINS_KEY = "user_hq_domains" +SESSION_OAUTH_RESPONSE_KEY = "oauth_response" +SESSION_DOMAIN_ROLE_LAST_SYNCED_AT = "domain_role_last_synced_at" diff --git a/hq_superset/hq_domain.py b/hq_superset/hq_domain.py index 0229124..83f6d81 100644 --- a/hq_superset/hq_domain.py +++ b/hq_superset/hq_domain.py @@ -1,10 +1,31 @@ -from flask import flash, g, redirect, request, session, url_for +from datetime import timedelta -from .utils import SESSION_USER_DOMAINS_KEY +import superset +from flask import current_app, flash, g, redirect, request, session, url_for +from superset.config import USER_DOMAIN_ROLE_EXPIRY +from superset.extensions import cache_manager + +from hq_superset.const import ( + SESSION_DOMAIN_ROLE_LAST_SYNCED_AT, + SESSION_USER_DOMAINS_KEY, +) +from hq_superset.utils import DomainSyncUtil, datetime_utcnow def before_request_hook(): - return ensure_domain_selected() + """ + Call all hooks functions set in sequence and + if any hook returns a response, + break the chain and return that response + """ + hooks = [ + ensure_domain_selected, + sync_user_domain_role + ] + for _function in hooks: + response = _function() + if response: + return response def after_request_hook(response): @@ -13,7 +34,7 @@ def after_request_hook(response): "AuthDBView.login", "AuthOAuthView.logout", ] - if (request.url_rule and request.url_rule.endpoint in logout_views): + if request.url_rule and (request.url_rule.endpoint in logout_views): response.set_cookie('hq_domain', '', expires=0) return response @@ -24,6 +45,8 @@ def after_request_hook(response): 'AuthOAuthView.login', 'AuthOAuthView.logout', 'AuthOAuthView.oauth_authorized', + 'CurrentUserRestApi.get_me', + 'Superset.log', 'DataSetChangeAPI.post_dataset_change', 'OAuth.issue_access_token', 'SelectDomainView.list', @@ -55,6 +78,58 @@ def ensure_domain_selected(): return redirect(url_for('SelectDomainView.list', next=request.url)) +def sync_user_domain_role(): + if is_user_admin() or ( + request.url_rule + and request.url_rule.endpoint in DOMAIN_EXCLUDED_VIEWS + ): + return + if _domain_role_expired(): + # only sync if another sync not in progress + if not _sync_in_progress(): + return _perform_sync_domain_role() + + +def _domain_role_expired(): + if not session.get(SESSION_DOMAIN_ROLE_LAST_SYNCED_AT): + return True + + time_since_last_sync = datetime_utcnow() - session[SESSION_DOMAIN_ROLE_LAST_SYNCED_AT] + return time_since_last_sync >= timedelta(minutes=USER_DOMAIN_ROLE_EXPIRY) + + +def _sync_in_progress(): + return cache_manager.cache.get(_sync_domain_role_cache_key()) + + +def _sync_domain_role_cache_key(): + return f"{g.user.id}_{g.hq_domain}_sync_domain_role" + + +def _perform_sync_domain_role(): + cache_key = _sync_domain_role_cache_key() + + # set cache for 30 seconds + cache_manager.cache.set(cache_key, True, timeout=30) + sync_domain_role_response = _sync_domain_role() + cache_manager.cache.delete(cache_key) + + return sync_domain_role_response + +def _sync_domain_role(): + if not DomainSyncUtil(superset.appbuilder.sm).sync_domain_role(g.hq_domain): + error_message = ( + f"Either your permissions for the project '{g.hq_domain}' were revoked or " + "your permissions failed to refresh. " + "Please select the project space again or login again to resolve. " + "If issue persists, please submit a support request." + ) + return current_app.response_class( + response=error_message, + status=400, + ) + + def is_valid_user_domain(hq_domain): # Admins have access to all domains return is_user_admin() or hq_domain in user_domains() @@ -71,7 +146,6 @@ def user_domains(): def add_domain_links(active_domain, domains): - import superset for domain in domains: superset.appbuilder.menu.add_link(domain, category=active_domain, href=url_for('SelectDomainView.select', hq_domain=domain)) diff --git a/hq_superset/migrations/versions/2024-02-24_23-53_56d0467ff6ff_added_oauth_tables.py b/hq_superset/migrations/versions/2024-02-24_23-53_56d0467ff6ff_added_oauth_tables.py index 0b962f5..0e120bf 100644 --- a/hq_superset/migrations/versions/2024-02-24_23-53_56d0467ff6ff_added_oauth_tables.py +++ b/hq_superset/migrations/versions/2024-02-24_23-53_56d0467ff6ff_added_oauth_tables.py @@ -6,9 +6,8 @@ """ from typing import Sequence, Union -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision: str = '56d0467ff6ff' diff --git a/hq_superset/models.py b/hq_superset/models.py index 4a35c2f..175a314 100644 --- a/hq_superset/models.py +++ b/hq_superset/models.py @@ -8,9 +8,13 @@ from cryptography.fernet import MultiFernet from superset import db -from .const import OAUTH2_DATABASE_NAME -from .exceptions import TableMissing -from .utils import cast_data_for_table, get_fernet_keys, get_hq_database +from hq_superset.const import OAUTH2_DATABASE_NAME +from hq_superset.exceptions import TableMissing +from hq_superset.utils import ( + cast_data_for_table, + get_fernet_keys, + get_hq_database, +) @dataclass diff --git a/hq_superset/oauth.py b/hq_superset/oauth.py index c6e8fcb..d724d77 100644 --- a/hq_superset/oauth.py +++ b/hq_superset/oauth.py @@ -6,11 +6,11 @@ from requests.exceptions import HTTPError from superset.security import SupersetSecurityManager -from .exceptions import OAuthSessionExpired -from .utils import ( +from hq_superset.const import ( SESSION_OAUTH_RESPONSE_KEY, SESSION_USER_DOMAINS_KEY, ) +from hq_superset.exceptions import OAuthSessionExpired logger = logging.getLogger(__name__) diff --git a/hq_superset/oauth2_server.py b/hq_superset/oauth2_server.py index f72285a..c1cbde1 100644 --- a/hq_superset/oauth2_server.py +++ b/hq_superset/oauth2_server.py @@ -13,7 +13,7 @@ ) from authlib.oauth2.rfc6749 import grants -from .models import OAuth2Client, OAuth2Token, db +from hq_superset.models import OAuth2Client, OAuth2Token, db def save_token(token: dict, request: FlaskOAuth2Request) -> None: diff --git a/hq_superset/services.py b/hq_superset/services.py index 284c478..7cd09ab 100644 --- a/hq_superset/services.py +++ b/hq_superset/services.py @@ -13,16 +13,16 @@ from superset.extensions import cache_manager from superset.sql_parse import Table -from .exceptions import HQAPIException -from .hq_requests import HQRequest -from .hq_url import ( +from hq_superset.exceptions import HQAPIException +from hq_superset.hq_requests import HQRequest +from hq_superset.hq_url import ( datasource_details, datasource_export, datasource_subscribe, datasource_unsubscribe, ) -from .models import OAuth2Client -from .utils import ( +from hq_superset.models import OAuth2Client +from hq_superset.utils import ( convert_to_array, generate_secret, get_column_dtypes, diff --git a/hq_superset/tasks.py b/hq_superset/tasks.py index 0797bee..09f0ed6 100644 --- a/hq_superset/tasks.py +++ b/hq_superset/tasks.py @@ -2,9 +2,9 @@ from superset.extensions import celery_app -from .exceptions import TableMissing -from .models import DataSetChange -from .services import AsyncImportHelper, refresh_hq_datasource +from hq_superset.exceptions import TableMissing +from hq_superset.models import DataSetChange +from hq_superset.services import AsyncImportHelper, refresh_hq_datasource @celery_app.task(name='refresh_hq_datasource_task') diff --git a/hq_superset/tests/base_test.py b/hq_superset/tests/base_test.py index b94e0be..4408b97 100644 --- a/hq_superset/tests/base_test.py +++ b/hq_superset/tests/base_test.py @@ -2,15 +2,17 @@ """ Base TestCase class """ - import os import shutil +import jwt from flask_testing import TestCase from sqlalchemy.sql import text from superset.app import create_app -from hq_superset.utils import DOMAIN_PREFIX, get_hq_database +from hq_superset.const import DOMAIN_PREFIX +from hq_superset.tests.utils import OAuthMock +from hq_superset.utils import get_hq_database superset_test_home = os.path.join(os.path.dirname(__file__), ".test_superset") shutil.rmtree(superset_test_home, ignore_errors=True) @@ -48,3 +50,33 @@ def tearDown(self): sql = "; ".join(domain_schemas) + ";" connection.execute(text(sql)) super(HQDBTestCase, self).tearDown() + + +class LoginUserTestMixin(object): + """ + Use this mixin by calling login function with client + & then logout once done for clearing the session + """ + def login(self, client): + self._setup_user() + # bypass oauth-workflow by skipping login and oauth flow + with client.session_transaction() as session_: + session_["oauth_state"] = "mock_state" + state = jwt.encode({}, "mock_state", algorithm="HS256") + return client.get(f"/oauth-authorized/commcare?state={state}", follow_redirects=True) + + def _setup_user(self): + self.app.appbuilder.add_permissions(update_perms=True) + self.app.appbuilder.sm.sync_role_definitions() + + self.oauth_mock = OAuthMock() + self.app.appbuilder.sm.oauth_remotes = {"commcare": self.oauth_mock} + + gamma_role = self.app.appbuilder.sm.find_role('Gamma') + self.user = self.app.appbuilder.sm.find_user(self.oauth_mock.user_json['username']) + if not self.user: + self.user = self.app.appbuilder.sm.add_user(**self.oauth_mock.user_json, role=[gamma_role]) + + @staticmethod + def logout(client): + return client.get("/logout/") diff --git a/hq_superset/tests/config_for_tests.py b/hq_superset/tests/config_for_tests.py index 5b43e63..c24a8d7 100644 --- a/hq_superset/tests/config_for_tests.py +++ b/hq_superset/tests/config_for_tests.py @@ -65,3 +65,4 @@ # CommCare Analytics extensions FLASK_APP_MUTATOR = flask_app_mutator CUSTOM_SECURITY_MANAGER = oauth.CommCareSecurityManager +USER_DOMAIN_ROLE_EXPIRY = 60 # minutes diff --git a/hq_superset/tests/const.py b/hq_superset/tests/const.py index 005ad7f..fbe1512 100644 --- a/hq_superset/tests/const.py +++ b/hq_superset/tests/const.py @@ -108,3 +108,16 @@ "id": "test1_ucr1", "resource_uri": "/a/demo/api/v0.5/ucr_data_source/52a134da12c9b801bd85d2122901b30c/" } + +TEST_UCR_CSV_V1 = """\ +doc_id,inserted_at,data_visit_date_eaece89e,data_visit_number_33d63739,data_lmp_date_5e24b993,data_visit_comment_fb984fda +a1, 2021-12-20, 2022-01-19, 100, 2022-02-20, some_text +a2, 2021-12-22, 2022-02-19, 10, 2022-03-20, some_other_text +""" + +TEST_UCR_CSV_V2 = """\ +doc_id,inserted_at,data_visit_date_eaece89e,data_visit_number_33d63739,data_lmp_date_5e24b993,data_visit_comment_fb984fda +a1, 2021-12-20, 2022-01-19, 100, 2022-02-20, some_text +a2, 2021-12-22, 2022-02-19, 10, 2022-03-20, some_other_text +a3, 2021-11-22, 2022-01-19, 10, 2022-03-20, some_other_text2 +""" diff --git a/hq_superset/tests/test_hq_domain.py b/hq_superset/tests/test_hq_domain.py index 68cd5ed..348eddc 100644 --- a/hq_superset/tests/test_hq_domain.py +++ b/hq_superset/tests/test_hq_domain.py @@ -2,6 +2,7 @@ from flask import g +from hq_superset.const import SESSION_USER_DOMAINS_KEY from hq_superset.hq_domain import ( DOMAIN_EXCLUDED_VIEWS, after_request_hook, @@ -10,15 +11,13 @@ is_valid_user_domain, user_domains, ) +from hq_superset.tests.base_test import HQDBTestCase, SupersetTestCase from hq_superset.utils import ( - SESSION_USER_DOMAINS_KEY, DomainSyncUtil, get_hq_database, get_schema_name_for_domain, ) -from .base_test import HQDBTestCase, SupersetTestCase - MOCK_DOMAIN_SESSION = { SESSION_USER_DOMAINS_KEY:[ { diff --git a/hq_superset/tests/test_oauth.py b/hq_superset/tests/test_oauth.py index 8182bc7..a1a2c91 100644 --- a/hq_superset/tests/test_oauth.py +++ b/hq_superset/tests/test_oauth.py @@ -3,14 +3,13 @@ from flask import session -from hq_superset.exceptions import OAuthSessionExpired -from hq_superset.oauth import get_valid_cchq_oauth_token -from hq_superset.utils import ( +from hq_superset.const import ( SESSION_OAUTH_RESPONSE_KEY, SESSION_USER_DOMAINS_KEY, ) - -from .base_test import SupersetTestCase +from hq_superset.exceptions import OAuthSessionExpired +from hq_superset.oauth import get_valid_cchq_oauth_token +from hq_superset.tests.base_test import SupersetTestCase class MockResponse: diff --git a/hq_superset/tests/test_utils.py b/hq_superset/tests/test_utils.py index 3260ee5..b07053f 100644 --- a/hq_superset/tests/test_utils.py +++ b/hq_superset/tests/test_utils.py @@ -1,10 +1,12 @@ import doctest from unittest.mock import patch -from hq_superset.utils import get_column_dtypes, DomainSyncUtil -from .base_test import SupersetTestCase -from .const import TEST_DATASOURCE -from hq_superset.const import READ_ONLY_ROLE_NAME +from flask import session + +from hq_superset.const import READ_ONLY_ROLE_NAME, SESSION_DOMAIN_ROLE_LAST_SYNCED_AT +from hq_superset.tests.base_test import LoginUserTestMixin, SupersetTestCase +from hq_superset.tests.const import TEST_DATASOURCE +from hq_superset.utils import DomainSyncUtil, get_column_dtypes def test_get_column_dtypes(): @@ -28,7 +30,7 @@ def test_doctests(): assert results.failed == 0 -class TestDomainSyncUtil(SupersetTestCase): +class TestDomainSyncUtil(LoginUserTestMixin, SupersetTestCase): PLATFORM_ROLE_NAMES = ["Gamma", "sql_lab", "dataset_editor"] @patch.object(DomainSyncUtil, "_get_domain_access") @@ -94,6 +96,26 @@ def test_permissions_change_updates_user_role(self, get_domain_access_mock): additional_roles = DomainSyncUtil(security_manager)._get_additional_user_roles("test-domain") assert not additional_roles + @patch('hq_superset.utils.datetime_utcnow') + @patch.object(DomainSyncUtil, "_get_domain_access") + def test_sync_domain_role(self, get_domain_access_mock, utcnow_mock): + client = self.app.test_client() + self.login(client) + + utcnow_mock_return = "2024-11-01 14:30:04.323000+00:00" + utcnow_mock.return_value = utcnow_mock_return + get_domain_access_mock.return_value = self._to_permissions_response( + can_write=False, + can_read=True, + roles=[], + ) + security_manager = self.app.appbuilder.sm + + self.assertIsNone(session.get(SESSION_DOMAIN_ROLE_LAST_SYNCED_AT)) + DomainSyncUtil(security_manager).sync_domain_role("test-domain") + self.assertEqual(session[SESSION_DOMAIN_ROLE_LAST_SYNCED_AT], utcnow_mock_return) + self.logout(client) + def _ensure_platform_roles_exist(self, sm): for role_name in self.PLATFORM_ROLE_NAMES: sm.add_role(role_name) diff --git a/hq_superset/tests/test_views.py b/hq_superset/tests/test_views.py index 96f0cf4..92765b3 100644 --- a/hq_superset/tests/test_views.py +++ b/hq_superset/tests/test_views.py @@ -8,112 +8,19 @@ from flask import redirect, session from sqlalchemy.sql import text -from hq_superset.exceptions import HQAPIException -from hq_superset.utils import ( +from hq_superset.const import ( + SESSION_DOMAIN_ROLE_LAST_SYNCED_AT, SESSION_USER_DOMAINS_KEY, - get_schema_name_for_domain, - DomainSyncUtil, ) - -from .base_test import HQDBTestCase -from .const import TEST_DATASOURCE - - -class MockResponse: - def __init__(self, json_data, status_code): - self.json_data = json_data - self.status_code = status_code - - def json(self): - return self.json_data - - @property - def content(self): - return pickle.dumps(self.json_data) - - -class UserMock(): - user_id = '123' - - def get_id(self): - return self.user_id - - -class OAuthMock(): - - def __init__(self): - self.user_json = { - 'username': 'testuser1', - 'first_name': 'user', - 'last_name': '1', - 'email': 'test@example.com', - } - self.domain_json = { - "objects": [ - { - "domain_name":"test1", - "project_name":"test1" - }, - { - "domain_name":"test2", - "project_name":"test 1" - }, - ] - } - self.test1_datasources = { - "objects": [ - { - "id": 'test1_ucr1', - "display_name": 'Test1 UCR1', - }, - { - "id": 'test1_ucr2', - "display_name": 'Test1 UCR2', - }, - ] - } - self.test2_datasources = { - "objects": [ - { - "id": 'test2_ucr1', - "display_name": 'Test2 UCR1', - } - ] - } - self.api_base_url = "https://cchq.org/" - self.user_domain_roles = { - "permissions": {"can_view": True, "can_edit": True}, - "roles": ["Gamma", "sql_lab"], - } - - def authorize_access_token(self): - return {"access_token": "some-key"} - - def get(self, url, token): - return { - 'api/v0.5/identity/': MockResponse(self.user_json, 200), - 'api/v0.5/user_domains?feature_flag=superset-analytics&can_view_reports=true': MockResponse(self.domain_json, 200), - 'a/test1/api/v0.5/ucr_data_source/': MockResponse(self.test1_datasources, 200), - 'a/test2/api/v0.5/ucr_data_source/': MockResponse(self.test2_datasources, 200), - 'a/test1/api/v0.5/ucr_data_source/test1_ucr1/': MockResponse(TEST_DATASOURCE, 200), - 'a/test1/configurable_reports/data_sources/export/test1_ucr1/?format=csv': MockResponse(TEST_UCR_CSV_V1, 200), - 'a/test1/api/analytics-roles/v1': MockResponse(self.user_domain_roles, 200), - 'a/test2/api/analytics-roles/v1': MockResponse(self.user_domain_roles, 200), - }[url] - - -TEST_UCR_CSV_V1 = """\ -doc_id,inserted_at,data_visit_date_eaece89e,data_visit_number_33d63739,data_lmp_date_5e24b993,data_visit_comment_fb984fda -a1, 2021-12-20, 2022-01-19, 100, 2022-02-20, some_text -a2, 2021-12-22, 2022-02-19, 10, 2022-03-20, some_other_text -""" - -TEST_UCR_CSV_V2 = """\ -doc_id,inserted_at,data_visit_date_eaece89e,data_visit_number_33d63739,data_lmp_date_5e24b993,data_visit_comment_fb984fda -a1, 2021-12-20, 2022-01-19, 100, 2022-02-20, some_text -a2, 2021-12-22, 2022-02-19, 10, 2022-03-20, some_other_text -a3, 2021-11-22, 2022-01-19, 10, 2022-03-20, some_other_text2 -""" +from hq_superset.exceptions import HQAPIException +from hq_superset.tests.base_test import HQDBTestCase +from hq_superset.tests.const import ( + TEST_DATASOURCE, + TEST_UCR_CSV_V1, + TEST_UCR_CSV_V2, +) +from hq_superset.tests.utils import MockResponse, OAuthMock, UserMock +from hq_superset.utils import DomainSyncUtil, get_schema_name_for_domain class TestViews(HQDBTestCase): @@ -226,8 +133,9 @@ def _do_assert(datasources): client.get('/domain/select/test2/', follow_redirects=True) client.get('/hq_datasource/list/', follow_redirects=True) _do_assert(self.oauth_mock.test2_datasources) + self.logout(client) - @patch.object(DomainSyncUtil, "sync_domain_role", return_value=True) + @patch.object(DomainSyncUtil, "_get_domain_access", return_value=({"can_write": True, "can_read": True}, [])) def test_datasource_upload(self, *args): client = self.app.test_client() self.login(client) @@ -241,6 +149,7 @@ def test_datasource_upload(self, *args): ucr_id, 'ds1' ) + self.logout(client) @patch.object(DomainSyncUtil, "sync_domain_role", return_value=True) def test_trigger_datasource_refresh_with_api_exception(self, *args): @@ -259,6 +168,7 @@ def test_trigger_datasource_refresh_with_api_exception(self, *args): ) self.assertEqual(response.status_code, 302) self.assertEqual(response.location, "/tablemodelview/list/") + self.logout(client) def test_trigger_datasource_refresh_with_errors(self, *args): from hq_superset.views import ( @@ -404,3 +314,75 @@ def _test_upload(test_data, expected_output): client.get('/hq_datasource/list/', follow_redirects=True) self.assert_context('ucr_id_to_pks', {}) + + @patch('hq_superset.hq_requests.get_valid_cchq_oauth_token', return_value={}) + @patch('hq_superset.hq_domain._sync_domain_role', return_value=None) + def test_sync_user_domain_role_calls(self, sync_domain_role_mock, *args): + with self.app.test_client() as client: + assert SESSION_USER_DOMAINS_KEY not in session + self.login(client) + self.assertIsNone(session.get(SESSION_DOMAIN_ROLE_LAST_SYNCED_AT)) + # not called on login + sync_domain_role_mock.assert_not_called() + + response = client.get('/', follow_redirects=True) + self.assertEqual(response.status, "200 OK") + self.assertTrue('/domain/list' in response.request.path) + self.assertIsNone(session.get(SESSION_DOMAIN_ROLE_LAST_SYNCED_AT)) + # not called on domain listing after login + sync_domain_role_mock.assert_not_called() + + client.get('/domain/select/test1/', follow_redirects=True) + self.assertIsNotNone(session.get(SESSION_DOMAIN_ROLE_LAST_SYNCED_AT)) + # not called on domain selection + sync_domain_role_mock.assert_not_called() + + client.get('/hq_datasource/list/', follow_redirects=True) + self.assertIsNotNone(session.get(SESSION_DOMAIN_ROLE_LAST_SYNCED_AT)) + # not called if recently synced + sync_domain_role_mock.assert_not_called() + + with patch('hq_superset.hq_domain.USER_DOMAIN_ROLE_EXPIRY', 0): + client.get('/hq_datasource/list/', follow_redirects=True) + self.assertIsNotNone(session.get(SESSION_DOMAIN_ROLE_LAST_SYNCED_AT)) + # called only if expired + sync_domain_role_mock.assert_called() + + self.logout(client) + + @patch('hq_superset.hq_requests.get_valid_cchq_oauth_token', return_value={}) + def test_sync_user_domain_before_request(self, *args): + with self.app.test_client() as client: + assert SESSION_USER_DOMAINS_KEY not in session + self.login(client) + + client.get('/domain/select/test1/', follow_redirects=True) + self.assertIsNotNone(session.get(SESSION_DOMAIN_ROLE_LAST_SYNCED_AT)) + + with patch('hq_superset.hq_domain.USER_DOMAIN_ROLE_EXPIRY', 0): + session_role_last_synced_at = session.get(SESSION_DOMAIN_ROLE_LAST_SYNCED_AT) + client.get('/hq_datasource/list/', follow_redirects=True) + # timestamp in session updated + self.assertNotEqual(session_role_last_synced_at, session.get(SESSION_DOMAIN_ROLE_LAST_SYNCED_AT)) + + with patch.object(DomainSyncUtil, 'sync_domain_role') as domain_sync_util_mock: + # in case of failure to sync + domain_sync_util_mock.return_value = False + response = client.get('/hq_datasource/list/', follow_redirects=True) + + # confirm function call for sync + domain_sync_util_mock.assert_called_once_with( + "test1" + ) + + self.assertEqual( + response.text, + "Either your permissions for the project 'test1' were revoked or " + "your permissions failed to refresh. " + "Please select the project space again or login again to resolve. " + "If issue persists, please submit a support request." + ) + self.assertEqual(response.status_code, 400) + + self.logout(client) + diff --git a/hq_superset/tests/utils.py b/hq_superset/tests/utils.py new file mode 100644 index 0000000..d406e6f --- /dev/null +++ b/hq_superset/tests/utils.py @@ -0,0 +1,91 @@ +import pickle + +from hq_superset.tests.const import TEST_DATASOURCE, TEST_UCR_CSV_V1 + + +class OAuthMock(object): + + def __init__(self): + self.user_json = { + 'username': 'testuser1', + 'first_name': 'user', + 'last_name': '1', + 'email': 'test@example.com', + } + self.domain_json = { + "objects": [ + { + "domain_name":"test1", + "project_name":"test1" + }, + { + "domain_name":"test2", + "project_name":"test 1" + }, + ] + } + self.test1_datasources = { + "objects": [ + { + "id": 'test1_ucr1', + "display_name": 'Test1 UCR1', + }, + { + "id": 'test1_ucr2', + "display_name": 'Test1 UCR2', + }, + ] + } + self.test2_datasources = { + "objects": [ + { + "id": 'test2_ucr1', + "display_name": 'Test2 UCR1', + } + ] + } + self.api_base_url = "https://cchq.org/" + self.user_domain_roles = { + "permissions": {"can_view": True, "can_edit": True}, + "roles": ["Gamma", "sql_lab"], + } + + def authorize_access_token(self): + return {"access_token": "some-key"} + + def get(self, url, token): + return { + 'api/v0.5/identity/': MockResponse(self.user_json, 200), + 'api/v0.5/user_domains?feature_flag=superset-analytics&can_view_reports=true': MockResponse( + self.domain_json, 200 + ), + 'a/test1/api/v0.5/ucr_data_source/': MockResponse(self.test1_datasources, 200), + 'a/test2/api/v0.5/ucr_data_source/': MockResponse(self.test2_datasources, 200), + 'a/test1/api/v0.5/ucr_data_source/test1_ucr1/': MockResponse(TEST_DATASOURCE, 200), + 'a/test1/configurable_reports/data_sources/export/test1_ucr1/?format=csv': MockResponse( + TEST_UCR_CSV_V1, 200 + ), + 'a/test1/api/analytics-roles/v1': MockResponse(self.user_domain_roles, 200), + 'a/test2/api/analytics-roles/v1': MockResponse(self.user_domain_roles, 200), + }[url] + + +class UserMock(object): + user_id = '123' + + def get_id(self): + return self.user_id + + +class MockResponse: + def __init__(self, json_data, status_code): + self.json_data = json_data + self.status_code = status_code + + def json(self): + return self.json_data + + @property + def content(self): + return pickle.dumps(self.json_data) + diff --git a/hq_superset/utils.py b/hq_superset/utils.py index c503985..af79803 100644 --- a/hq_superset/utils.py +++ b/hq_superset/utils.py @@ -9,28 +9,27 @@ from zipfile import ZipFile import pandas +import pytz import sqlalchemy from cryptography.fernet import Fernet -from flask import current_app +from flask import current_app, session from flask_login import current_user from sqlalchemy.sql import TableClause from superset.utils.database import get_or_create_db -from .const import ( +from hq_superset.const import ( + CAN_READ_PERMISSION, + CAN_WRITE_PERMISSION, + DOMAIN_PREFIX, GAMMA_ROLE_NAME, HQ_DATABASE_NAME, HQ_USER_ROLE_NAME, - SCHEMA_ACCESS_PERMISSION, - CAN_READ_PERMISSION, - CAN_WRITE_PERMISSION, READ_ONLY_MENU_PERMISSIONS, READ_ONLY_ROLE_NAME, + SCHEMA_ACCESS_PERMISSION, + SESSION_DOMAIN_ROLE_LAST_SYNCED_AT, ) -from .exceptions import DatabaseMissing - -DOMAIN_PREFIX = "hqdomain_" -SESSION_USER_DOMAINS_KEY = "user_hq_domains" -SESSION_OAUTH_RESPONSE_KEY = "oauth_response" +from hq_superset.exceptions import DatabaseMissing def get_hq_database(): @@ -152,6 +151,7 @@ def sync_domain_role(self, domain): self.sm.get_session.add(current_user) self.sm.get_session.commit() + session[SESSION_DOMAIN_ROLE_LAST_SYNCED_AT] = datetime_utcnow() return True def _ensure_hq_user_role(self): @@ -218,8 +218,8 @@ def _get_additional_user_roles(self, domain): @staticmethod def _get_domain_access(domain): - from .hq_url import user_domain_roles from .hq_requests import HQRequest + from .hq_url import user_domain_roles hq_request = HQRequest(url=user_domain_roles(domain)) response = hq_request.get() @@ -392,3 +392,7 @@ def cast_data_for_table( def generate_secret(): alphabet = string.ascii_letters + string.digits return ''.join(secrets.choice(alphabet) for __ in range(64)) + + +def datetime_utcnow(): + return datetime.utcnow().replace(tzinfo=pytz.UTC) diff --git a/hq_superset/views.py b/hq_superset/views.py index af02bfc..92455d5 100644 --- a/hq_superset/views.py +++ b/hq_superset/views.py @@ -16,19 +16,19 @@ from superset.connectors.sqla.models import SqlaTable from superset.views.base import BaseSupersetView -from .exceptions import HQAPIException -from .hq_domain import user_domains -from .hq_requests import HQRequest -from .hq_url import datasource_list -from .services import ( +from hq_superset.exceptions import HQAPIException +from hq_superset.hq_domain import user_domains +from hq_superset.hq_requests import HQRequest +from hq_superset.hq_url import datasource_list +from hq_superset.services import ( AsyncImportHelper, download_and_subscribe_to_datasource, get_datasource_defn, refresh_hq_datasource, unsubscribe_from_hq_datasource, ) -from .tasks import refresh_hq_datasource_task -from .utils import ( +from hq_superset.tasks import refresh_hq_datasource_task +from hq_superset.utils import ( DomainSyncUtil, get_hq_database, get_schema_name_for_domain, diff --git a/superset_config.example.py b/superset_config.example.py index bd283af..b62461f 100644 --- a/superset_config.example.py +++ b/superset_config.example.py @@ -180,3 +180,5 @@ class CeleryConfig: "session_cookie_secure": False, } +USER_DOMAIN_ROLE_EXPIRY = 60 # minutes +