Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Redesign communication client types #409

Merged
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
8ccc9e7
Redesign communication client types
robertbartel Aug 7, 2023
733ab9d
Add AuthClient and CachedAuthClient.
robertbartel Aug 9, 2023
3ef01cf
Modify ExternalRequestClient to use AuthClient
robertbartel Aug 9, 2023
6b7c6f1
Remove obsolete ExternalClient.
robertbartel Aug 10, 2023
52dfa9f
Un-generifying RequestClient and subtypes.
robertbartel Aug 10, 2023
072d086
Updating tests for client changes.
robertbartel Aug 10, 2023
7362f99
Updating comms package init.
robertbartel Aug 10, 2023
7e80278
Bump comms package to 1.0.0.
robertbartel Aug 10, 2023
7050ff6
Deprecating some RequestClient subclasses.
robertbartel Aug 10, 2023
f56bc98
Remove obsoleted generic type vars.
robertbartel Aug 10, 2023
5b6d015
Update externalrequests to latest comms dep.
robertbartel Aug 10, 2023
fbf6fa0
Update external request handlers for comms.
robertbartel Aug 10, 2023
3f421ab
Bump externalrequests package version to 0.5.0.
robertbartel Aug 10, 2023
2bc8fc3
Adjusting comms version to not be 1.0.0 yet.
robertbartel Aug 21, 2023
f05a09c
Adjusting comms dep to not be 1.0.0 yet.
robertbartel Aug 21, 2023
97548fa
Fix optional client session property type hints.
robertbartel Aug 21, 2023
b7ac988
For cached auth, use fix, common default basename.
robertbartel Aug 21, 2023
4be3cb1
Make WebSocketClient.build_endpoint_uri static.
robertbartel Aug 21, 2023
9e60aa0
Put SSLContext directly in TransportLayerClient.
robertbartel Aug 24, 2023
abd58db
Remove SSLSecuredTransportLayerClient.
robertbartel Aug 24, 2023
da0c407
Remove SSLSecuredTransportLayerClient from init.
robertbartel Aug 25, 2023
5b6160d
Refactor WebSocketClient after SSL client removal.
robertbartel Aug 25, 2023
549045f
Modify transport client to take URI components.
robertbartel Aug 25, 2023
2815193
Update request handlers for client class changes.
robertbartel Aug 25, 2023
b3ec89e
Update comms mock test class for client changes.
robertbartel Aug 25, 2023
5d834f4
Tweak transport client endpoint uri handling.
robertbartel Aug 28, 2023
3bc4cc6
Fix issues in client.py.
robertbartel Sep 8, 2023
828a1ea
Fix MockTransportLayerClient for unit tests.
robertbartel Sep 8, 2023
4dce763
Have transport client init only use keyword args.
robertbartel Sep 8, 2023
6d63be3
Remove public client_ssl_context property.
robertbartel Sep 8, 2023
583dbf3
Refactor endpoint_uri property to private func.
robertbartel Sep 8, 2023
0aaf94a
Clarify that endpoint_host does not include protocol.
robertbartel Sep 8, 2023
a1bbb5e
Update RequestClient init to keyword args only.
robertbartel Sep 8, 2023
92e58e8
Update scheduler client tests for endpoint uri.
robertbartel Sep 8, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions python/lib/communication/dmod/communication/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from ._version import __version__
from .client import DataServiceClient, InternalServiceClient, ModelExecRequestClient, ExternalRequestClient, \
PartitionerServiceClient, SchedulerClient
from .client import AuthClient, CachedAuthClient, DataServiceClient, ExternalRequestClient, PartitionerServiceClient, \
RequestClient, SchedulerClient, SSLSecuredTransportLayerClient, TransportLayerClient, WebSocketClient
from .maas_request import get_available_models, get_available_outputs, get_distribution_types, get_parameters, \
get_request, AbstractNgenRequest, Distribution, DmodJobRequest, ExternalRequest, ExternalRequestResponse,\
ModelExecRequest, ModelExecRequestResponse, NWMRequest, NWMRequestResponse, Scalar, NGENRequest, \
Expand Down
2 changes: 1 addition & 1 deletion python/lib/communication/dmod/communication/_version.py
aaraney marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = '0.14.0'
__version__ = '1.0.0'
1,117 changes: 572 additions & 545 deletions python/lib/communication/dmod/communication/client.py

Large diffs are not rendered by default.

76 changes: 42 additions & 34 deletions python/lib/communication/dmod/test/test_scheduler_client.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,16 @@
import asyncio
import json
import logging
import ssl
import unittest
from pathlib import Path
from typing import Optional, Union
from ..communication import NWMRequest, SchedulerClient, SchedulerRequestMessage, SchedulerRequestResponse
from ..communication import NWMRequest, SchedulerClient, SchedulerRequestMessage, SchedulerRequestResponse, \
TransportLayerClient


class MockSendTestingSchedulerClient(SchedulerClient):
"""
Customized extension of ``SchedulerClient`` for testing purposes, where the :meth:`async_send` method has been
overridden with a mock implementation to allow for testing without actually needing a real websocket connection.
"""
class MockTransportLayerClient(TransportLayerClient):

def __init__(self):
super().__init__(endpoint_uri='', ssl_directory=Path('.'))
super().__init__(endpoint_uri='')

self.test_responses = dict()

Expand Down Expand Up @@ -52,20 +48,52 @@ async def async_send(self, data: Union[str, bytearray], await_response: bool = F
else:
return str(response)

def set_scheduler_response_none(self):
async def async_recv(self) -> str:
pass

@property
def client_ssl_context(self) -> ssl.SSLContext:
pass

def set_client_response_none(self):
self.test_response_selection = 0

def set_scheduler_response_non_json_string(self):
def set_client_response_non_json_string(self):
self.test_response_selection = 1

def set_scheduler_response_unrecognized_json(self):
def set_client_response_unrecognized_json(self):
self.test_response_selection = 2

def set_scheduler_response_valid_obj_for_failure(self):
def set_client_response_valid_obj_for_failure(self):
self.test_response_selection = 3

def set_scheduler_response_valid_obj_for_success(self):
def set_client_response_valid_obj_for_success(self):
self.test_response_selection = 4


class MockSendTestingSchedulerClient(SchedulerClient):
"""
Customized extension of ``SchedulerClient`` for testing purposes, where the :meth:`async_send` method has been
overridden with a mock implementation to allow for testing without actually needing a real websocket connection.
"""

def __init__(self):
super().__init__(transport_client=MockTransportLayerClient())

def set_scheduler_response_none(self):
self._transport_client.test_response_selection = 0

def set_scheduler_response_non_json_string(self):
self._transport_client.test_response_selection = 1

def set_scheduler_response_unrecognized_json(self):
self._transport_client.test_response_selection = 2

def set_scheduler_response_valid_obj_for_failure(self):
self._transport_client.test_response_selection = 3

def set_scheduler_response_valid_obj_for_success(self):
self._transport_client.test_response_selection = 4


class TestSchedulerClient(unittest.TestCase):
Expand Down Expand Up @@ -103,26 +131,6 @@ def tearDown(self) -> None:
self.loop.stop()
self.loop.close()

def test_get_response_subtype_1_a(self):
"""
Test that ``get_response_subtype`` returns the right type.
"""
self.assertEqual(SchedulerRequestResponse, self.client.get_response_subtype())

def test_build_response_1_a(self):
"""
Basic test to ensure this function operates correctly.
"""
response = self.client.build_response(success=True, reason='Test Good', message='Test worked correctly')
self.assertTrue(isinstance(response, SchedulerRequestResponse))

def test_build_response_1_b(self):
"""
Basic test to ensure this response has the expected ``success`` value.
"""
response = self.client.build_response(success=True, reason='Test Good', message='Test worked correctly')
self.assertTrue(response.success)

def test_async_make_request_1_a(self):
"""
Test when function gets ``None`` returned over websocket that response object ``success`` is ``False``.
Expand Down
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = '0.4.1'
__version__ = '0.5.0'
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@

from dmod.access import Authorizer
from dmod.communication import AbstractRequestHandler, DataServiceClient, FullAuthSession, ExternalRequest, \
InitRequestResponseReason, InternalServiceClient, PartitionRequest, PartitionResponse, PartitionerServiceClient, \
Session, SessionManager
InitRequestResponseReason, RequestClient, PartitionRequest, PartitionResponse, PartitionerServiceClient, \
TransportLayerClient, Session, SessionManager, WebSocketClient
from dmod.communication.dataset_management_message import MaaSDatasetManagementMessage, MaaSDatasetManagementResponse, \
ManagementAction
from dmod.communication.data_transmit_message import DataTransmitMessage, DataTransmitResponse
Expand Down Expand Up @@ -38,6 +38,7 @@ def __init__(self, session_manager: SessionManager, authorizer: Authorizer, serv
self._service_port = service_port
self._service_ssl_dir = service_ssl_dir
self._service_url = None
self._transport_client = None

async def _is_authorized(self, request: ExternalRequest, session: FullAuthSession) -> bool:
"""
Expand Down Expand Up @@ -129,15 +130,22 @@ async def get_authorized_session(self, request: ExternalRequest) -> Tuple[
msg = None
return session, is_authorized, reason, msg

@property
def transport_client(self) -> TransportLayerClient:
if self._transport_client is None:
# TODO: parameterize whether to, e.g., use websocket uri/protocol, as opposed to something else
self._transport_client = WebSocketClient(endpoint_uri=self.service_url, ssl_directory=self.service_ssl_dir)
return self._transport_client

@property
@abstractmethod
def service_client(self) -> InternalServiceClient:
def service_client(self) -> RequestClient:
"""
Get the client for interacting with the service, which also is a context manager for connections.
Get the client for interacting with the service.

Returns
-------
InternalServiceClient
RequestClient
The client for interacting with the service.
"""
pass
Expand All @@ -148,6 +156,7 @@ def service_ssl_dir(self) -> Path:

@property
def service_url(self) -> str:
# TODO: parameterize whether to, e.g., use websocket uri/protocol, as opposed to something else
if self._service_url is None:
self._service_url = 'wss://{}:{}'.format(str(self._service_host), str(self._service_port))
return self._service_url
Expand Down Expand Up @@ -202,7 +211,7 @@ async def determine_required_access_types(self, request: PartitionRequest, user)
@property
def service_client(self) -> PartitionerServiceClient:
if self._service_client is None:
self._service_client = PartitionerServiceClient(self.service_url, self.service_ssl_dir)
self._service_client = PartitionerServiceClient(transport_client=self.transport_client)
return self._service_client

async def handle_request(self, request: PartitionRequest, **kwargs) -> PartitionResponse:
Expand Down Expand Up @@ -332,5 +341,5 @@ async def handle_request(self, request: MaaSDatasetManagementMessage, **kwargs)
@property
def service_client(self) -> DataServiceClient:
if self._service_client is None:
self._service_client = DataServiceClient(endpoint_uri=self.service_url, ssl_directory=self.service_ssl_dir)
self._service_client = DataServiceClient(transport_client=self.transport_client)
return self._service_client
2 changes: 1 addition & 1 deletion python/lib/externalrequests/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,6 @@
author_email='',
url='',
license='',
install_requires=['websockets', 'dmod-core>=0.1.0', 'dmod-communication>=0.4.2', 'dmod-access>=0.1.1'],
install_requires=['websockets', 'dmod-core>=0.1.0', 'dmod-communication>=1.0.0', 'dmod-access>=0.1.1'],
packages=find_namespace_packages(exclude=['dmod.test', 'schemas', 'ssl', 'src'])
)