diff --git a/inbox/events/abstract.py b/inbox/events/abstract.py index bda76ed4b..a4ed87594 100644 --- a/inbox/events/abstract.py +++ b/inbox/events/abstract.py @@ -1,6 +1,6 @@ import abc import datetime -from typing import Dict, List, Optional +from typing import Dict, Iterable, List, Optional from inbox.events.util import CalendarSyncResponse from inbox.logging import get_logger @@ -39,7 +39,7 @@ def sync_calendars(self) -> CalendarSyncResponse: @abc.abstractmethod def sync_events( self, calendar_uid: str, sync_from_time: Optional[datetime.datetime] = None - ) -> List[Event]: + ) -> Iterable[Event]: """ Fetch event data for an individual calendar. @@ -49,7 +49,7 @@ def sync_events( changed since this time. Returns: - A list of uncommited Event instances + An iterable of uncommited Event instances """ raise NotImplementedError() diff --git a/inbox/events/google.py b/inbox/events/google.py index 4d3bfecfa..1ed9a4962 100644 --- a/inbox/events/google.py +++ b/inbox/events/google.py @@ -73,7 +73,7 @@ def sync_calendars(self) -> CalendarSyncResponse: def sync_events( self, calendar_uid: str, sync_from_time: Optional[datetime.datetime] = None - ) -> List[Event]: + ) -> Iterable[Event]: """ Fetch event data for an individual calendar. @@ -88,22 +88,18 @@ def sync_events( all event data. Returns: - A list of uncommited Event instances + A generator of uncommited Event instances """ - updates = [] raw_events = self._get_raw_events(calendar_uid, sync_from_time) read_only_calendar = self.calendars_table.get(calendar_uid, True) for raw_event in iterate_and_periodically_switch_to_gevent(raw_events): try: - parsed = parse_event_response(raw_event, read_only_calendar) - updates.append(parsed) + yield parse_event_response(raw_event, read_only_calendar) except (arrow.parser.ParserError, ValueError): self.log.warning( "Skipping unparseable event", exc_info=True, raw=raw_event ) - return updates - def _get_raw_calendars(self) -> Iterable[Dict[str, Any]]: """Gets raw data for the user's calendars.""" return self._get_resource_list(CALENDARS_URL) diff --git a/inbox/events/microsoft/events_provider.py b/inbox/events/microsoft/events_provider.py index cd4d11988..55007c23f 100644 --- a/inbox/events/microsoft/events_provider.py +++ b/inbox/events/microsoft/events_provider.py @@ -110,7 +110,7 @@ def sync_calendars(self) -> CalendarSyncResponse: def sync_events( self, calendar_uid: str, sync_from_time: Optional[datetime.datetime] = None - ) -> List[Event]: + ) -> Iterable[Event]: """ Fetch event data for an individual calendar. @@ -120,7 +120,7 @@ def sync_events( changed since this time. Returns: - A list of uncommited Event instances + An iterator of uncommited Event instances """ if sync_from_time: # this got here from the database, we store them as naive @@ -128,7 +128,6 @@ def sync_events( # we attach timezone here. sync_from_time = sync_from_time.replace(tzinfo=pytz.UTC) - updates = [] raw_events = cast( Iterable[MsGraphEvent], self.client.iter_events( @@ -142,16 +141,14 @@ def sync_events( continue event = parse_event(raw_event, read_only=read_only) - updates.append(event) + yield event if isinstance(event, RecurringEvent): exceptions, cancellations = self._get_event_overrides( raw_event, event, read_only=read_only ) - updates.extend(exceptions) - updates.extend(cancellations) - - return updates + yield from exceptions + yield from cancellations def _get_event_overrides( self, raw_master_event: MsGraphEvent, master_event: RecurringEvent, *, read_only diff --git a/inbox/events/remote_sync.py b/inbox/events/remote_sync.py index 0d8c0680d..8bcb1f7a5 100644 --- a/inbox/events/remote_sync.py +++ b/inbox/events/remote_sync.py @@ -1,5 +1,5 @@ from datetime import datetime, timedelta -from typing import Any, List, Tuple, Type +from typing import Any, Iterable, List, Tuple, Type import more_itertools from requests.exceptions import HTTPError @@ -167,7 +167,11 @@ def handle_calendar_updates( def handle_event_updates( - namespace_id: int, calendar_id: int, events: List[Event], log: Any, db_session: Any + namespace_id: int, + calendar_id: int, + events: Iterable[Event], + log: Any, + db_session: Any, ) -> None: """Persists new or updated Event objects to the database.""" added_count = 0 diff --git a/tests/events/test_google_events.py b/tests/events/test_google_events.py index 98db06546..962a276c6 100644 --- a/tests/events/test_google_events.py +++ b/tests/events/test_google_events.py @@ -265,7 +265,7 @@ def test_event_parsing(): provider = GoogleEventsProvider(1, 1) provider.calendars_table = {"uid": False} provider._get_raw_events = mock.MagicMock(return_value=raw_response) - updates = provider.sync_events("uid", 1) + updates = list(provider.sync_events("uid", 1)) # deleted events are actually only marked as # cancelled. Look for them in the updates stream. @@ -317,7 +317,7 @@ def test_event_parsing(): # This is a read-only calendar provider.calendars_table = {"uid": True} provider._get_raw_events = mock.MagicMock(return_value=raw_response) - updates = provider.sync_events("uid", 1) + updates = list(provider.sync_events("uid", 1)) assert len(updates) == 1 assert updates[0].read_only is True @@ -508,7 +508,7 @@ def test_handle_unparseable_dates(): ) provider = GoogleEventsProvider(1, 1) provider._get_raw_events = mock.MagicMock(return_value=raw_response) - updates = provider.sync_events("uid", 1) + updates = list(provider.sync_events("uid", 1)) assert len(updates) == 0 @@ -856,5 +856,5 @@ def test_cancelled_override_creation(): provider = GoogleEventsProvider(1, 1) provider._get_raw_events = mock.MagicMock(return_value=raw_response) - updates = provider.sync_events("uid", 1) + updates = list(provider.sync_events("uid", 1)) assert updates[0].cancelled is True diff --git a/tests/events/test_sync.py b/tests/events/test_sync.py index fca47a0e9..d767c552b 100644 --- a/tests/events/test_sync.py +++ b/tests/events/test_sync.py @@ -71,7 +71,7 @@ def calendar_response_with_delete(): def event_response(calendar_uid, sync_from_time): if calendar_uid == "first_calendar_uid": - return [ + yield from [ Event.create( uid="first_event_uid", title="Plotting Meeting", **default_params ), @@ -83,7 +83,7 @@ def event_response(calendar_uid, sync_from_time): ), ] else: - return [ + yield from [ Event.create( uid="second_event_uid", title="Plotting Meeting", **default_params ), @@ -95,7 +95,7 @@ def event_response(calendar_uid, sync_from_time): def event_response_with_update(calendar_uid, sync_from_time): if calendar_uid == "first_calendar_uid": - return [ + yield from [ Event.create( uid="first_event_uid", title="Top Secret Plotting Meeting", @@ -110,12 +110,12 @@ def event_response_with_participants_update(calendar_uid, sync_from_time): new_events[0].participants = [ {"name": "Johnny Thunders", "email": "johnny@thunde.rs"} ] - return new_events + yield from new_events def event_response_with_delete(calendar_uid, sync_from_time): if calendar_uid == "first_calendar_uid": - return [ + yield from [ Event.create(uid="first_event_uid", status="cancelled", **default_params) ]