diff --git a/src/archive_reader/app.py b/src/archive_reader/app.py index 79d5729..11874ef 100644 --- a/src/archive_reader/app.py +++ b/src/archive_reader/app.py @@ -31,141 +31,11 @@ EmailItem, ) from .core import ListManager, ThreadsManager - +from .screens import ThreadReadScreen, MailingListAddScreen DEFAULT_NOTIFY_TIMEOUT = 2 -class ThreadReadScreen(Screen): - """The main screen to read Email threads. - - This is composed of multiple Emails, which are embedded inside a listview. - """ - - BINDINGS = [ - ('escape', 'app.pop_screen', 'Close thread'), - ('r', 'update_emails', 'Refresh Emails'), - ] - - DEFAULT_CSS = """ - .main { - layout: grid; - grid-size: 2; - grid-columns: 9fr 1fr; - } - .sender { - padding: 0 1; - } - """ - - def __init__(self, *args, thread=None, thread_mgr=None, **kw): - self.thread = thread - self.thread_mgr = thread_mgr - super().__init__(*args, **kw) - - def compose(self) -> ComposeResult: - header = Header() - header.text = self.thread.subject - yield header - yield LoadingIndicator() - with Horizontal(classes='main'): - yield ListView(id='thread-emails') - yield ListView(id='thread-authors') - yield Footer() - - @work - async def load_emails(self): - reply_objs = await self.thread_mgr.emails(self.thread) - reply_emails = [ - EmailItem( - email=reply, - id='message-id-{}'.format(reply.message_id_hash), - ) - for reply in reply_objs - ] - try: - self.add_emails(reply_emails) - self.add_email_authors(reply_emails) - except Exception as ex: - log(ex) - self._hide_loading() - - @work - async def action_update_emails(self): - await self.thread_mgr.update_emails(self.thread) - self.load_emails() - self.notify('Thread refresh complete.') - - def add_emails(self, emails): - view = self.query_one('#thread-emails', ListView) - for email in emails: - view.append(email) - - def add_email_authors(self, emails): - view = self.query_one('#thread-authors', ListView) - for email in emails: - view.append(ListItem(Static(f'{email.sender}', classes='sender'))) - - def on_mount(self): - self.load_emails() - - def _show_loading(self): - self.query_one(LoadingIndicator).display = True - - def _hide_loading(self): - self.query_one(LoadingIndicator).display = False - - -class MailingListAddScreen(Screen): - """A new screen where you can search and subscribe to MailingLists. - - This page will take the server as the input and load all the mailing lists on - that server. - """ - - DEFAULT_CSS = """ - Screen { - align: center middle; - } - """ - BINDINGS = [('escape', 'app.pop_screen', 'Pop screen')] - - def __init__(self, *args, **kw): - super().__init__(*args, **kw) - self.list_manager = ListManager() - self._list_cache = {} - - def compose(self): - yield Static('Hyperkitty Server URL', classes='label') - yield Input(placeholder='https://') - yield Static() - yield MailingListChoose(id='pick-mailinglist') - yield Footer() - - @work(exclusive=True) - async def update_mailinglists(self, base_url): - lists_json = await self.list_manager.fetch_lists(base_url) - selection_list = self.query_one(SelectionList) - for ml in lists_json.get('results'): - self._list_cache[ml.get('name')] = ml - selection_list.add_option( - ( - f"{ml.get('display_name')} <\"{ml.get('name')}\">", - ml.get('name'), - ) - ) - - async def on_input_submitted(self, message: Input.Submitted): - self.base_url = message.value - self.update_mailinglists(self.base_url) - - def on_mailing_list_choose_selected(self, message): - log(f'User chose {message.data=}') - self.dismiss( - [self._list_cache.get(listname) for listname in message.data] - ) - - class ArchiveApp(App): """Textual reader app to read Hyperkitty (GNU Mailman's official Archiver) email archives.""" @@ -248,7 +118,6 @@ async def _clear_threads(self): threads_container = self.query_one('#threads', ListView) # First, clear the threads. clear_resp = threads_container.clear() - log(type(clear_resp)) # .clear() returns an awaitable and gives the control back to # DOM to perform the action. await clear_resp @@ -260,7 +129,6 @@ async def action_update_threads(self): timeout=DEFAULT_NOTIFY_TIMEOUT, ) self.update_threads(self.current_mailinglist) - self._notify_update_complete() def _notify_update_complete(self): self.notify( @@ -281,7 +149,7 @@ def thread_mgr(self): @work() async def update_threads(self, ml): header = self.query_one('#header', Header) - header.text = ml.name + header.text = '{} ({})'.format(ml.display_name, ml.name) await self._clear_threads() self._show_loading() self.current_mailinglist = ml @@ -312,7 +180,7 @@ async def on_list_view_selected(self, item): log(f'Thread {item.item} was selected.') # Make sure that we cancel the workers so that nothing will interfere after # we have moved on to the next screen. - self.workers.cancel_all() + # self.workers.cancel_all() # Mark the threads as read. self.push_screen( ThreadReadScreen( diff --git a/src/archive_reader/core.py b/src/archive_reader/core.py index 94cff5e..f6fb286 100644 --- a/src/archive_reader/core.py +++ b/src/archive_reader/core.py @@ -1,10 +1,15 @@ """Core business logic.""" import asyncio +import httpx from textual import log from .models import MailingList, Thread, EmailManager from .hyperkitty import hyperktty_client, fetch_urls +class RemoteURLFetchException(Exception): + """Exception fetching remote URLs.""" + + class ThreadsManager: """The purpose of threads manager is to create Thread models and deal with local storage into sqlite3 database. @@ -19,6 +24,7 @@ class ThreadsManager: def __init__(self, mailinglist: MailingList) -> None: self.ml = mailinglist + self.email_mgr = EmailManager() # ================= Public API ================================= async def threads(self): @@ -36,11 +42,14 @@ async def emails(self, thread): async def update_emails(self, thread): """Load New Emails from remote.""" - replies, _ = await fetch_urls([thread.emails], log) + try: + replies, _ = await fetch_urls([thread.emails], log) + except httpx.ConnectError: + log(f'Failed to get Email URLs {thread.emails}') + raise RemoteURLFetchException(thread.emails) reply_urls = [each.get('url') for each in replies[0].get('results')] log(f'Retrieved email urls {reply_urls}') - email_manager = EmailManager() - existing_emails = await EmailManager.filter(thread=thread.url).all() + existing_emails = await self.email_mgr.filter(thread=thread.url) existing_email_urls = set(email.url for email in existing_emails) new_urls = list( url for url in reply_urls if url not in existing_email_urls @@ -51,15 +60,14 @@ async def update_emails(self, thread): replies, _ = await fetch_urls(new_urls) tasks = [] for reply in replies: - tasks.append(email_manager.create(reply)) + tasks.append(self.email_mgr.create(reply)) results = await asyncio.gather(*tasks) return [result[0] for result in results] # ================= Private API ================================ async def _load_emails_from_db(self, thread): - manager = EmailManager() - return await manager.filter(thread=thread.url).all() + return await self.email_mgr.filter(thread=thread.url) async def _load_threads_from_db(self): """Load all the existing threads from the db.""" diff --git a/src/archive_reader/hyperkitty.py b/src/archive_reader/hyperkitty.py index 4ac6a3e..38a98fa 100644 --- a/src/archive_reader/hyperkitty.py +++ b/src/archive_reader/hyperkitty.py @@ -45,9 +45,7 @@ async def fetch_urls(urls, logger=None): tasks.append( asyncio.ensure_future(client.get(url, follow_redirects=True)) ) - results = await asyncio.gather(*tasks) - for resp in results: if resp.status_code == 200: success.append(resp.json()) diff --git a/src/archive_reader/models.py b/src/archive_reader/models.py index 1a11c10..e7ef002 100644 --- a/src/archive_reader/models.py +++ b/src/archive_reader/models.py @@ -102,4 +102,5 @@ async def create(self, json_data): get = Email.objects.get - filter = Email.objects.filter + async def filter(self, *args, **kw): + return await Email.objects.filter(*args, **kw).all() diff --git a/src/archive_reader/screens.py b/src/archive_reader/screens.py new file mode 100644 index 0000000..5f9c016 --- /dev/null +++ b/src/archive_reader/screens.py @@ -0,0 +1,189 @@ +import asyncio +from contextlib import suppress +from collections import defaultdict + +from textual import log, work +from textual._node_list import DuplicateIds +from textual.app import App, ComposeResult +from textual.containers import Horizontal, Vertical +from textual.css.query import NoMatches + +from textual.reactive import var +from textual.screen import Screen +from textual.widgets import ( + Footer, + Input, + ListItem, + ListView, + LoadingIndicator, + SelectionList, + Static, +) + +from .models import initialize_database +from .widgets import ( + Threads, + MailingListItem, + MailingListChoose, + MailingLists, + ThreadItem, + Header, + EmailItem, +) +from .core import ListManager, ThreadsManager + +DEFAULT_NOTIFY_TIMEOUT = 2 + + +class ThreadReadScreen(Screen): + """The main screen to read Email threads. + + This is composed of multiple Emails, which are embedded inside a listview. + """ + + BINDINGS = [ + ('escape', 'app.pop_screen', 'Close thread'), + ('r', 'update_emails', 'Refresh Emails'), + ] + + DEFAULT_CSS = """ + .main { + layout: grid; + grid-size: 2; + grid-columns: 9fr 1fr; + } + .sender { + padding: 0 1; + } + """ + + def __init__(self, *args, thread=None, thread_mgr=None, **kw): + self.thread = thread + self.thread_mgr = thread_mgr + super().__init__(*args, **kw) + + def compose(self) -> ComposeResult: + header = Header() + header.text = self.thread.subject + yield header + yield LoadingIndicator() + with Horizontal(classes='main'): + yield ListView(id='thread-emails') + yield ListView(id='thread-authors') + yield Footer() + + def on_mount(self): + """Runs as soon as the Widget is mounted.""" + self.load_emails() + self.action_update_emails() + + @work + async def load_emails(self, show_loading=True): + """Load emails from Database and schedule emails to be fetched + from remote if needed. + + TODO: We don't currently have the if-needed criteria working too + well since we don't yet compare the replies_count. + """ + if show_loading: + self._show_loading() + reply_objs = await self.thread_mgr.emails(self.thread) + # if not reply_objs: + # self.notify(f'No saved emails for {self.thread.subject}. Fetching from remote.') + reply_emails = [ + EmailItem( + email=reply, + id='message-id-{}'.format(reply.message_id_hash), + ) + for reply in reply_objs + ] + try: + self.add_emails(reply_emails) + self.add_email_authors(reply_emails) + except Exception as ex: + log(ex) + self._hide_loading() + + @work + async def action_update_emails(self): + replies = await self.thread_mgr.update_emails(self.thread) + # self.load_emails(show_loading=False) + reply_emails = [ + EmailItem( + email=reply, + id='message-id-{}'.format(reply.message_id_hash), + ) + for reply in replies + ] + try: + self.add_emails(reply_emails) + self.add_email_authors(reply_emails) + except Exception as ex: + log(ex) + self.notify('Thread refresh complete.') + + def add_emails(self, emails): + view = self.query_one('#thread-emails', ListView) + for email in emails: + view.append(email) + + def add_email_authors(self, emails): + view = self.query_one('#thread-authors', ListView) + for email in emails: + view.append(ListItem(Static(f'{email.sender}', classes='sender'))) + + def _show_loading(self): + self.query_one(LoadingIndicator).display = True + + def _hide_loading(self): + self.query_one(LoadingIndicator).display = False + + +class MailingListAddScreen(Screen): + """A new screen where you can search and subscribe to MailingLists. + + This page will take the server as the input and load all the mailing lists on + that server. + """ + + DEFAULT_CSS = """ + Screen { + align: center middle; + } + """ + BINDINGS = [('escape', 'app.pop_screen', 'Pop screen')] + + def __init__(self, *args, **kw): + super().__init__(*args, **kw) + self.list_manager = ListManager() + self._list_cache = {} + + def compose(self): + yield Static('Hyperkitty Server URL', classes='label') + yield Input(placeholder='https://') + yield Static() + yield MailingListChoose(id='pick-mailinglist') + yield Footer() + + @work(exclusive=True) + async def update_mailinglists(self, base_url): + lists_json = await self.list_manager.fetch_lists(base_url) + selection_list = self.query_one(SelectionList) + for ml in lists_json.get('results'): + self._list_cache[ml.get('name')] = ml + selection_list.add_option( + ( + f"{ml.get('display_name')} <\"{ml.get('name')}\">", + ml.get('name'), + ) + ) + + async def on_input_submitted(self, message: Input.Submitted): + self.base_url = message.value + self.update_mailinglists(self.base_url) + + def on_mailing_list_choose_selected(self, message): + log(f'User chose {message.data=}') + self.dismiss( + [self._list_cache.get(listname) for listname in message.data] + ) diff --git a/src/archive_reader/widgets.py b/src/archive_reader/widgets.py index 855de53..2ff3139 100644 --- a/src/archive_reader/widgets.py +++ b/src/archive_reader/widgets.py @@ -241,7 +241,7 @@ def __init__(self, mlist): super().__init__() def render(self): - return repr(self.mlist) + return '{}\n{}'.format(self.get('display_name'), self.name) @property def name(self): @@ -266,6 +266,7 @@ class EmailItem(ListItem): width: 1fr; margin: 1 1; height: auto; + padding: 2; } Label { padding: 1 2;