fix: Fix the way in which we fetch existing emails

Previously, we were having issues with how we fetched emails as it
wasn't loading properly. The issue had to do with how th fetch
method was defined in the EmailManager.

src/archive_reader/app.py

@@ -31,141 +31,11 @@
     EmailItem,
 )
 from .core import ListManager, ThreadsManager
-
+from .screens import ThreadReadScreen, MailingListAddScreen
 
 DEFAULT_NOTIFY_TIMEOUT = 2
 
 
-class ThreadReadScreen(Screen):
-    """The main screen to read Email threads.
-
-    This is composed of multiple Emails, which are embedded inside a listview.
-    """
-
-    BINDINGS = [
-        ('escape', 'app.pop_screen', 'Close thread'),
-        ('r', 'update_emails', 'Refresh Emails'),
-    ]
-
-    DEFAULT_CSS = """
-    .main {
-        layout: grid;
-        grid-size: 2;
-        grid-columns: 9fr 1fr;
-    }
-    .sender {
-        padding: 0 1;
-    }
-    """
-
-    def __init__(self, *args, thread=None, thread_mgr=None, **kw):
-        self.thread = thread
-        self.thread_mgr = thread_mgr
-        super().__init__(*args, **kw)
-
-    def compose(self) -> ComposeResult:
-        header = Header()
-        header.text = self.thread.subject
-        yield header
-        yield LoadingIndicator()
-        with Horizontal(classes='main'):
-            yield ListView(id='thread-emails')
-            yield ListView(id='thread-authors')
-        yield Footer()
-
-    @work
-    async def load_emails(self):
-        reply_objs = await self.thread_mgr.emails(self.thread)
-        reply_emails = [
-            EmailItem(
-                email=reply,
-                id='message-id-{}'.format(reply.message_id_hash),
-            )
-            for reply in reply_objs
-        ]
-        try:
-            self.add_emails(reply_emails)
-            self.add_email_authors(reply_emails)
-        except Exception as ex:
-            log(ex)
-        self._hide_loading()
-
-    @work
-    async def action_update_emails(self):
-        await self.thread_mgr.update_emails(self.thread)
-        self.load_emails()
-        self.notify('Thread refresh complete.')
-
-    def add_emails(self, emails):
-        view = self.query_one('#thread-emails', ListView)
-        for email in emails:
-            view.append(email)
-
-    def add_email_authors(self, emails):
-        view = self.query_one('#thread-authors', ListView)
-        for email in emails:
-            view.append(ListItem(Static(f'{email.sender}', classes='sender')))
-
-    def on_mount(self):
-        self.load_emails()
-
-    def _show_loading(self):
-        self.query_one(LoadingIndicator).display = True
-
-    def _hide_loading(self):
-        self.query_one(LoadingIndicator).display = False
-
-
-class MailingListAddScreen(Screen):
-    """A new screen where you can search and subscribe to MailingLists.
-
-    This page will take the server as the input and load all the mailing lists on
-    that server.
-    """
-
-    DEFAULT_CSS = """
-    Screen {
-        align: center middle;
-    }
-    """
-    BINDINGS = [('escape', 'app.pop_screen', 'Pop screen')]
-
-    def __init__(self, *args, **kw):
-        super().__init__(*args, **kw)
-        self.list_manager = ListManager()
-        self._list_cache = {}
-
-    def compose(self):
-        yield Static('Hyperkitty Server URL', classes='label')
-        yield Input(placeholder='https://')
-        yield Static()
-        yield MailingListChoose(id='pick-mailinglist')
-        yield Footer()
-
-    @work(exclusive=True)
-    async def update_mailinglists(self, base_url):
-        lists_json = await self.list_manager.fetch_lists(base_url)
-        selection_list = self.query_one(SelectionList)
-        for ml in lists_json.get('results'):
-            self._list_cache[ml.get('name')] = ml
-            selection_list.add_option(
-                (
-                    f"{ml.get('display_name')} <\"{ml.get('name')}\">",
-                    ml.get('name'),
-                )
-            )
-
-    async def on_input_submitted(self, message: Input.Submitted):
-        self.base_url = message.value
-        self.update_mailinglists(self.base_url)
-
-    def on_mailing_list_choose_selected(self, message):
-        log(f'User chose {message.data=}')
-        self.dismiss(
-            [self._list_cache.get(listname) for listname in message.data]
-        )
-
-
 class ArchiveApp(App):
     """Textual reader app to read Hyperkitty (GNU Mailman's official Archiver) email archives."""
 
@@ -248,7 +118,6 @@ async def _clear_threads(self):
         threads_container = self.query_one('#threads', ListView)
         # First, clear the threads.
         clear_resp = threads_container.clear()
-        log(type(clear_resp))
         # .clear() returns an awaitable and gives the control back to
         # DOM to perform the action.
         await clear_resp
@@ -260,7 +129,6 @@ async def action_update_threads(self):
             timeout=DEFAULT_NOTIFY_TIMEOUT,
         )
         self.update_threads(self.current_mailinglist)
-        self._notify_update_complete()
 
     def _notify_update_complete(self):
         self.notify(
@@ -281,7 +149,7 @@ def thread_mgr(self):
     @work()
     async def update_threads(self, ml):
         header = self.query_one('#header', Header)
-        header.text = ml.name
+        header.text = '{} ({})'.format(ml.display_name, ml.name)
         await self._clear_threads()
         self._show_loading()
         self.current_mailinglist = ml
@@ -312,7 +180,7 @@ async def on_list_view_selected(self, item):
             log(f'Thread {item.item} was selected.')
             # Make sure that we cancel the workers so that nothing will interfere after
             # we have moved on to the next screen.
-            self.workers.cancel_all()
+            # self.workers.cancel_all()
             # Mark the threads as read.
             self.push_screen(
                 ThreadReadScreen(

src/archive_reader/core.py

@@ -1,10 +1,15 @@
 """Core business logic."""
 import asyncio
+import httpx
 from textual import log
 from .models import MailingList, Thread, EmailManager
 from .hyperkitty import hyperktty_client, fetch_urls
 
 
+class RemoteURLFetchException(Exception):
+    """Exception fetching remote URLs."""
+
+
 class ThreadsManager:
     """The purpose of threads manager is to create Thread models
     and deal with local storage into sqlite3 database.
@@ -19,6 +24,7 @@ class ThreadsManager:
 
     def __init__(self, mailinglist: MailingList) -> None:
         self.ml = mailinglist
+        self.email_mgr = EmailManager()
 
     # ================= Public API =================================
     async def threads(self):
@@ -36,11 +42,14 @@ async def emails(self, thread):
 
     async def update_emails(self, thread):
         """Load New Emails from remote."""
-        replies, _ = await fetch_urls([thread.emails], log)
+        try:
+            replies, _ = await fetch_urls([thread.emails], log)
+        except httpx.ConnectError:
+            log(f'Failed to get Email URLs {thread.emails}')
+            raise RemoteURLFetchException(thread.emails)
         reply_urls = [each.get('url') for each in replies[0].get('results')]
         log(f'Retrieved email urls {reply_urls}')
-        email_manager = EmailManager()
-        existing_emails = await EmailManager.filter(thread=thread.url).all()
+        existing_emails = await self.email_mgr.filter(thread=thread.url)
         existing_email_urls = set(email.url for email in existing_emails)
         new_urls = list(
             url for url in reply_urls if url not in existing_email_urls
@@ -51,15 +60,14 @@ async def update_emails(self, thread):
         replies, _ = await fetch_urls(new_urls)
         tasks = []
         for reply in replies:
-            tasks.append(email_manager.create(reply))
+            tasks.append(self.email_mgr.create(reply))
         results = await asyncio.gather(*tasks)
         return [result[0] for result in results]
 
     # ================= Private API ================================
 
     async def _load_emails_from_db(self, thread):
-        manager = EmailManager()
-        return await manager.filter(thread=thread.url).all()
+        return await self.email_mgr.filter(thread=thread.url)
 
     async def _load_threads_from_db(self):
         """Load all the existing threads from the db."""

src/archive_reader/hyperkitty.py

@@ -45,9 +45,7 @@ async def fetch_urls(urls, logger=None):
             tasks.append(
                 asyncio.ensure_future(client.get(url, follow_redirects=True))
             )
-
         results = await asyncio.gather(*tasks)
-
         for resp in results:
             if resp.status_code == 200:
                 success.append(resp.json())

src/archive_reader/models.py

@@ -102,4 +102,5 @@ async def create(self, json_data):
 
     get = Email.objects.get
 
-    filter = Email.objects.filter
+    async def filter(self, *args, **kw):
+        return await Email.objects.filter(*args, **kw).all()