Do not fetch oversized emails from IMAP (#643)

* Do not fetch oversized emails from IMAP

* Move constant

* move to try

* Rename constant

inbox/constants.py

@@ -0,0 +1 @@
+MAX_MESSAGE_BODY_LENGTH = 100 * 1024 * 1024  # 100 MB

inbox/crispin.py

@@ -25,6 +25,8 @@
 import imapclient.imapclient
 import imapclient.response_parser
 
+from inbox.constants import MAX_MESSAGE_BODY_LENGTH
+
 # Prevent "got more than 1000000 bytes" errors for servers that send more data.
 imaplib._MAXLINE = 10000000  # type: ignore
 
@@ -877,6 +879,17 @@ def uids(self, uids: List[int]) -> List[RawMessage]:
                 # corrupt other UID data. Also we don't always get a message
                 # back at the first try.
                 for _ in range(3):
+                    # We already reject parsing messages bigger than MAX_MESSAGE_BODY_PARSE_LENGTH.
+                    # We might as well not download them at all, save bandwidth, processing time
+                    # and prevent OOMs due to fetching oversized emails.
+                    fetched_size: Dict[int, Dict[bytes, Any]] = self.conn.fetch(
+                        uid, ["RFC822.SIZE"]
+                    )
+                    body_size = int(fetched_size.get(uid, {}).get(b"RFC822.SIZE", 0))
+                    if body_size > MAX_MESSAGE_BODY_LENGTH:
+                        log.warning("Skipping fetching of oversized message", uid=uid)
+                        continue
+
                     result: Dict[int, Dict[bytes, Any]] = self.conn.fetch(
                         uid, ["BODY.PEEK[]", "INTERNALDATE", "FLAGS"]
                     )

inbox/models/constants.py

@@ -3,4 +3,3 @@
 MAX_INDEXABLE_LENGTH = 191
 MAX_FOLDER_NAME_LENGTH = MAX_INDEXABLE_LENGTH
 MAX_LABEL_NAME_LENGTH = MAX_INDEXABLE_LENGTH
-MAX_MESSAGE_BODY_PARSE_LENGTH = 100 * 1024 * 1024  # 100 MB

inbox/models/message.py

@@ -34,11 +34,11 @@
 from sqlalchemy.sql.expression import false
 
 from inbox.config import config
+from inbox.constants import MAX_MESSAGE_BODY_LENGTH
 from inbox.logging import get_logger
 from inbox.models.account import Account
 from inbox.models.base import MailSyncBase
 from inbox.models.category import Category
-from inbox.models.constants import MAX_MESSAGE_BODY_PARSE_LENGTH
 from inbox.models.mixins import (
     DeletedAtMixin,
     HasPublicID,
@@ -318,7 +318,7 @@ def create_from_synced(
 
         try:
             body_length = len(body_string)
-            if body_length > MAX_MESSAGE_BODY_PARSE_LENGTH:
+            if body_length > MAX_MESSAGE_BODY_LENGTH:
                 raise MessageTooBigException(body_length)
             parsed: MimePart = mime.from_string(body_string)
             # Non-persisted instance attribute used by EAS.