From 8daf26f8b0ff499a941d64ae799c723af5e3ac6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Bartosi=C5=84ski?= Date: Thu, 12 Dec 2024 16:19:19 +0100 Subject: [PATCH 1/3] Remove dead code --- bin/inbox-api.py | 1 - inbox/api/wsgi.py | 56 --------------- inbox/contacts/vcard.py | 126 ---------------------------------- inbox/exceptions.py | 8 --- inbox/heartbeat/status.py | 3 - inbox/models/constants.py | 2 - inbox/models/session.py | 23 ------- inbox/sendmail/message.py | 2 - inbox/sendmail/smtp/postel.py | 3 - inbox/sqlalchemy_ext/util.py | 29 -------- inbox/util/misc.py | 21 ------ inbox/util/sharding.py | 35 ---------- inbox/util/startup.py | 7 -- inbox/util/testutils.py | 31 --------- 14 files changed, 347 deletions(-) diff --git a/bin/inbox-api.py b/bin/inbox-api.py index 643f1de2e..26f276e0d 100755 --- a/bin/inbox-api.py +++ b/bin/inbox-api.py @@ -28,7 +28,6 @@ from inbox.util.startup import load_overrides syncback = None -http_server = None @click.command() diff --git a/inbox/api/wsgi.py b/inbox/api/wsgi.py index 7f8778abc..5c45f92ea 100644 --- a/inbox/api/wsgi.py +++ b/inbox/api/wsgi.py @@ -1,7 +1,5 @@ -import logging import sys -import json_log_formatter from gunicorn.workers.gthread import ThreadWorker from inbox.error_handling import maybe_enable_rollbar @@ -28,58 +26,4 @@ def init_process(self) -> None: LOGLEVEL = config.get("LOGLEVEL", 10) -class JsonRequestFormatter(json_log_formatter.JSONFormatter): - """ - Custom JSON log formatter for gunicorn access logs. - - Adapted from https://til.codeinthehole.com/posts/how-to-get-gunicorn-to-log-as-json/ - """ - - def json_record( - self, - message: str, - extra: "dict[str, str | int | float]", - record: logging.LogRecord, - ) -> "dict[str, str | int | float]": - # Convert the log record to a JSON object. - # See https://docs.gunicorn.org/en/stable/settings.html#access-log-format - - url = record.args["U"] - if record.args["q"]: - url += f"?{record.args['q']}" - - method = record.args["m"] - log_context = record.args.get("{log_context}e", {}) - - return dict( - response_bytes=record.args["B"], - request_time=float(record.args["L"]), - remote_address=record.args["h"], - http_status=record.args["s"], - http_request=f"{method} {url}", - request_method=method, - **log_context, - ) - - -class JsonErrorFormatter(json_log_formatter.JSONFormatter): - """ - Custom JSON log formatter for gunicorn error logs. - - Adapted from https://til.codeinthehole.com/posts/how-to-get-gunicorn-to-log-as-json/ - """ - - def json_record( - self, - message: str, - extra: "dict[str, str | int | float]", - record: logging.LogRecord, - ) -> "dict[str, str | int | float]": - payload: "dict[str, str | int | float]" = super().json_record( - message, extra, record - ) - payload["level"] = record.levelname - return payload - - __all__ = ["NylasWSGIWorker"] diff --git a/inbox/contacts/vcard.py b/inbox/contacts/vcard.py index 10e8114e8..79ade36ab 100644 --- a/inbox/contacts/vcard.py +++ b/inbox/contacts/vcard.py @@ -33,116 +33,10 @@ import vobject - -def list_clean(string): # noqa: ANN201 - """ - Transforms a comma seperated string to a list, stripping whitespaces - "HOME, WORK,pref" -> ['HOME', 'WORK', 'pref'] - - string: string of comma seperated elements - returns: list() - """ # noqa: D401 - string = string.split(",") - rstring = list() - for element in string: - rstring.append(element.strip(" ")) - return rstring - - -NO_STRINGS = ["n", "no"] -YES_STRINGS = ["y", "yes"] - -PROPERTIES = ["EMAIL", "TEL"] -PROPS_ALL = [ - "FN", - "N", - "VERSION", - "NICKNAME", - "PHOTO", - "BDAY", - "ADR", - "LABEL", - "TEL", - "EMAIL", - "MAILER", - "TZ", - "GEO", - "TITLE", - "ROLE", - "LOGO", - "AGENT", - "ORG", - "NOTE", - "REV", - "SOUND", - "URL", - "UID", - "KEY", - "CATEGORIES", - "PRODID", - "REV", - "SORT-STRING", - "SOUND", - "URL", - "VERSION", - "UTC-OFFSET", -] -PROPS_ALLOWED = [ - "NICKNAME", - "BDAY", - "ADR", - "LABEL", - "TEL", - "EMAIL", - "MAILER", - "TZ", - "GEO", - "TITLE", - "ROLE", - "AGENT", - "ORG", - "NOTE", - "REV", - "SOUND", - "URL", - "UID", - "KEY", - "CATEGORIES", - "PRODID", - "REV", - "SORT-STRING", - "SOUND", - "URL", - "VERSION", - "UTC-OFFSET", -] -PROPS_ONCE = ["FN", "N", "VERSION"] -PROPS_LIST = ["NICKNAME", "CATEGORIES"] -PROPS_BIN = ["PHOTO", "LOGO", "SOUND", "KEY"] - - -RTEXT = "\x1b[7m" NTEXT = "\x1b[0m" BTEXT = "\x1b[1m" -def get_names(display_name): # noqa: ANN201 - first_name, last_name = "", display_name - - if display_name.find(",") > 0: - # Parsing something like 'Doe, John Abraham' - last_name, first_name = display_name.split(",") - - elif display_name.find(" "): - # Parsing something like 'John Abraham Doe' - # TODO: This fails for compound names. What is the most common case? - name_list = display_name.split(" ") - last_name = "".join(name_list[-1]) - first_name = " ".join(name_list[:-1]) - - return (first_name.strip().capitalize(), last_name.strip().capitalize()) - - def fix_vobject(vcard): # noqa: ANN201 """ Trying to fix some more or less common errors in vcards @@ -193,26 +87,6 @@ def vcard_from_string(vcard_string): # noqa: ANN201 return vcard_from_vobject(vcard) -def vcard_from_email(display_name, email): # noqa: ANN201 - fname, lname = get_names(display_name) - vcard = vobject.vCard() - vcard.add("n") - vcard.n.value = vobject.vcard.Name(family=lname, given=fname) - vcard.add("fn") - vcard.fn.value = display_name - vcard.add("email") - vcard.email.value = email - vcard.email.type_param = "INTERNET" - return vcard_from_vobject(vcard) - - -def cards_from_file(cards_f): # noqa: ANN201 - collector = list() - for vcard in vobject.readComponents(cards_f): - collector.append(vcard_from_vobject(vcard)) - return collector - - class VCard(defaultdict): """ internal representation of a VCard. This is dict with some diff --git a/inbox/exceptions.py b/inbox/exceptions.py index c654f4c74..a9d8e101b 100644 --- a/inbox/exceptions.py +++ b/inbox/exceptions.py @@ -22,18 +22,10 @@ class OAuthError(ValidationError): pass -class ConfigurationError(Exception): - pass - - class UserRecoverableConfigError(Exception): pass -class SettingUpdateError(Exception): - pass - - class GmailSettingError(ValidationError): pass diff --git a/inbox/heartbeat/status.py b/inbox/heartbeat/status.py index cf12290bc..b05902d1c 100644 --- a/inbox/heartbeat/status.py +++ b/inbox/heartbeat/status.py @@ -1,13 +1,10 @@ import time from collections import namedtuple -from datetime import timedelta from inbox.heartbeat.config import ALIVE_EXPIRY from inbox.heartbeat.store import HeartbeatStore from inbox.logging import get_logger -ALIVE_THRESHOLD = timedelta(seconds=ALIVE_EXPIRY) - log = get_logger() diff --git a/inbox/models/constants.py b/inbox/models/constants.py index 231bbba9b..3d9ca39e8 100644 --- a/inbox/models/constants.py +++ b/inbox/models/constants.py @@ -1,5 +1,3 @@ # Size constants, set in this file to avoid annoying circular import # errors MAX_INDEXABLE_LENGTH = 191 -MAX_FOLDER_NAME_LENGTH = MAX_INDEXABLE_LENGTH -MAX_LABEL_NAME_LENGTH = MAX_INDEXABLE_LENGTH diff --git a/inbox/models/session.py b/inbox/models/session.py index e4dd31e68..0181ee275 100644 --- a/inbox/models/session.py +++ b/inbox/models/session.py @@ -18,29 +18,6 @@ MAX_SANE_TRX_TIME_MS = 30000 -def two_phase_session( # noqa: ANN201, D417 - engine_map, versioned: bool = True -): - """ - Returns a session that implements two-phase-commit. - - Parameters - ---------- - engine_map: dict - Mapping of Table cls instance: database engine - - versioned: bool - - """ # noqa: D401 - session = Session( - binds=engine_map, twophase=True, autoflush=True, autocommit=False - ) - if versioned: - session = configure_versioning(session) - # TODO[k]: Metrics for transaction latencies! - return session - - def new_session(engine, versioned: bool = True): # noqa: ANN201 """Returns a session bound to the given engine.""" # noqa: D401 session = Session(bind=engine, autoflush=True, autocommit=False) diff --git a/inbox/sendmail/message.py b/inbox/sendmail/message.py index 8b29370ba..e272514ac 100644 --- a/inbox/sendmail/message.py +++ b/inbox/sendmail/message.py @@ -26,8 +26,6 @@ from inbox import VERSION -REPLYSTR = "Re: " - # Patch flanker to use base64 rather than quoted-printable encoding for # MIME parts with long lines. Flanker's implementation of quoted-printable diff --git a/inbox/sendmail/smtp/postel.py b/inbox/sendmail/smtp/postel.py index fdc32a9f6..929c17903 100644 --- a/inbox/sendmail/smtp/postel.py +++ b/inbox/sendmail/smtp/postel.py @@ -23,9 +23,6 @@ from .util import SMTP_ERRORS # noqa: E402 -# TODO[k]: Other types (LOGIN, XOAUTH, PLAIN-CLIENTTOKEN, CRAM-MD5) -AUTH_EXTNS = {"oauth2": "XOAUTH2", "password": "PLAIN"} - SMTP_MAX_RETRIES = 1 # Timeout in seconds for blocking operations. If no timeout is specified, # attempts to, say, connect to the wrong port may hang forever. diff --git a/inbox/sqlalchemy_ext/util.py b/inbox/sqlalchemy_ext/util.py index 6af47b98b..16fbefcb5 100644 --- a/inbox/sqlalchemy_ext/util.py +++ b/inbox/sqlalchemy_ext/util.py @@ -350,35 +350,6 @@ def receive_connect(dbapi_connection, connection_record) -> None: dbapi_connection.encoding = "utf8-surrogate-fix" -def safer_yield_per(query, id_field, start_id, count): # noqa: ANN201, D417 - """ - Incautious execution of 'for result in query.yield_per(N):' may cause - slowness or OOMing over large tables. This is a less general but less - dangerous alternative. - - Parameters - ---------- - query: sqlalchemy.Query - The query to yield windowed results from. - id_field: A SQLAlchemy attribute to use for windowing. E.g., - `Transaction.id` - start_id: The value of id_field at which to start iterating. - count: int - The number of results to fetch at a time. - - """ - cur_id = start_id - while True: - results = ( - query.filter(id_field >= cur_id) - .order_by(id_field) - .limit(count) - .all() - ) - yield from results - cur_id = results[-1].id + 1 - - def get_db_api_cursor_with_query(session, query): # noqa: ANN201 """ Return a DB-API cursor with the given SQLAlchemy query executed. diff --git a/inbox/util/misc.py b/inbox/util/misc.py index 46a4354f8..d0220150b 100644 --- a/inbox/util/misc.py +++ b/inbox/util/misc.py @@ -22,10 +22,6 @@ def __exit__( return False -class ProviderSpecificException(Exception): - pass - - def or_none(value, selector): # noqa: ANN201 if value is None: return None @@ -33,23 +29,6 @@ def or_none(value, selector): # noqa: ANN201 return selector(value) -def parse_ml_headers(headers): # noqa: ANN201 - """ - Parse the mailing list headers described in RFC 4021, - these headers are optional (RFC 2369). - - """ - return { - "List-Archive": headers.get("List-Archive"), - "List-Help": headers.get("List-Help"), - "List-Id": headers.get("List-Id"), - "List-Owner": headers.get("List-Owner"), - "List-Post": headers.get("List-Post"), - "List-Subscribe": headers.get("List-Subscribe"), - "List-Unsubscribe": headers.get("List-Unsubscribe"), - } - - def parse_references(references: str, in_reply_to: str) -> list[str]: """ Parse a References: header and returns an array of MessageIDs. diff --git a/inbox/util/sharding.py b/inbox/util/sharding.py index 69368e640..1b6045f73 100644 --- a/inbox/util/sharding.py +++ b/inbox/util/sharding.py @@ -1,26 +1,4 @@ -import random - from inbox.config import config -from inbox.ignition import engine_manager - - -def get_shards(): # noqa: ANN201 - return list(engine_manager.engines) - - -def get_open_shards(): # noqa: ANN201 - # Can't use engine_manager.engines here because it does not track - # shard state (open/ closed) - database_hosts = config.get_required("DATABASE_HOSTS") - open_shards: list[int] = [] - for host in database_hosts: - open_shards.extend( - shard["ID"] - for shard in host["SHARDS"] - if shard["OPEN"] and not shard.get("DISABLED") - ) - - return open_shards def get_shard_schemas(): # noqa: ANN201 @@ -35,16 +13,3 @@ def get_shard_schemas(): # noqa: ANN201 schema_name = shard["SCHEMA_NAME"] shard_schemas[shard_id] = schema_name return shard_schemas - - -def generate_open_shard_key(): # noqa: ANN201 - """ - Return the key that can be passed into session_scope() for an open shard, - picked at random. - - """ - open_shards = get_open_shards() - # TODO[k]: Always pick min()instead? - shard_id = random.choice(open_shards) - key = shard_id << 48 - return key diff --git a/inbox/util/startup.py b/inbox/util/startup.py index 5dea8f4c5..50b86a733 100644 --- a/inbox/util/startup.py +++ b/inbox/util/startup.py @@ -11,13 +11,6 @@ log = get_logger() -def _absolute_path(relative_path): - return os.path.join( # noqa: PTH118 - os.path.dirname(os.path.abspath(__file__)), # noqa: PTH100, PTH120 - relative_path, - ) - - def check_sudo() -> None: if os.getuid() == 0: raise Exception("Don't run the Nylas Sync Engine as root!") diff --git a/inbox/util/testutils.py b/inbox/util/testutils.py index b7f405034..ef9d9f472 100644 --- a/inbox/util/testutils.py +++ b/inbox/util/testutils.py @@ -107,37 +107,6 @@ def mock_dns_resolver(monkeypatch): # noqa: ANN201 monkeypatch.undo() -@pytest.fixture -def dump_dns_queries(monkeypatch): # noqa: ANN201 - original_query = dns.resolver.Resolver.query - query_results: dict[ - Literal["mx", "ns"], dict[str, dict[Literal["error"], str] | list[str]] - ] = {"ns": {}, "mx": {}} - - def mock_query(self, domain, record_type): - try: - result = original_query(self, domain, record_type) - except Exception as e: - query_results[record_type.lower()][domain] = { - "error": type(e).__name__ - } - raise - record_type = record_type.lower() - if record_type == "mx": - query_results["mx"][domain] = [ - str(r.exchange).lower() for r in result - ] - elif record_type == "ns": - query_results["ns"][domain] = [str(rdata) for rdata in result] - else: - raise RuntimeError(f"Unknown record type: {record_type}") - return result - - monkeypatch.setattr("dns.resolver.Resolver.query", mock_query) - yield - print(json.dumps(query_results, indent=4, sort_keys=True)) # noqa: T201 - - class MockIMAPClient: """ A bare-bones stand-in for an IMAPClient instance, used to test sync From f2f105949456fabfd4e3936f4b0017c08dd5985f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Bartosi=C5=84ski?= Date: Fri, 13 Dec 2024 06:20:15 +0100 Subject: [PATCH 2/3] Restore formatters --- inbox/api/wsgi.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/inbox/api/wsgi.py b/inbox/api/wsgi.py index 5c45f92ea..7f8778abc 100644 --- a/inbox/api/wsgi.py +++ b/inbox/api/wsgi.py @@ -1,5 +1,7 @@ +import logging import sys +import json_log_formatter from gunicorn.workers.gthread import ThreadWorker from inbox.error_handling import maybe_enable_rollbar @@ -26,4 +28,58 @@ def init_process(self) -> None: LOGLEVEL = config.get("LOGLEVEL", 10) +class JsonRequestFormatter(json_log_formatter.JSONFormatter): + """ + Custom JSON log formatter for gunicorn access logs. + + Adapted from https://til.codeinthehole.com/posts/how-to-get-gunicorn-to-log-as-json/ + """ + + def json_record( + self, + message: str, + extra: "dict[str, str | int | float]", + record: logging.LogRecord, + ) -> "dict[str, str | int | float]": + # Convert the log record to a JSON object. + # See https://docs.gunicorn.org/en/stable/settings.html#access-log-format + + url = record.args["U"] + if record.args["q"]: + url += f"?{record.args['q']}" + + method = record.args["m"] + log_context = record.args.get("{log_context}e", {}) + + return dict( + response_bytes=record.args["B"], + request_time=float(record.args["L"]), + remote_address=record.args["h"], + http_status=record.args["s"], + http_request=f"{method} {url}", + request_method=method, + **log_context, + ) + + +class JsonErrorFormatter(json_log_formatter.JSONFormatter): + """ + Custom JSON log formatter for gunicorn error logs. + + Adapted from https://til.codeinthehole.com/posts/how-to-get-gunicorn-to-log-as-json/ + """ + + def json_record( + self, + message: str, + extra: "dict[str, str | int | float]", + record: logging.LogRecord, + ) -> "dict[str, str | int | float]": + payload: "dict[str, str | int | float]" = super().json_record( + message, extra, record + ) + payload["level"] = record.levelname + return payload + + __all__ = ["NylasWSGIWorker"] From 7bc163eff93846a9c4e9ce5dcc417c12d9f656e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrzej=20Bartosi=C5=84ski?= Date: Mon, 16 Dec 2024 10:38:07 +0100 Subject: [PATCH 3/3] Remove comment --- inbox/api/validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/inbox/api/validation.py b/inbox/api/validation.py index 3b3f6ae1f..43e1e4d1a 100644 --- a/inbox/api/validation.py +++ b/inbox/api/validation.py @@ -485,7 +485,6 @@ def valid_display_name( # noqa: ANN201 display_name = display_name.rstrip() if len(display_name) > MAX_INDEXABLE_LENGTH: - # Set as MAX_FOLDER_LENGTH, MAX_LABEL_LENGTH raise InputError('"display_name" is too long') if (