Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ISort #24

Merged
merged 10 commits into from
Aug 18, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
5 changes: 5 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ jobs:
when: always
command: black --check .

- run:
name: isort
when: always
command: isort -c .

build:
machine:
image: ubuntu-1604:201903-01
Expand Down
12 changes: 6 additions & 6 deletions bin/backfix-duplicate-categories.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,17 @@
# associated. If two or more duplicate categories exist with associated
# messages, they are consolidated into a single category and the other
# categories are deleted
from itertools import chain

import click
from nylas.logging import configure_logging, get_logger
from sqlalchemy import func
from sqlalchemy.sql import and_, exists

from itertools import chain
from inbox.ignition import engine_manager
from nylas.logging import get_logger, configure_logging
from inbox.models import MessageCategory, Category
from inbox.models import Category, MessageCategory
from inbox.models.session import session_scope_by_shard_id

from sqlalchemy import func
from sqlalchemy.sql import exists, and_

configure_logging()
log = get_logger(purpose="duplicate-category-backfill")

Expand Down
5 changes: 2 additions & 3 deletions bin/backfix-generic-imap-separators.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,13 @@
# We previously didn't store IMAP path separators for generic imap accounts.
# This script backfixes the accounts.
import click
from nylas.logging import configure_logging, get_logger

from inbox.crispin import connection_pool

from nylas.logging import get_logger, configure_logging
from inbox.models.backends.generic import GenericAccount
from inbox.models.session import (
session_scope,
global_session_scope,
session_scope,
session_scope_by_shard_id,
)

Expand Down
19 changes: 11 additions & 8 deletions bin/balance-fleet
Original file line number Diff line number Diff line change
@@ -1,25 +1,28 @@
#!/usr/bin/env python
from gevent import monkey

monkey.patch_all()

import click
import json
import operator
import random
import time

import click
from munkres import Munkres, make_cost_matrix
from nylas.logging import get_logger, configure_logging
configure_logging()
log = get_logger()
from nylas.logging import configure_logging, get_logger

from inbox.config import config
from inbox.scheduling.deferred_migration import (DeferredAccountMigration,
DeferredAccountMigrationExecutor)
from inbox.models.session import global_session_scope
from inbox.models.account import Account
from inbox.models.session import global_session_scope, session_scope
from inbox.scheduling.deferred_migration import (
DeferredAccountMigration,
DeferredAccountMigrationExecutor,
)
from inbox.util import fleet
from inbox.models.session import session_scope

configure_logging()
log = get_logger()

# How long we should take to migrate all accounts (in seconds).
ACCOUNT_MIGRATION_TIMESPAN = 15 * 60 # 15 minutes
Expand Down
22 changes: 13 additions & 9 deletions bin/check-attachments
Original file line number Diff line number Diff line change
@@ -1,25 +1,29 @@
#!/usr/bin/env python
# Check that we can fetch attachments for 99.9% of our syncing accounts.
from gevent import monkey

monkey.patch_all()

import datetime
import traceback
from collections import defaultdict

import click
import gevent
from gevent.pool import Pool
import traceback
from nylas.logging import configure_logging, get_logger
from sqlalchemy.sql.expression import func

from collections import defaultdict
from inbox.models import Account, Block
from inbox.crispin import connection_pool
from inbox.models import Account, Block
from inbox.models.backends.generic import GenericAccount
from inbox.models.session import (
global_session_scope,
session_scope,
session_scope_by_shard_id,
)
from inbox.s3.base import get_raw_from_provider
from inbox.s3.exc import EmailFetchException, TemporaryEmailFetchException
from sqlalchemy.sql.expression import func

from nylas.logging import get_logger, configure_logging
from inbox.models.backends.generic import GenericAccount
from inbox.models.session import (session_scope, global_session_scope,
session_scope_by_shard_id)

configure_logging()
log = get_logger(purpose='separator-backfix')
Expand Down
15 changes: 10 additions & 5 deletions bin/clear-all-heartbeats
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
#!/usr/bin/env python

import click
from sys import exit
from redis import StrictRedis, BlockingConnectionPool

from inbox.heartbeat.config import (STATUS_DATABASE, get_redis_client,
MAX_CONNECTIONS, WAIT_TIMEOUT,
SOCKET_TIMEOUT)
import click
from redis import BlockingConnectionPool, StrictRedis

from inbox.heartbeat.config import (
MAX_CONNECTIONS,
SOCKET_TIMEOUT,
STATUS_DATABASE,
WAIT_TIMEOUT,
get_redis_client,
)


@click.command()
Expand Down
8 changes: 4 additions & 4 deletions bin/clear-db
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
#!/usr/bin/env python
from inbox.util.db import drop_everything

import sys
import argparse
import sys

from inbox.util.db import drop_everything


def main():
parser = argparse.ArgumentParser()
parser.add_argument('-u', '--with-users', action='store_true',
dest='with_users', default=False)
args = parser.parse_args()
from inbox.ignition import main_engine, init_db
from inbox.ignition import init_db, main_engine

engine = main_engine(pool_size=1)

Expand Down
5 changes: 3 additions & 2 deletions bin/clear-heartbeat-status
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
#!/usr/bin/env python

import click
from sys import exit

import click
from nylas.logging import configure_logging, get_logger

from inbox.config import config
from inbox.heartbeat.status import clear_heartbeat_status
from nylas.logging import configure_logging, get_logger

configure_logging(config.get('LOGLEVEL'))
log = get_logger()
Expand Down
11 changes: 8 additions & 3 deletions bin/clear-kv
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
#!/usr/bin/env python

import click
from sys import exit

from inbox.heartbeat.config import (STATUS_DATABASE, REPORT_DATABASE,
get_redis_client, _get_redis_client)
import click

from inbox.heartbeat.config import (
REPORT_DATABASE,
STATUS_DATABASE,
_get_redis_client,
get_redis_client,
)


@click.command()
Expand Down
9 changes: 6 additions & 3 deletions bin/contact-search-backfill
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
#!/usr/bin/env python
import click

from gevent import monkey

monkey.patch_all()

import gevent_openssl

gevent_openssl.monkey_patch()

import click
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we have a newline above this import (and above import gevent_openssl)? This applies in a few other cases within this commit as well.

from nylas.logging import configure_logging, get_logger

from inbox.contacts.search import index_namespace

from nylas.logging import get_logger, configure_logging
configure_logging()
log = get_logger()

Expand Down
2 changes: 1 addition & 1 deletion bin/contact-search-delete-index
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/usr/bin/env python
import click
from nylas.logging import configure_logging, get_logger

from inbox.contacts.search import delete_namespace_indexes as delete_indexes

from nylas.logging import get_logger, configure_logging
configure_logging()
log = get_logger()

Expand Down
19 changes: 11 additions & 8 deletions bin/contact-search-service
Original file line number Diff line number Diff line change
@@ -1,20 +1,23 @@
#!/usr/bin/env python
""" Start the contact search indexing service. """
import os
from setproctitle import setproctitle

import click
import gevent_openssl

gevent_openssl.monkey_patch()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

While this script is probably not used at all, ideally the only import above this line would be gevent_openssl.


from gevent import monkey

from inbox.config import config as inbox_config
from inbox.util.startup import preflight
monkey.patch_all()

import os

import click
from nylas.logging import configure_logging
from setproctitle import setproctitle

setproctitle('nylas-contact-search-index-service')
monkey.patch_all()
from inbox.config import config as inbox_config
from inbox.util.startup import preflight

setproctitle("nylas-contact-search-index-service")


@click.command()
Expand Down
6 changes: 3 additions & 3 deletions bin/create-db
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
#!/usr/bin/env python
import os

import alembic.command
import alembic.config
import click
import sqlalchemy
import alembic.config
import alembic.command

from inbox.config import config
from inbox.ignition import init_db, build_uri, EngineManager, verify_db
from inbox.ignition import EngineManager, build_uri, init_db, verify_db
from inbox.sqlalchemy_ext.util import ForceStrictMode


Expand Down
2 changes: 1 addition & 1 deletion bin/create-encryption-keys
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/usr/bin/env python
import binascii

import yaml
import nacl.secret
import nacl.utils
import yaml


def main():
Expand Down
6 changes: 3 additions & 3 deletions bin/create-event-contact-associations.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@
from __future__ import division

import click
from nylas.logging import configure_logging, get_logger
from sqlalchemy import asc

from inbox.ignition import engine_manager
from nylas.logging import get_logger, configure_logging
from inbox.contacts.processing import update_contacts_from_event
from inbox.ignition import engine_manager
from inbox.models import Event
from inbox.models.session import session_scope_by_shard_id
from inbox.models.util import limitlion
from sqlalchemy import asc

configure_logging()
log = get_logger(purpose="create-event-contact-associations")
Expand Down
5 changes: 4 additions & 1 deletion bin/deferred-migration-service
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@
"""Watches a redis priority queue for deferred account migrations to execute."""

import gevent.monkey

gevent.monkey.patch_all()

from nylas.logging import configure_logging
from setproctitle import setproctitle

from inbox.scheduling.deferred_migration import DeferredAccountMigrationExecutor
from nylas.logging import configure_logging

configure_logging()


Expand Down
4 changes: 2 additions & 2 deletions bin/delete-account-data
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ import time

import click

from inbox.models.session import session_scope
from inbox.heartbeat.status import clear_heartbeat_status
from inbox.models import Account
from inbox.models.session import session_scope
from inbox.models.util import delete_namespace
from inbox.heartbeat.status import clear_heartbeat_status


@click.command()
Expand Down
9 changes: 5 additions & 4 deletions bin/delete-marked-accounts
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,17 @@ Includes:

"""
from gevent import monkey

monkey.patch_all()

import logging

import click
import gevent
import logging
from nylas.logging import configure_logging, get_logger

from inbox.config import config
from inbox.models.util import get_accounts_to_delete, batch_delete_namespaces

from nylas.logging import get_logger, configure_logging
from inbox.models.util import batch_delete_namespaces, get_accounts_to_delete

configure_logging(logging.INFO)
log = get_logger()
Expand Down
1 change: 0 additions & 1 deletion bin/detect-missing-sync-host
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#!/usr/bin/env python

import click

from sqlalchemy.orm import load_only

from inbox.models.account import Account
Expand Down
4 changes: 3 additions & 1 deletion bin/get-account-loads
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
#!/usr/bin/env python
from gevent import monkey

monkey.patch_all()

import click
import json

import click
import requests

from inbox.config import config
Expand Down
2 changes: 1 addition & 1 deletion bin/get-accounts-for-host
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

import click

from inbox.models.session import global_session_scope
from inbox.models.account import Account
from inbox.models.session import global_session_scope


@click.command()
Expand Down
Loading