From 3994c54b7315d66c8b605ba9af293afbdc508e78 Mon Sep 17 00:00:00 2001 From: Alec Rosenbaum Date: Tue, 8 Oct 2019 11:33:17 -0400 Subject: [PATCH] Add local testing infrastructure (#149) * initial commit. This will likely need to be updated to work in ci * pull redis host from environment, use requirements file * formatting * downgrade pytest so it still supports python 2.7 --- .circleci/config.yml | 5 ++++- Dockerfile | 6 ++++++ README.rst | 16 ++++++++++++++++ docker-compose.yml | 16 ++++++++++++++++ requirements-test.txt | 3 +++ tests/config.py | 7 ++++++- tests/tasks.py | 18 +++++++++--------- tests/tasks_periodic.py | 4 ++-- tests/test_context_manager.py | 6 ++++-- tests/utils.py | 4 ++-- 10 files changed, 68 insertions(+), 17 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 requirements-test.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index 4eb31e0b..73abd987 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,11 +12,14 @@ workflows: defaults: &defaults working_directory: ~/code + environment: + # circleci exposes services on localhost + REDIS_HOST: localhost steps: - checkout - run: name: Install dependencies - command: sudo pip install -r requirements.txt freezefrog psutil pytest + command: sudo pip install -r requirements.txt -r requirements-test.txt - run: name: Test command: pytest diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..6fe300f6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM circleci/python:3.6 + +WORKDIR /src +COPY requirements.txt . +COPY requirements-test.txt . +RUN pip install --user -r requirements.txt -r requirements-test.txt diff --git a/README.rst b/README.rst index c6518326..4e1730bb 100644 --- a/README.rst +++ b/README.rst @@ -718,3 +718,19 @@ reported to Rollbar. Here is a custom worker launch script: tiger.log.addHandler(rollbar_handler) tiger.run_worker_with_args(sys.argv[1:]) + + +Running The Test Suite +---------------------- + +Tests can be run locally using the provided docker compose file. After installing docker, tests should be runnable with: + +.. code :: bash + + docker-compose run --rm tasktiger pytest + +Tests can be more granularly run using normal pytest flags. For example: + +.. code :: bash + + docker-compose run --rm tasktiger pytest tests/test_base.py::TestCase diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..89dff7b2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: "3.7" +services: + redis: + image: redis:4.0.6 + expose: + - 6379 + tasktiger: + build: + context: . + dockerfile: Dockerfile + environment: + REDIS_HOST: redis + volumes: + - .:/src + depends_on: + - redis diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..3c43ddd3 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,3 @@ +freezefrog==0.3.2 +psutil==5.6.3 +pytest==4.6.5 \ No newline at end of file diff --git a/tests/config.py b/tests/config.py index 9b118547..fcc6b860 100644 --- a/tests/config.py +++ b/tests/config.py @@ -1,5 +1,10 @@ +import os + # How much to delay scheduled tasks for testing purposes. DELAY = 0.2 # Redis database number which will be wiped and used for the tests -TEST_DB = 7 +TEST_DB = int(os.environ.get('REDIS_DB', 7)) + +# Redis hostname +REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost') diff --git a/tests/tasks.py b/tests/tasks.py index 5ff82413..ff8d372f 100644 --- a/tests/tasks.py +++ b/tests/tasks.py @@ -7,7 +7,7 @@ from tasktiger import RetryException from tasktiger.retry import fixed -from .config import DELAY, TEST_DB +from .config import DELAY, TEST_DB, REDIS_HOST from .utils import get_tiger @@ -52,7 +52,7 @@ def long_task_killed(): @tiger.task(hard_timeout=DELAY * 2) def long_task_ok(): # Signal task has started - conn = redis.Redis(db=TEST_DB, decode_responses=True) + conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True) conn.lpush(LONG_TASK_SIGNAL_KEY, '1') time.sleep(DELAY) @@ -60,20 +60,20 @@ def long_task_ok(): def wait_for_long_task(): """Waits for a long task to start.""" - conn = redis.Redis(db=TEST_DB, decode_responses=True) + conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True) result = conn.blpop(LONG_TASK_SIGNAL_KEY, int(ceil(DELAY * 3))) assert result[1] == '1' @tiger.task(unique=True) def unique_task(value=None): - conn = redis.Redis(db=TEST_DB, decode_responses=True) + conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True) conn.lpush('unique_task', value) @tiger.task(lock=True) def locked_task(key, other=None): - conn = redis.Redis(db=TEST_DB, decode_responses=True) + conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True) data = conn.getset(key, 1) if data is not None: raise Exception('task failed, key already set') @@ -83,7 +83,7 @@ def locked_task(key, other=None): @tiger.task(queue='batch', batch=True) def batch_task(params): - conn = redis.Redis(db=TEST_DB, decode_responses=True) + conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True) try: conn.rpush('batch_task', json.dumps(params)) except Exception: @@ -94,7 +94,7 @@ def batch_task(params): @tiger.task(queue='batch') def non_batch_task(arg): - conn = redis.Redis(db=TEST_DB, decode_responses=True) + conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True) conn.rpush('batch_task', arg) if arg == 10: raise Exception('exception') @@ -109,7 +109,7 @@ def retry_task_2(): def verify_current_task(): - conn = redis.Redis(db=TEST_DB, decode_responses=True) + conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True) try: tiger.current_tasks @@ -121,7 +121,7 @@ def verify_current_task(): @tiger.task(batch=True, queue='batch') def verify_current_tasks(tasks): - conn = redis.Redis(db=TEST_DB, decode_responses=True) + conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True) try: tasks = tiger.current_task diff --git a/tests/tasks_periodic.py b/tests/tasks_periodic.py index 25da7732..04c21a60 100644 --- a/tests/tasks_periodic.py +++ b/tests/tasks_periodic.py @@ -4,7 +4,7 @@ from tasktiger.schedule import periodic -from .config import TEST_DB +from .config import TEST_DB, REDIS_HOST from .utils import get_tiger tiger = get_tiger() @@ -13,7 +13,7 @@ @tiger.task(schedule=periodic(seconds=1), queue='periodic') def periodic_task(): """Periodic task.""" - conn = redis.Redis(db=TEST_DB, decode_responses=True) + conn = redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True) conn.incr('period_count', 1) diff --git a/tests/test_context_manager.py b/tests/test_context_manager.py index 25f7cf6e..b3310225 100644 --- a/tests/test_context_manager.py +++ b/tests/test_context_manager.py @@ -5,7 +5,7 @@ from .tasks import exception_task, simple_task from .test_base import BaseTestCase -from .config import TEST_DB +from .config import TEST_DB, REDIS_HOST class ContextManagerTester(object): @@ -17,7 +17,9 @@ class ContextManagerTester(object): def __init__(self, name): self.name = name - self.conn = redis.Redis(db=TEST_DB, decode_responses=True) + self.conn = redis.Redis( + host=REDIS_HOST, db=TEST_DB, decode_responses=True + ) self.conn.set('cm:{}:enter'.format(self.name), 0) self.conn.set('cm:{}:exit'.format(self.name), 0) diff --git a/tests/utils.py b/tests/utils.py index d3eba7b0..18c48be0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -3,7 +3,7 @@ import structlog from tasktiger import TaskTiger, Worker, fixed -from .config import DELAY, TEST_DB +from .config import DELAY, TEST_DB, REDIS_HOST TEST_TIGER_CONFIG = { # We need this 0 here so we don't pick up scheduled tasks when @@ -51,7 +51,7 @@ def setup_structlog(): def get_redis(): - return redis.Redis(db=TEST_DB, decode_responses=True) + return redis.Redis(host=REDIS_HOST, db=TEST_DB, decode_responses=True) def get_tiger():