Skip to content

Commit

Permalink
Consumer does not shutdown properly when embedded in gevent applicati…
Browse files Browse the repository at this point in the history
…on (celery#3746)

fixes celery#3745
  • Loading branch information
arcivanov authored and auvipy committed Jan 9, 2017
1 parent ce8ea16 commit c1fa9af
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 3 deletions.
6 changes: 4 additions & 2 deletions celery/worker/consumer/consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@
__all__ = ['Consumer', 'Evloop', 'dump_body']

CLOSE = bootsteps.CLOSE
TERMINATE = bootsteps.TERMINATE
STOP_CONDITIONS = {CLOSE, TERMINATE}
logger = get_logger(__name__)
debug, info, warn, error, crit = (logger.debug, logger.info, logger.warning,
logger.error, logger.critical)
Expand Down Expand Up @@ -305,7 +307,7 @@ def _limit_task(self, request, bucket, tokens):

def start(self):
blueprint = self.blueprint
while blueprint.state != CLOSE:
while blueprint.state not in STOP_CONDITIONS:
maybe_shutdown()
if self.restart_count:
try:
Expand All @@ -324,7 +326,7 @@ def start(self):
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
raise # Too many open files
maybe_shutdown()
if blueprint.state != CLOSE:
if blueprint.state not in STOP_CONDITIONS:
if self.connection:
self.on_connection_error_after_connected(exc)
else:
Expand Down
32 changes: 31 additions & 1 deletion t/unit/worker/test_consumer.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@
from billiard.exceptions import RestartFreqExceeded

from celery.worker.consumer.agent import Agent
from celery.worker.consumer.consumer import CLOSE, Consumer, dump_body
from celery.worker.consumer.consumer import (CLOSE, TERMINATE,
Consumer, dump_body)
from celery.worker.consumer.gossip import Gossip
from celery.worker.consumer.heart import Heart
from celery.worker.consumer.mingle import Mingle
Expand Down Expand Up @@ -153,6 +154,35 @@ def se(*args, **kwargs):
c.start()
sleep.assert_called_with(1)

def test_do_not_restart_when_closed(self):
c = self.get_consumer()

c.blueprint.state = None

def bp_start(*args, **kwargs):
c.blueprint.state = CLOSE

c.blueprint.start.side_effect = bp_start
with patch('celery.worker.consumer.consumer.sleep'):
c.start()

c.blueprint.start.assert_called_once_with(c)

def test_do_not_restart_when_terminated(self):
c = self.get_consumer()

c.blueprint.state = None

def bp_start(*args, **kwargs):
c.blueprint.state = TERMINATE

c.blueprint.start.side_effect = bp_start

with patch('celery.worker.consumer.consumer.sleep'):
c.start()

c.blueprint.start.assert_called_once_with(c)

def test_no_retry_raises_error(self):
self.app.conf.broker_connection_retry = False
c = self.get_consumer()
Expand Down

0 comments on commit c1fa9af

Please sign in to comment.