From a3728f0ab8c919e0f71f8c2d159c77f429432a80 Mon Sep 17 00:00:00 2001
From: James Lawrence <james.lawrence@backupify.com>
Date: Mon, 10 Mar 2014 00:46:29 -0400
Subject: [PATCH 01/72] simple throttle for qless

---
 Makefile     | 22 +++++++--------
 base.lua     | 76 ++++++++++++++++++++++++++++++++++++++++++++--------
 job.lua      | 67 ++++++++++++++++++++++++++++-----------------
 queue.lua    | 75 ++++++++++++++++++++++++++++++---------------------
 throttle.lua | 32 ++++++++++++++++++++++
 5 files changed, 196 insertions(+), 76 deletions(-)
 create mode 100644 throttle.lua

diff --git a/Makefile b/Makefile
index 748c504..a8cf62b 100644
--- a/Makefile
+++ b/Makefile
@@ -1,21 +1,21 @@
 all: qless.lua qless-lib.lua
 
 qless-lib.lua: base.lua config.lua job.lua queue.lua recurring.lua worker.lua
-	echo "-- Current SHA: `git rev-parse HEAD`" > qless-lib.lua
-	echo "-- This is a generated file" >> qless-lib.lua
-	cat base.lua config.lua job.lua queue.lua recurring.lua worker.lua >> qless-lib.lua
+  echo "-- Current SHA: `git rev-parse HEAD`" > qless-lib.lua
+  echo "-- This is a generated file" >> qless-lib.lua
+  cat base.lua config.lua job.lua queue.lua recurring.lua worker.lua resource.lua >> qless-lib.lua
 
 qless.lua: qless-lib.lua api.lua
-	# Cat these files out, but remove all the comments from the source
-	echo "-- Current SHA: `git rev-parse HEAD`" > qless.lua
-	echo "-- This is a generated file" >> qless.lua
-	cat qless-lib.lua api.lua | \
-		egrep -v '^[[:space:]]*--[^\[]' | \
-		egrep -v '^--$$' >> qless.lua
+  # Cat these files out, but remove all the comments from the source
+  echo "-- Current SHA: `git rev-parse HEAD`" > qless.lua
+  echo "-- This is a generated file" >> qless.lua
+  cat qless-lib.lua api.lua | \
+    egrep -v '^[[:space:]]*--[^\[]' | \
+    egrep -v '^--$$' >> qless.lua
 
 clean:
-	rm -f qless.lua qless-lib.lua
+  rm -f qless.lua qless-lib.lua
 
 .PHONY: test
 test: qless.lua *.lua
-	nosetests --exe -v
+  nosetests --exe -v
diff --git a/base.lua b/base.lua
index 3bc7293..f4a3d0e 100644
--- a/base.lua
+++ b/base.lua
@@ -23,6 +23,12 @@ local QlessJob = {
 }
 QlessJob.__index = QlessJob
 
+-- throttle forward declaration
+local Qlessthrottle = {
+  ns = Qless.ns .. 'rs:'
+}
+Qlessthrottle.__index = Qlessthrottle;
+
 -- RecurringJob forward declaration
 local QlessRecurringJob = {}
 QlessRecurringJob.__index = QlessRecurringJob
@@ -61,19 +67,64 @@ function Qless.recurring(jid)
   return job
 end
 
+-- Return a throttle object
+-- throttle objects are used for arbitrary throttling of jobs.
+function Qless.throttle(tid)
+  assert(tid, 'Throttle(): no tid provided')
+  local throttle = {}
+  setmetatable(throttle, QlessThrottle)
+  throttle.id = rid
+
+  -- Set maximum for this throttle, if no maximum is defined it defaults to 0 (unlimited)
+  throttle.maximum = Qless.config.get(tonumber(Qless.config.get(throttle.prefix .. '-maximum')) or 0
+
+  -- set of jids which have acquired a lock on this throttle.
+  throttle.locks = {
+    count = function()
+      redis.call('scard', throttle:prefix .. '-locks')
+    end, add = function(...)
+      if #arg > 0 then
+        redis.call('sadd', throttle:prefix .. '-locks', unpack(arg))
+      end
+    end, remove = function(...)
+      if #arg > 0 then
+        return redis.call('srem', throttle:prefix .. '-locks', unpack(arg))
+      end
+    end
+  }
+
+  -- set of jids waiting on this throttle to become available.
+  throttle.pending = {
+    count = function()
+      redis.call('scard', throttle:prefix .. '-pending')
+    end, add = function(...)
+      if #arg > 0 then
+        redis.call('sadd', throttle:prefix .. '-pending', unpack(arg))
+      end
+    end, remove = function(...)
+      if #arg > 0 then
+        redis.call('srem', throttle:prefix .. '-pending', unpack(arg))
+      end
+    end, pop = function()
+      return redis.call('spop', throttle:prefix .. '-pending')
+    end
+  }
+  return throttle
+end
+
 -- Failed([group, [start, [limit]]])
 -- ------------------------------------
 -- If no group is provided, this returns a JSON blob of the counts of the
 -- various groups of failures known. If a group is provided, it will report up
 -- to `limit` from `start` of the jobs affected by that issue.
--- 
+--
 --  # If no group, then...
 --  {
 --      'group1': 1,
 --      'group2': 5,
 --      ...
 --  }
---  
+--
 --  # If a group is provided, then...
 --  {
 --      'total': 20,
@@ -119,9 +170,9 @@ end
 -------------------------------------------------------------------------------
 -- Return all the job ids currently considered to be in the provided state
 -- in a particular queue. The response is a list of job ids:
--- 
+--
 --  [
---      jid1, 
+--      jid1,
 --      jid2,
 --      ...
 --  ]
@@ -167,7 +218,7 @@ end
 -- associated with that id, and 'untrack' stops tracking it. In this context,
 -- tracking is nothing more than saving the job to a list of jobs that are
 -- considered special.
--- 
+--
 --  {
 --      'jobs': [
 --          {
@@ -252,7 +303,7 @@ function Qless.tag(now, command, ...)
       tags = cjson.decode(tags)
       local _tags = {}
       for i,v in ipairs(tags) do _tags[v] = true end
-    
+
       -- Otherwise, add the job to the sorted set with that tags
       for i=2,#arg do
         local tag = arg[i]
@@ -263,7 +314,7 @@ function Qless.tag(now, command, ...)
         redis.call('zadd', 'ql:t:' .. tag, now, jid)
         redis.call('zincrby', 'ql:tags', 1, tag)
       end
-    
+
       tags = cjson.encode(tags)
       redis.call('hset', QlessJob.ns .. jid, 'tags', tags)
       return tags
@@ -279,7 +330,7 @@ function Qless.tag(now, command, ...)
       tags = cjson.decode(tags)
       local _tags = {}
       for i,v in ipairs(tags) do _tags[v] = true end
-    
+
       -- Otherwise, add the job to the sorted set with that tags
       for i=2,#arg do
         local tag = arg[i]
@@ -287,10 +338,10 @@ function Qless.tag(now, command, ...)
         redis.call('zrem', 'ql:t:' .. tag, jid)
         redis.call('zincrby', 'ql:tags', -1, tag)
       end
-    
+
       local results = {}
       for i,tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
-    
+
       tags = cjson.encode(results)
       redis.call('hset', QlessJob.ns .. jid, 'tags', tags)
       return results
@@ -343,6 +394,7 @@ function Qless.cancel(...)
   -- If we've made it this far, then we are good to go. We can now just
   -- remove any trace of all these jobs, as they form a dependent clique
   for _, jid in ipairs(arg) do
+    local namespaced_jid = QlessJob.ns .. jid
     -- Find any stage it's associated with and remove its from that stage
     local state, queue, failure, worker = unpack(redis.call(
       'hmget', QlessJob.ns .. jid, 'state', 'queue', 'failure', 'worker'))
@@ -373,6 +425,8 @@ function Qless.cancel(...)
         queue.depends.remove(jid)
       end
 
+      Qless.job(namespaced_jid):release_throttles()
+
       -- We should probably go through all our dependencies and remove
       -- ourselves from the list of dependents
       for i, j in ipairs(redis.call(
@@ -418,7 +472,7 @@ function Qless.cancel(...)
       redis.call('del', QlessJob.ns .. jid .. '-history')
     end
   end
-  
+
   return arg
 end
 
diff --git a/job.lua b/job.lua
index 9098015..986dc52 100644
--- a/job.lua
+++ b/job.lua
@@ -11,7 +11,7 @@ function QlessJob:data(...)
   local job = redis.call(
       'hmget', QlessJob.ns .. self.jid, 'jid', 'klass', 'state', 'queue',
       'worker', 'priority', 'expires', 'retries', 'remaining', 'data',
-      'tags', 'failure')
+      'tags', 'failure', 'throttle')
 
   -- Return nil if we haven't found it
   if not job[1] then
@@ -34,6 +34,8 @@ function QlessJob:data(...)
     tags         = cjson.decode(job[11]),
     history      = self:history(),
     failure      = cjson.decode(job[12] or '{}'),
+    -- default to queue throttle if no throttle was specified.
+    throttle     = job[13] or QlessQueue.ns .. job[4],
     dependents   = redis.call(
       'smembers', QlessJob.ns .. self.jid .. '-dependents'),
     dependencies = redis.call(
@@ -55,11 +57,11 @@ end
 
 -- Complete a job and optionally put it in another queue, either scheduled or
 -- to be considered waiting immediately. It can also optionally accept other
--- jids on which this job will be considered dependent before it's considered 
+-- jids on which this job will be considered dependent before it's considered
 -- valid.
 --
 -- The variable-length arguments may be pairs of the form:
--- 
+--
 --      ('next'   , queue) : The queue to advance it to next
 --      ('delay'  , delay) : The delay for the next queue
 --      ('depends',        : Json of jobs it depends on in the new queue
@@ -74,7 +76,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
   -- Read in all the optional parameters
   local options = {}
   for i = 1, #arg, 2 do options[arg[i]] = arg[i + 1] end
-  
+
   -- Sanity check on optional args
   local nextq   = options['next']
   local delay   = assert(tonumber(options['delay'] or 0))
@@ -130,6 +132,8 @@ function QlessJob:complete(now, worker, queue, data, ...)
   queue_obj.locks.remove(self.jid)
   queue_obj.scheduled.remove(self.jid)
 
+  self:release_throttle()
+
   ----------------------------------------------------------
   -- This is the massive stats update that we have to do
   ----------------------------------------------------------
@@ -167,7 +171,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
     if redis.call('zscore', 'ql:queues', nextq) == false then
       redis.call('zadd', 'ql:queues', now, nextq)
     end
-    
+
     redis.call('hmset', QlessJob.ns .. self.jid,
       'state', 'waiting',
       'worker', '',
@@ -175,7 +179,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
       'queue', nextq,
       'expires', 0,
       'remaining', tonumber(retries))
-    
+
     if (delay > 0) and (#depends == 0) then
       queue_obj.scheduled.add(now + delay, self.jid)
       return 'scheduled'
@@ -223,18 +227,18 @@ function QlessJob:complete(now, worker, queue, data, ...)
       'queue', '',
       'expires', 0,
       'remaining', tonumber(retries))
-    
+
     -- Do the completion dance
     local count = Qless.config.get('jobs-history-count')
     local time  = Qless.config.get('jobs-history')
-    
+
     -- These are the default values
     count = tonumber(count or 50000)
     time  = tonumber(time  or 7 * 24 * 60 * 60)
-    
+
     -- Schedule this job for destructination eventually
     redis.call('zadd', 'ql:completed', now, self.jid)
-    
+
     -- Now look at the expired job data. First, based on the current time
     local jids = redis.call('zrangebyscore', 'ql:completed', 0, now - time)
     -- Any jobs that need to be expired... delete
@@ -250,7 +254,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
     end
     -- And now remove those from the queued-for-cleanup queue
     redis.call('zremrangebyscore', 'ql:completed', 0, now - time)
-    
+
     -- Now take the all by the most recent 'count' ids
     jids = redis.call('zrange', 'ql:completed', 0, (-1-count))
     for index, jid in ipairs(jids) do
@@ -264,7 +268,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
       redis.call('del', QlessJob.ns .. jid .. '-history')
     end
     redis.call('zremrangebyrank', 'ql:completed', 0, (-1-count))
-    
+
     -- Alright, if this has any dependents, then we should go ahead
     -- and unstick those guys.
     for i, j in ipairs(redis.call(
@@ -288,10 +292,10 @@ function QlessJob:complete(now, worker, queue, data, ...)
         end
       end
     end
-    
+
     -- Delete our dependents key
     redis.call('del', QlessJob.ns .. self.jid .. '-dependents')
-    
+
     return 'complete'
   end
 end
@@ -302,14 +306,14 @@ end
 -- specific message. By `group`, we mean some phrase that might be one of
 -- several categorical modes of failure. The `message` is something more
 -- job-specific, like perhaps a traceback.
--- 
+--
 -- This method should __not__ be used to note that a job has been dropped or
 -- has failed in a transient way. This method __should__ be used to note that
 -- a job has something really wrong with it that must be remedied.
--- 
+--
 -- The motivation behind the `group` is so that similar errors can be grouped
 -- together. Optionally, updated data can be provided for the job. A job in
--- any state can be marked as failed. If it has been given to a worker as a 
+-- any state can be marked as failed. If it has been given to a worker as a
 -- job, then its subsequent requests to heartbeat or complete that job will
 -- fail. Failed jobs are kept until they are canceled or completed.
 --
@@ -380,7 +384,7 @@ function QlessJob:fail(now, worker, group, message, data)
   queue_obj.locks.remove(self.jid)
   queue_obj.scheduled.remove(self.jid)
 
-  -- The reason that this appears here is that the above will fail if the 
+  -- The reason that this appears here is that the above will fail if the
   -- job doesn't exist
   if data then
     redis.call('hset', QlessJob.ns .. self.jid, 'data', cjson.encode(data))
@@ -397,6 +401,8 @@ function QlessJob:fail(now, worker, group, message, data)
       ['worker']  = worker
     }))
 
+  self:release_throttle()
+
   -- Add this group of failure to the list of failures
   redis.call('sadd', 'ql:failures', group)
   -- And add this particular instance to the failed groups
@@ -417,7 +423,7 @@ end
 -- Throws an exception if:
 --      - the worker is not the worker with a lock on the job
 --      - the job is not actually running
--- 
+--
 -- Otherwise, it returns the number of retries remaining. If the allowed
 -- retries have been exhausted, then it is automatically failed, and a negative
 -- number is returned.
@@ -430,7 +436,7 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
   assert(worker, 'Retry(): Arg "worker" missing')
   delay = assert(tonumber(delay or 0),
     'Retry(): Arg "delay" not a number: ' .. tostring(delay))
-  
+
   -- Let's see what the old priority, and tags were
   local oldqueue, state, retries, oldworker, priority, failure = unpack(
     redis.call('hmget', QlessJob.ns .. self.jid, 'queue', 'state',
@@ -455,6 +461,9 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
   -- Remove it from the locks key of the old queue
   Qless.queue(oldqueue).locks.remove(self.jid)
 
+  -- Release the throttle for the job
+  self:release_throttle()
+
   -- Remove this job from the worker that was previously working it
   redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
 
@@ -463,7 +472,7 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
     -- queue it's in
     local group = group or 'failed-retries-' .. queue
     self:history(now, 'failed', {['group'] = group})
-    
+
     redis.call('hmset', QlessJob.ns .. self.jid, 'state', 'failed',
       'worker', '',
       'expires', '')
@@ -487,7 +496,7 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
         ['worker']  = unpack(self:data('worker'))
       }))
     end
-    
+
     -- Add this type of failure to the list of failures
     redis.call('sadd', 'ql:failures', group)
     -- And add this particular instance to the failed types
@@ -639,11 +648,11 @@ function QlessJob:heartbeat(now, worker, data)
       redis.call('hmset', QlessJob.ns .. self.jid,
         'expires', expires, 'worker', worker)
     end
-    
+
     -- Update hwen this job was last updated on that worker
     -- Add this job to the list of jobs handled by this worker
     redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, self.jid)
-    
+
     -- And now we should just update the locks
     local queue = Qless.queue(
       redis.call('hget', QlessJob.ns .. self.jid, 'queue'))
@@ -789,3 +798,13 @@ function QlessJob:history(now, what, item)
       cjson.encode({math.floor(now), what, item}))
   end
 end
+
+function QlessJob:release_throttle(now)
+  local tid = redis.call('hget', QlessJob.ns .. self.jid, 'throttle')
+  Qless.throttle(tid):release(now, self.jid)
+end
+
+function QlessJob:acquire_throttle(now)
+  local rid = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'throttle'))
+  return Qless.resource(rid):acquire(now, self.jid)
+end
diff --git a/queue.lua b/queue.lua
index fd42cd4..76291cf 100644
--- a/queue.lua
+++ b/queue.lua
@@ -77,6 +77,21 @@ function Qless.queue(name)
     end
   }
 
+  -- Access to our throttled jobs
+  queue.throttled = {
+    peek = function(now, offset, count)
+      return redis.call('srange', queue:prefix('throttled'), offset, offset + count - 1)
+    end, add = function(now, jid)
+      redis.call('sadd', queue:prefix('throttled'), jid)
+    end, remove = function(...)
+      if #arg > 0 then
+        return redis.call('srem', queue:prefix('throttled'), unpack(arg))
+      end
+    end, length = function()
+      return redis.call('scard', queue:prefix('throttled'))
+    end
+  }
+
   -- Access to our scheduled jobs
   queue.scheduled = {
     peek = function(now, offset, count)
@@ -182,11 +197,11 @@ function QlessQueue:stats(now, date)
 
     local key = 'ql:s:' .. name .. ':' .. bin .. ':' .. queue
     local count, mean, vk = unpack(redis.call('hmget', key, 'total', 'mean', 'vk'))
-    
+
     count = tonumber(count) or 0
     mean  = tonumber(mean) or 0
     vk    = tonumber(vk)
-    
+
     results.count     = count or 0
     results.mean      = mean  or 0
     results.histogram = {}
@@ -236,8 +251,8 @@ function QlessQueue:peek(now, count)
 
   -- Now we've checked __all__ the locks for this queue the could
   -- have expired, and are no more than the number requested. If
-  -- we still need values in order to meet the demand, then we 
-  -- should check if any scheduled items, and if so, we should 
+  -- we still need values in order to meet the demand, then we
+  -- should check if any scheduled items, and if so, we should
   -- insert them to ensure correctness when pulling off the next
   -- unit of work.
   self:check_scheduled(now, count - #jids)
@@ -311,8 +326,8 @@ function QlessQueue:pop(now, worker, count)
   -- look for all the recurring jobs that need jobs run
   self:check_recurring(now, count - #jids)
 
-  -- If we still need values in order to meet the demand, then we 
-  -- should check if any scheduled items, and if so, we should 
+  -- If we still need values in order to meet the demand, then we
+  -- should check if any scheduled items, and if so, we should
   -- insert them to ensure correctness when pulling off the next
   -- unit of work.
   self:check_scheduled(now, count - #jids)
@@ -334,19 +349,19 @@ function QlessQueue:pop(now, worker, count)
     self:stat(now, 'wait', waiting)
     redis.call('hset', QlessJob.ns .. jid,
       'time', string.format("%.20f", now))
-    
+
     -- Add this job to the list of jobs handled by this worker
     redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid)
-    
+
     -- Update the jobs data, and add its locks, and return the job
     job:update({
       worker  = worker,
       expires = expires,
       state   = 'running'
     })
-    
+
     self.locks.add(expires, jid)
-    
+
     local tracked = redis.call('zscore', 'ql:tracked', jid) ~= false
     if tracked then
       Qless.publish('popped', jid)
@@ -397,7 +412,7 @@ function QlessQueue:stat(now, stat, val)
     redis.call('hincrby', key, 'h' .. math.floor(val / 3600), 1)
   else -- days
     redis.call('hincrby', key, 'd' .. math.floor(val / 86400), 1)
-  end     
+  end
   redis.call('hmset', key, 'total', count, 'mean', mean, 'vk', vk)
 end
 
@@ -457,7 +472,7 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
     -- Now find what's in the original, but not the new
     local original = redis.call(
       'smembers', QlessJob.ns .. jid .. '-dependencies')
-    for _, dep in pairs(original) do 
+    for _, dep in pairs(original) do
       if new[dep] == nil then
         -- Remove k as a dependency
         redis.call('srem', QlessJob.ns .. dep .. '-dependents'  , jid)
@@ -580,7 +595,7 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
   end
 
   -- Lastly, we're going to make sure that this item is in the
-  -- set of known queues. We should keep this sorted by the 
+  -- set of known queues. We should keep this sorted by the
   -- order in which we saw each of these queues
   if redis.call('zscore', 'ql:queues', self.name) == false then
     redis.call('zadd', 'ql:queues', now, self.name)
@@ -650,7 +665,7 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
     if #arg % 2 == 1 then
       error('Odd number of additional args: ' .. tostring(arg))
     end
-    
+
     -- Read in all the optional parameters
     local options = {}
     for i = 3, #arg, 2 do options[arg[i]] = arg[i + 1] end
@@ -670,12 +685,12 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
     local count, old_queue = unpack(redis.call('hmget', 'ql:r:' .. jid, 'count', 'queue'))
     count = count or 0
 
-    -- If it has previously been in another queue, then we should remove 
+    -- If it has previously been in another queue, then we should remove
     -- some information about it
     if old_queue then
       Qless.queue(old_queue).recurring.remove(jid)
     end
-    
+
     -- Do some insertions
     redis.call('hmset', 'ql:r:' .. jid,
       'jid'     , jid,
@@ -693,14 +708,14 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
       'backlog' , options.backlog)
     -- Now, we should schedule the next run of the job
     self.recurring.add(now + offset, jid)
-    
+
     -- Lastly, we're going to make sure that this item is in the
-    -- set of known queues. We should keep this sorted by the 
+    -- set of known queues. We should keep this sorted by the
     -- order in which we saw each of these queues
     if redis.call('zscore', 'ql:queues', self.name) == false then
       redis.call('zadd', 'ql:queues', now, self.name)
     end
-    
+
     return jid
   else
     error('Recur(): schedule type "' .. tostring(spec) .. '" unknown')
@@ -746,20 +761,20 @@ function QlessQueue:check_recurring(now, count)
         )
       end
     end
-    
-    -- We're saving this value so that in the history, we can accurately 
+
+    -- We're saving this value so that in the history, we can accurately
     -- reflect when the job would normally have been scheduled
     while (score <= now) and (moved < count) do
       local count = redis.call('hincrby', 'ql:r:' .. jid, 'count', 1)
       moved = moved + 1
-      
+
       -- Add this job to the list of jobs tagged with whatever tags were
       -- supplied
       for i, tag in ipairs(_tags) do
         redis.call('zadd', 'ql:t:' .. tag, now, jid .. '-' .. count)
         redis.call('zincrby', 'ql:tags', 1, tag)
       end
-      
+
       -- First, let's save its data
       local child_jid = jid .. '-' .. count
       redis.call('hmset', QlessJob.ns .. child_jid,
@@ -776,12 +791,12 @@ function QlessQueue:check_recurring(now, count)
         'remaining', retries,
         'time'     , string.format("%.20f", score))
       Qless.job(child_jid):history(score, 'put', {q = self.name})
-      
+
       -- Now, if a delay was provided, and if it's in the future,
       -- then we'll have to schedule it. Otherwise, we're just
       -- going to add it to the work queue.
       self.work.add(score, priority, jid .. '-' .. count)
-      
+
       score = score + interval
       self.recurring.add(score, jid)
     end
@@ -796,7 +811,7 @@ function QlessQueue:check_scheduled(now, count)
   -- insert into the work queue
   local scheduled = self.scheduled.ready(now, 0, count)
   for index, jid in ipairs(scheduled) do
-    -- With these in hand, we'll have to go out and find the 
+    -- With these in hand, we'll have to go out and find the
     -- priorities of these jobs, and then we'll insert them
     -- into the work queue and then when that's complete, we'll
     -- remove them from the scheduled queue
@@ -881,7 +896,7 @@ function QlessQueue:invalidate_locks(now, count)
       -- See how many remaining retries the job has
       local remaining = tonumber(redis.call(
         'hincrby', QlessJob.ns .. jid, 'remaining', -1))
-      
+
       -- This is where we actually have to time out the work
       if remaining < 0 then
         -- Now remove the instance from the schedule, and work queues
@@ -889,7 +904,7 @@ function QlessQueue:invalidate_locks(now, count)
         self.work.remove(jid)
         self.locks.remove(jid)
         self.scheduled.remove(jid)
-        
+
         local group = 'failed-retries-' .. Qless.job(jid):data()['queue']
         local job = Qless.job(jid)
         job:history(now, 'failed', {group = group})
@@ -905,12 +920,12 @@ function QlessQueue:invalidate_locks(now, count)
           ['when']    = now,
           ['worker']  = unpack(job:data('worker'))
         }))
-        
+
         -- Add this type of failure to the list of failures
         redis.call('sadd', 'ql:failures', group)
         -- And add this particular instance to the failed types
         redis.call('lpush', 'ql:f:' .. group, jid)
-        
+
         if redis.call('zscore', 'ql:tracked', jid) ~= false then
           Qless.publish('failed', jid)
         end
diff --git a/throttle.lua b/throttle.lua
new file mode 100644
index 0000000..9ca1565
--- /dev/null
+++ b/throttle.lua
@@ -0,0 +1,32 @@
+function QlessThrottle:acquire(jid)
+  if self.available() then
+    self.locks.add(jid)
+    return true
+  else
+    queue_obj = Qless.queue(Qless.job(jid).queue)
+    queue_obj.throttled.add(jid)
+    self.pending.add(jid)
+    return false
+  end
+end
+
+function QlessThrottle:release(now, jid)
+  self.locks.remove(jid)
+  if self.available() then
+    next_jid = self.pending.pop
+    if next_jid then
+      queue_obj = Qless.queue(Qless.job(next_jid).queue)
+      queue_obj.throttled.remove(next_jid)
+      queue_obj.scheduled.add(now, next_jid)
+    end
+  end
+end
+
+function QlessThrottle:available
+  return self.maximum == 0 or self.locks.count < self.maximum
+end
+
+-- Return the prefix for this particular resource
+function QlessThrottle:prefix()
+  return QlessThrottle.ns .. self.id
+end

From 44df91cefc7830777d97c408f43b40c949914be8 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 10 Mar 2014 08:45:22 -0400
Subject: [PATCH 02/72] finished renaming resources to throttle

---
 base.lua     | 26 ++++++++++++++------------
 job.lua      | 12 ++++++------
 throttle.lua | 13 ++++---------
 3 files changed, 24 insertions(+), 27 deletions(-)

diff --git a/base.lua b/base.lua
index f4a3d0e..e510e50 100644
--- a/base.lua
+++ b/base.lua
@@ -24,10 +24,10 @@ local QlessJob = {
 QlessJob.__index = QlessJob
 
 -- throttle forward declaration
-local Qlessthrottle = {
-  ns = Qless.ns .. 'rs:'
+local QlessThrottle = {
+  ns = Qless.ns .. 't:'
 }
-Qlessthrottle.__index = Qlessthrottle;
+QlessThrottle.__index = QlessThrottle;
 
 -- RecurringJob forward declaration
 local QlessRecurringJob = {}
@@ -73,22 +73,24 @@ function Qless.throttle(tid)
   assert(tid, 'Throttle(): no tid provided')
   local throttle = {}
   setmetatable(throttle, QlessThrottle)
-  throttle.id = rid
+  throttle.id = tid
+  -- namedspaced throttle id
+  throttle.name = QlessThrottle.ns .. tid
 
   -- Set maximum for this throttle, if no maximum is defined it defaults to 0 (unlimited)
-  throttle.maximum = Qless.config.get(tonumber(Qless.config.get(throttle.prefix .. '-maximum')) or 0
+  throttle.maximum = Qless.config.get(tonumber(Qless.config.get(throttle.name .. '-maximum')) or 0
 
   -- set of jids which have acquired a lock on this throttle.
   throttle.locks = {
     count = function()
-      redis.call('scard', throttle:prefix .. '-locks')
+      redis.call('scard', throttle:name .. '-locks')
     end, add = function(...)
       if #arg > 0 then
-        redis.call('sadd', throttle:prefix .. '-locks', unpack(arg))
+        redis.call('sadd', throttle:name .. '-locks', unpack(arg))
       end
     end, remove = function(...)
       if #arg > 0 then
-        return redis.call('srem', throttle:prefix .. '-locks', unpack(arg))
+        return redis.call('srem', throttle:name .. '-locks', unpack(arg))
       end
     end
   }
@@ -96,17 +98,17 @@ function Qless.throttle(tid)
   -- set of jids waiting on this throttle to become available.
   throttle.pending = {
     count = function()
-      redis.call('scard', throttle:prefix .. '-pending')
+      redis.call('scard', throttle:name .. '-pending')
     end, add = function(...)
       if #arg > 0 then
-        redis.call('sadd', throttle:prefix .. '-pending', unpack(arg))
+        redis.call('sadd', throttle:name .. '-pending', unpack(arg))
       end
     end, remove = function(...)
       if #arg > 0 then
-        redis.call('srem', throttle:prefix .. '-pending', unpack(arg))
+        redis.call('srem', throttle:name .. '-pending', unpack(arg))
       end
     end, pop = function()
-      return redis.call('spop', throttle:prefix .. '-pending')
+      return redis.call('spop', throttle:name .. '-pending')
     end
   }
   return throttle
diff --git a/job.lua b/job.lua
index 986dc52..ec72c66 100644
--- a/job.lua
+++ b/job.lua
@@ -34,8 +34,7 @@ function QlessJob:data(...)
     tags         = cjson.decode(job[11]),
     history      = self:history(),
     failure      = cjson.decode(job[12] or '{}'),
-    -- default to queue throttle if no throttle was specified.
-    throttle     = job[13] or QlessQueue.ns .. job[4],
+    throttle     = job[13],
     dependents   = redis.call(
       'smembers', QlessJob.ns .. self.jid .. '-dependents'),
     dependencies = redis.call(
@@ -132,7 +131,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
   queue_obj.locks.remove(self.jid)
   queue_obj.scheduled.remove(self.jid)
 
-  self:release_throttle()
+  self:release_throttle(now)
 
   ----------------------------------------------------------
   -- This is the massive stats update that we have to do
@@ -462,7 +461,8 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
   Qless.queue(oldqueue).locks.remove(self.jid)
 
   -- Release the throttle for the job
-  self:release_throttle()
+  self:release_throttle(now)
+  self.acquire_throttle()
 
   -- Remove this job from the worker that was previously working it
   redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
@@ -804,7 +804,7 @@ function QlessJob:release_throttle(now)
   Qless.throttle(tid):release(now, self.jid)
 end
 
-function QlessJob:acquire_throttle(now)
+function QlessJob:acquire_throttle()
   local rid = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'throttle'))
-  return Qless.resource(rid):acquire(now, self.jid)
+  return Qless.resource(rid):acquire(self.jid)
 end
diff --git a/throttle.lua b/throttle.lua
index 9ca1565..8a43101 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -1,5 +1,5 @@
 function QlessThrottle:acquire(jid)
-  if self.available() then
+  if self.available then
     self.locks.add(jid)
     return true
   else
@@ -12,21 +12,16 @@ end
 
 function QlessThrottle:release(now, jid)
   self.locks.remove(jid)
-  if self.available() then
+  if self.available then
     next_jid = self.pending.pop
     if next_jid then
       queue_obj = Qless.queue(Qless.job(next_jid).queue)
       queue_obj.throttled.remove(next_jid)
-      queue_obj.scheduled.add(now, next_jid)
+      queue_obj.work.add(now, next_jid)
     end
   end
 end
 
-function QlessThrottle:available
+function QlessThrottle:available()
   return self.maximum == 0 or self.locks.count < self.maximum
 end
-
--- Return the prefix for this particular resource
-function QlessThrottle:prefix()
-  return QlessThrottle.ns .. self.id
-end

From 14d0fe3521ad7a9990ac5d834beee34f891866cd Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 10 Mar 2014 09:27:14 -0400
Subject: [PATCH 03/72] throttle tests wip

---
 Makefile              | 22 +++++++++++-----------
 base.lua              | 16 ++++++++--------
 test/test_throttle.py | 10 ++++++++++
 3 files changed, 29 insertions(+), 19 deletions(-)
 create mode 100644 test/test_throttle.py

diff --git a/Makefile b/Makefile
index a8cf62b..485864c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,21 +1,21 @@
 all: qless.lua qless-lib.lua
 
 qless-lib.lua: base.lua config.lua job.lua queue.lua recurring.lua worker.lua
-  echo "-- Current SHA: `git rev-parse HEAD`" > qless-lib.lua
-  echo "-- This is a generated file" >> qless-lib.lua
-  cat base.lua config.lua job.lua queue.lua recurring.lua worker.lua resource.lua >> qless-lib.lua
+	echo "-- Current SHA: `git rev-parse HEAD`" > qless-lib.lua
+	echo "-- This is a generated file" >> qless-lib.lua
+	cat base.lua config.lua job.lua queue.lua recurring.lua worker.lua throttle.lua >> qless-lib.lua
 
 qless.lua: qless-lib.lua api.lua
-  # Cat these files out, but remove all the comments from the source
-  echo "-- Current SHA: `git rev-parse HEAD`" > qless.lua
-  echo "-- This is a generated file" >> qless.lua
-  cat qless-lib.lua api.lua | \
-    egrep -v '^[[:space:]]*--[^\[]' | \
-    egrep -v '^--$$' >> qless.lua
+	# Cat these files out, but remove all the comments from the source
+	echo "-- Current SHA: `git rev-parse HEAD`" > qless.lua
+	echo "-- This is a generated file" >> qless.lua
+	cat qless-lib.lua api.lua | \
+		egrep -v '^[[:space:]]*--[^\[]' | \
+		egrep -v '^--$$' >> qless.lua
 
 clean:
-  rm -f qless.lua qless-lib.lua
+	rm -f qless.lua qless-lib.lua
 
 .PHONY: test
 test: qless.lua *.lua
-  nosetests --exe -v
+	nosetests --exe -v
diff --git a/base.lua b/base.lua
index e510e50..4d2a873 100644
--- a/base.lua
+++ b/base.lua
@@ -78,19 +78,19 @@ function Qless.throttle(tid)
   throttle.name = QlessThrottle.ns .. tid
 
   -- Set maximum for this throttle, if no maximum is defined it defaults to 0 (unlimited)
-  throttle.maximum = Qless.config.get(tonumber(Qless.config.get(throttle.name .. '-maximum')) or 0
+  throttle.maximum = Qless.config.get(tonumber(Qless.config.get(throttle.name .. '-maximum'))) or 0
 
   -- set of jids which have acquired a lock on this throttle.
   throttle.locks = {
     count = function()
-      redis.call('scard', throttle:name .. '-locks')
+      redis.call('scard', throttle.name .. '-locks')
     end, add = function(...)
       if #arg > 0 then
-        redis.call('sadd', throttle:name .. '-locks', unpack(arg))
+        redis.call('sadd', throttle.name .. '-locks', unpack(arg))
       end
     end, remove = function(...)
       if #arg > 0 then
-        return redis.call('srem', throttle:name .. '-locks', unpack(arg))
+        return redis.call('srem', throttle.name .. '-locks', unpack(arg))
       end
     end
   }
@@ -98,17 +98,17 @@ function Qless.throttle(tid)
   -- set of jids waiting on this throttle to become available.
   throttle.pending = {
     count = function()
-      redis.call('scard', throttle:name .. '-pending')
+      redis.call('scard', throttle.name .. '-pending')
     end, add = function(...)
       if #arg > 0 then
-        redis.call('sadd', throttle:name .. '-pending', unpack(arg))
+        redis.call('sadd', throttle.name .. '-pending', unpack(arg))
       end
     end, remove = function(...)
       if #arg > 0 then
-        redis.call('srem', throttle:name .. '-pending', unpack(arg))
+        redis.call('srem', throttle.name .. '-pending', unpack(arg))
       end
     end, pop = function()
-      return redis.call('spop', throttle:name .. '-pending')
+      return redis.call('spop', throttle.name .. '-pending')
     end
   }
   return throttle
diff --git a/test/test_throttle.py b/test/test_throttle.py
new file mode 100644
index 0000000..b5d7102
--- /dev/null
+++ b/test/test_throttle.py
@@ -0,0 +1,10 @@
+'''Test throttle-centric operations'''
+
+import redis
+from common import TestQless
+
+class TestAcquire(TestQless):
+  '''Test acquiring of a throttle lock'''
+  def test_acquire(self):
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', 0, 'throttle', 'tid')
+    self.assertEqual(self.lua('get', 0, 'jid')['throttle'], 'tid')

From 674353a580cee1122d87be2f1ded3fbe81bc4f7f Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 10 Mar 2014 09:38:33 -0400
Subject: [PATCH 04/72] test fixes

---
 base.lua     |  2 +-
 job.lua      | 15 ++++++++++-----
 throttle.lua |  4 ++--
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/base.lua b/base.lua
index 4d2a873..07c8e3c 100644
--- a/base.lua
+++ b/base.lua
@@ -427,7 +427,7 @@ function Qless.cancel(...)
         queue.depends.remove(jid)
       end
 
-      Qless.job(namespaced_jid):release_throttles()
+      Qless.job(namespaced_jid):release_throttle()
 
       -- We should probably go through all our dependencies and remove
       -- ourselves from the list of dependents
diff --git a/job.lua b/job.lua
index ec72c66..8fd0a48 100644
--- a/job.lua
+++ b/job.lua
@@ -34,7 +34,7 @@ function QlessJob:data(...)
     tags         = cjson.decode(job[11]),
     history      = self:history(),
     failure      = cjson.decode(job[12] or '{}'),
-    throttle     = job[13],
+    throttle     = job[13] or nil,
     dependents   = redis.call(
       'smembers', QlessJob.ns .. self.jid .. '-dependents'),
     dependencies = redis.call(
@@ -400,7 +400,7 @@ function QlessJob:fail(now, worker, group, message, data)
       ['worker']  = worker
     }))
 
-  self:release_throttle()
+  self:release_throttle(now)
 
   -- Add this group of failure to the list of failures
   redis.call('sadd', 'ql:failures', group)
@@ -801,10 +801,15 @@ end
 
 function QlessJob:release_throttle(now)
   local tid = redis.call('hget', QlessJob.ns .. self.jid, 'throttle')
-  Qless.throttle(tid):release(now, self.jid)
+  if tid then
+    Qless.throttle(tid):release(now, self.jid)
+  end
 end
 
 function QlessJob:acquire_throttle()
-  local rid = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'throttle'))
-  return Qless.resource(rid):acquire(self.jid)
+  local tid = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'throttle'))
+  if tid then
+    return Qless.resource(tid):acquire(self.jid)
+  end
+  return true
 end
diff --git a/throttle.lua b/throttle.lua
index 8a43101..8162f11 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -1,5 +1,5 @@
 function QlessThrottle:acquire(jid)
-  if self.available then
+  if self.available() then
     self.locks.add(jid)
     return true
   else
@@ -12,7 +12,7 @@ end
 
 function QlessThrottle:release(now, jid)
   self.locks.remove(jid)
-  if self.available then
+  if self.available() then
     next_jid = self.pending.pop
     if next_jid then
       queue_obj = Qless.queue(Qless.job(next_jid).queue)

From 32c795cface087b7bde640a54776e7e7c49ef340 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 10 Mar 2014 13:25:20 -0400
Subject: [PATCH 05/72] wip

---
 Makefile              |  4 ++--
 base.lua              | 28 +++++++++++++---------------
 job.lua               |  4 ++--
 queue.lua             | 30 ++++++++++++++++++++----------
 test/test_job.py      |  1 +
 test/test_throttle.py |  2 +-
 test/test_worker.py   |  3 +--
 throttle.lua          |  6 +++---
 8 files changed, 43 insertions(+), 35 deletions(-)

diff --git a/Makefile b/Makefile
index 485864c..8ab9e0a 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
 all: qless.lua qless-lib.lua
 
-qless-lib.lua: base.lua config.lua job.lua queue.lua recurring.lua worker.lua
+qless-lib.lua: base.lua config.lua job.lua queue.lua recurring.lua worker.lua throttle.lua
 	echo "-- Current SHA: `git rev-parse HEAD`" > qless-lib.lua
 	echo "-- This is a generated file" >> qless-lib.lua
 	cat base.lua config.lua job.lua queue.lua recurring.lua worker.lua throttle.lua >> qless-lib.lua
@@ -18,4 +18,4 @@ clean:
 
 .PHONY: test
 test: qless.lua *.lua
-	nosetests --exe -v
+	nosetests --exe -v $(TEST)
diff --git a/base.lua b/base.lua
index 07c8e3c..ff82868 100644
--- a/base.lua
+++ b/base.lua
@@ -27,7 +27,7 @@ QlessJob.__index = QlessJob
 local QlessThrottle = {
   ns = Qless.ns .. 't:'
 }
-QlessThrottle.__index = QlessThrottle;
+QlessThrottle.__index = QlessThrottle
 
 -- RecurringJob forward declaration
 local QlessRecurringJob = {}
@@ -71,26 +71,24 @@ end
 -- throttle objects are used for arbitrary throttling of jobs.
 function Qless.throttle(tid)
   assert(tid, 'Throttle(): no tid provided')
-  local throttle = {}
-  setmetatable(throttle, QlessThrottle)
-  throttle.id = tid
-  -- namedspaced throttle id
-  throttle.name = QlessThrottle.ns .. tid
+  local throttle = {
+    id = tid,
+    maximum = tonumber(Qless.config.get(QlessThrottle.ns .. tid .. '-maximum') or 0)
+  }
 
-  -- Set maximum for this throttle, if no maximum is defined it defaults to 0 (unlimited)
-  throttle.maximum = Qless.config.get(tonumber(Qless.config.get(throttle.name .. '-maximum'))) or 0
+  setmetatable(throttle, QlessThrottle)
 
   -- set of jids which have acquired a lock on this throttle.
   throttle.locks = {
     count = function()
-      redis.call('scard', throttle.name .. '-locks')
+      return (redis.call('scard', QlessThrottle.ns .. tid .. '-locks') or 0)
     end, add = function(...)
       if #arg > 0 then
-        redis.call('sadd', throttle.name .. '-locks', unpack(arg))
+        redis.call('sadd', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
       end
     end, remove = function(...)
       if #arg > 0 then
-        return redis.call('srem', throttle.name .. '-locks', unpack(arg))
+        return redis.call('srem', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
       end
     end
   }
@@ -98,17 +96,17 @@ function Qless.throttle(tid)
   -- set of jids waiting on this throttle to become available.
   throttle.pending = {
     count = function()
-      redis.call('scard', throttle.name .. '-pending')
+      redis.call('scard', QlessThrottle.ns .. tid .. '-pending')
     end, add = function(...)
       if #arg > 0 then
-        redis.call('sadd', throttle.name .. '-pending', unpack(arg))
+        redis.call('sadd', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
       end
     end, remove = function(...)
       if #arg > 0 then
-        redis.call('srem', throttle.name .. '-pending', unpack(arg))
+        redis.call('srem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
       end
     end, pop = function()
-      return redis.call('spop', throttle.name .. '-pending')
+      return redis.call('spop', QlessThrottle.ns .. tid .. '-pending')
     end
   }
   return throttle
diff --git a/job.lua b/job.lua
index 8fd0a48..bc70915 100644
--- a/job.lua
+++ b/job.lua
@@ -34,7 +34,7 @@ function QlessJob:data(...)
     tags         = cjson.decode(job[11]),
     history      = self:history(),
     failure      = cjson.decode(job[12] or '{}'),
-    throttle     = job[13] or nil,
+    throttle     = job[13],
     dependents   = redis.call(
       'smembers', QlessJob.ns .. self.jid .. '-dependents'),
     dependencies = redis.call(
@@ -809,7 +809,7 @@ end
 function QlessJob:acquire_throttle()
   local tid = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'throttle'))
   if tid then
-    return Qless.resource(tid):acquire(self.jid)
+    return Qless.throttle(tid):acquire(self.jid)
   end
   return true
 end
diff --git a/queue.lua b/queue.lua
index 76291cf..56139e4 100644
--- a/queue.lua
+++ b/queue.lua
@@ -452,14 +452,15 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
   end
 
   -- Sanity check on optional args
-  retries  = assert(tonumber(options['retries']  or retries or 5) ,
+  local retries  = assert(tonumber(options['retries']  or retries or 5) ,
     'Put(): Arg "retries" not a number: ' .. tostring(options['retries']))
-  tags     = assert(cjson.decode(options['tags'] or tags or '[]' ),
+  local tags     = assert(cjson.decode(options['tags'] or tags or '[]' ),
     'Put(): Arg "tags" not JSON'          .. tostring(options['tags']))
-  priority = assert(tonumber(options['priority'] or priority or 0),
+  local priority = assert(tonumber(options['priority'] or priority or 0),
     'Put(): Arg "priority" not a number'  .. tostring(options['priority']))
   local depends = assert(cjson.decode(options['depends'] or '[]') ,
     'Put(): Arg "depends" not JSON: '     .. tostring(options['depends']))
+  -- local throttle = options['throttle']
 
   -- If the job has old dependencies, determine which dependencies are
   -- in the new dependencies but not in the old ones, and which are in the
@@ -546,11 +547,10 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
     redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed'  , -1)
   end
 
-  -- First, let's save its data
-  redis.call('hmset', QlessJob.ns .. jid,
-    'jid'      , jid,
-    'klass'    , klass,
-    'data'     , raw_data,
+  data = {
+    'jid'      = jid,
+    'klass'    = klass,
+    'data'     = raw_data,
     'priority' , priority,
     'tags'     , cjson.encode(tags),
     'state'    , ((delay > 0) and 'scheduled') or 'waiting',
@@ -559,7 +559,15 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
     'queue'    , self.name,
     'retries'  , retries,
     'remaining', retries,
-    'time'     , string.format("%.20f", now))
+    'time'     , string.format("%.20f", now),
+  }
+
+  if options['throttle'] then
+    data['throttle'] = options['throttle']
+  end
+
+  -- First, let's save its data
+  redis.call('hmset', QlessJob.ns .. jid, unpack(data))
 
   -- These are the jids we legitimately have to wait on
   for i, j in ipairs(depends) do
@@ -590,7 +598,9 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
       self.depends.add(now, jid)
       redis.call('hset', QlessJob.ns .. jid, 'state', 'depends')
     else
-      self.work.add(now, priority, jid)
+      if job:acquire_throttle() then
+        self.work.add(now, priority, jid)
+      end
     end
   end
 
diff --git a/test/test_job.py b/test/test_job.py
index 1fe3398..1163d9b 100644
--- a/test/test_job.py
+++ b/test/test_job.py
@@ -1,6 +1,7 @@
 '''Test job-centric operations'''
 
 import redis
+import pdb
 from common import TestQless
 
 
diff --git a/test/test_throttle.py b/test/test_throttle.py
index b5d7102..1ce66da 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -6,5 +6,5 @@
 class TestAcquire(TestQless):
   '''Test acquiring of a throttle lock'''
   def test_acquire(self):
-    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
     self.assertEqual(self.lua('get', 0, 'jid')['throttle'], 'tid')
diff --git a/test/test_worker.py b/test/test_worker.py
index 5624fc5..1536507 100644
--- a/test/test_worker.py
+++ b/test/test_worker.py
@@ -172,8 +172,7 @@ def test_retry_worker(self):
         '''When retried, it removes a job from the worker's data'''
         self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0)
         self.lua('pop', 0, 'queue', 'worker', 10)
-        self.lua(
-            'retry', 0, 'jid', 'queue', 'worker', 0)
+        self.lua('retry', 0, 'jid', 'queue', 'worker', 0)
         self.assertEqual(self.lua('workers', 3600, 'worker'), {
             'jobs': {},
             'stalled': {}
diff --git a/throttle.lua b/throttle.lua
index 8162f11..66ba61e 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -1,5 +1,5 @@
 function QlessThrottle:acquire(jid)
-  if self.available() then
+  if self:available() then
     self.locks.add(jid)
     return true
   else
@@ -12,7 +12,7 @@ end
 
 function QlessThrottle:release(now, jid)
   self.locks.remove(jid)
-  if self.available() then
+  if self:available() then
     next_jid = self.pending.pop
     if next_jid then
       queue_obj = Qless.queue(Qless.job(next_jid).queue)
@@ -23,5 +23,5 @@ function QlessThrottle:release(now, jid)
 end
 
 function QlessThrottle:available()
-  return self.maximum == 0 or self.locks.count < self.maximum
+  return self.maximum == 0 or self.locks.count() < self.maximum
 end

From 4de319bea4eb819465cd1b5b3efb67ccdc0a5619 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 10 Mar 2014 14:15:00 -0400
Subject: [PATCH 06/72] general working

---
 base.lua              |  1 +
 job.lua               |  4 ++--
 queue.lua             | 14 +++++++++-----
 test/test_throttle.py |  1 +
 4 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/base.lua b/base.lua
index ff82868..2b09308 100644
--- a/base.lua
+++ b/base.lua
@@ -240,6 +240,7 @@ function Qless.track(now, command, jid)
     assert(jid, 'Track(): Arg "jid" missing')
     -- Verify that job exists
     assert(Qless.job(jid):exists(), 'Track(): Job does not exist')
+    redis.call('set', 'print_line_track_command', now .. command .. jid)
     if string.lower(command) == 'track' then
       Qless.publish('track', jid)
       return redis.call('zadd', 'ql:tracked', now, jid)
diff --git a/job.lua b/job.lua
index bc70915..6db8e6d 100644
--- a/job.lua
+++ b/job.lua
@@ -34,7 +34,7 @@ function QlessJob:data(...)
     tags         = cjson.decode(job[11]),
     history      = self:history(),
     failure      = cjson.decode(job[12] or '{}'),
-    throttle     = job[13],
+    throttle     = job[13] or nil,
     dependents   = redis.call(
       'smembers', QlessJob.ns .. self.jid .. '-dependents'),
     dependencies = redis.call(
@@ -462,7 +462,7 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
 
   -- Release the throttle for the job
   self:release_throttle(now)
-  self.acquire_throttle()
+  self:acquire_throttle()
 
   -- Remove this job from the worker that was previously working it
   redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
diff --git a/queue.lua b/queue.lua
index 56139e4..bc3e923 100644
--- a/queue.lua
+++ b/queue.lua
@@ -547,10 +547,11 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
     redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed'  , -1)
   end
 
+  redis.call('set', 'print_line_set_throttle', 'hello')
   data = {
-    'jid'      = jid,
-    'klass'    = klass,
-    'data'     = raw_data,
+    'jid'      , jid,
+    'klass'    , klass,
+    'data'     , raw_data,
     'priority' , priority,
     'tags'     , cjson.encode(tags),
     'state'    , ((delay > 0) and 'scheduled') or 'waiting',
@@ -559,11 +560,14 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
     'queue'    , self.name,
     'retries'  , retries,
     'remaining', retries,
-    'time'     , string.format("%.20f", now),
+    'time'     , string.format("%.20f", now)
   }
 
+  -- Insert the throttle resource into the array if it exists.
   if options['throttle'] then
-    data['throttle'] = options['throttle']
+    redis.call('set', 'print_line_set_throttle', 'true')
+    table.insert(data, 'throttle')
+    table.insert(data, options['throttle'])
   end
 
   -- First, let's save its data
diff --git a/test/test_throttle.py b/test/test_throttle.py
index 1ce66da..92ec057 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -7,4 +7,5 @@ class TestAcquire(TestQless):
   '''Test acquiring of a throttle lock'''
   def test_acquire(self):
     self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
+    print(self.lua('get', 0, 'jid'))
     self.assertEqual(self.lua('get', 0, 'jid')['throttle'], 'tid')

From 493d1571e91801ac3181da74778b9e6a6e54f698 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 10 Mar 2014 15:06:54 -0400
Subject: [PATCH 07/72] throttle api

---
 api.lua               | 27 +++++++++++++++++++++++++++
 queue.lua             |  2 --
 test/common.py        |  3 ++-
 test/test_throttle.py | 14 ++++++++++++--
 throttle.lua          | 14 ++++++++++++++
 5 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/api.lua b/api.lua
index 91da80b..35e85e8 100644
--- a/api.lua
+++ b/api.lua
@@ -193,6 +193,33 @@ QlessAPI['queue.forget'] = function(now, ...)
   QlessQueue.deregister(unpack(arg))
 end
 
+-- Throttle apis
+QlessAPI['throttle.set'] = function(now, tid, max)
+  local data = {
+    maximum = max
+  }
+  Qless.throttle(tid):set(data)
+end
+
+QlessAPI['throttle.get'] = function(now, tid)
+  local data = Qless.throttle(tid):data()
+  if not data then
+    return nil
+  end
+  return cjson.encode(data)
+end
+
+QlessAPI['throttle.unset'] = function(now, tid)
+  return Qless.throttle(tid):unset()
+end
+
+QlessAPI['throttle.locks'] = function(now, tid)
+  return Qless.throttle(tid):locks()
+end
+
+QlessAPI['throttle.pending'] = function(now, tid)
+  return Qless.throttle(tid):pending()
+end
 -------------------------------------------------------------------------------
 -- Function lookup
 -------------------------------------------------------------------------------
diff --git a/queue.lua b/queue.lua
index bc3e923..632a3d0 100644
--- a/queue.lua
+++ b/queue.lua
@@ -547,7 +547,6 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
     redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed'  , -1)
   end
 
-  redis.call('set', 'print_line_set_throttle', 'hello')
   data = {
     'jid'      , jid,
     'klass'    , klass,
@@ -565,7 +564,6 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
 
   -- Insert the throttle resource into the array if it exists.
   if options['throttle'] then
-    redis.call('set', 'print_line_set_throttle', 'true')
     table.insert(data, 'throttle')
     table.insert(data, options['throttle'])
   end
diff --git a/test/common.py b/test/common.py
index 70d38bd..3eb2c86 100644
--- a/test/common.py
+++ b/test/common.py
@@ -12,7 +12,8 @@ class TestQless(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         url = os.environ.get('REDIS_URL', 'redis://localhost:6379/')
-        cls.lua = qless.QlessRecorder(redis.Redis.from_url(url))
+        cls.redis = redis.Redis.from_url(url)
+        cls.lua = qless.QlessRecorder(cls.redis)
 
     def tearDown(self):
         self.lua.flush()
diff --git a/test/test_throttle.py b/test/test_throttle.py
index 92ec057..a9084ad 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -3,9 +3,19 @@
 import redis
 from common import TestQless
 
-class TestAcquire(TestQless):
+class TestThrottle(TestQless):
+  '''Test setting throttle data'''
+  def test_set(self):
+    self.lua('throttle.set', 0, 'tid', 5)
+    self.assertEqual(self.redis.hmget('ql:t:tid', 'id')[0], 'tid')
+    self.assertEqual(self.redis.hmget('ql:t:tid', 'maximum')[0], '5')
+
+  '''Test retrieving throttle data'''
+  def test_get(self):
+    self.redis.hmset('ql:t:tid', {'id': 'tid', 'maximum' : 5})
+    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : '5'})
+
   '''Test acquiring of a throttle lock'''
   def test_acquire(self):
     self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
-    print(self.lua('get', 0, 'jid'))
     self.assertEqual(self.lua('get', 0, 'jid')['throttle'], 'tid')
diff --git a/throttle.lua b/throttle.lua
index 66ba61e..b5a6d82 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -1,3 +1,17 @@
+function QlessThrottle:data()
+  local throttle = redis.call('hmget', QlessThrottle.ns .. self.id, 'tid', 'maximum')
+  local data = {
+    tid = throttle[0],
+    maximum = throttle[1]
+  }
+  redis.call('set', 'print_line', cjson.encode(data))
+  return data
+end
+
+function QlessThrottle:set(data)
+  redis.call('hmset', QlessThrottle.ns .. self.id, 'id', self.id, 'maximum', data.maximum)
+end
+
 function QlessThrottle:acquire(jid)
   if self:available() then
     self.locks.add(jid)

From f6b9a380561db2213881764e0502e51b3fba0890 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 10 Mar 2014 16:34:17 -0400
Subject: [PATCH 08/72] throttle changes

---
 api.lua               |  3 ++-
 test/test_throttle.py | 10 ++++++++++
 throttle.lua          | 41 ++++++++++++++++++++++++++++++-----------
 3 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/api.lua b/api.lua
index 35e85e8..d25f97a 100644
--- a/api.lua
+++ b/api.lua
@@ -203,13 +203,14 @@ end
 
 QlessAPI['throttle.get'] = function(now, tid)
   local data = Qless.throttle(tid):data()
+  redis.call('set', 'printline5', cjson.encode(data))
   if not data then
     return nil
   end
   return cjson.encode(data)
 end
 
-QlessAPI['throttle.unset'] = function(now, tid)
+QlessAPI['throttle.delete'] = function(now, tid)
   return Qless.throttle(tid):unset()
 end
 
diff --git a/test/test_throttle.py b/test/test_throttle.py
index a9084ad..b0f2804 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -15,7 +15,17 @@ def test_get(self):
     self.redis.hmset('ql:t:tid', {'id': 'tid', 'maximum' : 5})
     self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : '5'})
 
+  '''Test deleting the throttle data'''
+  def test_delete(self):
+    self.lua('throttle.set', 0, 'tid', 5)
+    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : '5'})
+    self.lua('throttle.delete', 0, 'tid')
+    self.assertEqual(self.lua('throttle.get', 0, 'tid'), None)
+
   '''Test acquiring of a throttle lock'''
   def test_acquire(self):
     self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
     self.assertEqual(self.lua('get', 0, 'jid')['throttle'], 'tid')
+
+  # '''Test that acquiring of a throttle lock properly limits the number of jobs'''
+  # def test_
diff --git a/throttle.lua b/throttle.lua
index b5a6d82..e71a88e 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -1,17 +1,32 @@
+-- Retrieve the data fro a throttled resource
 function QlessThrottle:data()
-  local throttle = redis.call('hmget', QlessThrottle.ns .. self.id, 'tid', 'maximum')
+  local throttle = redis.call('hmget', QlessThrottle.ns .. self.id, 'id', 'maximum')
+  -- Return nil if we haven't found it
+  if not throttle[1] then
+    return nil
+  end
+
   local data = {
-    tid = throttle[0],
-    maximum = throttle[1]
+    id = throttle[1],
+    maximum = throttle[2]
   }
-  redis.call('set', 'print_line', cjson.encode(data))
   return data
 end
 
+-- Set the data for a throttled resource
 function QlessThrottle:set(data)
   redis.call('hmset', QlessThrottle.ns .. self.id, 'id', self.id, 'maximum', data.maximum)
 end
 
+-- Delete a throttled resource
+function QlessThrottle:unset()
+  redis.call('del', QlessThrottle.ns .. self.id)
+end
+
+-- Acquire a throttled resource for a job.
+-- if the resource is at full capacity then add it to the pending
+-- set.
+-- Returns true of the job acquired the resource.
 function QlessThrottle:acquire(jid)
   if self:available() then
     self.locks.add(jid)
@@ -24,15 +39,19 @@ function QlessThrottle:acquire(jid)
   end
 end
 
+-- Release a throttled resource.
+-- This will take a currently pending job
+-- and attempt to acquire a lock.
+-- If it succeeds at acquiring a lock then
+-- the job will be moved from the throttled
+-- queue into the work queue
 function QlessThrottle:release(now, jid)
   self.locks.remove(jid)
-  if self:available() then
-    next_jid = self.pending.pop
-    if next_jid then
-      queue_obj = Qless.queue(Qless.job(next_jid).queue)
-      queue_obj.throttled.remove(next_jid)
-      queue_obj.work.add(now, next_jid)
-    end
+  next_jid = self.pending.pop
+  if next_jid and self:acquire(next_jid) then
+    queue_obj = Qless.queue(Qless.job(next_jid).queue)
+    queue_obj.throttled.remove(next_jid)
+    queue_obj.work.add(now, next_jid)
   end
 end
 

From 36fd0fba687c342d3f854a4db8855acb9b293c11 Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Mon, 10 Mar 2014 17:01:32 -0400
Subject: [PATCH 09/72] Add Throttle locks and pending member functions

---
 api.lua  | 4 ++--
 base.lua | 4 ++++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/api.lua b/api.lua
index d25f97a..3866377 100644
--- a/api.lua
+++ b/api.lua
@@ -215,11 +215,11 @@ QlessAPI['throttle.delete'] = function(now, tid)
 end
 
 QlessAPI['throttle.locks'] = function(now, tid)
-  return Qless.throttle(tid):locks()
+  return Qless.throttle(tid).locks:members()
 end
 
 QlessAPI['throttle.pending'] = function(now, tid)
-  return Qless.throttle(tid):pending()
+  return Qless.throttle(tid).pending:members()
 end
 -------------------------------------------------------------------------------
 -- Function lookup
diff --git a/base.lua b/base.lua
index 2b09308..95e0168 100644
--- a/base.lua
+++ b/base.lua
@@ -86,6 +86,8 @@ function Qless.throttle(tid)
       if #arg > 0 then
         redis.call('sadd', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
       end
+    end, members = function()
+      return redis.call('smembers', QlessThrottle.ns .. tid .. '-locks')
     end, remove = function(...)
       if #arg > 0 then
         return redis.call('srem', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
@@ -101,6 +103,8 @@ function Qless.throttle(tid)
       if #arg > 0 then
         redis.call('sadd', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
       end
+    end, members = function()
+      return redis.call('smembers', QlessThrottle.ns .. tid .. '-pending')
     end, remove = function(...)
       if #arg > 0 then
         redis.call('srem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))

From 3599db3c00095a6bce0ddea68d76ae93d0b1f83b Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 10 Mar 2014 17:46:52 -0400
Subject: [PATCH 10/72] wip

---
 api.lua               |  1 -
 base.lua              | 16 +++++++++-------
 queue.lua             |  6 +++---
 test/test_throttle.py | 11 +++++++++--
 throttle.lua          |  4 +---
 5 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/api.lua b/api.lua
index 3866377..4463098 100644
--- a/api.lua
+++ b/api.lua
@@ -203,7 +203,6 @@ end
 
 QlessAPI['throttle.get'] = function(now, tid)
   local data = Qless.throttle(tid):data()
-  redis.call('set', 'printline5', cjson.encode(data))
   if not data then
     return nil
   end
diff --git a/base.lua b/base.lua
index 95e0168..6ccb46e 100644
--- a/base.lua
+++ b/base.lua
@@ -71,11 +71,13 @@ end
 -- throttle objects are used for arbitrary throttling of jobs.
 function Qless.throttle(tid)
   assert(tid, 'Throttle(): no tid provided')
-  local throttle = {
-    id = tid,
-    maximum = tonumber(Qless.config.get(QlessThrottle.ns .. tid .. '-maximum') or 0)
-  }
-
+  local throttle = QlessThrottle.data({id = tid})
+  if not throttle then
+    throttle = {
+      id = tid,
+      maximum = 0
+    }
+  end
   setmetatable(throttle, QlessThrottle)
 
   -- set of jids which have acquired a lock on this throttle.
@@ -103,12 +105,12 @@ function Qless.throttle(tid)
       if #arg > 0 then
         redis.call('sadd', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
       end
-    end, members = function()
-      return redis.call('smembers', QlessThrottle.ns .. tid .. '-pending')
     end, remove = function(...)
       if #arg > 0 then
         redis.call('srem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
       end
+    end, members = function()
+      return redis.call('smembers', QlessThrottle.ns .. tid .. '-pending')
     end, pop = function()
       return redis.call('spop', QlessThrottle.ns .. tid .. '-pending')
     end
diff --git a/queue.lua b/queue.lua
index 632a3d0..fca60ce 100644
--- a/queue.lua
+++ b/queue.lua
@@ -599,10 +599,10 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
     if redis.call('scard', QlessJob.ns .. jid .. '-dependencies') > 0 then
       self.depends.add(now, jid)
       redis.call('hset', QlessJob.ns .. jid, 'state', 'depends')
+    elseif job:acquire_throttle() then
+      self.work.add(now, priority, jid)
     else
-      if job:acquire_throttle() then
-        self.work.add(now, priority, jid)
-      end
+      self.throttled.add(jid)
     end
   end
 
diff --git a/test/test_throttle.py b/test/test_throttle.py
index b0f2804..f926f42 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -22,10 +22,17 @@ def test_delete(self):
     self.lua('throttle.delete', 0, 'tid')
     self.assertEqual(self.lua('throttle.get', 0, 'tid'), None)
 
+class TestAcquire(TestQless):
   '''Test acquiring of a throttle lock'''
   def test_acquire(self):
     self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
     self.assertEqual(self.lua('get', 0, 'jid')['throttle'], 'tid')
 
-  # '''Test that acquiring of a throttle lock properly limits the number of jobs'''
-  # def test_
+  '''Test that acquiring of a throttle lock properly limits the number of jobs'''
+  def test_limit_number_of_locks(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttle', 'tid')
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
diff --git a/throttle.lua b/throttle.lua
index e71a88e..1fdc713 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -8,7 +8,7 @@ function QlessThrottle:data()
 
   local data = {
     id = throttle[1],
-    maximum = throttle[2]
+    maximum = tonumber(throttle[2])
   }
   return data
 end
@@ -32,8 +32,6 @@ function QlessThrottle:acquire(jid)
     self.locks.add(jid)
     return true
   else
-    queue_obj = Qless.queue(Qless.job(jid).queue)
-    queue_obj.throttled.add(jid)
     self.pending.add(jid)
     return false
   end

From 9f3bdb9803dc981498730f72315033b6e4c5d756 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 10 Mar 2014 18:08:38 -0400
Subject: [PATCH 11/72] wip

---
 queue.lua             |  1 +
 test/test_throttle.py | 26 ++++++++++++++++++++++++++
 throttle.lua          |  1 +
 3 files changed, 28 insertions(+)

diff --git a/queue.lua b/queue.lua
index fca60ce..14d1d37 100644
--- a/queue.lua
+++ b/queue.lua
@@ -600,6 +600,7 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
       self.depends.add(now, jid)
       redis.call('hset', QlessJob.ns .. jid, 'state', 'depends')
     elseif job:acquire_throttle() then
+      redis.call('set', 'printline5', 'adding job to work queue')
       self.work.add(now, priority, jid)
     else
       self.throttled.add(jid)
diff --git a/test/test_throttle.py b/test/test_throttle.py
index f926f42..e15abe1 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -36,3 +36,29 @@ def test_limit_number_of_locks(self):
     self.lua('put', 0, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttle', 'tid')
     self.lua('put', 0, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttle', 'tid')
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid2', 'jid3', 'jid4'])
+
+Class TestRelease(TestQless):
+  '''Test that when there are no pending jobs lock is properly released'''
+  def test_no_pending_jobs(self):
+    print("pending")
+
+  '''Test that releasing a lock properly another job in the work queue'''
+  def test_next_job_is_moved_into_work_qeueue(self):
+    print("pending")
+
+  '''Test that when a job completes it properly releases the lock'''
+  def test_on_complete_lock_is_released(self):
+    print("pending")
+
+  '''Test that when a job fails it properly releases the lock'''
+  def test_on_failure_lock_is_released(self):
+    print("pending")
+
+  '''Test that when a job retries it properly releases the lock
+     and goes back into pending'''
+  def test_on_retry_lock_is_released(self):
+    print("pending")
+
+
+# What about Recurring Jobs???
diff --git a/throttle.lua b/throttle.lua
index 1fdc713..11d4cdb 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -53,6 +53,7 @@ function QlessThrottle:release(now, jid)
   end
 end
 
+-- Returns true if the throttle has locks available, false otherwise.
 function QlessThrottle:available()
   return self.maximum == 0 or self.locks.count() < self.maximum
 end

From 623ebd94c213bbb217d5425837c475b0e7fd7f30 Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Tue, 11 Mar 2014 07:40:25 -0400
Subject: [PATCH 12/72] Fix tests

---
 test/test_throttle.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/test_throttle.py b/test/test_throttle.py
index e15abe1..4bb40ab 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -13,12 +13,12 @@ def test_set(self):
   '''Test retrieving throttle data'''
   def test_get(self):
     self.redis.hmset('ql:t:tid', {'id': 'tid', 'maximum' : 5})
-    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : '5'})
+    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 5})
 
   '''Test deleting the throttle data'''
   def test_delete(self):
     self.lua('throttle.set', 0, 'tid', 5)
-    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : '5'})
+    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 5})
     self.lua('throttle.delete', 0, 'tid')
     self.assertEqual(self.lua('throttle.get', 0, 'tid'), None)
 
@@ -38,7 +38,7 @@ def test_limit_number_of_locks(self):
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid2', 'jid3', 'jid4'])
 
-Class TestRelease(TestQless):
+class TestRelease(TestQless):
   '''Test that when there are no pending jobs lock is properly released'''
   def test_no_pending_jobs(self):
     print("pending")

From b21e5199a43fc1361a51d8099afb4396db21828b Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Tue, 11 Mar 2014 09:00:28 -0400
Subject: [PATCH 13/72] Switch to sorted set

---
 base.lua     | 20 ++++++++++----------
 queue.lua    |  9 ++++-----
 throttle.lua |  6 +++---
 3 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/base.lua b/base.lua
index 6ccb46e..a1b41a3 100644
--- a/base.lua
+++ b/base.lua
@@ -83,16 +83,16 @@ function Qless.throttle(tid)
   -- set of jids which have acquired a lock on this throttle.
   throttle.locks = {
     count = function()
-      return (redis.call('scard', QlessThrottle.ns .. tid .. '-locks') or 0)
+      return (redis.call('zcard', QlessThrottle.ns .. tid .. '-locks') or 0)
     end, add = function(...)
       if #arg > 0 then
-        redis.call('sadd', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
+        redis.call('zadd', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
       end
     end, members = function()
-      return redis.call('smembers', QlessThrottle.ns .. tid .. '-locks')
+      return redis.call('zmembers', QlessThrottle.ns .. tid .. '-locks')
     end, remove = function(...)
       if #arg > 0 then
-        return redis.call('srem', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
+        return redis.call('zrem', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
       end
     end
   }
@@ -100,19 +100,19 @@ function Qless.throttle(tid)
   -- set of jids waiting on this throttle to become available.
   throttle.pending = {
     count = function()
-      redis.call('scard', QlessThrottle.ns .. tid .. '-pending')
+      redis.call('zcard', QlessThrottle.ns .. tid .. '-pending')
     end, add = function(...)
       if #arg > 0 then
-        redis.call('sadd', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
+        redis.call('zadd', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
       end
+    end, members = function()
+      return redis.call('zmembers', QlessThrottle.ns .. tid .. '-pending')
     end, remove = function(...)
       if #arg > 0 then
-        redis.call('srem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
+        redis.call('zrem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
       end
-    end, members = function()
-      return redis.call('smembers', QlessThrottle.ns .. tid .. '-pending')
     end, pop = function()
-      return redis.call('spop', QlessThrottle.ns .. tid .. '-pending')
+      return redis.call('zremrangebyrank', QlessThrottle.ns .. tid .. '-pending', 0, 1)
     end
   }
   return throttle
diff --git a/queue.lua b/queue.lua
index 14d1d37..9ee6691 100644
--- a/queue.lua
+++ b/queue.lua
@@ -80,15 +80,15 @@ function Qless.queue(name)
   -- Access to our throttled jobs
   queue.throttled = {
     peek = function(now, offset, count)
-      return redis.call('srange', queue:prefix('throttled'), offset, offset + count - 1)
+      return redis.call('zrange', queue:prefix('throttled'), offset, offset + count - 1)
     end, add = function(now, jid)
-      redis.call('sadd', queue:prefix('throttled'), jid)
+      redis.call('zadd', queue:prefix('throttled'), jid)
     end, remove = function(...)
       if #arg > 0 then
-        return redis.call('srem', queue:prefix('throttled'), unpack(arg))
+        return redis.call('zrem', queue:prefix('throttled'), unpack(arg))
       end
     end, length = function()
-      return redis.call('scard', queue:prefix('throttled'))
+      return redis.call('zcard', queue:prefix('throttled'))
     end
   }
 
@@ -600,7 +600,6 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
       self.depends.add(now, jid)
       redis.call('hset', QlessJob.ns .. jid, 'state', 'depends')
     elseif job:acquire_throttle() then
-      redis.call('set', 'printline5', 'adding job to work queue')
       self.work.add(now, priority, jid)
     else
       self.throttled.add(jid)
diff --git a/throttle.lua b/throttle.lua
index 11d4cdb..c03f7e7 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -29,10 +29,10 @@ end
 -- Returns true of the job acquired the resource.
 function QlessThrottle:acquire(jid)
   if self:available() then
-    self.locks.add(jid)
+    self.locks.add(1, jid)
     return true
   else
-    self.pending.add(jid)
+    self.pending.add(1, jid)
     return false
   end
 end
@@ -45,7 +45,7 @@ end
 -- queue into the work queue
 function QlessThrottle:release(now, jid)
   self.locks.remove(jid)
-  next_jid = self.pending.pop
+  local pri, next_jid = self.pending.pop()
   if next_jid and self:acquire(next_jid) then
     queue_obj = Qless.queue(Qless.job(next_jid).queue)
     queue_obj.throttled.remove(next_jid)

From 69bd3e7a4aa04b2f8638e15ea660388451ca909a Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Tue, 11 Mar 2014 09:07:44 -0400
Subject: [PATCH 14/72] Fix throttle locks and pending member functions

---
 base.lua | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/base.lua b/base.lua
index a1b41a3..37a13ac 100644
--- a/base.lua
+++ b/base.lua
@@ -89,7 +89,7 @@ function Qless.throttle(tid)
         redis.call('zadd', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
       end
     end, members = function()
-      return redis.call('zmembers', QlessThrottle.ns .. tid .. '-locks')
+      return redis.call('zrange', QlessThrottle.ns .. tid .. '-locks', 0, -1)
     end, remove = function(...)
       if #arg > 0 then
         return redis.call('zrem', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
@@ -106,7 +106,7 @@ function Qless.throttle(tid)
         redis.call('zadd', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
       end
     end, members = function()
-      return redis.call('zmembers', QlessThrottle.ns .. tid .. '-pending')
+      return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', 0, -1)
     end, remove = function(...)
       if #arg > 0 then
         redis.call('zrem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))

From d3fe76608523ebffac3de313a57997214d6b4cb6 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 11 Mar 2014 10:27:08 -0400
Subject: [PATCH 15/72] lock fixes

---
 base.lua              | 20 ++++++++++++--------
 test/test_throttle.py | 20 +++++++++++++++++---
 throttle.lua          | 28 ++++++++++++++++++++++++----
 3 files changed, 53 insertions(+), 15 deletions(-)

diff --git a/base.lua b/base.lua
index 37a13ac..f6b8365 100644
--- a/base.lua
+++ b/base.lua
@@ -84,35 +84,39 @@ function Qless.throttle(tid)
   throttle.locks = {
     count = function()
       return (redis.call('zcard', QlessThrottle.ns .. tid .. '-locks') or 0)
+    end, members = function()
+      return redis.call('zrange', QlessThrottle.ns .. tid .. '-locks', 0, -1)
     end, add = function(...)
       if #arg > 0 then
         redis.call('zadd', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
       end
-    end, members = function()
-      return redis.call('zrange', QlessThrottle.ns .. tid .. '-locks', 0, -1)
     end, remove = function(...)
       if #arg > 0 then
         return redis.call('zrem', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
       end
+    end, pop = function(min, max)
+      return redis.call('zremrangebyscore', QlessThrottle.ns .. tid .. '-locks', min, max)
     end
   }
 
   -- set of jids waiting on this throttle to become available.
   throttle.pending = {
     count = function()
-      redis.call('zcard', QlessThrottle.ns .. tid .. '-pending')
+      return (redis.call('zcard', QlessThrottle.ns .. tid .. '-pending') or 0)
+    end, members = function()
+      return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', 0, -1)
+    end, peek = function(min, max)
+      return redis.call('zrangebyscore', QlessThrottle.ns .. tid .. '-pending', min, max)
     end, add = function(...)
       if #arg > 0 then
         redis.call('zadd', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
       end
-    end, members = function()
-      return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', 0, -1)
     end, remove = function(...)
       if #arg > 0 then
-        redis.call('zrem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
+        return redis.call('zrem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
       end
-    end, pop = function()
-      return redis.call('zremrangebyrank', QlessThrottle.ns .. tid .. '-pending', 0, 1)
+    end, pop = function(min, max)
+      return redis.call('zremrangebyscore', QlessThrottle.ns .. tid .. '-pending', min, max)
     end
   }
   return throttle
diff --git a/test/test_throttle.py b/test/test_throttle.py
index 4bb40ab..60c16e6 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -1,6 +1,7 @@
 '''Test throttle-centric operations'''
 
 import redis
+import code
 from common import TestQless
 
 class TestThrottle(TestQless):
@@ -41,11 +42,25 @@ def test_limit_number_of_locks(self):
 class TestRelease(TestQless):
   '''Test that when there are no pending jobs lock is properly released'''
   def test_no_pending_jobs(self):
-    print("pending")
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.lua('complete', 0, 'jid', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
 
   '''Test that releasing a lock properly another job in the work queue'''
   def test_next_job_is_moved_into_work_qeueue(self):
-    print("pending")
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttle', 'tid')
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid2'])
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.lua('complete', 0, 'jid1', 'worker', 'queue', {})
+    # code.interact(local=locals())
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+
 
   '''Test that when a job completes it properly releases the lock'''
   def test_on_complete_lock_is_released(self):
@@ -60,5 +75,4 @@ def test_on_failure_lock_is_released(self):
   def test_on_retry_lock_is_released(self):
     print("pending")
 
-
 # What about Recurring Jobs???
diff --git a/throttle.lua b/throttle.lua
index c03f7e7..ec533e6 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -29,9 +29,11 @@ end
 -- Returns true of the job acquired the resource.
 function QlessThrottle:acquire(jid)
   if self:available() then
+    redis.call('set', 'printline', jid .. ' is acquiring the lock for ' .. self.id)
     self.locks.add(1, jid)
     return true
   else
+    redis.call('set', 'printline', jid .. ' failed acquiring the lock for ' .. self.id .. ' marked as pending')
     self.pending.add(1, jid)
     return false
   end
@@ -45,15 +47,33 @@ end
 -- queue into the work queue
 function QlessThrottle:release(now, jid)
   self.locks.remove(jid)
-  local pri, next_jid = self.pending.pop()
+
+  -- 0th index does not exist, thanks redis!
+  local next_jid = unpack(self:pending_pop(1, 1))
   if next_jid and self:acquire(next_jid) then
-    queue_obj = Qless.queue(Qless.job(next_jid).queue)
-    queue_obj.throttled.remove(next_jid)
-    queue_obj.work.add(now, next_jid)
+    local job = Qless.job(next_jid):data()
+    local queue_obj = Qless.queue(job.queue)
+    queue_obj.throttled.remove(job.jid)
+    queue_obj.work.add(now, job.priority, job.jid)
   end
 end
 
+function QlessThrottle:lock_pop(min, max)
+  local lock = Qless.throttle(self.id).locks
+  local jid = lock.peek(min,max)
+  lock.pop(min,max)
+  return jid
+end
+
+function QlessThrottle:pending_pop(min, max)
+  local pending = Qless.throttle(self.id).pending
+  local jids = pending.peek(min,max)
+  pending.pop(min,max)
+  return jids
+end
+
 -- Returns true if the throttle has locks available, false otherwise.
 function QlessThrottle:available()
+  redis.call('set', 'printline', 'available ' .. self.maximum .. ' == 0 or ' .. self.locks.count() .. ' < self.maximum')
   return self.maximum == 0 or self.locks.count() < self.maximum
 end

From 1e003e5acbc9b880194de3bc57ec955eb667e07e Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 11 Mar 2014 10:37:06 -0400
Subject: [PATCH 16/72] finished basic tests

---
 test/test_throttle.py | 48 +++++++++++++++++++++++++++++++++++++++----
 1 file changed, 44 insertions(+), 4 deletions(-)

diff --git a/test/test_throttle.py b/test/test_throttle.py
index 60c16e6..14abaf5 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -3,6 +3,7 @@
 import redis
 import code
 from common import TestQless
+# code.interact(local=locals())
 
 class TestThrottle(TestQless):
   '''Test setting throttle data'''
@@ -40,6 +41,17 @@ def test_limit_number_of_locks(self):
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid2', 'jid3', 'jid4'])
 
 class TestRelease(TestQless):
+  '''Test that job retains lock while working'''
+  def test_retains_lock_while_working(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+    self.lua('complete', 0, 'jid', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+
   '''Test that when there are no pending jobs lock is properly released'''
   def test_no_pending_jobs(self):
     self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
@@ -57,22 +69,50 @@ def test_next_job_is_moved_into_work_qeueue(self):
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid2'])
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.lua('complete', 0, 'jid1', 'worker', 'queue', {})
-    # code.interact(local=locals())
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid2'])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
 
 
   '''Test that when a job completes it properly releases the lock'''
   def test_on_complete_lock_is_released(self):
-    print("pending")
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.lua('complete', 0, 'jid', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
 
   '''Test that when a job fails it properly releases the lock'''
   def test_on_failure_lock_is_released(self):
-    print("pending")
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.lua('fail', 0, 'jid', 'worker', 'failed', 'i failed', {})
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
 
   '''Test that when a job retries it properly releases the lock
      and goes back into pending'''
   def test_on_retry_lock_is_released(self):
-    print("pending")
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttle', 'tid')
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.lua('retry', 0, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid1'])
+
+  '''Test that when a job retries and no pending jobs it immediately acquires the lock again'''
+  def test_on_retry_no_pending_lock_is_reacquired(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.lua('retry', 0, 'jid', 'queue', 'worker', 0, 'retry', 'retrying')
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
 
 # What about Recurring Jobs???

From 4df412313097935d1d36bc0adc04ae09168cc53c Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 11 Mar 2014 10:53:13 -0400
Subject: [PATCH 17/72] switched to rank instead of score

---
 api.lua               | 4 ++--
 base.lua              | 6 +++---
 test/test_throttle.py | 2 +-
 throttle.lua          | 6 +++---
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/api.lua b/api.lua
index 4463098..7054433 100644
--- a/api.lua
+++ b/api.lua
@@ -214,11 +214,11 @@ QlessAPI['throttle.delete'] = function(now, tid)
 end
 
 QlessAPI['throttle.locks'] = function(now, tid)
-  return Qless.throttle(tid).locks:members()
+  return Qless.throttle(tid).locks.members()
 end
 
 QlessAPI['throttle.pending'] = function(now, tid)
-  return Qless.throttle(tid).pending:members()
+  return Qless.throttle(tid).pending.members()
 end
 -------------------------------------------------------------------------------
 -- Function lookup
diff --git a/base.lua b/base.lua
index f6b8365..46a7c68 100644
--- a/base.lua
+++ b/base.lua
@@ -95,7 +95,7 @@ function Qless.throttle(tid)
         return redis.call('zrem', QlessThrottle.ns .. tid .. '-locks', unpack(arg))
       end
     end, pop = function(min, max)
-      return redis.call('zremrangebyscore', QlessThrottle.ns .. tid .. '-locks', min, max)
+      return redis.call('zremrangebyrank', QlessThrottle.ns .. tid .. '-locks', min, max)
     end
   }
 
@@ -106,7 +106,7 @@ function Qless.throttle(tid)
     end, members = function()
       return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', 0, -1)
     end, peek = function(min, max)
-      return redis.call('zrangebyscore', QlessThrottle.ns .. tid .. '-pending', min, max)
+      return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', min, max)
     end, add = function(...)
       if #arg > 0 then
         redis.call('zadd', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
@@ -116,7 +116,7 @@ function Qless.throttle(tid)
         return redis.call('zrem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
       end
     end, pop = function(min, max)
-      return redis.call('zremrangebyscore', QlessThrottle.ns .. tid .. '-pending', min, max)
+      return redis.call('zremrangebyrank', QlessThrottle.ns .. tid .. '-pending', min, max)
     end
   }
   return throttle
diff --git a/test/test_throttle.py b/test/test_throttle.py
index 14abaf5..f64303e 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -115,4 +115,4 @@ def test_on_retry_no_pending_lock_is_reacquired(self):
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
 
-# What about Recurring Jobs???
+# class TestRecurring(TestQless):
diff --git a/throttle.lua b/throttle.lua
index ec533e6..06e7106 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -46,10 +46,10 @@ end
 -- the job will be moved from the throttled
 -- queue into the work queue
 function QlessThrottle:release(now, jid)
+  redis.call('set', 'printline', jid .. ' is releasing lock on ' .. self.id)
   self.locks.remove(jid)
-
-  -- 0th index does not exist, thanks redis!
-  local next_jid = unpack(self:pending_pop(1, 1))
+  redis.call('set', 'printline', 'retrieving next job from pending on ' .. self.id)
+  local next_jid = unpack(self:pending_pop(0, 0))
   if next_jid and self:acquire(next_jid) then
     local job = Qless.job(next_jid):data()
     local queue_obj = Qless.queue(job.queue)

From 7fff7848ebb20f756c5c130ee9bdd127568c0634 Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Tue, 11 Mar 2014 13:57:30 -0400
Subject: [PATCH 18/72] Acquire throttle on pop

---
 job.lua               |  1 -
 queue.lua             | 66 ++++++++++++++++++++-----------------
 test/test_throttle.py | 77 ++++++++++++++++++++++++++++---------------
 throttle.lua          |  4 +--
 4 files changed, 88 insertions(+), 60 deletions(-)

diff --git a/job.lua b/job.lua
index 6db8e6d..90e0454 100644
--- a/job.lua
+++ b/job.lua
@@ -462,7 +462,6 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
 
   -- Release the throttle for the job
   self:release_throttle(now)
-  self:acquire_throttle()
 
   -- Remove this job from the worker that was previously working it
   redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
diff --git a/queue.lua b/queue.lua
index 9ee6691..a708911 100644
--- a/queue.lua
+++ b/queue.lua
@@ -337,42 +337,48 @@ function QlessQueue:pop(now, worker, count)
   table.extend(jids, self.work.peek(count - #jids))
 
   local state
+  local popped = {}
   for index, jid in ipairs(jids) do
     local job = Qless.job(jid)
-    state = unpack(job:data('state'))
-    job:history(now, 'popped', {worker = worker})
-
-    -- Update the wait time statistics
-    local time = tonumber(
-      redis.call('hget', QlessJob.ns .. jid, 'time') or now)
-    local waiting = now - time
-    self:stat(now, 'wait', waiting)
-    redis.call('hset', QlessJob.ns .. jid,
-      'time', string.format("%.20f", now))
-
-    -- Add this job to the list of jobs handled by this worker
-    redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid)
-
-    -- Update the jobs data, and add its locks, and return the job
-    job:update({
-      worker  = worker,
-      expires = expires,
-      state   = 'running'
-    })
-
-    self.locks.add(expires, jid)
-
-    local tracked = redis.call('zscore', 'ql:tracked', jid) ~= false
-    if tracked then
-      Qless.publish('popped', jid)
+    if job:acquire_throttle() then
+      state = unpack(job:data('state'))
+      job:history(now, 'popped', {worker = worker})
+
+      -- Update the wait time statistics
+      local time = tonumber(
+        redis.call('hget', QlessJob.ns .. jid, 'time') or now)
+      local waiting = now - time
+      self:stat(now, 'wait', waiting)
+      redis.call('hset', QlessJob.ns .. jid,
+        'time', string.format("%.20f", now))
+
+      -- Add this job to the list of jobs handled by this worker
+      redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid)
+
+      -- Update the jobs data, and add its locks, and return the job
+      job:update({
+        worker  = worker,
+        expires = expires,
+        state   = 'running'
+      })
+
+      self.locks.add(expires, jid)
+
+      local tracked = redis.call('zscore', 'ql:tracked', jid) ~= false
+      if tracked then
+        Qless.publish('popped', jid)
+      end
+      popped[jid] = jid
+    else
+      job:history(now, 'throttled', {worker = worker})
     end
   end
 
   -- If we are returning any jobs, then we should remove them from the work
   -- queue
-  self.work.remove(unpack(jids))
+  self.work.remove(unpack(popped))
 
-  return jids
+  return popped
 end
 
 -- Update the stats for this queue
@@ -599,10 +605,8 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
     if redis.call('scard', QlessJob.ns .. jid .. '-dependencies') > 0 then
       self.depends.add(now, jid)
       redis.call('hset', QlessJob.ns .. jid, 'state', 'depends')
-    elseif job:acquire_throttle() then
-      self.work.add(now, priority, jid)
     else
-      self.throttled.add(jid)
+      self.work.add(now, priority, jid)
     end
   end
 
diff --git a/test/test_throttle.py b/test/test_throttle.py
index f64303e..be2eb6f 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -25,50 +25,51 @@ def test_delete(self):
     self.assertEqual(self.lua('throttle.get', 0, 'tid'), None)
 
 class TestAcquire(TestQless):
-  '''Test acquiring of a throttle lock'''
-  def test_acquire(self):
+  '''Test that job can specify a throttle'''
+  def test_specify_throttle(self):
     self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
     self.assertEqual(self.lua('get', 0, 'jid')['throttle'], 'tid')
 
+  '''Test that a job can acquire a throttle'''
+  def test_acquire_throttle(self):
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+
   '''Test that acquiring of a throttle lock properly limits the number of jobs'''
   def test_limit_number_of_locks(self):
     self.lua('throttle.set', 0, 'tid', 1)
     self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 0, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 0, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 0, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 2, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 3, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('pop', 0, 'queue', 'worker', 4)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid2', 'jid3', 'jid4'])
 
 class TestRelease(TestQless):
-  '''Test that job retains lock while working'''
-  def test_retains_lock_while_working(self):
-    self.lua('throttle.set', 0, 'tid', 1)
-    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
-    self.lua('pop', 0, 'queue', 'worker', 1)
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
-    self.lua('complete', 0, 'jid', 'worker', 'queue', {})
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
-
   '''Test that when there are no pending jobs lock is properly released'''
   def test_no_pending_jobs(self):
     self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
     self.lua('complete', 0, 'jid', 'worker', 'queue', {})
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
 
   '''Test that releasing a lock properly another job in the work queue'''
   def test_next_job_is_moved_into_work_qeueue(self):
     self.lua('throttle.set', 0, 'tid', 1)
     self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 0, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('pop', 2, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid2'])
-    self.lua('pop', 0, 'queue', 'worker', 1)
     self.lua('complete', 0, 'jid1', 'worker', 'queue', {})
+    # Lock should be empty until another job is popped
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.lua('pop', 2, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid2'])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
 
@@ -77,8 +78,8 @@ def test_next_job_is_moved_into_work_qeueue(self):
   def test_on_complete_lock_is_released(self):
     self.lua('throttle.set', 0, 'tid', 1)
     self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.lua('complete', 0, 'jid', 'worker', 'queue', {})
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
@@ -87,8 +88,8 @@ def test_on_complete_lock_is_released(self):
   def test_on_failure_lock_is_released(self):
     self.lua('throttle.set', 0, 'tid', 1)
     self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.lua('fail', 0, 'jid', 'worker', 'failed', 'i failed', {})
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
@@ -98,21 +99,45 @@ def test_on_failure_lock_is_released(self):
   def test_on_retry_lock_is_released(self):
     self.lua('throttle.set', 0, 'tid', 1)
     self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 0, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
+    self.lua('retry', 0, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+
+  '''Test that when a job retries it is able to reacquire the lock when next popped'''
+  def test_on_retry_lock_is_reacquired(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
     self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
     self.lua('retry', 0, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid2'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
 
-  '''Test that when a job retries and no pending jobs it immediately acquires the lock again'''
+  '''Test that when a job retries and no pending jobs it acquires the lock again on next pop'''
   def test_on_retry_no_pending_lock_is_reacquired(self):
     self.lua('throttle.set', 0, 'tid', 1)
     self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.lua('retry', 0, 'jid', 'queue', 'worker', 0, 'retry', 'retrying')
+    self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
 
+  '''Test that when a job retries and another job is pending, the pending job acquires the lock'''
+  def test_on_retry_no_pending_lock_is_reacquired(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('pop', 2, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 7, 'tid'), ['jid2'])
+    self.lua('retry', 4, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
+    self.assertEqual(self.lua('throttle.pending', 7, 'tid'), [])
+    self.lua('pop', 5, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 6, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('throttle.pending', 7, 'tid'), ['jid1'])
+
 # class TestRecurring(TestQless):
diff --git a/throttle.lua b/throttle.lua
index 06e7106..a49b870 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -50,7 +50,7 @@ function QlessThrottle:release(now, jid)
   self.locks.remove(jid)
   redis.call('set', 'printline', 'retrieving next job from pending on ' .. self.id)
   local next_jid = unpack(self:pending_pop(0, 0))
-  if next_jid and self:acquire(next_jid) then
+  if next_jid then
     local job = Qless.job(next_jid):data()
     local queue_obj = Qless.queue(job.queue)
     queue_obj.throttled.remove(job.jid)
@@ -74,6 +74,6 @@ end
 
 -- Returns true if the throttle has locks available, false otherwise.
 function QlessThrottle:available()
-  redis.call('set', 'printline', 'available ' .. self.maximum .. ' == 0 or ' .. self.locks.count() .. ' < self.maximum')
+  redis.call('set', 'printline', 'available ' .. self.maximum .. ' == 0 or ' .. self.locks.count() .. ' < ' .. self.maximum)
   return self.maximum == 0 or self.locks.count() < self.maximum
 end

From fe776efeba6310f4f7d96aabc8bf349aa25053f1 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 11 Mar 2014 14:56:06 -0400
Subject: [PATCH 19/72] test fixes

---
 base.lua  | 1 -
 queue.lua | 4 +++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/base.lua b/base.lua
index 46a7c68..02fbfd6 100644
--- a/base.lua
+++ b/base.lua
@@ -250,7 +250,6 @@ function Qless.track(now, command, jid)
     assert(jid, 'Track(): Arg "jid" missing')
     -- Verify that job exists
     assert(Qless.job(jid):exists(), 'Track(): Job does not exist')
-    redis.call('set', 'print_line_track_command', now .. command .. jid)
     if string.lower(command) == 'track' then
       Qless.publish('track', jid)
       return redis.call('zadd', 'ql:tracked', now, jid)
diff --git a/queue.lua b/queue.lua
index a708911..25a991c 100644
--- a/queue.lua
+++ b/queue.lua
@@ -368,9 +368,11 @@ function QlessQueue:pop(now, worker, count)
       if tracked then
         Qless.publish('popped', jid)
       end
-      popped[jid] = jid
+
+      table.insert(popped, jid)
     else
       job:history(now, 'throttled', {worker = worker})
+      self.throttled.add(now, jid)
     end
   end
 

From 4fc8e3c8c20cda0cc29d3bd45f33f61cbba24866 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 11 Mar 2014 15:05:17 -0400
Subject: [PATCH 20/72] test fixes

---
 queue.lua             | 8 ++++++--
 test/test_throttle.py | 1 -
 throttle.lua          | 4 ++--
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/queue.lua b/queue.lua
index 25a991c..b158c46 100644
--- a/queue.lua
+++ b/queue.lua
@@ -82,7 +82,7 @@ function Qless.queue(name)
     peek = function(now, offset, count)
       return redis.call('zrange', queue:prefix('throttled'), offset, offset + count - 1)
     end, add = function(now, jid)
-      redis.call('zadd', queue:prefix('throttled'), jid)
+      redis.call('zadd', queue:prefix('throttled'), now, jid)
     end, remove = function(...)
       if #arg > 0 then
         return redis.call('zrem', queue:prefix('throttled'), unpack(arg))
@@ -336,10 +336,12 @@ function QlessQueue:pop(now, worker, count)
   -- queue itself and the priorities therein
   table.extend(jids, self.work.peek(count - #jids))
 
+  redis.call('set', 'printline', 'before loop')
   local state
   local popped = {}
   for index, jid in ipairs(jids) do
     local job = Qless.job(jid)
+    redis.call('set', 'printline', 'pop acquiring throttle')
     if job:acquire_throttle() then
       state = unpack(job:data('state'))
       job:history(now, 'popped', {worker = worker})
@@ -371,11 +373,13 @@ function QlessQueue:pop(now, worker, count)
 
       table.insert(popped, jid)
     else
+      redis.call('set', 'printline', 'acquire failed')
       job:history(now, 'throttled', {worker = worker})
+      redis.call('set', 'printline', 'history set')
       self.throttled.add(now, jid)
     end
   end
-
+  redis.call('set', 'printline', 'before loop')
   -- If we are returning any jobs, then we should remove them from the work
   -- queue
   self.work.remove(unpack(popped))
diff --git a/test/test_throttle.py b/test/test_throttle.py
index be2eb6f..2f9c268 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -3,7 +3,6 @@
 import redis
 import code
 from common import TestQless
-# code.interact(local=locals())
 
 class TestThrottle(TestQless):
   '''Test setting throttle data'''
diff --git a/throttle.lua b/throttle.lua
index a49b870..feb8545 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -46,9 +46,9 @@ end
 -- the job will be moved from the throttled
 -- queue into the work queue
 function QlessThrottle:release(now, jid)
-  redis.call('set', 'printline', jid .. ' is releasing lock on ' .. self.id)
+  --redis.call('set', 'printline', jid .. ' is releasing lock on ' .. self.id)
   self.locks.remove(jid)
-  redis.call('set', 'printline', 'retrieving next job from pending on ' .. self.id)
+  --redis.call('set', 'printline', 'retrieving next job from pending on ' .. self.id)
   local next_jid = unpack(self:pending_pop(0, 0))
   if next_jid then
     local job = Qless.job(next_jid):data()

From 27aebd45e228a2a2dc13d8cfc64a51a835a05342 Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Tue, 11 Mar 2014 15:31:48 -0400
Subject: [PATCH 21/72] Add tests for dependent throttling

---
 test/test_throttle.py | 67 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 67 insertions(+)

diff --git a/test/test_throttle.py b/test/test_throttle.py
index 2f9c268..88d3107 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -139,4 +139,71 @@ def test_on_retry_no_pending_lock_is_reacquired(self):
     self.assertEqual(self.lua('throttle.locks', 6, 'tid'), ['jid2'])
     self.assertEqual(self.lua('throttle.pending', 7, 'tid'), ['jid1'])
 
+class TestDependents(TestQless):
+  def test_dependencies_can_acquire_lock_after_dependent_success(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'depends', ['jid1'], 'throttle', 'tid')
+    self.lua('put', 3, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'depends', ['jid2'], 'throttle', 'tid')
+
+    self.lua('pop', 4, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 5, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 6, 'tid'), [])
+    self.lua('complete', 7, 'jid1', 'worker', 'queue', {})
+
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.lua('complete', 0, 'jid2', 'worker', 'queue', {})
+
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid3'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.lua('complete', 0, 'jid3', 'worker', 'queue', {})
+
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+
+  def test_dependencies_can_acquire_lock_after_dependent_failure(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'depends', ['jid1'], 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'depends', ['jid2'], 'throttle', 'tid')
+
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.lua('fail', 0, 'jid1', 'worker', 'failed', 'i failed', {})
+
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+
+  def test_dependencies_do_not_acquire_lock_on_dependent_retry(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'depends', ['jid1'], 'throttle', 'tid')
+    self.lua('put', 3, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'depends', ['jid2'], 'throttle', 'tid')
+
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.lua('retry', 0, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
+
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.lua('complete', 0, 'jid1', 'worker', 'queue', {})
+
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.lua('complete', 0, 'jid2', 'worker', 'queue', {})
+
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid3'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.lua('complete', 0, 'jid3', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+
 # class TestRecurring(TestQless):

From a3244fe2b78b23186881f3ec7ed90d48d00aba1f Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Tue, 11 Mar 2014 17:02:46 -0400
Subject: [PATCH 22/72] Use throttles to handle max-queue-concurrency

---
 job.lua            |   7 +++
 queue.lua          | 124 +++++++++++++++++++++++----------------------
 test/test_queue.py |   8 +--
 3 files changed, 74 insertions(+), 65 deletions(-)

diff --git a/job.lua b/job.lua
index 90e0454..44fda23 100644
--- a/job.lua
+++ b/job.lua
@@ -131,6 +131,8 @@ function QlessJob:complete(now, worker, queue, data, ...)
   queue_obj.locks.remove(self.jid)
   queue_obj.scheduled.remove(self.jid)
 
+  -- Release queue throttle
+  Qless.throttle(QlessQueue.ns .. queue):release(now, self.jid)
   self:release_throttle(now)
 
   ----------------------------------------------------------
@@ -400,6 +402,8 @@ function QlessJob:fail(now, worker, group, message, data)
       ['worker']  = worker
     }))
 
+  -- Release queue throttle
+  Qless.throttle(QlessQueue.ns .. queue):release(now, self.jid)
   self:release_throttle(now)
 
   -- Add this group of failure to the list of failures
@@ -460,6 +464,9 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
   -- Remove it from the locks key of the old queue
   Qless.queue(oldqueue).locks.remove(self.jid)
 
+  -- Release the throttle for the queue
+  Qless.throttle(QlessQueue.ns .. queue):release(now, self.jid)
+
   -- Release the throttle for the job
   self:release_throttle(now)
 
diff --git a/queue.lua b/queue.lua
index b158c46..429aff2 100644
--- a/queue.lua
+++ b/queue.lua
@@ -292,11 +292,6 @@ function QlessQueue:pop(now, worker, count)
   count = assert(tonumber(count),
     'Pop(): Arg "count" missing or not a number: ' .. tostring(count))
 
-  -- We should find the heartbeat interval for this queue heartbeat
-  local expires = now + tonumber(
-    Qless.config.get(self.name .. '-heartbeat') or
-    Qless.config.get('heartbeat', 60))
-
   -- If this queue is paused, then return no jobs
   if self:paused() then
     return {}
@@ -305,88 +300,89 @@ function QlessQueue:pop(now, worker, count)
   -- Make sure we this worker to the list of seen workers
   redis.call('zadd', 'ql:workers', now, worker)
 
-  -- Check our max concurrency, and limit the count
-  local max_concurrency = tonumber(
-    Qless.config.get(self.name .. '-max-concurrency', 0))
-
-  if max_concurrency > 0 then
-    -- Allow at most max_concurrency - #running
-    local allowed = math.max(0, max_concurrency - self.locks.running(now))
-    count = math.min(allowed, count)
-    if count == 0 then
-      return {}
-    end
-  end
-
-  local jids = self:invalidate_locks(now, count)
+  local dead_jids = self:invalidate_locks(now, count) or {}
   -- Now we've checked __all__ the locks for this queue the could
   -- have expired, and are no more than the number requested.
 
   -- If we still need jobs in order to meet demand, then we should
   -- look for all the recurring jobs that need jobs run
-  self:check_recurring(now, count - #jids)
+  self:check_recurring(now, count - #dead_jids)
 
   -- If we still need values in order to meet the demand, then we
   -- should check if any scheduled items, and if so, we should
   -- insert them to ensure correctness when pulling off the next
   -- unit of work.
-  self:check_scheduled(now, count - #jids)
+  self:check_scheduled(now, count - #dead_jids)
 
   -- With these in place, we can expand this list of jids based on the work
   -- queue itself and the priorities therein
-  table.extend(jids, self.work.peek(count - #jids))
+  local jids = self.work.peek(count - #dead_jids) or {}
+
+  local queue_throttle = Qless.throttle(QlessQueue.ns .. self.name)
 
-  redis.call('set', 'printline', 'before loop')
-  local state
   local popped = {}
   for index, jid in ipairs(jids) do
     local job = Qless.job(jid)
-    redis.call('set', 'printline', 'pop acquiring throttle')
-    if job:acquire_throttle() then
-      state = unpack(job:data('state'))
-      job:history(now, 'popped', {worker = worker})
-
-      -- Update the wait time statistics
-      local time = tonumber(
-        redis.call('hget', QlessJob.ns .. jid, 'time') or now)
-      local waiting = now - time
-      self:stat(now, 'wait', waiting)
-      redis.call('hset', QlessJob.ns .. jid,
-        'time', string.format("%.20f", now))
-
-      -- Add this job to the list of jobs handled by this worker
-      redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid)
-
-      -- Update the jobs data, and add its locks, and return the job
-      job:update({
-        worker  = worker,
-        expires = expires,
-        state   = 'running'
-      })
-
-      self.locks.add(expires, jid)
-
-      local tracked = redis.call('zscore', 'ql:tracked', jid) ~= false
-      if tracked then
-        Qless.publish('popped', jid)
-      end
-
+    if queue_throttle:acquire(jid) and job:acquire_throttle() then
+      self:pop_job(now, worker, job)
       table.insert(popped, jid)
     else
-      redis.call('set', 'printline', 'acquire failed')
       job:history(now, 'throttled', {worker = worker})
-      redis.call('set', 'printline', 'history set')
-      self.throttled.add(now, jid)
     end
   end
-  redis.call('set', 'printline', 'before loop')
-  -- If we are returning any jobs, then we should remove them from the work
-  -- queue
+
+  -- If we are returning any jobs, then remove popped jobs from
+  -- work queue
   self.work.remove(unpack(popped))
 
+  -- Process dead jids after removing newly popped jids from work queue
+  -- This changes the order of returned jids
+  for index, jid in ipairs(dead_jids) do
+    self:pop_job(now, worker, Qless.job(jid))
+    table.insert(popped, jid)
+  end
+
   return popped
 end
 
+function QlessQueue:pop_job(now, worker, job)
+  local state
+  local jid = job.jid
+  state = unpack(job:data('state'))
+  job:history(now, 'popped', {worker = worker})
+
+  -- We should find the heartbeat interval for this queue heartbeat
+  local expires = now + tonumber(
+    Qless.config.get(self.name .. '-heartbeat') or
+    Qless.config.get('heartbeat', 60))
+
+  -- Update the wait time statistics
+  -- Just does job:data('time') do the same as this?
+  local time = tonumber(
+    redis.call('hget', QlessJob.ns .. jid, 'time') or now)
+  local waiting = now - time
+  self:stat(now, 'wait', waiting)
+  redis.call('hset', QlessJob.ns .. jid,
+    'time', string.format("%.20f", now))
+
+  -- Add this job to the list of jobs handled by this worker
+  redis.call('zadd', 'ql:w:' .. worker .. ':jobs', expires, jid)
+
+  -- Update the jobs data, and add its locks, and return the job
+  job:update({
+    worker  = worker,
+    expires = expires,
+    state   = 'running'
+  })
+
+  self.locks.add(expires, jid)
+
+  local tracked = redis.call('zscore', 'ql:tracked', jid) ~= false
+  if tracked then
+    Qless.publish('popped', jid)
+  end
+end
+
 -- Update the stats for this queue
 function QlessQueue:stat(now, stat, val)
   -- The bin is midnight of the provided day
@@ -927,8 +923,14 @@ function QlessQueue:invalidate_locks(now, count)
         self.locks.remove(jid)
         self.scheduled.remove(jid)
 
-        local group = 'failed-retries-' .. Qless.job(jid):data()['queue']
         local job = Qless.job(jid)
+        local job_data = Qless.job(jid):data()
+        local queue = job_data['queue']
+        local group = 'failed-retries-' .. queue
+
+        job:release_throttle(now)
+        Qless.throttle(QlessQueue.ns .. queue):release(now, jid)
+
         job:history(now, 'failed', {group = group})
         redis.call('hmset', QlessJob.ns .. jid, 'state', 'failed',
           'worker', '',
diff --git a/test/test_queue.py b/test/test_queue.py
index 144966e..e1e41a0 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -592,7 +592,7 @@ def test_move(self):
 
     def test_max_concurrency(self):
         '''We can control the maxinum number of jobs available in a queue'''
-        self.lua('config.set', 0, 'queue-max-concurrency', 5)
+        self.lua('throttle.set', 0, 'ql:q:queue', 5)
         for jid in xrange(10):
             self.lua('put', jid, 'worker', 'queue', jid, 'klass', {}, 0)
         self.assertEqual(len(self.lua('pop', 10, 'queue', 'worker', 10)), 5)
@@ -609,7 +609,7 @@ def test_reduce_max_concurrency(self):
         for jid in xrange(100):
             self.lua('put', jid, 'worker', 'queue', jid, 'klass', {}, 0)
         self.lua('pop', 100, 'queue', 'worker', 10)
-        self.lua('config.set', 100, 'queue-max-concurrency', 5)
+        self.lua('throttle.set', 100, 'ql:q:queue', 5)
         for jid in xrange(6):
             self.assertEqual(
                 len(self.lua('pop', 100, 'queue', 'worker', 10)), 0)
@@ -620,7 +620,7 @@ def test_reduce_max_concurrency(self):
 
     def test_stalled_max_concurrency(self):
         '''Stalled jobs can still be popped with max concurrency'''
-        self.lua('config.set', 0, 'queue-max-concurrency', 1)
+        self.lua('throttle.set', 0, 'ql:q:queue', 1)
         self.lua('config.set', 0, 'grace-period', 0)
         self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'retries', 5)
         job = self.lua('pop', 0, 'queue', 'worker', 10)[0]
@@ -630,7 +630,7 @@ def test_stalled_max_concurrency(self):
 
     def test_fail_max_concurrency(self):
         '''Failing a job makes space for a job in a queue with concurrency'''
-        self.lua('config.set', 0, 'queue-max-concurrency', 1)
+        self.lua('throttle.set', 0, 'ql:q:queue', 1)
         self.lua('put', 0, 'worker', 'queue', 'a', 'klass', {}, 0)
         self.lua('put', 1, 'worker', 'queue', 'b', 'klass', {}, 0)
         self.lua('pop', 2, 'queue', 'worker', 10)

From 81e21037eb84a843321b6bed41a6069162c9a627 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 11 Mar 2014 17:16:10 -0400
Subject: [PATCH 23/72] removed commented code

---
 test/test_throttle.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/test/test_throttle.py b/test/test_throttle.py
index 88d3107..2db2d80 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -205,5 +205,3 @@ def test_dependencies_do_not_acquire_lock_on_dependent_retry(self):
     self.lua('complete', 0, 'jid3', 'worker', 'queue', {})
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
-
-# class TestRecurring(TestQless):

From b4e7765a0b1b9f086880d30a57d8b81b9db7c493 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 11 Mar 2014 19:19:05 -0400
Subject: [PATCH 24/72] support multiple resources

---
 api.lua                |  2 +-
 base.lua               |  4 +--
 job.lua                | 49 ++++++++++++++++++------------
 queue.lua              | 55 +++++++++++++++++----------------
 test/test_fail.py      |  1 +
 test/test_job.py       |  1 +
 test/test_locks.py     |  3 ++
 test/test_queue.py     |  4 +++
 test/test_recurring.py |  2 ++
 test/test_throttle.py  | 69 ++++++++++++++++++++++--------------------
 test/test_track.py     |  1 +
 throttle.lua           |  9 +++++-
 12 files changed, 118 insertions(+), 82 deletions(-)

diff --git a/api.lua b/api.lua
index 7054433..4077862 100644
--- a/api.lua
+++ b/api.lua
@@ -135,7 +135,7 @@ QlessAPI.unpause = function(now, ...)
 end
 
 QlessAPI.cancel = function(now, ...)
-  return Qless.cancel(unpack(arg))
+  return Qless.cancel(now, unpack(arg))
 end
 
 QlessAPI.timeout = function(now, ...)
diff --git a/base.lua b/base.lua
index 02fbfd6..befacc3 100644
--- a/base.lua
+++ b/base.lua
@@ -382,7 +382,7 @@ end
 -- Cancel a job from taking place. It will be deleted from the system, and any
 -- attempts to renew a heartbeat will fail, and any attempts to complete it
 -- will fail. If you try to get the data on the object, you will get nothing.
-function Qless.cancel(...)
+function Qless.cancel(now, ...)
   -- Dependents is a mapping of a job to its dependent jids
   local dependents = {}
   for _, jid in ipairs(arg) do
@@ -435,7 +435,7 @@ function Qless.cancel(...)
         queue.depends.remove(jid)
       end
 
-      Qless.job(namespaced_jid):release_throttle()
+      Qless.job(namespaced_jid):release_throttles(now)
 
       -- We should probably go through all our dependencies and remove
       -- ourselves from the list of dependents
diff --git a/job.lua b/job.lua
index 44fda23..0016083 100644
--- a/job.lua
+++ b/job.lua
@@ -11,7 +11,7 @@ function QlessJob:data(...)
   local job = redis.call(
       'hmget', QlessJob.ns .. self.jid, 'jid', 'klass', 'state', 'queue',
       'worker', 'priority', 'expires', 'retries', 'remaining', 'data',
-      'tags', 'failure', 'throttle')
+      'tags', 'failure', 'throttles')
 
   -- Return nil if we haven't found it
   if not job[1] then
@@ -34,7 +34,7 @@ function QlessJob:data(...)
     tags         = cjson.decode(job[11]),
     history      = self:history(),
     failure      = cjson.decode(job[12] or '{}'),
-    throttle     = job[13] or nil,
+    throttles    = cjson.decode(job[13] or '[]'),
     dependents   = redis.call(
       'smembers', QlessJob.ns .. self.jid .. '-dependents'),
     dependencies = redis.call(
@@ -131,9 +131,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
   queue_obj.locks.remove(self.jid)
   queue_obj.scheduled.remove(self.jid)
 
-  -- Release queue throttle
-  Qless.throttle(QlessQueue.ns .. queue):release(now, self.jid)
-  self:release_throttle(now)
+  self:release_throttles(now)
 
   ----------------------------------------------------------
   -- This is the massive stats update that we have to do
@@ -402,9 +400,7 @@ function QlessJob:fail(now, worker, group, message, data)
       ['worker']  = worker
     }))
 
-  -- Release queue throttle
-  Qless.throttle(QlessQueue.ns .. queue):release(now, self.jid)
-  self:release_throttle(now)
+  self:release_throttles(now)
 
   -- Add this group of failure to the list of failures
   redis.call('sadd', 'ql:failures', group)
@@ -464,11 +460,8 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
   -- Remove it from the locks key of the old queue
   Qless.queue(oldqueue).locks.remove(self.jid)
 
-  -- Release the throttle for the queue
-  Qless.throttle(QlessQueue.ns .. queue):release(now, self.jid)
-
   -- Release the throttle for the job
-  self:release_throttle(now)
+  self:release_throttles(now)
 
   -- Remove this job from the worker that was previously working it
   redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
@@ -805,17 +798,33 @@ function QlessJob:history(now, what, item)
   end
 end
 
-function QlessJob:release_throttle(now)
-  local tid = redis.call('hget', QlessJob.ns .. self.jid, 'throttle')
-  if tid then
+function QlessJob:release_throttles(now)
+  local throttles = redis.call('hget', QlessJob.ns .. self.jid, 'throttles')
+  throttles = cjson.decode(throttles or '[]')
+
+  for _, tid in ipairs(throttles) do
+    redis.call('set', 'printline', 'releasing throttle : ' .. tid)
     Qless.throttle(tid):release(now, self.jid)
   end
 end
 
-function QlessJob:acquire_throttle()
-  local tid = unpack(redis.call('hmget', QlessJob.ns .. self.jid, 'throttle'))
-  if tid then
-    return Qless.throttle(tid):acquire(self.jid)
+function QlessJob:acquire_throttles(now)
+  local throttles = redis.call('hget', QlessJob.ns .. self.jid, 'throttles')
+  throttles = cjson.decode(throttles or '[]')
+
+  local acquired_all = true
+  local acquired_throttles = {}
+  for _, tid in ipairs(throttles) do
+    acquired_all = acquired_all and Qless.throttle(tid):acquire(self.jid)
+    table.insert(acquired_throttles, tid)
   end
-  return true
+
+  if not acquired_all then
+    redis.call('set', 'printline', 'rolling back acquired locks')
+    for _, tid in ipairs(acquired_throttles) do
+      Qless.throttle(tid):rollback_acquire(self.jid)
+    end
+  end
+
+  return acquired_all
 end
diff --git a/queue.lua b/queue.lua
index 429aff2..b23d1ea 100644
--- a/queue.lua
+++ b/queue.lua
@@ -323,7 +323,7 @@ function QlessQueue:pop(now, worker, count)
   local popped = {}
   for index, jid in ipairs(jids) do
     local job = Qless.job(jid)
-    if queue_throttle:acquire(jid) and job:acquire_throttle() then
+    if job:acquire_throttles(now) then
       self:pop_job(now, worker, job)
       table.insert(popped, jid)
     else
@@ -468,8 +468,10 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
     'Put(): Arg "priority" not a number'  .. tostring(options['priority']))
   local depends = assert(cjson.decode(options['depends'] or '[]') ,
     'Put(): Arg "depends" not JSON: '     .. tostring(options['depends']))
-  -- local throttle = options['throttle']
+  local throttles = assert(cjson.decode(options['throttles'] or '[]'),
+    'Put(): Arg "throttles" not JSON array: ' .. tostring(options['throttles']))
 
+  redis.call('set', 'printline', 'throttles : ' .. tostring(options['throttles']))
   -- If the job has old dependencies, determine which dependencies are
   -- in the new dependencies but not in the old ones, and which are in the
   -- old ones but not in the new
@@ -555,6 +557,9 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
     redis.call('hincrby', 'ql:s:stats:' .. bin .. ':' .. self.name, 'failed'  , -1)
   end
 
+  -- insert default queue throttle
+  table.insert(throttles, QlessQueue.ns .. self.name)
+
   data = {
     'jid'      , jid,
     'klass'    , klass,
@@ -567,15 +572,10 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
     'queue'    , self.name,
     'retries'  , retries,
     'remaining', retries,
-    'time'     , string.format("%.20f", now)
+    'time'     , string.format("%.20f", now),
+    'throttles', cjson.encode(throttles)
   }
 
-  -- Insert the throttle resource into the array if it exists.
-  if options['throttle'] then
-    table.insert(data, 'throttle')
-    table.insert(data, options['throttle'])
-  end
-
   -- First, let's save its data
   redis.call('hmset', QlessJob.ns .. jid, unpack(data))
 
@@ -699,6 +699,8 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
     options.backlog = assert(tonumber(options.backlog  or 0),
       'Recur(): Arg "backlog" not a number: ' .. tostring(
         options.backlog))
+    options.throttles = assert(cjson.decode(options['throttles'] or '[]'),
+      'Recur(): Arg "throttles" not JSON array: ' .. tostring(options['throttles']))
 
     local count, old_queue = unpack(redis.call('hmget', 'ql:r:' .. jid, 'count', 'queue'))
     count = count or 0
@@ -711,19 +713,20 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
 
     -- Do some insertions
     redis.call('hmset', 'ql:r:' .. jid,
-      'jid'     , jid,
-      'klass'   , klass,
-      'data'    , raw_data,
-      'priority', options.priority,
-      'tags'    , cjson.encode(options.tags or {}),
-      'state'   , 'recur',
-      'queue'   , self.name,
-      'type'    , 'interval',
+      'jid'      , jid,
+      'klass'    , klass,
+      'data'     , raw_data,
+      'priority' , options.priority,
+      'tags'     , cjson.encode(options.tags or {}),
+      'state'    , 'recur',
+      'queue'    , self.name,
+      'type'     , 'interval',
       -- How many jobs we've spawned from this
-      'count'   , count,
-      'interval', interval,
-      'retries' , options.retries,
-      'backlog' , options.backlog)
+      'count'    , count,
+      'interval' , interval,
+      'retries'  , options.retries,
+      'backlog'  , options.backlog,
+      'throttles', options.throttles)
     -- Now, we should schedule the next run of the job
     self.recurring.add(now + offset, jid)
 
@@ -759,9 +762,9 @@ function QlessQueue:check_recurring(now, count)
     -- get the last time each of them was run, and then increment
     -- it by its interval. While this time is less than now,
     -- we need to keep putting jobs on the queue
-    local klass, data, priority, tags, retries, interval, backlog = unpack(
+    local klass, data, priority, tags, retries, interval, backlog, throttles = unpack(
       redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority',
-        'tags', 'retries', 'interval', 'backlog'))
+        'tags', 'retries', 'interval', 'backlog', 'throttles'))
     local _tags = cjson.decode(tags)
     local score = math.floor(tonumber(self.recurring.score(jid)))
     interval = tonumber(interval)
@@ -807,7 +810,8 @@ function QlessQueue:check_recurring(now, count)
         'queue'    , self.name,
         'retries'  , retries,
         'remaining', retries,
-        'time'     , string.format("%.20f", score))
+        'time'     , string.format("%.20f", score),
+        'throttles', throttles)
       Qless.job(child_jid):history(score, 'put', {q = self.name})
 
       -- Now, if a delay was provided, and if it's in the future,
@@ -928,8 +932,7 @@ function QlessQueue:invalidate_locks(now, count)
         local queue = job_data['queue']
         local group = 'failed-retries-' .. queue
 
-        job:release_throttle(now)
-        Qless.throttle(QlessQueue.ns .. queue):release(now, jid)
+        job:release_throttles(now)
 
         job:history(now, 'failed', {group = group})
         redis.call('hmset', QlessJob.ns .. jid, 'state', 'failed',
diff --git a/test/test_fail.py b/test/test_fail.py
index 32f37d6..8e8ab4c 100644
--- a/test/test_fail.py
+++ b/test/test_fail.py
@@ -45,6 +45,7 @@ def test_basic(self):
             'state': 'failed',
             'tags': {},
             'tracked': False,
+            'throttles': ['ql:q:queue'],
             'worker': u''})
 
     def test_put(self):
diff --git a/test/test_job.py b/test/test_job.py
index 1163d9b..1ef1703 100644
--- a/test/test_job.py
+++ b/test/test_job.py
@@ -135,6 +135,7 @@ def test_basic(self):
             'state': 'complete',
             'tags': {},
             'tracked': False,
+            'throttles': ['ql:q:queue'],
             'worker': u''})
 
     def test_advance(self):
diff --git a/test/test_locks.py b/test/test_locks.py
index 37b101c..f261b2a 100644
--- a/test/test_locks.py
+++ b/test/test_locks.py
@@ -58,6 +58,7 @@ def test_lose_lock(self):
                 'state': 'running',
                 'tags': {},
                 'tracked': False,
+                'throttles': ['ql:q:queue'],
                 'worker': 'another'}])
         # When we try to heartbeat, it should raise an exception
         self.assertRaisesRegexp(redis.ResponseError, r'given out to another',
@@ -274,6 +275,7 @@ def test_retry_group_message(self):
             'state': 'failed',
             'tags': {},
             'tracked': False,
+            'throttles': ['ql:q:queue'],
             'worker': u''})
 
     def test_retry_delay(self):
@@ -323,6 +325,7 @@ def test_retry_failed_retries(self):
             'state': 'failed',
             'tags': {},
             'tracked': False,
+            'throttles': ['ql:q:queue'],
             'worker': u''
         })
 
diff --git a/test/test_queue.py b/test/test_queue.py
index e1e41a0..04a374f 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -323,6 +323,7 @@ def test_basic(self):
             'state': 'waiting',
             'tags': {},
             'tracked': False,
+            'throttles': ['ql:q:queue'],
             'worker': u''
         })
 
@@ -395,6 +396,7 @@ def test_move(self):
             'state': 'waiting',
             'tags': {},
             'tracked': False,
+            'throttles': ['ql:q:other'],
             'worker': u''})
 
     def test_move_update(self):
@@ -463,6 +465,7 @@ def test_basic(self):
             'state': 'waiting',
             'tags': {},
             'tracked': False,
+            'throttles': ['ql:q:foo'],
             'worker': u''
         }])
         # With several jobs in the queue, we should be able to see more
@@ -549,6 +552,7 @@ def test_basic(self):
             'state': 'running',
             'tags': {},
             'tracked': False,
+            'throttles': ['ql:q:queue'],
             'worker': 'worker'}])
 
     def test_pop_many(self):
diff --git a/test/test_recurring.py b/test/test_recurring.py
index 7834b7c..65a2b15 100644
--- a/test/test_recurring.py
+++ b/test/test_recurring.py
@@ -260,6 +260,7 @@ def test_rerecur_attributes(self):
             'state': 'running',
             'tags': ['foo'],
             'tracked': False,
+            'throttles': [],
             'worker': 'worker'})
         self.lua('recur', 60, 'queue', 'jid', 'class', {'foo': 'bar'},
             'interval', 10, 0, 'priority', 5, 'tags', ['bar'], 'retries', 5)
@@ -280,6 +281,7 @@ def test_rerecur_attributes(self):
             'state': 'running',
             'tags': ['bar'],
             'tracked': False,
+            'throttles': [],
             'worker': 'worker'})
 
     def test_rerecur_move(self):
diff --git a/test/test_throttle.py b/test/test_throttle.py
index 2db2d80..6d4e5ab 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -24,24 +24,29 @@ def test_delete(self):
     self.assertEqual(self.lua('throttle.get', 0, 'tid'), None)
 
 class TestAcquire(TestQless):
+  '''Test that a job has a default queue throttle'''
+  def test_default_queue_throttle(self):
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0)
+    self.assertEqual(self.lua('get', 0, 'jid')['throttles'], ['ql:q:queue'])
+
   '''Test that job can specify a throttle'''
   def test_specify_throttle(self):
-    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
-    self.assertEqual(self.lua('get', 0, 'jid')['throttle'], 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
+    self.assertEqual(self.lua('get', 0, 'jid')['throttles'], ['tid', 'ql:q:queue'])
 
   '''Test that a job can acquire a throttle'''
   def test_acquire_throttle(self):
-    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
 
   '''Test that acquiring of a throttle lock properly limits the number of jobs'''
   def test_limit_number_of_locks(self):
     self.lua('throttle.set', 0, 'tid', 1)
-    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 2, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 3, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 2, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 3, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 0, 'queue', 'worker', 4)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid2', 'jid3', 'jid4'])
@@ -49,7 +54,7 @@ def test_limit_number_of_locks(self):
 class TestRelease(TestQless):
   '''Test that when there are no pending jobs lock is properly released'''
   def test_no_pending_jobs(self):
-    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
@@ -60,8 +65,8 @@ def test_no_pending_jobs(self):
   '''Test that releasing a lock properly another job in the work queue'''
   def test_next_job_is_moved_into_work_qeueue(self):
     self.lua('throttle.set', 0, 'tid', 1)
-    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 2, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid2'])
@@ -76,7 +81,7 @@ def test_next_job_is_moved_into_work_qeueue(self):
   '''Test that when a job completes it properly releases the lock'''
   def test_on_complete_lock_is_released(self):
     self.lua('throttle.set', 0, 'tid', 1)
-    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.lua('complete', 0, 'jid', 'worker', 'queue', {})
@@ -86,7 +91,7 @@ def test_on_complete_lock_is_released(self):
   '''Test that when a job fails it properly releases the lock'''
   def test_on_failure_lock_is_released(self):
     self.lua('throttle.set', 0, 'tid', 1)
-    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.lua('fail', 0, 'jid', 'worker', 'failed', 'i failed', {})
@@ -97,7 +102,7 @@ def test_on_failure_lock_is_released(self):
      and goes back into pending'''
   def test_on_retry_lock_is_released(self):
     self.lua('throttle.set', 0, 'tid', 1)
-    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
     self.lua('retry', 0, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
@@ -107,7 +112,7 @@ def test_on_retry_lock_is_released(self):
   '''Test that when a job retries it is able to reacquire the lock when next popped'''
   def test_on_retry_lock_is_reacquired(self):
     self.lua('throttle.set', 0, 'tid', 1)
-    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
     self.lua('retry', 0, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
@@ -117,7 +122,7 @@ def test_on_retry_lock_is_reacquired(self):
   '''Test that when a job retries and no pending jobs it acquires the lock again on next pop'''
   def test_on_retry_no_pending_lock_is_reacquired(self):
     self.lua('throttle.set', 0, 'tid', 1)
-    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.lua('retry', 0, 'jid', 'queue', 'worker', 0, 'retry', 'retrying')
@@ -128,23 +133,23 @@ def test_on_retry_no_pending_lock_is_reacquired(self):
   '''Test that when a job retries and another job is pending, the pending job acquires the lock'''
   def test_on_retry_no_pending_lock_is_reacquired(self):
     self.lua('throttle.set', 0, 'tid', 1)
-    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 2, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('throttle.pending', 7, 'tid'), ['jid2'])
-    self.lua('retry', 4, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
-    self.assertEqual(self.lua('throttle.pending', 7, 'tid'), [])
-    self.lua('pop', 5, 'queue', 'worker', 2)
-    self.assertEqual(self.lua('throttle.locks', 6, 'tid'), ['jid2'])
-    self.assertEqual(self.lua('throttle.pending', 7, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 4, 'tid'), ['jid2'])
+    self.lua('retry', 5, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
+    self.assertEqual(self.lua('throttle.pending', 6, 'tid'), [])
+    self.lua('pop', 7, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 8, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('throttle.pending', 9, 'tid'), ['jid1'])
 
 class TestDependents(TestQless):
   def test_dependencies_can_acquire_lock_after_dependent_success(self):
     self.lua('throttle.set', 0, 'tid', 1)
-    self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'depends', ['jid1'], 'throttle', 'tid')
-    self.lua('put', 3, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'depends', ['jid2'], 'throttle', 'tid')
+    self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'depends', ['jid1'], 'throttles', ['tid'])
+    self.lua('put', 3, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'depends', ['jid2'], 'throttles', ['tid'])
 
     self.lua('pop', 4, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 5, 'tid'), ['jid1'])
@@ -166,9 +171,9 @@ def test_dependencies_can_acquire_lock_after_dependent_success(self):
 
   def test_dependencies_can_acquire_lock_after_dependent_failure(self):
     self.lua('throttle.set', 0, 'tid', 1)
-    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 0, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'depends', ['jid1'], 'throttle', 'tid')
-    self.lua('put', 0, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'depends', ['jid2'], 'throttle', 'tid')
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 0, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'depends', ['jid1'], 'throttles', ['tid'])
+    self.lua('put', 0, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'depends', ['jid2'], 'throttles', ['tid'])
 
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
@@ -180,9 +185,9 @@ def test_dependencies_can_acquire_lock_after_dependent_failure(self):
 
   def test_dependencies_do_not_acquire_lock_on_dependent_retry(self):
     self.lua('throttle.set', 0, 'tid', 1)
-    self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttle', 'tid')
-    self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'depends', ['jid1'], 'throttle', 'tid')
-    self.lua('put', 3, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'depends', ['jid2'], 'throttle', 'tid')
+    self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'depends', ['jid1'], 'throttles', ['tid'])
+    self.lua('put', 3, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'depends', ['jid2'], 'throttles', ['tid'])
 
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
diff --git a/test/test_track.py b/test/test_track.py
index 7bb057c..066a878 100644
--- a/test/test_track.py
+++ b/test/test_track.py
@@ -33,6 +33,7 @@ def test_track(self):
                 'dependencies': {},
                 'klass': 'klass',
                 'dependents': {},
+                'throttles': ['ql:q:queue'],
                 'data': '{}',
                 'remaining': 5,
                 'history': [{
diff --git a/throttle.lua b/throttle.lua
index feb8545..e6e93ea 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -29,7 +29,7 @@ end
 -- Returns true of the job acquired the resource.
 function QlessThrottle:acquire(jid)
   if self:available() then
-    redis.call('set', 'printline', jid .. ' is acquiring the lock for ' .. self.id)
+    redis.call('set', 'printline', jid .. ' acquired the lock for ' .. self.id)
     self.locks.add(1, jid)
     return true
   else
@@ -39,6 +39,12 @@ function QlessThrottle:acquire(jid)
   end
 end
 
+-- Rolls back an attempted lock acquisition.
+function QlessThrottle:rollback_acquire(jid)
+  self.locks.remove(jid)
+  self.pending.add(1, jid)
+end
+
 -- Release a throttled resource.
 -- This will take a currently pending job
 -- and attempt to acquire a lock.
@@ -48,6 +54,7 @@ end
 function QlessThrottle:release(now, jid)
   --redis.call('set', 'printline', jid .. ' is releasing lock on ' .. self.id)
   self.locks.remove(jid)
+
   --redis.call('set', 'printline', 'retrieving next job from pending on ' .. self.id)
   local next_jid = unpack(self:pending_pop(0, 0))
   if next_jid then

From 308a77ed2edcc1f5896d897c9d89924ca31621ae Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 11 Mar 2014 19:54:11 -0400
Subject: [PATCH 25/72] implemented multiple throttles per job

---
 job.lua                | 5 ++---
 queue.lua              | 7 +++++--
 test/test_recurring.py | 6 +++---
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/job.lua b/job.lua
index 0016083..f9a4f4c 100644
--- a/job.lua
+++ b/job.lua
@@ -800,7 +800,7 @@ end
 
 function QlessJob:release_throttles(now)
   local throttles = redis.call('hget', QlessJob.ns .. self.jid, 'throttles')
-  throttles = cjson.decode(throttles or '[]')
+  throttles = cjson.decode(throttles or '{}')
 
   for _, tid in ipairs(throttles) do
     redis.call('set', 'printline', 'releasing throttle : ' .. tid)
@@ -809,8 +809,7 @@ function QlessJob:release_throttles(now)
 end
 
 function QlessJob:acquire_throttles(now)
-  local throttles = redis.call('hget', QlessJob.ns .. self.jid, 'throttles')
-  throttles = cjson.decode(throttles or '[]')
+  local throttles = cjson.decode(redis.call('hget', QlessJob.ns .. self.jid, 'throttles'))
 
   local acquired_all = true
   local acquired_throttles = {}
diff --git a/queue.lua b/queue.lua
index b23d1ea..d48c6d5 100644
--- a/queue.lua
+++ b/queue.lua
@@ -699,7 +699,7 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
     options.backlog = assert(tonumber(options.backlog  or 0),
       'Recur(): Arg "backlog" not a number: ' .. tostring(
         options.backlog))
-    options.throttles = assert(cjson.decode(options['throttles'] or '[]'),
+    options.throttles = assert(cjson.decode(options['throttles'] or '{}'),
       'Recur(): Arg "throttles" not JSON array: ' .. tostring(options['throttles']))
 
     local count, old_queue = unpack(redis.call('hmget', 'ql:r:' .. jid, 'count', 'queue'))
@@ -726,7 +726,7 @@ function QlessQueue:recur(now, jid, klass, raw_data, spec, ...)
       'interval' , interval,
       'retries'  , options.retries,
       'backlog'  , options.backlog,
-      'throttles', options.throttles)
+      'throttles', cjson.encode(options.throttles or {}))
     -- Now, we should schedule the next run of the job
     self.recurring.add(now + offset, jid)
 
@@ -762,6 +762,9 @@ function QlessQueue:check_recurring(now, count)
     -- get the last time each of them was run, and then increment
     -- it by its interval. While this time is less than now,
     -- we need to keep putting jobs on the queue
+    local r = redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority',
+        'tags', 'retries', 'interval', 'backlog', 'throttles')
+    redis.call('set', 'printline', cjson.encode(r))
     local klass, data, priority, tags, retries, interval, backlog, throttles = unpack(
       redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority',
         'tags', 'retries', 'interval', 'backlog', 'throttles'))
diff --git a/test/test_recurring.py b/test/test_recurring.py
index 65a2b15..e25476d 100644
--- a/test/test_recurring.py
+++ b/test/test_recurring.py
@@ -260,10 +260,10 @@ def test_rerecur_attributes(self):
             'state': 'running',
             'tags': ['foo'],
             'tracked': False,
-            'throttles': [],
+            'throttles': {},
             'worker': 'worker'})
         self.lua('recur', 60, 'queue', 'jid', 'class', {'foo': 'bar'},
-            'interval', 10, 0, 'priority', 5, 'tags', ['bar'], 'retries', 5)
+            'interval', 10, 0, 'priority', 5, 'tags', ['bar'], 'retries', 5, 'throttles', ['lala'])
         self.assertEqual(self.lua('pop', 60, 'queue', 'worker', 10)[0], {
             'data': '{"foo": "bar"}',
             'dependencies': {},
@@ -281,7 +281,7 @@ def test_rerecur_attributes(self):
             'state': 'running',
             'tags': ['bar'],
             'tracked': False,
-            'throttles': [],
+            'throttles': ['lala'],
             'worker': 'worker'})
 
     def test_rerecur_move(self):

From 5d86598c8bf169b3e0ae6acec4214b845e2604d7 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 11 Mar 2014 20:00:26 -0400
Subject: [PATCH 26/72] documentation

---
 throttle.lua | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/throttle.lua b/throttle.lua
index e6e93ea..d4ec081 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -40,6 +40,11 @@ function QlessThrottle:acquire(jid)
 end
 
 -- Rolls back an attempted lock acquisition.
+-- Since jobs can acquire multiple locks and the acquire
+-- behavior is to either add them to the lock or pend them
+-- this method handles the rolling back an acquired lock
+-- on a job that failed to acquire all of its locks.
+-- without placing another pending job into the queue.
 function QlessThrottle:rollback_acquire(jid)
   self.locks.remove(jid)
   self.pending.add(1, jid)
@@ -52,10 +57,10 @@ end
 -- the job will be moved from the throttled
 -- queue into the work queue
 function QlessThrottle:release(now, jid)
-  --redis.call('set', 'printline', jid .. ' is releasing lock on ' .. self.id)
+  redis.call('set', 'printline', jid .. ' is releasing lock on ' .. self.id)
   self.locks.remove(jid)
 
-  --redis.call('set', 'printline', 'retrieving next job from pending on ' .. self.id)
+  redis.call('set', 'printline', 'retrieving next job from pending on ' .. self.id)
   local next_jid = unpack(self:pending_pop(0, 0))
   if next_jid then
     local job = Qless.job(next_jid):data()

From 5141cbfbcd4f1caca5036c2262d2fdfcf49b1ef8 Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Wed, 12 Mar 2014 07:57:32 -0400
Subject: [PATCH 27/72] Add job tests for multiple throttles

---
 test/test_job.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/test/test_job.py b/test/test_job.py
index 1ef1703..74ec485 100644
--- a/test/test_job.py
+++ b/test/test_job.py
@@ -253,3 +253,66 @@ def test_cancel_pop_retries(self):
         self.lua('pop', 2, 'queue', 'worker', 10)
         self.lua('cancel', 3, 'jid')
         self.assertEqual(self.lua('get', 4, 'jid'), None)
+
+
+class TestThrottles(TestQless):
+  '''Acquiring and releasing throttles'''
+  def test_acquire_throttles_acquires_all_throttles(self):
+    '''Can acquire locks for all throttles'''
+    # Should have throttles for queue and named throttles
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid', 'wid'])
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+    self.assertEqual(self.lua('throttle.locks', 0, 'wid'), ['jid'])
+    self.assertEqual(self.lua('throttle.locks', 0, 'ql:q:queue'), ['jid'])
+
+  def test_release_throttles_on_acquisition_failure(self):
+    '''Cancels locked throttles if locks can not be obtained for all locks'''
+    # Should have throttles for queue and named throttles
+    self.lua('throttle.set', 0, 'wid', 1)
+    self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['wid'])
+    self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid', 'wid'])
+    self.lua('pop', 3, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 4, 'wid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.locks', 5, 'tid'), [])
+    self.assertEqual(self.lua('throttle.locks', 6, 'ql:q:queue'), ['jid1'])
+    self.assertEqual(self.lua('get', 7, 'jid2')['state'], 'waiting')
+
+  def test_release_throttles_after_acquisition_on_completion(self):
+    '''Can acquire locks for all throttles and then release them when complete'''
+    # Should have throttles for queue and named throttles
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid', 'wid'])
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+    self.assertEqual(self.lua('throttle.locks', 0, 'wid'), ['jid'])
+    self.assertEqual(self.lua('throttle.locks', 0, 'ql:q:queue'), ['jid'])
+    self.lua('complete', 0, 'jid', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.locks', 0, 'wid'), [])
+    self.assertEqual(self.lua('throttle.locks', 0, 'ql:q:queue'), [])
+
+  def test_release_throttles_after_acquisition_on_retry(self):
+    '''Can acquire locks for all throttles and then release them on retry'''
+    # Should have throttles for queue and named throttles
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid', 'wid'])
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+    self.assertEqual(self.lua('throttle.locks', 0, 'wid'), ['jid'])
+    self.assertEqual(self.lua('throttle.locks', 0, 'ql:q:queue'), ['jid'])
+    self.lua('retry', 0, 'jid', 'queue', 'worker', 0, 'retry', 'retrying')
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.locks', 0, 'wid'), [])
+    self.assertEqual(self.lua('throttle.locks', 0, 'ql:q:queue'), [])
+
+  def test_release_throttles_after_acquisition_on_fail(self):
+    '''Can acquire locks for all throttles and then release them on failure'''
+    # Should have throttles for queue and named throttles
+    self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid', 'wid'])
+    self.lua('pop', 0, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
+    self.assertEqual(self.lua('throttle.locks', 0, 'wid'), ['jid'])
+    self.assertEqual(self.lua('throttle.locks', 0, 'ql:q:queue'), ['jid'])
+    self.lua('fail', 0, 'jid', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.locks', 0, 'wid'), [])
+    self.assertEqual(self.lua('throttle.locks', 0, 'ql:q:queue'), [])

From a27ac2e3c3a06bb234c81db7781a43a5a36d95ca Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Wed, 12 Mar 2014 09:10:19 -0400
Subject: [PATCH 28/72] implemeted queue throttled

---
 base.lua     |  4 ++--
 queue.lua    | 28 +++++++++++++++++++++++-----
 throttle.lua |  4 ++--
 3 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/base.lua b/base.lua
index befacc3..ae373ce 100644
--- a/base.lua
+++ b/base.lua
@@ -82,7 +82,7 @@ function Qless.throttle(tid)
 
   -- set of jids which have acquired a lock on this throttle.
   throttle.locks = {
-    count = function()
+    length = function()
       return (redis.call('zcard', QlessThrottle.ns .. tid .. '-locks') or 0)
     end, members = function()
       return redis.call('zrange', QlessThrottle.ns .. tid .. '-locks', 0, -1)
@@ -101,7 +101,7 @@ function Qless.throttle(tid)
 
   -- set of jids waiting on this throttle to become available.
   throttle.pending = {
-    count = function()
+    length = function()
       return (redis.call('zcard', QlessThrottle.ns .. tid .. '-pending') or 0)
     end, members = function()
       return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', 0, -1)
diff --git a/queue.lua b/queue.lua
index d48c6d5..729dabf 100644
--- a/queue.lua
+++ b/queue.lua
@@ -77,18 +77,21 @@ function Qless.queue(name)
     end
   }
 
-  -- Access to our throttled jobs
+
+  -- Access to the queue level throttled jobs.
+  -- We delegate down to a throttle here for the general queue methods.
+  local queue_throttle = Qless.throttle(QlessQueue.ns .. name)
   queue.throttled = {
     peek = function(now, offset, count)
-      return redis.call('zrange', queue:prefix('throttled'), offset, offset + count - 1)
+      return queue_throttle.pending.peek(offset, count)
     end, add = function(now, jid)
-      redis.call('zadd', queue:prefix('throttled'), now, jid)
+      return queue_throttle.pending.add(jid)
     end, remove = function(...)
       if #arg > 0 then
-        return redis.call('zrem', queue:prefix('throttled'), unpack(arg))
+        return queue_throttle.pending.remove(unpack(arg))
       end
     end, length = function()
-      return redis.call('zcard', queue:prefix('throttled'))
+      return queue_throttle.pending.length()
     end
   }
 
@@ -314,6 +317,11 @@ function QlessQueue:pop(now, worker, count)
   -- unit of work.
   self:check_scheduled(now, count - #dead_jids)
 
+  -- If we still need values in order to meet the demand, check our throttled
+  -- jobs. This has the side benefit of naturally updating other throttles
+  -- on the jobs checked.
+  self:check_throttled(now, count - #dead_jids)
+
   -- With these in place, we can expand this list of jids based on the work
   -- queue itself and the priorities therein
   local jids = self.work.peek(count - #dead_jids) or {}
@@ -851,6 +859,16 @@ function QlessQueue:check_scheduled(now, count)
   end
 end
 
+function QlessQueue:check_throttled(now, count)
+  local throttled = self.throttled.peek(now, 0, count)
+  for _, jid in ipairs(throttled) do
+    local priority = tonumber(redis.call('hget', QlessJob.ns .. jid, 'priority') or 0)
+    self.work.add(now, priority, jid)
+    self.throttled.remove(jid)
+    redis.call('hset', QlessJob.ns .. jid, 'state', 'waiting')
+  end
+end
+
 -- Check for and invalidate any locks that have been lost. Returns the
 -- list of jids that have been invalidated
 function QlessQueue:invalidate_locks(now, count)
diff --git a/throttle.lua b/throttle.lua
index d4ec081..04415d0 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -86,6 +86,6 @@ end
 
 -- Returns true if the throttle has locks available, false otherwise.
 function QlessThrottle:available()
-  redis.call('set', 'printline', 'available ' .. self.maximum .. ' == 0 or ' .. self.locks.count() .. ' < ' .. self.maximum)
-  return self.maximum == 0 or self.locks.count() < self.maximum
+  redis.call('set', 'printline', 'available ' .. self.maximum .. ' == 0 or ' .. self.locks.length() .. ' < ' .. self.maximum)
+  return self.maximum == 0 or self.locks.length() < self.maximum
 end

From b1599336a2a28922309e740c6b5cfe612208c909 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Wed, 12 Mar 2014 09:16:21 -0400
Subject: [PATCH 29/72] small changes

---
 queue.lua | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/queue.lua b/queue.lua
index 729dabf..7e7b268 100644
--- a/queue.lua
+++ b/queue.lua
@@ -860,6 +860,10 @@ function QlessQueue:check_scheduled(now, count)
 end
 
 function QlessQueue:check_throttled(now, count)
+  if not Qless.throttle(QlessQueue.ns .. self.name):available() then
+    return
+  end
+
   local throttled = self.throttled.peek(now, 0, count)
   for _, jid in ipairs(throttled) do
     local priority = tonumber(redis.call('hget', QlessJob.ns .. jid, 'priority') or 0)
@@ -1031,6 +1035,7 @@ function QlessQueue.counts(now, name)
       waiting   = queue.work.length(),
       stalled   = stalled,
       running   = queue.locks.length() - stalled,
+      throttled = queue.throttled.length(),
       scheduled = queue.scheduled.length(),
       depends   = queue.depends.length(),
       recurring = queue.recurring.length(),

From 6522fe702de4381261ee8e2d7b00483173b90744 Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Wed, 12 Mar 2014 09:06:51 -0400
Subject: [PATCH 30/72] Add tests for dynamically changing throttle concurrency
 level

---
 test/test_throttle.py | 79 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 79 insertions(+)

diff --git a/test/test_throttle.py b/test/test_throttle.py
index 6d4e5ab..a2b4914 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -210,3 +210,82 @@ def test_dependencies_do_not_acquire_lock_on_dependent_retry(self):
     self.lua('complete', 0, 'jid3', 'worker', 'queue', {})
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+
+
+class TestConcurrencyLevelChange(TestQless):
+  '''Test that changes to concurrency level are handled dynamically'''
+  def test_increasing_concurrency_level_activates_pending_jobs(self):
+    '''Activates pending jobs when concurrency level of throttle is increased'''
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 3, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttles', ['tid'])
+
+    self.lua('pop', 4, 'queue', 'worker', 3)
+    self.assertEqual(self.lua('throttle.locks', 5, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 6, 'tid'), ['jid2', 'jid3'])
+    self.lua('throttle.set', 7, 'tid', 3)
+    self.lua('pop', 8, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 9, 'tid'), ['jid1', 'jid2', 'jid3'])
+    self.assertEqual(self.lua('throttle.pending', 10, 'tid'), [])
+
+  def test_reducing_concurrency_level_without_pending(self):
+    '''Operates at reduced concurrency level after current jobs finish'''
+    self.lua('throttle.set', 0, 'tid', 3)
+    self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 3, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 4, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 5, 'worker', 'queue', 'jid5', 'klass', {}, 0, 'throttles', ['tid'])
+
+    self.lua('pop', 6, 'queue', 'worker', 3)
+    self.assertEqual(self.lua('throttle.locks', 7, 'tid'), ['jid1', 'jid2', 'jid3'])
+    self.assertEqual(self.lua('throttle.pending', 8, 'tid'), [])
+    self.lua('throttle.set', 9, 'tid', 1)
+    self.lua('pop', 10, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 11, 'tid'), ['jid1', 'jid2', 'jid3'])
+    self.assertEqual(self.lua('throttle.pending', 12, 'tid'), ['jid4', 'jid5'])
+    self.lua('complete', 13, 'jid1', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 14, 'tid'), ['jid2', 'jid3'])
+    self.assertEqual(self.lua('throttle.pending', 15, 'tid'), ['jid4', 'jid5'])
+    self.lua('complete', 16, 'jid2', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 17, 'tid'), ['jid3'])
+    self.assertEqual(self.lua('throttle.pending', 18, 'tid'), ['jid4', 'jid5'])
+    self.lua('complete', 19, 'jid3', 'worker', 'queue', {})
+    self.lua('pop', 20, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 21, 'tid'), ['jid4'])
+    self.assertEqual(self.lua('throttle.pending', 22, 'tid'), ['jid5'])
+    self.lua('pop', 23, 'queue', 'worker', 2)
+    self.lua('complete', 24, 'jid4', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 25, 'tid'), ['jid5'])
+    self.assertEqual(self.lua('throttle.pending', 26, 'tid'), [])
+
+  def test_reducing_concurrency_level_with_pending(self):
+    '''Operates at reduced concurrency level after current jobs finish'''
+    self.lua('throttle.set', 0, 'tid', 3)
+    self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 3, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 4, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 5, 'worker', 'queue', 'jid5', 'klass', {}, 0, 'throttles', ['tid'])
+
+    self.lua('pop', 6, 'queue', 'worker', 5)
+    self.assertEqual(self.lua('throttle.locks', 7, 'tid'), ['jid1', 'jid2', 'jid3'])
+    self.assertEqual(self.lua('throttle.pending', 8, 'tid'), ['jid4', 'jid5'])
+    self.lua('throttle.set', 9, 'tid', 1)
+    self.assertEqual(self.lua('throttle.locks', 10, 'tid'), ['jid1', 'jid2', 'jid3'])
+    self.assertEqual(self.lua('throttle.pending', 11, 'tid'), ['jid4', 'jid5'])
+    self.lua('complete', 12, 'jid1', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 13, 'tid'), ['jid2', 'jid3'])
+    self.assertEqual(self.lua('throttle.pending', 14, 'tid'), ['jid4', 'jid5'])
+    self.lua('complete', 15, 'jid2', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 16, 'tid'), ['jid3'])
+    self.assertEqual(self.lua('throttle.pending', 17, 'tid'), ['jid4', 'jid5'])
+    self.lua('complete', 18, 'jid3', 'worker', 'queue', {})
+    self.lua('pop', 19, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 20, 'tid'), ['jid4'])
+    self.assertEqual(self.lua('throttle.pending', 21, 'tid'), ['jid5'])
+    self.lua('pop', 22, 'queue', 'worker', 2)
+    self.lua('complete', 23, 'jid4', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 24, 'tid'), ['jid5'])
+    self.assertEqual(self.lua('throttle.pending', 25, 'tid'), [])

From eaf6be1f1955b31b89563e62a0ce84376fb6e685 Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Wed, 12 Mar 2014 09:07:26 -0400
Subject: [PATCH 31/72] Add tests to verify queue throttled

---
 test/test_queue.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/test/test_queue.py b/test/test_queue.py
index 04a374f..446a257 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -641,3 +641,26 @@ def test_fail_max_concurrency(self):
         self.lua('fail', 3, 'a', 'worker', 'group', 'message', {})
         job = self.lua('pop', 4, 'queue', 'worker', 10)[0]
         self.assertEqual(job['jid'], 'b')
+
+    def test_throttled_added(self):
+        '''New jobs are added to throttled when at concurrency limit'''
+        self.lua('throttle.set', 0, 'ql:q:queue', 1)
+        self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0)
+        self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0)
+        self.lua('pop', 2, 'queue', 'worker', 2)
+        self.assertEqual(self.lua('throttle.locks', 3, 'ql:q:queue'), ['jid1'])
+        self.assertEqual(self.redis.zrange('ql:q:queue-throttled', 0, -1), ['jid2'])
+
+    def test_throttled_removed(self):
+        '''Throttled jobs are removed from throttled when concurrency available'''
+        self.lua('throttle.set', 0, 'ql:q:queue', 1)
+        self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0)
+        self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0)
+        self.lua('pop', 2, 'queue', 'worker', 2)
+        self.assertEqual(self.lua('throttle.locks', 3, 'ql:q:queue'), ['jid1'])
+        self.assertEqual(self.redis.zrange('ql:q:queue-throttled', 0, -1), ['jid2'])
+        self.lua('complete', 4, 'jid1', 'worker', 'queue', {})
+        self.assertEqual(self.redis.zrange('ql:q:queue-throttled', 0, -1), ['jid2'])
+        self.lua('pop', 5, 'queue', 'worker', 1)
+        self.assertEqual(self.lua('throttle.locks', 6, 'ql:q:queue'), ['jid2'])
+        self.assertEqual(self.redis.zrange('ql:q:queue-throttled', 0, -1), [])

From 32a1408dd3ead382d492471e3f987a8d8d54fab6 Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Wed, 12 Mar 2014 10:52:16 -0400
Subject: [PATCH 32/72] Minor optimizations and test fixes

---
 queue.lua             | 34 +++++++++++++++-------------------
 test/test_queue.py    |  8 ++++----
 test/test_throttle.py | 20 +++++++++++---------
 throttle.lua          | 10 +---------
 4 files changed, 31 insertions(+), 41 deletions(-)

diff --git a/queue.lua b/queue.lua
index 7e7b268..6ce705c 100644
--- a/queue.lua
+++ b/queue.lua
@@ -304,6 +304,17 @@ function QlessQueue:pop(now, worker, count)
   redis.call('zadd', 'ql:workers', now, worker)
 
   local dead_jids = self:invalidate_locks(now, count) or {}
+  local popped = {}
+
+  for index, jid in ipairs(dead_jids) do
+    self:pop_job(now, worker, Qless.job(jid))
+    table.insert(popped, jid)
+  end
+
+  if not Qless.throttle(QlessQueue.ns .. self.name):available() then
+    return popped
+  end
+
   -- Now we've checked __all__ the locks for this queue the could
   -- have expired, and are no more than the number requested.
 
@@ -326,9 +337,6 @@ function QlessQueue:pop(now, worker, count)
   -- queue itself and the priorities therein
   local jids = self.work.peek(count - #dead_jids) or {}
 
-  local queue_throttle = Qless.throttle(QlessQueue.ns .. self.name)
-
-  local popped = {}
   for index, jid in ipairs(jids) do
     local job = Qless.job(jid)
     if job:acquire_throttles(now) then
@@ -339,16 +347,9 @@ function QlessQueue:pop(now, worker, count)
     end
   end
 
-  -- If we are returning any jobs, then remove popped jobs from
-  -- work queue
-  self.work.remove(unpack(popped))
-
-  -- Process dead jids after removing newly popped jids from work queue
-  -- This changes the order of returned jids
-  for index, jid in ipairs(dead_jids) do
-    self:pop_job(now, worker, Qless.job(jid))
-    table.insert(popped, jid)
-  end
+  -- All jobs should have acquired locks or be throttled,
+  -- ergo, remove all jids from work queue
+  self.work.remove(unpack(jids))
 
   return popped
 end
@@ -860,15 +861,10 @@ function QlessQueue:check_scheduled(now, count)
 end
 
 function QlessQueue:check_throttled(now, count)
-  if not Qless.throttle(QlessQueue.ns .. self.name):available() then
-    return
-  end
-
-  local throttled = self.throttled.peek(now, 0, count)
+  local throttled = self.throttled.peek(now, 0, count - 1)
   for _, jid in ipairs(throttled) do
     local priority = tonumber(redis.call('hget', QlessJob.ns .. jid, 'priority') or 0)
     self.work.add(now, priority, jid)
-    self.throttled.remove(jid)
     redis.call('hset', QlessJob.ns .. jid, 'state', 'waiting')
   end
 end
diff --git a/test/test_queue.py b/test/test_queue.py
index 446a257..67859c7 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -649,7 +649,7 @@ def test_throttled_added(self):
         self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0)
         self.lua('pop', 2, 'queue', 'worker', 2)
         self.assertEqual(self.lua('throttle.locks', 3, 'ql:q:queue'), ['jid1'])
-        self.assertEqual(self.redis.zrange('ql:q:queue-throttled', 0, -1), ['jid2'])
+        self.assertEqual(self.lua('throttle.pending', 3, 'ql:q:queue'), ['jid2'])
 
     def test_throttled_removed(self):
         '''Throttled jobs are removed from throttled when concurrency available'''
@@ -658,9 +658,9 @@ def test_throttled_removed(self):
         self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0)
         self.lua('pop', 2, 'queue', 'worker', 2)
         self.assertEqual(self.lua('throttle.locks', 3, 'ql:q:queue'), ['jid1'])
-        self.assertEqual(self.redis.zrange('ql:q:queue-throttled', 0, -1), ['jid2'])
+        self.assertEqual(self.lua('throttle.pending', 3, 'ql:q:queue'), ['jid2'])
         self.lua('complete', 4, 'jid1', 'worker', 'queue', {})
-        self.assertEqual(self.redis.zrange('ql:q:queue-throttled', 0, -1), ['jid2'])
+        self.assertEqual(self.lua('throttle.pending', 3, 'ql:q:queue'), ['jid2'])
         self.lua('pop', 5, 'queue', 'worker', 1)
         self.assertEqual(self.lua('throttle.locks', 6, 'ql:q:queue'), ['jid2'])
-        self.assertEqual(self.redis.zrange('ql:q:queue-throttled', 0, -1), [])
+        self.assertEqual(self.lua('throttle.pending', 3, 'ql:q:queue'), [])
diff --git a/test/test_throttle.py b/test/test_throttle.py
index a2b4914..cbc9cee 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -130,8 +130,10 @@ def test_on_retry_no_pending_lock_is_reacquired(self):
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
 
-  '''Test that when a job retries and another job is pending, the pending job acquires the lock'''
-  def test_on_retry_no_pending_lock_is_reacquired(self):
+  '''Test that when a job retries and another job is pending, the retrying job acquires the lock'''
+  def test_on_retry_with_pending_lock_is_reacquired(self):
+    # The retrying job will only re-acquire the lock if nothing is ahead of it in
+    # the work queue that requires that lock
     self.lua('throttle.set', 0, 'tid', 1)
     self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
@@ -139,10 +141,10 @@ def test_on_retry_no_pending_lock_is_reacquired(self):
     self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
     self.assertEqual(self.lua('throttle.pending', 4, 'tid'), ['jid2'])
     self.lua('retry', 5, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
-    self.assertEqual(self.lua('throttle.pending', 6, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 6, 'tid'), ['jid2'])
     self.lua('pop', 7, 'queue', 'worker', 2)
-    self.assertEqual(self.lua('throttle.locks', 8, 'tid'), ['jid2'])
-    self.assertEqual(self.lua('throttle.pending', 9, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.locks', 8, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 9, 'tid'), ['jid2'])
 
 class TestDependents(TestQless):
   def test_dependencies_can_acquire_lock_after_dependent_success(self):
@@ -252,11 +254,11 @@ def test_reducing_concurrency_level_without_pending(self):
     self.assertEqual(self.lua('throttle.locks', 17, 'tid'), ['jid3'])
     self.assertEqual(self.lua('throttle.pending', 18, 'tid'), ['jid4', 'jid5'])
     self.lua('complete', 19, 'jid3', 'worker', 'queue', {})
-    self.lua('pop', 20, 'queue', 'worker', 2)
+    self.lua('pop', 20, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 21, 'tid'), ['jid4'])
     self.assertEqual(self.lua('throttle.pending', 22, 'tid'), ['jid5'])
-    self.lua('pop', 23, 'queue', 'worker', 2)
     self.lua('complete', 24, 'jid4', 'worker', 'queue', {})
+    self.lua('pop', 23, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 25, 'tid'), ['jid5'])
     self.assertEqual(self.lua('throttle.pending', 26, 'tid'), [])
 
@@ -282,10 +284,10 @@ def test_reducing_concurrency_level_with_pending(self):
     self.assertEqual(self.lua('throttle.locks', 16, 'tid'), ['jid3'])
     self.assertEqual(self.lua('throttle.pending', 17, 'tid'), ['jid4', 'jid5'])
     self.lua('complete', 18, 'jid3', 'worker', 'queue', {})
-    self.lua('pop', 19, 'queue', 'worker', 2)
+    self.lua('pop', 19, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 20, 'tid'), ['jid4'])
     self.assertEqual(self.lua('throttle.pending', 21, 'tid'), ['jid5'])
-    self.lua('pop', 22, 'queue', 'worker', 2)
     self.lua('complete', 23, 'jid4', 'worker', 'queue', {})
+    self.lua('pop', 22, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 24, 'tid'), ['jid5'])
     self.assertEqual(self.lua('throttle.pending', 25, 'tid'), [])
diff --git a/throttle.lua b/throttle.lua
index 04415d0..c2b02cc 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -30,6 +30,7 @@ end
 function QlessThrottle:acquire(jid)
   if self:available() then
     redis.call('set', 'printline', jid .. ' acquired the lock for ' .. self.id)
+    self.pending.remove(jid)
     self.locks.add(1, jid)
     return true
   else
@@ -59,15 +60,6 @@ end
 function QlessThrottle:release(now, jid)
   redis.call('set', 'printline', jid .. ' is releasing lock on ' .. self.id)
   self.locks.remove(jid)
-
-  redis.call('set', 'printline', 'retrieving next job from pending on ' .. self.id)
-  local next_jid = unpack(self:pending_pop(0, 0))
-  if next_jid then
-    local job = Qless.job(next_jid):data()
-    local queue_obj = Qless.queue(job.queue)
-    queue_obj.throttled.remove(job.jid)
-    queue_obj.work.add(now, job.priority, job.jid)
-  end
 end
 
 function QlessThrottle:lock_pop(min, max)

From a70362b19f849af843d8685d6519db3595f553f2 Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Wed, 12 Mar 2014 11:26:03 -0400
Subject: [PATCH 33/72] Add test for queue throttled count

---
 test/test_queue.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/test/test_queue.py b/test/test_queue.py
index 67859c7..2e244ed 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -143,7 +143,8 @@ class TestQueue(TestQless):
         'running': 0,
         'depends': 0,
         'scheduled': 0,
-        'recurring': 0
+        'recurring': 0,
+        'throttled': 0
     }
 
     def setUp(self):
@@ -161,6 +162,18 @@ def test_stalled(self):
         self.assertEqual(self.lua('queues', expires, 'queue'), expected)
         self.assertEqual(self.lua('queues', expires), [expected])
 
+    def test_throttled(self):
+        '''Discern throttled job counts correctly'''
+        expected = dict(self.expected)
+        expected['throttled'] = 1
+        expected['running'] = 1
+        self.lua('throttle.set', 0, 'ql:q:queue', 1)
+        self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0)
+        self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0)
+        self.lua('pop', 3, 'queue', 'worker', 10)
+        self.assertEqual(self.lua('queues', 4, 'queue'), expected)
+        self.assertEqual(self.lua('queues', 5), [expected])
+
     def test_waiting(self):
         '''Discern waiting job counts correctly'''
         expected = dict(self.expected)

From fc332c90c61b3cb497d5afca2b745b8d243921fc Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Wed, 12 Mar 2014 14:51:17 -0400
Subject: [PATCH 34/72] Add api methods for setting queue throttle max

---
 api.lua               | 18 +++++++++++++-----
 base.lua              |  2 +-
 test/test_throttle.py | 12 ++++++++----
 throttle.lua          |  4 ++--
 4 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/api.lua b/api.lua
index 4077862..7d79f41 100644
--- a/api.lua
+++ b/api.lua
@@ -193,6 +193,18 @@ QlessAPI['queue.forget'] = function(now, ...)
   QlessQueue.deregister(unpack(arg))
 end
 
+QlessAPI['queue.throttle.get'] = function(now, queue)
+  local data = Qless.throttle(QlessQueue.ns .. queue):data()
+  if not data then
+    return nil
+  end
+  return cjson.encode(data)
+end
+
+QlessAPI['queue.throttle.set'] = function(now, queue, max)
+  Qless.throttle(QlessQueue.ns .. queue):set({maximum = max})
+end
+
 -- Throttle apis
 QlessAPI['throttle.set'] = function(now, tid, max)
   local data = {
@@ -202,11 +214,7 @@ QlessAPI['throttle.set'] = function(now, tid, max)
 end
 
 QlessAPI['throttle.get'] = function(now, tid)
-  local data = Qless.throttle(tid):data()
-  if not data then
-    return nil
-  end
-  return cjson.encode(data)
+  return cjson.encode(Qless.throttle(tid):data())
 end
 
 QlessAPI['throttle.delete'] = function(now, tid)
diff --git a/base.lua b/base.lua
index ae373ce..3abded0 100644
--- a/base.lua
+++ b/base.lua
@@ -25,7 +25,7 @@ QlessJob.__index = QlessJob
 
 -- throttle forward declaration
 local QlessThrottle = {
-  ns = Qless.ns .. 't:'
+  ns = Qless.ns .. 'th:'
 }
 QlessThrottle.__index = QlessThrottle
 
diff --git a/test/test_throttle.py b/test/test_throttle.py
index cbc9cee..2067302 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -8,20 +8,24 @@ class TestThrottle(TestQless):
   '''Test setting throttle data'''
   def test_set(self):
     self.lua('throttle.set', 0, 'tid', 5)
-    self.assertEqual(self.redis.hmget('ql:t:tid', 'id')[0], 'tid')
-    self.assertEqual(self.redis.hmget('ql:t:tid', 'maximum')[0], '5')
+    self.assertEqual(self.redis.hmget('ql:th:tid', 'id')[0], 'tid')
+    self.assertEqual(self.redis.hmget('ql:th:tid', 'maximum')[0], '5')
 
   '''Test retrieving throttle data'''
   def test_get(self):
-    self.redis.hmset('ql:t:tid', {'id': 'tid', 'maximum' : 5})
+    self.redis.hmset('ql:th:tid', {'id': 'tid', 'maximum' : 5})
     self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 5})
 
+  '''Test retrieving uninitiailized throttle data'''
+  def test_get(self):
+    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 0})
+
   '''Test deleting the throttle data'''
   def test_delete(self):
     self.lua('throttle.set', 0, 'tid', 5)
     self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 5})
     self.lua('throttle.delete', 0, 'tid')
-    self.assertEqual(self.lua('throttle.get', 0, 'tid'), None)
+    self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 0})
 
 class TestAcquire(TestQless):
   '''Test that a job has a default queue throttle'''
diff --git a/throttle.lua b/throttle.lua
index c2b02cc..1c0342d 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -1,9 +1,9 @@
 -- Retrieve the data fro a throttled resource
 function QlessThrottle:data()
   local throttle = redis.call('hmget', QlessThrottle.ns .. self.id, 'id', 'maximum')
-  -- Return nil if we haven't found it
+  -- Return default if it doesn't exist
   if not throttle[1] then
-    return nil
+    return {id = self.id, maximum = 0}
   end
 
   local data = {

From b54298f5ce5eeaa19bd7a9763f5a79cd7d19e69c Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Wed, 12 Mar 2014 16:29:14 -0400
Subject: [PATCH 35/72] throttles refactor/cleanup

---
 api.lua               |   4 --
 base.lua              |  40 +++++++++--------
 job.lua               |  27 ++++++-----
 queue.lua             |  40 ++++++++++++-----
 test/test_job.py      |   2 +-
 test/test_queue.py    |  14 +++---
 test/test_throttle.py | 101 +++++++++++++++++++++---------------------
 throttle.lua          |  47 +++-----------------
 8 files changed, 133 insertions(+), 142 deletions(-)

diff --git a/api.lua b/api.lua
index 7d79f41..7209954 100644
--- a/api.lua
+++ b/api.lua
@@ -224,10 +224,6 @@ end
 QlessAPI['throttle.locks'] = function(now, tid)
   return Qless.throttle(tid).locks.members()
 end
-
-QlessAPI['throttle.pending'] = function(now, tid)
-  return Qless.throttle(tid).pending.members()
-end
 -------------------------------------------------------------------------------
 -- Function lookup
 -------------------------------------------------------------------------------
diff --git a/base.lua b/base.lua
index 3abded0..65144c8 100644
--- a/base.lua
+++ b/base.lua
@@ -100,25 +100,25 @@ function Qless.throttle(tid)
   }
 
   -- set of jids waiting on this throttle to become available.
-  throttle.pending = {
-    length = function()
-      return (redis.call('zcard', QlessThrottle.ns .. tid .. '-pending') or 0)
-    end, members = function()
-      return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', 0, -1)
-    end, peek = function(min, max)
-      return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', min, max)
-    end, add = function(...)
-      if #arg > 0 then
-        redis.call('zadd', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
-      end
-    end, remove = function(...)
-      if #arg > 0 then
-        return redis.call('zrem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
-      end
-    end, pop = function(min, max)
-      return redis.call('zremrangebyrank', QlessThrottle.ns .. tid .. '-pending', min, max)
-    end
-  }
+  -- throttle.pending = {
+  --   length = function()
+  --     return (redis.call('zcard', QlessThrottle.ns .. tid .. '-pending') or 0)
+  --   end, members = function()
+  --     return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', 0, -1)
+  --   end, peek = function(min, max)
+  --     return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', min, max)
+  --   end, add = function(...)
+  --     if #arg > 0 then
+  --       redis.call('zadd', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
+  --     end
+  --   end, remove = function(...)
+  --     if #arg > 0 then
+  --       return redis.call('zrem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
+  --     end
+  --   end, pop = function(min, max)
+  --     return redis.call('zremrangebyrank', QlessThrottle.ns .. tid .. '-pending', min, max)
+  --   end
+  -- }
   return throttle
 end
 
@@ -207,6 +207,8 @@ function Qless.jobs(now, state, ...)
       return queue.locks.peek(now, offset, count)
     elseif state == 'stalled' then
       return queue.locks.expired(now, offset, count)
+    elseif state == 'throttled' then
+      return queue.throttled.peek(now, offset, count)
     elseif state == 'scheduled' then
       queue:check_scheduled(now, queue.scheduled.length())
       return queue.scheduled.peek(now, offset, count)
diff --git a/job.lua b/job.lua
index f9a4f4c..2a679e8 100644
--- a/job.lua
+++ b/job.lua
@@ -811,19 +811,26 @@ end
 function QlessJob:acquire_throttles(now)
   local throttles = cjson.decode(redis.call('hget', QlessJob.ns .. self.jid, 'throttles'))
 
-  local acquired_all = true
-  local acquired_throttles = {}
+  local all_locks_available = true
+
+  redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - checking availability')
   for _, tid in ipairs(throttles) do
-    acquired_all = acquired_all and Qless.throttle(tid):acquire(self.jid)
-    table.insert(acquired_throttles, tid)
+    redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - checking availability for ' .. tid)
+    all_locks_available = all_locks_available and Qless.throttle(tid):available()
+    redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - throttle available ' .. tid)
   end
 
-  if not acquired_all then
-    redis.call('set', 'printline', 'rolling back acquired locks')
-    for _, tid in ipairs(acquired_throttles) do
-      Qless.throttle(tid):rollback_acquire(self.jid)
-    end
+  redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - short circuit if we can not acquire locks ' .. tostring(all_locks_available))
+  if not all_locks_available then
+    return false
   end
 
-  return acquired_all
+  redis.call('set', 'printline', 'QlessJob:acquire_throttles - grabbing locks')
+  redis.call('set', 'printline', 'QlessJob:acquire_throttles - inside if')
+  for _, tid in ipairs(throttles) do
+    redis.call('set', 'printline', 'QlessJob:acquire_throttles - invoking QlessThrottle:acquire')
+    Qless.throttle(tid):acquire(self.jid)
+  end
+  redis.call('set', 'printline', 'QlessJob:acquire_throttles - successfully completed')
+  return true
 end
diff --git a/queue.lua b/queue.lua
index 6ce705c..1654f5d 100644
--- a/queue.lua
+++ b/queue.lua
@@ -79,19 +79,21 @@ function Qless.queue(name)
 
 
   -- Access to the queue level throttled jobs.
-  -- We delegate down to a throttle here for the general queue methods.
-  local queue_throttle = Qless.throttle(QlessQueue.ns .. name)
   queue.throttled = {
-    peek = function(now, offset, count)
-      return queue_throttle.pending.peek(offset, count)
-    end, add = function(now, jid)
-      return queue_throttle.pending.add(jid)
+    length = function()
+      return (redis.call('zcard', queue:prefix('throttled')) or 0)
+    end, peek = function(now, min, max)
+      return redis.call('zrange', queue:prefix('throttled'), min, max)
+    end, add = function(...)
+      if #arg > 0 then
+        redis.call('zadd', queue:prefix('throttled'), unpack(arg))
+      end
     end, remove = function(...)
       if #arg > 0 then
-        return queue_throttle.pending.remove(unpack(arg))
+        return redis.call('zrem', queue:prefix('throttled'), unpack(arg))
       end
-    end, length = function()
-      return queue_throttle.pending.length()
+    end, pop = function(min, max)
+      return redis.call('zremrangebyrank', queue:prefix('throttled'), min, max)
     end
   }
 
@@ -311,6 +313,7 @@ function QlessQueue:pop(now, worker, count)
     table.insert(popped, jid)
   end
 
+  -- if queue is at max capacity don't pop any further jobs.
   if not Qless.throttle(QlessQueue.ns .. self.name):available() then
     return popped
   end
@@ -336,14 +339,15 @@ function QlessQueue:pop(now, worker, count)
   -- With these in place, we can expand this list of jids based on the work
   -- queue itself and the priorities therein
   local jids = self.work.peek(count - #dead_jids) or {}
-
+  redis.call('set', 'printline', 'Pop - before acquire')
   for index, jid in ipairs(jids) do
     local job = Qless.job(jid)
     if job:acquire_throttles(now) then
       self:pop_job(now, worker, job)
       table.insert(popped, jid)
     else
-      job:history(now, 'throttled', {worker = worker})
+      redis.call('set', 'printline', 'QlessQueue:pop - throttling ' .. job.jid)
+      self:throttle(now, job)
     end
   end
 
@@ -354,6 +358,19 @@ function QlessQueue:pop(now, worker, count)
   return popped
 end
 
+-- Throttle a job
+function QlessQueue:throttle(now, job)
+  self.throttled.add(now, job.jid)
+  redis.call('set', 'printline', 'QlessQueue:throttle - get state')
+  local state = unpack(job:data('state'))
+  redis.call('set', 'printline', 'QlessQueue:throttle - check state')
+  if state ~= 'throttled' then
+    redis.call('set', 'printline', 'QlessQueue:throttle - update job')
+    job:update({state = 'throttled'})
+    job:history(now, 'throttled', {queue = self.name})
+  end
+end
+
 function QlessQueue:pop_job(now, worker, job)
   local state
   local jid = job.jid
@@ -865,6 +882,7 @@ function QlessQueue:check_throttled(now, count)
   for _, jid in ipairs(throttled) do
     local priority = tonumber(redis.call('hget', QlessJob.ns .. jid, 'priority') or 0)
     self.work.add(now, priority, jid)
+    self.throttled.remove(jid)
     redis.call('hset', QlessJob.ns .. jid, 'state', 'waiting')
   end
 end
diff --git a/test/test_job.py b/test/test_job.py
index 74ec485..1c12b37 100644
--- a/test/test_job.py
+++ b/test/test_job.py
@@ -276,7 +276,7 @@ def test_release_throttles_on_acquisition_failure(self):
     self.assertEqual(self.lua('throttle.locks', 4, 'wid'), ['jid1'])
     self.assertEqual(self.lua('throttle.locks', 5, 'tid'), [])
     self.assertEqual(self.lua('throttle.locks', 6, 'ql:q:queue'), ['jid1'])
-    self.assertEqual(self.lua('get', 7, 'jid2')['state'], 'waiting')
+    self.assertEqual(self.lua('get', 7, 'jid2')['state'], 'throttled')
 
   def test_release_throttles_after_acquisition_on_completion(self):
     '''Can acquire locks for all throttles and then release them when complete'''
diff --git a/test/test_queue.py b/test/test_queue.py
index 2e244ed..c51f3ad 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -662,7 +662,7 @@ def test_throttled_added(self):
         self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0)
         self.lua('pop', 2, 'queue', 'worker', 2)
         self.assertEqual(self.lua('throttle.locks', 3, 'ql:q:queue'), ['jid1'])
-        self.assertEqual(self.lua('throttle.pending', 3, 'ql:q:queue'), ['jid2'])
+        self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
 
     def test_throttled_removed(self):
         '''Throttled jobs are removed from throttled when concurrency available'''
@@ -671,9 +671,9 @@ def test_throttled_removed(self):
         self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0)
         self.lua('pop', 2, 'queue', 'worker', 2)
         self.assertEqual(self.lua('throttle.locks', 3, 'ql:q:queue'), ['jid1'])
-        self.assertEqual(self.lua('throttle.pending', 3, 'ql:q:queue'), ['jid2'])
-        self.lua('complete', 4, 'jid1', 'worker', 'queue', {})
-        self.assertEqual(self.lua('throttle.pending', 3, 'ql:q:queue'), ['jid2'])
-        self.lua('pop', 5, 'queue', 'worker', 1)
-        self.assertEqual(self.lua('throttle.locks', 6, 'ql:q:queue'), ['jid2'])
-        self.assertEqual(self.lua('throttle.pending', 3, 'ql:q:queue'), [])
+        self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
+        self.lua('complete', 5, 'jid1', 'worker', 'queue', {})
+        self.assertEqual(self.lua('jobs', 6, 'throttled', 'queue'), ['jid2'])
+        self.lua('pop', 7, 'queue', 'worker', 1)
+        self.assertEqual(self.lua('throttle.locks', 8, 'ql:q:queue'), ['jid2'])
+        self.assertEqual(self.lua('jobs', 9, 'throttled', 'queue'), [])
diff --git a/test/test_throttle.py b/test/test_throttle.py
index 2067302..e88416a 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -53,7 +53,7 @@ def test_limit_number_of_locks(self):
     self.lua('put', 3, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 0, 'queue', 'worker', 4)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid2', 'jid3', 'jid4'])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), ['jid2', 'jid3', 'jid4'])
 
 class TestRelease(TestQless):
   '''Test that when there are no pending jobs lock is properly released'''
@@ -61,10 +61,10 @@ def test_no_pending_jobs(self):
     self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
     self.lua('complete', 0, 'jid', 'worker', 'queue', {})
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
   '''Test that releasing a lock properly another job in the work queue'''
   def test_next_job_is_moved_into_work_qeueue(self):
@@ -72,14 +72,15 @@ def test_next_job_is_moved_into_work_qeueue(self):
     self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 2, 'queue', 'worker', 2)
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid2'])
-    self.lua('complete', 0, 'jid1', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
+    self.lua('complete', 5, 'jid1', 'worker', 'queue', {})
     # Lock should be empty until another job is popped
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
-    self.lua('pop', 2, 'queue', 'worker', 2)
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid2'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.locks', 6, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 7, 'throttled', 'queue'), ['jid2'])
+    self.lua('pop', 8, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 9, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 10, 'throttled', 'queue'), [])
 
 
   '''Test that when a job completes it properly releases the lock'''
@@ -90,7 +91,7 @@ def test_on_complete_lock_is_released(self):
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.lua('complete', 0, 'jid', 'worker', 'queue', {})
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
   '''Test that when a job fails it properly releases the lock'''
   def test_on_failure_lock_is_released(self):
@@ -100,7 +101,7 @@ def test_on_failure_lock_is_released(self):
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.lua('fail', 0, 'jid', 'worker', 'failed', 'i failed', {})
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
   '''Test that when a job retries it properly releases the lock
      and goes back into pending'''
@@ -111,7 +112,7 @@ def test_on_retry_lock_is_released(self):
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
     self.lua('retry', 0, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
   '''Test that when a job retries it is able to reacquire the lock when next popped'''
   def test_on_retry_lock_is_reacquired(self):
@@ -121,7 +122,7 @@ def test_on_retry_lock_is_reacquired(self):
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
     self.lua('retry', 0, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
   '''Test that when a job retries and no pending jobs it acquires the lock again on next pop'''
   def test_on_retry_no_pending_lock_is_reacquired(self):
@@ -132,7 +133,7 @@ def test_on_retry_no_pending_lock_is_reacquired(self):
     self.lua('retry', 0, 'jid', 'queue', 'worker', 0, 'retry', 'retrying')
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
   '''Test that when a job retries and another job is pending, the retrying job acquires the lock'''
   def test_on_retry_with_pending_lock_is_reacquired(self):
@@ -143,12 +144,12 @@ def test_on_retry_with_pending_lock_is_reacquired(self):
     self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 2, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('throttle.pending', 4, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
     self.lua('retry', 5, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
-    self.assertEqual(self.lua('throttle.pending', 6, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 6, 'throttled', 'queue'), ['jid2'])
     self.lua('pop', 7, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 8, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('throttle.pending', 9, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 9, 'throttled', 'queue'), ['jid2'])
 
 class TestDependents(TestQless):
   def test_dependencies_can_acquire_lock_after_dependent_success(self):
@@ -159,21 +160,21 @@ def test_dependencies_can_acquire_lock_after_dependent_success(self):
 
     self.lua('pop', 4, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 5, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('throttle.pending', 6, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 6, 'throttled', 'queue'), [])
     self.lua('complete', 7, 'jid1', 'worker', 'queue', {})
 
-    self.lua('pop', 0, 'queue', 'worker', 1)
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid2'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
-    self.lua('complete', 0, 'jid2', 'worker', 'queue', {})
+    self.lua('pop', 8, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 9, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 10, 'throttled', 'queue'), [])
+    self.lua('complete', 11, 'jid2', 'worker', 'queue', {})
 
-    self.lua('pop', 0, 'queue', 'worker', 1)
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid3'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
-    self.lua('complete', 0, 'jid3', 'worker', 'queue', {})
+    self.lua('pop', 12, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 13, 'tid'), ['jid3'])
+    self.assertEqual(self.lua('jobs', 14, 'throttled', 'queue'), [])
+    self.lua('complete', 15, 'jid3', 'worker', 'queue', {})
 
-    self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('throttle.locks', 16, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 17, 'throttled', 'queue'), [])
 
   def test_dependencies_can_acquire_lock_after_dependent_failure(self):
     self.lua('throttle.set', 0, 'tid', 1)
@@ -183,11 +184,11 @@ def test_dependencies_can_acquire_lock_after_dependent_failure(self):
 
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
     self.lua('fail', 0, 'jid1', 'worker', 'failed', 'i failed', {})
 
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
   def test_dependencies_do_not_acquire_lock_on_dependent_retry(self):
     self.lua('throttle.set', 0, 'tid', 1)
@@ -197,25 +198,25 @@ def test_dependencies_do_not_acquire_lock_on_dependent_retry(self):
 
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
     self.lua('retry', 0, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
 
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
     self.lua('complete', 0, 'jid1', 'worker', 'queue', {})
 
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid2'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
     self.lua('complete', 0, 'jid2', 'worker', 'queue', {})
 
     self.lua('pop', 0, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid3'])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
     self.lua('complete', 0, 'jid3', 'worker', 'queue', {})
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
-    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
 
 class TestConcurrencyLevelChange(TestQless):
@@ -229,11 +230,11 @@ def test_increasing_concurrency_level_activates_pending_jobs(self):
 
     self.lua('pop', 4, 'queue', 'worker', 3)
     self.assertEqual(self.lua('throttle.locks', 5, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('throttle.pending', 6, 'tid'), ['jid2', 'jid3'])
+    self.assertEqual(self.lua('jobs', 6, 'throttled', 'queue'), ['jid2', 'jid3'])
     self.lua('throttle.set', 7, 'tid', 3)
     self.lua('pop', 8, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 9, 'tid'), ['jid1', 'jid2', 'jid3'])
-    self.assertEqual(self.lua('throttle.pending', 10, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 10, 'throttled', 'queue'), [])
 
   def test_reducing_concurrency_level_without_pending(self):
     '''Operates at reduced concurrency level after current jobs finish'''
@@ -246,25 +247,25 @@ def test_reducing_concurrency_level_without_pending(self):
 
     self.lua('pop', 6, 'queue', 'worker', 3)
     self.assertEqual(self.lua('throttle.locks', 7, 'tid'), ['jid1', 'jid2', 'jid3'])
-    self.assertEqual(self.lua('throttle.pending', 8, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 8, 'throttled', 'queue'), [])
     self.lua('throttle.set', 9, 'tid', 1)
     self.lua('pop', 10, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 11, 'tid'), ['jid1', 'jid2', 'jid3'])
-    self.assertEqual(self.lua('throttle.pending', 12, 'tid'), ['jid4', 'jid5'])
+    self.assertEqual(self.lua('jobs', 12, 'throttled', 'queue'), ['jid4', 'jid5'])
     self.lua('complete', 13, 'jid1', 'worker', 'queue', {})
     self.assertEqual(self.lua('throttle.locks', 14, 'tid'), ['jid2', 'jid3'])
-    self.assertEqual(self.lua('throttle.pending', 15, 'tid'), ['jid4', 'jid5'])
+    self.assertEqual(self.lua('jobs', 15, 'throttled', 'queue'), ['jid4', 'jid5'])
     self.lua('complete', 16, 'jid2', 'worker', 'queue', {})
     self.assertEqual(self.lua('throttle.locks', 17, 'tid'), ['jid3'])
-    self.assertEqual(self.lua('throttle.pending', 18, 'tid'), ['jid4', 'jid5'])
+    self.assertEqual(self.lua('jobs', 18, 'throttled', 'queue'), ['jid4', 'jid5'])
     self.lua('complete', 19, 'jid3', 'worker', 'queue', {})
     self.lua('pop', 20, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 21, 'tid'), ['jid4'])
-    self.assertEqual(self.lua('throttle.pending', 22, 'tid'), ['jid5'])
+    self.assertEqual(self.lua('jobs', 22, 'throttled', 'queue'), ['jid5'])
     self.lua('complete', 24, 'jid4', 'worker', 'queue', {})
     self.lua('pop', 23, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 25, 'tid'), ['jid5'])
-    self.assertEqual(self.lua('throttle.pending', 26, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 26, 'throttled', 'queue'), [])
 
   def test_reducing_concurrency_level_with_pending(self):
     '''Operates at reduced concurrency level after current jobs finish'''
@@ -277,21 +278,21 @@ def test_reducing_concurrency_level_with_pending(self):
 
     self.lua('pop', 6, 'queue', 'worker', 5)
     self.assertEqual(self.lua('throttle.locks', 7, 'tid'), ['jid1', 'jid2', 'jid3'])
-    self.assertEqual(self.lua('throttle.pending', 8, 'tid'), ['jid4', 'jid5'])
+    self.assertEqual(self.lua('jobs', 8, 'throttled', 'queue'), ['jid4', 'jid5'])
     self.lua('throttle.set', 9, 'tid', 1)
     self.assertEqual(self.lua('throttle.locks', 10, 'tid'), ['jid1', 'jid2', 'jid3'])
-    self.assertEqual(self.lua('throttle.pending', 11, 'tid'), ['jid4', 'jid5'])
+    self.assertEqual(self.lua('jobs', 11, 'throttled', 'queue'), ['jid4', 'jid5'])
     self.lua('complete', 12, 'jid1', 'worker', 'queue', {})
     self.assertEqual(self.lua('throttle.locks', 13, 'tid'), ['jid2', 'jid3'])
-    self.assertEqual(self.lua('throttle.pending', 14, 'tid'), ['jid4', 'jid5'])
+    self.assertEqual(self.lua('jobs', 14, 'throttled', 'queue'), ['jid4', 'jid5'])
     self.lua('complete', 15, 'jid2', 'worker', 'queue', {})
     self.assertEqual(self.lua('throttle.locks', 16, 'tid'), ['jid3'])
-    self.assertEqual(self.lua('throttle.pending', 17, 'tid'), ['jid4', 'jid5'])
+    self.assertEqual(self.lua('jobs', 17, 'throttled', 'queue'), ['jid4', 'jid5'])
     self.lua('complete', 18, 'jid3', 'worker', 'queue', {})
     self.lua('pop', 19, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 20, 'tid'), ['jid4'])
-    self.assertEqual(self.lua('throttle.pending', 21, 'tid'), ['jid5'])
+    self.assertEqual(self.lua('jobs', 21, 'throttled', 'queue'), ['jid5'])
     self.lua('complete', 23, 'jid4', 'worker', 'queue', {})
     self.lua('pop', 22, 'queue', 'worker', 1)
     self.assertEqual(self.lua('throttle.locks', 24, 'tid'), ['jid5'])
-    self.assertEqual(self.lua('throttle.pending', 25, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 25, 'throttled', 'queue'), [])
diff --git a/throttle.lua b/throttle.lua
index 1c0342d..534c5a7 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -24,58 +24,25 @@ function QlessThrottle:unset()
 end
 
 -- Acquire a throttled resource for a job.
--- if the resource is at full capacity then add it to the pending
--- set.
--- Returns true of the job acquired the resource.
+-- Returns true of the job acquired the resource, false otherwise
 function QlessThrottle:acquire(jid)
-  if self:available() then
-    redis.call('set', 'printline', jid .. ' acquired the lock for ' .. self.id)
-    self.pending.remove(jid)
-    self.locks.add(1, jid)
-    return true
-  else
-    redis.call('set', 'printline', jid .. ' failed acquiring the lock for ' .. self.id .. ' marked as pending')
-    self.pending.add(1, jid)
+  redis.call('set', 'printline', 'QlessThrottle:acquire - checking availability')
+  if not self:available() then
+    redis.call('set', 'printline', jid .. ' failed to acquire lock on ' .. self.id)
     return false
   end
-end
 
--- Rolls back an attempted lock acquisition.
--- Since jobs can acquire multiple locks and the acquire
--- behavior is to either add them to the lock or pend them
--- this method handles the rolling back an acquired lock
--- on a job that failed to acquire all of its locks.
--- without placing another pending job into the queue.
-function QlessThrottle:rollback_acquire(jid)
-  self.locks.remove(jid)
-  self.pending.add(1, jid)
+  redis.call('set', 'printline', jid .. ' acquired a lock on ' .. self.id)
+  self.locks.add(1, jid)
+  return true
 end
 
 -- Release a throttled resource.
--- This will take a currently pending job
--- and attempt to acquire a lock.
--- If it succeeds at acquiring a lock then
--- the job will be moved from the throttled
--- queue into the work queue
 function QlessThrottle:release(now, jid)
   redis.call('set', 'printline', jid .. ' is releasing lock on ' .. self.id)
   self.locks.remove(jid)
 end
 
-function QlessThrottle:lock_pop(min, max)
-  local lock = Qless.throttle(self.id).locks
-  local jid = lock.peek(min,max)
-  lock.pop(min,max)
-  return jid
-end
-
-function QlessThrottle:pending_pop(min, max)
-  local pending = Qless.throttle(self.id).pending
-  local jids = pending.peek(min,max)
-  pending.pop(min,max)
-  return jids
-end
-
 -- Returns true if the throttle has locks available, false otherwise.
 function QlessThrottle:available()
   redis.call('set', 'printline', 'available ' .. self.maximum .. ' == 0 or ' .. self.locks.length() .. ' < ' .. self.maximum)

From 71ac5e095bcc91ed632bb5712b76676e5ea0c8dc Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Wed, 12 Mar 2014 18:57:56 -0400
Subject: [PATCH 36/72] Update queue.lua

fix off by
---
 queue.lua | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/queue.lua b/queue.lua
index 1654f5d..20319cb 100644
--- a/queue.lua
+++ b/queue.lua
@@ -878,7 +878,11 @@ function QlessQueue:check_scheduled(now, count)
 end
 
 function QlessQueue:check_throttled(now, count)
-  local throttled = self.throttled.peek(now, 0, count - 1)
+  if count == 0
+    return
+  end
+  
+  local throttled = self.throttled.peek(now, 0, count)
   for _, jid in ipairs(throttled) do
     local priority = tonumber(redis.call('hget', QlessJob.ns .. jid, 'priority') or 0)
     self.work.add(now, priority, jid)

From aadd9023cc2fae07a4f1d0e843ba2f660f117ea3 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Thu, 13 Mar 2014 05:33:35 -0400
Subject: [PATCH 37/72] fixed syntax error

---
 queue.lua | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/queue.lua b/queue.lua
index 20319cb..2d9cc9d 100644
--- a/queue.lua
+++ b/queue.lua
@@ -878,11 +878,12 @@ function QlessQueue:check_scheduled(now, count)
 end
 
 function QlessQueue:check_throttled(now, count)
-  if count == 0
+  if count == 0 then
     return
   end
-  
-  local throttled = self.throttled.peek(now, 0, count)
+
+  -- minus 1 since its inclusive
+  local throttled = self.throttled.peek(now, 0, count - 1)
   for _, jid in ipairs(throttled) do
     local priority = tonumber(redis.call('hget', QlessJob.ns .. jid, 'priority') or 0)
     self.work.add(now, priority, jid)

From 9595d5e29f565d7ceaf729ec25082bc0250502e1 Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Thu, 13 Mar 2014 06:09:50 -0400
Subject: [PATCH 38/72] Simplify job#acquire_throttles

---
 job.lua | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/job.lua b/job.lua
index 2a679e8..fea2c27 100644
--- a/job.lua
+++ b/job.lua
@@ -816,15 +816,13 @@ function QlessJob:acquire_throttles(now)
   redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - checking availability')
   for _, tid in ipairs(throttles) do
     redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - checking availability for ' .. tid)
-    all_locks_available = all_locks_available and Qless.throttle(tid):available()
+    if not Qless.throttle(tid):available() then
+      redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - short circuit if we can not acquire locks ' .. tostring(all_locks_available))
+      return false
+    end
     redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - throttle available ' .. tid)
   end
 
-  redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - short circuit if we can not acquire locks ' .. tostring(all_locks_available))
-  if not all_locks_available then
-    return false
-  end
-
   redis.call('set', 'printline', 'QlessJob:acquire_throttles - grabbing locks')
   redis.call('set', 'printline', 'QlessJob:acquire_throttles - inside if')
   for _, tid in ipairs(throttles) do

From 769d4f3728ad0cb6798b753f7bc17a02840ca56f Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Thu, 13 Mar 2014 06:13:47 -0400
Subject: [PATCH 39/72] Remove commented code

---
 base.lua | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/base.lua b/base.lua
index 65144c8..34a6076 100644
--- a/base.lua
+++ b/base.lua
@@ -99,26 +99,6 @@ function Qless.throttle(tid)
     end
   }
 
-  -- set of jids waiting on this throttle to become available.
-  -- throttle.pending = {
-  --   length = function()
-  --     return (redis.call('zcard', QlessThrottle.ns .. tid .. '-pending') or 0)
-  --   end, members = function()
-  --     return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', 0, -1)
-  --   end, peek = function(min, max)
-  --     return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', min, max)
-  --   end, add = function(...)
-  --     if #arg > 0 then
-  --       redis.call('zadd', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
-  --     end
-  --   end, remove = function(...)
-  --     if #arg > 0 then
-  --       return redis.call('zrem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
-  --     end
-  --   end, pop = function(min, max)
-  --     return redis.call('zremrangebyrank', QlessThrottle.ns .. tid .. '-pending', min, max)
-  --   end
-  -- }
   return throttle
 end
 

From f7a3b6f85dde853b3b110c6aa170f9458c6899dc Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Thu, 13 Mar 2014 07:03:15 -0400
Subject: [PATCH 40/72] misc fixes

---
 base.lua     | 22 +---------------------
 job.lua      | 53 +++++++++++++++++++++++++++++++---------------------
 queue.lua    | 21 +++++++++++++++------
 throttle.lua |  2 +-
 4 files changed, 49 insertions(+), 49 deletions(-)

diff --git a/base.lua b/base.lua
index 65144c8..d3adbc0 100644
--- a/base.lua
+++ b/base.lua
@@ -99,26 +99,6 @@ function Qless.throttle(tid)
     end
   }
 
-  -- set of jids waiting on this throttle to become available.
-  -- throttle.pending = {
-  --   length = function()
-  --     return (redis.call('zcard', QlessThrottle.ns .. tid .. '-pending') or 0)
-  --   end, members = function()
-  --     return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', 0, -1)
-  --   end, peek = function(min, max)
-  --     return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', min, max)
-  --   end, add = function(...)
-  --     if #arg > 0 then
-  --       redis.call('zadd', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
-  --     end
-  --   end, remove = function(...)
-  --     if #arg > 0 then
-  --       return redis.call('zrem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
-  --     end
-  --   end, pop = function(min, max)
-  --     return redis.call('zremrangebyrank', QlessThrottle.ns .. tid .. '-pending', min, max)
-  --   end
-  -- }
   return throttle
 end
 
@@ -437,7 +417,7 @@ function Qless.cancel(now, ...)
         queue.depends.remove(jid)
       end
 
-      Qless.job(namespaced_jid):release_throttles(now)
+      Qless.job(namespaced_jid):throttles_release(now)
 
       -- We should probably go through all our dependencies and remove
       -- ourselves from the list of dependents
diff --git a/job.lua b/job.lua
index 2a679e8..e6ec902 100644
--- a/job.lua
+++ b/job.lua
@@ -131,7 +131,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
   queue_obj.locks.remove(self.jid)
   queue_obj.scheduled.remove(self.jid)
 
-  self:release_throttles(now)
+  self:throttles_release(now)
 
   ----------------------------------------------------------
   -- This is the massive stats update that we have to do
@@ -400,7 +400,7 @@ function QlessJob:fail(now, worker, group, message, data)
       ['worker']  = worker
     }))
 
-  self:release_throttles(now)
+  self:throttles_release(now)
 
   -- Add this group of failure to the list of failures
   redis.call('sadd', 'ql:failures', group)
@@ -461,7 +461,7 @@ function QlessJob:retry(now, queue, worker, delay, group, message)
   Qless.queue(oldqueue).locks.remove(self.jid)
 
   -- Release the throttle for the job
-  self:release_throttles(now)
+  self:throttles_release(now)
 
   -- Remove this job from the worker that was previously working it
   redis.call('zrem', 'ql:w:' .. worker .. ':jobs', self.jid)
@@ -798,7 +798,7 @@ function QlessJob:history(now, what, item)
   end
 end
 
-function QlessJob:release_throttles(now)
+function QlessJob:throttles_release(now)
   local throttles = redis.call('hget', QlessJob.ns .. self.jid, 'throttles')
   throttles = cjson.decode(throttles or '{}')
 
@@ -808,29 +808,40 @@ function QlessJob:release_throttles(now)
   end
 end
 
-function QlessJob:acquire_throttles(now)
-  local throttles = cjson.decode(redis.call('hget', QlessJob.ns .. self.jid, 'throttles'))
-
-  local all_locks_available = true
-
-  redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - checking availability')
-  for _, tid in ipairs(throttles) do
-    redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - checking availability for ' .. tid)
-    all_locks_available = all_locks_available and Qless.throttle(tid):available()
-    redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - throttle available ' .. tid)
+function QlessJob:throttles_available()
+  redis.call('set', 'printline', 'QlessJob:throttles_available - ' .. self.jid .. ' - checking throttle availability')
+  for _, tid in ipairs(self:throttles()) do
+    redis.call('set', 'printline', 'QlessJob:throttles_available - ' .. self.jid .. ' - checking availability for ' .. tid)
+    if not Qless.throttle(tid):available() then
+      redis.call('set', 'printline', 'QlessJob:throttles_available - ' .. self.jid .. ' - throttle not available ' .. tid)
+      return false
+    end
+    redis.call('set', 'printline', 'QlessJob:throttles_available - ' .. self.jid .. ' - throttle available ' .. tid)
   end
 
-  redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - short circuit if we can not acquire locks ' .. tostring(all_locks_available))
-  if not all_locks_available then
+  return true
+end
+
+function QlessJob:throttles_acquire(now)
+  if not self:throttles_available() then
+    redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - throttles not avaible')
     return false
   end
 
-  redis.call('set', 'printline', 'QlessJob:acquire_throttles - grabbing locks')
-  redis.call('set', 'printline', 'QlessJob:acquire_throttles - inside if')
-  for _, tid in ipairs(throttles) do
-    redis.call('set', 'printline', 'QlessJob:acquire_throttles - invoking QlessThrottle:acquire')
+  for _, tid in ipairs(self:throttles()) do
+    redis.call('set', 'printline', 'QlessJob:acquire_throttles - acquiring ' .. tid)
     Qless.throttle(tid):acquire(self.jid)
   end
-  redis.call('set', 'printline', 'QlessJob:acquire_throttles - successfully completed')
+
+  redis.call('set', 'printline', 'QlessJob:acquire_throttles - throttles avaible')
   return true
 end
+
+function QlessJob:throttles()
+  -- memoize throttles for the job.
+  if not self._throttles then
+    self._throttles = cjson.decode(redis.call('hget', QlessJob.ns .. self.jid, 'throttles'))
+  end
+
+  return self._throttles
+end
diff --git a/queue.lua b/queue.lua
index 2d9cc9d..930ebc8 100644
--- a/queue.lua
+++ b/queue.lua
@@ -318,6 +318,7 @@ function QlessQueue:pop(now, worker, count)
     return popped
   end
 
+  redis.call('set', 'printline', 'dead_jids : ' .. tostring(#dead_jids))
   -- Now we've checked __all__ the locks for this queue the could
   -- have expired, and are no more than the number requested.
 
@@ -339,10 +340,10 @@ function QlessQueue:pop(now, worker, count)
   -- With these in place, we can expand this list of jids based on the work
   -- queue itself and the priorities therein
   local jids = self.work.peek(count - #dead_jids) or {}
-  redis.call('set', 'printline', 'Pop - before acquire')
+
   for index, jid in ipairs(jids) do
     local job = Qless.job(jid)
-    if job:acquire_throttles(now) then
+    if job:throttles_acquire(now) then
       self:pop_job(now, worker, job)
       table.insert(popped, jid)
     else
@@ -879,16 +880,24 @@ end
 
 function QlessQueue:check_throttled(now, count)
   if count == 0 then
+    redis.call('set', 'printline', 'count 0 not popping any throttled jobs')
     return
   end
 
   -- minus 1 since its inclusive
   local throttled = self.throttled.peek(now, 0, count - 1)
+  redis.call('set', 'printline', 'throttling the following jobs ' .. cjson.encode(throttled))
   for _, jid in ipairs(throttled) do
-    local priority = tonumber(redis.call('hget', QlessJob.ns .. jid, 'priority') or 0)
-    self.work.add(now, priority, jid)
     self.throttled.remove(jid)
-    redis.call('hset', QlessJob.ns .. jid, 'state', 'waiting')
+    if Qless.job(jid):throttles_available() then
+      local priority = tonumber(redis.call('hget', QlessJob.ns .. jid, 'priority') or 0)
+      self.work.add(now, priority, jid)
+      self.throttled.remove(jid)
+    else
+      -- shift jid to end of throttled jobs
+      -- use current time to make sure it gets added to the end of the sorted set.
+      self.throttled.add(now, jid)
+    end
   end
 end
 
@@ -976,7 +985,7 @@ function QlessQueue:invalidate_locks(now, count)
         local queue = job_data['queue']
         local group = 'failed-retries-' .. queue
 
-        job:release_throttles(now)
+        job:throttles_release(now)
 
         job:history(now, 'failed', {group = group})
         redis.call('hmset', QlessJob.ns .. jid, 'state', 'failed',
diff --git a/throttle.lua b/throttle.lua
index 534c5a7..11b8111 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -45,6 +45,6 @@ end
 
 -- Returns true if the throttle has locks available, false otherwise.
 function QlessThrottle:available()
-  redis.call('set', 'printline', 'available ' .. self.maximum .. ' == 0 or ' .. self.locks.length() .. ' < ' .. self.maximum)
+  redis.call('set', 'printline', self.id .. ' available ' .. self.maximum .. ' == 0 or ' .. self.locks.length() .. ' < ' .. self.maximum)
   return self.maximum == 0 or self.locks.length() < self.maximum
 end

From 7d85dc897eea7d46e0eaf3a1bb2c5b407209dd16 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Thu, 13 Mar 2014 07:30:54 -0400
Subject: [PATCH 41/72] attempt to throttle jobs immediately

---
 queue.lua          |  5 +++++
 test/test_queue.py | 11 ++++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/queue.lua b/queue.lua
index 930ebc8..3600e0b 100644
--- a/queue.lua
+++ b/queue.lua
@@ -631,9 +631,14 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
       self.scheduled.add(now + delay, jid)
     end
   else
+    -- to avoid false negatives when popping jobs check if the job should be
+    -- throttled immediately.
+    local job = Qless.job(jid)
     if redis.call('scard', QlessJob.ns .. jid .. '-dependencies') > 0 then
       self.depends.add(now, jid)
       redis.call('hset', QlessJob.ns .. jid, 'state', 'depends')
+    elseif not job:throttles_available() then
+      self:throttle(now, job)
     else
       self.work.add(now, priority, jid)
     end
diff --git a/test/test_queue.py b/test/test_queue.py
index c51f3ad..c1b2760 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -1,7 +1,7 @@
 '''Test the queue functionality'''
 
 from common import TestQless
-
+import code
 
 class TestJobs(TestQless):
     '''We should be able to list jobs in various states for a given queue'''
@@ -677,3 +677,12 @@ def test_throttled_removed(self):
         self.lua('pop', 7, 'queue', 'worker', 1)
         self.assertEqual(self.lua('throttle.locks', 8, 'ql:q:queue'), ['jid2'])
         self.assertEqual(self.lua('jobs', 9, 'throttled', 'queue'), [])
+
+    def test_throttled_additional_put(self):
+        '''put should attempt to throttle the job immediately'''
+        self.lua('throttle.set', 0, 'ql:q:queue', 1)
+        self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0)
+        self.lua('pop', 1, 'queue', 'worker', 1)
+        self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0)
+        self.assertEqual(self.lua('throttle.locks', 3, 'ql:q:queue'), ['jid1'])
+        self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])

From 2d9810c0f3546aa089954e970a45b7d0c8e99d80 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Thu, 13 Mar 2014 07:55:54 -0400
Subject: [PATCH 42/72] added optional expiration to a throttle

---
 api.lua               | 5 +++--
 test/test_throttle.py | 8 +++++++-
 throttle.lua          | 5 ++++-
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/api.lua b/api.lua
index 7209954..4ec1d5d 100644
--- a/api.lua
+++ b/api.lua
@@ -206,11 +206,12 @@ QlessAPI['queue.throttle.set'] = function(now, queue, max)
 end
 
 -- Throttle apis
-QlessAPI['throttle.set'] = function(now, tid, max)
+QlessAPI['throttle.set'] = function(now, tid, max, ...)
+  local expiration = unpack(arg)
   local data = {
     maximum = max
   }
-  Qless.throttle(tid):set(data)
+  Qless.throttle(tid):set(data, tonumber(expiration or 0))
 end
 
 QlessAPI['throttle.get'] = function(now, tid)
diff --git a/test/test_throttle.py b/test/test_throttle.py
index e88416a..f112bd0 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -7,9 +7,15 @@
 class TestThrottle(TestQless):
   '''Test setting throttle data'''
   def test_set(self):
-    self.lua('throttle.set', 0, 'tid', 5)
+    self.lua('throttle.set', 0, 'tid', 5, 0)
     self.assertEqual(self.redis.hmget('ql:th:tid', 'id')[0], 'tid')
     self.assertEqual(self.redis.hmget('ql:th:tid', 'maximum')[0], '5')
+    self.assertEqual(self.redis.ttl('ql:th:tid'), None)
+
+  '''Test setting a expiring throttle'''
+  def test_set_with_expiration(self):
+    self.lua('throttle.set', 0, 'tid', 5, 1000)
+    self.assertNotEqual(self.redis.ttl('ql:th:tid'), None)
 
   '''Test retrieving throttle data'''
   def test_get(self):
diff --git a/throttle.lua b/throttle.lua
index 11b8111..4a764b7 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -14,8 +14,11 @@ function QlessThrottle:data()
 end
 
 -- Set the data for a throttled resource
-function QlessThrottle:set(data)
+function QlessThrottle:set(data, expiration)
   redis.call('hmset', QlessThrottle.ns .. self.id, 'id', self.id, 'maximum', data.maximum)
+  if expiration > 0 then
+    redis.call('expire', QlessThrottle.ns .. self.id, expiration)
+  end
 end
 
 -- Delete a throttled resource

From 4728f1d2c8986415f57d35c635ce571bf2a8311d Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Thu, 13 Mar 2014 08:11:09 -0400
Subject: [PATCH 43/72] removed printlines

---
 job.lua      | 8 --------
 queue.lua    | 9 ---------
 throttle.lua | 5 -----
 3 files changed, 22 deletions(-)

diff --git a/job.lua b/job.lua
index e6ec902..ba2337a 100644
--- a/job.lua
+++ b/job.lua
@@ -803,20 +803,15 @@ function QlessJob:throttles_release(now)
   throttles = cjson.decode(throttles or '{}')
 
   for _, tid in ipairs(throttles) do
-    redis.call('set', 'printline', 'releasing throttle : ' .. tid)
     Qless.throttle(tid):release(now, self.jid)
   end
 end
 
 function QlessJob:throttles_available()
-  redis.call('set', 'printline', 'QlessJob:throttles_available - ' .. self.jid .. ' - checking throttle availability')
   for _, tid in ipairs(self:throttles()) do
-    redis.call('set', 'printline', 'QlessJob:throttles_available - ' .. self.jid .. ' - checking availability for ' .. tid)
     if not Qless.throttle(tid):available() then
-      redis.call('set', 'printline', 'QlessJob:throttles_available - ' .. self.jid .. ' - throttle not available ' .. tid)
       return false
     end
-    redis.call('set', 'printline', 'QlessJob:throttles_available - ' .. self.jid .. ' - throttle available ' .. tid)
   end
 
   return true
@@ -824,16 +819,13 @@ end
 
 function QlessJob:throttles_acquire(now)
   if not self:throttles_available() then
-    redis.call('set', 'printline', 'QlessJob:acquire_throttles - ' .. self.jid .. ' - throttles not avaible')
     return false
   end
 
   for _, tid in ipairs(self:throttles()) do
-    redis.call('set', 'printline', 'QlessJob:acquire_throttles - acquiring ' .. tid)
     Qless.throttle(tid):acquire(self.jid)
   end
 
-  redis.call('set', 'printline', 'QlessJob:acquire_throttles - throttles avaible')
   return true
 end
 
diff --git a/queue.lua b/queue.lua
index 3600e0b..0510830 100644
--- a/queue.lua
+++ b/queue.lua
@@ -318,7 +318,6 @@ function QlessQueue:pop(now, worker, count)
     return popped
   end
 
-  redis.call('set', 'printline', 'dead_jids : ' .. tostring(#dead_jids))
   -- Now we've checked __all__ the locks for this queue the could
   -- have expired, and are no more than the number requested.
 
@@ -347,7 +346,6 @@ function QlessQueue:pop(now, worker, count)
       self:pop_job(now, worker, job)
       table.insert(popped, jid)
     else
-      redis.call('set', 'printline', 'QlessQueue:pop - throttling ' .. job.jid)
       self:throttle(now, job)
     end
   end
@@ -362,11 +360,8 @@ end
 -- Throttle a job
 function QlessQueue:throttle(now, job)
   self.throttled.add(now, job.jid)
-  redis.call('set', 'printline', 'QlessQueue:throttle - get state')
   local state = unpack(job:data('state'))
-  redis.call('set', 'printline', 'QlessQueue:throttle - check state')
   if state ~= 'throttled' then
-    redis.call('set', 'printline', 'QlessQueue:throttle - update job')
     job:update({state = 'throttled'})
     job:history(now, 'throttled', {queue = self.name})
   end
@@ -498,7 +493,6 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
   local throttles = assert(cjson.decode(options['throttles'] or '[]'),
     'Put(): Arg "throttles" not JSON array: ' .. tostring(options['throttles']))
 
-  redis.call('set', 'printline', 'throttles : ' .. tostring(options['throttles']))
   -- If the job has old dependencies, determine which dependencies are
   -- in the new dependencies but not in the old ones, and which are in the
   -- old ones but not in the new
@@ -796,7 +790,6 @@ function QlessQueue:check_recurring(now, count)
     -- we need to keep putting jobs on the queue
     local r = redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority',
         'tags', 'retries', 'interval', 'backlog', 'throttles')
-    redis.call('set', 'printline', cjson.encode(r))
     local klass, data, priority, tags, retries, interval, backlog, throttles = unpack(
       redis.call('hmget', 'ql:r:' .. jid, 'klass', 'data', 'priority',
         'tags', 'retries', 'interval', 'backlog', 'throttles'))
@@ -885,13 +878,11 @@ end
 
 function QlessQueue:check_throttled(now, count)
   if count == 0 then
-    redis.call('set', 'printline', 'count 0 not popping any throttled jobs')
     return
   end
 
   -- minus 1 since its inclusive
   local throttled = self.throttled.peek(now, 0, count - 1)
-  redis.call('set', 'printline', 'throttling the following jobs ' .. cjson.encode(throttled))
   for _, jid in ipairs(throttled) do
     self.throttled.remove(jid)
     if Qless.job(jid):throttles_available() then
diff --git a/throttle.lua b/throttle.lua
index 4a764b7..b4c6a64 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -29,25 +29,20 @@ end
 -- Acquire a throttled resource for a job.
 -- Returns true of the job acquired the resource, false otherwise
 function QlessThrottle:acquire(jid)
-  redis.call('set', 'printline', 'QlessThrottle:acquire - checking availability')
   if not self:available() then
-    redis.call('set', 'printline', jid .. ' failed to acquire lock on ' .. self.id)
     return false
   end
 
-  redis.call('set', 'printline', jid .. ' acquired a lock on ' .. self.id)
   self.locks.add(1, jid)
   return true
 end
 
 -- Release a throttled resource.
 function QlessThrottle:release(now, jid)
-  redis.call('set', 'printline', jid .. ' is releasing lock on ' .. self.id)
   self.locks.remove(jid)
 end
 
 -- Returns true if the throttle has locks available, false otherwise.
 function QlessThrottle:available()
-  redis.call('set', 'printline', self.id .. ' available ' .. self.maximum .. ' == 0 or ' .. self.locks.length() .. ' < ' .. self.maximum)
   return self.maximum == 0 or self.locks.length() < self.maximum
 end

From 85350ea68c9836b810226dc737c08ed65e2c741b Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Thu, 13 Mar 2014 08:19:40 -0400
Subject: [PATCH 44/72] Add API to retrieve throttle ttl

---
 api.lua               | 4 ++++
 test/test_throttle.py | 5 +++++
 throttle.lua          | 5 +++++
 3 files changed, 14 insertions(+)

diff --git a/api.lua b/api.lua
index 4ec1d5d..635f5a9 100644
--- a/api.lua
+++ b/api.lua
@@ -225,6 +225,10 @@ end
 QlessAPI['throttle.locks'] = function(now, tid)
   return Qless.throttle(tid).locks.members()
 end
+
+QlessAPI['throttle.ttl'] = function(now, tid)
+  return Qless.throttle(tid):ttl()
+end
 -------------------------------------------------------------------------------
 -- Function lookup
 -------------------------------------------------------------------------------
diff --git a/test/test_throttle.py b/test/test_throttle.py
index f112bd0..d83a0f9 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -17,6 +17,11 @@ def test_set_with_expiration(self):
     self.lua('throttle.set', 0, 'tid', 5, 1000)
     self.assertNotEqual(self.redis.ttl('ql:th:tid'), None)
 
+  '''Test retrieving throttle ttl'''
+  def test_retrieve_ttl(self):
+    self.lua('throttle.set', 0, 'tid', 5, 1000)
+    self.assertEqual(self.lua('throttle.ttl', 1, 'tid'), self.redis.ttl('ql:th:tid'))
+
   '''Test retrieving throttle data'''
   def test_get(self):
     self.redis.hmset('ql:th:tid', {'id': 'tid', 'maximum' : 5})
diff --git a/throttle.lua b/throttle.lua
index b4c6a64..761bfb3 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -46,3 +46,8 @@ end
 function QlessThrottle:available()
   return self.maximum == 0 or self.locks.length() < self.maximum
 end
+
+-- Returns the TTL of the throttle
+function QlessThrottle:ttl()
+  return redis.call('ttl', QlessThrottle.ns .. self.id)
+end

From 5668c321c0a2433816aa3890b63c1a1eafeb96dc Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Thu, 13 Mar 2014 08:29:46 -0400
Subject: [PATCH 45/72] Set queue throttle expiration to 0

---
 api.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api.lua b/api.lua
index 635f5a9..03eba6c 100644
--- a/api.lua
+++ b/api.lua
@@ -202,7 +202,7 @@ QlessAPI['queue.throttle.get'] = function(now, queue)
 end
 
 QlessAPI['queue.throttle.set'] = function(now, queue, max)
-  Qless.throttle(QlessQueue.ns .. queue):set({maximum = max})
+  Qless.throttle(QlessQueue.ns .. queue):set({maximum = max}, 0)
 end
 
 -- Throttle apis

From 0d8290e6954c55bc801a495a7269938470360112 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Thu, 13 Mar 2014 10:30:08 -0400
Subject: [PATCH 46/72] Update test_job.py

removed import.
---
 test/test_job.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_job.py b/test/test_job.py
index 1c12b37..53dbb35 100644
--- a/test/test_job.py
+++ b/test/test_job.py
@@ -1,7 +1,6 @@
 '''Test job-centric operations'''
 
 import redis
-import pdb
 from common import TestQless
 
 

From 70a2779ec7ba909eab8ccc11f03c8ca941858d3a Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Thu, 13 Mar 2014 14:09:39 -0400
Subject: [PATCH 47/72] Catch when job has no throttles

---
 job.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/job.lua b/job.lua
index ba2337a..f9e932a 100644
--- a/job.lua
+++ b/job.lua
@@ -832,7 +832,7 @@ end
 function QlessJob:throttles()
   -- memoize throttles for the job.
   if not self._throttles then
-    self._throttles = cjson.decode(redis.call('hget', QlessJob.ns .. self.jid, 'throttles'))
+    self._throttles = cjson.decode(redis.call('hget', QlessJob.ns .. self.jid, 'throttles') or '[]')
   end
 
   return self._throttles

From dd2d9313ae1461775b23de93c6830883341ad1e9 Mon Sep 17 00:00:00 2001
From: Matt Conway <matt@conwaysplace.com>
Date: Fri, 14 Mar 2014 10:50:44 -0400
Subject: [PATCH 48/72] test exposing cancel bug

---
 test/test_throttle.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/test/test_throttle.py b/test/test_throttle.py
index d83a0f9..446a89a 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -94,6 +94,23 @@ def test_next_job_is_moved_into_work_qeueue(self):
     self.assertEqual(self.lua('jobs', 10, 'throttled', 'queue'), [])
 
 
+  '''Test that cancelling a job properly adds another job in the work queue'''
+  def test_on_cancel_next_job_is_moved_into_work_queue(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('pop', 2, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
+    self.lua('cancel', 5, 'jid1', 'worker', 'queue', {})
+    # Lock should be empty until another job is popped
+    self.assertEqual(self.lua('throttle.locks', 6, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 7, 'throttled', 'queue'), ['jid2'])
+    self.lua('pop', 8, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 9, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 10, 'throttled', 'queue'), [])
+
+
   '''Test that when a job completes it properly releases the lock'''
   def test_on_complete_lock_is_released(self):
     self.lua('throttle.set', 0, 'tid', 1)

From 8f004dc648f8b5f02c818f7694e62cee7808acf3 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 31 Mar 2014 11:10:45 -0400
Subject: [PATCH 49/72] fix for cancelled

---
 base.lua  |  8 ++------
 job.lua   | 10 +++-------
 queue.lua | 13 +++++++++----
 3 files changed, 14 insertions(+), 17 deletions(-)

diff --git a/base.lua b/base.lua
index d3adbc0..6e7563e 100644
--- a/base.lua
+++ b/base.lua
@@ -386,7 +386,6 @@ function Qless.cancel(now, ...)
   -- If we've made it this far, then we are good to go. We can now just
   -- remove any trace of all these jobs, as they form a dependent clique
   for _, jid in ipairs(arg) do
-    local namespaced_jid = QlessJob.ns .. jid
     -- Find any stage it's associated with and remove its from that stage
     local state, queue, failure, worker = unpack(redis.call(
       'hmget', QlessJob.ns .. jid, 'state', 'queue', 'failure', 'worker'))
@@ -411,13 +410,10 @@ function Qless.cancel(now, ...)
       -- Remove it from that queue
       if queue then
         local queue = Qless.queue(queue)
-        queue.work.remove(jid)
-        queue.locks.remove(jid)
-        queue.scheduled.remove(jid)
-        queue.depends.remove(jid)
+        queue:remove_job(jid)
       end
 
-      Qless.job(namespaced_jid):throttles_release(now)
+      Qless.job(jid):throttles_release(now)
 
       -- We should probably go through all our dependencies and remove
       -- ourselves from the list of dependents
diff --git a/job.lua b/job.lua
index f9e932a..8f6f2b7 100644
--- a/job.lua
+++ b/job.lua
@@ -127,9 +127,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
 
   -- Remove the job from the previous queue
   local queue_obj = Qless.queue(queue)
-  queue_obj.work.remove(self.jid)
-  queue_obj.locks.remove(self.jid)
-  queue_obj.scheduled.remove(self.jid)
+  queue_obj:remove_job(self.jid)
 
   self:throttles_release(now)
 
@@ -379,9 +377,7 @@ function QlessJob:fail(now, worker, group, message, data)
   -- Now remove the instance from the schedule, and work queues for the
   -- queue it's in
   local queue_obj = Qless.queue(queue)
-  queue_obj.work.remove(self.jid)
-  queue_obj.locks.remove(self.jid)
-  queue_obj.scheduled.remove(self.jid)
+  queue_obj:remove_job(self.jid)
 
   -- The reason that this appears here is that the above will fail if the
   -- job doesn't exist
@@ -800,7 +796,7 @@ end
 
 function QlessJob:throttles_release(now)
   local throttles = redis.call('hget', QlessJob.ns .. self.jid, 'throttles')
-  throttles = cjson.decode(throttles or '{}')
+  throttles = cjson.decode(throttles or '[]')
 
   for _, tid in ipairs(throttles) do
     Qless.throttle(tid):release(now, self.jid)
diff --git a/queue.lua b/queue.lua
index 0510830..ca6b0ad 100644
--- a/queue.lua
+++ b/queue.lua
@@ -526,10 +526,7 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
   -- If this item was previously in another queue, then we should remove it from there
   if oldqueue then
     local queue_obj = Qless.queue(oldqueue)
-    queue_obj.work.remove(jid)
-    queue_obj.locks.remove(jid)
-    queue_obj.depends.remove(jid)
-    queue_obj.scheduled.remove(jid)
+    queue_obj:remove_job(jid)
   end
 
   -- If this had previously been given out to a worker, make sure to remove it
@@ -777,6 +774,14 @@ end
 -------------------------------------------------------------------------------
 -- Housekeeping methods
 -------------------------------------------------------------------------------
+function QlessQueue:remove_job(jid)
+  self.work.remove(jid)
+  self.locks.remove(jid)
+  self.throttled.remove(jid)
+  self.depends.remove(jid)
+  self.scheduled.remove(jid)
+end
+
 -- Instantiate any recurring jobs that are ready
 function QlessQueue:check_recurring(now, count)
   -- This is how many jobs we've moved so far

From a9c3b988a5e3150a5d01b698d882bdbf8b264c42 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 1 Apr 2014 12:22:00 -0400
Subject: [PATCH 50/72] wip

---
 api.lua               |   4 ++
 base.lua              |  21 ++++++++
 job.lua               |  15 +++++-
 queue.lua             |  30 +----------
 test/test_queue.py    |  27 ++++++----
 test/test_throttle.py | 114 +++++++++++++++++++++++++++---------------
 throttle.lua          |  43 +++++++++++++++-
 7 files changed, 172 insertions(+), 82 deletions(-)

diff --git a/api.lua b/api.lua
index 03eba6c..5635a9f 100644
--- a/api.lua
+++ b/api.lua
@@ -226,6 +226,10 @@ QlessAPI['throttle.locks'] = function(now, tid)
   return Qless.throttle(tid).locks.members()
 end
 
+QlessAPI['throttle.pending'] = function(now, tid)
+  return Qless.throttle(tid).pending.members()
+end
+
 QlessAPI['throttle.ttl'] = function(now, tid)
   return Qless.throttle(tid):ttl()
 end
diff --git a/base.lua b/base.lua
index 6e7563e..ba2ff7a 100644
--- a/base.lua
+++ b/base.lua
@@ -96,6 +96,27 @@ function Qless.throttle(tid)
       end
     end, pop = function(min, max)
       return redis.call('zremrangebyrank', QlessThrottle.ns .. tid .. '-locks', min, max)
+    end, peek = function(min, max)
+      return redis.call('zrange', QlessThrottle.ns .. tid .. '-locks', min, max)
+    end
+  }
+
+  -- set of jids which are waiting for the throttle to become available.
+  throttle.pending = {
+    length = function()
+      return (redis.call('zcard', QlessThrottle.ns .. tid .. '-pending') or 0)
+    end, members = function()
+        return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', 0, -1)
+    end, add = function(now, jid)
+      redis.call('zadd', QlessThrottle.ns .. tid .. '-pending', now, jid)
+    end, remove = function(...)
+      if #arg > 0 then
+        return redis.call('zrem', QlessThrottle.ns .. tid .. '-pending', unpack(arg))
+      end
+    end, pop = function(min, max)
+      return redis.call('zremrangebyrank', QlessThrottle.ns .. tid .. '-pending', min, max)
+    end, peek = function(min, max)
+      return redis.call('zrange', QlessThrottle.ns .. tid .. '-pending', min, max)
     end
   }
 
diff --git a/job.lua b/job.lua
index 8f6f2b7..5cc7597 100644
--- a/job.lua
+++ b/job.lua
@@ -139,7 +139,7 @@ function QlessJob:complete(now, worker, queue, data, ...)
   local time = tonumber(
     redis.call('hget', QlessJob.ns .. self.jid, 'time') or now)
   local waiting = now - time
-  Qless.queue(queue):stat(now, 'run', waiting)
+  queue_obj:stat(now, 'run', waiting)
   redis.call('hset', QlessJob.ns .. self.jid,
     'time', string.format("%.20f", now))
 
@@ -799,6 +799,7 @@ function QlessJob:throttles_release(now)
   throttles = cjson.decode(throttles or '[]')
 
   for _, tid in ipairs(throttles) do
+    redis.call('set', 'printline', self.jid .. ' releasing throttle ' .. 'tid')
     Qless.throttle(tid):release(now, self.jid)
   end
 end
@@ -825,6 +826,18 @@ function QlessJob:throttles_acquire(now)
   return true
 end
 
+-- Finds the first unavailable throttle and adds the job to its pending job set.
+function QlessJob:throttle(now)
+  for _, tid in ipairs(self:throttles()) do
+    local throttle = Qless.throttle(tid)
+    if not throttle:available() then
+      redis.call('set', 'printline', 'pending ' .. self.jid .. ' for throttle ' .. tid)
+      throttle:pend(now, self.jid)
+      return
+    end
+  end
+end
+
 function QlessJob:throttles()
   -- memoize throttles for the job.
   if not self._throttles then
diff --git a/queue.lua b/queue.lua
index ca6b0ad..07fa3ce 100644
--- a/queue.lua
+++ b/queue.lua
@@ -331,11 +331,6 @@ function QlessQueue:pop(now, worker, count)
   -- unit of work.
   self:check_scheduled(now, count - #dead_jids)
 
-  -- If we still need values in order to meet the demand, check our throttled
-  -- jobs. This has the side benefit of naturally updating other throttles
-  -- on the jobs checked.
-  self:check_throttled(now, count - #dead_jids)
-
   -- With these in place, we can expand this list of jids based on the work
   -- queue itself and the priorities therein
   local jids = self.work.peek(count - #dead_jids) or {}
@@ -359,6 +354,7 @@ end
 
 -- Throttle a job
 function QlessQueue:throttle(now, job)
+  job:throttle(now)
   self.throttled.add(now, job.jid)
   local state = unpack(job:data('state'))
   if state ~= 'throttled' then
@@ -380,8 +376,7 @@ function QlessQueue:pop_job(now, worker, job)
 
   -- Update the wait time statistics
   -- Just does job:data('time') do the same as this?
-  local time = tonumber(
-    redis.call('hget', QlessJob.ns .. jid, 'time') or now)
+  local time = tonumber(redis.call('hget', QlessJob.ns .. jid, 'time') or now)
   local waiting = now - time
   self:stat(now, 'wait', waiting)
   redis.call('hset', QlessJob.ns .. jid,
@@ -881,27 +876,6 @@ function QlessQueue:check_scheduled(now, count)
   end
 end
 
-function QlessQueue:check_throttled(now, count)
-  if count == 0 then
-    return
-  end
-
-  -- minus 1 since its inclusive
-  local throttled = self.throttled.peek(now, 0, count - 1)
-  for _, jid in ipairs(throttled) do
-    self.throttled.remove(jid)
-    if Qless.job(jid):throttles_available() then
-      local priority = tonumber(redis.call('hget', QlessJob.ns .. jid, 'priority') or 0)
-      self.work.add(now, priority, jid)
-      self.throttled.remove(jid)
-    else
-      -- shift jid to end of throttled jobs
-      -- use current time to make sure it gets added to the end of the sorted set.
-      self.throttled.add(now, jid)
-    end
-  end
-end
-
 -- Check for and invalidate any locks that have been lost. Returns the
 -- list of jids that have been invalidated
 function QlessQueue:invalidate_locks(now, count)
diff --git a/test/test_queue.py b/test/test_queue.py
index c1b2760..fabe6bc 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -1,7 +1,6 @@
 '''Test the queue functionality'''
 
 from common import TestQless
-import code
 
 class TestJobs(TestQless):
     '''We should be able to list jobs in various states for a given queue'''
@@ -616,8 +615,7 @@ def test_max_concurrency(self):
         # But as we complete the jobs, we can pop more
         for jid in xrange(5):
             self.lua('complete', 10, jid, 'worker', 'queue', {})
-            self.assertEqual(
-                len(self.lua('pop', 10, 'queue', 'worker', 10)), 1)
+            self.assertEqual(len(self.lua('pop', 10, 'queue', 'worker', 10)), 1)
 
     def test_reduce_max_concurrency(self):
         '''We can reduce max_concurrency at any time'''
@@ -651,8 +649,10 @@ def test_fail_max_concurrency(self):
         self.lua('put', 0, 'worker', 'queue', 'a', 'klass', {}, 0)
         self.lua('put', 1, 'worker', 'queue', 'b', 'klass', {}, 0)
         self.lua('pop', 2, 'queue', 'worker', 10)
-        self.lua('fail', 3, 'a', 'worker', 'group', 'message', {})
-        job = self.lua('pop', 4, 'queue', 'worker', 10)[0]
+        self.assertEqual(self.lua('throttle.locks', 3, 'ql:q:queue'), ['a'])
+        self.assertEqual(self.lua('throttle.pending', 4, 'ql:q:queue'), ['b'])
+        self.lua('fail', 5, 'a', 'worker', 'group', 'message', {})
+        job = self.lua('pop', 6, 'queue', 'worker', 10)[0]
         self.assertEqual(job['jid'], 'b')
 
     def test_throttled_added(self):
@@ -671,12 +671,17 @@ def test_throttled_removed(self):
         self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0)
         self.lua('pop', 2, 'queue', 'worker', 2)
         self.assertEqual(self.lua('throttle.locks', 3, 'ql:q:queue'), ['jid1'])
-        self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
-        self.lua('complete', 5, 'jid1', 'worker', 'queue', {})
-        self.assertEqual(self.lua('jobs', 6, 'throttled', 'queue'), ['jid2'])
-        self.lua('pop', 7, 'queue', 'worker', 1)
-        self.assertEqual(self.lua('throttle.locks', 8, 'ql:q:queue'), ['jid2'])
-        self.assertEqual(self.lua('jobs', 9, 'throttled', 'queue'), [])
+        self.assertEqual(self.lua('throttle.pending', 4, 'ql:q:queue'), ['jid2'])
+        self.assertEqual(self.lua('jobs', 5, 'throttled', 'queue'), ['jid2'])
+        self.assertEqual(self.lua('jobs', 5, 'running', 'queue'), ['jid1'])
+        self.lua('complete', 7, 'jid1', 'worker', 'queue', {})
+        # code.interact()
+        self.assertEqual(self.lua('jobs', 8, 'throttled', 'queue'), [])
+        self.lua('pop', 10, 'queue', 'worker', 1)
+        self.assertEqual(self.lua('throttle.locks',11, 'ql:q:queue'), ['jid2'])
+        self.assertEqual(self.lua('throttle.pending', 12, 'ql:q:queue'), [])
+        self.assertEqual(self.lua('jobs', 13, 'throttled', 'queue'), [])
+        self.assertEqual(self.lua('jobs', 5, 'running', 'queue'), ['jid2'])
 
     def test_throttled_additional_put(self):
         '''put should attempt to throttle the job immediately'''
diff --git a/test/test_throttle.py b/test/test_throttle.py
index 446a89a..13c8ff3 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -64,7 +64,7 @@ def test_limit_number_of_locks(self):
     self.lua('put', 3, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 0, 'queue', 'worker', 4)
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), ['jid2', 'jid3', 'jid4'])
+    self.assertEqual(self.lua('throttle.pending', 0, 'tid'), ['jid2', 'jid3', 'jid4'])
 
 class TestRelease(TestQless):
   '''Test that when there are no pending jobs lock is properly released'''
@@ -77,21 +77,27 @@ def test_no_pending_jobs(self):
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
     self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
-  '''Test that releasing a lock properly another job in the work queue'''
+  '''Test that releasing a lock properly inserts another job in the work queue'''
   def test_next_job_is_moved_into_work_qeueue(self):
     self.lua('throttle.set', 0, 'tid', 1)
     self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 2, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
-    self.lua('complete', 5, 'jid1', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.pending', 4, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 5, 'throttled', 'queue'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 6, 'running', 'queue'), ['jid1'])
+    self.lua('complete', 7, 'jid1', 'worker', 'queue', {})
     # Lock should be empty until another job is popped
-    self.assertEqual(self.lua('throttle.locks', 6, 'tid'), [])
-    self.assertEqual(self.lua('jobs', 7, 'throttled', 'queue'), ['jid2'])
-    self.lua('pop', 8, 'queue', 'worker', 1)
-    self.assertEqual(self.lua('throttle.locks', 9, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('throttle.locks', 8, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 9, 'tid'), [])
     self.assertEqual(self.lua('jobs', 10, 'throttled', 'queue'), [])
+    self.assertEqual(self.lua('jobs', 11, 'running', 'queue'), [])
+    self.lua('pop', 12, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 13, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('throttle.pending', 14, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 15, 'throttled', 'queue'), [])
+    self.assertEqual(self.lua('jobs', 16, 'running', 'queue'), ['jid2'])
 
 
   '''Test that cancelling a job properly adds another job in the work queue'''
@@ -101,14 +107,18 @@ def test_on_cancel_next_job_is_moved_into_work_queue(self):
     self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 2, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
-    self.lua('cancel', 5, 'jid1', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.pending', 4, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 5, 'throttled', 'queue'), ['jid2'])
+    self.lua('cancel', 6, 'jid1', 'worker', 'queue', {})
     # Lock should be empty until another job is popped
-    self.assertEqual(self.lua('throttle.locks', 6, 'tid'), [])
-    self.assertEqual(self.lua('jobs', 7, 'throttled', 'queue'), ['jid2'])
-    self.lua('pop', 8, 'queue', 'worker', 1)
-    self.assertEqual(self.lua('throttle.locks', 9, 'tid'), ['jid2'])
-    self.assertEqual(self.lua('jobs', 10, 'throttled', 'queue'), [])
+    self.assertEqual(self.lua('throttle.locks', 7, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 8, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 9, 'throttled', 'queue'), [])
+    self.lua('pop', 10, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 11, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('throttle.pending', 12, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 13, 'throttled', 'queue'), [])
+    self.assertEqual(self.lua('jobs', 14, 'running', 'queue'), ['jid2'])
 
 
   '''Test that when a job completes it properly releases the lock'''
@@ -163,7 +173,7 @@ def test_on_retry_no_pending_lock_is_reacquired(self):
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), ['jid'])
     self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
-  '''Test that when a job retries and another job is pending, the retrying job acquires the lock'''
+  '''Test that when a job retries and another job is pending, the pending job acquires the lock'''
   def test_on_retry_with_pending_lock_is_reacquired(self):
     # The retrying job will only re-acquire the lock if nothing is ahead of it in
     # the work queue that requires that lock
@@ -172,12 +182,16 @@ def test_on_retry_with_pending_lock_is_reacquired(self):
     self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 2, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
-    self.lua('retry', 5, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
-    self.assertEqual(self.lua('jobs', 6, 'throttled', 'queue'), ['jid2'])
-    self.lua('pop', 7, 'queue', 'worker', 2)
-    self.assertEqual(self.lua('throttle.locks', 8, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('jobs', 9, 'throttled', 'queue'), ['jid2'])
+    self.assertEqual(self.lua('throttle.pending', 4, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 5, 'throttled', 'queue'), ['jid2'])
+    self.lua('retry', 6, 'jid1', 'queue', 'worker', 0, 'retry', 'retrying')
+    self.assertEqual(self.lua('throttle.locks', 7, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 8, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 9, 'throttled', 'queue'), [])
+    self.lua('pop', 10, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 11, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('throttle.pending', 11, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('jobs', 12, 'throttled', 'queue'), ['jid1'])
 
 class TestDependents(TestQless):
   def test_dependencies_can_acquire_lock_after_dependent_success(self):
@@ -255,14 +269,16 @@ def test_increasing_concurrency_level_activates_pending_jobs(self):
     self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('put', 3, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttles', ['tid'])
-
     self.lua('pop', 4, 'queue', 'worker', 3)
     self.assertEqual(self.lua('throttle.locks', 5, 'tid'), ['jid1'])
     self.assertEqual(self.lua('jobs', 6, 'throttled', 'queue'), ['jid2', 'jid3'])
     self.lua('throttle.set', 7, 'tid', 3)
-    self.lua('pop', 8, 'queue', 'worker', 2)
-    self.assertEqual(self.lua('throttle.locks', 9, 'tid'), ['jid1', 'jid2', 'jid3'])
-    self.assertEqual(self.lua('jobs', 10, 'throttled', 'queue'), [])
+    self.lua('complete', 8, 'jid1', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 9, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 10, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 11, 'throttled', 'queue'), [])
+    self.lua('pop', 12, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('jobs', 13, 'running', 'queue'), ['jid2', 'jid3'])
 
   def test_reducing_concurrency_level_without_pending(self):
     '''Operates at reduced concurrency level after current jobs finish'''
@@ -272,28 +288,44 @@ def test_reducing_concurrency_level_without_pending(self):
     self.lua('put', 3, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('put', 4, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('put', 5, 'worker', 'queue', 'jid5', 'klass', {}, 0, 'throttles', ['tid'])
-
     self.lua('pop', 6, 'queue', 'worker', 3)
     self.assertEqual(self.lua('throttle.locks', 7, 'tid'), ['jid1', 'jid2', 'jid3'])
     self.assertEqual(self.lua('jobs', 8, 'throttled', 'queue'), [])
     self.lua('throttle.set', 9, 'tid', 1)
     self.lua('pop', 10, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 11, 'tid'), ['jid1', 'jid2', 'jid3'])
-    self.assertEqual(self.lua('jobs', 12, 'throttled', 'queue'), ['jid4', 'jid5'])
-    self.lua('complete', 13, 'jid1', 'worker', 'queue', {})
-    self.assertEqual(self.lua('throttle.locks', 14, 'tid'), ['jid2', 'jid3'])
-    self.assertEqual(self.lua('jobs', 15, 'throttled', 'queue'), ['jid4', 'jid5'])
-    self.lua('complete', 16, 'jid2', 'worker', 'queue', {})
-    self.assertEqual(self.lua('throttle.locks', 17, 'tid'), ['jid3'])
+    self.assertEqual(self.lua('throttle.pending', 12, 'tid'), ['jid4', 'jid5'])
+    self.assertEqual(self.lua('jobs', 13, 'throttled', 'queue'), ['jid4', 'jid5'])
+    self.assertEqual(self.lua('jobs', 14, 'running', 'queue'), ['jid1', 'jid2', 'jid3'])
+    self.lua('complete', 15, 'jid1', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 16, 'tid'), ['jid2', 'jid3'])
+    self.assertEqual(self.lua('throttle.pending', 17, 'tid'), ['jid4', 'jid5'])
     self.assertEqual(self.lua('jobs', 18, 'throttled', 'queue'), ['jid4', 'jid5'])
-    self.lua('complete', 19, 'jid3', 'worker', 'queue', {})
-    self.lua('pop', 20, 'queue', 'worker', 1)
-    self.assertEqual(self.lua('throttle.locks', 21, 'tid'), ['jid4'])
-    self.assertEqual(self.lua('jobs', 22, 'throttled', 'queue'), ['jid5'])
-    self.lua('complete', 24, 'jid4', 'worker', 'queue', {})
-    self.lua('pop', 23, 'queue', 'worker', 2)
-    self.assertEqual(self.lua('throttle.locks', 25, 'tid'), ['jid5'])
-    self.assertEqual(self.lua('jobs', 26, 'throttled', 'queue'), [])
+    self.assertEqual(self.lua('jobs', 19, 'running', 'queue'), ['jid2', 'jid3'])
+    self.lua('complete', 20, 'jid2', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 21, 'tid'), ['jid3'])
+    self.assertEqual(self.lua('throttle.pending', 22, 'tid'), ['jid4', 'jid5'])
+    self.assertEqual(self.lua('jobs', 23, 'throttled', 'queue'), ['jid4', 'jid5'])
+    self.lua('complete', 24, 'jid3', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 25, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 26, 'tid'), ['jid5'])
+    self.assertEqual(self.lua('jobs', 27, 'throttled', 'queue'), ['jid5'])
+    self.assertEqual(self.lua('jobs', 28, 'running', 'queue'), [])
+    self.lua('pop', 29, 'queue', 'worker', 1)
+    self.assertEqual(self.lua('throttle.locks', 30, 'tid'), ['jid4'])
+    self.assertEqual(self.lua('throttle.pending', 31, 'tid'), ['jid5'])
+    self.assertEqual(self.lua('jobs', 32, 'throttled', 'queue'), ['jid5'])
+    self.assertEqual(self.lua('jobs', 33, 'running', 'queue'), ['jid4'])
+    self.lua('complete', 34, 'jid4', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 35, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 36, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 37, 'throttled', 'queue'), [])
+    self.assertEqual(self.lua('jobs', 38, 'running', 'queue'), [])
+    self.lua('pop', 39, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 40, 'tid'), ['jid5'])
+    self.assertEqual(self.lua('throttle.pending', 41, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 42, 'throttled', 'queue'), [])
+    self.assertEqual(self.lua('jobs', 43, 'running', 'queue'), ['jid5'])
 
   def test_reducing_concurrency_level_with_pending(self):
     '''Operates at reduced concurrency level after current jobs finish'''
diff --git a/throttle.lua b/throttle.lua
index 761bfb3..f318a86 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -37,9 +37,38 @@ function QlessThrottle:acquire(jid)
   return true
 end
 
--- Release a throttled resource.
+function QlessThrottle:pend(now, jid)
+  self.pending.add(now, jid)
+end
+
+-- Releases the lock taken by the specified jid.
+-- number of jobs released back into the queues is determined by the locks_available method.
 function QlessThrottle:release(now, jid)
+  redis.call('set', 'printline', self.id .. ' jid : ' .. jid .. ' removed from locks')
   self.locks.remove(jid)
+
+  local available_locks = self:locks_available()
+  redis.call('set', 'printline', self.id .. ' pending count ' .. self.pending.length() .. ' available_locks ' .. available_locks)
+  if self.pending.length() == 0 or available_locks < 1 then
+    return
+  end
+
+  -- subtract one to ensure we pop the correct amount. peek(0, 0) returns the first element
+  -- peek(0,1) return the first two.
+  for _, jid in ipairs(self.pending.peek(0, available_locks - 1)) do
+    redis.call('set', 'printline', self.id .. ' adding ' .. jid .. ' to work queue')
+    local job = Qless.job(jid)
+    local data = job:data()
+    local queue = Qless.queue(data['queue'])
+
+    queue.throttled.remove(jid)
+    queue.work.add(now, data.priority, jid)
+  end
+
+  -- subtract one to ensure we pop the correct amount. pop(0, 0) pops the first element
+  -- pop(0,1) pops the first two.
+  local popped = self.pending.pop(0, available_locks - 1)
+  redis.call('set', 'printline', self.id .. ' popped ' .. popped .. ' jids')
 end
 
 -- Returns true if the throttle has locks available, false otherwise.
@@ -51,3 +80,15 @@ end
 function QlessThrottle:ttl()
   return redis.call('ttl', QlessThrottle.ns .. self.id)
 end
+
+-- Returns the number of locks available for the throttle.
+-- calculated by maximum - locks.length(), if the throttle is unlimited
+-- then up to 10 jobs are released.
+function QlessThrottle:locks_available()
+  if self.maximum == 0 then
+    -- Arbitrarily chosen value. might want to make it configurable in the future.
+    return 10
+  end
+
+  return self.maximum - self.locks.length()
+end

From d7372ab2094b2c39086989ffef366cdbddb135a5 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Wed, 2 Apr 2014 07:48:05 -0400
Subject: [PATCH 51/72] removed printline statements

---
 job.lua            | 2 --
 test/test_queue.py | 2 +-
 throttle.lua       | 4 ----
 3 files changed, 1 insertion(+), 7 deletions(-)

diff --git a/job.lua b/job.lua
index 5cc7597..3d9045a 100644
--- a/job.lua
+++ b/job.lua
@@ -799,7 +799,6 @@ function QlessJob:throttles_release(now)
   throttles = cjson.decode(throttles or '[]')
 
   for _, tid in ipairs(throttles) do
-    redis.call('set', 'printline', self.jid .. ' releasing throttle ' .. 'tid')
     Qless.throttle(tid):release(now, self.jid)
   end
 end
@@ -831,7 +830,6 @@ function QlessJob:throttle(now)
   for _, tid in ipairs(self:throttles()) do
     local throttle = Qless.throttle(tid)
     if not throttle:available() then
-      redis.call('set', 'printline', 'pending ' .. self.jid .. ' for throttle ' .. tid)
       throttle:pend(now, self.jid)
       return
     end
diff --git a/test/test_queue.py b/test/test_queue.py
index fabe6bc..9c45d73 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -274,7 +274,7 @@ class TestPut(TestQless):
     '''Test putting jobs into a queue'''
     # For reference:
     #
-    #   Put(now, jid, klass, data, delay,
+    #   put(now, jid, klass, data, delay,
     #       [priority, p],
     #       [tags, t],
     #       [retries, r],
diff --git a/throttle.lua b/throttle.lua
index f318a86..f424cc3 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -44,11 +44,9 @@ end
 -- Releases the lock taken by the specified jid.
 -- number of jobs released back into the queues is determined by the locks_available method.
 function QlessThrottle:release(now, jid)
-  redis.call('set', 'printline', self.id .. ' jid : ' .. jid .. ' removed from locks')
   self.locks.remove(jid)
 
   local available_locks = self:locks_available()
-  redis.call('set', 'printline', self.id .. ' pending count ' .. self.pending.length() .. ' available_locks ' .. available_locks)
   if self.pending.length() == 0 or available_locks < 1 then
     return
   end
@@ -56,7 +54,6 @@ function QlessThrottle:release(now, jid)
   -- subtract one to ensure we pop the correct amount. peek(0, 0) returns the first element
   -- peek(0,1) return the first two.
   for _, jid in ipairs(self.pending.peek(0, available_locks - 1)) do
-    redis.call('set', 'printline', self.id .. ' adding ' .. jid .. ' to work queue')
     local job = Qless.job(jid)
     local data = job:data()
     local queue = Qless.queue(data['queue'])
@@ -68,7 +65,6 @@ function QlessThrottle:release(now, jid)
   -- subtract one to ensure we pop the correct amount. pop(0, 0) pops the first element
   -- pop(0,1) pops the first two.
   local popped = self.pending.pop(0, available_locks - 1)
-  redis.call('set', 'printline', self.id .. ' popped ' .. popped .. ' jids')
 end
 
 -- Returns true if the throttle has locks available, false otherwise.

From f1bf59f0b93c939ddebedce6a2f8eb5cafa1a3d0 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Wed, 2 Apr 2014 07:59:37 -0400
Subject: [PATCH 52/72] removed debugging code

---
 test/test_queue.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_queue.py b/test/test_queue.py
index 9c45d73..9b37250 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -675,7 +675,6 @@ def test_throttled_removed(self):
         self.assertEqual(self.lua('jobs', 5, 'throttled', 'queue'), ['jid2'])
         self.assertEqual(self.lua('jobs', 5, 'running', 'queue'), ['jid1'])
         self.lua('complete', 7, 'jid1', 'worker', 'queue', {})
-        # code.interact()
         self.assertEqual(self.lua('jobs', 8, 'throttled', 'queue'), [])
         self.lua('pop', 10, 'queue', 'worker', 1)
         self.assertEqual(self.lua('throttle.locks',11, 'ql:q:queue'), ['jid2'])

From 3108245a22bf30415f9f3db85059d238ef35c4b0 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 12 May 2014 09:28:03 -0400
Subject: [PATCH 53/72] fixes throttle release to properly remove a job from
 the throttled set

---
 test/test_throttle.py | 13 ++++++++++++-
 throttle.lua          |  1 +
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/test/test_throttle.py b/test/test_throttle.py
index 13c8ff3..2d73dba 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -77,6 +77,18 @@ def test_no_pending_jobs(self):
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
     self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
+  '''Test that releasing a pending throttled job correctly cleans up'''
+  def test_on_release_pending_job_is_removed_from_set(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('pop', 3, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 4, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 5, 'tid'), ['jid2'])
+    self.lua('cancel', 6, 'jid2', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 7, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 8, 'tid'), [])
+
   '''Test that releasing a lock properly inserts another job in the work queue'''
   def test_next_job_is_moved_into_work_qeueue(self):
     self.lua('throttle.set', 0, 'tid', 1)
@@ -120,7 +132,6 @@ def test_on_cancel_next_job_is_moved_into_work_queue(self):
     self.assertEqual(self.lua('jobs', 13, 'throttled', 'queue'), [])
     self.assertEqual(self.lua('jobs', 14, 'running', 'queue'), ['jid2'])
 
-
   '''Test that when a job completes it properly releases the lock'''
   def test_on_complete_lock_is_released(self):
     self.lua('throttle.set', 0, 'tid', 1)
diff --git a/throttle.lua b/throttle.lua
index f424cc3..6fa7721 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -45,6 +45,7 @@ end
 -- number of jobs released back into the queues is determined by the locks_available method.
 function QlessThrottle:release(now, jid)
   self.locks.remove(jid)
+  self.pending.remove(jid)
 
   local available_locks = self:locks_available()
   if self.pending.length() == 0 or available_locks < 1 then

From d05837436828199963653e86d3d51a0325ae1bb7 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 12 May 2014 09:35:44 -0400
Subject: [PATCH 54/72] small optimization

---
 throttle.lua | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/throttle.lua b/throttle.lua
index 6fa7721..8216991 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -44,8 +44,11 @@ end
 -- Releases the lock taken by the specified jid.
 -- number of jobs released back into the queues is determined by the locks_available method.
 function QlessThrottle:release(now, jid)
-  self.locks.remove(jid)
-  self.pending.remove(jid)
+  -- Only attempt to remove from the pending set if the job wasn't found in the
+  -- locks set
+  if self.locks.remove(jid) == 0 then
+    self.pending.remove(jid)
+  end
 
   local available_locks = self:locks_available()
   if self.pending.length() == 0 or available_locks < 1 then

From ee6f039090ff0b753532ee6bed18abcc90c66946 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 12 May 2014 09:40:42 -0400
Subject: [PATCH 55/72] test name change

---
 test/test_throttle.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_throttle.py b/test/test_throttle.py
index 2d73dba..4f01623 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -78,7 +78,7 @@ def test_no_pending_jobs(self):
     self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
   '''Test that releasing a pending throttled job correctly cleans up'''
-  def test_on_release_pending_job_is_removed_from_set(self):
+  def test_on_release_pending_job_is_removed_from_throttle(self):
     self.lua('throttle.set', 0, 'tid', 1)
     self.lua('put', 1, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])

From 42aa9beeabfbca768f5c5a899e1e71be973a4055 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 12 May 2014 09:43:32 -0400
Subject: [PATCH 56/72] whitespace fix

---
 test/test_throttle.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/test_throttle.py b/test/test_throttle.py
index 4f01623..f155275 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -111,7 +111,6 @@ def test_next_job_is_moved_into_work_qeueue(self):
     self.assertEqual(self.lua('jobs', 15, 'throttled', 'queue'), [])
     self.assertEqual(self.lua('jobs', 16, 'running', 'queue'), ['jid2'])
 
-
   '''Test that cancelling a job properly adds another job in the work queue'''
   def test_on_cancel_next_job_is_moved_into_work_queue(self):
     self.lua('throttle.set', 0, 'tid', 1)

From d9903a5bac0e765cae0377f9b03ea753eb45bd35 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 3 Jun 2014 09:31:26 -0400
Subject: [PATCH 57/72] fix tags work

---
 base.lua         | 39 +++++++-----------------------
 job.lua          | 63 ++++++++++++++++++++++++++++++++++++------------
 queue.lua        |  6 ++---
 test/test_job.py | 16 +++++++++++-
 throttle.lua     | 17 +++++++------
 5 files changed, 83 insertions(+), 58 deletions(-)

diff --git a/base.lua b/base.lua
index ba2ff7a..59c36e1 100644
--- a/base.lua
+++ b/base.lua
@@ -72,12 +72,6 @@ end
 function Qless.throttle(tid)
   assert(tid, 'Throttle(): no tid provided')
   local throttle = QlessThrottle.data({id = tid})
-  if not throttle then
-    throttle = {
-      id = tid,
-      maximum = 0
-    }
-  end
   setmetatable(throttle, QlessThrottle)
 
   -- set of jids which have acquired a lock on this throttle.
@@ -324,12 +318,10 @@ function Qless.tag(now, command, ...)
           _tags[tag] = true
           table.insert(tags, tag)
         end
-        redis.call('zadd', 'ql:t:' .. tag, now, jid)
-        redis.call('zincrby', 'ql:tags', 1, tag)
+        Qless.job(jid):insert_tag(now, tag)
       end
 
-      tags = cjson.encode(tags)
-      redis.call('hset', QlessJob.ns .. jid, 'tags', tags)
+      redis.call('hset', QlessJob.ns .. jid, 'tags', cjson.encode(tags))
       return tags
     else
       error('Tag(): Job ' .. jid .. ' does not exist')
@@ -348,15 +340,13 @@ function Qless.tag(now, command, ...)
       for i=2,#arg do
         local tag = arg[i]
         _tags[tag] = nil
-        redis.call('zrem', 'ql:t:' .. tag, jid)
-        redis.call('zincrby', 'ql:tags', -1, tag)
+        Qless.job(jid):remove_tag(tag)
       end
 
       local results = {}
       for i,tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
 
-      tags = cjson.encode(results)
-      redis.call('hset', QlessJob.ns .. jid, 'tags', tags)
+      redis.call('hset', QlessJob.ns .. jid, 'tags', cjson.encode(results))
       return results
     else
       error('Tag(): Job ' .. jid .. ' does not exist')
@@ -369,7 +359,7 @@ function Qless.tag(now, command, ...)
       'Tag(): Arg "count" not a number: ' .. tostring(arg[3]))
     return {
       total = redis.call('zcard', 'ql:t:' .. tag),
-      jobs  = redis.call('zrange', 'ql:t:' .. tag, offset, offset + count - 1)
+      jobs  = redis.call('zrange', 'ql:t:' .. tag, offset, offset + count - 1) or cjson.decode('[]')
     }
   elseif command == 'top' then
     local offset = assert(tonumber(arg[1] or 0) , 'Tag(): Arg "offset" not a number: ' .. tostring(arg[1]))
@@ -434,7 +424,9 @@ function Qless.cancel(now, ...)
         queue:remove_job(jid)
       end
 
-      Qless.job(jid):throttles_release(now)
+      local job = Qless.job(jid)
+
+      job:throttles_release(now)
 
       -- We should probably go through all our dependencies and remove
       -- ourselves from the list of dependents
@@ -443,9 +435,6 @@ function Qless.cancel(now, ...)
         redis.call('srem', QlessJob.ns .. j .. '-dependents', jid)
       end
 
-      -- Delete any notion of dependencies it has
-      redis.call('del', QlessJob.ns .. jid .. '-dependencies')
-
       -- If we're in the failed state, remove all of our data
       if state == 'failed' then
         failure = cjson.decode(failure)
@@ -463,22 +452,12 @@ function Qless.cancel(now, ...)
           'ql:s:stats:' .. bin .. ':' .. queue, 'failed', failed - 1)
       end
 
-      -- Remove it as a job that's tagged with this particular tag
-      local tags = cjson.decode(
-        redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}')
-      for i, tag in ipairs(tags) do
-        redis.call('zrem', 'ql:t:' .. tag, jid)
-        redis.call('zincrby', 'ql:tags', -1, tag)
-      end
+      job:delete()
 
       -- If the job was being tracked, we should notify
       if redis.call('zscore', 'ql:tracked', jid) ~= false then
         Qless.publish('canceled', jid)
       end
-
-      -- Just go ahead and delete our data
-      redis.call('del', QlessJob.ns .. jid)
-      redis.call('del', QlessJob.ns .. jid .. '-history')
     end
   end
 
diff --git a/job.lua b/job.lua
index 3d9045a..fa74092 100644
--- a/job.lua
+++ b/job.lua
@@ -240,29 +240,16 @@ function QlessJob:complete(now, worker, queue, data, ...)
     local jids = redis.call('zrangebyscore', 'ql:completed', 0, now - time)
     -- Any jobs that need to be expired... delete
     for index, jid in ipairs(jids) do
-      local tags = cjson.decode(
-        redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}')
-      for i, tag in ipairs(tags) do
-        redis.call('zrem', 'ql:t:' .. tag, jid)
-        redis.call('zincrby', 'ql:tags', -1, tag)
-      end
-      redis.call('del', QlessJob.ns .. jid)
-      redis.call('del', QlessJob.ns .. jid .. '-history')
+      Qless.job(jid):delete()
     end
+
     -- And now remove those from the queued-for-cleanup queue
     redis.call('zremrangebyscore', 'ql:completed', 0, now - time)
 
     -- Now take the all by the most recent 'count' ids
     jids = redis.call('zrange', 'ql:completed', 0, (-1-count))
     for index, jid in ipairs(jids) do
-      local tags = cjson.decode(
-        redis.call('hget', QlessJob.ns .. jid, 'tags') or '{}')
-      for i, tag in ipairs(tags) do
-        redis.call('zrem', 'ql:t:' .. tag, jid)
-        redis.call('zincrby', 'ql:tags', -1, tag)
-      end
-      redis.call('del', QlessJob.ns .. jid)
-      redis.call('del', QlessJob.ns .. jid .. '-history')
+      Qless.job(jid):delete()
     end
     redis.call('zremrangebyrank', 'ql:completed', 0, (-1-count))
 
@@ -844,3 +831,47 @@ function QlessJob:throttles()
 
   return self._throttles
 end
+
+-- Completely removes all the data
+-- associated with this job, use
+-- with care.
+function QlessJob:delete()
+  local tags = redis.call('hget', QlessJob.ns .. self.jid, 'tags') or '[]'
+  tags = cjson.decode(tags)
+  -- remove the jid from each tag
+  for i, tag in ipairs(tags) do
+    self:remove_tag(tag)
+  end
+  -- Delete the job's data
+  redis.call('del', QlessJob.ns .. self.jid)
+  -- Delete the job's history
+  redis.call('del', QlessJob.ns .. self.jid .. '-history')
+  -- Delete any notion of dependencies it has
+  redis.call('del', QlessJob.ns .. self.jid .. '-dependencies')
+end
+
+-- Inserts the jid into the specified tag.
+-- This should probably be moved to its own tag
+-- object.
+function QlessJob:insert_tag(now, tag)
+  redis.call('zadd', 'ql:t:' .. tag, now, self.jid)
+  redis.call('zincrby', 'ql:tags', 1, tag)
+end
+
+-- Removes the jid from the specified tag.
+-- this should probably be moved to its own tag
+-- object.
+function QlessJob:remove_tag(tag)
+  -- Remove the job from the specified tag
+  redis.call('zrem', 'ql:t:' .. tag, self.jid)
+
+  -- Decrement the tag in the set of all tags.
+  local score = redis.call('zincrby', 'ql:tags', -1, tag)
+
+  -- if the score for the specified tag is 0
+  -- it means we have no jobs with this tag anymore
+  -- and we should remove it from the set to prevent memory leaks.
+  if score == 0 then
+    redis.call('zrem', 'ql:tags', tag)
+  end
+end
diff --git a/queue.lua b/queue.lua
index 07fa3ce..0cd027c 100644
--- a/queue.lua
+++ b/queue.lua
@@ -550,8 +550,7 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
 
   -- Add this job to the list of jobs tagged with whatever tags were supplied
   for i, tag in ipairs(tags) do
-    redis.call('zadd', 'ql:t:' .. tag, now, jid)
-    redis.call('zincrby', 'ql:tags', 1, tag)
+    Qless.job(jid):insert_tag(now, tag)
   end
 
   -- If we're in the failed state, remove all of our data
@@ -820,8 +819,7 @@ function QlessQueue:check_recurring(now, count)
       -- Add this job to the list of jobs tagged with whatever tags were
       -- supplied
       for i, tag in ipairs(_tags) do
-        redis.call('zadd', 'ql:t:' .. tag, now, jid .. '-' .. count)
-        redis.call('zincrby', 'ql:tags', 1, tag)
+        Qless.job(jid .. '-' .. count):insert_tag(now, tag)
       end
 
       -- First, let's save its data
diff --git a/test/test_job.py b/test/test_job.py
index 53dbb35..f02e872 100644
--- a/test/test_job.py
+++ b/test/test_job.py
@@ -40,7 +40,6 @@ def test_history(self):
             {'q': 'queue', 'what': 'put', 'when': 98},
             {'q': 'queue', 'what': 'put', 'when': 99}])
 
-
 class TestComplete(TestQless):
     '''Test how we complete jobs'''
     def test_malformed(self):
@@ -183,6 +182,21 @@ def test_expire_complete_time(self):
         existing = [self.lua('get', 3, jid) for jid in range(10)]
         self.assertEqual([i for i in existing if i], [])
 
+    def test_expire_complete_tags_cleared(self):
+        '''Tag's should be removed once they no longer have any jobs'''
+        # Set all jobs to expire immediately
+        self.lua('config.set', 1, 'jobs-history', -1)
+        # When cancelled
+        self.lua('put', 2, 'worker', 'queue', 'jid', 'klass', {}, 0, 'tags', ['abc'])
+        self.assertEqual(self.lua('tag', 3, 'get', 'abc', 0, 0)['jobs'], ['jid'])
+        self.lua('cancel', 4, 'jid', 'worker', 'queue', {})
+        self.assertEqual(self.lua('tag', 5, 'get', 'abc', 0, 0)['jobs'], {})
+        # When complete
+        self.lua('put', 6, 'worker', 'queue', 'jid', 'klass', {}, 0, 'tags', ['abc'])
+        self.assertEqual(self.lua('tag', 7, 'get', 'abc', 0, 0)['jobs'], ['jid'])
+        self.lua('pop', 8, 'queue', 'worker', 1)
+        self.lua('complete', 9, 'jid', 'worker', 'queue', {})
+        self.assertEqual(self.lua('tag', 10, 'get', 'abc', 0, 0)['jobs'], {})
 
 class TestCancel(TestQless):
     '''Canceling jobs'''
diff --git a/throttle.lua b/throttle.lua
index 8216991..445f218 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -1,15 +1,18 @@
 -- Retrieve the data fro a throttled resource
 function QlessThrottle:data()
+  -- Default values for the data
+  local data = {
+    id = self.id,
+    maximum = 0
+  }
+
+  -- Retrieve data stored in redis
   local throttle = redis.call('hmget', QlessThrottle.ns .. self.id, 'id', 'maximum')
-  -- Return default if it doesn't exist
-  if not throttle[1] then
-    return {id = self.id, maximum = 0}
+
+  if throttle[2] then
+    data.maximum = tonumber(throttle[2])
   end
 
-  local data = {
-    id = throttle[1],
-    maximum = tonumber(throttle[2])
-  }
   return data
 end
 

From 56df3fc93da717fbb7a64ade1baaaeb17c376df2 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 3 Jun 2014 09:42:33 -0400
Subject: [PATCH 58/72] test fix

---
 job.lua          | 2 +-
 test/test_job.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/job.lua b/job.lua
index fa74092..1a38d0e 100644
--- a/job.lua
+++ b/job.lua
@@ -871,7 +871,7 @@ function QlessJob:remove_tag(tag)
   -- if the score for the specified tag is 0
   -- it means we have no jobs with this tag anymore
   -- and we should remove it from the set to prevent memory leaks.
-  if score == 0 then
+  if tonumber(score) == 0 then
     redis.call('zrem', 'ql:tags', tag)
   end
 end
diff --git a/test/test_job.py b/test/test_job.py
index f02e872..d60ff21 100644
--- a/test/test_job.py
+++ b/test/test_job.py
@@ -3,7 +3,6 @@
 import redis
 from common import TestQless
 
-
 class TestJob(TestQless):
     '''Some general jobby things'''
     def test_malformed(self):
@@ -191,12 +190,14 @@ def test_expire_complete_tags_cleared(self):
         self.assertEqual(self.lua('tag', 3, 'get', 'abc', 0, 0)['jobs'], ['jid'])
         self.lua('cancel', 4, 'jid', 'worker', 'queue', {})
         self.assertEqual(self.lua('tag', 5, 'get', 'abc', 0, 0)['jobs'], {})
+        self.assertEqual(self.redis.zrange('ql:tags', 0, -1), [])
         # When complete
         self.lua('put', 6, 'worker', 'queue', 'jid', 'klass', {}, 0, 'tags', ['abc'])
         self.assertEqual(self.lua('tag', 7, 'get', 'abc', 0, 0)['jobs'], ['jid'])
         self.lua('pop', 8, 'queue', 'worker', 1)
         self.lua('complete', 9, 'jid', 'worker', 'queue', {})
         self.assertEqual(self.lua('tag', 10, 'get', 'abc', 0, 0)['jobs'], {})
+        self.assertEqual(self.redis.zrange('ql:tags', 0, -1), [])
 
 class TestCancel(TestQless):
     '''Canceling jobs'''

From bd089c0d61a492036b6601a3e15c3fd4df84cd19 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 3 Jun 2014 09:49:57 -0400
Subject: [PATCH 59/72] removed unnecessary code

---
 base.lua | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/base.lua b/base.lua
index 59c36e1..f17e46c 100644
--- a/base.lua
+++ b/base.lua
@@ -346,7 +346,8 @@ function Qless.tag(now, command, ...)
       local results = {}
       for i,tag in ipairs(tags) do if _tags[tag] then table.insert(results, tag) end end
 
-      redis.call('hset', QlessJob.ns .. jid, 'tags', cjson.encode(results))
+      tags = cjson.encode(results)
+      redis.call('hset', QlessJob.ns .. jid, 'tags', tags)
       return results
     else
       error('Tag(): Job ' .. jid .. ' does not exist')
@@ -359,7 +360,7 @@ function Qless.tag(now, command, ...)
       'Tag(): Arg "count" not a number: ' .. tostring(arg[3]))
     return {
       total = redis.call('zcard', 'ql:t:' .. tag),
-      jobs  = redis.call('zrange', 'ql:t:' .. tag, offset, offset + count - 1) or cjson.decode('[]')
+      jobs  = redis.call('zrange', 'ql:t:' .. tag, offset, offset + count - 1)
     }
   elseif command == 'top' then
     local offset = assert(tonumber(arg[1] or 0) , 'Tag(): Arg "offset" not a number: ' .. tostring(arg[1]))

From ed4a0f85bbeaa8af682b46c46dfc07495e5ec060 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Tue, 3 Jun 2014 09:59:45 -0400
Subject: [PATCH 60/72] removed unnecessary change

---
 base.lua | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/base.lua b/base.lua
index f17e46c..42317ed 100644
--- a/base.lua
+++ b/base.lua
@@ -321,7 +321,8 @@ function Qless.tag(now, command, ...)
         Qless.job(jid):insert_tag(now, tag)
       end
 
-      redis.call('hset', QlessJob.ns .. jid, 'tags', cjson.encode(tags))
+      tags = cjson.encode(tags)
+      redis.call('hset', QlessJob.ns .. jid, 'tags', tags)
       return tags
     else
       error('Tag(): Job ' .. jid .. ' does not exist')

From 3550b9b2c051ea3fe251796efb8fbf221d8c75bc Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Wed, 4 Jun 2014 08:52:29 -0400
Subject: [PATCH 61/72] patches remove_tag method to be more reliable

---
 base.lua |  2 +-
 job.lua  | 23 +++++++++++++++--------
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/base.lua b/base.lua
index 42317ed..645e176 100644
--- a/base.lua
+++ b/base.lua
@@ -337,7 +337,7 @@ function Qless.tag(now, command, ...)
       local _tags = {}
       for i,v in ipairs(tags) do _tags[v] = true end
 
-      -- Otherwise, add the job to the sorted set with that tags
+      -- Otherwise, remove the job from the sorted set with that tags
       for i=2,#arg do
         local tag = arg[i]
         _tags[tag] = nil
diff --git a/job.lua b/job.lua
index 1a38d0e..e08f1d0 100644
--- a/job.lua
+++ b/job.lua
@@ -390,7 +390,7 @@ function QlessJob:fail(now, worker, group, message, data)
   -- And add this particular instance to the failed groups
   redis.call('lpush', 'ql:f:' .. group, self.jid)
 
-  -- Here is where we'd intcrement stats about the particular stage
+  -- Here is where we'd increment stats about the particular stage
   -- and possibly the workers
 
   return self.jid
@@ -862,16 +862,23 @@ end
 -- this should probably be moved to its own tag
 -- object.
 function QlessJob:remove_tag(tag)
+  -- namespace the tag
+  local namespaced_tag = 'ql:t:' .. tag
+
   -- Remove the job from the specified tag
-  redis.call('zrem', 'ql:t:' .. tag, self.jid)
+  redis.call('zrem', namespaced_tag, self.jid)
 
-  -- Decrement the tag in the set of all tags.
-  local score = redis.call('zincrby', 'ql:tags', -1, tag)
+  -- Check if any tags jids remain in the tag set.
+  local remaining = redis.call('zcard', namespaced_tag)
 
-  -- if the score for the specified tag is 0
-  -- it means we have no jobs with this tag anymore
-  -- and we should remove it from the set to prevent memory leaks.
-  if tonumber(score) == 0 then
+  -- If the number of jids in the tagged set
+  -- is 0 it means we have no jobs with this tag
+  -- and we should remove it from the set of all tags
+  -- to prevent memory leaks.
+  if tonumber(remaining) == 0 then
     redis.call('zrem', 'ql:tags', tag)
+  else
+    -- Decrement the tag in the set of all tags.
+    redis.call('zincrby', 'ql:tags', -1, tag)
   end
 end

From 3fe1fcdfeb0496d452d5f22826ab476a0051ee59 Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Fri, 8 Aug 2014 14:26:04 -0400
Subject: [PATCH 62/72] merged master

---
 test/test_recurring.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/test/test_recurring.py b/test/test_recurring.py
index c8e48f5..3425b4e 100644
--- a/test/test_recurring.py
+++ b/test/test_recurring.py
@@ -262,13 +262,9 @@ def test_rerecur_attributes(self):
             'state': 'running',
             'tags': ['foo'],
             'tracked': False,
-<<<<<<< HEAD
             'throttles': {},
-            'worker': 'worker'})
-=======
             'worker': 'worker',
             'spawned_from_jid': 'jid'})
->>>>>>> upstream-master
         self.lua('recur', 60, 'queue', 'jid', 'class', {'foo': 'bar'},
             'interval', 10, 0, 'priority', 5, 'tags', ['bar'], 'retries', 5, 'throttles', ['lala'])
         self.assertEqual(self.lua('pop', 60, 'queue', 'worker', 10)[0], {
@@ -288,13 +284,9 @@ def test_rerecur_attributes(self):
             'state': 'running',
             'tags': ['bar'],
             'tracked': False,
-<<<<<<< HEAD
             'throttles': ['lala'],
-            'worker': 'worker'})
-=======
             'worker': 'worker',
             'spawned_from_jid': 'jid'})
->>>>>>> upstream-master
 
     def test_rerecur_move(self):
         '''Re-recurring a job in a new queue works like a move'''

From cda5ed855a6caa1fb050ae657aa604ac64528724 Mon Sep 17 00:00:00 2001
From: james-lawrence <james-lawrence@users.noreply.github.com>
Date: Wed, 26 Nov 2014 18:47:26 -0500
Subject: [PATCH 63/72] Update throttle.lua

---
 throttle.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/throttle.lua b/throttle.lua
index 445f218..d388564 100644
--- a/throttle.lua
+++ b/throttle.lua
@@ -1,4 +1,4 @@
--- Retrieve the data fro a throttled resource
+-- Retrieve the data for a throttled resource
 function QlessThrottle:data()
   -- Default values for the data
   local data = {

From b6fd1aa8755ca3df9361e17d613dc122349e06db Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Mon, 4 May 2015 10:14:20 -0400
Subject: [PATCH 64/72] test fixes

---
 test/test_job.py      | 18 ++++++++++++++++++
 test/test_throttle.py | 26 ++++++++++++++++++++------
 2 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/test/test_job.py b/test/test_job.py
index 464a146..eb78c70 100644
--- a/test/test_job.py
+++ b/test/test_job.py
@@ -53,6 +53,24 @@ def test_requeue_cancelled_job(self):
         self.assertRaisesRegexp(redis.ResponseError, r'does not exist',
             self.lua, 'requeue', 2, 'worker', 'queue-2', 'jid', 'klass', {}, 0)
 
+    def test_requeue_throttled_job(self):
+        '''Requeueing  a throttled job should maintain correct state'''
+        self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
+        original = self.lua('multiget', 1, 'jid')[0]
+        self.lua('requeue', 2, 'worker', 'queue-2', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
+        updated = self.lua('multiget', 3, 'jid')[0]
+
+        # throttles and queue change during requeue
+        self.assertEqual(updated['throttles'], ['tid', 'ql:q:queue-2'])
+        self.assertEqual(updated['queue'], 'queue-2')
+        del updated['throttles']
+        del updated['queue']
+        del updated['history']
+        del original['throttles']
+        del original['queue']
+        del original['history']
+        self.assertEqual(updated, original)
+
 class TestComplete(TestQless):
     '''Test how we complete jobs'''
     def test_malformed(self):
diff --git a/test/test_throttle.py b/test/test_throttle.py
index f155275..b39a175 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -2,6 +2,7 @@
 
 import redis
 import code
+
 from common import TestQless
 
 class TestThrottle(TestQless):
@@ -172,8 +173,8 @@ def test_on_retry_lock_is_reacquired(self):
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
     self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
-  '''Test that when a job retries and no pending jobs it acquires the lock again on next pop'''
-  def test_on_retry_no_pending_lock_is_reacquired(self):
+  '''Test that when a job retries and has no pending jobs it acquires the lock again on next pop'''
+  def test_on_retry_without_pending_lock_is_reacquired(self):
     self.lua('throttle.set', 0, 'tid', 1)
     self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {}, 0, 'throttles', ['tid'])
     self.lua('pop', 0, 'queue', 'worker', 1)
@@ -184,7 +185,7 @@ def test_on_retry_no_pending_lock_is_reacquired(self):
     self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
   '''Test that when a job retries and another job is pending, the pending job acquires the lock'''
-  def test_on_retry_with_pending_lock_is_reacquired(self):
+  def test_on_retry_with_pending_lock_is_not_reacquired(self):
     # The retrying job will only re-acquire the lock if nothing is ahead of it in
     # the work queue that requires that lock
     self.lua('throttle.set', 0, 'tid', 1)
@@ -200,8 +201,22 @@ def test_on_retry_with_pending_lock_is_reacquired(self):
     self.assertEqual(self.lua('jobs', 9, 'throttled', 'queue'), [])
     self.lua('pop', 10, 'queue', 'worker', 2)
     self.assertEqual(self.lua('throttle.locks', 11, 'tid'), ['jid2'])
-    self.assertEqual(self.lua('throttle.pending', 11, 'tid'), ['jid1'])
-    self.assertEqual(self.lua('jobs', 12, 'throttled', 'queue'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 12, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('jobs', 13, 'throttled', 'queue'), ['jid1'])
+
+  '''Test that when a job is timed out to another queue it maintains its state'''
+  def test_on_timeout_locks_are_properly_released(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('pop', 2, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 4, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 5, 'throttled', 'queue'), ['jid2'])
+    self.lua('timeout', 6, 'jid1')
+    self.assertEqual(self.lua('throttle.locks', 7, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 8, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 9, 'throttled', 'queue'), ['jid2'])
 
 class TestDependents(TestQless):
   def test_dependencies_can_acquire_lock_after_dependent_success(self):
@@ -270,7 +285,6 @@ def test_dependencies_do_not_acquire_lock_on_dependent_retry(self):
     self.assertEqual(self.lua('throttle.locks', 0, 'tid'), [])
     self.assertEqual(self.lua('jobs', 0, 'throttled', 'queue'), [])
 
-
 class TestConcurrencyLevelChange(TestQless):
   '''Test that changes to concurrency level are handled dynamically'''
   def test_increasing_concurrency_level_activates_pending_jobs(self):

From 8fda126684f5a86f07521b30519127d51428def2 Mon Sep 17 00:00:00 2001
From: Matt Conway <matt@conwaysplace.com>
Date: Thu, 23 Jul 2015 10:55:11 -0400
Subject: [PATCH 65/72] retry pop up to config limit when pop quantity would be
 unfulfilled due to throttling

---
 queue.lua          | 39 ++++++++++++++++++--------
 test/test_queue.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+), 12 deletions(-)

diff --git a/queue.lua b/queue.lua
index c0e5ee9..436b2c8 100644
--- a/queue.lua
+++ b/queue.lua
@@ -333,21 +333,36 @@ function QlessQueue:pop(now, worker, count)
 
   -- With these in place, we can expand this list of jids based on the work
   -- queue itself and the priorities therein
-  local jids = self.work.peek(count - #dead_jids) or {}
 
-  for index, jid in ipairs(jids) do
-    local job = Qless.job(jid)
-    if job:throttles_acquire(now) then
-      self:pop_job(now, worker, job)
-      table.insert(popped, jid)
-    else
-      self:throttle(now, job)
+  -- Since throttles could prevent work queue items from being popped, we can
+  -- retry a number of times till we find work items that are not throttled
+  local pop_retry_limit = tonumber(
+    Qless.config.get(self.name .. '-max-pop-retry') or
+    Qless.config.get('max-pop-retry', 1)
+  )
+
+  -- Keep trying to fulfill fulfill jobs from the work queue until we reach
+  -- the desired count or exhaust our retry limit
+  while #popped < count and pop_retry_limit > 0 do
+
+    local jids = self.work.peek(count - #popped) or {}
+
+    for index, jid in ipairs(jids) do
+      local job = Qless.job(jid)
+      if job:throttles_acquire(now) then
+        self:pop_job(now, worker, job)
+        table.insert(popped, jid)
+      else
+        self:throttle(now, job)
+      end
     end
-  end
 
-  -- All jobs should have acquired locks or be throttled,
-  -- ergo, remove all jids from work queue
-  self.work.remove(unpack(jids))
+    -- All jobs should have acquired locks or be throttled,
+    -- ergo, remove all jids from work queue
+    self.work.remove(unpack(jids))
+
+    pop_retry_limit = pop_retry_limit - 1
+  end
 
   return popped
 end
diff --git a/test/test_queue.py b/test/test_queue.py
index ad1326b..16a442f 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -694,3 +694,71 @@ def test_throttled_additional_put(self):
         self.lua('put', 2, 'worker', 'queue', 'jid2', 'klass', {}, 0)
         self.assertEqual(self.lua('throttle.locks', 3, 'ql:q:queue'), ['jid1'])
         self.assertEqual(self.lua('jobs', 4, 'throttled', 'queue'), ['jid2'])
+
+    def test_pop_no_retry(self):
+        '''Pop is not retried when limit unset'''
+        self.lua('throttle.set', 0, 'tid1', 1)
+        self.lua('throttle.set', 0, 'tid2', 1)
+
+        self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid1'])
+        self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid1'])
+        self.lua('put', 2, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttles', ['tid2'])
+        self.lua('put', 3, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttles', ['tid2'])
+
+        jobs = self.lua('pop', 4, 'queue', 'worker', 2)
+        self.assertEqual(['jid1'], [job['jid'] for job in jobs])
+        self.assertEqual(self.lua('throttle.locks', 5, 'tid1'), ['jid1'])
+        self.assertEqual(self.lua('throttle.locks', 6, 'tid2'), [])
+        self.assertEqual(self.lua('throttle.pending', 7, 'tid1'), ['jid2'])
+
+    def test_pop_retry(self):
+        '''Pop is retried when jobs get throttled'''
+        self.lua('config.set', 0, 'max-pop-retry', 99)
+        self.lua('throttle.set', 0, 'tid1', 1)
+        self.lua('throttle.set', 0, 'tid2', 1)
+
+        self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid1'])
+        self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid1'])
+        self.lua('put', 2, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttles', ['tid2'])
+        self.lua('put', 3, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttles', ['tid2'])
+
+        jobs = self.lua('pop', 4, 'queue', 'worker', 2)
+        self.assertEqual(['jid1', 'jid3'], [job['jid'] for job in jobs])
+        self.assertEqual(self.lua('throttle.locks', 5, 'tid1'), ['jid1'])
+        self.assertEqual(self.lua('throttle.locks', 6, 'tid2'), ['jid3'])
+        self.assertEqual(self.lua('throttle.pending', 7, 'tid1'), ['jid2'])
+
+    def test_pop_retry_queue_config(self):
+        '''Pop is retried using queue limit if set'''
+        self.lua('config.set', 0, 'max-pop-retry', 1)
+        self.lua('config.set', 0, 'queue-max-pop-retry', 2)
+        self.lua('throttle.set', 0, 'tid1', 1)
+        self.lua('throttle.set', 0, 'tid2', 1)
+
+        self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid1'])
+        self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid1'])
+        self.lua('put', 2, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttles', ['tid2'])
+        self.lua('put', 3, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttles', ['tid2'])
+
+        jobs = self.lua('pop', 4, 'queue', 'worker', 2)
+        self.assertEqual(['jid1', 'jid3'], [job['jid'] for job in jobs])
+        self.assertEqual(self.lua('throttle.locks', 5, 'tid1'), ['jid1'])
+        self.assertEqual(self.lua('throttle.locks', 6, 'tid2'), ['jid3'])
+        self.assertEqual(self.lua('throttle.pending', 7, 'tid1'), ['jid2'])
+
+    def test_pop_retry_upto_limit(self):
+        '''Pop is retried up to limit when jobs get throttled'''
+        self.lua('config.set', 0, 'max-pop-retry', 2)
+        self.lua('throttle.set', 0, 'tid1', 1)
+        self.lua('throttle.set', 0, 'tid2', 1)
+
+        self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid1'])
+        self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid1'])
+        self.lua('put', 2, 'worker', 'queue', 'jid3', 'klass', {}, 0, 'throttles', ['tid1'])
+        self.lua('put', 3, 'worker', 'queue', 'jid4', 'klass', {}, 0, 'throttles', ['tid2'])
+
+        jobs = self.lua('pop', 4, 'queue', 'worker', 2)
+        self.assertEqual(['jid1'], [job['jid'] for job in jobs])
+        self.assertEqual(self.lua('throttle.locks', 5, 'tid1'), ['jid1'])
+        self.assertEqual(self.lua('throttle.locks', 6, 'tid2'), [])
+        self.assertEqual(self.lua('throttle.pending', 7, 'tid1'), ['jid2', 'jid3'])

From 98b71d6e188f20c34045bb296a460ed6d53df68d Mon Sep 17 00:00:00 2001
From: Matt Conway <matt@conwaysplace.com>
Date: Thu, 23 Jul 2015 11:45:26 -0400
Subject: [PATCH 66/72] short circuit retry if nothing in work queue

---
 queue.lua | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/queue.lua b/queue.lua
index 436b2c8..1ea1756 100644
--- a/queue.lua
+++ b/queue.lua
@@ -347,6 +347,12 @@ function QlessQueue:pop(now, worker, count)
 
     local jids = self.work.peek(count - #popped) or {}
 
+    -- If there is nothing in the work queue, then no need to keep looping
+    if #jids == 0 then
+      break
+    end
+
+
     for index, jid in ipairs(jids) do
       local job = Qless.job(jid)
       if job:throttles_acquire(now) then

From 471882427f9d02e686dd957309bfa532d8703f9e Mon Sep 17 00:00:00 2001
From: Matt Conway <matt@conwaysplace.com>
Date: Fri, 31 Jul 2015 14:04:50 -0400
Subject: [PATCH 67/72] verify contents of waiting jids in tests for
 max-pop-retry

---
 test/test_queue.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/test/test_queue.py b/test/test_queue.py
index 16a442f..63ea956 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -710,6 +710,8 @@ def test_pop_no_retry(self):
         self.assertEqual(self.lua('throttle.locks', 5, 'tid1'), ['jid1'])
         self.assertEqual(self.lua('throttle.locks', 6, 'tid2'), [])
         self.assertEqual(self.lua('throttle.pending', 7, 'tid1'), ['jid2'])
+        waiting_jobs = self.lua('peek', 8, 'queue', 99)
+        self.assertEqual([job['jid'] for job in waiting_jobs], ['jid3', 'jid4'])
 
     def test_pop_retry(self):
         '''Pop is retried when jobs get throttled'''
@@ -727,6 +729,8 @@ def test_pop_retry(self):
         self.assertEqual(self.lua('throttle.locks', 5, 'tid1'), ['jid1'])
         self.assertEqual(self.lua('throttle.locks', 6, 'tid2'), ['jid3'])
         self.assertEqual(self.lua('throttle.pending', 7, 'tid1'), ['jid2'])
+        waiting_jobs = self.lua('peek', 8, 'queue', 99)
+        self.assertEqual([job['jid'] for job in waiting_jobs], ['jid4'])
 
     def test_pop_retry_queue_config(self):
         '''Pop is retried using queue limit if set'''
@@ -762,3 +766,5 @@ def test_pop_retry_upto_limit(self):
         self.assertEqual(self.lua('throttle.locks', 5, 'tid1'), ['jid1'])
         self.assertEqual(self.lua('throttle.locks', 6, 'tid2'), [])
         self.assertEqual(self.lua('throttle.pending', 7, 'tid1'), ['jid2', 'jid3'])
+        waiting_jobs = self.lua('peek', 8, 'queue', 99)
+        self.assertEqual([job['jid'] for job in waiting_jobs], ['jid4'])

From 9e5e716c1b6ff43746989589258b6ffab7bafd77 Mon Sep 17 00:00:00 2001
From: James <james-lawrence@users.noreply.github.com>
Date: Thu, 10 Sep 2015 09:36:17 -0400
Subject: [PATCH 68/72] Update api.lua

---
 api.lua | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/api.lua b/api.lua
index bc8a99d..86f3727 100644
--- a/api.lua
+++ b/api.lua
@@ -239,6 +239,16 @@ end
 QlessAPI['throttle.ttl'] = function(now, tid)
   return Qless.throttle(tid):ttl()
 end
+
+-- releases the set of jids from the specified throttle.
+QlessAPI['throttle.release'] = function(now, tid, ...)
+  local jids = unpack(arg)
+  local throttle = Qless.throttle(tid)
+
+  for _, jid in ipairs(jids) do
+    throttle.release(jid)
+  end
+end
 -------------------------------------------------------------------------------
 -- Function lookup
 -------------------------------------------------------------------------------

From e8ed50c6435c3b7894c06f8074448b54c31d0ee1 Mon Sep 17 00:00:00 2001
From: James <james-lawrence@users.noreply.github.com>
Date: Thu, 10 Sep 2015 09:43:55 -0400
Subject: [PATCH 69/72] Update test_throttle.py

---
 test/test_throttle.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/test/test_throttle.py b/test/test_throttle.py
index b39a175..4165077 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -39,6 +39,27 @@ def test_delete(self):
     self.lua('throttle.delete', 0, 'tid')
     self.assertEqual(self.lua('throttle.get', 0, 'tid'), {'id' : 'tid', 'maximum' : 0})
 
+  '''Test release properly removes the jid from the throttle'''
+  def test_release(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('pop', 2, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 4, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 5, 'throttled', 'queue'), ['jid2'])
+    self.lua('throttle.release', 6, 'tid', 'jid1', 'jid2')
+    # Lock should be empty until another job is popped
+    self.assertEqual(self.lua('throttle.locks', 7, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 8, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 9, 'throttled', 'queue'), ['jid2'])
+    self.lua('cancel', 10, 'jid1', 'worker', 'queue', {})
+    self.lua('cancel', 11, 'jid2', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 12, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 13, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 13, 'throttled', 'queue'), [])
+    self.assertEqual(self.lua('jobs', 14, 'running', 'queue'), [])
+    
 class TestAcquire(TestQless):
   '''Test that a job has a default queue throttle'''
   def test_default_queue_throttle(self):

From 230a777aef8f6b2dcb5985f9a9c82bc68cff310f Mon Sep 17 00:00:00 2001
From: james-lawrence <james.lawrence@backupify.com>
Date: Thu, 10 Sep 2015 10:58:25 -0400
Subject: [PATCH 70/72] tests

---
 README.md             | 54 ++++++++++++++++++++++++++-----------------
 api.lua               |  5 ++--
 test/test_throttle.py | 25 +++++++++++++++++---
 3 files changed, 57 insertions(+), 27 deletions(-)

diff --git a/README.md b/README.md
index e836f6b..cd8375b 100644
--- a/README.md
+++ b/README.md
@@ -54,12 +54,24 @@ installed:
 pip install redis nose
 ```
 
-To run the tests, there is a directive included in the makefile:
+To run all the tests, there is a directive included in the makefile:
 
 ```bash
 make test
 ```
 
+You can run specific test files by passing the TEST environment variable to make:
+
+```bash
+make test TEST=test/test_worker.py
+```
+
+or for a single test case in that file:
+
+```bash
+make test TEST=test/test_worker.py:TestWorker.test_basic
+```
+
 If you have Redis running somewhere other than `localhost:6379`, you can supply
 the `REDIS_URL` environment variable:
 
@@ -107,7 +119,7 @@ the heartbeat should return `false` and the worker should yield.
 
 When a node attempts to heartbeat, the lua script should check to see if the
 node attempting to renew the lock is the same node that currently owns the
-lock. If so, then the lock's expiration should be pushed back accordingly, 
+lock. If so, then the lock's expiration should be pushed back accordingly,
 and the updated expiration returned. If not, an exception is raised.
 
 Stats
@@ -117,7 +129,7 @@ and job completion time (time completed - time popped). By 'statistics',
 I mean average, variange, count and a histogram. Stats for the number of
 failures and retries for a given queue are also available.
 
-Stats are grouped by day. In the case of job wait time, its stats are 
+Stats are grouped by day. In the case of job wait time, its stats are
 aggregated on the day when the job was popped. In the case of completion time,
 they are grouped by the day it was completed.
 
@@ -141,7 +153,7 @@ might have:
 	=============
 	upload error
 	widget failure
-	
+
 	ql:f:upload error
 	==================
 	deadbeef
@@ -211,45 +223,45 @@ job is stored in `ql:j:<jid>-dependents`. For example, `ql:j:<jid>`:
 		# This is the same id as identifies it in the key. It should be
 		# a hex value of a uuid
 		'jid'         : 'deadbeef...',
-		
+
 		# This is a 'type' identifier. Clients may choose to ignore it,
 		# or use it as a language-specific identifier for determining
 		# what code to run. For instance, it might be 'foo.bar.FooJob'
 		'type'        : '...',
-		
+
 		# This is the priority of the job -- lower means more priority.
 		# The default is 0
 		'priority'    : 0,
-		
+
 		# This is the user data associated with the job. (JSON blob)
 		'data'        : '{"hello": "how are you"}',
-		
+
 		# A JSON array of tags associated with this job
 		'tags'        : '["testing", "experimental"]',
-		
+
 		# The worker ID of the worker that owns it. Currently the worker
 		# id is <hostname>-<pid>
 		'worker'      : 'ec2-...-4925',
-		
+
 		# This is the time when it must next check in
 		'expires'     : 1352375209,
-		
+
 		# The current state of the job: 'waiting', 'pending', 'complete'
 		'state'       : 'waiting',
-		
+
 		# The queue that it's associated with. 'null' if complete
 		'queue'       : 'example',
-		
+
 		# The maximum number of retries this job is allowed per queue
 		'retries'     : 3,
 		# The number of retries remaining
 		'remaining'   : 3,
-		
+
 		# The jids that depend on this job's completion
 		'dependents'  : [...],
 		# The jids that this job is dependent upon
 		'dependencies': [...],
-		
+
 		# A list of all the things that have happened to a job. Each entry has
 		# the keys 'what' and 'when', but it may also have arbitrary keys
 		# associated with it.
@@ -274,11 +286,11 @@ A queue is a priority queue and consists of three parts:
 1. `ql:q:<name>-depends` -- sorted set of jobs in a queue, but waiting on
     other jobs
 
-When looking for a unit of work, the client should first choose from the 
+When looking for a unit of work, the client should first choose from the
 next expired lock. If none are expired, then we should next make sure that
 any jobs that should now be considered eligible (the scheduled time is in
-the past) are then inserted into the work queue. A sorted set of all the 
-known queues is maintained at `ql:queues`. Currently we're keeping it 
+the past) are then inserted into the work queue. A sorted set of all the
+known queues is maintained at `ql:queues`. Currently we're keeping it
 sorted based on the time when we first saw the queue, but that's a little
 bit at odd with only keeping queues around while they're being used.
 
@@ -293,7 +305,7 @@ an integer timestamp of midnight for that day:
 
 	<day> = time - (time % (24 * 60 * 60))
 
-Stats are stored under two hashes: `ql:s:wait:<day>:<queue>` and 
+Stats are stored under two hashes: `ql:s:wait:<day>:<queue>` and
 `ql:s:run:<day>:<queue>` respectively. Each has the keys:
 
 - `total` -- The total number of data points contained
@@ -363,8 +375,8 @@ something to be aware of when writing language bindings.
 
 Filesystem Access
 -----------------
-It's intended to be a common usecase that bindings provide a worker script or 
-binary that runs several worker subprocesses. These should run with their 
+It's intended to be a common usecase that bindings provide a worker script or
+binary that runs several worker subprocesses. These should run with their
 working directory as a sandbox.
 
 Forking Model
diff --git a/api.lua b/api.lua
index 86f3727..be06ff0 100644
--- a/api.lua
+++ b/api.lua
@@ -242,11 +242,10 @@ end
 
 -- releases the set of jids from the specified throttle.
 QlessAPI['throttle.release'] = function(now, tid, ...)
-  local jids = unpack(arg)
   local throttle = Qless.throttle(tid)
 
-  for _, jid in ipairs(jids) do
-    throttle.release(jid)
+  for _, jid in ipairs(arg) do
+    throttle:release(now, jid)
   end
 end
 -------------------------------------------------------------------------------
diff --git a/test/test_throttle.py b/test/test_throttle.py
index 4165077..fddd052 100644
--- a/test/test_throttle.py
+++ b/test/test_throttle.py
@@ -49,17 +49,36 @@ def test_release(self):
     self.assertEqual(self.lua('throttle.pending', 4, 'tid'), ['jid2'])
     self.assertEqual(self.lua('jobs', 5, 'throttled', 'queue'), ['jid2'])
     self.lua('throttle.release', 6, 'tid', 'jid1', 'jid2')
-    # Lock should be empty until another job is popped
     self.assertEqual(self.lua('throttle.locks', 7, 'tid'), [])
     self.assertEqual(self.lua('throttle.pending', 8, 'tid'), [])
-    self.assertEqual(self.lua('jobs', 9, 'throttled', 'queue'), ['jid2'])
+    self.assertEqual(self.lua('peek', 9,'queue', 1)[0]['jid'], 'jid2')
     self.lua('cancel', 10, 'jid1', 'worker', 'queue', {})
     self.lua('cancel', 11, 'jid2', 'worker', 'queue', {})
     self.assertEqual(self.lua('throttle.locks', 12, 'tid'), [])
     self.assertEqual(self.lua('throttle.pending', 13, 'tid'), [])
     self.assertEqual(self.lua('jobs', 13, 'throttled', 'queue'), [])
     self.assertEqual(self.lua('jobs', 14, 'running', 'queue'), [])
-    
+
+  '''Test release of pending jobs before lock holders'''
+  def test_release_pending_before_lock_holders(self):
+    self.lua('throttle.set', 0, 'tid', 1)
+    self.lua('put', 0, 'worker', 'queue', 'jid1', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('put', 1, 'worker', 'queue', 'jid2', 'klass', {}, 0, 'throttles', ['tid'])
+    self.lua('pop', 2, 'queue', 'worker', 2)
+    self.assertEqual(self.lua('throttle.locks', 3, 'tid'), ['jid1'])
+    self.assertEqual(self.lua('throttle.pending', 4, 'tid'), ['jid2'])
+    self.assertEqual(self.lua('jobs', 5, 'throttled', 'queue'), ['jid2'])
+    self.lua('throttle.release', 6, 'tid', 'jid2', 'jid1')
+    self.assertEqual(self.lua('throttle.locks', 7, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 8, 'tid'), [])
+    self.assertEqual(self.lua('peek', 9,'queue', 1), {})
+    self.lua('cancel', 10, 'jid1', 'worker', 'queue', {})
+    self.lua('cancel', 11, 'jid2', 'worker', 'queue', {})
+    self.assertEqual(self.lua('throttle.locks', 12, 'tid'), [])
+    self.assertEqual(self.lua('throttle.pending', 13, 'tid'), [])
+    self.assertEqual(self.lua('jobs', 13, 'throttled', 'queue'), [])
+    self.assertEqual(self.lua('jobs', 14, 'running', 'queue'), [])
+
 class TestAcquire(TestQless):
   '''Test that a job has a default queue throttle'''
   def test_default_queue_throttle(self):

From 6dbf028a915fb8c9b1df37310659adc8dc1762ca Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Tue, 5 Jul 2016 12:46:38 -0400
Subject: [PATCH 71/72] Ignore ghost jids that have no real job

---
 queue.lua | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/queue.lua b/queue.lua
index 1ea1756..d8c7923 100644
--- a/queue.lua
+++ b/queue.lua
@@ -309,8 +309,11 @@ function QlessQueue:pop(now, worker, count)
   local popped = {}
 
   for index, jid in ipairs(dead_jids) do
-    self:pop_job(now, worker, Qless.job(jid))
-    table.insert(popped, jid)
+    local success = self:pop_job(now, worker, Qless.job(jid))
+    -- only track jid if a job was popped and it's not a phantom jid
+    if success then
+      table.insert(popped, jid)
+    end
   end
 
   -- if queue is at max capacity don't pop any further jobs.
@@ -356,8 +359,11 @@ function QlessQueue:pop(now, worker, count)
     for index, jid in ipairs(jids) do
       local job = Qless.job(jid)
       if job:throttles_acquire(now) then
-        self:pop_job(now, worker, job)
-        table.insert(popped, jid)
+        local success = self:pop_job(now, worker, job)
+        -- only track jid if a job was popped and it's not a phantom jid
+        if success then
+          table.insert(popped, jid)
+        end
       else
         self:throttle(now, job)
       end
@@ -387,7 +393,13 @@ end
 function QlessQueue:pop_job(now, worker, job)
   local state
   local jid = job.jid
-  state = unpack(job:data('state'))
+  local job_state = job:data('state')
+  -- if the job doesn't exist, short circuit
+  if not job_state then
+    return false
+  end
+
+  state = unpack(job_state)
   job:history(now, 'popped', {worker = worker})
 
   -- We should find the heartbeat interval for this queue heartbeat
@@ -419,6 +431,7 @@ function QlessQueue:pop_job(now, worker, job)
   if tracked then
     Qless.publish('popped', jid)
   end
+  return true
 end
 
 -- Update the stats for this queue

From 20dc687832ad472f0a00899d26c285b893ff466c Mon Sep 17 00:00:00 2001
From: Danny Guinther <danny@backupify.com>
Date: Fri, 8 Jul 2016 11:56:56 -0400
Subject: [PATCH 72/72] Remove old queue throttle from reput job

---
 queue.lua          | 7 +++++++
 test/test_queue.py | 3 ++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/queue.lua b/queue.lua
index d8c7923..2cab39b 100644
--- a/queue.lua
+++ b/queue.lua
@@ -553,9 +553,16 @@ function QlessQueue:put(now, worker, jid, klass, raw_data, delay, ...)
   job:history(now, 'put', {q = self.name})
 
   -- If this item was previously in another queue, then we should remove it from there
+  -- and remove the associated throttle
   if oldqueue then
     local queue_obj = Qless.queue(oldqueue)
     queue_obj:remove_job(jid)
+    local old_qid = QlessQueue.ns .. oldqueue
+    for index, tname in ipairs(throttles) do
+      if tname == old_qid then
+        table.remove(throttles, index)
+      end
+    end
   end
 
   -- If this had previously been given out to a worker, make sure to remove it
diff --git a/test/test_queue.py b/test/test_queue.py
index 63ea956..c63daad 100644
--- a/test/test_queue.py
+++ b/test/test_queue.py
@@ -390,7 +390,8 @@ def test_put_depends_with_delay(self):
     def test_move(self):
         '''Move is described in terms of puts.'''
         self.lua('put', 0, 'worker', 'queue', 'jid', 'klass', {'foo': 'bar'}, 0)
-        self.lua('put', 0, 'worker', 'other', 'jid', 'klass', {'foo': 'bar'}, 0)
+        self.assertEqual(self.lua('get', 0, 'jid')['throttles'], ['ql:q:queue'])
+        self.lua('put', 0, 'worker', 'other', 'jid', 'klass', {'foo': 'bar'}, 0, 'throttles', ['ql:q:queue'])
         self.assertEqual(self.lua('get', 1, 'jid'), {
             'data': '{"foo": "bar"}',
             'dependencies': {},