Skip to content

Commit

Permalink
Improve repair performance in cases where there are a lot of finished…
Browse files Browse the repository at this point in the history
… jobs with dependencies
  • Loading branch information
kraih committed Nov 20, 2023
1 parent 1d68a04 commit cb9182a
Show file tree
Hide file tree
Showing 3 changed files with 9 additions and 12 deletions.
3 changes: 2 additions & 1 deletion Changes
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@

10.27 2023-11-11
10.27 2023-11-20
- Improved repair performance in cases where there are a lot of finished jobs with dependencies.

10.26 2023-11-10
- Added type information to worker status.
Expand Down
14 changes: 5 additions & 9 deletions lib/Minion/Backend/Pg.pm
Original file line number Diff line number Diff line change
Expand Up @@ -179,15 +179,11 @@ sub repair {
$db->query("DELETE FROM minion_workers WHERE notified < NOW() - INTERVAL '1 second' * ?", $minion->missing_after);

# Old jobs with no unresolved dependencies and expired jobs
$db->query(
"DELETE FROM minion_jobs WHERE id IN (
SELECT j.id FROM minion_jobs AS j LEFT JOIN minion_jobs AS children
ON children.state != 'finished' AND ARRAY_LENGTH(children.parents, 1) > 0 AND j.id = ANY(children.parents)
WHERE j.state = 'finished' AND j.finished <= NOW() - INTERVAL '1 second' * ? AND children.id IS NULL
UNION ALL
SELECT id FROM minion_jobs WHERE state = 'inactive' AND expires <= NOW()
)", $minion->remove_after
);
$db->query("DELETE FROM minion_jobs WHERE state = 'finished' AND finished <= NOW() - INTERVAL '1 second' * ?",
$minion->remove_after);

# Expired jobs
$db->query("DELETE FROM minion_jobs WHERE state = 'inactive' AND expires <= NOW()");

# Jobs with missing worker (can be retried)
$db->query(
Expand Down
4 changes: 2 additions & 2 deletions t/pg.t
Original file line number Diff line number Diff line change
Expand Up @@ -1116,9 +1116,9 @@ subtest 'Job dependencies' => sub {
is_deeply $job->info->{children}, [], 'right children';
is_deeply $job->info->{parents}, [$id, $id2], 'right parents';
is $minion->stats->{finished_jobs}, 2, 'two finished jobs';
is $minion->repair->stats->{finished_jobs}, 2, 'two finished jobs';
is $minion->repair->stats->{finished_jobs}, 0, 'no finished jobs';
ok $job->finish, 'job finished';
is $minion->stats->{finished_jobs}, 3, 'three finished jobs';
is $minion->stats->{finished_jobs}, 1, 'one finished job';
is $minion->repair->remove_after(172800)->stats->{finished_jobs}, 0, 'no finished jobs';
$id = $minion->enqueue(test => [] => {parents => [-1]});
ok $job = $worker->dequeue(0), 'job dequeued';
Expand Down

0 comments on commit cb9182a

Please sign in to comment.