diff --git a/src/Illuminate/Database/Query/Builder.php b/src/Illuminate/Database/Query/Builder.php index e5c0c8f2..ce831831 100755 --- a/src/Illuminate/Database/Query/Builder.php +++ b/src/Illuminate/Database/Query/Builder.php @@ -5,6 +5,7 @@ use Illuminate\Database\ConnectionInterface; use Illuminate\Database\Query\Grammars\Grammar; use Illuminate\Database\Query\Processors\Processor; +use RuntimeException; class Builder { @@ -1521,6 +1522,99 @@ public function chunk($count, callable $callback) } } + /** + * Chunk the results of a query by comparing IDs. + * + * @param int $count + * @param callable $callback + * @param string|null $column + * @param string|null $alias + * @return bool + */ + public function chunkById(int $count, callable $callback, string|null $column = null, string|null $alias = null): bool + { + $column ??= $this->defaultKeyName(); + + $alias ??= $column; + + $lastId = null; + + $page = 1; + + do { + $clone = clone $this; + + // We'll execute the query for the given page and get the results. If there are + // no results we can just break and return from here. When there are results + // we will call the callback with the current chunk of these results here. + $results = $clone->forPageAfterId($count, $lastId, $column)->get(); + + $countResults = count($results); + + if ($countResults === 0) { + break; + } + + // On each chunk result set, we will pass them to the callback and then let the + // developer take care of everything within the callback, which allows us to + // keep the memory low for spinning through large result sets for working. + if ($callback($results, $page) === false) { + return false; + } + + $lastId = data_get(end($results), $alias); + + if ($lastId === null) { + throw new RuntimeException("The chunkById operation was aborted because the [{$alias}] column is not present in the query result."); + } + + unset($results); + + $page++; + } while ($countResults === $count); + + return true; + } + + /** + * Constrain the query to the next "page" of results after a given ID. + * + * @param int $perPage + * @param int|null $lastId + * @param string $column + * @return $this + */ + protected function forPageAfterId(int $perPage = 15, int|null $lastId = 0, string $column = 'id'): Builder + { + $this->orders = $this->removeExistingOrdersFor($column); + + if (! is_null($lastId)) { + $this->where($column, '>', $lastId); + } + + return $this->orderBy($column, 'asc') + ->limit($perPage); + } + + /** + * Get an array with all orders with a given column removed. + * + * @param string $column + * @return array + */ + protected function removeExistingOrdersFor(string $column): array + { + return Collection::make($this->orders) + ->reject(function ($order) use ($column) { + return isset($order['column']) && $order['column'] === $column; + })->values()->all(); + } + + private function defaultKeyName(): string + { + return 'id'; + } + /** * Get an array with the values of a given column. * diff --git a/tests/Database/DatabaseQueryBuilderTest.php b/tests/Database/DatabaseQueryBuilderTest.php index 734d4c8e..513d200e 100755 --- a/tests/Database/DatabaseQueryBuilderTest.php +++ b/tests/Database/DatabaseQueryBuilderTest.php @@ -6,8 +6,10 @@ use Illuminate\Database\Query\Grammars\Grammar; use Illuminate\Database\Query\Processors\Processor; use Illuminate\Pagination\Factory; +use Illuminate\Support\Collection; use L4\Tests\BackwardCompatibleTestCase; use Mockery as m; +use Mockery\MockInterface; class DatabaseQueryBuilderTest extends BackwardCompatibleTestCase { @@ -1363,6 +1365,108 @@ public function testMergeBuildersBindingOrder(): void $this->assertEquals(['foo', 'bar', 'baz'], $builder->getBindings()); } + public function testChunkByIdOnArrays(): void + { + $builder = $this->getMockQueryBuilder(); + $builder->orders[] = ['column' => 'foobar', 'direction' => 'asc']; + + $chunk1 = [['someIdField' => 1], ['someIdField' => 2]]; + $chunk2 = [['someIdField' => 10], ['someIdField' => 11]]; + $chunk3 = []; + $builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturnSelf(); + $builder->shouldReceive('forPageAfterId')->once()->with(2, 2, 'someIdField')->andReturnSelf(); + $builder->shouldReceive('forPageAfterId')->once()->with(2, 11, 'someIdField')->andReturnSelf(); + $builder->shouldReceive('get')->times(3)->andReturn($chunk1, $chunk2, $chunk3); + + $callbackAssertor = m::mock(stdClass::class); + $callbackAssertor->shouldReceive('doSomething')->once()->with($chunk1); + $callbackAssertor->shouldReceive('doSomething')->once()->with($chunk2); + $callbackAssertor->shouldReceive('doSomething')->never()->with($chunk3); + + $builder->chunkById(2, function ($results) use ($callbackAssertor) { + $callbackAssertor->doSomething($results); + }, 'someIdField'); + } + + public function testChunkPaginatesUsingIdWithLastChunkComplete(): void + { + $builder = $this->getMockQueryBuilder(); + $builder->orders[] = ['column' => 'foobar', 'direction' => 'asc']; + + $chunk1 = [(object) ['someIdField' => 1], (object) ['someIdField' => 2]]; + $chunk2 = [(object) ['someIdField' => 10], (object) ['someIdField' => 11]]; + $chunk3 = []; + $builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturnSelf(); + $builder->shouldReceive('forPageAfterId')->once()->with(2, 2, 'someIdField')->andReturnSelf(); + $builder->shouldReceive('forPageAfterId')->once()->with(2, 11, 'someIdField')->andReturnSelf(); + $builder->shouldReceive('get')->times(3)->andReturn($chunk1, $chunk2, $chunk3); + + $callbackAssertor = m::mock(stdClass::class); + $callbackAssertor->shouldReceive('doSomething')->once()->with($chunk1); + $callbackAssertor->shouldReceive('doSomething')->once()->with($chunk2); + $callbackAssertor->shouldReceive('doSomething')->never()->with($chunk3); + + $builder->chunkById(2, function ($results) use ($callbackAssertor) { + $callbackAssertor->doSomething($results); + }, 'someIdField'); + } + + public function testChunkPaginatesUsingIdWithLastChunkPartial(): void + { + $builder = $this->getMockQueryBuilder(); + $builder->orders[] = ['column' => 'foobar', 'direction' => 'asc']; + + $chunk1 = [(object) ['someIdField' => 1], (object) ['someIdField' => 2]]; + $chunk2 = [(object) ['someIdField' => 10]]; + $builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'someIdField')->andReturnSelf(); + $builder->shouldReceive('forPageAfterId')->once()->with(2, 2, 'someIdField')->andReturnSelf(); + $builder->shouldReceive('get')->times(2)->andReturn($chunk1, $chunk2); + + $callbackAssertor = m::mock(stdClass::class); + $callbackAssertor->shouldReceive('doSomething')->once()->with($chunk1); + $callbackAssertor->shouldReceive('doSomething')->once()->with($chunk2); + + $builder->chunkById(2, function ($results) use ($callbackAssertor) { + $callbackAssertor->doSomething($results); + }, 'someIdField'); + } + + public function testChunkPaginatesUsingIdWithCountZero(): void + { + $builder = $this->getMockQueryBuilder(); + $builder->orders[] = ['column' => 'foobar', 'direction' => 'asc']; + + $chunk = []; + $builder->shouldReceive('forPageAfterId')->once()->with(0, 0, 'someIdField')->andReturnSelf(); + $builder->shouldReceive('get')->times(1)->andReturn($chunk); + + $callbackAssertor = m::mock(stdClass::class); + $callbackAssertor->shouldReceive('doSomething')->never(); + + $builder->chunkById(0, function ($results) use ($callbackAssertor) { + $callbackAssertor->doSomething($results); + }, 'someIdField'); + } + + public function testChunkPaginatesUsingIdWithAlias(): void + { + $builder = $this->getMockQueryBuilder(); + $builder->orders[] = ['column' => 'foobar', 'direction' => 'asc']; + + $chunk1 = [(object) ['table_id' => 1], (object) ['table_id' => 10]]; + $chunk2 = []; + $builder->shouldReceive('forPageAfterId')->once()->with(2, 0, 'table.id')->andReturnSelf(); + $builder->shouldReceive('forPageAfterId')->once()->with(2, 10, 'table.id')->andReturnSelf(); + $builder->shouldReceive('get')->times(2)->andReturn($chunk1, $chunk2); + + $callbackAssertor = m::mock(stdClass::class); + $callbackAssertor->shouldReceive('doSomething')->once()->with($chunk1); + $callbackAssertor->shouldReceive('doSomething')->never()->with($chunk2); + + $builder->chunkById(2, function ($results) use ($callbackAssertor) { + $callbackAssertor->doSomething($results); + }, 'table.id', 'table_id'); + } protected function getBuilder(): Builder { @@ -1411,4 +1515,16 @@ protected function getMySqlBuilderWithProcessor(): Builder return new Builder(m::mock(ConnectionInterface::class), $grammar, $processor); } + /** + * @return MockInterface|\Illuminate\Database\Query\Builder + */ + protected function getMockQueryBuilder(): MockInterface|Builder + { + return m::mock(Builder::class, [ + m::mock(ConnectionInterface::class), + new Grammar, + m::mock(Processor::class), + ])->makePartial()->shouldAllowMockingProtectedMethods(); + } + }