From d641a56bed92049e1fa2ac4dd0f5c425435d9825 Mon Sep 17 00:00:00 2001 From: Tigrov Date: Sat, 17 Feb 2024 16:26:30 +0700 Subject: [PATCH 01/10] Refactor `AbstractDMLQueryBuilder::batchInsert()` --- src/QueryBuilder/AbstractDMLQueryBuilder.php | 22 ++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/QueryBuilder/AbstractDMLQueryBuilder.php b/src/QueryBuilder/AbstractDMLQueryBuilder.php index dc048eb2c..1368c584c 100644 --- a/src/QueryBuilder/AbstractDMLQueryBuilder.php +++ b/src/QueryBuilder/AbstractDMLQueryBuilder.php @@ -4,6 +4,8 @@ namespace Yiisoft\Db\QueryBuilder; +use Iterator; +use IteratorAggregate; use JsonException; use Traversable; use Yiisoft\Db\Constraint\Constraint; @@ -19,7 +21,6 @@ use function array_combine; use function array_diff; -use function array_fill_keys; use function array_filter; use function array_key_exists; use function array_keys; @@ -142,7 +143,7 @@ protected function prepareBatchInsertValues(string $table, iterable $rows, array $values = []; /** @var string[] $columnNames */ $columnNames = array_values($columns); - $columnKeys = array_fill_keys($columnNames, false); + $columnKeys = array_combine($columnNames, $columnNames); $columnSchemas = $this->schema->getTableSchema($table)?->getColumns() ?? []; foreach ($rows as $row) { @@ -151,7 +152,7 @@ protected function prepareBatchInsertValues(string $table, iterable $rows, array /** @var int|string $key */ foreach ($row as $key => $value) { - $columnName = $columns[$key] ?? (isset($columnKeys[$key]) ? $key : $columnNames[$i] ?? $i); + $columnName = $columns[$key] ?? $columnKeys[$key] ?? $columnNames[$i] ?? $i; if (isset($columnSchemas[$columnName])) { $value = $columnSchemas[$columnName]->dbTypecast($value); @@ -175,7 +176,6 @@ protected function prepareBatchInsertValues(string $table, iterable $rows, array /** * Extract column names from columns and rows. * - * @param string $table The column schemas. * @param iterable $rows The rows to be batch inserted into the table. * @param string[] $columns The column names. * @@ -185,11 +185,21 @@ protected function extractColumnNames(iterable $rows, array $columns): array { $columns = $this->getNormalizeColumnNames('', $columns); - if ($columns !== [] || !is_array($rows)) { + if (!empty($columns)) { return $columns; } - $row = reset($rows); + while ($rows instanceof IteratorAggregate && !$rows instanceof Iterator) { + $rows = $rows->getIterator(); + } + + if ($rows instanceof Iterator) { + $row = $rows->current(); + } else { + /** @var array[] $rows */ + $row = reset($rows); + } + $row = match (true) { is_array($row) => $row, $row instanceof Traversable => iterator_to_array($row), From 499972a85b984e7d62a9ba58ad0f9adf5426f77c Mon Sep 17 00:00:00 2001 From: Tigrov Date: Sun, 7 Apr 2024 17:39:10 +0700 Subject: [PATCH 02/10] Allow skipped columns and empty columns for iterators --- src/QueryBuilder/AbstractDMLQueryBuilder.php | 20 +++++++++------ tests/Common/CommonCommandTest.php | 2 +- tests/Provider/CommandProvider.php | 26 +++++++++++++++++++- tests/Provider/QueryBuilderProvider.php | 12 +++++++++ 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/QueryBuilder/AbstractDMLQueryBuilder.php b/src/QueryBuilder/AbstractDMLQueryBuilder.php index 03ff04fdf..40809c8ae 100644 --- a/src/QueryBuilder/AbstractDMLQueryBuilder.php +++ b/src/QueryBuilder/AbstractDMLQueryBuilder.php @@ -67,6 +67,14 @@ public function batchInsert(string $table, array $columns, iterable $rows, array return ''; } + while ($rows instanceof IteratorAggregate) { + $rows = $rows->getIterator(); + } + + if ($rows instanceof Iterator && !$rows->valid()) { + return ''; + } + $columns = $this->extractColumnNames($rows, $columns); $values = $this->prepareBatchInsertValues($table, $rows, $columns, $params); @@ -149,11 +157,12 @@ protected function prepareBatchInsertValues(string $table, iterable $rows, array /** @var string[] $columnNames */ $columnNames = array_values($columns); $columnKeys = array_combine($columnNames, $columnNames); + $columnNulls = array_fill_keys($columnNames, 'NULL'); $columnSchemas = $this->schema->getTableSchema($table)?->getColumns() ?? []; foreach ($rows as $row) { $i = 0; - $placeholders = $columnKeys; + $placeholders = $columnNulls; /** @var int|string $key */ foreach ($row as $key => $value) { @@ -181,12 +190,12 @@ protected function prepareBatchInsertValues(string $table, iterable $rows, array /** * Extract column names from columns and rows. * - * @param iterable $rows The rows to be batch inserted into the table. + * @param array[]|Iterator $rows The rows to be batch inserted into the table. * @param string[] $columns The column names. * * @return string[] The column names. */ - protected function extractColumnNames(iterable $rows, array $columns): array + protected function extractColumnNames(array|Iterator $rows, array $columns): array { $columns = $this->getNormalizeColumnNames('', $columns); @@ -194,14 +203,9 @@ protected function extractColumnNames(iterable $rows, array $columns): array return $columns; } - while ($rows instanceof IteratorAggregate && !$rows instanceof Iterator) { - $rows = $rows->getIterator(); - } - if ($rows instanceof Iterator) { $row = $rows->current(); } else { - /** @var array[] $rows */ $row = reset($rows); } diff --git a/tests/Common/CommonCommandTest.php b/tests/Common/CommonCommandTest.php index 89710879f..1f09b5f59 100644 --- a/tests/Common/CommonCommandTest.php +++ b/tests/Common/CommonCommandTest.php @@ -305,7 +305,7 @@ public function testAddUnique(string $name, string $tableName, array|string $col public function testBatchInsert( string $table, array $columns, - array $values, + iterable $values, string $expected, array $expectedParams = [], int $insertedRow = 1 diff --git a/tests/Provider/CommandProvider.php b/tests/Provider/CommandProvider.php index 689426f80..b85319649 100644 --- a/tests/Provider/CommandProvider.php +++ b/tests/Provider/CommandProvider.php @@ -5,6 +5,8 @@ namespace Yiisoft\Db\Tests\Provider; use ArrayIterator; +use IteratorAggregate; +use Traversable; use Yiisoft\Db\Command\DataType; use Yiisoft\Db\Command\Param; use Yiisoft\Db\Expression\Expression; @@ -462,7 +464,7 @@ public static function batchInsert(): array ':qp3' => true, ], ], - 'empty columns and Traversable' => [ + 'empty columns and a Traversable value' => [ 'type', [], 'values' => [new ArrayIterator(['int_col' => '1.0', 'float_col' => '2', 'char_col' => 10, 'bool_col' => 1])], @@ -479,6 +481,28 @@ public static function batchInsert(): array ':qp3' => true, ], ], + 'empty columns and Traversable values' => [ + 'type', + [], + 'values' => new class implements IteratorAggregate { + public function getIterator(): Traversable + { + yield ['int_col' => '1.0', 'float_col' => '2', 'char_col' => 10, 'bool_col' => 1]; + } + }, + 'expected' => DbHelper::replaceQuotes( + << [ + ':qp0' => 1, + ':qp1' => 2.0, + ':qp2' => '10', + ':qp3' => true, + ], + ], ]; } diff --git a/tests/Provider/QueryBuilderProvider.php b/tests/Provider/QueryBuilderProvider.php index f37f97ebf..2db3eee8b 100644 --- a/tests/Provider/QueryBuilderProvider.php +++ b/tests/Provider/QueryBuilderProvider.php @@ -252,6 +252,18 @@ public static function batchInsert(): array ':qp3' => 1, ], ], + 'skipped columns' => [ + 'customer', + ['email', 'name', 'address'], + 'values' => [['email' => 'hello@localhost', 'address' => 'Earth, Solar System']], + 'expected' => DbHelper::replaceQuotes( + << [':qp0' => 'hello@localhost', ':qp1' => 'Earth, Solar System'], + ], ]; } From 605b5c477bf771c8a54128208af49fe04bb99ceb Mon Sep 17 00:00:00 2001 From: Tigrov Date: Sun, 14 Apr 2024 15:57:14 +0700 Subject: [PATCH 03/10] Improve --- src/QueryBuilder/AbstractDMLQueryBuilder.php | 38 ++++++++++++++------ tests/Provider/QueryBuilderProvider.php | 12 ------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/QueryBuilder/AbstractDMLQueryBuilder.php b/src/QueryBuilder/AbstractDMLQueryBuilder.php index 40809c8ae..a8d0a7916 100644 --- a/src/QueryBuilder/AbstractDMLQueryBuilder.php +++ b/src/QueryBuilder/AbstractDMLQueryBuilder.php @@ -22,6 +22,7 @@ use function array_combine; use function array_diff; +use function array_fill_keys; use function array_filter; use function array_key_exists; use function array_keys; @@ -63,15 +64,11 @@ public function __construct( public function batchInsert(string $table, array $columns, iterable $rows, array &$params = []): string { - if (empty($rows)) { - return ''; - } - - while ($rows instanceof IteratorAggregate) { - $rows = $rows->getIterator(); + if (!is_array($rows)) { + $rows = $this->prepareTraversable($rows); } - if ($rows instanceof Iterator && !$rows->valid()) { + if (empty($rows)) { return ''; } @@ -139,6 +136,26 @@ public function upsert( throw new NotSupportedException(__METHOD__ . ' is not supported by this DBMS.'); } + /** + * Prepare traversable for batch insert. + * + * @param Traversable $rows The rows to be batch inserted into the table. + * + * @return Iterator|array The prepared rows. + */ + protected function prepareTraversable(Traversable $rows): Iterator|array + { + while ($rows instanceof IteratorAggregate) { + $rows = $rows->getIterator(); + } + + if ($rows instanceof Iterator && !$rows->valid()) { + return []; + } + + return $rows; + } + /** * Prepare values for batch insert. * @@ -156,17 +173,16 @@ protected function prepareBatchInsertValues(string $table, iterable $rows, array $values = []; /** @var string[] $columnNames */ $columnNames = array_values($columns); - $columnKeys = array_combine($columnNames, $columnNames); - $columnNulls = array_fill_keys($columnNames, 'NULL'); + $columnKeys = array_fill_keys($columnNames, false); $columnSchemas = $this->schema->getTableSchema($table)?->getColumns() ?? []; foreach ($rows as $row) { $i = 0; - $placeholders = $columnNulls; + $placeholders = $columnKeys; /** @var int|string $key */ foreach ($row as $key => $value) { - $columnName = $columns[$key] ?? $columnKeys[$key] ?? $columnNames[$i] ?? $i; + $columnName = $columns[$key] ?? (isset($columnKeys[$key]) ? $key : $columnNames[$i] ?? $i); if (isset($columnSchemas[$columnName])) { $value = $columnSchemas[$columnName]->dbTypecast($value); diff --git a/tests/Provider/QueryBuilderProvider.php b/tests/Provider/QueryBuilderProvider.php index 2300425dd..f8f0b5356 100644 --- a/tests/Provider/QueryBuilderProvider.php +++ b/tests/Provider/QueryBuilderProvider.php @@ -255,18 +255,6 @@ public static function batchInsert(): array ':qp3' => 1, ], ], - 'skipped columns' => [ - 'customer', - ['email', 'name', 'address'], - 'values' => [['email' => 'hello@localhost', 'address' => 'Earth, Solar System']], - 'expected' => DbHelper::replaceQuotes( - << [':qp0' => 'hello@localhost', ':qp1' => 'Earth, Solar System'], - ], ]; } From 07e1a8d7164eb7191efef35aedc88319bdb27295 Mon Sep 17 00:00:00 2001 From: StyleCI Bot Date: Sun, 14 Apr 2024 09:51:17 +0000 Subject: [PATCH 04/10] Apply fixes from StyleCI --- src/QueryBuilder/AbstractDMLQueryBuilder.php | 2 +- tests/Provider/CommandProvider.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/QueryBuilder/AbstractDMLQueryBuilder.php b/src/QueryBuilder/AbstractDMLQueryBuilder.php index a8d0a7916..0db417372 100644 --- a/src/QueryBuilder/AbstractDMLQueryBuilder.php +++ b/src/QueryBuilder/AbstractDMLQueryBuilder.php @@ -141,7 +141,7 @@ public function upsert( * * @param Traversable $rows The rows to be batch inserted into the table. * - * @return Iterator|array The prepared rows. + * @return array|Iterator The prepared rows. */ protected function prepareTraversable(Traversable $rows): Iterator|array { diff --git a/tests/Provider/CommandProvider.php b/tests/Provider/CommandProvider.php index 1c247fec6..c0c1bee51 100644 --- a/tests/Provider/CommandProvider.php +++ b/tests/Provider/CommandProvider.php @@ -485,7 +485,7 @@ public static function batchInsert(): array 'empty columns and Traversable values' => [ 'type', [], - 'values' => new class implements IteratorAggregate { + 'values' => new class () implements IteratorAggregate { public function getIterator(): Traversable { yield ['int_col' => '1.0', 'float_col' => '2', 'char_col' => 10, 'bool_col' => 1]; From b9ff4ba6c936761088402ae084fef29d3848b14a Mon Sep 17 00:00:00 2001 From: Tigrov Date: Sun, 14 Apr 2024 18:16:46 +0700 Subject: [PATCH 05/10] Fix psalm issues --- src/QueryBuilder/AbstractDMLQueryBuilder.php | 5 +++++ src/QueryBuilder/DMLQueryBuilderInterface.php | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/QueryBuilder/AbstractDMLQueryBuilder.php b/src/QueryBuilder/AbstractDMLQueryBuilder.php index 0db417372..645ea7b5f 100644 --- a/src/QueryBuilder/AbstractDMLQueryBuilder.php +++ b/src/QueryBuilder/AbstractDMLQueryBuilder.php @@ -142,6 +142,8 @@ public function upsert( * @param Traversable $rows The rows to be batch inserted into the table. * * @return array|Iterator The prepared rows. + * + * @psalm-return Iterator|array> */ protected function prepareTraversable(Traversable $rows): Iterator|array { @@ -153,6 +155,7 @@ protected function prepareTraversable(Traversable $rows): Iterator|array return []; } + /** @var Iterator $rows */ return $rows; } @@ -210,6 +213,8 @@ protected function prepareBatchInsertValues(string $table, iterable $rows, array * @param string[] $columns The column names. * * @return string[] The column names. + * + * @psalm-param Iterator|non-empty-array> $rows */ protected function extractColumnNames(array|Iterator $rows, array $columns): array { diff --git a/src/QueryBuilder/DMLQueryBuilderInterface.php b/src/QueryBuilder/DMLQueryBuilderInterface.php index 24aa4c856..e00113a57 100644 --- a/src/QueryBuilder/DMLQueryBuilderInterface.php +++ b/src/QueryBuilder/DMLQueryBuilderInterface.php @@ -45,7 +45,7 @@ interface DMLQueryBuilderInterface * @return string The batch INSERT SQL statement. * * @psalm-param string[] $columns - * @psalm-param iterable> $rows + * @psalm-param iterable> $rows * @psalm-param ParamsType $params * * Note: From 24c277c807906214ebb6cae385ff78450176bb20 Mon Sep 17 00:00:00 2001 From: Tigrov Date: Sun, 14 Apr 2024 18:39:23 +0700 Subject: [PATCH 06/10] Remove redundant check --- src/QueryBuilder/AbstractDMLQueryBuilder.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/QueryBuilder/AbstractDMLQueryBuilder.php b/src/QueryBuilder/AbstractDMLQueryBuilder.php index 645ea7b5f..e7af38e74 100644 --- a/src/QueryBuilder/AbstractDMLQueryBuilder.php +++ b/src/QueryBuilder/AbstractDMLQueryBuilder.php @@ -75,10 +75,6 @@ public function batchInsert(string $table, array $columns, iterable $rows, array $columns = $this->extractColumnNames($rows, $columns); $values = $this->prepareBatchInsertValues($table, $rows, $columns, $params); - if (empty($values)) { - return ''; - } - $query = 'INSERT INTO ' . $this->quoter->quoteTableName($table); if (count($columns) > 0) { From 7bc5fccd2f15cb2e09a641ca30e1a16e5317609a Mon Sep 17 00:00:00 2001 From: Tigrov Date: Sun, 14 Apr 2024 18:39:52 +0700 Subject: [PATCH 07/10] Add line to CHANGELOG.md [skip ci] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5eab3b8c3..ed0fde599 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 2.0.0 under development +- Enh #820: Support `Traversable` values for `AbstractDMLQueryBuilder::batchInsert()` method with empty columns (@Tigrov) - Enh #816: Allow scalar values for `$columns` parameter of `Query::select()` and `Query::addSelect()` methods (@Tigrov) - Enh #806: Non-unique placeholder names inside `Expression::$params` will be replaced with unique names (@Tigrov) - Enh #806: Build `Expression` instances inside `Expression::$params` when build a query using `QueryBuilder` (@Tigrov) From 1b05d493fea952ee1c533f26be4196035117cbb6 Mon Sep 17 00:00:00 2001 From: Tigrov Date: Sun, 14 Apr 2024 21:09:24 +0700 Subject: [PATCH 08/10] Remove redundant check --- src/QueryBuilder/AbstractDMLQueryBuilder.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/QueryBuilder/AbstractDMLQueryBuilder.php b/src/QueryBuilder/AbstractDMLQueryBuilder.php index e7af38e74..6a1a290c1 100644 --- a/src/QueryBuilder/AbstractDMLQueryBuilder.php +++ b/src/QueryBuilder/AbstractDMLQueryBuilder.php @@ -147,11 +147,11 @@ protected function prepareTraversable(Traversable $rows): Iterator|array $rows = $rows->getIterator(); } - if ($rows instanceof Iterator && !$rows->valid()) { + /** @var Iterator $rows */ + if (!$rows->valid()) { return []; } - /** @var Iterator $rows */ return $rows; } From 8499d30e62423927cada149fce2214ad647ee71d Mon Sep 17 00:00:00 2001 From: Sergei Tigrov Date: Sat, 27 Apr 2024 08:52:06 +0700 Subject: [PATCH 09/10] Mark `AbstractDMLQueryBuilder::prepareTraversable()` as final by @vjik Co-authored-by: Sergei Predvoditelev --- src/QueryBuilder/AbstractDMLQueryBuilder.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/QueryBuilder/AbstractDMLQueryBuilder.php b/src/QueryBuilder/AbstractDMLQueryBuilder.php index 6a1a290c1..1a20cb8cd 100644 --- a/src/QueryBuilder/AbstractDMLQueryBuilder.php +++ b/src/QueryBuilder/AbstractDMLQueryBuilder.php @@ -141,7 +141,7 @@ public function upsert( * * @psalm-return Iterator|array> */ - protected function prepareTraversable(Traversable $rows): Iterator|array + final protected function prepareTraversable(Traversable $rows): Iterator|array { while ($rows instanceof IteratorAggregate) { $rows = $rows->getIterator(); From c4f61aaa27176e157858f01c34791600f289bc63 Mon Sep 17 00:00:00 2001 From: Tigrov Date: Sat, 27 Apr 2024 12:59:49 +0700 Subject: [PATCH 10/10] Fix tests --- tests/Provider/QueryBuilderProvider.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Provider/QueryBuilderProvider.php b/tests/Provider/QueryBuilderProvider.php index f8f0b5356..2ebd903ce 100644 --- a/tests/Provider/QueryBuilderProvider.php +++ b/tests/Provider/QueryBuilderProvider.php @@ -241,7 +241,7 @@ public static function batchInsert(): array 'empty columns and non-exists table' => [ 'non_exists_table', [], - 'values' => [['1.0', '2', 10, 1]], + [['1.0', '2', 10, 1]], 'expected' => DbHelper::replaceQuotes( <<