From c6aa29001c3dff12733d4d8d1610f86b22a68658 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sat, 13 Apr 2024 13:56:38 -0400 Subject: [PATCH 01/38] upgrade PHP and Laravel versions --- CHANGELOG.md | 6 ++++++ UPGRADING.md | 6 ++++++ composer.json | 14 +++++++------- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19dee9b..34b7c1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +# 6.0.0 +- Added support for PHP 8.3 +- Droppped support for PHP 8.0 and below +- Added support for Laravel 10 and 11 +- Dropped support Laravel versions 9 and below + # 5.0.1 - 2021-09-19 - Fixed `setManyMeta()` not properly serializing certain types of data. diff --git a/UPGRADING.md b/UPGRADING.md index 3864166..e9030d0 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -1,5 +1,11 @@ # Upgrading +## 5.X -> 6.X + +* Minimum PHP version moved to 8.1 +* Minimum Laravel version moved to 10 +* + ## 4.X -> 5.X - New migration new file added which adds a new composite unique index to the meta table on `metable_type`, `metable_id`, and `key`. Make sure that you have no duplicate keys for a given entity (previously possible as a race condition) before applying the new migration. diff --git a/composer.json b/composer.json index 7c10e61..baf8e16 100644 --- a/composer.json +++ b/composer.json @@ -15,21 +15,21 @@ } }, "require": { - "php": ">=7.3.0", + "php": ">=8.1", "ext-json": "*", - "illuminate/support": "^6.20.42|^8.22.1|^9.0", - "illuminate/database": "^6.20.42|^8.22.1|^9.0", + "illuminate/support": "^10.0|^11.0", + "illuminate/database": "^10.0|^11.0", "phpoption/phpoption": "^1.8" }, "require-dev": { - "symfony/symfony": "^5.4.1|^6.1", + "symfony/symfony": "^6.1|^7.0", "laravel/legacy-factories": "^1.0.4", - "orchestra/testbench": "^5.20|^6.23|^7.0", - "phpunit/phpunit": "^9.5.11", + "orchestra/testbench": "^8.0|^9.0", + "phpunit/phpunit": "^10.0", "guzzlehttp/guzzle": "^7.2", "guzzlehttp/promises": "^1.4", "mockery/mockery": "^1.4.2", - "php-coveralls/php-coveralls": "^2.4.2" + "php-coveralls/php-coveralls": "^2.5.2" }, "autoload-dev":{ "psr-4": { From b87c2dc6ae833dc15170be3b25b28d6fc1fa5cb8 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sat, 13 Apr 2024 13:57:14 -0400 Subject: [PATCH 02/38] upgrade phpunit version --- .gitignore | 3 ++- phpunit.xml | 22 ++++++---------------- tests/Integration/DataType/HandlerTest.php | 2 +- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index e7929dd..e9eab1d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ /composer.lock /coverage/ /.idea/ -.phpunit.result.cache +/.phpunit.result.cache +/.phpunit.cache diff --git a/phpunit.xml b/phpunit.xml index f9d78c7..c0dcf2e 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -1,20 +1,5 @@ - - - - ./src/ - - + ./tests/Integration/ @@ -23,4 +8,9 @@ + + + ./src/ + + diff --git a/tests/Integration/DataType/HandlerTest.php b/tests/Integration/DataType/HandlerTest.php index 187c1d4..9a9eb76 100644 --- a/tests/Integration/DataType/HandlerTest.php +++ b/tests/Integration/DataType/HandlerTest.php @@ -23,7 +23,7 @@ class HandlerTest extends TestCase { - public function handlerProvider() + static public function handlerProvider() { $timestamp = '2017-01-01 00:00:00.000000+0000'; $datetime = Carbon::createFromFormat('Y-m-d H:i:s.uO', $timestamp); From d498f9845d914c1f55d46c6d732ea4859428f247 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sat, 13 Apr 2024 13:59:19 -0400 Subject: [PATCH 03/38] update github actions --- .github/workflows/automated-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/automated-test.yml b/.github/workflows/automated-test.yml index 64c8941..98fbd53 100644 --- a/.github/workflows/automated-test.yml +++ b/.github/workflows/automated-test.yml @@ -5,12 +5,12 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - php-versions: ['7.4', '8.0', '8.1'] + php-versions: ['8.1', '8.2', '8.3'] prefer-lowest: ['','--prefer-lowest'] name: PHP ${{ matrix.php-versions }} ${{ matrix.prefer-lowest }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup PHP uses: shivammathur/setup-php@v2 @@ -22,10 +22,10 @@ jobs: - name: Get composer cache directory id: composer-cache - run: echo "::set-output name=dir::$(composer config cache-files-dir)" + run: echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT - name: Cache dependencies - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ${{ steps.composer-cache.outputs.dir }} key: ${{ runner.os }}-${{ matrix.php-version }}${{ matrix.prefer-lowest }}-composer-${{ hashFiles('**/composer.json') }} From 516e4c33eac302718d8c190473cc1719ac82529e Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sat, 13 Apr 2024 14:18:13 -0400 Subject: [PATCH 04/38] signature modernization --- CHANGELOG.md | 1 + UPGRADING.md | 2 +- src/DataType/ArrayHandler.php | 8 ++--- src/DataType/DateTimeHandler.php | 8 ++--- src/DataType/HandlerInterface.php | 6 ++-- src/DataType/ModelCollectionHandler.php | 8 ++--- src/DataType/ModelHandler.php | 12 ++++---- src/DataType/ObjectHandler.php | 8 ++--- src/DataType/ScalarHandler.php | 6 ++-- src/DataType/SerializableHandler.php | 8 ++--- src/Meta.php | 2 +- src/Metable.php | 39 ++++++++++++------------- 12 files changed, 53 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34b7c1e..39b3527 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - Droppped support for PHP 8.0 and below - Added support for Laravel 10 and 11 - Dropped support Laravel versions 9 and below +- adjusted some method signatures with PHP 8+ mixed and union types # 5.0.1 - 2021-09-19 - Fixed `setManyMeta()` not properly serializing certain types of data. diff --git a/UPGRADING.md b/UPGRADING.md index e9030d0..c5eb548 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -4,7 +4,7 @@ * Minimum PHP version moved to 8.1 * Minimum Laravel version moved to 10 -* +* Some methods have had their signatures adjusted to use PHP 8+ mixed and union types. If extending any class or implementing any interface from this package, method signatures may need to be updated. ## 4.X -> 5.X - New migration new file added which adds a new composite unique index to the meta table on `metable_type`, `metable_id`, and `key`. Make sure that you have no duplicate keys for a given entity (previously possible as a race condition) before applying the new migration. diff --git a/src/DataType/ArrayHandler.php b/src/DataType/ArrayHandler.php index 8c3c026..a0bd164 100644 --- a/src/DataType/ArrayHandler.php +++ b/src/DataType/ArrayHandler.php @@ -18,7 +18,7 @@ public function getDataType(): string /** * {@inheritdoc} */ - public function canHandleValue($value): bool + public function canHandleValue(mixed $value): bool { return is_array($value); } @@ -26,7 +26,7 @@ public function canHandleValue($value): bool /** * {@inheritdoc} */ - public function serializeValue($value): string + public function serializeValue(mixed $value): string { return json_encode($value); } @@ -34,8 +34,8 @@ public function serializeValue($value): string /** * {@inheritdoc} */ - public function unserializeValue(string $value) + public function unserializeValue(string $serializedValue): mixed { - return json_decode($value, true); + return json_decode($serializedValue, true); } } diff --git a/src/DataType/DateTimeHandler.php b/src/DataType/DateTimeHandler.php index c347e10..c9d1e96 100644 --- a/src/DataType/DateTimeHandler.php +++ b/src/DataType/DateTimeHandler.php @@ -28,7 +28,7 @@ public function getDataType(): string /** * {@inheritdoc} */ - public function canHandleValue($value): bool + public function canHandleValue(mixed $value): bool { return $value instanceof DateTimeInterface; } @@ -36,7 +36,7 @@ public function canHandleValue($value): bool /** * {@inheritdoc} */ - public function serializeValue($value): string + public function serializeValue(mixed $value): string { return $value->format($this->format); } @@ -44,8 +44,8 @@ public function serializeValue($value): string /** * {@inheritdoc} */ - public function unserializeValue(string $value) + public function unserializeValue(string $serializedValue): mixed { - return Carbon::createFromFormat($this->format, $value); + return Carbon::createFromFormat($this->format, $serializedValue); } } diff --git a/src/DataType/HandlerInterface.php b/src/DataType/HandlerInterface.php index 0be2be5..d9818fb 100644 --- a/src/DataType/HandlerInterface.php +++ b/src/DataType/HandlerInterface.php @@ -21,7 +21,7 @@ public function getDataType(): string; * * @return bool */ - public function canHandleValue($value): bool; + public function canHandleValue(mixed $value): bool; /** * Convert the value to a string, so that it can be stored in the database. @@ -30,7 +30,7 @@ public function canHandleValue($value): bool; * * @return string */ - public function serializeValue($value): string; + public function serializeValue(mixed $value): string; /** * Convert a serialized string back to its original value. @@ -39,5 +39,5 @@ public function serializeValue($value): string; * * @return mixed */ - public function unserializeValue(string $serializedValue); + public function unserializeValue(string $serializedValue): mixed; } diff --git a/src/DataType/ModelCollectionHandler.php b/src/DataType/ModelCollectionHandler.php index 2bdbcf6..41c4a9e 100644 --- a/src/DataType/ModelCollectionHandler.php +++ b/src/DataType/ModelCollectionHandler.php @@ -20,7 +20,7 @@ public function getDataType(): string /** * {@inheritdoc} */ - public function canHandleValue($value): bool + public function canHandleValue(mixed $value): bool { return $value instanceof Collection; } @@ -28,7 +28,7 @@ public function canHandleValue($value): bool /** * {@inheritdoc} */ - public function serializeValue($value): string + public function serializeValue(mixed $value): string { $items = []; foreach ($value as $key => $model) { @@ -44,9 +44,9 @@ public function serializeValue($value): string /** * {@inheritdoc} */ - public function unserializeValue(string $value) + public function unserializeValue(string $serializedValue): mixed { - $data = json_decode($value, true); + $data = json_decode($serializedValue, true); $collection = new $data['class'](); $models = $this->loadModels($data['items']); diff --git a/src/DataType/ModelHandler.php b/src/DataType/ModelHandler.php index bfb9994..9cc8943 100644 --- a/src/DataType/ModelHandler.php +++ b/src/DataType/ModelHandler.php @@ -20,7 +20,7 @@ public function getDataType(): string /** * {@inheritdoc} */ - public function canHandleValue($value): bool + public function canHandleValue(mixed $value): bool { return $value instanceof Model; } @@ -28,7 +28,7 @@ public function canHandleValue($value): bool /** * {@inheritdoc} */ - public function serializeValue($value): string + public function serializeValue(mixed $value): string { if ($value->exists) { return get_class($value) . '#' . $value->getKey(); @@ -40,15 +40,15 @@ public function serializeValue($value): string /** * {@inheritdoc} */ - public function unserializeValue(string $value) + public function unserializeValue(string $serializedValue): mixed { // Return blank instances. - if (strpos($value, '#') === false) { - return new $value(); + if (strpos($serializedValue, '#') === false) { + return new $serializedValue(); } // Fetch specific instances. - list($class, $id) = explode('#', $value); + list($class, $id) = explode('#', $serializedValue); return with(new $class())->findOrFail($id); } diff --git a/src/DataType/ObjectHandler.php b/src/DataType/ObjectHandler.php index 68c02de..7cb7049 100644 --- a/src/DataType/ObjectHandler.php +++ b/src/DataType/ObjectHandler.php @@ -18,7 +18,7 @@ public function getDataType(): string /** * {@inheritdoc} */ - public function canHandleValue($value): bool + public function canHandleValue(mixed $value): bool { return is_object($value); } @@ -26,7 +26,7 @@ public function canHandleValue($value): bool /** * {@inheritdoc} */ - public function serializeValue($value): string + public function serializeValue(mixed $value): string { return json_encode($value); } @@ -34,8 +34,8 @@ public function serializeValue($value): string /** * {@inheritdoc} */ - public function unserializeValue(string $value) + public function unserializeValue(string $serializedValue): mixed { - return json_decode($value, false); + return json_decode($serializedValue, false); } } diff --git a/src/DataType/ScalarHandler.php b/src/DataType/ScalarHandler.php index 5ad5955..f1b838f 100644 --- a/src/DataType/ScalarHandler.php +++ b/src/DataType/ScalarHandler.php @@ -25,7 +25,7 @@ public function getDataType(): string /** * {@inheritdoc} */ - public function canHandleValue($value): bool + public function canHandleValue(mixed $value): bool { return gettype($value) == $this->type; } @@ -33,7 +33,7 @@ public function canHandleValue($value): bool /** * {@inheritdoc} */ - public function serializeValue($value): string + public function serializeValue(mixed $value): string { settype($value, 'string'); @@ -43,7 +43,7 @@ public function serializeValue($value): string /** * {@inheritdoc} */ - public function unserializeValue(string $value) + public function unserializeValue(string $value): mixed { settype($value, $this->type); diff --git a/src/DataType/SerializableHandler.php b/src/DataType/SerializableHandler.php index 04b5ffd..6e8e6b6 100644 --- a/src/DataType/SerializableHandler.php +++ b/src/DataType/SerializableHandler.php @@ -20,7 +20,7 @@ public function getDataType(): string /** * {@inheritdoc} */ - public function canHandleValue($value): bool + public function canHandleValue(mixed $value): bool { return $value instanceof Serializable; } @@ -28,7 +28,7 @@ public function canHandleValue($value): bool /** * {@inheritdoc} */ - public function serializeValue($value): string + public function serializeValue(mixed $value): string { return serialize($value); } @@ -36,8 +36,8 @@ public function serializeValue($value): string /** * {@inheritdoc} */ - public function unserializeValue(string $value) + public function unserializeValue(string $serializedValue): mixed { - return unserialize($value); + return unserialize($serializedValue); } } diff --git a/src/Meta.php b/src/Meta.php index f385355..5b34218 100644 --- a/src/Meta.php +++ b/src/Meta.php @@ -47,7 +47,7 @@ class Meta extends Model * * @var mixed */ - protected $cachedValue; + protected mixed $cachedValue; /** * Metable Relation. diff --git a/src/Metable.php b/src/Metable.php index 149c1c7..e3dac21 100644 --- a/src/Metable.php +++ b/src/Metable.php @@ -5,29 +5,26 @@ use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Relations\MorphMany; -use Illuminate\Database\Query\Expression; use Illuminate\Database\Query\JoinClause; -use Illuminate\Support\Arr; use Illuminate\Support\Facades\DB; -use Traversable; /** * Trait for giving Eloquent models the ability to handle Meta. * - * @property Collection|Meta[] $meta - * @method static Builder whereHasMeta($key): void - * @method static Builder WhereDoesntHaveMeta($key) + * @property Collection $meta + * @method static Builder whereHasMeta(string|string[] $key): void + * @method static Builder WhereDoesntHaveMeta(string|string[] $key) * @method static Builder WhereHasMetaKeys(array $keys) - * @method static Builder WhereMeta(string $key, $operator, $value = null) - * @method static Builder WhereMetaNumeric(string $key, string $operator, $value) + * @method static Builder WhereMeta(string $key, $operator, mixed $value = null) + * @method static Builder WhereMetaNumeric(string $key, string $operator, mixed $value) * @method static Builder WhereMetaIn(string $key, array $values) - * @method static Builder OrderByMeta(string $key, string $direction = 'asc', $strict = false) - * @method static Builder OrderByMetaNumeric(string $key, string $direction = 'asc', $strict = false) + * @method static Builder OrderByMeta(string $key, string $direction = 'asc', bool $strict = false) + * @method static Builder OrderByMetaNumeric(string $key, string $direction = 'asc', bool $strict = false) */ trait Metable { /** - * @var Collection|Meta[] + * @var Collection */ private $indexedMetaCollection; @@ -63,7 +60,7 @@ public function meta(): MorphMany * @param string $key * @param mixed $value */ - public function setMeta(string $key, $value): void + public function setMeta(string $key, mixed $value): void { if ($this->hasMeta($key)) { $meta = $this->getMetaRecord($key); @@ -157,7 +154,7 @@ public function syncMeta(iterable $array): void * * @return mixed */ - public function getMeta(string $key, $default = null) + public function getMeta(string $key, mixed $default = null): mixed { if ($this->hasMeta($key)) { return $this->getMetaRecord($key)->getAttribute('value'); @@ -190,7 +187,7 @@ protected function hasDefaultMetaValue(string $key): bool * @param string $key * @return mixed */ - protected function getDefaultMetaValue(string $key) + protected function getDefaultMetaValue(string $key): mixed { return $this->defaultMetaValues[$key]; } @@ -285,11 +282,11 @@ public function getMetaRecord(string $key): ?Meta * If an array of keys is passed instead, will restrict the query to records having one or more Meta with any of the keys. * * @param Builder $q - * @param string|array $key + * @param string|string[] $key * * @return void */ - public function scopeWhereHasMeta(Builder $q, $key): void + public function scopeWhereHasMeta(Builder $q, string|array $key): void { $q->whereHas('meta', function (Builder $q) use ($key) { $q->whereIn('key', (array)$key); @@ -302,11 +299,11 @@ public function scopeWhereHasMeta(Builder $q, $key): void * If an array of keys is passed instead, will restrict the query to records having one or more Meta with any of the keys. * * @param Builder $q - * @param string|array $key + * @param string|string[] $key * * @return void */ - public function scopeWhereDoesntHaveMeta(Builder $q, $key): void + public function scopeWhereDoesntHaveMeta(Builder $q, string|array $key): void { $q->whereDoesntHave('meta', function (Builder $q) use ($key) { $q->whereIn('key', (array)$key); @@ -347,7 +344,7 @@ function (Builder $q) use ($keys) { * * @return void */ - public function scopeWhereMeta(Builder $q, string $key, $operator, $value = null): void + public function scopeWhereMeta(Builder $q, string $key, mixed $operator, mixed $value = null): void { // Shift arguments if no operator is present. if (!isset($value)) { @@ -378,7 +375,7 @@ public function scopeWhereMeta(Builder $q, string $key, $operator, $value = null * * @return void */ - public function scopeWhereMetaNumeric(Builder $q, string $key, string $operator, $value): void + public function scopeWhereMetaNumeric(Builder $q, string $key, string $operator, int|float $value): void { // Since we are manually interpolating into the query, // escape the operator to protect against injection. @@ -562,7 +559,7 @@ protected function getMetaClassName(): string * * @return Meta */ - protected function makeMeta(string $key = '', $value = ''): Meta + protected function makeMeta(string $key = '', mixed $value = ''): Meta { $className = $this->getMetaClassName(); From 256ce65a977d4e16b257ef58a25f656e2347b8f3 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sat, 13 Apr 2024 14:22:37 -0400 Subject: [PATCH 05/38] modernize migrations --- migrations/2017_01_01_000000_create_meta_table.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/2017_01_01_000000_create_meta_table.php b/migrations/2017_01_01_000000_create_meta_table.php index 52a7bf7..96ff75d 100644 --- a/migrations/2017_01_01_000000_create_meta_table.php +++ b/migrations/2017_01_01_000000_create_meta_table.php @@ -15,7 +15,7 @@ public function up() { if (!Schema::hasTable('meta')) { Schema::create('meta', function (Blueprint $table) { - $table->increments('id'); + $table->id(); $table->string('metable_type'); $table->unsignedInteger('metable_id'); $table->string('type')->default('null'); From 5810dbca939a4008095b37964d7a2a0a679d4216 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sat, 13 Apr 2024 14:32:17 -0400 Subject: [PATCH 06/38] remove unnecessary backwards compatibility code --- src/Metable.php | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/src/Metable.php b/src/Metable.php index e3dac21..509fc9b 100644 --- a/src/Metable.php +++ b/src/Metable.php @@ -91,32 +91,19 @@ public function setManyMeta(array $metaDictionary): void $builder = DB::table($prototype->getTable()); $needReload = $this->relationLoaded('meta'); - if (method_exists($builder, 'upsert')) { - // use upsert if available to store all data in a single query - // requires Laravel >8.0 - $metaModels = new Collection(); - foreach ($metaDictionary as $key => $value) { - $metaModels[$key] = $this->makeMeta($key, $value); - } - - $builder->upsert( - $metaModels->map(function (Meta $model) { - return method_exists($model, 'getAttributesForInsert') - ? $model->getAttributesForInsert() // Laravel >= 8.0 - : $model->getAttributes(); - })->all(), - ['metable_type', 'metable_id', 'key'], - ['type', 'value'] - ); - } else { - // otherwise insert manually. - // Clear local cache to speed things up since we will reload it afterwards - $this->unsetRelation('meta'); - foreach ($metaDictionary as $key => $value) { - $this->setMeta($key, $value); - } + $metaModels = new Collection(); + foreach ($metaDictionary as $key => $value) { + $metaModels[$key] = $this->makeMeta($key, $value); } + $builder->upsert( + $metaModels->map(function (Meta $model) { + return $model->getAttributesForInsert(); + })->all(), + ['metable_type', 'metable_id', 'key'], + ['type', 'value'] + ); + if ($needReload) { // reload media relation and indexed cache $this->load('meta'); From 09133c5e09da7602a5147afbffb9cc11d39e7583 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sat, 13 Apr 2024 14:49:15 -0400 Subject: [PATCH 07/38] update test suite typing --- tests/Integration/DataType/HandlerTest.php | 27 +++----- .../DataType/ModelCollectionHandlerTest.php | 2 +- .../Integration/DataType/ModelHandlerTest.php | 2 +- tests/Integration/DataType/RegistryTest.php | 10 +-- tests/Integration/MetaTest.php | 10 +-- .../MetableServiceProviderTest.php | 4 +- tests/Integration/MetableTest.php | 68 +++++++++---------- tests/Integration/MorphTest.php | 6 +- tests/Mocks/SampleSerializable.php | 10 +-- tests/factories/MetaFactory.php | 1 + tests/factories/MetableFactory.php | 2 + tests/factories/MorphFactory.php | 1 + ...01_000000_create_sample_classes_tables.php | 2 +- 13 files changed, 70 insertions(+), 75 deletions(-) diff --git a/tests/Integration/DataType/HandlerTest.php b/tests/Integration/DataType/HandlerTest.php index 9a9eb76..712c552 100644 --- a/tests/Integration/DataType/HandlerTest.php +++ b/tests/Integration/DataType/HandlerTest.php @@ -23,7 +23,7 @@ class HandlerTest extends TestCase { - static public function handlerProvider() + static public function handlerProvider(): array { $timestamp = '2017-01-01 00:00:00.000000+0000'; $datetime = Carbon::createFromFormat('Y-m-d H:i:s.uO', $timestamp); @@ -105,28 +105,19 @@ static public function handlerProvider() /** * @dataProvider handlerProvider */ - public function test_it_specifies_a_datatype_identifier(HandlerInterface $handler, $type) - { + public function test_it_can_verify_and_serialize_data( + HandlerInterface $handler, + string $type, + mixed $value, + array $incompatible + ): void { $this->assertEquals($type, $handler->getDataType()); - } - - /** - * @dataProvider handlerProvider - */ - public function test_it_can_verify_compatibility(HandlerInterface $handler, $type, $value, $incompatible) - { $this->assertTrue($handler->canHandleValue($value)); - foreach ($incompatible as $value) { - $this->assertFalse($handler->canHandleValue($value)); + foreach ($incompatible as $incompatibleValue) { + $this->assertFalse($handler->canHandleValue($incompatibleValue)); } - } - /** - * @dataProvider handlerProvider - */ - public function test_it_can_serialize_and_unserialize_values(HandlerInterface $handler, $type, $value) - { $serialized = $handler->serializeValue($value); $unserialized = $handler->unserializeValue($serialized); diff --git a/tests/Integration/DataType/ModelCollectionHandlerTest.php b/tests/Integration/DataType/ModelCollectionHandlerTest.php index 7962ded..165dbdf 100644 --- a/tests/Integration/DataType/ModelCollectionHandlerTest.php +++ b/tests/Integration/DataType/ModelCollectionHandlerTest.php @@ -9,7 +9,7 @@ class ModelCollectionHandlerTest extends TestCase { - public function test_it_reloads_model_instances() + public function test_it_reloads_model_instances(): void { $this->useDatabase(); diff --git a/tests/Integration/DataType/ModelHandlerTest.php b/tests/Integration/DataType/ModelHandlerTest.php index 53ca76f..23783c9 100644 --- a/tests/Integration/DataType/ModelHandlerTest.php +++ b/tests/Integration/DataType/ModelHandlerTest.php @@ -8,7 +8,7 @@ class ModelHandlerTest extends TestCase { - public function test_it_reloads_a_model_instance() + public function test_it_reloads_a_model_instance(): void { $this->useDatabase(); diff --git a/tests/Integration/DataType/RegistryTest.php b/tests/Integration/DataType/RegistryTest.php index 2787770..d06b72c 100644 --- a/tests/Integration/DataType/RegistryTest.php +++ b/tests/Integration/DataType/RegistryTest.php @@ -9,7 +9,7 @@ class RegistryTest extends TestCase { - public function test_it_can_set_a_handler() + public function test_it_can_set_a_handler(): void { $registry = new Registry(); $handler = $this->mockHandlerWithType('foo'); @@ -21,7 +21,7 @@ public function test_it_can_set_a_handler() $this->assertEquals($handler, $registry->getHandlerForType('foo')); } - public function test_it_can_remove_a_handler() + public function test_it_can_remove_a_handler(): void { $registry = new Registry(); $handler = $this->mockHandlerWithType('foo'); @@ -33,7 +33,7 @@ public function test_it_can_remove_a_handler() $this->assertFalse($registry->hasHandlerForType('foo')); } - public function test_it_throws_an_exception_if_no_handler_set() + public function test_it_throws_an_exception_if_no_handler_set(): void { $registry = new Registry(); @@ -41,7 +41,7 @@ public function test_it_throws_an_exception_if_no_handler_set() $registry->getHandlerForType('foo'); } - public function test_it_determines_best_handler_for_a_value() + public function test_it_determines_best_handler_for_a_value(): void { $stringHandler = $this->mockHandlerWithType('str'); $stringHandler->method('canHandleValue') @@ -64,7 +64,7 @@ public function test_it_determines_best_handler_for_a_value() $this->assertEquals('str', $type2); } - public function test_it_throws_an_exception_if_no_type_matches_value() + public function test_it_throws_an_exception_if_no_type_matches_value(): void { $registry = new Registry(); diff --git a/tests/Integration/MetaTest.php b/tests/Integration/MetaTest.php index 23e8e3f..4ff2b00 100644 --- a/tests/Integration/MetaTest.php +++ b/tests/Integration/MetaTest.php @@ -8,7 +8,7 @@ class MetaTest extends TestCase { - public function test_it_can_get_and_set_value() + public function test_it_can_get_and_set_value(): void { $meta = $this->makeMeta(); @@ -18,7 +18,7 @@ public function test_it_can_get_and_set_value() $this->assertEquals('string', $meta->type); } - public function test_it_exposes_its_serialized_value() + public function test_it_exposes_its_serialized_value(): void { $meta = $this->makeMeta(); $meta->value = 123; @@ -26,7 +26,7 @@ public function test_it_exposes_its_serialized_value() $this->assertEquals('123', $meta->getRawValue()); } - public function test_it_caches_unserialized_value() + public function test_it_caches_unserialized_value(): void { $meta = $this->makeMeta(); $meta->value = 'foo'; @@ -38,7 +38,7 @@ public function test_it_caches_unserialized_value() $this->assertEquals('bar', $meta->getRawValue()); } - public function test_it_clears_cache_on_set() + public function test_it_clears_cache_on_set(): void { $meta = $this->makeMeta(); $meta->value = 'foo'; @@ -49,7 +49,7 @@ public function test_it_clears_cache_on_set() $this->assertEquals('bar', $meta->value); } - public function test_it_can_get_its_model_relation() + public function test_it_can_get_its_model_relation(): void { $meta = $this->makeMeta(); diff --git a/tests/Integration/MetableServiceProviderTest.php b/tests/Integration/MetableServiceProviderTest.php index 8398461..0ddbca0 100644 --- a/tests/Integration/MetableServiceProviderTest.php +++ b/tests/Integration/MetableServiceProviderTest.php @@ -12,7 +12,7 @@ protected function getPackageProviders($app) return []; } - public function testBootSkipsMigrations() + public function testBootSkipsMigrations(): void { config()->set('metable.applyMigrations', false); $provider = new MetableServiceProvider(app()); @@ -20,7 +20,7 @@ public function testBootSkipsMigrations() $this->assertEmpty(app('migrator')->paths()); } - public function testBootAppliesMigrations() + public function testBootAppliesMigrations(): void { config()->set('metable.applyMigrations', true); $provider = new MetableServiceProvider(app()); diff --git a/tests/Integration/MetableTest.php b/tests/Integration/MetableTest.php index 8434d97..4c67688 100644 --- a/tests/Integration/MetableTest.php +++ b/tests/Integration/MetableTest.php @@ -11,7 +11,7 @@ class MetableTest extends TestCase { - public function test_it_can_get_and_set_meta_value_by_key() + public function test_it_can_get_and_set_meta_value_by_key(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -30,7 +30,7 @@ public function test_it_can_get_and_set_meta_value_by_key() $this->assertCount(1, $metable->meta); } - public function test_it_can_set_many_meta_values_at_once() + public function test_it_can_set_many_meta_values_at_once(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -53,7 +53,7 @@ public function test_it_can_set_many_meta_values_at_once() $this->assertEquals(['foo', 'bar'], $metable->getMeta('baz')); } - public function test_it_accepts_empty_array_for_set_many_meta() + public function test_it_accepts_empty_array_for_set_many_meta(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -62,7 +62,7 @@ public function test_it_accepts_empty_array_for_set_many_meta() $this->assertEquals('old', $metable->getMeta('foo')); } - public function test_it_can_set_uppercase_key() + public function test_it_can_set_uppercase_key(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -74,7 +74,7 @@ public function test_it_can_set_uppercase_key() $this->assertEquals('bar', $metable->getMeta('FOO')); } - public function test_it_can_get_meta_record() + public function test_it_can_get_meta_record(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -86,7 +86,7 @@ public function test_it_can_get_meta_record() $this->assertEquals(123, $record->value); } - public function test_it_can_get_meta_all_values() + public function test_it_can_get_meta_all_values(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -103,7 +103,7 @@ public function test_it_can_get_meta_all_values() ], $collection->toArray()); } - public function test_it_updates_existing_meta_records() + public function test_it_updates_existing_meta_records(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -117,7 +117,7 @@ public function test_it_updates_existing_meta_records() $this->assertEquals(321, $new_record->value); } - public function test_it_returns_default_value_if_no_meta_set() + public function test_it_returns_default_value_if_no_meta_set(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -127,7 +127,7 @@ public function test_it_returns_default_value_if_no_meta_set() $this->assertEquals('not-found', $result); } - public function test_it_can_replace_all_keys() + public function test_it_can_replace_all_keys(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -146,7 +146,7 @@ public function test_it_can_replace_all_keys() $this->assertEquals('d', $metable->getMeta('c')); } - public function test_it_can_delete_meta() + public function test_it_can_delete_meta(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -158,7 +158,7 @@ public function test_it_can_delete_meta() $this->assertFalse($metable->fresh()->hasMeta('foo')); } - public function test_it_can_delete_meta_not_set() + public function test_it_can_delete_meta_not_set(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -169,7 +169,7 @@ public function test_it_can_delete_meta_not_set() $this->assertFalse($metable->fresh()->hasMeta('foo')); } - public function test_it_can_delete_many_meta_at_once() + public function test_it_can_delete_many_meta_at_once(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -190,7 +190,7 @@ public function test_it_can_delete_many_meta_at_once() $this->assertFalse($metable->hasMeta('baz')); } - public function test_it_can_delete_all_meta() + public function test_it_can_delete_all_meta(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -203,7 +203,7 @@ public function test_it_can_delete_all_meta() $this->assertEquals(0, $metable->meta()->count()); } - public function test_it_clears_meta_on_deletion() + public function test_it_clears_meta_on_deletion(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -215,7 +215,7 @@ public function test_it_clears_meta_on_deletion() $this->assertEquals(0, $meta->count()); } - public function test_it_does_not_clear_meta_on_soft_deletion() + public function test_it_does_not_clear_meta_on_soft_deletion(): void { $this->useDatabase(); $metable = $this->createMetableSoftDeletes(); @@ -227,7 +227,7 @@ public function test_it_does_not_clear_meta_on_soft_deletion() $this->assertEquals(1, $meta->count()); } - public function test_it_does_clear_meta_on_force_deletion() + public function test_it_does_clear_meta_on_force_deletion(): void { $this->useDatabase(); $metable = $this->createMetableSoftDeletes(); @@ -239,7 +239,7 @@ public function test_it_does_clear_meta_on_force_deletion() $this->assertEquals(0, $meta->count()); } - public function test_it_can_be_queried_by_single_meta_key() + public function test_it_can_be_queried_by_single_meta_key(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -250,7 +250,7 @@ public function test_it_can_be_queried_by_single_meta_key() $this->assertEquals($metable->getKey(), $result->getKey()); } - public function test_it_can_retrieve_model_default_value() + public function test_it_can_retrieve_model_default_value(): void { $this->useDatabase(); $result = $this->makeMetable(); @@ -259,7 +259,7 @@ public function test_it_can_retrieve_model_default_value() $this->assertEquals(['foo' => 'bar'], $result->getAllMeta()->toArray()); } - public function test_it_can_get_database_before_default_value() + public function test_it_can_get_database_before_default_value(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -271,7 +271,7 @@ public function test_it_can_get_database_before_default_value() $this->assertEquals(['foo' => 'baz'], $result->getAllMeta()->toArray()); } - public function test_it_can_get_passed_default_before_model_default_value() + public function test_it_can_get_passed_default_before_model_default_value(): void { $this->useDatabase(); $this->createMetable(); @@ -281,7 +281,7 @@ public function test_it_can_get_passed_default_before_model_default_value() $this->assertEquals($result->getMeta('foo', null), null); } - public function test_it_can_be_queried_by_missing_single_meta_key() + public function test_it_can_be_queried_by_missing_single_meta_key(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -296,7 +296,7 @@ public function test_it_can_be_queried_by_missing_single_meta_key() $this->assertNotEquals($metable->getKey(), $result->getKey()); } - public function test_it_can_be_queried_by_any_meta_keys() + public function test_it_can_be_queried_by_any_meta_keys(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -310,7 +310,7 @@ public function test_it_can_be_queried_by_any_meta_keys() $this->assertEquals($metable->getKey(), $result2->getKey()); } - public function test_it_can_be_queried_by_any_missing_meta_keys() + public function test_it_can_be_queried_by_any_missing_meta_keys(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -331,7 +331,7 @@ public function test_it_can_be_queried_by_any_missing_meta_keys() $this->assertNotEquals($metable->getKey(), $result2->getKey()); } - public function test_it_can_be_queried_by_all_meta_keys() + public function test_it_can_be_queried_by_all_meta_keys(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -345,7 +345,7 @@ public function test_it_can_be_queried_by_all_meta_keys() $this->assertNull($result2); } - public function test_it_can_be_queried_by_meta_value() + public function test_it_can_be_queried_by_meta_value(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -361,7 +361,7 @@ public function test_it_can_be_queried_by_meta_value() $this->assertEquals($metable->getKey(), $result3->getKey()); } - public function test_it_can_be_queried_by_numeric_meta_value() + public function test_it_can_be_queried_by_numeric_meta_value(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -372,7 +372,7 @@ public function test_it_can_be_queried_by_numeric_meta_value() $this->assertEquals($metable->getKey(), $result->getKey()); } - public function test_it_can_be_queried_by_in_array() + public function test_it_can_be_queried_by_in_array(): void { $this->useDatabase(); $metable = $this->createMetable(); @@ -385,7 +385,7 @@ public function test_it_can_be_queried_by_in_array() $this->assertNull($result2); } - public function test_it_can_order_query_by_meta_value() + public function test_it_can_order_query_by_meta_value(): void { $this->useDatabase(); $metable1 = $this->createMetable(['id' => 1]); @@ -402,7 +402,7 @@ public function test_it_can_order_query_by_meta_value() $this->assertEquals([2, 1, 3], $results2->pluck('id')->toArray()); } - public function test_it_can_order_query_by_meta_value_strict() + public function test_it_can_order_query_by_meta_value_strict(): void { $this->useDatabase(); $metable1 = $this->createMetable(['id' => 1]); @@ -419,7 +419,7 @@ public function test_it_can_order_query_by_meta_value_strict() $this->assertEquals([1, 3], $results2->pluck('id')->toArray()); } - public function test_it_can_order_query_by_numeric_meta_value() + public function test_it_can_order_query_by_numeric_meta_value(): void { $this->useDatabase(); $metable1 = $this->createMetable(['id' => 1]); @@ -436,7 +436,7 @@ public function test_it_can_order_query_by_numeric_meta_value() $this->assertEquals([1, 3, 2], $results2->pluck('id')->toArray()); } - public function test_it_can_order_query_by_numeric_meta_value_strict() + public function test_it_can_order_query_by_numeric_meta_value_strict(): void { $this->useDatabase(); $metable1 = $this->createMetable(['id' => 1]); @@ -453,7 +453,7 @@ public function test_it_can_order_query_by_numeric_meta_value_strict() $this->assertEquals([1, 3], $results2->pluck('id')->toArray()); } - public function test_set_relation_updates_index() + public function test_set_relation_updates_index(): void { $metable = $this->makeMetable(); $meta = $this->makeMeta(['key' => 'foo', 'value' => 'bar']); @@ -477,7 +477,7 @@ public function test_set_relation_updates_index() $this->assertEquals($emptyCollection, $method->invoke($metable)); } - public function test_set_relations_updates_index() + public function test_set_relations_updates_index(): void { $metable = $this->makeMetable(); $meta = $this->makeMeta(['key' => 'foo', 'value' => 'bar']); @@ -502,7 +502,7 @@ public function test_set_relations_updates_index() $this->assertEquals($emptyCollection, $method->invoke($metable)); } - public function test_it_can_serialize_properly() + public function test_it_can_serialize_properly(): void { $metable = $this->makeMetable(); $meta = $this->makeMeta(['key' => 'foo', 'value' => 'baz']); diff --git a/tests/Integration/MorphTest.php b/tests/Integration/MorphTest.php index 8b85571..d99f2fc 100644 --- a/tests/Integration/MorphTest.php +++ b/tests/Integration/MorphTest.php @@ -9,7 +9,7 @@ class MorphTest extends TestCase { - public function test_it_can_get_and_set_meta_value_by_key() + public function test_it_can_get_and_set_meta_value_by_key(): void { $this->useDatabase(); $child = $this->createChild(); @@ -21,7 +21,7 @@ public function test_it_can_get_and_set_meta_value_by_key() $this->assertEquals('bar', $child->getMeta('foo')); } - public function test_it_can_get_meta_record() + public function test_it_can_get_meta_record(): void { $this->useDatabase(); $child = $this->createChild(); @@ -35,7 +35,7 @@ public function test_it_can_get_meta_record() $this->assertEquals($class, $record->metable_type); } - public function test_it_can_join_correctly_from_morphed_class() + public function test_it_can_join_correctly_from_morphed_class(): void { $this->useDatabase(); $metable1 = $this->createMetable(['id' => 1]); diff --git a/tests/Mocks/SampleSerializable.php b/tests/Mocks/SampleSerializable.php index 67eef00..34a8d68 100644 --- a/tests/Mocks/SampleSerializable.php +++ b/tests/Mocks/SampleSerializable.php @@ -8,7 +8,7 @@ class SampleSerializable implements Serializable { public $data; - public function __construct($data) + public function __construct(mixed $data) { $this->data = $data; } @@ -18,9 +18,9 @@ public function serialize() return serialize($this->data); } - public function unserialize($serialized) + public function unserialize(string $data): void { - $this->data = unserialize($serialized); + $this->data = unserialize($data); } public function __serialize(): array @@ -28,8 +28,8 @@ public function __serialize(): array return $this->data; } - public function __unserialize(array $data) + public function __unserialize(array $data): void { - return $this->data = $data; + $this->data = $data; } } diff --git a/tests/factories/MetaFactory.php b/tests/factories/MetaFactory.php index 28ce5a1..70ef499 100644 --- a/tests/factories/MetaFactory.php +++ b/tests/factories/MetaFactory.php @@ -2,6 +2,7 @@ use Plank\Metable\Meta; +$factory = app(Illuminate\Database\Eloquent\Factory::class); $factory->define(Meta::class, function (Faker\Generator $faker) { return []; }); diff --git a/tests/factories/MetableFactory.php b/tests/factories/MetableFactory.php index 4a1422c..aac121c 100644 --- a/tests/factories/MetableFactory.php +++ b/tests/factories/MetableFactory.php @@ -3,6 +3,8 @@ use Plank\Metable\Tests\Mocks\SampleMetable; use Plank\Metable\Tests\Mocks\SampleMetableSoftDeletes; +$factory = app(Illuminate\Database\Eloquent\Factory::class); + $factory->define(SampleMetable::class, function (Faker\Generator $faker) { return []; }); diff --git a/tests/factories/MorphFactory.php b/tests/factories/MorphFactory.php index a2f3da7..3dd3060 100644 --- a/tests/factories/MorphFactory.php +++ b/tests/factories/MorphFactory.php @@ -2,6 +2,7 @@ use Plank\Metable\Tests\Mocks\SampleMorph; +$factory = app(Illuminate\Database\Eloquent\Factory::class); $factory->define(SampleMorph::class, function (Faker\Generator $faker) { return []; }); diff --git a/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php b/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php index 6ec45ce..6403a2b 100644 --- a/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php +++ b/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php @@ -14,7 +14,7 @@ class CreateSampleClassesTables extends Migration public function up() { Schema::create('sample_metables', function (Blueprint $table) { - $table->increments('id'); + $table->id(); $table->softDeletes(); $table->timestamps(); }); From d367be69d724268798c7f8a419a7190b00649745 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sat, 13 Apr 2024 23:59:59 -0400 Subject: [PATCH 08/38] added SerializeHandler data type deprecated ArrayHandler, ObjectHandler, and SerializableHandler, as the new type is more secure and consistent than these --- CHANGELOG.md | 15 +++- UPGRADING.md | 5 +- config/metable.php | 40 +++++++++- docs/source/datatypes.rst | 75 ++++++++++++------- src/DataType/ArrayHandler.php | 1 + src/DataType/ModelCollectionHandler.php | 38 ++++++++-- src/DataType/ModelHandler.php | 8 +- src/DataType/ObjectHandler.php | 1 + src/DataType/SerializableHandler.php | 4 +- src/DataType/SerializeHandler.php | 29 +++++++ tests/Integration/DataType/HandlerTest.php | 19 +++++ .../DataType/ModelCollectionHandlerTest.php | 16 ++++ .../Integration/DataType/ModelHandlerTest.php | 9 +++ .../DataType/SerializableHandlerTest.php | 45 +++++++++++ tests/Integration/MetableTest.php | 6 +- tests/TestCase.php | 9 ++- 16 files changed, 273 insertions(+), 47 deletions(-) create mode 100644 src/DataType/SerializeHandler.php create mode 100644 tests/Integration/DataType/SerializableHandlerTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 39b3527..a4b9f00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,25 @@ # Changelog -# 6.0.0 +## 6.0.0 + +### Compatibility + - Added support for PHP 8.3 - Droppped support for PHP 8.0 and below - Added support for Laravel 10 and 11 - Dropped support Laravel versions 9 and below - adjusted some method signatures with PHP 8+ mixed and union types +### Data Types + +- Added `SerializeHandler` as a catch-all datatype, which will attempt to serialize the data using PHP's `serialize()` function. The payload is encrypted before being stored in the database to prevent unserializing untrusted data. +- Deprecated `SerializableHandler` in favor of the new `SerializeHandler` datatype. The `SerializableHandler` will be removed in a future release. In the interim, added the `metable.options.serializable.allowedClasses` config to protect against unserializing untrusted data. +- Deprecated `ArrayHandler` and `ObjectHandler`, due to the ambiguity of nested array/objects switching type. These will be removed in a future release. The `SerializeHandler` should be used instead. +- `ModelHandler` will now validate that the encoded class is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, the meta value will return `null`. +- `ModelHandler` will no longer throw a model not found exception if the model no longer exists. Instead, the meta value will return `null`. This is more in line with the existing behavior of the `ModelCollectionHandler`. +- `ModelCollectionHandler` will now validate that the encoded collection class is a valid Eloquent collection before attempting to instantiate it during unserialization. If the class is invalid, an instance of `Illuminate\Database\Eloquent\Collection` will be used instead. +- `ModelCollectionHandler` will now validate that the encoded class of each entry is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, that entry in the collection will be omitted. + # 5.0.1 - 2021-09-19 - Fixed `setManyMeta()` not properly serializing certain types of data. diff --git a/UPGRADING.md b/UPGRADING.md index c5eb548..272e031 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -5,9 +5,12 @@ * Minimum PHP version moved to 8.1 * Minimum Laravel version moved to 10 * Some methods have had their signatures adjusted to use PHP 8+ mixed and union types. If extending any class or implementing any interface from this package, method signatures may need to be updated. +* Recommended to add the `SerializeHandler` to the end of `datatypes` config (catch-all). +* The `SerializableHandler`, `ArrayHandler`, and `ObjectHandler` data types have been deprecated in favor of the new `SerializeHandler`. If you have any Meta encoded using any of these data types, you should continue to include them in the `datatypes` config _after_ the `SerializeHandler` to ensure that existing values will continue to be properly decoded, but new values will use the new encoding. Once all old values have been migrated, you may remove the deprecated data types from the `datatypes` config. +* For security reasons, if you have any existing Meta encoded using `SerializableHandler`, you must configure the `metable.options.serializable.allowedClasses` config to list classes that are allowed to be unserialized. Otherwise, all objects will be returned as `__PHP_Incomplete_Class`. This config may be set to `true` to disable this security check and allow any class, but this is not recommended. ## 4.X -> 5.X -- New migration new file added which adds a new composite unique index to the meta table on `metable_type`, `metable_id`, and `key`. Make sure that you have no duplicate keys for a given entity (previously possible as a race condition) before applying the new migration. +- New migration file added which adds a new composite unique index to the meta table on `metable_type`, `metable_id`, and `key`. Make sure that you have no duplicate keys for a given entity (previously possible as a race condition) before applying the new migration. ## 3.X -> 4.X - Database migration files are now served from within the package. In your migrations table, rename the `XXXX_XX_XX_XXXXXX_create_meta_table.php` entry to `2017_01_01_000000_create_meta_table.php` and delete your local copy of the migration file from the /database/migrations directory. If any customizations were made to the table, those should be defined as one or more separate ALTER table migrations. \ No newline at end of file diff --git a/config/metable.php b/config/metable.php index 43f4544..1c6e6ee 100644 --- a/config/metable.php +++ b/config/metable.php @@ -6,6 +6,11 @@ */ 'model' => Plank\Metable\Meta::class, + /** + * Whether to apply migrations from this package automatically. + */ + 'applyMigrations' => true, + /* * List of handlers for recognized data types. * @@ -19,12 +24,39 @@ Plank\Metable\DataType\FloatHandler::class, Plank\Metable\DataType\StringHandler::class, Plank\Metable\DataType\DateTimeHandler::class, - Plank\Metable\DataType\ArrayHandler::class, Plank\Metable\DataType\ModelHandler::class, Plank\Metable\DataType\ModelCollectionHandler::class, - Plank\Metable\DataType\SerializableHandler::class, - Plank\Metable\DataType\ObjectHandler::class, + + /* + * The following handlers are catch-all handlers that will encode anything. + * Only one of these should be enabled at a time. + */ + Plank\Metable\DataType\SerializeHandler::class, + // Plank\Metable\DataType\JsonHandler::class, + + /* + * The following handlers are deprecated and will be removed in a future release. + * They are kept for backwards compatibility, but should not be used in new code. + */ + // Plank\Metable\DataType\ArrayHandler::class, + // Plank\Metable\DataType\ObjectHandler::class, + // Plank\Metable\DataType\SerializableHandler::class, ], - 'applyMigrations' => true + 'options' => [ + 'serializable' => [ + /* + * List of classes that may be stored and retrieved using PHP serialization. + * + * Must explicitly list all classes that may be unserialized. + * Child classes of listed classes are not allowed, unless they are listed. + * + * May be set to an empty array or `false` to disallow object unserialization. + * May be set to `true` to allow serialization of all classes (strongly discouraged). + */ + 'allowedClasses' => [ + // \SampleClass::class, + ], + ], + ], ]; diff --git a/docs/source/datatypes.rst b/docs/source/datatypes.rst index 2d6e5bd..50f25ca 100644 --- a/docs/source/datatypes.rst +++ b/docs/source/datatypes.rst @@ -12,29 +12,6 @@ Scalar Values The following scalar values are supported. -Array -^^^^^^^^ - -Arrays of scalar values. Nested arrays are supported. - -:: - - setMeta('information', [ - 'address' => [ - 'street' => '123 Somewhere Ave.', - 'city' => 'Somewhereville', - 'country' => 'Somewhereland', - 'postal' => '123456', - ], - 'contact' => [ - 'phone' => '555-555-5555', - 'email' => 'email@example.com' - ] - ]); - -.. warning:: Laravel-Metable uses ``json_encode()`` and ``json_decode()`` under the hood for array serialization. This will cause any objects nested within the array to be cast to an array. - Boolean ^^^^^^^^ @@ -128,10 +105,54 @@ Any object implementing the ``DateTimeInterface``. Object will be converted to setMeta('last_viewed', \Carbon\Carbon::now()); +Other +^^^^^ + +Objects and arrays will be serialized using PHP's `serialize()` function, to allow for the storage and retrieval of complex data structures. The serialized value is encrypted before being stored in the database, and decrypted when retrieved to prevent tampered data from being unserialized. + +:: + + setMeta('data', ['key' => 'value']); + $metable->setMeta('data', new MyValueObject(123)); + +.. note:: The ``Plank\Metable\DataType\SerializeHandler`` class should always be the last entry the ``config/metable.php`` datatypes array, as it will accept data of any type, causing any handlers below it to be ignored. + +Deprecated +---------- + +The following data types are deprecated and should not be used in new code. They are still supported for backwards compatibility, but will be removed in a future release. + +Array +^^^^^^^^ + +.. warning:: The ``ArrayHandler`` datatype is deprecated. The ``SerializeHandler`` should be used for handling arrays. + +Arrays of scalar values. Nested arrays are supported. + +:: + + setMeta('information', [ + 'address' => [ + 'street' => '123 Somewhere Ave.', + 'city' => 'Somewhereville', + 'country' => 'Somewhereland', + 'postal' => '123456', + ], + 'contact' => [ + 'phone' => '555-555-5555', + 'email' => 'email@example.com' + ] + ]); + +.. warning:: the ``ArrayHandler`` class uses ``json_encode()`` and ``json_decode()`` under the hood for array serialization. This will cause any objects nested within the array to be cast to an array. This is not a concern for the ``SerializeHandler``. Serializable ^^^^^^^^^^^^^ +.. warning:: The ``SerializableHandler`` datatype is deprecated. The ``SerializeHandler`` should be used for handling all objects. + Any object implementing the PHP ``Serializable`` interface. :: @@ -146,9 +167,13 @@ Any object implementing the PHP ``Serializable`` interface. $metable->setMeta('example', $serializable); +For security reasons, it is necessary to list any classes that can be unserialized in the ``metable.options.serializable.allowedClasses`` key in the ``config/metable.php`` file. This is to prevent arbitrary code execution when unserializing untrusted data. This config can be set to true to allow all classes, but this is not recommended. + Plain Objects ^^^^^^^^^^^^^^ +.. warning:: The ``ObjectHandler`` datatype is deprecated. The ``SerializeHandler`` should be used for handling all objects. + Any other objects will be converted to ``stdClass`` plain objects. You can control what properties are stored by implementing the ``JsonSerializable`` interface on the class of your stored object. :: @@ -157,9 +182,7 @@ Any other objects will be converted to ``stdClass`` plain objects. You can contr $metable->setMeta('weight', new Weight(10, 'kg')); $weight = $metable->getMeta('weight') // stdClass($amount = 10; $unit => 'kg'); -.. note:: The ``Plank\Metable\DataType\ObjectHandler`` class should always be the last entry the ``config/metable.php`` datatypes array, as it will accept any object, causing any handlers below it to be ignored. - -.. warning:: Laravel-Metable uses ``json_encode()`` and ``json_decode()`` under the hood for plain object serialization. This will cause any arrays within the object's properties to be cast to a ``stdClass`` object. +.. warning:: Laravel-Metable uses ``json_encode()`` and ``json_decode()`` under the hood for plain object serialization. This will cause any arrays within the object's properties to be cast to a ``stdClass`` object. This is not a concern for the ``SerializeHandler``. Adding Custom Data Types diff --git a/src/DataType/ArrayHandler.php b/src/DataType/ArrayHandler.php index a0bd164..059ae9f 100644 --- a/src/DataType/ArrayHandler.php +++ b/src/DataType/ArrayHandler.php @@ -4,6 +4,7 @@ /** * Handle serialization of arrays. + * @deprecated Use SerializeHandler instead. */ class ArrayHandler implements HandlerInterface { diff --git a/src/DataType/ModelCollectionHandler.php b/src/DataType/ModelCollectionHandler.php index 41c4a9e..13e26b1 100644 --- a/src/DataType/ModelCollectionHandler.php +++ b/src/DataType/ModelCollectionHandler.php @@ -3,6 +3,7 @@ namespace Plank\Metable\DataType; use Illuminate\Database\Eloquent\Collection; +use Illuminate\Database\Eloquent\Model; /** * Handle serialization of Eloquent collections. @@ -48,12 +49,29 @@ public function unserializeValue(string $serializedValue): mixed { $data = json_decode($serializedValue, true); - $collection = new $data['class'](); + $collectionClass = (string)($data['class'] ?? ''); + + if (class_exists($collectionClass) + && is_a($collectionClass, Collection::class, true) + ) { + $collection = new $collectionClass(); + } else { + // attempt to gracefully fall back to a standard collection + // if the defined collection class is not found + $collection = new Collection(); + } + $models = $this->loadModels($data['items']); // Repopulate collection keys with loaded models. foreach ($data['items'] as $key => $item) { - if (is_null($item['key'])) { + if (empty($item['key'])) { + $class = (string)($item['class'] ?? ''); + if (!class_exists($class) + || !is_a($class, Model::class, true) + ) { + continue; + } $collection[$key] = new $item['class'](); } elseif (isset($models[$item['class']][$item['key']])) { $collection[$key] = $models[$item['class']][$item['key']]; @@ -77,15 +95,23 @@ private function loadModels(array $items) // Retrieve a list of keys to load from each class. foreach ($items as $item) { - if (!is_null($item['key'])) { - $classes[$item['class']][] = $item['key']; + $class = (string)($item['class'] ?? ''); + + if (!empty($item['key'])) { + $classes[$class][] = $item['key']; } } // Iterate list of classes and load all records matching a key. foreach ($classes as $class => $keys) { - $model = new $class(); - $results[$class] = $model->whereIn($model->getKeyName(), $keys)->get()->keyBy($model->getKeyName()); + if (!class_exists($class) + || !is_a($class, Model::class, true) + ) { + continue; + } + + $results[$class] = $class::query()->findMany($keys) + ->keyBy(fn(Model $model) => $model->getKey()); } return $results; diff --git a/src/DataType/ModelHandler.php b/src/DataType/ModelHandler.php index 9cc8943..2611343 100644 --- a/src/DataType/ModelHandler.php +++ b/src/DataType/ModelHandler.php @@ -48,8 +48,12 @@ public function unserializeValue(string $serializedValue): mixed } // Fetch specific instances. - list($class, $id) = explode('#', $serializedValue); + /** @var class-string $class */ + [$class, $id] = explode('#', $serializedValue); + if (!is_a($class, Model::class, true)) { + return null; + } - return with(new $class())->findOrFail($id); + return $class::query()->find($id); } } diff --git a/src/DataType/ObjectHandler.php b/src/DataType/ObjectHandler.php index 7cb7049..7582c49 100644 --- a/src/DataType/ObjectHandler.php +++ b/src/DataType/ObjectHandler.php @@ -4,6 +4,7 @@ /** * Handle serialization of plain objects. + * @deprecated Use SerializeHandler instead. */ class ObjectHandler implements HandlerInterface { diff --git a/src/DataType/SerializableHandler.php b/src/DataType/SerializableHandler.php index 6e8e6b6..ed9fce9 100644 --- a/src/DataType/SerializableHandler.php +++ b/src/DataType/SerializableHandler.php @@ -6,6 +6,7 @@ /** * Handle serialization of Serializable objects. + * @deprecated Use SerializeHandler instead. */ class SerializableHandler implements HandlerInterface { @@ -38,6 +39,7 @@ public function serializeValue(mixed $value): string */ public function unserializeValue(string $serializedValue): mixed { - return unserialize($serializedValue); + $allowedClasses = config('metable.options.serializable.allowedClasses', false); + return unserialize($serializedValue, ['allowed_classes' => $allowedClasses]); } } diff --git a/src/DataType/SerializeHandler.php b/src/DataType/SerializeHandler.php new file mode 100644 index 0000000..9ebe0e8 --- /dev/null +++ b/src/DataType/SerializeHandler.php @@ -0,0 +1,29 @@ +encrypt($value, true); + } + + public function unserializeValue(string $serializedValue): mixed + { + return app('encrypter')->decrypt($serializedValue, true); + } +} \ No newline at end of file diff --git a/tests/Integration/DataType/HandlerTest.php b/tests/Integration/DataType/HandlerTest.php index 712c552..288c88f 100644 --- a/tests/Integration/DataType/HandlerTest.php +++ b/tests/Integration/DataType/HandlerTest.php @@ -15,6 +15,7 @@ use Plank\Metable\DataType\NullHandler; use Plank\Metable\DataType\ObjectHandler; use Plank\Metable\DataType\SerializableHandler; +use Plank\Metable\DataType\SerializeHandler; use Plank\Metable\DataType\StringHandler; use Plank\Metable\Tests\Mocks\SampleMetable; use Plank\Metable\Tests\Mocks\SampleSerializable; @@ -23,6 +24,7 @@ class HandlerTest extends TestCase { + private static $resource; static public function handlerProvider(): array { $timestamp = '2017-01-01 00:00:00.000000+0000'; @@ -32,6 +34,8 @@ static public function handlerProvider(): array $object->foo = 'bar'; $object->baz = 3; + self::$resource = fopen('php://memory', 'r'); + return [ 'array' => [ new ArrayHandler(), @@ -87,6 +91,12 @@ static public function handlerProvider(): array $object, [[]], ], + 'serialize' => [ + new SerializeHandler(), + 'serialized', + ['foo' => 'bar', 'baz' => [3]], + [self::$resource], + ], 'serializable' => [ new SerializableHandler(), 'serializable', @@ -102,6 +112,15 @@ static public function handlerProvider(): array ]; } + public static function tearDownAfterClass(): void + { + if (self::$resource) { + fclose(self::$resource); + self::$resource = null; + } + parent::tearDownAfterClass(); + } + /** * @dataProvider handlerProvider */ diff --git a/tests/Integration/DataType/ModelCollectionHandlerTest.php b/tests/Integration/DataType/ModelCollectionHandlerTest.php index 165dbdf..bb2f869 100644 --- a/tests/Integration/DataType/ModelCollectionHandlerTest.php +++ b/tests/Integration/DataType/ModelCollectionHandlerTest.php @@ -31,4 +31,20 @@ public function test_it_reloads_model_instances(): void $this->assertFalse($unserialized[1]->exists); $this->assertEquals(1, $unserialized['foo']->getKey()); } + + public function test_it_handles_invalid_model_class(): void + { + $handler = new ModelCollectionHandler(); + $serialized = json_encode([ + 'class' => 'stdClass', + 'items' => [ + 'class' => 'stdClass', + 'key' => '1' + ] + ]); + $unserialized = $handler->unserializeValue($serialized); + + $this->assertInstanceOf(Collection::class, $unserialized); + $this->assertEmpty($unserialized); + } } diff --git a/tests/Integration/DataType/ModelHandlerTest.php b/tests/Integration/DataType/ModelHandlerTest.php index 23783c9..b77ab9a 100644 --- a/tests/Integration/DataType/ModelHandlerTest.php +++ b/tests/Integration/DataType/ModelHandlerTest.php @@ -22,4 +22,13 @@ public function test_it_reloads_a_model_instance(): void $this->assertEquals(12, $unserialized->getKey()); $this->assertTrue($unserialized->exists); } + + public function test_it_handles_invalid_model_class(): void + { + $handler = new ModelHandler(); + $serialized = 'stdClass#1'; + $unserialized = $handler->unserializeValue($serialized); + + $this->assertNull($unserialized); + } } diff --git a/tests/Integration/DataType/SerializableHandlerTest.php b/tests/Integration/DataType/SerializableHandlerTest.php new file mode 100644 index 0000000..4d53925 --- /dev/null +++ b/tests/Integration/DataType/SerializableHandlerTest.php @@ -0,0 +1,45 @@ + 'bar']); + + $handler = new SerializableHandler(); + + $serialized = $handler->serializeValue($original); + + $incomplete = unserialize(serialize($original), ['allowed_classes' => false]); + + config()->set( + 'metable.options.serializable.allowedClasses', + [SampleSerializable::class] + ); + $this->assertEquals($original, $handler->unserializeValue($serialized)); + + config()->set( + 'metable.options.serializable.allowedClasses', + true + ); + $this->assertEquals($original, $handler->unserializeValue($serialized)); + + config()->set( + 'metable.options.serializable.allowedClasses', + [] + ); + $this->assertEquals($incomplete, $handler->unserializeValue($serialized)); + + config()->set( + 'metable.options.serializable.allowedClasses', + false + ); + $this->assertEquals($incomplete, $handler->unserializeValue($serialized)); + } +} \ No newline at end of file diff --git a/tests/Integration/MetableTest.php b/tests/Integration/MetableTest.php index 4c67688..0578d70 100644 --- a/tests/Integration/MetableTest.php +++ b/tests/Integration/MetableTest.php @@ -2,6 +2,7 @@ namespace Plank\Metable\Tests\Integration; +use Carbon\Carbon; use Illuminate\Database\Eloquent\Collection; use Plank\Metable\Meta; use Plank\Metable\Tests\Mocks\SampleMetable; @@ -347,14 +348,15 @@ public function test_it_can_be_queried_by_all_meta_keys(): void public function test_it_can_be_queried_by_meta_value(): void { + $now = Carbon::now(); $this->useDatabase(); $metable = $this->createMetable(); $metable->setMeta('foo', 'bar'); - $metable->setMeta('array', ['a' => 'b']); + $metable->setMeta('datetime', $now); $result1 = SampleMetable::whereMeta('foo', 'bar')->first(); $result2 = SampleMetable::whereMeta('foo', 'baz')->first(); - $result3 = SampleMetable::whereMeta('array', ['a' => 'b'])->first(); + $result3 = SampleMetable::whereMeta('datetime', $now)->first(); $this->assertEquals($metable->getKey(), $result1->getKey()); $this->assertNull($result2); diff --git a/tests/TestCase.php b/tests/TestCase.php index 3f328b9..3fc9f1b 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -4,6 +4,7 @@ use Orchestra\Testbench\TestCase as BaseTestCase; use Plank\Metable\MetableServiceProvider; +use Plank\Metable\Tests\Mocks\SampleSerializable; use ReflectionClass; class TestCase extends BaseTestCase @@ -29,6 +30,8 @@ protected function getPackageAliases($app) protected function getEnvironmentSetUp($app) { date_default_timezone_set('GMT'); + $app['config']->set('app.key', 'base64:'.base64_encode(random_bytes(32))); + //use in-memory database $app['config']->set('database.connections.testing', [ 'driver' => 'sqlite', @@ -48,14 +51,14 @@ protected function getEnvironmentSetUp($app) 'strict' => false, ]); $app['config']->set('database.default', 'testing'); + + $app['config']->set('metable.options.serializable.allowedClasses', [SampleSerializable::class]); } protected function getPrivateProperty($class, $property_name) { $reflector = new ReflectionClass($class); $property = $reflector->getProperty($property_name); - $property->setAccessible(true); - return $property; } @@ -63,8 +66,6 @@ protected function getPrivateMethod($class, $method_name) { $reflector = new ReflectionClass($class); $method = $reflector->getMethod($method_name); - $method->setAccessible(true); - return $method; } From eaa18ca8cd9a9f750a76f9fa3843ce6fa54bb305 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sun, 14 Apr 2024 14:30:55 -0400 Subject: [PATCH 09/38] add indexed columns for string/numeric comparison and sorting --- docs/source/datatypes.rst | 2 +- ...4_04_14_000000_add_meta_search_columns.php | 40 ++++++++++++++++ src/DataType/ArrayHandler.php | 19 +++++++- src/DataType/BooleanHandler.php | 10 ++++ src/DataType/DateTimeHandler.php | 12 +++++ src/DataType/FloatHandler.php | 10 ++++ src/DataType/HandlerInterface.php | 4 ++ src/DataType/IntegerHandler.php | 10 ++++ src/DataType/ModelCollectionHandler.php | 10 ++++ src/DataType/ModelHandler.php | 10 ++++ src/DataType/NullHandler.php | 10 ++++ src/DataType/ObjectHandler.php | 10 ++++ src/DataType/ScalarHandler.php | 6 +-- src/DataType/SerializableHandler.php | 10 ++++ src/DataType/SerializeHandler.php | 10 ++++ src/DataType/StringHandler.php | 13 +++++ src/Meta.php | 27 +++++++++-- tests/Integration/DataType/HandlerTest.php | 47 +++++++++++++++++-- 18 files changed, 246 insertions(+), 14 deletions(-) create mode 100644 migrations/2024_04_14_000000_add_meta_search_columns.php diff --git a/docs/source/datatypes.rst b/docs/source/datatypes.rst index 50f25ca..2b27037 100644 --- a/docs/source/datatypes.rst +++ b/docs/source/datatypes.rst @@ -105,7 +105,7 @@ Any object implementing the ``DateTimeInterface``. Object will be converted to setMeta('last_viewed', \Carbon\Carbon::now()); -Other +Objects and Arrays ^^^^^ Objects and arrays will be serialized using PHP's `serialize()` function, to allow for the storage and retrieval of complex data structures. The serialized value is encrypted before being stored in the database, and decrypted when retrieved to prevent tampered data from being unserialized. diff --git a/migrations/2024_04_14_000000_add_meta_search_columns.php b/migrations/2024_04_14_000000_add_meta_search_columns.php new file mode 100644 index 0000000..11ba084 --- /dev/null +++ b/migrations/2024_04_14_000000_add_meta_search_columns.php @@ -0,0 +1,40 @@ +decimal('numeric_value', 18, 9)->nullable(); + $table->string('string_value', 255)->nullable(); + $table->dropIndex(['key', 'metable_type']); + $table->index(['key', 'metable_type', 'numeric_value']); + $table->index(['key', 'metable_type', 'string_value']); + }); + } + + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::table('meta', function (Blueprint $table) { + $table->dropIndex(['key', 'metable_type', 'string_value']); + $table->dropIndex(['key', 'metable_type', 'numeric_value']); + $table->index(['metable_type', 'metable_id']); + $table->dropColumn('numeric_value'); + $table->dropColumn('string_value'); + }); + } +} diff --git a/src/DataType/ArrayHandler.php b/src/DataType/ArrayHandler.php index 059ae9f..16ed86f 100644 --- a/src/DataType/ArrayHandler.php +++ b/src/DataType/ArrayHandler.php @@ -29,7 +29,7 @@ public function canHandleValue(mixed $value): bool */ public function serializeValue(mixed $value): string { - return json_encode($value); + return json_encode($value, JSON_THROW_ON_ERROR); } /** @@ -37,6 +37,21 @@ public function serializeValue(mixed $value): string */ public function unserializeValue(string $serializedValue): mixed { - return json_decode($serializedValue, true); + return json_decode( + $serializedValue, + true, + 512, + JSON_THROW_ON_ERROR + ); + } + + public function getNumericValue(mixed $value, string $serializedValue): null|int|float + { + return null; + } + + public function getStringValue(mixed $value, string $serializedValue): null|string + { + return null; } } diff --git a/src/DataType/BooleanHandler.php b/src/DataType/BooleanHandler.php index f435b45..fc8476c 100644 --- a/src/DataType/BooleanHandler.php +++ b/src/DataType/BooleanHandler.php @@ -11,4 +11,14 @@ class BooleanHandler extends ScalarHandler * {@inheritdoc} */ protected $type = 'boolean'; + + public function getNumericValue(mixed $value, string $serializedValue): null|int|float + { + return $value ? 1 : 0; + } + + public function getStringValue(mixed $value, string $serializedValue): null|string + { + return $value ? 'true' : 'false'; + } } diff --git a/src/DataType/DateTimeHandler.php b/src/DataType/DateTimeHandler.php index c9d1e96..5e67000 100644 --- a/src/DataType/DateTimeHandler.php +++ b/src/DataType/DateTimeHandler.php @@ -48,4 +48,16 @@ public function unserializeValue(string $serializedValue): mixed { return Carbon::createFromFormat($this->format, $serializedValue); } + + public function getNumericValue(mixed $value, string $serializedValue): null|int|float + { + return $value instanceof DateTimeInterface + ? $value->getTimestamp() + : null; + } + + public function getStringValue(mixed $value, string $serializedValue): null|string + { + return $serializedValue; + } } diff --git a/src/DataType/FloatHandler.php b/src/DataType/FloatHandler.php index 7838160..40feb91 100644 --- a/src/DataType/FloatHandler.php +++ b/src/DataType/FloatHandler.php @@ -19,4 +19,14 @@ public function getDataType(): string { return 'float'; } + + public function getNumericValue(mixed $value, string $serializedValue): null|int|float + { + return $value; + } + + public function getStringValue(mixed $value, string $serializedValue): null|string + { + return (string) $value; + } } diff --git a/src/DataType/HandlerInterface.php b/src/DataType/HandlerInterface.php index d9818fb..6714842 100644 --- a/src/DataType/HandlerInterface.php +++ b/src/DataType/HandlerInterface.php @@ -32,6 +32,10 @@ public function canHandleValue(mixed $value): bool; */ public function serializeValue(mixed $value): string; + public function getNumericValue(mixed $value, string $serializedValue): null|int|float; + + public function getStringValue(mixed $value, string $serializedValue): null|string; + /** * Convert a serialized string back to its original value. * diff --git a/src/DataType/IntegerHandler.php b/src/DataType/IntegerHandler.php index b4f1abb..3722769 100644 --- a/src/DataType/IntegerHandler.php +++ b/src/DataType/IntegerHandler.php @@ -11,4 +11,14 @@ class IntegerHandler extends ScalarHandler * {@inheritdoc} */ protected $type = 'integer'; + + public function getNumericValue(mixed $value, string $serializedValue): null|int|float + { + return $value; + } + + public function getStringValue(mixed $value, string $serializedValue): null|string + { + return (string) $value; + } } diff --git a/src/DataType/ModelCollectionHandler.php b/src/DataType/ModelCollectionHandler.php index 13e26b1..ef1895f 100644 --- a/src/DataType/ModelCollectionHandler.php +++ b/src/DataType/ModelCollectionHandler.php @@ -116,4 +116,14 @@ private function loadModels(array $items) return $results; } + + public function getNumericValue(mixed $value, string $serializedValue): null|int|float + { + return null; + } + + public function getStringValue(mixed $value, string $serializedValue): null|string + { + return null; + } } diff --git a/src/DataType/ModelHandler.php b/src/DataType/ModelHandler.php index 2611343..b90c5ea 100644 --- a/src/DataType/ModelHandler.php +++ b/src/DataType/ModelHandler.php @@ -56,4 +56,14 @@ public function unserializeValue(string $serializedValue): mixed return $class::query()->find($id); } + + public function getNumericValue(mixed $value, string $serializedValue): null|int|float + { + return null; + } + + public function getStringValue(mixed $value, string $serializedValue): null|string + { + return $serializedValue; + } } diff --git a/src/DataType/NullHandler.php b/src/DataType/NullHandler.php index acd1a66..b8fe689 100644 --- a/src/DataType/NullHandler.php +++ b/src/DataType/NullHandler.php @@ -19,4 +19,14 @@ public function getDataType(): string { return 'null'; } + + public function getNumericValue(mixed $value, string $serializedValue): null|int|float + { + return null; + } + + public function getStringValue(mixed $value, string $serializedValue): null|string + { + return null; + } } diff --git a/src/DataType/ObjectHandler.php b/src/DataType/ObjectHandler.php index 7582c49..319c719 100644 --- a/src/DataType/ObjectHandler.php +++ b/src/DataType/ObjectHandler.php @@ -39,4 +39,14 @@ public function unserializeValue(string $serializedValue): mixed { return json_decode($serializedValue, false); } + + public function getNumericValue(mixed $value, string $serializedValue): null|int|float + { + return null; + } + + public function getStringValue(mixed $value, string $serializedValue): null|string + { + return null; + } } diff --git a/src/DataType/ScalarHandler.php b/src/DataType/ScalarHandler.php index f1b838f..853a552 100644 --- a/src/DataType/ScalarHandler.php +++ b/src/DataType/ScalarHandler.php @@ -43,10 +43,10 @@ public function serializeValue(mixed $value): string /** * {@inheritdoc} */ - public function unserializeValue(string $value): mixed + public function unserializeValue(string $serializedValue): mixed { - settype($value, $this->type); + settype($serializedValue, $this->type); - return $value; + return $serializedValue; } } diff --git a/src/DataType/SerializableHandler.php b/src/DataType/SerializableHandler.php index ed9fce9..dd4e8a0 100644 --- a/src/DataType/SerializableHandler.php +++ b/src/DataType/SerializableHandler.php @@ -42,4 +42,14 @@ public function unserializeValue(string $serializedValue): mixed $allowedClasses = config('metable.options.serializable.allowedClasses', false); return unserialize($serializedValue, ['allowed_classes' => $allowedClasses]); } + + public function getNumericValue(mixed $value, string $serializedValue): null|int|float + { + return null; + } + + public function getStringValue(mixed $value, string $serializedValue): null|string + { + return null; + } } diff --git a/src/DataType/SerializeHandler.php b/src/DataType/SerializeHandler.php index 9ebe0e8..4fc1a96 100644 --- a/src/DataType/SerializeHandler.php +++ b/src/DataType/SerializeHandler.php @@ -26,4 +26,14 @@ public function unserializeValue(string $serializedValue): mixed { return app('encrypter')->decrypt($serializedValue, true); } + + public function getNumericValue(mixed $value, string $serializedValue): null|int|float + { + return null; + } + + public function getStringValue(mixed $value, string $serializedValue): null|string + { + return null; + } } \ No newline at end of file diff --git a/src/DataType/StringHandler.php b/src/DataType/StringHandler.php index c7e8b4b..0caf97b 100644 --- a/src/DataType/StringHandler.php +++ b/src/DataType/StringHandler.php @@ -11,4 +11,17 @@ class StringHandler extends ScalarHandler * {@inheritdoc} */ protected $type = 'string'; + + public function getNumericValue(mixed $value, string $serializedValue): null|int|float + { + if (is_numeric($value)) { + return (float)$value; + } + return null; + } + + public function getStringValue(mixed $value, string $serializedValue): null|string + { + return substr($value, 0, 255); + } } diff --git a/src/Meta.php b/src/Meta.php index 5b34218..40a4d3a 100644 --- a/src/Meta.php +++ b/src/Meta.php @@ -14,7 +14,9 @@ * @property int $metable_id * @property string $type * @property string $key - * @property string $value + * @property mixed $value + * @property null|string $string_value + * @property null|int|float $numeric_value * @property Model $metable */ class Meta extends Model @@ -32,7 +34,14 @@ class Meta extends Model /** * {@inheritdoc} */ - protected $guarded = ['id', 'metable_type', 'metable_id', 'type']; + protected $guarded = [ + 'id', + 'metable_type', + 'metable_id', + 'type', + 'string_value', + 'numeric_value' + ]; /** * {@inheritdoc} @@ -93,8 +102,18 @@ public function setValueAttribute($value): void $registry = $this->getDataTypeRegistry(); $this->attributes['type'] = $registry->getTypeForValue($value); - $this->attributes['value'] = $registry->getHandlerForType($this->type) - ->serializeValue($value); + $handler = $registry->getHandlerForType($this->type); + $serializedValue = $handler->serializeValue($value); + + $this->attributes['value'] = $serializedValue; + $this->attributes['string_value'] = $handler->getStringValue( + $value, + $serializedValue + ); + $this->attributes['numeric_value'] = $handler->getNumericValue( + $value, + $serializedValue + ); $this->cachedValue = null; } diff --git a/tests/Integration/DataType/HandlerTest.php b/tests/Integration/DataType/HandlerTest.php index 288c88f..97527ed 100644 --- a/tests/Integration/DataType/HandlerTest.php +++ b/tests/Integration/DataType/HandlerTest.php @@ -27,13 +27,16 @@ class HandlerTest extends TestCase private static $resource; static public function handlerProvider(): array { - $timestamp = '2017-01-01 00:00:00.000000+0000'; - $datetime = Carbon::createFromFormat('Y-m-d H:i:s.uO', $timestamp); + $dateString = '2017-01-01 00:00:00.000000+0000'; + $datetime = Carbon::createFromFormat('Y-m-d H:i:s.uO', $dateString); + $timestamp = $datetime->getTimestamp(); $object = new stdClass(); $object->foo = 'bar'; $object->baz = 3; + $model = new SampleMetable(); + self::$resource = fopen('php://memory', 'r'); return [ @@ -42,72 +45,104 @@ static public function handlerProvider(): array 'array', ['foo' => ['bar'], 'baz'], [new stdClass()], + null, + null, ], 'boolean' => [ new BooleanHandler(), 'boolean', true, [1, 0, '', [], null], + 1, + 'true' ], 'datetime' => [ new DateTimeHandler(), 'datetime', $datetime, [2017, '2017-01-01'], + $timestamp, + $dateString, ], 'float' => [ new FloatHandler(), 'float', 1.1, ['1.1', 1], + 1.1, + '1.1', ], 'integer' => [ new IntegerHandler(), 'integer', 3, [1.1, '1'], + 3, + '3', ], 'model' => [ new ModelHandler(), 'model', - new SampleMetable(), + $model, [new stdClass()], + null, + SampleMetable::class, ], 'model collection' => [ new ModelCollectionHandler(), 'collection', new Collection([new SampleMetable()]), [collect()], + null, + null, ], 'null' => [ new NullHandler(), 'null', null, [0, '', 'null', [], false], + null, + null, ], 'object' => [ new ObjectHandler(), 'object', $object, [[]], + null, + null, ], 'serialize' => [ new SerializeHandler(), 'serialized', ['foo' => 'bar', 'baz' => [3]], [self::$resource], + null, + null, ], 'serializable' => [ new SerializableHandler(), 'serializable', new SampleSerializable(['foo' => 'bar']), [], + null, + null, ], 'string' => [ new StringHandler(), 'string', 'foo', [1, 1.1], + null, + 'foo', + ], + 'numeric-string' => [ + new StringHandler(), + 'string', + '1.2345', + [1, 1.1], + 1.2345, + '1.2345', ], ]; } @@ -128,7 +163,9 @@ public function test_it_can_verify_and_serialize_data( HandlerInterface $handler, string $type, mixed $value, - array $incompatible + array $incompatible, + null|int|float $numericValue, + null|string $stringValue ): void { $this->assertEquals($type, $handler->getDataType()); $this->assertTrue($handler->canHandleValue($value)); @@ -141,5 +178,7 @@ public function test_it_can_verify_and_serialize_data( $unserialized = $handler->unserializeValue($serialized); $this->assertEquals($value, $unserialized); + $this->assertEquals($numericValue, $handler->getNumericValue($value, $serialized)); + $this->assertEquals($stringValue, $handler->getStringValue($value, $serialized)); } } From 0d6838ac44375adaf6b17e3c631c5d99b09b10c4 Mon Sep 17 00:00:00 2001 From: anil kumar thakur Date: Tue, 16 Apr 2024 01:20:11 +0545 Subject: [PATCH 10/38] migration changes --- migrations/2017_01_01_000000_create_meta_table.php | 10 ++++------ src/Metable.php | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/migrations/2017_01_01_000000_create_meta_table.php b/migrations/2017_01_01_000000_create_meta_table.php index 96ff75d..b7b8352 100644 --- a/migrations/2017_01_01_000000_create_meta_table.php +++ b/migrations/2017_01_01_000000_create_meta_table.php @@ -11,17 +11,15 @@ class CreateMetaTable extends Migration * * @return void */ - public function up() + public function up(): void { if (!Schema::hasTable('meta')) { Schema::create('meta', function (Blueprint $table) { $table->id(); - $table->string('metable_type'); - $table->unsignedInteger('metable_id'); - $table->string('type')->default('null'); + $table->morphs('metable'); + $table->string('type')->nullable(); $table->string('key')->index(); $table->longtext('value'); - $table->index(['metable_type', 'metable_id']); }); } @@ -32,7 +30,7 @@ public function up() * * @return void */ - public function down() + public function down(): void { Schema::dropIfExists('meta'); } diff --git a/src/Metable.php b/src/Metable.php index 509fc9b..04d9159 100644 --- a/src/Metable.php +++ b/src/Metable.php @@ -33,7 +33,7 @@ trait Metable * * @return void */ - public static function bootMetable() + public static function bootMetable(): void { // delete all attached meta on deletion static::deleted(function (self $model) { From 762489a5deb31e25218dae04767edb5503af6e26 Mon Sep 17 00:00:00 2001 From: anil kumar thakur Date: Tue, 16 Apr 2024 01:24:52 +0545 Subject: [PATCH 11/38] ci fixes --- .github/workflows/automated-test.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/automated-test.yml b/.github/workflows/automated-test.yml index 98fbd53..20a3dca 100644 --- a/.github/workflows/automated-test.yml +++ b/.github/workflows/automated-test.yml @@ -20,16 +20,15 @@ jobs: env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Get composer cache directory - id: composer-cache - run: echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT - - - name: Cache dependencies + - name: Cache Composer dependencies uses: actions/cache@v4 with: - path: ${{ steps.composer-cache.outputs.dir }} - key: ${{ runner.os }}-${{ matrix.php-version }}${{ matrix.prefer-lowest }}-composer-${{ hashFiles('**/composer.json') }} - restore-keys: ${{ runner.os }}-${{ matrix.php-version }}${{ matrix.prefer-lowest }}-composer- + path: | + ~/.composer/cache/files + ~/.cache/composer/files + key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} + restore-keys: | + ${{ runner.os }}-composer- - name: Install dependencies run: composer update --prefer-dist ${{ matrix.prefer-lowest }} From 561789413c43c94dc581dd08aff233fec7dc6aec Mon Sep 17 00:00:00 2001 From: anil kumar thakur Date: Tue, 16 Apr 2024 01:26:18 +0545 Subject: [PATCH 12/38] ci fixes --- .github/workflows/automated-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/automated-test.yml b/.github/workflows/automated-test.yml index 20a3dca..1a1f78f 100644 --- a/.github/workflows/automated-test.yml +++ b/.github/workflows/automated-test.yml @@ -31,7 +31,7 @@ jobs: ${{ runner.os }}-composer- - name: Install dependencies - run: composer update --prefer-dist ${{ matrix.prefer-lowest }} + run: composer install --no-ansi --no-interaction --no-scripts --no-progress --prefer-dist env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 6b5296727b764345a579ad0d68847f9aaffac035 Mon Sep 17 00:00:00 2001 From: anil kumar thakur Date: Tue, 16 Apr 2024 01:34:26 +0545 Subject: [PATCH 13/38] ci fixes --- .github/workflows/automated-test.yml | 2 +- migrations/2017_01_01_000000_create_meta_table.php | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/automated-test.yml b/.github/workflows/automated-test.yml index 1a1f78f..13a937c 100644 --- a/.github/workflows/automated-test.yml +++ b/.github/workflows/automated-test.yml @@ -36,7 +36,7 @@ jobs: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run phpunit - run: vendor/bin/phpunit --coverage-clover build/logs/clover.xml -v + run: vendor/bin/phpunit --coverage-clover build/logs/clover.xml --debug - name: Upload coverage results to Coveralls env: diff --git a/migrations/2017_01_01_000000_create_meta_table.php b/migrations/2017_01_01_000000_create_meta_table.php index b7b8352..ea8e736 100644 --- a/migrations/2017_01_01_000000_create_meta_table.php +++ b/migrations/2017_01_01_000000_create_meta_table.php @@ -20,7 +20,6 @@ public function up(): void $table->string('type')->nullable(); $table->string('key')->index(); $table->longtext('value'); - $table->index(['metable_type', 'metable_id']); }); } } From 3ad04669354170597bb9081db7583559322f6a5c Mon Sep 17 00:00:00 2001 From: anil kumar thakur Date: Tue, 16 Apr 2024 01:41:55 +0545 Subject: [PATCH 14/38] ci fixes --- .github/workflows/automated-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/automated-test.yml b/.github/workflows/automated-test.yml index 13a937c..58c6440 100644 --- a/.github/workflows/automated-test.yml +++ b/.github/workflows/automated-test.yml @@ -31,7 +31,7 @@ jobs: ${{ runner.os }}-composer- - name: Install dependencies - run: composer install --no-ansi --no-interaction --no-scripts --no-progress --prefer-dist + run: composer update --prefer-dist ${{ matrix.prefer-lowest }} env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 989e0e148354bc5053c9be2040b8cc2b569405df Mon Sep 17 00:00:00 2001 From: anil kumar thakur Date: Tue, 16 Apr 2024 01:47:39 +0545 Subject: [PATCH 15/38] ci fixes --- .github/workflows/automated-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/automated-test.yml b/.github/workflows/automated-test.yml index 58c6440..13a937c 100644 --- a/.github/workflows/automated-test.yml +++ b/.github/workflows/automated-test.yml @@ -31,7 +31,7 @@ jobs: ${{ runner.os }}-composer- - name: Install dependencies - run: composer update --prefer-dist ${{ matrix.prefer-lowest }} + run: composer install --no-ansi --no-interaction --no-scripts --no-progress --prefer-dist env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c539f0c54156388425c17d38938940429b17c52d Mon Sep 17 00:00:00 2001 From: anil kumar thakur Date: Tue, 16 Apr 2024 01:57:00 +0545 Subject: [PATCH 16/38] return types added --- src/Meta.php | 4 ++-- src/Metable.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Meta.php b/src/Meta.php index 5b34218..06788bb 100644 --- a/src/Meta.php +++ b/src/Meta.php @@ -69,7 +69,7 @@ public function metable(): MorphTo * @return mixed * @throws Exceptions\DataTypeException */ - public function getValueAttribute() + public function getValueAttribute(): mixed { if (!isset($this->cachedValue)) { $this->cachedValue = $this->getDataTypeRegistry() @@ -88,7 +88,7 @@ public function getValueAttribute() * @param mixed $value * @throws Exceptions\DataTypeException */ - public function setValueAttribute($value): void + public function setValueAttribute(mixed $value): void { $registry = $this->getDataTypeRegistry(); diff --git a/src/Metable.php b/src/Metable.php index 04d9159..be6524e 100644 --- a/src/Metable.php +++ b/src/Metable.php @@ -488,7 +488,7 @@ private function joinMetaTable(Builder $q, string $key, string $type = 'left'): * * @return mixed */ - private function getMetaCollection() + private function getMetaCollection(): mixed { // load meta relation if not loaded. if (!$this->relationLoaded('meta')) { From 2473e6854381c1e41ea922fdbca5731bb5c34321 Mon Sep 17 00:00:00 2001 From: anil kumar thakur Date: Wed, 17 Apr 2024 09:25:19 +0545 Subject: [PATCH 17/38] Update automated-test.yml --- .github/workflows/automated-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/automated-test.yml b/.github/workflows/automated-test.yml index 13a937c..c5858ce 100644 --- a/.github/workflows/automated-test.yml +++ b/.github/workflows/automated-test.yml @@ -31,7 +31,7 @@ jobs: ${{ runner.os }}-composer- - name: Install dependencies - run: composer install --no-ansi --no-interaction --no-scripts --no-progress --prefer-dist + run: composer install --no-ansi --no-interaction --no-scripts --no-progress --${{ matrix.prefer-lowest }} env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c963e6428182b6fa68aa71e7add7f7d5a664d940 Mon Sep 17 00:00:00 2001 From: anil kumar thakur Date: Wed, 17 Apr 2024 09:27:33 +0545 Subject: [PATCH 18/38] Update automated-test.yml --- .github/workflows/automated-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/automated-test.yml b/.github/workflows/automated-test.yml index c5858ce..2f9adb5 100644 --- a/.github/workflows/automated-test.yml +++ b/.github/workflows/automated-test.yml @@ -31,7 +31,7 @@ jobs: ${{ runner.os }}-composer- - name: Install dependencies - run: composer install --no-ansi --no-interaction --no-scripts --no-progress --${{ matrix.prefer-lowest }} + run: composer install --no-ansi --no-interaction --no-scripts --no-progress ${{ matrix.prefer-lowest }} env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 6f9e26c20755d90fed7f1c45ab7ce019a95ecbe8 Mon Sep 17 00:00:00 2001 From: anil kumar thakur Date: Wed, 17 Apr 2024 09:30:27 +0545 Subject: [PATCH 19/38] ci fixes --- .github/workflows/automated-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/automated-test.yml b/.github/workflows/automated-test.yml index 2f9adb5..58c6440 100644 --- a/.github/workflows/automated-test.yml +++ b/.github/workflows/automated-test.yml @@ -31,7 +31,7 @@ jobs: ${{ runner.os }}-composer- - name: Install dependencies - run: composer install --no-ansi --no-interaction --no-scripts --no-progress ${{ matrix.prefer-lowest }} + run: composer update --prefer-dist ${{ matrix.prefer-lowest }} env: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 1e799de355d1ec5fe529c73fb009f4f3f67ee949 Mon Sep 17 00:00:00 2001 From: anil kumar thakur Date: Wed, 17 Apr 2024 09:32:44 +0545 Subject: [PATCH 20/38] ci fixes --- .github/workflows/automated-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/automated-test.yml b/.github/workflows/automated-test.yml index 58c6440..7c0ab88 100644 --- a/.github/workflows/automated-test.yml +++ b/.github/workflows/automated-test.yml @@ -36,7 +36,7 @@ jobs: COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run phpunit - run: vendor/bin/phpunit --coverage-clover build/logs/clover.xml --debug + run: vendor/bin/phpunit --coverage-clover build/logs/clover.xml - name: Upload coverage results to Coveralls env: From 918021781872f2fbf62c554a0292e4504ca9c7cb Mon Sep 17 00:00:00 2001 From: anil kumar thakur Date: Wed, 17 Apr 2024 09:37:33 +0545 Subject: [PATCH 21/38] php unit run on upgraded to latest --- .github/workflows/automated-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/automated-test.yml b/.github/workflows/automated-test.yml index 7c0ab88..465a3bc 100644 --- a/.github/workflows/automated-test.yml +++ b/.github/workflows/automated-test.yml @@ -2,7 +2,7 @@ name: PHPUnit Tests on: [push, pull_request] jobs: phpunit: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: php-versions: ['8.1', '8.2', '8.3'] From 99ead304cccbb42a9e30904b136233bbba763678 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Wed, 17 Apr 2024 00:30:53 -0400 Subject: [PATCH 22/38] optimize querying by meta value with indexed lookup columns --- CHANGELOG.md | 19 ++ UPGRADING.md | 4 + config/metable.php | 18 ++ docs/source/datatypes.rst | 84 +++++- docs/source/querying_meta.rst | 77 ++++-- ...4_04_14_000000_add_meta_search_columns.php | 14 +- src/DataType/ArrayHandler.php | 19 +- src/DataType/BooleanHandler.php | 4 +- src/DataType/DateTimeHandler.php | 19 +- src/DataType/FloatHandler.php | 4 +- src/DataType/HandlerInterface.php | 9 +- src/DataType/IntegerHandler.php | 4 +- src/DataType/ModelCollectionHandler.php | 9 +- src/DataType/ModelHandler.php | 11 +- src/DataType/NullHandler.php | 4 +- src/DataType/ObjectHandler.php | 19 +- src/DataType/Registry.php | 5 + src/DataType/ScalarHandler.php | 5 + src/DataType/SerializableHandler.php | 19 +- src/DataType/SerializeHandler.php | 19 +- src/DataType/StringHandler.php | 10 +- src/Meta.php | 15 +- src/Metable.php | 258 +++++++++++++++--- tests/Integration/DataType/HandlerTest.php | 4 +- tests/Integration/MetableTest.php | 178 +++++++++++- 25 files changed, 695 insertions(+), 136 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4b9f00..7e88fde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,11 @@ - Added support for Laravel 10 and 11 - Dropped support Laravel versions 9 and below - adjusted some method signatures with PHP 8+ mixed and union types +- New schema migration adding two new columns and improving indexing for searching by meta values. See [UPGRADING.md](UPGRADING.md) for details. ### Data Types +- `getStringValue(): ?string` and `getNumericValue(): null|int|float` methods added to `HandlerInterface` which should convert the original value into a format that can be indexed, if possible. - Added `SerializeHandler` as a catch-all datatype, which will attempt to serialize the data using PHP's `serialize()` function. The payload is encrypted before being stored in the database to prevent unserializing untrusted data. - Deprecated `SerializableHandler` in favor of the new `SerializeHandler` datatype. The `SerializableHandler` will be removed in a future release. In the interim, added the `metable.options.serializable.allowedClasses` config to protect against unserializing untrusted data. - Deprecated `ArrayHandler` and `ObjectHandler`, due to the ambiguity of nested array/objects switching type. These will be removed in a future release. The `SerializeHandler` should be used instead. @@ -20,6 +22,23 @@ - `ModelCollectionHandler` will now validate that the encoded collection class is a valid Eloquent collection before attempting to instantiate it during unserialization. If the class is invalid, an instance of `Illuminate\Database\Eloquent\Collection` will be used instead. - `ModelCollectionHandler` will now validate that the encoded class of each entry is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, that entry in the collection will be omitted. +### Mediable trait + +- `whereMeta()`, `whereMetaIn()`, and `orderByMeta()` query scopes will now scan the indexed `string_value` column instead of the serialized `value` column. This greatly improves performance when searching for meta values against larger datasets. +- `whereMetaNumeric()` and `orderByMetaNumeric()` query scopes will now scan the indexed `numeric_value` column instead of the serialized `value` column. This greatly improves performance when searching for meta values against larger datasets. +- `whereMetaNumeric()` query scope will now accept a value of any type. It will be converted to an integer or float by the handler. This is more consistent with the behaviour of the other query scopes. +- Added additional query scopes to more easily search meta values based on different criteria: + - `whereMetaInNumeric()` + - `whereMetaNotIn()` + - `whereMetaNotInNumeric()` + - `whereMetaBetween()` + - `whereMetaBetweenNumeric()` + - `whereMetaNotBetween()` + - `whereMetaNotBetweenNumeric()` + - `whereMetaIsNull()` + - `whereMetaIsModel()` +- If the data type handlers cannot convert the search value provided to a whereMeta* query scope to a string or numeric value (as appropriate for the scope), then an exception will be thrown. + # 5.0.1 - 2021-09-19 - Fixed `setManyMeta()` not properly serializing certain types of data. diff --git a/UPGRADING.md b/UPGRADING.md index 272e031..ddafcb7 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -5,9 +5,13 @@ * Minimum PHP version moved to 8.1 * Minimum Laravel version moved to 10 * Some methods have had their signatures adjusted to use PHP 8+ mixed and union types. If extending any class or implementing any interface from this package, method signatures may need to be updated. +* A new schema migration has been added which adds two new columns to the meta table and improves indexing for querying by meta values. * Recommended to add the `SerializeHandler` to the end of `datatypes` config (catch-all). * The `SerializableHandler`, `ArrayHandler`, and `ObjectHandler` data types have been deprecated in favor of the new `SerializeHandler`. If you have any Meta encoded using any of these data types, you should continue to include them in the `datatypes` config _after_ the `SerializeHandler` to ensure that existing values will continue to be properly decoded, but new values will use the new encoding. Once all old values have been migrated, you may remove the deprecated data types from the `datatypes` config. * For security reasons, if you have any existing Meta encoded using `SerializableHandler`, you must configure the `metable.options.serializable.allowedClasses` config to list classes that are allowed to be unserialized. Otherwise, all objects will be returned as `__PHP_Incomplete_Class`. This config may be set to `true` to disable this security check and allow any class, but this is not recommended. +* If you have any custom data types, you will need to implement the `getStringValue()` and `getNumericValue()` methods in your handler class to populate those indexes. You may return `null` if the value does not need to be searchable. +* Once you have applied the schema migration and configured the changes to the `datatypes` config, you should run the `meta:refresh` command to update all existing meta values to use the new types and populate the index columns. +* Review the documentation about which data types can be queried with the various `whereMeta*` and `whereMeta*Numeric` query scopes. If you are querying the serialized `value` column directly, be aware that the formatting of array/object data types has changed. ## 4.X -> 5.X - New migration file added which adds a new composite unique index to the meta table on `metable_type`, `metable_id`, and `key`. Make sure that you have no duplicate keys for a given entity (previously possible as a race condition) before applying the new migration. diff --git a/config/metable.php b/config/metable.php index 1c6e6ee..0ac563d 100644 --- a/config/metable.php +++ b/config/metable.php @@ -59,4 +59,22 @@ ], ], ], + + /** + * Whether to index complex data types (arrays, objects, etc). + * If enabled the value will be serialized and the first 255 characters will be indexed. + * This allows for using whereMeta*() query scopes on serialized values, but may have + * performance and disk usage implications for large data sets. + * + * If you do not intend to query meta values containing complex data types, you should leave this disabled. + */ + 'indexComplexDataTypes' => false, + + /** + * Number of bytes to index for strings and complex data types. + * This value is used to determine the length of the index column in the database. + * Higher values allow for better precision when querying, + * but will use more disk space in the database. + */ + 'stringValueIndexLength' => 255, ]; diff --git a/docs/source/datatypes.rst b/docs/source/datatypes.rst index 2b27037..ffeca8e 100644 --- a/docs/source/datatypes.rst +++ b/docs/source/datatypes.rst @@ -7,6 +7,8 @@ Data Types You can attach a number of different kinds of values to a ``Metable`` model. The data types that are supported by Laravel-Mediable out of the box are the following. +Meta encoded with different data types support different query scopes for filtering by meta value. See :ref:`querying_meta` for more information on available query scopes. + Scalar Values --------------- @@ -14,6 +16,12 @@ The following scalar values are supported. Boolean ^^^^^^^^ ++----------------------+-----+ +| Handler | \Plank\Metable\DataType\BooleanHandler | +| String Query Scopes | Yes | +| Numeric Query Scopes | Yes | +| Other Query Scopes | | ++----------------------+-----+ :: @@ -22,6 +30,12 @@ Boolean Integer ^^^^^^^^ ++----------------------+-----+ +| Handler | \Plank\Metable\DataType\IntegerHandler | +| String Query Scopes | Yes | +| Numeric Query Scopes | Yes | +| Other Query Scopes | | ++----------------------+-----+ :: @@ -30,6 +44,12 @@ Integer Float ^^^^^^^^ ++----------------------+-----+ +| Handler | \Plank\Metable\DataType\FloatHandler | +| String Query Scopes | Yes | +| Numeric Query Scopes | Yes | +| Other Query Scopes | | ++----------------------+-----+ :: @@ -38,6 +58,12 @@ Float Null ^^^^^^^^ ++----------------------+-----+ +| Handler | \Plank\Metable\DataType\NullHandler | +| String Query Scopes | No | +| Numeric Query Scopes | No | +| Other Query Scopes | whereMetaIsNull() | ++----------------------+-----+ :: @@ -46,6 +72,12 @@ Null String ^^^^^^^^ ++----------------------+-----+ +| Handler | \Plank\Metable\DataType\StringHandler | +| String Query Scopes | Yes, first `metable.stringValueIndexLength` characters indexed | +| Numeric Query Scopes | if string is numeric | +| Other Query Scopes | | ++----------------------+-----+ :: @@ -62,6 +94,13 @@ The following classes and interfaces are supported. Eloquent Models ^^^^^^^^^^^^^^^^^ ++----------------------+-----+ +| Handler | \Plank\Metable\DataType\ModelHandler | +| String Query Scopes | Yes | +| Numeric Query Scopes | No | +| Other Query Scopes | whereMetaIsModel() | ++----------------------+-----+ + It is possible to attach another Eloquent model to a ``Metable`` model. :: @@ -85,6 +124,13 @@ When ``$metable->getMeta()`` is called, a fresh instance of the class will be cr Eloquent Collections ^^^^^^^^^^^^^^^^^^^^ ++----------------------+-----+ +| Handler | \Plank\Metable\DataType\ModelCollectionHandler | +| String Query Scopes | No | +| Numeric Query Scopes | No | +| Other Query Scopes | | ++----------------------+-----+ + Similarly, it is possible to attach multiple models to a key by providing an instance of ``Illuminate\Database\Eloquent\Collection`` containing the models. As with individual models, both existing and unsaved instances can be stored. @@ -97,8 +143,14 @@ As with individual models, both existing and unsaved instances can be stored. DateTime & Carbon ^^^^^^^^^^^^^^^^^^ ++----------------------+-----+ +| Handler | \Plank\Metable\DataType\DateTimeHandler | +| String Query Scopes | Yes (UTC format) | +| Numeric Query Scopes | Yes (timestamp) | +| Other Query Scopes | | ++----------------------+-----+ -Any object implementing the ``DateTimeInterface``. Object will be converted to a ``Carbon`` instance. +Any object implementing the ``DateTimeInterface``. Object will be converted to a ``Carbon`` instance when unserialized. :: @@ -108,6 +160,13 @@ Any object implementing the ``DateTimeInterface``. Object will be converted to Objects and Arrays ^^^^^ ++----------------------+-----+ +| Handler | \Plank\Metable\DataType\IntegerHandler | +| String Query Scopes | if `metable.indexComplexDataTypes` is enabled | +| Numeric Query Scopes | No | +| Other Query Scopes | | ++----------------------+-----+ + Objects and arrays will be serialized using PHP's `serialize()` function, to allow for the storage and retrieval of complex data structures. The serialized value is encrypted before being stored in the database, and decrypted when retrieved to prevent tampered data from being unserialized. :: @@ -126,6 +185,13 @@ The following data types are deprecated and should not be used in new code. They Array ^^^^^^^^ ++----------------------+-----+ +| Handler | \Plank\Metable\DataType\ArrayHandler | +| String Query Scopes | if `metable.indexComplexDataTypes` is enabled | +| Numeric Query Scopes | No | +| Other Query Scopes | | ++----------------------+-----+ + .. warning:: The ``ArrayHandler`` datatype is deprecated. The ``SerializeHandler`` should be used for handling arrays. Arrays of scalar values. Nested arrays are supported. @@ -151,6 +217,13 @@ Arrays of scalar values. Nested arrays are supported. Serializable ^^^^^^^^^^^^^ ++----------------------+-----+ +| Handler | \Plank\Metable\DataType\ArrayHandler | +| String Query Scopes | if `metable.indexComplexDataTypes` is enabled | +| Numeric Query Scopes | No | +| Other Query Scopes | | ++----------------------+-----+ + .. warning:: The ``SerializableHandler`` datatype is deprecated. The ``SerializeHandler`` should be used for handling all objects. Any object implementing the PHP ``Serializable`` interface. @@ -167,11 +240,18 @@ Any object implementing the PHP ``Serializable`` interface. $metable->setMeta('example', $serializable); -For security reasons, it is necessary to list any classes that can be unserialized in the ``metable.options.serializable.allowedClasses`` key in the ``config/metable.php`` file. This is to prevent arbitrary code execution when unserializing untrusted data. This config can be set to true to allow all classes, but this is not recommended. +For security reasons, it is necessary to list any classes that can be unserialized in the ``metable.options.serializable.allowedClasses`` key in the ``config/metable.php`` file. This is to prevent PHP Object Injection vulnerabilities when unserializing untrusted data. This config can be set to true to allow all classes, but this is not recommended. Plain Objects ^^^^^^^^^^^^^^ ++----------------------+-----+ +| Handler | \Plank\Metable\DataType\ArrayHandler | +| String Query Scopes | if `metable.indexComplexDataTypes` is enabled | +| Numeric Query Scopes | No | +| Other Query Scopes | | ++----------------------+-----+ + .. warning:: The ``ObjectHandler`` datatype is deprecated. The ``SerializeHandler`` should be used for handling all objects. Any other objects will be converted to ``stdClass`` plain objects. You can control what properties are stored by implementing the ``JsonSerializable`` interface on the class of your stored object. diff --git a/docs/source/querying_meta.rst b/docs/source/querying_meta.rst index da6764f..dbe4844 100644 --- a/docs/source/querying_meta.rst +++ b/docs/source/querying_meta.rst @@ -34,61 +34,88 @@ You can also query for records that does not contain a meta key using the ``wher Comparing value --------------- -You can restrict your query based on the value stored at a meta key. The ``whereMeta()`` method can be used to compare the value using any of the operators accepted by the Laravel query builder's ``where()`` method. +You can restrict your query based on the value stored for a particular meta key. Query scopes for selecting records based on the value of attached meta come in two main flavors: string-based and numeric-based, which informs which indexes are used to lookup the data efficiently. Different data types may support filtering and ordering by different query scopes. Refer to the :ref:`Data Types ` section for more information. + +The value passed to to the query scope will be converted to the appropriate data type before being passed to the database. As such any value that you can pass to the ``Metable::setMeta()`` method can be passed to the query scope, as long as the data type is supported by the operation. + +String Value Query Scopes +^^^^^^^^^^^^^^^^^^^^^^^^^ + +All string-based query scopes using lexicographic comparison to look up values. This means that the values are compared alphabetically as strings. This can lead to unexpected results when comparing numbers, e.g. ``'11'`` is greater than ``'100'``. + +By default, only the first 255 characters of a string are indexed (can be adjusted with the ``metable.stringValueIndexLength`` config). When querying by longer values, characters exceeding the limit will be ignored when determining if the criteria matches. The ``whereMeta()`` method will attempt to work around this by comparing the entire serialized value after the results have been filtered by the indexed portion (other query scopes will not do this). + + +The ``whereMeta()`` method can be used to compare the value using any of the operators accepted by the Laravel query builder's ``where()`` method. :: get(); + $models = MyModel::whereMeta('status', 'success')->get(); // greater than $models = MyModel::whereMeta('name', '>', 'M')->get(); // like - $models = MyModel::whereMeta('summary', 'like', '%bacon%')->get(); + $models = MyModel::whereMeta('summary', 'like', 'Once upon a time%')->get(); //etc. -The ``whereMetaIn()`` method is also available to find records where the value is matches one of a predefined set of options. +The ``whereMetaIn()`` method and its inverse are also available to find records where the value is matches one of a predefined set of options. :: get(); + $models = MyModel::whereMetaNotIn('currency', ['USD', 'GBP', 'EUR'])->get(); - -The ``whereMeta()`` and ``whereMetaIn()`` methods perform string comparison (lexicographic ordering). Any non-string values passed to these methods will be serialized to a string. This is useful for evaluating equality (``=``) or inequality (``<>``), but may behave unpredictably with some other operators for non-string data types. +The ``whereMetaBetween()`` and its inverse method can be used to compare records to a range. :: setMeta('letters', ['a', 'b', 'c']); + $models = MyModel::whereMetaBetween('country_code', 'AD', 'AZ')->get(); + $models = MyModel::whereMetaNotBetween('name', 'a', 'm')->get(); - // array argument will be serialized using the same mechanism - // the original model will be found. - $model = MyModel::whereMeta('letters', ['a', 'b', 'c'])->first(); +Numeric Value Query Scopes +^^^^^^^^^^^^^^^^^^^^^^^^^ -Depending on the format of the original data, it may be possible to compare against subsets of the data using the SQL ``like`` operator and a string argument. +Numeric values are indexed in a decimal column that supports up to 20 integral digits and 16 fractional digits (enough to support 64-bit integers and floats at full precision). This allows for a wide range of values to be stored and queried efficiently. The numeric query scopes use numeric comparison to look up values. +Query scopes are available for numeric values as for string values. :: setMeta('letters', ['a', 'b', 'c']); + $models = MyModel::whereMetaNumeric('counter', '>', 42)->get(); + $models = MyModel::whereMetaInNumeric('http_code', [401, 403])->get(); + $models = MyModel::whereMetaNotInNumeric('department', [])->get(); + $models = MyModel::whereMetaBetweenNumeric('completed_at', Carbon::yesterday(), Carbon::today())->get(); + $models = MyModel::whereMetaNotBetweenNumeric('percentile', 90, 100)->get(); - // check for the presence of one value within the json encoded array - // the original model will be found - $model = MyModel::whereMeta('letters', 'like', '%"b"%' )->first(); +Other Query Scopes +^^^^^^^^^^^^^^^^^^ +You can look up if a meta key contains a reference to a particular model using the ``whereMetaIsModel()`` method. -When comparing integer or float values with the ``<``, ``<=``, ``>=`` or ``>`` operators, use the ``whereMetaNumeric()`` method. This will cast the values to a number before performing the comparison, in order to avoid common pitfalls of lexicographic ordering (e.g. ``'11'`` is greater than ``'100'``). +:: + + get(); + $models = MyModel::whereMetaIsModel($otherModelInstance)->get(); + + // find models that reference a any instance of a model class + $models = MyModel::whereMetaIsModel(\App\MyOtherModel::class)->get(); + +If you specifically assigned a meta key to `null`, you can query for models that have a `null` value for that key using the ``whereMetaNull()`` method. :: ', 42)->get(); + $models = MyModel::whereMetaNull('notes')->get(); + Ordering results ---------------- @@ -98,10 +125,10 @@ You can apply an order by clause to the query to sort the results by the value o :: get(); - //order by numeric value + // numeric order $models = MyModel::orderByMetaNumeric('score', 'desc')->get(); By default, all records matching the rest of the query will be ordered. Any records which have no meta assigned to the key being sorted on will be considered to have a value of ``null``. @@ -118,11 +145,9 @@ To automatically exclude all records that do not have meta assigned to the sorte $models = MyModel::whereHasMeta('score') ->orderByMetaNumeric('score', 'desc')->get(); -A Note on Optimization ----------------------- -Laravel-Metable is intended a convenient means for handling data of many different shapes and sizes. It was designed for dealing with data that only a subset of all models in a table would have any need for. +Querying by Complex Data Types +------------------------------- -For example, you have a Page model with a template field and each template needs some number of additional fields to modify how it displays. If you have X templates which each have up to Y fields, adding all of these as columns to pages table will quickly get out of hand. Instead, appending these template fields to the Page model as meta can make handling this use case trivial. +By default, meta containing complex data types (e.g. objects and arrays) are not indexed and cannot be filtered or ordered with any of the above methods. If you need to query by these values, you can enable the ``metable.indexComplexDataTypes`` config option. This will cause a truncated version of the serialized value to be indexed. This can be useful for exact matches, but may not work predictably for other operations. Given the database overhead of indexing complex data types, it is recommended to only enable this feature if you need it. -Laravel-Metable makes it very easy to append just about any data to your models. However, for sufficiently large data sets or data that is queried very frequently, it will often be more efficient to use regular database columns instead in order to take advantage of native SQL data types and indexes. The optimal solution will depend on your use case. diff --git a/migrations/2024_04_14_000000_add_meta_search_columns.php b/migrations/2024_04_14_000000_add_meta_search_columns.php index 11ba084..fba14f6 100644 --- a/migrations/2024_04_14_000000_add_meta_search_columns.php +++ b/migrations/2024_04_14_000000_add_meta_search_columns.php @@ -2,6 +2,7 @@ use Illuminate\Database\Migrations\Migration; use Illuminate\Database\Schema\Blueprint; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; class AddMetaSearchColumns extends Migration @@ -14,9 +15,13 @@ class AddMetaSearchColumns extends Migration public function up() { Schema::table('meta', function (Blueprint $table) { - $table->decimal('numeric_value', 18, 9)->nullable(); - $table->string('string_value', 255)->nullable(); + $table->decimal('numeric_value', 36, 16)->nullable(); + $table->string( + 'string_value', + config('metable.stringValueIndexLength', 255) + )->nullable(); $table->dropIndex(['key', 'metable_type']); + $table->dropIndex(['key']); $table->index(['key', 'metable_type', 'numeric_value']); $table->index(['key', 'metable_type', 'string_value']); }); @@ -32,9 +37,10 @@ public function down() Schema::table('meta', function (Blueprint $table) { $table->dropIndex(['key', 'metable_type', 'string_value']); $table->dropIndex(['key', 'metable_type', 'numeric_value']); - $table->index(['metable_type', 'metable_id']); - $table->dropColumn('numeric_value'); + $table->index(['key']); + $table->index(['key', 'metable_type']); $table->dropColumn('string_value'); + $table->dropColumn('numeric_value'); }); } } diff --git a/src/DataType/ArrayHandler.php b/src/DataType/ArrayHandler.php index 16ed86f..3d951d3 100644 --- a/src/DataType/ArrayHandler.php +++ b/src/DataType/ArrayHandler.php @@ -45,13 +45,26 @@ public function unserializeValue(string $serializedValue): mixed ); } - public function getNumericValue(mixed $value, string $serializedValue): null|int|float + public function getNumericValue(mixed $value): null|int|float { return null; } - public function getStringValue(mixed $value, string $serializedValue): null|string + public function getStringValue(mixed $value): null|string { - return null; + if (!config('metable.indexComplexDataTypes', false)) { + return null; + } + + return substr( + json_encode($value, JSON_THROW_ON_ERROR), + 0, + config('metable.stringValueIndexLength', 255) + ); + } + + public function isIdempotent(): bool + { + return true; } } diff --git a/src/DataType/BooleanHandler.php b/src/DataType/BooleanHandler.php index fc8476c..700ab18 100644 --- a/src/DataType/BooleanHandler.php +++ b/src/DataType/BooleanHandler.php @@ -12,12 +12,12 @@ class BooleanHandler extends ScalarHandler */ protected $type = 'boolean'; - public function getNumericValue(mixed $value, string $serializedValue): null|int|float + public function getNumericValue(mixed $value): null|int|float { return $value ? 1 : 0; } - public function getStringValue(mixed $value, string $serializedValue): null|string + public function getStringValue(mixed $value): null|string { return $value ? 'true' : 'false'; } diff --git a/src/DataType/DateTimeHandler.php b/src/DataType/DateTimeHandler.php index 5e67000..b055ebf 100644 --- a/src/DataType/DateTimeHandler.php +++ b/src/DataType/DateTimeHandler.php @@ -15,7 +15,7 @@ class DateTimeHandler implements HandlerInterface * * @var string */ - protected $format = 'Y-m-d H:i:s.uO'; + const FORMAT = 'Y-m-d H:i:s.uO'; /** * {@inheritdoc} @@ -38,7 +38,7 @@ public function canHandleValue(mixed $value): bool */ public function serializeValue(mixed $value): string { - return $value->format($this->format); + return $value->format(self::FORMAT); } /** @@ -46,18 +46,25 @@ public function serializeValue(mixed $value): string */ public function unserializeValue(string $serializedValue): mixed { - return Carbon::createFromFormat($this->format, $serializedValue); + return Carbon::createFromFormat(self::FORMAT, $serializedValue); } - public function getNumericValue(mixed $value, string $serializedValue): null|int|float + public function getNumericValue(mixed $value): null|int|float { return $value instanceof DateTimeInterface ? $value->getTimestamp() : null; } - public function getStringValue(mixed $value, string $serializedValue): null|string + public function getStringValue(mixed $value): null|string { - return $serializedValue; + return $value instanceof DateTimeInterface + ? $value->copy()->setTimezone('UTC')->format(self::FORMAT) + : null; + } + + public function isIdempotent(): bool + { + return true; } } diff --git a/src/DataType/FloatHandler.php b/src/DataType/FloatHandler.php index 40feb91..a7d7ac2 100644 --- a/src/DataType/FloatHandler.php +++ b/src/DataType/FloatHandler.php @@ -20,12 +20,12 @@ public function getDataType(): string return 'float'; } - public function getNumericValue(mixed $value, string $serializedValue): null|int|float + public function getNumericValue(mixed $value): null|int|float { return $value; } - public function getStringValue(mixed $value, string $serializedValue): null|string + public function getStringValue(mixed $value): null|string { return (string) $value; } diff --git a/src/DataType/HandlerInterface.php b/src/DataType/HandlerInterface.php index 6714842..94179a0 100644 --- a/src/DataType/HandlerInterface.php +++ b/src/DataType/HandlerInterface.php @@ -32,9 +32,9 @@ public function canHandleValue(mixed $value): bool; */ public function serializeValue(mixed $value): string; - public function getNumericValue(mixed $value, string $serializedValue): null|int|float; + public function getNumericValue(mixed $value): null|int|float; - public function getStringValue(mixed $value, string $serializedValue): null|string; + public function getStringValue(mixed $value): null|string; /** * Convert a serialized string back to its original value. @@ -44,4 +44,9 @@ public function getStringValue(mixed $value, string $serializedValue): null|stri * @return mixed */ public function unserializeValue(string $serializedValue): mixed; + + /** + * Indicate whether multiple serializations of the same value will produce the same result. + */ + public function isIdempotent(): bool; } diff --git a/src/DataType/IntegerHandler.php b/src/DataType/IntegerHandler.php index 3722769..cf3cae1 100644 --- a/src/DataType/IntegerHandler.php +++ b/src/DataType/IntegerHandler.php @@ -12,12 +12,12 @@ class IntegerHandler extends ScalarHandler */ protected $type = 'integer'; - public function getNumericValue(mixed $value, string $serializedValue): null|int|float + public function getNumericValue(mixed $value): null|int|float { return $value; } - public function getStringValue(mixed $value, string $serializedValue): null|string + public function getStringValue(mixed $value): null|string { return (string) $value; } diff --git a/src/DataType/ModelCollectionHandler.php b/src/DataType/ModelCollectionHandler.php index ef1895f..b25a06d 100644 --- a/src/DataType/ModelCollectionHandler.php +++ b/src/DataType/ModelCollectionHandler.php @@ -117,13 +117,18 @@ private function loadModels(array $items) return $results; } - public function getNumericValue(mixed $value, string $serializedValue): null|int|float + public function getNumericValue(mixed $value): null|int|float { return null; } - public function getStringValue(mixed $value, string $serializedValue): null|string + public function getStringValue(mixed $value): null|string { return null; } + + public function isIdempotent(): bool + { + return true; + } } diff --git a/src/DataType/ModelHandler.php b/src/DataType/ModelHandler.php index b90c5ea..222dc55 100644 --- a/src/DataType/ModelHandler.php +++ b/src/DataType/ModelHandler.php @@ -57,13 +57,18 @@ public function unserializeValue(string $serializedValue): mixed return $class::query()->find($id); } - public function getNumericValue(mixed $value, string $serializedValue): null|int|float + public function getNumericValue(mixed $value): null|int|float { return null; } - public function getStringValue(mixed $value, string $serializedValue): null|string + public function getStringValue(mixed $value): null|string { - return $serializedValue; + return $this->serializeValue($value); + } + + public function isIdempotent(): bool + { + return true; } } diff --git a/src/DataType/NullHandler.php b/src/DataType/NullHandler.php index b8fe689..1feb3ed 100644 --- a/src/DataType/NullHandler.php +++ b/src/DataType/NullHandler.php @@ -20,12 +20,12 @@ public function getDataType(): string return 'null'; } - public function getNumericValue(mixed $value, string $serializedValue): null|int|float + public function getNumericValue(mixed $value): null|int|float { return null; } - public function getStringValue(mixed $value, string $serializedValue): null|string + public function getStringValue(mixed $value): null|string { return null; } diff --git a/src/DataType/ObjectHandler.php b/src/DataType/ObjectHandler.php index 319c719..f5d53f2 100644 --- a/src/DataType/ObjectHandler.php +++ b/src/DataType/ObjectHandler.php @@ -40,13 +40,26 @@ public function unserializeValue(string $serializedValue): mixed return json_decode($serializedValue, false); } - public function getNumericValue(mixed $value, string $serializedValue): null|int|float + public function getNumericValue(mixed $value): null|int|float { return null; } - public function getStringValue(mixed $value, string $serializedValue): null|string + public function getStringValue(mixed $value): null|string { - return null; + if (!config('metable.indexComplexDataTypes', false)) { + return null; + } + + return substr( + json_encode($value, JSON_THROW_ON_ERROR), + 0, + config('metable.stringValueIndexLength', 255) + ); + } + + public function isIdempotent(): bool + { + return true; } } diff --git a/src/DataType/Registry.php b/src/DataType/Registry.php index b8ab30a..588b8a3 100644 --- a/src/DataType/Registry.php +++ b/src/DataType/Registry.php @@ -90,4 +90,9 @@ public function getTypeForValue($value): string throw DataTypeException::handlerNotFoundForValue($value); } + + public function getHandlerForValue($value): HandlerInterface + { + return $this->getHandlerForType($this->getTypeForValue($value)); + } } diff --git a/src/DataType/ScalarHandler.php b/src/DataType/ScalarHandler.php index 853a552..2c62706 100644 --- a/src/DataType/ScalarHandler.php +++ b/src/DataType/ScalarHandler.php @@ -49,4 +49,9 @@ public function unserializeValue(string $serializedValue): mixed return $serializedValue; } + + public function isIdempotent(): bool + { + return true; + } } diff --git a/src/DataType/SerializableHandler.php b/src/DataType/SerializableHandler.php index dd4e8a0..a803468 100644 --- a/src/DataType/SerializableHandler.php +++ b/src/DataType/SerializableHandler.php @@ -43,13 +43,26 @@ public function unserializeValue(string $serializedValue): mixed return unserialize($serializedValue, ['allowed_classes' => $allowedClasses]); } - public function getNumericValue(mixed $value, string $serializedValue): null|int|float + public function getNumericValue(mixed $value): null|int|float { return null; } - public function getStringValue(mixed $value, string $serializedValue): null|string + public function getStringValue(mixed $value): null|string { - return null; + if (!config('metable.indexComplexDataTypes', false)) { + return null; + } + + return substr( + serialize($value), + 0, + config('metable.stringValueIndexLength', 255) + ); + } + + public function isIdempotent(): bool + { + return true; } } diff --git a/src/DataType/SerializeHandler.php b/src/DataType/SerializeHandler.php index 4fc1a96..d7877ec 100644 --- a/src/DataType/SerializeHandler.php +++ b/src/DataType/SerializeHandler.php @@ -27,13 +27,26 @@ public function unserializeValue(string $serializedValue): mixed return app('encrypter')->decrypt($serializedValue, true); } - public function getNumericValue(mixed $value, string $serializedValue): null|int|float + public function getNumericValue(mixed $value): null|int|float { return null; } - public function getStringValue(mixed $value, string $serializedValue): null|string + public function getStringValue(mixed $value): null|string { - return null; + if (!config('metable.indexComplexDataTypes', false)) { + return null; + } + + return substr( + serialize($value), + 0, + config('metable.stringValueIndexLength', 255) + ); + } + + public function isIdempotent(): bool + { + return false; } } \ No newline at end of file diff --git a/src/DataType/StringHandler.php b/src/DataType/StringHandler.php index 0caf97b..9e1ebc1 100644 --- a/src/DataType/StringHandler.php +++ b/src/DataType/StringHandler.php @@ -12,7 +12,7 @@ class StringHandler extends ScalarHandler */ protected $type = 'string'; - public function getNumericValue(mixed $value, string $serializedValue): null|int|float + public function getNumericValue(mixed $value): null|int|float { if (is_numeric($value)) { return (float)$value; @@ -20,8 +20,12 @@ public function getNumericValue(mixed $value, string $serializedValue): null|int return null; } - public function getStringValue(mixed $value, string $serializedValue): null|string + public function getStringValue(mixed $value): null|string { - return substr($value, 0, 255); + return substr( + $value, + 0, + config('metable.stringValueIndexLength', 255) + ); } } diff --git a/src/Meta.php b/src/Meta.php index 40a4d3a..6ee0a92 100644 --- a/src/Meta.php +++ b/src/Meta.php @@ -103,17 +103,10 @@ public function setValueAttribute($value): void $this->attributes['type'] = $registry->getTypeForValue($value); $handler = $registry->getHandlerForType($this->type); - $serializedValue = $handler->serializeValue($value); - - $this->attributes['value'] = $serializedValue; - $this->attributes['string_value'] = $handler->getStringValue( - $value, - $serializedValue - ); - $this->attributes['numeric_value'] = $handler->getNumericValue( - $value, - $serializedValue - ); + + $this->attributes['value'] = $handler->serializeValue($value); + $this->attributes['string_value'] = $handler->getStringValue($value); + $this->attributes['numeric_value'] = $handler->getNumericValue($value); $this->cachedValue = null; } diff --git a/src/Metable.php b/src/Metable.php index 509fc9b..69f4665 100644 --- a/src/Metable.php +++ b/src/Metable.php @@ -4,22 +4,34 @@ use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Collection; +use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\MorphMany; use Illuminate\Database\Query\JoinClause; use Illuminate\Support\Facades\DB; +use Plank\Metable\DataType\HandlerInterface; +use Plank\Metable\DataType\Registry; /** * Trait for giving Eloquent models the ability to handle Meta. * * @property Collection $meta * @method static Builder whereHasMeta(string|string[] $key): void - * @method static Builder WhereDoesntHaveMeta(string|string[] $key) - * @method static Builder WhereHasMetaKeys(array $keys) - * @method static Builder WhereMeta(string $key, $operator, mixed $value = null) - * @method static Builder WhereMetaNumeric(string $key, string $operator, mixed $value) - * @method static Builder WhereMetaIn(string $key, array $values) - * @method static Builder OrderByMeta(string $key, string $direction = 'asc', bool $strict = false) - * @method static Builder OrderByMetaNumeric(string $key, string $direction = 'asc', bool $strict = false) + * @method static Builder whereDoesntHaveMeta(string|string[] $key) + * @method static Builder whereHasMetaKeys(array $keys) + * @method static Builder whereMeta(string $key, mixed $operator, mixed $value = null) + * @method static Builder whereMetaNumeric(string $key, mixed $operator, mixed $value = null) + * @method static Builder whereMetaIn(string $key, array $values) + * @method static Builder whereMetaInNumeric(string $key, array $values) + * @method static Builder whereMetaNotIn(string $key, array $values) + * @method static Builder whereMetaNotInNumeric(string $key, array $values) + * @method static Builder whereMetaBetween(string $key, mixed $min, mixed $max, bool $not = false) + * @method static Builder whereMetaBetweenNumeric(string $key, mixed $min, mixed $max, bool $not = false) + * @method static Builder whereMetaNotBetween(string $key, mixed $min, mixed $max) + * @method static Builder whereMetaNotBetweenNumeric(string $key, mixed $min, mixed $max) + * @method static Builder whereMetaIsNull(string $key) + * @method static Builder whereMetaIsModel(string $key, Model|string $classOrInstance, null|int|string $id = null) + * @method static Builder orderByMeta(string $key, string $direction = 'asc', bool $strict = false) + * @method static Builder orderByMetaNumeric(string $key, string $direction = 'asc', bool $strict = false) */ trait Metable { @@ -33,7 +45,7 @@ trait Metable * * @return void */ - public static function bootMetable() + public static function bootMetable(): void { // delete all attached meta on deletion static::deleted(function (self $model) { @@ -248,7 +260,7 @@ public function removeManyMeta(array $keys): void public function purgeMeta(): void { $this->meta()->delete(); - $this->setRelation('meta', $this->makeMeta()->newCollection([])); + $this->setRelation('meta', $this->makeMeta()->newCollection()); } /** @@ -339,15 +351,28 @@ public function scopeWhereMeta(Builder $q, string $key, mixed $operator, mixed $ $operator = '='; } - // Convert value to its serialized version for comparison. - if (!is_string($value)) { - $value = $this->makeMeta($key, $value)->getRawValue(); - } - - $q->whereHas('meta', function (Builder $q) use ($key, $operator, $value) { - $q->where('key', $key); - $q->where('value', $operator, $value); - }); + $stringValue = $this->valueToString($value); + $q->whereHas( + 'meta', + function (Builder $q) use ($key, $operator, $stringValue, $value) { + $q->where('key', $key); + $q->where('string_value', $operator, $stringValue); + + // If the value is a string and the string value is at the maximum length, + // we can optimize the query by looking up using the index first + // then compare the serialized value (not indexed) afterward to ensure correctness. + if (strlen($stringValue) === config( + 'metable.stringValueIndexLength', + 255 + ) + ) { + $handler = $this->getHandlerForValue($value); + if ($handler->isIdempotent()) { + $q->where('value', $operator, $handler->serializeValue($value)); + } + } + } + ); } /** @@ -357,48 +382,173 @@ public function scopeWhereMeta(Builder $q, string $key, mixed $operator, mixed $ * * @param Builder $q * @param string $key - * @param string $operator - * @param int|float $value + * @param mixed|string $operator + * @param mixed $value * * @return void */ - public function scopeWhereMetaNumeric(Builder $q, string $key, string $operator, int|float $value): void + public function scopeWhereMetaNumeric(Builder $q, string $key, mixed $operator, mixed $value = null): void + { + // Shift arguments if no operator is present. + if (!isset($value)) { + $value = $operator; + $operator = '='; + } + + $numericValue = $this->valueToNumeric($value); + $q->whereHas('meta', function (Builder $q) use ($key, $operator, $numericValue) { + $q->where('key', $key); + $q->where('numeric_value', $operator, $numericValue); + }); + } + + public function scopeWhereMetaBetween( + Builder $q, + string $key, + mixed $min, + mixed $max, + bool $not = false + ): void { + $min = $this->valueToString($min); + $max = $this->valueToString($max); + + $q->whereHas( + 'meta', + function (Builder $q) use ($key, $min, $max, $not) { + $q->where('key', $key); + $q->whereBetween('string_value', [$min, $max], 'and', $not); + } + ); + } + + public function scopeWhereMetaNotBetween( + Builder $q, + string $key, + mixed $min, + mixed $max, + ): void { + $this->scopeWhereMetaBetween($q, $key, $min, $max, true); + } + + public function scopeWhereMetaBetweenNumeric( + Builder $q, + string $key, + mixed $min, + mixed $max, + bool $not = false + ): void { + $min = $this->valueToNumeric($min); + $max = $this->valueToNumeric($max); + + $q->whereHas('meta', function (Builder $q) use ($key, $min, $max, $not) { + $q->where('key', $key); + $q->whereBetween('numeric_value', [$min, $max], 'and', $not); + }); + } + + public function scopeWhereMetaNotBetweenNumeric( + Builder $q, + string $key, + mixed $min, + mixed $max + ): void { + $this->scopeWhereMetaBetweenNumeric($q, $key, $min, $max, true); + } + + /** + * Query scope to restrict the query to records which have `Meta` with a specific key and a `null` value. + * @param Builder $q + * @param string $key + * @return void + */ + public function scopeWhereMetaIsNull(Builder $q, string $key): void { - // Since we are manually interpolating into the query, - // escape the operator to protect against injection. - $validOperators = ['<', '<=', '>', '>=', '=', '<>', '!=']; - $operator = in_array($operator, $validOperators) ? $operator : '='; - $field = $q->getQuery() - ->getGrammar() - ->wrap($this->meta()->getRelated()->getTable() . '.value'); - - $q->whereHas('meta', function (Builder $q) use ($key, $operator, $value, $field) { + $q->whereHas('meta', function (Builder $q) use ($key) { $q->where('key', $key); - $q->whereRaw("cast({$field} as decimal) {$operator} ?", [(float)$value]); + $q->whereNull('string_value'); + $q->where('type', 'null'); }); } + + public function scopeWhereMetaIsModel( + Builder $q, + string $key, + Model|string $classOrInstance, + null|int|string $id = null + ): void { + if ($classOrInstance instanceof Model) { + $id = $classOrInstance->getKey(); + $classOrInstance = get_class($classOrInstance); + } + $value = $classOrInstance; + if ($id) { + $value .= '#' . $id; + } else { + $value .= '%'; + } + + $this->scopeWhereMeta($q, $key, 'like', $value); + } + /** * Query scope to restrict the query to records which have `Meta` with a specific key and a value within a specified set of options. * * @param Builder $q * @param string $key * @param array $values + * @param bool $not * * @return void */ - public function scopeWhereMetaIn(Builder $q, string $key, array $values): void - { + public function scopeWhereMetaIn( + Builder $q, + string $key, + array $values, + bool $not = false + ): void { + $values = array_map(function ($val) use ($key) { + return $this->valueToString($val); + }, $values); + + $q->whereHas('meta', function (Builder $q) use ($key, $values, $not) { + $q->where('key', $key); + $q->whereIn('string_value', $values, 'and', $not); + }); + } + + public function scopeWhereMetaNotIn( + Builder $q, + string $key, + array $values + ): void { + $this->scopeWhereMetaIn($q, $key, $values, true); + } + + public function scopeWhereMetaInNumeric( + Builder $q, + string $key, + array $values, + bool $not = false + ): void { $values = array_map(function ($val) use ($key) { - return is_string($val) ? $val : $this->makeMeta($key, $val)->getRawValue(); + return $this->valueToNumeric($val); }, $values); - $q->whereHas('meta', function (Builder $q) use ($key, $values) { + $q->whereHas('meta', function (Builder $q) use ($key, $values, $not) { $q->where('key', $key); - $q->whereIn('value', $values); + $q->whereIn('numeric_value', $values, 'and', $not); }); } + public function scopeWhereMetaNotInNumeric( + Builder $q, + string $key, + array $values + ): void { + $this->scopeWhereMetaInNumeric($q, $key, $values, true); + } + /** * Query scope to order the query results by the string value of an attached meta. * @@ -416,7 +566,7 @@ public function scopeOrderByMeta( bool $strict = false ): void { $table = $this->joinMetaTable($q, $key, $strict ? 'inner' : 'left'); - $q->orderBy("{$table}.value", $direction); + $q->orderBy("{$table}.string_value", $direction); } /** @@ -436,10 +586,7 @@ public function scopeOrderByMetaNumeric( bool $strict = false ): void { $table = $this->joinMetaTable($q, $key, $strict ? 'inner' : 'left'); - $direction = strtolower($direction) == 'asc' ? 'asc' : 'desc'; - $field = $q->getQuery()->getGrammar()->wrap("{$table}.value"); - - $q->orderByRaw("cast({$field} as decimal) $direction"); + $q->orderBy("{$table}.numeric_value", $direction); } /** @@ -559,4 +706,33 @@ protected function makeMeta(string $key = '', mixed $value = ''): Meta return $meta; } + + private function valueToString(mixed $value): string + { + $stringValue = $this->getHandlerForValue($value)->getStringValue($value); + + if ($stringValue === null) { + throw new \InvalidArgumentException('Cannot convert to a numeric value'); + } + + return $stringValue; + } + + private function valueToNumeric(mixed $value): int|float + { + $numericValue = $this->getHandlerForValue($value)->getNumericValue($value); + + if ($numericValue === null) { + throw new \InvalidArgumentException('Cannot convert to a numeric value'); + } + + return $numericValue; + } + + private function getHandlerForValue(mixed $value): HandlerInterface + { + /** @var Registry $registry */ + $registry = app('metable.datatype.registry'); + return $registry->getHandlerForValue($value); + } } diff --git a/tests/Integration/DataType/HandlerTest.php b/tests/Integration/DataType/HandlerTest.php index 97527ed..62f8918 100644 --- a/tests/Integration/DataType/HandlerTest.php +++ b/tests/Integration/DataType/HandlerTest.php @@ -178,7 +178,7 @@ public function test_it_can_verify_and_serialize_data( $unserialized = $handler->unserializeValue($serialized); $this->assertEquals($value, $unserialized); - $this->assertEquals($numericValue, $handler->getNumericValue($value, $serialized)); - $this->assertEquals($stringValue, $handler->getStringValue($value, $serialized)); + $this->assertEquals($numericValue, $handler->getNumericValue($value)); + $this->assertEquals($stringValue, $handler->getStringValue($value)); } } diff --git a/tests/Integration/MetableTest.php b/tests/Integration/MetableTest.php index 0578d70..e450d1d 100644 --- a/tests/Integration/MetableTest.php +++ b/tests/Integration/MetableTest.php @@ -369,9 +369,11 @@ public function test_it_can_be_queried_by_numeric_meta_value(): void $metable = $this->createMetable(); $metable->setMeta('foo', 123); - $result = SampleMetable::whereMetaNumeric('foo', '>', 4)->first(); + $result = SampleMetable::whereMetaNumeric('foo', '>', 4)->get(); + $result2 = SampleMetable::whereMetaNumeric('foo', '<', 4)->get(); - $this->assertEquals($metable->getKey(), $result->getKey()); + $this->assertEquals([$metable->getKey()], $result->modelKeys()); + $this->assertEquals([], $result2->modelKeys()); } public function test_it_can_be_queried_by_in_array(): void @@ -380,11 +382,159 @@ public function test_it_can_be_queried_by_in_array(): void $metable = $this->createMetable(); $metable->setMeta('foo', 'bar'); - $result1 = SampleMetable::whereMetaIn('foo', ['baz', 'bar'])->first(); - $result2 = SampleMetable::whereMetaIn('foo', ['baz', 'bat'])->first(); + $result1 = SampleMetable::whereMetaIn('foo', ['baz', 'bar'])->get(); + $result2 = SampleMetable::whereMetaIn('foo', ['baz', 'bat'])->get(); - $this->assertEquals($metable->getKey(), $result1->getKey()); - $this->assertNull($result2); + $this->assertEquals([$metable->getKey()], $result1->modelKeys()); + $this->assertEquals([], $result2->modelKeys()); + } + + + public function test_it_can_be_queried_by_not_in_array(): void + { + $this->useDatabase(); + $metable = $this->createMetable(); + $metable->setMeta('foo', 'bar'); + + $result1 = SampleMetable::whereMetaNotIn('foo', ['baz', 'bar'])->get(); + $result2 = SampleMetable::whereMetaNotIn('foo', ['baz', 'bat'])->get(); + + $this->assertEquals([], $result1->modelKeys()); + $this->assertEquals([$metable->getKey()], $result2->modelKeys()); + } + + public function test_it_can_be_queried_by_in_array_numeric(): void + { + $this->useDatabase(); + $metable = $this->createMetable(); + $metable->setMeta('foo', 1.1); + + $result1 = SampleMetable::whereMetaInNumeric('foo', [1.1, 2.2])->get(); + $result2 = SampleMetable::whereMetaInNumeric('foo', [1, 2])->get(); + + $this->assertEquals([$metable->getKey()], $result1->modelKeys()); + $this->assertEquals([], $result2->modelKeys()); + } + + public function test_it_can_be_queried_by_not_in_array_numeric(): void + { + $this->useDatabase(); + $metable = $this->createMetable(); + $metable->setMeta('foo', 1.1); + + $result1 = SampleMetable::whereMetaNotInNumeric('foo', [1.1, 2.2])->get(); + $result2 = SampleMetable::whereMetaNotInNumeric('foo', [1, 2])->get(); + + $this->assertEquals([], $result1->modelKeys()); + $this->assertEquals([$metable->getKey()], $result2->modelKeys()); + } + + public function test_it_can_be_queried_by_meta_between(): void + { + $this->useDatabase(); + $metable = $this->createMetable(); + $metable->setMeta('foo', 'c'); + + $result1 = SampleMetable::whereMetaBetween( + 'foo', + 'a', + 'd' + )->get(); + $result2 = SampleMetable::whereMetaBetween( + 'foo', + 'd', + 'z' + )->get(); + + $this->assertEquals([$metable->getKey()], $result1->modelKeys()); + $this->assertEquals([], $result2->modelKeys()); + } + + public function test_it_can_be_queried_by_meta_between_numeric(): void + { + $this->useDatabase(); + $metable = $this->createMetable(); + $date = Carbon::now(); + $metable->setMeta('foo', $date); + + $result1 = SampleMetable::whereMetaBetweenNumeric( + 'foo', + $date->clone()->subDay(), + $date->clone()->addDay() + )->get(); + $result2 = SampleMetable::whereMetaBetweenNumeric( + 'foo', + $date->subDays(2), + $date->subDay() + )->get(); + + $this->assertEquals([$metable->getKey()], $result1->modelKeys()); + $this->assertEquals([], $result2->modelKeys()); + } + + public function test_it_can_be_queried_by_meta_not_between_numeric(): void + { + $this->useDatabase(); + $metable = $this->createMetable(); + $date = Carbon::now(); + $metable->setMeta('foo', $date); + + $result1 = SampleMetable::whereMetaNotBetweenNumeric( + 'foo', + $date->clone()->subDay(), + $date->clone()->addDay() + )->get(); + $result2 = SampleMetable::whereMetaNotBetweenNumeric( + 'foo', + $date->subDays(2), + $date->subDay() + )->get(); + + $this->assertEquals([], $result1->modelKeys()); + $this->assertEquals([$metable->getKey()], $result2->modelKeys()); + } + + public function test_it_can_be_queried_by_null(): void + { + $this->useDatabase(); + $metable1 = $this->createMetable(); + $metable1->setMeta('foo', null); + + $metable2 = $this->createMetable(); + $metable2->setMeta('foo', 1); + + $result1 = SampleMetable::whereMetaIsNull('foo')->get(); + + $this->assertEquals([$metable1->getKey()], $result1->modelKeys()); + } + + public function test_it_can_be_queried_by_model(): void + { + $this->useDatabase(); + $metable1 = $this->createMetable(); + $metable2 = $this->createMetable(); + $metable1->setMeta('foo', $metable2); + $metable2->setMeta('foo', $metable1); + + $result1 = SampleMetable::whereMetaIsModel('foo', $metable1)->get(); + $result2 = SampleMetable::whereMetaIsModel('foo', $metable2)->get(); + $result3 = SampleMetable::whereMetaIsModel( + 'foo', + SampleMetable::class, + $metable1->getKey() + )->get(); + $result4 = SampleMetable::whereMetaIsModel( + 'foo', + SampleMetable::class, + $metable2->getKey() + )->get(); + $result5 = SampleMetable::whereMetaIsModel('foo', SampleMetable::class)->get(); + + $this->assertEquals([$metable2->getKey()], $result1->modelKeys()); + $this->assertEquals([$metable1->getKey()], $result2->modelKeys()); + $this->assertEquals([$metable2->getKey()], $result3->modelKeys()); + $this->assertEquals([$metable1->getKey()], $result4->modelKeys()); + $this->assertEquals([$metable1->getKey(), $metable2->getKey()], $result5->modelKeys()); } public function test_it_can_order_query_by_meta_value(): void @@ -400,8 +550,8 @@ public function test_it_can_order_query_by_meta_value(): void $results1 = SampleMetable::orderByMeta('foo', 'asc')->get(); $results2 = SampleMetable::orderByMeta('foo', 'desc')->get(); - $this->assertEquals([3, 1, 2], $results1->pluck('id')->toArray()); - $this->assertEquals([2, 1, 3], $results2->pluck('id')->toArray()); + $this->assertEquals([3, 1, 2], $results1->modelKeys()); + $this->assertEquals([2, 1, 3], $results2->modelKeys()); } public function test_it_can_order_query_by_meta_value_strict(): void @@ -417,8 +567,8 @@ public function test_it_can_order_query_by_meta_value_strict(): void $results1 = SampleMetable::orderByMeta('foo', 'asc', true)->get(); $results2 = SampleMetable::orderByMeta('foo', 'desc', true)->get(); - $this->assertEquals([3, 1], $results1->pluck('id')->toArray()); - $this->assertEquals([1, 3], $results2->pluck('id')->toArray()); + $this->assertEquals([3, 1], $results1->modelKeys()); + $this->assertEquals([1, 3], $results2->modelKeys()); } public function test_it_can_order_query_by_numeric_meta_value(): void @@ -434,8 +584,8 @@ public function test_it_can_order_query_by_numeric_meta_value(): void $results1 = SampleMetable::orderByMetaNumeric('foo', 'asc')->get(); $results2 = SampleMetable::orderByMetaNumeric('foo', 'desc')->get(); - $this->assertEquals([2, 3, 1], $results1->pluck('id')->toArray()); - $this->assertEquals([1, 3, 2], $results2->pluck('id')->toArray()); + $this->assertEquals([2, 3, 1], $results1->modelKeys()); + $this->assertEquals([1, 3, 2], $results2->modelKeys()); } public function test_it_can_order_query_by_numeric_meta_value_strict(): void @@ -451,8 +601,8 @@ public function test_it_can_order_query_by_numeric_meta_value_strict(): void $results1 = SampleMetable::orderByMetaNumeric('foo', 'asc', true)->get(); $results2 = SampleMetable::orderByMetaNumeric('foo', 'desc', true)->get(); - $this->assertEquals([3, 1], $results1->pluck('id')->toArray()); - $this->assertEquals([1, 3], $results2->pluck('id')->toArray()); + $this->assertEquals([3, 1], $results1->modelKeys()); + $this->assertEquals([1, 3], $results2->modelKeys()); } public function test_set_relation_updates_index(): void From e21ef5b3c4315a19e0b9575a61af3719a4f5a183 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Wed, 17 Apr 2024 20:20:05 -0400 Subject: [PATCH 23/38] fix tests --- src/DataType/DateTimeHandler.php | 2 +- src/DataType/ModelCollectionHandler.php | 2 +- src/DataType/SerializeHandler.php | 1 + src/Metable.php | 56 +++++++++++++------ tests/Integration/DataType/HandlerTest.php | 2 +- .../DataType/SerializableHandlerTest.php | 1 + 6 files changed, 44 insertions(+), 20 deletions(-) diff --git a/src/DataType/DateTimeHandler.php b/src/DataType/DateTimeHandler.php index b055ebf..5253a9e 100644 --- a/src/DataType/DateTimeHandler.php +++ b/src/DataType/DateTimeHandler.php @@ -58,7 +58,7 @@ public function getNumericValue(mixed $value): null|int|float public function getStringValue(mixed $value): null|string { - return $value instanceof DateTimeInterface + return $value instanceof DateTimeInterface ? $value->copy()->setTimezone('UTC')->format(self::FORMAT) : null; } diff --git a/src/DataType/ModelCollectionHandler.php b/src/DataType/ModelCollectionHandler.php index b25a06d..883eaf6 100644 --- a/src/DataType/ModelCollectionHandler.php +++ b/src/DataType/ModelCollectionHandler.php @@ -111,7 +111,7 @@ private function loadModels(array $items) } $results[$class] = $class::query()->findMany($keys) - ->keyBy(fn(Model $model) => $model->getKey()); + ->keyBy(fn (Model $model) => $model->getKey()); } return $results; diff --git a/src/DataType/SerializeHandler.php b/src/DataType/SerializeHandler.php index d7877ec..532ecfd 100644 --- a/src/DataType/SerializeHandler.php +++ b/src/DataType/SerializeHandler.php @@ -49,4 +49,5 @@ public function isIdempotent(): bool { return false; } + } \ No newline at end of file diff --git a/src/Metable.php b/src/Metable.php index 095d1f1..3023b07 100644 --- a/src/Metable.php +++ b/src/Metable.php @@ -341,8 +341,12 @@ function (Builder $q) use ($keys) { * * @return void */ - public function scopeWhereMeta(Builder $q, string $key, mixed $operator, mixed $value = null): void - { + public function scopeWhereMeta( + Builder $q, + string $key, + mixed $operator, + mixed $value = null + ): void { // Shift arguments if no operator is present. if (!isset($value)) { $value = $operator; @@ -359,11 +363,10 @@ function (Builder $q) use ($key, $operator, $stringValue, $value) { // If the value is a string and the string value is at the maximum length, // we can optimize the query by looking up using the index first // then compare the serialized value (not indexed) afterward to ensure correctness. - if (strlen($stringValue) === config( - 'metable.stringValueIndexLength', - 255 - ) - ) { + if (strlen($stringValue) >= config( + 'metable.stringValueIndexLength', + 255 + )) { $handler = $this->getHandlerForValue($value); if ($handler->isIdempotent()) { $q->where('value', $operator, $handler->serializeValue($value)); @@ -385,8 +388,12 @@ function (Builder $q) use ($key, $operator, $stringValue, $value) { * * @return void */ - public function scopeWhereMetaNumeric(Builder $q, string $key, mixed $operator, mixed $value = null): void - { + public function scopeWhereMetaNumeric( + Builder $q, + string $key, + mixed $operator, + mixed $value = null + ): void { // Shift arguments if no operator is present. if (!isset($value)) { $value = $operator; @@ -468,7 +475,6 @@ public function scopeWhereMetaIsNull(Builder $q, string $key): void }); } - public function scopeWhereMetaIsModel( Builder $q, string $key, @@ -612,11 +618,25 @@ private function joinMetaTable(Builder $q, string $key, string $type = 'left'): } // Join the meta table to the query - $q->join("{$metaTable} as {$alias}", function (JoinClause $q) use ($relation, $key, $alias) { - $q->on($relation->getQualifiedParentKeyName(), '=', $alias . '.' . $relation->getForeignKeyName()) - ->where($alias . '.key', '=', $key) - ->where($alias . '.' . $relation->getMorphType(), '=', $this->getMorphClass()); - }, null, null, $type); + $q->join( + "{$metaTable} as {$alias}", + function (JoinClause $q) use ($relation, $key, $alias) { + $q->on( + $relation->getQualifiedParentKeyName(), + '=', + $alias . '.' . $relation->getForeignKeyName() + ) + ->where($alias . '.key', '=', $key) + ->where( + $alias . '.' . $relation->getMorphType(), + '=', + $this->getMorphClass() + ); + }, + null, + null, + $type + ); // Return the alias so that the calling context can // reference the table. @@ -660,7 +680,7 @@ public function setRelation($relation, $value) /** * Set the entire relations array on the model. * - * @param array $relations + * @param array $relations * @return $this */ public function setRelations(array $relations) @@ -707,7 +727,9 @@ protected function makeMeta(string $key = '', mixed $value = ''): Meta protected function getAllDefaultMeta(): array { - return property_exists($this, 'defaultMetaValues') ? $this->defaultMetaValues : []; + return property_exists($this, 'defaultMetaValues') + ? $this->defaultMetaValues + : []; } private function valueToString(mixed $value): string diff --git a/tests/Integration/DataType/HandlerTest.php b/tests/Integration/DataType/HandlerTest.php index 62f8918..45bf672 100644 --- a/tests/Integration/DataType/HandlerTest.php +++ b/tests/Integration/DataType/HandlerTest.php @@ -25,7 +25,7 @@ class HandlerTest extends TestCase { private static $resource; - static public function handlerProvider(): array + public static function handlerProvider(): array { $dateString = '2017-01-01 00:00:00.000000+0000'; $datetime = Carbon::createFromFormat('Y-m-d H:i:s.uO', $dateString); diff --git a/tests/Integration/DataType/SerializableHandlerTest.php b/tests/Integration/DataType/SerializableHandlerTest.php index 4d53925..b7cf968 100644 --- a/tests/Integration/DataType/SerializableHandlerTest.php +++ b/tests/Integration/DataType/SerializableHandlerTest.php @@ -42,4 +42,5 @@ public function test_it_configures_allowed_classes(): void ); $this->assertEquals($incomplete, $handler->unserializeValue($serialized)); } + } \ No newline at end of file From 4b165f9e5fb393127d7179c200c2e744c1471754 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Wed, 17 Apr 2024 21:24:03 -0400 Subject: [PATCH 24/38] added metable:refresh artisan command --- CHANGELOG.md | 6 +- UPGRADING.md | 4 +- src/Commands/RefreshMeta.php | 52 ++++++++++++ src/MetableServiceProvider.php | 6 +- .../Integration/Commands/RefreshMetaTest.php | 83 +++++++++++++++++++ 5 files changed, 147 insertions(+), 4 deletions(-) create mode 100644 src/Commands/RefreshMeta.php create mode 100644 tests/Integration/Commands/RefreshMetaTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e88fde..f0c0b02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ ### Data Types -- `getStringValue(): ?string` and `getNumericValue(): null|int|float` methods added to `HandlerInterface` which should convert the original value into a format that can be indexed, if possible. +- Added `getStringValue(): ?string` and `getNumericValue(): null|int|float` methods to `HandlerInterface` which should convert the original value into a format that can be indexed, if possible. - Added `SerializeHandler` as a catch-all datatype, which will attempt to serialize the data using PHP's `serialize()` function. The payload is encrypted before being stored in the database to prevent unserializing untrusted data. - Deprecated `SerializableHandler` in favor of the new `SerializeHandler` datatype. The `SerializableHandler` will be removed in a future release. In the interim, added the `metable.options.serializable.allowedClasses` config to protect against unserializing untrusted data. - Deprecated `ArrayHandler` and `ObjectHandler`, due to the ambiguity of nested array/objects switching type. These will be removed in a future release. The `SerializeHandler` should be used instead. @@ -22,6 +22,10 @@ - `ModelCollectionHandler` will now validate that the encoded collection class is a valid Eloquent collection before attempting to instantiate it during unserialization. If the class is invalid, an instance of `Illuminate\Database\Eloquent\Collection` will be used instead. - `ModelCollectionHandler` will now validate that the encoded class of each entry is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, that entry in the collection will be omitted. +### Commands + +- Added `metable:refresh` artisan command which will descode and re-encode all meta values in the database. This is useful if you have changed the data type handlers and need to update the serialized data and indexes in the database. + ### Mediable trait - `whereMeta()`, `whereMetaIn()`, and `orderByMeta()` query scopes will now scan the indexed `string_value` column instead of the serialized `value` column. This greatly improves performance when searching for meta values against larger datasets. diff --git a/UPGRADING.md b/UPGRADING.md index ddafcb7..03e2fc5 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -9,8 +9,8 @@ * Recommended to add the `SerializeHandler` to the end of `datatypes` config (catch-all). * The `SerializableHandler`, `ArrayHandler`, and `ObjectHandler` data types have been deprecated in favor of the new `SerializeHandler`. If you have any Meta encoded using any of these data types, you should continue to include them in the `datatypes` config _after_ the `SerializeHandler` to ensure that existing values will continue to be properly decoded, but new values will use the new encoding. Once all old values have been migrated, you may remove the deprecated data types from the `datatypes` config. * For security reasons, if you have any existing Meta encoded using `SerializableHandler`, you must configure the `metable.options.serializable.allowedClasses` config to list classes that are allowed to be unserialized. Otherwise, all objects will be returned as `__PHP_Incomplete_Class`. This config may be set to `true` to disable this security check and allow any class, but this is not recommended. -* If you have any custom data types, you will need to implement the `getStringValue()` and `getNumericValue()` methods in your handler class to populate those indexes. You may return `null` if the value does not need to be searchable. -* Once you have applied the schema migration and configured the changes to the `datatypes` config, you should run the `meta:refresh` command to update all existing meta values to use the new types and populate the index columns. +* If you have any custom data types, you will need to implement the `getStringValue()` and `getNumericValue()` methods in your handler class to populate those indexes. You may return `null` if the value cannot be conerted into a searchable format or does not need to be searchable. +* Once you have applied the schema migration and configured the changes to the `datatypes` config, you should run the `metable:refresh` command to update all existing meta values to use the new types and populate the index columns. After this command has been run, you may remove the deprecated data types from the `datatypes` config. * Review the documentation about which data types can be queried with the various `whereMeta*` and `whereMeta*Numeric` query scopes. If you are querying the serialized `value` column directly, be aware that the formatting of array/object data types has changed. ## 4.X -> 5.X diff --git a/src/Commands/RefreshMeta.php b/src/Commands/RefreshMeta.php new file mode 100644 index 0000000..8afa881 --- /dev/null +++ b/src/Commands/RefreshMeta.php @@ -0,0 +1,52 @@ +info('Refreshing meta values...'); + + $count = 0; + $total = DB::table('meta')->count(); + $lastId = null; + + $progress = $this->output->createProgressBar($total); + $progress->start(); + + while ($count < $total) { + $query = Meta::query() + ->orderBy('id') + ->limit(100); + if ($lastId) { + $query->where('id', '>', $lastId); + } + + $collection = $query->get(); + /** @var Meta $meta */ + foreach ($collection as $meta) { + $value = $meta->value; + $meta->setValueAttribute(null); + $meta->setValueAttribute($value); + $meta->save(); + $count++; + $progress->advance(); + $lastId = $meta->id; + } + } + + $progress->finish(); + + $this->info('Refresh complete.'); + } + +} \ No newline at end of file diff --git a/src/MetableServiceProvider.php b/src/MetableServiceProvider.php index 0c05f12..e52b2dc 100644 --- a/src/MetableServiceProvider.php +++ b/src/MetableServiceProvider.php @@ -2,8 +2,8 @@ namespace Plank\Metable; -use CreateMetaTable; use Illuminate\Support\ServiceProvider; +use Plank\Metable\Commands\RefreshMeta; use Plank\Metable\DataType\Registry; /** @@ -25,6 +25,10 @@ public function boot(): void if (config('metable.applyMigrations', true)) { $this->loadMigrationsFrom(dirname(__DIR__) . '/migrations'); } + + $this->commands([ + RefreshMeta::class + ]); } /** diff --git a/tests/Integration/Commands/RefreshMetaTest.php b/tests/Integration/Commands/RefreshMetaTest.php new file mode 100644 index 0000000..1a2fde5 --- /dev/null +++ b/tests/Integration/Commands/RefreshMetaTest.php @@ -0,0 +1,83 @@ +useDatabase(); + + config()->set('metable.datatypes', [ + StringHandler::class, + DateTimeHandler::class, + SerializeHandler::class, + ArrayHandler::class, + ]); + config()->set('metable.indexComplexDataTypes', true); + + $complexValue = ['a' => 'b']; + + DB::table('meta')->insert([ + [ + 'metable_type' => 'foo', + 'metable_id' => 1, + 'type' => 'array', + 'key' => 'foo', + 'value' => json_encode($complexValue), + 'string_value' => null, + 'numeric_value' => null, + ], + [ + 'metable_type' => 'foo', + 'metable_id' => 2, + 'type' => 'string', + 'key' => 'bar', + 'value' => 'blah', + 'string_value' => null, + 'numeric_value' => null, + ], + [ + 'metable_type' => 'foo', + 'metable_id' => 3, + 'type' => 'datetime', + 'key' => 'baz', + 'value' => '2020-01-01 00:00:00.000000+0000', + 'string_value' => null, + 'numeric_value' => null, + ], + ]); + + $this->artisan('metable:refresh') + ->expectsOutput('Refreshing meta values...') + ->expectsOutput('Refresh complete.') + ->assertExitCode(0); + + + $result = DB::table('meta')->get(); + $this->assertCount(3, $result); + + $this->assertEquals('serialized', $result[0]->type); + $this->assertEquals($complexValue, app('encrypter')->decrypt($result[0]->value)); + $this->assertEquals(serialize($complexValue), $result[0]->string_value); + $this->assertNull($result[0]->numeric_value); + + $this->assertEquals('string', $result[1]->type); + $this->assertEquals('blah', $result[1]->value); + $this->assertEquals('blah', $result[1]->string_value); + $this->assertNull($result[1]->numeric_value); + + $this->assertEquals('datetime', $result[2]->type); + $this->assertEquals('2020-01-01 00:00:00.000000+0000', $result[2]->value); + $this->assertEquals('2020-01-01 00:00:00.000000+0000', $result[2]->string_value); + $this->assertEquals(1577836800, $result[2]->numeric_value); + } + +} \ No newline at end of file From 63ab5f3243753557803da3790215a81818e64b55 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Wed, 17 Apr 2024 22:05:07 -0400 Subject: [PATCH 25/38] test improvements --- .github/workflows/automated-test.yml | 2 +- src/Commands/RefreshMeta.php | 5 +- src/DataType/ModelCollectionHandler.php | 1 + .../Integration/Commands/RefreshMetaTest.php | 5 +- tests/Integration/DataType/HandlerTest.php | 201 +++++++++++------- .../DataType/ModelCollectionHandlerTest.php | 39 +++- tests/Integration/MetableTest.php | 38 ++++ 7 files changed, 204 insertions(+), 87 deletions(-) diff --git a/.github/workflows/automated-test.yml b/.github/workflows/automated-test.yml index 465a3bc..941fab3 100644 --- a/.github/workflows/automated-test.yml +++ b/.github/workflows/automated-test.yml @@ -1,5 +1,5 @@ name: PHPUnit Tests -on: [push, pull_request] +on: [push] jobs: phpunit: runs-on: ubuntu-latest diff --git a/src/Commands/RefreshMeta.php b/src/Commands/RefreshMeta.php index 8afa881..a6e47a2 100644 --- a/src/Commands/RefreshMeta.php +++ b/src/Commands/RefreshMeta.php @@ -26,7 +26,7 @@ public function handle(): void while ($count < $total) { $query = Meta::query() ->orderBy('id') - ->limit(100); + ->limit(config('metable.refreshPageSize', 100)); if ($lastId) { $query->where('id', '>', $lastId); } @@ -48,5 +48,4 @@ public function handle(): void $this->info('Refresh complete.'); } - -} \ No newline at end of file +} diff --git a/src/DataType/ModelCollectionHandler.php b/src/DataType/ModelCollectionHandler.php index 883eaf6..eb33bee 100644 --- a/src/DataType/ModelCollectionHandler.php +++ b/src/DataType/ModelCollectionHandler.php @@ -63,6 +63,7 @@ public function unserializeValue(string $serializedValue): mixed $models = $this->loadModels($data['items']); + // Repopulate collection keys with loaded models. foreach ($data['items'] as $key => $item) { if (empty($item['key'])) { diff --git a/tests/Integration/Commands/RefreshMetaTest.php b/tests/Integration/Commands/RefreshMetaTest.php index 1a2fde5..5e0599c 100644 --- a/tests/Integration/Commands/RefreshMetaTest.php +++ b/tests/Integration/Commands/RefreshMetaTest.php @@ -23,6 +23,8 @@ public function test_it_refreshes_all_meta_values(): void ]); config()->set('metable.indexComplexDataTypes', true); + config()->set('metable.refreshPageSize', 2); + $complexValue = ['a' => 'b']; DB::table('meta')->insert([ @@ -79,5 +81,4 @@ public function test_it_refreshes_all_meta_values(): void $this->assertEquals('2020-01-01 00:00:00.000000+0000', $result[2]->string_value); $this->assertEquals(1577836800, $result[2]->numeric_value); } - -} \ No newline at end of file +} diff --git a/tests/Integration/DataType/HandlerTest.php b/tests/Integration/DataType/HandlerTest.php index 45bf672..707a523 100644 --- a/tests/Integration/DataType/HandlerTest.php +++ b/tests/Integration/DataType/HandlerTest.php @@ -41,108 +41,144 @@ public static function handlerProvider(): array return [ 'array' => [ - new ArrayHandler(), - 'array', - ['foo' => ['bar'], 'baz'], - [new stdClass()], - null, - null, + 'handler' => new ArrayHandler(), + 'type' => 'array', + 'value' => ['foo' => ['bar'], 'baz'], + 'invalid' => [new stdClass()], + 'numericValue' => null, + 'stringValue' => null, + 'stringValueComplex' => json_encode(['foo' => ['bar'], 'baz']), + 'isIdempotent' => true, ], 'boolean' => [ - new BooleanHandler(), - 'boolean', - true, - [1, 0, '', [], null], - 1, - 'true' + 'handler' => new BooleanHandler(), + 'type' => 'boolean', + 'value' => true, + 'invalid' => [1, 0, '', [], null], + 'numericValue' => 1, + 'stringValue' => 'true', + 'stringValueComplex' => 'true', + 'isIdempotent' => true, ], 'datetime' => [ - new DateTimeHandler(), - 'datetime', - $datetime, - [2017, '2017-01-01'], - $timestamp, - $dateString, + 'handler' => new DateTimeHandler(), + 'type' => 'datetime', + 'value' => $datetime, + 'invalid' => [2017, '2017-01-01'], + 'numericValue' => $timestamp, + 'stringValue' => $dateString, + 'stringValueComplex' => $dateString, + 'isIdempotent' => true, ], 'float' => [ - new FloatHandler(), - 'float', - 1.1, - ['1.1', 1], - 1.1, - '1.1', + 'handler' => new FloatHandler(), + 'type' => 'float', + 'value' => 1.1, + 'invalid' => ['1.1', 1], + 'numericValue' => 1.1, + 'stringValue' => '1.1', + 'stringValueComplex' => '1.1', + 'isIdempotent' => true, ], 'integer' => [ - new IntegerHandler(), - 'integer', - 3, - [1.1, '1'], - 3, - '3', + 'handler' => new IntegerHandler(), + 'type' => 'integer', + 'value' => 3, + 'invalid' => [1.1, '1'], + 'numericValue' => 3, + 'stringValue' => '3', + 'stringValueComplex' => '3', + 'isIdempotent' => true, ], 'model' => [ - new ModelHandler(), - 'model', - $model, - [new stdClass()], - null, - SampleMetable::class, + 'handler' => new ModelHandler(), + 'type' => 'model', + 'value' => $model, + 'invalid' => [new stdClass()], + 'numericValue' => null, + 'stringValue' => SampleMetable::class, + 'stringValueComplex' => SampleMetable::class, + 'isIdempotent' => true, ], 'model collection' => [ - new ModelCollectionHandler(), - 'collection', - new Collection([new SampleMetable()]), - [collect()], - null, - null, + 'handler' => new ModelCollectionHandler(), + 'type' => 'collection', + 'value' => new Collection([new SampleMetable()]), + 'invalid' => [collect()], + 'numericValue' => null, + 'stringValue' => null, + 'stringValueComplex' => null, + 'isIdempotent' => true, ], 'null' => [ - new NullHandler(), - 'null', - null, - [0, '', 'null', [], false], - null, - null, + 'handler' => new NullHandler(), + 'type' => 'null', + 'value' => null, + 'invalid' => [0, '', 'null', [], false], + 'numericValue' => null, + 'stringValue' => null, + 'stringValueComplex' => null, + 'isIdempotent' => true, ], 'object' => [ - new ObjectHandler(), - 'object', - $object, - [[]], - null, - null, + 'handler' => new ObjectHandler(), + 'type' => 'object', + 'value' => $object, + 'invalid' => [[]], + 'numericValue' => null, + 'stringValue' => null, + 'stringValueComplex' => json_encode($object), + 'isIdempotent' => true, ], 'serialize' => [ - new SerializeHandler(), - 'serialized', - ['foo' => 'bar', 'baz' => [3]], - [self::$resource], - null, - null, + 'handler' => new SerializeHandler(), + 'type' => 'serialized', + 'value' => ['foo' => 'bar', 'baz' => [3]], + 'invalid' => [self::$resource], + 'numericValue' => null, + 'stringValue' => null, + 'stringValueComplex' => serialize(['foo' => 'bar', 'baz' => [3]]), + 'isIdempotent' => false, ], 'serializable' => [ - new SerializableHandler(), - 'serializable', - new SampleSerializable(['foo' => 'bar']), - [], - null, - null, + 'handler' => new SerializableHandler(), + 'type' => 'serializable', + 'value' => new SampleSerializable(['foo' => 'bar']), + 'invalid' => [], + 'numericValue' => null, + 'stringValue' => null, + 'stringValueComplex' => serialize(new SampleSerializable(['foo' => 'bar'])), + 'isIdempotent' => true, ], 'string' => [ - new StringHandler(), - 'string', - 'foo', - [1, 1.1], - null, - 'foo', + 'handler' => new StringHandler(), + 'type' => 'string', + 'value' => 'foo', + 'invalid' => [1, 1.1], + 'numericValue' => null, + 'stringValue' => 'foo', + 'stringValueComplex' => 'foo', + 'isIdempotent' => true, + ], + 'long-string' => [ + 'handler' => new StringHandler(), + 'type' => 'string', + 'value' => str_repeat('a', 300), + 'invalid' => [1, 1.1], + 'numericValue' => null, + 'stringValue' => str_repeat('a', 255), + 'stringValueComplex' => str_repeat('a', 255), + 'isIdempotent' => true, ], 'numeric-string' => [ - new StringHandler(), - 'string', - '1.2345', - [1, 1.1], - 1.2345, - '1.2345', + 'handler' => new StringHandler(), + 'type' => 'string', + 'value' => '1.2345', + 'invalid' => [1, 1.1], + 'numericValue' => 1.2345, + 'stringValue' => '1.2345', + 'stringValueComplex' => '1.2345', + 'isIdempotent' => true, ], ]; } @@ -165,7 +201,9 @@ public function test_it_can_verify_and_serialize_data( mixed $value, array $incompatible, null|int|float $numericValue, - null|string $stringValue + null|string $stringValue, + null|string $stringValueComplex, + bool $isIdempotent ): void { $this->assertEquals($type, $handler->getDataType()); $this->assertTrue($handler->canHandleValue($value)); @@ -179,6 +217,11 @@ public function test_it_can_verify_and_serialize_data( $this->assertEquals($value, $unserialized); $this->assertEquals($numericValue, $handler->getNumericValue($value)); + config()->set('metable.indexComplexDataTypes', false); $this->assertEquals($stringValue, $handler->getStringValue($value)); + config()->set('metable.indexComplexDataTypes', true); + $this->assertEquals($stringValueComplex, $handler->getStringValue($value)); + + $this->assertEquals($isIdempotent, $handler->isIdempotent()); } } diff --git a/tests/Integration/DataType/ModelCollectionHandlerTest.php b/tests/Integration/DataType/ModelCollectionHandlerTest.php index bb2f869..8eaf1ca 100644 --- a/tests/Integration/DataType/ModelCollectionHandlerTest.php +++ b/tests/Integration/DataType/ModelCollectionHandlerTest.php @@ -32,11 +32,31 @@ public function test_it_reloads_model_instances(): void $this->assertEquals(1, $unserialized['foo']->getKey()); } - public function test_it_handles_invalid_model_class(): void + public function test_it_handles_invalid_collection_class(): void { + $this->useDatabase(); + $metable = SampleMetable::create(); $handler = new ModelCollectionHandler(); $serialized = json_encode([ 'class' => 'stdClass', + 'items' => [ + [ + 'class' => SampleMetable::class, + 'key' => $metable->getKey() + ] + ] + ]); + $unserialized = $handler->unserializeValue($serialized); + + $this->assertInstanceOf(Collection::class, $unserialized); + $this->assertEquals([$metable->getKey()], $unserialized->modelKeys()); + } + + public function test_it_handles_invalid_model_class(): void + { + $handler = new ModelCollectionHandler(); + $serialized = json_encode([ + 'class' => Collection::class, 'items' => [ 'class' => 'stdClass', 'key' => '1' @@ -45,6 +65,21 @@ public function test_it_handles_invalid_model_class(): void $unserialized = $handler->unserializeValue($serialized); $this->assertInstanceOf(Collection::class, $unserialized); - $this->assertEmpty($unserialized); + $this->assertCount(0, $unserialized); + } + + public function test_it_handles_invalid_model_class_no_key(): void + { + $handler = new ModelCollectionHandler(); + $serialized = json_encode([ + 'class' => Collection::class, + 'items' => [ + 'class' => 'stdClass', + ] + ]); + $unserialized = $handler->unserializeValue($serialized); + + $this->assertInstanceOf(Collection::class, $unserialized); + $this->assertCount(0, $unserialized); } } diff --git a/tests/Integration/MetableTest.php b/tests/Integration/MetableTest.php index e450d1d..4509c52 100644 --- a/tests/Integration/MetableTest.php +++ b/tests/Integration/MetableTest.php @@ -353,14 +353,17 @@ public function test_it_can_be_queried_by_meta_value(): void $metable = $this->createMetable(); $metable->setMeta('foo', 'bar'); $metable->setMeta('datetime', $now); + $metable->setMeta('long', str_repeat('a', 300)); $result1 = SampleMetable::whereMeta('foo', 'bar')->first(); $result2 = SampleMetable::whereMeta('foo', 'baz')->first(); $result3 = SampleMetable::whereMeta('datetime', $now)->first(); + $result4 = SampleMetable::whereMeta('long', str_repeat('a', 300))->first(); $this->assertEquals($metable->getKey(), $result1->getKey()); $this->assertNull($result2); $this->assertEquals($metable->getKey(), $result3->getKey()); + $this->assertEquals($metable->getKey(), $result4->getKey()); } public function test_it_can_be_queried_by_numeric_meta_value(): void @@ -371,9 +374,11 @@ public function test_it_can_be_queried_by_numeric_meta_value(): void $result = SampleMetable::whereMetaNumeric('foo', '>', 4)->get(); $result2 = SampleMetable::whereMetaNumeric('foo', '<', 4)->get(); + $result3 = SampleMetable::whereMetaNumeric('foo', 123)->get(); $this->assertEquals([$metable->getKey()], $result->modelKeys()); $this->assertEquals([], $result2->modelKeys()); + $this->assertEquals([$metable->getKey()], $result3->modelKeys()); } public function test_it_can_be_queried_by_in_array(): void @@ -450,6 +455,27 @@ public function test_it_can_be_queried_by_meta_between(): void $this->assertEquals([], $result2->modelKeys()); } + public function test_it_can_be_queried_by_meta_not_between(): void + { + $this->useDatabase(); + $metable = $this->createMetable(); + $metable->setMeta('foo', 'c'); + + $result1 = SampleMetable::whereMetaNotBetween( + 'foo', + 'a', + 'd' + )->get(); + $result2 = SampleMetable::whereMetaNotBetween( + 'foo', + 'd', + 'z' + )->get(); + + $this->assertEquals([], $result1->modelKeys()); + $this->assertEquals([$metable->getKey()], $result2->modelKeys()); + } + public function test_it_can_be_queried_by_meta_between_numeric(): void { $this->useDatabase(); @@ -665,6 +691,18 @@ public function test_it_can_serialize_properly(): void $this->assertEquals('baz', $result->getMeta('foo')); } + public function test_it_throws_for_param_that_cannot_be_converted_to_string(): void + { + $this->expectException(\LogicException::class); + SampleMetable::whereMeta('foo', null)->get(); + } + + public function test_it_throws_for_param_that_cannot_be_converted_to_numeric(): void + { + $this->expectException(\LogicException::class); + SampleMetable::whereMetaNumeric('foo', null)->get(); + } + private function makeMeta(array $attributes = []): Meta { return factory(Meta::class)->make($attributes); From 2ca308705b426cb94557a76437090c899f6178f8 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Fri, 19 Apr 2024 22:49:37 -0400 Subject: [PATCH 26/38] use hash_hmac to verify serialized payloads are trusted --- CHANGELOG.md | 10 ++-- UPGRADING.md | 17 ++++--- config/metable.php | 47 +++++++++++-------- docs/source/datatypes.rst | 20 ++++---- ...4_04_14_000000_add_meta_search_columns.php | 1 + src/DataType/ArrayHandler.php | 7 ++- src/DataType/DateTimeHandler.php | 5 ++ src/DataType/HandlerInterface.php | 2 + src/DataType/ModelCollectionHandler.php | 5 ++ src/DataType/ModelHandler.php | 5 ++ src/DataType/ObjectHandler.php | 7 ++- src/DataType/ScalarHandler.php | 5 ++ src/DataType/SerializableHandler.php | 9 +++- ...Handler.php => SignedSerializeHandler.php} | 21 +++++++-- src/Exceptions/DataTypeException.php | 4 +- src/Exceptions/SecurityException.php | 11 +++++ src/Meta.php | 41 ++++++++++++++-- .../Integration/Commands/RefreshMetaTest.php | 6 +-- tests/Integration/DataType/HandlerTest.php | 31 ++++++++---- .../DataType/ModelCollectionHandlerTest.php | 17 +++++++ .../DataType/SerializableHandlerTest.php | 8 ++-- .../DataType/SignedSerializeHandlerTest.php | 27 +++++++++++ tests/Integration/MetaTest.php | 12 +++++ tests/TestCase.php | 2 +- 24 files changed, 248 insertions(+), 72 deletions(-) rename src/DataType/{SerializeHandler.php => SignedSerializeHandler.php} (67%) create mode 100644 src/Exceptions/SecurityException.php create mode 100644 tests/Integration/DataType/SignedSerializeHandlerTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index f0c0b02..6d6a435 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,15 +8,15 @@ - Droppped support for PHP 8.0 and below - Added support for Laravel 10 and 11 - Dropped support Laravel versions 9 and below -- adjusted some method signatures with PHP 8+ mixed and union types -- New schema migration adding two new columns and improving indexing for searching by meta values. See [UPGRADING.md](UPGRADING.md) for details. +- Adjusted some method signatures with PHP 8+ mixed and union types +- New schema migration adding two new columns and improving indexing for searching by meta values. See [UPGRADING.md](UPGRADING.md) for details ### Data Types - Added `getStringValue(): ?string` and `getNumericValue(): null|int|float` methods to `HandlerInterface` which should convert the original value into a format that can be indexed, if possible. -- Added `SerializeHandler` as a catch-all datatype, which will attempt to serialize the data using PHP's `serialize()` function. The payload is encrypted before being stored in the database to prevent unserializing untrusted data. -- Deprecated `SerializableHandler` in favor of the new `SerializeHandler` datatype. The `SerializableHandler` will be removed in a future release. In the interim, added the `metable.options.serializable.allowedClasses` config to protect against unserializing untrusted data. -- Deprecated `ArrayHandler` and `ObjectHandler`, due to the ambiguity of nested array/objects switching type. These will be removed in a future release. The `SerializeHandler` should be used instead. +- Added `SignedSerializeHandler` as a catch-all datatype, which will attempt to serialize the data using PHP's `serialize()` function. The payload is cryptographically signed with an HMAC before being stored in the database to prevent PHP object injection attacks. +- Deprecated `SerializableHandler` in favor of the new `SignedSerializeHandler` datatype. The `SerializableHandler` will be removed in a future release. In the interim, added the `metable.options.serializable.allowedClasses` config to protect against unserializing untrusted data. +- Deprecated `ArrayHandler` and `ObjectHandler`, due to the ambiguity of nested array/objects switching type. These will be removed in a future release. The `SignedSerializeHandler` should be used instead. - `ModelHandler` will now validate that the encoded class is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, the meta value will return `null`. - `ModelHandler` will no longer throw a model not found exception if the model no longer exists. Instead, the meta value will return `null`. This is more in line with the existing behavior of the `ModelCollectionHandler`. - `ModelCollectionHandler` will now validate that the encoded collection class is a valid Eloquent collection before attempting to instantiate it during unserialization. If the class is invalid, an instance of `Illuminate\Database\Eloquent\Collection` will be used instead. diff --git a/UPGRADING.md b/UPGRADING.md index 03e2fc5..029ba36 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -5,13 +5,16 @@ * Minimum PHP version moved to 8.1 * Minimum Laravel version moved to 10 * Some methods have had their signatures adjusted to use PHP 8+ mixed and union types. If extending any class or implementing any interface from this package, method signatures may need to be updated. -* A new schema migration has been added which adds two new columns to the meta table and improves indexing for querying by meta values. -* Recommended to add the `SerializeHandler` to the end of `datatypes` config (catch-all). -* The `SerializableHandler`, `ArrayHandler`, and `ObjectHandler` data types have been deprecated in favor of the new `SerializeHandler`. If you have any Meta encoded using any of these data types, you should continue to include them in the `datatypes` config _after_ the `SerializeHandler` to ensure that existing values will continue to be properly decoded, but new values will use the new encoding. Once all old values have been migrated, you may remove the deprecated data types from the `datatypes` config. -* For security reasons, if you have any existing Meta encoded using `SerializableHandler`, you must configure the `metable.options.serializable.allowedClasses` config to list classes that are allowed to be unserialized. Otherwise, all objects will be returned as `__PHP_Incomplete_Class`. This config may be set to `true` to disable this security check and allow any class, but this is not recommended. -* If you have any custom data types, you will need to implement the `getStringValue()` and `getNumericValue()` methods in your handler class to populate those indexes. You may return `null` if the value cannot be conerted into a searchable format or does not need to be searchable. -* Once you have applied the schema migration and configured the changes to the `datatypes` config, you should run the `metable:refresh` command to update all existing meta values to use the new types and populate the index columns. After this command has been run, you may remove the deprecated data types from the `datatypes` config. -* Review the documentation about which data types can be queried with the various `whereMeta*` and `whereMeta*Numeric` query scopes. If you are querying the serialized `value` column directly, be aware that the formatting of array/object data types has changed. +* A new schema migration has been added which adds three new columns to the meta table and improves indexing for querying by meta values. +* Recommended to add the `SignedSerializeHandler` to the end of `datatypes` config (catch-all). +* The `SerializableHandler`, `ArrayHandler`, and `ObjectHandler` data types have been deprecated in favor of the new `SignedSerializeHandler`. If you have any Meta encoded using any of these data types, you should continue to include them in the `datatypes` config _after_ the `SignedSerializeHandler` to ensure that existing values will continue to be properly decoded, but new values will use the new encoding. Once all old values have been migrated, you may remove the deprecated data types from the `datatypes` config. +* For security reasons, if you have any existing Meta encoded using `SerializableHandler`, you must configure the `metable.serializableHandlerAllowedClasses` config to list classes that are allowed to be unserialized. Otherwise, all objects will be returned as `__PHP_Incomplete_Class`. This config may be set to `true` to disable this security check and allow any class, but this is not recommended. +* If you have any custom data types, you will need to implement the new methods from the `HandlerInterface`: + * `getStringValue(): ?string` and `getNumericValue(): null|int|float`: These are used to populate the new indexed search columns. You may return `null` if the value cannot be converted into the specified format or does not need to be searchable. + * `isIdempotent(): bool`: This method should indicate whether multiple calls to the `serialize()` method with the same value will produce the same serialized output. This is used to determine if the complete serialized value can be used when searching for meta values. + * `useHmacVerification(): bool`: if the integrity of the serialized data should be verified with a HMAC, return `true`. If unserializing this data type is safe without HMAC verification, you may return `false`. +* Once you have applied the schema migration and configured the changes to the `datatypes` config, you should run the `metable:refresh` Artisan command to update all existing meta values to use the new types and populate the index columns. After this command has been run, you may remove the deprecated data types from the `datatypes` config. +* Review the documentation about which data types can be queried with the various `whereMeta*` and `whereMeta*Numeric` query scopes. If you are querying the serialized `value` column directly, be aware that the formatting of array/object data types may have changed. ## 4.X -> 5.X - New migration file added which adds a new composite unique index to the meta table on `metable_type`, `metable_id`, and `key`. Make sure that you have no duplicate keys for a given entity (previously possible as a race condition) before applying the new migration. diff --git a/config/metable.php b/config/metable.php index 0ac563d..3aed2d7 100644 --- a/config/metable.php +++ b/config/metable.php @@ -16,6 +16,8 @@ * * Handlers will be evaluated in order, so a value will be handled * by the first appropriate handler in the list. + * + * If you change this list, it may be necessary to refresh the meta table with the `artisan metable:refresh` command. */ 'datatypes' => [ Plank\Metable\DataType\BooleanHandler::class, @@ -28,11 +30,12 @@ Plank\Metable\DataType\ModelCollectionHandler::class, /* - * The following handlers are catch-all handlers that will encode anything. - * Only one of these should be enabled at a time. + * The following handler is a catch-all that will encode anything. + * It should come after all other handlers in active use + * + * Any handlers listed after this one will only be used for unserializing existing meta */ - Plank\Metable\DataType\SerializeHandler::class, - // Plank\Metable\DataType\JsonHandler::class, + Plank\Metable\DataType\SignedSerializeHandler::class, /* * The following handlers are deprecated and will be removed in a future release. @@ -43,21 +46,26 @@ // Plank\Metable\DataType\SerializableHandler::class, ], - 'options' => [ - 'serializable' => [ - /* - * List of classes that may be stored and retrieved using PHP serialization. - * - * Must explicitly list all classes that may be unserialized. - * Child classes of listed classes are not allowed, unless they are listed. - * - * May be set to an empty array or `false` to disallow object unserialization. - * May be set to `true` to allow serialization of all classes (strongly discouraged). - */ - 'allowedClasses' => [ - // \SampleClass::class, - ], - ], + /* + * List of classes that are allowed to be unserialized by the SignedSerializeHandler. + * If true, all classes are allowed. If false, no classes are allowed. + * If an array, only classes listed in the array are allowed. + * + * SignedSerializeHandler employs hmac verification to prevent PHP object injection attacks, + * so allowing all classes is generally safe. + */ + 'signedSerializeHandlerAllowedClasses' => true, + + /* + * List of classes that are allowed to be unserialized by the deprecated SerializableHandler. + * If true, all classes are allowed. If false, no classes are allowed. + * If an array, only classes listed in the array are allowed. + * + * This is the only protection against PHP object injection attacks, so it is strongly + * recommended to list allowed classes or set to false. + */ + 'serializableHandlerAllowedClasses' => [ + // \SampleClass::class, ], /** @@ -67,6 +75,7 @@ * performance and disk usage implications for large data sets. * * If you do not intend to query meta values containing complex data types, you should leave this disabled. + * If you change this value, it may be necessary to refresh the meta table with the `artisan metable:refresh` command. */ 'indexComplexDataTypes' => false, diff --git a/docs/source/datatypes.rst b/docs/source/datatypes.rst index ffeca8e..a5bed10 100644 --- a/docs/source/datatypes.rst +++ b/docs/source/datatypes.rst @@ -161,13 +161,13 @@ Objects and Arrays ^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\IntegerHandler | +| Handler | \Plank\Metable\DataType\SignedSerializeHandler | | String Query Scopes | if `metable.indexComplexDataTypes` is enabled | | Numeric Query Scopes | No | | Other Query Scopes | | +----------------------+-----+ -Objects and arrays will be serialized using PHP's `serialize()` function, to allow for the storage and retrieval of complex data structures. The serialized value is encrypted before being stored in the database, and decrypted when retrieved to prevent tampered data from being unserialized. +Objects and arrays will be serialized using PHP's `serialize()` function, to allow for the storage and retrieval of complex data structures. The serialized value is cryptographically signed with an HMAC which is verified before the data is unserialized to prevent PHP object injection attacks. The application's ``APP_KEY`` is used as the HMAC signing key. :: @@ -175,7 +175,9 @@ Objects and arrays will be serialized using PHP's `serialize()` function, to all $metable->setMeta('data', ['key' => 'value']); $metable->setMeta('data', new MyValueObject(123)); -.. note:: The ``Plank\Metable\DataType\SerializeHandler`` class should always be the last entry the ``config/metable.php`` datatypes array, as it will accept data of any type, causing any handlers below it to be ignored. +HMAC verification is generally sufficient for preventing PHP object injection attacks, but it possible to further restrict what can be unserialized by specifying an array or class name in the ``metable.serializableHandlerAllowedClasses`` config in the ``config/metable.php`` file. + +.. note:: The ``Plank\Metable\DataType\SignedSerializeHandler`` class should generally be the last entry the ``config/metable.php`` datatypes array, as it will accept data of any type, causing any handlers below it to be ignored for serializing new meta values. Any handlers defined below it will still be used for unserializing existing meta values. This can be used to temporarily provide backwards compatibility for deprecated data types. Deprecated ---------- @@ -192,7 +194,7 @@ Array | Other Query Scopes | | +----------------------+-----+ -.. warning:: The ``ArrayHandler`` datatype is deprecated. The ``SerializeHandler`` should be used for handling arrays. +.. warning:: The ``ArrayHandler`` datatype is deprecated. The ``SignedSerializeHandler`` should be used for handling arrays. Arrays of scalar values. Nested arrays are supported. @@ -212,7 +214,7 @@ Arrays of scalar values. Nested arrays are supported. ] ]); -.. warning:: the ``ArrayHandler`` class uses ``json_encode()`` and ``json_decode()`` under the hood for array serialization. This will cause any objects nested within the array to be cast to an array. This is not a concern for the ``SerializeHandler``. +.. warning:: the ``ArrayHandler`` class uses ``json_encode()`` and ``json_decode()`` under the hood for array serialization. This will cause any objects nested within the array to be cast to an array. This is not a concern for the ``SignedSerializeHandler``. Serializable ^^^^^^^^^^^^^ @@ -224,7 +226,7 @@ Serializable | Other Query Scopes | | +----------------------+-----+ -.. warning:: The ``SerializableHandler`` datatype is deprecated. The ``SerializeHandler`` should be used for handling all objects. +.. warning:: The ``SerializableHandler`` datatype is deprecated. The ``SignedSerializeHandler`` should be used for handling all objects. Any object implementing the PHP ``Serializable`` interface. @@ -240,7 +242,7 @@ Any object implementing the PHP ``Serializable`` interface. $metable->setMeta('example', $serializable); -For security reasons, it is necessary to list any classes that can be unserialized in the ``metable.options.serializable.allowedClasses`` key in the ``config/metable.php`` file. This is to prevent PHP Object Injection vulnerabilities when unserializing untrusted data. This config can be set to true to allow all classes, but this is not recommended. +For security reasons, it is necessary to list any classes that can be unserialized in the ``metable.serializableHandlerAllowedClasses`` key in the ``config/metable.php`` file. This is to prevent PHP Object Injection vulnerabilities when unserializing untrusted data. This config can be set to true to allow all classes, but this is not recommended. Plain Objects ^^^^^^^^^^^^^^ @@ -252,7 +254,7 @@ Plain Objects | Other Query Scopes | | +----------------------+-----+ -.. warning:: The ``ObjectHandler`` datatype is deprecated. The ``SerializeHandler`` should be used for handling all objects. +.. warning:: The ``ObjectHandler`` datatype is deprecated. The ``SignedSerializeHandler`` should be used for handling all objects. Any other objects will be converted to ``stdClass`` plain objects. You can control what properties are stored by implementing the ``JsonSerializable`` interface on the class of your stored object. @@ -262,7 +264,7 @@ Any other objects will be converted to ``stdClass`` plain objects. You can contr $metable->setMeta('weight', new Weight(10, 'kg')); $weight = $metable->getMeta('weight') // stdClass($amount = 10; $unit => 'kg'); -.. warning:: Laravel-Metable uses ``json_encode()`` and ``json_decode()`` under the hood for plain object serialization. This will cause any arrays within the object's properties to be cast to a ``stdClass`` object. This is not a concern for the ``SerializeHandler``. +.. warning:: ``ObjectHandler`` class uses ``json_encode()`` and ``json_decode()`` under the hood for plain object serialization. This will cause any arrays within the object's properties to be cast to a ``stdClass`` object. This is not a concern for the ``SignedSerializeHandler``. Adding Custom Data Types diff --git a/migrations/2024_04_14_000000_add_meta_search_columns.php b/migrations/2024_04_14_000000_add_meta_search_columns.php index fba14f6..3dc2c35 100644 --- a/migrations/2024_04_14_000000_add_meta_search_columns.php +++ b/migrations/2024_04_14_000000_add_meta_search_columns.php @@ -20,6 +20,7 @@ public function up() 'string_value', config('metable.stringValueIndexLength', 255) )->nullable(); + $table->string('hmac', 64)->nullable(); $table->dropIndex(['key', 'metable_type']); $table->dropIndex(['key']); $table->index(['key', 'metable_type', 'numeric_value']); diff --git a/src/DataType/ArrayHandler.php b/src/DataType/ArrayHandler.php index 3d951d3..d632c07 100644 --- a/src/DataType/ArrayHandler.php +++ b/src/DataType/ArrayHandler.php @@ -4,7 +4,7 @@ /** * Handle serialization of arrays. - * @deprecated Use SerializeHandler instead. + * @deprecated Use SignedSerializeHandler instead. */ class ArrayHandler implements HandlerInterface { @@ -67,4 +67,9 @@ public function isIdempotent(): bool { return true; } + + public function useHmacVerification(): bool + { + return false; + } } diff --git a/src/DataType/DateTimeHandler.php b/src/DataType/DateTimeHandler.php index 5253a9e..2ecd2ec 100644 --- a/src/DataType/DateTimeHandler.php +++ b/src/DataType/DateTimeHandler.php @@ -67,4 +67,9 @@ public function isIdempotent(): bool { return true; } + + public function useHmacVerification(): bool + { + return false; + } } diff --git a/src/DataType/HandlerInterface.php b/src/DataType/HandlerInterface.php index 94179a0..b15b561 100644 --- a/src/DataType/HandlerInterface.php +++ b/src/DataType/HandlerInterface.php @@ -49,4 +49,6 @@ public function unserializeValue(string $serializedValue): mixed; * Indicate whether multiple serializations of the same value will produce the same result. */ public function isIdempotent(): bool; + + public function useHmacVerification(): bool; } diff --git a/src/DataType/ModelCollectionHandler.php b/src/DataType/ModelCollectionHandler.php index eb33bee..164aea4 100644 --- a/src/DataType/ModelCollectionHandler.php +++ b/src/DataType/ModelCollectionHandler.php @@ -132,4 +132,9 @@ public function isIdempotent(): bool { return true; } + + public function useHmacVerification(): bool + { + return false; + } } diff --git a/src/DataType/ModelHandler.php b/src/DataType/ModelHandler.php index 222dc55..990f9cb 100644 --- a/src/DataType/ModelHandler.php +++ b/src/DataType/ModelHandler.php @@ -71,4 +71,9 @@ public function isIdempotent(): bool { return true; } + + public function useHmacVerification(): bool + { + return false; + } } diff --git a/src/DataType/ObjectHandler.php b/src/DataType/ObjectHandler.php index f5d53f2..aba893f 100644 --- a/src/DataType/ObjectHandler.php +++ b/src/DataType/ObjectHandler.php @@ -4,7 +4,7 @@ /** * Handle serialization of plain objects. - * @deprecated Use SerializeHandler instead. + * @deprecated Use SignedSerializeHandler instead. */ class ObjectHandler implements HandlerInterface { @@ -62,4 +62,9 @@ public function isIdempotent(): bool { return true; } + + public function useHmacVerification(): bool + { + return false; + } } diff --git a/src/DataType/ScalarHandler.php b/src/DataType/ScalarHandler.php index 2c62706..a362a1c 100644 --- a/src/DataType/ScalarHandler.php +++ b/src/DataType/ScalarHandler.php @@ -54,4 +54,9 @@ public function isIdempotent(): bool { return true; } + + public function useHmacVerification(): bool + { + return false; + } } diff --git a/src/DataType/SerializableHandler.php b/src/DataType/SerializableHandler.php index a803468..4dd3698 100644 --- a/src/DataType/SerializableHandler.php +++ b/src/DataType/SerializableHandler.php @@ -6,7 +6,7 @@ /** * Handle serialization of Serializable objects. - * @deprecated Use SerializeHandler instead. + * @deprecated Use SignedSerializeHandler instead. */ class SerializableHandler implements HandlerInterface { @@ -39,7 +39,7 @@ public function serializeValue(mixed $value): string */ public function unserializeValue(string $serializedValue): mixed { - $allowedClasses = config('metable.options.serializable.allowedClasses', false); + $allowedClasses = config('metable.serializableHandlerAllowedClasses', false); return unserialize($serializedValue, ['allowed_classes' => $allowedClasses]); } @@ -65,4 +65,9 @@ public function isIdempotent(): bool { return true; } + + public function useHmacVerification(): bool + { + return false; + } } diff --git a/src/DataType/SerializeHandler.php b/src/DataType/SignedSerializeHandler.php similarity index 67% rename from src/DataType/SerializeHandler.php rename to src/DataType/SignedSerializeHandler.php index c6e816d..32c21a6 100644 --- a/src/DataType/SerializeHandler.php +++ b/src/DataType/SignedSerializeHandler.php @@ -5,7 +5,7 @@ /** * Securely handle any type of value using php serialize with encryption. */ -class SerializeHandler implements HandlerInterface +final class SignedSerializeHandler implements HandlerInterface { public function getDataType(): string { @@ -19,12 +19,20 @@ public function canHandleValue(mixed $value): bool public function serializeValue(mixed $value): string { - return app('encrypter')->encrypt($value, true); + return serialize($value); } public function unserializeValue(string $serializedValue): mixed { - return app('encrypter')->decrypt($serializedValue, true); + return unserialize( + $serializedValue, + [ + 'allowed_classes' => config( + 'metable.signedSerializeHandlerAllowedClasses', + true + ) + ] + ); } public function getNumericValue(mixed $value): null|int|float @@ -47,6 +55,11 @@ public function getStringValue(mixed $value): null|string public function isIdempotent(): bool { - return false; + return true; + } + + public function useHmacVerification(): bool + { + return true; } } diff --git a/src/Exceptions/DataTypeException.php b/src/Exceptions/DataTypeException.php index bbba937..2dc133a 100644 --- a/src/Exceptions/DataTypeException.php +++ b/src/Exceptions/DataTypeException.php @@ -2,12 +2,10 @@ namespace Plank\Metable\Exceptions; -use Exception; - /** * Data Type registry exception. */ -class DataTypeException extends Exception +class DataTypeException extends \LogicException { public static function handlerNotFound(string $type): self { diff --git a/src/Exceptions/SecurityException.php b/src/Exceptions/SecurityException.php new file mode 100644 index 0000000..bd83917 --- /dev/null +++ b/src/Exceptions/SecurityException.php @@ -0,0 +1,11 @@ +cachedValue)) { - $this->cachedValue = $this->getDataTypeRegistry() - ->getHandlerForType($this->type) - ->unserializeValue($this->attributes['value']); + $handler = $this->getDataTypeRegistry()->getHandlerForType($this->type); + + if ($handler->useHmacVerification()) { + $this->verifyHmac($this->attributes['value'], $this->attributes['hmac']); + } + + $this->cachedValue = $handler->unserializeValue( + $this->attributes['value'] + ); } return $this->cachedValue; @@ -102,15 +112,23 @@ public function setValueAttribute(mixed $value): void $registry = $this->getDataTypeRegistry(); $this->attributes['type'] = $registry->getTypeForValue($value); - $handler = $registry->getHandlerForType($this->type); + $handler = $registry->getHandlerForType($this->attributes['type']); $this->attributes['value'] = $handler->serializeValue($value); + $this->attributes['hmac'] = $handler->useHmacVerification() + ? $this->computeHmac($this->attributes['value']) + : null; $this->attributes['string_value'] = $handler->getStringValue($value); $this->attributes['numeric_value'] = $handler->getNumericValue($value); $this->cachedValue = null; } + public function getRawValueAttribute(): string + { + return $this->attributes['value']; + } + /** * Retrieve the underlying serialized value. * @@ -130,4 +148,17 @@ protected function getDataTypeRegistry(): Registry { return app('metable.datatype.registry'); } + + protected function verifyHmac(string $serializedValue, string $hmac): void + { + $expectedHash = $this->computeHmac($serializedValue); + if (!hash_equals($expectedHash, $hmac)) { + throw SecurityException::hmacVerificationFailed(); + } + } + + protected function computeHmac(string $serializedValue): string + { + return hash_hmac('sha256', $serializedValue, config('app.key')); + } } diff --git a/tests/Integration/Commands/RefreshMetaTest.php b/tests/Integration/Commands/RefreshMetaTest.php index 5e0599c..8b34182 100644 --- a/tests/Integration/Commands/RefreshMetaTest.php +++ b/tests/Integration/Commands/RefreshMetaTest.php @@ -5,7 +5,7 @@ use Illuminate\Support\Facades\DB; use Plank\Metable\DataType\ArrayHandler; use Plank\Metable\DataType\DateTimeHandler; -use Plank\Metable\DataType\SerializeHandler; +use Plank\Metable\DataType\SignedSerializeHandler; use Plank\Metable\DataType\StringHandler; use Plank\Metable\Tests\TestCase; @@ -18,7 +18,7 @@ public function test_it_refreshes_all_meta_values(): void config()->set('metable.datatypes', [ StringHandler::class, DateTimeHandler::class, - SerializeHandler::class, + SignedSerializeHandler::class, ArrayHandler::class, ]); config()->set('metable.indexComplexDataTypes', true); @@ -67,7 +67,7 @@ public function test_it_refreshes_all_meta_values(): void $this->assertCount(3, $result); $this->assertEquals('serialized', $result[0]->type); - $this->assertEquals($complexValue, app('encrypter')->decrypt($result[0]->value)); + $this->assertEquals($complexValue, unserialize($result[0]->value)); $this->assertEquals(serialize($complexValue), $result[0]->string_value); $this->assertNull($result[0]->numeric_value); diff --git a/tests/Integration/DataType/HandlerTest.php b/tests/Integration/DataType/HandlerTest.php index 707a523..7f5163a 100644 --- a/tests/Integration/DataType/HandlerTest.php +++ b/tests/Integration/DataType/HandlerTest.php @@ -15,7 +15,7 @@ use Plank\Metable\DataType\NullHandler; use Plank\Metable\DataType\ObjectHandler; use Plank\Metable\DataType\SerializableHandler; -use Plank\Metable\DataType\SerializeHandler; +use Plank\Metable\DataType\SignedSerializeHandler; use Plank\Metable\DataType\StringHandler; use Plank\Metable\Tests\Mocks\SampleMetable; use Plank\Metable\Tests\Mocks\SampleSerializable; @@ -49,6 +49,7 @@ public static function handlerProvider(): array 'stringValue' => null, 'stringValueComplex' => json_encode(['foo' => ['bar'], 'baz']), 'isIdempotent' => true, + 'usesHmac' => false, ], 'boolean' => [ 'handler' => new BooleanHandler(), @@ -59,6 +60,7 @@ public static function handlerProvider(): array 'stringValue' => 'true', 'stringValueComplex' => 'true', 'isIdempotent' => true, + 'usesHmac' => false, ], 'datetime' => [ 'handler' => new DateTimeHandler(), @@ -69,6 +71,7 @@ public static function handlerProvider(): array 'stringValue' => $dateString, 'stringValueComplex' => $dateString, 'isIdempotent' => true, + 'usesHmac' => false, ], 'float' => [ 'handler' => new FloatHandler(), @@ -79,6 +82,7 @@ public static function handlerProvider(): array 'stringValue' => '1.1', 'stringValueComplex' => '1.1', 'isIdempotent' => true, + 'usesHmac' => false, ], 'integer' => [ 'handler' => new IntegerHandler(), @@ -89,6 +93,7 @@ public static function handlerProvider(): array 'stringValue' => '3', 'stringValueComplex' => '3', 'isIdempotent' => true, + 'usesHmac' => false, ], 'model' => [ 'handler' => new ModelHandler(), @@ -99,6 +104,7 @@ public static function handlerProvider(): array 'stringValue' => SampleMetable::class, 'stringValueComplex' => SampleMetable::class, 'isIdempotent' => true, + 'usesHmac' => false, ], 'model collection' => [ 'handler' => new ModelCollectionHandler(), @@ -109,6 +115,7 @@ public static function handlerProvider(): array 'stringValue' => null, 'stringValueComplex' => null, 'isIdempotent' => true, + 'usesHmac' => false, ], 'null' => [ 'handler' => new NullHandler(), @@ -119,6 +126,7 @@ public static function handlerProvider(): array 'stringValue' => null, 'stringValueComplex' => null, 'isIdempotent' => true, + 'usesHmac' => false, ], 'object' => [ 'handler' => new ObjectHandler(), @@ -129,16 +137,18 @@ public static function handlerProvider(): array 'stringValue' => null, 'stringValueComplex' => json_encode($object), 'isIdempotent' => true, + 'usesHmac' => false, ], - 'serialize' => [ - 'handler' => new SerializeHandler(), + 'signedSerialize' => [ + 'handler' => new SignedSerializeHandler(), 'type' => 'serialized', 'value' => ['foo' => 'bar', 'baz' => [3]], 'invalid' => [self::$resource], 'numericValue' => null, 'stringValue' => null, 'stringValueComplex' => serialize(['foo' => 'bar', 'baz' => [3]]), - 'isIdempotent' => false, + 'isIdempotent' => true, + 'usesHmac' => true, ], 'serializable' => [ 'handler' => new SerializableHandler(), @@ -149,6 +159,7 @@ public static function handlerProvider(): array 'stringValue' => null, 'stringValueComplex' => serialize(new SampleSerializable(['foo' => 'bar'])), 'isIdempotent' => true, + 'usesHmac' => false, ], 'string' => [ 'handler' => new StringHandler(), @@ -159,6 +170,7 @@ public static function handlerProvider(): array 'stringValue' => 'foo', 'stringValueComplex' => 'foo', 'isIdempotent' => true, + 'usesHmac' => false, ], 'long-string' => [ 'handler' => new StringHandler(), @@ -169,6 +181,7 @@ public static function handlerProvider(): array 'stringValue' => str_repeat('a', 255), 'stringValueComplex' => str_repeat('a', 255), 'isIdempotent' => true, + 'usesHmac' => false, ], 'numeric-string' => [ 'handler' => new StringHandler(), @@ -179,6 +192,7 @@ public static function handlerProvider(): array 'stringValue' => '1.2345', 'stringValueComplex' => '1.2345', 'isIdempotent' => true, + 'usesHmac' => false, ], ]; } @@ -201,9 +215,10 @@ public function test_it_can_verify_and_serialize_data( mixed $value, array $incompatible, null|int|float $numericValue, - null|string $stringValue, - null|string $stringValueComplex, - bool $isIdempotent + ?string $stringValue, + ?string $stringValueComplex, + bool $isIdempotent, + bool $usesHmac ): void { $this->assertEquals($type, $handler->getDataType()); $this->assertTrue($handler->canHandleValue($value)); @@ -215,13 +230,13 @@ public function test_it_can_verify_and_serialize_data( $serialized = $handler->serializeValue($value); $unserialized = $handler->unserializeValue($serialized); + $this->assertEquals($usesHmac, $handler->useHmacVerification()); $this->assertEquals($value, $unserialized); $this->assertEquals($numericValue, $handler->getNumericValue($value)); config()->set('metable.indexComplexDataTypes', false); $this->assertEquals($stringValue, $handler->getStringValue($value)); config()->set('metable.indexComplexDataTypes', true); $this->assertEquals($stringValueComplex, $handler->getStringValue($value)); - $this->assertEquals($isIdempotent, $handler->isIdempotent()); } } diff --git a/tests/Integration/DataType/ModelCollectionHandlerTest.php b/tests/Integration/DataType/ModelCollectionHandlerTest.php index 8eaf1ca..c9fc4ce 100644 --- a/tests/Integration/DataType/ModelCollectionHandlerTest.php +++ b/tests/Integration/DataType/ModelCollectionHandlerTest.php @@ -52,6 +52,23 @@ public function test_it_handles_invalid_collection_class(): void $this->assertEquals([$metable->getKey()], $unserialized->modelKeys()); } + public function test_it_handles_invalid_collection_class_no_key(): void + { + $handler = new ModelCollectionHandler(); + $serialized = json_encode([ + 'class' => 'stdClass', + 'items' => [ + [ + 'class' => SampleMetable::class, + ] + ] + ]); + $unserialized = $handler->unserializeValue($serialized); + + $this->assertInstanceOf(Collection::class, $unserialized); + $this->assertEquals(new Collection([new SampleMetable()]), $unserialized); + } + public function test_it_handles_invalid_model_class(): void { $handler = new ModelCollectionHandler(); diff --git a/tests/Integration/DataType/SerializableHandlerTest.php b/tests/Integration/DataType/SerializableHandlerTest.php index 4de1bea..f9a98e1 100644 --- a/tests/Integration/DataType/SerializableHandlerTest.php +++ b/tests/Integration/DataType/SerializableHandlerTest.php @@ -19,25 +19,25 @@ public function test_it_configures_allowed_classes(): void $incomplete = unserialize(serialize($original), ['allowed_classes' => false]); config()->set( - 'metable.options.serializable.allowedClasses', + 'metable.serializableHandlerAllowedClasses', [SampleSerializable::class] ); $this->assertEquals($original, $handler->unserializeValue($serialized)); config()->set( - 'metable.options.serializable.allowedClasses', + 'metable.serializableHandlerAllowedClasses', true ); $this->assertEquals($original, $handler->unserializeValue($serialized)); config()->set( - 'metable.options.serializable.allowedClasses', + 'metable.serializableHandlerAllowedClasses', [] ); $this->assertEquals($incomplete, $handler->unserializeValue($serialized)); config()->set( - 'metable.options.serializable.allowedClasses', + 'metable.serializableHandlerAllowedClasses', false ); $this->assertEquals($incomplete, $handler->unserializeValue($serialized)); diff --git a/tests/Integration/DataType/SignedSerializeHandlerTest.php b/tests/Integration/DataType/SignedSerializeHandlerTest.php new file mode 100644 index 0000000..870de12 --- /dev/null +++ b/tests/Integration/DataType/SignedSerializeHandlerTest.php @@ -0,0 +1,27 @@ + 'bar']); + $serialized = $handler->serializeValue($value); + $hmac = $handler->useHmacVerification(); + + config()->set('metable.signedSerializeHandlerAllowedClasses', false); + $unserialized = $handler->unserializeValue($serialized); + $this->assertInstanceOf(\__PHP_Incomplete_Class::class, $unserialized); + + config()->set('metable.signedSerializeHandlerAllowedClasses', [SampleSerializable::class]); + $unserialized = $handler->unserializeValue($serialized); + $this->assertInstanceOf(SampleSerializable::class, $unserialized); + $this->assertEquals($value, $unserialized); + } +} diff --git a/tests/Integration/MetaTest.php b/tests/Integration/MetaTest.php index 4ff2b00..6d47115 100644 --- a/tests/Integration/MetaTest.php +++ b/tests/Integration/MetaTest.php @@ -3,6 +3,7 @@ namespace Plank\Metable\Tests\Integration; use Illuminate\Database\Eloquent\Relations\MorphTo; +use Plank\Metable\Exceptions\SecurityException; use Plank\Metable\Meta; use Plank\Metable\Tests\TestCase; @@ -60,6 +61,17 @@ public function test_it_can_get_its_model_relation(): void $this->assertEquals('metable_id', $relation->getForeignKeyName()); } + public function test_it_verifies_hmac(): void + { + $this->expectException(SecurityException::class); + $meta = $this->makeMeta(); + $meta->type = 'serialized'; + $meta->value = ['foo']; + $this->assertEquals('serialized', $meta->type); + $meta->hmac = hash_hmac('sha256', 'foo', 'badsecret'); + $meta->getValueAttribute(); + } + private function makeMeta(array $attributes = []): Meta { return factory(Meta::class)->make($attributes); diff --git a/tests/TestCase.php b/tests/TestCase.php index 3fc9f1b..2b4b726 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -52,7 +52,7 @@ protected function getEnvironmentSetUp($app) ]); $app['config']->set('database.default', 'testing'); - $app['config']->set('metable.options.serializable.allowedClasses', [SampleSerializable::class]); + $app['config']->set('metable.serializableHandlerAllowedClasses', [SampleSerializable::class]); } protected function getPrivateProperty($class, $property_name) From 76506949eca1d885d0f9a7619f73b46ccaca510d Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Fri, 19 Apr 2024 23:01:56 -0400 Subject: [PATCH 27/38] update changelog --- CHANGELOG.md | 12 ++++++++++-- src/Meta.php | 2 +- tests/Integration/MetaTest.php | 1 + 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d6a435..7b72b11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## 6.0.0 +Version 6 contains a number of changes to improve the security and performance of the package. Refer to the [UPGRADING.md](UPGRADING.md) file for detailed instructions on how to upgrade from version 5. + ### Compatibility - Added support for PHP 8.3 @@ -13,18 +15,21 @@ ### Data Types -- Added `getStringValue(): ?string` and `getNumericValue(): null|int|float` methods to `HandlerInterface` which should convert the original value into a format that can be indexed, if possible. - Added `SignedSerializeHandler` as a catch-all datatype, which will attempt to serialize the data using PHP's `serialize()` function. The payload is cryptographically signed with an HMAC before being stored in the database to prevent PHP object injection attacks. + - Deprecated `SerializableHandler` in favor of the new `SignedSerializeHandler` datatype. The `SerializableHandler` will be removed in a future release. In the interim, added the `metable.options.serializable.allowedClasses` config to protect against unserializing untrusted data. - Deprecated `ArrayHandler` and `ObjectHandler`, due to the ambiguity of nested array/objects switching type. These will be removed in a future release. The `SignedSerializeHandler` should be used instead. - `ModelHandler` will now validate that the encoded class is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, the meta value will return `null`. - `ModelHandler` will no longer throw a model not found exception if the model no longer exists. Instead, the meta value will return `null`. This is more in line with the existing behavior of the `ModelCollectionHandler`. - `ModelCollectionHandler` will now validate that the encoded collection class is a valid Eloquent collection before attempting to instantiate it during unserialization. If the class is invalid, an instance of `Illuminate\Database\Eloquent\Collection` will be used instead. - `ModelCollectionHandler` will now validate that the encoded class of each entry is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, that entry in the collection will be omitted. +- Added `getStringValue(): ?string` and `getNumericValue(): null|int|float` methods to `HandlerInterface` which should convert the original value into a format that can be indexed, if possible. +- Added `isIdempotent(): bool` method to `HandlerInterface` which should indicate whether multiple calls to the `serialize()` method with the same value will produce the same serialized output. This is used to determine if the complete serialized value can be used when searching for meta values. +- Added `useHmacVerification(): bool` method to `HandlerInterface` which should indicate whether the integrity of the serialized data should be verified with an HMAC. ### Commands -- Added `metable:refresh` artisan command which will descode and re-encode all meta values in the database. This is useful if you have changed the data type handlers and need to update the serialized data and indexes in the database. +- Added `metable:refresh` artisan command which will decode and re-encode all meta values in the database. This is useful if you have changed the data type handlers and need to update the serialized data and indexes in the database. ### Mediable trait @@ -43,6 +48,9 @@ - `whereMetaIsModel()` - If the data type handlers cannot convert the search value provided to a whereMeta* query scope to a string or numeric value (as appropriate for the scope), then an exception will be thrown. +### Meta +- Added `$meta->raw_value` property which exposes the raw serialized value of the meta key. This is useful for debugging purposes. + # 5.0.1 - 2021-09-19 - Fixed `setManyMeta()` not properly serializing certain types of data. diff --git a/src/Meta.php b/src/Meta.php index c9f8fab..1682286 100644 --- a/src/Meta.php +++ b/src/Meta.php @@ -126,7 +126,7 @@ public function setValueAttribute(mixed $value): void public function getRawValueAttribute(): string { - return $this->attributes['value']; + return $this->getRawValue(); } /** diff --git a/tests/Integration/MetaTest.php b/tests/Integration/MetaTest.php index 6d47115..aba386a 100644 --- a/tests/Integration/MetaTest.php +++ b/tests/Integration/MetaTest.php @@ -16,6 +16,7 @@ public function test_it_can_get_and_set_value(): void $meta->value = 'foo'; $this->assertEquals('foo', $meta->value); + $this->assertEquals('foo', $meta->raw_value); $this->assertEquals('string', $meta->type); } From 254bd3e5a1d03984164d875ed3a924f35aae9ef9 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sat, 20 Apr 2024 17:30:30 -0400 Subject: [PATCH 28/38] initial attempt at casts --- src/Metable.php | 43 +++++++++++++++++++++++-------- tests/Integration/MetableTest.php | 34 +++++++++++++++++------- tests/Mocks/SampleMetable.php | 4 +++ 3 files changed, 60 insertions(+), 21 deletions(-) diff --git a/src/Metable.php b/src/Metable.php index 358d0cc..0378add 100644 --- a/src/Metable.php +++ b/src/Metable.php @@ -7,7 +7,6 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\MorphMany; use Illuminate\Database\Query\JoinClause; -use Illuminate\Support\Facades\DB; use Plank\Metable\DataType\HandlerInterface; use Plank\Metable\DataType\Registry; @@ -40,6 +39,12 @@ trait Metable */ private $indexedMetaCollection; + + public function __construct(array $attributes = []) + { + parent::__construct($attributes); + } + /** * Initialize the trait. * @@ -112,7 +117,7 @@ public function setManyMeta(array $metaDictionary): void return $model->getAttributesForInsert(); })->all(), ['metable_type', 'metable_id', 'key'], - ['type', 'value'] + ['type', 'value', 'string_value', 'numeric_value', 'hmac'] ); if ($needReload) { @@ -155,16 +160,17 @@ public function syncMeta(iterable $array): void public function getMeta(string $key, mixed $default = null): mixed { if ($this->hasMeta($key)) { - return $this->getMetaRecord($key)->getAttribute('value'); + $value = $this->getMetaRecord($key)->getAttribute('value'); } - // If we have only one argument provided (i.e. default is not set) // then we check the model for the defaultMetaValues - if (func_num_args() == 1 && $this->hasDefaultMetaValue($key)) { - return $this->getDefaultMetaValue($key); + elseif (func_num_args() == 1 && $this->hasDefaultMetaValue($key)) { + $value = $this->getDefaultMetaValue($key); + } else { + $value = $default; } - return $default; + return $this->castMetaValue($key, $value); } /** @@ -197,10 +203,10 @@ protected function getDefaultMetaValue(string $key): mixed public function getAllMeta(): \Illuminate\Support\Collection { return collect($this->getAllDefaultMeta())->merge( - $this->getMetaCollection()->toBase()->map(function (Meta $meta) { - return $meta->getAttribute('value'); - }) - ); + $this->getMetaCollection()->toBase()->map( + fn(Meta $meta) => $meta->getAttribute('value') + ) + )->map(fn(mixed $value, string $key) => $this->castMetaValue($key, $value)); } /** @@ -732,6 +738,21 @@ protected function getAllDefaultMeta(): array : []; } + protected function castMetaValue(string $key, mixed $value): mixed + { + if (!property_exists($this, 'castsMeta') + || !empty($this->castsMeta[$key]) + ) { + return $value; + } + $castKey = "meta.$key"; + + $this->casts[$castKey] = $this->castsMeta[$key]; + $value = $this->castAttribute($castKey, $value); + unset($this->casts[$castKey]); + return $value; + } + private function valueToString(mixed $value): string { $stringValue = $this->getHandlerForValue($value)->getStringValue($value); diff --git a/tests/Integration/MetableTest.php b/tests/Integration/MetableTest.php index 4509c52..4297d1f 100644 --- a/tests/Integration/MetableTest.php +++ b/tests/Integration/MetableTest.php @@ -41,17 +41,21 @@ public function test_it_can_set_many_meta_values_at_once(): void $this->assertFalse($metable->hasMeta('baz')); $metable->setManyMeta([ - 'foo' => 'bar', - 'bar' => 'baz', - 'baz' => ['foo', 'bar'], - ]); + 'foo' => 'bar', + 'bar' => 33, + 'baz' => ['foo', 'bar'], + ]); $this->assertTrue($metable->hasMeta('foo')); $this->assertTrue($metable->hasMeta('bar')); $this->assertTrue($metable->hasMeta('baz')); $this->assertEquals('bar', $metable->getMeta('foo')); - $this->assertEquals('baz', $metable->getMeta('bar')); + $this->assertEquals(33, $metable->getMeta('bar')); $this->assertEquals(['foo', 'bar'], $metable->getMeta('baz')); + + $this->assertEquals('bar', $metable->getMetaRecord('foo')->string_value); + $this->assertEquals(33, $metable->getMetaRecord('bar')->numeric_value); + $this->assertNotEmpty($metable->getMetaRecord('baz')->hmac); } public function test_it_accepts_empty_array_for_set_many_meta(): void @@ -98,10 +102,10 @@ public function test_it_can_get_meta_all_values(): void $collection = $metable->getAllMeta(); $this->assertEquals([ - 'foo' => 123, - 'bar' => 'hello', - 'baz' => ['a', 'b', 'c'], - ], $collection->toArray()); + 'foo' => 123, + 'bar' => 'hello', + 'baz' => ['a', 'b', 'c'], + ], $collection->toArray()); } public function test_it_updates_existing_meta_records(): void @@ -394,7 +398,6 @@ public function test_it_can_be_queried_by_in_array(): void $this->assertEquals([], $result2->modelKeys()); } - public function test_it_can_be_queried_by_not_in_array(): void { $this->useDatabase(); @@ -703,6 +706,17 @@ public function test_it_throws_for_param_that_cannot_be_converted_to_numeric(): SampleMetable::whereMetaNumeric('foo', null)->get(); } + public function test_it_can_cast_meta_values(): void + { + $this->useDatabase(); + $metable = $this->createMetable(); + $metable->setMeta('castable', 123); + + $result = SampleMetable::first(); + + $this->assertSame('123', $result->getMeta('castable')); + } + private function makeMeta(array $attributes = []): Meta { return factory(Meta::class)->make($attributes); diff --git a/tests/Mocks/SampleMetable.php b/tests/Mocks/SampleMetable.php index 60441b9..f50c898 100644 --- a/tests/Mocks/SampleMetable.php +++ b/tests/Mocks/SampleMetable.php @@ -12,4 +12,8 @@ class SampleMetable extends Model protected $defaultMetaValues = [ 'foo' => 'bar' ]; + + protected $castsMeta = [ + 'castable' => 'string', + ]; } From c9ffd50bb0e70795c571e8f7777401d36710f20b Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sat, 20 Apr 2024 20:27:32 -0400 Subject: [PATCH 29/38] add enum support --- CHANGELOG.md | 1 + UPGRADING.md | 1 + config/metable.php | 2 + docs/source/datatypes.rst | 54 ++++++++++------- src/DataType/BackedEnumHandler.php | 61 +++++++++++++++++++ src/DataType/PureEnumHandler.php | 58 ++++++++++++++++++ tests/Integration/DataType/HandlerTest.php | 69 +++++++++++++++++++++- tests/Mocks/SampleIntBackedEnum.php | 8 +++ tests/Mocks/SamplePureEnum.php | 8 +++ tests/Mocks/SampleStringBackedEnum.php | 10 ++++ 10 files changed, 251 insertions(+), 21 deletions(-) create mode 100644 src/DataType/BackedEnumHandler.php create mode 100644 src/DataType/PureEnumHandler.php create mode 100644 tests/Mocks/SampleIntBackedEnum.php create mode 100644 tests/Mocks/SamplePureEnum.php create mode 100644 tests/Mocks/SampleStringBackedEnum.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b72b11..a70b3c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Version 6 contains a number of changes to improve the security and performance o - Deprecated `SerializableHandler` in favor of the new `SignedSerializeHandler` datatype. The `SerializableHandler` will be removed in a future release. In the interim, added the `metable.options.serializable.allowedClasses` config to protect against unserializing untrusted data. - Deprecated `ArrayHandler` and `ObjectHandler`, due to the ambiguity of nested array/objects switching type. These will be removed in a future release. The `SignedSerializeHandler` should be used instead. +- Added `PureEnumHandler` and `BackedEnumHandler` which adds support for storing enum values as Meta. - `ModelHandler` will now validate that the encoded class is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, the meta value will return `null`. - `ModelHandler` will no longer throw a model not found exception if the model no longer exists. Instead, the meta value will return `null`. This is more in line with the existing behavior of the `ModelCollectionHandler`. - `ModelCollectionHandler` will now validate that the encoded collection class is a valid Eloquent collection before attempting to instantiate it during unserialization. If the class is invalid, an instance of `Illuminate\Database\Eloquent\Collection` will be used instead. diff --git a/UPGRADING.md b/UPGRADING.md index 029ba36..1d1d7c6 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -6,6 +6,7 @@ * Minimum Laravel version moved to 10 * Some methods have had their signatures adjusted to use PHP 8+ mixed and union types. If extending any class or implementing any interface from this package, method signatures may need to be updated. * A new schema migration has been added which adds three new columns to the meta table and improves indexing for querying by meta values. +* Add the `PureEnumHandler` and `BackedEnumHandler` classes to the `datatypes` config. These handlers provide support for storing enum values as Meta. * Recommended to add the `SignedSerializeHandler` to the end of `datatypes` config (catch-all). * The `SerializableHandler`, `ArrayHandler`, and `ObjectHandler` data types have been deprecated in favor of the new `SignedSerializeHandler`. If you have any Meta encoded using any of these data types, you should continue to include them in the `datatypes` config _after_ the `SignedSerializeHandler` to ensure that existing values will continue to be properly decoded, but new values will use the new encoding. Once all old values have been migrated, you may remove the deprecated data types from the `datatypes` config. * For security reasons, if you have any existing Meta encoded using `SerializableHandler`, you must configure the `metable.serializableHandlerAllowedClasses` config to list classes that are allowed to be unserialized. Otherwise, all objects will be returned as `__PHP_Incomplete_Class`. This config may be set to `true` to disable this security check and allow any class, but this is not recommended. diff --git a/config/metable.php b/config/metable.php index 3aed2d7..907b0d7 100644 --- a/config/metable.php +++ b/config/metable.php @@ -28,6 +28,8 @@ Plank\Metable\DataType\DateTimeHandler::class, Plank\Metable\DataType\ModelHandler::class, Plank\Metable\DataType\ModelCollectionHandler::class, + Plank\Metable\DataType\BackedEnumHandler::class, + Plank\Metable\DataType\PureEnumHandler::class, /* * The following handler is a catch-all that will encode anything. diff --git a/docs/source/datatypes.rst b/docs/source/datatypes.rst index a5bed10..d956111 100644 --- a/docs/source/datatypes.rst +++ b/docs/source/datatypes.rst @@ -17,7 +17,7 @@ The following scalar values are supported. Boolean ^^^^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\BooleanHandler | +| Handler | ``\Plank\Metable\DataType\BooleanHandler`` | | String Query Scopes | Yes | | Numeric Query Scopes | Yes | | Other Query Scopes | | @@ -31,7 +31,7 @@ Boolean Integer ^^^^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\IntegerHandler | +| Handler | ``\Plank\Metable\DataType\IntegerHandler`` | | String Query Scopes | Yes | | Numeric Query Scopes | Yes | | Other Query Scopes | | @@ -45,7 +45,7 @@ Integer Float ^^^^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\FloatHandler | +| Handler | ``\Plank\Metable\DataType\FloatHandler`` | | String Query Scopes | Yes | | Numeric Query Scopes | Yes | | Other Query Scopes | | @@ -59,7 +59,7 @@ Float Null ^^^^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\NullHandler | +| Handler | ``\Plank\Metable\DataType\NullHandler`` | | String Query Scopes | No | | Numeric Query Scopes | No | | Other Query Scopes | whereMetaIsNull() | @@ -73,8 +73,8 @@ Null String ^^^^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\StringHandler | -| String Query Scopes | Yes, first `metable.stringValueIndexLength` characters indexed | +| Handler | ``\Plank\Metable\DataType\StringHandler`` | +| String Query Scopes | Yes, first ``metable.stringValueIndexLength`` characters indexed | | Numeric Query Scopes | if string is numeric | | Other Query Scopes | | +----------------------+-----+ @@ -84,8 +84,8 @@ String setMeta('attachment', '/var/www/html/public/attachment.pdf'); -Objects ---------------- +Composite Values +---------------- The following classes and interfaces are supported. @@ -95,7 +95,7 @@ Eloquent Models ^^^^^^^^^^^^^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\ModelHandler | +| Handler | ``\Plank\Metable\DataType\ModelHandler`` | | String Query Scopes | Yes | | Numeric Query Scopes | No | | Other Query Scopes | whereMetaIsModel() | @@ -125,7 +125,7 @@ Eloquent Collections ^^^^^^^^^^^^^^^^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\ModelCollectionHandler | +| Handler | ``\Plank\Metable\DataType\ModelCollectionHandler`` | | String Query Scopes | No | | Numeric Query Scopes | No | | Other Query Scopes | | @@ -144,7 +144,7 @@ As with individual models, both existing and unsaved instances can be stored. DateTime & Carbon ^^^^^^^^^^^^^^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\DateTimeHandler | +| Handler | ``\Plank\Metable\DataType\DateTimeHandler`` | | String Query Scopes | Yes (UTC format) | | Numeric Query Scopes | Yes (timestamp) | | Other Query Scopes | | @@ -157,17 +157,31 @@ Any object implementing the ``DateTimeInterface``. Object will be converted to setMeta('last_viewed', \Carbon\Carbon::now()); +Enums +^^^^^^^^ ++----------------------+-----+ +| Handler | ``\Plank\Metable\DataType\PureEnumHandler``
``\Plank\Metable\DataType\BackedEnumHandler`` | +| String Query Scopes | Yes | +| Numeric Query Scopes | If backed with integer or numeric-string | +| Other Query Scopes | | ++----------------------+-----+ + +:: + + setMeta('status', Status::ACTIVE); + Objects and Arrays ^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\SignedSerializeHandler | -| String Query Scopes | if `metable.indexComplexDataTypes` is enabled | +| Handler | ``\Plank\Metable\DataType\SignedSerializeHandler`` | +| String Query Scopes | if ``metable.indexComplexDataTypes`` is enabled | | Numeric Query Scopes | No | | Other Query Scopes | | +----------------------+-----+ -Objects and arrays will be serialized using PHP's `serialize()` function, to allow for the storage and retrieval of complex data structures. The serialized value is cryptographically signed with an HMAC which is verified before the data is unserialized to prevent PHP object injection attacks. The application's ``APP_KEY`` is used as the HMAC signing key. +Objects and arrays will be serialized using PHP's ``serialize()`` function, to allow for the storage and retrieval of complex data structures. The serialized value is cryptographically signed with an HMAC which is verified before the data is unserialized to prevent PHP object injection attacks. The application's ``APP_KEY`` is used as the HMAC signing key. :: @@ -188,8 +202,8 @@ Array ^^^^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\ArrayHandler | -| String Query Scopes | if `metable.indexComplexDataTypes` is enabled | +| Handler | ``\Plank\Metable\DataType\ArrayHandler`` | +| String Query Scopes | if ``metable.indexComplexDataTypes`` is enabled | | Numeric Query Scopes | No | | Other Query Scopes | | +----------------------+-----+ @@ -220,8 +234,8 @@ Serializable ^^^^^^^^^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\ArrayHandler | -| String Query Scopes | if `metable.indexComplexDataTypes` is enabled | +| Handler | ``\Plank\Metable\DataType\ArrayHandler`` | +| String Query Scopes | if ``metable.indexComplexDataTypes`` is enabled | | Numeric Query Scopes | No | | Other Query Scopes | | +----------------------+-----+ @@ -248,8 +262,8 @@ Plain Objects ^^^^^^^^^^^^^^ +----------------------+-----+ -| Handler | \Plank\Metable\DataType\ArrayHandler | -| String Query Scopes | if `metable.indexComplexDataTypes` is enabled | +| Handler | ``\Plank\Metable\DataType\ArrayHandler`` | +| String Query Scopes | if ``metable.indexComplexDataTypes`` is enabled | | Numeric Query Scopes | No | | Other Query Scopes | | +----------------------+-----+ diff --git a/src/DataType/BackedEnumHandler.php b/src/DataType/BackedEnumHandler.php new file mode 100644 index 0000000..16f79b3 --- /dev/null +++ b/src/DataType/BackedEnumHandler.php @@ -0,0 +1,61 @@ +value + ); + } + + public function unserializeValue(string $serializedValue): mixed + { + [$class, $value] = explode('#', $serializedValue, 2); + + if (!class_exists($class) + || !is_a($class, \BackedEnum::class, true) + ) { + return null; + } + + return $class::tryFrom($value); + } + + public function getNumericValue(mixed $value): null|int|float + { + if(is_numeric($value->value)) { + return $value->value; + } + return null; + } + + public function getStringValue(mixed $value): null|string + { + return (string)$value->value; + } + + public function isIdempotent(): bool + { + return true; + } + + public function useHmacVerification(): bool + { + return false; + } +} diff --git a/src/DataType/PureEnumHandler.php b/src/DataType/PureEnumHandler.php new file mode 100644 index 0000000..54a6d07 --- /dev/null +++ b/src/DataType/PureEnumHandler.php @@ -0,0 +1,58 @@ +name + ); + } + + public function unserializeValue(string $serializedValue): mixed + { + [$class, $name] = explode('#', $serializedValue, 2); + + if (!class_exists($class) + || !is_a($class, \UnitEnum::class, true) + ) { + return null; + } + + return constant("$class::$name"); + } + + public function getNumericValue(mixed $value): null|int|float + { + return null; + } + + public function getStringValue(mixed $value): null|string + { + return $value->name; + } + + public function isIdempotent(): bool + { + return true; + } + + public function useHmacVerification(): bool + { + return false; + } +} diff --git a/tests/Integration/DataType/HandlerTest.php b/tests/Integration/DataType/HandlerTest.php index 7f5163a..0a9aa92 100644 --- a/tests/Integration/DataType/HandlerTest.php +++ b/tests/Integration/DataType/HandlerTest.php @@ -5,6 +5,7 @@ use Carbon\Carbon; use Illuminate\Database\Eloquent\Collection; use Plank\Metable\DataType\ArrayHandler; +use Plank\Metable\DataType\BackedEnumHandler; use Plank\Metable\DataType\BooleanHandler; use Plank\Metable\DataType\DateTimeHandler; use Plank\Metable\DataType\FloatHandler; @@ -17,8 +18,12 @@ use Plank\Metable\DataType\SerializableHandler; use Plank\Metable\DataType\SignedSerializeHandler; use Plank\Metable\DataType\StringHandler; +use Plank\Metable\DataType\PureEnumHandler; +use Plank\Metable\Tests\Mocks\SampleIntBackedEnum; use Plank\Metable\Tests\Mocks\SampleMetable; use Plank\Metable\Tests\Mocks\SampleSerializable; +use Plank\Metable\Tests\Mocks\SampleStringBackedEnum; +use Plank\Metable\Tests\Mocks\SamplePureEnum; use Plank\Metable\Tests\TestCase; use stdClass; @@ -194,6 +199,68 @@ public static function handlerProvider(): array 'isIdempotent' => true, 'usesHmac' => false, ], + 'unitEnum' => [ + 'handler' => new PureEnumHandler(), + 'type' => 'enum', + 'value' => SamplePureEnum::Alpha, + 'invalid' => [ + SampleIntBackedEnum::One, + SampleStringBackedEnum::Alpha, + 'Alpha', + 1, + new stdClass() + ], + 'numericValue' => null, + 'stringValue' => 'Alpha', + 'stringValueComplex' => 'Alpha', + 'isIdempotent' => true, + 'usesHmac' => false, + ], + 'stringBackedEnum' => [ + 'handler' => new BackedEnumHandler(), + 'type' => 'backed_enum', + 'value' => SampleStringBackedEnum::Alpha, + 'invalid' => [ + SamplePureEnum::Alpha, + 'Alpha', + new stdClass() + ], + 'numericValue' => null, + 'stringValue' => 'alpha', + 'stringValueComplex' => 'alpha', + 'isIdempotent' => true, + 'usesHmac' => false, + ], + 'numericStringBackedEnum' => [ + 'handler' => new BackedEnumHandler(), + 'type' => 'backed_enum', + 'value' => SampleStringBackedEnum::Numeric, + 'invalid' => [ + SamplePureEnum::Alpha, + 'Alpha', + new stdClass() + ], + 'numericValue' => 1, + 'stringValue' => '1', + 'stringValueComplex' => '1', + 'isIdempotent' => true, + 'usesHmac' => false, + ], + 'intBackedEnum' => [ + 'handler' => new BackedEnumHandler(), + 'type' => 'backed_enum', + 'value' => SampleIntBackedEnum::One, + 'invalid' => [ + SamplePureEnum::Alpha, + 1, + new stdClass() + ], + 'numericValue' => 1, + 'stringValue' => '1', + 'stringValueComplex' => '1', + 'isIdempotent' => true, + 'usesHmac' => false, + ], ]; } @@ -224,7 +291,7 @@ public function test_it_can_verify_and_serialize_data( $this->assertTrue($handler->canHandleValue($value)); foreach ($incompatible as $incompatibleValue) { - $this->assertFalse($handler->canHandleValue($incompatibleValue)); + $this->assertFalse($handler->canHandleValue($incompatibleValue), "Failed for ". get_debug_type($incompatibleValue)); } $serialized = $handler->serializeValue($value); diff --git a/tests/Mocks/SampleIntBackedEnum.php b/tests/Mocks/SampleIntBackedEnum.php new file mode 100644 index 0000000..c10c082 --- /dev/null +++ b/tests/Mocks/SampleIntBackedEnum.php @@ -0,0 +1,8 @@ + Date: Sat, 20 Apr 2024 21:06:56 -0400 Subject: [PATCH 30/38] update MetableInterface --- src/MetableInterface.php | 13 +++++++++++-- tests/Mocks/SampleMetable.php | 3 ++- tests/Mocks/SampleMetableSoftDeletes.php | 3 ++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/MetableInterface.php b/src/MetableInterface.php index 38a5460..cf9dd5c 100644 --- a/src/MetableInterface.php +++ b/src/MetableInterface.php @@ -5,15 +5,24 @@ use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Relations\MorphMany; use Illuminate\Support\Collection; -use Plank\Metable\Meta; +use Illuminate\Database\Eloquent\Model; /** - * @method static Builder whereHasMeta($key): void + * @method static Builder whereHasMeta($key) * @method static Builder whereDoesntHaveMeta($key) * @method static Builder whereHasMetaKeys(array $keys) * @method static Builder whereMeta(string $key, $operator, $value = null) * @method static Builder whereMetaNumeric(string $key, string $operator, $value) * @method static Builder whereMetaIn(string $key, array $values) + * @method static Builder whereMetaNotIn(string $key, array $values) + * @method static Builder whereMetaInNumeric(string $key, array $values) + * @method static Builder whereMetaNotInNumeric(string $key, array $values) + * @method static Builder whereMetaBetween(string $key, array $values) + * @method static Builder whereMetaNotBetween(string $key, array $values) + * @method static Builder whereMetaBetweenNumeric(string $key, array $values) + * @method static Builder whereMetaNotBetweenNumeric(string $key, array $values) + * @method static Builder whereMetaIsModel(string $key, string|Model $classOrInstance, string $modelId = null) + * @method static Builder whereMetaIsNull(string $key) * @method static Builder orderByMeta(string $key, string $direction = 'asc', $strict = false) * @method static Builder orderByMetaNumeric(string $key, string $direction = 'asc', $strict = false) */ diff --git a/tests/Mocks/SampleMetable.php b/tests/Mocks/SampleMetable.php index f50c898..04aed50 100644 --- a/tests/Mocks/SampleMetable.php +++ b/tests/Mocks/SampleMetable.php @@ -4,8 +4,9 @@ use Illuminate\Database\Eloquent\Model; use Plank\Metable\Metable; +use Plank\Metable\MetableInterface; -class SampleMetable extends Model +class SampleMetable extends Model implements MetableInterface { use Metable; diff --git a/tests/Mocks/SampleMetableSoftDeletes.php b/tests/Mocks/SampleMetableSoftDeletes.php index fdb4221..2769e09 100644 --- a/tests/Mocks/SampleMetableSoftDeletes.php +++ b/tests/Mocks/SampleMetableSoftDeletes.php @@ -5,8 +5,9 @@ use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\SoftDeletes; use Plank\Metable\Metable; +use Plank\Metable\MetableInterface; -class SampleMetableSoftDeletes extends Model +class SampleMetableSoftDeletes extends Model implements MetableInterface { use Metable; use SoftDeletes; From fab3910ebde4cad648e3023274e5c0f697bc3979 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Tue, 23 Apr 2024 22:20:47 -0400 Subject: [PATCH 31/38] add support for casting meta --- CHANGELOG.md | 23 ++- UPGRADING.md | 24 ++- composer.json | 4 +- config/metable.php | 8 +- docs/source/datatypes.rst | 37 +++- docs/source/handling_meta.rst | 45 +++++ src/DataType/BackedEnumHandler.php | 2 +- src/DataType/DateTimeHandler.php | 2 +- src/DataType/DateTimeImmutableHandler.php | 76 ++++++++ src/DataType/StringableHandler.php | 52 ++++++ src/Exceptions/CastException.php | 17 ++ src/Meta.php | 41 ++++- src/Metable.php | 196 ++++++++++++++++++--- tests/Integration/DataType/HandlerTest.php | 25 +++ tests/Integration/MetaTest.php | 16 ++ tests/Integration/MetableTest.php | 155 ++++++++++++++++ tests/Mocks/SampleMetable.php | 2 +- tests/Mocks/SampleStringBackedEnum.php | 1 - tests/TestCase.php | 1 + 19 files changed, 677 insertions(+), 50 deletions(-) create mode 100644 src/DataType/DateTimeImmutableHandler.php create mode 100644 src/DataType/StringableHandler.php create mode 100644 src/Exceptions/CastException.php diff --git a/CHANGELOG.md b/CHANGELOG.md index a70b3c2..15cad40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,10 +16,11 @@ Version 6 contains a number of changes to improve the security and performance o ### Data Types - Added `SignedSerializeHandler` as a catch-all datatype, which will attempt to serialize the data using PHP's `serialize()` function. The payload is cryptographically signed with an HMAC before being stored in the database to prevent PHP object injection attacks. - - Deprecated `SerializableHandler` in favor of the new `SignedSerializeHandler` datatype. The `SerializableHandler` will be removed in a future release. In the interim, added the `metable.options.serializable.allowedClasses` config to protect against unserializing untrusted data. - Deprecated `ArrayHandler` and `ObjectHandler`, due to the ambiguity of nested array/objects switching type. These will be removed in a future release. The `SignedSerializeHandler` should be used instead. - Added `PureEnumHandler` and `BackedEnumHandler` which adds support for storing enum values as Meta. +- Added `StringableHandler` which adds support for storing `Illuminate\Support\Stringable` objects as Meta. +- Added `DateTimeImmutableHandler` which adds support for storing `DateTimeImmutable`/`CarbonImmutable` objects as Meta. - `ModelHandler` will now validate that the encoded class is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, the meta value will return `null`. - `ModelHandler` will no longer throw a model not found exception if the model no longer exists. Instead, the meta value will return `null`. This is more in line with the existing behavior of the `ModelCollectionHandler`. - `ModelCollectionHandler` will now validate that the encoded collection class is a valid Eloquent collection before attempting to instantiate it during unserialization. If the class is invalid, an instance of `Illuminate\Database\Eloquent\Collection` will be used instead. @@ -28,13 +29,13 @@ Version 6 contains a number of changes to improve the security and performance o - Added `isIdempotent(): bool` method to `HandlerInterface` which should indicate whether multiple calls to the `serialize()` method with the same value will produce the same serialized output. This is used to determine if the complete serialized value can be used when searching for meta values. - Added `useHmacVerification(): bool` method to `HandlerInterface` which should indicate whether the integrity of the serialized data should be verified with an HMAC. -### Commands +### New Commands - Added `metable:refresh` artisan command which will decode and re-encode all meta values in the database. This is useful if you have changed the data type handlers and need to update the serialized data and indexes in the database. -### Mediable trait +### Searching Metables By Meta Value -- `whereMeta()`, `whereMetaIn()`, and `orderByMeta()` query scopes will now scan the indexed `string_value` column instead of the serialized `value` column. This greatly improves performance when searching for meta values against larger datasets. +- the Metable `whereMeta()`, `whereMetaIn()`, and `orderByMeta()` query scopes will now scan the indexed `string_value` column instead of the serialized `value` column. This greatly improves performance when searching for meta values against larger datasets. - `whereMetaNumeric()` and `orderByMetaNumeric()` query scopes will now scan the indexed `numeric_value` column instead of the serialized `value` column. This greatly improves performance when searching for meta values against larger datasets. - `whereMetaNumeric()` query scope will now accept a value of any type. It will be converted to an integer or float by the handler. This is more consistent with the behaviour of the other query scopes. - Added additional query scopes to more easily search meta values based on different criteria: @@ -47,10 +48,20 @@ Version 6 contains a number of changes to improve the security and performance o - `whereMetaNotBetweenNumeric()` - `whereMetaIsNull()` - `whereMetaIsModel()` -- If the data type handlers cannot convert the search value provided to a whereMeta* query scope to a string or numeric value (as appropriate for the scope), then an exception will be thrown. +- If the data type handlers cannot convert the search value provided to a whereMeta* query scope to a string or numeric value (as appropriate for the method), then an exception will be thrown. + +### Metable Casting + +- Added support for casting meta values to specific types by defining the `$castMeta` property or `castMeta(): array` method on the model. This is similar to the `casts` property of Eloquent Models used for attributes. All cast types supported by Eloquent Models are also available for Meta values.Values will be cast before values are stored in the database to ensure that they are indexed consistently +- the `encrypted:` cast prefix can be combined with any other cast type to cast to the desired type before the value is encrypted to be stored it in the database. Encrypted values are not searchable. +- A value of `null` is ignored by all cast types. +- Added `mergeMetaCasts()` method which can be used to override the defined cast on a meta key at runtime. ### Meta -- Added `$meta->raw_value` property which exposes the raw serialized value of the meta key. This is useful for debugging purposes. +- Added `$meta->string_value` and `$meta->numeric_value` attributes, which are used for optimizing queries filtering by meta value +- Added `$meta->hmac` attribute, which is used by some data type handlers to validate that the payload has not been tampered with. +- Added `$meta->raw_value` virtual attribute, which exposes the raw serialized value of the meta key. This is useful for debugging purposes. +- Added `encrypt()` method, used internally by the `Metable::setMetaEncrypted()` method # 5.0.1 - 2021-09-19 - Fixed `setManyMeta()` not properly serializing certain types of data. diff --git a/UPGRADING.md b/UPGRADING.md index 1d1d7c6..b9cb473 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -2,19 +2,37 @@ ## 5.X -> 6.X +### Compatibility + * Minimum PHP version moved to 8.1 * Minimum Laravel version moved to 10 * Some methods have had their signatures adjusted to use PHP 8+ mixed and union types. If extending any class or implementing any interface from this package, method signatures may need to be updated. + +### Schema Changes + * A new schema migration has been added which adds three new columns to the meta table and improves indexing for querying by meta values. -* Add the `PureEnumHandler` and `BackedEnumHandler` classes to the `datatypes` config. These handlers provide support for storing enum values as Meta. -* Recommended to add the `SignedSerializeHandler` to the end of `datatypes` config (catch-all). + +### Configuration Changes + +* Add the `Plank\Metable\DateType\PureEnumHandler`, `Plank\Metable\DateType\BackedEnumHandler`, `Plank\Metable\DateType\DateTimeImmutableHandler`, `Plank\Metable\DateType\StringableHandler` classes to the `datatypes` config. The order of these handlers is not important, except for `DateTimeImmutableHandler` which must come before `DateTimeHandler` if both are used. +* Recommended to add the `Plank\Metable\DateType\SignedSerializeHandler` class to the end of `datatypes` config list (catch-all). * The `SerializableHandler`, `ArrayHandler`, and `ObjectHandler` data types have been deprecated in favor of the new `SignedSerializeHandler`. If you have any Meta encoded using any of these data types, you should continue to include them in the `datatypes` config _after_ the `SignedSerializeHandler` to ensure that existing values will continue to be properly decoded, but new values will use the new encoding. Once all old values have been migrated, you may remove the deprecated data types from the `datatypes` config. * For security reasons, if you have any existing Meta encoded using `SerializableHandler`, you must configure the `metable.serializableHandlerAllowedClasses` config to list classes that are allowed to be unserialized. Otherwise, all objects will be returned as `__PHP_Incomplete_Class`. This config may be set to `true` to disable this security check and allow any class, but this is not recommended. + +### Handlers + * If you have any custom data types, you will need to implement the new methods from the `HandlerInterface`: * `getStringValue(): ?string` and `getNumericValue(): null|int|float`: These are used to populate the new indexed search columns. You may return `null` if the value cannot be converted into the specified format or does not need to be searchable. * `isIdempotent(): bool`: This method should indicate whether multiple calls to the `serialize()` method with the same value will produce the same serialized output. This is used to determine if the complete serialized value can be used when searching for meta values. * `useHmacVerification(): bool`: if the integrity of the serialized data should be verified with a HMAC, return `true`. If unserializing this data type is safe without HMAC verification, you may return `false`. -* Once you have applied the schema migration and configured the changes to the `datatypes` config, you should run the `metable:refresh` Artisan command to update all existing meta values to use the new types and populate the index columns. After this command has been run, you may remove the deprecated data types from the `datatypes` config. + +### Update Existing Data + +* Once you have applied the schema migration and updated the `datatypes` config, you should run the `metable:refresh` Artisan command to update all existing meta values to use the new types and populate the index columns. +* After this command has been run, you may remove the deprecated data types from the `datatypes` config. + +### Query Scopes + * Review the documentation about which data types can be queried with the various `whereMeta*` and `whereMeta*Numeric` query scopes. If you are querying the serialized `value` column directly, be aware that the formatting of array/object data types may have changed. ## 4.X -> 5.X diff --git a/composer.json b/composer.json index e7867b4..9564dec 100644 --- a/composer.json +++ b/composer.json @@ -17,8 +17,8 @@ "require": { "php": ">=8.1", "ext-json": "*", - "illuminate/support": "^10.0|^11.0", - "illuminate/database": "^10.0|^11.0", + "illuminate/support": "^10.10|^11.0", + "illuminate/database": "^10.10|^11.0", "phpoption/phpoption": "^1.8" }, "require-dev": { diff --git a/config/metable.php b/config/metable.php index 907b0d7..7385758 100644 --- a/config/metable.php +++ b/config/metable.php @@ -20,16 +20,18 @@ * If you change this list, it may be necessary to refresh the meta table with the `artisan metable:refresh` command. */ 'datatypes' => [ - Plank\Metable\DataType\BooleanHandler::class, Plank\Metable\DataType\NullHandler::class, + Plank\Metable\DataType\BooleanHandler::class, Plank\Metable\DataType\IntegerHandler::class, Plank\Metable\DataType\FloatHandler::class, Plank\Metable\DataType\StringHandler::class, + Plank\Metable\DataType\StringableHandler::class, + Plank\Metable\DataType\DateTimeImmutableHandler::class, Plank\Metable\DataType\DateTimeHandler::class, - Plank\Metable\DataType\ModelHandler::class, - Plank\Metable\DataType\ModelCollectionHandler::class, Plank\Metable\DataType\BackedEnumHandler::class, Plank\Metable\DataType\PureEnumHandler::class, + Plank\Metable\DataType\ModelHandler::class, + Plank\Metable\DataType\ModelCollectionHandler::class, /* * The following handler is a catch-all that will encode anything. diff --git a/docs/source/datatypes.rst b/docs/source/datatypes.rst index d956111..9d37044 100644 --- a/docs/source/datatypes.rst +++ b/docs/source/datatypes.rst @@ -157,6 +157,39 @@ Any object implementing the ``DateTimeInterface``. Object will be converted to setMeta('last_viewed', \Carbon\Carbon::now()); +DateTimeImmutable & CarbonImmutable +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ++----------------------+-----+ +| Handler | ``\Plank\Metable\DataType\DateTimeImmutableHandler`` | +| String Query Scopes | Yes (UTC format) | +| Numeric Query Scopes | Yes (timestamp) | +| Other Query Scopes | | ++----------------------+-----+ + +Any object extending the ``DateTimeImmutable`` class. Object will be converted to a ``CarbonImmutable`` instance when unserialized. + +:: + + setMeta('completed_at', \Carbon\CarbonImmutable::now()); + +Stringable +^^^^^^^^^^ ++----------------------+-----+ +| Handler | ``\Plank\Metable\DataType\StringableHandler`` | +| String Query Scopes | Yes | +| Numeric Query Scopes | If numeric string | +| Other Query Scopes | | ++----------------------+-----+ + +Strings wrapped in Laravel's ``Illuminate\Support\Stringable`` fluent interface. + +:: + + setMeta('address', Str::of('123 Somewhere St.')); + Enums ^^^^^^^^ +----------------------+-----+ @@ -181,7 +214,7 @@ Objects and Arrays | Other Query Scopes | | +----------------------+-----+ -Objects and arrays will be serialized using PHP's ``serialize()`` function, to allow for the storage and retrieval of complex data structures. The serialized value is cryptographically signed with an HMAC which is verified before the data is unserialized to prevent PHP object injection attacks. The application's ``APP_KEY`` is used as the HMAC signing key. +Objects and arrays will be serialized using PHP's ``serialize()`` function, to allow for the storage and retrieval of complex data structures. :: @@ -189,7 +222,7 @@ Objects and arrays will be serialized using PHP's ``serialize()`` function, to a $metable->setMeta('data', ['key' => 'value']); $metable->setMeta('data', new MyValueObject(123)); -HMAC verification is generally sufficient for preventing PHP object injection attacks, but it possible to further restrict what can be unserialized by specifying an array or class name in the ``metable.serializableHandlerAllowedClasses`` config in the ``config/metable.php`` file. +The serialized value is cryptographically signed with an HMAC which is verified before the data is unserialized to prevent PHP object injection attacks. The application's ``APP_KEY`` is used as the HMAC signing key. HMAC verification is generally sufficient for preventing PHP object injection attacks, but it possible to further restrict what can be unserialized by specifying an array or class name in the ``metable.SignedSerializeHandlerAllowedClasses`` config in the ``config/metable.php`` file. .. note:: The ``Plank\Metable\DataType\SignedSerializeHandler`` class should generally be the last entry the ``config/metable.php`` datatypes array, as it will accept data of any type, causing any handlers below it to be ignored for serializing new meta values. Any handlers defined below it will still be used for unserializing existing meta values. This can be used to temporarily provide backwards compatibility for deprecated data types. diff --git a/docs/source/handling_meta.rst b/docs/source/handling_meta.rst index be77201..2913d3f 100644 --- a/docs/source/handling_meta.rst +++ b/docs/source/handling_meta.rst @@ -54,6 +54,19 @@ To replace existing meta with a new set of meta, you can pass an associative arr 'age' => 18, ]); +Encrypting Meta +--------------- + +If storing sensitive data, you can instruct the package to encrypt a meta value when it is stored in the database. Encrypted values are automatically decrypted when retrieved. To encrypt a value, use the ``setMetaEncrypted()`` method or pass ``true`` as the third argument to the ``setMeta()`` method. + +:: + + setMetaEncrypted('secret', 'sensitive data'); + $model->setMeta('secret', 'sensitive data', true); + +Data of any type can be encrypted. Encrypted values are never searchable or sortable with query scopes. + Retrieving Meta --------------- @@ -106,6 +119,7 @@ Alternatively, you may set default values as key-value pairs on the model itself //... } + :: 'boolean', + 'age' => 'integer', + 'secret' => 'encrypted:string', + 'parent' => ExampleMetable::class, + 'children' => 'collection:\App\ExampleMetable', + ]; + + //... + } + +All `cast types supported by Eloquent`_ are supported, with the following modifications: + +- Casts are applied on write, not read. This means that the value will be serialized and stored in the database in the specified format, and indexes will be populated in a consistent manner. However, old data prior to the addition of the cast will not be automatically converted. +- All casts ignore values of ``null``. If a value is set to ``null``, it will be stored as ``null`` in the database, and will not be cast to the specified type. +- The ``encrypted`` cast will tell the package to always encrypt the value of that key, even if the 3rd parameter of ``setMeta()`` is not set to ``true``. +- The ``encrypted:`` cast prefix can be combined with any other cast type to convert the value to the specified type before encrypting it. +- when a class name is provided as a cast, if it implements ``\Illuminate\Contracts\Database\Eloquent\Castable``, it will be used to cast the value per the interface. Otherwise, it will enforce that the value is an instance of that class. If an instance of a different class is provided, an exception will be thrown. If the class is an Eloquent model, and an an integer or string is provided, it will attempt to retrieve the model from the database. +- The ``collection`` cast will preserve ``Illuminate\Database\Eloquent\Collection`` instances and contents, using the ``Plank\Metable\DataType\ModelCollection`` data type to store them. If passed a single model instance, it will be wrapped in an eloquent collection. A class name can be provided as an argument to enforce that the collection contains only instances of that class. + Retrieving All Meta ------------------- diff --git a/src/DataType/BackedEnumHandler.php b/src/DataType/BackedEnumHandler.php index 16f79b3..1517b7f 100644 --- a/src/DataType/BackedEnumHandler.php +++ b/src/DataType/BackedEnumHandler.php @@ -38,7 +38,7 @@ public function unserializeValue(string $serializedValue): mixed public function getNumericValue(mixed $value): null|int|float { - if(is_numeric($value->value)) { + if (is_numeric($value->value)) { return $value->value; } return null; diff --git a/src/DataType/DateTimeHandler.php b/src/DataType/DateTimeHandler.php index 2ecd2ec..92ee322 100644 --- a/src/DataType/DateTimeHandler.php +++ b/src/DataType/DateTimeHandler.php @@ -15,7 +15,7 @@ class DateTimeHandler implements HandlerInterface * * @var string */ - const FORMAT = 'Y-m-d H:i:s.uO'; + public const FORMAT = 'Y-m-d H:i:s.uO'; /** * {@inheritdoc} diff --git a/src/DataType/DateTimeImmutableHandler.php b/src/DataType/DateTimeImmutableHandler.php new file mode 100644 index 0000000..c18554b --- /dev/null +++ b/src/DataType/DateTimeImmutableHandler.php @@ -0,0 +1,76 @@ +format(self::FORMAT); + } + + /** + * {@inheritdoc} + */ + public function unserializeValue(string $serializedValue): mixed + { + return CarbonImmutable::createFromFormat(self::FORMAT, $serializedValue); + } + + public function getNumericValue(mixed $value): null|int|float + { + return $value instanceof DateTimeInterface + ? $value->getTimestamp() + : null; + } + + public function getStringValue(mixed $value): null|string + { + return $value instanceof DateTimeInterface + ? $value->copy()->setTimezone('UTC')->format(self::FORMAT) + : null; + } + + public function isIdempotent(): bool + { + return true; + } + + public function useHmacVerification(): bool + { + return false; + } +} diff --git a/src/DataType/StringableHandler.php b/src/DataType/StringableHandler.php new file mode 100644 index 0000000..0812c67 --- /dev/null +++ b/src/DataType/StringableHandler.php @@ -0,0 +1,52 @@ +cachedValue)) { - $handler = $this->getDataTypeRegistry()->getHandlerForType($this->type); + $type = $this->type; + $value = $this->attributes['value']; + + if (str_starts_with($type, self::ENCRYPTED_PREFIX)) { + $value = $this->getEncrypter()->decrypt($value); + $type = substr($this->type, strlen(self::ENCRYPTED_PREFIX)); + } + + $registry = $this->getDataTypeRegistry(); + $handler = $registry->getHandlerForType($type); if ($handler->useHmacVerification()) { - $this->verifyHmac($this->attributes['value'], $this->attributes['hmac']); + $this->verifyHmac($value, $this->attributes['hmac']); } $this->cachedValue = $handler->unserializeValue( - $this->attributes['value'] + $value ); } @@ -124,6 +137,23 @@ public function setValueAttribute(mixed $value): void $this->cachedValue = null; } + public function encrypt(): void + { + if ($this->type === 'null') { + return; + } + + if (str_starts_with($this->type, self::ENCRYPTED_PREFIX)) { + return; + } + + $this->attributes['value'] = $this->getEncrypter() + ->encrypt($this->attributes['value']); + $this->type = self::ENCRYPTED_PREFIX . $this->type; + $this->string_value = null; + $this->numeric_value = null; + } + public function getRawValueAttribute(): string { return $this->getRawValue(); @@ -161,4 +191,9 @@ protected function computeHmac(string $serializedValue): string { return hash_hmac('sha256', $serializedValue, config('app.key')); } + + protected function getEncrypter(): Encrypter + { + return self::$encrypter ?? Crypt::getFacadeRoot(); + } } diff --git a/src/Metable.php b/src/Metable.php index 0378add..17e2550 100644 --- a/src/Metable.php +++ b/src/Metable.php @@ -2,6 +2,7 @@ namespace Plank\Metable; +use Illuminate\Contracts\Database\Eloquent\Castable; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Collection; use Illuminate\Database\Eloquent\Model; @@ -9,6 +10,7 @@ use Illuminate\Database\Query\JoinClause; use Plank\Metable\DataType\HandlerInterface; use Plank\Metable\DataType\Registry; +use Plank\Metable\Exceptions\CastException; /** * Trait for giving Eloquent models the ability to handle Meta. @@ -39,6 +41,8 @@ trait Metable */ private $indexedMetaCollection; + private array $mergedMetaCasts = []; + public function __construct(array $attributes = []) { @@ -77,20 +81,28 @@ public function meta(): MorphMany * @param string $key * @param mixed $value */ - public function setMeta(string $key, mixed $value): void + public function setMeta(string $key, mixed $value, bool $encrypt = false): void { if ($this->hasMeta($key)) { $meta = $this->getMetaRecord($key); - $meta->setAttribute('value', $value); + $meta->setAttribute('value', $this->castMetaValueIfNeeded($key, $value)); + if ($encrypt || $this->hasEncryptedMetaCast($key)) { + $meta->encrypt(); + } $meta->save(); } else { - $meta = $this->makeMeta($key, $value); + $meta = $this->makeMeta($key, $value, $encrypt); $this->meta()->save($meta); $this->meta[] = $meta; $this->indexedMetaCollection[$key] = $meta; } } + public function setMetaEncrypted(string $key, mixed $value): void + { + $this->setMeta($key, $value, true); + } + /** * Add or update many `Meta` values. * @@ -145,7 +157,7 @@ public function syncMeta(iterable $array): void $this->meta()->saveMany($meta); // Update cached relationship. - $collection = $this->makeMeta()->newCollection($meta); + $collection = $this->getMetaInstance()->newCollection($meta); $this->setRelation('meta', $collection); } @@ -160,17 +172,16 @@ public function syncMeta(iterable $array): void public function getMeta(string $key, mixed $default = null): mixed { if ($this->hasMeta($key)) { - $value = $this->getMetaRecord($key)->getAttribute('value'); + return $this->getMetaRecord($key)->getAttribute('value'); } + // If we have only one argument provided (i.e. default is not set) // then we check the model for the defaultMetaValues - elseif (func_num_args() == 1 && $this->hasDefaultMetaValue($key)) { - $value = $this->getDefaultMetaValue($key); - } else { - $value = $default; + if (func_num_args() == 1 && $this->hasDefaultMetaValue($key)) { + return $this->getDefaultMetaValue($key); } - return $this->castMetaValue($key, $value); + return $default; } /** @@ -204,9 +215,9 @@ public function getAllMeta(): \Illuminate\Support\Collection { return collect($this->getAllDefaultMeta())->merge( $this->getMetaCollection()->toBase()->map( - fn(Meta $meta) => $meta->getAttribute('value') + fn (Meta $meta) => $meta->getAttribute('value') ) - )->map(fn(mixed $value, string $key) => $this->castMetaValue($key, $value)); + ); } /** @@ -264,7 +275,7 @@ public function removeManyMeta(array $keys): void public function purgeMeta(): void { $this->meta()->delete(); - $this->setRelation('meta', $this->makeMeta()->newCollection()); + $this->setRelation('meta', $this->getMetaInstance()->newCollection()); } /** @@ -709,6 +720,12 @@ protected function getMetaClassName(): string return config('metable.model', Meta::class); } + protected function getMetaInstance(): Meta + { + $class = $this->getMetaClassName(); + return new $class; + } + /** * Create a new `Meta` record. * @@ -717,17 +734,21 @@ protected function getMetaClassName(): string * * @return Meta */ - protected function makeMeta(string $key = '', mixed $value = ''): Meta - { - $className = $this->getMetaClassName(); - - $meta = new $className([ - 'key' => $key, - 'value' => $value, - ]); + protected function makeMeta( + string $key = null, + mixed $value = null, + bool $encrypt = false + ): Meta { + $meta = $this->getMetaInstance(); + $meta->key = $key; + $meta->value = $this->castMetaValueIfNeeded($key, $value); $meta->metable_type = $this->getMorphClass(); $meta->metable_id = $this->getKey(); + if ($encrypt || $this->hasEncryptedMetaCast($key)) { + $meta->encrypt(); + } + return $meta; } @@ -738,21 +759,128 @@ protected function getAllDefaultMeta(): array : []; } - protected function castMetaValue(string $key, mixed $value): mixed + protected function hasEncryptedMetaCast(string $key): bool { - if (!property_exists($this, 'castsMeta') - || !empty($this->castsMeta[$key]) - ) { + $cast = $this->getCastForMetaKey($key); + return $cast === 'encrypted' + || str_starts_with((string)$cast, 'encrypted:'); + } + + protected function castMetaValueIfNeeded(string $key, mixed $value): mixed + { + $cast = $this->getCastForMetaKey($key); + if ($cast === null || $value === null) { return $value; } - $castKey = "meta.$key"; - $this->casts[$castKey] = $this->castsMeta[$key]; + if ($cast == 'encrypted') { + return $value; + } + + if (str_starts_with($cast, 'encrypted:')) { + $cast = substr($cast, 10); + } + + return $this->castMetaValue($key, $value, $cast); + } + + protected function castMetaValue(string $key, mixed $value, string $cast): mixed + { + if ($cast == 'array' || $cast == 'object') { + $assoc = $cast == 'array'; + if (is_string($value)) { + $value = json_decode($value, $assoc, 512, JSON_THROW_ON_ERROR); + } + return json_decode( + json_encode($value, JSON_THROW_ON_ERROR), + $assoc, + 512, + JSON_THROW_ON_ERROR + ); + } + + if ($cast == 'hashed') { + return $this->castAttributeAsHashedString($key, $value); + } + + if ($cast == 'collection' || str_starts_with($cast, 'collection:')) { + if ($value instanceof \Illuminate\Support\Collection) { + $collection = $value; + } elseif ($value instanceof Model) { + $collection = $value->newCollection([$value]); + } else { + $collection = collect($value); + } + + if (str_starts_with($cast, 'collection:')) { + $class = substr($cast, 11); + $collection->each(function ($item) use ($class): void { + if (!$item instanceof $class) { + throw CastException::invalidClassCast($class, $item); + } + }); + } + + return $collection; + } + + if (class_exists($cast) + && !is_a($cast, Castable::class, true) + && $cast != 'datetime' + ) { + if ($value instanceof $cast) { + return $value; + } + + if (is_a($cast, Model::class, true) + && (is_string($value) || is_int($value)) + ) { + return $cast::find($value); + } + + throw CastException::invalidClassCast($cast, $value); + } + + // leverage Eloquent built-in casting functionality + $castKey = "meta.$key"; + $this->casts[$castKey] = $cast; $value = $this->castAttribute($castKey, $value); + + // cleanup to avoid polluting the model's casts unset($this->casts[$castKey]); + unset($this->attributeCastCache[$castKey]); + unset($this->classCastCache[$castKey]); + return $value; } + protected function getCastForMetaKey(string $key): ?string + { + if (isset($this->mergedMetaCasts[$key])) { + return $this->mergedMetaCasts[$key]; + } + + if (method_exists($this, 'metaCasts')) { + $casts = $this->metaCasts(); + if (isset($casts[$key])) { + return $casts[$key]; + } + } + + if (property_exists($this, 'metaCasts') + && isset($this->metaCasts[$key]) + ) { + return $this->metaCasts[$key]; + } + + return null; + } + + public function mergeMetaCasts(array $casts): void + { + $this->mergedMetaCasts = array_merge($this->mergedMetaCasts, $casts); + } + private function valueToString(mixed $value): string { $stringValue = $this->getHandlerForValue($value)->getStringValue($value); @@ -781,4 +909,18 @@ private function getHandlerForValue(mixed $value): HandlerInterface $registry = app('metable.datatype.registry'); return $registry->getHandlerForValue($value); } + + abstract public function getKey(); + + abstract public function getMorphClass(); + + abstract protected function castAttribute($key, $value); + + abstract public function morphMany($related, $name, $type = null, $id = null, $localKey = null); + + abstract public function load($relations); + + abstract public function relationLoaded($key); + + abstract protected function castAttributeAsHashedString($key, $value); } diff --git a/tests/Integration/DataType/HandlerTest.php b/tests/Integration/DataType/HandlerTest.php index 0a9aa92..8c7539f 100644 --- a/tests/Integration/DataType/HandlerTest.php +++ b/tests/Integration/DataType/HandlerTest.php @@ -4,10 +4,12 @@ use Carbon\Carbon; use Illuminate\Database\Eloquent\Collection; +use Illuminate\Support\Stringable; use Plank\Metable\DataType\ArrayHandler; use Plank\Metable\DataType\BackedEnumHandler; use Plank\Metable\DataType\BooleanHandler; use Plank\Metable\DataType\DateTimeHandler; +use Plank\Metable\DataType\DateTimeImmutableHandler; use Plank\Metable\DataType\FloatHandler; use Plank\Metable\DataType\HandlerInterface; use Plank\Metable\DataType\IntegerHandler; @@ -17,6 +19,7 @@ use Plank\Metable\DataType\ObjectHandler; use Plank\Metable\DataType\SerializableHandler; use Plank\Metable\DataType\SignedSerializeHandler; +use Plank\Metable\DataType\StringableHandler; use Plank\Metable\DataType\StringHandler; use Plank\Metable\DataType\PureEnumHandler; use Plank\Metable\Tests\Mocks\SampleIntBackedEnum; @@ -78,6 +81,17 @@ public static function handlerProvider(): array 'isIdempotent' => true, 'usesHmac' => false, ], + 'datetimeImmutable' => [ + 'handler' => new DateTimeImmutableHandler(), + 'type' => 'datetime_immutable', + 'value' => $datetime->toImmutable(), + 'invalid' => [2017, '2017-01-01'], + 'numericValue' => $timestamp, + 'stringValue' => $dateString, + 'stringValueComplex' => $dateString, + 'isIdempotent' => true, + 'usesHmac' => false, + ], 'float' => [ 'handler' => new FloatHandler(), 'type' => 'float', @@ -261,6 +275,17 @@ public static function handlerProvider(): array 'isIdempotent' => true, 'usesHmac' => false, ], + 'Stringable' => [ + 'handler' => new StringableHandler(), + 'type' => 'stringable', + 'value' => new Stringable('foo'), + 'invalid' => ['foo'], + 'numericValue' => null, + 'stringValue' => 'foo', + 'stringValueComplex' => 'foo', + 'isIdempotent' => true, + 'usesHmac' => false, + ] ]; } diff --git a/tests/Integration/MetaTest.php b/tests/Integration/MetaTest.php index aba386a..e3cfd3c 100644 --- a/tests/Integration/MetaTest.php +++ b/tests/Integration/MetaTest.php @@ -73,6 +73,22 @@ public function test_it_verifies_hmac(): void $meta->getValueAttribute(); } + public function test_it_can_encrypt_its_value(): void + { + $meta = $this->makeMeta(); + $meta->value = 'foo'; + $meta->hmac = $hmac = random_bytes(64); + + $meta->encrypt(); + + $this->assertEquals('foo', $meta->value); + $this->assertNotEquals('foo', $meta->raw_value); + $this->assertEquals($hmac, $meta->hmac); + $this->assertEquals('encrypted:string', $meta->type); + $this->assertNull($meta->string_value); + $this->assertNull($meta->numeric_value); + } + private function makeMeta(array $attributes = []): Meta { return factory(Meta::class)->make($attributes); diff --git a/tests/Integration/MetableTest.php b/tests/Integration/MetableTest.php index 4297d1f..24df438 100644 --- a/tests/Integration/MetableTest.php +++ b/tests/Integration/MetableTest.php @@ -3,10 +3,14 @@ namespace Plank\Metable\Tests\Integration; use Carbon\Carbon; +use Illuminate\Database\Eloquent\Casts\AsStringable; use Illuminate\Database\Eloquent\Collection; +use Illuminate\Support\Facades\Hash; +use Illuminate\Support\Stringable; use Plank\Metable\Meta; use Plank\Metable\Tests\Mocks\SampleMetable; use Plank\Metable\Tests\Mocks\SampleMetableSoftDeletes; +use Plank\Metable\Tests\Mocks\SampleStringBackedEnum; use Plank\Metable\Tests\TestCase; use ReflectionClass; @@ -31,6 +35,26 @@ public function test_it_can_get_and_set_meta_value_by_key(): void $this->assertCount(1, $metable->meta); } + public function test_it_can_set_meta_encrypted(): void + { + $this->useDatabase(); + $metable = $this->createMetable(); + $metable->load('meta'); + $this->assertFalse($metable->hasMeta('foo')); + + $metable->setMeta('foo', 'bar', true); + + $this->assertTrue($metable->hasMeta('foo')); + $this->assertEquals('bar', $metable->getMeta('foo')); + $this->assertEquals('encrypted:string', $metable->getMetaRecord('foo')->type); + + $metable->setMetaEncrypted('baz', [1 => 2]); + + $this->assertTrue($metable->hasMeta('baz')); + $this->assertEquals([1 => 2], $metable->getMeta('baz')); + $this->assertEquals('encrypted:serialized', $metable->getMetaRecord('baz')->type); + } + public function test_it_can_set_many_meta_values_at_once(): void { $this->useDatabase(); @@ -706,6 +730,137 @@ public function test_it_throws_for_param_that_cannot_be_converted_to_numeric(): SampleMetable::whereMetaNumeric('foo', null)->get(); } + public static function castProvider(): array + { + $date = Carbon::now(); + $object = new \stdClass(); + $object->foo = 'bar'; + $model = new SampleMetable(); + $model->id = 99; + $model->exists = true; + $modelCollection = new Collection([$model]); + return [ + 'string - string' => ['string', 'foo', 'foo', 'string'], + 'string - int' => ['string', 123, '123', 'string'], + 'string - float' => ['string', 123.45, '123.45', 'string'], + 'string - true' => ['string', true, '1', 'string'], + 'string - false' => ['string', false, '', 'string'], + 'string - null' => ['string', null, null, 'null'], + 'string - dateTime' => ['string', $date, (string)$date, 'string'], + 'array - array' => ['array', ['foo', 'bar'], ['foo', 'bar'], 'serialized'], + 'array - json' => ['array', json_encode(['foo' => 'bar']), ['foo' => 'bar'], 'serialized'], + 'array - object' => ['array', $object, ['foo' => 'bar'], 'serialized'], + 'array - null' => ['array', null, null, 'null'], + 'boolean - true' => ['boolean', true, true, 'boolean'], + 'boolean - false' => ['boolean', false, false, 'boolean'], + 'boolean - 1' => ['boolean', 1, true, 'boolean'], + 'boolean - 0' => ['boolean', 0, false, 'boolean'], + 'boolean - null' => ['boolean', null, null, 'null'], + 'boolean - string' => ['boolean', 'abc', true, 'boolean'], + 'boolean - empty string' => ['boolean', '', false, 'boolean'], + 'boolean - string 1' => ['boolean', '1', true, 'boolean'], + 'boolean - string 0' => ['boolean', '0', false, 'boolean'], + 'decimal - int' => ['decimal:2', 123, '123.00', 'string'], + 'decimal - float' => ['decimal:2', 123.456, '123.46', 'string'], + 'decimal - string' => ['decimal:2', '123.456', '123.46', 'string'], + 'decimal - null' => ['decimal:2', null, null, 'null'], + 'double - int' => ['double', 123, 123.0, 'float'], + 'double - float' => ['double', 123.456, 123.456, 'float'], + 'double - string' => ['double', '123.456', 123.456, 'float'], + 'double - string int' => ['double', '123', 123.0, 'float'], + 'double - null' => ['double', null, null, 'null'], + 'float - int' => ['float', 123, 123.0, 'float'], + 'float - float' => ['float', 123.456, 123.456, 'float'], + 'float - string' => ['float', '123.456', 123.456, 'float'], + 'float - string int' => ['float', '123', 123.0, 'float'], + 'float - null' => ['float', null, null, 'null'], + 'integer - int' => ['integer', 123, 123, 'integer'], + 'integer - float' => ['integer', 123.456, 123, 'integer'], + 'integer - string' => ['integer', '123', 123, 'integer'], + 'integer - null' => ['integer', null, null, 'null'], + 'object - object' => ['object', $object, $object, 'serialized', false], + 'object - array' => ['object', ['foo' => 'bar'], $object, 'serialized', false], + 'object - json' => ['object', json_encode(['foo' => 'bar']), $object, 'serialized', false], + 'object - null' => ['object', null, null, 'null'], + 'real - int' => ['real', 123, 123.0, 'float'], + 'real - float' => ['real', 123.456, 123.456, 'float'], + 'real - string' => ['real', '123.456', 123.456, 'float'], + 'real - string int' => ['real', '123', 123.0, 'float'], + 'real - null' => ['real', null, null, 'null'], + 'timestamp - dateTime' => ['timestamp', $date, $date->timestamp, 'integer'], + 'timestamp - int' => ['timestamp', 123, 123, 'integer'], + 'timestamp - string' => ['timestamp', '2020-01-01 00:00:00', strtotime('2020-01-01 00:00:00'), 'integer'], + 'timestamp - null' => ['timestamp', null, null, 'null'], + 'date - dateTime' => ['date', $date, $date->copy()->startOfDay(), 'datetime', false], + 'date - string' => ['date', (string)$date, $date->copy()->startOfDay(), 'datetime', false], + 'date - timestamp' => ['date', $date->timestamp, $date->copy()->startOfDay(), 'datetime', false], + 'date - string timestamp' => ['date', (string)$date->timestamp, $date->copy()->startOfDay(), 'datetime', false], + 'date - null' => ['date', null, null, 'null'], + 'datetime - dateTime' => ['datetime', $date, $date, 'datetime', false], + 'datetime - string' => ['datetime', $date->format('Y-m-d H:i:s.uO'), $date, 'datetime', false], + 'datetime - timestamp' => ['datetime', $date->timestamp, $date->copy()->startOfSecond(), 'datetime', false], + 'datetime - string timestamp' => ['datetime', (string)$date->timestamp, $date->copy()->startOfSecond(), 'datetime', false], + 'datetime - null' => ['datetime', null, null, 'null'], + 'immutable_date - dateTime' => ['immutable_date', $date, $date->copy()->startOfDay()->toImmutable(), 'datetime_immutable', false], + 'immutable_date - string' => ['immutable_date', $date->format('Y-m-d H:i:s.uO'), $date->copy()->startOfDay()->toImmutable(), 'datetime_immutable', false], + 'immutable_date - timestamp' => ['immutable_date', $date->timestamp, $date->copy()->startOfDay()->toImmutable(), 'datetime_immutable', false], + 'immutable_date - string timestamp' => ['immutable_date', (string)$date->timestamp, $date->copy()->startOfDay()->toImmutable(), 'datetime_immutable', false], + 'immutable_date - null' => ['immutable_date', null, null, 'null'], + 'immutable_datetime - dateTime' => ['immutable_datetime', $date, $date->toImmutable(), 'datetime_immutable', false], + 'immutable_datetime - string' => ['immutable_datetime', $date->format('Y-m-d H:i:s.uO'), $date->toImmutable(), 'datetime_immutable', false], + 'immutable_datetime - timestamp' => ['immutable_datetime', $date->timestamp, $date->copy()->startOfSecond()->toImmutable(), 'datetime_immutable', false], + 'immutable_datetime - string timestamp' => ['immutable_datetime', (string)$date->timestamp, $date->copy()->startOfSecond()->toImmutable(), 'datetime_immutable', false], + 'immutable_datetime - null' => ['immutable_datetime', null, null, 'null'], + 'hashed - string' => ['hashed', 'foo', fn ($result) => password_verify('foo', $result), 'string'], + 'hashed - int' => ['hashed', 123, fn ($result) => password_verify('123', $result), 'string'], + 'hashed - null' => ['hashed', null, null, 'null'], + 'collection - array' => ['collection', ['foo', 'bar'], collect(['foo', 'bar']), 'serialized', false], + 'collection - eloquent' => ['collection', $model, fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), 'collection', false], + 'collection - eloquent collection' => ['collection', $modelCollection, fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), 'collection', false], + 'collection - null' => ['collection', null, null, 'null'], + 'stringable - string' => [AsStringable::class, 'foo', new Stringable('foo'), 'stringable', false], + 'stringable - int' => [AsStringable::class, 123, new Stringable('123'), 'stringable', false], + 'stringable - null' => [AsStringable::class, null, null, 'null'], + 'encrypted - string' => ['encrypted', 'foo', 'foo', 'encrypted:string'], + 'encrypted - array' => ['encrypted', ['foo' => 'bar'], ['foo' => 'bar'], 'encrypted:serialized'], + 'encrypted - null' => ['encrypted', null, null, 'null'], + 'encrypted:collection - array' => ['encrypted:collection', ['foo', 'bar'], collect(['foo', 'bar']), 'encrypted:serialized', false], + 'encrypted:collection - eloquent' => ['encrypted:collection', $model, fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), 'encrypted:collection', false], + 'encrypted:collection - eloquent collection' => ['encrypted:collection', $modelCollection, fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), 'encrypted:collection', false], + 'encrypted:string - int' => ['encrypted:string', 123, '123', 'encrypted:string'], + ]; + } + + /** @dataProvider castProvider */ + public function test_it_casts_meta_values( + string $cast, + mixed $original, + mixed $expected, + string $expectedHandlerType, + bool $strict = true + ): void { + $this->useDatabase(); + + if ($cast === 'collection' || $cast === 'encrypted:collection') { + $model = new SampleMetable(); + $model->id = 99; + $model->save(); + } + + $key = 'castable'; + $metable = $this->createMetable(); + $metable->mergeMetaCasts([$key => $cast]); + $metable->setMeta($key, $original); + if ($expected instanceof \Closure) { + $this->assertTrue($expected($metable->getMeta($key))); + } elseif ($strict) { + $this->assertSame($expected, $metable->getMeta($key)); + } else { + $this->assertEquals($expected, $metable->getMeta($key)); + } + $this->assertSame($expectedHandlerType, $metable->getMetaRecord($key)->type); + } + public function test_it_can_cast_meta_values(): void { $this->useDatabase(); diff --git a/tests/Mocks/SampleMetable.php b/tests/Mocks/SampleMetable.php index 04aed50..bb26315 100644 --- a/tests/Mocks/SampleMetable.php +++ b/tests/Mocks/SampleMetable.php @@ -14,7 +14,7 @@ class SampleMetable extends Model implements MetableInterface 'foo' => 'bar' ]; - protected $castsMeta = [ + protected $metaCasts = [ 'castable' => 'string', ]; } diff --git a/tests/Mocks/SampleStringBackedEnum.php b/tests/Mocks/SampleStringBackedEnum.php index c8a0fd9..a415980 100644 --- a/tests/Mocks/SampleStringBackedEnum.php +++ b/tests/Mocks/SampleStringBackedEnum.php @@ -6,5 +6,4 @@ enum SampleStringBackedEnum: string { case Alpha = 'alpha'; case Numeric = '1'; - } diff --git a/tests/TestCase.php b/tests/TestCase.php index 2b4b726..e985f3d 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -2,6 +2,7 @@ namespace Plank\Metable\Tests; +use Illuminate\Hashing\HashServiceProvider; use Orchestra\Testbench\TestCase as BaseTestCase; use Plank\Metable\MetableServiceProvider; use Plank\Metable\Tests\Mocks\SampleSerializable; From 99e5219b8c2052afe27e3fb57228fc521779d383 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Wed, 24 Apr 2024 00:06:09 -0400 Subject: [PATCH 32/38] added MetableAttributes trait --- CHANGELOG.md | 4 + UPGRADING.md | 4 + docs/source/handling_meta.rst | 51 ++++++++ src/MetableAttributes.php | 116 ++++++++++++++++++ tests/Integration/MetableAttributesTest.php | 96 +++++++++++++++ tests/Integration/MorphTest.php | 2 +- tests/Mocks/SampleMetable.php | 6 + ...01_000000_create_sample_classes_tables.php | 1 + 8 files changed, 279 insertions(+), 1 deletion(-) create mode 100644 src/MetableAttributes.php create mode 100644 tests/Integration/MetableAttributesTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 15cad40..09d4d06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,10 @@ Version 6 contains a number of changes to improve the security and performance o - A value of `null` is ignored by all cast types. - Added `mergeMetaCasts()` method which can be used to override the defined cast on a meta key at runtime. +### Meta Attributes + +- Added optional trait `MetableAttributes` which can further extend the `Metable` trait allowing access to meta values as model attributes using a `meta_` prefix. This can be useful for type hinting, IDE autocompletion, static analysis, and usage in Blade templates. + ### Meta - Added `$meta->string_value` and `$meta->numeric_value` attributes, which are used for optimizing queries filtering by meta value - Added `$meta->hmac` attribute, which is used by some data type handlers to validate that the payload has not been tampered with. diff --git a/UPGRADING.md b/UPGRADING.md index b9cb473..3e9368a 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -35,6 +35,10 @@ * Review the documentation about which data types can be queried with the various `whereMeta*` and `whereMeta*Numeric` query scopes. If you are querying the serialized `value` column directly, be aware that the formatting of array/object data types may have changed. +### Metable Attributes + +* Optional: if you intend to access meta with property access, add the new `\Plank\Metable\MetableAttributes` traits to your `Metable`. + ## 4.X -> 5.X - New migration file added which adds a new composite unique index to the meta table on `metable_type`, `metable_id`, and `key`. Make sure that you have no duplicate keys for a given entity (previously possible as a race condition) before applying the new migration. diff --git a/docs/source/handling_meta.rst b/docs/source/handling_meta.rst index 2913d3f..1dece1e 100644 --- a/docs/source/handling_meta.rst +++ b/docs/source/handling_meta.rst @@ -254,3 +254,54 @@ You can also instruct your model class to `always` eager load the meta relations protected $with = ['meta']; } + + +Meta As Attributes +------------------ + +If you prefer to access meta as if they were attributes of the model, you can use the ``MetableAttributes`` trait insin addition to the ``Metable`` trait. This will allow you to access meta as if they were attributes of the model by prefixing them with ``meta_``. Meta attributes can be combined with type annotations, casts and/or default values to provide consistent typing. This can be useful for IDE completions and static analysis, as well as for use in Blade templates. + +:: + + 'boolean', + 'published_at' => 'datetime', + 'likes' => 'integer', + ]; + + $defaultMetaValues = [ + 'approved' => false, + 'published_at' => null, + 'likes' => 0, + ]; + + // ... + } + + $page = new Page(); + $page->meta_likes = 5; // equivalent to $page->setMeta('likes', 5); + $page->fill(['meta_approved' => true, 'meta_published_at' => now()]); // equivalent to $page->setManyMeta([...]); + if ($page->meta_likes > 0) {} // equivalent $page->getMeta('likes'); + unset($page->meta_likes); // equivalent to $page->removeMeta('likes'); + + +Most attribute operations will translate meta attributes to their corresponding meta operations. However, the ``getAttributes()`` method will **not** include meta attributes. The ``getMetaAttributes()`` method can be used to retrieve all meta values keyed by their attribute name. + +The ``toArray()`` method will include meta attributes by default. The ``$visible``/``$hidden`` properties of the model will be respected if any meta attributes are listed. The ``makeMetaHidden()`` method can be used to quickly hide all currently assigned meta attributes from the array representation of the model. \ No newline at end of file diff --git a/src/MetableAttributes.php b/src/MetableAttributes.php new file mode 100644 index 0000000..dd309fa --- /dev/null +++ b/src/MetableAttributes.php @@ -0,0 +1,116 @@ +isMetaAttribute($key)) { + return $this->getMeta($this->metaAttributeToKey($key)); + } + + return parent::getAttribute($key); + } + + public function setAttribute($key, $value) + { + if ($this->isMetaAttribute($key)) { + $this->setMeta($this->metaAttributeToKey($key), $value); + return; + } + + parent::setAttribute($key, $value); + } + + public function fill(array $attributes) + { + foreach ($attributes as $key => $value) { + if ($this->isMetaAttribute($key)) { + $this->setMeta($this->metaAttributeToKey($key), $value); + unset($attributes[$key]); + } + } + + parent::fill($attributes); + } + + public function getMetaAttributes(): Collection + { + $attributes = []; + foreach ($this->getAllMeta() as $key => $value) { + $attributes[$this->metaKeyToAttribute($key)] = $value; + } + return collect($attributes); + } + + public function offsetExists($key): bool + { + if ($this->isMetaAttribute($key)) { + return $this->hasMeta($this->metaAttributeToKey($key)); + } + + return parent::isset($key); + } + + public function offsetUnset($key): void + { + if ($this->isMetaAttribute($key)) { + $this->removeMeta($this->metaAttributeToKey($key)); + return; + } + + parent::offsetUnset($key); + } + + protected function isMetaAttribute($key): bool + { + return str_starts_with($key, 'meta_') + && !array_key_exists($key, $this->attributes) + && !array_key_exists($key, $this->casts); + } + + public function toArray() + { + if (property_exists($this, 'includeMetaInArray') + && !$this->includeMetaInArray + ) { + return parent::toArray(); + } + + return array_merge( + parent::toArray(), + $this->getArrayableItems($this->getMetaAttributes()->toArray()) + ); + } + + public function makeMetaHidden(): void + { + $this->hidden = array_merge( + $this->hidden, + array_keys($this->getMetaAttributes()->toArray()) + ); + } + + protected function metaAttributeToKey(string $attribute): string + { + return substr($attribute, 5); + } + + protected function metaKeyToAttribute(string $key): string + { + return 'meta_' . $key; + } + + abstract public function getMeta(string $key, mixed $default = null): mixed; + + abstract public function setMeta(string $key, mixed $value, bool $encrypt = false): void; + + abstract public function removeMeta(string $key): void; + + abstract public function getAllMeta(): \Illuminate\Support\Collection; + + abstract protected function getArrayableItems(array $values); +} diff --git a/tests/Integration/MetableAttributesTest.php b/tests/Integration/MetableAttributesTest.php new file mode 100644 index 0000000..c69cc39 --- /dev/null +++ b/tests/Integration/MetableAttributesTest.php @@ -0,0 +1,96 @@ +useDatabase(); + + $model = $this->createMetable(); + $this->assertFalse($model->hasMeta('var')); + $this->assertFalse(isset($model->meta_var)); + $this->assertNull($model->getAttribute('meta_var')); + + $model->setAttribute('meta_var', 'bar'); + $this->assertEquals('bar', $model->getMeta('var')); + $this->assertTrue(isset($model->meta_var)); + $this->assertEquals('bar', $model->getAttribute('meta_var')); + + $model->meta_var = 'baz'; + $this->assertEquals('baz', $model->getMeta('var')); + $this->assertTrue(isset($model->meta_var)); + $this->assertEquals('baz', $model->getAttribute('meta_var')); + + $model->offsetUnset('meta_var'); + $this->assertFalse($model->hasMeta('var')); + $this->assertFalse(isset($model->meta_var)); + $this->assertNull($model->getAttribute('meta_var')); + + $model->fill(['meta_var' => 'qux']); + $this->assertEquals('qux', $model->getMeta('var')); + $this->assertTrue(isset($model->meta_var)); + $this->assertEquals('qux', $model->getAttribute('meta_var')); + + unset($model['meta_var']); + $this->assertFalse($model->hasMeta('var')); + $this->assertFalse(isset($model->meta_var)); + $this->assertNull($model->getAttribute('meta_var')); + } + + public function test_it_doesnt_overwrite_existing_attributes() + { + $this->useDatabase(); + + $model = $this->createMetable(); + $model->meta_attribute = 'bar'; + $this->assertFalse($model->hasMeta('attribute')); + + $model->setAttribute('meta_attribute', 'baz'); + $this->assertFalse($model->hasMeta('attribute')); + } + + public function test_it_converts_to_array() + { + $this->useDatabase(); + $model = $this->createMetable(); + $model->meta_var = 'foo'; + $model->meta_var2 = 'foo2'; + + $this->assertEquals( + collect([ + 'meta_var' => 'foo', + 'meta_var2' => 'foo2', + 'meta_foo' => 'bar' // default value + ]), + $model->getMetaAttributes() + ); + + $model->makeHidden('meta_var2', 'created_at', 'updated_at', 'meta'); + + $array = $model->toArray(); + $this->assertEquals([ + 'meta_attribute' => '', + 'id' => $model->getKey(), + 'meta_foo' => 'bar', + 'meta_var' => 'foo' + ], $array); + + $model->makeMetaHidden(); + + $array = $model->toArray(); + $this->assertEquals([ + 'meta_attribute' => '', + 'id' => $model->getKey(), + ], $array); + } + + private function createMetable(array $attributes = []): SampleMetable + { + return factory(SampleMetable::class)->create($attributes); + } +} diff --git a/tests/Integration/MorphTest.php b/tests/Integration/MorphTest.php index d99f2fc..b3e41dc 100644 --- a/tests/Integration/MorphTest.php +++ b/tests/Integration/MorphTest.php @@ -49,7 +49,7 @@ public function test_it_can_join_correctly_from_morphed_class(): void $results1 = SampleMorph::orderByMeta('foo', 'asc')->get(); $results2 = Meta::select('metable_type')->get(); - $this->assertCount(3, $results1->pluck('id')->toArray()); + $this->assertEquals([3, 1, 2], $results1->modelKeys()); $this->assertEquals([$class, $class, $class], $results2->pluck('metable_type')->toArray()); } diff --git a/tests/Mocks/SampleMetable.php b/tests/Mocks/SampleMetable.php index bb26315..adf1848 100644 --- a/tests/Mocks/SampleMetable.php +++ b/tests/Mocks/SampleMetable.php @@ -4,11 +4,17 @@ use Illuminate\Database\Eloquent\Model; use Plank\Metable\Metable; +use Plank\Metable\MetableAttributes; use Plank\Metable\MetableInterface; class SampleMetable extends Model implements MetableInterface { use Metable; + use MetableAttributes; + + protected $attributes = [ + 'meta_attribute' => '', + ]; protected $defaultMetaValues = [ 'foo' => 'bar' diff --git a/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php b/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php index 6403a2b..4c6441b 100644 --- a/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php +++ b/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php @@ -15,6 +15,7 @@ public function up() { Schema::create('sample_metables', function (Blueprint $table) { $table->id(); + $table->string('meta_attribute')->nullable(); $table->softDeletes(); $table->timestamps(); }); From 7657070086d4a1b21dcc71df413a1cda60a0c14e Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Thu, 25 Apr 2024 23:37:30 -0400 Subject: [PATCH 33/38] use prefix index on value instead of separate string_value column --- CHANGELOG.md | 22 +++--- UPGRADING.md | 3 +- config/metable.php | 20 ++---- docs/source/datatypes.rst | 16 ++--- docs/source/querying_meta.rst | 3 - ...4_04_14_000000_add_meta_search_columns.php | 48 +++++++++++-- src/DataType/ArrayHandler.php | 18 ----- src/DataType/BackedEnumHandler.php | 10 --- src/DataType/BooleanHandler.php | 5 -- src/DataType/DateTimeHandler.php | 12 ---- src/DataType/DateTimeImmutableHandler.php | 12 ---- src/DataType/FloatHandler.php | 5 -- src/DataType/HandlerInterface.php | 7 -- src/DataType/IntegerHandler.php | 5 -- src/DataType/ModelCollectionHandler.php | 10 --- src/DataType/ModelHandler.php | 10 --- src/DataType/NullHandler.php | 5 -- src/DataType/ObjectHandler.php | 18 ----- src/DataType/PureEnumHandler.php | 10 --- src/DataType/ScalarHandler.php | 5 -- src/DataType/SerializableHandler.php | 18 ----- src/DataType/SignedSerializeHandler.php | 18 ----- src/DataType/StringHandler.php | 9 --- src/DataType/StringableHandler.php | 14 ---- src/Meta.php | 4 +- src/Metable.php | 43 ++++-------- .../Integration/Commands/RefreshMetaTest.php | 7 -- tests/Integration/DataType/HandlerTest.php | 68 ------------------- tests/Integration/MetaTest.php | 1 - tests/Integration/MetableTest.php | 24 +++---- 30 files changed, 95 insertions(+), 355 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09d4d06..680e0c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,19 +25,18 @@ Version 6 contains a number of changes to improve the security and performance o - `ModelHandler` will no longer throw a model not found exception if the model no longer exists. Instead, the meta value will return `null`. This is more in line with the existing behavior of the `ModelCollectionHandler`. - `ModelCollectionHandler` will now validate that the encoded collection class is a valid Eloquent collection before attempting to instantiate it during unserialization. If the class is invalid, an instance of `Illuminate\Database\Eloquent\Collection` will be used instead. - `ModelCollectionHandler` will now validate that the encoded class of each entry is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, that entry in the collection will be omitted. -- Added `getStringValue(): ?string` and `getNumericValue(): null|int|float` methods to `HandlerInterface` which should convert the original value into a format that can be indexed, if possible. -- Added `isIdempotent(): bool` method to `HandlerInterface` which should indicate whether multiple calls to the `serialize()` method with the same value will produce the same serialized output. This is used to determine if the complete serialized value can be used when searching for meta values. +- Added `getNumericValue(): null|int|float` method to `HandlerInterface` which should convert the original value into a numeric format for indexing, if relevant for the data type. - Added `useHmacVerification(): bool` method to `HandlerInterface` which should indicate whether the integrity of the serialized data should be verified with an HMAC. ### New Commands - Added `metable:refresh` artisan command which will decode and re-encode all meta values in the database. This is useful if you have changed the data type handlers and need to update the serialized data and indexes in the database. -### Searching Metables By Meta Value +### Efficient Value Search -- the Metable `whereMeta()`, `whereMetaIn()`, and `orderByMeta()` query scopes will now scan the indexed `string_value` column instead of the serialized `value` column. This greatly improves performance when searching for meta values against larger datasets. +- The Metable `whereMeta()`, `whereMetaIn()`, and `orderByMeta()` query scopes can now leverage a prefix index on the ``meta.value`` column. This greatly improves performance when searching for meta values against larger datasets when using applicable operators, e.g. `=`, `%`, `>`, `>=`, `<`, `<=`, `<>`, `LIKE` (no leading wildcard). - `whereMetaNumeric()` and `orderByMetaNumeric()` query scopes will now scan the indexed `numeric_value` column instead of the serialized `value` column. This greatly improves performance when searching for meta values against larger datasets. -- `whereMetaNumeric()` query scope will now accept a value of any type. It will be converted to an integer or float by the handler. This is more consistent with the behaviour of the other query scopes. +- `whereMetaNumeric()` query scope will now accept a value of any type. It will be converted to an integer or float by the handler. This is more consistent with the behaviour of the other query scopes. - Added additional query scopes to more easily search meta values based on different criteria: - `whereMetaInNumeric()` - `whereMetaNotIn()` @@ -48,21 +47,24 @@ Version 6 contains a number of changes to improve the security and performance o - `whereMetaNotBetweenNumeric()` - `whereMetaIsNull()` - `whereMetaIsModel()` -- If the data type handlers cannot convert the search value provided to a whereMeta* query scope to a string or numeric value (as appropriate for the method), then an exception will be thrown. +- If the data type handlers cannot convert the search value provided to a ``whereMeta*Numeric()`` query scope to a numeric value, then an exception will be thrown. ### Metable Casting - Added support for casting meta values to specific types by defining the `$castMeta` property or `castMeta(): array` method on the model. This is similar to the `casts` property of Eloquent Models used for attributes. All cast types supported by Eloquent Models are also available for Meta values.Values will be cast before values are stored in the database to ensure that they are indexed consistently -- the `encrypted:` cast prefix can be combined with any other cast type to cast to the desired type before the value is encrypted to be stored it in the database. Encrypted values are not searchable. -- A value of `null` is ignored by all cast types. - Added `mergeMetaCasts()` method which can be used to override the defined cast on a meta key at runtime. -### Meta Attributes +### Encrypt Meta + +- Added the `setMetaEncrypted()` method which will encrypt data before storing it in the database and decrypt it when retrieving it. This is useful for storing sensitive data in the meta table. +- prefixing a meta cast with `encrypted:` will automatically encrypt all values for that meta key. + +### Metable Attributes - Added optional trait `MetableAttributes` which can further extend the `Metable` trait allowing access to meta values as model attributes using a `meta_` prefix. This can be useful for type hinting, IDE autocompletion, static analysis, and usage in Blade templates. ### Meta -- Added `$meta->string_value` and `$meta->numeric_value` attributes, which are used for optimizing queries filtering by meta value +- Added `$meta->numeric_value` attributes, which are used for optimizing queries filtering by meta value - Added `$meta->hmac` attribute, which is used by some data type handlers to validate that the payload has not been tampered with. - Added `$meta->raw_value` virtual attribute, which exposes the raw serialized value of the meta key. This is useful for debugging purposes. - Added `encrypt()` method, used internally by the `Metable::setMetaEncrypted()` method diff --git a/UPGRADING.md b/UPGRADING.md index 3e9368a..4529eaa 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -22,8 +22,7 @@ ### Handlers * If you have any custom data types, you will need to implement the new methods from the `HandlerInterface`: - * `getStringValue(): ?string` and `getNumericValue(): null|int|float`: These are used to populate the new indexed search columns. You may return `null` if the value cannot be converted into the specified format or does not need to be searchable. - * `isIdempotent(): bool`: This method should indicate whether multiple calls to the `serialize()` method with the same value will produce the same serialized output. This is used to determine if the complete serialized value can be used when searching for meta values. + * `getNumericValue(): null|int|float`: used to populate the new indexed numeric search column. You may return `null` if the value cannot be converted into a meaningful numeric value or does not need to be searchable. * `useHmacVerification(): bool`: if the integrity of the serialized data should be verified with a HMAC, return `true`. If unserializing this data type is safe without HMAC verification, you may return `false`. ### Update Existing Data diff --git a/config/metable.php b/config/metable.php index 7385758..9cd43ea 100644 --- a/config/metable.php +++ b/config/metable.php @@ -73,21 +73,13 @@ ], /** - * Whether to index complex data types (arrays, objects, etc). - * If enabled the value will be serialized and the first 255 characters will be indexed. - * This allows for using whereMeta*() query scopes on serialized values, but may have - * performance and disk usage implications for large data sets. + * Number of bytes of the to index for strings + * This value is used to determine the length of the prefix index on the value column in the database. + * Higher values allow for better precision when querying, but will use more disk space in the database. * - * If you do not intend to query meta values containing complex data types, you should leave this disabled. - * If you change this value, it may be necessary to refresh the meta table with the `artisan metable:refresh` command. - */ - 'indexComplexDataTypes' => false, - - /** - * Number of bytes to index for strings and complex data types. - * This value is used to determine the length of the index column in the database. - * Higher values allow for better precision when querying, - * but will use more disk space in the database. + * Prefix index is only supported on the 'mysql', 'mariadb', 'pgsql', and 'sqlite' database drivers. + * + * Set to 0 before running the migration to disable the index. */ 'stringValueIndexLength' => 255, ]; diff --git a/docs/source/datatypes.rst b/docs/source/datatypes.rst index 9d37044..c3b64eb 100644 --- a/docs/source/datatypes.rst +++ b/docs/source/datatypes.rst @@ -74,7 +74,7 @@ String ^^^^^^^^ +----------------------+-----+ | Handler | ``\Plank\Metable\DataType\StringHandler`` | -| String Query Scopes | Yes, first ``metable.stringValueIndexLength`` characters indexed | +| String Query Scopes | Yes | | Numeric Query Scopes | if string is numeric | | Other Query Scopes | | +----------------------+-----+ @@ -126,7 +126,7 @@ Eloquent Collections +----------------------+-----+ | Handler | ``\Plank\Metable\DataType\ModelCollectionHandler`` | -| String Query Scopes | No | +| String Query Scopes | Yes | | Numeric Query Scopes | No | | Other Query Scopes | | +----------------------+-----+ @@ -145,7 +145,7 @@ DateTime & Carbon ^^^^^^^^^^^^^^^^^^ +----------------------+-----+ | Handler | ``\Plank\Metable\DataType\DateTimeHandler`` | -| String Query Scopes | Yes (UTC format) | +| String Query Scopes | Yes | | Numeric Query Scopes | Yes (timestamp) | | Other Query Scopes | | +----------------------+-----+ @@ -162,7 +162,7 @@ DateTimeImmutable & CarbonImmutable +----------------------+-----+ | Handler | ``\Plank\Metable\DataType\DateTimeImmutableHandler`` | -| String Query Scopes | Yes (UTC format) | +| String Query Scopes | Yes | | Numeric Query Scopes | Yes (timestamp) | | Other Query Scopes | | +----------------------+-----+ @@ -209,7 +209,7 @@ Objects and Arrays +----------------------+-----+ | Handler | ``\Plank\Metable\DataType\SignedSerializeHandler`` | -| String Query Scopes | if ``metable.indexComplexDataTypes`` is enabled | +| String Query Scopes | Yes | | Numeric Query Scopes | No | | Other Query Scopes | | +----------------------+-----+ @@ -236,7 +236,7 @@ Array +----------------------+-----+ | Handler | ``\Plank\Metable\DataType\ArrayHandler`` | -| String Query Scopes | if ``metable.indexComplexDataTypes`` is enabled | +| String Query Scopes | Yes | | Numeric Query Scopes | No | | Other Query Scopes | | +----------------------+-----+ @@ -268,7 +268,7 @@ Serializable +----------------------+-----+ | Handler | ``\Plank\Metable\DataType\ArrayHandler`` | -| String Query Scopes | if ``metable.indexComplexDataTypes`` is enabled | +| String Query Scopes | Yes | | Numeric Query Scopes | No | | Other Query Scopes | | +----------------------+-----+ @@ -296,7 +296,7 @@ Plain Objects +----------------------+-----+ | Handler | ``\Plank\Metable\DataType\ArrayHandler`` | -| String Query Scopes | if ``metable.indexComplexDataTypes`` is enabled | +| String Query Scopes | Yes | | Numeric Query Scopes | No | | Other Query Scopes | | +----------------------+-----+ diff --git a/docs/source/querying_meta.rst b/docs/source/querying_meta.rst index dbe4844..e3a6755 100644 --- a/docs/source/querying_meta.rst +++ b/docs/source/querying_meta.rst @@ -43,9 +43,6 @@ String Value Query Scopes All string-based query scopes using lexicographic comparison to look up values. This means that the values are compared alphabetically as strings. This can lead to unexpected results when comparing numbers, e.g. ``'11'`` is greater than ``'100'``. -By default, only the first 255 characters of a string are indexed (can be adjusted with the ``metable.stringValueIndexLength`` config). When querying by longer values, characters exceeding the limit will be ignored when determining if the criteria matches. The ``whereMeta()`` method will attempt to work around this by comparing the entire serialized value after the results have been filtered by the indexed portion (other query scopes will not do this). - - The ``whereMeta()`` method can be used to compare the value using any of the operators accepted by the Laravel query builder's ``where()`` method. :: diff --git a/migrations/2024_04_14_000000_add_meta_search_columns.php b/migrations/2024_04_14_000000_add_meta_search_columns.php index 3dc2c35..4de70d0 100644 --- a/migrations/2024_04_14_000000_add_meta_search_columns.php +++ b/migrations/2024_04_14_000000_add_meta_search_columns.php @@ -16,15 +16,26 @@ public function up() { Schema::table('meta', function (Blueprint $table) { $table->decimal('numeric_value', 36, 16)->nullable(); - $table->string( - 'string_value', - config('metable.stringValueIndexLength', 255) - )->nullable(); $table->string('hmac', 64)->nullable(); + $table->dropIndex(['key', 'metable_type']); $table->dropIndex(['key']); $table->index(['key', 'metable_type', 'numeric_value']); - $table->index(['key', 'metable_type', 'string_value']); + + $stringIndexLength = (int)config('metable.stringValueIndexLength', 255); + if ($stringIndexLength > 0 && $driver = $this->detectDriverName()) { + if (in_array($driver, ['mysql', 'mariadb'])) { + $table->rawIndex( + "metable_type, key, value($stringIndexLength)", + 'value_string_prefix_index' + ); + } elseif (in_array($driver, ['pgsql', 'sqlite'])) { + $table->rawIndex( + "metable_type, key, substr(value, 1, $stringIndexLength)", + 'value_string_prefix_index' + ); + } + } }); } @@ -36,12 +47,35 @@ public function up() public function down() { Schema::table('meta', function (Blueprint $table) { - $table->dropIndex(['key', 'metable_type', 'string_value']); + $stringIndexLength = (int)config('metable.stringValueIndexLength', 255); + if ($stringIndexLength > 0 + && in_array($this->detectDriverName(), ['mysql', 'mariadb', 'pgsql', 'sqlite']) + ) { + $table->dropIndex('value_string_prefix_index'); + } + $table->dropIndex(['key', 'metable_type', 'numeric_value']); $table->index(['key']); $table->index(['key', 'metable_type']); - $table->dropColumn('string_value'); + $table->dropColumn('hmac'); $table->dropColumn('numeric_value'); }); } + + private function detectDriverName(): ?string + { + /** @var \Illuminate\Database\Migrations\Migrator $migrator */ + $migrator = app('migrator'); + $repository = $migrator->getRepository(); + + if (method_exists($repository, 'getConnectionResolver')) { + $resolver = $repository->getConnectionResolver(); + } else { + $resolver = DB::getFacadeRoot(); + } + + return $resolver->connection( + $this->getConnection() ?? $migrator->getConnection() + )->getDriverName(); + } } diff --git a/src/DataType/ArrayHandler.php b/src/DataType/ArrayHandler.php index d632c07..9069b67 100644 --- a/src/DataType/ArrayHandler.php +++ b/src/DataType/ArrayHandler.php @@ -50,24 +50,6 @@ public function getNumericValue(mixed $value): null|int|float return null; } - public function getStringValue(mixed $value): null|string - { - if (!config('metable.indexComplexDataTypes', false)) { - return null; - } - - return substr( - json_encode($value, JSON_THROW_ON_ERROR), - 0, - config('metable.stringValueIndexLength', 255) - ); - } - - public function isIdempotent(): bool - { - return true; - } - public function useHmacVerification(): bool { return false; diff --git a/src/DataType/BackedEnumHandler.php b/src/DataType/BackedEnumHandler.php index 1517b7f..34c7229 100644 --- a/src/DataType/BackedEnumHandler.php +++ b/src/DataType/BackedEnumHandler.php @@ -44,16 +44,6 @@ public function getNumericValue(mixed $value): null|int|float return null; } - public function getStringValue(mixed $value): null|string - { - return (string)$value->value; - } - - public function isIdempotent(): bool - { - return true; - } - public function useHmacVerification(): bool { return false; diff --git a/src/DataType/BooleanHandler.php b/src/DataType/BooleanHandler.php index 700ab18..30a2b4b 100644 --- a/src/DataType/BooleanHandler.php +++ b/src/DataType/BooleanHandler.php @@ -16,9 +16,4 @@ public function getNumericValue(mixed $value): null|int|float { return $value ? 1 : 0; } - - public function getStringValue(mixed $value): null|string - { - return $value ? 'true' : 'false'; - } } diff --git a/src/DataType/DateTimeHandler.php b/src/DataType/DateTimeHandler.php index 92ee322..2495f74 100644 --- a/src/DataType/DateTimeHandler.php +++ b/src/DataType/DateTimeHandler.php @@ -56,18 +56,6 @@ public function getNumericValue(mixed $value): null|int|float : null; } - public function getStringValue(mixed $value): null|string - { - return $value instanceof DateTimeInterface - ? $value->copy()->setTimezone('UTC')->format(self::FORMAT) - : null; - } - - public function isIdempotent(): bool - { - return true; - } - public function useHmacVerification(): bool { return false; diff --git a/src/DataType/DateTimeImmutableHandler.php b/src/DataType/DateTimeImmutableHandler.php index c18554b..bbddd17 100644 --- a/src/DataType/DateTimeImmutableHandler.php +++ b/src/DataType/DateTimeImmutableHandler.php @@ -57,18 +57,6 @@ public function getNumericValue(mixed $value): null|int|float : null; } - public function getStringValue(mixed $value): null|string - { - return $value instanceof DateTimeInterface - ? $value->copy()->setTimezone('UTC')->format(self::FORMAT) - : null; - } - - public function isIdempotent(): bool - { - return true; - } - public function useHmacVerification(): bool { return false; diff --git a/src/DataType/FloatHandler.php b/src/DataType/FloatHandler.php index a7d7ac2..c91327e 100644 --- a/src/DataType/FloatHandler.php +++ b/src/DataType/FloatHandler.php @@ -24,9 +24,4 @@ public function getNumericValue(mixed $value): null|int|float { return $value; } - - public function getStringValue(mixed $value): null|string - { - return (string) $value; - } } diff --git a/src/DataType/HandlerInterface.php b/src/DataType/HandlerInterface.php index b15b561..729333c 100644 --- a/src/DataType/HandlerInterface.php +++ b/src/DataType/HandlerInterface.php @@ -34,8 +34,6 @@ public function serializeValue(mixed $value): string; public function getNumericValue(mixed $value): null|int|float; - public function getStringValue(mixed $value): null|string; - /** * Convert a serialized string back to its original value. * @@ -45,10 +43,5 @@ public function getStringValue(mixed $value): null|string; */ public function unserializeValue(string $serializedValue): mixed; - /** - * Indicate whether multiple serializations of the same value will produce the same result. - */ - public function isIdempotent(): bool; - public function useHmacVerification(): bool; } diff --git a/src/DataType/IntegerHandler.php b/src/DataType/IntegerHandler.php index cf3cae1..237b9e8 100644 --- a/src/DataType/IntegerHandler.php +++ b/src/DataType/IntegerHandler.php @@ -16,9 +16,4 @@ public function getNumericValue(mixed $value): null|int|float { return $value; } - - public function getStringValue(mixed $value): null|string - { - return (string) $value; - } } diff --git a/src/DataType/ModelCollectionHandler.php b/src/DataType/ModelCollectionHandler.php index 164aea4..afb8358 100644 --- a/src/DataType/ModelCollectionHandler.php +++ b/src/DataType/ModelCollectionHandler.php @@ -123,16 +123,6 @@ public function getNumericValue(mixed $value): null|int|float return null; } - public function getStringValue(mixed $value): null|string - { - return null; - } - - public function isIdempotent(): bool - { - return true; - } - public function useHmacVerification(): bool { return false; diff --git a/src/DataType/ModelHandler.php b/src/DataType/ModelHandler.php index 990f9cb..3264415 100644 --- a/src/DataType/ModelHandler.php +++ b/src/DataType/ModelHandler.php @@ -62,16 +62,6 @@ public function getNumericValue(mixed $value): null|int|float return null; } - public function getStringValue(mixed $value): null|string - { - return $this->serializeValue($value); - } - - public function isIdempotent(): bool - { - return true; - } - public function useHmacVerification(): bool { return false; diff --git a/src/DataType/NullHandler.php b/src/DataType/NullHandler.php index 1feb3ed..318f6a6 100644 --- a/src/DataType/NullHandler.php +++ b/src/DataType/NullHandler.php @@ -24,9 +24,4 @@ public function getNumericValue(mixed $value): null|int|float { return null; } - - public function getStringValue(mixed $value): null|string - { - return null; - } } diff --git a/src/DataType/ObjectHandler.php b/src/DataType/ObjectHandler.php index aba893f..0085dfa 100644 --- a/src/DataType/ObjectHandler.php +++ b/src/DataType/ObjectHandler.php @@ -45,24 +45,6 @@ public function getNumericValue(mixed $value): null|int|float return null; } - public function getStringValue(mixed $value): null|string - { - if (!config('metable.indexComplexDataTypes', false)) { - return null; - } - - return substr( - json_encode($value, JSON_THROW_ON_ERROR), - 0, - config('metable.stringValueIndexLength', 255) - ); - } - - public function isIdempotent(): bool - { - return true; - } - public function useHmacVerification(): bool { return false; diff --git a/src/DataType/PureEnumHandler.php b/src/DataType/PureEnumHandler.php index 54a6d07..4f57e7c 100644 --- a/src/DataType/PureEnumHandler.php +++ b/src/DataType/PureEnumHandler.php @@ -41,16 +41,6 @@ public function getNumericValue(mixed $value): null|int|float return null; } - public function getStringValue(mixed $value): null|string - { - return $value->name; - } - - public function isIdempotent(): bool - { - return true; - } - public function useHmacVerification(): bool { return false; diff --git a/src/DataType/ScalarHandler.php b/src/DataType/ScalarHandler.php index a362a1c..bd6c0f5 100644 --- a/src/DataType/ScalarHandler.php +++ b/src/DataType/ScalarHandler.php @@ -50,11 +50,6 @@ public function unserializeValue(string $serializedValue): mixed return $serializedValue; } - public function isIdempotent(): bool - { - return true; - } - public function useHmacVerification(): bool { return false; diff --git a/src/DataType/SerializableHandler.php b/src/DataType/SerializableHandler.php index 4dd3698..3f5dbb2 100644 --- a/src/DataType/SerializableHandler.php +++ b/src/DataType/SerializableHandler.php @@ -48,24 +48,6 @@ public function getNumericValue(mixed $value): null|int|float return null; } - public function getStringValue(mixed $value): null|string - { - if (!config('metable.indexComplexDataTypes', false)) { - return null; - } - - return substr( - serialize($value), - 0, - config('metable.stringValueIndexLength', 255) - ); - } - - public function isIdempotent(): bool - { - return true; - } - public function useHmacVerification(): bool { return false; diff --git a/src/DataType/SignedSerializeHandler.php b/src/DataType/SignedSerializeHandler.php index 32c21a6..e0fa8e4 100644 --- a/src/DataType/SignedSerializeHandler.php +++ b/src/DataType/SignedSerializeHandler.php @@ -40,24 +40,6 @@ public function getNumericValue(mixed $value): null|int|float return null; } - public function getStringValue(mixed $value): null|string - { - if (!config('metable.indexComplexDataTypes', false)) { - return null; - } - - return substr( - serialize($value), - 0, - config('metable.stringValueIndexLength', 255) - ); - } - - public function isIdempotent(): bool - { - return true; - } - public function useHmacVerification(): bool { return true; diff --git a/src/DataType/StringHandler.php b/src/DataType/StringHandler.php index 9e1ebc1..5c1012d 100644 --- a/src/DataType/StringHandler.php +++ b/src/DataType/StringHandler.php @@ -19,13 +19,4 @@ public function getNumericValue(mixed $value): null|int|float } return null; } - - public function getStringValue(mixed $value): null|string - { - return substr( - $value, - 0, - config('metable.stringValueIndexLength', 255) - ); - } } diff --git a/src/DataType/StringableHandler.php b/src/DataType/StringableHandler.php index 0812c67..5e12858 100644 --- a/src/DataType/StringableHandler.php +++ b/src/DataType/StringableHandler.php @@ -26,25 +26,11 @@ public function getNumericValue(mixed $value): null|int|float return is_numeric((string)$value) ? (float)(string)$value : null; } - public function getStringValue(mixed $value): null|string - { - return substr( - (string)$value, - 0, - config('metable.stringValueIndexLength', 255) - ); - } - public function unserializeValue(string $serializedValue): mixed { return new Stringable($serializedValue); } - public function isIdempotent(): bool - { - return true; - } - public function useHmacVerification(): bool { return false; diff --git a/src/Meta.php b/src/Meta.php index 80be87e..a2a57a7 100644 --- a/src/Meta.php +++ b/src/Meta.php @@ -128,11 +128,10 @@ public function setValueAttribute(mixed $value): void $handler = $registry->getHandlerForType($this->attributes['type']); $this->attributes['value'] = $handler->serializeValue($value); + $this->attributes['numeric_value'] = $handler->getNumericValue($value); $this->attributes['hmac'] = $handler->useHmacVerification() ? $this->computeHmac($this->attributes['value']) : null; - $this->attributes['string_value'] = $handler->getStringValue($value); - $this->attributes['numeric_value'] = $handler->getNumericValue($value); $this->cachedValue = null; } @@ -150,7 +149,6 @@ public function encrypt(): void $this->attributes['value'] = $this->getEncrypter() ->encrypt($this->attributes['value']); $this->type = self::ENCRYPTED_PREFIX . $this->type; - $this->string_value = null; $this->numeric_value = null; } diff --git a/src/Metable.php b/src/Metable.php index 17e2550..e4a6903 100644 --- a/src/Metable.php +++ b/src/Metable.php @@ -129,7 +129,7 @@ public function setManyMeta(array $metaDictionary): void return $model->getAttributesForInsert(); })->all(), ['metable_type', 'metable_id', 'key'], - ['type', 'value', 'string_value', 'numeric_value', 'hmac'] + ['type', 'value', 'numeric_value', 'hmac'] ); if ($needReload) { @@ -375,19 +375,14 @@ public function scopeWhereMeta( 'meta', function (Builder $q) use ($key, $operator, $stringValue, $value) { $q->where('key', $key); - $q->where('string_value', $operator, $stringValue); - - // If the value is a string and the string value is at the maximum length, - // we can optimize the query by looking up using the index first - // then compare the serialized value (not indexed) afterward to ensure correctness. - if (strlen($stringValue) >= config( - 'metable.stringValueIndexLength', - 255 - )) { - $handler = $this->getHandlerForValue($value); - if ($handler->isIdempotent()) { - $q->where('value', $operator, $handler->serializeValue($value)); - } + $q->where('value', $operator, $stringValue); + + // null and empty string look the same in the database, + // use the type column to differentiate. + if ($value === null) { + $q->where('type', 'null'); + } elseif ($value === '') { + $q->where('type', '!=', 'null'); } } ); @@ -438,7 +433,7 @@ public function scopeWhereMetaBetween( 'meta', function (Builder $q) use ($key, $min, $max, $not) { $q->where('key', $key); - $q->whereBetween('string_value', [$min, $max], 'and', $not); + $q->whereBetween('value', [$min, $max], 'and', $not); } ); } @@ -485,11 +480,7 @@ public function scopeWhereMetaNotBetweenNumeric( */ public function scopeWhereMetaIsNull(Builder $q, string $key): void { - $q->whereHas('meta', function (Builder $q) use ($key) { - $q->where('key', $key); - $q->whereNull('string_value'); - $q->where('type', 'null'); - }); + $this->scopeWhereMeta($q, $key, null); } public function scopeWhereMetaIsModel( @@ -534,7 +525,7 @@ public function scopeWhereMetaIn( $q->whereHas('meta', function (Builder $q) use ($key, $values, $not) { $q->where('key', $key); - $q->whereIn('string_value', $values, 'and', $not); + $q->whereIn('value', $values, 'and', $not); }); } @@ -587,7 +578,7 @@ public function scopeOrderByMeta( bool $strict = false ): void { $table = $this->joinMetaTable($q, $key, $strict ? 'inner' : 'left'); - $q->orderBy("{$table}.string_value", $direction); + $q->orderBy("{$table}.value", $direction); } /** @@ -883,13 +874,7 @@ public function mergeMetaCasts(array $casts): void private function valueToString(mixed $value): string { - $stringValue = $this->getHandlerForValue($value)->getStringValue($value); - - if ($stringValue === null) { - throw new \InvalidArgumentException('Cannot convert to a numeric value'); - } - - return $stringValue; + return $this->getHandlerForValue($value)->serializeValue($value); } private function valueToNumeric(mixed $value): int|float diff --git a/tests/Integration/Commands/RefreshMetaTest.php b/tests/Integration/Commands/RefreshMetaTest.php index 8b34182..6bbb71b 100644 --- a/tests/Integration/Commands/RefreshMetaTest.php +++ b/tests/Integration/Commands/RefreshMetaTest.php @@ -21,7 +21,6 @@ public function test_it_refreshes_all_meta_values(): void SignedSerializeHandler::class, ArrayHandler::class, ]); - config()->set('metable.indexComplexDataTypes', true); config()->set('metable.refreshPageSize', 2); @@ -34,7 +33,6 @@ public function test_it_refreshes_all_meta_values(): void 'type' => 'array', 'key' => 'foo', 'value' => json_encode($complexValue), - 'string_value' => null, 'numeric_value' => null, ], [ @@ -43,7 +41,6 @@ public function test_it_refreshes_all_meta_values(): void 'type' => 'string', 'key' => 'bar', 'value' => 'blah', - 'string_value' => null, 'numeric_value' => null, ], [ @@ -52,7 +49,6 @@ public function test_it_refreshes_all_meta_values(): void 'type' => 'datetime', 'key' => 'baz', 'value' => '2020-01-01 00:00:00.000000+0000', - 'string_value' => null, 'numeric_value' => null, ], ]); @@ -68,17 +64,14 @@ public function test_it_refreshes_all_meta_values(): void $this->assertEquals('serialized', $result[0]->type); $this->assertEquals($complexValue, unserialize($result[0]->value)); - $this->assertEquals(serialize($complexValue), $result[0]->string_value); $this->assertNull($result[0]->numeric_value); $this->assertEquals('string', $result[1]->type); $this->assertEquals('blah', $result[1]->value); - $this->assertEquals('blah', $result[1]->string_value); $this->assertNull($result[1]->numeric_value); $this->assertEquals('datetime', $result[2]->type); $this->assertEquals('2020-01-01 00:00:00.000000+0000', $result[2]->value); - $this->assertEquals('2020-01-01 00:00:00.000000+0000', $result[2]->string_value); $this->assertEquals(1577836800, $result[2]->numeric_value); } } diff --git a/tests/Integration/DataType/HandlerTest.php b/tests/Integration/DataType/HandlerTest.php index 8c7539f..480d7bd 100644 --- a/tests/Integration/DataType/HandlerTest.php +++ b/tests/Integration/DataType/HandlerTest.php @@ -54,9 +54,6 @@ public static function handlerProvider(): array 'value' => ['foo' => ['bar'], 'baz'], 'invalid' => [new stdClass()], 'numericValue' => null, - 'stringValue' => null, - 'stringValueComplex' => json_encode(['foo' => ['bar'], 'baz']), - 'isIdempotent' => true, 'usesHmac' => false, ], 'boolean' => [ @@ -65,9 +62,6 @@ public static function handlerProvider(): array 'value' => true, 'invalid' => [1, 0, '', [], null], 'numericValue' => 1, - 'stringValue' => 'true', - 'stringValueComplex' => 'true', - 'isIdempotent' => true, 'usesHmac' => false, ], 'datetime' => [ @@ -76,9 +70,6 @@ public static function handlerProvider(): array 'value' => $datetime, 'invalid' => [2017, '2017-01-01'], 'numericValue' => $timestamp, - 'stringValue' => $dateString, - 'stringValueComplex' => $dateString, - 'isIdempotent' => true, 'usesHmac' => false, ], 'datetimeImmutable' => [ @@ -87,9 +78,6 @@ public static function handlerProvider(): array 'value' => $datetime->toImmutable(), 'invalid' => [2017, '2017-01-01'], 'numericValue' => $timestamp, - 'stringValue' => $dateString, - 'stringValueComplex' => $dateString, - 'isIdempotent' => true, 'usesHmac' => false, ], 'float' => [ @@ -98,9 +86,6 @@ public static function handlerProvider(): array 'value' => 1.1, 'invalid' => ['1.1', 1], 'numericValue' => 1.1, - 'stringValue' => '1.1', - 'stringValueComplex' => '1.1', - 'isIdempotent' => true, 'usesHmac' => false, ], 'integer' => [ @@ -109,9 +94,6 @@ public static function handlerProvider(): array 'value' => 3, 'invalid' => [1.1, '1'], 'numericValue' => 3, - 'stringValue' => '3', - 'stringValueComplex' => '3', - 'isIdempotent' => true, 'usesHmac' => false, ], 'model' => [ @@ -120,9 +102,6 @@ public static function handlerProvider(): array 'value' => $model, 'invalid' => [new stdClass()], 'numericValue' => null, - 'stringValue' => SampleMetable::class, - 'stringValueComplex' => SampleMetable::class, - 'isIdempotent' => true, 'usesHmac' => false, ], 'model collection' => [ @@ -131,9 +110,6 @@ public static function handlerProvider(): array 'value' => new Collection([new SampleMetable()]), 'invalid' => [collect()], 'numericValue' => null, - 'stringValue' => null, - 'stringValueComplex' => null, - 'isIdempotent' => true, 'usesHmac' => false, ], 'null' => [ @@ -142,9 +118,6 @@ public static function handlerProvider(): array 'value' => null, 'invalid' => [0, '', 'null', [], false], 'numericValue' => null, - 'stringValue' => null, - 'stringValueComplex' => null, - 'isIdempotent' => true, 'usesHmac' => false, ], 'object' => [ @@ -153,9 +126,6 @@ public static function handlerProvider(): array 'value' => $object, 'invalid' => [[]], 'numericValue' => null, - 'stringValue' => null, - 'stringValueComplex' => json_encode($object), - 'isIdempotent' => true, 'usesHmac' => false, ], 'signedSerialize' => [ @@ -164,9 +134,6 @@ public static function handlerProvider(): array 'value' => ['foo' => 'bar', 'baz' => [3]], 'invalid' => [self::$resource], 'numericValue' => null, - 'stringValue' => null, - 'stringValueComplex' => serialize(['foo' => 'bar', 'baz' => [3]]), - 'isIdempotent' => true, 'usesHmac' => true, ], 'serializable' => [ @@ -175,9 +142,6 @@ public static function handlerProvider(): array 'value' => new SampleSerializable(['foo' => 'bar']), 'invalid' => [], 'numericValue' => null, - 'stringValue' => null, - 'stringValueComplex' => serialize(new SampleSerializable(['foo' => 'bar'])), - 'isIdempotent' => true, 'usesHmac' => false, ], 'string' => [ @@ -186,9 +150,6 @@ public static function handlerProvider(): array 'value' => 'foo', 'invalid' => [1, 1.1], 'numericValue' => null, - 'stringValue' => 'foo', - 'stringValueComplex' => 'foo', - 'isIdempotent' => true, 'usesHmac' => false, ], 'long-string' => [ @@ -197,9 +158,6 @@ public static function handlerProvider(): array 'value' => str_repeat('a', 300), 'invalid' => [1, 1.1], 'numericValue' => null, - 'stringValue' => str_repeat('a', 255), - 'stringValueComplex' => str_repeat('a', 255), - 'isIdempotent' => true, 'usesHmac' => false, ], 'numeric-string' => [ @@ -208,9 +166,6 @@ public static function handlerProvider(): array 'value' => '1.2345', 'invalid' => [1, 1.1], 'numericValue' => 1.2345, - 'stringValue' => '1.2345', - 'stringValueComplex' => '1.2345', - 'isIdempotent' => true, 'usesHmac' => false, ], 'unitEnum' => [ @@ -225,9 +180,6 @@ public static function handlerProvider(): array new stdClass() ], 'numericValue' => null, - 'stringValue' => 'Alpha', - 'stringValueComplex' => 'Alpha', - 'isIdempotent' => true, 'usesHmac' => false, ], 'stringBackedEnum' => [ @@ -240,9 +192,6 @@ public static function handlerProvider(): array new stdClass() ], 'numericValue' => null, - 'stringValue' => 'alpha', - 'stringValueComplex' => 'alpha', - 'isIdempotent' => true, 'usesHmac' => false, ], 'numericStringBackedEnum' => [ @@ -255,9 +204,6 @@ public static function handlerProvider(): array new stdClass() ], 'numericValue' => 1, - 'stringValue' => '1', - 'stringValueComplex' => '1', - 'isIdempotent' => true, 'usesHmac' => false, ], 'intBackedEnum' => [ @@ -270,9 +216,6 @@ public static function handlerProvider(): array new stdClass() ], 'numericValue' => 1, - 'stringValue' => '1', - 'stringValueComplex' => '1', - 'isIdempotent' => true, 'usesHmac' => false, ], 'Stringable' => [ @@ -281,9 +224,6 @@ public static function handlerProvider(): array 'value' => new Stringable('foo'), 'invalid' => ['foo'], 'numericValue' => null, - 'stringValue' => 'foo', - 'stringValueComplex' => 'foo', - 'isIdempotent' => true, 'usesHmac' => false, ] ]; @@ -307,9 +247,6 @@ public function test_it_can_verify_and_serialize_data( mixed $value, array $incompatible, null|int|float $numericValue, - ?string $stringValue, - ?string $stringValueComplex, - bool $isIdempotent, bool $usesHmac ): void { $this->assertEquals($type, $handler->getDataType()); @@ -325,10 +262,5 @@ public function test_it_can_verify_and_serialize_data( $this->assertEquals($usesHmac, $handler->useHmacVerification()); $this->assertEquals($value, $unserialized); $this->assertEquals($numericValue, $handler->getNumericValue($value)); - config()->set('metable.indexComplexDataTypes', false); - $this->assertEquals($stringValue, $handler->getStringValue($value)); - config()->set('metable.indexComplexDataTypes', true); - $this->assertEquals($stringValueComplex, $handler->getStringValue($value)); - $this->assertEquals($isIdempotent, $handler->isIdempotent()); } } diff --git a/tests/Integration/MetaTest.php b/tests/Integration/MetaTest.php index e3cfd3c..f913836 100644 --- a/tests/Integration/MetaTest.php +++ b/tests/Integration/MetaTest.php @@ -85,7 +85,6 @@ public function test_it_can_encrypt_its_value(): void $this->assertNotEquals('foo', $meta->raw_value); $this->assertEquals($hmac, $meta->hmac); $this->assertEquals('encrypted:string', $meta->type); - $this->assertNull($meta->string_value); $this->assertNull($meta->numeric_value); } diff --git a/tests/Integration/MetableTest.php b/tests/Integration/MetableTest.php index 24df438..419c8c3 100644 --- a/tests/Integration/MetableTest.php +++ b/tests/Integration/MetableTest.php @@ -5,12 +5,10 @@ use Carbon\Carbon; use Illuminate\Database\Eloquent\Casts\AsStringable; use Illuminate\Database\Eloquent\Collection; -use Illuminate\Support\Facades\Hash; use Illuminate\Support\Stringable; use Plank\Metable\Meta; use Plank\Metable\Tests\Mocks\SampleMetable; use Plank\Metable\Tests\Mocks\SampleMetableSoftDeletes; -use Plank\Metable\Tests\Mocks\SampleStringBackedEnum; use Plank\Metable\Tests\TestCase; use ReflectionClass; @@ -77,7 +75,6 @@ public function test_it_can_set_many_meta_values_at_once(): void $this->assertEquals(33, $metable->getMeta('bar')); $this->assertEquals(['foo', 'bar'], $metable->getMeta('baz')); - $this->assertEquals('bar', $metable->getMetaRecord('foo')->string_value); $this->assertEquals(33, $metable->getMetaRecord('bar')->numeric_value); $this->assertNotEmpty($metable->getMetaRecord('baz')->hmac); } @@ -382,16 +379,27 @@ public function test_it_can_be_queried_by_meta_value(): void $metable->setMeta('foo', 'bar'); $metable->setMeta('datetime', $now); $metable->setMeta('long', str_repeat('a', 300)); + $metable->setMeta('empty', ''); + $metable->setMeta('null', null); $result1 = SampleMetable::whereMeta('foo', 'bar')->first(); $result2 = SampleMetable::whereMeta('foo', 'baz')->first(); $result3 = SampleMetable::whereMeta('datetime', $now)->first(); $result4 = SampleMetable::whereMeta('long', str_repeat('a', 300))->first(); + $result5 = SampleMetable::whereMeta('empty', '')->first(); + $result6 = SampleMetable::whereMeta('empty', null)->first(); + $result7 = SampleMetable::whereMeta('null', '')->first(); + $result8 = SampleMetable::whereMeta('null', null)->first(); + $this->assertEquals($metable->getKey(), $result1->getKey()); $this->assertNull($result2); $this->assertEquals($metable->getKey(), $result3->getKey()); $this->assertEquals($metable->getKey(), $result4->getKey()); + $this->assertEquals($metable->getKey(), $result5->getKey()); + $this->assertNull($result6); + $this->assertNull($result7); + $this->assertEquals($metable->getKey(), $result8->getKey()); } public function test_it_can_be_queried_by_numeric_meta_value(): void @@ -668,7 +676,6 @@ public function test_set_relation_updates_index(): void $method = (new ReflectionClass($metable)) ->getMethod('getMetaCollection'); - $method->setAccessible(true); $this->assertEquals($emptyCollection, $method->invoke($metable)); @@ -693,7 +700,6 @@ public function test_set_relations_updates_index(): void $method = (new ReflectionClass($metable)) ->getMethod('getMetaCollection'); - $method->setAccessible(true); $this->assertEquals($emptyCollection, $method->invoke($metable)); @@ -718,16 +724,10 @@ public function test_it_can_serialize_properly(): void $this->assertEquals('baz', $result->getMeta('foo')); } - public function test_it_throws_for_param_that_cannot_be_converted_to_string(): void - { - $this->expectException(\LogicException::class); - SampleMetable::whereMeta('foo', null)->get(); - } - public function test_it_throws_for_param_that_cannot_be_converted_to_numeric(): void { $this->expectException(\LogicException::class); - SampleMetable::whereMetaNumeric('foo', null)->get(); + SampleMetable::query()->whereMetaNumeric('foo', null)->get(); } public static function castProvider(): array From cfe53031c1ba8c22b7c175aecd8ebb6e7f1b819f Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Thu, 25 Apr 2024 23:53:16 -0400 Subject: [PATCH 34/38] update changelog --- CHANGELOG.md | 19 +++++++++---------- UPGRADING.md | 5 +++-- docs/source/datatypes.rst | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 680e0c8..a3d59c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 6.0.0 -Version 6 contains a number of changes to improve the security and performance of the package. Refer to the [UPGRADING.md](UPGRADING.md) file for detailed instructions on how to upgrade from version 5. +Version 6 contains a number of changes to improve the usability, performance and security of the package. Refer to the [UPGRADING.md](UPGRADING.md) file for detailed instructions on how to upgrade from version 5. ### Compatibility @@ -10,8 +10,7 @@ Version 6 contains a number of changes to improve the security and performance o - Droppped support for PHP 8.0 and below - Added support for Laravel 10 and 11 - Dropped support Laravel versions 9 and below -- Adjusted some method signatures with PHP 8+ mixed and union types -- New schema migration adding two new columns and improving indexing for searching by meta values. See [UPGRADING.md](UPGRADING.md) for details +- New schema migration adding new columns and improving indexing for searching by meta values. See [UPGRADING.md](UPGRADING.md) for details ### Data Types @@ -21,10 +20,10 @@ Version 6 contains a number of changes to improve the security and performance o - Added `PureEnumHandler` and `BackedEnumHandler` which adds support for storing enum values as Meta. - Added `StringableHandler` which adds support for storing `Illuminate\Support\Stringable` objects as Meta. - Added `DateTimeImmutableHandler` which adds support for storing `DateTimeImmutable`/`CarbonImmutable` objects as Meta. -- `ModelHandler` will now validate that the encoded class is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, the meta value will return `null`. -- `ModelHandler` will no longer throw a model not found exception if the model no longer exists. Instead, the meta value will return `null`. This is more in line with the existing behavior of the `ModelCollectionHandler`. -- `ModelCollectionHandler` will now validate that the encoded collection class is a valid Eloquent collection before attempting to instantiate it during unserialization. If the class is invalid, an instance of `Illuminate\Database\Eloquent\Collection` will be used instead. -- `ModelCollectionHandler` will now validate that the encoded class of each entry is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, that entry in the collection will be omitted. +- The `ModelHandler` will now validate that the encoded class is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, the meta value will return `null`. +- The `ModelHandler` will no longer throw a model not found exception if the model no longer exists. Instead, the meta value will return `null`. This is more in line with the existing behavior of the `ModelCollectionHandler`. +- The `ModelCollectionHandler` will now validate that the encoded collection class is a valid Eloquent collection before attempting to instantiate it during unserialization. If the class is invalid, an instance of `Illuminate\Database\Eloquent\Collection` will be used instead. +- The `ModelCollectionHandler` will now validate that the encoded class of each entry is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, that entry in the collection will be omitted. - Added `getNumericValue(): null|int|float` method to `HandlerInterface` which should convert the original value into a numeric format for indexing, if relevant for the data type. - Added `useHmacVerification(): bool` method to `HandlerInterface` which should indicate whether the integrity of the serialized data should be verified with an HMAC. @@ -34,8 +33,8 @@ Version 6 contains a number of changes to improve the security and performance o ### Efficient Value Search -- The Metable `whereMeta()`, `whereMetaIn()`, and `orderByMeta()` query scopes can now leverage a prefix index on the ``meta.value`` column. This greatly improves performance when searching for meta values against larger datasets when using applicable operators, e.g. `=`, `%`, `>`, `>=`, `<`, `<=`, `<>`, `LIKE` (no leading wildcard). -- `whereMetaNumeric()` and `orderByMetaNumeric()` query scopes will now scan the indexed `numeric_value` column instead of the serialized `value` column. This greatly improves performance when searching for meta values against larger datasets. +- The Metable `whereMeta()`, `whereMetaIn()`, and `orderByMeta()` query scopes can now leverage a prefix index on the ``meta.value`` column. This greatly improves performance when searching for meta values against larger datasets when using applicable operators, e.g. `=`, `%`, `>`, `>=`, `<`, `<=`, `<>`, `LIKE` (no leading wildcard). This index is only supported by the `'mysql'`, `'mariadb'`, `'pgsql'`, and `'sqlite'` drivers. +- The `whereMetaNumeric()` and `orderByMetaNumeric()` query scopes will now scan the indexed `numeric_value` column instead of the serialized `value` column. This greatly improves performance when searching for meta values against larger datasets. - `whereMetaNumeric()` query scope will now accept a value of any type. It will be converted to an integer or float by the handler. This is more consistent with the behaviour of the other query scopes. - Added additional query scopes to more easily search meta values based on different criteria: - `whereMetaInNumeric()` @@ -57,7 +56,7 @@ Version 6 contains a number of changes to improve the security and performance o ### Encrypt Meta - Added the `setMetaEncrypted()` method which will encrypt data before storing it in the database and decrypt it when retrieving it. This is useful for storing sensitive data in the meta table. -- prefixing a meta cast with `encrypted:` will automatically encrypt all values for that meta key. +- Prefixing a meta cast with `encrypted:` will automatically encrypt all values for that meta key. ### Metable Attributes diff --git a/UPGRADING.md b/UPGRADING.md index 4529eaa..5560e8c 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -10,7 +10,8 @@ ### Schema Changes -* A new schema migration has been added which adds three new columns to the meta table and improves indexing for querying by meta values. +* A new schema migration has been added which adds two new columns to the meta table and improves indexing for querying by meta values. +* Before running the migration, you may choose to tune the `metable.stringValueIndexLength` config to adjust the length of the index on the `value` column. The default value of 255 is suitable for most use cases. ### Configuration Changes @@ -36,7 +37,7 @@ ### Metable Attributes -* Optional: if you intend to access meta with property access, add the new `\Plank\Metable\MetableAttributes` traits to your `Metable`. +* (Optional) If you intend to access meta with property access, add the new `\Plank\Metable\MetableAttributes` traits to your `Metable`. ## 4.X -> 5.X - New migration file added which adds a new composite unique index to the meta table on `metable_type`, `metable_id`, and `key`. Make sure that you have no duplicate keys for a given entity (previously possible as a race condition) before applying the new migration. diff --git a/docs/source/datatypes.rst b/docs/source/datatypes.rst index c3b64eb..9b92733 100644 --- a/docs/source/datatypes.rst +++ b/docs/source/datatypes.rst @@ -60,7 +60,7 @@ Null ^^^^^^^^ +----------------------+-----+ | Handler | ``\Plank\Metable\DataType\NullHandler`` | -| String Query Scopes | No | +| String Query Scopes | Yes | | Numeric Query Scopes | No | | Other Query Scopes | whereMetaIsNull() | +----------------------+-----+ From 2005c8d53e24368b41ff0454ef070dcd8e31bcab Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sun, 28 Apr 2024 15:18:59 -0400 Subject: [PATCH 35/38] convert migration files to anonymous classes --- migrations/2017_01_01_000000_create_meta_table.php | 4 ++-- migrations/2020_01_24_000000_modify_meta_indexes.php | 4 ++-- migrations/2024_04_14_000000_add_meta_search_columns.php | 4 ++-- .../2017_01_01_000000_create_sample_classes_tables.php | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/migrations/2017_01_01_000000_create_meta_table.php b/migrations/2017_01_01_000000_create_meta_table.php index ea8e736..f004902 100644 --- a/migrations/2017_01_01_000000_create_meta_table.php +++ b/migrations/2017_01_01_000000_create_meta_table.php @@ -4,7 +4,7 @@ use Illuminate\Database\Schema\Blueprint; use Illuminate\Support\Facades\Schema; -class CreateMetaTable extends Migration +return new class extends Migration { /** * Run the migrations. @@ -33,4 +33,4 @@ public function down(): void { Schema::dropIfExists('meta'); } -} +}; diff --git a/migrations/2020_01_24_000000_modify_meta_indexes.php b/migrations/2020_01_24_000000_modify_meta_indexes.php index 481377b..c60e008 100644 --- a/migrations/2020_01_24_000000_modify_meta_indexes.php +++ b/migrations/2020_01_24_000000_modify_meta_indexes.php @@ -4,7 +4,7 @@ use Illuminate\Database\Schema\Blueprint; use Illuminate\Support\Facades\Schema; -class ModifyMetaIndexes extends Migration +return new class extends Migration { /** * Run the migrations. @@ -33,4 +33,4 @@ public function down() $table->index(['metable_type', 'metable_id']); }); } -} +}; diff --git a/migrations/2024_04_14_000000_add_meta_search_columns.php b/migrations/2024_04_14_000000_add_meta_search_columns.php index 4de70d0..a14195b 100644 --- a/migrations/2024_04_14_000000_add_meta_search_columns.php +++ b/migrations/2024_04_14_000000_add_meta_search_columns.php @@ -5,7 +5,7 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; -class AddMetaSearchColumns extends Migration +return new class extends Migration { /** * Run the migrations. @@ -78,4 +78,4 @@ private function detectDriverName(): ?string $this->getConnection() ?? $migrator->getConnection() )->getDriverName(); } -} +}; diff --git a/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php b/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php index 4c6441b..b139396 100644 --- a/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php +++ b/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php @@ -4,7 +4,7 @@ use Illuminate\Database\Schema\Blueprint; use Illuminate\Support\Facades\Schema; -class CreateSampleClassesTables extends Migration +return new class extends Migration { /** * Run the migrations. @@ -30,4 +30,4 @@ public function down() { Schema::dropIfExists('sample_metables'); } -} +}; From 3862d243ea8fb5b84d615dda16478d7f3d0bbe7a Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sun, 28 Apr 2024 16:45:01 -0400 Subject: [PATCH 36/38] account for sqlite/pgsql expression indexes in meta value query scopes --- ...4_04_14_000000_add_meta_search_columns.php | 2 +- src/Metable.php | 103 +++++++++++++++++- tests/Integration/MetableTest.php | 62 +++++++++++ 3 files changed, 163 insertions(+), 4 deletions(-) diff --git a/migrations/2024_04_14_000000_add_meta_search_columns.php b/migrations/2024_04_14_000000_add_meta_search_columns.php index a14195b..0512b43 100644 --- a/migrations/2024_04_14_000000_add_meta_search_columns.php +++ b/migrations/2024_04_14_000000_add_meta_search_columns.php @@ -31,7 +31,7 @@ public function up() ); } elseif (in_array($driver, ['pgsql', 'sqlite'])) { $table->rawIndex( - "metable_type, key, substr(value, 1, $stringIndexLength)", + "metable_type, key, SUBSTR(value, 1, $stringIndexLength)", 'value_string_prefix_index' ); } diff --git a/src/Metable.php b/src/Metable.php index e4a6903..414d8df 100644 --- a/src/Metable.php +++ b/src/Metable.php @@ -375,7 +375,23 @@ public function scopeWhereMeta( 'meta', function (Builder $q) use ($key, $operator, $stringValue, $value) { $q->where('key', $key); - $q->where('value', $operator, $stringValue); + [ + $needPartialMatch, + $needExactMatch + ] = $this->determineQueryValueMatchTypes($q, [$stringValue]); + + if ($needPartialMatch) { + $indexLength = (int)config('metable.stringValueIndexLength', 255); + $q->where( + $q->raw("SUBSTR(value, 1, $indexLength)"), + $operator, + substr($stringValue, 0, $indexLength) + ); + } + + if ($needExactMatch) { + $q->where('value', $operator, $stringValue); + } // null and empty string look the same in the database, // use the type column to differentiate. @@ -433,7 +449,27 @@ public function scopeWhereMetaBetween( 'meta', function (Builder $q) use ($key, $min, $max, $not) { $q->where('key', $key); - $q->whereBetween('value', [$min, $max], 'and', $not); + + [ + $needPartialMatch, + $needExactMatch + ] = $this->determineQueryValueMatchTypes($q, [$min, $max]); + + if ($needPartialMatch) { + $indexLength = (int)config('metable.stringValueIndexLength', 255); + $q->whereBetween( + $q->raw("SUBSTR(value, 1, $indexLength)"), + [ + substr($min, 0, $indexLength), + substr($max, 0, $indexLength) + ], + 'and', + $not + ); + } + if ($needExactMatch) { + $q->whereBetween('value', [$min, $max], 'and', $not); + } } ); } @@ -525,7 +561,27 @@ public function scopeWhereMetaIn( $q->whereHas('meta', function (Builder $q) use ($key, $values, $not) { $q->where('key', $key); - $q->whereIn('value', $values, 'and', $not); + + [ + $needPartialMatch, + $needExactMatch + ] = $this->determineQueryValueMatchTypes($q, $values); + if ($needPartialMatch) { + $indexLength = (int)config('metable.stringValueIndexLength', 255); + $q->whereIn( + $q->raw("SUBSTR(value, 1, $indexLength)"), + array_map( + fn ($val) => substr($val, 0, $indexLength), + $values + ), + 'and', + $not + ); + } + + if ($needExactMatch) { + $q->whereIn('value', $values, 'and', $not); + } }); } @@ -578,6 +634,16 @@ public function scopeOrderByMeta( bool $strict = false ): void { $table = $this->joinMetaTable($q, $key, $strict ? 'inner' : 'left'); + + [$needPartialMatch] = $this->determineQueryValueMatchTypes($q, []); + if ($needPartialMatch) { + $indexLength = (int)config('metable.stringValueIndexLength', 255); + $q->orderBy( + $q->raw("SUBSTR({$table}.value, 1, $indexLength)"), + $direction + ); + } + $q->orderBy("{$table}.value", $direction); } @@ -895,6 +961,37 @@ private function getHandlerForValue(mixed $value): HandlerInterface return $registry->getHandlerForValue($value); } + /** + * @param Builder $q + * @param string[] $stringValues + * @return array{bool, bool} [needPartialMatch, needExactMatch] + */ + protected function determineQueryValueMatchTypes( + Builder $q, + array $stringValues + ): array { + $driver = $q->getConnection()->getDriverName(); + $indexLength = (int)config('metable.stringValueIndexLength', 255); + + // only sqlite and pgsql support expression indexes, which must be partially matched + // mysql and mariadb support prefix indexes, which works with the entire value + // sqlserv does not support any substring indexing mechanism + if (!in_array($driver, ['sqlite', 'pgsql'])) { + return [false, true]; + } + // if any value is longer than the index length, we need to do both a + // substring match to leverage the index and an exact match to avoid false positives + foreach ($stringValues as $stringValue) { + if (strlen($stringValue) > $indexLength) { + return [true, true]; + } + } + + // if all values are shorter than the index length, + // we only need to do a substring match + return [true, false]; + } + abstract public function getKey(); abstract public function getMorphClass(); diff --git a/tests/Integration/MetableTest.php b/tests/Integration/MetableTest.php index 419c8c3..65b46e5 100644 --- a/tests/Integration/MetableTest.php +++ b/tests/Integration/MetableTest.php @@ -666,6 +666,68 @@ public function test_it_can_order_query_by_numeric_meta_value_strict(): void $this->assertEquals([1, 3], $results2->modelKeys()); } + public function test_it_can_query_long_strings(): void + { + config()->set('metable.stringValueIndexLength', 255); + $this->useDatabase(); + $metable1 = $this->createMetable(); + $metable1->setMeta('foo', $val1 = str_repeat('a', 255) . 'm'); + $metable2 = $this->createMetable(); + $metable2->setMeta('foo', $val2 = str_repeat('a', 255) . 'f'); + + $this->assertSame( + [$metable1->getKey()], + SampleMetable::whereMeta('foo', $val1)->get()->modelKeys() + ); + $this->assertSame( + [$metable2->getKey()], + SampleMetable::whereMeta('foo', $val2)->get()->modelKeys() + ); + + $this->assertSame( + [$metable1->getKey()], + SampleMetable::whereMetaIn('foo', [$val1])->get()->modelKeys() + ); + + $this->assertSame( + [$metable2->getKey()], + SampleMetable::whereMetaIn('foo', [$val2])->get()->modelKeys() + ); + + $this->assertSame( + [$metable1->getKey(), $metable2->getKey()], + SampleMetable::whereMetaIn('foo', [$val1, $val2])->get()->modelKeys() + ); + + $this->assertSame( + [$metable2->getKey()], + SampleMetable::whereMetaBetween( + 'foo', + str_repeat('a', 256), + str_repeat('a', 255) . 'l' + )->get()->modelKeys() + ); + + $this->assertSame( + [$metable1->getKey()], + SampleMetable::whereMetaBetween( + 'foo', + str_repeat('a', 255) . 'm', + str_repeat('a', 255) . 'z' + )->get()->modelKeys() + ); + + $this->assertSame( + [$metable2->getKey(), $metable1->getKey()], + SampleMetable::orderByMeta('foo', 'asc')->get()->modelKeys() + ); + + $this->assertSame( + [$metable1->getKey(), $metable2->getKey()], + SampleMetable::orderByMeta('foo', 'desc')->get()->modelKeys() + ); + } + public function test_set_relation_updates_index(): void { $metable = $this->makeMetable(); From 34c26681b4ee33db297873b2dd9d9e3d98564c70 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sun, 28 Apr 2024 16:46:27 -0400 Subject: [PATCH 37/38] style fixes --- migrations/2017_01_01_000000_create_meta_table.php | 3 +-- migrations/2020_01_24_000000_modify_meta_indexes.php | 3 +-- migrations/2024_04_14_000000_add_meta_search_columns.php | 3 +-- .../2017_01_01_000000_create_sample_classes_tables.php | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/migrations/2017_01_01_000000_create_meta_table.php b/migrations/2017_01_01_000000_create_meta_table.php index f004902..6774167 100644 --- a/migrations/2017_01_01_000000_create_meta_table.php +++ b/migrations/2017_01_01_000000_create_meta_table.php @@ -4,8 +4,7 @@ use Illuminate\Database\Schema\Blueprint; use Illuminate\Support\Facades\Schema; -return new class extends Migration -{ +return new class extends Migration { /** * Run the migrations. * diff --git a/migrations/2020_01_24_000000_modify_meta_indexes.php b/migrations/2020_01_24_000000_modify_meta_indexes.php index c60e008..11e7501 100644 --- a/migrations/2020_01_24_000000_modify_meta_indexes.php +++ b/migrations/2020_01_24_000000_modify_meta_indexes.php @@ -4,8 +4,7 @@ use Illuminate\Database\Schema\Blueprint; use Illuminate\Support\Facades\Schema; -return new class extends Migration -{ +return new class extends Migration { /** * Run the migrations. * diff --git a/migrations/2024_04_14_000000_add_meta_search_columns.php b/migrations/2024_04_14_000000_add_meta_search_columns.php index 0512b43..0263546 100644 --- a/migrations/2024_04_14_000000_add_meta_search_columns.php +++ b/migrations/2024_04_14_000000_add_meta_search_columns.php @@ -5,8 +5,7 @@ use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; -return new class extends Migration -{ +return new class extends Migration { /** * Run the migrations. * diff --git a/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php b/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php index b139396..8676505 100644 --- a/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php +++ b/tests/migrations/2017_01_01_000000_create_sample_classes_tables.php @@ -4,8 +4,7 @@ use Illuminate\Database\Schema\Blueprint; use Illuminate\Support\Facades\Schema; -return new class extends Migration -{ +return new class extends Migration { /** * Run the migrations. * From 9f3dcd67bcd752c95e01b454bd5e1f515bc20b71 Mon Sep 17 00:00:00 2001 From: Sean Fraser Date: Sun, 28 Apr 2024 21:48:30 -0400 Subject: [PATCH 38/38] add additional tests --- docs/source/handling_meta.rst | 13 +- src/Metable.php | 121 +++++-- src/MetableAttributes.php | 2 +- .../Integration/DataType/EnumHandlerTest.php | 34 ++ tests/Integration/MetaTest.php | 7 + tests/Integration/MetableAttributesTest.php | 18 +- tests/Integration/MetableTest.php | 338 ++++++++++++++++-- tests/Mocks/SampleMetable.php | 13 +- 8 files changed, 461 insertions(+), 85 deletions(-) create mode 100644 tests/Integration/DataType/EnumHandlerTest.php diff --git a/docs/source/handling_meta.rst b/docs/source/handling_meta.rst index 1dece1e..ed0ab71 100644 --- a/docs/source/handling_meta.rst +++ b/docs/source/handling_meta.rst @@ -149,7 +149,18 @@ You can enforce that any meta attached to a particular key is always of a partic 'children' => 'collection:\App\ExampleMetable', ]; - //... + // equivalent to: + protected function metaCasts(): array + { + return [ + 'optin' => 'boolean', + 'age' => 'integer', + 'secret' => 'encrypted:string', + 'parent' => ExampleMetable::class, + 'children' => 'collection:\App\ExampleMetable', + ]; + } + } All `cast types supported by Eloquent`_ are supported, with the following modifications: diff --git a/src/Metable.php b/src/Metable.php index 414d8df..7dc2408 100644 --- a/src/Metable.php +++ b/src/Metable.php @@ -844,16 +844,7 @@ protected function castMetaValueIfNeeded(string $key, mixed $value): mixed protected function castMetaValue(string $key, mixed $value, string $cast): mixed { if ($cast == 'array' || $cast == 'object') { - $assoc = $cast == 'array'; - if (is_string($value)) { - $value = json_decode($value, $assoc, 512, JSON_THROW_ON_ERROR); - } - return json_decode( - json_encode($value, JSON_THROW_ON_ERROR), - $assoc, - 512, - JSON_THROW_ON_ERROR - ); + return $this->castMetaToJson($cast, $value); } if ($cast == 'hashed') { @@ -861,41 +852,14 @@ protected function castMetaValue(string $key, mixed $value, string $cast): mixed } if ($cast == 'collection' || str_starts_with($cast, 'collection:')) { - if ($value instanceof \Illuminate\Support\Collection) { - $collection = $value; - } elseif ($value instanceof Model) { - $collection = $value->newCollection([$value]); - } else { - $collection = collect($value); - } - - if (str_starts_with($cast, 'collection:')) { - $class = substr($cast, 11); - $collection->each(function ($item) use ($class): void { - if (!$item instanceof $class) { - throw CastException::invalidClassCast($class, $item); - } - }); - } - - return $collection; + return $this->castMetaToCollection($cast, $value); } if (class_exists($cast) && !is_a($cast, Castable::class, true) && $cast != 'datetime' ) { - if ($value instanceof $cast) { - return $value; - } - - if (is_a($cast, Model::class, true) - && (is_string($value) || is_int($value)) - ) { - return $cast::find($value); - } - - throw CastException::invalidClassCast($cast, $value); + return $this->castMetaToClass($value, $cast); } // leverage Eloquent built-in casting functionality @@ -1005,4 +969,83 @@ abstract public function load($relations); abstract public function relationLoaded($key); abstract protected function castAttributeAsHashedString($key, $value); + + /** + * @param mixed $value + * @param string $cast + * @return Collection|\Illuminate\Support\Collection + */ + protected function castMetaToCollection(string $cast, mixed $value): \Illuminate\Support\Collection + { + if ($value instanceof \Illuminate\Support\Collection) { + $collection = $value; + } elseif ($value instanceof Model) { + $collection = $value->newCollection([$value]); + } elseif (is_iterable($value)) { + $isEloquentModels = true; + $notEmpty = false; + + foreach ($value as $item) { + $notEmpty = true; + if (!$item instanceof Model) { + $isEloquentModels = false; + break; + } + } + $collection = $isEloquentModels && $notEmpty + ? $value[0]->newCollection($value) + : collect($value); + } + + if (str_starts_with($cast, 'collection:')) { + $class = substr($cast, 11); + $collection->each(function ($item) use ($class): void { + if (!$item instanceof $class) { + throw CastException::invalidClassCast($class, $item); + } + }); + } + + return $collection; + } + + /** + * @param string $cast + * @param mixed $value + * @return mixed + * @throws \JsonException + */ + protected function castMetaToJson(string $cast, mixed $value): mixed + { + $assoc = $cast == 'array'; + if (is_string($value)) { + $value = json_decode($value, $assoc, 512, JSON_THROW_ON_ERROR); + } + return json_decode( + json_encode($value, JSON_THROW_ON_ERROR), + $assoc, + 512, + JSON_THROW_ON_ERROR + ); + } + + /** + * @param mixed $value + * @param string $cast + * @return mixed + */ + protected function castMetaToClass(mixed $value, string $cast): mixed + { + if ($value instanceof $cast) { + return $value; + } + + if (is_a($cast, Model::class, true) + && (is_string($value) || is_int($value)) + ) { + return $cast::findOrFail($value); + } + + throw CastException::invalidClassCast($cast, $value); + } } diff --git a/src/MetableAttributes.php b/src/MetableAttributes.php index dd309fa..22b837f 100644 --- a/src/MetableAttributes.php +++ b/src/MetableAttributes.php @@ -52,7 +52,7 @@ public function offsetExists($key): bool return $this->hasMeta($this->metaAttributeToKey($key)); } - return parent::isset($key); + return parent::offsetExists($key); } public function offsetUnset($key): void diff --git a/tests/Integration/DataType/EnumHandlerTest.php b/tests/Integration/DataType/EnumHandlerTest.php new file mode 100644 index 0000000..fbaa37f --- /dev/null +++ b/tests/Integration/DataType/EnumHandlerTest.php @@ -0,0 +1,34 @@ +assertNull($handler->unserializeValue('baz#value')); + } + + public function test_back_enum_handles_non_enum() + { + $handler = new BackedEnumHandler(); + $this->assertNull($handler->unserializeValue('stdClass#value')); + } + + public function test_pure_enum_handles_unknown_class() + { + $handler = new PureEnumHandler(); + $this->assertNull($handler->unserializeValue('baz#value')); + } + + public function test_pure_enum_handles_non_enum() + { + $handler = new PureEnumHandler(); + $this->assertNull($handler->unserializeValue('stdClass#value')); + } +} diff --git a/tests/Integration/MetaTest.php b/tests/Integration/MetaTest.php index f913836..a6557ce 100644 --- a/tests/Integration/MetaTest.php +++ b/tests/Integration/MetaTest.php @@ -86,6 +86,13 @@ public function test_it_can_encrypt_its_value(): void $this->assertEquals($hmac, $meta->hmac); $this->assertEquals('encrypted:string', $meta->type); $this->assertNull($meta->numeric_value); + + + $rawValue = $meta->getRawValue(); + // should not re-encrypt + $meta->encrypt(); + $this->assertEquals($rawValue, $meta->getRawValue()); + $this->assertEquals('encrypted:string', $meta->type); } private function makeMeta(array $attributes = []): Meta diff --git a/tests/Integration/MetableAttributesTest.php b/tests/Integration/MetableAttributesTest.php index c69cc39..48ce979 100644 --- a/tests/Integration/MetableAttributesTest.php +++ b/tests/Integration/MetableAttributesTest.php @@ -52,6 +52,11 @@ public function test_it_doesnt_overwrite_existing_attributes() $model->setAttribute('meta_attribute', 'baz'); $this->assertFalse($model->hasMeta('attribute')); + + $model->meta_attribute = 'qux'; + $this->assertTrue($model->offsetExists('meta_attribute')); + $model->offsetUnset('meta_attribute'); + $this->assertNull($model->meta_attribute); } public function test_it_converts_to_array() @@ -72,21 +77,26 @@ public function test_it_converts_to_array() $model->makeHidden('meta_var2', 'created_at', 'updated_at', 'meta'); - $array = $model->toArray(); $this->assertEquals([ 'meta_attribute' => '', 'id' => $model->getKey(), 'meta_foo' => 'bar', 'meta_var' => 'foo' - ], $array); + ], $model->toArray()); + + $model->includeMetaInArray = false; + $this->assertEquals([ + 'meta_attribute' => '', + 'id' => $model->getKey(), + ], $model->toArray()); + $model->includeMetaInArray = true; $model->makeMetaHidden(); - $array = $model->toArray(); $this->assertEquals([ 'meta_attribute' => '', 'id' => $model->getKey(), - ], $array); + ], $model->toArray()); } private function createMetable(array $attributes = []): SampleMetable diff --git a/tests/Integration/MetableTest.php b/tests/Integration/MetableTest.php index 65b46e5..d409854 100644 --- a/tests/Integration/MetableTest.php +++ b/tests/Integration/MetableTest.php @@ -5,10 +5,13 @@ use Carbon\Carbon; use Illuminate\Database\Eloquent\Casts\AsStringable; use Illuminate\Database\Eloquent\Collection; +use Illuminate\Database\Eloquent\ModelNotFoundException; use Illuminate\Support\Stringable; +use Plank\Metable\Exceptions\CastException; use Plank\Metable\Meta; use Plank\Metable\Tests\Mocks\SampleMetable; use Plank\Metable\Tests\Mocks\SampleMetableSoftDeletes; +use Plank\Metable\Tests\Mocks\SampleSerializable; use Plank\Metable\Tests\TestCase; use ReflectionClass; @@ -810,7 +813,12 @@ public static function castProvider(): array 'string - null' => ['string', null, null, 'null'], 'string - dateTime' => ['string', $date, (string)$date, 'string'], 'array - array' => ['array', ['foo', 'bar'], ['foo', 'bar'], 'serialized'], - 'array - json' => ['array', json_encode(['foo' => 'bar']), ['foo' => 'bar'], 'serialized'], + 'array - json' => [ + 'array', + json_encode(['foo' => 'bar']), + ['foo' => 'bar'], + 'serialized' + ], 'array - object' => ['array', $object, ['foo' => 'bar'], 'serialized'], 'array - null' => ['array', null, null, 'null'], 'boolean - true' => ['boolean', true, true, 'boolean'], @@ -841,8 +849,20 @@ public static function castProvider(): array 'integer - string' => ['integer', '123', 123, 'integer'], 'integer - null' => ['integer', null, null, 'null'], 'object - object' => ['object', $object, $object, 'serialized', false], - 'object - array' => ['object', ['foo' => 'bar'], $object, 'serialized', false], - 'object - json' => ['object', json_encode(['foo' => 'bar']), $object, 'serialized', false], + 'object - array' => [ + 'object', + ['foo' => 'bar'], + $object, + 'serialized', + false + ], + 'object - json' => [ + 'object', + json_encode(['foo' => 'bar']), + $object, + 'serialized', + false + ], 'object - null' => ['object', null, null, 'null'], 'real - int' => ['real', 123, 123.0, 'float'], 'real - float' => ['real', 123.456, 123.456, 'float'], @@ -851,45 +871,245 @@ public static function castProvider(): array 'real - null' => ['real', null, null, 'null'], 'timestamp - dateTime' => ['timestamp', $date, $date->timestamp, 'integer'], 'timestamp - int' => ['timestamp', 123, 123, 'integer'], - 'timestamp - string' => ['timestamp', '2020-01-01 00:00:00', strtotime('2020-01-01 00:00:00'), 'integer'], + 'timestamp - string' => [ + 'timestamp', + '2020-01-01 00:00:00', + strtotime('2020-01-01 00:00:00'), + 'integer' + ], 'timestamp - null' => ['timestamp', null, null, 'null'], - 'date - dateTime' => ['date', $date, $date->copy()->startOfDay(), 'datetime', false], - 'date - string' => ['date', (string)$date, $date->copy()->startOfDay(), 'datetime', false], - 'date - timestamp' => ['date', $date->timestamp, $date->copy()->startOfDay(), 'datetime', false], - 'date - string timestamp' => ['date', (string)$date->timestamp, $date->copy()->startOfDay(), 'datetime', false], + 'date - dateTime' => [ + 'date', + $date, + $date->copy()->startOfDay(), + 'datetime', + false + ], + 'date - string' => [ + 'date', + (string)$date, + $date->copy()->startOfDay(), + 'datetime', + false + ], + 'date - timestamp' => [ + 'date', + $date->timestamp, + $date->copy()->startOfDay(), + 'datetime', + false + ], + 'date - string timestamp' => [ + 'date', + (string)$date->timestamp, + $date->copy()->startOfDay(), + 'datetime', + false + ], 'date - null' => ['date', null, null, 'null'], 'datetime - dateTime' => ['datetime', $date, $date, 'datetime', false], - 'datetime - string' => ['datetime', $date->format('Y-m-d H:i:s.uO'), $date, 'datetime', false], - 'datetime - timestamp' => ['datetime', $date->timestamp, $date->copy()->startOfSecond(), 'datetime', false], - 'datetime - string timestamp' => ['datetime', (string)$date->timestamp, $date->copy()->startOfSecond(), 'datetime', false], + 'datetime - string' => [ + 'datetime', + $date->format('Y-m-d H:i:s.uO'), + $date, + 'datetime', + false + ], + 'datetime - timestamp' => [ + 'datetime', + $date->timestamp, + $date->copy()->startOfSecond(), + 'datetime', + false + ], + 'datetime - string timestamp' => [ + 'datetime', + (string)$date->timestamp, + $date->copy()->startOfSecond(), + 'datetime', + false + ], 'datetime - null' => ['datetime', null, null, 'null'], - 'immutable_date - dateTime' => ['immutable_date', $date, $date->copy()->startOfDay()->toImmutable(), 'datetime_immutable', false], - 'immutable_date - string' => ['immutable_date', $date->format('Y-m-d H:i:s.uO'), $date->copy()->startOfDay()->toImmutable(), 'datetime_immutable', false], - 'immutable_date - timestamp' => ['immutable_date', $date->timestamp, $date->copy()->startOfDay()->toImmutable(), 'datetime_immutable', false], - 'immutable_date - string timestamp' => ['immutable_date', (string)$date->timestamp, $date->copy()->startOfDay()->toImmutable(), 'datetime_immutable', false], + 'immutable_date - dateTime' => [ + 'immutable_date', + $date, + $date->copy()->startOfDay()->toImmutable(), + 'datetime_immutable', + false + ], + 'immutable_date - string' => [ + 'immutable_date', + $date->format('Y-m-d H:i:s.uO'), + $date->copy()->startOfDay()->toImmutable(), + 'datetime_immutable', + false + ], + 'immutable_date - timestamp' => [ + 'immutable_date', + $date->timestamp, + $date->copy()->startOfDay()->toImmutable(), + 'datetime_immutable', + false + ], + 'immutable_date - string timestamp' => [ + 'immutable_date', + (string)$date->timestamp, + $date->copy()->startOfDay()->toImmutable(), + 'datetime_immutable', + false + ], 'immutable_date - null' => ['immutable_date', null, null, 'null'], - 'immutable_datetime - dateTime' => ['immutable_datetime', $date, $date->toImmutable(), 'datetime_immutable', false], - 'immutable_datetime - string' => ['immutable_datetime', $date->format('Y-m-d H:i:s.uO'), $date->toImmutable(), 'datetime_immutable', false], - 'immutable_datetime - timestamp' => ['immutable_datetime', $date->timestamp, $date->copy()->startOfSecond()->toImmutable(), 'datetime_immutable', false], - 'immutable_datetime - string timestamp' => ['immutable_datetime', (string)$date->timestamp, $date->copy()->startOfSecond()->toImmutable(), 'datetime_immutable', false], + 'immutable_datetime - dateTime' => [ + 'immutable_datetime', + $date, + $date->toImmutable(), + 'datetime_immutable', + false + ], + 'immutable_datetime - string' => [ + 'immutable_datetime', + $date->format('Y-m-d H:i:s.uO'), + $date->toImmutable(), + 'datetime_immutable', + false + ], + 'immutable_datetime - timestamp' => [ + 'immutable_datetime', + $date->timestamp, + $date->copy()->startOfSecond()->toImmutable(), + 'datetime_immutable', + false + ], + 'immutable_datetime - string timestamp' => [ + 'immutable_datetime', + (string)$date->timestamp, + $date->copy()->startOfSecond()->toImmutable(), + 'datetime_immutable', + false + ], 'immutable_datetime - null' => ['immutable_datetime', null, null, 'null'], - 'hashed - string' => ['hashed', 'foo', fn ($result) => password_verify('foo', $result), 'string'], - 'hashed - int' => ['hashed', 123, fn ($result) => password_verify('123', $result), 'string'], + 'hashed - string' => [ + 'hashed', + 'foo', + fn ($result) => password_verify('foo', $result), + 'string' + ], + 'hashed - int' => [ + 'hashed', + 123, + fn ($result) => password_verify('123', $result), + 'string' + ], 'hashed - null' => ['hashed', null, null, 'null'], - 'collection - array' => ['collection', ['foo', 'bar'], collect(['foo', 'bar']), 'serialized', false], - 'collection - eloquent' => ['collection', $model, fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), 'collection', false], - 'collection - eloquent collection' => ['collection', $modelCollection, fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), 'collection', false], + 'collection - array' => [ + 'collection', + ['foo', 'bar'], + collect(['foo', 'bar']), + 'serialized', + false + ], + 'collection - eloquent' => [ + 'collection', + $model, + fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), + 'collection', + false + ], + 'collection - eloquent collection' => [ + 'collection', + $modelCollection, + fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), + 'collection', + false + ], 'collection - null' => ['collection', null, null, 'null'], - 'stringable - string' => [AsStringable::class, 'foo', new Stringable('foo'), 'stringable', false], - 'stringable - int' => [AsStringable::class, 123, new Stringable('123'), 'stringable', false], + 'collection:class - eloquent object' => [ + 'collection:' . SampleMetable::class, + $model, + fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), + 'collection', + false + ], + 'collection:class - eloquent array' => [ + 'collection:' . SampleMetable::class, + [$model], + fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), + 'collection', + false + ], + 'collection:class - eloquent collection' => [ + 'collection:' . SampleMetable::class, + $modelCollection, + fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), + 'collection', + false + ], + 'collection:class - object collection' => [ + 'collection:' . \stdClass::class, + collect([$object]), + collect([$object]), + 'serialized', + false + ], + 'stringable - string' => [ + AsStringable::class, + 'foo', + new Stringable('foo'), + 'stringable', + false + ], + 'stringable - int' => [ + AsStringable::class, + 123, + new Stringable('123'), + 'stringable', + false + ], 'stringable - null' => [AsStringable::class, null, null, 'null'], 'encrypted - string' => ['encrypted', 'foo', 'foo', 'encrypted:string'], - 'encrypted - array' => ['encrypted', ['foo' => 'bar'], ['foo' => 'bar'], 'encrypted:serialized'], + 'encrypted - array' => [ + 'encrypted', + ['foo' => 'bar'], + ['foo' => 'bar'], + 'encrypted:serialized' + ], 'encrypted - null' => ['encrypted', null, null, 'null'], - 'encrypted:collection - array' => ['encrypted:collection', ['foo', 'bar'], collect(['foo', 'bar']), 'encrypted:serialized', false], - 'encrypted:collection - eloquent' => ['encrypted:collection', $model, fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), 'encrypted:collection', false], - 'encrypted:collection - eloquent collection' => ['encrypted:collection', $modelCollection, fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), 'encrypted:collection', false], - 'encrypted:string - int' => ['encrypted:string', 123, '123', 'encrypted:string'], + 'encrypted:collection - array' => [ + 'encrypted:collection', + ['foo', 'bar'], + collect(['foo', 'bar']), + 'encrypted:serialized', + false + ], + 'encrypted:collection - eloquent' => [ + 'encrypted:collection', + $model, + fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), + 'encrypted:collection', + false + ], + 'encrypted:collection - eloquent collection' => [ + 'encrypted:collection', + $modelCollection, + fn ($result) => $result->modelKeys() === $modelCollection->modelKeys(), + 'encrypted:collection', + false + ], + 'encrypted:string - int' => [ + 'encrypted:string', + 123, + '123', + 'encrypted:string' + ], + 'class - class' => [\stdClass::class, $object, $object, 'serialized', false], + 'class - eloquent id' => [ + SampleMetable::class, + 99, + fn ($result) => $result instanceof SampleMetable + && $result->getKey() === $model->getKey(), + 'model', + false + ], ]; } @@ -903,7 +1123,11 @@ public function test_it_casts_meta_values( ): void { $this->useDatabase(); - if ($cast === 'collection' || $cast === 'encrypted:collection') { + if ($cast === 'collection' + || $cast === 'encrypted:collection' + || str_starts_with($cast, 'collection:') + || $cast === SampleMetable::class + ) { $model = new SampleMetable(); $model->id = 99; $model->save(); @@ -923,15 +1147,55 @@ public function test_it_casts_meta_values( $this->assertSame($expectedHandlerType, $metable->getMetaRecord($key)->type); } - public function test_it_can_cast_meta_values(): void + public static function invalidClassCastProvider(): array + { + return [ + 'collection:class - string' => ['collection:stdClass', collect('bar')], + 'collection:class - int' => ['collection:stdClass', collect(123)], + 'collection:class - other class' => ['collection:stdClass', collect(new SampleSerializable([]))], + 'class - string' => [\stdClass::class, 'bar'], + 'class - int' => [\stdClass::class, 123], + 'class - other class' => [\stdClass::class, new SampleSerializable([])], + 'eloquent - int' => [SampleMetable::class, 999, ModelNotFoundException::class], + 'eloquent - string' => [SampleMetable::class, 'abc', ModelNotFoundException::class], + 'eloquent - other class' => [SampleMetable::class, new \stdClass()], + ]; + } + + public function test_cast_source_hierarchy(): void { $this->useDatabase(); $metable = $this->createMetable(); - $metable->setMeta('castable', 123); - - $result = SampleMetable::first(); + $metable->metaCasts = [ + 'prop_cast' => 'string', + 'cast' => 'string', + ]; + $metable->methodMetaCasts = ['cast' => 'integer']; + + $metable->setMeta('prop_cast', 123); + $this->assertSame('123', $metable->getMeta('prop_cast')); + $metable->setMeta('cast', 123); + $this->assertSame(123, $metable->getMeta('cast')); + + $date = Carbon::now()->startOfSecond(); + $metable->mergeMetaCasts(['prop_cast' => 'datetime', 'cast' => 'datetime']); + $metable->setMeta('prop_cast', $date->timestamp); + $metable->setMeta('cast', $date->timestamp); + $this->assertEquals($date, $metable->getMeta('prop_cast')); + $this->assertEquals($date, $metable->getMeta('cast')); + } - $this->assertSame('123', $result->getMeta('castable')); + /** @dataProvider invalidClassCastProvider */ + public function test_it_throws_for_invalid_class_cast( + string $cast, + mixed $invalidValue, + string $expectedException = CastException::class + ): void { + $this->expectException($expectedException); + $this->useDatabase(); + $metable = $this->createMetable(); + $metable->mergeMetaCasts(['foo' => $cast]); + $metable->setMeta('foo', $invalidValue); } private function makeMeta(array $attributes = []): Meta diff --git a/tests/Mocks/SampleMetable.php b/tests/Mocks/SampleMetable.php index adf1848..d621423 100644 --- a/tests/Mocks/SampleMetable.php +++ b/tests/Mocks/SampleMetable.php @@ -20,7 +20,14 @@ class SampleMetable extends Model implements MetableInterface 'foo' => 'bar' ]; - protected $metaCasts = [ - 'castable' => 'string', - ]; + public $metaCasts = []; + + public $methodMetaCasts = []; + + public $includeMetaInArray = true; + + public function metaCasts(): array + { + return $this->methodMetaCasts; + } }