Merge pull request #102 from plank/v6

V6

.github/workflows/automated-test.yml

@@ -1,42 +1,48 @@
 name: PHPUnit Tests
-on: [push, pull_request]
+on: [push]
 jobs:
   phpunit:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        php: [7.4, 8.0, 8.1, 8.2]
-        laravel: [10.*, 9.*]
-        stability: [prefer-lowest, prefer-stable]
-        exclude:
-          - laravel: 10.*
-            php: 7.4
-          - laravel: 10.*
-            php: 8.0
-    name: PHP ${{ matrix.php }} - L${{ matrix.laravel }} - ${{ matrix.stability}}
+        php-versions: ['8.1', '8.2', '8.3']
+        prefer-lowest: ['','--prefer-lowest']
+    name: PHP ${{ matrix.php-versions }} ${{ matrix.prefer-lowest }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup PHP
         uses: shivammathur/setup-php@v2
         with:
-          php-version: ${{ matrix.php }}
+          php-version: ${{ matrix.php-versions }}
           coverage: pcov
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Install dependencies
-        uses: ramsey/composer-install@v2
+      - name: Cache Composer dependencies
+        uses: actions/cache@v4
         with:
-          composer-options: "--prefer-dist --${{ matrix.stability }}"
+          path: |
+            ~/.composer/cache/files
+            ~/.cache/composer/files
+          key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-composer-
+
+      - name: Install dependencies
+        run: composer update --prefer-dist ${{ matrix.prefer-lowest }}
+        env:
+          COMPOSER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run phpunit
-        run: vendor/bin/phpunit --coverage-clover build/logs/clover.xml -v
+        run: vendor/bin/phpunit --coverage-clover build/logs/clover.xml
 
       - name: Upload coverage results to Coveralls
         env:
           COVERALLS_REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           COVERALLS_PARALLEL: true
-          COVERALLS_FLAG_NAME: php-${{ matrix.php }} - L${{ matrix.laravel }} - ${{ matrix.stability}}
+          COVERALLS_FLAG_NAME: php-${{ matrix.php-versions }}${{ matrix.prefer-lowest }}
         run: vendor/bin/php-coveralls --coverage_clover=build/logs/clover.xml -vvv
 
   finish-coverage:

.gitignore

@@ -2,4 +2,5 @@
 /composer.lock
 /coverage/
 /.idea/
-.phpunit.result.cache
+/.phpunit.result.cache
+/.phpunit.cache

CHANGELOG.md

@@ -1,5 +1,73 @@
 # Changelog
 
+## 6.0.0
+
+Version 6 contains a number of changes to improve the usability, performance and security of the package. Refer to the [UPGRADING.md](UPGRADING.md) file for detailed instructions on how to upgrade from version 5.
+
+### Compatibility
+
+- Added support for PHP 8.3
+- Droppped support for PHP 8.0 and below
+- Added support for Laravel 10 and 11
+- Dropped support Laravel versions 9 and below
+- New schema migration adding new columns and improving indexing for searching by meta values. See [UPGRADING.md](UPGRADING.md) for details
+
+### Data Types
+
+- Added `SignedSerializeHandler` as a catch-all datatype, which will attempt to serialize the data using PHP's `serialize()` function. The payload is cryptographically signed with an HMAC before being stored in the database to prevent PHP object injection attacks.
+- Deprecated `SerializableHandler` in favor of the new `SignedSerializeHandler` datatype. The `SerializableHandler` will be removed in a future release. In the interim, added the `metable.options.serializable.allowedClasses` config to protect against unserializing untrusted data.
+- Deprecated `ArrayHandler` and `ObjectHandler`, due to the ambiguity of nested array/objects switching type. These will be removed in a future release. The `SignedSerializeHandler` should be used instead.
+- Added `PureEnumHandler` and `BackedEnumHandler` which adds support for storing enum values as Meta.
+- Added `StringableHandler` which adds support for storing `Illuminate\Support\Stringable` objects as Meta.
+- Added `DateTimeImmutableHandler` which adds support for storing `DateTimeImmutable`/`CarbonImmutable` objects as Meta.
+- The `ModelHandler` will now validate that the encoded class is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, the meta value will return `null`.
+- The `ModelHandler` will no longer throw a model not found exception if the model no longer exists. Instead, the meta value will return `null`. This is more in line with the existing behavior of the `ModelCollectionHandler`.
+- The `ModelCollectionHandler` will now validate that the encoded collection class is a valid Eloquent collection before attempting to instantiate it during unserialization. If the class is invalid,  an instance of `Illuminate\Database\Eloquent\Collection` will be used instead.
+- The `ModelCollectionHandler` will now validate that the encoded class of each entry is a valid Eloquent Model before attempting to instantiate it during unserialization. If the class is invalid, that entry in the collection will be omitted.
+- Added `getNumericValue(): null|int|float` method to `HandlerInterface` which should convert the original value into a numeric format for indexing, if relevant for the data type.
+- Added `useHmacVerification(): bool` method to `HandlerInterface` which should indicate whether the integrity of the serialized data should be verified with an HMAC.
+
+### New Commands
+
+- Added `metable:refresh` artisan command which will decode and re-encode all meta values in the database. This is useful if you have changed the data type handlers and need to update the serialized data and indexes in the database.
+
+### Efficient Value Search 
+
+- The Metable `whereMeta()`, `whereMetaIn()`, and `orderByMeta()` query scopes can now leverage a prefix index on the ``meta.value`` column. This greatly improves performance when searching for meta values against larger datasets when using applicable operators, e.g. `=`, `%`, `>`, `>=`, `<`, `<=`, `<>`, `LIKE` (no leading wildcard). This index is only supported by the `'mysql'`, `'mariadb'`, `'pgsql'`, and `'sqlite'` drivers.
+- The `whereMetaNumeric()` and `orderByMetaNumeric()` query scopes will now scan the indexed `numeric_value` column instead of the serialized `value` column. This greatly improves performance when searching for meta values against larger datasets.
+- `whereMetaNumeric()` query scope will now accept a value of any type. It will be converted to an integer or float by the handler. This is more consistent with the behaviour of the other query scopes.
+- Added additional query scopes to more easily search meta values based on different criteria:
+  - `whereMetaInNumeric()`
+  - `whereMetaNotIn()`
+  - `whereMetaNotInNumeric()`
+  - `whereMetaBetween()`
+  - `whereMetaBetweenNumeric()`
+  - `whereMetaNotBetween()`
+  - `whereMetaNotBetweenNumeric()`
+  - `whereMetaIsNull()`
+  - `whereMetaIsModel()`
+- If the data type handlers cannot convert the search value provided to a ``whereMeta*Numeric()`` query scope to a numeric value, then an exception will be thrown.
+
+### Metable Casting
+
+- Added support for casting meta values to specific types by defining the `$castMeta` property or `castMeta(): array` method on the model. This is similar to the `casts` property of Eloquent Models used for attributes. All cast types supported by Eloquent Models are also available for Meta values.Values will be cast before values are stored in the database to ensure that they are indexed consistently
+- Added `mergeMetaCasts()` method which can be used to override the defined cast on a meta key at runtime.
+
+### Encrypt Meta
+
+- Added the `setMetaEncrypted()` method which will encrypt data before storing it in the database and decrypt it when retrieving it. This is useful for storing sensitive data in the meta table. 
+- Prefixing a meta cast with `encrypted:` will automatically encrypt all values for that meta key.
+
+### Metable Attributes
+
+- Added optional trait `MetableAttributes` which can further extend the `Metable` trait allowing access to meta values as model attributes using a `meta_` prefix. This can be useful for type hinting, IDE autocompletion, static analysis, and usage in Blade templates. 
+
+### Meta
+- Added `$meta->numeric_value` attributes, which are used for optimizing queries filtering by meta value
+- Added `$meta->hmac` attribute, which is used by some data type handlers to validate that the payload has not been tampered with.
+- Added `$meta->raw_value` virtual attribute, which exposes the raw serialized value of the meta key. This is useful for debugging purposes.
+- Added `encrypt()` method, used internally by the `Metable::setMetaEncrypted()` method
+
 # 5.0.1 - 2021-09-19
 - Fixed `setManyMeta()` not properly serializing certain types of data.
 

UPGRADING.md

@@ -1,7 +1,46 @@
 # Upgrading
 
+## 5.X -> 6.X
+
+### Compatibility
+
+* Minimum PHP version moved to 8.1
+* Minimum Laravel version moved to 10
+* Some methods have had their signatures adjusted to use PHP 8+ mixed and union types. If extending any class or implementing any interface from this package, method signatures may need to be updated.
+
+### Schema Changes
+
+* A new schema migration has been added which adds two new columns to the meta table and improves indexing for querying by meta values. 
+* Before running the migration, you may choose to tune the `metable.stringValueIndexLength` config to adjust the length of the index on the `value` column. The default value of 255 is suitable for most use cases.
+
+### Configuration Changes
+
+* Add the `Plank\Metable\DateType\PureEnumHandler`, `Plank\Metable\DateType\BackedEnumHandler`, `Plank\Metable\DateType\DateTimeImmutableHandler`, `Plank\Metable\DateType\StringableHandler` classes to the `datatypes` config. The order of these handlers is not important, except for `DateTimeImmutableHandler` which must come before `DateTimeHandler` if both are used. 
+* Recommended to add the `Plank\Metable\DateType\SignedSerializeHandler` class to the end of `datatypes` config list (catch-all).
+* The `SerializableHandler`, `ArrayHandler`, and `ObjectHandler` data types have been deprecated in favor of the new `SignedSerializeHandler`. If you have any Meta encoded using any of these data types, you should continue to include them in the `datatypes` config _after_ the `SignedSerializeHandler` to ensure that existing values will continue to be properly decoded, but new values will use the new encoding. Once all old values have been migrated, you may remove the deprecated data types from the `datatypes` config.
+* For security reasons, if you have any existing Meta encoded using `SerializableHandler`, you must configure the `metable.serializableHandlerAllowedClasses` config to list classes that are allowed to be unserialized. Otherwise, all objects will be returned as `__PHP_Incomplete_Class`. This config may be set to `true` to disable this security check and allow any class, but this is not recommended.
+
+### Handlers
+
+* If you have any custom data types, you will need to implement the new methods from the `HandlerInterface`:
+  * `getNumericValue(): null|int|float`: used to populate the new indexed numeric search column. You may return `null` if the value cannot be converted into a meaningful numeric value or does not need to be searchable.
+  * `useHmacVerification(): bool`: if the integrity of the serialized data should be verified with a HMAC, return `true`. If unserializing this data type is safe without HMAC verification, you may return `false`. 
+
+### Update Existing Data
+
+* Once you have applied the schema migration and updated the `datatypes` config, you should run the `metable:refresh` Artisan command to update all existing meta values to use the new types and populate the index columns. 
+* After this command has been run, you may remove the deprecated data types from the `datatypes` config. 
+
+### Query Scopes
+
+* Review the documentation about which data types can be queried with the various `whereMeta*` and `whereMeta*Numeric` query scopes. If you are querying the serialized `value` column directly, be aware that the formatting of array/object data types may have changed.
+
+### Metable Attributes
+
+* (Optional) If you intend to access meta with property access, add the new `\Plank\Metable\MetableAttributes` traits to your `Metable`.
+
 ## 4.X -> 5.X
-- New migration new file added which adds a new composite unique index to the meta table on `metable_type`, `metable_id`, and `key`. Make sure that you have no duplicate keys for a given entity (previously possible as a race condition) before applying the new migration.  
+- New migration file added which adds a new composite unique index to the meta table on `metable_type`, `metable_id`, and `key`. Make sure that you have no duplicate keys for a given entity (previously possible as a race condition) before applying the new migration.  
 
 ## 3.X -> 4.X
 - Database migration files are now served from within the package. In your migrations table, rename the `XXXX_XX_XX_XXXXXX_create_meta_table.php` entry to `2017_01_01_000000_create_meta_table.php` and delete your local copy of the migration file from the /database/migrations directory. If any customizations were made to the table, those should be defined as one or more separate ALTER table migrations.

composer.json

@@ -15,22 +15,22 @@
         }
     },
     "require": {
-        "php": ">=7.3.0",
+        "php": ">=8.1",
         "ext-json": "*",
-        "illuminate/support": "^6.20.42|^8.22.1|^9.0|^10.0",
-        "illuminate/database": "^6.20.42|^8.22.1|^9.0|^10.0",
+        "illuminate/support": "^10.10|^11.0",
+        "illuminate/database": "^10.10|^11.0",
         "phpoption/phpoption": "^1.8"
     },
     "require-dev": {
-        "symfony/symfony": "^5.4.1|^6.1",
+        "symfony/symfony": "^6.1|^7.0",
         "laravel/legacy-factories": "^1.0.4",
-        "orchestra/testbench": "^5.20|^6.23|^7.0|^8.0",
-        "phpunit/phpunit": "^9.5.11",
+        "orchestra/testbench": "^8.0|^9.0",
+        "phpunit/phpunit": "^10.5",
         "guzzlehttp/guzzle": "^7.2",
         "guzzlehttp/promises": "^1.4",
         "mockery/mockery": "^1.4.2",
         "nesbot/carbon" : "^2.62.1",
-        "php-coveralls/php-coveralls": "^2.4.2"
+        "php-coveralls/php-coveralls": "^2.5.2"
     },
     "autoload-dev":{
         "psr-4": {

config/metable.php

@@ -6,25 +6,80 @@
      */
     'model' => Plank\Metable\Meta::class,
 
+    /**
+     * Whether to apply migrations from this package automatically.
+     */
+    'applyMigrations' => true,
+
     /*
      * List of handlers for recognized data types.
      *
      * Handlers will be evaluated in order, so a value will be handled
      * by the first appropriate handler in the list.
+     *
+     * If you change this list, it may be necessary to refresh the meta table with the `artisan metable:refresh` command.
      */
     'datatypes' => [
-        Plank\Metable\DataType\BooleanHandler::class,
         Plank\Metable\DataType\NullHandler::class,
+        Plank\Metable\DataType\BooleanHandler::class,
         Plank\Metable\DataType\IntegerHandler::class,
         Plank\Metable\DataType\FloatHandler::class,
         Plank\Metable\DataType\StringHandler::class,
+        Plank\Metable\DataType\StringableHandler::class,
+        Plank\Metable\DataType\DateTimeImmutableHandler::class,
         Plank\Metable\DataType\DateTimeHandler::class,
-        Plank\Metable\DataType\ArrayHandler::class,
+        Plank\Metable\DataType\BackedEnumHandler::class,
+        Plank\Metable\DataType\PureEnumHandler::class,
         Plank\Metable\DataType\ModelHandler::class,
         Plank\Metable\DataType\ModelCollectionHandler::class,
-        Plank\Metable\DataType\SerializableHandler::class,
-        Plank\Metable\DataType\ObjectHandler::class,
+
+        /*
+         * The following handler is a catch-all that will encode anything.
+         * It should come after all other handlers in active use
+         *
+         * Any handlers listed after this one will only be used for unserializing existing meta
+         */
+        Plank\Metable\DataType\SignedSerializeHandler::class,
+
+        /*
+         * The following handlers are deprecated and will be removed in a future release.
+         * They are kept for backwards compatibility, but should not be used in new code.
+         */
+         // Plank\Metable\DataType\ArrayHandler::class,
+         // Plank\Metable\DataType\ObjectHandler::class,
+         // Plank\Metable\DataType\SerializableHandler::class,
     ],
 
-    'applyMigrations' => true
+    /*
+     * List of classes that are allowed to be unserialized by the SignedSerializeHandler.
+     * If true, all classes are allowed. If false, no classes are allowed.
+     * If an array, only classes listed in the array are allowed.
+     *
+     * SignedSerializeHandler employs hmac verification to prevent PHP object injection attacks,
+     * so allowing all classes is generally safe.
+     */
+    'signedSerializeHandlerAllowedClasses' => true,
+
+    /*
+     * List of classes that are allowed to be unserialized by the deprecated SerializableHandler.
+     * If true, all classes are allowed. If false, no classes are allowed.
+     * If an array, only classes listed in the array are allowed.
+     *
+     * This is the only protection against PHP object injection attacks, so it is strongly
+     * recommended to list allowed classes or set to false.
+     */
+    'serializableHandlerAllowedClasses' => [
+        // \SampleClass::class,
+    ],
+
+    /**
+     * Number of bytes of the to index for strings
+     * This value is used to determine the length of the prefix index on the value column in the database.
+     * Higher values allow for better precision when querying, but will use more disk space in the database.
+     *
+     * Prefix index is only supported on the 'mysql', 'mariadb', 'pgsql', and 'sqlite' database drivers.
+     *
+     * Set to 0 before running the migration to disable the index.
+     */
+    'stringValueIndexLength' => 255,
 ];