diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 4bc0ba42..4f0a98a5 100644 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -23,7 +23,7 @@ build: nodes: analysis: services: - elasticsearch: 6.6.2 + elasticsearch: 7.4.2 mysql: 5.7 tests: override: diff --git a/.travis.yml b/.travis.yml index 91dee885..f82d302e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ services: - mysql env: - - ELASTICSEARCH_HOST=localhost:9200 DB_DATABASE=scout_database DB_USERNAME=travis DB_PASSWORD="" + - ELASTICSEARCH_BUILD=7.1.1-amd64.deb ELASTICSEARCH_HOST=localhost:9200 DB_DATABASE=scout_database DB_USERNAME=travis DB_PASSWORD="" php: - 7.1 @@ -19,10 +19,17 @@ cache: matrix: allow_failures: - php: nightly + - env: ELASTICSEARCH_BUILD=7.1.1 before_script: - mysql -e "create database $DB_DATABASE;" - - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.6.2.deb && sudo dpkg -i --force-confnew elasticsearch-6.6.2.deb && sudo service elasticsearch restart + - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ELASTICSEARCH_BUILD + - sudo dpkg -i --force-confnew elasticsearch-$ELASTICSEARCH_BUILD + - sudo sed -i.old 's/-Xms1g/-Xms128m/' /etc/elasticsearch/jvm.options + - sudo sed -i.old 's/-Xmx1g/-Xmx128m/' /etc/elasticsearch/jvm.options + - echo -e '-XX:+DisableExplicitGC\n-Djdk.io.permissionsUseCanonicalPath=true\n-Dlog4j.skipJansi=true\n-server\n' | sudo tee -a /etc/elasticsearch/jvm.options + - sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch + - sudo service elasticsearch restart - sleep 30 - curl -X GET "localhost:9200/_cluster/health?wait_for_status=yellow&timeout=50s" - travis_retry composer self-update diff --git a/CHANGELOG.md b/CHANGELOG.md index f899acd3..ee5c068d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/) ## [Unreleased] +## [3.0.0] - 2019-11-17 +### Added +- Elasticsearch 7 support +- Added interface binding for HitsIteratorAggregate for custom implementation + ## [2.1.0] - 2019-11-13 ### Added - Import source factory diff --git a/Makefile b/Makefile index 38faeb17..ebb73c81 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ help: ## Show this help up: ## Start all containers (in background) for development sudo sysctl -w vm.max_map_count=262144 - $(docker_compose_bin) up --no-recreate -d + $(docker_compose_bin) up -d down: ## Stop all started for development containers $(docker_compose_bin) down diff --git a/README.md b/README.md index a2081150..4ee73293 100644 --- a/README.md +++ b/README.md @@ -25,12 +25,13 @@ great features, and at the same time leverage the complete set of ElasticSearch If you need any help, [stack overflow](https://stackoverflow.com/questions/tagged/laravel-scout%20laravel%20elasticsearch) is the preferred and recommended way to ask support questions. -## :two_hearts: Features +## :two_hearts: Features +Don't forget to :star: the package if you like it. :pray: - [Search amongst multiple models](#search-amongst-multiple-models) - [**Zero downtime** reimport](#zero-downtime-reimport) - it’s a breeze to import data in production. - [Eager load relations](#eager-load) - speed up your import. -- Elasticsearch **7.0** ready - Use [elasticsearch-7](https://github.com/matchish/laravel-scout-elasticsearch/tree/elasticsearch-7) branch instead. +- Elasticsearch **7.0** ready - Import all searchable models at once. - A fully configurable mapping for each model. - Full power of ElasticSearch in your queries. @@ -39,7 +40,11 @@ If you need any help, [stack overflow](https://stackoverflow.com/questions/tagge - PHP version >= 7.1.3 - Laravel Framework version >= 5.6 -- Elasticsearch version >= 6 + +| Elasticsearch version | ElasticsearchDSL version | +| --------------------- | --------------------------- | +| >= 7.0 | >= 3.0.0 | +| >= 6.0, < 7.0 | < 3.0.0 | ## :rocket: Installation @@ -190,7 +195,11 @@ Mixed::search('title:Barcelona or to:Barcelona') In this example you will get collection of `Ticket` and `Book` models where ticket's arrival city or book title is `Barcelona` ->Don't forget to :star: the package if you like it. :pray: +### Working with results +Often your response isn't collection of models but aggregations or models with higlights an so on. +In this case you need to implement your own implementation of `HitsIteratorAggregate` and bind it in your service provider + +[Here is a case](https://github.com/matchish/laravel-scout-elasticsearch/issues/28) ## :free: License Scout ElasticSearch is an open-sourced software licensed under the [MIT license](LICENSE.md). diff --git a/composer.json b/composer.json index 4624687d..afd73744 100644 --- a/composer.json +++ b/composer.json @@ -1,7 +1,7 @@ { "name": "matchish/laravel-scout-elasticsearch", "description": "This package extends Laravel Scout adding full power of ElasticSearch", - "version": "2.1.0", + "version": "3.0.0", "keywords": [ "laravel", "scout", @@ -18,9 +18,9 @@ ], "require": { "php": "^7.1.3", - "elasticsearch/elasticsearch": "~6.0", + "elasticsearch/elasticsearch": ">=7.2", "laravel/scout": "^6.1.1|^7.0", - "ongr/elasticsearch-dsl": "^6.0" + "ongr/elasticsearch-dsl": "^7.0" }, "require-dev": { "fzaninotto/faker": "^1.8", diff --git a/config/elasticsearch.php b/config/elasticsearch.php index 90b8c2fc..9a4a1dd9 100644 --- a/config/elasticsearch.php +++ b/config/elasticsearch.php @@ -5,11 +5,9 @@ 'indices' => [ 'mappings' => [ 'default' => [ - '_doc' => [ - 'properties' => [ - 'id' => [ - 'type' => 'keyword', - ], + 'properties' => [ + 'id' => [ + 'type' => 'keyword', ], ], ], diff --git a/docker-compose.yml b/docker-compose.yml index 253132d4..9583a224 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,16 +36,13 @@ services: networks: - default elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0 + image: docker.elastic.co/elasticsearch/elasticsearch:7.1.1 user: "1000:1000" volumes: - elasticsearch-data:/usr/share/elasticsearch/data environment: - - cluster.name=docker-cluster - - cluster.routing.allocation.disk.threshold_enabled=true - - cluster.routing.allocation.disk.watermark.flood_stage=200mb - - cluster.routing.allocation.disk.watermark.low=500mb - - cluster.routing.allocation.disk.watermark.high=300mb + - discovery.type=single-node + - cluster.routing.allocation.disk.threshold_enabled=false - bootstrap.memory_lock=true - "ES_JAVA_OPTS=-Xms512m -Xmx512m" ulimits: @@ -60,7 +57,8 @@ services: image: lmenezes/cerebro ports: - "9002:9000" - + networks: + - default networks: default: driver: bridge diff --git a/phpstan.neon.dist b/phpstan.neon.dist index 6ef936f3..d20fdfcf 100644 --- a/phpstan.neon.dist +++ b/phpstan.neon.dist @@ -12,3 +12,9 @@ parameters: - path: %currentWorkingDirectory%/src/Engines/ElasticSearchEngine.php message: '#message.*ServerErrorResponseException constructor expects string, string|false given.#' + - + path: %currentWorkingDirectory%/src/Engines/ElasticSearchEngine.php + message: '#Parameter \#2 \$search of function array_key_exists expects array, array\|\(callable\) given\.#' + - + path: %currentWorkingDirectory%/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php + message: '#Argument of an invalid type array\|\(callable\) supplied for foreach, only iterables are supported\.#' diff --git a/src/ElasticSearch/HitsIteratorAggregate.php b/src/ElasticSearch/HitsIteratorAggregate.php new file mode 100644 index 00000000..20032ea4 --- /dev/null +++ b/src/ElasticSearch/HitsIteratorAggregate.php @@ -0,0 +1,10 @@ +app->bind(Client::class, function () { return ClientBuilder::create()->setHosts([config('elasticsearch.host')])->build(); }); + + $this->app->bind( + 'Matchish\ScoutElasticSearch\ElasticSearch\HitsIteratorAggregate', + 'Matchish\ScoutElasticSearch\ElasticSearch\EloquentHitsIteratorAggregate' + ); } /** diff --git a/src/Engines/ElasticSearchEngine.php b/src/Engines/ElasticSearchEngine.php index 5bf010dd..5f2b83ea 100644 --- a/src/Engines/ElasticSearchEngine.php +++ b/src/Engines/ElasticSearchEngine.php @@ -7,7 +7,7 @@ use Laravel\Scout\Builder as BaseBuilder; use Laravel\Scout\Engines\Engine; use Laravel\Scout\Searchable; -use Matchish\ScoutElasticSearch\ElasticSearch\EloquentHitsIteratorAggregate; +use Matchish\ScoutElasticSearch\ElasticSearch\HitsIteratorAggregate; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Bulk; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Refresh; use Matchish\ScoutElasticSearch\ElasticSearch\Params\Search as SearchParams; @@ -28,7 +28,7 @@ final class ElasticSearchEngine extends Engine /** * Create a new engine instance. * - * @param \Elasticsearch\Client $elasticsearch + * @param \Elasticsearch\Client $elasticsearch * @return void */ public function __construct(\Elasticsearch\Client $elasticsearch) @@ -107,7 +107,10 @@ public function mapIds($results) */ public function map(BaseBuilder $builder, $results, $model) { - $hits = new EloquentHitsIteratorAggregate($results, $builder->queryCallback); + $hits = app()->makeWith(HitsIteratorAggregate::class, + ['results' => $results, + 'callback' => $builder->queryCallback, + ]); return new Collection($hits); } @@ -117,7 +120,7 @@ public function map(BaseBuilder $builder, $results, $model) */ public function getTotalCount($results) { - return $results['hits']['total']; + return $results['hits']['total']['value']; } /** diff --git a/src/Jobs/Stages/CleanUp.php b/src/Jobs/Stages/CleanUp.php index ff11ecb3..6308385e 100644 --- a/src/Jobs/Stages/CleanUp.php +++ b/src/Jobs/Stages/CleanUp.php @@ -31,7 +31,8 @@ public function handle(Client $elasticsearch): void $source = $this->source; $params = GetAliasParams::anyIndex($source->searchableAs()); try { - $response = $elasticsearch->indices()->getAliases($params->toArray()); + /** @var array $response */ + $response = $elasticsearch->indices()->getAlias($params->toArray()); } catch (Missing404Exception $e) { $response = []; } diff --git a/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php b/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php index 9257a4c0..05b726de 100644 --- a/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php +++ b/src/Jobs/Stages/SwitchToNewAndRemoveOldIndex.php @@ -36,7 +36,7 @@ public function handle(Client $elasticsearch): void { $source = $this->source; $params = Get::anyIndex($source->searchableAs()); - $response = $elasticsearch->indices()->getAliases($params->toArray()); + $response = $elasticsearch->indices()->getAlias($params->toArray()); $params = new Update(); foreach ($response as $indexName => $alias) { diff --git a/tests/Feature/FlushCommandTest.php b/tests/Feature/FlushCommandTest.php index 6f066c9e..0b3a9d93 100644 --- a/tests/Feature/FlushCommandTest.php +++ b/tests/Feature/FlushCommandTest.php @@ -38,6 +38,6 @@ public function test_clear_index(): void ]; $response = $this->elasticsearch->search($params); - $this->assertEquals(0, $response['hits']['total']); + $this->assertEquals(0, $response['hits']['total']['value']); } } diff --git a/tests/Feature/ImportCommandTest.php b/tests/Feature/ImportCommandTest.php index 3d98a254..22ca3e0d 100644 --- a/tests/Feature/ImportCommandTest.php +++ b/tests/Feature/ImportCommandTest.php @@ -40,7 +40,7 @@ public function test_import_entites(): void ], ]; $response = $this->elasticsearch->search($params); - $this->assertEquals($productsAmount, $response['hits']['total']); + $this->assertEquals($productsAmount, $response['hits']['total']['value']); } public function test_import_entites_in_queue(): void @@ -65,7 +65,7 @@ public function test_import_entites_in_queue(): void ], ]; $response = $this->elasticsearch->search($params); - $this->assertEquals($productsAmount, $response['hits']['total']); + $this->assertEquals($productsAmount, $response['hits']['total']['value']); } public function test_import_all_pages(): void @@ -89,7 +89,7 @@ public function test_import_all_pages(): void ], ]; $response = $this->elasticsearch->search($params); - $this->assertEquals($productsAmount, $response['hits']['total']); + $this->assertEquals($productsAmount, $response['hits']['total']['value']); } public function test_import_with_custom_key_all_pages(): void @@ -115,7 +115,7 @@ public function test_import_with_custom_key_all_pages(): void ], ]; $response = $this->elasticsearch->search($params); - $this->assertEquals($booksAmount, $response['hits']['total']); + $this->assertEquals($booksAmount, $response['hits']['total']['value']); } public function test_remove_old_index_after_switching_to_new(): void diff --git a/tests/Integration/Elasticsearch/HitsIteratorAggregateTest.php b/tests/Integration/Elasticsearch/HitsIteratorAggregateTest.php new file mode 100644 index 00000000..6a89eca4 --- /dev/null +++ b/tests/Integration/Elasticsearch/HitsIteratorAggregateTest.php @@ -0,0 +1,33 @@ +app->makeWith(HitsIteratorAggregate::class, [ + 'results' => [], + 'callback' => null, + ]); + + $this->assertEquals(EloquentHitsIteratorAggregate::class, get_class($iteratorAggregate)); + } + + public function test_override_bind_for_custom_iterator_aggregate_implementation() + { + $this->app->bind(HitsIteratorAggregate::class, CustomHitsIteratorAggregate::class); + + $iteratorAggregate = $this->app->makeWith(HitsIteratorAggregate::class, [ + 'results' => [], + 'callback' => null, + ]); + + $this->assertEquals(CustomHitsIteratorAggregate::class, get_class($iteratorAggregate)); + } +} diff --git a/tests/Integration/Engines/ElasticSearchEngineTest.php b/tests/Integration/Engines/ElasticSearchEngineTest.php index 58a3c0f6..0cd08d58 100644 --- a/tests/Integration/Engines/ElasticSearchEngineTest.php +++ b/tests/Integration/Engines/ElasticSearchEngineTest.php @@ -52,7 +52,7 @@ public function test_update() ], ]; $response = $this->elasticsearch->search($params); - $this->assertEquals($models->count(), $response['hits']['total']); + $this->assertEquals($models->count(), $response['hits']['total']['value']); foreach ($response['hits']['hits'] as $doc) { $this->assertEquals('Scout', $doc['_source']['title']); } @@ -93,7 +93,7 @@ public function test_delete() ], ]; $response = $this->elasticsearch->search($params); - $this->assertEquals(1, $response['hits']['total']); + $this->assertEquals(1, $response['hits']['total']['value']); foreach ($response['hits']['hits'] as $doc) { $this->assertEquals($shouldBeNotDeleted->getScoutKey(), $doc['_id']); } @@ -115,7 +115,7 @@ public function test_flush() ], ]; $response = $this->elasticsearch->search($params); - $this->assertEquals(0, $response['hits']['total']); + $this->assertEquals(0, $response['hits']['total']['value']); } public function test_map_with_custom_key_name() diff --git a/tests/Integration/Jobs/Stages/CreateWriteIndexTest.php b/tests/Integration/Jobs/Stages/CreateWriteIndexTest.php index dd54ea42..cf9325d5 100644 --- a/tests/Integration/Jobs/Stages/CreateWriteIndexTest.php +++ b/tests/Integration/Jobs/Stages/CreateWriteIndexTest.php @@ -18,7 +18,7 @@ public function test_create_write_index(): void $elasticsearch = $this->app->make(Client::class); $stage = new CreateWriteIndex(DefaultImportSourceFactory::from(Product::class), Index::fromSource(DefaultImportSourceFactory::from(Product::class))); $stage->handle($elasticsearch); - $response = $elasticsearch->indices()->getAliases(['index' => '*', 'name' => 'products']); + $response = $elasticsearch->indices()->getAlias(['index' => '*', 'name' => 'products']); $this->assertTrue($this->containsWriteIndex($response, 'products')); } diff --git a/tests/Integration/Jobs/Stages/PullFromSourceTest.php b/tests/Integration/Jobs/Stages/PullFromSourceTest.php index 8cbe31f6..d99f7bb6 100644 --- a/tests/Integration/Jobs/Stages/PullFromSourceTest.php +++ b/tests/Integration/Jobs/Stages/PullFromSourceTest.php @@ -40,7 +40,7 @@ public function test_put_all_entites_to_index(): void ], ]; $response = $this->elasticsearch->search($params); - $this->assertEquals($productsAmount, $response['hits']['total']); + $this->assertEquals($productsAmount, $response['hits']['total']['value']); } public function test_dont_put_entities_if_no_entities_in_collection(): void @@ -63,7 +63,7 @@ public function test_dont_put_entities_if_no_entities_in_collection(): void ], ]; $response = $this->elasticsearch->search($params); - $this->assertEquals(0, $response['hits']['total']); + $this->assertEquals(0, $response['hits']['total']['value']); } public function test_put_all_to_index_if_amount_of_entities_more_than_chunk_size(): void @@ -95,7 +95,7 @@ public function test_put_all_to_index_if_amount_of_entities_more_than_chunk_size ], ]; $response = $this->elasticsearch->search($params); - $this->assertEquals($productsAmount, $response['hits']['total']); + $this->assertEquals($productsAmount, $response['hits']['total']['value']); } public function test_pull_soft_delete_meta_data() @@ -163,7 +163,7 @@ public function test_pull_soft_deleted() ], ]; $response = $this->elasticsearch->search($params); - $this->assertEquals(3, $response['hits']['total']); + $this->assertEquals(3, $response['hits']['total']['value']); } public function test_no_searchables_no_chunks() @@ -199,6 +199,6 @@ public function test_chunked_pull_only_one_page() ]; $response = $this->elasticsearch->search($params); - $this->assertEquals(3, $response['hits']['total']); + $this->assertEquals(3, $response['hits']['total']['value']); } } diff --git a/tests/Integration/Jobs/Stages/RefreshIndexTest.php b/tests/Integration/Jobs/Stages/RefreshIndexTest.php index 39139b7e..deaec211 100644 --- a/tests/Integration/Jobs/Stages/RefreshIndexTest.php +++ b/tests/Integration/Jobs/Stages/RefreshIndexTest.php @@ -41,6 +41,6 @@ public function test_refresh_index(): void ], ]; $response = $this->elasticsearch->search($params); - $this->assertEquals(1, $response['hits']['total']); + $this->assertEquals(1, $response['hits']['total']['value']); } } diff --git a/tests/TestCase.php b/tests/TestCase.php index aeb605da..a236aaea 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -39,16 +39,14 @@ protected function getEnvironmentSetUp($app) $app['config']->set('database.default', 'mysql'); $app['config']->set('elasticsearch', require(__DIR__.'/../config/elasticsearch.php')); $app['config']->set('elasticsearch.indices.mappings.products', [ - '_doc' => [ - 'properties' => [ - 'type' => [ - 'type' => 'keyword', - ], - 'price' => [ - 'type' => 'integer', - ], - ], + 'properties' => [ + 'type' => [ + 'type' => 'keyword', ], + 'price' => [ + 'type' => 'integer', + ], + ], ]); } diff --git a/tests/Unit/Engines/ElasticSearchEngineTest.php b/tests/Unit/Engines/ElasticSearchEngineTest.php index 5287a208..ca886efe 100644 --- a/tests/Unit/Engines/ElasticSearchEngineTest.php +++ b/tests/Unit/Engines/ElasticSearchEngineTest.php @@ -7,7 +7,6 @@ use Laravel\Scout\Builder; use Matchish\ScoutElasticSearch\Engines\ElasticSearchEngine; use Mockery as m; -use ONGR\ElasticsearchDSL\Search; use Tests\TestCase; class ElasticSearchEngineTest extends TestCase @@ -36,10 +35,10 @@ public function test_pass_search_builder_to_callback() { $client = m::mock(Client::class); $engine = new ElasticSearchEngine($client); - $client->shouldReceive('search')->once()->with(m::type(Search::class)); + $client->shouldReceive('search')->once()->with(m::type('array')); $query = 'zonda'; $builder = new Builder(new Product(), $query, function ($client, $query) { - return $client->search($query); + return $client->search($query->toArray()); }); $engine->search($builder); } diff --git a/tests/laravel/app/library/CustomHitsIteratorAggregate.php b/tests/laravel/app/library/CustomHitsIteratorAggregate.php new file mode 100644 index 00000000..99bc2ee5 --- /dev/null +++ b/tests/laravel/app/library/CustomHitsIteratorAggregate.php @@ -0,0 +1,26 @@ +results = $results; + + $this->callback = $callback; + } + + public function getIterator() + { + $hits = ['test1', 'test2', 'test3']; + + return new \ArrayIterator($hits); + } +} diff --git a/tests/laravel/config/elasticsearch.php b/tests/laravel/config/elasticsearch.php index 8cfb8955..b414dbbc 100644 --- a/tests/laravel/config/elasticsearch.php +++ b/tests/laravel/config/elasticsearch.php @@ -4,17 +4,15 @@ 'indices' => [ 'mappings' => [ 'default' => [ - '_doc' => [ - 'properties' => [ - 'created_at' => [ - 'type' => 'date', - ], - 'updated_at' => [ - 'type' => 'date', - ], - 'deleted_at' => [ - 'type' => 'date', - ], + 'properties' => [ + 'created_at' => [ + 'type' => 'date', + ], + 'updated_at' => [ + 'type' => 'date', + ], + 'deleted_at' => [ + 'type' => 'date', ], ], ],