Merge pull request #15 from matchish/progress

Add progress report

CHANGELOG.md

@@ -5,8 +5,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/)
 
 ## [Unreleased]
+
+## [2.0.1] - 2019-05-06
 ### Added
-- Progress bar for console commands
+- Progress report for console commands
 
 ## [2.0.0] - 2019-04-09
 ### Added

README.md

@@ -2,6 +2,8 @@
   <a href="https://github.com/matchish/laravel-scout-elasticsearch">
     <img alt="Scout ElasticSearch" src="https://raw.githubusercontent.com/matchish/laravel-scout-elasticsearch/master/docs/banner.svg?sanitize=true" >
   </a>
+
+  <img alt="Import progress report" src="https://raw.githubusercontent.com/matchish/laravel-scout-elasticsearch/master/docs/demo.gif" >
 
   <p align="center">
     <a href="https://travis-ci.com/matchish/laravel-scout-elasticsearch"><img src="https://img.shields.io/travis/com/matchish/laravel-scout-elasticsearch.svg" alt="Build Status"></img></a>

composer.json

@@ -1,7 +1,7 @@
 {
     "name": "matchish/laravel-scout-elasticsearch",
     "description": "This package extends Laravel Scout adding full power of ElasticSearch",
-    "version": "2.0.0",
+    "version": "2.0.1",
     "keywords": [
         "laravel",
         "scout",
@@ -20,7 +20,6 @@
         "php": "^7.1.3",
         "elasticsearch/elasticsearch": "~6.0",
         "laravel/scout": "^6.1.1|^7.0",
-        "league/pipeline": "^1.0",
         "ongr/elasticsearch-dsl": "^6.0"
     },
     "require-dev": {

resources/lang/en/flush.php

@@ -0,0 +1,5 @@
+<?php
+
+return [
+    'done' => 'All [:searchable] records have been flushed.',
+];

resources/lang/en/import.php

@@ -0,0 +1,7 @@
+<?php
+
+return [
+    'start' => 'Importing [:searchable]',
+    'done' => 'All [:searchable] records have been imported.',
+    'done.queue' => 'Import job dispatched to the queue.',
+];

src/Console/Commands/FlushCommand.php

@@ -32,7 +32,10 @@ public function handle(): void
         });
         $searchableList->each(function ($searchable) {
             $searchable::removeAllFromSearch();
-            $this->output->success('All ['.$searchable.'] records have been flushed.');
+            $doneMessage = trans('scout::flush.done', [
+                'searchable' => $searchable,
+            ]);
+            $this->output->success($doneMessage);
         });
     }
 }

src/Console/Commands/ImportCommand.php

@@ -6,6 +6,7 @@
 
 use Illuminate\Console\Command;
 use Matchish\ScoutElasticSearch\Jobs\Import;
+use Matchish\ScoutElasticSearch\Jobs\QueueableJob;
 use Matchish\ScoutElasticSearch\Searchable\SearchableListFactory;
 
 final class ImportCommand extends Command
@@ -24,22 +25,41 @@ final class ImportCommand extends Command
      */
     public function handle(): void
     {
-        $command = $this;
-        $searchableList = collect($command->argument('searchable'))->whenEmpty(function () {
+        $this->searchableList($this->argument('searchable'))
+        ->each(function ($searchable) {
+            $this->import($searchable);
+        });
+    }
+
+    private function searchableList($argument)
+    {
+        return collect($argument)->whenEmpty(function () {
             $factory = new SearchableListFactory(app()->getNamespace(), app()->path());
 
             return $factory->make();
         });
-        $searchableList->each(function ($searchable) {
-            $job = new Import($searchable);
-            if (config('scout.queue')) {
-                dispatch($job)->allOnQueue((new $searchable)->syncWithSearchUsingQueue())
-                    ->allOnConnection(config((new $searchable)->syncWithSearchUsing()));
-                $this->output->success('All ['.$searchable.'] records have been dispatched to import job.');
-            } else {
-                dispatch_now($job);
-                $this->output->success('All ['.$searchable.'] records have been searchable.');
-            }
-        });
+    }
+
+    private function import($searchable)
+    {
+        $job = new Import($searchable);
+
+        if (config('scout.queue')) {
+            $job = (new QueueableJob())->chain([$job]);
+        }
+
+        $bar = (new ProgressBarFactory($this->output))->create();
+        $job->withProgressReport($bar);
+
+        $startMessage = trans('scout::import.start', ['searchable' => "<comment>$searchable</comment>"]);
+        $this->line($startMessage);
+
+        dispatch($job)->allOnQueue((new $searchable)->syncWithSearchUsingQueue())
+            ->allOnConnection(config((new $searchable)->syncWithSearchUsing()));
+
+        $doneMessage = trans(config('scout.queue') ? 'scout::import.done.queue' : 'scout::import.done', [
+            'searchable' => $searchable,
+        ]);
+        $this->output->success($doneMessage);
     }
 }

src/Console/Commands/ProgressBarFactory.php

@@ -0,0 +1,34 @@
+<?php
+
+namespace Matchish\ScoutElasticSearch\Console\Commands;
+
+use Symfony\Component\Console\Style\OutputStyle;
+
+class ProgressBarFactory
+{
+    /**
+     * @var OutputStyle
+     */
+    private $output;
+
+    /**
+     * @param OutputStyle $output
+     */
+    public function __construct(OutputStyle $output)
+    {
+        $this->output = $output;
+    }
+
+    public function create(int $max = 0)
+    {
+        $bar = $this->output->createProgressBar($max);
+        $bar->setBarCharacter('<fg=green>⚬</>');
+        $bar->setEmptyBarCharacter('<fg=red>⚬</>');
+        $bar->setProgressCharacter('<fg=green>➤</>');
+        $bar->setFormat(
+            "%message%\n%current%/%max% [%bar%] %percent:3s%%\n"
+        );
+
+        return $bar;
+    }
+}

src/Engines/ElasticSearchEngine.php

@@ -8,7 +8,6 @@
 use Laravel\Scout\Builder as BaseBuilder;
 use ONGR\ElasticsearchDSL\Query\MatchAllQuery;
 use Matchish\ScoutElasticSearch\ElasticSearch\Index;
-use Matchish\ScoutElasticSearch\Pipelines\ImportPipeline;
 use Matchish\ScoutElasticSearch\ElasticSearch\Params\Bulk;
 use Matchish\ScoutElasticSearch\ElasticSearch\SearchFactory;
 use Matchish\ScoutElasticSearch\ElasticSearch\SearchResults;
@@ -62,10 +61,13 @@ public function delete($models)
     public function flush($model)
     {
         $indexName = $model->searchableAs();
-        $body = (new Search())->addQuery(new MatchAllQuery())->toArray();
-        $params = new SearchParams($indexName, $body);
-        $this->elasticsearch->deleteByQuery($params->toArray());
-        $this->elasticsearch->indices()->refresh((new Refresh($indexName))->toArray());
+        $exist = $this->elasticsearch->indices()->exists(['index' => $indexName]);
+        if ($exist) {
+            $body = (new Search())->addQuery(new MatchAllQuery())->toArray();
+            $params = new SearchParams($indexName, $body);
+            $this->elasticsearch->deleteByQuery($params->toArray());
+            $this->elasticsearch->indices()->refresh((new Refresh($indexName))->toArray());
+        }
     }
 
     /**
@@ -114,15 +116,6 @@ public function getTotalCount($results)
         return $results['hits']['total'];
     }
 
-    /**
-     * @internal
-     */
-    public function sync($model)
-    {
-        $pipeline = new ImportPipeline($this->elasticsearch);
-        $pipeline->process([Index::fromSearchable($model), $model]);
-    }
-
     /**
      * @param BaseBuilder $builder
      * @param array $options

src/Jobs/Import.php

@@ -2,16 +2,18 @@
 
 namespace Matchish\ScoutElasticSearch\Jobs;
 
+use Elasticsearch\Client;
 use Illuminate\Bus\Queueable;
-use Illuminate\Contracts\Queue\ShouldQueue;
-use Matchish\ScoutElasticSearch\Engines\ElasticSearchEngine;
+use Illuminate\Support\Collection;
+use Matchish\ScoutElasticSearch\ProgressReportable;
 
 /**
  * @internal
  */
-final class Import implements ShouldQueue
+final class Import
 {
     use Queueable;
+    use ProgressReportable;
 
     /**
      * @var string
@@ -27,10 +29,22 @@ public function __construct(string $searchable)
     }
 
     /**
-     * @param ElasticSearchEngine $engine
+     * @param Client $elasticsearch
      */
-    public function handle(ElasticSearchEngine $engine): void
+    public function handle(Client $elasticsearch): void
     {
-        $engine->sync(new $this->searchable);
+        $stages = $this->stages();
+        $estimate = $stages->sum->estimate();
+        $this->progressBar()->setMaxSteps($estimate);
+        $stages->each(function ($stage) use ($elasticsearch) {
+            $this->progressBar()->setMessage($stage->title());
+            $stage->handle($elasticsearch);
+            $this->progressBar()->advance($stage->estimate());
+        });
+    }
+
+    private function stages(): Collection
+    {
+        return ImportStages::fromSearchable(new $this->searchable);
     }
 }

src/Jobs/ImportStages.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace Matchish\ScoutElasticSearch\Jobs;
+
+use Illuminate\Support\Collection;
+use Illuminate\Database\Eloquent\Model;
+use Matchish\ScoutElasticSearch\ElasticSearch\Index;
+use Matchish\ScoutElasticSearch\Jobs\Stages\CleanUp;
+use Matchish\ScoutElasticSearch\Jobs\Stages\RefreshIndex;
+use Matchish\ScoutElasticSearch\Jobs\Stages\PullFromSource;
+use Matchish\ScoutElasticSearch\Jobs\Stages\CreateWriteIndex;
+use Matchish\ScoutElasticSearch\Jobs\Stages\SwitchToNewAndRemoveOldIndex;
+
+class ImportStages extends Collection
+{
+    /**
+     * @param Model $searchable
+     * @return Collection
+     */
+    public static function fromSearchable(Model $searchable)
+    {
+        $index = Index::fromSearchable($searchable);
+        $main = PullFromSource::chunked($searchable);
+        if ($main->count()) {
+            return (new static([
+                new CleanUp($searchable),
+                new CreateWriteIndex($searchable, $index),
+                PullFromSource::chunked($searchable),
+                new RefreshIndex($index),
+                new SwitchToNewAndRemoveOldIndex($searchable, $index),
+            ]))->flatten();
+        } else {
+            return new static;
+        }
+    }
+}

src/Jobs/QueueableJob.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace Matchish\ScoutElasticSearch\Jobs;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Matchish\ScoutElasticSearch\ProgressReportable;
+
+class QueueableJob implements ShouldQueue
+{
+    use Queueable;
+    use ProgressReportable;
+
+    public function handle()
+    {
+    }
+}

src/Jobs/Stages/CleanUp.php

@@ -0,0 +1,60 @@
+<?php
+
+namespace Matchish\ScoutElasticSearch\Jobs\Stages;
+
+use Elasticsearch\Client;
+use Laravel\Scout\Searchable;
+use Illuminate\Database\Eloquent\Model;
+use Elasticsearch\Common\Exceptions\Missing404Exception;
+use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Alias\Get as GetAliasParams;
+use Matchish\ScoutElasticSearch\ElasticSearch\Params\Indices\Delete as DeleteIndexParams;
+
+/**
+ * @internal
+ */
+final class CleanUp
+{
+    /**
+     * @var Model
+     */
+    private $searchable;
+
+    /**
+     * @param Model $searchable
+     */
+    public function __construct(Model $searchable)
+    {
+        $this->searchable = $searchable;
+    }
+
+    public function handle(Client $elasticsearch): void
+    {
+        /** @var Searchable $searchable */
+        $searchable = $this->searchable;
+        $params = GetAliasParams::anyIndex($searchable->searchableAs());
+        try {
+            $response = $elasticsearch->indices()->getAliases($params->toArray());
+        } catch (Missing404Exception $e) {
+            $response = [];
+        }
+        foreach ($response as $indexName => $data) {
+            foreach ($data['aliases'] as $alias => $config) {
+                if (array_key_exists('is_write_index', $config) && $config['is_write_index']) {
+                    $params = new DeleteIndexParams((string) $indexName);
+                    $elasticsearch->indices()->delete($params->toArray());
+                    continue 2;
+                }
+            }
+        }
+    }
+
+    public function title(): string
+    {
+        return 'Clean up';
+    }
+
+    public function estimate(): int
+    {
+        return 1;
+    }
+}