Merge pull request #55 from Jeroen-G/index-updater

Index updater

changelog.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - The new match_phrase query (#73)
 - Ability to optionally set more parameters for a few queries (#68)
+- Index aliases, useful for zero downtime indexing
 
 ### Changed
 - Connection configuration now can include API ID and key (#74)

config/explorer.php

@@ -21,4 +21,10 @@
     'indexes' => [
         // \App\Models\Post::class
     ],
+
+    /**
+     * You may opt to keep the old indices after the alias is pointed to a new index.
+     * A model is only using index aliases if it implements the Aliased interface.
+     */
+    'prune_old_aliases' => true,
 ];

docs/index-aliases.md

@@ -0,0 +1,56 @@
+# Index aliases
+Aliases allow you to use an index under a different name.This very useful for zero downtime deployments.
+
+There are three aliases created: a read alias, a write alias and a history alias.
+The read alias is used for reading, the write index is used for writing.
+The history aggregates all old indices, which can be pruned.
+When updating an index it is recreated to a new index with a unique name.
+The "write" alias is pointed to the new index and all Scout updates will be forwarded to the "write" index.
+After all entities are imported the "read" alias will also be pointed to the new index.
+
+If you wish to keep the old indices set `prune_old_aliases` to false in `config/explorer.php`
+
+## Using aliases
+A model is only using index aliases if it implements the Aliased interface or enabled in the configuration (see [mapping](mapping.md)).
+
+After that, any time you use the `scout:import` command a new index will be created and when the insertion of models is done the alias will be pointed to the new index.
+
+```php
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+use JeroenG\Explorer\Application\Explored;
+use JeroenG\Explorer\Application\Aliased;
+use Laravel\Scout\Searchable;
+
+class Post extends Model implements Explored, Aliased
+{
+    use HasFactory;
+    use Searchable;
+
+    //...
+}
+```
+
+```php
+return [
+    'indexes' => [
+        'posts' => [
+            'aliased' => true,
+            'properties' => [
+                'id' => 'keyword',
+                'title' => 'text',
+                'created_at' => 'date',
+                'published' => 'boolean',
+                'author' => 'nested',
+            ],
+        ],
+    ],
+];
+```
+
+Be aware that if you currently already have indices and would like to move to using aliases you will need to delete those indices before configuring the aliases.
+In Elasticsearch a given name can only be either an index or alias, not both and this cannot be changed on-the-fly. 

docs/index.md

@@ -34,6 +34,7 @@ Also do not forget to follow the [installation instructions for Laravel Scout](h
 - [Preparing data](preparing-data.md)
 - [Advanced queries](advanced-queries.md)
 - [Advanced index settings](index-settings.md)
+- [Index aliases](index-aliases.md)
 - [Aggregations](aggregations.md)
 
 [ico-version]: https://img.shields.io/packagist/v/jeroen-g/explorer.svg?style=flat-square

src/Application/Aliased.php

@@ -0,0 +1,9 @@
+<?php
+
+declare(strict_types=1);
+
+namespace JeroenG\Explorer\Application;
+
+interface Aliased
+{
+}

src/Application/IndexAdapterInterface.php

@@ -8,9 +8,17 @@
 
 interface IndexAdapterInterface
 {
+    public function getRemoteConfiguration(IndexConfigurationInterface $indexConfiguration): ?IndexConfigurationInterface;
+
+    public function getInactiveIndexForAlias(IndexConfigurationInterface $indexConfiguration): ?string;
+
     public function create(IndexConfigurationInterface $indexConfiguration): void;
 
+    public function pointToAlias(IndexConfigurationInterface $indexConfiguration): void;
+
     public function delete(IndexConfigurationInterface $indexConfiguration): void;
 
     public function flush(string $index): void;
+
+    public function createNewInactiveIndex(IndexConfigurationInterface $indexConfiguration): string;
 }

src/Application/Operations/Bulk/BulkUpdateOperation.php

@@ -7,14 +7,21 @@
 use JeroenG\Explorer\Application\BePrepared;
 use JeroenG\Explorer\Application\Explored;
 
-class BulkUpdateOperation implements BulkOperationInterface
+final class BulkUpdateOperation implements BulkOperationInterface
 {
     /** @var Explored[] */
     private array $models = [];
 
-    public static function from(iterable $iterable): self
+    private static string $indexName;
+
+    public function __construct(string $indexName)
+    {
+        self::$indexName = $indexName;
+    }
+
+    public static function from(iterable $iterable, string $indexName): self
     {
-        $operation = new self();
+        $operation = new self($indexName);
 
         if ($iterable instanceof \Traversable) {
             $operation->models = iterator_to_array($iterable);
@@ -34,17 +41,17 @@ public function build(): array
     {
         $payload = [];
         foreach ($this->models as $model) {
-            $payload[] = self::modelToBulkAction($model);
+            $payload[] = self::bulkActionSettings($model);
             $payload[] = self::modelToData($model);
         }
         return $payload;
     }
 
-    private static function modelToBulkAction(Explored $model): array
+    private static function bulkActionSettings(Explored $model): array
     {
         return [
             'index' => [
-                '_index' => $model->searchableAs(),
+                '_index' => self::$indexName,
                 '_id' => $model->getScoutKey(),
             ]
         ];

src/Domain/IndexManagement/IndexAliasConfiguration.php

@@ -0,0 +1,48 @@
+<?php
+
+declare(strict_types=1);
+
+namespace JeroenG\Explorer\Domain\IndexManagement;
+
+final class IndexAliasConfiguration implements IndexAliasConfigurationInterface
+{
+    private string $name;
+
+    private bool $pruneOldAliases;
+
+    private function __construct(string $name, bool $pruneOldAliases)
+    {
+        $this->name = $name;
+        $this->pruneOldAliases = $pruneOldAliases;
+    }
+
+    public static function create(string $name, bool $pruneOldAliases = true): IndexAliasConfiguration
+    {
+        return new self($name, $pruneOldAliases);
+    }
+
+    public function shouldOldAliasesBePruned(): bool
+    {
+        return $this->pruneOldAliases;
+    }
+
+    public function getIndexName(): string
+    {
+        return $this->name;
+    }
+
+    public function getAliasName(): string
+    {
+        return $this->name;
+    }
+
+    public function getHistoryAliasName(): string
+    {
+        return $this->name . '-history';
+    }
+
+    public function getWriteAliasName(): string
+    {
+        return $this->name . '-write';
+    }
+}

src/Domain/IndexManagement/IndexAliasConfigurationInterface.php

@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+namespace JeroenG\Explorer\Domain\IndexManagement;
+
+interface IndexAliasConfigurationInterface
+{
+    public function shouldOldAliasesBePruned(): bool;
+
+    public function getIndexName(): string;
+
+    public function getAliasName(): string;
+
+    public function getHistoryAliasName(): string;
+
+    public function getWriteAliasName(): string;
+}

src/Domain/IndexManagement/IndexConfiguration.php

@@ -4,22 +4,35 @@
 
 namespace JeroenG\Explorer\Domain\IndexManagement;
 
+use Webmozart\Assert\Assert;
+
 final class IndexConfiguration implements IndexConfigurationInterface
 {
     private string $name;
 
+    private ?string $model;
+
     private array $settings = [];
 
     private array $properties = [];
 
-    private function __construct(string $name)
+    private ?IndexAliasConfigurationInterface $aliasConfiguration;
+
+    private function __construct(string $name, ?string $model, ?IndexAliasConfigurationInterface $aliasConfiguration = null)
     {
+        $this->model = $model;
         $this->name = $name;
+        $this->aliasConfiguration = $aliasConfiguration;
     }
 
-    public static function create(string $name, array $properties, array $settings): self
-    {
-        $config = new self($name);
+    public static function create(
+        string $name,
+        array $properties,
+        array $settings,
+        ?string $model = null,
+        ?IndexAliasConfigurationInterface $aliasConfiguration = null
+    ): self {
+        $config = new self($name, $model, $aliasConfiguration);
         $config->properties = $properties;
         $config->settings = $settings;
 
@@ -31,6 +44,28 @@ public function getName(): string
         return $this->name;
     }
 
+    public function getModel(): string
+    {
+        return $this->model;
+    }
+
+    public function isAliased(): bool
+    {
+        return !is_null($this->aliasConfiguration);
+    }
+
+    public function getAliasConfiguration(): IndexAliasConfigurationInterface
+    {
+        Assert::notNull($this->aliasConfiguration);
+
+        return $this->aliasConfiguration;
+    }
+
+    public function getReadIndexName(): string
+    {
+        return $this->isAliased() ? $this->getAliasConfiguration()->getIndexName() : $this->getName();
+    }
+
     public function getProperties(): array
     {
         return $this->properties;
@@ -41,22 +76,13 @@ public function getSettings(): array
         return $this->settings;
     }
 
-    public function toArray(): array
+    public function getWriteIndexName()
     {
-        $config = [
-            'index' => $this->getName(),
-        ];
-
-        if (!empty($this->settings)) {
-            $config['body']['settings'] = $this->getSettings();
-        }
-
-        if (!empty($this->properties)) {
-            $config['body']['mappings'] = [
-                'properties' => $this->getProperties()
-            ];
-        }
+        return $this->isAliased() ? $this->getAliasConfiguration()->getWriteAliasName() : $this->getName();
+    }
 
-        return $config;
+    private function getAliasedName(): string
+    {
+        return sprintf('%s-%d', $this->name, time());
     }
 }

src/Domain/IndexManagement/IndexConfigurationInterface.php

@@ -8,9 +8,17 @@ interface IndexConfigurationInterface
 {
     public function getName(): string;
 
+    public function getModel(): ?string;
+
+    public function isAliased(): bool;
+
+    public function getAliasConfiguration(): IndexAliasConfigurationInterface;
+
     public function getProperties(): array;
 
     public function getSettings(): array;
 
-    public function toArray(): array;
+    public function getReadIndexName(): string;
+
+    public function getWriteIndexName();
 }

src/Domain/IndexManagement/IndexConfigurationNotFoundException.php

@@ -4,7 +4,6 @@
 
 namespace JeroenG\Explorer\Domain\IndexManagement;
 
-
 class IndexConfigurationNotFoundException extends \InvalidArgumentException
 {
     public static function index(string $name): IndexConfigurationNotFoundException

src/Domain/Query/Query.php

@@ -4,8 +4,6 @@
 
 namespace JeroenG\Explorer\Domain\Query;
 
-use JeroenG\Explorer\Domain\Aggregates\AggregationItem;
-use JeroenG\Explorer\Domain\Aggregates\Aggregations;
 use JeroenG\Explorer\Domain\Aggregations\AggregationSyntaxInterface;
 use JeroenG\Explorer\Domain\Syntax\Sort;
 use JeroenG\Explorer\Domain\Syntax\SyntaxInterface;

src/Domain/Syntax/Wildcard.php

@@ -12,7 +12,7 @@ class Wildcard implements SyntaxInterface
 
     private float $boost;
 
-    public function __construct (
+    public function __construct(
         string $field,
         string $value,
         float $boost = 1.0