Implement keyword search backend (#8539)

* add tha package * add migration * add columns from diff tables * change the model * configure engine * update user model * generate scout.php * modify scopeGeneralSearch * make searchable column as tsvector type * accept array of string as search term * add only necessary columns to index * improve the migration to be readable * fix linting * fix model so that UserSeeder would run * allow re-index of users after seeding so existing users are searchable * add missing attributes in user model * Modify the searching from OR clause to AND clause * ensure changes to experiences updates user model * sync the updates to notes column with the index on the user model * remove unnecessary composer files * address the feedback the migration * php linting fixes * fix user.php for php linting * update deps * add null verification * fix the test for the new change * fix linting * trim it * add translation * trigger searchable on deleting exp and user creation * update index on user deletion * remove leading slashes * add a test file for keyword search * change copy for the dropdown label and add translation to it * add tests for search after create,update,delete * add tests for search by each type of work experience * address feedback * fix linting * fix linting errors * fix the null value clean up * optimize updating notes * address feedback on migration * simplify searchable array for readability * improve tests as per the pr feedback * Revert "simplify searchable array for readability" This reverts commit 46b35f1. * optimize the query to get only ids for the first time * array refactor and fix empty array bug * move take to ID query * turn after_commit to true * add drop index * limit IDs being passed to in clause * fix extra database query when making searchable array --------- Co-authored-by: Peter Giles <[email protected]>
GCTC-NTGC · Dec 22, 2023 · b96b1cb · b96b1cb
1 parent 408b3a0
commit b96b1cb
Show file tree

Hide file tree

Showing 23 changed files with 1,022 additions and 44 deletions.
diff --git a/api/.env.example b/api/.env.example
@@ -103,3 +103,6 @@ FEATURE_RECORD_OF_DECISION=false
 # MAINTENANCE_BANNER_PUBLIC_DATE="2023-09-07 00:00:00"
 # MAINTENANCE_BANNER_START_DATE="2023-09-09 12:00:00"
 # MAINTENANCE_BANNER_DURATION=1 # duration unit is hours (e.g. 4 hours)
+
+# Laravel Scout Driver
+SCOUT_DRIVER=pgsql
diff --git a/api/app/Models/AwardExperience.php b/api/app/Models/AwardExperience.php
@@ -3,6 +3,7 @@
 namespace App\Models;
 
 use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
 use Illuminate\Database\Eloquent\SoftDeletes;
 
 /**
@@ -32,4 +33,10 @@ class AwardExperience extends Experience
     protected $casts = [
         'awarded_date' => 'date',
     ];
+
+    // Define the relationship to the User model
+    public function user(): BelongsTo
+    {
+        return $this->belongsTo(User::class);
+    }
 }
diff --git a/api/app/Models/CommunityExperience.php b/api/app/Models/CommunityExperience.php
@@ -3,6 +3,7 @@
 namespace App\Models;
 
 use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
 use Illuminate\Database\Eloquent\SoftDeletes;
 
 /**
@@ -33,4 +34,10 @@ class CommunityExperience extends Experience
         'start_date' => 'date',
         'end_date' => 'date',
     ];
+
+    // Define the relationship to the User model
+    public function user(): BelongsTo
+    {
+        return $this->belongsTo(User::class);
+    }
 }
diff --git a/api/app/Models/EducationExperience.php b/api/app/Models/EducationExperience.php
@@ -3,6 +3,7 @@
 namespace App\Models;
 
 use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
 use Illuminate\Database\Eloquent\SoftDeletes;
 
 /**
@@ -35,4 +36,10 @@ class EducationExperience extends Experience
         'start_date' => 'date',
         'end_date' => 'date',
     ];
+
+    // Define the relationship to the User model
+    public function user(): BelongsTo
+    {
+        return $this->belongsTo(User::class);
+    }
 }
diff --git a/api/app/Models/Experience.php b/api/app/Models/Experience.php
@@ -147,4 +147,23 @@ public function disconnectSkills($skillIds)
         // If this experience instance continues to be used, ensure the in-memory instance is updated.
         $this->refresh();
     }
+
+    protected static function boot()
+    {
+        parent::boot();
+
+        static::saving(function ($experience) {
+            $user = $experience->user;
+            if ($user) {
+                $user->searchable();
+            }
+        });
+
+        static::deleted(function ($experience) {
+            $user = $experience->user;
+            if ($user) {
+                $user->searchable();
+            }
+        });
+    }
 }
diff --git a/api/app/Models/PersonalExperience.php b/api/app/Models/PersonalExperience.php
@@ -3,6 +3,7 @@
 namespace App\Models;
 
 use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
 use Illuminate\Database\Eloquent\SoftDeletes;
 
 /**
@@ -32,4 +33,10 @@ class PersonalExperience extends Experience
         'start_date' => 'date',
         'end_date' => 'date',
     ];
+
+    // Define the relationship to the User model
+    public function user(): BelongsTo
+    {
+        return $this->belongsTo(User::class);
+    }
 }
diff --git a/api/app/Models/PoolCandidate.php b/api/app/Models/PoolCandidate.php
@@ -81,6 +81,8 @@ class PoolCandidate extends Model
         'pool_candidate_status',
     ];
 
+    protected $touches = ['user'];
+
     /**
      * The "booted" method of the model.
      */
@@ -89,6 +91,20 @@ protected static function booted(): void
         PoolCandidate::observe(PoolCandidateObserver::class);
     }
 
+    public static function boot()
+    {
+        parent::boot();
+
+        static::updating(function ($model) {
+            // Check if the 'notes' attribute is being updated and if so, update the searchable user model
+            // Seems to work without this but not sure why
+            if ($model->user()->exists() && $model->isDirty('notes')) {
+                $model->user()->searchable();
+            }
+
+        });
+    }
+
     public function getActivitylogOptions(): LogOptions
     {
         return LogOptions::defaults()
@@ -346,17 +362,15 @@ public function scopeEquity(Builder $query, ?array $equity): Builder
         return $query;
     }
 
-    public function scopeGeneralSearch(Builder $query, ?string $search): Builder
+    public function scopeGeneralSearch(Builder $query, ?array $searchTerms): Builder
     {
-        if (empty($search)) {
+        if (empty($searchTerms)) {
             return $query;
         }
 
-        $query->where(function ($query) use ($search) {
-            $query->whereHas('user', function ($query) use ($search) {
-                User::scopeGeneralSearch($query, $search);
-            })->orWhere(function ($query) use ($search) {
-                self::scopeNotes($query, $search);
+        $query->where(function ($query) use ($searchTerms) {
+            $query->whereHas('user', function ($query) use ($searchTerms) {
+                User::scopeGeneralSearch($query, $searchTerms);
             });
         });
 

diff --git a/api/app/Models/User.php b/api/app/Models/User.php
@@ -22,6 +22,7 @@
 use Illuminate\Support\Facades\DB;
 use Laratrust\Contracts\LaratrustUser;
 use Laratrust\Traits\HasRolesAndPermissions;
+use Laravel\Scout\Searchable;
 use Spatie\Activitylog\LogOptions;
 use Spatie\Activitylog\Traits\CausesActivity;
 use Spatie\Activitylog\Traits\LogsActivity;
@@ -81,6 +82,7 @@ class User extends Model implements Authenticatable, LaratrustUser
     use HasRolesAndPermissions;
     use LogsActivity;
     use Notifiable;
+    use Searchable;
     use SoftDeletes;
 
     protected $keyType = 'string';
@@ -95,8 +97,64 @@ class User extends Model implements Authenticatable, LaratrustUser
     protected $fillable = [
         'email',
         'sub',
+        'searchable',
     ];
 
+    protected $hidden = [
+        'searchable',
+    ];
+
+    /**
+     * Get the indexable data array for the model.
+     *
+     * @return array<string, mixed>
+     */
+    public function toSearchableArray(): array
+    {
+        $this->loadMissing([
+            'poolCandidates',
+            'workExperiences',
+            'educationExperiences',
+            'personalExperiences',
+            'communityExperiences',
+            'awardExperiences',
+        ]);
+
+        $result = collect([
+            $this->email, $this->first_name, $this->last_name, $this->telephone, $this->current_province, $this->current_city,
+            $this->poolCandidates->pluck('notes'),
+            $this->workExperiences->pluck('role'),
+            $this->workExperiences->pluck('organization'),
+            $this->workExperiences->pluck('division'),
+            $this->workExperiences->pluck('details'),
+            $this->educationExperiences->pluck('thesis_title'),
+            $this->educationExperiences->pluck('institution'),
+            $this->educationExperiences->pluck('details'),
+            $this->educationExperiences->pluck('area_of_study'),
+            $this->personalExperiences->pluck('title'),
+            $this->personalExperiences->pluck('description'),
+            $this->personalExperiences->pluck('details'),
+            $this->communityExperiences->pluck('title'),
+            $this->communityExperiences->pluck('organization'),
+            $this->communityExperiences->pluck('project'),
+            $this->communityExperiences->pluck('details'),
+            $this->awardExperiences->pluck('title'),
+            $this->awardExperiences->pluck('details'),
+            $this->awardExperiences->pluck('issued_by'),
+        ])
+            ->flatten()
+            ->reject(function ($value) {
+                return is_null($value) || $value === '';
+            })->toArray();
+
+        if (! $result) {
+            // SQL query doesn't handle empty arrays for some reason?
+            $result = [' '];
+        }
+
+        return $result;
+    }
+
     public function getActivitylogOptions(): LogOptions
     {
         return LogOptions::defaults()
@@ -202,6 +260,7 @@ public function addSkills($skill_ids)
         }
         // If this User instance continues to be used, ensure the in-memory instance has the updated skills.
         $this->refresh();
+        $this->searchable();
     }
 
     // getIsProfileCompleteAttribute function is correspondent to isProfileComplete attribute in graphql schema
@@ -279,7 +338,9 @@ public static function scopeIsProfileComplete(Builder $query, ?bool $isProfileCo
     protected static function boot()
     {
         parent::boot();
-
+        static::created(function (User $user) {
+            $user->searchable();
+        });
         static::deleting(function (User $user) {
             // We only need to run this if the user is being soft deleted
             if (! $user->isForceDeleting()) {
@@ -292,6 +353,7 @@ protected static function boot()
                 $newEmail = $user->email.'-deleted-at-'.Carbon::now()->format('Y-m-d');
                 $user->update(['email' => $newEmail]);
             }
+            $user->searchable();
         });
 
         static::restoring(function (User $user) {
@@ -618,18 +680,19 @@ public static function scopeEquity(Builder $query, ?array $equity): Builder
         return $query;
     }
 
-    public static function scopeGeneralSearch(Builder $query, ?string $search): Builder
+    public static function scopeGeneralSearch(Builder $query, ?array $searchTerms): Builder
     {
-        if ($search) {
-            $query->where(function ($query) use ($search) {
-                self::scopeName($query, $search);
-                $query->orWhere(function ($query) use ($search) {
-                    self::scopeEmail($query, $search);
-                });
-                $query->orWhere(function ($query) use ($search) {
-                    self::scopeTelephone($query, $search);
-                });
-            });
+        if ($searchTerms && is_array($searchTerms)) {
+            $combinedSearchTerm = implode('&', array_map('trim', $searchTerms));
+            $resultIds = self::search($combinedSearchTerm)->usingWebSearchQuery()
+                ->get(['id'])
+                ->pluck('id')
+                ->unique()
+                ->take(32766)
+                ->toArray();
+
+            // Use Eloquent builder to filter results based on unique IDs
+            $query->whereIn('id', $resultIds);
         }
 
         return $query;

diff --git a/api/app/Models/WorkExperience.php b/api/app/Models/WorkExperience.php
@@ -3,6 +3,7 @@
 namespace App\Models;
 
 use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Relations\BelongsTo;
 use Illuminate\Database\Eloquent\SoftDeletes;
 
 /**
@@ -33,4 +34,10 @@ class WorkExperience extends Experience
         'start_date' => 'date',
         'end_date' => 'date',
     ];
+
+    // Define the relationship to the User model
+    public function user(): BelongsTo
+    {
+        return $this->belongsTo(User::class);
+    }
 }
diff --git a/api/composer.json b/api/composer.json
@@ -10,9 +10,11 @@
   "license": "AGPL-3.0",
   "require": {
     "php": "^8.2",
+    "devnoiseconsulting/laravel-scout-postgres-tsvector": "^9.1",
     "doctrine/dbal": "^3.1",
     "guzzlehttp/guzzle": "^7.4",
     "laravel/framework": "^10.0",
+    "laravel/scout": "^10.5",
     "laravel/tinker": "^2.6",
     "lcobucci/clock": "^3.0.0",
     "lcobucci/jwt": "^5.0.0",