Feat: Sample Searching using SearchKick (Part One) (#846)

* feat: add Searchkick and update Sample model to enable Searchkick indexing and update Samples table to use Searchkick

* chore: add opensearch setup to github actions and attempt to get tests working again

* chore: add in Sample indexing to db/seeds and updated README.md with opensearch instructions.

* chore: refactor Sample::Query to have a single results method with a type option to select between ransack and searchkick

* chore: refactor group samples table to resolve rubocop warning

* chore: add in  to SearchKick order so that Samples table loads when no Samples exist.

* chore: add in migration to index existing Samples using SearchKick

* chore: remove puts and update test helper

* chore: update samples destroy service to refresh the search index after destroy

.github/workflows/ruby.yml

@@ -48,6 +48,10 @@ jobs:
           cache: "pnpm"
       - name: Install dependencies
         run: pnpm install
+      - name: Set up OpenSearch
+        uses: ankane/setup-opensearch@v1
+        with:
+          opensearch-version: 2
       - name: Precompile assets
         run: |
           bin/rails assets:precompile

Gemfile

@@ -134,6 +134,11 @@ gem 'caxlsx'
 # renders client's local time zone
 gem 'local_time', '~> 3.0', '>= 3.0.2'
 
+# Advanced searching using SearchKick
+gem 'searchkick'
+
+gem 'opensearch-ruby'
+
 group :development, :test do
   # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem
   gem 'debug', platforms: %i[mri windows], require: 'debug/prelude'

Gemfile.lock

@@ -410,6 +410,9 @@ GEM
     omniauth-saml (2.1.2)
       omniauth (~> 2.1)
       ruby-saml (~> 1.17)
+    opensearch-ruby (3.4.0)
+      faraday (>= 1.0, < 3)
+      multi_json (>= 1.0)
     orm_adapter (0.5.0)
     os (1.1.4)
     pagy (9.0.5)
@@ -538,6 +541,9 @@ GEM
     rubyzip (2.3.2)
     search_syntax (0.1.3)
       treetop (~> 1.6)
+    searchkick (5.4.0)
+      activemodel (>= 6.1)
+      hashie
     securerandom (0.3.1)
     signet (0.19.0)
       addressable (~> 2.8)
@@ -682,6 +688,7 @@ DEPENDENCIES
   omniauth-entra-id
   omniauth-rails_csrf_protection
   omniauth-saml
+  opensearch-ruby
   pagy (~> 9.0.5)
   paranoia
   pathogen_view_components!
@@ -696,6 +703,7 @@ DEPENDENCIES
   rubocop-graphql
   rubocop-rails
   search_syntax
+  searchkick
   simplecov
   sprockets-rails
   stimulus-rails

README.md

@@ -56,6 +56,7 @@ bundle && pnpm install
 ```
 
 Generate credentials:
+
 ```bash
 EDITOR=nano bin/rails credentials:edit
 ```
@@ -98,6 +99,15 @@ Start postgresql service:
 sudo systemctl start postgresql.service
 ```
 
+## OpenSearch Setup
+
+Install [OpenSearch](https://opensearch.org/downloads.html). For [Homebrew](https://brew.sh/), use:
+
+```sh
+brew install opensearch
+brew services start opensearch
+```
+
 ## Serve
 
 ```bash

app/components/samples/table_component.html.erb

@@ -101,7 +101,7 @@
                   <% end %>
                   <% if column == :puid || column == :name %>
                     <%= link_to(
-                    project_sample_path(sample.project, sample),
+                    namespace_project_sample_path(sample.project.namespace.parent, sample.project, sample),
                     data: { turbo: false },
                     class: "text-slate-700 dark:text-slate-300 font-semibold hover:underline"
                   ) do %>
@@ -116,7 +116,7 @@
                     <% end %>
                   <% elsif column == :project_id %>
                     <%= link_to sample.project.puid,
-                    project_samples_path(sample.project),
+                    namespace_project_samples_path(sample.project.namespace.parent, sample.project),
                     data: {
                       turbo: false,
                     },

app/controllers/groups/samples_controller.rb

@@ -11,7 +11,7 @@ class SamplesController < Groups::ApplicationController
 
     def index
       @timestamp = DateTime.current
-      @pagy, @samples = pagy(@query.results, limit: params[:limit] || 20)
+      @pagy, @samples = pagy_searchkick(@query.results(:searchkick_pagy), limit: params[:limit] || 20)
       @has_samples = authorized_samples.count.positive?
     end
 
@@ -26,7 +26,9 @@ def select
       respond_to do |format|
         format.turbo_stream do
           if params[:select].present?
-            @sample_ids = @query.results.where(updated_at: ..params[:timestamp].to_datetime).select(:id).pluck(:id)
+            @sample_ids = @query.results(:searchkick)
+                                .where(updated_at: ..params[:timestamp].to_datetime)
+                                .select(:id).pluck(:id)
           end
         end
       end
@@ -38,10 +40,6 @@ def group
       @group = Group.find_by_full_path(params[:group_id]) # rubocop:disable Rails/DynamicFindBy
     end
 
-    def authorized_projects
-      authorized_scope(Project, type: :relation, as: :group_projects, scope_options: { group: @group })
-    end
-
     def authorized_samples
       authorized_scope(Sample, type: :relation, as: :namespace_samples,
                                scope_options: { namespace: @group }).includes(project: { namespace: [{ parent: :route },
@@ -80,7 +78,10 @@ def query
       @search_params = search_params
       set_metadata_fields
 
-      @query = Sample::Query.new(@search_params.except(:metadata).merge({ project_ids: authorized_projects.select(:id) }))
+      project_ids =
+        authorized_scope(Project, type: :relation, as: :group_projects, scope_options: { group: @group }).pluck(:id)
+
+      @query = Sample::Query.new(@search_params.except(:metadata).merge({ project_ids: project_ids }))
     end
 
     def search_params

app/controllers/projects/samples_controller.rb

@@ -13,7 +13,7 @@ class SamplesController < Projects::ApplicationController # rubocop:disable Metr
 
     def index
       @timestamp = DateTime.current
-      @pagy, @samples = pagy(@query.results, limit: params[:limit] || 20)
+      @pagy, @samples = pagy_searchkick(@query.results(:searchkick_pagy), limit: params[:limit] || 20)
       @has_samples = @project.samples.size.positive?
     end
 
@@ -83,7 +83,9 @@ def select
       respond_to do |format|
         format.turbo_stream do
           if params[:select].present?
-            @sample_ids = @query.results.where(updated_at: ..params[:timestamp].to_datetime).select(:id).pluck(:id)
+            @sample_ids = @query.results(:searchkick)
+                                .where(updated_at: ..params[:timestamp].to_datetime)
+                                .select(:id).pluck(:id)
           end
         end
       end

app/models/sample.rb

@@ -6,9 +6,15 @@ class Sample < ApplicationRecord
   include HasPuid
   include History
 
+  extend Pagy::Searchkick
+
   has_logidze
   acts_as_paranoid
 
+  searchkick \
+    deep_paging: true,
+    text_middle: %i[name puid]
+
   belongs_to :project, counter_cache: true
 
   broadcasts_refreshes_to :project
@@ -79,4 +85,20 @@ def sort_files
 
     { singles:, pe_forward:, pe_reverse: }
   end
+
+  def search_data
+    {
+      name: name,
+      puid: puid,
+      project_id: project_id,
+      metadata: metadata.as_json,
+      created_at: created_at,
+      updated_at: updated_at,
+      attachments_updated_at: attachments_updated_at
+    }
+  end
+
+  def should_index?
+    !deleted?
+  end
 end

app/models/sample/query.rb

@@ -5,6 +5,8 @@ class Sample::Query # rubocop:disable Style/ClassAndModuleChildren
   include ActiveModel::Model
   include ActiveModel::Attributes
 
+  ResultTypeError = Class.new(StandardError)
+
   attribute :column, :string
   attribute :direction, :string
   attribute :name_or_puid_cont, :string
@@ -23,16 +25,57 @@ def initialize(...)
   def sort=(value)
     super
     column, direction = sort.split
+    column = column.gsub('metadata_', 'metadata.') if column.match?(/metadata_/)
     assign_attributes(column:, direction:)
   end
 
-  def results
+  def results(type = :ransack)
+    case type
+    when :ransack
+      ransack_results
+    when :searchkick
+      searchkick_results
+    when :searchkick_pagy
+      searchkick_pagy_results
+    else
+      raise ResultTypeError, "Unrecognized type: #{type}"
+    end
+  end
+
+  private
+
+  def ransack_results
     return Sample.none unless valid?
 
     sort_samples.ransack(ransack_params).result
   end
 
-  private
+  def searchkick_pagy_results
+    return Sample.pagy_search('') unless valid?
+
+    Sample.pagy_search(name_or_puid_cont.presence || '*', **searchkick_kwargs)
+  end
+
+  def searchkick_results
+    return Sample.search('') unless valid?
+
+    Sample.search(name_or_puid_cont.presence || '*', **searchkick_kwargs)
+  end
+
+  def searchkick_kwargs
+    { fields: [{ name: :text_middle }, { puid: :text_middle }],
+      misspellings: false,
+      where: { project_id: project_ids }.merge((
+       if name_or_puid_in.present?
+         { _or: [{ name: name_or_puid_in },
+                 { puid: name_or_puid_in }] }
+       else
+         {}
+       end
+     )),
+      order: { "#{column}": { order: direction, unmapped_type: 'long' } },
+      includes: [project: { namespace: [{ parent: :route }, :route] }] }
+  end
 
   def ransack_params
     {
@@ -42,8 +85,8 @@ def ransack_params
   end
 
   def sort_samples(scope = Sample.where(project_id: project_ids))
-    if column.starts_with? 'metadata_'
-      field = column.gsub('metadata_', '')
+    if column.starts_with? 'metadata.'
+      field = column.gsub('metadata.', '')
       scope.order(Sample.metadata_sort(field, direction))
     else
       scope.order("#{column} #{direction}")

app/services/samples/destroy_service.rb

@@ -24,6 +24,8 @@ def destroy_single
       sample_destroyed = sample.destroy
 
       if sample_destroyed
+        Sample.search_index.refresh
+
         @project.namespace.create_activity key: 'namespaces_project_namespace.samples.destroy',
                                            owner: current_user,
                                            parameters:
@@ -46,10 +48,13 @@ def destroy_multiple # rubocop:disable Metrics/MethodLength
       samples = samples.destroy_all
 
       samples.each do |sample|
+        # Sample.searchkick_index.remove(sample)
         update_metadata_summary(sample)
         samples_deleted_puids << sample.puid
       end
 
+      Sample.search_index.refresh
+
       update_samples_count(samples_to_delete_count) if @project.parent.type == 'Group'
 
       @project.namespace.create_activity key: 'namespaces_project_namespace.samples.destroy_multiple',

config/initializers/pagy.rb

@@ -103,9 +103,9 @@
 # DEFAULT[:searchkick_pagy_search] = :pagy_search
 # Default original :search method called internally to do the actual search
 # Pagy::DEFAULT[:searchkick_search] = :search
-# require 'pagy/extras/searchkick'
+require 'pagy/extras/searchkick'
 # uncomment if you are going to use Searchkick.pagy_search
-# Searchkick.extend Pagy::Searchkick
+Searchkick.extend Pagy::Searchkick
 
 # Frontend Extras
 

db/migrate/20241212164410_index_existing_samples.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Index existing Samples using SearchKick
+class IndexExistingSamples < ActiveRecord::Migration[7.2]
+  def up
+    Sample.reindex
+  end
+end

db/seeds.rb

@@ -803,4 +803,7 @@ def seed_exports # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metric
   DataExport.suppressing_turbo_broadcasts do
     seed_exports
   end
+
+  # index samples using searchkick
+  Sample.reindex
 end