From 9af6adc1969011296a22bb019aa8688d08198cab Mon Sep 17 00:00:00 2001 From: woylie <13847569+woylie@users.noreply.github.com> Date: Thu, 12 Sep 2024 08:55:44 +0900 Subject: [PATCH 1/2] update sqlite tests for case-insensitive like operator --- test/adapters/ecto/cases/flop_test.exs | 25 +++- test/adapters/ecto/postgres/test_helper.exs | 4 + test/adapters/ecto/sqlite/test_helper.exs | 4 + test/support/test_util.ex | 154 ++++++++++++++------ 4 files changed, 132 insertions(+), 55 deletions(-) diff --git a/test/adapters/ecto/cases/flop_test.exs b/test/adapters/ecto/cases/flop_test.exs index 362e72d1..1c5e4690 100644 --- a/test/adapters/ecto/cases/flop_test.exs +++ b/test/adapters/ecto/cases/flop_test.exs @@ -387,14 +387,14 @@ defmodule Flop.Adapters.Ecto.FlopTest do end end - property "applies like filter" do + property "applies like filter", %{ecto_adapter: ecto_adapter} do check all pet_count <- integer(@pet_count_range), pets = insert_list_and_sort(pet_count, :pet_with_owner), field <- filterable_pet_field(:string), pet <- member_of(pets), value = Pet.get_field(pet, field), query_value <- substring(value) do - expected = filter_items(pets, field, :like, query_value) + expected = filter_items(pets, field, :like, query_value, ecto_adapter) assert query_pets_with_owners(%{ filters: [%{field: field, op: :like, value: query_value}] @@ -434,14 +434,15 @@ defmodule Flop.Adapters.Ecto.FlopTest do end end - property "applies not like filter" do + property "applies not like filter", %{ecto_adapter: ecto_adapter} do check all pet_count <- integer(@pet_count_range), pets = insert_list_and_sort(pet_count, :pet_with_owner), field <- filterable_pet_field(:string), pet <- member_of(pets), value = Pet.get_field(pet, field), query_value <- substring(value) do - expected = filter_items(pets, field, :not_like, query_value) + expected = + filter_items(pets, field, :not_like, query_value, ecto_adapter) assert query_pets_with_owners(%{ filters: [%{field: field, op: :not_like, value: query_value}] @@ -486,14 +487,21 @@ defmodule Flop.Adapters.Ecto.FlopTest do end end - property "applies like_and filter" do + property "applies like_and filter", %{ecto_adapter: ecto_adapter} do check all pet_count <- integer(@pet_count_range), pets = insert_list_and_sort(pet_count, :pet_with_owner), field <- filterable_pet_field(:string), pet <- member_of(pets), value = Pet.get_field(pet, field), search_text_or_list <- search_text_or_list(value) do - expected = filter_items(pets, field, :like_and, search_text_or_list) + expected = + filter_items( + pets, + field, + :like_and, + search_text_or_list, + ecto_adapter + ) assert query_pets_with_owners(%{ filters: [ @@ -505,14 +513,15 @@ defmodule Flop.Adapters.Ecto.FlopTest do end end - property "applies like_or filter" do + property "applies like_or filter", %{ecto_adapter: ecto_adapter} do check all pet_count <- integer(@pet_count_range), pets = insert_list_and_sort(pet_count, :pet_with_owner), field <- filterable_pet_field(:string), pet <- member_of(pets), value = Pet.get_field(pet, field), search_text_or_list <- search_text_or_list(value) do - expected = filter_items(pets, field, :like_or, search_text_or_list) + expected = + filter_items(pets, field, :like_or, search_text_or_list, ecto_adapter) assert query_pets_with_owners(%{ filters: [ diff --git a/test/adapters/ecto/postgres/test_helper.exs b/test/adapters/ecto/postgres/test_helper.exs index d0d4f5e3..be2d073e 100644 --- a/test/adapters/ecto/postgres/test_helper.exs +++ b/test/adapters/ecto/postgres/test_helper.exs @@ -22,6 +22,10 @@ defmodule Flop.Integration.Case do setup do :ok = Sandbox.checkout(Flop.Repo) end + + setup do + %{ecto_adapter: :postgres} + end end Code.require_file("migration.exs", __DIR__) diff --git a/test/adapters/ecto/sqlite/test_helper.exs b/test/adapters/ecto/sqlite/test_helper.exs index 3544bbc1..50811900 100644 --- a/test/adapters/ecto/sqlite/test_helper.exs +++ b/test/adapters/ecto/sqlite/test_helper.exs @@ -20,6 +20,10 @@ defmodule Flop.Integration.Case do setup do :ok = Sandbox.checkout(Flop.Repo) end + + setup do + %{ecto_adapter: :sqlite} + end end Code.require_file("migration.exs", __DIR__) diff --git a/test/support/test_util.ex b/test/support/test_util.ex index e8610cd4..d187c79a 100644 --- a/test/support/test_util.ex +++ b/test/support/test_util.ex @@ -22,16 +22,22 @@ defmodule Flop.TestUtil do The function supports regular fields, join fields and compound fields. The associations need to be preloaded if join fields are used. """ - def filter_items(items, field, op, value \\ nil) + def filter_items(items, field, op, value \\ nil, ecto_adapter \\ nil) - def filter_items([], _, _, _), do: [] + def filter_items([], _, _, _, _), do: [] - def filter_items([%module{} = struct | _] = items, field, op, value) + def filter_items( + [struct | _] = items, + field, + op, + value, + ecto_adapter + ) when is_atom(field) do case Flop.Schema.field_info(struct, field) do %FieldInfo{ecto_type: ecto_type, extra: %{type: :join}} = field_info when not is_nil(ecto_type) -> - filter_func = matches?(op, value, ecto_type) + filter_func = matches?(op, value, ecto_adapter) Enum.filter(items, fn item -> item |> get_field(field_info) |> filter_func.() @@ -39,8 +45,7 @@ defmodule Flop.TestUtil do %FieldInfo{extra: %{type: type}} = field_info when type in [:normal, :join] -> - ecto_type = module.__schema__(:type, field) - filter_func = matches?(op, value, ecto_type) + filter_func = matches?(op, value, ecto_adapter) Enum.filter(items, fn item -> item |> get_field(field_info) |> filter_func.() @@ -49,12 +54,12 @@ defmodule Flop.TestUtil do %FieldInfo{extra: %{type: :compound, fields: fields}} -> Enum.filter( items, - &apply_filter_to_compound_fields(&1, fields, op, value) + &apply_filter_to_compound_fields(&1, fields, op, value, ecto_adapter) ) end end - defp apply_filter_to_compound_fields(_pet, _fields, op, _value) + defp apply_filter_to_compound_fields(_pet, _fields, op, _value, _ecto_adapter) when op in [ :==, :=~, @@ -70,8 +75,8 @@ defmodule Flop.TestUtil do true end - defp apply_filter_to_compound_fields(pet, fields, :empty, value) do - filter_func = matches?(:empty, value) + defp apply_filter_to_compound_fields(pet, fields, :empty, value, ecto_adapter) do + filter_func = matches?(:empty, value, ecto_adapter) Enum.all?(fields, fn field -> field_info = Flop.Schema.field_info(%Pet{}, field) @@ -79,11 +84,17 @@ defmodule Flop.TestUtil do end) end - defp apply_filter_to_compound_fields(pet, fields, :like_and, value) do + defp apply_filter_to_compound_fields( + pet, + fields, + :like_and, + value, + ecto_adapter + ) do value = if is_binary(value), do: String.split(value), else: value Enum.all?(value, fn substring -> - filter_func = matches?(:like, substring) + filter_func = matches?(:like, substring, ecto_adapter) Enum.any?(fields, fn field -> field_info = Flop.Schema.field_info(%Pet{}, field) @@ -92,11 +103,17 @@ defmodule Flop.TestUtil do end) end - defp apply_filter_to_compound_fields(pet, fields, :ilike_and, value) do + defp apply_filter_to_compound_fields( + pet, + fields, + :ilike_and, + value, + ecto_adapter + ) do value = if is_binary(value), do: String.split(value), else: value Enum.all?(value, fn substring -> - filter_func = matches?(:ilike, substring) + filter_func = matches?(:ilike, substring, ecto_adapter) Enum.any?(fields, fn field -> field_info = Flop.Schema.field_info(%Pet{}, field) @@ -105,11 +122,17 @@ defmodule Flop.TestUtil do end) end - defp apply_filter_to_compound_fields(pet, fields, :like_or, value) do + defp apply_filter_to_compound_fields( + pet, + fields, + :like_or, + value, + ecto_adapter + ) do value = if is_binary(value), do: String.split(value), else: value Enum.any?(value, fn substring -> - filter_func = matches?(:like, substring) + filter_func = matches?(:like, substring, ecto_adapter) Enum.any?(fields, fn field -> field_info = Flop.Schema.field_info(%Pet{}, field) @@ -118,11 +141,17 @@ defmodule Flop.TestUtil do end) end - defp apply_filter_to_compound_fields(pet, fields, :ilike_or, value) do + defp apply_filter_to_compound_fields( + pet, + fields, + :ilike_or, + value, + ecto_adapter + ) do value = if is_binary(value), do: String.split(value), else: value Enum.any?(value, fn substring -> - filter_func = matches?(:ilike, substring) + filter_func = matches?(:ilike, substring, ecto_adapter) Enum.any?(fields, fn field -> field_info = Flop.Schema.field_info(%Pet{}, field) @@ -131,8 +160,8 @@ defmodule Flop.TestUtil do end) end - defp apply_filter_to_compound_fields(pet, fields, op, value) do - filter_func = matches?(op, value) + defp apply_filter_to_compound_fields(pet, fields, op, value, ecto_adapter) do + filter_func = matches?(op, value, ecto_adapter) Enum.any?(fields, fn field -> field_info = Flop.Schema.field_info(%Pet{}, field) @@ -146,63 +175,94 @@ defmodule Flop.TestUtil do defp get_field(pet, %FieldInfo{extra: %{type: :join, path: [a, b]}}), do: pet |> Map.fetch!(a) |> Map.fetch!(b) - defp matches?(op, v, _), do: matches?(op, v) - defp matches?(:==, v), do: &(&1 == v) - defp matches?(:!=, v), do: &(&1 != v) - defp matches?(:empty, _), do: &empty?(&1) - defp matches?(:not_empty, _), do: &(!empty?(&1)) - defp matches?(:<=, v), do: &(&1 <= v) - defp matches?(:<, v), do: &(&1 < v) - defp matches?(:>, v), do: &(&1 > v) - defp matches?(:>=, v), do: &(&1 >= v) - defp matches?(:in, v), do: &(&1 in v) - defp matches?(:not_in, v), do: &(&1 not in v) - defp matches?(:contains, v), do: &(v in &1) - defp matches?(:not_contains, v), do: &(v not in &1) - defp matches?(:like, v), do: &(&1 =~ v) - defp matches?(:not_like, v), do: &(&1 =~ v == false) - defp matches?(:=~, v), do: matches?(:ilike, v) - - defp matches?(:ilike, v) do + defp matches?(:==, v, _), do: &(&1 == v) + defp matches?(:!=, v, _), do: &(&1 != v) + defp matches?(:empty, _, _), do: &empty?(&1) + defp matches?(:not_empty, _, _), do: &(!empty?(&1)) + defp matches?(:<=, v, _), do: &(&1 <= v) + defp matches?(:<, v, _), do: &(&1 < v) + defp matches?(:>, v, _), do: &(&1 > v) + defp matches?(:>=, v, _), do: &(&1 >= v) + defp matches?(:in, v, _), do: &(&1 in v) + defp matches?(:not_in, v, _), do: &(&1 not in v) + defp matches?(:contains, v, _), do: &(v in &1) + defp matches?(:not_contains, v, _), do: &(v not in &1) + + defp matches?(:like, v, :sqlite) do + v = String.downcase(v) + &(String.downcase(&1) =~ v) + end + + defp matches?(:like, v, _), do: &(&1 =~ v) + + defp matches?(:not_like, v, :sqlite) do + v = String.downcase(v) + &(String.downcase(&1) =~ v == false) + end + + defp matches?(:not_like, v, _), do: &(&1 =~ v == false) + defp matches?(:=~, v, ecto_adapter), do: matches?(:ilike, v, ecto_adapter) + + defp matches?(:ilike, v, _) do v = String.downcase(v) &(String.downcase(&1) =~ v) end - defp matches?(:not_ilike, v) do + defp matches?(:not_ilike, v, _) do v = String.downcase(v) &(String.downcase(&1) =~ v == false) end - defp matches?(:like_and, v) when is_binary(v) do + defp matches?(:like_and, v, :sqlite) when is_binary(v) do + values = v |> String.downcase() |> String.split() + &Enum.all?(values, fn v -> String.downcase(&1) =~ v end) + end + + defp matches?(:like_and, v, :sqlite) do + values = Enum.map(v, &String.downcase/1) + &Enum.all?(values, fn v -> String.downcase(&1) =~ v end) + end + + defp matches?(:like_and, v, _) when is_binary(v) do values = String.split(v) &Enum.all?(values, fn v -> &1 =~ v end) end - defp matches?(:like_and, v), do: &Enum.all?(v, fn v -> &1 =~ v end) + defp matches?(:like_and, v, _), do: &Enum.all?(v, fn v -> &1 =~ v end) + + defp matches?(:like_or, v, :sqlite) when is_binary(v) do + values = v |> String.downcase() |> String.split() + &Enum.any?(values, fn v -> String.downcase(&1) =~ v end) + end + + defp matches?(:like_or, v, :sqlite) do + values = Enum.map(v, &String.downcase/1) + &Enum.any?(values, fn v -> String.downcase(&1) =~ v end) + end - defp matches?(:like_or, v) when is_binary(v) do + defp matches?(:like_or, v, _) when is_binary(v) do values = String.split(v) &Enum.any?(values, fn v -> &1 =~ v end) end - defp matches?(:like_or, v), do: &Enum.any?(v, fn v -> &1 =~ v end) + defp matches?(:like_or, v, _), do: &Enum.any?(v, fn v -> &1 =~ v end) - defp matches?(:ilike_and, v) when is_binary(v) do + defp matches?(:ilike_and, v, _) when is_binary(v) do values = v |> String.downcase() |> String.split() &Enum.all?(values, fn v -> String.downcase(&1) =~ v end) end - defp matches?(:ilike_and, v) do + defp matches?(:ilike_and, v, _) do values = Enum.map(v, &String.downcase/1) &Enum.all?(values, fn v -> String.downcase(&1) =~ v end) end - defp matches?(:ilike_or, v) when is_binary(v) do + defp matches?(:ilike_or, v, _) when is_binary(v) do values = v |> String.downcase() |> String.split() &Enum.any?(values, fn v -> String.downcase(&1) =~ v end) end - defp matches?(:ilike_or, v) do + defp matches?(:ilike_or, v, _) do values = Enum.map(v, &String.downcase/1) &Enum.any?(values, fn v -> String.downcase(&1) =~ v end) end From c5145214233cc2126b0adc0e033ee8030b1b3e39 Mon Sep 17 00:00:00 2001 From: woylie <13847569+woylie@users.noreply.github.com> Date: Thu, 12 Sep 2024 09:12:28 +0900 Subject: [PATCH 2/2] skip ilike operator tests for sqlite --- lib/flop.ex | 2 +- test/adapters/ecto/cases/flop_test.exs | 49 ++++++++++++++++++++--- test/adapters/ecto/sqlite/test_helper.exs | 2 +- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/lib/flop.ex b/lib/flop.ex index 5e1fc4ea..bcf671c2 100644 --- a/lib/flop.ex +++ b/lib/flop.ex @@ -232,7 +232,7 @@ defmodule Flop do ...> %{name: "Patty", age: 2, species: "C. aegagrus"} ...> ]) iex> - iex> params = %{filters: [%{field: :name, op: :=~, value: "Mag"}]} + iex> params = %{filters: [%{field: :name, op: :like, value: "Mag"}]} iex> {:ok, {results, meta}} = Flop.validate_and_run(MyApp.Pet, params) iex> meta.total_count 1 diff --git a/test/adapters/ecto/cases/flop_test.exs b/test/adapters/ecto/cases/flop_test.exs index 1c5e4690..5a4be393 100644 --- a/test/adapters/ecto/cases/flop_test.exs +++ b/test/adapters/ecto/cases/flop_test.exs @@ -404,31 +404,64 @@ defmodule Flop.Adapters.Ecto.FlopTest do end end - test "escapes % in (i)like queries" do + test "escapes % in like queries" do %{id: _id1} = insert(:pet, name: "abc") %{id: id2} = insert(:pet, name: "a%c") - for op <- [:like, :ilike, :like_and, :like_or, :ilike_and, :ilike_or] do + for op <- [:like, :like_and, :like_or] do flop = %Flop{filters: [%Filter{field: :name, op: op, value: "a%c"}]} assert [%Pet{id: ^id2}] = Flop.all(Pet, flop) end end - test "escapes _ in (i)like queries" do + @tag :ilike + test "escapes % in ilike queries" do + %{id: _id1} = insert(:pet, name: "abc") + %{id: id2} = insert(:pet, name: "a%c") + + for op <- [:ilike, :ilike_and, :ilike_or] do + flop = %Flop{filters: [%Filter{field: :name, op: op, value: "a%c"}]} + assert [%Pet{id: ^id2}] = Flop.all(Pet, flop) + end + end + + test "escapes _ in like queries" do + %{id: _id1} = insert(:pet, name: "abc") + %{id: id2} = insert(:pet, name: "a_c") + + for op <- [:like, :like_and, :like_or] do + flop = %Flop{filters: [%Filter{field: :name, op: op, value: "a_c"}]} + assert [%Pet{id: ^id2}] = Flop.all(Pet, flop) + end + end + + @tag :ilike + test "escapes _ in ilike queries" do %{id: _id1} = insert(:pet, name: "abc") %{id: id2} = insert(:pet, name: "a_c") - for op <- [:like, :ilike, :like_and, :like_or, :ilike_and, :ilike_or] do + for op <- [:ilike, :ilike_and, :ilike_or] do flop = %Flop{filters: [%Filter{field: :name, op: op, value: "a_c"}]} assert [%Pet{id: ^id2}] = Flop.all(Pet, flop) end end - test "escapes \\ in (i)like queries" do + test "escapes \\ in like queries" do + %{id: _id1} = insert(:pet, name: "abc") + %{id: id2} = insert(:pet, name: "a\\c") + + for op <- [:like, :like_and, :like_or] do + flop = %Flop{filters: [%Filter{field: :name, op: op, value: "a\\c"}]} + assert [%Pet{id: ^id2}] = Flop.all(Pet, flop) + end + end + + @tag :ilike + test "escapes \\ in ilike queries" do %{id: _id1} = insert(:pet, name: "abc") %{id: id2} = insert(:pet, name: "a\\c") - for op <- [:like, :ilike, :like_and, :like_or, :ilike_and, :ilike_or] do + for op <- [:ilike, :ilike_and, :ilike_or] do flop = %Flop{filters: [%Filter{field: :name, op: op, value: "a\\c"}]} assert [%Pet{id: ^id2}] = Flop.all(Pet, flop) end @@ -452,6 +485,7 @@ defmodule Flop.Adapters.Ecto.FlopTest do end end + @tag :ilike property "applies ilike filter" do check all pet_count <- integer(@pet_count_range), pets = insert_list_and_sort(pet_count, :pet_with_owner), @@ -470,6 +504,7 @@ defmodule Flop.Adapters.Ecto.FlopTest do end end + @tag :ilike property "applies not ilike filter" do check all pet_count <- integer(@pet_count_range), pets = insert_list_and_sort(pet_count, :pet_with_owner), @@ -533,6 +568,7 @@ defmodule Flop.Adapters.Ecto.FlopTest do end end + @tag :ilike property "applies ilike_and filter" do check all pet_count <- integer(@pet_count_range), pets = insert_list_and_sort(pet_count, :pet_with_owner), @@ -552,6 +588,7 @@ defmodule Flop.Adapters.Ecto.FlopTest do end end + @tag :ilike property "applies ilike_or filter" do check all pet_count <- integer(@pet_count_range), pets = insert_list_and_sort(pet_count, :pet_with_owner), diff --git a/test/adapters/ecto/sqlite/test_helper.exs b/test/adapters/ecto/sqlite/test_helper.exs index 50811900..6e1fd871 100644 --- a/test/adapters/ecto/sqlite/test_helper.exs +++ b/test/adapters/ecto/sqlite/test_helper.exs @@ -40,4 +40,4 @@ _ = Ecto.Adapters.SQLite3.storage_down(Flop.Repo.config()) :ok = Ecto.Migrator.up(Flop.Repo, 0, Flop.Repo.SQLite.Migration, log: false) {:ok, _} = Application.ensure_all_started(:ex_machina) -ExUnit.start(exclude: [:composite_type, :prefix]) +ExUnit.start(exclude: [:composite_type, :ilike, :prefix])