feat: add "totsv" modifier to the "fts" operator to explicitly apply …

…"to_tsvector()" to the filtered column

CHANGELOG.md

@@ -14,6 +14,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
  - #3727, Log maximum pool size - @steve-chavez
  - #1536, Add string comparison feature for jwt-role-claim-key - @taimoorzaeem
  - #3747, Allow `not_null` value for the `is` operator - @taimoorzaeem
+ - #2255, Add `totsv` modifier to `fts` operators to explicitly apply `to_tsvector()` to the filtered column, e.g. `?col=totsv.fts(simple).val` - @laurenceisla
 
 ### Fixed
 

docs/postgrest.dict

@@ -166,6 +166,7 @@ systemd
 todo
 todos
 tos
+totsv
 tsquery
 tx
 TypeScript

docs/references/api/tables_views.rst

@@ -78,6 +78,7 @@ fts           :code:`@@`                :ref:`fts` using to_tsquery
 plfts         :code:`@@`                :ref:`fts` using plainto_tsquery
 phfts         :code:`@@`                :ref:`fts` using phraseto_tsquery
 wfts          :code:`@@`                :ref:`fts` using websearch_to_tsquery
+totsv         :code:`to_tsvector()`     :ref:`fts` modifier that converts the filtered column to :code:`tsvector`
 cs            :code:`@>`                contains e.g. :code:`?tags=cs.{example, new}`
 cd            :code:`<@`                contained in e.g. :code:`?values=cd.{1,2,3}`
 ov            :code:`&&`                overlap (have points in common), e.g. :code:`?period=ov.[2017-01-01,2017-06-30]` –
@@ -193,7 +194,16 @@ The :code:`fts` filter mentioned above has a number of options to support flexib
 
   curl "http://localhost:3000/tsearch?my_tsv=not.wfts(french).amusant"
 
-Using `websearch_to_tsquery` requires PostgreSQL of version at least 11.0 and will raise an error in earlier versions of the database.
+For other column types like ``text`` or ``json``, you may want to explicitly convert these columns using ``to_tsvector()``, specially when a language is specified.
+To do so, use the ``totsv`` modifier:
+
+.. code-block:: bash
+
+  curl "http://localhost:3000/tsearch?text_column=totsv.fts(french).amusant"
+
+.. code-block:: bash
+
+  curl "http://localhost:3000/tsearch?json_column=not.totsv.phfts(english).The%20Fat%20Cats"
 
 .. _v_filter:
 

src/PostgREST/ApiRequest/QueryParams.hs

@@ -660,13 +660,15 @@ pOpExpr pSVal = do
           <?> "isVal: (null, not_null, true, false, unknown)"
 
     pFts = do
+      toTsVector <- try (string "totsv" *> pDelimiter $> True) <|> pure False
+
       op <-  try (string "fts"   $> FilterFts)
          <|> try (string "plfts" $> FilterFtsPlain)
          <|> try (string "phfts" $> FilterFtsPhrase)
          <|> try (string "wfts"  $> FilterFtsWebsearch)
 
       lang <- optionMaybe $ try (between (char '(') (char ')') pIdentifier)
-      pDelimiter >> Fts op (toS <$> lang) <$> pSVal
+      pDelimiter >> Fts op (toS <$> lang) toTsVector <$> pSVal
 
     -- case insensitive char and string
     ciChar :: Char -> GenParser Char state Char

src/PostgREST/ApiRequest/Types.hs

@@ -220,7 +220,7 @@ data Operation
   | In ListVal
   | Is IsVal
   | IsDistinctFrom SingleVal
-  | Fts FtsOperator (Maybe Language) SingleVal
+  | Fts FtsOperator (Maybe Language) Bool SingleVal
   deriving (Eq, Show)
 
 type Language = Text

src/PostgREST/Query/SqlFragment.hs

@@ -370,7 +370,7 @@ pgFmtArrayLiteralForField values _ = unknownLiteral (pgBuildArrayLiteral values)
 pgFmtFilter :: QualifiedIdentifier -> CoercibleFilter -> SQL.Snippet
 pgFmtFilter _ (CoercibleFilterNullEmbed hasNot fld) = pgFmtIdent fld <> " IS " <> (if not hasNot then "NOT " else mempty) <> "DISTINCT FROM NULL"
 pgFmtFilter _ (CoercibleFilter _ (NoOpExpr _)) = mempty -- TODO unreachable because NoOpExpr is filtered on QueryParams
-pgFmtFilter table (CoercibleFilter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtField table fld <> case oper of
+pgFmtFilter table (CoercibleFilter fld (OpExpr hasNot oper)) = notOp <> " " <> pgFmtFieldOper <> case oper of
    Op op val  -> " " <> simpleOperator op <> " " <> pgFmtUnknownLiteralForField (unknownLiteral val) fld
 
    OpQuant op quant val -> " " <> quantOperator op <> " " <> case op of
@@ -399,8 +399,11 @@ pgFmtFilter table (CoercibleFilter fld (OpExpr hasNot oper)) = notOp <> " " <> p
       [""] -> "= ANY('{}') "
       _    -> "= ANY (" <> pgFmtArrayLiteralForField vals fld <> ") "
 
-   Fts op lang val -> " " <> ftsOperator op <> "(" <> ftsLang lang <> unknownLiteral val <> ") "
+   Fts op lang _ val -> " " <> ftsOperator op <> "(" <> ftsLang lang <> unknownLiteral val <> ") "
  where
+   pgFmtFieldOper = case oper of
+     Fts _ lang True _ -> "to_tsvector(" <> ftsLang lang <> pgFmtField table fld <> ")"
+     _                 -> pgFmtField table fld
    ftsLang = maybe mempty (\l -> unknownLiteral l <> ", ")
    notOp = if hasNot then "NOT" else mempty
    star c = if c == '*' then '%' else c

test/spec/Feature/Query/AndOrParamsSpec.hs

@@ -90,6 +90,17 @@ spec =
               {"text_search_vector": "'amus':5 'fair':7 'impossibl':9 'peu':4" },
               {"text_search_vector": "'art':4 'spass':5 'unmog':7"}
             ]|] { matchHeaders = [matchContentTypeJson] }
+        it "can handle fts with totsv modifier" $ do
+          get "/grandchild_entities?or=(jsonb_col.totsv.fts.bar,jsonb_col.totsv.fts.foo)&select=jsonb_col" `shouldRespondWith`
+            [json|[
+              { "jsonb_col": {"a": {"b":"foo"}} },
+              { "jsonb_col": {"b":"bar"} }]
+            |] { matchHeaders = [matchContentTypeJson] }
+          get "/tsearch_to_tsvector?and=(text_search.not.totsv.plfts(german).Art%20Spass, text_search.not.totsv.plfts(french).amusant%20impossible, text_search.not.totsv.fts(english).impossible)&select=text_search" `shouldRespondWith`
+            [json|[
+              { "text_search": "But also fun to do what is possible" },
+              { "text_search": "Fat cats ate rats" }]
+            |] { matchHeaders = [matchContentTypeJson] }
         it "can handle isdistinct" $
           get "/entities?and=(id.gte.2,arr.isdistinct.{1,2})&select=id" `shouldRespondWith`
             [json|[{ "id": 3 }, { "id": 4 }]|] { matchHeaders = [matchContentTypeJson] }