break: remove binary field logic, raw-media-types

BREAKING CHANGE

Can be done later with custom media types

CHANGELOG.md

@@ -81,6 +81,13 @@ This project adheres to [Semantic Versioning](http://semver.org/).
    + The `/table?select=*,other!fk(*)` must be used to disambiguate
    + The server aids in choosing the `!fk` by sending a `hint` on the error whenever an ambiguous request happens.
 
+### Changed
+
+ - #1462, #1548, Removed [raw-media-types config](https://postgrest.org/en/v11.1/references/configuration.html#raw-media-types) - @steve-chavez
+   + Can be replaced with custom media types
+ - #1462, #1548, Removed `application/octet-stream`, `text/plain`, `text/xml` [builtin support for scalar results](https://postgrest.org/en/v11.1/references/api/resource_representation.html#scalar-function-response-format) - @steve-chavez
+   + Can be replaced with custom media types
+
 ## [11.1.0] - 2023-06-07
 
 ### Added

src/PostgREST/CLI.hs

@@ -218,9 +218,6 @@ exampleConfigFile =
       |## Base url for the OpenAPI output
       |openapi-server-proxy-uri = ""
       |
-      |## Content types to produce raw output
-      |# raw-media-types="image/png, image/jpg"
-      |
       |## Configurable CORS origins
       |# server-cors-allowed-origins = ""
       |

src/PostgREST/Config.hs

@@ -61,7 +61,6 @@ import PostgREST.Config.JSPath           (JSPath, JSPathExp (..),
                                           dumpJSPath, pRoleClaimKey)
 import PostgREST.Config.Proxy            (Proxy (..),
                                           isMalformedProxyUri, toURI)
-import PostgREST.MediaType               (MediaType (..), toMime)
 import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier, dumpQi,
                                           toQi)
 
@@ -102,7 +101,6 @@ data AppConfig = AppConfig
   , configOpenApiMode              :: OpenAPIMode
   , configOpenApiSecurityActive    :: Bool
   , configOpenApiServerProxyUri    :: Maybe Text
-  , configRawMediaTypes            :: [MediaType]
   , configServerCorsAllowedOrigins :: Maybe [Text]
   , configServerHost               :: Text
   , configServerPort               :: Int
@@ -169,7 +167,6 @@ toText conf =
       ,("openapi-mode",              q . dumpOpenApiMode . configOpenApiMode)
       ,("openapi-security-active",       T.toLower . show . configOpenApiSecurityActive)
       ,("openapi-server-proxy-uri",  q . fromMaybe mempty . configOpenApiServerProxyUri)
-      ,("raw-media-types",           q . T.decodeUtf8 . BS.intercalate "," . fmap toMime . configRawMediaTypes)
       ,("server-cors-allowed-origins",      q . maybe "" (T.intercalate ",") . configServerCorsAllowedOrigins)
       ,("server-host",               q . configServerHost)
       ,("server-port",                   show . configServerPort)
@@ -274,7 +271,6 @@ parser optPath env dbSettings roleSettings roleIsolationLvl =
     <*> parseOpenAPIMode "openapi-mode"
     <*> (fromMaybe False <$> optBool "openapi-security-active")
     <*> parseOpenAPIServerProxyURI "openapi-server-proxy-uri"
-    <*> (maybe [] (fmap (MTOther . encodeUtf8) . splitOnCommas) <$> optValue "raw-media-types")
     <*> (fmap splitOnCommas <$> optValue "server-cors-allowed-origins")
     <*> (fromMaybe "!4" <$> optString "server-host")
     <*> (fromMaybe 3000 <$> optInt "server-port")

src/PostgREST/Plan.hs

@@ -124,18 +124,16 @@ wrappedReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest
 wrappedReadPlan  identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{..},..} = do
   rPlan <- readPlan identifier conf sCache apiRequest
   mediaType <- mapLeft ApiRequestError $ negotiateContent conf iAction iPathInfo iAcceptMediaType
-  binField <- mapLeft ApiRequestError $ binaryField conf mediaType Nothing rPlan
   if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right ()
-  return $ WrappedReadPlan rPlan SQL.Read (mediaToAggregate mediaType binField apiRequest) mediaType
+  return $ WrappedReadPlan rPlan SQL.Read (mediaToAggregate mediaType apiRequest) mediaType
 
 mutateReadPlan :: Mutation -> ApiRequest -> QualifiedIdentifier -> AppConfig -> SchemaCache -> Either Error MutateReadPlan
 mutateReadPlan  mutation apiRequest@ApiRequest{iPreferences=Preferences{..},..} identifier conf sCache = do
   rPlan <- readPlan identifier conf sCache apiRequest
   mPlan <- mutatePlan mutation identifier apiRequest sCache rPlan
   mediaType <- mapLeft ApiRequestError $ negotiateContent conf iAction iPathInfo iAcceptMediaType
-  binField <- mapLeft ApiRequestError $ binaryField conf mediaType Nothing rPlan
   if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right ()
-  return $ MutateReadPlan rPlan mPlan SQL.Write (mediaToAggregate mediaType binField apiRequest) mediaType
+  return $ MutateReadPlan rPlan mPlan SQL.Write (mediaToAggregate mediaType apiRequest) mediaType
 
 callReadPlan :: QualifiedIdentifier -> AppConfig -> SchemaCache -> ApiRequest -> InvokeMethod -> Either Error CallReadPlan
 callReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferences{..},..} invMethod = do
@@ -160,9 +158,8 @@ callReadPlan identifier conf sCache apiRequest@ApiRequest{iPreferences=Preferenc
           (InvPost, Routine.Volatile)  -> SQL.Write
       cPlan = callPlan proc apiRequest paramKeys args rPlan
   mediaType <- mapLeft ApiRequestError $ negotiateContent conf iAction iPathInfo iAcceptMediaType
-  binField <- mapLeft ApiRequestError $ binaryField conf mediaType (Just proc) rPlan
   if not (null invalidPrefs) && preferHandling == Just Strict then Left $ ApiRequestError $ InvalidPreferences invalidPrefs else Right ()
-  return $ CallReadPlan rPlan cPlan txMode proc (mediaToAggregate mediaType binField apiRequest) mediaType
+  return $ CallReadPlan rPlan cPlan txMode proc (mediaToAggregate mediaType apiRequest) mediaType
   where
     qsParams' = QueryParams.qsParams iQueryParams
 
@@ -828,40 +825,8 @@ inferColsEmbedNeeds (Node ReadPlan{select} forest) pkCols
 addFilterToLogicForest :: CoercibleFilter -> [CoercibleLogicTree] -> [CoercibleLogicTree]
 addFilterToLogicForest flt lf = CoercibleStmnt flt : lf
 
--- | If raw(binary) output is requested, check that MediaType is one of the
--- admitted rawMediaTypes and that`?select=...` contains only one field other
--- than `*`
-binaryField :: AppConfig -> MediaType -> Maybe Routine -> ReadPlanTree -> Either ApiRequestError (Maybe FieldName)
-binaryField AppConfig{configRawMediaTypes} acceptMediaType proc rpTree
-  | isRawMediaType =
-    if (funcReturnsScalar <$> proc) == Just True ||
-       (funcReturnsSetOfScalar <$> proc) == Just True
-      then Right $ Just "pgrst_scalar"
-      else
-        let
-          fieldName = fstFieldName rpTree
-        in
-        case fieldName of
-          Just fld -> Right $ Just fld
-          Nothing  -> Left $ BinaryFieldError acceptMediaType
-  | otherwise =
-      Right Nothing
-  where
-    isRawMediaType = acceptMediaType `elem` configRawMediaTypes `L.union` [MTOctetStream, MTTextPlain, MTTextXML] || isRawPlan acceptMediaType
-    isRawPlan mt = case mt of
-      MTPlan MTOctetStream _ _ -> True
-      MTPlan MTTextPlain _ _   -> True
-      MTPlan MTTextXML _ _     -> True
-      _                        -> False
-
-    fstFieldName :: ReadPlanTree -> Maybe FieldName
-    fstFieldName (Node ReadPlan{select=(CoercibleField{cfName="*", cfJsonPath=[]}, _, _):_} [])  = Nothing
-    fstFieldName (Node ReadPlan{select=[(CoercibleField{cfName=fld, cfJsonPath=[]}, _, _)]} [])  = Just fld
-    fstFieldName _                                               = Nothing
-
-
-mediaToAggregate :: MediaType -> Maybe FieldName -> ApiRequest -> ResultAggregate
-mediaToAggregate mt binField apiReq@ApiRequest{iAction=act, iPreferences=Preferences{preferRepresentation=rep}} =
+mediaToAggregate :: MediaType -> ApiRequest -> ResultAggregate
+mediaToAggregate mt apiReq@ApiRequest{iAction=act, iPreferences=Preferences{preferRepresentation=rep}} =
   if noAgg then NoAgg
   else case mt of
     MTApplicationJSON     -> BuiltinAggJson
@@ -873,16 +838,11 @@ mediaToAggregate mt binField apiReq@ApiRequest{iAction=act, iPreferences=Prefere
     MTOpenAPI             -> BuiltinAggJson
     MTUrlEncoded          -> NoAgg -- TODO: unreachable since a previous step (producedMediaTypes) whitelists the media types that can become aggregates.
 
-    -- binary types
-    MTTextPlain           -> BuiltinAggBinary binField
-    MTTextXML             -> BuiltinAggXml binField
-    MTOctetStream         -> BuiltinAggBinary binField
-    MTOther _             -> BuiltinAggBinary binField
-
     -- Doing `Accept: application/vnd.pgrst.plan; for="application/vnd.pgrst.plan"` doesn't make sense, so we just empty the body.
     -- TODO: fail instead to be more strict
     MTPlan (MTPlan{}) _ _ -> NoAgg
-    MTPlan media      _ _ -> mediaToAggregate media binField apiReq
+    MTPlan media      _ _ -> mediaToAggregate media apiReq
+    _                     -> NoAgg
   where
     noAgg = case act of
       ActionMutate _         -> rep == Just HeadersOnly || rep == Just None || isNothing rep
@@ -904,7 +864,7 @@ negotiateContent conf action path accepts =
 producedMediaTypes :: AppConfig -> Action -> PathInfo -> [MediaType]
 producedMediaTypes conf action path =
   case action of
-    ActionRead _    -> defaultMediaTypes ++ rawMediaTypes
+    ActionRead _    -> defaultMediaTypes
     ActionInvoke _  -> invokeMediaTypes
     ActionInfo      -> defaultMediaTypes
     ActionMutate _  -> defaultMediaTypes
@@ -913,9 +873,7 @@ producedMediaTypes conf action path =
     inspectMediaTypes = [MTOpenAPI, MTApplicationJSON, MTArrayJSONStrip, MTAny]
     invokeMediaTypes =
       defaultMediaTypes
-        ++ rawMediaTypes
         ++ [MTOpenAPI | pathIsRootSpec path]
     defaultMediaTypes =
       [MTApplicationJSON, MTArrayJSONStrip, MTSingularJSON True, MTSingularJSON False, MTGeoJSON, MTTextCSV] ++
       [MTPlan MTApplicationJSON PlanText mempty | configDbPlanEnabled conf] ++ [MTAny]
-    rawMediaTypes = configRawMediaTypes conf `L.union` [MTOctetStream, MTTextPlain, MTTextXML]

src/PostgREST/Query.hs

@@ -66,7 +66,7 @@ import Protolude hiding (Handler)
 type DbHandler = ExceptT Error SQL.Transaction
 
 readQuery :: WrappedReadPlan -> AppConfig -> ApiRequest -> DbHandler ResultSet
-readQuery WrappedReadPlan{wrReadPlan, wrMedia, wrResAgg} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} = do
+readQuery WrappedReadPlan{..} conf@AppConfig{..} apiReq@ApiRequest{iPreferences=Preferences{..}} = do
   let countQuery = QueryBuilder.readPlanToCountQuery wrReadPlan
   resultSet <-
      lift . SQL.statement mempty $
@@ -151,7 +151,7 @@ deleteQuery mrPlan@MutateReadPlan{mrMedia} apiReq@ApiRequest{..} conf = do
   pure resultSet
 
 invokeQuery :: Routine -> CallReadPlan -> ApiRequest -> AppConfig -> PgVersion -> DbHandler ResultSet
-invokeQuery rout CallReadPlan{crReadPlan, crCallPlan, crResAgg, crMedia} apiReq@ApiRequest{iPreferences=Preferences{..}} conf@AppConfig{..} pgVer = do
+invokeQuery rout CallReadPlan{..} apiReq@ApiRequest{iPreferences=Preferences{..}} conf@AppConfig{..} pgVer = do
   resultSet <-
     lift . SQL.statement mempty $
       Statements.prepareCall

src/PostgREST/Query/SqlFragment.hs

@@ -209,20 +209,9 @@ asJsonF rout strip
       Just r  -> (funcReturnsSingleComposite r, funcReturnsScalar r, funcReturnsSetOfScalar r)
       Nothing -> (False, False, False)
 
-
-asXmlF :: Maybe FieldName -> SQL.Snippet
-asXmlF (Just fieldName) = "coalesce(xmlagg(_postgrest_t." <> pgFmtIdent fieldName <> "), '')"
--- TODO unreachable because a previous step(binaryField) will validate that there's a field. This will be cleared once custom media types are implemented.
-asXmlF Nothing          = "coalesce(xmlagg(_postgrest_t), '')"
-
 asGeoJsonF ::  SQL.Snippet
 asGeoJsonF = "json_build_object('type', 'FeatureCollection', 'features', coalesce(json_agg(ST_AsGeoJSON(_postgrest_t)::json), '[]'))"
 
-asBinaryF :: Maybe FieldName -> SQL.Snippet
-asBinaryF (Just fieldName) = "coalesce(string_agg(_postgrest_t." <> pgFmtIdent fieldName <> ", ''), '')"
--- TODO unreachable because a previous step(binaryField) will validate that there's a field. This will be cleared once custom media types are implemented.
-asBinaryF Nothing          = "coalesce(string_agg(_postgrest_t, ''), '')"
-
 locationF :: [Text] -> SQL.Snippet
 locationF pKeys = [qc|(
   WITH data AS (SELECT row_to_json(_) AS row FROM {sourceCTEName} AS _ LIMIT 1)
@@ -504,6 +493,4 @@ aggF rout = \case
   BuiltinAggSingleJson strip -> asJsonSingleF rout strip
   BuiltinAggGeoJson          -> asGeoJsonF
   BuiltinAggCsv              -> asCsvF
-  BuiltinAggXml    bField    -> asXmlF    bField
-  BuiltinAggBinary bField    -> asBinaryF bField
   NoAgg                      -> "''::text"

src/PostgREST/SchemaCache/Routine.hs

@@ -23,8 +23,7 @@ import qualified Data.Aeson                 as JSON
 import qualified Data.HashMap.Strict        as HM
 import qualified Hasql.Transaction.Sessions as SQL
 
-import PostgREST.SchemaCache.Identifiers (FieldName,
-                                          QualifiedIdentifier (..),
+import PostgREST.SchemaCache.Identifiers (QualifiedIdentifier (..),
                                           Schema, TableName)
 
 import Protolude
@@ -95,8 +94,6 @@ data ResultAggregate
    | BuiltinAggArrayJsonStrip
    | BuiltinAggGeoJson
    | BuiltinAggCsv
-   | BuiltinAggXml    (Maybe FieldName)
-   | BuiltinAggBinary (Maybe FieldName)
    | NoAgg
    deriving (Eq, Show)
 

test/io/configs/expected/aliases.config

@@ -27,7 +27,6 @@ log-level = "error"
 openapi-mode = "follow-privileges"
 openapi-security-active = false
 openapi-server-proxy-uri = ""
-raw-media-types = ""
 server-cors-allowed-origins = ""
 server-host = "!4"
 server-port = 3000

test/io/configs/expected/boolean-numeric.config

@@ -27,7 +27,6 @@ log-level = "error"
 openapi-mode = "follow-privileges"
 openapi-security-active = false
 openapi-server-proxy-uri = ""
-raw-media-types = ""
 server-cors-allowed-origins = ""
 server-host = "!4"
 server-port = 3000

test/io/configs/expected/boolean-string.config

@@ -27,7 +27,6 @@ log-level = "error"
 openapi-mode = "follow-privileges"
 openapi-security-active = false
 openapi-server-proxy-uri = ""
-raw-media-types = ""
 server-cors-allowed-origins = ""
 server-host = "!4"
 server-port = 3000

test/io/configs/expected/defaults.config

@@ -27,7 +27,6 @@ log-level = "error"
 openapi-mode = "follow-privileges"
 openapi-security-active = false
 openapi-server-proxy-uri = ""
-raw-media-types = ""
 server-cors-allowed-origins = ""
 server-host = "!4"
 server-port = 3000

test/io/configs/expected/no-defaults-with-db-other-authenticator.config

@@ -27,7 +27,6 @@ log-level = "info"
 openapi-mode = "disabled"
 openapi-security-active = false
 openapi-server-proxy-uri = "https://otherexample.org/api"
-raw-media-types = "application/vnd.pgrst.other-db-config"
 server-cors-allowed-origins = "http://example.com"
 server-host = "0.0.0.0"
 server-port = 80

test/io/configs/expected/no-defaults-with-db.config

@@ -27,7 +27,6 @@ log-level = "info"
 openapi-mode = "ignore-privileges"
 openapi-security-active = true
 openapi-server-proxy-uri = "https://example.org/api"
-raw-media-types = "application/vnd.pgrst.db-config"
 server-cors-allowed-origins = "http://example.com"
 server-host = "0.0.0.0"
 server-port = 80

test/io/configs/expected/no-defaults.config

@@ -27,7 +27,6 @@ log-level = "info"
 openapi-mode = "ignore-privileges"
 openapi-security-active = true
 openapi-server-proxy-uri = "https://postgrest.org"
-raw-media-types = "application/vnd.pgrst.config"
 server-cors-allowed-origins = "http://example.com"
 server-host = "0.0.0.0"
 server-port = 80

test/io/configs/expected/types.config

@@ -27,7 +27,6 @@ log-level = "error"
 openapi-mode = "follow-privileges"
 openapi-security-active = false
 openapi-server-proxy-uri = ""
-raw-media-types = ""
 server-cors-allowed-origins = ""
 server-host = "!4"
 server-port = 3000

test/io/configs/no-defaults-env.yaml

@@ -29,7 +29,6 @@ PGRST_LOG_LEVEL: info
 PGRST_OPENAPI_MODE: 'ignore-privileges'
 PGRST_OPENAPI_SECURITY_ACTIVE: true
 PGRST_OPENAPI_SERVER_PROXY_URI: 'https://postgrest.org'
-PGRST_RAW_MEDIA_TYPES: application/vnd.pgrst.config
 PGRST_SERVER_CORS_ALLOWED_ORIGINS: "http://example.com"
 PGRST_SERVER_HOST: 0.0.0.0
 PGRST_SERVER_PORT: 80

test/io/configs/no-defaults.config

@@ -27,7 +27,6 @@ log-level = "info"
 openapi-mode = "ignore-privileges"
 openapi-security-active = true
 openapi-server-proxy-uri = "https://postgrest.org"
-raw-media-types = "application/vnd.pgrst.config"
 server-cors-allowed-origins = "http://example.com"
 server-host = "0.0.0.0"
 server-port = 80

test/io/configs/types.config

@@ -8,6 +8,3 @@ db-channel-enabled = 13
 
 # expects integer or string
 db-max-rows = true
-
-# expects string
-raw-media-types = true

test/io/db_config.sql

@@ -3,7 +3,6 @@ CREATE ROLE db_config_authenticator LOGIN NOINHERIT;
 -- reloadable config options
 ALTER ROLE db_config_authenticator SET pgrst.jwt_aud = 'https://example.org';
 ALTER ROLE db_config_authenticator SET pgrst.openapi_server_proxy_uri = 'https://example.org/api';
-ALTER ROLE db_config_authenticator SET pgrst.raw_media_types = 'application/vnd.pgrst.db-config';
 ALTER ROLE db_config_authenticator SET pgrst.jwt_secret = 'REALLY=REALLY=REALLY=REALLY=VERY=SAFE';
 ALTER ROLE db_config_authenticator SET pgrst.jwt_secret_is_base64 = 'false';
 ALTER ROLE db_config_authenticator SET pgrst.jwt_role_claim_key = '."a"."role"';
@@ -51,7 +50,6 @@ ALTER ROLE db_config_authenticator SET pgrst.db_config = 'true';
 CREATE ROLE other_authenticator LOGIN NOINHERIT;
 ALTER ROLE other_authenticator SET pgrst.jwt_aud = 'https://otherexample.org';
 ALTER ROLE other_authenticator SET pgrst.openapi_server_proxy_uri = 'https://otherexample.org/api';
-ALTER ROLE other_authenticator SET pgrst.raw_media_types = 'application/vnd.pgrst.other-db-config';
 ALTER ROLE other_authenticator SET pgrst.jwt_secret = 'ODERREALLYREALLYREALLYREALLYVERYSAFE';
 ALTER ROLE other_authenticator SET pgrst.jwt_secret_is_base64 = 'true';
 ALTER ROLE other_authenticator SET pgrst.db_schemas = 'test, other_tenant1, other_tenant2';

test/io/fixtures.yaml

@@ -119,11 +119,6 @@ cli:
     use_defaultenv: true
     env:
       PGRST_DB_TX_END: rollback
-  - name: raw-media-types list
-    expect: 'raw-media-types = "image/png,image/jpeg"'
-    use_defaultenv: true
-    env:
-      PGRST_RAW_MEDIA_TYPES: ' image/png ,  image/jpeg '
 
 roleclaims:
   - key: '.postgrest.a_role'

test/spec/Feature/Query/HtmlRawOutputSpec.hs

@@ -12,7 +12,8 @@ import SpecHelper (acceptHdrs)
 
 spec :: SpecWith ((), Application)
 spec = describe "When raw-media-types is set to \"text/html\"" $
-  it "can get raw output with Accept: text/html" $
+  it "can get raw output with Accept: text/html" $ do
+    pendingWith "TBD"
     request methodGet "/rpc/welcome.html" (acceptHdrs "text/html") ""
       `shouldRespondWith`
       [str|

test/spec/Feature/Query/PlanSpec.hs

@@ -276,6 +276,7 @@ spec actualPgVersion = do
         totalCost `shouldBe` 68.56
 
     it "outputs the plan for text/xml" $ do
+      pendingWith "TBD"
       r <- request methodGet "/rpc/return_scalar_xml"
         (acceptHdrs "application/vnd.pgrst.plan+json; for=\"text/xml\"; options=verbose") ""