Catch-all rules in the Gateway

manifest/Scarf/Manifest.hs

@@ -35,6 +35,7 @@ import Scarf.Gateway.ImagePattern qualified as ImagePattern
 import Scarf.Gateway.Regex (Regex)
 import Scarf.Gateway.Rule
   ( Rule,
+    newCatchAllRule,
     newDockerRuleV1,
     newDockerRuleV2,
     newFileRuleV2,
@@ -144,6 +145,18 @@ data ManifestRule
         -- | Manifest backend index. If not present, assume https://pypi.org/simple/
         manifestRuleBackendSimpleIndex :: !(Maybe Text)
       }
+  | ManifestRuleCatchAllV1
+      { -- | Package name
+        manifestRulePackageName :: !Text,
+        -- | Owner of the package
+        manifestRuleOwner :: !Text,
+        -- | e.g. cr.l5d.io
+        manifestRuleDomain :: !Domain,
+        -- | Package id this file belongs to
+        manifestRulePackageId :: !Text,
+        -- | Target domain to redirect to
+        manifestRuleRedirectTargetDomain :: !(Maybe Text)
+      }
   deriving (Eq, Show)
 
 data PythonFileHashV1 = PythonFileHashV1
@@ -227,6 +240,13 @@ instance FromJSON ManifestRule where
           <*> o .: "backend-url"
           <*> o .: "package-id"
           <*> o .:? "backend-simple-index-url"
+      "catch-all-v1" ->
+        ManifestRuleCatchAllV1
+          <$> o .:? "package-name" .!= ""
+          <*> o .:? "owner" .!= ""
+          <*> o .: "domain"
+          <*> o .: "package-id"
+          <*> o .:? "redirect-target-domain"
       _ ->
         fail "invalid manifest rule type"
 
@@ -290,6 +310,16 @@ instance ToJSON ManifestRule where
           "package-id" .= manifestRulePackageId,
           "backend-simple-index-url" .= manifestRuleBackendSimpleIndex
         ]
+  toJSON ManifestRuleCatchAllV1 {..} =
+    object $
+      dropNull
+        [ "type" .= ("catch-all-v1" :: Text),
+          "package-name" .= manifestRulePackageName,
+          "owner" .= manifestRuleOwner,
+          "domain" .= manifestRuleDomain,
+          "package-id" .= manifestRulePackageId,
+          "redirect-target-domain" .= manifestRuleRedirectTargetDomain
+        ]
 
 dropNull :: [(a, Value)] -> [(a, Value)]
 dropNull = filter $ \(_, x) -> case x of
@@ -357,3 +387,7 @@ manifestRuleToRule manifestRule = case manifestRule of
       manifestRuleRequiresPython
       manifestRuleBackendURL
       manifestRulePackageId
+  ManifestRuleCatchAllV1 {..} ->
+    newCatchAllRule
+      manifestRulePackageId
+      manifestRuleRedirectTargetDomain

src/Scarf/Gateway/Rule.hs

@@ -17,6 +17,7 @@ module Scarf.Gateway.Rule
     newPixelRule,
     newScarfJsRule,
     newPythonRule,
+    newCatchAllRule,
     optimizeRules,
     RedirectOrProxy (..),
     ResponseHeaders (..),
@@ -44,7 +45,14 @@ import Data.HashSet qualified as HashSet
 import Data.Text (Text)
 import Data.Text qualified as Text
 import Data.Text.Encoding qualified as Text
-import Network.HTTP.Types.URI (Query, parseQuery, parseQueryText, renderQuery)
+import Network.HTTP.Types.URI
+  ( Query,
+    parseQuery,
+    parseQueryText,
+    renderQuery,
+    renderQueryBuilder,
+    urlEncodeBuilder,
+  )
 import Network.URI (URI (..), parseRelativeReference, parseURI)
 import Network.Wai
   ( pathInfo,
@@ -116,6 +124,20 @@ data FileRuleV2 = FileRuleV2
 instance Ord FileRuleV2 where
   a `compare` b = ruleIncomingPathRegex a `compare` ruleIncomingPathRegex b
 
+data CatchAllRuleV1 = CatchAllRuleV1
+  { -- | Package this rule belongs to.
+    rulePackage :: Text,
+    -- | Domain to redirect to. The request path and query string
+    -- will of the incoming request will be passed to this domain.
+    -- If not present, the Gateway returns a 200 instead of a
+    -- redirect via 302.
+    ruleRedirectTargetDomain :: Maybe Text
+  }
+  deriving (Eq, Show)
+
+instance Ord CatchAllRuleV1 where
+  a `compare` b = ruleRedirectTargetDomain a `compare` ruleRedirectTargetDomain b
+
 -- | Matching Scarf's documentation pixels. We don't validate the pixel-id
 -- at request time. Just log the access and carry on.
 data PixelRule = PixelRule
@@ -182,6 +204,7 @@ data Rule
   | RulePixel PixelRule
   | RulePythonV1 PythonRuleV1
   | RuleScarfJs
+  | RuleCatchAllV1 CatchAllRuleV1
   deriving (Show, Eq)
 
 instance Ord Rule where
@@ -215,6 +238,8 @@ instance Ord Rule where
     (RuleScarfJs, RuleScarfJs) -> EQ
     (RuleScarfJs, _) -> LT
     (_, RuleScarfJs) -> GT
+    (RuleCatchAllV1 {}, _) -> GT
+    (_, RuleCatchAllV1 {}) -> LT
 
 data ResponseHeaders = ResponseHeaders
   { contentType :: ByteString,
@@ -323,6 +348,19 @@ newFileRuleV2 package incoming backend =
         ruleBackendTemplate = backend
       }
 
+newCatchAllRule ::
+  -- | Package identifier
+  Text ->
+  -- | Redirect target domain, if any
+  Maybe Text ->
+  Rule
+newCatchAllRule package redirectTargetDomain =
+  RuleCatchAllV1
+    CatchAllRuleV1
+      { rulePackage = package,
+        ruleRedirectTargetDomain = redirectTargetDomain
+      }
+
 newPixelRule ::
   -- | Content-type of pixel
   ByteString ->
@@ -537,6 +575,8 @@ matchRule rule request = case rule of
     matchPython pythonRule request
   RuleScarfJs ->
     matchScarfJsPackageEvent request
+  RuleCatchAllV1 rule ->
+    matchCatchAllV1 rule request
   where
     responseBuilder =
       Scarf.Gateway.Rule.Response.ResponseBuilder
@@ -806,6 +846,67 @@ matchPixel PixelRule {..} Request {requestWai}
   | otherwise =
       pure Nothing
 
+-- | Match a catch-all rule. In its current form the only thing it does is redirect a request 
+-- to separate domain. Something that, in the past, has been done using file rules and catch-all
+-- patterns like /{+path}. While effective URI templates are not great at dealing with encoding
+-- specifics.
+matchCatchAllV1 ::
+  (MonadMatch m) =>
+  CatchAllRuleV1 ->
+  Request ->
+  m (Maybe RedirectOrProxy)
+matchCatchAllV1 CatchAllRuleV1 {..} Request {requestWai}
+  | Just targetDomain <- ruleRedirectTargetDomain = do
+      -- Construct the absolute URL to redirect to.
+      let !absoluteUrl =
+            LBS.toStrict $
+              toLazyByteString $
+                -- Just using the protocol that the request came in with 
+                -- allows for convenient testing.
+                (if Wai.isSecure requestWai then "https://" else "http://")
+                  <> Text.encodeUtf8Builder targetDomain
+                  <> (
+                       -- We would just like to use encodePath which is provided by http-types
+                       -- but that doesn't apply all the escaping out of the box. Instead, we
+                       -- are essentially inlining its implementation here with the necessary
+                       -- adjustment.
+                       let encodedSegments =
+                             foldr
+                               (\x -> ((char7 '/' <> urlEncodeBuilder True (Text.encodeUtf8 x)) <>))
+                               mempty
+                               (Wai.pathInfo requestWai)
+                        in case Wai.queryString requestWai of
+                             [] ->
+                               encodedSegments
+                             query ->
+                               encodedSegments <> renderQueryBuilder True query
+                     )
+      pure
+        ( Just
+            ( RedirectTo
+                ( FlatfileCapture
+                    { filePackage = rulePackage,
+                      fileAbsoluteUrl = Just absoluteUrl,
+                      fileVariables = mempty
+                    }
+                )
+                absoluteUrl
+            )
+        )
+  | otherwise =
+      -- No target domain, just capture the request.
+      pure
+        ( Just
+            ( RespondOk
+                ( FlatfileCapture
+                    { filePackage = rulePackage,
+                      fileAbsoluteUrl = Nothing,
+                      fileVariables = mempty
+                    }
+                )
+            )
+        )
+
 -- | Etag matching for caching of Python Simple index. This allows clients (e.g. pip) to
 -- avoid downloading the whole index again.
 ifNoneMatch ::

test/golden/catch-all-1.output.yaml

@@ -0,0 +1,10 @@
+capture: |-
+  Just
+    FlatfileCapture
+      { fileAbsoluteUrl = Just "http://downloads.test.sh/hello%20world"
+      , fileVariables = fromList []
+      , filePackage = "8717953c-3452-4ef7-9a14-b64dc19163b4"
+      }
+headers:
+  Location: http://downloads.test.sh/hello%20world
+status: 302

test/golden/catch-all-1.yaml

@@ -0,0 +1,9 @@
+path: /hello%20world
+headers: 
+  Host: downloads.test.io
+manifest:
+  rules:
+    - type:  catch-all-v1
+      domain: downloads.test.io
+      redirect-target-domain: downloads.test.sh
+      package-id: "8717953c-3452-4ef7-9a14-b64dc19163b4"

test/golden/catch-all-2.output.yaml

@@ -0,0 +1,12 @@
+capture: |-
+  Just
+    FlatfileCapture
+      { fileAbsoluteUrl =
+          Just "http://downloads.test.sh/search?q=hello%20world"
+      , fileVariables = fromList []
+      , filePackage = "8717953c-3452-4ef7-9a14-b64dc19163b4"
+      }
+headers:
+  Location: http://downloads.test.sh/search?q=hello%20world
+query: ?q=hello%20world
+status: 302

test/golden/catch-all-2.yaml

@@ -0,0 +1,9 @@
+path: /search?q=hello%20world
+headers: 
+  Host: downloads.test.io
+manifest:
+  rules:
+    - type:  catch-all-v1
+      domain: downloads.test.io
+      redirect-target-domain: downloads.test.sh
+      package-id: "8717953c-3452-4ef7-9a14-b64dc19163b4"