From fa9a3a0afba71852d9e18b01909b8981ef3d4496 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Tue, 19 Dec 2023 00:40:18 -0800 Subject: [PATCH 01/12] Handle HTTP 103 Early Hints --- http-client/Network/HTTP/Client/Connection.hs | 6 ++++ http-client/Network/HTTP/Client/Headers.hs | 33 +++++++++++++++---- .../Network/HTTP/Client/HeadersSpec.hs | 18 ++++++++++ 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/http-client/Network/HTTP/Client/Connection.hs b/http-client/Network/HTTP/Client/Connection.hs index f4c42df7..a57da553 100644 --- a/http-client/Network/HTTP/Client/Connection.hs +++ b/http-client/Network/HTTP/Client/Connection.hs @@ -5,6 +5,7 @@ module Network.HTTP.Client.Connection ( connectionReadLine , connectionReadLineWith , connectionDropTillBlankLine + , connectionUnreadLine , dummyConnection , openSocketConnection , openSocketConnectionSize @@ -60,6 +61,11 @@ connectionReadLineWith mhl conn bs0 = unless (S.null y) $! connectionUnread conn y return $! killCR $! S.concat $! front [x] +connectionUnreadLine :: Connection -> ByteString -> IO () +connectionUnreadLine conn line = do + connectionUnread conn (S.pack [charCR, charLF]) + connectionUnread conn line + charLF, charCR :: Word8 charLF = 10 charCR = 13 diff --git a/http-client/Network/HTTP/Client/Headers.hs b/http-client/Network/HTTP/Client/Headers.hs index 087653d6..f11452f6 100644 --- a/http-client/Network/HTTP/Client/Headers.hs +++ b/http-client/Network/HTTP/Client/Headers.hs @@ -1,4 +1,6 @@ {-# LANGUAGE DisambiguateRecordFields #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE MultiWayIf #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ViewPatterns #-} module Network.HTTP.Client.Headers @@ -45,11 +47,17 @@ parseStatusHeaders mhl conn timeout' cont Just s -> return s Nothing -> sendBody >> getStatus + nextStatusHeaders :: IO (Maybe StatusHeaders) nextStatusHeaders = do (s, v) <- nextStatusLine mhl - if statusCode s == 100 - then connectionDropTillBlankLine mhl conn >> return Nothing - else Just . StatusHeaders s v A.<$> parseHeaders (0 :: Int) id + if | statusCode s == 100 -> connectionDropTillBlankLine mhl conn >> return Nothing + | statusCode s == 103 -> do + linkHeaders <- parseHeadersUntilFailure 0 id + nextStatusHeaders >>= \case + Nothing -> return Nothing + Just (StatusHeaders s' v' reqHeaders) -> + return $ Just $ StatusHeaders s' v' (linkHeaders <> reqHeaders) + | otherwise -> Just . StatusHeaders s v A.<$> parseHeaders 0 id nextStatusLine :: Maybe MaxHeaderLength -> IO (Status, HttpVersion) nextStatusLine mhl = do @@ -82,14 +90,14 @@ parseStatusHeaders mhl conn timeout' cont Just (i, "") -> Just i _ -> Nothing + parseHeaders :: Int -> ([Header] -> [Header]) -> IO [Header] parseHeaders 100 _ = throwHttp OverlongHeaders parseHeaders count front = do line <- connectionReadLine mhl conn if S.null line then return $ front [] - else do - mheader <- parseHeader line - case mheader of + else + parseHeader line >>= \case Just header -> parseHeaders (count + 1) $ front . (header:) Nothing -> @@ -97,6 +105,19 @@ parseStatusHeaders mhl conn timeout' cont -- an exception, ignore it for robustness. parseHeaders count front + parseHeadersUntilFailure :: Int -> ([Header] -> [Header]) -> IO [Header] + parseHeadersUntilFailure 100 _ = throwHttp OverlongHeaders + parseHeadersUntilFailure count front = do + line <- connectionReadLine mhl conn + if S.null line + then return $ front [] + else + parseHeader line >>= \case + Just header -> parseHeadersUntilFailure (count + 1) $ front . (header:) + Nothing -> do + connectionUnreadLine conn line + return $ front [] + parseHeader :: S.ByteString -> IO (Maybe Header) parseHeader bs = do let (key, bs2) = S.break (== charColon) bs diff --git a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs index 08304b9f..a3284612 100644 --- a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs @@ -60,3 +60,21 @@ spec = describe "HeadersSpec" $ do statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [ ("foo", "bar") ] out >>= (`shouldBe` []) inp >>= (`shouldBe` ["result"]) + + it "103 early hints" $ do + let input = + [ "HTTP/1.1 103 Early Hints\r\n" + , "Link: \r\n" + , "Link: \r\n\r\n" + , "HTTP/1.1 200 OK\r\n" + , "Content-Type: text/html\r\n\r\n" + , "
" + ] + (conn, _, inp) <- dummyConnection input + statusHeaders <- parseStatusHeaders Nothing conn Nothing Nothing + statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [ + ("Link", "") + , ("Link", "") + , ("Content-Type", "text/html") + ] + inp >>= (`shouldBe` ["
"]) From 76e46f7ba9b6f62832454ddcc65e65bc0d8c1768 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Tue, 19 Dec 2023 01:14:48 -0800 Subject: [PATCH 02/12] Put early hints into `responseEarlyHints` field --- http-client/Network/HTTP/Client/Headers.hs | 8 ++++---- http-client/Network/HTTP/Client/Manager.hs | 2 +- http-client/Network/HTTP/Client/Response.hs | 3 ++- http-client/Network/HTTP/Client/Types.hs | 7 ++++++- .../Network/HTTP/Client/HeadersSpec.hs | 19 ++++++++++--------- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/http-client/Network/HTTP/Client/Headers.hs b/http-client/Network/HTTP/Client/Headers.hs index f11452f6..7379a005 100644 --- a/http-client/Network/HTTP/Client/Headers.hs +++ b/http-client/Network/HTTP/Client/Headers.hs @@ -52,12 +52,12 @@ parseStatusHeaders mhl conn timeout' cont (s, v) <- nextStatusLine mhl if | statusCode s == 100 -> connectionDropTillBlankLine mhl conn >> return Nothing | statusCode s == 103 -> do - linkHeaders <- parseHeadersUntilFailure 0 id + earlyHeaders <- parseHeadersUntilFailure 0 id nextStatusHeaders >>= \case Nothing -> return Nothing - Just (StatusHeaders s' v' reqHeaders) -> - return $ Just $ StatusHeaders s' v' (linkHeaders <> reqHeaders) - | otherwise -> Just . StatusHeaders s v A.<$> parseHeaders 0 id + Just (StatusHeaders s' v' earlyHeaders' reqHeaders) -> + return $ Just $ StatusHeaders s' v' (earlyHeaders <> earlyHeaders') reqHeaders + | otherwise -> (Just <$>) $ StatusHeaders s v mempty A.<$> parseHeaders 0 id nextStatusLine :: Maybe MaxHeaderLength -> IO (Status, HttpVersion) nextStatusLine mhl = do diff --git a/http-client/Network/HTTP/Client/Manager.hs b/http-client/Network/HTTP/Client/Manager.hs index 57462f1e..fbf4eb04 100644 --- a/http-client/Network/HTTP/Client/Manager.hs +++ b/http-client/Network/HTTP/Client/Manager.hs @@ -259,7 +259,7 @@ mkCreateConnection ms = do , "\r\n" ] parse conn = do - StatusHeaders status _ _ <- parseStatusHeaders (managerMaxHeaderLength ms) conn Nothing Nothing + StatusHeaders status _ _ _ <- parseStatusHeaders (managerMaxHeaderLength ms) conn Nothing Nothing unless (status == status200) $ throwHttp $ ProxyConnectException ultHost ultPort status in tlsProxyConnection diff --git a/http-client/Network/HTTP/Client/Response.hs b/http-client/Network/HTTP/Client/Response.hs index 242723fc..7c589e90 100644 --- a/http-client/Network/HTTP/Client/Response.hs +++ b/http-client/Network/HTTP/Client/Response.hs @@ -121,7 +121,7 @@ getResponse :: Maybe MaxHeaderLength -> IO (Response BodyReader) getResponse mhl timeout' req@(Request {..}) mconn cont = do let conn = managedResource mconn - StatusHeaders s version hs <- parseStatusHeaders mhl conn timeout' cont + StatusHeaders s version earlyHs hs <- parseStatusHeaders mhl conn timeout' cont let mcl = lookup "content-length" hs >>= readPositiveInt . S8.unpack isChunked = ("transfer-encoding", CI.mk "chunked") `elem` map (second CI.mk) hs @@ -162,6 +162,7 @@ getResponse mhl timeout' req@(Request {..}) mconn cont = do , responseCookieJar = Data.Monoid.mempty , responseClose' = ResponseClose (cleanup False) , responseOriginalRequest = req {requestBody = ""} + , responseEarlyHints = earlyHs } -- | Does this response have no body? diff --git a/http-client/Network/HTTP/Client/Types.hs b/http-client/Network/HTTP/Client/Types.hs index 7a142cfc..8c8d0bfc 100644 --- a/http-client/Network/HTTP/Client/Types.hs +++ b/http-client/Network/HTTP/Client/Types.hs @@ -90,7 +90,7 @@ data Connection = Connection } deriving T.Typeable -data StatusHeaders = StatusHeaders Status HttpVersion RequestHeaders +data StatusHeaders = StatusHeaders Status HttpVersion RequestHeaders RequestHeaders deriving (Show, Eq, Ord, T.Typeable) -- | A newtype wrapper which is not exported from this library but is an @@ -715,6 +715,11 @@ data Response body = Response -- via @getOriginalRequest@ instead. -- -- Since 0.7.8 + , responseEarlyHints :: ResponseHeaders + -- ^ Early response headers sent by the server, as part of an HTTP + -- 103 Early Hints section. + -- + -- Since 0.7.16 } deriving (Show, T.Typeable, Functor, Data.Foldable.Foldable, Data.Traversable.Traversable) diff --git a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs index a3284612..6f96cf6a 100644 --- a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs @@ -21,7 +21,7 @@ spec = describe "HeadersSpec" $ do ] (connection, _, _) <- dummyConnection input statusHeaders <- parseStatusHeaders Nothing connection Nothing Nothing - statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) + statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) mempty [ ("foo", "bar") , ("baz", "bin") ] @@ -35,7 +35,7 @@ spec = describe "HeadersSpec" $ do (conn, out, _) <- dummyConnection input let sendBody = connectionWrite conn "data" statusHeaders <- parseStatusHeaders Nothing conn Nothing (Just sendBody) - statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [ ("foo", "bar") ] + statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [] [ ("foo", "bar") ] out >>= (`shouldBe` ["data"]) it "Expect: 100-continue (failure)" $ do @@ -45,7 +45,7 @@ spec = describe "HeadersSpec" $ do (conn, out, _) <- dummyConnection input let sendBody = connectionWrite conn "data" statusHeaders <- parseStatusHeaders Nothing conn Nothing (Just sendBody) - statusHeaders `shouldBe` StatusHeaders status417 (HttpVersion 1 1) [] + statusHeaders `shouldBe` StatusHeaders status417 (HttpVersion 1 1) [] [] out >>= (`shouldBe` []) it "100 Continue without expectation is OK" $ do @@ -57,7 +57,7 @@ spec = describe "HeadersSpec" $ do ] (conn, out, inp) <- dummyConnection input statusHeaders <- parseStatusHeaders Nothing conn Nothing Nothing - statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [ ("foo", "bar") ] + statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [] [ ("foo", "bar") ] out >>= (`shouldBe` []) inp >>= (`shouldBe` ["result"]) @@ -72,9 +72,10 @@ spec = describe "HeadersSpec" $ do ] (conn, _, inp) <- dummyConnection input statusHeaders <- parseStatusHeaders Nothing conn Nothing Nothing - statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [ - ("Link", "") - , ("Link", "") - , ("Content-Type", "text/html") - ] + statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) + [("Link", "") + , ("Link", "") + ] + [("Content-Type", "text/html") + ] inp >>= (`shouldBe` ["
"]) From 07043c23e67add25419aa54fa351d9bc239b2199 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Tue, 19 Dec 2023 01:18:24 -0800 Subject: [PATCH 03/12] Expose responseEarlyHints constructor --- http-client/Network/HTTP/Client.hs | 1 + http-conduit/Network/HTTP/Conduit.hs | 1 + 2 files changed, 2 insertions(+) diff --git a/http-client/Network/HTTP/Client.hs b/http-client/Network/HTTP/Client.hs index 813c85ca..34e6c315 100644 --- a/http-client/Network/HTTP/Client.hs +++ b/http-client/Network/HTTP/Client.hs @@ -184,6 +184,7 @@ module Network.HTTP.Client , responseBody , responseCookieJar , getOriginalRequest + , responseEarlyHints , throwErrorStatusCodes -- ** Response body , BodyReader diff --git a/http-conduit/Network/HTTP/Conduit.hs b/http-conduit/Network/HTTP/Conduit.hs index 5f0dc07f..ba6f88db 100644 --- a/http-conduit/Network/HTTP/Conduit.hs +++ b/http-conduit/Network/HTTP/Conduit.hs @@ -179,6 +179,7 @@ module Network.HTTP.Conduit , responseHeaders , responseBody , responseCookieJar + , responseEarlyHints -- * Manager , Manager , newManager From f51be27fca9f762c9cecbce1f20692b90660e0cc Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Fri, 22 Dec 2023 00:26:43 -0700 Subject: [PATCH 04/12] Bump cabal file version to 0.7.16 and update ChangeLog.md --- http-client/ChangeLog.md | 4 ++++ http-client/http-client.cabal | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/http-client/ChangeLog.md b/http-client/ChangeLog.md index fc45385d..99242993 100644 --- a/http-client/ChangeLog.md +++ b/http-client/ChangeLog.md @@ -1,5 +1,9 @@ # Changelog for http-client +## 0.7.16 + +* Add `responseEarlyHints` field to `Response`, containing any HTTP 103 Early Hints headers from the server. + ## 0.7.15 * Adds `shouldStripHeaderOnRedirectIfOnDifferentHostOnly` option to `Request` [#520](https://github.com/snoyberg/http-client/pull/520) diff --git a/http-client/http-client.cabal b/http-client/http-client.cabal index a6c6bbdb..f8d4f0d4 100644 --- a/http-client/http-client.cabal +++ b/http-client/http-client.cabal @@ -1,5 +1,5 @@ name: http-client -version: 0.7.15 +version: 0.7.16 synopsis: An HTTP client engine description: Hackage documentation generation is not reliable. For up to date documentation, please see: . homepage: https://github.com/snoyberg/http-client From 1dfea22390ee974d05262785c1cbe9ec90e236b3 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Thu, 28 Dec 2023 11:25:39 -0700 Subject: [PATCH 05/12] Plumb early hints callback through and add test --- http-client/Network/HTTP/Client/Headers.hs | 4 +-- http-client/Network/HTTP/Client/Manager.hs | 2 +- http-client/Network/HTTP/Client/Request.hs | 1 + http-client/Network/HTTP/Client/Response.hs | 2 +- http-client/Network/HTTP/Client/Types.hs | 5 ++++ .../Network/HTTP/Client/HeadersSpec.hs | 26 +++++++++++++++---- 6 files changed, 31 insertions(+), 9 deletions(-) diff --git a/http-client/Network/HTTP/Client/Headers.hs b/http-client/Network/HTTP/Client/Headers.hs index 7379a005..2ef98818 100644 --- a/http-client/Network/HTTP/Client/Headers.hs +++ b/http-client/Network/HTTP/Client/Headers.hs @@ -28,8 +28,8 @@ charColon = 58 charPeriod = 46 -parseStatusHeaders :: Maybe MaxHeaderLength -> Connection -> Maybe Int -> Maybe (IO ()) -> IO StatusHeaders -parseStatusHeaders mhl conn timeout' cont +parseStatusHeaders :: Maybe MaxHeaderLength -> Connection -> Maybe Int -> (Header -> IO ()) -> Maybe (IO ()) -> IO StatusHeaders +parseStatusHeaders mhl conn timeout' onEarlyHintHeader cont | Just k <- cont = getStatusExpectContinue k | otherwise = getStatus where diff --git a/http-client/Network/HTTP/Client/Manager.hs b/http-client/Network/HTTP/Client/Manager.hs index fbf4eb04..fc21ac90 100644 --- a/http-client/Network/HTTP/Client/Manager.hs +++ b/http-client/Network/HTTP/Client/Manager.hs @@ -259,7 +259,7 @@ mkCreateConnection ms = do , "\r\n" ] parse conn = do - StatusHeaders status _ _ _ <- parseStatusHeaders (managerMaxHeaderLength ms) conn Nothing Nothing + StatusHeaders status _ _ _ <- parseStatusHeaders (managerMaxHeaderLength ms) conn Nothing (\_ -> return ()) Nothing unless (status == status200) $ throwHttp $ ProxyConnectException ultHost ultPort status in tlsProxyConnection diff --git a/http-client/Network/HTTP/Client/Request.hs b/http-client/Network/HTTP/Client/Request.hs index f6f3fad2..55647a34 100644 --- a/http-client/Network/HTTP/Client/Request.hs +++ b/http-client/Network/HTTP/Client/Request.hs @@ -306,6 +306,7 @@ defaultRequest = Request , shouldStripHeaderOnRedirectIfOnDifferentHostOnly = False , proxySecureMode = ProxySecureWithConnect , redactHeaders = Set.singleton "Authorization" + , earlyHintHeaderReceived = \_ -> return () } -- | Parses a URL via 'parseRequest_' diff --git a/http-client/Network/HTTP/Client/Response.hs b/http-client/Network/HTTP/Client/Response.hs index 7c589e90..d4d593d5 100644 --- a/http-client/Network/HTTP/Client/Response.hs +++ b/http-client/Network/HTTP/Client/Response.hs @@ -121,7 +121,7 @@ getResponse :: Maybe MaxHeaderLength -> IO (Response BodyReader) getResponse mhl timeout' req@(Request {..}) mconn cont = do let conn = managedResource mconn - StatusHeaders s version earlyHs hs <- parseStatusHeaders mhl conn timeout' cont + StatusHeaders s version earlyHs hs <- parseStatusHeaders mhl conn timeout' earlyHintHeaderReceived cont let mcl = lookup "content-length" hs >>= readPositiveInt . S8.unpack isChunked = ("transfer-encoding", CI.mk "chunked") `elem` map (second CI.mk) hs diff --git a/http-client/Network/HTTP/Client/Types.hs b/http-client/Network/HTTP/Client/Types.hs index 8c8d0bfc..5eb0a5b0 100644 --- a/http-client/Network/HTTP/Client/Types.hs +++ b/http-client/Network/HTTP/Client/Types.hs @@ -634,6 +634,11 @@ data Request = Request -- ^ List of header values being redacted in case we show Request. -- -- @since 0.7.13 + + , earlyHintHeaderReceived :: Header -> IO () + -- ^ Called every time an HTTP 103 Early Hints header is received from the server. + -- + -- @since 0.7.16 } deriving T.Typeable diff --git a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs index 6f96cf6a..dbf863bd 100644 --- a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs @@ -1,6 +1,9 @@ {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} module Network.HTTP.Client.HeadersSpec where +import Control.Concurrent.MVar +import qualified Data.Sequence as Seq import Network.HTTP.Client.Internal import Network.HTTP.Types import Test.Hspec @@ -20,7 +23,7 @@ spec = describe "HeadersSpec" $ do , "\nignored" ] (connection, _, _) <- dummyConnection input - statusHeaders <- parseStatusHeaders Nothing connection Nothing Nothing + statusHeaders <- parseStatusHeaders Nothing connection Nothing (\_ -> return ()) Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) mempty [ ("foo", "bar") , ("baz", "bin") @@ -34,7 +37,7 @@ spec = describe "HeadersSpec" $ do ] (conn, out, _) <- dummyConnection input let sendBody = connectionWrite conn "data" - statusHeaders <- parseStatusHeaders Nothing conn Nothing (Just sendBody) + statusHeaders <- parseStatusHeaders Nothing conn Nothing (\_ -> return ()) (Just sendBody) statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [] [ ("foo", "bar") ] out >>= (`shouldBe` ["data"]) @@ -44,7 +47,7 @@ spec = describe "HeadersSpec" $ do ] (conn, out, _) <- dummyConnection input let sendBody = connectionWrite conn "data" - statusHeaders <- parseStatusHeaders Nothing conn Nothing (Just sendBody) + statusHeaders <- parseStatusHeaders Nothing conn Nothing (\_ -> return ()) (Just sendBody) statusHeaders `shouldBe` StatusHeaders status417 (HttpVersion 1 1) [] [] out >>= (`shouldBe` []) @@ -56,7 +59,7 @@ spec = describe "HeadersSpec" $ do , "result" ] (conn, out, inp) <- dummyConnection input - statusHeaders <- parseStatusHeaders Nothing conn Nothing Nothing + statusHeaders <- parseStatusHeaders Nothing conn Nothing (\_ -> return ()) Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [] [ ("foo", "bar") ] out >>= (`shouldBe` []) inp >>= (`shouldBe` ["result"]) @@ -71,11 +74,24 @@ spec = describe "HeadersSpec" $ do , "
" ] (conn, _, inp) <- dummyConnection input - statusHeaders <- parseStatusHeaders Nothing conn Nothing Nothing + + callbackResults :: MVar (Seq.Seq Header) <- newMVar mempty + let onEarlyHintHeader h = modifyMVar_ callbackResults (return . (Seq.|> h)) + + statusHeaders <- parseStatusHeaders Nothing conn Nothing onEarlyHintHeader Nothing statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) [("Link", "") , ("Link", "") ] [("Content-Type", "text/html") ] + inp >>= (`shouldBe` ["
"]) + + readMVar callbackResults + >>= ( `shouldBe` + Seq.fromList + [ ("Link", ""), + ("Link", "") + ] + ) From 4c56e9558a5d60e610d02f281408c87f229fea01 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Thu, 28 Dec 2023 11:28:19 -0700 Subject: [PATCH 06/12] Call early hint callback in Headers.hs --- http-client/Network/HTTP/Client/Headers.hs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/http-client/Network/HTTP/Client/Headers.hs b/http-client/Network/HTTP/Client/Headers.hs index 2ef98818..13bc1d19 100644 --- a/http-client/Network/HTTP/Client/Headers.hs +++ b/http-client/Network/HTTP/Client/Headers.hs @@ -52,7 +52,7 @@ parseStatusHeaders mhl conn timeout' onEarlyHintHeader cont (s, v) <- nextStatusLine mhl if | statusCode s == 100 -> connectionDropTillBlankLine mhl conn >> return Nothing | statusCode s == 103 -> do - earlyHeaders <- parseHeadersUntilFailure 0 id + earlyHeaders <- parseEarlyHintHeadersUntilFailure 0 id nextStatusHeaders >>= \case Nothing -> return Nothing Just (StatusHeaders s' v' earlyHeaders' reqHeaders) -> @@ -105,15 +105,17 @@ parseStatusHeaders mhl conn timeout' onEarlyHintHeader cont -- an exception, ignore it for robustness. parseHeaders count front - parseHeadersUntilFailure :: Int -> ([Header] -> [Header]) -> IO [Header] - parseHeadersUntilFailure 100 _ = throwHttp OverlongHeaders - parseHeadersUntilFailure count front = do + parseEarlyHintHeadersUntilFailure :: Int -> ([Header] -> [Header]) -> IO [Header] + parseEarlyHintHeadersUntilFailure 100 _ = throwHttp OverlongHeaders + parseEarlyHintHeadersUntilFailure count front = do line <- connectionReadLine mhl conn if S.null line then return $ front [] else parseHeader line >>= \case - Just header -> parseHeadersUntilFailure (count + 1) $ front . (header:) + Just header -> do + onEarlyHintHeader header + parseEarlyHintHeadersUntilFailure (count + 1) $ front . (header:) Nothing -> do connectionUnreadLine conn line return $ front [] From 29bdb3fbd84badbdff1234e08420300935158ac1 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Thu, 28 Dec 2023 11:31:54 -0700 Subject: [PATCH 07/12] Export earlyHintHeaderReceived --- http-client/Network/HTTP/Client.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/http-client/Network/HTTP/Client.hs b/http-client/Network/HTTP/Client.hs index 34e6c315..ed6d9d62 100644 --- a/http-client/Network/HTTP/Client.hs +++ b/http-client/Network/HTTP/Client.hs @@ -168,6 +168,7 @@ module Network.HTTP.Client , cookieJar , requestVersion , redactHeaders + , earlyHintHeaderReceived -- ** Request body , RequestBody (..) , Popper From 345ed727782e2ed834d7a0d1ca14c3fb3cd81ec8 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Thu, 28 Dec 2023 11:33:41 -0700 Subject: [PATCH 08/12] Update CHANGELOG again --- http-client/ChangeLog.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/http-client/ChangeLog.md b/http-client/ChangeLog.md index 99242993..a44ea58c 100644 --- a/http-client/ChangeLog.md +++ b/http-client/ChangeLog.md @@ -2,7 +2,8 @@ ## 0.7.16 -* Add `responseEarlyHints` field to `Response`, containing any HTTP 103 Early Hints headers from the server. +* Add `responseEarlyHints` field to `Response`, containing a list of HTTP 103 Early Hints headers received from the server. +* Add `earlyHintHeaderReceived` callback to `Request`, which will be called on each HTTP 103 Early Hints header received. ## 0.7.15 From 8daeeddf6d04db04b50dbb67496cee29f47bbcbc Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Fri, 29 Dec 2023 12:59:37 -0700 Subject: [PATCH 09/12] Switch callback signature to [Header] -> IO () --- http-client/ChangeLog.md | 4 ++-- http-client/Network/HTTP/Client.hs | 2 +- http-client/Network/HTTP/Client/Headers.hs | 22 +++++++++---------- http-client/Network/HTTP/Client/Request.hs | 2 +- http-client/Network/HTTP/Client/Response.hs | 2 +- http-client/Network/HTTP/Client/Types.hs | 2 +- .../Network/HTTP/Client/HeadersSpec.hs | 12 +++++----- 7 files changed, 22 insertions(+), 24 deletions(-) diff --git a/http-client/ChangeLog.md b/http-client/ChangeLog.md index a44ea58c..e29dcd5a 100644 --- a/http-client/ChangeLog.md +++ b/http-client/ChangeLog.md @@ -2,8 +2,8 @@ ## 0.7.16 -* Add `responseEarlyHints` field to `Response`, containing a list of HTTP 103 Early Hints headers received from the server. -* Add `earlyHintHeaderReceived` callback to `Request`, which will be called on each HTTP 103 Early Hints header received. +* Add `responseEarlyHints` field to `Response`, containing a list of all HTTP 103 Early Hints headers received from the server. +* Add `earlyHintHeadersReceived` callback to `Request`, which will be called on each HTTP 103 Early Hints header section received. ## 0.7.15 diff --git a/http-client/Network/HTTP/Client.hs b/http-client/Network/HTTP/Client.hs index ed6d9d62..ab1b8c06 100644 --- a/http-client/Network/HTTP/Client.hs +++ b/http-client/Network/HTTP/Client.hs @@ -168,7 +168,7 @@ module Network.HTTP.Client , cookieJar , requestVersion , redactHeaders - , earlyHintHeaderReceived + , earlyHintHeadersReceived -- ** Request body , RequestBody (..) , Popper diff --git a/http-client/Network/HTTP/Client/Headers.hs b/http-client/Network/HTTP/Client/Headers.hs index 13bc1d19..1eab92d0 100644 --- a/http-client/Network/HTTP/Client/Headers.hs +++ b/http-client/Network/HTTP/Client/Headers.hs @@ -16,11 +16,11 @@ import qualified Data.ByteString.Char8 as S8 import qualified Data.CaseInsensitive as CI import Data.Maybe (mapMaybe) import Data.Monoid +import Data.Word (Word8) import Network.HTTP.Client.Connection import Network.HTTP.Client.Types -import System.Timeout (timeout) import Network.HTTP.Types -import Data.Word (Word8) +import System.Timeout (timeout) charSpace, charColon, charPeriod :: Word8 charSpace = 32 @@ -28,8 +28,8 @@ charColon = 58 charPeriod = 46 -parseStatusHeaders :: Maybe MaxHeaderLength -> Connection -> Maybe Int -> (Header -> IO ()) -> Maybe (IO ()) -> IO StatusHeaders -parseStatusHeaders mhl conn timeout' onEarlyHintHeader cont +parseStatusHeaders :: Maybe MaxHeaderLength -> Connection -> Maybe Int -> ([Header] -> IO ()) -> Maybe (IO ()) -> IO StatusHeaders +parseStatusHeaders mhl conn timeout' onEarlyHintHeaders cont | Just k <- cont = getStatusExpectContinue k | otherwise = getStatus where @@ -52,11 +52,12 @@ parseStatusHeaders mhl conn timeout' onEarlyHintHeader cont (s, v) <- nextStatusLine mhl if | statusCode s == 100 -> connectionDropTillBlankLine mhl conn >> return Nothing | statusCode s == 103 -> do - earlyHeaders <- parseEarlyHintHeadersUntilFailure 0 id - nextStatusHeaders >>= \case - Nothing -> return Nothing - Just (StatusHeaders s' v' earlyHeaders' reqHeaders) -> - return $ Just $ StatusHeaders s' v' (earlyHeaders <> earlyHeaders') reqHeaders + earlyHeaders <- parseEarlyHintHeadersUntilFailure 0 id + onEarlyHintHeaders earlyHeaders + nextStatusHeaders >>= \case + Nothing -> return Nothing + Just (StatusHeaders s' v' earlyHeaders' reqHeaders) -> + return $ Just $ StatusHeaders s' v' (earlyHeaders <> earlyHeaders') reqHeaders | otherwise -> (Just <$>) $ StatusHeaders s v mempty A.<$> parseHeaders 0 id nextStatusLine :: Maybe MaxHeaderLength -> IO (Status, HttpVersion) @@ -113,8 +114,7 @@ parseStatusHeaders mhl conn timeout' onEarlyHintHeader cont then return $ front [] else parseHeader line >>= \case - Just header -> do - onEarlyHintHeader header + Just header -> parseEarlyHintHeadersUntilFailure (count + 1) $ front . (header:) Nothing -> do connectionUnreadLine conn line diff --git a/http-client/Network/HTTP/Client/Request.hs b/http-client/Network/HTTP/Client/Request.hs index 55647a34..99f37edd 100644 --- a/http-client/Network/HTTP/Client/Request.hs +++ b/http-client/Network/HTTP/Client/Request.hs @@ -306,7 +306,7 @@ defaultRequest = Request , shouldStripHeaderOnRedirectIfOnDifferentHostOnly = False , proxySecureMode = ProxySecureWithConnect , redactHeaders = Set.singleton "Authorization" - , earlyHintHeaderReceived = \_ -> return () + , earlyHintHeadersReceived = \_ -> return () } -- | Parses a URL via 'parseRequest_' diff --git a/http-client/Network/HTTP/Client/Response.hs b/http-client/Network/HTTP/Client/Response.hs index d4d593d5..ceb878ef 100644 --- a/http-client/Network/HTTP/Client/Response.hs +++ b/http-client/Network/HTTP/Client/Response.hs @@ -121,7 +121,7 @@ getResponse :: Maybe MaxHeaderLength -> IO (Response BodyReader) getResponse mhl timeout' req@(Request {..}) mconn cont = do let conn = managedResource mconn - StatusHeaders s version earlyHs hs <- parseStatusHeaders mhl conn timeout' earlyHintHeaderReceived cont + StatusHeaders s version earlyHs hs <- parseStatusHeaders mhl conn timeout' earlyHintHeadersReceived cont let mcl = lookup "content-length" hs >>= readPositiveInt . S8.unpack isChunked = ("transfer-encoding", CI.mk "chunked") `elem` map (second CI.mk) hs diff --git a/http-client/Network/HTTP/Client/Types.hs b/http-client/Network/HTTP/Client/Types.hs index 5eb0a5b0..51a53c52 100644 --- a/http-client/Network/HTTP/Client/Types.hs +++ b/http-client/Network/HTTP/Client/Types.hs @@ -635,7 +635,7 @@ data Request = Request -- -- @since 0.7.13 - , earlyHintHeaderReceived :: Header -> IO () + , earlyHintHeadersReceived :: [Header] -> IO () -- ^ Called every time an HTTP 103 Early Hints header is received from the server. -- -- @since 0.7.16 diff --git a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs index dbf863bd..ae639715 100644 --- a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs @@ -75,7 +75,7 @@ spec = describe "HeadersSpec" $ do ] (conn, _, inp) <- dummyConnection input - callbackResults :: MVar (Seq.Seq Header) <- newMVar mempty + callbackResults :: MVar (Seq.Seq [Header]) <- newMVar mempty let onEarlyHintHeader h = modifyMVar_ callbackResults (return . (Seq.|> h)) statusHeaders <- parseStatusHeaders Nothing conn Nothing onEarlyHintHeader Nothing @@ -89,9 +89,7 @@ spec = describe "HeadersSpec" $ do inp >>= (`shouldBe` ["
"]) readMVar callbackResults - >>= ( `shouldBe` - Seq.fromList - [ ("Link", ""), - ("Link", "") - ] - ) + >>= (`shouldBe` Seq.fromList [ + [("Link", "") + , ("Link", "") + ]]) From 1041fe8645aa34b34007209cd7b771874173be1c Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Fri, 29 Dec 2023 13:02:06 -0700 Subject: [PATCH 10/12] Add test of multiple 103 sections --- .../Network/HTTP/Client/HeadersSpec.hs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs index ae639715..0d242d44 100644 --- a/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs +++ b/http-client/test-nonet/Network/HTTP/Client/HeadersSpec.hs @@ -93,3 +93,38 @@ spec = describe "HeadersSpec" $ do [("Link", "") , ("Link", "") ]]) + + it "103 early hints (multiple sections)" $ do + let input = + [ "HTTP/1.1 103 Early Hints\r\n" + , "Link: \r\n" + , "Link: \r\n\r\n" + , "HTTP/1.1 103 Early Hints\r\n" + , "Link: \r\n\r\n" + , "HTTP/1.1 200 OK\r\n" + , "Content-Type: text/html\r\n\r\n" + , "
" + ] + (conn, _, inp) <- dummyConnection input + + callbackResults :: MVar (Seq.Seq [Header]) <- newMVar mempty + let onEarlyHintHeader h = modifyMVar_ callbackResults (return . (Seq.|> h)) + + statusHeaders <- parseStatusHeaders Nothing conn Nothing onEarlyHintHeader Nothing + statusHeaders `shouldBe` StatusHeaders status200 (HttpVersion 1 1) + [("Link", "") + , ("Link", "") + , ("Link", "") + ] + [("Content-Type", "text/html") + ] + + inp >>= (`shouldBe` ["
"]) + + readMVar callbackResults + >>= (`shouldBe` Seq.fromList [ + [("Link", "") + , ("Link", "") + ] + , [("Link", "")] + ]) From 4fd5eba28d75c8104eb7b4e3506a2b06560a9516 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Fri, 29 Dec 2023 13:06:26 -0700 Subject: [PATCH 11/12] Fix an import indentation in Headers.hs --- http-client/Network/HTTP/Client/Headers.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http-client/Network/HTTP/Client/Headers.hs b/http-client/Network/HTTP/Client/Headers.hs index 1eab92d0..fefe808c 100644 --- a/http-client/Network/HTTP/Client/Headers.hs +++ b/http-client/Network/HTTP/Client/Headers.hs @@ -16,7 +16,7 @@ import qualified Data.ByteString.Char8 as S8 import qualified Data.CaseInsensitive as CI import Data.Maybe (mapMaybe) import Data.Monoid -import Data.Word (Word8) +import Data.Word (Word8) import Network.HTTP.Client.Connection import Network.HTTP.Client.Types import Network.HTTP.Types From 487bebf535252f7deb163c409598b64461053f87 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Fri, 29 Dec 2023 13:07:59 -0700 Subject: [PATCH 12/12] Update comment in Types.hs --- http-client/Network/HTTP/Client/Types.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http-client/Network/HTTP/Client/Types.hs b/http-client/Network/HTTP/Client/Types.hs index 51a53c52..646dac79 100644 --- a/http-client/Network/HTTP/Client/Types.hs +++ b/http-client/Network/HTTP/Client/Types.hs @@ -636,7 +636,7 @@ data Request = Request -- @since 0.7.13 , earlyHintHeadersReceived :: [Header] -> IO () - -- ^ Called every time an HTTP 103 Early Hints header is received from the server. + -- ^ Called every time an HTTP 103 Early Hints header section is received from the server. -- -- @since 0.7.16 }