-
Notifications
You must be signed in to change notification settings - Fork 162
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Improve offchain #1860
base: master
Are you sure you want to change the base?
Improve offchain #1860
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -38,6 +38,7 @@ import qualified Data.Text.Encoding as Text | |
import GHC.Show (show) | ||
import Network.HTTP.Client (HttpException (..)) | ||
import qualified Network.HTTP.Client as Http | ||
import Network.HTTP.Client.TLS (tlsManagerSettings) | ||
import qualified Network.HTTP.Types as Http | ||
|
||
------------------------------------------------------------------------------------- | ||
|
@@ -78,13 +79,33 @@ httpGetOffChainPoolData manager request purl expectedMetaHash = do | |
url = OffChainPoolUrl purl | ||
|
||
httpGetOffChainVoteData :: | ||
Http.Manager -> | ||
Http.Request -> | ||
[Text] -> | ||
VoteUrl -> | ||
Maybe VoteMetaHash -> | ||
DB.AnchorType -> | ||
ExceptT OffChainFetchError IO SimplifiedOffChainVoteData | ||
httpGetOffChainVoteData gateways vurl metaHash anchorType = do | ||
case useIpfsGatewayMaybe vurl gateways of | ||
Nothing -> httpGetOffChainVoteDataSingle vurl metaHash anchorType | ||
Just [] -> left $ OCFErrNoIpfsGateway (OffChainVoteUrl vurl) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why not just make this a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It can be empty There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I might be reading this wrong, but it looks to me like having empty There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's an error that doesn't bubble up. It ends up in the off_chain_vote_fetch_error table. |
||
Just urls -> tryAllGatewaysRec urls [] | ||
where | ||
tryAllGatewaysRec [] acc = left $ OCFErrIpfsGatewayFailures (OffChainVoteUrl vurl) (reverse acc) | ||
tryAllGatewaysRec (url : rest) acc = do | ||
msocd <- liftIO $ runExceptT $ httpGetOffChainVoteDataSingle url metaHash anchorType | ||
case msocd of | ||
Right socd -> pure socd | ||
Left err -> tryAllGatewaysRec rest (err : acc) | ||
|
||
httpGetOffChainVoteDataSingle :: | ||
VoteUrl -> | ||
Maybe VoteMetaHash -> | ||
DB.AnchorType -> | ||
ExceptT OffChainFetchError IO SimplifiedOffChainVoteData | ||
httpGetOffChainVoteData manager request vurl metaHash anchorType = do | ||
httpGetOffChainVoteDataSingle vurl metaHash anchorType = do | ||
manager <- liftIO $ Http.newManager tlsManagerSettings | ||
request <- parseOffChainUrl url | ||
let req = httpGetBytes manager request 10000 30000 url | ||
httpRes <- handleExceptT (convertHttpException url) req | ||
(respBS, respLBS, mContentType) <- hoistEither httpRes | ||
(ocvd, decodedValue, metadataHash, mWarning) <- parseAndValidateVoteData respBS respLBS metaHash anchorType (Just $ OffChainVoteUrl vurl) | ||
|
@@ -98,7 +119,6 @@ httpGetOffChainVoteData manager request vurl metaHash anchorType = do | |
, sovaWarning = mWarning | ||
} | ||
where | ||
req = httpGetBytes manager request 10000 30000 url | ||
url = OffChainVoteUrl vurl | ||
|
||
parseAndValidateVoteData :: ByteString -> LBS.ByteString -> Maybe VoteMetaHash -> DB.AnchorType -> Maybe OffChainUrlType -> ExceptT OffChainFetchError IO (Vote.OffChainVoteData, Aeson.Value, ByteString, Maybe Text) | ||
|
@@ -152,6 +172,8 @@ httpGetBytes manager request bytesToRead maxBytes url = | |
OCFErrBadContentTypeHtml url (Text.decodeLatin1 ct) | ||
unless | ||
( "application/json" | ||
`BS.isInfixOf` ct | ||
|| "application/ld+json" | ||
`BS.isInfixOf` ct | ||
|| "text/plain" | ||
`BS.isInfixOf` ct | ||
|
@@ -217,3 +239,9 @@ convertHttpException url he = | |
case url of | ||
OffChainPoolUrl _ -> OCFErrUrlParseFail (OffChainPoolUrl $ PoolUrl $ Text.pack urlx) (Text.pack err) | ||
OffChainVoteUrl _ -> OCFErrUrlParseFail (OffChainVoteUrl $ VoteUrl $ Text.pack urlx) (Text.pack err) | ||
|
||
useIpfsGatewayMaybe :: VoteUrl -> [Text] -> Maybe [VoteUrl] | ||
useIpfsGatewayMaybe vu gateways = | ||
case Text.stripPrefix "ipfs://" (unVoteUrl vu) of | ||
Just sf -> Just $ VoteUrl . (<> sf) <$> gateways | ||
Nothing -> Nothing |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -56,6 +56,7 @@ syncPreConfig = | |
<*> Gen.bool | ||
<*> Gen.int (Range.linear 0 10000) | ||
<*> syncInsertConfig | ||
<*> pure [] | ||
|
||
syncNodeParams :: MonadGen m => m SyncNodeParams | ||
syncNodeParams = | ||
|
@@ -103,6 +104,7 @@ syncNodeConfig loggingCfg = | |
<*> triggerHardFork | ||
<*> triggerHardFork | ||
<*> syncInsertOptions | ||
<*> pure [] | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It would probably be useful to generate There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Any suggestions for a generator to be used here? Not bvery familiar with these tests |
||
|
||
syncInsertConfig :: Gen SyncInsertConfig | ||
syncInsertConfig = | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
hlint always complains about
_ ->
that's the only reason I've been doing_otherwise
🤷