From 7eecf34fd38b90f734a544ce7343237a7d2d19a8 Mon Sep 17 00:00:00 2001 From: peachbits <41451710+peachbits@users.noreply.github.com> Date: Wed, 21 Apr 2021 23:00:41 -0700 Subject: [PATCH] Convert exchange rate provider to query multiple rates per request (#127) * Bulkify Nomics query In the process I hardcoded the convert parameter to USD.. Leaving it as a variable and coding for it would result in hundred of queries on new installations. * Bulkify Coincap * Bulkify currencyconverterapi --- src/rate/coincap.js | 65 +++++++++++++++++++++++++------- src/rate/currencyconverterapi.js | 50 +++++++++++++++--------- src/rate/nomics.js | 65 ++++++++++++++++++++------------ 3 files changed, 124 insertions(+), 56 deletions(-) diff --git a/src/rate/coincap.js b/src/rate/coincap.js index 5f6eff77..2bcaa8ab 100644 --- a/src/rate/coincap.js +++ b/src/rate/coincap.js @@ -1,15 +1,23 @@ // @flow -import { asArray, asObject, asString } from 'cleaners' +import { asArray, asNumber, asObject, asOptional, asString } from 'cleaners' import { type EdgeCorePluginOptions, type EdgeRatePlugin } from 'edge-core-js/types' const asCoincapResponse = asObject({ - data: asObject({ - priceUsd: asString - }) + data: asArray( + asObject({ + symbol: asString, + priceUsd: asString + }) + ) +}) + +const asCoincapError = asObject({ + error: asOptional(asString), + timestamp: asNumber }) const asCoincapAssets = asArray( @@ -33,26 +41,55 @@ export function makeCoincapPlugin(opts: EdgeCorePluginOptions): EdgeRatePlugin { async fetchRates(pairsHint) { const pairs = [] + // Create unique ID map if (Object.keys(currencyMap).length === 0) { const assets = await fetch(`https://api.coincap.io/v2/assets/`) const assetsJson = await assets.json() const assetIds = asCoincapAssets(assetsJson.data) assetIds.forEach(code => (currencyMap[code.symbol] = code.id)) } - for (const pair of pairsHint) { - // Coincap only provides prices in USD and must be queried by unique identifier rather that currency code - if (!currencyMap[pair.fromCurrency]) continue + + // Create query strings + const queryStrings = [] + let filteredPairs = [] + for (let i = 0; i < pairsHint.length; i++) { + if (!currencyMap[pairsHint[i].fromCurrency]) continue + if (pairsHint[i].fromCurrency.indexOf('iso:') >= 0) continue + if ( + filteredPairs.some( + cc => cc === currencyMap[pairsHint[i].fromCurrency] + ) + ) + continue + filteredPairs.push(currencyMap[pairsHint[i].fromCurrency]) + if (filteredPairs.length === 100 || i === pairsHint.length - 1) { + queryStrings.push(filteredPairs.join(',')) + filteredPairs = [] + } + } + + for (const query of queryStrings) { + // Coincap only provides prices in USD try { const reply = await fetch( - `https://api.coincap.io/v2/assets/${currencyMap[pair.fromCurrency]}` + `https://api.coincap.io/v2/assets?ids=${query}` ) const json = await reply.json() - const rate = parseFloat(asCoincapResponse(json).data.priceUsd) - pairs.push({ - fromCurrency: pair.fromCurrency, - toCurrency: 'iso:USD', - rate - }) + const { error } = asCoincapError(json) + if ((error != null && error !== '') || reply.ok === false) { + throw new Error( + `CoincapHistorical returned code ${JSON.stringify( + error ?? reply.status + )}` + ) + } + asCoincapResponse(json).data.forEach(rate => + pairs.push({ + fromCurrency: rate.symbol, + toCurrency: 'iso:USD', + rate: Number(rate.priceUsd) + }) + ) } catch (e) { log.warn(`Issue with Coincap rate data structure ${e}`) } diff --git a/src/rate/currencyconverterapi.js b/src/rate/currencyconverterapi.js index edf33dab..43d2b52b 100644 --- a/src/rate/currencyconverterapi.js +++ b/src/rate/currencyconverterapi.js @@ -1,15 +1,22 @@ // @flow +import { asMap, asNumber, asObject, asOptional, asString } from 'cleaners' import { type EdgeCorePluginOptions, type EdgeRatePlugin } from 'edge-core-js/types' +const asCurrencyConverterResponse = asObject({ + status: asOptional(asNumber), + error: asOptional(asString), + ...asMap(asNumber) +}) + const checkAndPush = (isoCc, ccArray) => { if (isoCc !== 'iso:USD' && isoCc.slice(0, 4) === 'iso:') { const cc = isoCc.slice(4).toUpperCase() - if (!ccArray.includes(cc)) { - ccArray.push(cc) + if (!ccArray.includes(`USD_${cc}`)) { + ccArray.push(`USD_${cc}`) } } } @@ -43,27 +50,34 @@ export function makeCurrencyconverterapiPlugin( } const pairs = [] - for (const isoCode of isoCodesWanted) { - try { - const query = `USD_${isoCode}` - const response = await fetchCors( - `https://api.currencyconverterapi.com/api/v6/convert?q=${query}&compact=ultra&apiKey=${apiKey}` + const query = isoCodesWanted.join(',') + try { + const response = await fetchCors( + `https://api.currconv.com/api/v7/convert?q=${query}&compact=ultra&apiKey=${apiKey}` + ) + const { status, error, ...rates } = asCurrencyConverterResponse( + await response.json() + ) + if ( + (status != null && status !== 200) || + (error != null && error !== '') || + response.ok === false + ) { + throw new Error( + `CurrencyConvertor returned with status: ${JSON.stringify( + status ?? response.status + )} and error: ${JSON.stringify(error)}` ) - if (!response.ok) continue - const json = await response.json() - if (json == null || json[query] == null) continue - const rate = json[query] + } + for (const rate of Object.keys(rates)) { pairs.push({ fromCurrency: 'iso:USD', - toCurrency: `iso:${isoCode}`, - rate + toCurrency: `iso:${rate.split('_')[1]}`, + rate: rates[rate] }) - } catch (e) { - log.warn( - `Failed to get ${isoCode} rate from currencyconverterapi.com`, - e - ) } + } catch (e) { + log.warn(`Failed to get ${query} from currencyconverterapi.com`, e) } return pairs } diff --git a/src/rate/nomics.js b/src/rate/nomics.js index 20e2fca5..b3a68d8a 100644 --- a/src/rate/nomics.js +++ b/src/rate/nomics.js @@ -1,6 +1,6 @@ // @flow -import { asArray, asObject, asString } from 'cleaners' +import { asArray, asObject, asOptional, asString } from 'cleaners' import { type EdgeCorePluginOptions, type EdgeRatePlugin @@ -8,15 +8,11 @@ import { const asNomicsResponse = asArray( asObject({ - price: asString + price: asOptional(asString), + symbol: asString }) ) -function checkIfFiat(code: string): boolean { - if (code.indexOf('iso:') >= 0) return true - return false -} - export function makeNomicsPlugin(opts: EdgeCorePluginOptions): EdgeRatePlugin { const { io, initOptions, log } = opts const { fetchCors = io.fetch } = io @@ -33,29 +29,50 @@ export function makeNomicsPlugin(opts: EdgeCorePluginOptions): EdgeRatePlugin { async fetchRates(pairsHint) { const pairs = [] - for (const pair of pairsHint) { - // Skip if codes are both fiat or crypto - if ( - (checkIfFiat(pair.fromCurrency) && checkIfFiat(pair.toCurrency)) || - (!checkIfFiat(pair.fromCurrency) && !checkIfFiat(pair.toCurrency)) - ) - continue - const fiatCode = pair.toCurrency.split(':') + + // Create query strings + const queryStrings = [] + let filteredPairs = [] + for (let i = 0; i < pairsHint.length; i++) { + if (pairsHint[i].fromCurrency.indexOf('iso:') >= 0) continue + if (filteredPairs.some(cc => cc === pairsHint[i].fromCurrency)) continue + filteredPairs.push(pairsHint[i].fromCurrency) + if (filteredPairs.length === 100 || i === pairsHint.length - 1) { + queryStrings.push(filteredPairs.join(',')) + filteredPairs = [] + } + } + + for (const query of queryStrings) { try { const reply = await fetchCors( - `https://api.nomics.com/v1/currencies/ticker?key=${apiKey}&ids=${pair.fromCurrency}&convert=${fiatCode[1]}` + `https://api.nomics.com/v1/currencies/ticker?key=${apiKey}&ids=${query}&convert=USD` + ) + const replyJson = await reply.json() + if ( + reply.status === 429 || + reply.status === 401 || + reply.ok === false ) - if (reply.status === 429) continue - const jsonData = await reply.json() - const rate = Number(asNomicsResponse(jsonData)[0].price) - pairs.push({ - fromCurrency: pair.fromCurrency, - toCurrency: pair.toCurrency, - rate + throw new Error( + `Nomics returned with status: ${JSON.stringify( + reply.status + )} and error: ${JSON.stringify(replyJson)}` + ) + asNomicsResponse(replyJson).forEach(rate => { + // When Nomics considers a coin "dead" they don't return a price + if (rate.price) + pairs.push({ + fromCurrency: rate.symbol, + toCurrency: 'iso:USD', + rate: Number(rate.price) + }) }) } catch (e) { log.warn( - `Issue with Nomics rate data structure for ${pair.fromCurrency}/${pair.toCurrency} pair. Error: ${e}` + `Issue with Nomics rate data structure. Querystrings ${JSON.stringify( + queryStrings + )} Error: ${e}` ) } }