Convert exchange rate provider to query multiple rates per request (#127

) * Bulkify Nomics query In the process I hardcoded the convert parameter to USD.. Leaving it as a variable and coding for it would result in hundred of queries on new installations. * Bulkify Coincap * Bulkify currencyconverterapi
EdgeApp · Apr 22, 2021 · 7eecf34 · 7eecf34
1 parent b2e1d1b
commit 7eecf34
Show file tree

Hide file tree

Showing 3 changed files with 124 additions and 56 deletions.
diff --git a/src/rate/coincap.js b/src/rate/coincap.js
@@ -1,15 +1,23 @@
 // @flow
 
-import { asArray, asObject, asString } from 'cleaners'
+import { asArray, asNumber, asObject, asOptional, asString } from 'cleaners'
 import {
   type EdgeCorePluginOptions,
   type EdgeRatePlugin
 } from 'edge-core-js/types'
 
 const asCoincapResponse = asObject({
-  data: asObject({
-    priceUsd: asString
-  })
+  data: asArray(
+    asObject({
+      symbol: asString,
+      priceUsd: asString
+    })
+  )
+})
+
+const asCoincapError = asObject({
+  error: asOptional(asString),
+  timestamp: asNumber
 })
 
 const asCoincapAssets = asArray(
@@ -33,26 +41,55 @@ export function makeCoincapPlugin(opts: EdgeCorePluginOptions): EdgeRatePlugin {
 
     async fetchRates(pairsHint) {
       const pairs = []
+      // Create unique ID map
       if (Object.keys(currencyMap).length === 0) {
         const assets = await fetch(`https://api.coincap.io/v2/assets/`)
         const assetsJson = await assets.json()
         const assetIds = asCoincapAssets(assetsJson.data)
         assetIds.forEach(code => (currencyMap[code.symbol] = code.id))
       }
-      for (const pair of pairsHint) {
-        // Coincap only provides prices in USD and must be queried by unique identifier rather that currency code
-        if (!currencyMap[pair.fromCurrency]) continue
+
+      // Create query strings
+      const queryStrings = []
+      let filteredPairs = []
+      for (let i = 0; i < pairsHint.length; i++) {
+        if (!currencyMap[pairsHint[i].fromCurrency]) continue
+        if (pairsHint[i].fromCurrency.indexOf('iso:') >= 0) continue
+        if (
+          filteredPairs.some(
+            cc => cc === currencyMap[pairsHint[i].fromCurrency]
+          )
+        )
+          continue
+        filteredPairs.push(currencyMap[pairsHint[i].fromCurrency])
+        if (filteredPairs.length === 100 || i === pairsHint.length - 1) {
+          queryStrings.push(filteredPairs.join(','))
+          filteredPairs = []
+        }
+      }
+
+      for (const query of queryStrings) {
+        // Coincap only provides prices in USD
         try {
           const reply = await fetch(
-            `https://api.coincap.io/v2/assets/${currencyMap[pair.fromCurrency]}`
+            `https://api.coincap.io/v2/assets?ids=${query}`
           )
           const json = await reply.json()
-          const rate = parseFloat(asCoincapResponse(json).data.priceUsd)
-          pairs.push({
-            fromCurrency: pair.fromCurrency,
-            toCurrency: 'iso:USD',
-            rate
-          })
+          const { error } = asCoincapError(json)
+          if ((error != null && error !== '') || reply.ok === false) {
+            throw new Error(
+              `CoincapHistorical returned code ${JSON.stringify(
+                error ?? reply.status
+              )}`
+            )
+          }
+          asCoincapResponse(json).data.forEach(rate =>
+            pairs.push({
+              fromCurrency: rate.symbol,
+              toCurrency: 'iso:USD',
+              rate: Number(rate.priceUsd)
+            })
+          )
         } catch (e) {
           log.warn(`Issue with Coincap rate data structure ${e}`)
         }

diff --git a/src/rate/currencyconverterapi.js b/src/rate/currencyconverterapi.js
@@ -1,15 +1,22 @@
 // @flow
 
+import { asMap, asNumber, asObject, asOptional, asString } from 'cleaners'
 import {
   type EdgeCorePluginOptions,
   type EdgeRatePlugin
 } from 'edge-core-js/types'
 
+const asCurrencyConverterResponse = asObject({
+  status: asOptional(asNumber),
+  error: asOptional(asString),
+  ...asMap(asNumber)
+})
+
 const checkAndPush = (isoCc, ccArray) => {
   if (isoCc !== 'iso:USD' && isoCc.slice(0, 4) === 'iso:') {
     const cc = isoCc.slice(4).toUpperCase()
-    if (!ccArray.includes(cc)) {
-      ccArray.push(cc)
+    if (!ccArray.includes(`USD_${cc}`)) {
+      ccArray.push(`USD_${cc}`)
     }
   }
 }
@@ -43,27 +50,34 @@ export function makeCurrencyconverterapiPlugin(
       }
 
       const pairs = []
-      for (const isoCode of isoCodesWanted) {
-        try {
-          const query = `USD_${isoCode}`
-          const response = await fetchCors(
-            `https://api.currencyconverterapi.com/api/v6/convert?q=${query}&compact=ultra&apiKey=${apiKey}`
+      const query = isoCodesWanted.join(',')
+      try {
+        const response = await fetchCors(
+          `https://api.currconv.com/api/v7/convert?q=${query}&compact=ultra&apiKey=${apiKey}`
+        )
+        const { status, error, ...rates } = asCurrencyConverterResponse(
+          await response.json()
+        )
+        if (
+          (status != null && status !== 200) ||
+          (error != null && error !== '') ||
+          response.ok === false
+        ) {
+          throw new Error(
+            `CurrencyConvertor returned with status: ${JSON.stringify(
+              status ?? response.status
+            )} and error: ${JSON.stringify(error)}`
           )
-          if (!response.ok) continue
-          const json = await response.json()
-          if (json == null || json[query] == null) continue
-          const rate = json[query]
+        }
+        for (const rate of Object.keys(rates)) {
           pairs.push({
             fromCurrency: 'iso:USD',
-            toCurrency: `iso:${isoCode}`,
-            rate
+            toCurrency: `iso:${rate.split('_')[1]}`,
+            rate: rates[rate]
           })
-        } catch (e) {
-          log.warn(
-            `Failed to get ${isoCode} rate from currencyconverterapi.com`,
-            e
-          )
         }
+      } catch (e) {
+        log.warn(`Failed to get ${query} from currencyconverterapi.com`, e)
       }
       return pairs
     }

diff --git a/src/rate/nomics.js b/src/rate/nomics.js
@@ -1,22 +1,18 @@
 // @flow
 
-import { asArray, asObject, asString } from 'cleaners'
+import { asArray, asObject, asOptional, asString } from 'cleaners'
 import {
   type EdgeCorePluginOptions,
   type EdgeRatePlugin
 } from 'edge-core-js/types'
 
 const asNomicsResponse = asArray(
   asObject({
-    price: asString
+    price: asOptional(asString),
+    symbol: asString
   })
 )
 
-function checkIfFiat(code: string): boolean {
-  if (code.indexOf('iso:') >= 0) return true
-  return false
-}
-
 export function makeNomicsPlugin(opts: EdgeCorePluginOptions): EdgeRatePlugin {
   const { io, initOptions, log } = opts
   const { fetchCors = io.fetch } = io
@@ -33,29 +29,50 @@ export function makeNomicsPlugin(opts: EdgeCorePluginOptions): EdgeRatePlugin {
 
     async fetchRates(pairsHint) {
       const pairs = []
-      for (const pair of pairsHint) {
-        // Skip if codes are both fiat or crypto
-        if (
-          (checkIfFiat(pair.fromCurrency) && checkIfFiat(pair.toCurrency)) ||
-          (!checkIfFiat(pair.fromCurrency) && !checkIfFiat(pair.toCurrency))
-        )
-          continue
-        const fiatCode = pair.toCurrency.split(':')
+
+      // Create query strings
+      const queryStrings = []
+      let filteredPairs = []
+      for (let i = 0; i < pairsHint.length; i++) {
+        if (pairsHint[i].fromCurrency.indexOf('iso:') >= 0) continue
+        if (filteredPairs.some(cc => cc === pairsHint[i].fromCurrency)) continue
+        filteredPairs.push(pairsHint[i].fromCurrency)
+        if (filteredPairs.length === 100 || i === pairsHint.length - 1) {
+          queryStrings.push(filteredPairs.join(','))
+          filteredPairs = []
+        }
+      }
+
+      for (const query of queryStrings) {
         try {
           const reply = await fetchCors(
-            `https://api.nomics.com/v1/currencies/ticker?key=${apiKey}&ids=${pair.fromCurrency}&convert=${fiatCode[1]}`
+            `https://api.nomics.com/v1/currencies/ticker?key=${apiKey}&ids=${query}&convert=USD`
+          )
+          const replyJson = await reply.json()
+          if (
+            reply.status === 429 ||
+            reply.status === 401 ||
+            reply.ok === false
           )
-          if (reply.status === 429) continue
-          const jsonData = await reply.json()
-          const rate = Number(asNomicsResponse(jsonData)[0].price)
-          pairs.push({
-            fromCurrency: pair.fromCurrency,
-            toCurrency: pair.toCurrency,
-            rate
+            throw new Error(
+              `Nomics returned with status: ${JSON.stringify(
+                reply.status
+              )} and error: ${JSON.stringify(replyJson)}`
+            )
+          asNomicsResponse(replyJson).forEach(rate => {
+            // When Nomics considers a coin "dead" they don't return a price
+            if (rate.price)
+              pairs.push({
+                fromCurrency: rate.symbol,
+                toCurrency: 'iso:USD',
+                rate: Number(rate.price)
+              })
           })
         } catch (e) {
           log.warn(
-            `Issue with Nomics rate data structure for ${pair.fromCurrency}/${pair.toCurrency} pair. Error: ${e}`
+            `Issue with Nomics rate data structure. Querystrings ${JSON.stringify(
+              queryStrings
+            )} Error: ${e}`
           )
         }
       }