Add excludeMempoolSpent Query Parameter to /blockchain/box/unspent/byAddress

src/main/resources/api/openapi-ai.yaml

@@ -1390,6 +1390,13 @@ paths:
           schema:
             type: string
             default: desc
+        - in: query
+          name: excludeMempoolSpent
+          required: false
+          description: if true exclude spent inputs from mempool
+          schema:
+            type: boolean
+            default: false
       responses:
         '200':
           description: unspent boxes associated with wanted address
@@ -1479,4 +1486,4 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/ApiError'
+                $ref: '#/components/schemas/ApiError'

src/main/resources/api/openapi.yaml

@@ -6413,6 +6413,13 @@ paths:
           schema:
             type: boolean
             default: false
+        - in: query
+          name: excludeMempoolSpent
+          required: false
+          description: if true exclude spent inputs from mempool
+          schema:
+            type: boolean
+            default: false
       responses:
         '200':
           description: unspent boxes associated with wanted address

src/main/scala/org/ergoplatform/http/api/BlockchainApiRoute.scala

@@ -242,39 +242,69 @@ case class BlockchainApiRoute(readersHolder: ActorRef, ergoSettings: ErgoSetting
     validateAndGetBoxesByAddress(address, offset, limit)
   }
 
-  private def getBoxesByAddressUnspent(addr: ErgoAddress, offset: Int, limit: Int, sortDir: Direction, unconfirmed: Boolean): Future[Seq[IndexedErgoBox]] =
-    getHistoryWithMempool.map { case (history, mempool) =>
-      getAddress(addr)(history)
+  private def getBoxesByAddressUnspent(
+  addr: ErgoAddress,
+  offset: Int,
+  limit: Int,
+  sortDir: Direction,
+  unconfirmed: Boolean,
+  excludeMempoolSpent: Boolean
+): Future[Seq[IndexedErgoBox]] = {
+
+  val originalLimit = limit
+
+  def fetchAndFilter(limit: Int, accumulated: Seq[IndexedErgoBox] = Seq.empty): Future[Seq[IndexedErgoBox]] = {
+    getHistoryWithMempool.flatMap { case (history, mempool) =>
+      val spentBoxesIdsInMempool = if (excludeMempoolSpent) mempool.spentInputs.map(bytesToId).toSet else Set.empty[ModifierId]
+
+      val addressUtxos = getAddress(addr)(history)
         .getOrElse(IndexedErgoAddress(hashErgoTree(addr.script)))
-        .retrieveUtxos(history, mempool, offset, limit, sortDir, unconfirmed)
+        .retrieveUtxos(history, mempool, offset + accumulated.length, limit, sortDir, unconfirmed, spentBoxesIdsInMempool)
+
+      val updatedAccumulated = accumulated ++ addressUtxos
+      if (updatedAccumulated.length >= originalLimit || addressUtxos.length < limit) {
+        Future.successful(updatedAccumulated.take(originalLimit))
+      } else {
+        val maxLimit = 200
+        val newLimit = Math.min(limit * 2, maxLimit)
+        fetchAndFilter(newLimit, updatedAccumulated)
+      }
     }
+  }
+
+  fetchAndFilter(originalLimit)
+}
 
   private def validateAndGetBoxesByAddressUnspent(address: ErgoAddress,
                                                   offset: Int,
                                                   limit: Int,
                                                   dir: Direction,
-                                                  unconfirmed: Boolean): Route = {
+                                                  unconfirmed: Boolean, 
+                                                  excludeMempoolSpent: Boolean): Route = {
     if (limit > MaxItems) {
       BadRequest(s"No more than $MaxItems boxes can be requested")
     } else if (dir == SortDirection.INVALID) {
       BadRequest("Invalid parameter for sort direction, valid values are \"ASC\" and \"DESC\"")
     } else {
-      ApiResponse(getBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed))
+      ApiResponse(getBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed, excludeMempoolSpent))
     }
   }
 
   private def getBoxesByAddressUnspentR: Route =
-    (post & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging & sortDir & unconfirmed) {
-      (address, offset, limit, dir, unconfirmed) =>
-        validateAndGetBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed)
+    (post & pathPrefix("box" / "unspent" / "byAddress") & ergoAddress & paging & sortDir & unconfirmed & parameter('excludeMempoolSpent.as[Boolean].?)) {
+      (address, offset, limit, dir, unconfirmed, excludeMempoolSpentOption) =>
+        val excludeMempoolSpent = excludeMempoolSpentOption.getOrElse(false)
+        validateAndGetBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed, excludeMempoolSpent)
     }
 
   private def getBoxesByAddressUnspentGetRoute: Route =
-    (pathPrefix("box" / "unspent" / "byAddress") & get & addressPass & paging & sortDir & unconfirmed) {
-      (address, offset, limit, dir, unconfirmed) =>
-        validateAndGetBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed)
+    (pathPrefix("box" / "unspent" / "byAddress") & get & addressPass & paging & sortDir & unconfirmed & parameter('excludeMempoolSpent.as[Boolean].?)) {
+      (address, offset, limit, dir, unconfirmed, excludeMempoolSpentOption) =>
+        val excludeMempoolSpent = excludeMempoolSpentOption.getOrElse(false)
+        validateAndGetBoxesByAddressUnspent(address, offset, limit, dir, unconfirmed, excludeMempoolSpent)
     }
 
+
   private def getBoxRange(offset: Int, limit: Int): Future[Seq[ModifierId]] =
     getHistory.map { history =>
       val base: Long = getIndex(GlobalBoxIndexKey, history).getLong - offset

src/main/scala/org/ergoplatform/nodeView/history/extra/Segment.scala

@@ -287,6 +287,60 @@ abstract class Segment[T <: Segment[_] : ClassTag](val parentId: ModifierId,
       confirmedBoxes
   }
 
+
+  /**
+   * Overloaded retrieveUtxos for mempool filtering
+   * Get a range of the boxes associated with the parent that are NOT spent
+   *
+   * @param history     - history to use
+   * @param mempool     - mempool to use, if unconfirmed is true
+   * @param offset      - items to skip from the start
+   * @param limit       - items to retrieve
+   * @param sortDir     - whether to start retreival from newest box ([[DESC]]) or oldest box ([[ASC]])
+   * @param unconfirmed - whether to include unconfirmed boxes
+   * @param spentBoxesIdsInMempool - Set of box IDs that are spent in the mempool (to be excluded if necessary)
+   * @return array of unspent boxes
+   */
+  def retrieveUtxos(history: ErgoHistoryReader,
+                    mempool: ErgoMemPoolReader,
+                    offset: Int,
+                    limit: Int,
+                    sortDir: Direction,
+                    unconfirmed: Boolean,
+                    spentBoxesIdsInMempool: Set[ModifierId]): Seq[IndexedErgoBox] = {
+    val data: ArrayBuffer[IndexedErgoBox] = ArrayBuffer.empty[IndexedErgoBox]
+    val confirmedBoxes: Seq[IndexedErgoBox] = sortDir match {
+      case DESC =>
+        data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filterNot(box => spentBoxesIdsInMempool.contains(box.id))
+        var segment: Int = boxSegmentCount
+        while(data.length < (limit + offset) && segment > 0) {
+          segment -= 1
+          history.typedExtraIndexById[T](idMod(boxSegmentId(parentId, segment))).get.boxes
+            .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filterNot(box => spentBoxesIdsInMempool.contains(box.id)) ++=: data
+        }
+        data.reverse.slice(offset, offset + limit)
+      case ASC =>
+        var segment: Int = 0
+        while(data.length < (limit + offset) && segment < boxSegmentCount) {
+          data ++= history.typedExtraIndexById[T](idMod(boxSegmentId(parentId, segment))).get.boxes
+            .filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filterNot(box => spentBoxesIdsInMempool.contains(box.id))
+          segment += 1
+        }
+        if (data.length < (limit + offset))
+          data ++= boxes.filter(_ > 0).map(n => NumericBoxIndex.getBoxByNumber(history, n).get).filterNot(box => spentBoxesIdsInMempool.contains(box.id))
+        data.slice(offset, offset + limit)
+    }
+    if(unconfirmed) {
+      val mempoolBoxes = filterMempool(mempool.getAll.flatMap(_.transaction.outputs))
+      val unconfirmedBoxes = mempoolBoxes.map(new IndexedErgoBox(0, None, None, _, 0)).filterNot(box => spentBoxesIdsInMempool.contains(box.id))
+      sortDir match {
+        case DESC => unconfirmedBoxes ++ confirmedBoxes
+        case ASC => confirmedBoxes ++ unconfirmedBoxes
+      }
+    } else
+      confirmedBoxes
+  }
+
   /**
    * Logic for [[Segment.rollback]]
    *