Merge pull request #1674 from ergoplatform/v4.0.26

Candidate for 4.0.26

.github/workflows/docker-publish.yml

@@ -31,17 +31,3 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-
-      - name: Docker Metadata action for sigma-rust integration test node
-        uses: docker/[email protected]
-        id: metasigma
-        with:
-          images: ergoplatform/ergo-integration-test
-
-      - name: Build and push Docker images for integration-test node
-        uses: docker/[email protected]
-        with:
-          context: "{{defaultContext}}:sigma-rust-integration-test" 
-          push: true
-          tags: ${{ steps.metasigma.outputs.tags }}
-          labels: ${{ steps.metasigma.outputs.labels }}

README.md

@@ -82,7 +82,7 @@ To run specific Ergo version `<VERSION>` as a service with custom config `/path/
         -e MAX_HEAP=3G \
         ergoplatform/ergo:<VERSION> --<networkId> -c /etc/myergo.conf
 
-Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.25`.
+Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.26`.
 
 This will connect to the Ergo mainnet or testnet following your configuration passed in `myergo.conf` and network flag `--<networkId>`. Every default config value would be overwritten with corresponding value in `myergo.conf`. `MAX_HEAP` variable can be used to control how much memory can the node consume.
 

avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala

@@ -33,11 +33,11 @@ object Helper {
     inserts ++ updates
   }
 
-  def persistentProverWithVersionedStore(keepVersions: Int,
+  def persistentProverWithVersionedStore(initialKeepVersions: Int,
                                          baseOperationsCount: Int = 0): (Prover, LDBVersionedStore, VersionedLDBAVLStorage[Digest32]) = {
     val dir = java.nio.file.Files.createTempDirectory("bench_testing_" + scala.util.Random.alphanumeric.take(15)).toFile
     dir.deleteOnExit()
-    val store = new LDBVersionedStore(dir, initialKeepVersions = keepVersions)
+    val store = new LDBVersionedStore(dir, initialKeepVersions = initialKeepVersions)
     val storage = new VersionedLDBAVLStorage(store, NodeParameters(kl, Some(vl), ll))
     require(storage.isEmpty)
     val prover = new BatchAVLProver[Digest32, HF](kl, Some(vl))

avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala

@@ -30,7 +30,9 @@ class VersionedLDBAVLStorage[D <: Digest](store: LDBVersionedStore,
   private val fixedSizeValueMode = nodeParameters.valueSize.isDefined
 
   override def rollback(version: ADDigest): Try[(ProverNodes[D], Int)] = Try {
-    store.rollbackTo(version)
+    if (!this.version.contains(version)) { // do not rollback to self
+      store.rollbackTo(version)
+    }
 
     val top = VersionedLDBAVLStorage.fetch[D](ADKey @@ store.get(TopNodeKey).get)(hf, store, nodeParameters)
     val topHeight = Ints.fromByteArray(store.get(TopNodeHeight).get)

avldb/src/main/scala/scorex/db/LDBKVStore.scala

@@ -27,6 +27,21 @@ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging
     }
   }
 
+  /**
+    * Insert single key-value into database
+    * @param id - key to insert
+    * @param value - value to insert
+    * @return - Success(()) in case of successful insertion, Failure otherwise
+    */
+  def insert(id: K,  value: V): Try[Unit] = {
+    try {
+      db.put(id, value)
+      Success(())
+    } catch {
+      case t: Throwable => Failure(t)
+    }
+  }
+
   def insert(values: Seq[(K, V)]): Try[Unit] = update(values, Seq.empty)
 
   def remove(keys: Seq[K]): Try[Unit] = update(Seq.empty, keys)

avldb/src/main/scala/scorex/db/LDBVersionedStore.scala

@@ -9,6 +9,8 @@ import java.nio.ByteBuffer
 import scala.collection.mutable.ArrayBuffer
 import java.util.concurrent.locks.ReentrantReadWriteLock
 
+import scorex.crypto.hash.Blake2b256
+
 import scala.util.Try
 
 
@@ -29,7 +31,9 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e
 
   type LSN = Long // logical serial number: type used to provide order of records in undo list
 
-  private var keepVersions: Int = initialKeepVersions;
+  private val last_version_key = Blake2b256("last_version")
+
+  private var keepVersions: Int = initialKeepVersions
 
   override val db: DB = createDB(dir, "ldb_main") // storage for main data
   override val lock = new ReentrantReadWriteLock()
@@ -42,6 +46,7 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e
   private val versions: ArrayBuffer[VersionID] = getAllVersions
   private var lastVersion: Option[VersionID] = versions.lastOption
 
+
   //default write options, no sync!
   private val writeOptions = new WriteOptions()
 
@@ -67,6 +72,8 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e
     oldKeepVersions
   }
 
+  def getKeepVersions: Int = keepVersions
+
   /** returns value associated with the key or throws `NoSuchElementException` */
   def apply(key: K): V = getOrElse(key, {
     throw new NoSuchElementException()
@@ -175,7 +182,17 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e
     versionLsn += lastLsn // first LSN of oldest version
     versionLsn = versionLsn.reverse // LSNs should be in ascending order
     versionLsn.remove(versionLsn.size - 1) // remove last element which corresponds to next assigned LSN
-    versions.reverse
+
+    if (versions.nonEmpty) {
+      versions.reverse
+    } else {
+      val dbVersion = db.get(last_version_key)
+      if (dbVersion != null) {
+        versions += dbVersion
+        versionLsn += lastLsn
+      }
+      versions
+    }
   }
 
   /**
@@ -189,7 +206,7 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e
     val versionSize = versionID.length
     val keySize = key.length
     val packed = new Array[Byte](2 + versionSize + keySize + valueSize)
-    assert(keySize <= 0xFF)
+    require(keySize <= 0xFF)
     packed(0) = versionSize.asInstanceOf[Byte]
     packed(1) = keySize.asInstanceOf[Byte]
     Array.copy(versionID, 0, packed, 2, versionSize)
@@ -232,14 +249,14 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e
         }
       })
       for ((key, v) <- toUpdate) {
-        assert(key.length != 0) // empty keys are not allowed
+        require(key.length != 0) // empty keys are not allowed
         if (keepVersions > 0) {
           val old = db.get(key)
           undoBatch.put(newLSN(), serializeUndo(versionID, key, old))
         }
         batch.put(key, v)
       }
-      db.write(batch, writeOptions)
+
       if (keepVersions > 0) {
         if (lsn == lastLsn) { // no records were written for this version: generate dummy record
           undoBatch.put(newLSN(), serializeUndo(versionID, new Array[Byte](0), null))
@@ -250,7 +267,19 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e
           versionLsn += lastLsn + 1 // first LSN for this version
           cleanStart(keepVersions)
         }
+      } else {
+        //keepVersions = 0
+        if (lastVersion.isEmpty || !versionID.sameElements(lastVersion.get)) {
+          batch.put(last_version_key, versionID)
+          versions.clear()
+          versions += versionID
+          if (versionLsn.isEmpty) {
+            versionLsn += lastLsn
+          }
+        }
       }
+
+      db.write(batch, writeOptions)
       lastVersion = Some(versionID)
     } finally {
       // Make sure you close the batch to avoid resource leaks.
@@ -265,12 +294,12 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e
   def remove(versionID: VersionID, toRemove: Seq[K]): Try[Unit] = update(versionID, toRemove, Seq.empty)
 
 
-  // Keep last "count" versions and remove undo information for older versions
+  // Keep last "count"+1 versions and remove undo information for older versions
   private def cleanStart(count: Int): Unit = {
-    val deteriorated = versions.size - count
-    if (deteriorated > 0) {
+    val deteriorated = versions.size - count - 1
+    if (deteriorated >= 0) {
       val fromLsn = versionLsn(0)
-      val tillLsn = versionLsn(deteriorated)
+      val tillLsn = if (deteriorated+1 < versions.size) versionLsn(deteriorated+1) else lsn+1
       val batch = undo.createWriteBatch()
       try {
         for (lsn <- fromLsn until tillLsn) {
@@ -280,9 +309,12 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e
       } finally {
         batch.close()
       }
+
       versions.remove(0, deteriorated)
       versionLsn.remove(0, deteriorated)
-      lastVersion = versions.lastOption
+      if (count == 0) {
+        db.put(last_version_key, versions(0))
+      }
     }
   }
 
@@ -318,49 +350,52 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e
     try {
       val versionIndex = versions.indexWhere(_.sameElements(versionID))
       if (versionIndex >= 0) {
-        val batch = db.createWriteBatch()
-        val undoBatch = undo.createWriteBatch()
-        var nUndoRecords: Long = 0
-        val iterator = undo.iterator()
-        var lastLsn: LSN = 0
-        try {
-          var undoing = true
-          iterator.seekToFirst()
-          while (undoing) {
-            assert(iterator.hasNext)
-            val entry = iterator.next()
-            val undo = deserializeUndo(entry.getValue)
-            if (undo.versionID.sameElements(versionID)) {
-              undoing = false
-              lastLsn = decodeLSN(entry.getKey)
-            } else {
-              undoBatch.delete(entry.getKey)
-              nUndoRecords += 1
-              if (undo.value == null) {
-                if (undo.key.length != 0) { // dummy record
-                  batch.delete(undo.key)
-                }
+        if (versionIndex != versions.size-1) {
+          val batch = db.createWriteBatch()
+          val undoBatch = undo.createWriteBatch()
+          var nUndoRecords: Long = 0
+          val iterator = undo.iterator()
+          var lastLsn: LSN = 0
+          try {
+            var undoing = true
+            iterator.seekToFirst()
+            while (undoing && iterator.hasNext) {
+              val entry = iterator.next()
+              val undo = deserializeUndo(entry.getValue)
+              if (undo.versionID.sameElements(versionID)) {
+                undoing = false
+                lastLsn = decodeLSN(entry.getKey)
               } else {
-                batch.put(undo.key, undo.value)
+                undoBatch.delete(entry.getKey)
+                nUndoRecords += 1
+                if (undo.value == null) {
+                  if (undo.key.length != 0) { // dummy record
+                    batch.delete(undo.key)
+                  }
+                } else {
+                  batch.put(undo.key, undo.value)
+                }
               }
             }
+            db.write(batch, writeOptions)
+            undo.write(undoBatch, writeOptions)
+          } finally {
+            // Make sure you close the batch to avoid resource leaks.
+            iterator.close()
+            batch.close()
+            undoBatch.close()
           }
-          db.write(batch, writeOptions)
-          undo.write(undoBatch, writeOptions)
-        } finally {
-          // Make sure you close the batch to avoid resource leaks.
-          iterator.close()
-          batch.close()
-          undoBatch.close()
+          val nVersions = versions.size
+          require((versionIndex + 1 == nVersions && nUndoRecords == 0) || (versionIndex + 1 < nVersions && lsn - versionLsn(versionIndex + 1) + 1 == nUndoRecords))
+          versions.remove(versionIndex + 1, nVersions - versionIndex - 1)
+          versionLsn.remove(versionIndex + 1, nVersions - versionIndex - 1)
+          lsn -= nUndoRecords // reuse deleted LSN to avoid holes in LSNs
+          require(lastLsn == 0 || lsn == lastLsn)
+          require(versions.last.sameElements(versionID))
+          lastVersion = Some(versionID)
+        } else {
+          require(lastVersion.get.sameElements(versionID))
         }
-        val nVersions = versions.size
-        assert((versionIndex + 1 == nVersions && nUndoRecords == 0) || (versionIndex + 1 < nVersions && lsn - versionLsn(versionIndex + 1) + 1 == nUndoRecords))
-        versions.remove(versionIndex + 1, nVersions - versionIndex - 1)
-        versionLsn.remove(versionIndex + 1, nVersions - versionIndex - 1)
-        lsn -= nUndoRecords // reuse deleted LSN to avoid holes in LSNs
-        assert(lsn == lastLsn)
-        assert(versions.last.sameElements(versionID))
-        lastVersion = Some(versionID)
       } else {
         throw new NoSuchElementException("versionID not found, can not rollback")
       }

avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala

@@ -23,9 +23,9 @@ trait TestHelper extends FileHelper {
 
   implicit val hf: HF = Blake2b256
 
-  def createVersionedStore(keepVersions: Int = 10): LDBVersionedStore = {
+  def createVersionedStore(initialKeepVersions: Int = 10): LDBVersionedStore = {
     val dir = getRandomTempDir
-    new LDBVersionedStore(dir, initialKeepVersions = keepVersions)
+    new LDBVersionedStore(dir, initialKeepVersions = initialKeepVersions)
   }
 
   def createVersionedStorage(store: LDBVersionedStore): STORAGE =

avldb/src/test/scala/scorex/db/LDBVersionedStoreSpec.scala

@@ -86,15 +86,22 @@ class LDBVersionedStoreSpec extends AnyPropSpec with Matchers {
   property("alter keepVersions") {
     val version1 = Longs.toByteArray(Long.MaxValue + 1)
     val version2 = Longs.toByteArray(Long.MaxValue + 2)
+    val version3 = Longs.toByteArray(Long.MaxValue + 3)
     val k1 = Longs.toByteArray(1)
     val v1 = Longs.toByteArray(100)
     store.update(version1, Seq.empty, Seq(k1 -> v1)).get
     store.update(version2, Seq.empty, Seq(k1 -> v1)).get
     store.versionIdExists(version1) shouldBe true
     store.versionIdExists(version2) shouldBe true
-    store.setKeepVersions(1) shouldBe 100
+    store.setKeepVersions(0) shouldBe 100
     store.versionIdExists(version1) shouldBe false
     store.versionIdExists(version2) shouldBe true
-    store.setKeepVersions(10) shouldBe 1
-  }
+    store.setKeepVersions(10) shouldBe 0
+    store.update(version3, Seq.empty, Seq(k1 -> v1)).get
+    store.rollbackTo(version2).isSuccess shouldBe true
+    store.versionIdExists(version2) shouldBe true
+    store.versionIdExists(version3) shouldBe false
+    store.update(version3, Seq.empty, Seq(k1 -> v1)).get
+    store.versionIdExists(version3) shouldBe true
+   }
 }

benchmarks/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateBenchmark.scala

@@ -25,7 +25,7 @@ object UtxoStateBenchmark extends HistoryTestHelpers with NVBenchmark {
       val state = ErgoState.generateGenesisUtxoState(createTempDir, StateConstants(realNetworkSetting), parameters)._1
       Utils.time {
         mods.foldLeft(state) { case (st, mod) =>
-          st.applyModifier(mod)(_ => ()).get
+          st.applyModifier(mod, None)(_ => ()).get
         }
       }.toLong
     }

ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/ErgoLikeParameters.scala

@@ -44,7 +44,7 @@ trait ErgoLikeParameters {
   /**
     * @return computation units limit per block
     */
-  def maxBlockCost: Long
+  def maxBlockCost: Int
 
   /**
     * @return height when voting for a soft-fork had been started

ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala

@@ -1,9 +1,8 @@
 package org.ergoplatform.wallet.secrets
 
-import java.io.{File, PrintWriter}
+import java.io.{File, FileNotFoundException, PrintWriter}
 import java.util
 import java.util.UUID
-
 import io.circe.parser._
 import io.circe.syntax._
 import org.ergoplatform.wallet.crypto
@@ -137,7 +136,7 @@ object JsonSecretStorage {
           Failure(new Exception(s"Cannot readSecretStorage: Secret file not found in dir '$dir'"))
       }
     } else {
-      Failure(new Exception(s"Cannot readSecretStorage: dir '$dir' doesn't exist"))
+      Failure(new FileNotFoundException(s"Cannot readSecretStorage: dir '$dir' doesn't exist"))
     }
   }
 

ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala

@@ -26,7 +26,7 @@ trait InterpreterSpecCommon {
 
     override def outputCost: Int = 100
 
-    override def maxBlockCost: Long = 1000000
+    override def maxBlockCost: Int = 1000000
 
     override def softForkStartingHeight: Option[Int] = None