diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 8857471c48..d222647106 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -31,17 +31,3 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - - name: Docker Metadata action for sigma-rust integration test node - uses: docker/metadata-action@v3.5.0 - id: metasigma - with: - images: ergoplatform/ergo-integration-test - - - name: Build and push Docker images for integration-test node - uses: docker/build-push-action@v2.7.0 - with: - context: "{{defaultContext}}:sigma-rust-integration-test" - push: true - tags: ${{ steps.metasigma.outputs.tags }} - labels: ${{ steps.metasigma.outputs.labels }} diff --git a/README.md b/README.md index e61436ce25..9aa5a4a0d2 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ To run specific Ergo version `` as a service with custom config `/path/ -e MAX_HEAP=3G \ ergoplatform/ergo: -- -c /etc/myergo.conf -Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.25`. +Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.26`. This will connect to the Ergo mainnet or testnet following your configuration passed in `myergo.conf` and network flag `--`. Every default config value would be overwritten with corresponding value in `myergo.conf`. `MAX_HEAP` variable can be used to control how much memory can the node consume. diff --git a/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala b/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala index f71795a506..a3e1ead7af 100644 --- a/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala +++ b/avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala @@ -33,11 +33,11 @@ object Helper { inserts ++ updates } - def persistentProverWithVersionedStore(keepVersions: Int, + def persistentProverWithVersionedStore(initialKeepVersions: Int, baseOperationsCount: Int = 0): (Prover, LDBVersionedStore, VersionedLDBAVLStorage[Digest32]) = { val dir = java.nio.file.Files.createTempDirectory("bench_testing_" + scala.util.Random.alphanumeric.take(15)).toFile dir.deleteOnExit() - val store = new LDBVersionedStore(dir, initialKeepVersions = keepVersions) + val store = new LDBVersionedStore(dir, initialKeepVersions = initialKeepVersions) val storage = new VersionedLDBAVLStorage(store, NodeParameters(kl, Some(vl), ll)) require(storage.isEmpty) val prover = new BatchAVLProver[Digest32, HF](kl, Some(vl)) diff --git a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala index ffd4be9e43..e7c1bc5688 100644 --- a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala +++ b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala @@ -30,7 +30,9 @@ class VersionedLDBAVLStorage[D <: Digest](store: LDBVersionedStore, private val fixedSizeValueMode = nodeParameters.valueSize.isDefined override def rollback(version: ADDigest): Try[(ProverNodes[D], Int)] = Try { - store.rollbackTo(version) + if (!this.version.contains(version)) { // do not rollback to self + store.rollbackTo(version) + } val top = VersionedLDBAVLStorage.fetch[D](ADKey @@ store.get(TopNodeKey).get)(hf, store, nodeParameters) val topHeight = Ints.fromByteArray(store.get(TopNodeHeight).get) diff --git a/avldb/src/main/scala/scorex/db/LDBKVStore.scala b/avldb/src/main/scala/scorex/db/LDBKVStore.scala index cda42dda19..bdf3ade613 100644 --- a/avldb/src/main/scala/scorex/db/LDBKVStore.scala +++ b/avldb/src/main/scala/scorex/db/LDBKVStore.scala @@ -27,6 +27,21 @@ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging } } + /** + * Insert single key-value into database + * @param id - key to insert + * @param value - value to insert + * @return - Success(()) in case of successful insertion, Failure otherwise + */ + def insert(id: K, value: V): Try[Unit] = { + try { + db.put(id, value) + Success(()) + } catch { + case t: Throwable => Failure(t) + } + } + def insert(values: Seq[(K, V)]): Try[Unit] = update(values, Seq.empty) def remove(keys: Seq[K]): Try[Unit] = update(Seq.empty, keys) diff --git a/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala b/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala index e2a41d2635..0cb937bb04 100644 --- a/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala +++ b/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala @@ -9,6 +9,8 @@ import java.nio.ByteBuffer import scala.collection.mutable.ArrayBuffer import java.util.concurrent.locks.ReentrantReadWriteLock +import scorex.crypto.hash.Blake2b256 + import scala.util.Try @@ -29,7 +31,9 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e type LSN = Long // logical serial number: type used to provide order of records in undo list - private var keepVersions: Int = initialKeepVersions; + private val last_version_key = Blake2b256("last_version") + + private var keepVersions: Int = initialKeepVersions override val db: DB = createDB(dir, "ldb_main") // storage for main data override val lock = new ReentrantReadWriteLock() @@ -42,6 +46,7 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e private val versions: ArrayBuffer[VersionID] = getAllVersions private var lastVersion: Option[VersionID] = versions.lastOption + //default write options, no sync! private val writeOptions = new WriteOptions() @@ -67,6 +72,8 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e oldKeepVersions } + def getKeepVersions: Int = keepVersions + /** returns value associated with the key or throws `NoSuchElementException` */ def apply(key: K): V = getOrElse(key, { throw new NoSuchElementException() @@ -175,7 +182,17 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e versionLsn += lastLsn // first LSN of oldest version versionLsn = versionLsn.reverse // LSNs should be in ascending order versionLsn.remove(versionLsn.size - 1) // remove last element which corresponds to next assigned LSN - versions.reverse + + if (versions.nonEmpty) { + versions.reverse + } else { + val dbVersion = db.get(last_version_key) + if (dbVersion != null) { + versions += dbVersion + versionLsn += lastLsn + } + versions + } } /** @@ -189,7 +206,7 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e val versionSize = versionID.length val keySize = key.length val packed = new Array[Byte](2 + versionSize + keySize + valueSize) - assert(keySize <= 0xFF) + require(keySize <= 0xFF) packed(0) = versionSize.asInstanceOf[Byte] packed(1) = keySize.asInstanceOf[Byte] Array.copy(versionID, 0, packed, 2, versionSize) @@ -232,14 +249,14 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e } }) for ((key, v) <- toUpdate) { - assert(key.length != 0) // empty keys are not allowed + require(key.length != 0) // empty keys are not allowed if (keepVersions > 0) { val old = db.get(key) undoBatch.put(newLSN(), serializeUndo(versionID, key, old)) } batch.put(key, v) } - db.write(batch, writeOptions) + if (keepVersions > 0) { if (lsn == lastLsn) { // no records were written for this version: generate dummy record undoBatch.put(newLSN(), serializeUndo(versionID, new Array[Byte](0), null)) @@ -250,7 +267,19 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e versionLsn += lastLsn + 1 // first LSN for this version cleanStart(keepVersions) } + } else { + //keepVersions = 0 + if (lastVersion.isEmpty || !versionID.sameElements(lastVersion.get)) { + batch.put(last_version_key, versionID) + versions.clear() + versions += versionID + if (versionLsn.isEmpty) { + versionLsn += lastLsn + } + } } + + db.write(batch, writeOptions) lastVersion = Some(versionID) } finally { // Make sure you close the batch to avoid resource leaks. @@ -265,12 +294,12 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e def remove(versionID: VersionID, toRemove: Seq[K]): Try[Unit] = update(versionID, toRemove, Seq.empty) - // Keep last "count" versions and remove undo information for older versions + // Keep last "count"+1 versions and remove undo information for older versions private def cleanStart(count: Int): Unit = { - val deteriorated = versions.size - count - if (deteriorated > 0) { + val deteriorated = versions.size - count - 1 + if (deteriorated >= 0) { val fromLsn = versionLsn(0) - val tillLsn = versionLsn(deteriorated) + val tillLsn = if (deteriorated+1 < versions.size) versionLsn(deteriorated+1) else lsn+1 val batch = undo.createWriteBatch() try { for (lsn <- fromLsn until tillLsn) { @@ -280,9 +309,12 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e } finally { batch.close() } + versions.remove(0, deteriorated) versionLsn.remove(0, deteriorated) - lastVersion = versions.lastOption + if (count == 0) { + db.put(last_version_key, versions(0)) + } } } @@ -318,49 +350,52 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) e try { val versionIndex = versions.indexWhere(_.sameElements(versionID)) if (versionIndex >= 0) { - val batch = db.createWriteBatch() - val undoBatch = undo.createWriteBatch() - var nUndoRecords: Long = 0 - val iterator = undo.iterator() - var lastLsn: LSN = 0 - try { - var undoing = true - iterator.seekToFirst() - while (undoing) { - assert(iterator.hasNext) - val entry = iterator.next() - val undo = deserializeUndo(entry.getValue) - if (undo.versionID.sameElements(versionID)) { - undoing = false - lastLsn = decodeLSN(entry.getKey) - } else { - undoBatch.delete(entry.getKey) - nUndoRecords += 1 - if (undo.value == null) { - if (undo.key.length != 0) { // dummy record - batch.delete(undo.key) - } + if (versionIndex != versions.size-1) { + val batch = db.createWriteBatch() + val undoBatch = undo.createWriteBatch() + var nUndoRecords: Long = 0 + val iterator = undo.iterator() + var lastLsn: LSN = 0 + try { + var undoing = true + iterator.seekToFirst() + while (undoing && iterator.hasNext) { + val entry = iterator.next() + val undo = deserializeUndo(entry.getValue) + if (undo.versionID.sameElements(versionID)) { + undoing = false + lastLsn = decodeLSN(entry.getKey) } else { - batch.put(undo.key, undo.value) + undoBatch.delete(entry.getKey) + nUndoRecords += 1 + if (undo.value == null) { + if (undo.key.length != 0) { // dummy record + batch.delete(undo.key) + } + } else { + batch.put(undo.key, undo.value) + } } } + db.write(batch, writeOptions) + undo.write(undoBatch, writeOptions) + } finally { + // Make sure you close the batch to avoid resource leaks. + iterator.close() + batch.close() + undoBatch.close() } - db.write(batch, writeOptions) - undo.write(undoBatch, writeOptions) - } finally { - // Make sure you close the batch to avoid resource leaks. - iterator.close() - batch.close() - undoBatch.close() + val nVersions = versions.size + require((versionIndex + 1 == nVersions && nUndoRecords == 0) || (versionIndex + 1 < nVersions && lsn - versionLsn(versionIndex + 1) + 1 == nUndoRecords)) + versions.remove(versionIndex + 1, nVersions - versionIndex - 1) + versionLsn.remove(versionIndex + 1, nVersions - versionIndex - 1) + lsn -= nUndoRecords // reuse deleted LSN to avoid holes in LSNs + require(lastLsn == 0 || lsn == lastLsn) + require(versions.last.sameElements(versionID)) + lastVersion = Some(versionID) + } else { + require(lastVersion.get.sameElements(versionID)) } - val nVersions = versions.size - assert((versionIndex + 1 == nVersions && nUndoRecords == 0) || (versionIndex + 1 < nVersions && lsn - versionLsn(versionIndex + 1) + 1 == nUndoRecords)) - versions.remove(versionIndex + 1, nVersions - versionIndex - 1) - versionLsn.remove(versionIndex + 1, nVersions - versionIndex - 1) - lsn -= nUndoRecords // reuse deleted LSN to avoid holes in LSNs - assert(lsn == lastLsn) - assert(versions.last.sameElements(versionID)) - lastVersion = Some(versionID) } else { throw new NoSuchElementException("versionID not found, can not rollback") } diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala index baa6403a84..a3e1c8ce60 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala @@ -23,9 +23,9 @@ trait TestHelper extends FileHelper { implicit val hf: HF = Blake2b256 - def createVersionedStore(keepVersions: Int = 10): LDBVersionedStore = { + def createVersionedStore(initialKeepVersions: Int = 10): LDBVersionedStore = { val dir = getRandomTempDir - new LDBVersionedStore(dir, initialKeepVersions = keepVersions) + new LDBVersionedStore(dir, initialKeepVersions = initialKeepVersions) } def createVersionedStorage(store: LDBVersionedStore): STORAGE = diff --git a/avldb/src/test/scala/scorex/db/LDBVersionedStoreSpec.scala b/avldb/src/test/scala/scorex/db/LDBVersionedStoreSpec.scala index 53dc8b948b..59863452ab 100644 --- a/avldb/src/test/scala/scorex/db/LDBVersionedStoreSpec.scala +++ b/avldb/src/test/scala/scorex/db/LDBVersionedStoreSpec.scala @@ -86,15 +86,22 @@ class LDBVersionedStoreSpec extends AnyPropSpec with Matchers { property("alter keepVersions") { val version1 = Longs.toByteArray(Long.MaxValue + 1) val version2 = Longs.toByteArray(Long.MaxValue + 2) + val version3 = Longs.toByteArray(Long.MaxValue + 3) val k1 = Longs.toByteArray(1) val v1 = Longs.toByteArray(100) store.update(version1, Seq.empty, Seq(k1 -> v1)).get store.update(version2, Seq.empty, Seq(k1 -> v1)).get store.versionIdExists(version1) shouldBe true store.versionIdExists(version2) shouldBe true - store.setKeepVersions(1) shouldBe 100 + store.setKeepVersions(0) shouldBe 100 store.versionIdExists(version1) shouldBe false store.versionIdExists(version2) shouldBe true - store.setKeepVersions(10) shouldBe 1 - } + store.setKeepVersions(10) shouldBe 0 + store.update(version3, Seq.empty, Seq(k1 -> v1)).get + store.rollbackTo(version2).isSuccess shouldBe true + store.versionIdExists(version2) shouldBe true + store.versionIdExists(version3) shouldBe false + store.update(version3, Seq.empty, Seq(k1 -> v1)).get + store.versionIdExists(version3) shouldBe true + } } diff --git a/benchmarks/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateBenchmark.scala b/benchmarks/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateBenchmark.scala index 49909ad177..78addff3f8 100644 --- a/benchmarks/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateBenchmark.scala +++ b/benchmarks/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateBenchmark.scala @@ -25,7 +25,7 @@ object UtxoStateBenchmark extends HistoryTestHelpers with NVBenchmark { val state = ErgoState.generateGenesisUtxoState(createTempDir, StateConstants(realNetworkSetting), parameters)._1 Utils.time { mods.foldLeft(state) { case (st, mod) => - st.applyModifier(mod)(_ => ()).get + st.applyModifier(mod, None)(_ => ()).get } }.toLong } diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/ErgoLikeParameters.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/ErgoLikeParameters.scala index 5b4bd4d9f9..3f5f893ca7 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/ErgoLikeParameters.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/ErgoLikeParameters.scala @@ -44,7 +44,7 @@ trait ErgoLikeParameters { /** * @return computation units limit per block */ - def maxBlockCost: Long + def maxBlockCost: Int /** * @return height when voting for a soft-fork had been started diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala index 160e4b9690..105822af3f 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala @@ -1,9 +1,8 @@ package org.ergoplatform.wallet.secrets -import java.io.{File, PrintWriter} +import java.io.{File, FileNotFoundException, PrintWriter} import java.util import java.util.UUID - import io.circe.parser._ import io.circe.syntax._ import org.ergoplatform.wallet.crypto @@ -137,7 +136,7 @@ object JsonSecretStorage { Failure(new Exception(s"Cannot readSecretStorage: Secret file not found in dir '$dir'")) } } else { - Failure(new Exception(s"Cannot readSecretStorage: dir '$dir' doesn't exist")) + Failure(new FileNotFoundException(s"Cannot readSecretStorage: dir '$dir' doesn't exist")) } } diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala index 59b4f985d8..b1095e160e 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala @@ -26,7 +26,7 @@ trait InterpreterSpecCommon { override def outputCost: Int = 100 - override def maxBlockCost: Long = 1000000 + override def maxBlockCost: Int = 1000000 override def softForkStartingHeight: Option[Int] = None diff --git a/sigma-rust-integration-test/Dockerfile b/sigma-rust-integration-test/Dockerfile deleted file mode 100644 index f8154bd4d5..0000000000 --- a/sigma-rust-integration-test/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM ergoplatform/ergo:latest -COPY ergo.conf /etc/myergo.conf -ENTRYPOINT ["java", "-jar", "/home/ergo/ergo.jar", "--devnet", "-c", "/etc/myergo.conf"] \ No newline at end of file diff --git a/sigma-rust-integration-test/ergo.conf b/sigma-rust-integration-test/ergo.conf deleted file mode 100644 index 6848e3acea..0000000000 --- a/sigma-rust-integration-test/ergo.conf +++ /dev/null @@ -1,104 +0,0 @@ -ergo { - # Settings for node view holder regime. See papers.yellow.ModifiersProcessing.md - node { - # State type. Possible options are: - # "utxo" - keep full utxo set, that allows to validate arbitrary block and generate ADProofs - # "digest" - keep state root hash only and validate transactions via ADProofs - stateType = "utxo" - - # Download block transactions and verify them (requires BlocksToKeep == 0 if disabled) - verifyTransactions = true - - # Number of last blocks to keep with transactions and ADproofs, for all other blocks only header will be stored. - # Keep all blocks from genesis if negative - blocksToKeep = -1 - - # Download PoPoW proof on node bootstrap - PoPoWBootstrap = false - - # Minimal suffix size for PoPoW proof (may be pre-defined constant or settings parameter) - minimalSuffix = 10 - - # Is the node is doing mining - mining = true - - # Use external miner, native miner is used if set to `false` - useExternalMiner = false - - # If true, a node generate blocks being offline. The only really useful case for it probably is to start a new - # blockchain - offlineGeneration = true - - # internal miner's interval of polling for a candidate - internalMinerPollingInterval = 5s - - mempoolCapacity = 10000 - } - - chain { - # Difficulty network start with - initialDifficultyHex = "01" - - powScheme { - powType = "autolykos" - k = 32 - n = 26 - } - - } - - wallet { - - secretStorage { - - secretDir = ${ergo.directory}"/wallet/keystore" - - encryption { - - # Pseudo-random function with output of length `dkLen` (PBKDF2 param) - prf = "HmacSHA256" - - # Number of PBKDF2 iterations (PBKDF2 param) - c = 128000 - - # Desired bit-length of the derived key (PBKDF2 param) - dkLen = 256 - } - - } - - # Generating seed length in bits - # Options: 128, 160, 192, 224, 256 - seedStrengthBits = 160 - - # Language to be used in mnemonic seed - # Options: "chinese_simplified", "chinese_traditional", "english", "french", "italian", "japanese", "korean", "spanish" - mnemonicPhraseLanguage = "english" - - defaultTransactionFee = 10000 - - # Save used boxes (may consume additinal disk space) or delete them immediately - keepSpentBoxes = false - - # Mnemonic seed used in wallet for tests - testMnemonic = "ozone drill grab fiber curtain grace pudding thank cruise elder eight picnic" - - # Number of keys to be generated for tests - testKeysQty = 5 - } -} - -scorex { - network { - maxPacketSize = 2048576 - maxInvObjects = 400 - bindAddress = "0.0.0.0:9001" - knownPeers = [] - agentName = "ergo-integration-test" - } - restApi { - bindAddress = "0.0.0.0:9053" - # Hash of "hello", taken from mainnet config - apiKeyHash = "324dcf027dd4a30a932c441f365a25e86b173defa4b8e58948253471b81b72cf" - } -} diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 8bcb06ec8b..ee3477b5e4 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.2" info: - version: "4.0.25" + version: "4.0.26" title: Ergo Node API description: API docs for Ergo Node. Models are shared between all Ergo products contact: @@ -13,7 +13,7 @@ info: url: https://raw.githubusercontent.com/ergoplatform/ergo/master/LICENSE servers: - - url: http://127.0.0.1:9053 + - url: / description: Local full node - url: http://213.239.193.208:9053 description: Known public node @@ -1778,7 +1778,7 @@ components: example: '127.0.0.1:5673' version: type: string - example: '4.0.25' + example: '4.0.26' checks: type: integer description: How many times we checked for modifier delivery status @@ -1799,7 +1799,7 @@ components: example: '127.0.0.1:5673' version: type: string - example: '4.0.25' + example: '4.0.26' lastMessage: $ref: '#/components/schemas/Timestamp' diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index cdb07b2130..cb716f8155 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -260,6 +260,9 @@ ergo { # Use pre-EIP3 key derivation scheme usePreEip3Derivation = false + # Boxes with value smaller than dustLimit are disregarded in wallet scan logic + dustLimit = 999999 + # Maximum number of inputs per transaction generated by the wallet maxInputs = 100 @@ -360,7 +363,7 @@ scorex { nodeName = "ergo-node" # Network protocol version to be sent in handshakes - appVersion = 4.0.25 + appVersion = 4.0.26 # Network agent name. May contain information about client code # stack, starting from core code-base up to the end graphical interface. diff --git a/src/main/resources/mainnet.conf b/src/main/resources/mainnet.conf index 6cac98b0e7..f9b43b2e97 100644 --- a/src/main/resources/mainnet.conf +++ b/src/main/resources/mainnet.conf @@ -49,7 +49,7 @@ scorex { network { magicBytes = [1, 0, 2, 4] bindAddress = "0.0.0.0:9030" - nodeName = "ergo-mainnet-4.0.25" + nodeName = "ergo-mainnet-4.0.26" nodeName = ${?NODENAME} knownPeers = [ "213.239.193.208:9030", diff --git a/src/main/resources/testnet.conf b/src/main/resources/testnet.conf index fbe668ce42..da5c6ee2de 100644 --- a/src/main/resources/testnet.conf +++ b/src/main/resources/testnet.conf @@ -61,7 +61,7 @@ scorex { network { magicBytes = [2, 0, 0, 2] bindAddress = "0.0.0.0:9020" - nodeName = "ergo-testnet-4.0.25" + nodeName = "ergo-testnet-4.0.26" nodeName = ${?NODENAME} knownPeers = [ "213.239.193.208:9020", diff --git a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala index 4980ded478..ad5c3c0c94 100644 --- a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala +++ b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala @@ -579,7 +579,7 @@ object CandidateGenerator extends ScorexLogging { /** * Transaction and its cost. */ - type CostedTransaction = (ErgoTransaction, Long) + type CostedTransaction = (ErgoTransaction, Int) //TODO move ErgoMiner to mining package and make `collectTxs` and `fixTxsConflicts` private[mining] @@ -680,8 +680,8 @@ object CandidateGenerator extends ScorexLogging { */ def collectTxs( minerPk: ProveDlog, - maxBlockCost: Long, - maxBlockSize: Long, + maxBlockCost: Int, + maxBlockSize: Int, us: UtxoStateReader, upcomingContext: ErgoStateContext, transactions: Seq[ErgoTransaction] diff --git a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala index a512b52bf9..6d3558e466 100644 --- a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala @@ -139,7 +139,7 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input], ) // Cost limit per block - val maxCost = stateContext.currentParameters.maxBlockCost + val maxCost = stateContext.currentParameters.maxBlockCost.toLong // We sum up previously accumulated cost and transaction initialization cost val startCost = addExact(initialCost, accumulatedCost) @@ -250,8 +250,8 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input], dataBoxes: IndexedSeq[ErgoBox], stateContext: ErgoStateContext, accumulatedCost: Long = 0L) - (implicit verifier: ErgoInterpreter): Try[Long] = { - validateStateful(boxesToSpend, dataBoxes, stateContext, accumulatedCost).result.toTry + (implicit verifier: ErgoInterpreter): Try[Int] = { + validateStateful(boxesToSpend, dataBoxes, stateContext, accumulatedCost).result.toTry.map(_.toInt) } override type M = ErgoTransaction diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 7a26d23f39..15b1a582b1 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -300,7 +300,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, log.debug(s"$remote has equal header-chain") } - if ((oldStatus != status) || syncTracker.isOutdated(remote) || status == Older || status == Fork) { + if ((oldStatus != status) || syncTracker.notSyncedOrOutdated(remote) || status == Older || status == Fork) { val ownSyncInfo = getV1SyncInfo(hr) sendSyncToPeer(remote, ownSyncInfo) } @@ -350,7 +350,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, log.debug(s"$remote has equal header-chain") } - if ((oldStatus != status) || syncTracker.isOutdated(remote) || status == Older || status == Fork) { + if ((oldStatus != status) || syncTracker.notSyncedOrOutdated(remote) || status == Older || status == Fork) { val ownSyncInfo = getV2SyncInfo(hr, full = true) sendSyncToPeer(remote, ownSyncInfo) } diff --git a/src/main/scala/org/ergoplatform/network/ErgoPeerStatus.scala b/src/main/scala/org/ergoplatform/network/ErgoPeerStatus.scala index 6f7372d625..ebb9811efa 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoPeerStatus.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoPeerStatus.scala @@ -5,6 +5,7 @@ import org.ergoplatform.nodeView.history.ErgoHistory.Height import scorex.core.app.Version import scorex.core.consensus.History.HistoryComparisonResult import scorex.core.network.ConnectedPeer +import scorex.core.utils.TimeProvider.Time /** * Container for status of another peer @@ -12,10 +13,14 @@ import scorex.core.network.ConnectedPeer * @param peer - peer information (public address, exposed info on operating mode etc) * @param status - peer's blockchain status (is it ahead or behind our, or on fork) * @param height - peer's height + * @param lastSyncSentTime - last time peer was asked to sync, None if never + * @param lastSyncGetTime - last time peer received sync, None if never */ case class ErgoPeerStatus(peer: ConnectedPeer, status: HistoryComparisonResult, - height: Height) { + height: Height, + lastSyncSentTime: Option[Time], + lastSyncGetTime: Option[Time]) { val mode: Option[ModeFeature] = ErgoPeerStatus.mode(peer) def version: Option[Version] = peer.peerInfo.map(_.peerSpec.protocolVersion) diff --git a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala index bc6acb50c2..f2d47dd383 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala @@ -10,46 +10,54 @@ import org.ergoplatform.network.ErgoNodeViewSynchronizer.Events.{BetterNeighbour import scorex.core.network.ConnectedPeer import scorex.core.settings.NetworkSettings import scorex.core.utils.TimeProvider -import scorex.core.utils.TimeProvider.Time import scorex.util.ScorexLogging import scala.collection.mutable import scala.concurrent.duration._ - +import scorex.core.utils.MapPimp final case class ErgoSyncTracker(system: ActorSystem, networkSettings: NetworkSettings, timeProvider: TimeProvider) extends ScorexLogging { - val MinSyncInterval: FiniteDuration = 20.seconds - val SyncThreshold: FiniteDuration = 1.minute - - private[network] val statuses = mutable.Map[ConnectedPeer, ErgoPeerStatus]() - private[network] val lastSyncSentTime = mutable.Map[ConnectedPeer, Time]() - private[network] val lastSyncGetTime = mutable.Map[ConnectedPeer, Time]() + private val MinSyncInterval: FiniteDuration = 20.seconds + private val SyncThreshold: FiniteDuration = 1.minute - protected var lastSyncInfoSentTime: Time = 0L + protected[network] val statuses: mutable.Map[ConnectedPeer, ErgoPeerStatus] = + mutable.Map[ConnectedPeer, ErgoPeerStatus]() - val heights: mutable.Map[ConnectedPeer, Height] = mutable.Map[ConnectedPeer, Height]() + def fullInfo(): Iterable[ErgoPeerStatus] = statuses.values // returns diff def updateLastSyncGetTime(peer: ConnectedPeer): Long = { - val prevSyncGetTime = lastSyncGetTime.getOrElse(peer, 0L) + val prevSyncGetTime = statuses.get(peer).flatMap(_.lastSyncGetTime).getOrElse(0L) val currentTime = timeProvider.time() - lastSyncGetTime(peer) = currentTime + statuses.get(peer).foreach { status => + statuses.update(peer, status.copy(lastSyncGetTime = Option(currentTime))) + } currentTime - prevSyncGetTime } - def fullInfo: Iterable[ErgoPeerStatus] = statuses.values - - def isOutdated(peer: ConnectedPeer): Boolean = { - (timeProvider.time() - lastSyncSentTime.getOrElse(peer, 0L)).millis > SyncThreshold + def notSyncedOrOutdated(peer: ConnectedPeer): Boolean = { + val peerOpt = statuses.get(peer) + val notSyncedOrMissing = peerOpt.forall(_.lastSyncSentTime.isEmpty) + val outdated = + peerOpt + .flatMap(_.lastSyncSentTime) + .exists(syncTime => (timeProvider.time() - syncTime).millis > SyncThreshold) + notSyncedOrMissing || outdated } def updateStatus(peer: ConnectedPeer, status: HistoryComparisonResult, height: Option[Height]): Unit = { val seniorsBefore = numOfSeniors() - statuses += peer -> ErgoPeerStatus(peer, status, height.getOrElse(ErgoHistory.EmptyHistoryHeight)) + statuses.adjust(peer){ + case None => + ErgoPeerStatus(peer, status, height.getOrElse(ErgoHistory.EmptyHistoryHeight), None, None) + case Some(existingPeer) => + existingPeer.copy(status = status, height = height.getOrElse(existingPeer.height)) + } + val seniorsAfter = numOfSeniors() // todo: we should also send NoBetterNeighbour signal when all the peers around are not seniors initially @@ -69,33 +77,26 @@ final case class ErgoSyncTracker(system: ActorSystem, statuses.get(peer).map(_.status) } - //todo: combine both? def clearStatus(remote: InetSocketAddress): Unit = { statuses.find(_._1.connectionId.remoteAddress == remote) match { case Some((peer, _)) => statuses -= peer case None => log.warn(s"Trying to clear status for $remote, but it is not found") } - - lastSyncSentTime.find(_._1.connectionId.remoteAddress == remote) match { - case Some((peer, _)) => lastSyncSentTime -= peer - case None => log.warn(s"Trying to clear last sync time for $remote, but it is not found") - } - - lastSyncGetTime.find(_._1.connectionId.remoteAddress == remote) match { - case Some((peer, _)) => lastSyncGetTime -= peer - case None => log.warn(s"Trying to clear last sync time for $remote, but it is not found") - } } def updateLastSyncSentTime(peer: ConnectedPeer): Unit = { val currentTime = timeProvider.time() - lastSyncSentTime(peer) = currentTime - lastSyncInfoSentTime = currentTime + statuses.get(peer).foreach { status => + statuses.update(peer, status.copy(lastSyncSentTime = Option(currentTime))) + } } - protected def outdatedPeers(): Seq[ConnectedPeer] = - lastSyncSentTime.filter(t => (timeProvider.time() - t._2).millis > SyncThreshold).keys.toSeq - + protected[network] def outdatedPeers: IndexedSeq[ConnectedPeer] = { + val currentTime = timeProvider.time() + statuses.filter { case (_, status) => + status.lastSyncSentTime.exists(syncTime => (currentTime - syncTime).millis > SyncThreshold) + }.keys.toVector + } def peersByStatus: Map[HistoryComparisonResult, Iterable[ConnectedPeer]] = statuses.groupBy(_._2.status).mapValues(_.keys).view.force @@ -108,18 +109,21 @@ final case class ErgoSyncTracker(system: ActorSystem, * `Older` status. * Updates lastSyncSentTime for all returned peers as a side effect */ - def peersToSyncWith(): Seq[ConnectedPeer] = { - val outdated = outdatedPeers() + def peersToSyncWith(): IndexedSeq[ConnectedPeer] = { + val outdated = outdatedPeers val peers = if (outdated.nonEmpty) { outdated } else { - val unknowns = statuses.filter(_._2.status == Unknown).keys.toSeq - val forks = statuses.filter(_._2.status == Fork).keys - val elders = statuses.filter(_._2.status == Older).keys.toSeq + val currentTime = timeProvider.time() + val unknowns = statuses.filter(_._2.status == Unknown).toVector + val forks = statuses.filter(_._2.status == Fork).toVector + val elders = statuses.filter(_._2.status == Older).toVector val nonOutdated = (if (elders.nonEmpty) elders(scala.util.Random.nextInt(elders.size)) +: unknowns else unknowns) ++ forks - nonOutdated.filter(p => (timeProvider.time() - lastSyncSentTime.getOrElse(p, 0L)).millis >= MinSyncInterval) + nonOutdated.filter { case (_, status) => + (currentTime - status.lastSyncSentTime.getOrElse(0L)).millis >= MinSyncInterval + }.map(_._1) } peers.foreach(updateLastSyncSentTime) @@ -128,9 +132,9 @@ final case class ErgoSyncTracker(system: ActorSystem, override def toString: String = { val now = System.currentTimeMillis() - lastSyncSentTime.toSeq.sortBy(_._2)(Ordering[Long].reverse).map { - case (peer, syncTimestamp) => - (peer.connectionId.remoteAddress, statuses.get(peer), now - syncTimestamp) + statuses.toSeq.sortBy(_._2.lastSyncSentTime.getOrElse(0L))(Ordering[Long].reverse).map { + case (peer, status) => + (peer.connectionId.remoteAddress, statuses.get(peer), status.lastSyncSentTime.map(now - _)) }.map { case (address, status, millisSinceLastSync) => s"$address, height: ${status.map(_.height)}, status: ${status.map(_.status)}, lastSync: $millisSinceLastSync ms ago" }.mkString("\n") diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala index f59f3a95d0..58456377bf 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala @@ -14,7 +14,7 @@ import org.ergoplatform.nodeView.mempool.ErgoMemPool.ProcessingOutcome import org.ergoplatform.nodeView.state._ import org.ergoplatform.nodeView.wallet.ErgoWallet import org.ergoplatform.wallet.utils.FileUtils -import org.ergoplatform.settings.{Algos, Constants, ErgoSettings, Parameters} +import org.ergoplatform.settings.{Algos, Constants, ErgoSettings, NetworkType, Parameters} import scorex.core._ import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages._ import org.ergoplatform.nodeView.ErgoNodeViewHolder.{BlockAppliedTransactions, CurrentView, DownloadRequest} @@ -26,6 +26,12 @@ import scorex.core.validation.RecoverableModifierError import scorex.util.ScorexLogging import spire.syntax.all.cfor import java.io.File + +import org.ergoplatform.modifiers.history.{ADProofs, HistoryModifierSerializer} + + +import org.ergoplatform.nodeView.history.ErgoHistory.Height + import scala.annotation.tailrec import scala.util.{Failure, Success, Try} @@ -210,6 +216,20 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti } } + /** + * Get estimated height of headers-chain, if it is synced + * @return height of last header known, if headers-chain is synced, or None if not synced + */ + private def estimatedTip(): Option[Height] = { + Try { //error may happen if history not initialized + if(history().isHeadersChainSynced) { + Some(history().headersHeight) + } else { + None + } + }.getOrElse(None) + } + private def applyState(history: ErgoHistory, stateToApply: State, suffixTrimmed: IndexedSeq[ErgoPersistentModifier], @@ -221,7 +241,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti f case (success@Success(updateInfo), modToApply) => if (updateInfo.failedMod.isEmpty) { - updateInfo.state.applyModifier(modToApply)(lm => pmodModify(lm.pmod, local = true)) match { + updateInfo.state.applyModifier(modToApply, estimatedTip())(lm => pmodModify(lm.pmod, local = true)) match { case Success(stateAfterApply) => history.reportModifierIsValid(modToApply).map { newHis => context.system.eventStream.publish(SemanticallySuccessfulModifier(modToApply)) @@ -280,14 +300,12 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti if (sorted.head.asInstanceOf[Header].height == history().headersHeight + 1) { // we apply sorted headers while headers sequence is not broken - var expectedHeight = history().headersHeight + 1 var linkBroken = false cfor(0)(_ < sorted.length, _ + 1) { idx => val header = sorted(idx).asInstanceOf[Header] - if (!linkBroken && header.height == expectedHeight) { + if (!linkBroken && header.height == history().headersHeight + 1) { pmodModify(header, local = false) - expectedHeight += 1 } else { if (!linkBroken) { linkBroken = true @@ -297,7 +315,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti } } } else { - mods.foreach(h => modifiersCache.put(h.id, h)) + sorted.foreach(h => modifiersCache.put(h.id, h)) } applyFromCacheLoop() @@ -383,75 +401,87 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti //todo: update state in async way? /** * Remote and local persistent modifiers need to be appended to history, applied to state - * which also needs to be propagated to mempool and wallet + * which also needs to be git propagated to mempool and wallet * @param pmod Remote or local persistent modifier + * @param local whether the modifier was generated locally or not */ - protected def pmodModify(pmod: ErgoPersistentModifier, local: Boolean): Unit = - if (!history().contains(pmod.id)) { - context.system.eventStream.publish(StartingPersistentModifierApplication(pmod)) - - log.info(s"Apply modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} to nodeViewHolder") - - history().append(pmod) match { - case Success((historyBeforeStUpdate, progressInfo)) => - log.debug(s"Going to apply modifications to the state: $progressInfo") - context.system.eventStream.publish(SyntacticallySuccessfulModifier(pmod)) - - if (progressInfo.toApply.nonEmpty) { - val (newHistory, newStateTry, blocksApplied) = - updateState(historyBeforeStUpdate, minimalState(), progressInfo, IndexedSeq.empty) - - newStateTry match { - case Success(newMinState) => - val newMemPool = updateMemPool(progressInfo.toRemove, blocksApplied, memoryPool(), newMinState) - - @SuppressWarnings(Array("org.wartremover.warts.OptionPartial")) - val v = vault() - val newVault = if (progressInfo.chainSwitchingNeeded) { - v.rollback(idToVersion(progressInfo.branchPoint.get)) match { - case Success(nv) => nv - case Failure(e) => log.warn("Wallet rollback failed: ", e); v + protected def pmodModify(pmod: ErgoPersistentModifier, local: Boolean): Unit = { + if (!history().contains(pmod.id)) { // todo: .contains reads modifier pmod fully here if in db + + // if ADProofs block section generated locally, just dump it into the database + if (pmod.modifierTypeId == ADProofs.modifierTypeId && local && settings.networkType == NetworkType.MainNet) { + val bytes = HistoryModifierSerializer.toBytes(pmod) //todo: extra allocation here, eliminate + history().dumpToDb(pmod.serializedId, bytes) + context.system.eventStream.publish(SyntacticallySuccessfulModifier(pmod)) + context.system.eventStream.publish(SemanticallySuccessfulModifier(pmod)) + } else { + + context.system.eventStream.publish(StartingPersistentModifierApplication(pmod)) + + log.info(s"Apply modifier ${pmod.encodedId} of type ${pmod.modifierTypeId} to nodeViewHolder") + + history().append(pmod) match { + case Success((historyBeforeStUpdate, progressInfo)) => + log.debug(s"Going to apply modifications to the state: $progressInfo") + context.system.eventStream.publish(SyntacticallySuccessfulModifier(pmod)) + + if (progressInfo.toApply.nonEmpty) { + val (newHistory, newStateTry, blocksApplied) = + updateState(historyBeforeStUpdate, minimalState(), progressInfo, IndexedSeq.empty) + + newStateTry match { + case Success(newMinState) => + val newMemPool = updateMemPool(progressInfo.toRemove, blocksApplied, memoryPool(), newMinState) + + @SuppressWarnings(Array("org.wartremover.warts.OptionPartial")) + val v = vault() + val newVault = if (progressInfo.chainSwitchingNeeded) { + v.rollback(idToVersion(progressInfo.branchPoint.get)) match { + case Success(nv) => nv + case Failure(e) => log.warn("Wallet rollback failed: ", e); v + } + } else { + v } - } else { - v - } - // we assume that wallet scan may be started if fullblocks-chain is no more - // than 20 blocks behind headers-chain - val almostSyncedGap = 20 + // we assume that wallet scan may be started if fullblocks-chain is no more + // than 20 blocks behind headers-chain + val almostSyncedGap = 20 - val headersHeight = newHistory.headersHeight - val fullBlockHeight = newHistory.fullBlockHeight - if((headersHeight - fullBlockHeight) < almostSyncedGap) { - blocksApplied.foreach(newVault.scanPersistent) - } + val headersHeight = newHistory.headersHeight + val fullBlockHeight = newHistory.fullBlockHeight + if ((headersHeight - fullBlockHeight) < almostSyncedGap) { + blocksApplied.foreach(newVault.scanPersistent) + } - log.info(s"Persistent modifier ${pmod.encodedId} applied successfully") - updateNodeView(Some(newHistory), Some(newMinState), Some(newVault), Some(newMemPool)) - chainProgress = - Some(ChainProgress(pmod, headersHeight, fullBlockHeight, timeProvider.time())) - case Failure(e) => - log.warn(s"Can`t apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod) to minimal state", e) - updateNodeView(updatedHistory = Some(newHistory)) - context.system.eventStream.publish(SemanticallyFailedModification(pmod, e)) + log.info(s"Persistent modifier ${pmod.encodedId} applied successfully") + updateNodeView(Some(newHistory), Some(newMinState), Some(newVault), Some(newMemPool)) + chainProgress = + Some(ChainProgress(pmod, headersHeight, fullBlockHeight, timeProvider.time())) + case Failure(e) => + log.warn(s"Can`t apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod) to minimal state", e) + updateNodeView(updatedHistory = Some(newHistory)) + context.system.eventStream.publish(SemanticallyFailedModification(pmod, e)) + } + } else { + requestDownloads(progressInfo) + updateNodeView(updatedHistory = Some(historyBeforeStUpdate)) } - } else { - requestDownloads(progressInfo) - updateNodeView(updatedHistory = Some(historyBeforeStUpdate)) - } - case Failure(CriticalSystemException(error)) => - log.error(error) - ErgoApp.shutdownSystem()(context.system) - case Failure(e: RecoverableModifierError) => - log.warn(s"Can`t yet apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod) to history", e) - context.system.eventStream.publish(RecoverableFailedModification(pmod, e)) - case Failure(e) => - log.warn(s"Can`t apply invalid persistent modifier (id: ${pmod.encodedId}, contents: $pmod) to history", e) - context.system.eventStream.publish(SyntacticallyFailedModification(pmod, e)) + case Failure(CriticalSystemException(error)) => + log.error(error) + ErgoApp.shutdownSystem()(context.system) + case Failure(e: RecoverableModifierError) => + log.warn(s"Can`t yet apply persistent modifier (id: ${pmod.encodedId}, contents: $pmod) to history", e) + context.system.eventStream.publish(RecoverableFailedModification(pmod, e)) + case Failure(e) => + log.warn(s"Can`t apply invalid persistent modifier (id: ${pmod.encodedId}, contents: $pmod) to history", e) + context.system.eventStream.publish(SyntacticallyFailedModification(pmod, e)) + } } } else { log.warn(s"Trying to apply modifier ${pmod.encodedId} that's already in history") } + } @SuppressWarnings(Array("AsInstanceOf")) private def recreatedState(): State = { @@ -495,7 +525,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti } toApply.foldLeft[Try[State]](Success(initState)) { case (acc, m) => log.info(s"Applying modifier during node start-up to restore consistent state: ${m.id}") - acc.flatMap(_.applyModifier(m)(lm => self ! lm)) + acc.flatMap(_.applyModifier(m, estimatedTip())(lm => self ! lm)) } } } @@ -527,14 +557,14 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti case Success(state) => log.info("Recovering state using current epoch") chainToApply.foldLeft[Try[DigestState]](Success(state)) { case (acc, m) => - acc.flatMap(_.applyModifier(m)(lm => self ! lm)) + acc.flatMap(_.applyModifier(m, estimatedTip())(lm => self ! lm)) } case Failure(exception) => // recover using whole headers chain log.warn(s"Failed to recover state from current epoch, using whole chain: ${exception.getMessage}") val wholeChain = history.headerChainBack(Int.MaxValue, bestFullBlock.header, _.isGenesis).headers val genesisState = DigestState.create(None, None, stateDir(settings), constants, parameters) wholeChain.foldLeft[Try[DigestState]](Success(genesisState)) { case (acc, m) => - acc.flatMap(_.applyModifier(m)(lm => self ! lm)) + acc.flatMap(_.applyModifier(m, estimatedTip())(lm => self ! lm)) } } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala index 83e3914b98..906a445daa 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala @@ -46,6 +46,19 @@ trait ErgoHistory def closeStorage(): Unit = historyStorage.close() + /** + * Dump modifier identifier and bytes to database. + * + * Used to dump ADProofs generated locally. + * + * @param mId - modifier identifier + * @param bytes - modifier bytes + * @return Success if modifier inserted into database successfully, Failure otherwise + */ + def dumpToDb(mId: Array[Byte], bytes: Array[Byte]): Try[Unit] = { + historyStorage.insert(mId, bytes) + } + /** * Append ErgoPersistentModifier to History if valid */ diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index b0544a9355..91ed99e2c7 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -94,6 +94,19 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, c } } + /** + * Insert single object to database. This version allows for efficient insert + * when identifier and bytes of object (i.e. modifier, a block section) are known. + * + * @param objectIdToInsert - object id to insert + * @param objectToInsert - object bytes to insert + * @return - Success if insertion was successful, Failure otherwise + */ + def insert(objectIdToInsert: Array[Byte], + objectToInsert: Array[Byte]): Try[Unit] = { + objectsStore.insert(objectIdToInsert, objectToInsert) + } + /** * Remove elements from stored indices and modifiers * diff --git a/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala b/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala index 097ffb7417..ef585a4eb4 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala @@ -1,7 +1,9 @@ package org.ergoplatform.nodeView.state import java.io.File + import org.ergoplatform.ErgoBox +import org.ergoplatform.ErgoLikeContext.Height import org.ergoplatform.modifiers.history.ADProofs import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.ErgoTransaction @@ -84,7 +86,7 @@ class DigestState protected(override val version: VersionTag, Failure(new Exception(s"Modifier not validated: $a")) } - override def applyModifier(mod: ErgoPersistentModifier)(generate: LocallyGeneratedModifier => Unit): Try[DigestState] = + override def applyModifier(mod: ErgoPersistentModifier, estimatedTip: Option[Height])(generate: LocallyGeneratedModifier => Unit): Try[DigestState] = (processFullBlock orElse processHeader orElse processOther) (mod) @SuppressWarnings(Array("OptionGet")) diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala index b4b5bc65c3..5372bd9ef3 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala @@ -2,6 +2,7 @@ package org.ergoplatform.nodeView.state import java.io.File import org.ergoplatform.ErgoBox.{AdditionalRegisters, R4, TokenId} +import org.ergoplatform.ErgoLikeContext.Height import org.ergoplatform._ import org.ergoplatform.mining.emission.EmissionRules import org.ergoplatform.mining.groupElemFromBytes @@ -48,10 +49,11 @@ trait ErgoState[IState <: ErgoState[IState]] extends ErgoStateReader { /** * * @param mod modifire to apply to the state + * @param estimatedTip - estimated height of blockchain tip * @param generate function that handles newly created modifier as a result of application the current one * @return new State */ - def applyModifier(mod: ErgoPersistentModifier)(generate: LocallyGeneratedModifier => Unit): Try[IState] + def applyModifier(mod: ErgoPersistentModifier, estimatedTip: Option[Height])(generate: LocallyGeneratedModifier => Unit): Try[IState] def rollbackTo(version: VersionTag): Try[IState] diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala index 32873b8df2..569bab26d1 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala @@ -4,6 +4,7 @@ import java.io.File import cats.Traverse import org.ergoplatform.ErgoBox +import org.ergoplatform.ErgoLikeContext.Height import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.history.ADProofs import org.ergoplatform.modifiers.mempool.ErgoTransaction @@ -57,7 +58,6 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 val rollbackResult = p.rollback(rootHash).map { _ => new UtxoState(p, version, store, constants, parameters) } - store.clean(constants.keepVersions) rollbackResult case None => Failure(new Error(s"Unable to get root hash at version ${Algos.encoder.encode(version)}")) @@ -92,8 +92,23 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 } } - override def applyModifier(mod: ErgoPersistentModifier)(generate: LocallyGeneratedModifier => Unit): Try[UtxoState] = mod match { + override def applyModifier(mod: ErgoPersistentModifier, estimatedTip: Option[Height]) + (generate: LocallyGeneratedModifier => Unit): Try[UtxoState] = mod match { case fb: ErgoFullBlock => + + // avoid storing versioned information in the database when block being processed is behind + // blockchain tip by `keepVersions` blocks at least + // we store `keepVersions` diffs in the database if chain tip is not known yet + if (fb.height >= estimatedTip.getOrElse(0) - constants.keepVersions) { + if (store.getKeepVersions < constants.keepVersions) { + store.setKeepVersions(constants.keepVersions) + } + } else { + if (store.getKeepVersions > 0) { + store.setKeepVersions(0) + } + } + persistentProver.synchronized { val height = fb.header.height diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala index 1006955c88..72d66565ec 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala @@ -39,30 +39,27 @@ trait UtxoStateReader extends ErgoStateReader with TransactionValidation { */ def validateWithCost(tx: ErgoTransaction, stateContextOpt: Option[ErgoStateContext], - costLimit: Long, - interpreterOpt: Option[ErgoInterpreter]): Try[Long] = { + costLimit: Int, + interpreterOpt: Option[ErgoInterpreter]): Try[Int] = { val context = stateContextOpt.getOrElse(stateContext) - - val verifier = interpreterOpt.getOrElse(ErgoInterpreter(context.currentParameters)) - - val maxBlockCost = context.currentParameters.maxBlockCost - val startCost = maxBlockCost - costLimit + val parameters = context.currentParameters.withBlockCost(costLimit) + val verifier = interpreterOpt.getOrElse(ErgoInterpreter(parameters)) tx.statelessValidity().flatMap { _ => val boxesToSpend = tx.inputs.flatMap(i => boxById(i.boxId)) tx.statefulValidity( - boxesToSpend, - tx.dataInputs.flatMap(i => boxById(i.boxId)), - context, - startCost)(verifier).map(_ - startCost) match { - case Success(txCost) if txCost > costLimit => - Failure(TooHighCostError(s"Transaction $tx has too high cost $txCost")) - case Success(txCost) => - Success(txCost) - case Failure(mme: MalformedModifierError) if mme.message.contains("CostLimitException") => - Failure(TooHighCostError(s"Transaction $tx has too high cost")) - case f: Failure[_] => f - } + boxesToSpend, + tx.dataInputs.flatMap(i => boxById(i.boxId)), + context, + accumulatedCost = 0L)(verifier) match { + case Success(txCost) if txCost > costLimit => + Failure(TooHighCostError(s"Transaction $tx has too high cost $txCost")) + case Success(txCost) => + Success(txCost) + case Failure(mme: MalformedModifierError) if mme.message.contains("CostLimitException") => + Failure(TooHighCostError(s"Transaction $tx has too high cost")) + case f: Failure[_] => f + } } } @@ -73,7 +70,7 @@ trait UtxoStateReader extends ErgoStateReader with TransactionValidation { * * Used in mempool. */ - override def validateWithCost(tx: ErgoTransaction, maxTxCost: Long): Try[Long] = { + override def validateWithCost(tx: ErgoTransaction, maxTxCost: Int): Try[Int] = { validateWithCost(tx, None, maxTxCost, None) } diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala index ec3886ed04..1bb7616fe9 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala @@ -182,7 +182,15 @@ class ErgoWalletActor(settings: ErgoSettings, state.storage.updateStateContext(s.stateContext) match { case Success(_) => val cp = s.stateContext.currentParameters - val newState = ergoWalletService.updateUtxoState(state.copy(stateReaderOpt = Some(s), parameters = cp)) + + val newWalletVars = state.walletVars.withParameters(cp) match { + case Success(res) => res + case Failure(t) => + log.warn("Can not update wallet vars: ", t) + state.walletVars + } + val updState = state.copy(stateReaderOpt = Some(s), parameters = cp, walletVars = newWalletVars) + val newState = ergoWalletService.updateUtxoState(updState) context.become(loadedWallet(newState)) case Failure(t) => val errorMsg = s"Updating wallet state context failed : ${t.getMessage}" @@ -193,7 +201,8 @@ class ErgoWalletActor(settings: ErgoSettings, /** SCAN COMMANDS */ //scan mempool transaction case ScanOffChain(tx) => - val newWalletBoxes = WalletScanLogic.extractWalletOutputs(tx, None, state.walletVars) + val dustLimit = settings.walletSettings.dustLimit + val newWalletBoxes = WalletScanLogic.extractWalletOutputs(tx, None, state.walletVars, dustLimit) val inputs = WalletScanLogic.extractInputBoxes(tx) val newState = state.copy(offChainRegistry = state.offChainRegistry.updateOnTransaction(newWalletBoxes, inputs, state.walletVars.externalScans) @@ -207,7 +216,7 @@ class ErgoWalletActor(settings: ErgoSettings, historyReader.bestFullBlockAt(blockHeight) match { case Some(block) => log.info(s"Wallet is going to scan a block ${block.id} in the past at height ${block.height}") - ergoWalletService.scanBlockUpdate(state, block) match { + ergoWalletService.scanBlockUpdate(state, block, settings.walletSettings.dustLimit) match { case Failure(ex) => val errorMsg = s"Scanning block ${block.id} at height $blockHeight failed : ${ex.getMessage}" log.error(errorMsg, ex) @@ -231,7 +240,7 @@ class ErgoWalletActor(settings: ErgoSettings, if (nextBlockHeight == newBlock.height) { log.info(s"Wallet is going to scan a block ${newBlock.id} on chain at height ${newBlock.height}") val newState = - ergoWalletService.scanBlockUpdate(state, newBlock) match { + ergoWalletService.scanBlockUpdate(state, newBlock, settings.walletSettings.dustLimit) match { case Failure(ex) => val errorMsg = s"Scanning new block ${newBlock.id} on chain at height ${newBlock.height} failed : ${ex.getMessage}" log.error(errorMsg, ex) diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala index 4eebec073e..f3988c16fc 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala @@ -24,6 +24,7 @@ import scorex.util.encode.Base16 import scorex.util.{ModifierId, bytesToId} import sigmastate.Values.SigmaBoolean +import java.io.FileNotFoundException import scala.util.{Failure, Success, Try} /** @@ -179,8 +180,9 @@ trait ErgoWalletService { * * @param state current wallet state * @param block - block to scan + * @param dustLimit - Boxes with value smaller than dustLimit are disregarded in wallet scan logic */ - def scanBlockUpdate(state: ErgoWalletState, block: ErgoFullBlock): Try[ErgoWalletState] + def scanBlockUpdate(state: ErgoWalletState, block: ErgoFullBlock, dustLimit: Option[Long]): Try[ErgoWalletState] /** * Sign a transaction @@ -243,7 +245,12 @@ class ErgoWalletServiceImpl extends ErgoWalletService with ErgoWalletSupport wit log.info("Trying to read wallet in secure mode ..") JsonSecretStorage.readFile(secretStorageSettings).fold( e => { - log.warn(s"Failed to read wallet. Manual initialization is required. Details: ", e) + e match { + case e: FileNotFoundException => + log.info(s"Wallet secret storage not found. Details: {}", e.getMessage) + case _ => + log.warn(s"Failed to read wallet. Manual initialization is required. Details: ", e) + } state }, secretStorage => { @@ -521,8 +528,8 @@ class ErgoWalletServiceImpl extends ErgoWalletService with ErgoWalletSupport wit Failure(new Exception("Unable to derive key, wallet is not initialized")) } - def scanBlockUpdate(state: ErgoWalletState, block: ErgoFullBlock): Try[ErgoWalletState] = - WalletScanLogic.scanBlockTransactions(state.registry, state.offChainRegistry, state.walletVars, block, state.outputsFilter) + def scanBlockUpdate(state: ErgoWalletState, block: ErgoFullBlock, dustLimit: Option[Long]): Try[ErgoWalletState] = + WalletScanLogic.scanBlockTransactions(state.registry, state.offChainRegistry, state.walletVars, block, state.outputsFilter, dustLimit) .map { case (reg, offReg, updatedOutputsFilter) => state.copy(registry = reg, offChainRegistry = offReg, outputsFilter = Some(updatedOutputsFilter)) } def updateUtxoState(state: ErgoWalletState): ErgoWalletState = { diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/WalletScanLogic.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/WalletScanLogic.scala index 36606a1068..13fa45e892 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/WalletScanLogic.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/WalletScanLogic.scala @@ -43,11 +43,12 @@ object WalletScanLogic extends ScorexLogging { offChainRegistry: OffChainRegistry, walletVars: WalletVars, block: ErgoFullBlock, - cachedOutputsFilter: Option[BloomFilter[Array[Byte]]] + cachedOutputsFilter: Option[BloomFilter[Array[Byte]]], + dustLimit: Option[Long] ): Try[(WalletRegistry, OffChainRegistry, BloomFilter[Array[Byte]])] = { scanBlockTransactions( registry, offChainRegistry, walletVars, - block.height, block.id, block.transactions, cachedOutputsFilter) + block.height, block.id, block.transactions, cachedOutputsFilter, dustLimit) } /** @@ -68,7 +69,8 @@ object WalletScanLogic extends ScorexLogging { height: Int, blockId: ModifierId, transactions: Seq[ErgoTransaction], - cachedOutputsFilter: Option[BloomFilter[Array[Byte]]] + cachedOutputsFilter: Option[BloomFilter[Array[Byte]]], + dustLimit: Option[Long] ): Try[(WalletRegistry, OffChainRegistry, BloomFilter[Array[Byte]])] = { // Take unspent wallet outputs Bloom Filter from cache @@ -105,7 +107,7 @@ object WalletScanLogic extends ScorexLogging { val scanRes = transactions.foldLeft(initialScanResults) { case (scanResults, tx) => // extract wallet- (and external scans) related outputs from the transaction - val myOutputs = extractWalletOutputs(tx, Some(height), walletVars) + val myOutputs = extractWalletOutputs(tx, Some(height), walletVars, dustLimit) // add extracted outputs to the filter myOutputs.foreach { out => @@ -180,13 +182,14 @@ object WalletScanLogic extends ScorexLogging { */ def extractWalletOutputs(tx: ErgoTransaction, inclusionHeight: Option[Int], - walletVars: WalletVars): Seq[TrackedBox] = { + walletVars: WalletVars, + dustLimit: Option[Long]): Seq[TrackedBox] = { val trackedBytes: Seq[Array[Byte]] = walletVars.trackedBytes val miningScriptsBytes: Seq[Array[Byte]] = walletVars.miningScriptsBytes val externalScans: Seq[Scan] = walletVars.externalScans - tx.outputs.flatMap { bx => + tx.outputs.flatMap { bx => // First, we check apps triggered by the tx output val appsTriggered = @@ -242,9 +245,14 @@ object WalletScanLogic extends ScorexLogging { } if (statuses.nonEmpty) { - val tb = TrackedBox(tx.id, bx.index, inclusionHeight, None, None, bx, statuses) - log.debug("New tracked box: " + tb.boxId, " scans: " + tb.scans) - Some(tb) + if (dustLimit.exists(bx.value <= _)){ + // filter out boxes with value that is considered dust + None + } else { + val tb = TrackedBox(tx.id, bx.index, inclusionHeight, None, None, bx, statuses) + log.debug("New tracked box: " + tb.boxId, " scans: " + tb.scans) + Some(tb) + } } else { None } diff --git a/src/main/scala/org/ergoplatform/settings/Parameters.scala b/src/main/scala/org/ergoplatform/settings/Parameters.scala index 658c2a82b1..1be7d6abe7 100644 --- a/src/main/scala/org/ergoplatform/settings/Parameters.scala +++ b/src/main/scala/org/ergoplatform/settings/Parameters.scala @@ -62,7 +62,7 @@ class Parameters(val height: Height, /** * Max total computation cost of a block. */ - lazy val maxBlockCost: Long = parametersTable(MaxBlockCostIncrease) + lazy val maxBlockCost: Int = parametersTable(MaxBlockCostIncrease) lazy val softForkStartingHeight: Option[Height] = parametersTable.get(SoftForkStartingHeight) lazy val softForkVotesCollected: Option[Int] = parametersTable.get(SoftForkVotesCollected) @@ -210,6 +210,10 @@ class Parameters(val height: Height, ExtensionCandidate(paramFields ++ rulesToDisableFields) } + def withBlockCost(cost: Int): Parameters = { + Parameters(height, parametersTable.updated(MaxBlockCostIncrease, cost), proposedUpdate) + } + override def toString: String = s"Parameters(height: $height; ${parametersTable.mkString("; ")}; $proposedUpdate)" def canEqual(o: Any): Boolean = o.isInstanceOf[Parameters] diff --git a/src/main/scala/org/ergoplatform/settings/WalletSettings.scala b/src/main/scala/org/ergoplatform/settings/WalletSettings.scala index ccab3d3d99..77971f0ec0 100644 --- a/src/main/scala/org/ergoplatform/settings/WalletSettings.scala +++ b/src/main/scala/org/ergoplatform/settings/WalletSettings.scala @@ -9,6 +9,7 @@ case class WalletSettings(secretStorage: SecretStorageSettings, usePreEip3Derivation: Boolean = false, keepSpentBoxes: Boolean = false, defaultTransactionFee: Long = 1000000L, + dustLimit: Option[Long] = None, maxInputs: Int = 100, optimalInputs: Int = 3, testMnemonic: Option[String] = None, diff --git a/src/main/scala/scorex/core/network/DeliveryTracker.scala b/src/main/scala/scorex/core/network/DeliveryTracker.scala index 25a45da169..7060c56918 100644 --- a/src/main/scala/scorex/core/network/DeliveryTracker.scala +++ b/src/main/scala/scorex/core/network/DeliveryTracker.scala @@ -10,7 +10,7 @@ import scorex.core.ModifierTypeId import scorex.core.consensus.ContainsModifiers import scorex.core.network.DeliveryTracker._ import scorex.core.network.ModifiersStatus._ -import scorex.core.utils.ScorexEncoding +import scorex.core.utils._ import scorex.util.{ModifierId, ScorexLogging} import scala.collection.mutable @@ -375,26 +375,4 @@ object DeliveryTracker { ) } - implicit class MapPimp[K, V](underlying: mutable.Map[K, V]) { - /** - * One liner for updating a Map with the possibility to handle case of missing Key - * @param k map key - * @param f function that is passed Option depending on Key being present or missing, returning new Value - * @return Option depending on map being updated or not - */ - def adjust(k: K)(f: Option[V] => V): Option[V] = underlying.put(k, f(underlying.get(k))) - - /** - * One liner for updating a Map with the possibility to handle case of missing Key - * @param k map key - * @param f function that is passed Option depending on Key being present or missing, - * returning Option signaling whether to update or not - * @return new Map with value updated under given key - */ - def flatAdjust(k: K)(f: Option[V] => Option[V]): Option[V] = - f(underlying.get(k)) match { - case None => None - case Some(v) => underlying.put(k, v) - } - } } diff --git a/src/main/scala/scorex/core/transaction/state/StateFeature.scala b/src/main/scala/scorex/core/transaction/state/StateFeature.scala index 3c2ceecab4..ebccd8a23f 100644 --- a/src/main/scala/scorex/core/transaction/state/StateFeature.scala +++ b/src/main/scala/scorex/core/transaction/state/StateFeature.scala @@ -12,7 +12,7 @@ trait StateFeature * Instance of this trait supports stateful validation of any transaction */ trait TransactionValidation extends StateFeature { - def validateWithCost(tx: ErgoTransaction, maxTxCost: Long): Try[Long] + def validateWithCost(tx: ErgoTransaction, maxTxCost: Int): Try[Int] } object TransactionValidation { diff --git a/src/main/scala/scorex/core/utils/utils.scala b/src/main/scala/scorex/core/utils/utils.scala index f2a37c899c..a3532d56fd 100644 --- a/src/main/scala/scorex/core/utils/utils.scala +++ b/src/main/scala/scorex/core/utils/utils.scala @@ -1,8 +1,8 @@ package scorex.core import java.security.SecureRandom - import scala.annotation.tailrec +import scala.collection.mutable import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} @@ -74,4 +74,27 @@ package object utils { result } + + implicit class MapPimp[K, V](underlying: mutable.Map[K, V]) { + /** + * One liner for updating a Map with the possibility to handle case of missing Key + * @param k map key + * @param f function that is passed Option depending on Key being present or missing, returning new Value + * @return Option depending on map being updated or not + */ + def adjust(k: K)(f: Option[V] => V): Option[V] = underlying.put(k, f(underlying.get(k))) + + /** + * One liner for updating a Map with the possibility to handle case of missing Key + * @param k map key + * @param f function that is passed Option depending on Key being present or missing, + * returning Option signaling whether to update or not + * @return new Map with value updated under given key + */ + def flatAdjust(k: K)(f: Option[V] => Option[V]): Option[V] = + f(underlying.get(k)) match { + case None => None + case Some(v) => underlying.put(k, v) + } + } } diff --git a/src/test/resources/application.conf b/src/test/resources/application.conf index b874e974ec..5971fbf227 100644 --- a/src/test/resources/application.conf +++ b/src/test/resources/application.conf @@ -169,6 +169,9 @@ ergo { defaultTransactionFee = 10000 + # Boxes with value smaller than dustLimit are disregarded in wallet scan logic + dustLimit = 999 + # Save used boxes (may consume additinal disk space) or delete them immediately keepSpentBoxes = false diff --git a/src/test/scala/org/ergoplatform/db/VersionedStoreSpec.scala b/src/test/scala/org/ergoplatform/db/VersionedStoreSpec.scala index b1096baed0..59aa72d8b0 100644 --- a/src/test/scala/org/ergoplatform/db/VersionedStoreSpec.scala +++ b/src/test/scala/org/ergoplatform/db/VersionedStoreSpec.scala @@ -65,7 +65,9 @@ class VersionedStoreSpec extends AnyPropSpec with Matchers with DBSpec { store.versionIdExists(v1) shouldBe true store.insert(v3, Seq(keyC -> valC)).get + store.insert(v4, Seq(keyA -> valA)).get + store.versionIdExists(v4) shouldBe true store.versionIdExists(v3) shouldBe true store.versionIdExists(v2) shouldBe true store.versionIdExists(v1) shouldBe false diff --git a/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala b/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala index 7b2f10e932..76d9cf1df0 100644 --- a/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala +++ b/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala @@ -95,7 +95,7 @@ class MempoolAuditorSpec extends AnyFlatSpec with NodeViewTestOps with ErgoTestH val (txs0, bh1) = validTransactionsFromBoxHolder(bh0) val b1 = validFullBlock(None, us0, txs0) - val us = us0.applyModifier(b1)(_ => ()).get + val us = us0.applyModifier(b1, None)(_ => ()).get val bxs = bh1.boxes.values.toList.filter(_.proposition != genesisEmissionBox.proposition) val txs = validTransactionsFromBoxes(200000, bxs, new RandomWrapper)._1 diff --git a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala index 18eae2d466..35ed17da81 100644 --- a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala +++ b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala @@ -67,8 +67,8 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest { defaultMinerPk ) - us.applyModifier(validFullBlock(None, us, incorrectTxs))(_ => ()) shouldBe 'failure - us.applyModifier(validFullBlock(None, us, txs))(_ => ()) shouldBe 'success + us.applyModifier(validFullBlock(None, us, incorrectTxs), None)(_ => ()) shouldBe 'failure + us.applyModifier(validFullBlock(None, us, txs), None)(_ => ()) shouldBe 'success } property("collect reward from transaction fees only") { @@ -93,8 +93,8 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest { defaultMinerPk ) - us.applyModifier(validFullBlock(None, us, blockTx +: incorrect))(_ => ()) shouldBe 'failure - us.applyModifier(validFullBlock(None, us, blockTx +: txs))(_ => ()) shouldBe 'success + us.applyModifier(validFullBlock(None, us, blockTx +: incorrect), None)(_ => ()) shouldBe 'failure + us.applyModifier(validFullBlock(None, us, blockTx +: txs), None)(_ => ()) shouldBe 'success } property("filter out double spend txs") { @@ -116,7 +116,7 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest { property("should only collect valid transactions") { def checkCollectTxs( - maxCost: Long, + maxCost: Int, maxSize: Int, withTokens: Boolean = false ): Unit = { @@ -166,7 +166,7 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest { ._1 val newBoxes = fromBigMempool.flatMap(_.outputs) - val costs: Seq[Long] = fromBigMempool.map { tx => + val costs: Seq[Int] = fromBigMempool.map { tx => us.validateWithCost(tx, Some(upcomingContext), Int.MaxValue, Some(verifier)).getOrElse { val boxesToSpend = tx.inputs.map(i => newBoxes.find(b => b.id sameElements i.boxId).get) @@ -184,7 +184,7 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest { checkCollectTxs(parameters.maxBlockCost, Int.MaxValue) // transactions reach block size limit - checkCollectTxs(Long.MaxValue, 4096) + checkCollectTxs(Int.MaxValue, 4096) // miner collects correct transactions from mempool even if they have tokens checkCollectTxs(Int.MaxValue, Int.MaxValue, withTokens = true) @@ -215,7 +215,7 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest { .toSeq val block = validFullBlock(None, us, blockTx +: txs) - us = us.applyModifier(block)(_ => ()).get + us = us.applyModifier(block, None)(_ => ()).get val blockTx2 = validTransactionFromBoxes(txBoxes(1), outputsProposition = feeProposition) @@ -227,9 +227,9 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest { val invalidBlock2 = validFullBlock(Some(block), us, IndexedSeq(earlySpendingTx, blockTx2)) - us.applyModifier(invalidBlock2)(_ => ()) shouldBe 'failure + us.applyModifier(invalidBlock2, None)(_ => ()) shouldBe 'failure - us = us.applyModifier(block2)(_ => ()).get + us = us.applyModifier(block2, None)(_ => ()).get val earlySpendingTx2 = validTransactionFromBoxes(txs.head.outputs, stateCtxOpt = Some(us.stateContext)) @@ -238,7 +238,7 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest { validTransactionFromBoxes(txBoxes(2), outputsProposition = feeProposition) val block3 = validFullBlock(Some(block2), us, IndexedSeq(earlySpendingTx2, blockTx3)) - us.applyModifier(block3)(_ => ()) shouldBe 'success + us.applyModifier(block3, None)(_ => ()) shouldBe 'success } property("collect reward from both emission box and fees") { diff --git a/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala b/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala index 73216bd9ab..5d6c4f94c7 100644 --- a/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala +++ b/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala @@ -451,7 +451,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { boxCandidate.additionalRegisters) } - private def checkTx(from: IndexedSeq[ErgoBox], wrongTx: ErgoTransaction): Try[Long] = { + private def checkTx(from: IndexedSeq[ErgoBox], wrongTx: ErgoTransaction): Try[Int] = { wrongTx.statelessValidity().flatMap(_ => wrongTx.statefulValidity(from, emptyDataBoxes, emptyStateContext)) } diff --git a/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala b/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala index 211d864ae0..ac5d0f3869 100644 --- a/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala +++ b/src/test/scala/org/ergoplatform/network/ErgoSyncTrackerSpecification.scala @@ -2,7 +2,7 @@ package org.ergoplatform.network import akka.actor.ActorSystem import org.ergoplatform.utils.ErgoPropertyTest -import scorex.core.consensus.History.Older +import scorex.core.consensus.History.{Older, Younger} import scorex.core.network.{ConnectedPeer, ConnectionId, Incoming} import scorex.core.network.peer.PeerInfo @@ -14,19 +14,29 @@ class ErgoSyncTrackerSpecification extends ErgoPropertyTest { val connectedPeer = ConnectedPeer(cid, handlerRef = null, lastMessage = 5L, Some(peerInfo)) val syncTracker = ErgoSyncTracker(ActorSystem(), settings.scorexSettings.network, timeProvider) - val status = Older val height = 1000 - syncTracker.updateStatus(connectedPeer, status, Some(height)) + // add peer to sync + syncTracker.updateStatus(connectedPeer, Younger, Some(height)) + syncTracker.statuses(connectedPeer) shouldBe ErgoPeerStatus(connectedPeer, Younger, height, None, None) + // updating status should change status and height of existing peer + syncTracker.updateStatus(connectedPeer, Older, Some(height+1)) + syncTracker.getStatus(connectedPeer) shouldBe Some(Older) + syncTracker.fullInfo().head.height shouldBe height+1 - syncTracker.getStatus(connectedPeer) shouldBe Some(status) - syncTracker.peersByStatus.apply(status).head shouldBe connectedPeer - syncTracker.isOutdated(connectedPeer) shouldBe true + syncTracker.peersByStatus.apply(Older).head shouldBe connectedPeer + // peer should not be synced yet + syncTracker.notSyncedOrOutdated(connectedPeer) shouldBe true + syncTracker.outdatedPeers shouldBe Vector.empty + // peer should be ready for sync syncTracker.peersToSyncWith().head shouldBe connectedPeer - + syncTracker.updateLastSyncSentTime(connectedPeer) + // peer should be synced now + syncTracker.notSyncedOrOutdated(connectedPeer) shouldBe false syncTracker.clearStatus(connectedPeer.connectionId.remoteAddress) + // peer should not be tracked anymore syncTracker.getStatus(connectedPeer) shouldBe None syncTracker.peersByStatus.isEmpty shouldBe true - syncTracker.lastSyncSentTime.get(connectedPeer) shouldBe None + syncTracker.statuses.get(connectedPeer) shouldBe None syncTracker.peersToSyncWith().length shouldBe 0 } } diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/ScriptsSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/ScriptsSpec.scala index b313bc8f3d..72c38d6337 100644 --- a/src/test/scala/org/ergoplatform/nodeView/mempool/ScriptsSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/mempool/ScriptsSpec.scala @@ -76,6 +76,6 @@ class ScriptsSpec extends ErgoPropertyTest { assert(us.boxById(boxId).isDefined, s"Box ${Algos.encode(boxId)} missed") } val block = validFullBlock(None, us, tx, Some(1234L)) - us.applyModifier(block)(_ => ()) + us.applyModifier(block, None)(_ => ()) } } diff --git a/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala index 6f543d374a..7679337105 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala @@ -22,7 +22,7 @@ class DigestStateSpecification extends ErgoPropertyTest { val fb = validFullBlock(parentOpt = None, us, bh) val dir2 = createTempDir val ds = DigestState.create(Some(us.version), Some(us.rootHash), dir2, stateConstants, parameters) - ds.applyModifier(fb)(_ => ()) shouldBe 'success + ds.applyModifier(fb, None)(_ => ()) shouldBe 'success ds.close() val state = DigestState.create(None, None, dir2, stateConstants, parameters) @@ -40,8 +40,8 @@ class DigestStateSpecification extends ErgoPropertyTest { val blBh = validFullBlockWithBoxHolder(parentOpt, us, bh, new RandomWrapper(Some(seed))) val block = blBh._1 bh = blBh._2 - ds = ds.applyModifier(block)(_ => ()).get - us = us.applyModifier(block)(_ => ()).get + ds = ds.applyModifier(block, None)(_ => ()).get + us = us.applyModifier(block, None)(_ => ()).get parentOpt = Some(block) } } @@ -62,14 +62,14 @@ class DigestStateSpecification extends ErgoPropertyTest { block.blockTransactions.transactions.exists(_.dataInputs.nonEmpty) shouldBe true val ds = createDigestState(us.version, us.rootHash, parameters) - ds.applyModifier(block)(_ => ()) shouldBe 'success + ds.applyModifier(block, None)(_ => ()) shouldBe 'success } } property("applyModifier() - invalid block") { forAll(invalidErgoFullBlockGen) { b => val state = createDigestState(emptyVersion, emptyAdDigest, parameters) - state.applyModifier(b)(_ => ()).isFailure shouldBe true + state.applyModifier(b, None)(_ => ()).isFailure shouldBe true } } @@ -84,7 +84,7 @@ class DigestStateSpecification extends ErgoPropertyTest { ds.rollbackVersions.size shouldEqual 1 - val ds2 = ds.applyModifier(block)(_ => ()).get + val ds2 = ds.applyModifier(block, None)(_ => ()).get ds2.rollbackVersions.size shouldEqual 2 @@ -97,7 +97,7 @@ class DigestStateSpecification extends ErgoPropertyTest { ds3.stateContext.lastHeaders.size shouldEqual 0 - ds3.applyModifier(block)(_ => ()).get.rootHash shouldBe ds2.rootHash + ds3.applyModifier(block, None)(_ => ()).get.rootHash shouldBe ds2.rootHash } } diff --git a/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala index 7b51aad33d..68f396a5db 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala @@ -31,11 +31,11 @@ class ErgoStateSpecification extends ErgoPropertyTest { val bt = BlockTransactions(dsHeader.id, version, dsTxs) val doubleSpendBlock = ErgoFullBlock(dsHeader, bt, validBlock.extension, validBlock.adProofs) - us.applyModifier(doubleSpendBlock)(_ => ()) shouldBe 'failure - us.applyModifier(validBlock)(_ => ()) shouldBe 'success + us.applyModifier(doubleSpendBlock, None)(_ => ()) shouldBe 'failure + us.applyModifier(validBlock, None)(_ => ()) shouldBe 'success - ds.applyModifier(doubleSpendBlock)(_ => ()) shouldBe 'failure - ds.applyModifier(validBlock)(_ => ()) shouldBe 'success + ds.applyModifier(doubleSpendBlock, None)(_ => ()) shouldBe 'failure + ds.applyModifier(validBlock, None)(_ => ()) shouldBe 'success } } @@ -58,8 +58,8 @@ class ErgoStateSpecification extends ErgoPropertyTest { val blBh = validFullBlockWithBoxHolder(lastBlocks.headOption, us, bh, new RandomWrapper(Some(seed))) val block = blBh._1 bh = blBh._2 - ds = ds.applyModifier(block)(_ => ()).get - us = us.applyModifier(block)(_ => ()).get + ds = ds.applyModifier(block, None)(_ => ()).get + us = us.applyModifier(block, None)(_ => ()).get lastBlocks = block +: lastBlocks requireEqualStateContexts(us.stateContext, ds.stateContext, lastBlocks.map(_.header)) } @@ -82,7 +82,7 @@ class ErgoStateSpecification extends ErgoPropertyTest { val block = blBh._1 parentOpt = Some(block) bh = blBh._2 - us = us.applyModifier(block)(_ => ()).get + us = us.applyModifier(block, None)(_ => ()).get val changes1 = ErgoState.boxChanges(block.transactions).get val changes2 = ErgoState.boxChanges(block.transactions).get diff --git a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala index 57d64d73f1..d4d06386a1 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala @@ -36,7 +36,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera var (us, bh) = createUtxoState(parameters) var foundersBox = genesisBoxes.last var lastBlock = validFullBlock(parentOpt = None, us, bh) - us = us.applyModifier(lastBlock)(_ => ()).get + us = us.applyModifier(lastBlock, None)(_ => ()).get // spent founders box, leaving the same proposition (0 until 10) foreach { _ => @@ -49,9 +49,9 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera val unsignedTx = new UnsignedErgoTransaction(inputs, IndexedSeq(), newBoxes) val tx: ErgoTransaction = ErgoTransaction(defaultProver.sign(unsignedTx, IndexedSeq(foundersBox), emptyDataBoxes, us.stateContext).get) val txCostLimit = initSettings.nodeSettings.maxTransactionCost - us.validateWithCost(tx, None, txCostLimit, None).get should be <= 100000L + us.validateWithCost(tx, None, txCostLimit, None).get should be <= 100000 val block1 = validFullBlock(Some(lastBlock), us, Seq(ErgoTransaction(tx))) - us = us.applyModifier(block1)(_ => ()).get + us = us.applyModifier(block1, None)(_ => ()).get foundersBox = tx.outputs.head lastBlock = block1 } @@ -81,7 +81,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera val adProofs = ADProofs(realHeader.id, adProofBytes) val bt = BlockTransactions(realHeader.id, Header.InitialVersion, txs) val fb = ErgoFullBlock(realHeader, bt, genExtension(realHeader, us.stateContext), Some(adProofs)) - us = us.applyModifier(fb)(_ => ()).get + us = us.applyModifier(fb, None)(_ => ()).get val remaining = emission.remainingFoundationRewardAtHeight(height) // check validity of transaction, spending founders box @@ -142,7 +142,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera us.extractEmissionBox(block) should not be None lastBlockOpt = Some(block) bh = blBh._2 - us = us.applyModifier(block)(_ => ()).get + us = us.applyModifier(block, None)(_ => ()).get } } @@ -170,7 +170,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera val adProofs = ADProofs(realHeader.id, adProofBytes) val bt = BlockTransactions(realHeader.id, 1: Byte, txs) val fb = ErgoFullBlock(realHeader, bt, genExtension(realHeader, us.stateContext), Some(adProofs)) - us = us.applyModifier(fb)(_ => ()).get + us = us.applyModifier(fb, None)(_ => ()).get height = height + 1 } } @@ -197,7 +197,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera height = height + 1 val bt = BlockTransactions(realHeader.id, Header.InitialVersion, txs) val fb = ErgoFullBlock(realHeader, bt, genExtension(realHeader, us.stateContext), Some(adProofs)) - us = us.applyModifier(fb)(_ => ()).get + us = us.applyModifier(fb, None)(_ => ()).get fb } // create new genesis state @@ -215,7 +215,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera } // apply chain of headers full block to state chain.foreach { fb => - us2 = us2.applyModifier(fb)(_ => ()).get + us2 = us2.applyModifier(fb, None)(_ => ()).get } Await.result(f, Duration.Inf) } @@ -427,14 +427,14 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera bh.sortedBoxes.foreach(box => us.boxById(box.id) should not be None) val block = validFullBlock(parentOpt = None, us, bh) - us.applyModifier(block)(_ => ()).get + us.applyModifier(block, None)(_ => ()).get } } property("applyModifier() - invalid block") { forAll(invalidErgoFullBlockGen) { b => val state = createUtxoState(parameters)._1 - state.applyModifier(b)(_ => ()).isFailure shouldBe true + state.applyModifier(b, None)(_ => ()).isFailure shouldBe true } } @@ -452,8 +452,8 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera } val invalidBlock = validFullBlock(parentOpt = None, us2, bh2) - us.applyModifier(invalidBlock)(_ => ()).isSuccess shouldBe false - us.applyModifier(validBlock)(_ => ()).isSuccess shouldBe true + us.applyModifier(invalidBlock, None)(_ => ()).isSuccess shouldBe false + us.applyModifier(validBlock, None)(_ => ()).isSuccess shouldBe true } @@ -472,17 +472,17 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera val chain2block2 = validFullBlock(Some(chain2block1), wusChain2Block1) var (state, _) = createUtxoState(parameters) - state = state.applyModifier(genesis)(_ => ()).get + state = state.applyModifier(genesis, None)(_ => ()).get - state = state.applyModifier(chain1block1)(_ => ()).get + state = state.applyModifier(chain1block1, None)(_ => ()).get state = state.rollbackTo(idToVersion(genesis.id)).get - state = state.applyModifier(chain2block1)(_ => ()).get - state = state.applyModifier(chain2block2)(_ => ()).get + state = state.applyModifier(chain2block1, None)(_ => ()).get + state = state.applyModifier(chain2block2, None)(_ => ()).get state = state.rollbackTo(idToVersion(genesis.id)).get - state = state.applyModifier(chain1block1)(_ => ()).get - state = state.applyModifier(chain1block2)(_ => ()).get + state = state.applyModifier(chain1block1, None)(_ => ()).get + state = state.applyModifier(chain1block2, None)(_ => ()).get } diff --git a/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedDigestState.scala b/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedDigestState.scala index 26d2949c0d..8210569695 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedDigestState.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedDigestState.scala @@ -1,5 +1,6 @@ package org.ergoplatform.nodeView.state.wrapped +import org.ergoplatform.ErgoLikeContext.Height import org.ergoplatform.modifiers.ErgoPersistentModifier import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import org.ergoplatform.nodeView.state.DigestState @@ -14,9 +15,9 @@ class WrappedDigestState(val digestState: DigestState, override val parameters: Parameters) extends DigestState(digestState.version, digestState.rootHash, digestState.store, parameters, settings) { - override def applyModifier(mod: ErgoPersistentModifier) + override def applyModifier(mod: ErgoPersistentModifier, estimatedTip: Option[Height]) (generate: LocallyGeneratedModifier => Unit): Try[WrappedDigestState] = { - wrapped(super.applyModifier(mod)(_ => ()), wrappedUtxoState.applyModifier(mod)(_ => ())) + wrapped(super.applyModifier(mod, estimatedTip)(_ => ()), wrappedUtxoState.applyModifier(mod, estimatedTip)(_ => ())) } override def rollbackTo(version: VersionTag): Try[WrappedDigestState] = { diff --git a/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedUtxoState.scala b/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedUtxoState.scala index e3ffc99869..445fed09a6 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedUtxoState.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedUtxoState.scala @@ -4,6 +4,7 @@ import java.io.File import akka.actor.ActorRef import org.ergoplatform.ErgoBox +import org.ergoplatform.ErgoLikeContext.Height import org.ergoplatform.modifiers.ErgoPersistentModifier import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import org.ergoplatform.nodeView.state._ @@ -36,8 +37,9 @@ class WrappedUtxoState(prover: PersistentBatchAVLProver[Digest32, HF], case Failure(e) => Failure(e) } - override def applyModifier(mod: ErgoPersistentModifier)(generate: LocallyGeneratedModifier => Unit): Try[WrappedUtxoState] = - super.applyModifier(mod)(generate) match { + override def applyModifier(mod: ErgoPersistentModifier, estimatedTip: Option[Height] = None) + (generate: LocallyGeneratedModifier => Unit): Try[WrappedUtxoState] = + super.applyModifier(mod, estimatedTip)(generate) match { case Success(us) => mod match { case ct: TransactionsCarryingPersistentNodeViewModifier => diff --git a/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala b/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala index 19fb66fadc..c5c034bd1b 100644 --- a/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala @@ -67,6 +67,31 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi getBestHeaderOpt shouldBe Some(block.header) } + private val t3a = TestCase("do not apply block headers in invalid order") { fixture => + import fixture._ + val (us, bh) = createUtxoState(parameters) + val parentBlock = validFullBlock(None, us, bh) + val block = validFullBlock(Some(parentBlock), us, bh) + + getBestHeaderOpt shouldBe None + getHistoryHeight shouldBe ErgoHistory.EmptyHistoryHeight + + subscribeEvents(classOf[SyntacticallySuccessfulModifier]) + + //sending child header without parent header + nodeViewHolderRef ! ModifiersFromRemote(List(block.header)) + expectNoMsg() + + // sende correct header sequence + nodeViewHolderRef ! ModifiersFromRemote(List(parentBlock.header)) + expectMsgType[SyntacticallySuccessfulModifier] + + nodeViewHolderRef ! ModifiersFromRemote(List(block.header)) + expectMsgType[SyntacticallySuccessfulModifier] + + getHistoryHeight shouldBe 2 + } + private val t4 = TestCase("apply valid block as genesis") { fixture => import fixture._ val (us, bh) = createUtxoState(parameters) @@ -491,7 +516,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi } } - val cases: List[TestCase] = List(t0, t1, t2, t3, t4, t5, t6, t7, t8, t9) + val cases: List[TestCase] = List(t0, t1, t2, t3, t3a, t4, t5, t6, t7, t8, t9) NodeViewTestConfig.allConfigs.foreach { c => cases.foreach { t => diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala index 08aa7bfe73..a825bf1f55 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala @@ -13,7 +13,6 @@ import scorex.util.encode.Base16 import sigmastate.eval._ import sigmastate.eval.Extensions._ -import scala.util.Random import org.ergoplatform.wallet.boxes.BoxSelector.MinBoxValue import org.ergoplatform.wallet.boxes.ErgoBoxSerializer import org.ergoplatform.wallet.secrets.PrimitiveSecretKey @@ -302,7 +301,7 @@ class ErgoWalletSpec extends ErgoPropertyTest with WalletTestOps with Eventually bs0.walletBalance shouldBe 0 bs0.walletAssetBalances shouldBe empty - val balance1 = Random.nextInt(1000) + 1 + val balance1 = settings.walletSettings.dustLimit.getOrElse(1000000L) + 1 val box1 = IndexedSeq(new ErgoBoxCandidate(balance1, pubKey, startHeight, randomNewAsset.toColl)) wallet.scanOffchain(ErgoTransaction(fakeInputs, box1)) @@ -313,7 +312,7 @@ class ErgoWalletSpec extends ErgoPropertyTest with WalletTestOps with Eventually bs1.walletAssetBalances shouldBe assetAmount(box1) } - val balance2 = Random.nextInt(1000) + 1 + val balance2 = settings.walletSettings.dustLimit.getOrElse(1000000L) + 1 val box2 = IndexedSeq(new ErgoBoxCandidate(balance2, pubKey, startHeight, randomNewAsset.toColl)) wallet.scanOffchain(ErgoTransaction(fakeInputs, IndexedSeq(), box2)) diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/WalletScanLogicSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/WalletScanLogicSpec.scala index 9080bdc4d2..24493711d5 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/WalletScanLogicSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/WalletScanLogicSpec.scala @@ -91,7 +91,7 @@ class WalletScanLogicSpec extends ErgoPropertyTest with DBSpec with WalletTestOp val inclusionHeightOpt = if (height <= 0) None else Some(height) forAll(trackedTransactionGen, walletVarsGen) { case (trackedTransaction, walletVars) => - val foundBoxes = extractWalletOutputs(trackedTransaction.tx, inclusionHeightOpt, walletVars) + val foundBoxes = extractWalletOutputs(trackedTransaction.tx, inclusionHeightOpt, walletVars, None) foundBoxes.length shouldBe trackedTransaction.scriptsCount foundBoxes.map(_.inclusionHeightOpt).forall(_ == inclusionHeightOpt) shouldBe true foundBoxes.map(_.value).sum shouldBe trackedTransaction.valuesSum @@ -105,6 +105,19 @@ class WalletScanLogicSpec extends ErgoPropertyTest with DBSpec with WalletTestOp } } + property("extractWalletOutputs filters out boxes with dust") { + val height = Random.nextInt(200) - 100 + val inclusionHeightOpt = if (height <= 0) None else Some(height) + forAll(trackedTransactionGen, walletVarsGen) { case (trackedTransaction, walletVars) => + val highDustLimit = Some(Long.MaxValue) + val foundBoxes1 = extractWalletOutputs(trackedTransaction.tx, inclusionHeightOpt, walletVars, highDustLimit) + foundBoxes1 shouldBe empty + val lowDustLimit = Some(1L) + val foundBoxes2 = extractWalletOutputs(trackedTransaction.tx, inclusionHeightOpt, walletVars, lowDustLimit) + foundBoxes2.forall(_.value > 1) shouldBe true + } + } + property("scanBlockTransactions") { withVersionedStore(10) { store => val walletVars = walletVarsGen.sample.get @@ -115,7 +128,7 @@ class WalletScanLogicSpec extends ErgoPropertyTest with DBSpec with WalletTestOp val height0 = 5 //simplest case - we're scanning an empty block val (r0, o0, f0) = - scanBlockTransactions(emptyReg, emptyOff, walletVars, height0, blockId, Seq.empty, None).get + scanBlockTransactions(emptyReg, emptyOff, walletVars, height0, blockId, Seq.empty, None, None).get val r0digest = r0.fetchDigest() r0digest.walletBalance shouldBe 0 r0digest.walletAssetBalances.size shouldBe 0 @@ -139,7 +152,7 @@ class WalletScanLogicSpec extends ErgoPropertyTest with DBSpec with WalletTestOp val offDigestBefore = off.digest.walletBalance val (r1, o1, f1) = - scanBlockTransactions(registry, off, walletVars, height1, blockId, txs, Some(f0)).get + scanBlockTransactions(registry, off, walletVars, height1, blockId, txs, Some(f0), None).get val r1digest = r1.fetchDigest() r1digest.walletBalance shouldBe (regDigestBefore + trackedTransaction.paymentValues.sum) r1digest.walletAssetBalances.size shouldBe 0 @@ -159,7 +172,7 @@ class WalletScanLogicSpec extends ErgoPropertyTest with DBSpec with WalletTestOp val spendingTx = ErgoTransaction(inputs, IndexedSeq.empty, tx.outputCandidates) val (r2, o2, f2) = - scanBlockTransactions(registry, off, walletVars, height1 + 1, blockId, Seq(spendingTx), Some(f1)).get + scanBlockTransactions(registry, off, walletVars, height1 + 1, blockId, Seq(spendingTx), Some(f1), None).get val r2digest = r2.fetchDigest() r2digest.walletBalance shouldBe (regDigestBefore + trackedTransaction.paymentValues.sum) @@ -180,7 +193,7 @@ class WalletScanLogicSpec extends ErgoPropertyTest with DBSpec with WalletTestOp val spendingTx2 = new ErgoTransaction(inputs2, IndexedSeq.empty, outputs2) val (r3, o3, f3) = - scanBlockTransactions(registry, off, walletVars, height1 + 2, blockId, Seq(spendingTx2), Some(f2)).get + scanBlockTransactions(registry, off, walletVars, height1 + 2, blockId, Seq(spendingTx2), Some(f2), None).get val r3digest = r3.fetchDigest() r3digest.walletBalance shouldBe regDigestBefore @@ -199,7 +212,7 @@ class WalletScanLogicSpec extends ErgoPropertyTest with DBSpec with WalletTestOp val threeTxs = Seq(creatingTx, spendingTx, spendingTx2) val (r4, o4, _) = - scanBlockTransactions(registry, off, walletVars, height1 + 3, blockId, threeTxs, Some(f3)).get + scanBlockTransactions(registry, off, walletVars, height1 + 3, blockId, threeTxs, Some(f3), None).get val r4digest = r4.fetchDigest() r4digest.walletBalance shouldBe regDigestBefore @@ -230,7 +243,7 @@ class WalletScanLogicSpec extends ErgoPropertyTest with DBSpec with WalletTestOp val paymentScanReq = ScanRequest("Payment scan", paymentPredicate, Some(intFlag), Some(true)) val walletVars = WalletVars(None, Seq(paymentScanReq.toScan(scanId).get), Some(cache))(s) - val boxes = extractWalletOutputs(tx, Some(1), walletVars) + val boxes = extractWalletOutputs(tx, Some(1), walletVars, None) if (intFlag == ScanWalletInteraction.Shared || intFlag == ScanWalletInteraction.Forced) { boxes.size shouldBe 1 @@ -254,7 +267,7 @@ class WalletScanLogicSpec extends ErgoPropertyTest with DBSpec with WalletTestOp val paymentScanReq = ScanRequest("Payment scan", paymentPredicate, Some(ScanWalletInteraction.Forced), Some(false)) val walletVars = WalletVars(None, Seq(paymentScanReq.toScan(scanId).get), Some(cache))(s) - val boxes = extractWalletOutputs(tx, Some(1), walletVars) + val boxes = extractWalletOutputs(tx, Some(1), walletVars, None) boxes.size shouldBe 1 boxes.head.scans.size shouldBe 2 diff --git a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala index 97a312b2b2..418765b82d 100644 --- a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala +++ b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala @@ -112,7 +112,7 @@ object ChainGenerator extends App with ErgoTestHelpers { log.info( s"Block ${block.id} with ${block.transactions.size} transactions at height ${block.header.height} generated") - loop(state.applyModifier(block)(_ => ()).get, outToPassNext, Some(block.header), acc :+ block.id) + loop(state.applyModifier(block, None)(_ => ()).get, outToPassNext, Some(block.header), acc :+ block.id) } else { acc } diff --git a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala index 4a3588b895..b87eafa58b 100644 --- a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala +++ b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala @@ -41,7 +41,7 @@ trait ErgoTestConstants extends ScorexLogging { val extendedParameters: Parameters = { // Randomness in tests is causing occasional cost overflow in the state context and insufficient box value val extension = Map( - MaxBlockCostIncrease -> Math.ceil(parameters.parametersTable(MaxBlockCostIncrease) * 1.2).toInt, + MaxBlockCostIncrease -> Math.ceil(parameters.parametersTable(MaxBlockCostIncrease) * 1.3).toInt, MinValuePerByteIncrease -> (parameters.parametersTable(MinValuePerByteIncrease) - 30) ) new Parameters( diff --git a/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala index 507e55c631..b69d08b322 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala @@ -121,12 +121,12 @@ trait ValidBlocksGenerators loop(emptyStateContext.currentParameters.maxBlockCost, stateBoxesIn, mutable.WrappedArray.empty, mutable.WrappedArray.empty, rnd) } - protected def getTxCost(tx: ErgoTransaction, boxesToSpend: Seq[ErgoBox], dataBoxesToUse: Seq[ErgoBox]): Long = { + protected def getTxCost(tx: ErgoTransaction, boxesToSpend: Seq[ErgoBox], dataBoxesToUse: Seq[ErgoBox]): Int = { tx.statefulValidity( tx.inputs.flatMap(i => boxesToSpend.find(_.id == i.boxId)), tx.dataInputs.flatMap(i => dataBoxesToUse.find(_.id == i.boxId)), emptyStateContext, - -20000000)(emptyVerifier).getOrElse(0L) + -2000000)(emptyVerifier).getOrElse(0) } /** diff --git a/src/test/scala/scorex/testkit/properties/state/StateApplicationTest.scala b/src/test/scala/scorex/testkit/properties/state/StateApplicationTest.scala index dc67dd387c..f4b3dc5557 100644 --- a/src/test/scala/scorex/testkit/properties/state/StateApplicationTest.scala +++ b/src/test/scala/scorex/testkit/properties/state/StateApplicationTest.scala @@ -21,7 +21,7 @@ trait StateApplicationTest[ST <: ErgoState[ST]] extends StateTests[ST] { property(propertyNameGenerator("apply modifier")) { forAll(stateGenWithValidModifier) { case (s, m) => val ver = s.version - val sTry = s.applyModifier(m)(_ => ()) + val sTry = s.applyModifier(m, None)(_ => ()) sTry.isSuccess shouldBe true sTry.get.version == ver shouldBe false } @@ -30,17 +30,17 @@ trait StateApplicationTest[ST <: ErgoState[ST]] extends StateTests[ST] { property(propertyNameGenerator("do not apply same valid modifier twice")) { forAll(stateGenWithValidModifier) { case (s, m) => val ver = s.version - val sTry = s.applyModifier(m)(_ => ()) + val sTry = s.applyModifier(m, None)(_ => ()) sTry.isSuccess shouldBe true val s2 = sTry.get s2.version == ver shouldBe false - s2.applyModifier(m)(_ => ()).isSuccess shouldBe false + s2.applyModifier(m, None)(_ => ()).isSuccess shouldBe false } } property(propertyNameGenerator("do not apply invalid modifier")) { forAll(stateGenWithInvalidModifier) { case (s, m) => - val sTry = s.applyModifier(m)(_ => ()) + val sTry = s.applyModifier(m, None)(_ => ()) sTry.isSuccess shouldBe false } } @@ -48,7 +48,8 @@ trait StateApplicationTest[ST <: ErgoState[ST]] extends StateTests[ST] { property(propertyNameGenerator("apply valid modifier after rollback")) { forAll(stateGenWithValidModifier) { case (s, m) => val ver = s.version - val sTry = s.applyModifier(m)(_ => ()) + s.store.setKeepVersions(10) + val sTry = s.applyModifier(m, Some(0))(_ => ()) sTry.isSuccess shouldBe true val s2 = sTry.get s2.version == ver shouldBe false @@ -57,7 +58,7 @@ trait StateApplicationTest[ST <: ErgoState[ST]] extends StateTests[ST] { val s3 = s2.rollbackTo(ver).get s3.version == ver shouldBe true - val sTry2 = s3.applyModifier(m)(_ => ()) + val sTry2 = s3.applyModifier(m, None)(_ => ()) sTry2.isSuccess shouldBe true val s4 = sTry2.get s4.version == ver shouldBe false @@ -67,7 +68,7 @@ trait StateApplicationTest[ST <: ErgoState[ST]] extends StateTests[ST] { property(propertyNameGenerator("application after rollback is possible")) { forAll(stateGen) { s => - + s.store.setKeepVersions(10) val maxRollbackDepth = s match { case ds: DigestState => ds.store.rollbackVersions().size @@ -82,7 +83,7 @@ trait StateApplicationTest[ST <: ErgoState[ST]] extends StateTests[ST] { val s2 = (0 until rollbackDepth).foldLeft(s) { case (state, _) => val modifier = semanticallyValidModifier(state) buf += modifier - val sTry = state.applyModifier(modifier)(_ => ()) + val sTry = state.applyModifier(modifier, Some(rollbackDepth))(_ => ()) sTry shouldBe 'success sTry.get } @@ -94,7 +95,7 @@ trait StateApplicationTest[ST <: ErgoState[ST]] extends StateTests[ST] { s3.version == ver shouldBe true val s4 = buf.foldLeft(s3) { case (state, m) => - val sTry = state.applyModifier(m)(_ => ()) + val sTry = state.applyModifier(m, Some(0))(_ => ()) sTry shouldBe 'success sTry.get }