Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

problem: slow request for full blocks on old heights #255

Merged
merged 1 commit into from
Aug 15, 2023
Merged

problem: slow request for full blocks on old heights #255

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/main/kotlin/io/emeraldpay/dshackle/cache/Caches.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,10 @@ open class Caches(
return memHeightByHash
}

fun getHeightByHash(id: BlockId): Long? {
return memHeightByHash.get(id)
}

fun getRedisHeightByHash(): HeightByHashCache? {
return redisHeightByHashCache
}
Expand Down
4 changes: 4 additions & 0 deletions src/main/kotlin/io/emeraldpay/dshackle/cache/HeightByHashMemCache.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ class HeightByHashMemCache(
return Mono.justOrEmpty(heights.getIfPresent(key))
}

fun get(key: BlockId): Long? {
return heights.getIfPresent(key)
}

fun add(block: BlockContainer) {
heights.put(block.hash, block.height)
}
Expand Down
6 changes: 6 additions & 0 deletions src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/EthereumFullBlocksReader.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,12 @@ class EthereumFullBlocksReader(
private val log = LoggerFactory.getLogger(EthereumFullBlocksReader::class.java)
}

enum class Mode {
Auto,
Always,
Disabled
}

constructor(dataReaders: DataReaders) : this(dataReaders.blockReaderById, dataReaders.blockByHeightReader, dataReaders.txReaderById)

val byHash: Reader<BlockId, BlockContainer> = BlockMerge(blocks, txes)
Expand Down
48 changes: 43 additions & 5 deletions src/main/kotlin/io/emeraldpay/dshackle/upstream/ethereum/NormalizingReader.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,20 @@ import reactor.core.publisher.Mono
import java.util.concurrent.atomic.AtomicReference

class NormalizingReader(
head: AtomicReference<Head>,
caches: Caches,
private val head: AtomicReference<Head>,
private val caches: Caches,
fullBlocksReader: EthereumFullBlocksReader,
) : MethodSpecificReader(), DshackleRpcReader {

companion object {
private val log = LoggerFactory.getLogger(NormalizingReader::class.java)

private val HEX_REGEX = Regex("^0x[0-9a-fA-F]+$")
}

// TODO make configurable, and it supposed to go into the cache config but read from here
private val blockInCache = 6

private val blockByHashFull = BlockByHash(fullBlocksReader)
private val blockByHashAuto = BlockByHashAuto(blockByHashFull)
private val blockByNumber = BlockByNumber(fullBlocksReader, caches.getBlockHashByHeight(), blockByHashAuto)
Expand All @@ -31,21 +36,54 @@ class NormalizingReader(
init {
register(
"eth_getBlockByHash",
{ params -> params.size >= 2 && params[1] == true },
{ params -> params.size >= 2 && acceptBlock(params[0].toString()) && params[1] == true },
blockByHashFull
)
register(
"eth_getBlockByNumber",
{ params -> params.size >= 2 && params[0] is String && params[0].toString().startsWith("0x") && params[1] == true },
{ params -> params.size >= 2 && params[0] is String && acceptBlock(params[0].toString()) && params[1] == true },
blockByNumber
)
register(
"eth_getBlockByNumber",
{ params -> params.size >= 2 && params[0] is String && !params[0].toString().startsWith("0x") && params[1] == true },
{ params -> params.size >= 2 && params[0] is String && !isBlockOrNumber(params[0].toString()) && acceptBlock(params[0].toString()) && params[1] == true },
blockByTag
)
}

fun isBlockOrNumber(id: String): Boolean {
return id.matches(HEX_REGEX) && id.length == 66 || id.length <= 18
}

fun acceptHeight(height: Long): Boolean {
val current = head.get().getCurrentHeight() ?: return false
return height >= current - blockInCache
}

fun acceptBlock(id: String): Boolean {
if (id == "latest") {
return true
}
if (!isBlockOrNumber(id)) {
return false
}
val height = if (id.length == 66) {
val blockId = try {
BlockId.from(id)
} catch (t: Throwable) {
return false
}
caches.getHeightByHash(blockId) ?: return false
} else {
try {
HexQuantity.from(id).value.longValueExact()
} catch (t: Throwable) {
return false
}
}
return acceptHeight(height)
}

class BlockByHash(private val fullBlocksReader: EthereumFullBlocksReader) : DshackleRpcReader {

override fun read(key: DshackleRequest): Mono<DshackleResponse> {
Expand Down
28 changes: 28 additions & 0 deletions src/test/kotlin/io/emeraldpay/dshackle/test/FixedHead.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package io.emeraldpay.dshackle.test

import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.upstream.Head
import org.slf4j.LoggerFactory
import reactor.core.publisher.Flux
import reactor.core.publisher.Mono

class FixedHead(
var height: Long? = null,
var block: BlockContainer? = null,
) : Head {

companion object {
private val log = LoggerFactory.getLogger(FixedHead::class.java)
}

override fun getFlux(): Flux<BlockContainer> {
return Flux.from(Mono.justOrEmpty(block))
}

override fun onBeforeBlock(handler: Runnable) {
}

override fun getCurrentHeight(): Long? {
return height ?: block?.height
}
}
110 changes: 107 additions & 3 deletions src/test/kotlin/io/emeraldpay/dshackle/upstream/ethereum/NormalizingReaderTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import io.emeraldpay.dshackle.cache.Caches
import io.emeraldpay.dshackle.data.BlockContainer
import io.emeraldpay.dshackle.data.BlockId
import io.emeraldpay.dshackle.reader.Reader
import io.emeraldpay.dshackle.test.FixedHead
import io.emeraldpay.dshackle.upstream.EmptyHead
import io.emeraldpay.dshackle.upstream.rpcclient.DshackleRequest
import io.emeraldpay.etherjar.domain.BlockHash
Expand Down Expand Up @@ -38,8 +39,88 @@ class NormalizingReaderTest : ShouldSpec({
every { fullBlock.byHeight } returns heightReader
every { fullBlock.byHash } returns hashReader

context("Predicated") {
should("Accept block") {
val reader = NormalizingReader(AtomicReference(EmptyHead()), Caches.default(), fullBlock)

reader.isBlockOrNumber("0x9e1891c70836f4d5817924a801e3e74fdd794842dfb762ff6767db503208a409") shouldBe true
reader.isBlockOrNumber("0x515846D0F6CC07C99B60F6A910723D5C91817E3C3570D198427222D4ED9F4FB7") shouldBe true
}

should("Accept height") {
val reader = NormalizingReader(AtomicReference(EmptyHead()), Caches.default(), fullBlock)

reader.isBlockOrNumber("0x0") shouldBe true
reader.isBlockOrNumber("0x1D4C00") shouldBe true
reader.isBlockOrNumber("0x1d4c00") shouldBe true
reader.isBlockOrNumber("0x1112e74") shouldBe true
}

should("Deny invalid block") {
val reader = NormalizingReader(AtomicReference(EmptyHead()), Caches.default(), fullBlock)

reader.isBlockOrNumber("0x9e1891c70836f4d5817924") shouldBe false
reader.isBlockOrNumber("0x515846D0F6CC07C99B60F6A910723D5C91817E3C3570D198427222D4ED9F4FB") shouldBe false
}

should("Accept latest tag block") {
val head = FixedHead(height = 100)
val reader = NormalizingReader(AtomicReference(head), Caches.default(), fullBlock)

reader.acceptBlock("latest") shouldBe true
}

should("Accept known recent block") {
val head = FixedHead(block = BlockContainer.from(blockJson))
val caches = Caches.default()
caches.memoizeBlock(BlockContainer.from(blockJson))
val reader = NormalizingReader(AtomicReference(head), caches, fullBlock)

reader.acceptBlock("0xf85b826fdf98ee0f48f7db001be00472e63ceb056846f4ecac5f0c32878b8ab2") shouldBe true
}

should("Deny unknown block") {
val head = FixedHead(block = BlockContainer.from(blockJson))
val caches = Caches.default()
caches.memoizeBlock(BlockContainer.from(blockJson))
val reader = NormalizingReader(AtomicReference(head), caches, fullBlock)

reader.acceptBlock("0x720c6f4ecac5f8f7db001be032878b8ab2f85b826fdf98ee0f4e63ceb0568404") shouldBe false
}

should("Accept recent block") {
val head = FixedHead(height = 100)
val reader = NormalizingReader(AtomicReference(head), Caches.default(), fullBlock)

reader.acceptHeight(100) shouldBe true
reader.acceptHeight(99) shouldBe true
reader.acceptHeight(98) shouldBe true
reader.acceptHeight(97) shouldBe true
reader.acceptHeight(96) shouldBe true
reader.acceptHeight(95) shouldBe true
}

should("Accept future block") {
val head = FixedHead(height = 100)
val reader = NormalizingReader(AtomicReference(head), Caches.default(), fullBlock)

reader.acceptHeight(105) shouldBe true
}

should("Deny old block") {
val head = FixedHead(height = 100)
val reader = NormalizingReader(AtomicReference(head), Caches.default(), fullBlock)

reader.acceptHeight(75) shouldBe false
reader.acceptHeight(0) shouldBe false
}
}

context("eth_getBlockByHash") {
val reader = NormalizingReader(AtomicReference(EmptyHead()), Caches.default(), fullBlock)
val head = FixedHead(block = BlockContainer.from(blockJson))
val caches = Caches.default()
caches.memoizeBlock(BlockContainer.from(blockJson))
val reader = NormalizingReader(AtomicReference(head), caches, fullBlock)

should("Use Full Blocks to rebuild transactions when requested") {
every { hashReader.read(blockId) } returns
Expand All @@ -52,6 +133,17 @@ class NormalizingReaderTest : ShouldSpec({
act.hasResult shouldBe true
}

should("Return nothing when old block requests") {
head.height = 200
every { hashReader.read(blockId) } returns
Mono.just(BlockContainer.from(blockJson))

val act = reader.read(DshackleRequest("eth_getBlockByHash", listOf(blockId.toHexWithPrefix(), true)))
.block(Duration.ofSeconds(1))

act shouldBe null
}

should("Return nothing is full block not requested") {
val act = reader.read(DshackleRequest("eth_getBlockByHash", listOf(blockId.toHexWithPrefix(), false)))
.block(Duration.ofSeconds(1))
Expand All @@ -63,7 +155,8 @@ class NormalizingReaderTest : ShouldSpec({
context("eth_getBlockByNumber") {

should("Use Full Blocks to rebuild transactions when requested and no height is cached") {
val reader = NormalizingReader(AtomicReference(EmptyHead()), Caches.default(), fullBlock)
val head = FixedHead(height = 101)
val reader = NormalizingReader(AtomicReference(head), Caches.default(), fullBlock)
every { heightReader.read(101) } returns
Mono.just(BlockContainer.from(blockJson))

Expand All @@ -75,9 +168,10 @@ class NormalizingReaderTest : ShouldSpec({
}

should("Use Full Blocks to rebuild transactions when requested and height is cached") {
val head = FixedHead(height = 101)
val caches = Caches.default()
caches.memoizeBlock(BlockContainer.from(blockJson))
val reader = NormalizingReader(AtomicReference(EmptyHead()), caches, fullBlock)
val reader = NormalizingReader(AtomicReference(head), caches, fullBlock)

every { hashReader.read(blockId) } returns
Mono.just(BlockContainer.from(blockJson))
Expand All @@ -88,5 +182,15 @@ class NormalizingReaderTest : ShouldSpec({
act shouldNotBe null
act.hasResult shouldBe true
}

should("Return nothing when old block requests") {
val head = FixedHead(height = 101)
val reader = NormalizingReader(AtomicReference(head), Caches.default(), fullBlock)

val act = reader.read(DshackleRequest("eth_getBlockByNumber", listOf("0x25", true)))
.block(Duration.ofSeconds(1))

act shouldBe null
}
}
})
Loading