diff --git a/coordinator/clients/prover-client/serialization/src/test/kotlin/net/consensys/zkevm/coordinator/clients/prover/serialization/ProofToFinalizeJsonResponseTest.kt b/coordinator/clients/prover-client/serialization/src/test/kotlin/net/consensys/zkevm/coordinator/clients/prover/serialization/ProofToFinalizeJsonResponseTest.kt index f986fa147..cacdc7602 100644 --- a/coordinator/clients/prover-client/serialization/src/test/kotlin/net/consensys/zkevm/coordinator/clients/prover/serialization/ProofToFinalizeJsonResponseTest.kt +++ b/coordinator/clients/prover-client/serialization/src/test/kotlin/net/consensys/zkevm/coordinator/clients/prover/serialization/ProofToFinalizeJsonResponseTest.kt @@ -60,7 +60,7 @@ class ProofToFinalizeJsonResponseTest { val jsonParser = jsonNode.traverse() while (!jsonParser.isClosed) { if (jsonParser.nextToken() == JsonToken.FIELD_NAME) { - keys.add(jsonParser.currentName) + keys.add(jsonParser.currentName()) } } return keys diff --git a/coordinator/ethereum/blob-submitter/src/main/kotlin/net/consensys/zkevm/ethereum/submission/LoggingHelper.kt b/coordinator/ethereum/blob-submitter/src/main/kotlin/net/consensys/zkevm/ethereum/submission/LoggingHelper.kt index d68e6059f..814fc33b0 100644 --- a/coordinator/ethereum/blob-submitter/src/main/kotlin/net/consensys/zkevm/ethereum/submission/LoggingHelper.kt +++ b/coordinator/ethereum/blob-submitter/src/main/kotlin/net/consensys/zkevm/ethereum/submission/LoggingHelper.kt @@ -2,6 +2,20 @@ package net.consensys.zkevm.ethereum.submission import org.apache.logging.log4j.Logger +private const val insufficientGasFeeRegexStr = + "(max fee per (blob )?gas less than block (blob gas|base) fee:" + + " address 0x[a-fA-F0-9]{40},? (blobGasFeeCap|maxFeePerGas): [0-9]+," + + " (blobBaseFee|baseFee): [0-9]+ \\(supplied gas [0-9]+\\))" + +private val insufficientGasFeeRegex = Regex(insufficientGasFeeRegexStr) +private val maxFeePerBlobGasRegex = Regex("max fee per blob gas|blobGasFeeCap") + +private fun rewriteInsufficientGasFeeErrorMessage(errorMessage: String): String? { + return insufficientGasFeeRegex.find(errorMessage)?.groupValues?.first() + ?.replace("max fee per gas", "maxFeePerGas") + ?.replace(maxFeePerBlobGasRegex, "maxFeePerBlobGas") +} + fun logUnhandledError( log: Logger, errorOrigin: String, @@ -30,13 +44,32 @@ fun logSubmissionError( error: Throwable, isEthCall: Boolean = false ) { + var matchedInsufficientGasFeeRegex = false val ethMethod = if (isEthCall) "eth_call" else "eth_sendRawTransaction" + val errorMessage = if (isEthCall) { + error.message?.let { + rewriteInsufficientGasFeeErrorMessage(it)?.also { + matchedInsufficientGasFeeRegex = true + } + } ?: error.message + } else { + error.message + } - log.error( - logMessage, - ethMethod, - intervalString, - error.message, - error - ) + if (matchedInsufficientGasFeeRegex) { + log.info( + logMessage, + ethMethod, + intervalString, + errorMessage + ) + } else { + log.error( + logMessage, + ethMethod, + intervalString, + errorMessage, + error + ) + } } diff --git a/coordinator/ethereum/blob-submitter/src/test/kotlin/net/consensys/zkevm/ethereum/submission/LoggingHelperTest.kt b/coordinator/ethereum/blob-submitter/src/test/kotlin/net/consensys/zkevm/ethereum/submission/LoggingHelperTest.kt new file mode 100644 index 000000000..b288799ab --- /dev/null +++ b/coordinator/ethereum/blob-submitter/src/test/kotlin/net/consensys/zkevm/ethereum/submission/LoggingHelperTest.kt @@ -0,0 +1,144 @@ +package net.consensys.zkevm.ethereum.submission + +import org.apache.logging.log4j.Logger +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.mockito.Mockito.mock +import org.mockito.kotlin.eq +import org.mockito.kotlin.verify + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class LoggingHelperTest { + private lateinit var logger: Logger + + private val blobSubmissionFailedLogMsg = "{} for blob submission failed: blob={} errorMessage={}" + private val blobSubmissionFailedIntervalStr = "[4025752..4031499]5748" + private val insufficientMaxFeePerBlobGasErrMsg = "org.web3j.tx.exceptions.ContractCallException:" + + " Contract Call has been reverted by the EVM with the reason:" + + " 'err: max fee per blob gas less than block blob gas fee:" + + " address 0x47C63d1E391FcB3dCdC40C4d7fA58ADb172f8c38 blobGasFeeCap: 1875810596," + + " blobBaseFee: 1962046498 (supplied gas 1000000)'. revertReason=UNKNOWN errorData=null" + + private val insufficientMaxFeePerGasErrMsg = "org.web3j.tx.exceptions.ContractCallException:" + + " Contract Call has been reverted by the EVM with the reason:" + + " 'err: max fee per gas less than block base fee:" + + " address 0x47C63d1E391FcB3dCdC40C4d7fA58ADb172f8c38, maxFeePerGas: 300000000000," + + " baseFee: 302246075616 (supplied gas 1000000)'. revertReason=UNKNOWN errorData=null" + + private val unknownErrMsg = "org.web3j.tx.exceptions.ContractCallException:" + + " Contract Call has been reverted by the EVM with the reason:" + + " 'err: unknown error:" + + " address 0x47C63d1E391FcB3dCdC40C4d7fA58ADb172f8c38'. revertReason=UNKNOWN errorData=null" + + @BeforeEach + fun setUp() { + logger = mock() + } + + @Test + fun `insufficient max fee per gas with isEthCall is true triggers rewrite info message`() { + val error = RuntimeException(insufficientMaxFeePerGasErrMsg) + logSubmissionError( + log = logger, + logMessage = blobSubmissionFailedLogMsg, + intervalString = blobSubmissionFailedIntervalStr, + error = error, + isEthCall = true + ) + val expectedErrorMessage = + "maxFeePerGas less than block base fee:" + + " address 0x47C63d1E391FcB3dCdC40C4d7fA58ADb172f8c38, maxFeePerGas: 300000000000," + + " baseFee: 302246075616 (supplied gas 1000000)" + + verify(logger).info( + eq(blobSubmissionFailedLogMsg), + eq("eth_call"), + eq(blobSubmissionFailedIntervalStr), + eq(expectedErrorMessage) + ) + } + + @Test + fun `insufficient max fee per blob gas with isEthCall is true triggers rewrite info message`() { + val error = RuntimeException(insufficientMaxFeePerBlobGasErrMsg) + logSubmissionError( + log = logger, + logMessage = blobSubmissionFailedLogMsg, + intervalString = blobSubmissionFailedIntervalStr, + error = error, + isEthCall = true + ) + val expectedErrorMessage = + "maxFeePerBlobGas less than block blob gas fee:" + + " address 0x47C63d1E391FcB3dCdC40C4d7fA58ADb172f8c38 maxFeePerBlobGas: 1875810596," + + " blobBaseFee: 1962046498 (supplied gas 1000000)" + + verify(logger).info( + eq(blobSubmissionFailedLogMsg), + eq("eth_call"), + eq(blobSubmissionFailedIntervalStr), + eq(expectedErrorMessage) + ) + } + + @Test + fun `insufficient max fee per gas with isEthCall is false do not trigger rewrite error message`() { + val error = RuntimeException(insufficientMaxFeePerGasErrMsg) + logSubmissionError( + log = logger, + logMessage = blobSubmissionFailedLogMsg, + intervalString = blobSubmissionFailedIntervalStr, + error = error, + isEthCall = false + ) + + verify(logger).error( + eq(blobSubmissionFailedLogMsg), + eq("eth_sendRawTransaction"), + eq(blobSubmissionFailedIntervalStr), + eq(insufficientMaxFeePerGasErrMsg), + eq(error) + ) + } + + @Test + fun `insufficient max fee per blob gas with isEthCall is false do not trigger rewrite error message`() { + val error = RuntimeException(insufficientMaxFeePerBlobGasErrMsg) + logSubmissionError( + log = logger, + logMessage = blobSubmissionFailedLogMsg, + intervalString = blobSubmissionFailedIntervalStr, + error = error, + isEthCall = false + ) + + verify(logger).error( + eq(blobSubmissionFailedLogMsg), + eq("eth_sendRawTransaction"), + eq(blobSubmissionFailedIntervalStr), + eq(insufficientMaxFeePerBlobGasErrMsg), + eq(error) + ) + } + + @Test + fun `Other error with isEthCall is true do not trigger rewrite error message`() { + val error = RuntimeException(unknownErrMsg) + logSubmissionError( + log = logger, + logMessage = blobSubmissionFailedLogMsg, + intervalString = blobSubmissionFailedIntervalStr, + error = error, + isEthCall = true + ) + + verify(logger).error( + eq(blobSubmissionFailedLogMsg), + eq("eth_call"), + eq(blobSubmissionFailedIntervalStr), + eq(unknownErrMsg), + eq(error) + ) + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9a5be47e1..725044e1a 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -8,7 +8,7 @@ jreleaser = {id = "org.jreleaser", version = "1.13.1"} besu = "22.4.2" caffeine = "3.1.6" hoplite = "2.7.5" -jackson = "2.14.2" +jackson = "2.18.0" jna = "5.14.0" junit = "5.10.1" kotlinxDatetime = "0.4.0" @@ -23,5 +23,6 @@ tuweni = "2.3.1" vertx = "4.5.0" web3j = "4.12.0" wiremock = "3.0.1" +jsonUnit = "3.4.1" blobCompressor = "0.0.4" blobShnarfCalculator = "0.0.4" diff --git a/jvm-libs/generic/serialization/jackson/build.gradle b/jvm-libs/generic/serialization/jackson/build.gradle new file mode 100644 index 000000000..6c14517dc --- /dev/null +++ b/jvm-libs/generic/serialization/jackson/build.gradle @@ -0,0 +1,13 @@ +plugins { + id 'net.consensys.zkevm.kotlin-library-conventions' +} + +dependencies { + implementation(project(':jvm-libs:kotlin-extensions')) + api "com.fasterxml.jackson.core:jackson-annotations:${libs.versions.jackson.get()}" + api "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.get()}" + api "com.fasterxml.jackson.module:jackson-module-kotlin:${libs.versions.jackson.get()}" + api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${libs.versions.jackson.get()}" + + testImplementation "net.javacrumbs.json-unit:json-unit-assertj:${libs.versions.jsonUnit.get()}" +} diff --git a/jvm-libs/generic/serialization/jackson/src/main/kotlin/build/linea/s11n/jackson/BytesHexSerDe.kt b/jvm-libs/generic/serialization/jackson/src/main/kotlin/build/linea/s11n/jackson/BytesHexSerDe.kt new file mode 100644 index 000000000..e3e6874d6 --- /dev/null +++ b/jvm-libs/generic/serialization/jackson/src/main/kotlin/build/linea/s11n/jackson/BytesHexSerDe.kt @@ -0,0 +1,51 @@ +package build.linea.s11n.jackson + +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonDeserializer +import com.fasterxml.jackson.databind.JsonSerializer +import com.fasterxml.jackson.databind.SerializerProvider +import java.util.HexFormat + +private val hexFormatter = HexFormat.of() + +object ByteArrayToHexSerializer : JsonSerializer() { + override fun serialize(value: ByteArray, gen: JsonGenerator, serializers: SerializerProvider?) { + gen.writeString("0x" + hexFormatter.formatHex(value)) + } + + override fun handledType(): Class { + return ByteArray::class.java + } +} + +object ByteToHexSerializer : JsonSerializer() { + override fun serialize(value: Byte, gen: JsonGenerator, serializers: SerializerProvider?) { + gen.writeString("0x" + hexFormatter.toHexDigits(value)) + } +} + +object UByteToHexSerializer : JsonSerializer() { + override fun serialize(value: UByte, gen: JsonGenerator, serializers: SerializerProvider?) { + gen.writeString("0x" + hexFormatter.toHexDigits(value.toByte())) + } +} + +object ByteArrayToHexDeserializer : JsonDeserializer() { + override fun deserialize(parser: JsonParser, contex: DeserializationContext): ByteArray { + return hexFormatter.parseHex(parser.text.removePrefix("0x")) + } +} + +object ByteToHexDeserializer : JsonDeserializer() { + override fun deserialize(parser: JsonParser, contex: DeserializationContext): Byte { + return hexFormatter.parseHex(parser.text.removePrefix("0x"))[0] + } +} + +object UByteToHexDeserializer : JsonDeserializer() { + override fun deserialize(parser: JsonParser, contex: DeserializationContext): UByte { + return hexFormatter.parseHex(parser.text.removePrefix("0x"))[0].toUByte() + } +} diff --git a/jvm-libs/generic/serialization/jackson/src/main/kotlin/build/linea/s11n/jackson/InstantISO8601SerDe.kt b/jvm-libs/generic/serialization/jackson/src/main/kotlin/build/linea/s11n/jackson/InstantISO8601SerDe.kt new file mode 100644 index 000000000..3c1d671ec --- /dev/null +++ b/jvm-libs/generic/serialization/jackson/src/main/kotlin/build/linea/s11n/jackson/InstantISO8601SerDe.kt @@ -0,0 +1,22 @@ +package build.linea.s11n.jackson + +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.core.JsonParser +import com.fasterxml.jackson.databind.DeserializationContext +import com.fasterxml.jackson.databind.JsonDeserializer +import com.fasterxml.jackson.databind.JsonSerializer +import com.fasterxml.jackson.databind.SerializerProvider +import kotlinx.datetime.Instant + +object InstantISO8601Serializer : JsonSerializer() { + override fun serialize(value: Instant, gen: JsonGenerator, serializers: SerializerProvider) { + gen.writeString(value.toString()) + } +} + +// To uncomment and add the tests when necessary +object InstantISO8601Deserializer : JsonDeserializer() { + override fun deserialize(p: JsonParser, ctxt: DeserializationContext): Instant { + return Instant.parse(p.text) + } +} diff --git a/jvm-libs/generic/serialization/jackson/src/main/kotlin/build/linea/s11n/jackson/NumbersHexSerDe.kt b/jvm-libs/generic/serialization/jackson/src/main/kotlin/build/linea/s11n/jackson/NumbersHexSerDe.kt new file mode 100644 index 000000000..50496b5cd --- /dev/null +++ b/jvm-libs/generic/serialization/jackson/src/main/kotlin/build/linea/s11n/jackson/NumbersHexSerDe.kt @@ -0,0 +1,47 @@ +package build.linea.s11n.jackson + +import com.fasterxml.jackson.core.JsonGenerator +import com.fasterxml.jackson.databind.JsonSerializer +import com.fasterxml.jackson.databind.SerializerProvider +import java.math.BigInteger + +internal fun Number.toHex(): String = "0x" + BigInteger(toString()).toString(16) +internal fun ULong.toHex(): String = "0x" + BigInteger(toString()).toString(16) + +object IntToHexSerializer : JsonSerializer() { + override fun serialize(value: Int, gen: JsonGenerator, serializers: SerializerProvider) { + gen.writeString(value.toHex()) + } +} + +@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN") +object JIntegerToHexSerializer : JsonSerializer() { + override fun serialize(value: Integer, gen: JsonGenerator, serializers: SerializerProvider) { + gen.writeString(value.toHex()) + } +} + +object LongToHexSerializer : JsonSerializer() { + override fun serialize(value: Long, gen: JsonGenerator, serializers: SerializerProvider) { + gen.writeString(value.toHex()) + } +} + +@Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN") +object JLongToHexSerializer : JsonSerializer() { + override fun serialize(value: java.lang.Long, gen: JsonGenerator, serializers: SerializerProvider) { + gen.writeString(value.toHex()) + } +} + +object ULongToHexSerializer : JsonSerializer() { + override fun serialize(value: ULong, gen: JsonGenerator, serializers: SerializerProvider) { + gen.writeString(value.toHex()) + } +} + +object BigIntegerToHexSerializer : JsonSerializer() { + override fun serialize(value: BigInteger, gen: JsonGenerator, serializers: SerializerProvider) { + gen.writeString(value.toHex()) + } +} diff --git a/jvm-libs/generic/serialization/jackson/src/main/kotlin/build/linea/s11n/jackson/ObjectMappers.kt b/jvm-libs/generic/serialization/jackson/src/main/kotlin/build/linea/s11n/jackson/ObjectMappers.kt new file mode 100644 index 000000000..dcd093ab3 --- /dev/null +++ b/jvm-libs/generic/serialization/jackson/src/main/kotlin/build/linea/s11n/jackson/ObjectMappers.kt @@ -0,0 +1,36 @@ +package build.linea.s11n.jackson + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.module.SimpleModule +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import kotlinx.datetime.Instant +import java.math.BigInteger + +val ethNumberAsHexSerialisersModule = SimpleModule().apply { + this.addSerializer(Instant::class.java, InstantISO8601Serializer) + this.addDeserializer(Instant::class.java, InstantISO8601Deserializer) + this.addSerializer(Int::class.java, IntToHexSerializer) + this.addSerializer(Integer::class.java, JIntegerToHexSerializer) + this.addSerializer(Long::class.java, LongToHexSerializer) + this.addSerializer(java.lang.Long::class.java, JLongToHexSerializer) + this.addSerializer(ULong::class.java, ULongToHexSerializer) + this.addSerializer(BigInteger::class.java, BigIntegerToHexSerializer) +} + +val ethByteAsHexSerialisersModule = SimpleModule().apply { + this.addSerializer(Byte::class.java, ByteToHexSerializer) + this.addSerializer(UByte::class.java, UByteToHexSerializer) + this.addSerializer(ByteArray::class.java, ByteArrayToHexSerializer) +} + +val ethByteAsHexDeserialisersModule = SimpleModule().apply { + this.addDeserializer(Byte::class.java, ByteToHexDeserializer) + this.addDeserializer(UByte::class.java, UByteToHexDeserializer) + this.addDeserializer(ByteArray::class.java, ByteArrayToHexDeserializer) +} + +val ethApiObjectMapper: ObjectMapper = jacksonObjectMapper() + .registerModules( + ethNumberAsHexSerialisersModule, + ethByteAsHexSerialisersModule + ) diff --git a/jvm-libs/generic/serialization/jackson/src/test/kotlin/build/linea/s11n/jackson/BytesSerDeTest.kt b/jvm-libs/generic/serialization/jackson/src/test/kotlin/build/linea/s11n/jackson/BytesSerDeTest.kt new file mode 100644 index 000000000..cbafa2c3b --- /dev/null +++ b/jvm-libs/generic/serialization/jackson/src/test/kotlin/build/linea/s11n/jackson/BytesSerDeTest.kt @@ -0,0 +1,160 @@ +package build.linea.s11n.jackson + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import net.consensys.decodeHex +import net.consensys.encodeHex +import net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class BytesSerDeTest { + private lateinit var objectMapper: ObjectMapper + + private val jsonObj = """ + { + "nullBytes": null, + "emptyBytes": "0x", + "someBytes": "0x01aaff04", + "listOfByteArray": ["0x01aaff04", "0x01aaff05"], + "nullUByte": null, + "someUByte": "0xaf", + "minUByte": "0x00", + "maxUByte": "0xff", + "nullByte": null, + "someByte": "0xaf", + "minByte": "0x80", + "maxByte": "0x7f" + } + """.trimIndent() + private val objWithBytesFields = SomeObject( + // ByteArray + nullBytes = null, + emptyBytes = byteArrayOf(), + someBytes = "0x01aaff04".decodeHex(), + listOfByteArray = listOf("0x01aaff04", "0x01aaff05").map { it.decodeHex() }, + + // UByte + nullUByte = null, + someUByte = 0xaf.toUByte(), + minUByte = UByte.MIN_VALUE, + maxUByte = UByte.MAX_VALUE, + + // Byte + nullByte = null, + someByte = 0xaf.toByte(), + minByte = Byte.MIN_VALUE, + maxByte = Byte.MAX_VALUE + ) + + @BeforeEach + fun setUp() { + objectMapper = jacksonObjectMapper() + .registerModules(ethByteAsHexSerialisersModule) + .registerModules(ethByteAsHexDeserialisersModule) + } + + @Test + fun bytesSerDe() { + assertThatJson(objectMapper.writeValueAsString(objWithBytesFields)).isEqualTo(jsonObj) + assertThat(objectMapper.readValue(jsonObj)).isEqualTo(objWithBytesFields) + } + + @Test + fun testBytes() { + val list1 = listOf("0x01aaff04", "0x01aaff05").map { it.decodeHex() } + val list2 = listOf("0x01aaff04", "0x01aaff05", "0x01aaff06").map { it.decodeHex() } + list1.zip(list2).also { println(it) } + println(list1.zip(list2).all { (arr1, arr2) -> arr1.contentEquals(arr2) }) + + println(list1 == list2) + println(list1 != list2) + } + + private data class SomeObject( + // ByteArray + val nullBytes: ByteArray?, + val emptyBytes: ByteArray, + val someBytes: ByteArray, + val listOfByteArray: List, + + // UByte + val nullUByte: UByte?, + val someUByte: UByte, + val minUByte: UByte, + val maxUByte: UByte, + // Byte + val nullByte: Byte?, + val someByte: Byte, + val minByte: Byte, + val maxByte: Byte + ) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as SomeObject + + if (nullBytes != null) { + if (other.nullBytes == null) return false + if (!nullBytes.contentEquals(other.nullBytes)) return false + } else if (other.nullBytes != null) return false + if (!emptyBytes.contentEquals(other.emptyBytes)) return false + if (!someBytes.contentEquals(other.someBytes)) return false + if (!contentEquals(listOfByteArray, other.listOfByteArray)) return false + if (nullUByte != other.nullUByte) return false + if (someUByte != other.someUByte) return false + if (minUByte != other.minUByte) return false + if (maxUByte != other.maxUByte) return false + if (nullByte != other.nullByte) return false + if (someByte != other.someByte) return false + if (minByte != other.minByte) return false + if (maxByte != other.maxByte) return false + + return true + } + + override fun hashCode(): Int { + var result = nullBytes?.contentHashCode() ?: 0 + result = 31 * result + emptyBytes.contentHashCode() + result = 31 * result + someBytes.contentHashCode() + result = 31 * result + listOfByteArray.hashCode() + result = 31 * result + (nullUByte?.hashCode() ?: 0) + result = 31 * result + someUByte.hashCode() + result = 31 * result + minUByte.hashCode() + result = 31 * result + maxUByte.hashCode() + result = 31 * result + (nullByte ?: 0) + result = 31 * result + someByte + result = 31 * result + minByte + result = 31 * result + maxByte + return result + } + + override fun toString(): String { + return "SomeObject(" + + "nullBytes=${nullBytes?.contentToString()}, " + + "emptyBytes=${emptyBytes.contentToString()}, " + + "someByte=${someBytes.contentToString()}, " + + "listOfByteArray=${listOfByteArray.joinToString(",", "[", "]") { it.encodeHex() }}, " + + "nullUByte=$nullUByte, " + + "someUByte=$someUByte, " + + "minUByte=$minUByte, " + + "maxUByte=$maxUByte, " + + "nullByte=$nullByte, " + + "someByte=$someByte, " + + "minByte=$minByte, " + + "maxByte=$maxByte" + + ")" + } + } + + companion object { + fun contentEquals(list1: List, list2: List): Boolean { + if (list1.size != list2.size) return false + + return list1.zip(list2).all { (arr1, arr2) -> arr1.contentEquals(arr2) } + } + } +} diff --git a/jvm-libs/generic/serialization/jackson/src/test/kotlin/build/linea/s11n/jackson/InstantISO8601SerDeTest.kt b/jvm-libs/generic/serialization/jackson/src/test/kotlin/build/linea/s11n/jackson/InstantISO8601SerDeTest.kt new file mode 100644 index 000000000..21e5d732a --- /dev/null +++ b/jvm-libs/generic/serialization/jackson/src/test/kotlin/build/linea/s11n/jackson/InstantISO8601SerDeTest.kt @@ -0,0 +1,65 @@ +package build.linea.s11n.jackson + +import com.fasterxml.jackson.databind.ObjectMapper +import kotlinx.datetime.Instant +import net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class InstantISO8601SerDeTest { + private lateinit var objectMapper: ObjectMapper + + @BeforeEach + fun setUp() { + objectMapper = ethApiObjectMapper + } + + @Test + fun instantSerialization() { + data class SomeObject( + // Int + val instantNull: Instant? = null, + val instantUTC: Instant = Instant.parse("2021-01-02T09:00:45Z"), + val instantUTCPlus: Instant = Instant.parse("2021-01-02T09:00:45+01:30"), + val instantUTCMinus: Instant = Instant.parse("2021-01-02T09:00:45-01:30") + ) + + val json = objectMapper.writeValueAsString(SomeObject()) + assertThatJson(json).isEqualTo( + """ + { + "instantNull": null, + "instantUTC": "2021-01-02T09:00:45Z", + "instantUTCPlus": "2021-01-02T07:30:45Z", + "instantUTCMinus": "2021-01-02T10:30:45Z" + } + """.trimIndent() + ) + } + + @Test + fun instantSerDeDeSerialization() { + data class SomeObject( + // Int + val instantNull: Instant? = null, + val instantUTC: Instant = Instant.parse("2021-01-02T09:00:45Z"), + val instantUTCPlus: Instant = Instant.parse("2021-01-02T09:00:45+01:30"), + val instantUTCMinus: Instant = Instant.parse("2021-01-02T09:00:45-01:30") + ) + + val expectedJson = """ + { + "instantNull": null, + "instantUTC": "2021-01-02T09:00:45Z", + "instantUTCPlus": "2021-01-02T07:30:45Z", + "instantUTCMinus": "2021-01-02T10:30:45Z" + } + """.trimIndent() + + // assert serialization + assertThatJson(objectMapper.writeValueAsString(SomeObject())).isEqualTo(expectedJson) + + // assert deserialization + assertThatJson(objectMapper.readValue(expectedJson, SomeObject::class.java)).isEqualTo(SomeObject()) + } +} diff --git a/jvm-libs/generic/serialization/jackson/src/test/kotlin/build/linea/s11n/jackson/NumbersSerDeTest.kt b/jvm-libs/generic/serialization/jackson/src/test/kotlin/build/linea/s11n/jackson/NumbersSerDeTest.kt new file mode 100644 index 000000000..bba6e82dd --- /dev/null +++ b/jvm-libs/generic/serialization/jackson/src/test/kotlin/build/linea/s11n/jackson/NumbersSerDeTest.kt @@ -0,0 +1,80 @@ +package build.linea.s11n.jackson + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.databind.ObjectMapper +import net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.math.BigInteger + +class NumbersSerDeTest { + private lateinit var objectMapper: ObjectMapper + + @BeforeEach + fun setUp() { + objectMapper = ethApiObjectMapper + } + + @Test + fun numbersSerialization() { + data class SomeObject( + // Int + val intNull: Int? = null, + val intZero: Int = 0, + val intMaxValue: Int = Int.MAX_VALUE, + val intSomeValue: Int = 0xff00aa, + + // Long + val longNull: Long? = null, + val longZero: Long = 0, + val longMaxValue: Long = Long.MAX_VALUE, + val longSomeValue: Long = 0xff00aa, + + // ULong + // jackson has a bug and serializes any cAmelCase as camelCase, need to set it with @JsonProperty + // it's only on 1st character, so we can't use uLongNull, uLongZero, without annotation etc. + @get:JsonProperty("uLongNull") val uLongNull: ULong? = null, + @get:JsonProperty("uLongZero") val uLongZero: ULong = 0uL, + @get:JsonProperty("uLongMaxValue") val uLongMaxValue: ULong = ULong.MAX_VALUE, + @get:JsonProperty("uLongSomeValue") val uLongSomeValue: ULong = 0xff00aa.toULong(), + + // BigInteger + val bigIntegerNull: BigInteger? = null, + val bigIntegerZero: BigInteger = BigInteger.ZERO, + val bigIntegerSomeValue: BigInteger = BigInteger.valueOf(0xff00aaL), + + // nested Structures + val listOfInts: List = listOf(1, 10), + val listOfLongs: List = listOf(1, 10), + val listOfULongs: List = listOf(1UL, 10UL), + val listOfBigIntegers: List = listOf(1L, 10L).map(BigInteger::valueOf) + ) + + val json = objectMapper.writeValueAsString(SomeObject()) + assertThatJson(json).isEqualTo( + """ + { + "intNull": null, + "intZero": "0x0", + "intMaxValue": "0x7fffffff", + "intSomeValue": "0xff00aa", + "longNull": null, + "longZero": "0x0", + "longMaxValue": "0x7fffffffffffffff", + "longSomeValue": "0xff00aa", + "uLongNull": null, + "uLongZero": "0x0", + "uLongMaxValue": "0xffffffffffffffff", + "uLongSomeValue": "0xff00aa", + "bigIntegerNull": null, + "bigIntegerZero": "0x0", + "bigIntegerSomeValue": "0xff00aa", + "listOfInts": ["0x1", "0xa"], + "listOfLongs": ["0x1", "0xa"], + "listOfULongs": ["0x1", "0xa"], + "listOfBigIntegers": ["0x1", "0xa"] + } + """.trimIndent() + ) + } +} diff --git a/prover/protocol/compiler/innerproduct/compiler.go b/prover/protocol/compiler/innerproduct/compiler.go index 2e5291d52..bfe76159f 100644 --- a/prover/protocol/compiler/innerproduct/compiler.go +++ b/prover/protocol/compiler/innerproduct/compiler.go @@ -31,9 +31,11 @@ func Compile(comp *wizard.CompiledIOP) { // same protocol if the same step of compilation are applied in the same // order. sizes = []int{} - // contextsForSize list all the sub-compilation context in the same - // order as `sizes` - proverTask proverTask + // contextsForSize list all the sub-compilation context + // in the same order as `sizes`. + // proverTaskCollaps indicates when we have more than one pair of inner-product with the same size + // and thus collapsing all pairs to a single column is required. + proverTaskNoCollaps, proverTaskCollpas proverTask ) for _, qName := range comp.QueriesParams.AllUnignoredKeys() { @@ -60,9 +62,24 @@ func Compile(comp *wizard.CompiledIOP) { } for _, size := range sizes { - proverTask = append(proverTask, compileForSize(comp, round, queryMap[size])) + ctx := compileForSize(comp, round, queryMap[size]) + switch ctx.round { + case round: + proverTaskNoCollaps = append(proverTaskNoCollaps, ctx) + case round + 1: + proverTaskCollpas = append(proverTaskCollpas, ctx) + default: + utils.Panic("round before compilation was %v and after compilation %v", round, ctx.round) + } + + } + // run the prover of the relevant round + if len(proverTaskNoCollaps) >= 1 { + comp.RegisterProverAction(round, proverTaskNoCollaps) } - comp.RegisterProverAction(round+1, proverTask) + if len(proverTaskCollpas) >= 1 { + comp.RegisterProverAction(round+1, proverTaskCollpas) + } } diff --git a/prover/protocol/compiler/innerproduct/context.go b/prover/protocol/compiler/innerproduct/context.go index 789b1d82f..14eb78737 100644 --- a/prover/protocol/compiler/innerproduct/context.go +++ b/prover/protocol/compiler/innerproduct/context.go @@ -34,6 +34,9 @@ type contextForSize struct { // entry of [Summation]. It is compared to the alleged inner-product values // by the verifier to finalize the compilation step.s SummationOpening query.LocalOpening + + // round after compilation + round int } // compileForSize applies the compilation step on a range of queries such that @@ -41,6 +44,7 @@ type contextForSize struct { // list of queries. // // It returns the compilation context of the query +// the round indicate the round of the last inner-product query, independent of its size. func compileForSize( comp *wizard.CompiledIOP, round int, @@ -60,10 +64,12 @@ func compileForSize( if hasMoreThan1Pair { round = round + 1 } + //set the round + ctx.round = round ctx.Summation = comp.InsertCommit( - round+1, - deriveName[ifaces.ColID]("SUMMATION", comp.SelfRecursionCount), + round, + deriveName[ifaces.ColID]("SUMMATION", size, comp.SelfRecursionCount), size, ) @@ -74,8 +80,8 @@ func compileForSize( ) batchingCoin = comp.InsertCoin( - round+1, - deriveName[coin.Name]("BATCHING_COIN", comp.SelfRecursionCount), + round, + deriveName[coin.Name]("BATCHING_COIN", size, comp.SelfRecursionCount), coin.Field, ) @@ -85,8 +91,16 @@ func compileForSize( } } - ctx.Collapsed = symbolic.NewPolyEval(batchingCoin.AsVariable(), pairProduct) - ctx.Collapsed.Board() + // @Azam the following function is commented out due to the issue https://github.com/Consensys/linea-monorepo/issues/192 + // ctx.Collapsed = symbolic.NewPolyEval(batchingCoin.AsVariable(), pairProduct) + res := symbolic.NewConstant(0) + for i := len(pairProduct) - 1; i >= 0; i-- { + res = symbolic.Mul(res, batchingCoin) + res = symbolic.Add(res, pairProduct[i]) + } + + ctx.Collapsed = res + ctx.CollapsedBoard = ctx.Collapsed.Board() } if !hasMoreThan1Pair { @@ -96,8 +110,8 @@ func compileForSize( // This constraints set the recurrent property of summation comp.InsertGlobal( - round+1, - deriveName[ifaces.QueryID]("SUMMATION_CONSISTENCY", comp.SelfRecursionCount), + round, + deriveName[ifaces.QueryID]("SUMMATION_CONSISTENCY", size, comp.SelfRecursionCount), symbolic.Sub( ctx.Summation, column.Shift(ctx.Summation, -1), @@ -107,20 +121,21 @@ func compileForSize( // This constraint ensures that summation has the correct initial value comp.InsertLocal( - round+1, - deriveName[ifaces.QueryID]("SUMMATION_INIT", comp.SelfRecursionCount), + round, + deriveName[ifaces.QueryID]("SUMMATION_INIT", size, comp.SelfRecursionCount), symbolic.Sub(ctx.Collapsed, ctx.Summation), ) // The opening of the final position of ctx.Summation should be equal to // the linear combinations of the alleged openings of the inner-products. ctx.SummationOpening = comp.InsertLocalOpening( - round+1, - deriveName[ifaces.QueryID]("SUMMATION_END", comp.SelfRecursionCount), + round, + deriveName[ifaces.QueryID]("SUMMATION_END", size, comp.SelfRecursionCount), column.Shift(ctx.Summation, -1), ) - comp.RegisterVerifierAction(round+1, &verifierForSize{ + lastRound := comp.NumRounds() - 1 + comp.RegisterVerifierAction(lastRound, &verifierForSize{ Queries: queries, SummationOpening: ctx.SummationOpening, BatchOpening: batchingCoin, diff --git a/prover/protocol/compiler/innerproduct/innerproduct_test.go b/prover/protocol/compiler/innerproduct/innerproduct_test.go new file mode 100644 index 000000000..0a9afb899 --- /dev/null +++ b/prover/protocol/compiler/innerproduct/innerproduct_test.go @@ -0,0 +1,107 @@ +package innerproduct + +import ( + "testing" + + "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" + "github.com/consensys/linea-monorepo/prover/maths/field" + "github.com/consensys/linea-monorepo/prover/protocol/coin" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/dummy" + "github.com/consensys/linea-monorepo/prover/protocol/ifaces" + "github.com/consensys/linea-monorepo/prover/protocol/wizard" + "github.com/stretchr/testify/assert" +) + +func TestInnerProduct(t *testing.T) { + define := func(b *wizard.Builder) { + for i, c := range testCases { + bs := make([]ifaces.Column, len(c.bName)) + a := b.RegisterCommit(c.aName, c.size) + for i, name := range c.bName { + bs[i] = b.RegisterCommit(name, c.size) + } + b.InnerProduct(c.qName, a, bs...) + // go to the next round + _ = b.RegisterRandomCoin(coin.Namef("Coin_%v", i), coin.Field) + } + } + prover := func(run *wizard.ProverRuntime) { + for j, c := range testCases { + run.AssignColumn(c.aName, c.a) + for i, name := range c.bName { + run.AssignColumn(name, c.b[i]) + } + run.AssignInnerProduct(c.qName, c.expected...) + run.GetRandomCoinField(coin.Namef("Coin_%v", j)) + } + } + + comp := wizard.Compile(define, Compile, dummy.Compile) + proof := wizard.Prove(comp, prover) + assert.NoErrorf(t, wizard.Verify(comp, proof), "invalid proof") +} + +var testCases = []struct { + qName ifaces.QueryID + aName ifaces.ColID + bName []ifaces.ColID + size int + a smartvectors.SmartVector + b []smartvectors.SmartVector + expected []field.Element +}{ + {qName: "Quey1", + aName: "ColA1", + bName: []ifaces.ColID{"ColB1"}, + size: 4, + a: smartvectors.ForTest(1, 1, 1, 1), + b: []smartvectors.SmartVector{ + smartvectors.ForTest(0, 3, 0, 2), + }, + expected: []field.Element{field.NewElement(5)}, + }, + {qName: "Quey2", + aName: "ColA2", + bName: []ifaces.ColID{"ColB2_0", "ColB2_1"}, + size: 4, + a: smartvectors.ForTest(1, 1, 1, 1), + b: []smartvectors.SmartVector{ + smartvectors.ForTest(0, 3, 0, 2), + smartvectors.ForTest(1, 0, 0, 2), + }, + expected: []field.Element{field.NewElement(5), field.NewElement(3)}, + }, + {qName: "Quey3", + aName: "ColA3", + bName: []ifaces.ColID{"ColB3_0", "ColB3_1"}, + size: 8, + a: smartvectors.ForTest(1, 1, 1, 1, 2, 0, 2, 0), + b: []smartvectors.SmartVector{ + smartvectors.ForTest(0, 3, 0, 2, 1, 0, 0, 0), + smartvectors.ForTest(1, 0, 0, 2, 1, 0, 0, 0), + }, + expected: []field.Element{field.NewElement(7), field.NewElement(5)}, + }, + {qName: "Quey4", + aName: "ColA4", + bName: []ifaces.ColID{"ColB4"}, + size: 16, + a: smartvectors.ForTest(1, 1, 1, 1, 2, 0, 2, 0, 1, 1, 1, 1, 1, 1, 1, 1), + b: []smartvectors.SmartVector{ + smartvectors.ForTest(0, 3, 0, 2, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1), + }, + expected: []field.Element{field.NewElement(15)}, + }, + + {qName: "Quey", + + aName: "ColA", + bName: []ifaces.ColID{"ColB"}, + size: 32, + a: smartvectors.ForTest(1, 1, 1, 1, 2, 0, 2, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 0, 2, 0, 1, 1, 1, 1, 1, 1, 1, 1), + b: []smartvectors.SmartVector{ + smartvectors.ForTest(0, 3, 0, 2, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 3, 0, 2, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1), + }, + expected: []field.Element{field.NewElement(30)}, + }, +} diff --git a/prover/protocol/compiler/lookup/compiler.go b/prover/protocol/compiler/lookup/compiler.go index 2bb571bc8..6b4f8fea6 100644 --- a/prover/protocol/compiler/lookup/compiler.go +++ b/prover/protocol/compiler/lookup/compiler.go @@ -28,7 +28,7 @@ func CompileLogDerivative(comp *wizard.CompiledIOP) { var ( mainLookupCtx = captureLookupTables(comp) - lastRound = comp.NumRounds() + lastRound = comp.NumRounds() - 1 proverActions = make([]proverTaskAtRound, comp.NumRounds()+1) // zCatalog stores a mapping (round, size) into ZCtx and helps finding // which Z context should be used to handle a part of a given permutation @@ -101,9 +101,11 @@ func CompileLogDerivative(comp *wizard.CompiledIOP) { // z-packing compile zC.compile(comp) // entry[0]:round, entry[1]: size + // the round that Gamma was registered. round := entry[0] - proverActions[round+1].pushZAssignment(zAssignmentTask(*zC)) + proverActions[round].pushZAssignment(zAssignmentTask(*zC)) va.ZOpenings = append(va.ZOpenings, zC.ZOpenings...) + va.Name = zC.Name } for round := range proverActions { @@ -209,6 +211,8 @@ func captureLookupTables(comp *wizard.CompiledIOP) mainLookupCtx { // - (3) The verifier makes a `Local` query : $(\Sigma_T)[0] = \frac{M_0}{T_0 + \gamma}$ // - (4) **(For all k)** The verifier makes a `Global` query : $\left((\Sigma_{S,k})[i] - (\Sigma_{S,k})[i-1]\right)(S_{k,i} + \gamma) = 1$ // - (5) The verier makes a `Global` query : $\left((\Sigma_T)[i] - (\Sigma_T)[i-1]\right)(T_i + \gamma) = M_i$ + +// here we are looking up set of columns S in a single column T func compileLookupTable( comp *wizard.CompiledIOP, round int, @@ -228,7 +232,7 @@ func compileLookupTable( var ( // isMultiColumn indicates whether the lookup table (and thus the // checked tables) have the same number of - isMultiColumn = len(lookupTable) > 1 + isMultiColumn = len(lookupTable[0]) > 1 ) if !isMultiColumn { @@ -285,7 +289,7 @@ func compileLookupTable( func (stc *singleTableCtx) pushToZCatalog(zCatalog map[[2]int]*zCtx) { var ( - round = stc.M[0].Round() + round = stc.Gamma.Round ) // tableCtx push to -> zCtx @@ -298,6 +302,7 @@ func (stc *singleTableCtx) pushToZCatalog(zCatalog map[[2]int]*zCtx) { zCatalog[key] = &zCtx{ Size: size, Round: round, + Name: stc.TableName, } } @@ -307,14 +312,14 @@ func (stc *singleTableCtx) pushToZCatalog(zCatalog map[[2]int]*zCtx) { } // Process the S columns - for frag := range stc.S { + for table := range stc.S { var ( - _, _, size = wizardutils.AsExpr(stc.S[frag]) + _, _, size = wizardutils.AsExpr(stc.S[table]) sFilter = symbolic.NewConstant(1) ) - if stc.SFilters[frag] != nil { - sFilter = symbolic.NewVariable(stc.SFilters[frag]) + if stc.SFilters[table] != nil { + sFilter = symbolic.NewVariable(stc.SFilters[table]) } key := [2]int{round, size} @@ -322,11 +327,12 @@ func (stc *singleTableCtx) pushToZCatalog(zCatalog map[[2]int]*zCtx) { zCatalog[key] = &zCtx{ Size: size, Round: round, + Name: stc.TableName, } } zCtxEntry := zCatalog[key] zCtxEntry.SigmaNumerator = append(zCtxEntry.SigmaNumerator, sFilter) - zCtxEntry.SigmaDenominator = append(zCtxEntry.SigmaDenominator, symbolic.Add(stc.Gamma, stc.S[frag])) + zCtxEntry.SigmaDenominator = append(zCtxEntry.SigmaDenominator, symbolic.Add(stc.Gamma, stc.S[table])) } } diff --git a/prover/protocol/compiler/lookup/compiler_test.go b/prover/protocol/compiler/lookup/compiler_test.go index 638329308..867836b59 100644 --- a/prover/protocol/compiler/lookup/compiler_test.go +++ b/prover/protocol/compiler/lookup/compiler_test.go @@ -4,6 +4,8 @@ import ( "testing" "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" + "github.com/consensys/linea-monorepo/prover/maths/field" + "github.com/consensys/linea-monorepo/prover/protocol/coin" "github.com/consensys/linea-monorepo/prover/protocol/compiler/dummy" "github.com/consensys/linea-monorepo/prover/protocol/ifaces" "github.com/consensys/linea-monorepo/prover/protocol/wizard" @@ -78,9 +80,11 @@ func TestLogDerivativeLookupManyChecksOneTable(t *testing.T) { define := func(b *wizard.Builder) { cola := b.RegisterCommit("S", sizeA) cola2 := b.RegisterCommit("S2", sizeA) + cola3 := b.RegisterCommit("S3", sizeA) colb := b.RegisterCommit("T", sizeB) b.Inclusion("LOOKUP", []ifaces.Column{colb}, []ifaces.Column{cola}) b.Inclusion("LOOKUP2", []ifaces.Column{colb}, []ifaces.Column{cola2}) + b.Inclusion("LOOKUP3", []ifaces.Column{colb}, []ifaces.Column{cola3}) } prover := func(run *wizard.ProverRuntime) { @@ -88,10 +92,12 @@ func TestLogDerivativeLookupManyChecksOneTable(t *testing.T) { // assign a and b cola := smartvectors.ForTest(1, 1, 1, 2, 3, 0, 0, 1, 1, 1, 1, 2, 3, 0, 0, 1) cola2 := smartvectors.ForTest(2, 2, 2, 1, 0, 3, 3, 2, 2, 2, 2, 1, 0, 3, 3, 2) + cola3 := smartvectors.ForTest(2, 2, 2, 1, 0, 3, 3, 2, 2, 2, 2, 1, 0, 3, 3, 3) colb := smartvectors.ForTest(0, 1, 2, 3) // m expected to be run.AssignColumn("S", cola) run.AssignColumn("S2", cola2) + run.AssignColumn("S3", cola3) run.AssignColumn("T", colb) } @@ -99,7 +105,7 @@ func TestLogDerivativeLookupManyChecksOneTable(t *testing.T) { proof := wizard.Prove(comp, prover) // m should be - expectedM := smartvectors.ForTest(6, 10, 10, 6) + expectedM := smartvectors.ForTest(8, 12, 17, 11) t.Logf("all columns = %v", runtime.Columns.ListAllKeys()) actualM := runtime.GetColumn("TABLE_T_0_LOGDERIVATIVE_M") @@ -155,7 +161,7 @@ func TestLogDerivativeLookupOneXor(t *testing.T) { // m should be expectedM := smartvectors.ForTest(0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0) t.Logf("all columns = %v", runtime.Columns.ListAllKeys()) - actualM := runtime.GetColumn("TABLE_XOR_TABLE_X_XOR_TABLE_XXORY_XOR_TABLE_Y_0_LOGDERIVATIVE_M") + actualM := runtime.GetColumn("TABLE_XOR_TABLE_X,XOR_TABLE_XXORY,XOR_TABLE_Y_0_LOGDERIVATIVE_M") assert.Equal(t, expectedM.Pretty(), actualM.Pretty(), "m does not match the expected value") @@ -222,7 +228,7 @@ func TestLogDerivativeLookupMultiXor(t *testing.T) { // m should be expectedM := smartvectors.ForTest(1, 1, 1, 2, 2, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0) t.Logf("all column names = %v", runtime.Columns.ListAllKeys()) - actualM := runtime.GetColumn("TABLE_XOR_TABLE_X_XOR_TABLE_XXORY_XOR_TABLE_Y_0_LOGDERIVATIVE_M") + actualM := runtime.GetColumn("TABLE_XOR_TABLE_X,XOR_TABLE_XXORY,XOR_TABLE_Y_0_LOGDERIVATIVE_M") assert.Equal(t, expectedM.Pretty(), actualM.Pretty(), "m does not match the expected value") @@ -230,6 +236,58 @@ func TestLogDerivativeLookupMultiXor(t *testing.T) { require.NoError(t, err) } +func TestLogDerivativeLookupRandomLinComb(t *testing.T) { + + var sizeA, sizeB int = 16, 8 + var col1, col2 ifaces.Column + define := func(b *wizard.Builder) { + col1 = b.RegisterPrecomputed("P1", smartvectors.ForTest(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)) + col2 = b.RegisterPrecomputed("P2", smartvectors.ForTest(12, 6, 8, 0, 3, 12, 13, 23, 17, 9, 8, 7, 6, 5, 4, 3)) + colI := b.RegisterPrecomputed("I", smartvectors.ForTest(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)) + + _ = b.RegisterRandomCoin("COIN", coin.Field) + + uCol := b.InsertProof(1, "LC", sizeA) + + _ = b.RegisterRandomCoin("COIN1", coin.Field) + + colQ := b.RegisterCommit("Q", sizeB) + uChosen := b.RegisterCommit("UChosen", sizeB) + + // multi-col query + b.Inclusion("LOOKUP", []ifaces.Column{colI, uCol}, []ifaces.Column{colQ, uChosen}) + } + + prover := func(run *wizard.ProverRuntime) { + // assign a and b + + coin := run.GetRandomCoinField("COIN") + + a := col1.GetColAssignment(run) + b := col2.GetColAssignment(run) + lc := smartvectors.PolyEval([]smartvectors.SmartVector{a, b}, coin) + + run.AssignColumn("LC", lc) + + run.GetRandomCoinField("COIN1") + + colQ := smartvectors.ForTest(0, 1, 2, 3, 4, 5, 6, 7) + run.AssignColumn("Q", colQ) + + colQFr := colQ.IntoRegVecSaveAlloc() + var t []field.Element + for _, q := range colQFr { + t = append(t, lc.Get(int(q.Uint64()))) + } + run.AssignColumn("UChosen", smartvectors.NewRegular(t)) + } + + comp := wizard.Compile(define, CompileLogDerivative, dummy.Compile) + proof := wizard.Prove(comp, prover) + err := wizard.Verify(comp, proof) + require.NoError(t, err) +} + func BenchmarkLogDeriveLookupMultiXor(b *testing.B) { for i := 0; i < b.N; i++ { diff --git a/prover/protocol/compiler/lookup/conditional_test.go b/prover/protocol/compiler/lookup/conditional_test.go index d3b02a6dc..d20894309 100644 --- a/prover/protocol/compiler/lookup/conditional_test.go +++ b/prover/protocol/compiler/lookup/conditional_test.go @@ -119,7 +119,7 @@ func TestConditionalLogDerivativeLookupSimple2(t *testing.T) { // in the last appearance of 0 in filteredT expectedM := smartvectors.ForTest(1, 2, 0, 4) t.Logf("the list of columns is: %v", runtime.Columns.ListAllKeys()) - actualM := runtime.GetColumn("TABLE_filterB_T_0_LOGDERIVATIVE_M") + actualM := runtime.GetColumn("TABLE_filterB,T_0_LOGDERIVATIVE_M") assert.Equal(t, expectedM.Pretty(), actualM.Pretty(), "m does not match the expected value") @@ -172,7 +172,7 @@ func TestConditionalLogDerivativeLookupManyChecksOneTable(t *testing.T) { // m should be expectedM := smartvectors.ForTest(0, 4, 4, 3) t.Logf("the list of columns is: %v", runtime.Columns.ListAllKeys()) - actualM := runtime.GetColumn("TABLE_filterT_T_0_LOGDERIVATIVE_M") + actualM := runtime.GetColumn("TABLE_filterT,T_0_LOGDERIVATIVE_M") assert.Equal(t, expectedM.Pretty(), actualM.Pretty(), "m does not match the expected value") @@ -238,7 +238,7 @@ func TestConditionalLogDerivativeLookupOneXor(t *testing.T) { expectedM := smartvectors.ForTest(0, 1, 1, 1, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0) t.Logf("the list of columns is: %v", runtime.Columns.ListAllKeys()) - actualM := runtime.GetColumn("TABLE_filterT_XOR_TABLE_X_XOR_TABLE_XXORY_XOR_TABLE_Y_0_LOGDERIVATIVE_M") + actualM := runtime.GetColumn("TABLE_filterT,XOR_TABLE_X,XOR_TABLE_XXORY,XOR_TABLE_Y_0_LOGDERIVATIVE_M") assert.Equal(t, expectedM.Pretty(), actualM.Pretty(), "m does not match the expected value") @@ -318,7 +318,7 @@ func TestConditionalLogDerivativeLookupMultiXor(t *testing.T) { // m should be expectedM := smartvectors.ForTest(1, 1, 1, 2, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0) t.Logf("the list of columns is: %v", runtime.Columns.ListAllKeys()) - actualM := runtime.GetColumn("TABLE_FILTER_T_XOR_TABLE_X_XOR_TABLE_XXORY_XOR_TABLE_Y_0_LOGDERIVATIVE_M") + actualM := runtime.GetColumn("TABLE_FILTER_T,XOR_TABLE_X,XOR_TABLE_XXORY,XOR_TABLE_Y_0_LOGDERIVATIVE_M") assert.Equal(t, expectedM.Pretty(), actualM.Pretty(), "m does not match the expected value") @@ -432,7 +432,7 @@ func TestMixedConditionalLogDerivativeLookupMultiXor(t *testing.T) { // m should be expectedM := smartvectors.ForTest(2, 2, 1, 2, 2, 0, 1, 3, 0, 0, 0, 0, 2, 1, 1, 0) // 16 rows are included t.Logf("the list of columns is: %v", runtime.Columns.ListAllKeys()) - actualM := runtime.GetColumn("TABLE_FILTER_T_XOR_TABLE_X_XOR_TABLE_XXORY_XOR_TABLE_Y_0_LOGDERIVATIVE_M") + actualM := runtime.GetColumn("TABLE_FILTER_T,XOR_TABLE_X,XOR_TABLE_XXORY,XOR_TABLE_Y_0_LOGDERIVATIVE_M") assert.Equal(t, expectedM.Pretty(), actualM.Pretty(), "m does not match the expected value") diff --git a/prover/protocol/compiler/lookup/prover.go b/prover/protocol/compiler/lookup/prover.go index 6d7807c06..063fa5a6b 100644 --- a/prover/protocol/compiler/lookup/prover.go +++ b/prover/protocol/compiler/lookup/prover.go @@ -171,7 +171,7 @@ func (a mAssignmentTask) run(run *wizard.ProverRuntime) { // otherwise. tCollapsed = make([]sv.SmartVector, len(a.T)) - // tCollapsed contains either the assignment of the Ss if the table is a + // sCollapsed contains either the assignment of the Ss if the table is a // single column (e.g. isMultiColumn=false) or their collapsed version // otherwise. sCollapsed = make([]sv.SmartVector, len(a.S)) @@ -211,6 +211,7 @@ func (a mAssignmentTask) run(run *wizard.ProverRuntime) { } var ( + // m is associated with tCollapsed // m stores the assignment to the column M as we build it. m = make([][]field.Element, len(a.T)) diff --git a/prover/protocol/compiler/lookup/utils.go b/prover/protocol/compiler/lookup/utils.go index b569c04c1..a6d4eeb27 100644 --- a/prover/protocol/compiler/lookup/utils.go +++ b/prover/protocol/compiler/lookup/utils.go @@ -102,7 +102,7 @@ func nameTable(t []table) string { for col := range t[0] { colNames[col] = string(t[0][col].GetColID()) } - return fmt.Sprintf("TABLE_%v", strings.Join(colNames, "_")) + return fmt.Sprintf("TABLE_%v", strings.Join(colNames, ",")) } fragNames := make([]string, len(t)) diff --git a/prover/protocol/compiler/lookup/verifier.go b/prover/protocol/compiler/lookup/verifier.go index ceb0d31f3..b5179574b 100644 --- a/prover/protocol/compiler/lookup/verifier.go +++ b/prover/protocol/compiler/lookup/verifier.go @@ -17,9 +17,7 @@ import ( // // The current implementation is for packed Zs type finalEvaluationCheck struct { - // Name is a string repr of the verifier action. It is used to format errors - // so that we can easily know which verifier action is at fault during the - // verification is at fault. + // the name of a lookupTable in the pack, this can help for debugging. Name string // ZOpenings lists all the openings of all the zCtx ZOpenings []query.LocalOpening @@ -37,7 +35,7 @@ func (f *finalEvaluationCheck) Run(run *wizard.VerifierRuntime) error { } if zSum != field.Zero() { - return fmt.Errorf("log-derivate lookup, the final evaluation check failed") + return fmt.Errorf("log-derivate lookup, the final evaluation check failed for %v,", f.Name) } return nil diff --git a/prover/protocol/compiler/lookup/z_packing.go b/prover/protocol/compiler/lookup/z_packing.go index 1be03c51c..d6d12ffbe 100644 --- a/prover/protocol/compiler/lookup/z_packing.go +++ b/prover/protocol/compiler/lookup/z_packing.go @@ -36,6 +36,7 @@ type zCtx struct { Zs []ifaces.Column // ZOpenings are the opening queries to the end of each Z. ZOpenings []query.LocalOpening + Name string } // check permutation and see how/where compile is called (see how to constracut z there) @@ -80,32 +81,32 @@ func (z *zCtx) compile(comp *wizard.CompiledIOP) { z.ZDenominatorBoarded[i] = zDenominator.Board() z.Zs[i] = comp.InsertCommit( - z.Round+1, + z.Round, deriveName[ifaces.ColID]("Z", comp.SelfRecursionCount, z.Round, z.Size, i), z.Size, ) - // consistency check - comp.InsertGlobal( - z.Round+1, - deriveName[ifaces.QueryID]("Z_CONSISTENCY", comp.SelfRecursionCount, z.Round, z.Size, i), + // initial condition + comp.InsertLocal( + z.Round, + deriveName[ifaces.QueryID]("Z_CONSISTENCY_START", comp.SelfRecursionCount, z.Round, z.Size, i), sym.Sub( zNumerator, sym.Mul( - sym.Sub(z.Zs[i], column.Shift(z.Zs[i], -1)), + z.Zs[i], zDenominator, ), ), ) - // complete the consistency by adding the edge-case at position 0 - comp.InsertLocal( - z.Round+1, - deriveName[ifaces.QueryID]("Z_CONSISTENCY_START", comp.SelfRecursionCount, z.Round, z.Size, i), + // consistency check + comp.InsertGlobal( + z.Round, + deriveName[ifaces.QueryID]("Z_CONSISTENCY", comp.SelfRecursionCount, z.Round, z.Size, i), sym.Sub( zNumerator, sym.Mul( - z.Zs[i], + sym.Sub(z.Zs[i], column.Shift(z.Zs[i], -1)), zDenominator, ), ), @@ -113,7 +114,7 @@ func (z *zCtx) compile(comp *wizard.CompiledIOP) { // local opening of the final value of the Z polynomial z.ZOpenings[i] = comp.InsertLocalOpening( - z.Round+1, + z.Round, deriveName[ifaces.QueryID]("Z_FINAL", comp.SelfRecursionCount, z.Round, z.Size, i), column.Shift(z.Zs[i], -1), ) diff --git a/prover/protocol/compiler/selfrecursion/selfrecursion_test.go b/prover/protocol/compiler/selfrecursion/selfrecursion_test.go index 21fe71019..de57faab5 100644 --- a/prover/protocol/compiler/selfrecursion/selfrecursion_test.go +++ b/prover/protocol/compiler/selfrecursion/selfrecursion_test.go @@ -227,7 +227,7 @@ func TestSelfRecursionMultiLayered(t *testing.T) { define, vortex.Compile( 2, - vortex.ForceNumOpenedColumns(16), + vortex.ForceNumOpenedColumns(tc.NumOpenCol), vortex.WithSISParams(&tc.SisInstance), ), selfrecursion.SelfRecurse, @@ -235,7 +235,7 @@ func TestSelfRecursionMultiLayered(t *testing.T) { compiler.Arcane(1<<8, 1<<10, false), vortex.Compile( 2, - vortex.ForceNumOpenedColumns(16), + vortex.ForceNumOpenedColumns(tc.NumOpenCol), vortex.WithSISParams(&tc.SisInstance), ), selfrecursion.SelfRecurse, @@ -243,7 +243,7 @@ func TestSelfRecursionMultiLayered(t *testing.T) { compiler.Arcane(1<<11, 1<<13, false), vortex.Compile( 2, - vortex.ForceNumOpenedColumns(16), + vortex.ForceNumOpenedColumns(tc.NumOpenCol), vortex.WithSISParams(&tc.SisInstance), ), dummy.Compile, diff --git a/prover/protocol/compiler/specialqueries/range_test.go b/prover/protocol/compiler/specialqueries/range_test.go new file mode 100644 index 000000000..cc2d25190 --- /dev/null +++ b/prover/protocol/compiler/specialqueries/range_test.go @@ -0,0 +1,37 @@ +package specialqueries + +import ( + "testing" + + "github.com/consensys/linea-monorepo/prover/maths/common/smartvectors" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/dummy" + "github.com/consensys/linea-monorepo/prover/protocol/compiler/lookup" + "github.com/consensys/linea-monorepo/prover/protocol/wizard" + "github.com/stretchr/testify/require" +) + +func TestRangWithLogDerivCompiler(t *testing.T) { + + define := func(b *wizard.Builder) { + cola := b.RegisterCommit("A", 8) + colb := b.RegisterCommit("B", 8) + + b.Range("RANG1", cola, 1<<4) + b.Range("RANG2", colb, 1<<4) + } + + prover := func(run *wizard.ProverRuntime) { + // assign a and b + cola := smartvectors.ForTest(15, 0, 12, 14, 9, 6, 2, 1) + + colb := smartvectors.ForTest(0, 6, 9, 1, 1, 5, 11, 1) + + run.AssignColumn("A", cola) + run.AssignColumn("B", colb) + } + + comp := wizard.Compile(define, RangeProof, lookup.CompileLogDerivative, dummy.Compile) + proof := wizard.Prove(comp, prover) + err := wizard.Verify(comp, proof) + require.NoError(t, err) +} diff --git a/prover/protocol/wizardutils/evaluation.go b/prover/protocol/wizardutils/evaluation.go index 7381060ac..d2a7096be 100644 --- a/prover/protocol/wizardutils/evaluation.go +++ b/prover/protocol/wizardutils/evaluation.go @@ -51,13 +51,22 @@ func EvalExprColumn(run *wizard.ProverRuntime, board symbolic.ExpressionBoard) s // returns the symbolic expression of a column obtained as a random linear combinations of differents handles // without committing to the column itself -func RandLinCombColSymbolic(x coin.Info, hs []ifaces.Column) *symbolic.Expression { +// @Azam this function is temporarily ignored and addressed in issue https://github.com/Consensys/linea-monorepo/issues/192 +/*func RandLinCombColSymbolic(x coin.Info, hs []ifaces.Column) *symbolic.Expression { cols := make([]*symbolic.Expression, len(hs)) for c := range cols { cols[c] = ifaces.ColumnAsVariable(hs[c]) } expr := symbolic.NewPolyEval(x.AsVariable(), cols) return expr +}*/ +func RandLinCombColSymbolic(x coin.Info, hs []ifaces.Column) *symbolic.Expression { + res := symbolic.NewConstant(0) + for i := len(hs) - 1; i >= 0; i-- { + res = symbolic.Mul(res, x) + res = symbolic.Add(res, hs[i]) + } + return res } // return the runtime assignments of a linear combination column diff --git a/settings.gradle b/settings.gradle index 674fb5b9b..73082e51f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -21,6 +21,7 @@ include 'jvm-libs:metrics:micrometer' include 'jvm-libs:teku-execution-client' include 'jvm-libs:testing:l1-blob-and-proof-submission' include 'jvm-libs:testing:file-system' +include 'jvm-libs:generic:serialization:jackson' include 'coordinator:app' include 'coordinator:core' include 'coordinator:utilities'