Decompressor Library

.github/workflows/prover-native-lib-blob-compressor-release.yml

@@ -7,6 +7,7 @@ on:
         description: 'Version (e.g. v1.2.3)'
         required: true
         default: 'v0.0.0'
+        type: string
       draft-release:
         description: 'Draft Release'
         required: false
@@ -37,13 +38,16 @@ jobs:
           VERSION: ${{ github.event.inputs.version }}
           SRC_SHNARF: "./lib/shnarf_calculator/shnarf_calculator.go"
           TARGET_SHNARF: "shnarf_calculator"
-          SRC_COMPRESSOR: "./lib/compressor/libcompressor.go"
+          SRC_COMPRESSOR: "./lib/compressor/libcompressor/libcompressor.go"
           TARGET_COMPRESSOR: "blob_compressor"
+          SRC_DECOMPRESSOR: "./lib/compressor/libdecompressor/libdecompressor.go"
+          TARGET_DECOMPRESSOR: "blob_decompressor"
         run: |
           cd prover
           mkdir target
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_SHNARF}_${VERSION}_linux_x86_64.so ${SRC_SHNARF}
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_linux_x86_64.so ${SRC_COMPRESSOR}
+          GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_linux_x86_64.so ${SRC_DECOMPRESSOR}
 
       - name: Cache built binaries
         uses: actions/upload-artifact@master
@@ -69,13 +73,16 @@ jobs:
           VERSION: ${{ github.event.inputs.version }}
           SRC_SHNARF: "./lib/shnarf_calculator/shnarf_calculator.go"
           TARGET_SHNARF: "shnarf_calculator"
-          SRC_COMPRESSOR: "./lib/compressor/libcompressor.go"
+          SRC_COMPRESSOR: "./lib/compressor/libcompressor/libcompressor.go"
           TARGET_COMPRESSOR: "blob_compressor"
+          SRC_DECOMPRESSOR: "./lib/compressor/libdecompressor/libdecompressor.go"
+          TARGET_DECOMPRESSOR: "blob_decompressor"
         run: |
           cd prover
           mkdir target
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_SHNARF}_${VERSION}_linux_arm64.so ${SRC_SHNARF}
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_linux_arm64.so ${SRC_COMPRESSOR}
+          GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_linux_arm64.so ${SRC_DECOMPRESSOR}
       - name: Cache built binaries
         uses: actions/upload-artifact@master
         with:
@@ -98,15 +105,19 @@ jobs:
           VERSION: ${{ github.event.inputs.version }}
           SRC_SHNARF: "./lib/shnarf_calculator/shnarf_calculator.go"
           TARGET_SHNARF: "shnarf_calculator"
-          SRC_COMPRESSOR: "./lib/compressor/libcompressor.go"
+          SRC_COMPRESSOR: "./lib/compressor/libcompressor/libcompressor.go"
           TARGET_COMPRESSOR: "blob_compressor"
+          SRC_DECOMPRESSOR: "./lib/compressor/libdecompressor/libdecompressor.go"
+          TARGET_DECOMPRESSOR: "blob_decompressor"
         run: |
           cd prover
           mkdir target
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_SHNARF}_${VERSION}_darwin_x86_64.dylib ${SRC_SHNARF}
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_SHNARF}_${VERSION}_darwin_arm64.dylib ${SRC_SHNARF}
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_darwin_x86_64.dylib ${SRC_COMPRESSOR}
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_darwin_arm64.dylib ${SRC_COMPRESSOR}
+          GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_darwin_x86_64.dylib ${SRC_DECOMPRESSOR}
+          GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_darwin_arm64.dylib ${SRC_DECOMPRESSOR}
 
       - name: Cache built binaries
         uses: actions/upload-artifact@v4

jvm-libs/blob-compressor/build.gradle

@@ -10,7 +10,7 @@ apply from: rootProject.file("gradle/publishing.gradle")
 
 dependencies {
   implementation "net.java.dev.jna:jna:${libs.versions.jna.get()}"
-  testImplementation project(":jvm-libs:kotlin-extensions")
+  implementation project(":jvm-libs:kotlin-extensions")
   testImplementation project(":jvm-libs:blob-shnarf-calculator")
 }
 

jvm-libs/blob-decompressor/build.gradle

@@ -0,0 +1,52 @@
+plugins {
+  id 'net.consensys.zkevm.kotlin-common-minimal-conventions'
+  id 'net.consensys.zkevm.linea-native-libs-helper'
+  alias(libs.plugins.jreleaser)
+  id 'java-test-fixtures'
+}
+
+description = 'Java JNA wrapper for Linea Blob Decompressor Library implemented in GO Lang'
+apply from: rootProject.file("gradle/publishing.gradle")
+
+dependencies {
+  implementation "net.java.dev.jna:jna:${libs.versions.jna.get()}"
+  implementation project(":jvm-libs:kotlin-extensions")
+  testImplementation project(":jvm-libs:blob-compressor")
+//  testImplementation project(":jvm-libs:web3j-extensions")
+  testImplementation(testFixtures(project(":jvm-libs:blob-compressor")))
+}
+
+jar {
+  dependsOn configurations.runtimeClasspath
+}
+
+test {
+  // we cannot have more 1 compressor per JVM, hence we disable parallel execution
+  // because multiple threads would cause issues with the native library
+  systemProperties["junit.jupiter.execution.parallel.enabled"] = false
+  maxParallelForks = 1
+}
+
+def libsZipDownloadOutputDir = project.parent.layout.buildDirectory.asFile.get().absolutePath
+
+task downloadNativeLibs {
+  doLast {
+    fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v1.1.0-test6/linea-blob-libs-v1.1.0-test6.zip", "blob_decompressor", libsZipDownloadOutputDir)
+  }
+}
+
+compileKotlin {
+  dependsOn tasks.downloadNativeLibs
+}
+
+task cleanResources(type: Delete) {
+  fileTree(project.layout.projectDirectory.dir('src/main/resources'))
+    .filter {
+      it.name.endsWith(".so") || it.name.endsWith(".dll") || it.name.endsWith(".dylib")
+    }.each {
+      println("Deleting: ${it}")
+      delete it
+    }
+}
+
+clean.dependsOn cleanResources

jvm-libs/blob-decompressor/src/main/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressor.kt

@@ -0,0 +1,101 @@
+package net.consensys.linea.blob
+
+import com.sun.jna.Library
+import com.sun.jna.Native
+import net.consensys.jvm.ResourcesUtil.copyResourceToTmpDir
+import java.nio.file.Path
+
+class DecompressionException(message: String) : RuntimeException(message)
+
+interface BlobDecompressor {
+  fun decompress(blob: ByteArray): ByteArray
+}
+
+internal class Adapter(
+  private val delegate: GoNativeBlobDecompressorJnaBinding,
+  private val maxExpectedCompressionRatio: Int = 10,
+  dictionaries: List<Path>
+) : BlobDecompressor {
+  init {
+    delegate.Init()
+
+    val paths = dictionaries.joinToString(separator = ":") { path -> path.toString() }
+
+    if (delegate.LoadDictionaries(paths) != dictionaries.size) {
+      throw DecompressionException("Failed to load dictionaries '$paths', error='${delegate.Error()}'")
+    }
+  }
+
+  override fun decompress(blob: ByteArray): ByteArray {
+    val decompressionBuffer = ByteArray(blob.size * maxExpectedCompressionRatio)
+    val decompressedSize = delegate.Decompress(blob, blob.size, decompressionBuffer, decompressionBuffer.size)
+    if (decompressedSize < 0) {
+      throw DecompressionException("Decompression failed, error='${delegate.Error()}'")
+    }
+    return decompressionBuffer.copyOf(decompressedSize)
+  }
+}
+
+internal interface GoNativeBlobDecompressorJnaBinding {
+
+  /**
+   * Init initializes the Decompressor. Must be run before anything else.
+   */
+  fun Init()
+
+  /**
+   * LoadDictionaries attempts to cache dictionaries from given paths, separated by colons,
+   * e.g. "../compressor_dict.bin:./other_dict"
+   * Returns the number of dictionaries successfully loaded, and -1 in case of failure, in which case Error() will
+   * return a description of the error.
+   *
+   * @param dictPaths a colon-separated list of paths to dictionaries, to be loaded into the decompressor
+   * @return the number of dictionaries loaded if successful, -1 if not.
+   */
+  fun LoadDictionaries(dictPaths: String): Int
+
+  /**
+   * Decompress a blob b and writes the resulting blocks in out, serialized in the format of
+   * prover/backend/ethereum.
+   * Returns the number of bytes in out, or -1 in case of failure
+   * If -1 is returned, the Error() method will return a string describing the error.
+   *
+   * @param blob to be decompressed
+   * @param blob_len length of the blob
+   * @param out buffer to write the decompressed data
+   * @param out_max_len maximum length of the out buffer
+   * @return number of bytes in out, or -1 in case of failure
+   */
+  fun Decompress(blob: ByteArray, blob_len: Int, out: ByteArray, out_max_len: Int): Int
+
+  /**
+   * Error returns the last error message. Should be checked if Write returns false.
+   */
+  fun Error(): String?
+}
+
+internal interface GoNativeBlobDecompressorJnaLib : GoNativeBlobDecompressorJnaBinding, Library
+
+enum class BlobDecompressorVersion(val version: String) {
+  V1_1_0("v1.1.0")
+}
+
+class GoNativeBlobDecompressorFactory {
+  companion object {
+    private const val DICTIONARY_NAME = "compressor_dict.bin"
+    private val dictionaryPath = copyResourceToTmpDir(DICTIONARY_NAME)
+
+    private fun getLibFileName(version: String) = "blob_decompressor_jna_$version"
+
+    fun getInstance(
+      version: BlobDecompressorVersion
+    ): BlobDecompressor {
+      return Native.load(
+        Native.extractFromResourcePath(getLibFileName(version.version)).toString(),
+        GoNativeBlobDecompressorJnaLib::class.java
+      ).let {
+        Adapter(delegate = it, dictionaries = listOf(dictionaryPath))
+      }
+    }
+  }
+}

jvm-libs/blob-decompressor/src/main/resources/.gitignore

@@ -0,0 +1,4 @@
+linux-aarch64/*
+linux-x86-64/*
+darwin-aarch64/*
+darwin-x86-64/*

jvm-libs/blob-decompressor/src/test/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressorTest.kt

@@ -0,0 +1,41 @@
+package net.consensys.linea.blob
+
+import net.consensys.linea.nativecompressor.CompressorTestData
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+
+class GoNativeBlobDecompressorTest {
+  private val blobCompressedLimit = 10 * 1024
+  private lateinit var compressor: GoNativeBlobCompressor
+  private lateinit var decompressor: BlobDecompressor
+
+  @BeforeEach
+  fun beforeEach() {
+    compressor = GoNativeBlobCompressorFactory
+      .getInstance(BlobCompressorVersion.V1_0_1)
+      .apply {
+        Init(
+          dataLimit = blobCompressedLimit,
+          dictPath = GoNativeBlobCompressorFactory.dictionaryPath.toAbsolutePath().toString()
+        )
+        Reset()
+      }
+    decompressor = GoNativeBlobDecompressorFactory.getInstance(BlobDecompressorVersion.V1_1_0)
+  }
+
+  @Test
+  fun `when blocks are compressed with compressor shall decompress them back`() {
+    val blocks = CompressorTestData.blocksRlpEncoded.slice(0..1)
+    assertTrue(compressor.Write(blocks[0], blocks[0].size))
+    assertTrue(compressor.Write(blocks[1], blocks[1].size))
+
+    val compressedData = ByteArray(compressor.Len())
+    compressor.Bytes(compressedData)
+
+    val decompressedBlob = decompressor.decompress(compressedData)
+    assertThat(decompressedBlob.size).isGreaterThan(compressedData.size)
+    // TODO: assert decompressedDataBuffer content
+  }
+}

jvm-libs/kotlin-extensions/src/main/kotlin/net/consensys/jvm/jvm/ResourcesUtil.kt

@@ -0,0 +1,24 @@
+package net.consensys.jvm
+
+import java.io.File
+import java.nio.file.Files
+import java.nio.file.Path
+
+object ResourcesUtil {
+  @JvmStatic
+  fun copyResourceToTmpDir(
+    resourcePath: String,
+    tmpDirPrefix: String = "linea-resources-"
+  ): Path {
+    val fileDestination = File(
+      Files.createTempDirectory(tmpDirPrefix)
+        .resolve(Path.of(resourcePath).fileName)
+        .toString()
+    )
+    Files.copy(
+      Thread.currentThread().getContextClassLoader().getResourceAsStream(resourcePath),
+      fileDestination.toPath()
+    )
+    return fileDestination.toPath()
+  }
+}

prover/backend/blobsubmission/blobcompression_test.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"testing"
 
-	blob "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/v1"
+	"github.com/consensys/linea-monorepo/prover/lib/compressor/blob/encode"
 
 	fr381 "github.com/consensys/gnark-crypto/ecc/bls12-381/fr"
 	"github.com/consensys/linea-monorepo/prover/utils"
@@ -270,7 +270,7 @@ func TestKZGWithPoint(t *testing.T) {
 	}
 
 	// Compute all the prover fields
-	snarkHash, err := blob.MiMCChecksumPackedData(blobBytes[:], fr381.Bits-1, blob.NoTerminalSymbol())
+	snarkHash, err := encode.MiMCChecksumPackedData(blobBytes[:], fr381.Bits-1, encode.NoTerminalSymbol())
 	assert.NoError(t, err)
 
 	xUnreduced := evaluationChallenge(snarkHash, blobHash[:])

prover/backend/blobsubmission/craft.go

@@ -7,9 +7,9 @@ import (
 	"hash"
 
 	"github.com/consensys/linea-monorepo/prover/crypto/mimc"
+	"github.com/consensys/linea-monorepo/prover/lib/compressor/blob/encode"
 
 	fr381 "github.com/consensys/gnark-crypto/ecc/bls12-381/fr"
-	blob "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/v1"
 	"github.com/consensys/linea-monorepo/prover/utils"
 	"golang.org/x/crypto/sha3"
 )
@@ -72,7 +72,7 @@ func CraftResponseCalldata(req *Request) (*Response, error) {
 	}
 
 	// Compute all the prover fields
-	snarkHash, err := blob.MiMCChecksumPackedData(compressedStream, fr381.Bits-1, blob.NoTerminalSymbol())
+	snarkHash, err := encode.MiMCChecksumPackedData(compressedStream, fr381.Bits-1, encode.NoTerminalSymbol())
 	if err != nil {
 		return nil, fmt.Errorf("crafting response: could not compute snark hash: %w", err)
 	}

prover/backend/blobsubmission/craft_eip4844.go

@@ -4,6 +4,7 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
+	"github.com/consensys/linea-monorepo/prover/lib/compressor/blob/encode"
 
 	blob "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/v1"
 
@@ -91,7 +92,7 @@ func CraftResponse(req *Request) (*Response, error) {
 	}
 
 	// Compute all the prover fields
-	snarkHash, err := blob.MiMCChecksumPackedData(append(compressedStream, make([]byte, blob.MaxUsableBytes-len(compressedStream))...), fr381.Bits-1, blob.NoTerminalSymbol())
+	snarkHash, err := encode.MiMCChecksumPackedData(append(compressedStream, make([]byte, blob.MaxUsableBytes-len(compressedStream))...), fr381.Bits-1, encode.NoTerminalSymbol())
 	if err != nil {
 		return nil, fmt.Errorf("crafting response: could not compute snark hash: %w", err)
 	}

prover/backend/ethereum/signature_test.go

@@ -117,10 +117,11 @@ func TestTransactionSigning(t *testing.T) {
 			assert.Equal(t, from.Hex(), recovered.Hex(), "Mismatch of the recovered address")
 
 			// Simulates the decoding of the transaction
-			var decodedTx types.Transaction
-			err = DecodeTxFromBytes(bytes.NewReader(rlp), &decodedTx)
-
+			decodedTxData, err := DecodeTxFromBytes(bytes.NewReader(rlp))
 			require.NoError(t, err)
+
+			decodedTx := types.NewTx(decodedTxData)
+
 			assert.Equal(t, tx.To(), decodedTx.To())
 			assert.Equal(t, tx.Nonce(), decodedTx.Nonce())
 			assert.Equal(t, tx.Data(), decodedTx.Data())