Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: support conditional compress #23

Merged
merged 12 commits into from
Aug 22, 2024
Merged

feat: support conditional compress #23

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
ad42cd9
feat: support conditional encode
colinlyguo Aug 14, 2024
62758c8
move append conditionalEncode flag after validity check
colinlyguo Aug 14, 2024
6901956
update da-codec
colinlyguo Aug 18, 2024
e4bf12e
align naming
colinlyguo Aug 18, 2024
030349d
add ConvertBlobToBlobBytes utility functions
colinlyguo Aug 19, 2024
ed4de9e
kept blob bytes
colinlyguo Aug 19, 2024
c6af3bb
rename enableEncode to enableCompress
colinlyguo Aug 19, 2024
a5691d4
refactor: move some common functions to encoding (#24)
colinlyguo Aug 20, 2024
9532963
move symbol replace script to zstd folder
colinlyguo Aug 20, 2024
e49e96d
Update encoding/codecv4/codecv4_test.go
colinlyguo Aug 22, 2024
6d08fd3
Merge branch 'main' into feat-support-conditional-encode
colinlyguo Aug 22, 2024
7fd9672
fix typos
colinlyguo Aug 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 5 additions & 62 deletions encoding/codecv1/codecv1.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,6 @@ import (
"github.com/scroll-tech/da-codec/encoding/codecv0"
)

// BLSModulus is the BLS modulus defined in EIP-4844.
var BLSModulus = new(big.Int).SetBytes(common.FromHex("0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001"))

// MaxNumChunks is the maximum number of chunks that a batch can contain.
const MaxNumChunks = 15

Expand Down Expand Up @@ -261,7 +258,7 @@ func constructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484
copy(challengePreimage[0:], hash[:])

// convert raw data to BLSFieldElements
blob, err := MakeBlobCanonical(blobBytes)
blob, err := encoding.MakeBlobCanonical(blobBytes)
if err != nil {
return nil, common.Hash{}, nil, err
}
Expand All @@ -278,7 +275,7 @@ func constructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484

// compute z = challenge_digest % BLS_MODULUS
challengeDigest := crypto.Keccak256Hash(challengePreimage)
pointBigInt := new(big.Int).Mod(new(big.Int).SetBytes(challengeDigest[:]), BLSModulus)
pointBigInt := new(big.Int).Mod(new(big.Int).SetBytes(challengeDigest[:]), encoding.BLSModulus)
pointBytes := pointBigInt.Bytes()

// the challenge point z
Expand All @@ -289,31 +286,6 @@ func constructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484
return blob, blobVersionedHash, &z, nil
}

// MakeBlobCanonical converts the raw blob data into the canonical blob representation of 4096 BLSFieldElements.
func MakeBlobCanonical(blobBytes []byte) (*kzg4844.Blob, error) {
// blob contains 131072 bytes but we can only utilize 31/32 of these
if len(blobBytes) > 126976 {
return nil, fmt.Errorf("oversized batch payload, blob bytes length: %v, max length: %v", len(blobBytes), 126976)
}

// the canonical (padded) blob payload
var blob kzg4844.Blob

// encode blob payload by prepending every 31 bytes with 1 zero byte
index := 0

for from := 0; from < len(blobBytes); from += 31 {
to := from + 31
if to > len(blobBytes) {
to = len(blobBytes)
}
copy(blob[index+1:], blobBytes[from:to])
index += 32
}

return &blob, nil
}

// NewDABatchFromBytes decodes the given byte slice into a DABatch.
// Note: This function only populates the batch header, it leaves the blob-related fields empty.
func NewDABatchFromBytes(data []byte) (*DABatch, error) {
Expand Down Expand Up @@ -374,24 +346,7 @@ func (b *DABatch) BlobDataProof() ([]byte, error) {
return nil, fmt.Errorf("failed to create KZG proof at point, err: %w, z: %v", err, hex.EncodeToString(b.z[:]))
}

return BlobDataProofFromValues(*b.z, y, commitment, proof), nil
}

// BlobDataProofFromValues creates the blob data proof from the given values.
// Memory layout of ``_blobDataProof``:
// | z | y | kzg_commitment | kzg_proof |
// |---------|---------|----------------|-----------|
// | bytes32 | bytes32 | bytes48 | bytes48 |

func BlobDataProofFromValues(z kzg4844.Point, y kzg4844.Claim, commitment kzg4844.Commitment, proof kzg4844.Proof) []byte {
result := make([]byte, 32+32+48+48)

copy(result[0:32], z[:])
copy(result[32:64], y[:])
copy(result[64:112], commitment[:])
copy(result[112:160], proof[:])

return result
return encoding.BlobDataProofFromValues(*b.z, y, commitment, proof), nil
}

// Blob returns the blob of the batch.
Expand All @@ -406,7 +361,7 @@ func EstimateChunkL1CommitBlobSize(c *encoding.Chunk) (uint64, error) {
if err != nil {
return 0, err
}
return CalculatePaddedBlobSize(metadataSize + chunkDataSize), nil
return encoding.CalculatePaddedBlobSize(metadataSize + chunkDataSize), nil
}

// EstimateBatchL1CommitBlobSize estimates the total size of the L1 commit blob for a batch.
Expand All @@ -420,7 +375,7 @@ func EstimateBatchL1CommitBlobSize(b *encoding.Batch) (uint64, error) {
}
batchDataSize += chunkDataSize
}
return CalculatePaddedBlobSize(metadataSize + batchDataSize), nil
return encoding.CalculatePaddedBlobSize(metadataSize + batchDataSize), nil
}

func chunkL1CommitBlobDataSize(c *encoding.Chunk) (uint64, error) {
Expand Down Expand Up @@ -558,15 +513,3 @@ func EstimateBatchL1CommitCalldataSize(b *encoding.Batch) uint64 {
}
return totalL1CommitCalldataSize
}

// CalculatePaddedBlobSize calculates the required size on blob storage
// where every 32 bytes can store only 31 bytes of actual data, with the first byte being zero.
func CalculatePaddedBlobSize(dataSize uint64) uint64 {
paddedSize := (dataSize / 31) * 32

if dataSize%31 != 0 {
paddedSize += 1 + dataSize%31 // Add 1 byte for the first empty byte plus the remainder bytes
}

return paddedSize
}
125 changes: 28 additions & 97 deletions encoding/codecv2/codecv2.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,12 @@
package codecv2

/*
#include <stdint.h>
char* compress_scroll_batch_bytes(uint8_t* src, uint64_t src_size, uint8_t* output_buf, uint64_t *output_buf_size);
*/
import "C"

import (
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"math/big"
"unsafe"

"github.com/scroll-tech/go-ethereum/common"
"github.com/scroll-tech/go-ethereum/core/types"
Expand All @@ -23,11 +16,9 @@ import (

"github.com/scroll-tech/da-codec/encoding"
"github.com/scroll-tech/da-codec/encoding/codecv1"
"github.com/scroll-tech/da-codec/encoding/zstd"
)

// BLSModulus is the BLS modulus defined in EIP-4844.
var BLSModulus = codecv1.BLSModulus

// MaxNumChunks is the maximum number of chunks that a batch can contain.
const MaxNumChunks = 45

Expand Down Expand Up @@ -88,7 +79,7 @@ func NewDABatch(batch *encoding.Batch) (*DABatch, error) {
}

// blob payload
blob, blobVersionedHash, z, err := ConstructBlobPayload(batch.Chunks, false /* no mock */)
blob, blobVersionedHash, z, _, err := ConstructBlobPayload(batch.Chunks, false /* no mock */)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -118,7 +109,7 @@ func ComputeBatchDataHash(chunks []*encoding.Chunk, totalL1MessagePoppedBefore u
}

// ConstructBlobPayload constructs the 4844 blob payload.
func ConstructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg4844.Blob, common.Hash, *kzg4844.Point, error) {
func ConstructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg4844.Blob, common.Hash, *kzg4844.Point, []byte, error) {
// metadata consists of num_chunks (2 bytes) and chunki_size (4 bytes per chunk)
metadataLength := 2 + MaxNumChunks*4

Expand Down Expand Up @@ -149,7 +140,7 @@ func ConstructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484
// encode L2 txs into blob payload
rlpTxData, err := encoding.ConvertTxDataToRLPEncoding(tx, useMockTxData)
if err != nil {
return nil, common.Hash{}, nil, err
return nil, common.Hash{}, nil, nil, err
}
batchBytes = append(batchBytes, rlpTxData...)
}
Expand Down Expand Up @@ -178,30 +169,35 @@ func ConstructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484
copy(challengePreimage[0:], hash[:])

// blobBytes represents the compressed blob payload (batchBytes)
blobBytes, err := compressScrollBatchBytes(batchBytes)
blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes)
if err != nil {
return nil, common.Hash{}, nil, err
return nil, common.Hash{}, nil, nil, err
}

// Only apply this check when the uncompressed batch data has exceeded 128 KiB.
if !useMockTxData && len(batchBytes) > 131072 {
// Check compressed data compatibility.
if err = encoding.CheckCompressedDataCompatibility(blobBytes); err != nil {
log.Error("ConstructBlobPayload: compressed data compatibility check failed", "err", err, "batchBytes", hex.EncodeToString(batchBytes), "blobBytes", hex.EncodeToString(blobBytes))
return nil, common.Hash{}, nil, err
return nil, common.Hash{}, nil, nil, err
}
}

if len(blobBytes) > 126976 {
log.Error("ConstructBlobPayload: Blob payload exceeds maximum size", "size", len(blobBytes), "blobBytes", hex.EncodeToString(blobBytes))
return nil, common.Hash{}, nil, nil, errors.New("Blob payload exceeds maximum size")
}

// convert raw data to BLSFieldElements
blob, err := MakeBlobCanonical(blobBytes)
blob, err := encoding.MakeBlobCanonical(blobBytes)
if err != nil {
return nil, common.Hash{}, nil, err
return nil, common.Hash{}, nil, nil, err
}

// compute blob versioned hash
c, err := kzg4844.BlobToCommitment(blob)
if err != nil {
return nil, common.Hash{}, nil, errors.New("failed to create blob commitment")
return nil, common.Hash{}, nil, nil, errors.New("failed to create blob commitment")
}
blobVersionedHash := kzg4844.CalcBlobHashV1(sha256.New(), &c)

Expand All @@ -210,20 +206,15 @@ func ConstructBlobPayload(chunks []*encoding.Chunk, useMockTxData bool) (*kzg484

// compute z = challenge_digest % BLS_MODULUS
challengeDigest := crypto.Keccak256Hash(challengePreimage)
pointBigInt := new(big.Int).Mod(new(big.Int).SetBytes(challengeDigest[:]), BLSModulus)
pointBigInt := new(big.Int).Mod(new(big.Int).SetBytes(challengeDigest[:]), encoding.BLSModulus)
pointBytes := pointBigInt.Bytes()

// the challenge point z
var z kzg4844.Point
start := 32 - len(pointBytes)
copy(z[start:], pointBytes)

return blob, blobVersionedHash, &z, nil
}

// MakeBlobCanonical converts the raw blob data into the canonical blob representation of 4096 BLSFieldElements.
func MakeBlobCanonical(blobBytes []byte) (*kzg4844.Blob, error) {
return codecv1.MakeBlobCanonical(blobBytes)
return blob, blobVersionedHash, &z, blobBytes, nil
}

// NewDABatchFromBytes decodes the given byte slice into a DABatch.
Expand Down Expand Up @@ -286,7 +277,7 @@ func (b *DABatch) BlobDataProof() ([]byte, error) {
return nil, fmt.Errorf("failed to create KZG proof at point, err: %w, z: %v", err, hex.EncodeToString(b.z[:]))
}

return codecv1.BlobDataProofFromValues(*b.z, y, commitment, proof), nil
return encoding.BlobDataProofFromValues(*b.z, y, commitment, proof), nil
}

// Blob returns the blob of the batch.
Expand All @@ -296,38 +287,38 @@ func (b *DABatch) Blob() *kzg4844.Blob {

// EstimateChunkL1CommitBatchSizeAndBlobSize estimates the L1 commit uncompressed batch size and compressed blob size for a single chunk.
func EstimateChunkL1CommitBatchSizeAndBlobSize(c *encoding.Chunk) (uint64, uint64, error) {
batchBytes, err := constructBatchPayload([]*encoding.Chunk{c})
batchBytes, err := encoding.ConstructBatchPayloadInBlob([]*encoding.Chunk{c}, MaxNumChunks)
if err != nil {
return 0, 0, err
}
blobBytes, err := compressScrollBatchBytes(batchBytes)
blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes)
if err != nil {
return 0, 0, err
}
return uint64(len(batchBytes)), CalculatePaddedBlobSize(uint64(len(blobBytes))), nil
return uint64(len(batchBytes)), encoding.CalculatePaddedBlobSize(uint64(len(blobBytes))), nil
}

// EstimateBatchL1CommitBatchSizeAndBlobSize estimates the L1 commit uncompressed batch size and compressed blob size for a batch.
func EstimateBatchL1CommitBatchSizeAndBlobSize(b *encoding.Batch) (uint64, uint64, error) {
batchBytes, err := constructBatchPayload(b.Chunks)
batchBytes, err := encoding.ConstructBatchPayloadInBlob(b.Chunks, MaxNumChunks)
if err != nil {
return 0, 0, err
}
blobBytes, err := compressScrollBatchBytes(batchBytes)
blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes)
if err != nil {
return 0, 0, err
}
return uint64(len(batchBytes)), CalculatePaddedBlobSize(uint64(len(blobBytes))), nil
return uint64(len(batchBytes)), encoding.CalculatePaddedBlobSize(uint64(len(blobBytes))), nil
}

// CheckChunkCompressedDataCompatibility checks the compressed data compatibility for a batch built from a single chunk.
// It constructs a batch payload, compresses the data, and checks the compressed data compatibility if the uncompressed data exceeds 128 KiB.
func CheckChunkCompressedDataCompatibility(c *encoding.Chunk) (bool, error) {
batchBytes, err := constructBatchPayload([]*encoding.Chunk{c})
batchBytes, err := encoding.ConstructBatchPayloadInBlob([]*encoding.Chunk{c}, MaxNumChunks)
if err != nil {
return false, err
}
blobBytes, err := compressScrollBatchBytes(batchBytes)
blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes)
if err != nil {
return false, err
}
Expand All @@ -345,11 +336,11 @@ func CheckChunkCompressedDataCompatibility(c *encoding.Chunk) (bool, error) {
// CheckBatchCompressedDataCompatibility checks the compressed data compatibility for a batch.
// It constructs a batch payload, compresses the data, and checks the compressed data compatibility if the uncompressed data exceeds 128 KiB.
func CheckBatchCompressedDataCompatibility(b *encoding.Batch) (bool, error) {
batchBytes, err := constructBatchPayload(b.Chunks)
batchBytes, err := encoding.ConstructBatchPayloadInBlob(b.Chunks, MaxNumChunks)
if err != nil {
return false, err
}
blobBytes, err := compressScrollBatchBytes(batchBytes)
blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes)
if err != nil {
return false, err
}
Expand Down Expand Up @@ -388,63 +379,3 @@ func EstimateChunkL1CommitGas(c *encoding.Chunk) uint64 {
func EstimateBatchL1CommitGas(b *encoding.Batch) uint64 {
return codecv1.EstimateBatchL1CommitGas(b)
}

// constructBatchPayload constructs the batch payload.
// This function is only used in compressed batch payload length estimation.
func constructBatchPayload(chunks []*encoding.Chunk) ([]byte, error) {
// metadata consists of num_chunks (2 bytes) and chunki_size (4 bytes per chunk)
metadataLength := 2 + MaxNumChunks*4

// batchBytes represents the raw (un-compressed and un-padded) blob payload
batchBytes := make([]byte, metadataLength)

// batch metadata: num_chunks
binary.BigEndian.PutUint16(batchBytes[0:], uint16(len(chunks)))

// encode batch metadata and L2 transactions,
for chunkID, chunk := range chunks {
currentChunkStartIndex := len(batchBytes)

for _, block := range chunk.Blocks {
for _, tx := range block.Transactions {
if tx.Type == types.L1MessageTxType {
continue
}

// encode L2 txs into batch payload
rlpTxData, err := encoding.ConvertTxDataToRLPEncoding(tx, false /* no mock */)
if err != nil {
return nil, err
}
batchBytes = append(batchBytes, rlpTxData...)
}
}

// batch metadata: chunki_size
if chunkSize := len(batchBytes) - currentChunkStartIndex; chunkSize != 0 {
binary.BigEndian.PutUint32(batchBytes[2+4*chunkID:], uint32(chunkSize))
}
}
return batchBytes, nil
}

// compressScrollBatchBytes compresses the given batch of bytes.
// The output buffer is allocated with an extra 128 bytes to accommodate metadata overhead or error message.
func compressScrollBatchBytes(batchBytes []byte) ([]byte, error) {
srcSize := C.uint64_t(len(batchBytes))
outbufSize := C.uint64_t(len(batchBytes) + 128) // Allocate output buffer with extra 128 bytes
outbuf := make([]byte, outbufSize)

if err := C.compress_scroll_batch_bytes((*C.uchar)(unsafe.Pointer(&batchBytes[0])), srcSize,
(*C.uchar)(unsafe.Pointer(&outbuf[0])), &outbufSize); err != nil {
return nil, fmt.Errorf("failed to compress scroll batch bytes: %s", C.GoString(err))
}

return outbuf[:int(outbufSize)], nil
}

// CalculatePaddedBlobSize calculates the required size on blob storage
// where every 32 bytes can store only 31 bytes of actual data, with the first byte being zero.
func CalculatePaddedBlobSize(dataSize uint64) uint64 {
return codecv1.CalculatePaddedBlobSize(dataSize)
}
Loading
Loading