Add compression for receipts

Makefile

@@ -220,3 +220,7 @@ split-test-packages:$(BUILDDIR)/packages.txt
 	split -d -n l/$(NUM_SPLIT) $< $<.
 test-group-%:split-test-packages
 	cat $(BUILDDIR)/packages.txt.$* | xargs go test -parallel 4 -mod=readonly -timeout=10m -race -coverprofile=$*.profile.out -covermode=atomic
+
+.PHONY: proto
+proto:
+	ignite generate proto-go -y

go.mod

@@ -126,6 +126,7 @@ require (
 	github.com/fzipp/gocyclo v0.5.1 // indirect
 	github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff // indirect
 	github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-critic/go-critic v0.6.3 // indirect
 	github.com/go-kit/kit v0.12.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
@@ -173,6 +174,7 @@ require (
 	github.com/gostaticanalysis/nilerr v0.1.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-bexpr v0.1.10 // indirect

go.sum

@@ -457,6 +457,7 @@ github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 h1:BAIP2Gihuqh
 github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46/go.mod h1:QNpY22eby74jVhqH4WhDLDwxc/vqsern6pW+u2kbkpc=
 github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
 github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
@@ -765,6 +766,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
 github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
 github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU=
 github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0=
 github.com/guptarohit/asciigraph v0.5.5/go.mod h1:dYl5wwK4gNsnFf9Zp+l06rFiDZ5YtXM6x7SRWZ3KGag=

proto/evm/compression.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+package seiprotocol.seichain.evm;
+
+import "gogoproto/gogo.proto";
+
+option go_package = "github.com/sei-protocol/sei-chain/x/evm/types";
+
+// CompressedData is only used for receipts as the moment
+// It is not meant to be used outside of storage and retrieval
+message CompressedData {
+  enum Algorithm {
+    SNAPPY = 0;
+  }
+
+  bytes data = 1;
+  Algorithm algorithm = 2;
+}

utils/compression/compress.go

@@ -0,0 +1,52 @@
+package compression
+
+import (
+	"fmt"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/sei-protocol/sei-chain/x/evm/types"
+)
+
+func CompressMessage(message proto.Message) ([]byte, error) {
+	b, err := proto.Marshal(message)
+	if err != nil {
+		return nil, err
+	}
+
+	bCompressed, err := compressSnappy(b)
+	if err != nil {
+		return nil, err
+	}
+
+	cd := &types.CompressedData{
+		Data:      bCompressed,
+		Algorithm: types.CompressedData_SNAPPY,
+	}
+	return cd.Marshal()
+}
+
+func DecompressMessage(target proto.Message, compressed []byte) error {
+	// this will work if the type is CompressedData
+	// if not, then it will try to unmarshal it as a regular proto message
+	var cd types.CompressedData
+	if err := cd.Unmarshal(compressed); err != nil {
+		// fall back to non-compress unmarshal
+		return proto.Unmarshal(compressed, target)
+	}
+
+	// unmarshal was successful, but no data, treat as non-compressed
+	if cd.Data == nil {
+		return proto.Unmarshal(compressed, target)
+	}
+
+	// add other algorithms here if we need to change it
+	if cd.Algorithm == types.CompressedData_SNAPPY {
+		decompressed, err := decompressSnappy(cd.Data)
+		if err != nil {
+			return err
+		}
+		return proto.Unmarshal(decompressed, target)
+	}
+
+	return fmt.Errorf("unsupported compression algorithm: %d (%s)", cd.Algorithm, cd.Algorithm.String())
+}

utils/compression/compress_test.go

@@ -0,0 +1,242 @@
+package compression
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/gogo/protobuf/jsonpb"
+	"github.com/sei-protocol/sei-chain/x/evm/types"
+	"github.com/stretchr/testify/require"
+)
+
+type compressFunc func([]byte) ([]byte, error)
+type uncompressFunc func([]byte) ([]byte, error)
+
+func printCompressionRatio(algo string, b, bCompressed []byte) {
+	originalSize := len(b)
+	compressedSize := len(bCompressed)
+	compressionRatio := float64(compressedSize) / float64(originalSize) * 100
+
+	fmt.Printf("[Debug] (%s) Original size: %d bytes\n", algo, originalSize)
+	fmt.Printf("[Debug] (%s), compressed size: %d bytes\n", algo, compressedSize)
+	fmt.Printf("[Debug] (%s), compression ratio: %.2f%%\n", algo, compressionRatio)
+}
+
+const receiptJson = `{
+	"tx_type": 0,
+	"cumulative_gas_used": 0,
+	"contract_address": null,
+	"tx_hash_hex": "50000603341192e9af2688fcd052dfbba333d3196d40df121ca8bb92a345a2b4",
+	"gas_used": 243714,
+	"effective_gas_price": 1000000000,
+	"block_number": 12345,
+	"transaction_index": 0,
+	"status": 1,
+	"from": "0x70f67735d4b4d9fcfb3014da2470e2f82a8744c7",
+	"to": "0x2880ab155794e7179c9ee2e38200202908c17b43",
+	"vm_error": "",
+	"logs": [
+		{
+			"address": "0x2880ab155794e7179c9ee2e38200202908c17b43",
+			"topics": [
+				"0xd06a6b7f4918494b3719217d1802786c1f5112a6c1d88fe2cfec00b4584f6aec",
+				"0x53614f1cb0c031d4af66c04cb9c756234adad0e1cee85303795091499a4084eb"
+			],
+			"data": "00000000000000000000000000000000000000000000000000000000667072e700000000000000000000000000000000000000000000000000000000024c88ca000000000000000000000000000000000000000000000000000000000000d23a",
+			"index": 0
+		}
+	],
+	"logsBloom": "00000400000000000000002000000000000000000000000000000000000000000200000000000000000000000008000000000000000000000000000000000000000000000000000000400000000010000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000010001400000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000100008000000000800000000000"
+}`
+
+func TestBackwardsCompatibility(t *testing.T) {
+	var r types.Receipt
+	err := jsonpb.Unmarshal(bytes.NewReader([]byte(receiptJson)), &r)
+	require.NoError(t, err)
+	receiptBytes, err := r.Marshal()
+	require.NoError(t, err)
+
+	type compressTest struct {
+		Name     string
+		Stored   func() []byte
+		Expected []byte
+		WantErr  bool
+	}
+
+	for _, test := range []compressTest{
+		{
+			Name: "not-compressed",
+			Stored: func() []byte {
+				return receiptBytes
+			},
+			Expected: receiptBytes,
+		},
+		{
+			Name: "compressed",
+			Stored: func() []byte {
+				cb, err := CompressMessage(&r)
+				require.NoError(t, err)
+				return cb
+			},
+			Expected: receiptBytes,
+		},
+		{
+			Name: "bad data",
+			Stored: func() []byte {
+				// not a receipt
+				block := &types.Log{
+					Address: "0x1230ab155794e7179c9ee2e38200202908c17b43",
+				}
+				b, err := block.Marshal()
+				require.NoError(t, err)
+				return b
+			},
+			WantErr: true,
+		},
+	} {
+		t.Run(test.Name, func(t *testing.T) {
+			var result types.Receipt
+			err := DecompressMessage(&result, test.Stored())
+			if test.WantErr {
+				require.Error(t, err)
+			} else {
+				// should have same bytes as expected bytes
+				require.NoError(t, err)
+				resultBytes, err := result.Marshal()
+				require.NoError(t, err)
+				require.Equal(t, test.Expected, resultBytes)
+			}
+		})
+	}
+}
+
+func TestCompressMessage(t *testing.T) {
+	var r types.Receipt
+	err := jsonpb.Unmarshal(bytes.NewReader([]byte(receiptJson)), &r)
+	require.NoError(t, err)
+
+	raw, err := r.Marshal()
+	require.NoError(t, err)
+
+	compressed, err := CompressMessage(&r)
+	require.NoError(t, err)
+
+	var r2 types.Receipt
+	err = DecompressMessage(&r2, compressed)
+	require.NoError(t, err)
+
+	require.Equal(t, r, r2)
+	printCompressionRatio("zlib", raw, compressed)
+}
+
+func TestCompressionRatio(t *testing.T) {
+	r := types.Receipt{}
+	err := jsonpb.Unmarshal(bytes.NewReader([]byte(receiptJson)), &r)
+	require.NoError(t, err)
+
+	b, err := r.Marshal()
+	require.NoError(t, err)
+
+	tests := []struct {
+		Name       string
+		Compress   compressFunc
+		Uncompress uncompressFunc
+	}{
+		//{
+		//	Name:       "brotli",
+		//	Compress:   compressBrotli,
+		//	Uncompress: decompressBrotli,
+		//},
+		//{
+		//	Name:       "bzip2",
+		//	Compress:   compressBzip2,
+		//	Uncompress: decompressBzip2,
+		//},
+		//{
+		//	Name:       "lzma",
+		//	Compress:   compressLzma,
+		//	Uncompress: decompressLzma,
+		//},
+		//{
+		//	Name:       "lz4",
+		//	Compress:   compressLz4,
+		//	Uncompress: decompressLz4,
+		//},
+		{
+			Name:       "zlib",
+			Compress:   compressZLib,
+			Uncompress: decompressZLib,
+		},
+		//{
+		//	Name:       "zstd",
+		//	Compress:   compressZstd,
+		//	Uncompress: decompressZstd,
+		//},
+		{
+			Name:       "snappy",
+			Compress:   compressSnappy,
+			Uncompress: decompressSnappy,
+		},
+		//{
+		//	Name:       "gzip",
+		//	Compress:   compressGzip,
+		//	Uncompress: decompressGzip,
+		//},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			bCompressed, err := test.Compress(b)
+			require.NoError(t, err)
+
+			bUncompressed, err := test.Uncompress(bCompressed)
+			require.NoError(t, err)
+
+			require.Equal(t, string(b), string(bUncompressed), "%s: expected %s but got %s", test.Name, b, bUncompressed)
+			printCompressionRatio(test.Name, b, bCompressed)
+		})
+	}
+}
+
+func benchmarkCompression(b *testing.B, name string, compress compressFunc, uncompress uncompressFunc) {
+	r := types.Receipt{}
+	err := jsonpb.Unmarshal(bytes.NewReader([]byte(receiptJson)), &r)
+	require.NoError(b, err)
+
+	data, err := r.Marshal()
+	require.NoError(b, err)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		compressedData, err := compress(data)
+		require.NoError(b, err)
+
+		_, err = uncompress(compressedData)
+		require.NoError(b, err)
+	}
+}
+
+func BenchmarkCompression(b *testing.B) {
+	benchmarks := []struct {
+		name       string
+		compress   compressFunc
+		uncompress uncompressFunc
+	}{
+		//{"brotli", compressBrotli, decompressBrotli},
+		//{"bzip2", compressBzip2, decompressBzip2},
+		//{"lzma", compressLzma, decompressLzma},
+		{"zlib", compressZLib, decompressZLib},
+		//{"lz4", compressLz4, decompressLz4},
+		//{"zstd", compressZstd, decompressZstd},
+		{"snappy", compressSnappy, decompressSnappy},
+		//{"gzip", compressGzip, decompressGzip},
+	}
+
+	for _, bm := range benchmarks {
+		b.Run(bm.name, func(b *testing.B) {
+			benchmarkCompression(b, bm.name, bm.compress, bm.uncompress)
+		})
+	}
+}

utils/compression/snappy.go

@@ -0,0 +1,30 @@
+package compression
+
+import (
+	"bytes"
+
+	"github.com/golang/snappy"
+)
+
+func compressSnappy(b []byte) ([]byte, error) {
+	var buf bytes.Buffer
+	writer := snappy.NewBufferedWriter(&buf)
+	_, err := writer.Write(b)
+	if err != nil {
+		return nil, err
+	}
+	writer.Close()
+
+	return buf.Bytes(), nil
+}
+
+func decompressSnappy(b []byte) ([]byte, error) {
+	r := snappy.NewReader(bytes.NewReader(b))
+
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(r); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}