feat: Support encoding/emitting nested structures with binary fields

go.mod

@@ -13,6 +13,9 @@ require (
 	github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect
 	github.com/gookit/color v1.5.4 // indirect
 	github.com/gorilla/websocket v1.5.1 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/onsi/ginkgo/v2 v2.12.0 // indirect
 	github.com/quic-go/qpack v0.4.0 // indirect
 	github.com/quic-go/quic-go v0.44.0 // indirect

go.sum

@@ -11,14 +11,21 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ=
 github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
 github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
 github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
 github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI=
 github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ=
 github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
@@ -32,6 +39,7 @@ github.com/quic-go/quic-go v0.44.0/go.mod h1:z4cx/9Ny9UtGITIPzmPTXh1ULfOyWh4qGQl
 github.com/quic-go/webtransport-go v0.8.0 h1:HxSrwun11U+LlmwpgM1kEqIqH90IT4N8auv/cD7QFJg=
 github.com/quic-go/webtransport-go v0.8.0/go.mod h1:N99tjprW432Ut5ONql/aUhSLT0YVSlwHohQsuac9WaM=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=

parser/binary.go

@@ -1,9 +1,11 @@
 package parser
 
 import (
+	"bytes"
 	"errors"
-	"io"
+	"unsafe"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/mitchellh/mapstructure"
 	"github.com/zishang520/engine.io-go-parser/types"
 )
@@ -13,54 +15,46 @@ type Placeholder struct {
 	Num         int  `json:"num" mapstructure:"num" msgpack:"num"`
 }
 
+func init() {
+	jsoniter.RegisterTypeEncoderFunc("types.BytesBuffer", func(ptr unsafe.Pointer, stream *jsoniter.Stream) {
+		bb := ((*types.BytesBuffer)(ptr))
+
+		bufList := stream.Attachment.([]types.BufferInterface)
+		_placeholder := &Placeholder{Placeholder: true, Num: len(bufList)}
+		stream.WriteVal(_placeholder)
+		stream.Attachment = append(bufList, bb)
+	}, nil)
+
+	jsoniter.RegisterTypeEncoderFunc("[]uint8", func(ptr unsafe.Pointer, stream *jsoniter.Stream) {
+		bb := types.NewBytesBuffer(nil)
+		barr := ((*[]byte)(ptr))
+		bb.Write(*barr)
+
+		bufList := stream.Attachment.([]types.BufferInterface)
+		_placeholder := &Placeholder{Placeholder: true, Num: len(bufList)}
+		stream.WriteVal(_placeholder)
+		stream.Attachment = append(bufList, bb)
+	}, nil)
+}
+
 // Replaces every io.Reader | []byte in packet with a numbered placeholder.
 func DeconstructPacket(packet *Packet) (pack *Packet, buffers []types.BufferInterface) {
 	pack = packet
-	pack.Data = _deconstructPacket(packet.Data, &buffers)
+
+	// Run the serialization now, replacing any bytebuffers/[]byte found along the way with placeholders
+	buf := &bytes.Buffer{}
+	ns := jsoniter.NewStream(jsoniter.ConfigDefault, buf, buf.Cap())
+	ns.Attachment = buffers
+	ns.WriteVal(pack.Data)
+	buffers = ns.Attachment.([]types.BufferInterface)
+	ns.Flush()
+	pack.preSerializedData = buf.String()
+
 	attachments := uint64(len(buffers))
 	pack.Attachments = &attachments // number of binary 'attachments'
 	return pack, buffers
 }
 
-func _deconstructPacket(data any, buffers *[]types.BufferInterface) any {
-	if data == nil {
-		return nil
-	}
-
-	if IsBinary(data) {
-		_placeholder := &Placeholder{Placeholder: true, Num: len(*buffers)}
-		rdata := types.NewBytesBuffer(nil)
-		switch tdata := data.(type) {
-		case io.Reader:
-			if c, ok := data.(io.Closer); ok {
-				defer c.Close()
-			}
-			rdata.ReadFrom(tdata)
-		case []byte:
-			rdata.Write(tdata)
-		}
-		*buffers = append(*buffers, rdata)
-		return _placeholder
-	}
-
-	switch tdata := data.(type) {
-	case []any:
-		newData := make([]any, 0, len(tdata))
-		for _, v := range tdata {
-			newData = append(newData, _deconstructPacket(v, buffers))
-		}
-		return newData
-	case map[string]any:
-		newData := map[string]any{}
-		for k, v := range tdata {
-			newData[k] = _deconstructPacket(v, buffers)
-		}
-		return newData
-	default:
-		return data
-	}
-}
-
 // Reconstructs a binary packet from its placeholder packet and buffers
 func ReconstructPacket(packet *Packet, buffers []types.BufferInterface) (*Packet, error) {
 	data, err := _reconstructPacket(packet.Data, &buffers)

parser/encoder.go

@@ -53,9 +53,9 @@ func _encodeData(data any) any {
 			newData[k] = _encodeData(v)
 		}
 		return newData
-	default:
-		return data
 	}
+
+	return data
 }
 
 // Encode packet as string.
@@ -79,8 +79,13 @@ func (e *encoder) encodeAsString(packet *Packet) types.BufferInterface {
 	}
 	// json data
 	if nil != packet.Data {
-		if b, err := json.Marshal(_encodeData(packet.Data)); err == nil {
-			str.Write(b)
+		if packet.preSerializedData != "" {
+			// Already serialized in the DeconstructPacket function
+			str.WriteString(packet.preSerializedData)
+		} else {
+			if b, err := json.Marshal(_encodeData(packet.Data)); err == nil {
+				str.Write(b)
+			}
 		}
 	}
 	parser_log.Debug("encoded %v as %v", packet, str)

parser/is-binary.go

@@ -2,6 +2,7 @@ package parser
 
 import (
 	"io"
+	"reflect"
 	"strings"
 
 	"github.com/zishang520/engine.io-go-parser/types"
@@ -30,14 +31,51 @@ func HasBinary(data any) bool {
 				return true
 			}
 		}
+		return false
 	case map[string]any:
 		for _, value := range v {
 			if HasBinary(value) {
 				return true
 			}
 		}
+		return false
+	}
+
+	if IsBinary(data) {
+		return true
+	}
+
+	dv := reflect.ValueOf(data)
+	switch dv.Kind() {
+	case reflect.Pointer:
+		return IsBinary(dv.Elem().Interface())
+	case reflect.Struct:
+		for fi := range dv.NumField() {
+			dfv := dv.Field(fi)
+			if dfv.CanInterface() && HasBinary(dfv.Interface()) {
+				return true
+			}
+		}
+		return false
+	case reflect.Array, reflect.Slice:
+		for i := range dv.Len() {
+			av := dv.Index(i)
+			if av.CanInterface() && HasBinary(av.Interface()) {
+				return true
+			}
+		}
+		return false
+	case reflect.Map:
+		mr := dv.MapRange()
+		for mr.Next() {
+			// Keys can't be binary blobs in json, so only check values
+			mv := mr.Value()
+			if mv.CanInterface() && HasBinary(mv.Interface()) {
+				return true
+			}
+		}
+		return false
 	default:
-		return IsBinary(data)
+		return false
 	}
-	return false
 }

parser/type.go

@@ -4,11 +4,12 @@ type (
 	PacketType byte
 
 	Packet struct {
-		Type        PacketType `json:"type" mapstructure:"type" msgpack:"type"`
-		Nsp         string     `json:"nsp" mapstructure:"nsp" msgpack:"nsp"`
-		Data        any        `json:"data,omitempty" mapstructure:"data,omitempty" msgpack:"data,omitempty"`
-		Id          *uint64    `json:"id,omitempty" mapstructure:"id,omitempty" msgpack:"id,omitempty"`
-		Attachments *uint64    `json:"attachments,omitempty" mapstructure:"attachments,omitempty" msgpack:"attachments,omitempty"`
+		Type              PacketType `json:"type" mapstructure:"type" msgpack:"type"`
+		Nsp               string     `json:"nsp" mapstructure:"nsp" msgpack:"nsp"`
+		preSerializedData string
+		Data              any     `json:"data,omitempty" mapstructure:"data,omitempty" msgpack:"data,omitempty"`
+		Id                *uint64 `json:"id,omitempty" mapstructure:"id,omitempty" msgpack:"id,omitempty"`
+		Attachments       *uint64 `json:"attachments,omitempty" mapstructure:"attachments,omitempty" msgpack:"attachments,omitempty"`
 	}
 )