From 45972119cade27039590a094ecfa23c0f9bab47f Mon Sep 17 00:00:00 2001 From: David de Regt Date: Sun, 21 Jul 2024 01:14:18 -0400 Subject: [PATCH 1/4] feat: Support encoding/emitting nested structures with binary fields I've pulled in jsoniter to support passing a context into a json serialization, so we can stop walking and rewriting the entire passed structure to replace with placeholders. Instead, for the registered types, we replace them with placeholders inline as they're serialized, writing their attachments to a stream-context-passed attachments list, so it's multiple-simultaneous-encoding-safe. --- go.mod | 3 ++ go.sum | 8 +++++ parser/binary.go | 76 +++++++++++++++++++++------------------------ parser/encoder.go | 13 +++++--- parser/is-binary.go | 14 +++++++++ 5 files changed, 69 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index 054ff8a..5604a0b 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,9 @@ require ( github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/gookit/color v1.5.4 // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/onsi/ginkgo/v2 v2.12.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/quic-go v0.44.0 // indirect diff --git a/go.sum b/go.sum index b15988a..9a2d43e 100644 --- a/go.sum +++ b/go.sum @@ -11,14 +11,21 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= @@ -32,6 +39,7 @@ github.com/quic-go/quic-go v0.44.0/go.mod h1:z4cx/9Ny9UtGITIPzmPTXh1ULfOyWh4qGQl github.com/quic-go/webtransport-go v0.8.0 h1:HxSrwun11U+LlmwpgM1kEqIqH90IT4N8auv/cD7QFJg= github.com/quic-go/webtransport-go v0.8.0/go.mod h1:N99tjprW432Ut5ONql/aUhSLT0YVSlwHohQsuac9WaM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= diff --git a/parser/binary.go b/parser/binary.go index 58a3c6f..51cfeb9 100644 --- a/parser/binary.go +++ b/parser/binary.go @@ -1,9 +1,11 @@ package parser import ( + "bytes" "errors" - "io" + "unsafe" + jsoniter "github.com/json-iterator/go" "github.com/mitchellh/mapstructure" "github.com/zishang520/engine.io-go-parser/types" ) @@ -13,54 +15,46 @@ type Placeholder struct { Num int `json:"num" mapstructure:"num" msgpack:"num"` } +func init() { + jsoniter.RegisterTypeEncoderFunc("types.BytesBuffer", func(ptr unsafe.Pointer, stream *jsoniter.Stream) { + bb := ((*types.BytesBuffer)(ptr)) + + bufList := stream.Attachment.([]types.BufferInterface) + _placeholder := &Placeholder{Placeholder: true, Num: len(bufList)} + stream.WriteVal(_placeholder) + stream.Attachment = append(bufList, bb) + }, nil) + + jsoniter.RegisterTypeEncoderFunc("[]byte", func(ptr unsafe.Pointer, stream *jsoniter.Stream) { + bb := types.NewBytesBuffer(nil) + barr := ((*[]byte)(ptr)) + bb.Write(*barr) + + bufList := stream.Attachment.([]types.BufferInterface) + _placeholder := &Placeholder{Placeholder: true, Num: len(bufList)} + stream.WriteVal(_placeholder) + stream.Attachment = append(bufList, bb) + }, nil) +} + // Replaces every io.Reader | []byte in packet with a numbered placeholder. func DeconstructPacket(packet *Packet) (pack *Packet, buffers []types.BufferInterface) { pack = packet - pack.Data = _deconstructPacket(packet.Data, &buffers) + + // Run the serialization now, replacing any bytebuffers/[]byte found along the way with placeholders + buf := &bytes.Buffer{} + ns := jsoniter.NewStream(jsoniter.ConfigDefault, buf, buf.Cap()) + ns.Attachment = buffers + ns.WriteVal(pack.Data) + buffers = ns.Attachment.([]types.BufferInterface) + ns.Flush() + pack.Data = buf.String() + attachments := uint64(len(buffers)) pack.Attachments = &attachments // number of binary 'attachments' return pack, buffers } -func _deconstructPacket(data any, buffers *[]types.BufferInterface) any { - if data == nil { - return nil - } - - if IsBinary(data) { - _placeholder := &Placeholder{Placeholder: true, Num: len(*buffers)} - rdata := types.NewBytesBuffer(nil) - switch tdata := data.(type) { - case io.Reader: - if c, ok := data.(io.Closer); ok { - defer c.Close() - } - rdata.ReadFrom(tdata) - case []byte: - rdata.Write(tdata) - } - *buffers = append(*buffers, rdata) - return _placeholder - } - - switch tdata := data.(type) { - case []any: - newData := make([]any, 0, len(tdata)) - for _, v := range tdata { - newData = append(newData, _deconstructPacket(v, buffers)) - } - return newData - case map[string]any: - newData := map[string]any{} - for k, v := range tdata { - newData[k] = _deconstructPacket(v, buffers) - } - return newData - default: - return data - } -} - // Reconstructs a binary packet from its placeholder packet and buffers func ReconstructPacket(packet *Packet, buffers []types.BufferInterface) (*Packet, error) { data, err := _reconstructPacket(packet.Data, &buffers) diff --git a/parser/encoder.go b/parser/encoder.go index df470a4..b8d293f 100644 --- a/parser/encoder.go +++ b/parser/encoder.go @@ -53,9 +53,9 @@ func _encodeData(data any) any { newData[k] = _encodeData(v) } return newData - default: - return data } + + return data } // Encode packet as string. @@ -79,8 +79,13 @@ func (e *encoder) encodeAsString(packet *Packet) types.BufferInterface { } // json data if nil != packet.Data { - if b, err := json.Marshal(_encodeData(packet.Data)); err == nil { - str.Write(b) + if pds, is := packet.Data.(string); is { + // Already serialized in the DeconstructPacket function + str.WriteString(pds) + } else { + if b, err := json.Marshal(_encodeData(packet.Data)); err == nil { + str.Write(b) + } } } parser_log.Debug("encoded %v as %v", packet, str) diff --git a/parser/is-binary.go b/parser/is-binary.go index 68701b2..de3d050 100644 --- a/parser/is-binary.go +++ b/parser/is-binary.go @@ -2,6 +2,7 @@ package parser import ( "io" + "reflect" "strings" "github.com/zishang520/engine.io-go-parser/types" @@ -30,12 +31,25 @@ func HasBinary(data any) bool { return true } } + return false case map[string]any: for _, v := range o { if HasBinary(v) { return true } } + return false + } + dv := reflect.ValueOf(data) + if dv.Kind() == reflect.Struct { + for fi := range dv.NumField() { + dfv := dv.Field(fi) + if dfv.CanInterface() && HasBinary(dfv.Interface()) { + return true + } + } + return false } + return IsBinary(data) } From 111a35d8d97584e24f2c751ae5c81c24fcac2bfc Mon Sep 17 00:00:00 2001 From: David de Regt Date: Sun, 21 Jul 2024 23:14:27 -0400 Subject: [PATCH 2/4] PR feedback --- parser/binary.go | 2 +- parser/encoder.go | 4 ++-- parser/is-binary.go | 3 +++ parser/type.go | 11 ++++++----- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/parser/binary.go b/parser/binary.go index 51cfeb9..bb721cb 100644 --- a/parser/binary.go +++ b/parser/binary.go @@ -48,7 +48,7 @@ func DeconstructPacket(packet *Packet) (pack *Packet, buffers []types.BufferInte ns.WriteVal(pack.Data) buffers = ns.Attachment.([]types.BufferInterface) ns.Flush() - pack.Data = buf.String() + pack.preSerializedData = buf.String() attachments := uint64(len(buffers)) pack.Attachments = &attachments // number of binary 'attachments' diff --git a/parser/encoder.go b/parser/encoder.go index b8d293f..ee034bc 100644 --- a/parser/encoder.go +++ b/parser/encoder.go @@ -79,9 +79,9 @@ func (e *encoder) encodeAsString(packet *Packet) types.BufferInterface { } // json data if nil != packet.Data { - if pds, is := packet.Data.(string); is { + if packet.preSerializedData != "" { // Already serialized in the DeconstructPacket function - str.WriteString(pds) + str.WriteString(packet.preSerializedData) } else { if b, err := json.Marshal(_encodeData(packet.Data)); err == nil { str.Write(b) diff --git a/parser/is-binary.go b/parser/is-binary.go index de3d050..39c6287 100644 --- a/parser/is-binary.go +++ b/parser/is-binary.go @@ -41,6 +41,9 @@ func HasBinary(data any) bool { return false } dv := reflect.ValueOf(data) + if dv.Kind() == reflect.Pointer { + return IsBinary(dv.Elem().Interface()) + } if dv.Kind() == reflect.Struct { for fi := range dv.NumField() { dfv := dv.Field(fi) diff --git a/parser/type.go b/parser/type.go index 63a8e5a..677368b 100644 --- a/parser/type.go +++ b/parser/type.go @@ -4,11 +4,12 @@ type ( PacketType byte Packet struct { - Type PacketType `json:"type" mapstructure:"type" msgpack:"type"` - Nsp string `json:"nsp" mapstructure:"nsp" msgpack:"nsp"` - Data any `json:"data,omitempty" mapstructure:"data,omitempty" msgpack:"data,omitempty"` - Id *uint64 `json:"id,omitempty" mapstructure:"id,omitempty" msgpack:"id,omitempty"` - Attachments *uint64 `json:"attachments,omitempty" mapstructure:"attachments,omitempty" msgpack:"attachments,omitempty"` + Type PacketType `json:"type" mapstructure:"type" msgpack:"type"` + Nsp string `json:"nsp" mapstructure:"nsp" msgpack:"nsp"` + preSerializedData any + Data any `json:"data,omitempty" mapstructure:"data,omitempty" msgpack:"data,omitempty"` + Id *uint64 `json:"id,omitempty" mapstructure:"id,omitempty" msgpack:"id,omitempty"` + Attachments *uint64 `json:"attachments,omitempty" mapstructure:"attachments,omitempty" msgpack:"attachments,omitempty"` } ) From 005578a56f99f1f15b4a3b13989bce2cf4529078 Mon Sep 17 00:00:00 2001 From: David de Regt Date: Sun, 21 Jul 2024 23:15:30 -0400 Subject: [PATCH 3/4] forgot to commit this --- parser/type.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/type.go b/parser/type.go index 677368b..2db154f 100644 --- a/parser/type.go +++ b/parser/type.go @@ -6,7 +6,7 @@ type ( Packet struct { Type PacketType `json:"type" mapstructure:"type" msgpack:"type"` Nsp string `json:"nsp" mapstructure:"nsp" msgpack:"nsp"` - preSerializedData any + preSerializedData string Data any `json:"data,omitempty" mapstructure:"data,omitempty" msgpack:"data,omitempty"` Id *uint64 `json:"id,omitempty" mapstructure:"id,omitempty" msgpack:"id,omitempty"` Attachments *uint64 `json:"attachments,omitempty" mapstructure:"attachments,omitempty" msgpack:"attachments,omitempty"` From 379e950146b56437ae739f260e1d58a7bf042f9c Mon Sep 17 00:00:00 2001 From: David de Regt Date: Mon, 22 Jul 2024 00:30:51 -0400 Subject: [PATCH 4/4] fixed byte arrays and arrays-that-are-binary detection --- parser/binary.go | 2 +- parser/is-binary.go | 33 ++++++++++++++++++++++++++++----- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/parser/binary.go b/parser/binary.go index bb721cb..6ff21c4 100644 --- a/parser/binary.go +++ b/parser/binary.go @@ -25,7 +25,7 @@ func init() { stream.Attachment = append(bufList, bb) }, nil) - jsoniter.RegisterTypeEncoderFunc("[]byte", func(ptr unsafe.Pointer, stream *jsoniter.Stream) { + jsoniter.RegisterTypeEncoderFunc("[]uint8", func(ptr unsafe.Pointer, stream *jsoniter.Stream) { bb := types.NewBytesBuffer(nil) barr := ((*[]byte)(ptr)) bb.Write(*barr) diff --git a/parser/is-binary.go b/parser/is-binary.go index 39c6287..80421b6 100644 --- a/parser/is-binary.go +++ b/parser/is-binary.go @@ -40,11 +40,16 @@ func HasBinary(data any) bool { } return false } + + if IsBinary(data) { + return true + } + dv := reflect.ValueOf(data) - if dv.Kind() == reflect.Pointer { + switch dv.Kind() { + case reflect.Pointer: return IsBinary(dv.Elem().Interface()) - } - if dv.Kind() == reflect.Struct { + case reflect.Struct: for fi := range dv.NumField() { dfv := dv.Field(fi) if dfv.CanInterface() && HasBinary(dfv.Interface()) { @@ -52,7 +57,25 @@ func HasBinary(data any) bool { } } return false + case reflect.Array, reflect.Slice: + for i := range dv.Len() { + av := dv.Index(i) + if av.CanInterface() && HasBinary(av.Interface()) { + return true + } + } + return false + case reflect.Map: + mr := dv.MapRange() + for mr.Next() { + // Keys can't be binary blobs in json, so only check values + mv := mr.Value() + if mv.CanInterface() && HasBinary(mv.Interface()) { + return true + } + } + return false + default: + return false } - - return IsBinary(data) }