Implement GOB (and avro) encoder and parser

common.go

@@ -40,7 +40,6 @@ This is needed because the math package doesn't handle 32-bit variants.
 
 The pattern is:
 
-
 seeeeeee e0000000 00000000 00000000
 01234567 01234567 01234567 01234567
 
@@ -52,11 +51,13 @@ Value of 0, exponent of 1's, mean infinity, sign will determine, well, the
 sign (e.g. Inf vs -Inf).
 
 Could also set this as:
+
 	uvneginf = 0b11111111100000000000000000000000
 	uvinf    = 0b01111111100000000000000000000000
 
 Hex equivalents are just weird, so I left it as binary after first figuring
 out the hex encoding...
+
 	uvneginf = 0xFF800000
 	uvinf = 0x7F800000
 

docs/examples/avro/avro_schema

@@ -0,0 +1,33 @@
+{
+	"type": "record",
+	"name": "simple",
+	"namespace": "org.hamba.avro",
+	"values": "string",
+	"fields": [
+		{
+		"name": "Metrics", "type": {
+			"type": "array",
+			"items": {
+				"type": "record",
+				"name": "metrics",
+				"fields": [
+					{
+						"name": "Metadata",
+						"type": {
+							"type": "map",
+							"values": ["string", "int", "long", "float", "double" ]
+						}
+					},
+					{
+						"name": "Data",
+						"type": {
+							"type": "map",
+							"values": ["string", "int", "long", "float", "double" ]
+						}
+					}
+				]
+			}
+		}
+	}
+]
+}

docs/examples/gob/README.rst

@@ -0,0 +1,15 @@
+GOB examples
+==============
+
+WARNING: The GOB implementation is very much "MVP" and hast not _yet_
+been tested extensively.
+
+http_gob_to_stdout.json
+--------------------
+Receives the GOB data format and parses the GOB and prints the output in the stdout. 
+All communication is http. 
+
+tester_to_http_gob.json
+--------------------
+Encodes the incoming json data into the GOB data format. 
+All communication is http. 

docs/examples/gob/http_gob_to_stdout.json

@@ -0,0 +1,17 @@
+{
+	"receivers": {
+		"r": {
+			"type": "http",
+			"address": "localhost:1339",
+			"handlers": {
+				"/": "something"
+			}
+		}
+	},
+	"handlers": {
+		"something": {
+			"parser": "gob",
+			"sender": "print"
+		}
+	}
+}

docs/examples/gob/tester_to_http_gob.json

@@ -0,0 +1,25 @@
+{
+	"receivers": {
+		"r": {
+			"type": "test",
+			"delay": "1s",
+			"metrics": 2,
+			"values": 2,
+			"threads": 1,
+			"handler": "something"
+		}
+	},
+	"handlers": {
+		"something": {
+			"parser": "json",
+			"sender": "tohttps"
+		}
+	},
+	"senders": {
+		"tohttps": {
+			"type": "http",
+			"encoder": "gob",
+			"url": "http://localhost:1339/"
+		}
+	}
+}

encoder/auto.go

@@ -1,10 +1,10 @@
 /*
  * skogul, Encoder automation
  *
- * Copyright (c) 2019-2020 Telenor Norge AS
+ * Copyright (c) 2022 Telenor Norge AS
  * Author(s):
+ *  - Roshini NarasimhaRaghavan([email protected])
  *  - Kristian Lyngstøl <[email protected]>
- *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
@@ -54,4 +54,16 @@ func init() {
 		Help:     "Encodes the standard Skogul JSON format with indentation.",
 		AutoMake: true,
 	})
+	Auto.Add(skogul.Module{
+		Name:     "gob",
+		Alloc:    func() interface{} { return &GOB{} },
+		Help:     "Encodes the GOB format.",
+		AutoMake: true,
+	})
+	Auto.Add(skogul.Module{
+		Name:  "avro",
+		Alloc: func() interface{} { return &AVRO{} },
+		Help:  "Encodes the avro format.",
+	})
+
 }

encoder/avro.go

@@ -0,0 +1,61 @@
+/*
+ * skogul, test avro encoder
+ *
+ * Copyright (c) 2022 Telenor Norge AS
+ * Author:
+ *  - Roshini Narasimha Raghavan <[email protected]>
+  *  - Kristian Lyngstøl <[email protected]>
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+*/
+
+package encoder
+
+import (
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/hamba/avro"
+	"github.com/telenornms/skogul"
+)
+
+type AVRO struct {
+	Schema string
+	s      avro.Schema
+	err    error
+	once   sync.Once
+}
+
+func (x *AVRO) Encode(c *skogul.Container) ([]byte, error) {
+	x.once.Do(func() {
+		b, err := os.ReadFile(x.Schema)
+		x.err = err
+		if x.err == nil {
+			x.s = avro.MustParse(string(b))
+			if x.s == nil {
+				x.err = fmt.Errorf("parsed schema is nil")
+			}
+		}
+	})
+	if x.err != nil {
+		return nil, x.err
+	}
+	return avro.Marshal(x.s, c)
+}
+
+func (x *AVRO) EncodeMetric(m *skogul.Metric) ([]byte, error) {
+	return nil, fmt.Errorf("not supported")
+}

encoder/gob.go

@@ -0,0 +1,28 @@
+package encoder
+
+import (
+	"bytes"
+	"encoding/gob"
+
+	"github.com/telenornms/skogul"
+)
+
+type GOB struct{}
+
+// encode the content in the skogul container as a gob format
+func (x GOB) Encode(c *skogul.Container) ([]byte, error) {
+	var buf bytes.Buffer
+	enc := gob.NewEncoder(&buf)
+	err := enc.Encode(*c)
+	b := buf.Bytes()
+	return b, err
+}
+
+// encode the metrics in the skogul container as a gob format
+func (x GOB) EncodeMetric(m *skogul.Metric) ([]byte, error) {
+	var buf bytes.Buffer
+	enc := gob.NewEncoder(&buf)
+	err := enc.Encode(*m)
+	b := buf.Bytes()
+	return b, err
+}

encoder/gob_test.go

@@ -0,0 +1,128 @@
+/*
+ * skogul, test gob encoder
+ *
+ * Copyright (c) 2022 Telenor Norge AS
+ * Author:
+ *  - Roshini Narasimha Raghavan <[email protected]>
+ *  - Kristian Lyngstøl <[email protected]>
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+package encoder_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/telenornms/skogul"
+	"github.com/telenornms/skogul/encoder"
+	"github.com/telenornms/skogul/parser"
+)
+
+// TestGOBEncode tests encoding of a simple GOB format from a skogul
+// container
+func TestGOBEncode(t *testing.T) {
+	testGOB(t, "./testdata/testdata.gob", true)
+}
+
+func testGOB(t *testing.T, file string, match bool) {
+	t.Helper()
+	c, orig := parseGOB(t, file)
+	b, err := encoder.GOB{}.Encode(c)
+
+	if err != nil {
+		t.Errorf("Encoding %s failed: %v", file, err)
+		return
+	}
+	if len(b) <= 0 {
+		t.Errorf("Encoding %s failed: zero length data", file)
+		return
+	}
+	if !match {
+		return
+	}
+
+	//The test gob data is generated using a separate program and there are differences in 3 locations compared to the encoded data.
+	//These differences are fixed by replacing the differences with exact matches before the comparison.
+	sorig := string(orig)
+	var origstring1 string
+	var old1 string
+	var old2 string
+	var old3 string
+	var new1 string
+	var new2 string
+	var new3 string
+	var result1, result2, result3 []byte
+	var n int
+	origstring1 = sorig
+	old1 = "\x1d"
+	old2 = "\x0e"
+	old3 = "main"
+	n = 1
+	new1 = "\x1f"
+	new2 = "\x10"
+	new3 = "skogul"
+	result1 = bytes.Replace([]byte(origstring1), []byte(old1), []byte(new1), n)
+	result2 = bytes.Replace([]byte(result1), []byte(old2), []byte(new2), n)
+	result3 = bytes.Replace([]byte(result2), []byte(old3), []byte(new3), n)
+
+	sorig_trim := string(result3)
+	snew := string(b)
+
+	if len(sorig) < 2 {
+		t.Logf("Encoding %s failed: original pre-encoded length way too short. Shouldn't happen.", file)
+		t.FailNow()
+	}
+
+	result := strings.Compare(sorig_trim, snew)
+	if result != 0 {
+		t.Errorf("Encoding %s failed: original and newly encoded container doesn't match", file)
+		t.Logf("orig:\n'%s'", sorig_trim)
+		t.Logf("new:\n'%s'", snew)
+		t.Logf("result\n %d", result)
+		return
+	}
+
+}
+
+func parseGOB(t *testing.T, file string) (*skogul.Container, []byte) {
+	t.Helper()
+
+	b, err := ioutil.ReadFile(file)
+
+	if err != nil {
+		t.Logf("Failed to read test data file: %v", err)
+		t.FailNow()
+		return nil, nil
+	}
+	container, err := parser.GOB{}.Parse(b)
+
+	if err != nil {
+		t.Logf("Failed to parse GOB data: %v", err)
+		t.FailNow()
+		return nil, nil
+	}
+
+	if container == nil || container.Metrics == nil || len(container.Metrics) == 0 {
+		t.Logf("Expected parsed GOB to return a container with at least 1 metric")
+		t.FailNow()
+		return nil, nil
+	}
+	return container, b
+
+}

go.mod

@@ -3,19 +3,24 @@ module github.com/telenornms/skogul
 go 1.18
 
 require (
-	github.com/eclipse/paho.mqtt.golang v1.3.5
+	github.com/eclipse/paho.mqtt.golang v1.4.1
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/gogo/protobuf v1.3.2
-	github.com/lib/pq v1.10.4
-	github.com/segmentio/kafka-go v0.4.31
+	github.com/hamba/avro v1.8.0
+	github.com/lib/pq v1.10.6
+	github.com/segmentio/kafka-go v0.4.32
 	github.com/sirupsen/logrus v1.8.1
 	google.golang.org/protobuf v1.28.0
 )
 
 require (
-	github.com/gorilla/websocket v1.4.2 // indirect
-	github.com/klauspost/compress v1.14.2 // indirect
-	github.com/pierrec/lz4/v4 v4.1.14 // indirect
-	golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.15.6 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
+	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
+	golang.org/x/sys v0.0.0-20220624220833-87e55d714810 // indirect
 )