Implement GOB (and avro) encoder and parser

config/https_gob_to_stdout.json

@@ -0,0 +1,17 @@
+{
+	"receivers": {
+		"r": {
+			"type": "http",
+			"address": "localhost:1339",
+			"handlers": {
+				"/": "something"
+			}
+		}
+	},
+	"handlers": {
+		"something": {
+			"parser": "gob",
+			"sender": "print"
+		}
+	}
+}

config/tester_to_https_gob.json

@@ -0,0 +1,25 @@
+{
+	"receivers": {
+		"r": {
+			"type": "test",
+			"delay": "1s",
+			"metrics": 2,
+			"values": 2,
+			"threads": 1,
+			"handler": "something"
+		}
+	},
+	"handlers": {
+		"something": {
+			"parser": "json",
+			"sender": "tohttps"
+		}
+	},
+	"senders": {
+		"tohttps": {
+			"type": "http",
+			"encoder": "gob",
+			"url": "http://localhost:1339/"
+		}
+	}
+}

encoder/auto.go

@@ -1,9 +1,9 @@
 /*
  * skogul, Encoder automation
  *
- * Copyright (c) 2019-2020 Telenor Norge AS
+ * Copyright (c) 2022 Telenor Norge AS
  * Author(s):
- *  - Kristian Lyngstøl <[email protected]>
+ *  - Roshini NarasimhaRaghavan([email protected])
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -54,4 +54,12 @@ func init() {
 		Help:     "Encodes the standard Skogul JSON format with indentation.",
 		AutoMake: true,
 	})
+	Auto.Add(skogul.Module{
+		Name:     "GOBskogul",
+		Aliases:  []string{"gob"},
+		Alloc:    func() interface{} { return &GOB{} },
+		Help:     "Encodes the GOB format.",
+		AutoMake: true,
+	})
+
 }

encoder/avro.go

@@ -0,0 +1,44 @@
+/*
+ * skogul, test avro encoder
+ *
+ * Copyright (c) 2022 Telenor Norge AS
+ * Author:
+ *  - Roshini Narasimha Raghavan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+package encoder
+
+import (
+	"os"
+
+	"github.com/hamba/avro"
+	"github.com/telenornms/skogul"
+)
+
+type AVRO struct {
+	Schema avro.Schema
+	In     skogul.Container
+}
+
+func (x AVRO) Encode(c *skogul.Container) ([]byte, error) {
+	var A AVRO
+	b, _ := os.ReadFile("./schema/avro_schema")
+	A.Schema = avro.MustParse(string(b))
+	A.In = *c
+	return avro.Marshal(A.Schema, A.In)
+}

encoder/gob.go

@@ -0,0 +1,51 @@
+/*
+ * skogul, test gob encoder
+ *
+ * Copyright (c) 2022 Telenor Norge AS
+ * Author:
+ *  - Roshini Narasimha Raghavan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+package encoder
+
+import (
+	"bytes"
+	"encoding/gob"
+
+	"github.com/telenornms/skogul"
+)
+
+type GOB struct{}
+
+// encode the content in the skogul container as a gob format
+func (x GOB) Encode(c *skogul.Container) ([]byte, error) {
+	var buf bytes.Buffer
+	enc := gob.NewEncoder(&buf)
+	err := enc.Encode(*c)
+	b := buf.Bytes()
+	return b, err
+}
+
+// encode the metrics in the skogul container as a gob format
+func (x GOB) EncodeMetric(m *skogul.Metric) ([]byte, error) {
+	var buf bytes.Buffer
+	enc := gob.NewEncoder(&buf)
+	err := enc.Encode(*m)
+	b := buf.Bytes()
+	return b, err
+}

encoder/gob_test.go

@@ -0,0 +1,128 @@
+/*
+ * skogul, test gob encoder
+ *
+ * Copyright (c) 2022 Telenor Norge AS
+ * Author:
+ *  - Roshini Narasimha Raghavan <[email protected]>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+package encoder_test
+
+import (
+	"bytes"
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"github.com/telenornms/skogul"
+	"github.com/telenornms/skogul/encoder"
+	"github.com/telenornms/skogul/parser"
+)
+
+// TestGOBEncode tests encoding of a simple GOB format from a skogul
+// container
+func TestGOBEncode(t *testing.T) {
+	testGOB(t, "./testdata/testdata.gob", true)
+}
+
+func testGOB(t *testing.T, file string, match bool) {
+	t.Helper()
+	c, orig := parseGOB(t, file)
+	b, err := encoder.GOB{}.Encode(c)
+
+	if err != nil {
+		t.Errorf("Encoding %s failed: %v", file, err)
+		return
+	}
+	if len(b) <= 0 {
+		t.Errorf("Encoding %s failed: zero length data", file)
+		return
+	}
+	if !match {
+		return
+	}
+	sorig := string(orig)
+	var origstring1 string
+	var old1 string
+	var old2 string
+	var old3 string
+	var new1 string
+	var new2 string
+	var new3 string
+	var result1, result2, result3 []byte
+	var n int
+	origstring1 = sorig
+	old1 = "\x1d"
+	old2 = "\x0e"
+	old3 = "main"
+	n = 1
+	new1 = "\x1f"
+	new2 = "\x10"
+	new3 = "skogul"
+	result1 = bytes.Replace([]byte(origstring1), []byte(old1), []byte(new1), n)
+	result2 = bytes.Replace([]byte(result1), []byte(old2), []byte(new2), n)
+	result3 = bytes.Replace([]byte(result2), []byte(old3), []byte(new3), n)
+
+	sorig_trim := string(result3)
+	snew := string(b)
+
+	//sorig_trim := strings.TrimSpace(string(result3))
+	//snew_trim := strings.TrimSpace(snew)
+
+	if len(sorig) < 2 {
+		t.Logf("Encoding %s failed: original pre-encoded length way too short. Shouldn't happen.", file)
+		t.FailNow()
+	}
+
+	result := strings.Compare(sorig_trim, snew)
+	if result != 0 {
+		t.Errorf("Encoding %s failed: original and newly encoded container doesn't match", file)
+		t.Logf("orig:\n'%s'", sorig_trim)
+		t.Logf("new:\n'%s'", snew)
+		t.Logf("result\n %d", result)
+		return
+	}
+
+}
+
+func parseGOB(t *testing.T, file string) (*skogul.Container, []byte) {
+	t.Helper()
+
+	b, err := ioutil.ReadFile(file)
+
+	if err != nil {
+		t.Logf("Failed to read test data file: %v", err)
+		t.FailNow()
+		return nil, nil
+	}
+	container, err := parser.GOB{}.Parse(b)
+
+	if err != nil {
+		t.Logf("Failed to parse GOB data: %v", err)
+		t.FailNow()
+		return nil, nil
+	}
+
+	if container == nil || container.Metrics == nil || len(container.Metrics) == 0 {
+		t.Logf("Expected parsed GOB to return a container with at least 1 metric")
+		t.FailNow()
+		return nil, nil
+	}
+	return container, b
+
+}

encoder/schema/avro_schema

@@ -0,0 +1,39 @@
+{
+	    "type": "record",
+	    "name": "simple",
+	    "namespace": "org.hamba.avro",
+	    "values": "string",
+	    "fields": [
+	    	{
+			"name": "Metrics", "type": {
+				"type": "array",
+				"items": {
+					"type": "record",
+					"name": "metrics",
+					"fields": [
+                        {
+							"name": "Time",
+							"type": "long.timestamp-micros"
+
+						},
+						{
+							"name": "Metadata",
+							"type": {
+								"type": "map",
+								"values": "string"
+							}
+						},
+                        {
+							"name": "Data",
+							"type": {
+								"type": "map",
+								"values": ["string", "int", "long", "float", "double" ]
+							}
+						}
+
+					]
+				}
+			}
+		}
+	]
+}

go.mod

@@ -3,19 +3,24 @@ module github.com/telenornms/skogul
 go 1.18
 
 require (
-	github.com/eclipse/paho.mqtt.golang v1.3.5
+	github.com/eclipse/paho.mqtt.golang v1.4.1
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/gogo/protobuf v1.3.2
-	github.com/lib/pq v1.10.4
-	github.com/segmentio/kafka-go v0.4.31
+	github.com/lib/pq v1.10.6
+	github.com/segmentio/kafka-go v0.4.32
 	github.com/sirupsen/logrus v1.8.1
 	google.golang.org/protobuf v1.28.0
 )
 
 require (
-	github.com/gorilla/websocket v1.4.2 // indirect
-	github.com/klauspost/compress v1.14.2 // indirect
-	github.com/pierrec/lz4/v4 v4.1.14 // indirect
-	golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/hamba/avro v1.8.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.15.6 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
+	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
+	golang.org/x/sys v0.0.0-20220624220833-87e55d714810 // indirect
 )