diff --git a/assets/docs/configuration/transformations/snowplow-builtin/spCollectorPayloadThriftToJSON-full-example.hcl b/assets/docs/configuration/transformations/snowplow-builtin/spCollectorPayloadThriftToJSON-full-example.hcl new file mode 100644 index 00000000..7b33cc4b --- /dev/null +++ b/assets/docs/configuration/transformations/snowplow-builtin/spCollectorPayloadThriftToJSON-full-example.hcl @@ -0,0 +1,4 @@ +transform { + use "spCollectorPayloadThriftToJSON" { + } +} diff --git a/assets/docs/configuration/transformations/snowplow-builtin/spCollectorPayloadThriftToJSON-minimal-example.hcl b/assets/docs/configuration/transformations/snowplow-builtin/spCollectorPayloadThriftToJSON-minimal-example.hcl new file mode 100644 index 00000000..7b33cc4b --- /dev/null +++ b/assets/docs/configuration/transformations/snowplow-builtin/spCollectorPayloadThriftToJSON-minimal-example.hcl @@ -0,0 +1,4 @@ +transform { + use "spCollectorPayloadThriftToJSON" { + } +} diff --git a/assets/docs/configuration/transformations/snowplow-builtin/spJSONToCollectorPayloadThrift-full-example.hcl b/assets/docs/configuration/transformations/snowplow-builtin/spJSONToCollectorPayloadThrift-full-example.hcl new file mode 100644 index 00000000..93ee69e0 --- /dev/null +++ b/assets/docs/configuration/transformations/snowplow-builtin/spJSONToCollectorPayloadThrift-full-example.hcl @@ -0,0 +1,6 @@ +transform { + use "spJSONToCollectorPayloadThrift" { + # Whether the output thrift should be further encoded with base64 + base_64_encode = true + } +} diff --git a/assets/docs/configuration/transformations/snowplow-builtin/spJSONToCollectorPayloadThrift-minimal-example.hcl b/assets/docs/configuration/transformations/snowplow-builtin/spJSONToCollectorPayloadThrift-minimal-example.hcl new file mode 100644 index 00000000..892d49c1 --- /dev/null +++ b/assets/docs/configuration/transformations/snowplow-builtin/spJSONToCollectorPayloadThrift-minimal-example.hcl @@ -0,0 +1,4 @@ +transform { + use "spJSONToCollectorPayloadThrift" { + } +} diff --git a/docs/configuration_transformations_docs_test.go b/docs/configuration_transformations_docs_test.go index 80349fc5..b73d60d1 100644 --- a/docs/configuration_transformations_docs_test.go +++ b/docs/configuration_transformations_docs_test.go @@ -95,12 +95,10 @@ func TestScriptTransformationCustomScripts(t *testing.T) { testLuaScriptCompiles(t, file) case ".hcl": isFull := strings.Contains(file, "full-example") - testTransformationConfig(t, file, isFull) case "": // If there's no extension, fail the test. assert.Fail("File with no extension found: %v", file) - default: // Otherwise it's likely a typo or error. assert.Fail("unexpected file extension found: %v", file) @@ -141,6 +139,12 @@ func testTransformationConfig(t *testing.T, filepath string, fullExample bool) { configObject = &transform.SetPkConfig{} case "spEnrichedToJson": configObject = &transform.EnrichedToJSONConfig{} + case "spCollectorPayloadThriftToJSON": + configObject = &transform.CollectorPayloadThriftToJSONConfig{} + case "spJSONToCollectorPayloadThrift": + configObject = &transform.JSONToCollectorPayloadThriftConfig{} + case "JSONManipulator": + configObject = &transform.JSONManipulatorConfig{} case "js": configObject = &engine.JSEngineConfig{} case "lua": @@ -183,9 +187,7 @@ func testJSScriptCompiles(t *testing.T, scriptPath string) { assert.NotNil(jsTransformationFunc, scriptPath) if err != nil { t.Fatalf("JSConfigFunction failed with error: %s. Script: %s", err.Error(), string(scriptPath)) - } - } func testLuaScriptCompiles(t *testing.T, scriptPath string) { diff --git a/go.mod b/go.mod index 5582bea3..364d1160 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( ) require ( + github.com/apache/thrift v0.17.0 github.com/davecgh/go-spew v1.1.1 github.com/dop251/goja v0.0.0-20220722151623-4765a9872229 github.com/hashicorp/hcl/v2 v2.13.0 diff --git a/go.sum b/go.sum index 262d0250..b87a11b5 100644 --- a/go.sum +++ b/go.sum @@ -26,7 +26,6 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw1U= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= @@ -47,11 +46,9 @@ cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9U cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/kms v1.4.0 h1:iElbfoE61VeLhnZcGOltqL8HIly8Nhbe5t6JlH9GXjo= -cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -129,7 +126,6 @@ github.com/Shopify/sarama v1.34.1 h1:pVCQO7BMAK3s1jWhgi5v1W6lwZ6Veiekfc2vsgRS06Y github.com/Shopify/sarama v1.34.1/go.mod h1:NZSNswsnStpq8TUdFaqnpXm2Do6KRzTIjdBdVlL1YRM= github.com/Shopify/toxiproxy/v2 v2.4.0 h1:O1e4Jfvr/hefNTNu+8VtdEG5lSeamJRo4aKhMOKNM64= github.com/Shopify/toxiproxy/v2 v2.4.0/go.mod h1:3ilnjng821bkozDRxNoo64oI/DKqM+rOyJzb564+bvg= -github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= @@ -139,13 +135,11 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= -github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0= -github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= +github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo= +github.com/apache/thrift v0.17.0/go.mod h1:OLxhMRJxomX+1I/KUw03qoV3mMz16BwaKI+d4fPBx7Q= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go v1.25.19/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.44.60 h1:KTTogelVR+4dWiIPl7eyxoxaJkziChON6/Y/hVfTipk= github.com/aws/aws-sdk-go v1.44.60/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= @@ -154,7 +148,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/caarlos0/env/v6 v6.9.3 h1:Tyg69hoVXDnpO5Qvpsu8EoquarbPyQb+YwExWHP8wWU= github.com/caarlos0/env/v6 v6.9.3/go.mod h1:hvp/ryKXKipEkcuYjs9mI4bBCg+UI0Yhgm5Zu0ddvwc= -github.com/cactus/go-statsd-client/statsd v0.0.0-20190922113730-52b467de415c/go.mod h1:D4RDtP0MffJ3+R36OkGul0LwJLIN8nRb0Ac6jZmJCmo= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -176,7 +169,6 @@ github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcju github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= @@ -232,9 +224,7 @@ github.com/getsentry/sentry-go v0.13.0 h1:20dgTiUSfxRB/EhMPtxcL9ZEbM1ZdR+W/7f7NW github.com/getsentry/sentry-go v0.13.0/go.mod h1:EOsfu5ZdvKPfeHYV6pTVQnsjfp30+XA7//UooKNumH0= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= -github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= @@ -248,21 +238,14 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= -github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= @@ -282,7 +265,6 @@ github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -343,7 +325,6 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -386,7 +367,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.13.0 h1:0Apadu1w6M11dyGFxWnmhhcMjkbAiKCv7G1r/2QgCNc= github.com/hashicorp/hcl/v2 v2.13.0/go.mod h1:e4z5nxYlWNPdDSNYX+ph14EvWYMFm3eP0zIUqPc2jr0= @@ -414,7 +394,6 @@ github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJz github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -460,20 +439,13 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= -github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= -github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -483,7 +455,6 @@ github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -544,7 +515,6 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -580,7 +550,6 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -606,9 +575,7 @@ github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKn github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -635,11 +602,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= -github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= -github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zclconf/go-cty v1.10.0 h1:mp9ZXQeIcN8kAwuqorjH+Q+njbJKjLrvB2yIh4q7U+0= github.com/zclconf/go-cty v1.10.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= -github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -661,11 +625,9 @@ golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -704,7 +666,6 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -722,7 +683,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191028085509-fe3aa8a45271/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -747,7 +707,6 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -802,7 +761,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -821,7 +779,6 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -848,7 +805,6 @@ golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -860,8 +816,6 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -895,8 +849,6 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/transform/json_manipulator.go b/pkg/transform/json_manipulator.go new file mode 100644 index 00000000..4fff1161 --- /dev/null +++ b/pkg/transform/json_manipulator.go @@ -0,0 +1,212 @@ +// +// Copyright (c) 2023-present Snowplow Analytics Ltd. All rights reserved. +// +// This program is licensed to you under the Snowplow Community License Version 1.0, +// and you may not use this file except in compliance with the Snowplow Community License Version 1.0. +// You may obtain a copy of the Snowplow Community License Version 1.0 at https://docs.snowplow.io/community-license-1.0 + +package transform + +import ( + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/snowplow/snowbridge/config" + "github.com/snowplow/snowbridge/pkg/models" +) + +const ( + snowplowPayloadDataSchema = "iglu:com.snowplowanalytics.snowplow/payload_data/jsonschema/1-0-4" +) + +// JSONManipulatorConfig is a configuration object for the JSONManipulator transformation +type JSONManipulatorConfig struct { + KeyRename map[string]string `hcl:"key_rename"` + KeyValueFunc map[string]string `hcl:"key_value_func"` +} + +// JSONManipulatorAdapter is a configuration object for the JSONManipulator transformation +type JSONManipulatorAdapter func(i interface{}) (interface{}, error) + +// Create implements the ComponentCreator interface. +func (f JSONManipulatorAdapter) Create(i interface{}) (interface{}, error) { + return f(i) +} + +// ProvideDefault implements the ComponentConfigurable interface +func (f JSONManipulatorAdapter) ProvideDefault() (interface{}, error) { + // Provide defaults + cfg := &JSONManipulatorConfig{ + KeyRename: make(map[string]string), + KeyValueFunc: make(map[string]string), + } + + return cfg, nil +} + +// JSONManipulatorAdapterGenerator returns a JSONManipulator transformation adapter. +func JSONManipulatorAdapterGenerator(f func(c *JSONManipulatorConfig) (TransformationFunction, error)) JSONManipulatorAdapter { + return func(i interface{}) (interface{}, error) { + cfg, ok := i.(*JSONManipulatorConfig) + if !ok { + return nil, errors.New("invalid input, expected JSONManipulatorConfig") + } + + return f(cfg) + } +} + +// JSONManipulatorConfigFunction returns an JSONManipulator transformation function, from an JSONManipulatorConfig. +func JSONManipulatorConfigFunction(c *JSONManipulatorConfig) (TransformationFunction, error) { + return NewJSONManipulator( + c.KeyRename, + c.KeyValueFunc, + ) +} + +// JSONManipulatorConfigPair is a configuration pair for the JSONManipulator transformation +var JSONManipulatorConfigPair = config.ConfigurationPair{ + Name: "JSONManipulator", + Handle: JSONManipulatorAdapterGenerator(JSONManipulatorConfigFunction), +} + +// --- Manipulator Value Functions + +// toEpochMillis attempts to convert an RFC3339 string to a Unix Timestamp in milliseconds +func toEpochMillis(v interface{}) (int64, error) { + switch v.(type) { + case string: + vTime, err := time.Parse(time.RFC3339, v.(string)) + if err != nil { + return -1, err + } + return vTime.UnixNano() / (int64(time.Millisecond) / int64(time.Nanosecond)), nil + default: + return -1, errors.New(fmt.Sprintf("input value for 'toEpochMillis' must be a 'string' was '%T'", v)) + } +} + +// stringValueFromMap tries to extract a value from a map and then casts the +// value to a string type or returns an error +func stringValueFromMap(input map[string]interface{}, key string) (string, error) { + keyValue, ok := input[key] + if !ok { + return "", errors.New(fmt.Sprintf("key '%s' does not exist in input map", key)) + } + keyValueStr, ok := keyValue.(string) + if !ok { + return "", errors.New(fmt.Sprintf("key '%s' must be a 'string' was '%T'", key, keyValue)) + } + return keyValueStr, nil +} + +// toSnowplowPayloadData converts an input []map[string]string which contains +// a set of key-value pair objects into a valid Snowplow Payload data structure +// encoded as a JSON string +func toSnowplowPayloadData(v interface{}) (string, error) { + switch v.(type) { + case []interface{}: + dataMap := make(map[string]string) + for _, param := range v.([]interface{}) { + paramMap, ok := param.(map[string]interface{}) + if !ok { + return "", errors.New(fmt.Sprintf("input values for 'toSnowplowPayloadData' within array must be a 'map[string]interface {}' was '%T'", param)) + } + name, err := stringValueFromMap(paramMap, "name") + if err != nil { + return "", err + } + value, err := stringValueFromMap(paramMap, "value") + if err != nil { + return "", err + } + dataMap[name] = value + } + + dataArray := make([]map[string]string, 0) + dataArray = append(dataArray, dataMap) + + snowplowPayload := make(map[string]interface{}) + snowplowPayload["schema"] = snowplowPayloadDataSchema + snowplowPayload["data"] = dataArray + + snowplowPayloadStr, err := json.Marshal(snowplowPayload) + if err != nil { + return "", err + } + return string(snowplowPayloadStr), nil + default: + return "", errors.New(fmt.Sprintf("input value for 'toSnowplowPayloadData' must be a '[]interface {}' was '%T'", v)) + } +} + +// --- Manipulator Functions + +// mapKeyRename takes an input map and renames keys it finds in the replace instructions +func mapKeyRename(input map[string]interface{}, keyRename map[string]string) map[string]interface{} { + for old, new := range keyRename { + if _, ok := input[old]; ok { + input[new] = input[old] + delete(input, old) + } + } + return input +} + +// mapKeyValueFunc runs pre-defined functions against a value specified by the input key +func mapKeyValueFunc(input map[string]interface{}, keyValueFunc map[string]string) (map[string]interface{}, error) { + for key, funcToRun := range keyValueFunc { + var valNew interface{} + var err error + if val, ok := input[key]; ok { + switch funcToRun { + case "toEpochMillis": + valNew, err = toEpochMillis(val) + case "toSnowplowPayloadData": + valNew, err = toSnowplowPayloadData(val) + default: + return nil, errors.New(fmt.Sprintf("value func '%s' is not defined", funcToRun)) + } + } + if err != nil { + return nil, err + } + input[key] = valNew + } + return input, nil +} + +// NewJSONManipulator returns a transformation implementation to transform an input JSON string according to the configured manipulation +// instructions provided in the configuration +func NewJSONManipulator(keyRename map[string]string, keyValueFunc map[string]string) (TransformationFunction, error) { + return func(message *models.Message, intermediateState interface{}) (*models.Message, *models.Message, *models.Message, interface{}) { + // 1. Unmarshal inbound message to a map + var input map[string]interface{} + unmarshallErr := json.Unmarshal(message.Data, &input) + if unmarshallErr != nil { + message.SetError(unmarshallErr) + return nil, nil, message, nil + } + + // 2. Rename keys in input JSON + renamed := mapKeyRename(input, keyRename) + + // 3. Apply value functions on renamed JSON + manipulated, valueFuncErr := mapKeyValueFunc(renamed, keyValueFunc) + if valueFuncErr != nil { + message.SetError(valueFuncErr) + return nil, nil, message, nil + } + + // 4. Marshal back to a JSON string + res, jsonErr := json.Marshal(manipulated) + if jsonErr != nil { + message.SetError(jsonErr) + return nil, nil, message, nil + } + message.Data = res + return message, nil, nil, intermediateState + }, nil +} diff --git a/pkg/transform/snowplow_collector_payload_thrift_to_json.go b/pkg/transform/snowplow_collector_payload_thrift_to_json.go new file mode 100644 index 00000000..b4bef34c --- /dev/null +++ b/pkg/transform/snowplow_collector_payload_thrift_to_json.go @@ -0,0 +1,83 @@ +// +// Copyright (c) 2023-present Snowplow Analytics Ltd. All rights reserved. +// +// This program is licensed to you under the Snowplow Community License Version 1.0, +// and you may not use this file except in compliance with the Snowplow Community License Version 1.0. +// You may obtain a copy of the Snowplow Community License Version 1.0 at https://docs.snowplow.io/community-license-1.0 + +package transform + +import ( + "context" + "errors" + + "github.com/snowplow/snowbridge/config" + "github.com/snowplow/snowbridge/pkg/models" + + collectorpayload "github.com/snowplow/snowbridge/third_party/snowplow/collectorpayload" +) + +// CollectorPayloadThriftToJSONConfig is a configuration object for the spCollectorPayloadThriftToJSON transformation +type CollectorPayloadThriftToJSONConfig struct { +} + +type collectorPayloadThriftToJSONAdapter func(i interface{}) (interface{}, error) + +// Create implements the ComponentCreator interface. +func (f collectorPayloadThriftToJSONAdapter) Create(i interface{}) (interface{}, error) { + return f(i) +} + +// ProvideDefault implements the ComponentConfigurable interface +func (f collectorPayloadThriftToJSONAdapter) ProvideDefault() (interface{}, error) { + // Provide defaults + cfg := &CollectorPayloadThriftToJSONConfig{} + + return cfg, nil +} + +// adapterGenerator returns a spCollectorPayloadThriftToJSON transformation adapter. +func collectorPayloadThriftToJSONAdapterGenerator(f func(c *CollectorPayloadThriftToJSONConfig) (TransformationFunction, error)) collectorPayloadThriftToJSONAdapter { + return func(i interface{}) (interface{}, error) { + cfg, ok := i.(*CollectorPayloadThriftToJSONConfig) + if !ok { + return nil, errors.New("invalid input, expected collectorPayloadThriftToJSONConfig") + } + + return f(cfg) + } +} + +// collectorPayloadThriftToJSONConfigFunction returns an spCollectorPayloadThriftToJSON transformation function, from an collectorPayloadThriftToJSONConfig. +func collectorPayloadThriftToJSONConfigFunction(c *CollectorPayloadThriftToJSONConfig) (TransformationFunction, error) { + return SpCollectorPayloadThriftToJSON, nil +} + +// CollectorPayloadThriftToJSONConfigPair is a configuration pair for the spCollectorPayloadThriftToJSON transformation +var CollectorPayloadThriftToJSONConfigPair = config.ConfigurationPair{ + Name: "spCollectorPayloadThriftToJSON", + Handle: collectorPayloadThriftToJSONAdapterGenerator(collectorPayloadThriftToJSONConfigFunction), +} + +// SpCollectorPayloadThriftToJSON is a specific transformation implementation to transform a Thrift encoded Collector Payload +// to a JSON string representation. +func SpCollectorPayloadThriftToJSON(message *models.Message, intermediateState interface{}) (*models.Message, *models.Message, *models.Message, interface{}) { + ctx := context.Background() + + // Deserialize the Collector Payload to a struct + res, deserializeErr := collectorpayload.BinaryDeserializer(ctx, message.Data) + if deserializeErr != nil { + message.SetError(deserializeErr) + return nil, nil, message, nil + } + + // Re-encode as a JSON string to be able to leverage it downstream + resJSON, jsonErr := collectorpayload.ToJSON(res) + if jsonErr != nil { + message.SetError(jsonErr) + return nil, nil, message, nil + } + + message.Data = resJSON + return message, nil, nil, intermediateState +} diff --git a/pkg/transform/snowplow_collector_payload_thrift_to_json_test.go b/pkg/transform/snowplow_collector_payload_thrift_to_json_test.go new file mode 100644 index 00000000..a548d121 --- /dev/null +++ b/pkg/transform/snowplow_collector_payload_thrift_to_json_test.go @@ -0,0 +1,57 @@ +// +// Copyright (c) 2023-present Snowplow Analytics Ltd. All rights reserved. +// +// This program is licensed to you under the Snowplow Community License Version 1.0, +// and you may not use this file except in compliance with the Snowplow Community License Version 1.0. +// You may obtain a copy of the Snowplow Community License Version 1.0 at https://docs.snowplow.io/community-license-1.0 + +package transform + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/snowplow/snowbridge/pkg/models" +) + +func TestSpCollectorPayloadThriftToJSON(t *testing.T) { + assert := assert.New(t) + + var messageGood = models.Message{ + Data: []byte(`CwBkAAAACjE3Mi4xNy4wLjEKAMgAAAGF27kNagsA0gAAAAVVVEYtOAsA3AAAABFzc2MtMi44LjItc3Rkb3V0JAsBLAAAAAtjdXJsLzcuODUuMAsBQAAAAAIvaQsBSgAAAAplPXB2JnA9YWlkDwFeCwAAAAQAAAAbVGltZW91dC1BY2Nlc3M6IDxmdW5jdGlvbjE+AAAAFEhvc3Q6IGxvY2FsaG9zdDo4MDgwAAAAF1VzZXItQWdlbnQ6IGN1cmwvNy44NS4wAAAAC0FjY2VwdDogKi8qCwGQAAAACWxvY2FsaG9zdAsBmgAAACQ1ZmZjODFlMS03YmYyLTQwODQtODBhYS1mODg3NGE1M2NlNTALemkAAABBaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvQ29sbGVjdG9yUGF5bG9hZC90aHJpZnQvMS0wLTAA`), + PartitionKey: "some-key", + } + + var messageBad = models.Message{ + Data: []byte(`not-a-json`), + PartitionKey: "some-key4", + } + + var expectedGood = models.Message{ + Data: []byte(`{"ipAddress":"172.17.0.1","timestamp":1674428616042,"encoding":"UTF-8","collector":"ssc-2.8.2-stdout$","userAgent":"curl/7.85.0","path":"/i","querystring":"e=pv&p=aid","headers":["Timeout-Access: ","Host: localhost:8080","User-Agent: curl/7.85.0","Accept: */*"],"hostname":"localhost","networkUserId":"5ffc81e1-7bf2-4084-80aa-f8874a53ce50","schema":"iglu:com.snowplowanalytics.snowplow/CollectorPayload/thrift/1-0-0"}`), + PartitionKey: "some-key", + } + + // Simple success case + transformSuccess, _, failure, _ := SpCollectorPayloadThriftToJSON(&messageGood, nil) + + assert.Equal(expectedGood.PartitionKey, transformSuccess.PartitionKey) + assert.JSONEq(string(expectedGood.Data), string(transformSuccess.Data)) + assert.Nil(failure) + + // Simple failure case + success, _, transformFailure, _ := SpCollectorPayloadThriftToJSON(&messageBad, nil) + + // Not matching equivalence of whole object because error stacktrace makes it unfeasible. Doing each component part instead. + assert.NotNil(transformFailure.GetError()) + if transformFailure.GetError() != nil { + assert.Equal("Unknown data type 110", transformFailure.GetError().Error()) + } + assert.Equal([]byte("not-a-json"), transformFailure.Data) + assert.Equal("some-key4", transformFailure.PartitionKey) + assert.Nil(success) + + // Check that the input has not been altered + assert.Nil(messageGood.GetError()) +} diff --git a/pkg/transform/snowplow_json_to_collector_payload_thrift.go b/pkg/transform/snowplow_json_to_collector_payload_thrift.go new file mode 100644 index 00000000..8cf5956f --- /dev/null +++ b/pkg/transform/snowplow_json_to_collector_payload_thrift.go @@ -0,0 +1,99 @@ +// +// Copyright (c) 2023-present Snowplow Analytics Ltd. All rights reserved. +// +// This program is licensed to you under the Snowplow Community License Version 1.0, +// and you may not use this file except in compliance with the Snowplow Community License Version 1.0. +// You may obtain a copy of the Snowplow Community License Version 1.0 at https://docs.snowplow.io/community-license-1.0 + +package transform + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + + "github.com/snowplow/snowbridge/config" + "github.com/snowplow/snowbridge/pkg/models" + + collectorpayload "github.com/snowplow/snowbridge/third_party/snowplow/collectorpayload" + collectorpayloadmodel1 "github.com/snowplow/snowbridge/third_party/snowplow/collectorpayload/gen-go/model1" +) + +// JSONToCollectorPayloadThriftConfig is a configuration object for the spJSONToCollectorPayloadThrift transformation +type JSONToCollectorPayloadThriftConfig struct { + Base64Encode bool `hcl:"base_64_encode"` +} + +// JSONToCollectorPayloadThriftAdapter is a configuration object for the spJSONToCollectorPayloadThrift transformation +type JSONToCollectorPayloadThriftAdapter func(i interface{}) (interface{}, error) + +// Create implements the ComponentCreator interface. +func (f JSONToCollectorPayloadThriftAdapter) Create(i interface{}) (interface{}, error) { + return f(i) +} + +// ProvideDefault implements the ComponentConfigurable interface +func (f JSONToCollectorPayloadThriftAdapter) ProvideDefault() (interface{}, error) { + // Provide defaults + cfg := &JSONToCollectorPayloadThriftConfig{ + Base64Encode: false, + } + + return cfg, nil +} + +// JSONToCollectorPayloadThriftAdapterGenerator returns a spJSONToCollectorPayloadThrift transformation adapter. +func JSONToCollectorPayloadThriftAdapterGenerator(f func(c *JSONToCollectorPayloadThriftConfig) (TransformationFunction, error)) JSONToCollectorPayloadThriftAdapter { + return func(i interface{}) (interface{}, error) { + cfg, ok := i.(*JSONToCollectorPayloadThriftConfig) + if !ok { + return nil, errors.New("invalid input, expected JSONToCollectorPayloadThriftConfig") + } + + return f(cfg) + } +} + +// JSONToCollectorPayloadThriftConfigFunction returns an spJSONToCollectorPayloadThrift transformation function, from an JSONToCollectorPayloadThriftConfig. +func JSONToCollectorPayloadThriftConfigFunction(c *JSONToCollectorPayloadThriftConfig) (TransformationFunction, error) { + return NewSpJSONToCollectorPayloadThrift( + c.Base64Encode, + ) +} + +// JSONToCollectorPayloadThriftConfigPair is a configuration pair for the spJSONToCollectorPayloadThrift transformation +var JSONToCollectorPayloadThriftConfigPair = config.ConfigurationPair{ + Name: "spJSONToCollectorPayloadThrift", + Handle: JSONToCollectorPayloadThriftAdapterGenerator(JSONToCollectorPayloadThriftConfigFunction), +} + +// NewSpJSONToCollectorPayloadThrift returns a transformation implementation to transform a raw message into a valid Thrift encoded Collector Payload +// so that it can be pushed directly into the egress stream of a Collector. +func NewSpJSONToCollectorPayloadThrift(base64Encode bool) (TransformationFunction, error) { + return func(message *models.Message, intermediateState interface{}) (*models.Message, *models.Message, *models.Message, interface{}) { + var p *collectorpayloadmodel1.CollectorPayload + unmarshallErr := json.Unmarshal(message.Data, &p) + if unmarshallErr != nil { + message.SetError(unmarshallErr) + return nil, nil, message, nil + } + + ctx := context.Background() + + res, serializeErr := collectorpayload.BinarySerializer(ctx, p) + if serializeErr != nil { + message.SetError(serializeErr) + return nil, nil, message, nil + } + + // Optionally base64 encode the output + if base64Encode { + message.Data = []byte(base64.StdEncoding.EncodeToString(res)) + } else { + message.Data = res + } + + return message, nil, nil, intermediateState + }, nil +} diff --git a/pkg/transform/snowplow_json_to_collector_payload_thrift_test.go b/pkg/transform/snowplow_json_to_collector_payload_thrift_test.go new file mode 100644 index 00000000..84e769bf --- /dev/null +++ b/pkg/transform/snowplow_json_to_collector_payload_thrift_test.go @@ -0,0 +1,59 @@ +// +// Copyright (c) 2023-present Snowplow Analytics Ltd. All rights reserved. +// +// This program is licensed to you under the Snowplow Community License Version 1.0, +// and you may not use this file except in compliance with the Snowplow Community License Version 1.0. +// You may obtain a copy of the Snowplow Community License Version 1.0 at https://docs.snowplow.io/community-license-1.0 + +package transform + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/snowplow/snowbridge/pkg/models" +) + +func TestSpJSONToCollectorPayloadThrift(t *testing.T) { + assert := assert.New(t) + + var messageGood = models.Message{ + Data: []byte(`{"ipAddress":"172.17.0.1","timestamp":1674428616042,"encoding":"UTF-8","collector":"ssc-2.8.2-stdout$","userAgent":"curl/7.85.0","path":"/i","querystring":"e=pv&p=aid","headers":["Timeout-Access: ","Host: localhost:8080","User-Agent: curl/7.85.0","Accept: */*"],"hostname":"localhost","networkUserId":"5ffc81e1-7bf2-4084-80aa-f8874a53ce50","schema":"iglu:com.snowplowanalytics.snowplow/CollectorPayload/thrift/1-0-0"}`), + PartitionKey: "some-key", + } + + var messageBad = models.Message{ + Data: []byte(`not-a-json`), + PartitionKey: "some-key4", + } + + var expectedGood = models.Message{ + Data: []byte(`CwBkAAAACjE3Mi4xNy4wLjEKAMgAAAGF27kNagsA0gAAAAVVVEYtOAsA3AAAABFzc2MtMi44LjItc3Rkb3V0JAsBLAAAAAtjdXJsLzcuODUuMAsBQAAAAAIvaQsBSgAAAAplPXB2JnA9YWlkDwFeCwAAAAQAAAAbVGltZW91dC1BY2Nlc3M6IDxmdW5jdGlvbjE+AAAAFEhvc3Q6IGxvY2FsaG9zdDo4MDgwAAAAF1VzZXItQWdlbnQ6IGN1cmwvNy44NS4wAAAAC0FjY2VwdDogKi8qCwGQAAAACWxvY2FsaG9zdAsBmgAAACQ1ZmZjODFlMS03YmYyLTQwODQtODBhYS1mODg3NGE1M2NlNTALemkAAABBaWdsdTpjb20uc25vd3Bsb3dhbmFseXRpY3Muc25vd3Bsb3cvQ29sbGVjdG9yUGF5bG9hZC90aHJpZnQvMS0wLTAA`), + PartitionKey: "some-key", + } + + base64EncodeFunc, _ := NewSpJSONToCollectorPayloadThrift(true) + + // Simple success case + transformSuccess, _, failure, _ := base64EncodeFunc(&messageGood, nil) + + assert.Equal(expectedGood.PartitionKey, transformSuccess.PartitionKey) + assert.Equal(string(expectedGood.Data), string(transformSuccess.Data)) + assert.Nil(failure) + + // Simple failure case + success, _, transformFailure, _ := base64EncodeFunc(&messageBad, nil) + + // Not matching equivalence of whole object because error stacktrace makes it unfeasible. Doing each component part instead. + assert.NotNil(transformFailure.GetError()) + if transformFailure.GetError() != nil { + assert.Equal("invalid character 'o' in literal null (expecting 'u')", transformFailure.GetError().Error()) + } + assert.Equal([]byte("not-a-json"), transformFailure.Data) + assert.Equal("some-key4", transformFailure.PartitionKey) + assert.Nil(success) + + // Check that the input has not been altered + assert.Nil(messageGood.GetError()) +} diff --git a/pkg/transform/transformconfig/transform_config.go b/pkg/transform/transformconfig/transform_config.go index 7de543bc..d3fb0905 100644 --- a/pkg/transform/transformconfig/transform_config.go +++ b/pkg/transform/transformconfig/transform_config.go @@ -23,6 +23,9 @@ var SupportedTransformations = []config.ConfigurationPair{ filter.ContextFilterConfigPair, transform.SetPkConfigPair, transform.EnrichedToJSONConfigPair, + transform.CollectorPayloadThriftToJSONConfigPair, + transform.JSONToCollectorPayloadThriftConfigPair, + transform.JSONManipulatorConfigPair, engine.LuaConfigPair, engine.JSConfigPair, } diff --git a/third_party/snowplow/collectorpayload/README.md b/third_party/snowplow/collectorpayload/README.md new file mode 100644 index 00000000..ee4deede --- /dev/null +++ b/third_party/snowplow/collectorpayload/README.md @@ -0,0 +1,12 @@ +# collectorpayload + +This module is forked from the [Scala equivalent](https://github.com/snowplow/snowplow/tree/master/2-collectors/thrift-schemas/collector-payload-1). + +To generate the `gen-go/model1` directory: + +1. Install `thrift`: https://thrift.apache.org/download +2. From this directory run `thrift -r --gen go collector_payload_1.thrift` + +_NOTE_: Running `make format` will fix formatting issues in the auto-generated code. + +To see how to use the library have a look at `collector_payload_test.go` which contains helper functions for seralizing and deserializing payloads. diff --git a/third_party/snowplow/collectorpayload/collector_payload.go b/third_party/snowplow/collectorpayload/collector_payload.go new file mode 100644 index 00000000..1fc2024a --- /dev/null +++ b/third_party/snowplow/collectorpayload/collector_payload.go @@ -0,0 +1,70 @@ +// +// Copyright (c) 2023-present Snowplow Analytics Ltd. All rights reserved. +// +// This program is licensed to you under the Snowplow Community License Version 1.0, +// and you may not use this file except in compliance with the Snowplow Community License Version 1.0. +// You may obtain a copy of the Snowplow Community License Version 1.0 at https://docs.snowplow.io/community-license-1.0 + +package collectorpayload + +import ( + "context" + "encoding/base64" + "encoding/json" + + thrift "github.com/apache/thrift/lib/go/thrift" + + model1 "github.com/snowplow/snowbridge/third_party/snowplow/collectorpayload/gen-go/model1" +) + +const ( + schema = "iglu:com.snowplowanalytics.snowplow/CollectorPayload/thrift/1-0-0" +) + +// BinarySerializer serializes a CollectorPayload into a byte array ready for transport +func BinarySerializer(ctx context.Context, collectorPayload *model1.CollectorPayload) ([]byte, error) { + collectorPayload.Schema = schema + + t := thrift.NewTMemoryBufferLen(1024) + p := thrift.NewTBinaryProtocolFactoryDefault().GetProtocol(t) + + serializer := &thrift.TSerializer{ + Transport: t, + Protocol: p, + } + + return serializer.Write(ctx, collectorPayload) +} + +// BinaryDeserializer deserializes a CollectorPayload byte array back to a struct +func BinaryDeserializer(ctx context.Context, collectorPayloadBytes []byte) (*model1.CollectorPayload, error) { + var inputBytes []byte + + // Attempt to decode from base64 as most payloads will arrive with the thrift string re-encoded + base64DecodedCollectorPayload, base64Err := base64.StdEncoding.DecodeString(string(collectorPayloadBytes)) + if base64Err != nil { + inputBytes = collectorPayloadBytes + } else { + inputBytes = []byte(base64DecodedCollectorPayload) + } + + t := thrift.NewTMemoryBufferLen(1024) + p := thrift.NewTBinaryProtocolFactoryDefault().GetProtocol(t) + + deserializer := &thrift.TDeserializer{ + Transport: t, + Protocol: p, + } + + collectorPayload := model1.NewCollectorPayload() + err := deserializer.Read(ctx, collectorPayload, inputBytes) + + collectorPayload.Schema = schema + + return collectorPayload, err +} + +// ToJSON converts the collector payload struct to a JSON representation for simpler portability +func ToJSON(collectorPayload *model1.CollectorPayload) ([]byte, error) { + return json.Marshal(collectorPayload) +} diff --git a/third_party/snowplow/collectorpayload/collector_payload_1.thrift b/third_party/snowplow/collectorpayload/collector_payload_1.thrift new file mode 100644 index 00000000..fb0364cf --- /dev/null +++ b/third_party/snowplow/collectorpayload/collector_payload_1.thrift @@ -0,0 +1,33 @@ +// +// Copyright (c) 2023-present Snowplow Analytics Ltd. All rights reserved. +// +// This program is licensed to you under the Snowplow Community License Version 1.0, +// and you may not use this file except in compliance with the Snowplow Community License Version 1.0. +// You may obtain a copy of the Snowplow Community License Version 1.0 at https://docs.snowplow.io/community-license-1.0 + +namespace go model1 + +struct CollectorPayload { + 31337: string schema + + // Required fields which are intrinsic properties of HTTP + 100: string ipAddress + + // Required fields which are Snowplow-specific + 200: i64 timestamp + 210: string encoding + 220: string collector + + // Optional fields which are intrinsic properties of HTTP + 300: optional string userAgent + 310: optional string refererUri + 320: optional string path + 330: optional string querystring + 340: optional string body + 350: optional list headers + 360: optional string contentType + + // Optional fields which are Snowplow-specific + 400: optional string hostname + 410: optional string networkUserId +} diff --git a/third_party/snowplow/collectorpayload/collector_payload_test.go b/third_party/snowplow/collectorpayload/collector_payload_test.go new file mode 100644 index 00000000..513dbf5d --- /dev/null +++ b/third_party/snowplow/collectorpayload/collector_payload_test.go @@ -0,0 +1,36 @@ +// +// Copyright (c) 2023-present Snowplow Analytics Ltd. All rights reserved. +// +// This program is licensed to you under the Snowplow Community License Version 1.0, +// and you may not use this file except in compliance with the Snowplow Community License Version 1.0. +// You may obtain a copy of the Snowplow Community License Version 1.0 at https://docs.snowplow.io/community-license-1.0 + +package collectorpayload + +import ( + "context" + "testing" + + model1 "github.com/snowplow/snowbridge/third_party/snowplow/collectorpayload/gen-go/model1" + + "github.com/stretchr/testify/assert" +) + +// TestBinarySerializer +func TestBinarySerializerAndDeserializer(t *testing.T) { + ctx := context.Background() + + assert := assert.New(t) + + payload := model1.NewCollectorPayload() + payload.IpAddress = "192.168.0.1" + + res, err := BinarySerializer(ctx, payload) + assert.Nil(err) + assert.NotNil(res) + + res1, err1 := BinaryDeserializer(ctx, res) + assert.Nil(err1) + assert.NotNil(res1) + assert.Equal("192.168.0.1", res1.IpAddress) +} diff --git a/third_party/snowplow/collectorpayload/gen-go/model1/GoUnusedProtection__.go b/third_party/snowplow/collectorpayload/gen-go/model1/GoUnusedProtection__.go new file mode 100644 index 00000000..7f95c7d8 --- /dev/null +++ b/third_party/snowplow/collectorpayload/gen-go/model1/GoUnusedProtection__.go @@ -0,0 +1,5 @@ +// Code generated by Thrift Compiler (0.17.0). DO NOT EDIT. + +package model1 + +var GoUnusedProtection__ int diff --git a/third_party/snowplow/collectorpayload/gen-go/model1/collector_payload_1-consts.go b/third_party/snowplow/collectorpayload/gen-go/model1/collector_payload_1-consts.go new file mode 100644 index 00000000..bc927fbf --- /dev/null +++ b/third_party/snowplow/collectorpayload/gen-go/model1/collector_payload_1-consts.go @@ -0,0 +1,23 @@ +// Code generated by Thrift Compiler (0.17.0). DO NOT EDIT. + +package model1 + +import ( + "bytes" + "context" + "errors" + "fmt" + thrift "github.com/apache/thrift/lib/go/thrift" + "time" +) + +// (needed to ensure safety because of naive import list construction.) +var _ = thrift.ZERO +var _ = fmt.Printf +var _ = errors.New +var _ = context.Background +var _ = time.Now +var _ = bytes.Equal + +func init() { +} diff --git a/third_party/snowplow/collectorpayload/gen-go/model1/collector_payload_1.go b/third_party/snowplow/collectorpayload/gen-go/model1/collector_payload_1.go new file mode 100644 index 00000000..a8dfeeff --- /dev/null +++ b/third_party/snowplow/collectorpayload/gen-go/model1/collector_payload_1.go @@ -0,0 +1,880 @@ +// Code generated by Thrift Compiler (0.17.0). DO NOT EDIT. + +package model1 + +import ( + "bytes" + "context" + "errors" + "fmt" + thrift "github.com/apache/thrift/lib/go/thrift" + "time" +) + +// (needed to ensure safety because of naive import list construction.) +var _ = thrift.ZERO +var _ = fmt.Printf +var _ = errors.New +var _ = context.Background +var _ = time.Now +var _ = bytes.Equal + +// Attributes: +// - Schema +// - IpAddress +// - Timestamp +// - Encoding +// - Collector +// - UserAgent +// - RefererUri +// - Path +// - Querystring +// - Body +// - Headers +// - ContentType +// - Hostname +// - NetworkUserId +type CollectorPayload struct { + // unused fields # 1 to 99 + IpAddress string `thrift:"ipAddress,100" db:"ipAddress" json:"ipAddress"` + // unused fields # 101 to 199 + Timestamp int64 `thrift:"timestamp,200" db:"timestamp" json:"timestamp"` + // unused fields # 201 to 209 + Encoding string `thrift:"encoding,210" db:"encoding" json:"encoding"` + // unused fields # 211 to 219 + Collector string `thrift:"collector,220" db:"collector" json:"collector"` + // unused fields # 221 to 299 + UserAgent *string `thrift:"userAgent,300" db:"userAgent" json:"userAgent,omitempty"` + // unused fields # 301 to 309 + RefererUri *string `thrift:"refererUri,310" db:"refererUri" json:"refererUri,omitempty"` + // unused fields # 311 to 319 + Path *string `thrift:"path,320" db:"path" json:"path,omitempty"` + // unused fields # 321 to 329 + Querystring *string `thrift:"querystring,330" db:"querystring" json:"querystring,omitempty"` + // unused fields # 331 to 339 + Body *string `thrift:"body,340" db:"body" json:"body,omitempty"` + // unused fields # 341 to 349 + Headers []string `thrift:"headers,350" db:"headers" json:"headers,omitempty"` + // unused fields # 351 to 359 + ContentType *string `thrift:"contentType,360" db:"contentType" json:"contentType,omitempty"` + // unused fields # 361 to 399 + Hostname *string `thrift:"hostname,400" db:"hostname" json:"hostname,omitempty"` + // unused fields # 401 to 409 + NetworkUserId *string `thrift:"networkUserId,410" db:"networkUserId" json:"networkUserId,omitempty"` + // unused fields # 411 to 31336 + Schema string `thrift:"schema,31337" db:"schema" json:"schema"` +} + +func NewCollectorPayload() *CollectorPayload { + return &CollectorPayload{} +} + +func (p *CollectorPayload) GetSchema() string { + return p.Schema +} + +func (p *CollectorPayload) GetIpAddress() string { + return p.IpAddress +} + +func (p *CollectorPayload) GetTimestamp() int64 { + return p.Timestamp +} + +func (p *CollectorPayload) GetEncoding() string { + return p.Encoding +} + +func (p *CollectorPayload) GetCollector() string { + return p.Collector +} + +var CollectorPayload_UserAgent_DEFAULT string + +func (p *CollectorPayload) GetUserAgent() string { + if !p.IsSetUserAgent() { + return CollectorPayload_UserAgent_DEFAULT + } + return *p.UserAgent +} + +var CollectorPayload_RefererUri_DEFAULT string + +func (p *CollectorPayload) GetRefererUri() string { + if !p.IsSetRefererUri() { + return CollectorPayload_RefererUri_DEFAULT + } + return *p.RefererUri +} + +var CollectorPayload_Path_DEFAULT string + +func (p *CollectorPayload) GetPath() string { + if !p.IsSetPath() { + return CollectorPayload_Path_DEFAULT + } + return *p.Path +} + +var CollectorPayload_Querystring_DEFAULT string + +func (p *CollectorPayload) GetQuerystring() string { + if !p.IsSetQuerystring() { + return CollectorPayload_Querystring_DEFAULT + } + return *p.Querystring +} + +var CollectorPayload_Body_DEFAULT string + +func (p *CollectorPayload) GetBody() string { + if !p.IsSetBody() { + return CollectorPayload_Body_DEFAULT + } + return *p.Body +} + +var CollectorPayload_Headers_DEFAULT []string + +func (p *CollectorPayload) GetHeaders() []string { + return p.Headers +} + +var CollectorPayload_ContentType_DEFAULT string + +func (p *CollectorPayload) GetContentType() string { + if !p.IsSetContentType() { + return CollectorPayload_ContentType_DEFAULT + } + return *p.ContentType +} + +var CollectorPayload_Hostname_DEFAULT string + +func (p *CollectorPayload) GetHostname() string { + if !p.IsSetHostname() { + return CollectorPayload_Hostname_DEFAULT + } + return *p.Hostname +} + +var CollectorPayload_NetworkUserId_DEFAULT string + +func (p *CollectorPayload) GetNetworkUserId() string { + if !p.IsSetNetworkUserId() { + return CollectorPayload_NetworkUserId_DEFAULT + } + return *p.NetworkUserId +} +func (p *CollectorPayload) IsSetUserAgent() bool { + return p.UserAgent != nil +} + +func (p *CollectorPayload) IsSetRefererUri() bool { + return p.RefererUri != nil +} + +func (p *CollectorPayload) IsSetPath() bool { + return p.Path != nil +} + +func (p *CollectorPayload) IsSetQuerystring() bool { + return p.Querystring != nil +} + +func (p *CollectorPayload) IsSetBody() bool { + return p.Body != nil +} + +func (p *CollectorPayload) IsSetHeaders() bool { + return p.Headers != nil +} + +func (p *CollectorPayload) IsSetContentType() bool { + return p.ContentType != nil +} + +func (p *CollectorPayload) IsSetHostname() bool { + return p.Hostname != nil +} + +func (p *CollectorPayload) IsSetNetworkUserId() bool { + return p.NetworkUserId != nil +} + +func (p *CollectorPayload) Read(ctx context.Context, iprot thrift.TProtocol) error { + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 31337: + if fieldTypeId == thrift.STRING { + if err := p.ReadField31337(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 100: + if fieldTypeId == thrift.STRING { + if err := p.ReadField100(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 200: + if fieldTypeId == thrift.I64 { + if err := p.ReadField200(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 210: + if fieldTypeId == thrift.STRING { + if err := p.ReadField210(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 220: + if fieldTypeId == thrift.STRING { + if err := p.ReadField220(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 300: + if fieldTypeId == thrift.STRING { + if err := p.ReadField300(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 310: + if fieldTypeId == thrift.STRING { + if err := p.ReadField310(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 320: + if fieldTypeId == thrift.STRING { + if err := p.ReadField320(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 330: + if fieldTypeId == thrift.STRING { + if err := p.ReadField330(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 340: + if fieldTypeId == thrift.STRING { + if err := p.ReadField340(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 350: + if fieldTypeId == thrift.LIST { + if err := p.ReadField350(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 360: + if fieldTypeId == thrift.STRING { + if err := p.ReadField360(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 400: + if fieldTypeId == thrift.STRING { + if err := p.ReadField400(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + case 410: + if fieldTypeId == thrift.STRING { + if err := p.ReadField410(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + return nil +} + +func (p *CollectorPayload) ReadField31337(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 31337: ", err) + } else { + p.Schema = v + } + return nil +} + +func (p *CollectorPayload) ReadField100(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 100: ", err) + } else { + p.IpAddress = v + } + return nil +} + +func (p *CollectorPayload) ReadField200(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI64(ctx); err != nil { + return thrift.PrependError("error reading field 200: ", err) + } else { + p.Timestamp = v + } + return nil +} + +func (p *CollectorPayload) ReadField210(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 210: ", err) + } else { + p.Encoding = v + } + return nil +} + +func (p *CollectorPayload) ReadField220(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 220: ", err) + } else { + p.Collector = v + } + return nil +} + +func (p *CollectorPayload) ReadField300(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 300: ", err) + } else { + p.UserAgent = &v + } + return nil +} + +func (p *CollectorPayload) ReadField310(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 310: ", err) + } else { + p.RefererUri = &v + } + return nil +} + +func (p *CollectorPayload) ReadField320(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 320: ", err) + } else { + p.Path = &v + } + return nil +} + +func (p *CollectorPayload) ReadField330(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 330: ", err) + } else { + p.Querystring = &v + } + return nil +} + +func (p *CollectorPayload) ReadField340(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 340: ", err) + } else { + p.Body = &v + } + return nil +} + +func (p *CollectorPayload) ReadField350(ctx context.Context, iprot thrift.TProtocol) error { + _, size, err := iprot.ReadListBegin(ctx) + if err != nil { + return thrift.PrependError("error reading list begin: ", err) + } + tSlice := make([]string, 0, size) + p.Headers = tSlice + for i := 0; i < size; i++ { + var _elem0 string + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 0: ", err) + } else { + _elem0 = v + } + p.Headers = append(p.Headers, _elem0) + } + if err := iprot.ReadListEnd(ctx); err != nil { + return thrift.PrependError("error reading list end: ", err) + } + return nil +} + +func (p *CollectorPayload) ReadField360(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 360: ", err) + } else { + p.ContentType = &v + } + return nil +} + +func (p *CollectorPayload) ReadField400(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 400: ", err) + } else { + p.Hostname = &v + } + return nil +} + +func (p *CollectorPayload) ReadField410(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadString(ctx); err != nil { + return thrift.PrependError("error reading field 410: ", err) + } else { + p.NetworkUserId = &v + } + return nil +} + +func (p *CollectorPayload) Write(ctx context.Context, oprot thrift.TProtocol) error { + if err := oprot.WriteStructBegin(ctx, "CollectorPayload"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField100(ctx, oprot); err != nil { + return err + } + if err := p.writeField200(ctx, oprot); err != nil { + return err + } + if err := p.writeField210(ctx, oprot); err != nil { + return err + } + if err := p.writeField220(ctx, oprot); err != nil { + return err + } + if err := p.writeField300(ctx, oprot); err != nil { + return err + } + if err := p.writeField310(ctx, oprot); err != nil { + return err + } + if err := p.writeField320(ctx, oprot); err != nil { + return err + } + if err := p.writeField330(ctx, oprot); err != nil { + return err + } + if err := p.writeField340(ctx, oprot); err != nil { + return err + } + if err := p.writeField350(ctx, oprot); err != nil { + return err + } + if err := p.writeField360(ctx, oprot); err != nil { + return err + } + if err := p.writeField400(ctx, oprot); err != nil { + return err + } + if err := p.writeField410(ctx, oprot); err != nil { + return err + } + if err := p.writeField31337(ctx, oprot); err != nil { + return err + } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil +} + +func (p *CollectorPayload) writeField100(ctx context.Context, oprot thrift.TProtocol) (err error) { + if err := oprot.WriteFieldBegin(ctx, "ipAddress", thrift.STRING, 100); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 100:ipAddress: ", p), err) + } + if err := oprot.WriteString(ctx, string(p.IpAddress)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.ipAddress (100) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 100:ipAddress: ", p), err) + } + return err +} + +func (p *CollectorPayload) writeField200(ctx context.Context, oprot thrift.TProtocol) (err error) { + if err := oprot.WriteFieldBegin(ctx, "timestamp", thrift.I64, 200); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 200:timestamp: ", p), err) + } + if err := oprot.WriteI64(ctx, int64(p.Timestamp)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.timestamp (200) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 200:timestamp: ", p), err) + } + return err +} + +func (p *CollectorPayload) writeField210(ctx context.Context, oprot thrift.TProtocol) (err error) { + if err := oprot.WriteFieldBegin(ctx, "encoding", thrift.STRING, 210); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 210:encoding: ", p), err) + } + if err := oprot.WriteString(ctx, string(p.Encoding)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.encoding (210) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 210:encoding: ", p), err) + } + return err +} + +func (p *CollectorPayload) writeField220(ctx context.Context, oprot thrift.TProtocol) (err error) { + if err := oprot.WriteFieldBegin(ctx, "collector", thrift.STRING, 220); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 220:collector: ", p), err) + } + if err := oprot.WriteString(ctx, string(p.Collector)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.collector (220) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 220:collector: ", p), err) + } + return err +} + +func (p *CollectorPayload) writeField300(ctx context.Context, oprot thrift.TProtocol) (err error) { + if p.IsSetUserAgent() { + if err := oprot.WriteFieldBegin(ctx, "userAgent", thrift.STRING, 300); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 300:userAgent: ", p), err) + } + if err := oprot.WriteString(ctx, string(*p.UserAgent)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.userAgent (300) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 300:userAgent: ", p), err) + } + } + return err +} + +func (p *CollectorPayload) writeField310(ctx context.Context, oprot thrift.TProtocol) (err error) { + if p.IsSetRefererUri() { + if err := oprot.WriteFieldBegin(ctx, "refererUri", thrift.STRING, 310); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 310:refererUri: ", p), err) + } + if err := oprot.WriteString(ctx, string(*p.RefererUri)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.refererUri (310) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 310:refererUri: ", p), err) + } + } + return err +} + +func (p *CollectorPayload) writeField320(ctx context.Context, oprot thrift.TProtocol) (err error) { + if p.IsSetPath() { + if err := oprot.WriteFieldBegin(ctx, "path", thrift.STRING, 320); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 320:path: ", p), err) + } + if err := oprot.WriteString(ctx, string(*p.Path)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.path (320) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 320:path: ", p), err) + } + } + return err +} + +func (p *CollectorPayload) writeField330(ctx context.Context, oprot thrift.TProtocol) (err error) { + if p.IsSetQuerystring() { + if err := oprot.WriteFieldBegin(ctx, "querystring", thrift.STRING, 330); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 330:querystring: ", p), err) + } + if err := oprot.WriteString(ctx, string(*p.Querystring)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.querystring (330) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 330:querystring: ", p), err) + } + } + return err +} + +func (p *CollectorPayload) writeField340(ctx context.Context, oprot thrift.TProtocol) (err error) { + if p.IsSetBody() { + if err := oprot.WriteFieldBegin(ctx, "body", thrift.STRING, 340); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 340:body: ", p), err) + } + if err := oprot.WriteString(ctx, string(*p.Body)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.body (340) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 340:body: ", p), err) + } + } + return err +} + +func (p *CollectorPayload) writeField350(ctx context.Context, oprot thrift.TProtocol) (err error) { + if p.IsSetHeaders() { + if err := oprot.WriteFieldBegin(ctx, "headers", thrift.LIST, 350); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 350:headers: ", p), err) + } + if err := oprot.WriteListBegin(ctx, thrift.STRING, len(p.Headers)); err != nil { + return thrift.PrependError("error writing list begin: ", err) + } + for _, v := range p.Headers { + if err := oprot.WriteString(ctx, string(v)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T. (0) field write error: ", p), err) + } + } + if err := oprot.WriteListEnd(ctx); err != nil { + return thrift.PrependError("error writing list end: ", err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 350:headers: ", p), err) + } + } + return err +} + +func (p *CollectorPayload) writeField360(ctx context.Context, oprot thrift.TProtocol) (err error) { + if p.IsSetContentType() { + if err := oprot.WriteFieldBegin(ctx, "contentType", thrift.STRING, 360); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 360:contentType: ", p), err) + } + if err := oprot.WriteString(ctx, string(*p.ContentType)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.contentType (360) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 360:contentType: ", p), err) + } + } + return err +} + +func (p *CollectorPayload) writeField400(ctx context.Context, oprot thrift.TProtocol) (err error) { + if p.IsSetHostname() { + if err := oprot.WriteFieldBegin(ctx, "hostname", thrift.STRING, 400); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 400:hostname: ", p), err) + } + if err := oprot.WriteString(ctx, string(*p.Hostname)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.hostname (400) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 400:hostname: ", p), err) + } + } + return err +} + +func (p *CollectorPayload) writeField410(ctx context.Context, oprot thrift.TProtocol) (err error) { + if p.IsSetNetworkUserId() { + if err := oprot.WriteFieldBegin(ctx, "networkUserId", thrift.STRING, 410); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 410:networkUserId: ", p), err) + } + if err := oprot.WriteString(ctx, string(*p.NetworkUserId)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.networkUserId (410) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 410:networkUserId: ", p), err) + } + } + return err +} + +func (p *CollectorPayload) writeField31337(ctx context.Context, oprot thrift.TProtocol) (err error) { + if err := oprot.WriteFieldBegin(ctx, "schema", thrift.STRING, 31337); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 31337:schema: ", p), err) + } + if err := oprot.WriteString(ctx, string(p.Schema)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.schema (31337) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 31337:schema: ", p), err) + } + return err +} + +func (p *CollectorPayload) Equals(other *CollectorPayload) bool { + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.IpAddress != other.IpAddress { + return false + } + if p.Timestamp != other.Timestamp { + return false + } + if p.Encoding != other.Encoding { + return false + } + if p.Collector != other.Collector { + return false + } + if p.UserAgent != other.UserAgent { + if p.UserAgent == nil || other.UserAgent == nil { + return false + } + if (*p.UserAgent) != (*other.UserAgent) { + return false + } + } + if p.RefererUri != other.RefererUri { + if p.RefererUri == nil || other.RefererUri == nil { + return false + } + if (*p.RefererUri) != (*other.RefererUri) { + return false + } + } + if p.Path != other.Path { + if p.Path == nil || other.Path == nil { + return false + } + if (*p.Path) != (*other.Path) { + return false + } + } + if p.Querystring != other.Querystring { + if p.Querystring == nil || other.Querystring == nil { + return false + } + if (*p.Querystring) != (*other.Querystring) { + return false + } + } + if p.Body != other.Body { + if p.Body == nil || other.Body == nil { + return false + } + if (*p.Body) != (*other.Body) { + return false + } + } + if len(p.Headers) != len(other.Headers) { + return false + } + for i, _tgt := range p.Headers { + _src1 := other.Headers[i] + if _tgt != _src1 { + return false + } + } + if p.ContentType != other.ContentType { + if p.ContentType == nil || other.ContentType == nil { + return false + } + if (*p.ContentType) != (*other.ContentType) { + return false + } + } + if p.Hostname != other.Hostname { + if p.Hostname == nil || other.Hostname == nil { + return false + } + if (*p.Hostname) != (*other.Hostname) { + return false + } + } + if p.NetworkUserId != other.NetworkUserId { + if p.NetworkUserId == nil || other.NetworkUserId == nil { + return false + } + if (*p.NetworkUserId) != (*other.NetworkUserId) { + return false + } + } + if p.Schema != other.Schema { + return false + } + return true +} + +func (p *CollectorPayload) String() string { + if p == nil { + return "" + } + return fmt.Sprintf("CollectorPayload(%+v)", *p) +}