From 08364bdfb33c20a4039725f55ec811107f13e2a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Jim=C3=A9nez=20Pascual?= Date: Thu, 13 Apr 2023 23:28:45 +0200 Subject: [PATCH 1/3] as: Add CO2 and LightIntensity Sync Air's data model and validation with the latest JSON schema --- pkg/messageprocessors/normalizedpayload/uplink.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/messageprocessors/normalizedpayload/uplink.go b/pkg/messageprocessors/normalizedpayload/uplink.go index 20a4396627..1c09b7baae 100644 --- a/pkg/messageprocessors/normalizedpayload/uplink.go +++ b/pkg/messageprocessors/normalizedpayload/uplink.go @@ -29,6 +29,8 @@ type Air struct { Temperature *float64 RelativeHumidity *float64 Pressure *float64 + CO2 *float64 + LightIntensity *float64 } // Wind is a wind measurement. @@ -211,6 +213,19 @@ var fieldParsers = map[string]fieldParser{ minimum(900.0), maximum(1100.0), ), + "air.co2": parseNumber( + func(dst *Measurement) **float64 { + return &dst.Air.CO2 + }, + minimum(0.0), + maximum(1000000.0), + ), + "air.lightIntensity": parseNumber( + func(dst *Measurement) **float64 { + return &dst.Air.LightIntensity + }, + minimum(0.0), + ), "wind": object( func(dst *Measurement) *Wind { return &dst.Wind From 9fc8242e141180a0ab0e065b75b0455451d0581f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Jim=C3=A9nez=20Pascual?= Date: Thu, 13 Apr 2023 23:31:10 +0200 Subject: [PATCH 2/3] as: Add Soil data model and validation --- .../normalizedpayload/uplink.go | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/pkg/messageprocessors/normalizedpayload/uplink.go b/pkg/messageprocessors/normalizedpayload/uplink.go index 1c09b7baae..1cf083fda0 100644 --- a/pkg/messageprocessors/normalizedpayload/uplink.go +++ b/pkg/messageprocessors/normalizedpayload/uplink.go @@ -24,6 +24,18 @@ import ( "google.golang.org/protobuf/types/known/structpb" ) +// Soil is an soil measurement. +type Soil struct { + Depth *float64 + Moisture *float64 + Temperature *float64 + ElectricalConductivity *float64 + PHLevel *float64 + Nitrogen *float64 + Phosphorus *float64 + Potassium *float64 +} + // Air is an air measurement. type Air struct { Temperature *float64 @@ -42,6 +54,7 @@ type Wind struct { // Measurement is a measurement. type Measurement struct { Time *time.Time + Soil Soil Air Air Wind Wind } @@ -188,6 +201,65 @@ var fieldParsers = map[string]fieldParser{ return &dst.Time }, ), + "soil": object( + func(dst *Measurement) *Soil { + return &dst.Soil + }, + ), + "soil.depth": parseNumber( + func(dst *Measurement) **float64 { + return &dst.Soil.Depth + }, + minimum(0.0), + ), + "soil.moisture": parseNumber( + func(dst *Measurement) **float64 { + return &dst.Soil.Moisture + }, + minimum(0.0), + maximum(100.0), + ), + "soil.temperature": parseNumber( + func(dst *Measurement) **float64 { + return &dst.Soil.Temperature + }, + minimum(-273.15), + ), + "soil.ec": parseNumber( + func(dst *Measurement) **float64 { + return &dst.Soil.ElectricalConductivity + }, + minimum(0.0), + maximum(621.0), + ), + "soil.pH": parseNumber( + func(dst *Measurement) **float64 { + return &dst.Soil.PHLevel + }, + minimum(0.0), + maximum(14.0), + ), + "soil.n": parseNumber( + func(dst *Measurement) **float64 { + return &dst.Soil.Nitrogen + }, + minimum(0.0), + maximum(1000000.0), + ), + "soil.p": parseNumber( + func(dst *Measurement) **float64 { + return &dst.Soil.Phosphorus + }, + minimum(0.0), + maximum(1000000.0), + ), + "soil.k": parseNumber( + func(dst *Measurement) **float64 { + return &dst.Soil.Potassium + }, + minimum(0.0), + maximum(1000000.0), + ), "air": object( func(dst *Measurement) *Air { return &dst.Air From 76a4ecd6763cc3c385e7dd7763cb7fc9e7f70714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Jim=C3=A9nez=20Pascual?= Date: Fri, 14 Apr 2023 00:09:30 +0200 Subject: [PATCH 3/3] as: Abstract percentages and concentration parsing These kind of of validation checks are so common and might appear even more as the schema grows. In the same way that definitions are used in the JSON schema for defining these types, here I've abstracted them to separate functions to facilitate the maintenance down the road. I've also added some test to increase coverage and explicitly test the latest additions. --- .../normalizedpayload/uplink.go | 42 +++++++------ .../normalizedpayload/uplink_test.go | 62 +++++++++++++++++++ 2 files changed, 86 insertions(+), 18 deletions(-) diff --git a/pkg/messageprocessors/normalizedpayload/uplink.go b/pkg/messageprocessors/normalizedpayload/uplink.go index 1cf083fda0..2bd121322d 100644 --- a/pkg/messageprocessors/normalizedpayload/uplink.go +++ b/pkg/messageprocessors/normalizedpayload/uplink.go @@ -141,6 +141,24 @@ func parseNumber(selector func(dst *Measurement) **float64, vals ...fieldValidat } } +// parsePercentage parses and validates a percentage. +func parsePercentage(selector func(dst *Measurement) **float64) fieldParser { + return parseNumber( + selector, + minimum(0.0), + maximum(100.0), + ) +} + +// parseConcentration parses and validates a concentration. Concentration must be in ppm between 0 and 1000000. +func parseConcentration(selector func(dst *Measurement) **float64) fieldParser { + return parseNumber( + selector, + minimum(0.0), + maximum(1000000.0), + ) +} + // minimum returns a field validator that checks the inclusive minimum. func minimum[T constraints.Ordered](min T) fieldValidator[T] { return func(v T, path string) error { @@ -212,12 +230,10 @@ var fieldParsers = map[string]fieldParser{ }, minimum(0.0), ), - "soil.moisture": parseNumber( + "soil.moisture": parsePercentage( func(dst *Measurement) **float64 { return &dst.Soil.Moisture }, - minimum(0.0), - maximum(100.0), ), "soil.temperature": parseNumber( func(dst *Measurement) **float64 { @@ -239,26 +255,20 @@ var fieldParsers = map[string]fieldParser{ minimum(0.0), maximum(14.0), ), - "soil.n": parseNumber( + "soil.n": parseConcentration( func(dst *Measurement) **float64 { return &dst.Soil.Nitrogen }, - minimum(0.0), - maximum(1000000.0), ), - "soil.p": parseNumber( + "soil.p": parseConcentration( func(dst *Measurement) **float64 { return &dst.Soil.Phosphorus }, - minimum(0.0), - maximum(1000000.0), ), - "soil.k": parseNumber( + "soil.k": parseConcentration( func(dst *Measurement) **float64 { return &dst.Soil.Potassium }, - minimum(0.0), - maximum(1000000.0), ), "air": object( func(dst *Measurement) *Air { @@ -271,12 +281,10 @@ var fieldParsers = map[string]fieldParser{ }, minimum(-273.15), ), - "air.relativeHumidity": parseNumber( + "air.relativeHumidity": parsePercentage( func(dst *Measurement) **float64 { return &dst.Air.RelativeHumidity }, - minimum(0.0), - maximum(100.0), ), "air.pressure": parseNumber( func(dst *Measurement) **float64 { @@ -285,12 +293,10 @@ var fieldParsers = map[string]fieldParser{ minimum(900.0), maximum(1100.0), ), - "air.co2": parseNumber( + "air.co2": parseConcentration( func(dst *Measurement) **float64 { return &dst.Air.CO2 }, - minimum(0.0), - maximum(1000000.0), ), "air.lightIntensity": parseNumber( func(dst *Measurement) **float64 { diff --git a/pkg/messageprocessors/normalizedpayload/uplink_test.go b/pkg/messageprocessors/normalizedpayload/uplink_test.go index 286f520a7f..e023720cff 100644 --- a/pkg/messageprocessors/normalizedpayload/uplink_test.go +++ b/pkg/messageprocessors/normalizedpayload/uplink_test.go @@ -62,6 +62,35 @@ func TestUplink(t *testing.T) { }, }, }, + { + name: "one soil nutrient concentration", + normalizedPayload: []*structpb.Struct{ + { + Fields: map[string]*structpb.Value{ + "soil": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "n": { + Kind: &structpb.Value_NumberValue{ + NumberValue: 999999.99, + }, + }, + }, + }, + }, + }, + }, + }, + }, + expected: []normalizedpayload.Measurement{ + { + Soil: normalizedpayload.Soil{ + Nitrogen: float64Ptr(999999.99), + }, + }, + }, + }, { name: "two air temperatures", normalizedPayload: []*structpb.Struct{ @@ -122,6 +151,39 @@ func TestUplink(t *testing.T) { {}, }, }, + { + name: "above 100 percent soil moisture", + normalizedPayload: []*structpb.Struct{ + { + Fields: map[string]*structpb.Value{ + "soil": { + Kind: &structpb.Value_StructValue{ + StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "moisture": { + Kind: &structpb.Value_NumberValue{ + NumberValue: 120, + }, + }, + }, + }, + }, + }, + }, + }, + }, + expected: []normalizedpayload.Measurement{ + {}, + }, + expectedValidationErrors: [][]error{ + { + normalizedpayload.ErrFieldMaximum.WithAttributes( + "path", "soil.moisture", + "maximum", 100.0, + ), + }, + }, + }, { name: "below absolute zero", normalizedPayload: []*structpb.Struct{