Support payload normalization (#508)

* Fix link * Fix downlink decoder examples * Update dependencies * Initial support for payload normalization * Upgrade to Go 1.18 * Add validation for atmospheric pressure * Extract normalized payload to own schema file * Use percentage for relative humidity * Refer to TTS documentation for codecs
TheThingsNetwork · Aug 29, 2022 · d810556 · d810556
1 parent cfe6c7a
commit d810556
Show file tree

Hide file tree

Showing 12 changed files with 431 additions and 1,100 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -15,7 +15,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: '~1.17'
+          go-version: '~1.18'
       - name: Set up Node
         uses: actions/setup-node@v2
         with:

diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml
@@ -10,7 +10,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: '~1.17'
+          go-version: '~1.18'
       - uses: actions/setup-node@v2
         with:
           node-version: '16'

diff --git a/README.md b/README.md
@@ -26,7 +26,7 @@ Development dependencies:
 
 - Node.js version 16.x
 - npm version 8.x
-- Go version 1.17.x
+- Go version 1.18.x
 
 To check your Node.js, npm and Go versions:
 
@@ -288,6 +288,12 @@ uplinkDecoder:
         data:
           direction: 'N'
           speed: 32
+      # Normalized output, uses the normalizeUplink function (optional)
+      normalizedOutput:
+        data:
+          - wind:
+              speed: 16.4608
+              direction: 0
 # Downlink encoder encodes JSON object into a binary data downlink (optional)
 downlinkEncoder:
   fileName: codec.js
@@ -316,108 +322,8 @@ downlinkDecoder:
 
 The actual **Payload codec implementation** is in the referenced filename: `vendor/<vendor-id>/<codec-filename>`.
 
-An example codec for a wind direction and speed sensor with controllable LED looks like this:
-
-```js
-var directions = ["N", "E", "S", "W"];
-var colors = ["red", "green"];
-
-// input = { fPort: 1, bytes: [1, 62] }
-function decodeUplink(input) {
-  switch (input.fPort) {
-  case 1:
-    return {
-      // Decoded data
-      data: {
-        direction: directions[input.bytes[0]],
-        speed: input.bytes[1]
-      }
-    }
-  default:
-    return {
-      errors: ["unknown FPort"]
-    }
-  }
-}
-
-// input = { data: { led: "green" } }
-function encodeDownlink(input) {
-  var i = colors.indexOf(input.data.led);
-  if (i === -1) {
-    return {
-      errors: ["invalid LED color"]
-    }
-  }
-  return {
-    // LoRaWAN FPort used for the downlink message
-    fPort: 2,
-    // Encoded bytes
-    bytes: [i]
-  }
-}
-
-// input = { fPort: 2, bytes: [1] }
-function decodeDownlink(input) {
-  switch (input.fPort) {
-  case 2:
-    return {
-      // Decoded downlink (must be symmetric with encodeDownlink)
-      data: {
-        led: colors[input.bytes[0]]
-      }
-    }
-  default:
-    return {
-      errors: ["invalid FPort"]
-    }
-  }
-}
-```
+See [The Things Stack documentation](https://www.thethingsindustries.com/docs/integrations/payload-formatters/javascript) for how to write JavaScript functions for decoding, normalizing and encoding data.
 
-#### Errors and Warnings
-
-Scripts can return warnings and errors to inform the application layer of potential issues with the data or indicate that the payload is malformatted.
-
-The warnings and errors are string arrays. If there are any errors, the message fails. Any warnings are added to the message.
-
-Example warning:
-
-```js
-// input = { fPort: 1, bytes: [1, 2, 3] }
-function decodeUplink(input) {
-  var warnings = [];
-  var battery = input.bytes[0] << 8 | input.bytes[1];
-  if (battery < 2000) {
-    warnings.push("unreliable battery level");
-  }
-  return {
-    // Decoded data
-    data: {
-      battery: battery
-    },
-    // Warnings
-    warnings: warnings
-  }
-}
-```
-
-Example error:
-
-```js
-function encodeDownlink(input) {
-  if (typeof input.data.gate !== 'boolean') {
-    return {
-      errors: [
-        "missing required field: gate"
-      ]
-    }
-  }
-  return {
-    fPort: 1,
-    bytes: [input.data.gate ? 1 : 0]
-  }
-}
-```
 ## Legal
 
 The API is distributed under [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0). See `LICENSE` for more information.

diff --git a/bin/validate.js b/bin/validate.js
@@ -10,7 +10,7 @@ const isEqual = require('lodash.isequal');
 const readChunk = require('read-chunk');
 const imageType = require('image-type');
 
-const ajv = new Ajv({ schemas: [require('../schema.json')] });
+const ajv = new Ajv({ schemas: [require('../lib/payload.json'), require('../schema.json')] });
 
 const options = yargs.usage('Usage: --vendor <file>').option('v', {
   alias: 'vendor',
@@ -78,15 +78,36 @@ function validatePayloadCodecs(vendorId, payloadEncoding) {
     { def: payloadEncoding.downlinkDecoder, routine: 'decodeDownlink' },
   ].forEach((d) => {
     if (d.def) {
-      let fileName = `${vendorId}/${d.def.fileName}`;
+      const { routine } = d;
+      const fileName = `${vendorId}/${d.def.fileName}`;
       promises.push(requireFile(fileName));
       if (d.def.examples) {
         d.def.examples.forEach((e) => {
+          const { input, output, description, normalizedOutput } = e;
           runs.push({
-            fileName: fileName,
-            routine: d.routine,
-            ...e,
+            fileName,
+            routine,
+            description,
+            input,
+            output,
           });
+          if (normalizedOutput && d.routine === 'decodeUplink') {
+            runs.push({
+              fileName,
+              routine: 'normalizeUplink',
+              description: `${description} (normalized)`,
+              input: output,
+              output: normalizedOutput,
+              transformOutput: (output) => {
+                // The normalizer can return an object or an array of objects.
+                // If it's an object, convert it to an array with a single item.
+                if (output.data && !Array.isArray(output.data)) {
+                  output.data = [output.data];
+                }
+                return output;
+              },
+            });
+          }
         });
       }
     }
@@ -104,7 +125,10 @@ function validatePayloadCodecs(vendorId, payloadEncoding) {
               reject(stderr);
             } else {
               const expected = r.output;
-              const actual = JSON.parse(stdout);
+              let actual = JSON.parse(stdout);
+              if (r.transformOutput) {
+                actual = r.transformOutput(actual);
+              }
               if (isEqual(expected, actual)) {
                 console.debug(`${r.fileName}:${r.routine}: ${r.description} has correct output`);
                 resolve();

diff --git a/lib/payload.json b/lib/payload.json
@@ -0,0 +1,86 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://schema.thethings.network/devicerepository/1/payload/schema",
+  "title": "LoRaWAN Device Repository Payload",
+  "description": "Payload definitions for the LoRaWAN Device Repository",
+  "definitions": {
+    "temperature": {
+      "type": "number",
+      "description": "Temperature (°C)",
+      "minimum": -273.15
+    },
+    "direction": {
+      "type": "number",
+      "description": "Direction (°)",
+      "minimum": 0,
+      "exclusiveMaximum": 360
+    },
+    "speed": {
+      "type": "number",
+      "description": "Speed (m/s)",
+      "minimum": 0
+    },
+    "percentage": {
+      "type": "number",
+      "description": "Percentage (%)",
+      "minimum": 0,
+      "maximum": 100
+    },
+    "measurement": {
+      "type": "object",
+      "properties": {
+        "time": {
+          "type": "string",
+          "format": "date-time",
+          "description": "Date and time of the measurement (RFC3339)"
+        },
+        "air": {
+          "type": "object",
+          "properties": {
+            "temperature": {
+              "description": "Air temperature (°C)",
+              "$ref": "#/definitions/temperature"
+            },
+            "relativeHumidity": {
+              "description": "Relative humidity (%)",
+              "$ref": "#/definitions/percentage"
+            },
+            "pressure": {
+              "type": "number",
+              "description": "Atmospheric pressure (hPa)",
+              "minimum": 900,
+              "maximum": 1100
+            }
+          },
+          "additionalProperties": false
+        },
+        "wind": {
+          "type": "object",
+          "properties": {
+            "speed": {
+              "description": "Wind speed (m/s)",
+              "$ref": "#/definitions/speed"
+            },
+            "direction": {
+              "description": "Wind direction (°)",
+              "$ref": "#/definitions/direction"
+            }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+    "uplinkPayload": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/measurement"
+      }
+    }
+  },
+  "oneOf": [
+    {
+      "$ref": "#/definitions/uplinkPayload"
+    }
+  ]
+}
diff --git a/schema.json b/schema.json
@@ -423,6 +423,43 @@
               }
             }
           ]
+        },
+        "normalizeUplinkInput": {
+          "type": "object",
+          "properties": {
+            "data": {
+              "type": "object"
+            }
+          },
+          "required": ["data"]
+        },
+        "normalizeUplinkOutput": {
+          "allOf": [
+            {
+              "$ref": "#/definitions/endDevicePayloadCodec/definitions/scriptOutput"
+            },
+            {
+              "type": "object",
+              "properties": {
+                "data": { "$ref": "https://schema.thethings.network/devicerepository/1/payload/schema#/definitions/uplinkPayload" }
+              }
+            },
+            {
+              "if": {
+                "type": "object",
+                "properties": {
+                  "errors": {
+                    "type": "array",
+                    "maxItems": 0
+                  }
+                }
+              },
+              "then": {
+                "type": "object",
+                "required": ["data"]
+              }
+            }
+          ]
         }
       },
       "properties": {
@@ -442,6 +479,9 @@
                       },
                       "output": {
                         "$ref": "#/definitions/endDevicePayloadCodec/definitions/decodeOutput"
+                      },
+                      "normalizedOutput": {
+                        "$ref": "#/definitions/endDevicePayloadCodec/definitions/normalizeUplinkOutput"
                       }
                     }
                   }
@@ -498,7 +538,8 @@
             }
           ]
         }
-      }
+      },
+      "additionalProperties": false
     },
     "endDeviceProfile": {
       "title": "End Device Profile",
@@ -1176,5 +1217,5 @@
       "required": ["name", "firmwareVersions"]
     }
   },
-  "anyOf": [{ "$ref": "#/definitions/endDevice" }, { "$ref": "#/definitions/endDeviceProfile" }, { "$ref": "#/definitions/vendorsIndex" }, { "$ref": "#/definitions/vendorIndex" }]
+  "oneOf": [{ "$ref": "#/definitions/endDevice" }, { "$ref": "#/definitions/endDeviceProfile" }, { "$ref": "#/definitions/vendorsIndex" }, { "$ref": "#/definitions/vendorIndex" }]
 }