From 100a50e116a29119a4e08365309fb91f258373db Mon Sep 17 00:00:00 2001 From: Levi Lesches Date: Fri, 3 May 2024 00:30:04 -0400 Subject: [PATCH] Improved Science post-processing sketch --- bin/science.dart | 67 ++++++++++++++++++++------------ lib/src/models/view/science.dart | 2 +- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/bin/science.dart b/bin/science.dart index 8cdc862f00..130eb94f40 100644 --- a/bin/science.dart +++ b/bin/science.dart @@ -1,3 +1,5 @@ +// ignore_for_file: avoid_print + import "dart:convert"; import "dart:io"; @@ -7,16 +9,14 @@ import "package:burt_network/generated.dart"; /// A cleaner name for any message generated by Protobuf. typedef Message = proto.GeneratedMessage; -/// Change these paths as needed -const oldPath = "/Users/aidanahram/Downloads/FakeData1.log"; -const newPath = "/Users/aidanahram/Downloads/FakeData1New.log"; - /// Return true to keep this data in the dataset, or false to remove it. /// /// Not only are the sensors on the rover wonky, but the CAN bus can corrupt data along the way. /// This function can be used to remove any unwanted bad data. -bool shouldKeepData(ScienceData data) { - // Filter data based on reasonable thresholds: +bool shouldKeepData(Timestamp timestamp, ScienceData data) { + final elapsed = firstTimestamp! - timestamp; + + // Filter data based on reasonable thresholds, eg: if (data.co2.isOutOfBounds(min: 400, max: 2000)) return false; if (data.methane.isOutOfBounds(min: 100, max: 500)) return false; if (data.pH.isOutOfBounds(min: 0, max: 14)) return false; @@ -24,6 +24,7 @@ bool shouldKeepData(ScienceData data) { if (data.temperature.isOutOfBounds(min: 0, max: 100)) return false; // Any other conditions should go here: + if (elapsed >= 30 * 60) return false; return true; // if none of the above rules are broken, then keep this data } @@ -46,6 +47,7 @@ WrappedMessage modifyData(Timestamp timestamp, ScienceData data) { // ignore: p // Wrap the data and return it. Do not delete. return data.wrap(timestamp.toDateTime()); } + /* /// Use this to add new data to your dataset. List newData = [ @@ -68,22 +70,16 @@ List newData = [ pH: t + s + (random.nextInt(15).toDouble()), ).wrap(DateTime.now().add(Duration(seconds: t + 1))) ]; - -// ==================== Do not edit unless you're Aidan or Levi ==================== - -final random = Random(); */ +// ==================== Do not edit below this line unless you know what you're doing ==================== + extension on num { bool isOutOfBounds({required num min, required num max}) => (this > max) || (this != 0 && this < min); } -/// Outputs log data to the correct file based on message -Future logData(WrappedMessage message) async{ - final List bytes = message.writeToBuffer(); - final line = base64.encode(bytes); - final file = File(newPath); - await file.writeAsString("$line\n", mode: FileMode.append, flush: true); +extension on Timestamp { + int operator -(Timestamp other) => (seconds - other.seconds).toInt(); } /// Reads logs from the given file. @@ -92,16 +88,37 @@ Future> readLogs(File file) async => [ WrappedMessage.fromBuffer(base64.decode(line)), ]; -void main() async { - var pruned = 0; - if (File(newPath).existsSync()) await File(newPath).delete(); - final oldData = await readLogs(File(oldPath)); +Timestamp? firstTimestamp; + +void main(List args) async { + if (args.isEmpty) { + print("Usage: dart run :science path/to/ScienceData.log"); + return; + } + final inputFile = File(args.first); + if (!inputFile.existsSync()) { + print("Error: Could not read log file: ${inputFile.absolute}"); + return; + } + final outputFile = File("${inputFile.parent.path}/processed/ScienceData.log"); + if (outputFile.existsSync()) await outputFile.delete(); + await outputFile.create(recursive: true); + var total = 0; + var written = 0; + final oldData = await readLogs(inputFile); + final sink = outputFile.openWrite(mode: FileMode.append); for (final wrapper in oldData) { + total++; final data = ScienceData.fromBuffer(wrapper.data); - if (!shouldKeepData(data)) { pruned++; continue; } - else { await logData(modifyData(wrapper.timestamp, data)); } + firstTimestamp ??= wrapper.timestamp; + if (!shouldKeepData(wrapper.timestamp, data)) continue; + final message = modifyData(wrapper.timestamp, data); + final List bytes = message.writeToBuffer(); + final line = base64.encode(bytes); + sink.writeln(line); + written++; } - print("Pruned $pruned rows of data"); // ignore: avoid_print + await sink.flush(); + await sink.close(); + print("Read $total lines, output $written lines"); } - -// ==================== Do not edit unless you're Aidan or Levi ==================== diff --git a/lib/src/models/view/science.dart b/lib/src/models/view/science.dart index 9aee0018c2..62015b6206 100644 --- a/lib/src/models/view/science.dart +++ b/lib/src/models/view/science.dart @@ -149,7 +149,7 @@ class ScienceModel with ChangeNotifier { /// Adds a [WrappedMessage] containing a [ScienceData] to the UI. void addMessage(WrappedMessage wrapper) { - if (wrapper.name != ScienceData().messageName) throw ArgumentError("Incorrect log type: ${wrapper.name}"); + if (!wrapper.name.contains(ScienceData().messageName)) throw ArgumentError("Incorrect log type: ${wrapper.name}"); final data = wrapper.decode(ScienceData.fromBuffer); final sample = data.sample; if (!wrapper.hasTimestamp()) { throw ArgumentError("Data is missing a timestamp"); }