Improved Science post-processing sketch

bin/science.dart

@@ -1,3 +1,5 @@
+// ignore_for_file: avoid_print
+
 import "dart:convert";
 import "dart:io";
 
@@ -7,23 +9,22 @@ import "package:burt_network/generated.dart";
 /// A cleaner name for any message generated by Protobuf.
 typedef Message = proto.GeneratedMessage;
 
-/// Change these paths as needed
-const oldPath = "/Users/aidanahram/Downloads/FakeData1.log";
-const newPath = "/Users/aidanahram/Downloads/FakeData1New.log";
-
 /// Return true to keep this data in the dataset, or false to remove it.
 /// 
 /// Not only are the sensors on the rover wonky, but the CAN bus can corrupt data along the way.
 /// This function can be used to remove any unwanted bad data.
-bool shouldKeepData(ScienceData data) {
-	// Filter data based on reasonable thresholds:
+bool shouldKeepData(Timestamp timestamp, ScienceData data) {
+  final elapsed = firstTimestamp! - timestamp;
+
+	// Filter data based on reasonable thresholds, eg:
 	if (data.co2.isOutOfBounds(min: 400, max: 2000)) return false;
 	if (data.methane.isOutOfBounds(min: 100, max: 500)) return false;
 	if (data.pH.isOutOfBounds(min: 0, max: 14)) return false;
 	if (data.humidity.isOutOfBounds(min: 0, max: 50)) return false;
 	if (data.temperature.isOutOfBounds(min: 0, max: 100)) return false;
 
 	// Any other conditions should go here: 
+  if (elapsed >= 30 * 60) return false;
 
 	return true;  // if none of the above rules are broken, then keep this data
 }
@@ -46,6 +47,7 @@ WrappedMessage modifyData(Timestamp timestamp, ScienceData data) {  // ignore: p
 	// Wrap the data and return it. Do not delete.
 	return data.wrap(timestamp.toDateTime());
 }
+
 /*
 /// Use this to add new data to your dataset.
 List<WrappedMessage> newData = [
@@ -68,22 +70,16 @@ List<WrappedMessage> newData = [
 			pH: t + s + (random.nextInt(15).toDouble()),
 		).wrap(DateTime.now().add(Duration(seconds: t + 1)))
 ];
-
-// ==================== Do not edit unless you're Aidan or Levi ====================
-
-final random = Random();
 */
 
+// ==================== Do not edit below this line unless you know what you're doing ====================
+
 extension on num {
 	bool isOutOfBounds({required num min, required num max}) => (this > max) || (this != 0 && this < min);
 }
 
-/// Outputs log data to the correct file based on message
-Future<void> logData(WrappedMessage message) async{
-  final List<int> bytes = message.writeToBuffer();
-  final line = base64.encode(bytes);
-  final file = File(newPath);
-  await file.writeAsString("$line\n", mode: FileMode.append, flush: true);
+extension on Timestamp {
+  int operator -(Timestamp other) => (seconds - other.seconds).toInt();
 }
 
 /// Reads logs from the given file.
@@ -92,16 +88,37 @@ Future<List<WrappedMessage>> readLogs(File file) async => [
     WrappedMessage.fromBuffer(base64.decode(line)),
 ];
 
-void main() async {
-	var pruned = 0;
-	if (File(newPath).existsSync()) await File(newPath).delete();
-	final oldData = await readLogs(File(oldPath));
+Timestamp? firstTimestamp;
+
+void main(List<String> args) async {
+  if (args.isEmpty) {
+    print("Usage: dart run :science path/to/ScienceData.log");
+    return;
+  }
+  final inputFile = File(args.first);
+  if (!inputFile.existsSync()) {
+    print("Error: Could not read log file: ${inputFile.absolute}");
+    return;
+  }
+  final outputFile = File("${inputFile.parent.path}/processed/ScienceData.log");
+	if (outputFile.existsSync()) await outputFile.delete();
+  await outputFile.create(recursive: true);
+	var total = 0;
+  var written = 0;
+	final oldData = await readLogs(inputFile);
+  final sink = outputFile.openWrite(mode: FileMode.append);
 	for (final wrapper in oldData) {
+    total++;
 		final data = ScienceData.fromBuffer(wrapper.data);
-		if (!shouldKeepData(data)) { pruned++; continue; }
-		else { await logData(modifyData(wrapper.timestamp, data)); }
+    firstTimestamp ??= wrapper.timestamp;
+		if (!shouldKeepData(wrapper.timestamp, data)) continue;
+		final message = modifyData(wrapper.timestamp, data);
+    final List<int> bytes = message.writeToBuffer();
+    final line = base64.encode(bytes);
+    sink.writeln(line);
+    written++;
 	}
-	print("Pruned $pruned rows of data");  // ignore: avoid_print
+  await sink.flush();
+  await sink.close();
+	print("Read $total lines, output $written lines");
 }
-
-// ==================== Do not edit unless you're Aidan or Levi ====================

lib/src/models/view/science.dart

@@ -149,7 +149,7 @@ class ScienceModel with ChangeNotifier {
 
 	/// Adds a [WrappedMessage] containing a [ScienceData] to the UI.
 	void addMessage(WrappedMessage wrapper) {
-		if (wrapper.name != ScienceData().messageName) throw ArgumentError("Incorrect log type: ${wrapper.name}");
+		if (!wrapper.name.contains(ScienceData().messageName)) throw ArgumentError("Incorrect log type: ${wrapper.name}");
 		final data = wrapper.decode(ScienceData.fromBuffer);
 		final sample = data.sample;
 		if (!wrapper.hasTimestamp()) { throw ArgumentError("Data is missing a timestamp"); }