Nsi processor (#29)

* initial DepthHazard

* renaming in Result, added unit test file

* added parameter checking to Get

* implemented Has for LifeLosshazard, need to test more

* changed unit tests to test individual methods

* implemented get for LifeLossHazard, added tests #15

* implemented IResultsWriter #16

* Implemented ConsoleWriter and unit tests for it #17

* Implemented functionality and tests for first unit test of Structure #18

* Added more tests for Structure, fixed issues with testing console output

* implemented Location and BoundingBox, not sure about GDAL format for BB yet though #19

* implemented IHazardProvider

* implemented RandomDepthHazardProvider and tests #21, and removed check on console output from StructureTest (test had weird behavior, console output passes intermittently but values are correct) #18

* removed unneeded dependencies auto-generated by visual studio

* added fields to Structure

* changed floats to doubles to match float64 specification

* removed filter from CI

* removed no-build

* changed unit tests for #17 and #18 to circumvent weird behavior with console output

* added initial interfaces and implementations for processors after the video call #23 #24 #25

* added GetLocation to ConsequenceReceptor and changed Location coordinate types to double to match NSI #27

* added auto properties where needed

* renamed indiviual tests to be more descriptive

* added API functionality

* added scratchpaper testing environment

* implemented feature stream processor

* added URL construction method + compatiblity with custom bounding boxes

* added bounding box to test larger area

* changed tests

* various refactors to make code cleaner

* muted warnings in Process, cleaned up Process test

Consequences.sln

@@ -14,6 +14,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ConsequencesTest", "ConsequencesTest\ConsequencesTest.csproj", "{95BCC49B-7780-41E9-8365-C51B5E1B3D5E}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ScratchPaper", "ScratchPaper\ScratchPaper.csproj", "{31071E1B-DA08-40CF-8F16-95982E85B45D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -28,6 +30,10 @@ Global
 		{95BCC49B-7780-41E9-8365-C51B5E1B3D5E}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{95BCC49B-7780-41E9-8365-C51B5E1B3D5E}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{95BCC49B-7780-41E9-8365-C51B5E1B3D5E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{31071E1B-DA08-40CF-8F16-95982E85B45D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{31071E1B-DA08-40CF-8F16-95982E85B45D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{31071E1B-DA08-40CF-8F16-95982E85B45D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{31071E1B-DA08-40CF-8F16-95982E85B45D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

Consequences/Consequences.csproj

@@ -4,6 +4,7 @@
 		<TargetFramework>net8.0</TargetFramework>
 		<ImplicitUsings>enable</ImplicitUsings>
 		<IsAotCompatible>true</IsAotCompatible>
+		<PublishAot>true</PublishAot>
 		<RootNamespace>USACE.HEC</RootNamespace>
 		<AssemblyName>USACE.HEC.Consequences</AssemblyName>
 	</PropertyGroup>

Consequences/Consequences/IBBoxStreamingProcessor.cs

@@ -3,5 +3,5 @@
 namespace USACE.HEC.Consequences;
 public interface IBBoxStreamingProcessor
 {
-  public void Process(BoundingBox boundingBox, Action<IConsequencesReceptor> consequenceReceptorProcess);
+  public Task Process(BoundingBox boundingBox, Action<IConsequencesReceptor> consequenceReceptorProcess);
 }

Consequences/Consequences/NSIStreamingProcessor.cs

@@ -1,51 +1,101 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+using System.Text;
+using System.Text.Json;
 using USACE.HEC.Geography;
 
 namespace USACE.HEC.Consequences;
+
 public class NSIStreamingProcessor : IBBoxStreamingProcessor
 {
-  public void Process(BoundingBox boundingBox, Action<IConsequencesReceptor> ConsequenceReceptorProcess)
+  public async Task Process(BoundingBox boundingBox, Action<IConsequencesReceptor> ConsequenceReceptorProcess)
   {
-    Structure s = new Structure();
-    ConsequenceReceptorProcess(s);
+    await ProcessStream(boundingBox, ConsequenceReceptorProcess);
   }
 
+  private static string ConstructURL(BoundingBox boundingBox, string directive)
+  {
+    StringBuilder url = new();
+
+    url.Append("https://nsi.sec.usace.army.mil/nsiapi/structures?bbox=");
+    url.Append(boundingBox.NSIFormat());
+
+    // directive to specify collection or stream
+    url.Append(directive);
+
+    return url.ToString();
+  }
 
 
-  /*
-  static async Task Test()
+  // this method processes an entire batch from the NSI at once rather than streaming them one by one
+  // was tested against ProcessStream and took used significantly more memory with similar times, so we opted to use ProcessStream instead
+  private static async Task ProcessCollection(BoundingBox boundingBox, Action<IConsequencesReceptor> ConsequenceReceptorProcess)
   {
-    // Define the API endpoint
-    string apiEndpoint = "https://nsi.sec.usace.army.mil/nsiapi/structures?bbox=-81.58418,30.25165,-81.58161,30.26939,-81.55898,30.26939,-81.55281,30.24998,-81.58418,30.25165";
+    string apiUrl = ConstructURL(boundingBox, "&fmt=fc");
 
-    // Create an instance of HttpClient
-    using (HttpClient client = new HttpClient())
+    using HttpClient client = new();
+    try
     {
-      try
-      {
-        // Make the GET request
-        HttpResponseMessage response = await client.GetAsync(apiEndpoint);
+      string jsonResponse = await client.GetStringAsync(apiUrl);
+
+      using JsonDocument doc = JsonDocument.Parse(jsonResponse);
 
-        // Check if the request was successful
-        response.EnsureSuccessStatusCode();
+      JsonElement featureCollection = doc.RootElement.GetProperty("features");
+      foreach (JsonElement structure in featureCollection.EnumerateArray())
+      {
+        // access the properties of each structure
+        JsonElement propertiesElement = structure.GetProperty("properties");
 
-        // Read and process the response content
-        string responseBody = await response.Content.ReadAsStringAsync();
+        // disable the warnings generated by Deserialize
+        #pragma warning disable IL2026
+        #pragma warning disable IL3050
+        Structure s = JsonSerializer.Deserialize<Structure>(propertiesElement.GetRawText());
+        #pragma warning restore IL2026
+        #pragma warning restore IL3050
 
-        // Output the response content (for demonstration purposes)
-        Console.WriteLine("API Response:");
-        Console.WriteLine(responseBody);
+        // apply the action to the deserialized structure
+        ConsequenceReceptorProcess(s);
       }
-      catch (HttpRequestException e)
+    }
+    catch (Exception e)
+    {
+      Console.WriteLine($"Request error: {e.Message}");
+    }
+  }
+
+  // processes a stream of JSONs from the NSI one by one as opposed to loading in the entire batch
+  private static async Task ProcessStream(BoundingBox boundingBox, Action<IConsequencesReceptor> ConsequenceReceptorProcess)
+  {
+    string apiURL = ConstructURL(boundingBox, "&fmt=fs");
+
+    using HttpClient httpClient = new();
+    try
+    {
+      HttpResponseMessage response = await httpClient.GetAsync(apiURL);
+      response.EnsureSuccessStatusCode();
+
+      using StreamReader reader = new(await response.Content.ReadAsStreamAsync());
+
+      // read in a line from the stream on by one
+      // each individual structure JSON is contained in one line
+      string line;
+      while ((line = await reader.ReadLineAsync()) != null)
       {
-        // Handle any errors that occur during the request
-        Console.WriteLine($"Request error: {e.Message}");
+        using JsonDocument structure = JsonDocument.Parse(line);
+        JsonElement propertiesElement = structure.RootElement.GetProperty("properties");
+
+        // disable the warnings generated by Deserialize
+        #pragma warning disable IL2026
+        #pragma warning disable IL3050
+        Structure s = JsonSerializer.Deserialize<Structure>(propertiesElement.GetRawText());
+        #pragma warning restore IL2026
+        #pragma warning restore IL3050
+
+        // apply the action to the deserialized structure
+        ConsequenceReceptorProcess(s);
       }
     }
+    catch (Exception e)
+    {
+      Console.WriteLine($"Request error: {e.Message}");
+    }
   }
-  */
 }

Consequences/Consequences/Structure.cs

@@ -67,7 +67,7 @@ public Location GetLocation()
 
   public Result Compute(IHazard hazard) 
   {
-    List<ResultItem> resultItems = new List<ResultItem>();
+    List<ResultItem> resultItems = [];
 
     if (hazard.Has(HazardParameter.Depth))
     {
@@ -101,6 +101,6 @@ public Result Compute(IHazard hazard)
                                      });
     }
 
-    return new Result(resultItems.ToArray());
+    return new Result([.. resultItems]);
   }
 }

Consequences/Results/ConsoleWriter.cs

@@ -27,7 +27,7 @@ private void CheckIfSameHeaders(Result res)
 
   public string WriteString(Result res)
   {
-    StringBuilder output = new StringBuilder();
+    StringBuilder output = new();
     if (!hasHeaderWritten)
     {
       // write the headers to the top of the file

Consequences/Results/Result.cs

@@ -15,8 +15,6 @@ public ResultItem Fetch(string name)
       if (ResultItems[i].ResultName == name)
         return ResultItems[i];
     // return empty ResultItem if not found
-    ResultItem item = new ResultItem();
-    item.ResultName = name;
-    return item;
+    return new ResultItem { ResultName = name };
   }
 }

ConsequencesTest/NSIStreamingProcessorTest.cs

@@ -1,28 +1,47 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization;
+using System.Text.Json;
 using USACE.HEC.Consequences;
 using USACE.HEC.Geography;
 using USACE.HEC.Hazards;
 using USACE.HEC.Results;
 
 namespace ConsequencesTest;
+
 public class NSIStreamingProcessorTest
 {
   [Fact]
-  public void Test()
+  public async Task Test()
   {
     IBBoxStreamingProcessor sp = new NSIStreamingProcessor();
-    // int i = 0;
     IHazardProvider depthHazardProvider = new RandomDepthHazardProvider(25);
+    Location upperLeft1 = new Location { X = -122.475275, Y = 37.752394 };
+    Location lowerRight1 = new Location { X = -122.473523, Y = 37.750642 };
+    depthHazardProvider.Extent = new BoundingBox(upperLeft1, lowerRight1);
     IResultsWriter consoleWriter = new ConsoleWriter();
 
-    sp.Process(depthHazardProvider.Extent, (IConsequencesReceptor s) => {
+    await sp.Process(depthHazardProvider.Extent, (IConsequencesReceptor s) => {
       Result r = s.Compute(depthHazardProvider.Hazard(s.GetLocation()));
       consoleWriter.Write(r);
     });
   }
+
+  [Fact]
+  public void JSON_Deserialize()
+  {
+    string jsonString = """{"type": "Feature","geometry": {"type": "Point","coordinates": [-81.563689, 30.265468]},"properties":{"fd_id":498054345,"bid":"862W7C8P+5GM-0-0-0-0","occtype":"RES1-1SNB","st_damcat":"RES","bldgtype":"M","found_type":"S","cbfips":"120310159233002","pop2amu65":2,"pop2amo65":0,"pop2pmu65":1,"pop2pmo65":0,"sqft":1195,"num_story":1,"ftprntid":"none_242828","ftprntsrc":null,"students":0,"found_ht":0.5,"val_struct":124643.72,"val_cont":62321.8604,"val_vehic":27000,"source":"P","med_yr_blt":2004,"firmzone":null,"o65disable":0.26,"u65disable":0.05,"x":-81.56368862,"y":30.265468241,"ground_elv":27.445583771209716,"ground_elv_m":8.365413665771484}}""";
+    Structure? s = new Structure();
+    using (JsonDocument doc = JsonDocument.Parse(jsonString))
+    {
+      // Extract the "properties" section
+      JsonElement root = doc.RootElement;
+      JsonElement propertiesElement = root.GetProperty("properties");
+
+      // Convert the "properties" section to a JSON string
+      string propertiesJson = propertiesElement.GetRawText();
+
+      // Deserialize the "properties" section into the Properties class
+      s = JsonSerializer.Deserialize<Structure>(propertiesJson);
+
+    }
+    Assert.Equal(498054345, s?.Name);
+  }
 }

ScratchPaper/Program.cs

@@ -0,0 +1,33 @@
+using USACE.HEC.Consequences;
+using USACE.HEC.Geography;
+
+internal class Program
+{
+  private static async Task Main(string[] args)
+  {
+    // city block in Sunset District, SF
+    Location upperLeft1 = new Location { X = -122.475275, Y = 37.752394 };
+    Location lowerRight1 = new Location { X = -122.473523, Y = 37.750642 };
+    BoundingBox boundingBox1 = new BoundingBox(upperLeft1, lowerRight1);
+
+
+    Location upperLeft2 = new Location { X = -122.5, Y = 37.8 };
+    Location lowerRight2 = new Location { X = -122, Y = 37.3 };
+    BoundingBox boundingBox2 = new BoundingBox(upperLeft2, lowerRight2);
+
+    IBBoxStreamingProcessor sp = new NSIStreamingProcessor();
+
+    int count = 0;
+    var watch = System.Diagnostics.Stopwatch.StartNew();
+    await sp.Process(boundingBox2, (IConsequencesReceptor s) => {
+      //Console.WriteLine(((Structure)s).Name);
+      count++;
+    });
+    watch.Stop();
+    var elapsedMs = watch.ElapsedMilliseconds;
+
+    Console.WriteLine(count);
+    Console.WriteLine("Time elapsed: " + elapsedMs.ToString() + " milliseconds");
+    Console.Read(); 
+  }
+}

ScratchPaper/ScratchPaper.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Consequences\Consequences.csproj" />
+  </ItemGroup>
+
+</Project>