From 57c974ef55e5c3da2d3630771a4e6d14096f9267 Mon Sep 17 00:00:00 2001 From: Jack Schonherr Date: Wed, 28 Aug 2024 15:49:56 -0700 Subject: [PATCH] Nsi processor (#29) * initial DepthHazard * renaming in Result, added unit test file * added parameter checking to Get * implemented Has for LifeLosshazard, need to test more * changed unit tests to test individual methods * implemented get for LifeLossHazard, added tests #15 * implemented IResultsWriter #16 * Implemented ConsoleWriter and unit tests for it #17 * Implemented functionality and tests for first unit test of Structure #18 * Added more tests for Structure, fixed issues with testing console output * implemented Location and BoundingBox, not sure about GDAL format for BB yet though #19 * implemented IHazardProvider * implemented RandomDepthHazardProvider and tests #21, and removed check on console output from StructureTest (test had weird behavior, console output passes intermittently but values are correct) #18 * removed unneeded dependencies auto-generated by visual studio * added fields to Structure * changed floats to doubles to match float64 specification * removed filter from CI * removed no-build * changed unit tests for #17 and #18 to circumvent weird behavior with console output * added initial interfaces and implementations for processors after the video call #23 #24 #25 * added GetLocation to ConsequenceReceptor and changed Location coordinate types to double to match NSI #27 * added auto properties where needed * renamed indiviual tests to be more descriptive * added API functionality * added scratchpaper testing environment * implemented feature stream processor * added URL construction method + compatiblity with custom bounding boxes * added bounding box to test larger area * changed tests * various refactors to make code cleaner * muted warnings in Process, cleaned up Process test --- Consequences.sln | 6 + Consequences/Consequences.csproj | 1 + .../Consequences/IBBoxStreamingProcessor.cs | 2 +- .../Consequences/NSIStreamingProcessor.cs | 108 +++++++++++++----- Consequences/Consequences/Structure.cs | 4 +- Consequences/Results/ConsoleWriter.cs | 2 +- Consequences/Results/Result.cs | 4 +- ConsequencesTest/NSIStreamingProcessorTest.cs | 37 ++++-- ScratchPaper/Program.cs | 33 ++++++ ScratchPaper/ScratchPaper.csproj | 14 +++ 10 files changed, 166 insertions(+), 45 deletions(-) create mode 100644 ScratchPaper/Program.cs create mode 100644 ScratchPaper/ScratchPaper.csproj diff --git a/Consequences.sln b/Consequences.sln index 08b8004..e148dd9 100644 --- a/Consequences.sln +++ b/Consequences.sln @@ -14,6 +14,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ConsequencesTest", "ConsequencesTest\ConsequencesTest.csproj", "{95BCC49B-7780-41E9-8365-C51B5E1B3D5E}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ScratchPaper", "ScratchPaper\ScratchPaper.csproj", "{31071E1B-DA08-40CF-8F16-95982E85B45D}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -28,6 +30,10 @@ Global {95BCC49B-7780-41E9-8365-C51B5E1B3D5E}.Debug|Any CPU.Build.0 = Debug|Any CPU {95BCC49B-7780-41E9-8365-C51B5E1B3D5E}.Release|Any CPU.ActiveCfg = Release|Any CPU {95BCC49B-7780-41E9-8365-C51B5E1B3D5E}.Release|Any CPU.Build.0 = Release|Any CPU + {31071E1B-DA08-40CF-8F16-95982E85B45D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {31071E1B-DA08-40CF-8F16-95982E85B45D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {31071E1B-DA08-40CF-8F16-95982E85B45D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {31071E1B-DA08-40CF-8F16-95982E85B45D}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Consequences/Consequences.csproj b/Consequences/Consequences.csproj index 04adbd9..ae3fea7 100644 --- a/Consequences/Consequences.csproj +++ b/Consequences/Consequences.csproj @@ -4,6 +4,7 @@ net8.0 enable true + true USACE.HEC USACE.HEC.Consequences diff --git a/Consequences/Consequences/IBBoxStreamingProcessor.cs b/Consequences/Consequences/IBBoxStreamingProcessor.cs index d7d036d..0dea8f3 100644 --- a/Consequences/Consequences/IBBoxStreamingProcessor.cs +++ b/Consequences/Consequences/IBBoxStreamingProcessor.cs @@ -3,5 +3,5 @@ namespace USACE.HEC.Consequences; public interface IBBoxStreamingProcessor { - public void Process(BoundingBox boundingBox, Action consequenceReceptorProcess); + public Task Process(BoundingBox boundingBox, Action consequenceReceptorProcess); } diff --git a/Consequences/Consequences/NSIStreamingProcessor.cs b/Consequences/Consequences/NSIStreamingProcessor.cs index 0939417..c287d6a 100644 --- a/Consequences/Consequences/NSIStreamingProcessor.cs +++ b/Consequences/Consequences/NSIStreamingProcessor.cs @@ -1,51 +1,101 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using System.Text; +using System.Text.Json; using USACE.HEC.Geography; namespace USACE.HEC.Consequences; + public class NSIStreamingProcessor : IBBoxStreamingProcessor { - public void Process(BoundingBox boundingBox, Action ConsequenceReceptorProcess) + public async Task Process(BoundingBox boundingBox, Action ConsequenceReceptorProcess) { - Structure s = new Structure(); - ConsequenceReceptorProcess(s); + await ProcessStream(boundingBox, ConsequenceReceptorProcess); } + private static string ConstructURL(BoundingBox boundingBox, string directive) + { + StringBuilder url = new(); + + url.Append("https://nsi.sec.usace.army.mil/nsiapi/structures?bbox="); + url.Append(boundingBox.NSIFormat()); + + // directive to specify collection or stream + url.Append(directive); + + return url.ToString(); + } - /* - static async Task Test() + // this method processes an entire batch from the NSI at once rather than streaming them one by one + // was tested against ProcessStream and took used significantly more memory with similar times, so we opted to use ProcessStream instead + private static async Task ProcessCollection(BoundingBox boundingBox, Action ConsequenceReceptorProcess) { - // Define the API endpoint - string apiEndpoint = "https://nsi.sec.usace.army.mil/nsiapi/structures?bbox=-81.58418,30.25165,-81.58161,30.26939,-81.55898,30.26939,-81.55281,30.24998,-81.58418,30.25165"; + string apiUrl = ConstructURL(boundingBox, "&fmt=fc"); - // Create an instance of HttpClient - using (HttpClient client = new HttpClient()) + using HttpClient client = new(); + try { - try - { - // Make the GET request - HttpResponseMessage response = await client.GetAsync(apiEndpoint); + string jsonResponse = await client.GetStringAsync(apiUrl); + + using JsonDocument doc = JsonDocument.Parse(jsonResponse); - // Check if the request was successful - response.EnsureSuccessStatusCode(); + JsonElement featureCollection = doc.RootElement.GetProperty("features"); + foreach (JsonElement structure in featureCollection.EnumerateArray()) + { + // access the properties of each structure + JsonElement propertiesElement = structure.GetProperty("properties"); - // Read and process the response content - string responseBody = await response.Content.ReadAsStringAsync(); + // disable the warnings generated by Deserialize + #pragma warning disable IL2026 + #pragma warning disable IL3050 + Structure s = JsonSerializer.Deserialize(propertiesElement.GetRawText()); + #pragma warning restore IL2026 + #pragma warning restore IL3050 - // Output the response content (for demonstration purposes) - Console.WriteLine("API Response:"); - Console.WriteLine(responseBody); + // apply the action to the deserialized structure + ConsequenceReceptorProcess(s); } - catch (HttpRequestException e) + } + catch (Exception e) + { + Console.WriteLine($"Request error: {e.Message}"); + } + } + + // processes a stream of JSONs from the NSI one by one as opposed to loading in the entire batch + private static async Task ProcessStream(BoundingBox boundingBox, Action ConsequenceReceptorProcess) + { + string apiURL = ConstructURL(boundingBox, "&fmt=fs"); + + using HttpClient httpClient = new(); + try + { + HttpResponseMessage response = await httpClient.GetAsync(apiURL); + response.EnsureSuccessStatusCode(); + + using StreamReader reader = new(await response.Content.ReadAsStreamAsync()); + + // read in a line from the stream on by one + // each individual structure JSON is contained in one line + string line; + while ((line = await reader.ReadLineAsync()) != null) { - // Handle any errors that occur during the request - Console.WriteLine($"Request error: {e.Message}"); + using JsonDocument structure = JsonDocument.Parse(line); + JsonElement propertiesElement = structure.RootElement.GetProperty("properties"); + + // disable the warnings generated by Deserialize + #pragma warning disable IL2026 + #pragma warning disable IL3050 + Structure s = JsonSerializer.Deserialize(propertiesElement.GetRawText()); + #pragma warning restore IL2026 + #pragma warning restore IL3050 + + // apply the action to the deserialized structure + ConsequenceReceptorProcess(s); } } + catch (Exception e) + { + Console.WriteLine($"Request error: {e.Message}"); + } } - */ } diff --git a/Consequences/Consequences/Structure.cs b/Consequences/Consequences/Structure.cs index 7e6b9ab..559cd1b 100644 --- a/Consequences/Consequences/Structure.cs +++ b/Consequences/Consequences/Structure.cs @@ -67,7 +67,7 @@ public Location GetLocation() public Result Compute(IHazard hazard) { - List resultItems = new List(); + List resultItems = []; if (hazard.Has(HazardParameter.Depth)) { @@ -101,6 +101,6 @@ public Result Compute(IHazard hazard) }); } - return new Result(resultItems.ToArray()); + return new Result([.. resultItems]); } } diff --git a/Consequences/Results/ConsoleWriter.cs b/Consequences/Results/ConsoleWriter.cs index 50f550f..831bdb5 100644 --- a/Consequences/Results/ConsoleWriter.cs +++ b/Consequences/Results/ConsoleWriter.cs @@ -27,7 +27,7 @@ private void CheckIfSameHeaders(Result res) public string WriteString(Result res) { - StringBuilder output = new StringBuilder(); + StringBuilder output = new(); if (!hasHeaderWritten) { // write the headers to the top of the file diff --git a/Consequences/Results/Result.cs b/Consequences/Results/Result.cs index bf4004d..f49f980 100644 --- a/Consequences/Results/Result.cs +++ b/Consequences/Results/Result.cs @@ -15,8 +15,6 @@ public ResultItem Fetch(string name) if (ResultItems[i].ResultName == name) return ResultItems[i]; // return empty ResultItem if not found - ResultItem item = new ResultItem(); - item.ResultName = name; - return item; + return new ResultItem { ResultName = name }; } } diff --git a/ConsequencesTest/NSIStreamingProcessorTest.cs b/ConsequencesTest/NSIStreamingProcessorTest.cs index f7de2c8..0e29897 100644 --- a/ConsequencesTest/NSIStreamingProcessorTest.cs +++ b/ConsequencesTest/NSIStreamingProcessorTest.cs @@ -1,28 +1,47 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization; +using System.Text.Json; using USACE.HEC.Consequences; using USACE.HEC.Geography; using USACE.HEC.Hazards; using USACE.HEC.Results; namespace ConsequencesTest; + public class NSIStreamingProcessorTest { [Fact] - public void Test() + public async Task Test() { IBBoxStreamingProcessor sp = new NSIStreamingProcessor(); - // int i = 0; IHazardProvider depthHazardProvider = new RandomDepthHazardProvider(25); + Location upperLeft1 = new Location { X = -122.475275, Y = 37.752394 }; + Location lowerRight1 = new Location { X = -122.473523, Y = 37.750642 }; + depthHazardProvider.Extent = new BoundingBox(upperLeft1, lowerRight1); IResultsWriter consoleWriter = new ConsoleWriter(); - sp.Process(depthHazardProvider.Extent, (IConsequencesReceptor s) => { + await sp.Process(depthHazardProvider.Extent, (IConsequencesReceptor s) => { Result r = s.Compute(depthHazardProvider.Hazard(s.GetLocation())); consoleWriter.Write(r); }); } + + [Fact] + public void JSON_Deserialize() + { + string jsonString = """{"type": "Feature","geometry": {"type": "Point","coordinates": [-81.563689, 30.265468]},"properties":{"fd_id":498054345,"bid":"862W7C8P+5GM-0-0-0-0","occtype":"RES1-1SNB","st_damcat":"RES","bldgtype":"M","found_type":"S","cbfips":"120310159233002","pop2amu65":2,"pop2amo65":0,"pop2pmu65":1,"pop2pmo65":0,"sqft":1195,"num_story":1,"ftprntid":"none_242828","ftprntsrc":null,"students":0,"found_ht":0.5,"val_struct":124643.72,"val_cont":62321.8604,"val_vehic":27000,"source":"P","med_yr_blt":2004,"firmzone":null,"o65disable":0.26,"u65disable":0.05,"x":-81.56368862,"y":30.265468241,"ground_elv":27.445583771209716,"ground_elv_m":8.365413665771484}}"""; + Structure? s = new Structure(); + using (JsonDocument doc = JsonDocument.Parse(jsonString)) + { + // Extract the "properties" section + JsonElement root = doc.RootElement; + JsonElement propertiesElement = root.GetProperty("properties"); + + // Convert the "properties" section to a JSON string + string propertiesJson = propertiesElement.GetRawText(); + + // Deserialize the "properties" section into the Properties class + s = JsonSerializer.Deserialize(propertiesJson); + + } + Assert.Equal(498054345, s?.Name); + } } diff --git a/ScratchPaper/Program.cs b/ScratchPaper/Program.cs new file mode 100644 index 0000000..dcf3604 --- /dev/null +++ b/ScratchPaper/Program.cs @@ -0,0 +1,33 @@ +using USACE.HEC.Consequences; +using USACE.HEC.Geography; + +internal class Program +{ + private static async Task Main(string[] args) + { + // city block in Sunset District, SF + Location upperLeft1 = new Location { X = -122.475275, Y = 37.752394 }; + Location lowerRight1 = new Location { X = -122.473523, Y = 37.750642 }; + BoundingBox boundingBox1 = new BoundingBox(upperLeft1, lowerRight1); + + + Location upperLeft2 = new Location { X = -122.5, Y = 37.8 }; + Location lowerRight2 = new Location { X = -122, Y = 37.3 }; + BoundingBox boundingBox2 = new BoundingBox(upperLeft2, lowerRight2); + + IBBoxStreamingProcessor sp = new NSIStreamingProcessor(); + + int count = 0; + var watch = System.Diagnostics.Stopwatch.StartNew(); + await sp.Process(boundingBox2, (IConsequencesReceptor s) => { + //Console.WriteLine(((Structure)s).Name); + count++; + }); + watch.Stop(); + var elapsedMs = watch.ElapsedMilliseconds; + + Console.WriteLine(count); + Console.WriteLine("Time elapsed: " + elapsedMs.ToString() + " milliseconds"); + Console.Read(); + } +} \ No newline at end of file diff --git a/ScratchPaper/ScratchPaper.csproj b/ScratchPaper/ScratchPaper.csproj new file mode 100644 index 0000000..9014475 --- /dev/null +++ b/ScratchPaper/ScratchPaper.csproj @@ -0,0 +1,14 @@ + + + + Exe + net8.0 + enable + enable + + + + + + +