From c39007246147b77d6bdba95e6f60ad9051a33c89 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 28 Aug 2024 14:27:27 -0700 Subject: [PATCH] various refactors to make code cleaner --- .../Consequences/NSIStreamingProcessor.cs | 132 +++++++----------- Consequences/Consequences/Structure.cs | 4 +- Consequences/Results/ConsoleWriter.cs | 2 +- Consequences/Results/Result.cs | 4 +- ConsequencesTest/NSIStreamingProcessorTest.cs | 83 +---------- 5 files changed, 54 insertions(+), 171 deletions(-) diff --git a/Consequences/Consequences/NSIStreamingProcessor.cs b/Consequences/Consequences/NSIStreamingProcessor.cs index 4f73997..d9ac455 100644 --- a/Consequences/Consequences/NSIStreamingProcessor.cs +++ b/Consequences/Consequences/NSIStreamingProcessor.cs @@ -1,35 +1,17 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Reflection.Emit; -using System.Text; +using System.Text; using System.Text.Json; -using System.Text.Json.Nodes; -using System.Threading.Tasks; using USACE.HEC.Geography; - namespace USACE.HEC.Consequences; + public class NSIStreamingProcessor : IBBoxStreamingProcessor { public async Task Process(BoundingBox boundingBox, Action ConsequenceReceptorProcess) - { - await ProcessStream(boundingBox, ConsequenceReceptorProcess); - //await ProcessCollection(boundingBox, ConsequenceReceptorProcess); - } - - public async Task TestProcessStream(BoundingBox boundingBox, Action ConsequenceReceptorProcess) { await ProcessStream(boundingBox, ConsequenceReceptorProcess); } - public async Task TestProcessCollection(BoundingBox boundingBox, Action ConsequenceReceptorProcess) - { - await ProcessCollection(boundingBox, ConsequenceReceptorProcess); - } - - private string ConstructURL(BoundingBox boundingBox, string directive) + private static string ConstructURL(BoundingBox boundingBox, string directive) { StringBuilder url = new StringBuilder(); @@ -42,84 +24,66 @@ private string ConstructURL(BoundingBox boundingBox, string directive) return url.ToString(); } - private async Task ProcessCollection(BoundingBox boundingBox, Action ConsequenceReceptorProcess) + + // this method processes an entire batch from the NSI at once rather than streaming them one by one + // was tested against ProcessStream and took used significantly more memory with similar times, so we opted to use ProcessStream instead + private static async Task ProcessCollection(BoundingBox boundingBox, Action ConsequenceReceptorProcess) { - // endpoint URL string apiUrl = ConstructURL(boundingBox, "&fmt=fc"); - // Create an instance of HttpClient - using (HttpClient client = new HttpClient()) + using HttpClient client = new HttpClient(); + try { - try - { - // send HTTP GET request, read in the entire json into a string - string jsonResponse = await client.GetStringAsync(apiUrl); - - using (JsonDocument doc = JsonDocument.Parse(jsonResponse)) - { - // access the FeatureCollection - JsonElement featureCollection = doc.RootElement.GetProperty("features"); - - // iterate through the FeatureCollection - foreach (JsonElement structure in featureCollection.EnumerateArray()) - { - // access the properties of each structure - JsonElement propertiesElement = structure.GetProperty("properties"); - - // deserialize the properties JSON and create new Structure() object - Structure s = JsonSerializer.Deserialize(propertiesElement.GetRawText()); - - // apply the action to the deserialized structure - ConsequenceReceptorProcess(s); - } - } - } - catch (Exception e) + string jsonResponse = await client.GetStringAsync(apiUrl); + + using JsonDocument doc = JsonDocument.Parse(jsonResponse); + + JsonElement featureCollection = doc.RootElement.GetProperty("features"); + foreach (JsonElement structure in featureCollection.EnumerateArray()) { - Console.WriteLine($"Request error: {e.Message}"); + // access the properties of each structure + JsonElement propertiesElement = structure.GetProperty("properties"); + Structure s = JsonSerializer.Deserialize(propertiesElement.GetRawText()); + + // apply the action to the deserialized structure + ConsequenceReceptorProcess(s); } } + catch (Exception e) + { + Console.WriteLine($"Request error: {e.Message}"); + } } - private async Task ProcessStream(BoundingBox boundingBox, Action ConsequenceReceptorProcess) + // processes a stream of JSONs from the NSI one by one as opposed to loading in the entire batch + private static async Task ProcessStream(BoundingBox boundingBox, Action ConsequenceReceptorProcess) { - // endpoint URL string apiURL = ConstructURL(boundingBox, "&fmt=fs"); - using (HttpClient httpClient = new HttpClient()) + using HttpClient httpClient = new(); + try { - try - { - // send HTTP GET request - HttpResponseMessage response = await httpClient.GetAsync(apiURL); - response.EnsureSuccessStatusCode(); - - using (StreamReader reader = new StreamReader(await response.Content.ReadAsStreamAsync())) - { - // read in a line from the stream on by one - // each individual structure JSON is contained in one line - string line; - while ((line = await reader.ReadLineAsync()) != null) - { - //Console.WriteLine(line); - //Console.WriteLine("********************"); - using (JsonDocument structure = JsonDocument.Parse(line)) - { - JsonElement propertiesElement = structure.RootElement.GetProperty("properties"); - // deserialize the properties JSON and create new Structure() object - Structure s = JsonSerializer.Deserialize(propertiesElement.GetRawText()); - - // apply the action to the deserialized structure - ConsequenceReceptorProcess(s); - } - } - } - } - catch (Exception e) + HttpResponseMessage response = await httpClient.GetAsync(apiURL); + response.EnsureSuccessStatusCode(); + + using StreamReader reader = new(await response.Content.ReadAsStreamAsync()); + + // read in a line from the stream on by one + // each individual structure JSON is contained in one line + string line; + while ((line = await reader.ReadLineAsync()) != null) { - // Handle any errors that occur during the request - Console.WriteLine($"Request error: {e.Message}"); + using JsonDocument structure = JsonDocument.Parse(line); + JsonElement propertiesElement = structure.RootElement.GetProperty("properties"); + Structure s = JsonSerializer.Deserialize(propertiesElement.GetRawText()); + + // apply the action to the deserialized structure + ConsequenceReceptorProcess(s); } } + catch (Exception e) + { + Console.WriteLine($"Request error: {e.Message}"); + } } } diff --git a/Consequences/Consequences/Structure.cs b/Consequences/Consequences/Structure.cs index 7e6b9ab..559cd1b 100644 --- a/Consequences/Consequences/Structure.cs +++ b/Consequences/Consequences/Structure.cs @@ -67,7 +67,7 @@ public Location GetLocation() public Result Compute(IHazard hazard) { - List resultItems = new List(); + List resultItems = []; if (hazard.Has(HazardParameter.Depth)) { @@ -101,6 +101,6 @@ public Result Compute(IHazard hazard) }); } - return new Result(resultItems.ToArray()); + return new Result([.. resultItems]); } } diff --git a/Consequences/Results/ConsoleWriter.cs b/Consequences/Results/ConsoleWriter.cs index 50f550f..831bdb5 100644 --- a/Consequences/Results/ConsoleWriter.cs +++ b/Consequences/Results/ConsoleWriter.cs @@ -27,7 +27,7 @@ private void CheckIfSameHeaders(Result res) public string WriteString(Result res) { - StringBuilder output = new StringBuilder(); + StringBuilder output = new(); if (!hasHeaderWritten) { // write the headers to the top of the file diff --git a/Consequences/Results/Result.cs b/Consequences/Results/Result.cs index bf4004d..f49f980 100644 --- a/Consequences/Results/Result.cs +++ b/Consequences/Results/Result.cs @@ -15,8 +15,6 @@ public ResultItem Fetch(string name) if (ResultItems[i].ResultName == name) return ResultItems[i]; // return empty ResultItem if not found - ResultItem item = new ResultItem(); - item.ResultName = name; - return item; + return new ResultItem { ResultName = name }; } } diff --git a/ConsequencesTest/NSIStreamingProcessorTest.cs b/ConsequencesTest/NSIStreamingProcessorTest.cs index 97e02dc..ce319b0 100644 --- a/ConsequencesTest/NSIStreamingProcessorTest.cs +++ b/ConsequencesTest/NSIStreamingProcessorTest.cs @@ -1,17 +1,10 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; -using System.Transactions; -using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Serialization; +using System.Text.Json; using USACE.HEC.Consequences; -using USACE.HEC.Geography; using USACE.HEC.Hazards; using USACE.HEC.Results; namespace ConsequencesTest; + public class NSIStreamingProcessorTest { [Fact] @@ -48,76 +41,4 @@ public void JSON_Deserialize() } Assert.Equal(498054345, s?.Name); } - - [Fact] - public void ProcessCollection() - {; - IBBoxStreamingProcessor sp = new NSIStreamingProcessor(); - sp.Process(null, (IConsequencesReceptor s) => { - - }); - - } - /* - [Fact] - public static async Task Ping() - { - // endpoint URL - string apiUrl = "https://nsi.sec.usace.army.mil/nsiapi/structures?bbox=-81.58418,30.25165,-81.58161,30.26939,-81.55898,30.26939,-81.55281,30.24998,-81.58418,30.25165"; - - using (var client = new HttpClient()) - { - // get the json from the NSI - string jsonResponse = await client.GetStringAsync(apiUrl); - - int count = 0; - using (JsonDocument doc = JsonDocument.Parse(jsonResponse)) - { - // access the FeatureCollection - JsonElement featureCollection = doc.RootElement.GetProperty("features"); - - // iterate through the FeatureCollection - foreach (JsonElement structure in featureCollection.EnumerateArray()) - { - // access the properties of each structure - JsonElement propertiesElement = structure.GetProperty("properties"); - - // deserialize the properties JSON and create new Structure() object - Structure? s = JsonSerializer.Deserialize(propertiesElement.GetRawText()); - count++; - } - } - Assert.Equal(2735, count); - } - } - - [Fact] - public static async Task ProcessCollection() - { - // endpoint URL - string apiUrl = "https://nsi.sec.usace.army.mil/nsiapi/structures?bbox=-81.58418,30.25165,-81.58161,30.26939,-81.55898,30.26939,-81.55281,30.24998,-81.58418,30.25165"; - - using (var client = new HttpClient()) - { - // get the json from the NSI - string jsonResponse = await client.GetStringAsync(apiUrl); - - using (JsonDocument doc = JsonDocument.Parse(jsonResponse)) - { - // access the FeatureCollection - JsonElement featureCollection = doc.RootElement.GetProperty("features"); - - // iterate through the FeatureCollection - foreach (JsonElement structure in featureCollection.EnumerateArray()) - { - // access the properties of each structure - JsonElement propertiesElement = structure.GetProperty("properties"); - - // deserialize the properties JSON and create new Structure() object - Structure? s = JsonSerializer.Deserialize(propertiesElement.GetRawText()); - } - } - } - } - */ }