fix: deal with 429 and extra logging

Informatievlaanderen · Oct 24, 2024 · a317fa2 · a317fa2
1 parent e54e1d5
commit a317fa2
Showing 1 changed file with 71 additions and 40 deletions.
diff --git a/src/Ops.Tickets.Console/Program.cs b/src/Ops.Tickets.Console/Program.cs
@@ -6,6 +6,7 @@
     using System.Globalization;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Net.Http.Json;
     using System.Text;
@@ -60,64 +61,94 @@ await Parallel.ForEachAsync(
                     tickets,
                     new ParallelOptions { MaxDegreeOfParallelism = 5, CancellationToken = cts.Token },
                     async (ticket, token) =>
-                {
-                    if(token.IsCancellationRequested)
-                        return;
-
-                    Interlocked.Increment(ref countProcessed);
-                    Console.WriteLine($"Processing ticket: {ticket.TicketUrl} ({Math.Round((double)countProcessed / tickets.Count * 100, 2)}%)");
-
-                    try
                     {
-                        var done = false;
-                        while (!done)
-                        {
-                            var response = await client.GetFromJsonAsync<TicketResponse>(ticket.TicketUrl, token);
-
-                            done = response!.Status.Equals("complete") || response.Status.Equals("error");
-                            if (!done)
-                            {
-                                Console.WriteLine($"Ticket has status {response.Status}. Waiting 2 seconds.");
-                                await Task.Delay(TimeSpan.FromSeconds(2), token);
-                                continue;
-                            }
+                        if (token.IsCancellationRequested)
+                            return;
 
-                            if (response.Status.Equals("complete"))
-                            {
-                                completedTickets.Add(ticket);
-                            }
+                        Interlocked.Increment(ref countProcessed);
+                        Console.WriteLine($"Processing ticket: {ticket.TicketUrl} ({Math.Round((double)countProcessed / tickets.Count * 100, 2)}%)");
 
-                            if (response.Status.Equals("error"))
+                        try
+                        {
+                            var done = false;
+                            while (!done)
                             {
-                                errorTickets.Add(new TicketErrorRecord(ticket.Id, ticket.TicketUrl, response.Result.Json));
+                                try
+                                {
+                                    var response = await client.GetFromJsonAsync<TicketResponse>(ticket.TicketUrl, token);
+
+                                    done = response!.Status.Equals("complete") || response.Status.Equals("error");
+                                    if (!done)
+                                    {
+                                        Console.WriteLine($"Ticket has status {response.Status}. Waiting 2 seconds.");
+                                        await Task.Delay(TimeSpan.FromSeconds(2), token);
+                                        continue;
+                                    }
+
+                                    if (response.Status.Equals("complete"))
+                                    {
+                                        completedTickets.Add(ticket);
+                                    }
+
+                                    if (response.Status.Equals("error"))
+                                    {
+                                        errorTickets.Add(new TicketErrorRecord(ticket.Id, ticket.TicketUrl, response.Result.Json));
+                                    }
+                                }
+                                catch (WebException ex)
+                                {
+                                    if (ex.Response is HttpWebResponse response
+                                        && response.StatusCode == HttpStatusCode.TooManyRequests)
+                                    {
+                                        await Task.Delay(50);
+                                        continue;
+                                    }
+
+                                    throw;
+                                }
+
+                                done = true;
                             }
-
-                            done = true;
                         }
-
-                    }
-                    catch (Exception e)
-                    {
-                        Console.WriteLine($"Failed to process id: {ticket}");
-                        Console.WriteLine(e);
-                        await cts.CancelAsync();
-                        throw;
-                    }
-                });
+                        catch (Exception e)
+                        {
+                            Console.WriteLine($"Failed to process id: {ticket.TicketUrl}");
+                            Console.WriteLine(e);
+                            await cts.CancelAsync();
+                            throw;
+                        }
+                    });
             }
             finally
             {
-                var completedTicketFileStream = new FileStream(Path.Join(outputCsvFolder, "completed.csv"), FileMode.CreateNew);
+                Directory.CreateDirectory(outputCsvFolder);
+
+                var completedPath = Path.Join(outputCsvFolder, "completed.csv");
+                if (File.Exists(completedPath))
+                {
+                    File.Delete(completedPath);
+                }
+
+                var errorPath = Path.Join(outputCsvFolder, "error.csv");
+                if (File.Exists(errorPath))
+                {
+                    File.Delete(errorPath);
+                }
+
+                var completedTicketFileStream = new FileStream(completedPath, FileMode.CreateNew);
                 await using var completedTicketStreamWriter = new StreamWriter(completedTicketFileStream);
                 await using var completedTicketCsvWriter = new CsvWriter(completedTicketStreamWriter, CsvConfiguration);
 
                 completedTicketCsvWriter.Context.RegisterClassMap<TicketRecordMap>();
                 await completedTicketCsvWriter.WriteRecordsAsync(completedTickets, CancellationToken.None);
                 await completedTicketCsvWriter.FlushAsync();
 
+                Console.WriteLine($"Completed tickets: {completedTickets.Count}");
+                Console.WriteLine($"Error tickets: {errorTickets.Count}");
+
                 if (errorTickets.Any())
                 {
-                    var errorTicketFileStream = new FileStream(Path.Join(outputCsvFolder, "error.csv"), FileMode.CreateNew);
+                    var errorTicketFileStream = new FileStream(errorPath, FileMode.CreateNew);
                     await using var errorTicketStreamWriter = new StreamWriter(errorTicketFileStream);
                     await using var errorTicketCsvWriter = new CsvWriter(errorTicketStreamWriter, CsvConfiguration);