From d9e347db35cc8ea3ac310c3da11db808adf5f7cb Mon Sep 17 00:00:00 2001 From: miyaji255 <84168445+miyaji255@users.noreply.github.com> Date: Fri, 15 Mar 2024 23:32:36 +0900 Subject: [PATCH 1/5] =?UTF-8?q?#19=20=E3=82=B9=E3=82=AF=E3=83=AC=E3=82=A4?= =?UTF-8?q?=E3=83=94=E3=83=B3=E3=82=B0=E7=94=A8=E3=82=AF=E3=83=A9=E3=82=A4?= =?UTF-8?q?=E3=82=A2=E3=83=B3=E3=83=88=E3=82=92=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Services/IScrapingClientService.cs | 10 +++++++++ .../Services/ScrapingAozoraService.cs | 5 ++++- .../Services/ScrapingClientService.cs | 22 +++++++++++++++++++ .../Services/ScrapingNaroService.cs | 7 ++++-- KoeBook/App.xaml.cs | 9 +++++++- 5 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 Epub/KoeBook.Epub/Contracts/Services/IScrapingClientService.cs create mode 100644 Epub/KoeBook.Epub/Services/ScrapingClientService.cs diff --git a/Epub/KoeBook.Epub/Contracts/Services/IScrapingClientService.cs b/Epub/KoeBook.Epub/Contracts/Services/IScrapingClientService.cs new file mode 100644 index 0000000..44cff57 --- /dev/null +++ b/Epub/KoeBook.Epub/Contracts/Services/IScrapingClientService.cs @@ -0,0 +1,10 @@ +namespace KoeBook.Epub.Contracts.Services; + +public interface IScrapingClientService +{ + /// + /// スクレイピングでGETする用 + /// APIは不要 + /// + ValueTask GetAsync(string url, CancellationToken ct); +} diff --git a/Epub/KoeBook.Epub/Services/ScrapingAozoraService.cs b/Epub/KoeBook.Epub/Services/ScrapingAozoraService.cs index 3516dd9..7ea75c7 100644 --- a/Epub/KoeBook.Epub/Services/ScrapingAozoraService.cs +++ b/Epub/KoeBook.Epub/Services/ScrapingAozoraService.cs @@ -5,13 +5,16 @@ using KoeBook.Core; using KoeBook.Epub.Contracts.Services; using KoeBook.Epub.Models; +using Microsoft.Extensions.DependencyInjection; using static KoeBook.Epub.Utility.ScrapingHelper; namespace KoeBook.Epub.Services { - public partial class ScrapingAozoraService : IScrapingService + public partial class ScrapingAozoraService([FromKeyedServices(nameof(ScrapingAozoraService))] IScrapingClientService scrapingClientService) : IScrapingService { + private readonly IScrapingClientService _scrapingClientService = scrapingClientService; + public bool IsMatchSite(Uri uri) { return uri.Host == "www.aozora.gr.jp"; diff --git a/Epub/KoeBook.Epub/Services/ScrapingClientService.cs b/Epub/KoeBook.Epub/Services/ScrapingClientService.cs new file mode 100644 index 0000000..9a0ab20 --- /dev/null +++ b/Epub/KoeBook.Epub/Services/ScrapingClientService.cs @@ -0,0 +1,22 @@ +using KoeBook.Epub.Contracts.Services; + +namespace KoeBook.Epub.Services; + +public sealed class ScrapingClientService(IHttpClientFactory httpClientFactory, TimeProvider timeProvider) : IScrapingClientService, IDisposable +{ + private readonly IHttpClientFactory _httpClientFactory = httpClientFactory; + private readonly PeriodicTimer _periodicTimer = new(TimeSpan.FromSeconds(10), timeProvider); + + public void Dispose() + { + _periodicTimer.Dispose(); + } + + public async ValueTask GetAsync(string url, CancellationToken ct) + { + await _periodicTimer.WaitForNextTickAsync(ct).ConfigureAwait(false); + + var httpClient = _httpClientFactory.CreateClient(); + return await httpClient.GetAsync(url, ct).ConfigureAwait(false); + } +} diff --git a/Epub/KoeBook.Epub/Services/ScrapingNaroService.cs b/Epub/KoeBook.Epub/Services/ScrapingNaroService.cs index c04694f..c631a3d 100644 --- a/Epub/KoeBook.Epub/Services/ScrapingNaroService.cs +++ b/Epub/KoeBook.Epub/Services/ScrapingNaroService.cs @@ -6,13 +6,15 @@ using KoeBook.Core; using KoeBook.Epub.Contracts.Services; using KoeBook.Epub.Models; +using Microsoft.Extensions.DependencyInjection; using static KoeBook.Epub.Utility.ScrapingHelper; namespace KoeBook.Epub.Services { - public partial class ScrapingNaroService(IHttpClientFactory httpClientFactory) : IScrapingService + public partial class ScrapingNaroService(IHttpClientFactory httpClientFactory, [FromKeyedServices(nameof(ScrapingNaroService))] IScrapingClientService scrapingClientService) : IScrapingService { private readonly IHttpClientFactory _httpCliantFactory = httpClientFactory; + private readonly IScrapingClientService _scrapingClientService = scrapingClientService; public bool IsMatchSite(Uri uri) { @@ -22,6 +24,7 @@ public bool IsMatchSite(Uri uri) public async ValueTask ScrapingAsync(string url, string coverFilePath, string imageDirectory, Guid id, CancellationToken ct) { var config = Configuration.Default.WithDefaultLoader(); + using var context = BrowsingContext.New(config); var doc = await context.OpenAsync(url, ct).ConfigureAwait(false); @@ -133,7 +136,7 @@ public record BookInfo(int? allcount, int? noveltype, int? general_all_no); private record SectionWithChapterTitle(string? title, Section section); - private static async Task ReadPageAsync(string url, bool isRensai, string imageDirectory, CancellationToken ct) + private async Task ReadPageAsync(string url, bool isRensai, string imageDirectory, CancellationToken ct) { var config = Configuration.Default.WithDefaultLoader(); using var context = BrowsingContext.New(config); diff --git a/KoeBook/App.xaml.cs b/KoeBook/App.xaml.cs index 0ed636f..10b0523 100644 --- a/KoeBook/App.xaml.cs +++ b/KoeBook/App.xaml.cs @@ -60,6 +60,9 @@ public App() .UseContentRoot(AppContext.BaseDirectory) .ConfigureServices((context, services) => { + // System + services.AddSingleton(TimeProvider.System); + // Default Activation Handler services.AddTransient, DefaultActivationHandler>(); @@ -99,7 +102,11 @@ public App() services.AddSingleton(); services.AddSingleton(); - services.AddSingleton() + // Epub Services + services + .AddKeyedSingleton(nameof(ScrapingAozoraService)) + .AddKeyedSingleton(nameof(ScrapingNaroService)) + .AddSingleton() .AddSingleton() .AddSingleton(); services.AddSingleton(); From 0441c1dbed8029e24e9991872b5967fd34c2fa61 Mon Sep 17 00:00:00 2001 From: miyaji255 <84168445+miyaji255@users.noreply.github.com> Date: Fri, 15 Mar 2024 23:34:08 +0900 Subject: [PATCH 2/5] =?UTF-8?q?#19=20=E4=B8=8D=E8=A6=81=E3=81=AA=E5=A4=89?= =?UTF-8?q?=E6=9B=B4=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Epub/KoeBook.Epub/Services/ScrapingNaroService.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Epub/KoeBook.Epub/Services/ScrapingNaroService.cs b/Epub/KoeBook.Epub/Services/ScrapingNaroService.cs index c631a3d..70dd1e4 100644 --- a/Epub/KoeBook.Epub/Services/ScrapingNaroService.cs +++ b/Epub/KoeBook.Epub/Services/ScrapingNaroService.cs @@ -24,7 +24,6 @@ public bool IsMatchSite(Uri uri) public async ValueTask ScrapingAsync(string url, string coverFilePath, string imageDirectory, Guid id, CancellationToken ct) { var config = Configuration.Default.WithDefaultLoader(); - using var context = BrowsingContext.New(config); var doc = await context.OpenAsync(url, ct).ConfigureAwait(false); @@ -136,7 +135,7 @@ public record BookInfo(int? allcount, int? noveltype, int? general_all_no); private record SectionWithChapterTitle(string? title, Section section); - private async Task ReadPageAsync(string url, bool isRensai, string imageDirectory, CancellationToken ct) + private static async Task ReadPageAsync(string url, bool isRensai, string imageDirectory, CancellationToken ct) { var config = Configuration.Default.WithDefaultLoader(); using var context = BrowsingContext.New(config); From 119de8e163bc1b8c1b85b4ec1a3cd1748bfc5ea0 Mon Sep 17 00:00:00 2001 From: miyaji255 <84168445+miyaji255@users.noreply.github.com> Date: Sat, 16 Mar 2024 01:19:13 +0900 Subject: [PATCH 3/5] =?UTF-8?q?#19=20Queue-Consumer=E6=96=B9=E5=BC=8F?= =?UTF-8?q?=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Services/IScrapingClientService.cs | 14 ++- .../Services/ScrapingClientService.cs | 97 +++++++++++++++++-- 2 files changed, 100 insertions(+), 11 deletions(-) diff --git a/Epub/KoeBook.Epub/Contracts/Services/IScrapingClientService.cs b/Epub/KoeBook.Epub/Contracts/Services/IScrapingClientService.cs index 44cff57..c2a0b81 100644 --- a/Epub/KoeBook.Epub/Contracts/Services/IScrapingClientService.cs +++ b/Epub/KoeBook.Epub/Contracts/Services/IScrapingClientService.cs @@ -1,10 +1,18 @@ -namespace KoeBook.Epub.Contracts.Services; +using System.Net.Http.Headers; + +namespace KoeBook.Epub.Contracts.Services; public interface IScrapingClientService { /// /// スクレイピングでGETする用 - /// APIは不要 + /// APIを叩く際は不要 + /// + Task GetAsStringAsync(string url, CancellationToken ct); + + /// + /// スクレイピングでGETする用 + /// APIを叩く際は不要 /// - ValueTask GetAsync(string url, CancellationToken ct); + Task GetAsStreamAsync(string url, Stream destination, CancellationToken ct); } diff --git a/Epub/KoeBook.Epub/Services/ScrapingClientService.cs b/Epub/KoeBook.Epub/Services/ScrapingClientService.cs index 9a0ab20..1367980 100644 --- a/Epub/KoeBook.Epub/Services/ScrapingClientService.cs +++ b/Epub/KoeBook.Epub/Services/ScrapingClientService.cs @@ -1,22 +1,103 @@ -using KoeBook.Epub.Contracts.Services; +using System.Net.Http.Headers; +using KoeBook.Epub.Contracts.Services; namespace KoeBook.Epub.Services; -public sealed class ScrapingClientService(IHttpClientFactory httpClientFactory, TimeProvider timeProvider) : IScrapingClientService, IDisposable +public sealed class ScrapingClientService : IScrapingClientService, IDisposable { - private readonly IHttpClientFactory _httpClientFactory = httpClientFactory; - private readonly PeriodicTimer _periodicTimer = new(TimeSpan.FromSeconds(10), timeProvider); + private readonly IHttpClientFactory _httpClientFactory; + private readonly PeriodicTimer _periodicTimer; + private readonly Queue> _actionQueue = []; + private bool _workerActivated; + + public ScrapingClientService(IHttpClientFactory httpClientFactory, TimeProvider timeProvider) + { + _httpClientFactory = httpClientFactory; + _periodicTimer = new(TimeSpan.FromSeconds(10), timeProvider); + + Worker(); + } public void Dispose() { _periodicTimer.Dispose(); } - public async ValueTask GetAsync(string url, CancellationToken ct) + private async void Worker() + { + lock (_actionQueue) + { + _workerActivated = true; + } + + while (await _periodicTimer.WaitForNextTickAsync().ConfigureAwait(false) && _actionQueue.Count > 0) + { + if (_actionQueue.TryDequeue(out var action)) + { + await action(_httpClientFactory.CreateClient()).ConfigureAwait(false); + } + } + + lock (_actionQueue) + { + _workerActivated = false; + } + } + + public Task GetAsStringAsync(string url, CancellationToken ct) { - await _periodicTimer.WaitForNextTickAsync(ct).ConfigureAwait(false); + var taskCompletion = new TaskCompletionSource(); + _actionQueue.Enqueue(async httpClient => + { + if (ct.IsCancellationRequested) + taskCompletion.SetCanceled(ct); + + try + { + var response = await httpClient.GetAsync(url, ct).ConfigureAwait(false); + taskCompletion.SetResult(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); + } + catch (Exception ex) + { + taskCompletion.SetException(ex); + } + }); + + lock (_actionQueue) + { + if (!_workerActivated) + Worker(); + } + + return taskCompletion.Task; + } + + public Task GetAsStreamAsync(string url, Stream destination, CancellationToken ct) + { + var taskCompletion = new TaskCompletionSource(); + _actionQueue.Enqueue(async httpClient => + { + if (ct.IsCancellationRequested) + taskCompletion.SetCanceled(ct); + + try + { + var response = await httpClient.GetAsync(url, ct).ConfigureAwait(false); + await response.Content.CopyToAsync(destination, ct).ConfigureAwait(false); + taskCompletion.SetResult(response.Content.Headers.ContentDisposition); + } + catch (Exception ex) + { + taskCompletion.SetException(ex); + } + }); + + lock (_actionQueue) + { + if (!_workerActivated) + Worker(); + } - var httpClient = _httpClientFactory.CreateClient(); - return await httpClient.GetAsync(url, ct).ConfigureAwait(false); + return taskCompletion.Task; } } From d63ea333f0c9952348b206bd717538ba82c58534 Mon Sep 17 00:00:00 2001 From: miyaji255 <84168445+miyaji255@users.noreply.github.com> Date: Sat, 16 Mar 2024 13:14:32 +0900 Subject: [PATCH 4/5] =?UTF-8?q?#19=20lock=E6=96=87=E3=81=AE=E7=AE=87?= =?UTF-8?q?=E6=89=80=E3=82=92=E8=AA=BF=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Services/ScrapingClientService.cs | 137 ++++++++++-------- 1 file changed, 76 insertions(+), 61 deletions(-) diff --git a/Epub/KoeBook.Epub/Services/ScrapingClientService.cs b/Epub/KoeBook.Epub/Services/ScrapingClientService.cs index 1367980..4a6ac00 100644 --- a/Epub/KoeBook.Epub/Services/ScrapingClientService.cs +++ b/Epub/KoeBook.Epub/Services/ScrapingClientService.cs @@ -7,7 +7,7 @@ public sealed class ScrapingClientService : IScrapingClientService, IDisposable { private readonly IHttpClientFactory _httpClientFactory; private readonly PeriodicTimer _periodicTimer; - private readonly Queue> _actionQueue = []; + private readonly Queue> _actionQueue = []; private bool _workerActivated; public ScrapingClientService(IHttpClientFactory httpClientFactory, TimeProvider timeProvider) @@ -18,86 +18,101 @@ public ScrapingClientService(IHttpClientFactory httpClientFactory, TimeProvider Worker(); } - public void Dispose() + public Task GetAsStringAsync(string url, CancellationToken ct) { - _periodicTimer.Dispose(); - } + var taskCompletion = new TaskCompletionSource(); - private async void Worker() - { lock (_actionQueue) - { - _workerActivated = true; - } - - while (await _periodicTimer.WaitForNextTickAsync().ConfigureAwait(false) && _actionQueue.Count > 0) - { - if (_actionQueue.TryDequeue(out var action)) + _actionQueue.Enqueue(async httpClient => { - await action(_httpClientFactory.CreateClient()).ConfigureAwait(false); - } - } + if (ct.IsCancellationRequested) + taskCompletion.SetCanceled(ct); + + try + { + var response = await httpClient.GetAsync(url, ct).ConfigureAwait(false); + taskCompletion.SetResult(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); + } + catch (Exception ex) + { + taskCompletion.SetException(ex); + } + }); + + EnsureWorkerActivated(); - lock (_actionQueue) - { - _workerActivated = false; - } + return taskCompletion.Task; } - public Task GetAsStringAsync(string url, CancellationToken ct) + public Task GetAsStreamAsync(string url, Stream destination, CancellationToken ct) { - var taskCompletion = new TaskCompletionSource(); - _actionQueue.Enqueue(async httpClient => - { - if (ct.IsCancellationRequested) - taskCompletion.SetCanceled(ct); - - try - { - var response = await httpClient.GetAsync(url, ct).ConfigureAwait(false); - taskCompletion.SetResult(await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false)); - } - catch (Exception ex) - { - taskCompletion.SetException(ex); - } - }); + var taskCompletion = new TaskCompletionSource(); lock (_actionQueue) - { - if (!_workerActivated) - Worker(); - } + _actionQueue.Enqueue(async httpClient => + { + if (ct.IsCancellationRequested) + taskCompletion.SetCanceled(ct); + + try + { + var response = await httpClient.GetAsync(url, ct).ConfigureAwait(false); + await response.Content.CopyToAsync(destination, ct).ConfigureAwait(false); + taskCompletion.SetResult(response.Content.Headers.ContentDisposition); + } + catch (Exception ex) + { + taskCompletion.SetException(ex); + } + }); + + EnsureWorkerActivated(); return taskCompletion.Task; } - public Task GetAsStreamAsync(string url, Stream destination, CancellationToken ct) + /// + /// が起動していない場合は起動します + /// + private void EnsureWorkerActivated() { - var taskCompletion = new TaskCompletionSource(); - _actionQueue.Enqueue(async httpClient => - { - if (ct.IsCancellationRequested) - taskCompletion.SetCanceled(ct); + bool activateWorker; + lock (_actionQueue) activateWorker = !_workerActivated; - try - { - var response = await httpClient.GetAsync(url, ct).ConfigureAwait(false); - await response.Content.CopyToAsync(destination, ct).ConfigureAwait(false); - taskCompletion.SetResult(response.Content.Headers.ContentDisposition); - } - catch (Exception ex) - { - taskCompletion.SetException(ex); - } - }); + if (activateWorker) + Worker(); + } + /// + /// のConsumer + /// 別スレッドでループさせるためにvoid + /// + private async void Worker() + { lock (_actionQueue) + _workerActivated = true; + + try { - if (!_workerActivated) - Worker(); + while (await _periodicTimer.WaitForNextTickAsync().ConfigureAwait(false) && _actionQueue.Count > 0) + { + Func? action; + lock (_actionQueue) + if (!_actionQueue.TryDequeue(out action)) + continue; + + await action(_httpClientFactory.CreateClient()).ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + } } + finally + { + lock (_actionQueue) + _workerActivated = false; + } + } - return taskCompletion.Task; + public void Dispose() + { + _periodicTimer.Dispose(); } } From e93ac8e5962151c6e84bdccf50d4e1cab59a577f Mon Sep 17 00:00:00 2001 From: miyaji255 <84168445+miyaji255@users.noreply.github.com> Date: Sat, 16 Mar 2024 13:15:11 +0900 Subject: [PATCH 5/5] =?UTF-8?q?#19=20=E3=82=B3=E3=83=B3=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=83=A9=E3=82=AF=E3=82=BF=E3=81=A7=E3=81=AEWorker=E3=81=AE?= =?UTF-8?q?=E8=B5=B7=E5=8B=95=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Epub/KoeBook.Epub/Services/ScrapingClientService.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Epub/KoeBook.Epub/Services/ScrapingClientService.cs b/Epub/KoeBook.Epub/Services/ScrapingClientService.cs index 4a6ac00..164898e 100644 --- a/Epub/KoeBook.Epub/Services/ScrapingClientService.cs +++ b/Epub/KoeBook.Epub/Services/ScrapingClientService.cs @@ -14,8 +14,6 @@ public ScrapingClientService(IHttpClientFactory httpClientFactory, TimeProvider { _httpClientFactory = httpClientFactory; _periodicTimer = new(TimeSpan.FromSeconds(10), timeProvider); - - Worker(); } public Task GetAsStringAsync(string url, CancellationToken ct)