Skip to content

Commit

Permalink
Tags is now moved off tagService, which means slower scans but much l…
Browse files Browse the repository at this point in the history 
…ess memory usage. All that's left is css and fixing broken unit tests.
  • Loading branch information
@majora2007
majora2007 committed Oct 13, 2024
1 parent 3bbe4f2 commit 1de838d
Show file tree
Hide file tree
Showing 7 changed files with 77 additions and 348 deletions.
2 changes: 1 addition & 1 deletion API.Tests/Services/ScannerServiceTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ private async Task<Library> GenerateScannerData(string testcase)
Substitute.For<IMetadataService>(),
Substitute.For<IWordCountAnalyzerService>(),
Substitute.For<IReadingListService>(),
Substitute.For<IExternalMetadataService>(), new TagManagerService(_unitOfWork, Substitute.For<ILogger<TagManagerService>>()));
Substitute.For<IExternalMetadataService>());

var scanner = new ScannerService(_unitOfWork, Substitute.For<ILogger<ScannerService>>(),
Substitute.For<IMetadataService>(),
Expand Down
2 changes: 1 addition & 1 deletion API/Data/Repositories/CollectionTagRepository.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ public interface ICollectionTagRepository
Task<IEnumerable<AppUserCollection>> GetCollectionsByIds(IEnumerable<int> tags, CollectionIncludes includes = CollectionIncludes.None);
Task<IList<AppUserCollection>> GetAllCollectionsForSyncing(DateTime expirationTime);
}

public class CollectionTagRepository : ICollectionTagRepository
{
private readonly DataContext _context;
Expand Down Expand Up @@ -219,7 +220,6 @@ public async Task<IList<AppUserCollection>> GetAllCollectionsForSyncing(DateTime
.ToListAsync();
}


public async Task<AppUserCollection?> GetCollectionAsync(int tagId, CollectionIncludes includes = CollectionIncludes.None)
{
return await _context.AppUserCollection
Expand Down
1 change: 0 additions & 1 deletion API/Extensions/ApplicationServiceExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ public static void AddApplicationServices(this IServiceCollection services, ICon
services.AddScoped<ILibraryWatcher, LibraryWatcher>();
services.AddScoped<ITachiyomiService, TachiyomiService>();
services.AddScoped<ICollectionTagService, CollectionTagService>();
services.AddScoped<ITagManagerService, TagManagerService>();

services.AddScoped<IFileSystem, FileSystem>();
services.AddScoped<IDirectoryService, DirectoryService>();
Expand Down
11 changes: 8 additions & 3 deletions API/Helpers/Builders/PersonBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@ public PersonBuilder(string name)
{
Name = name.Trim(),
NormalizedName = name.ToNormalized(),
//Role = role,
// ChapterMetadatas = new List<Chapter>(),
// SeriesMetadatas = new List<SeriesMetadata>()
SeriesMetadataPeople = new List<SeriesMetadataPeople>(),
ChapterPeople = new List<ChapterPeople>()
};
}

Expand All @@ -34,4 +33,10 @@ public PersonBuilder WithId(int id)
return this;
}

public PersonBuilder WithSeriesMetadata(SeriesMetadataPeople seriesMetadataPeople)
{
_person.SeriesMetadataPeople.Add(seriesMetadataPeople);
return this;
}

}
117 changes: 66 additions & 51 deletions API/Services/Tasks/Scanner/ProcessSeries.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
using API.Services.Tasks.Metadata;
using API.Services.Tasks.Scanner.Parser;
using API.SignalR;
using Hangfire;
using Kavita.Common;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
Expand All @@ -27,13 +26,6 @@ namespace API.Services.Tasks.Scanner;

public interface IProcessSeries
{
/// <summary>
/// Do not allow this Prime to be invoked by multiple threads. It will break the DB.
/// </summary>
/// <returns></returns>
Task Prime();

void Reset();
Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, int totalToProcess, bool forceUpdate = false);
}

Expand All @@ -53,7 +45,6 @@ public class ProcessSeries : IProcessSeries
private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
private readonly IReadingListService _readingListService;
private readonly IExternalMetadataService _externalMetadataService;
private readonly ITagManagerService _tagManagerService;


public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
Expand All @@ -73,31 +64,8 @@ public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEve
_wordCountAnalyzerService = wordCountAnalyzerService;
_readingListService = readingListService;
_externalMetadataService = externalMetadataService;
_tagManagerService = new TagManagerService(_unitOfWork, _logger);
}

/// <summary>
/// Invoke this before processing any series, just once to prime all the needed data during a scan
/// </summary>
public async Task Prime()
{
try
{
await _tagManagerService.Prime();
}
catch (Exception ex)
{
_logger.LogCritical(ex, "Unable to prime tag manager. Scan cannot proceed. Report to Kavita dev");
}
}

/// <summary>
/// Frees up memory
/// </summary>
public void Reset()
{
_tagManagerService.Reset();
}

public async Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, int totalToProcess, bool forceUpdate = false)
{
Expand Down Expand Up @@ -343,29 +311,35 @@ private async Task UpdateSeriesMetadata(Series series, Library library)
series.Metadata.Language = firstChapter.Language;
}

// if (!string.IsNullOrEmpty(firstChapter?.SeriesGroup) && library.ManageCollections)
// {
// // Get the default admin to associate these tags to
// var defaultAdmin = await _unitOfWork.UserRepository.GetDefaultAdminUser(AppUserIncludes.Collections);
// if (defaultAdmin == null) return;
//
// _logger.LogDebug("Collection tag(s) found for {SeriesName}, updating collections", series.Name);
// foreach (var collection in firstChapter.SeriesGroup.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
// {
// // TODO: Remove this _tagManagerService
// var (tag, _) = await _tagManagerService.GetCollectionTag(collection, defaultAdmin);
// if (tag == null) continue;
//
// // Check if the Series is already on the tag
// if (tag.Items.Any(s => s.MatchesSeriesByName(series.NormalizedName, series.NormalizedLocalizedName)))
// {
// continue;
// }
//
// tag.Items.Add(series);
// await _unitOfWork.CollectionTagRepository.UpdateCollectionAgeRating(tag);
// }
// }
if (!string.IsNullOrEmpty(firstChapter?.SeriesGroup) && library.ManageCollections)
{
// Get the default admin to associate these tags to
var defaultAdmin = await _unitOfWork.UserRepository.GetDefaultAdminUser(AppUserIncludes.Collections);
if (defaultAdmin == null) return;
await UpdateCollectionTags(series, firstChapter);
}

_logger.LogDebug("Collection tag(s) found for {SeriesName}, updating collections", series.Name);
foreach (var collection in firstChapter.SeriesGroup.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
{
// TODO: Remove this _tagManagerService
var (tag, _) = await _tagManagerService.GetCollectionTag(collection, defaultAdmin);
if (tag == null) continue;

// Check if the Series is already on the tag
if (tag.Items.Any(s => s.MatchesSeriesByName(series.NormalizedName, series.NormalizedLocalizedName)))
{
continue;
}

tag.Items.Add(series);
await _unitOfWork.CollectionTagRepository.UpdateCollectionAgeRating(tag);
}
}

#region PeopleAndTagsAndGenres
if (!series.Metadata.WriterLocked)
Expand Down Expand Up @@ -462,6 +436,47 @@ private async Task UpdateSeriesMetadata(Series series, Library library)
#endregion
}

private async Task UpdateCollectionTags(Series series, Chapter firstChapter)
{
// Get the default admin to associate these tags to
var defaultAdmin = await _unitOfWork.UserRepository.GetDefaultAdminUser(AppUserIncludes.Collections);
if (defaultAdmin == null) return;

_logger.LogDebug("Collection tag(s) found for {SeriesName}, updating collections", series.Name);

foreach (var collection in firstChapter.SeriesGroup.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
{
// Try to find an existing collection tag by its normalized name
var normalizedCollectionName = collection.ToNormalized();
var collectionTag = defaultAdmin.Collections.FirstOrDefault(c => c.NormalizedTitle == normalizedCollectionName);

// If the collection tag does not exist, create a new one
if (collectionTag == null)
{
_logger.LogDebug("Creating new collection tag for {Tag}", collection);

collectionTag = new AppUserCollectionBuilder(collection).Build();
defaultAdmin.Collections.Add(collectionTag);

_unitOfWork.UserRepository.Update(defaultAdmin);

await _unitOfWork.CommitAsync();
}

// Check if the Series is already associated with this collection
if (collectionTag.Items.Any(s => s.MatchesSeriesByName(series.NormalizedName, series.NormalizedLocalizedName)))
{
continue;
}

// Add the series to the collection tag
collectionTag.Items.Add(series);

// Update the collection age rating
await _unitOfWork.CollectionTagRepository.UpdateCollectionAgeRating(collectionTag);
}
}


private static void UpdateSeriesMetadataTags(ICollection<Tag> metadataTags, IEnumerable<Tag> chapterTags)
{
Expand Down
Loading

0 comments on commit 1de838d

Please sign in to comment.