diff --git a/XLWebServices/Controllers/PlogonController.cs b/XLWebServices/Controllers/PlogonController.cs index 7c1bd4d..d21808f 100644 --- a/XLWebServices/Controllers/PlogonController.cs +++ b/XLWebServices/Controllers/PlogonController.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using Discord; using Microsoft.AspNetCore.Mvc; +using Sentry; using XLWebServices.Data; using XLWebServices.Data.Models; using XLWebServices.Services; @@ -247,11 +248,14 @@ private async ValueTask BuildCommitWorkItemAsync(CancellationToken token, IServi var dbPlugin = db.Plugins.FirstOrDefault(x => x.InternalName == pluginInfo.InternalName); if (dbPlugin != null) { + if (!Version.TryParse(pluginInfo.Version, out var versionNum)) + throw new Exception($"Could not parse plugin version ({pluginInfo.Version}"); + var version = new PluginVersion { Plugin = dbPlugin, Dip17Track = pluginInfo.Dip17Track, - Version = pluginInfo.Version, + Version = versionNum, PrNumber = pluginInfo.PrNumber, Changelog = pluginInfo.Changelog, PublishedAt = DateTime.Now, @@ -272,7 +276,7 @@ private async ValueTask BuildCommitWorkItemAsync(CancellationToken token, IServi // Send discord notification if (pluginInfo.Changelog == null || !shallExplicitlyHideChangelog) { - var isOtherRepo = pluginInfo.Dip17Track != Dip17SystemDefine.MainTrack; + var isOtherRepo = pluginInfo.Dip17Track != Dip17SystemDefine.StableTrack; var embed = new EmbedBuilder() .WithTitle($"{manifest.Name} (v{pluginInfo.Version})") @@ -286,12 +290,14 @@ private async ValueTask BuildCommitWorkItemAsync(CancellationToken token, IServi } await db.SaveChangesAsync(token); + await data.PostProcessD17Masters(); _logger.LogInformation("Committed {NumPlogons} in {Secs}s", staged.Count, stopwatch.Elapsed.TotalSeconds); } catch (Exception ex) { _logger.LogError(ex, "Could not process plogon commit job"); + SentrySdk.CaptureException(ex); } } diff --git a/XLWebServices/Controllers/PluginController.cs b/XLWebServices/Controllers/PluginController.cs index ee7039f..189917e 100644 --- a/XLWebServices/Controllers/PluginController.cs +++ b/XLWebServices/Controllers/PluginController.cs @@ -41,7 +41,10 @@ public PluginController(ILogger logger, FallibleService Download(string internalName, [FromQuery(Name = "isTesting")] bool isTesting = false, [FromQuery(Name = "isDip17")] bool isDip17 = false) + public async Task Download(string internalName, + [FromQuery(Name = "isTesting")] bool isTesting = false, + [FromQuery(Name = "isUpdate")] bool isUpdate = false, + [FromQuery(Name = "isDip17")] bool isDip17 = false) { if (this.pluginData.HasFailed&& this.pluginData.Get()?.PluginMaster == null) return StatusCode(500, "Precondition failed"); @@ -52,34 +55,27 @@ public async Task Download(string internalName, [FromQuery(Name = if (manifest == null) return BadRequest("Invalid plugin"); - DownloadsOverTime.WithLabels(internalName.ToLower(), isTesting.ToString()).Inc(); - - if (!this.redis.HasFailed) + if (!isUpdate) { - await this.redis.Get()!.IncrementCount(internalName); - await this.redis.Get()!.IncrementCount(RedisCumulativeKey); + DownloadsOverTime.WithLabels(internalName.ToLower(), isTesting.ToString()).Inc(); + + if (!this.redis.HasFailed) + { + await this.redis.Get()!.IncrementCount(internalName); + await this.redis.Get()!.IncrementCount(RedisCumulativeKey); + } } - if (isDip17) - { - const string githubPath = "https://raw.githubusercontent.com/goatcorp/PluginDistD17/{0}/{1}/{2}/latest.zip"; - var folder = isTesting ? "testing-live" : "stable"; - var version = isTesting && manifest.TestingAssemblyVersion != null ? manifest.TestingAssemblyVersion : manifest.AssemblyVersion; - var cachedFile = await this.cache.CacheFile(internalName, $"{version}-{folder}-{this.pluginData.Get()!.RepoShaDip17}", - string.Format(githubPath, this.pluginData.Get()!.RepoShaDip17, folder, internalName), FileCacheService.CachedFile.FileCategory.Plugin); + if (!isDip17) + return BadRequest("Legacy plugin downloads are no longer available"); - return new RedirectResult($"{this.configuration["HostedUrl"]}/File/Get/{cachedFile.Id}"); - } - else - { - const string githubPath = "https://raw.githubusercontent.com/goatcorp/DalamudPlugins/{0}/{1}/{2}/latest.zip"; - var folder = isTesting ? "testing" : "plugins"; - var version = isTesting && manifest.TestingAssemblyVersion != null ? manifest.TestingAssemblyVersion : manifest.AssemblyVersion; - var cachedFile = await this.cache.CacheFile(internalName, $"{version}-{folder}-{this.pluginData.Get()!.RepoSha}", - string.Format(githubPath, this.pluginData.Get()!.RepoSha, folder, internalName), FileCacheService.CachedFile.FileCategory.Plugin); + const string githubPath = "https://raw.githubusercontent.com/goatcorp/PluginDistD17/{0}/{1}/{2}/latest.zip"; + var folder = isTesting ? "testing-live" : "stable"; + var version = isTesting && manifest.TestingAssemblyVersion != null ? manifest.TestingAssemblyVersion : manifest.AssemblyVersion; + var cachedFile = await this.cache.CacheFile(internalName, $"{version}-{folder}-{this.pluginData.Get()!.RepoShaDip17}", + string.Format(githubPath, this.pluginData.Get()!.RepoShaDip17, folder, internalName), FileCacheService.CachedFile.FileCategory.Plugin); - return new RedirectResult($"{this.configuration["HostedUrl"]}/File/Get/{cachedFile.Id}"); - } + return new RedirectResult($"{this.configuration["HostedUrl"]}/File/Get/{cachedFile.Id}"); } [HttpGet] @@ -115,7 +111,9 @@ public IActionResult PluginMaster([FromQuery] bool proxy = true, [FromQuery] int pluginMaster = this.pluginData.Get()!.PluginMaster; } - pluginMaster ??= Array.Empty(); + if (pluginMaster == null) + return StatusCode(500, "No plugin data"); + if (minApiLevel > 0) { pluginMaster = pluginMaster.Where(manifest => manifest.DalamudApiLevel >= minApiLevel).ToArray(); @@ -160,7 +158,7 @@ public IActionResult History(string internalName, [FromQuery(Name = "track")] st if (string.IsNullOrEmpty(dip17Track)) { - dip17Track = Dip17SystemDefine.MainTrack; + dip17Track = Dip17SystemDefine.StableTrack; } var dbPlugin = _dbContext.Plugins.Include(x => x.VersionHistory).FirstOrDefault(x => x.InternalName == internalName); @@ -202,7 +200,6 @@ public async Task Meta() NumPlugins = this.pluginData.Get()!.PluginMaster!.Count, LastUpdate = this.pluginData.Get()!.LastUpdate, CumulativeDownloads = await this.redis.Get()!.GetCount(RedisCumulativeKey), - Sha = this.pluginData.Get()!.RepoSha, Dip17Sha = this.pluginData.Get()!.RepoShaDip17, }); } @@ -212,7 +209,6 @@ public class PluginMeta public int NumPlugins { get; init; } public long CumulativeDownloads { get; init; } public DateTime LastUpdate { get; init; } - public string Sha { get; init; } public string Dip17Sha { get; init; } } } \ No newline at end of file diff --git a/XLWebServices/Data/Migrations/20240415181820_VersionString-To-SystemVersion.Designer.cs b/XLWebServices/Data/Migrations/20240415181820_VersionString-To-SystemVersion.Designer.cs new file mode 100644 index 0000000..26f224f --- /dev/null +++ b/XLWebServices/Data/Migrations/20240415181820_VersionString-To-SystemVersion.Designer.cs @@ -0,0 +1,107 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using XLWebServices.Data; + +#nullable disable + +namespace XLWebServices.Data.Migrations +{ + [DbContext(typeof(WsDbContext))] + [Migration("20240415181820_VersionString-To-SystemVersion")] + partial class VersionStringToSystemVersion + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "7.0.2"); + + modelBuilder.Entity("XLWebServices.Data.Models.Plugin", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("InternalName") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("Plugins"); + }); + + modelBuilder.Entity("XLWebServices.Data.Models.PluginVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Changelog") + .HasColumnType("TEXT"); + + b.Property("DiffLinesAdded") + .HasColumnType("INTEGER"); + + b.Property("DiffLinesRemoved") + .HasColumnType("INTEGER"); + + b.Property("Dip17Track") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("IsHidden") + .HasColumnType("INTEGER"); + + b.Property("IsInitialRelease") + .HasColumnType("INTEGER"); + + b.Property("PluginId") + .HasColumnType("TEXT"); + + b.Property("PrNumber") + .HasColumnType("INTEGER"); + + b.Property("PublishedAt") + .HasColumnType("TEXT"); + + b.Property("PublishedBy") + .HasColumnType("TEXT"); + + b.Property("TimeToMerge") + .HasColumnType("TEXT"); + + b.Property("Version") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("PluginId"); + + b.ToTable("PluginVersions"); + }); + + modelBuilder.Entity("XLWebServices.Data.Models.PluginVersion", b => + { + b.HasOne("XLWebServices.Data.Models.Plugin", "Plugin") + .WithMany("VersionHistory") + .HasForeignKey("PluginId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Plugin"); + }); + + modelBuilder.Entity("XLWebServices.Data.Models.Plugin", b => + { + b.Navigation("VersionHistory"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/XLWebServices/Data/Migrations/20240415181820_VersionString-To-SystemVersion.cs b/XLWebServices/Data/Migrations/20240415181820_VersionString-To-SystemVersion.cs new file mode 100644 index 0000000..1a2717c --- /dev/null +++ b/XLWebServices/Data/Migrations/20240415181820_VersionString-To-SystemVersion.cs @@ -0,0 +1,22 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace XLWebServices.Data.Migrations +{ + /// + public partial class VersionStringToSystemVersion : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + + } + } +} diff --git a/XLWebServices/Data/Models/PluginVersion.cs b/XLWebServices/Data/Models/PluginVersion.cs index ab9ec38..66c04e4 100644 --- a/XLWebServices/Data/Models/PluginVersion.cs +++ b/XLWebServices/Data/Models/PluginVersion.cs @@ -8,7 +8,7 @@ public class PluginVersion public Guid Id { get; set; } public Plugin Plugin { get; set; } - public string Version { get; set; } + public Version Version { get; set; } public string Dip17Track { get; set; } public string? Changelog { get; set; } public DateTime PublishedAt { get; set; } diff --git a/XLWebServices/Data/WsDbContext.cs b/XLWebServices/Data/WsDbContext.cs index 10bb1ea..1901132 100644 --- a/XLWebServices/Data/WsDbContext.cs +++ b/XLWebServices/Data/WsDbContext.cs @@ -25,5 +25,11 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .WithOne(e => e.Plugin) .IsRequired() .OnDelete(DeleteBehavior.Cascade); + + modelBuilder.Entity() + .Property(x => x.Version) + .HasConversion( + v => v.ToString(), + v => Version.Parse(v)); } } \ No newline at end of file diff --git a/XLWebServices/Dip17SystemDefine.cs b/XLWebServices/Dip17SystemDefine.cs index b8dff25..d600913 100644 --- a/XLWebServices/Dip17SystemDefine.cs +++ b/XLWebServices/Dip17SystemDefine.cs @@ -2,6 +2,6 @@ namespace XLWebServices; public static class Dip17SystemDefine { - public const string MainTrack = "stable"; + public const string StableTrack = "stable"; public const string ChangelogMarkerHide = "nofranz"; } \ No newline at end of file diff --git a/XLWebServices/Services/PluginData/PluginDataService.cs b/XLWebServices/Services/PluginData/PluginDataService.cs index 4dde623..9365955 100644 --- a/XLWebServices/Services/PluginData/PluginDataService.cs +++ b/XLWebServices/Services/PluginData/PluginDataService.cs @@ -1,6 +1,5 @@ +using System.Diagnostics; using System.Text.Json; -using System.Text.RegularExpressions; -using System.Threading.Channels; using Octokit; using Tomlyn; using XLWebServices.Data; @@ -19,12 +18,11 @@ public class PluginDataService private readonly HttpClient _client; + // All plugins public IReadOnlyList? PluginMaster { get; private set; } - //public IReadOnlyList? PluginMasterNoProxy { get; private set; } - - public IReadOnlyDictionary> PluginMastersDip17 { get; private set; } - public string RepoSha { get; private set; } + // Per track + public IReadOnlyDictionary> PluginMastersDip17 { get; private set; } public string RepoShaDip17 { get; private set; } @@ -54,119 +52,26 @@ public async Task ClearCache() try { - var repoOwner = _configuration["GitHub:PluginRepository:Owner"]; - var repoName = _configuration["GitHub:PluginRepository:Name"]; - var apiLevel = _configuration["ApiLevel"]; - - var commit = await _github.Client.Repository.Commit.Get(repoOwner, repoName, _configuration["PluginRepoBranch"]); - var sha = commit.Sha; - - var downloadTemplate = _configuration["TemplateDownload"]; - var updateTemplate = _configuration["TemplateUpdate"]; - + /* var bannedPlugins = await _client.GetFromJsonAsync(_configuration["BannedPlugin"]); if (bannedPlugins == null) throw new Exception("Failed to load banned plugins"); - - var pluginsDir = - await _github.Client.Repository.Content.GetAllContentsByRef(repoOwner, repoName, "plugins/", sha); - var testingDir = - await _github.Client.Repository.Content.GetAllContentsByRef(repoOwner, repoName, "testing/", sha); + */ var pluginMaster = new List(); var d17 = await ClearCacheD17(pluginMaster); - foreach (var repositoryContent in pluginsDir) - { - if (pluginMaster.Any(x => x.InternalName == repositoryContent.Name)) - continue; - - var manifest = await this.GetManifest(repoOwner, repoName, "plugins", repositoryContent.Name, sha); - if (manifest == null) - throw new Exception($"Could not fetch manifest for release plugin: {repositoryContent.Name}"); - - var banned = bannedPlugins.LastOrDefault(x => x.Name == manifest.InternalName); - var isHide = banned != null && manifest.AssemblyVersion <= banned.AssemblyVersion || - manifest.DalamudApiLevel.ToString() != apiLevel; - manifest.IsHide = isHide; - - var testingVersion = testingDir.FirstOrDefault(x => x.Name == repositoryContent.Name); - if (testingVersion != null) - { - var testingManifest = await this.GetManifest(repoOwner, repoName, "testing", repositoryContent.Name, sha); - if (testingManifest == null) - throw new Exception( - $"Plugin had testing version, but could not fetch manifest: {repositoryContent.Name}"); - - manifest.TestingAssemblyVersion = testingManifest.AssemblyVersion; - } - - manifest.IsTestingExclusive = false; - - var (lastUpdate, changelog) = await this.GetPluginInfo(manifest, repositoryContent, repoOwner, repoName); - manifest.LastUpdate = lastUpdate; - manifest.Changelog = changelog; - - if (!_redis.HasFailed) - { - var dlCount = await _redis.RunFallibleAsync(s => s.GetCount(manifest.InternalName)); - if (dlCount.HasValue) - { - manifest.DownloadCount = dlCount.Value; - } - } - - manifest.DownloadLinkInstall = string.Format(downloadTemplate, manifest.InternalName, false, apiLevel, false); - manifest.DownloadLinkTesting = string.Format(downloadTemplate, manifest.InternalName, true, apiLevel, false); - manifest.DownloadLinkUpdate = string.Format(updateTemplate, "plugins", manifest.InternalName, apiLevel); - - pluginMaster.Add(manifest); - } - - foreach (var repositoryContent in testingDir) - { - if (pluginMaster.Any(x => x.InternalName == repositoryContent.Name)) - continue; - - var manifest = await this.GetManifest(repoOwner, repoName, "testing", repositoryContent.Name, sha); - if (manifest == null) - throw new Exception($"Could not fetch manifest for testing plugin: {repositoryContent.Name}"); - - manifest.TestingAssemblyVersion = manifest.AssemblyVersion; - manifest.IsTestingExclusive = true; - - var (lastUpdate, changelog) = await this.GetPluginInfo(manifest, repositoryContent, repoOwner, repoName); - manifest.LastUpdate = lastUpdate; - manifest.Changelog = changelog; - - if (!_redis.HasFailed) - { - var dlCount = await _redis.RunFallibleAsync(s => s.GetCount(manifest.InternalName)); - if (dlCount.HasValue) - { - manifest.DownloadCount = dlCount.Value; - } - } - - manifest.DownloadLinkInstall = string.Format(downloadTemplate, manifest.InternalName, false, apiLevel, false); - manifest.DownloadLinkTesting = string.Format(downloadTemplate, manifest.InternalName, true, apiLevel, false); - manifest.DownloadLinkUpdate = string.Format(updateTemplate, manifest.InternalName, "plugins", apiLevel); - - pluginMaster.Add(manifest); - } - PluginMaster = pluginMaster; PluginMastersDip17 = d17.Manifests; - RepoSha = sha; RepoShaDip17 = d17.Sha; LastUpdate = DateTime.Now; - EnsureDatabaseConsistent(); + await EnsureDatabaseConsistent(); _logger.LogInformation("Plugin list updated, {Count} plugins found", this.PluginMaster.Count); - await _discord.AdminSendSuccess($"Plugin list updated, {this.PluginMaster.Count} plugins loaded\nSHA: {sha}\nSHA(D17): {RepoShaDip17}", + await _discord.AdminSendSuccess($"Plugin list updated, {this.PluginMaster.Count} plugins loaded\nSHA(D17): {RepoShaDip17}", "PluginMaster updated"); } catch (Exception e) @@ -177,7 +82,43 @@ await _discord.AdminSendSuccess($"Plugin list updated, {this.PluginMaster.Count} } } - public async Task<(Dictionary> Manifests, string Sha)> ClearCacheD17(List pluginMaster) + public async Task PostProcessD17Masters() + { + Debug.Assert(PluginMaster != null); + Debug.Assert(PluginMastersDip17 != null); + + // Patch changelogs into the "main" PluginMaster that has normal and testing versions. + // This is the only manifest that will have testing changelogs. + // I don't think this approach scales but I don't care. + foreach (var plugin in PluginMaster) + { + var versionStable = _dbContext.PluginVersions + .OrderByDescending(x => x.PublishedAt) + .FirstOrDefault(x => x.Version == plugin.AssemblyVersion && + x.Dip17Track == Dip17SystemDefine.StableTrack); + var versionTesting = _dbContext.PluginVersions + .OrderByDescending(x => x.PublishedAt) + .FirstOrDefault(x => x.Version == plugin.AssemblyVersion && + x.Dip17Track != Dip17SystemDefine.StableTrack); + + plugin.Changelog = versionStable?.Changelog; + plugin.TestingChangelog = versionTesting?.Changelog; + } + + // Patch changelogs into individual tracks. These only have the main changelog appropriate for the track. + foreach (var track in PluginMastersDip17) + foreach (var plugin in track.Value) + { + var version = _dbContext.PluginVersions + .OrderByDescending(x => x.PublishedAt) + .FirstOrDefault(x => x.Version == plugin.AssemblyVersion && + x.Dip17Track == track.Key); + + plugin.Changelog = version?.Changelog; + } + } + + private async Task<(Dictionary> Manifests, string Sha)> ClearCacheD17(List pluginMaster) { var repoOwner = _configuration["GitHub:PluginDistD17:Owner"]; var repoName = _configuration["GitHub:PluginDistD17:Name"]; @@ -219,6 +160,9 @@ async Task ProcessPluginsInChannel(Dip17State.Channel channel, string channelNam manifest.DalamudApiLevel != apiLevel; manifest.IsHide = isHide; + // This is NECESSARY here for changelog fallback logic. + // PlogonController.BuildCommitWorkItemAsync() will take this value and use it as the + // committed changelog for the plugin if the PR description does not apply. manifest.Changelog = pluginState.Changelogs?.FirstOrDefault(x => x.Key == manifest.AssemblyVersion.ToString()).Value?.Changelog; manifest.LastUpdate = ((DateTimeOffset)pluginState.TimeBuilt).ToUnixTimeSeconds(); diff --git a/XLWebServices/Services/PluginData/PluginManifest.cs b/XLWebServices/Services/PluginData/PluginManifest.cs index 9faa629..ea82232 100644 --- a/XLWebServices/Services/PluginData/PluginManifest.cs +++ b/XLWebServices/Services/PluginData/PluginManifest.cs @@ -66,6 +66,12 @@ public PluginManifest(PluginManifest toCopy) /// [JsonPropertyName("Changelog")] public string? Changelog { get; set; } + + /// + /// Gets a changelog for the current testing version, if applicable. + /// + [JsonPropertyName("TestingChangelog")] + public string? TestingChangelog { get; set; } /// /// Gets a list of tags defined on the plugin.