From 9849e898b31d05a3c75f7ee54d03475082c07f13 Mon Sep 17 00:00:00 2001 From: sonicfind Date: Thu, 26 Sep 2024 16:11:16 -0500 Subject: [PATCH 01/12] Scanning Architecture Refactor + Add DTAEntry type that siphons the old ParseDTA code - providing an avenue to pre-loading information for a DTA node before loading a new song entry. + Streamline the hierarchy of construction to the most basic form. All the processing of data that produces the member variables for new entries will be performed solely through static functions. Additionally, we will now mark all the values in any of the `SongEntry` types as `readonly`. This was already the case for the "ini` subtypes, but the RBCON structures prior to this change required the live modification of an existing entry. The only exception is Song Length, as processing a missing length value requires loading the mixer - which requires setting all subtype information. + Provide `SongMetadata` & `RBMetadata` types with constructors & `Serialize()` functions holding the code that used to exist in the `SongEntry` & `RBCONEntry` constructors and `Serialize()` functions. + Streamline the `Serialize()` functions so that they all share the same signature - allowing for proper virtualization. For the ini subtypes, this meant moving the relative path and EntryType boolean writes to the `IniGroup` `SerializeEntries` function instead. + To add to the above: instead of passing around a BinaryWriter wrapper around the underlying MemoryStream, I've added StreamExtensions to implement directly writing other values to streams - allowing use to pass the streams themselves to all the serialize functions. + Alter the return types of the Midi & .Chart scanning functions to return the tick resolution of the underlying file. We require that value so that we can calculate the sustain cutoff or hopo thresholds where necessary. By setting the values during the scan, we dodge the necessity of re-evaluating the values at song load time - which also means that the songcache more accurately depicts the true state of the songs it contains. + Replace the ParseSettings variable in song entries with a trimmed down LoaderSettings variable. This new type only carries the two threshold types and the overdrive midinote. On song load, we cna gather all the other values used in ParseSettings through the other states present in the entry instance. + Add new `ScanResult` entries to catch songs that try to be funny and set their resolutions to zero. + Provide `YARGTextContainer` with a `Null` variable similar to FixedArray alongside an `IsActive` property to allow the simplification of functions that had to use a boolean variable as the return type with a container as an `out` variable. + Add DTAEntry type + Optimize the RBCON update and upgrade utilization: ++ Firstly, instead of holding all the updates and upgrades in separate lists during the scanning of new items, use the new CONModification type to combine the updates and upgrades that apply to the same DTA node name into a single instance. This , combined with the new DTAEntry type, allows for more-optimal deferment of the processing of DTA data without the downside of re-evaluating the data for song entries that share the same node name. However, upgrades loaded from cache will be held in a separate list. ++Secondly, disallow separate SongUpdate folders to hold references to the same node names. Only the update with the most recent write date will be used to modify files. This simplified the DTA parsing scheme so that no looping occurs over multiple update nodes at different locations. + Re-convert CONFile to a static class that contains a function that generates a `List` from a CON. The "TryGetListing" method becomes an extension on that type. --- TestConsole/CacheLoader.cs | 2 +- YARG.Core/Extensions/StreamExtensions.cs | 58 +- YARG.Core/IO/AbridgedFileInfo.cs | 6 +- YARG.Core/IO/ConHandler/CONFile.cs | 99 +- YARG.Core/IO/ConHandler/CONFileListing.cs | 8 +- YARG.Core/IO/DTA/DTAEntry.cs | 310 +++++ YARG.Core/IO/{ => DTA}/YARGDTAReader.cs | 12 +- YARG.Core/IO/FixedArray.cs | 7 +- YARG.Core/IO/Midi/YARGMidiFile.cs | 16 +- YARG.Core/IO/SngHandler/SngFile.cs | 4 +- YARG.Core/IO/TextReader/YARGTextContainer.cs | 24 +- YARG.Core/Song/Cache/CONModification.cs | 16 + .../Song/Cache/CacheGroups/CacheGroup.cs | 33 +- YARG.Core/Song/Cache/CacheGroups/ConGroup.cs | 97 +- .../Cache/CacheGroups/IModificationGroup.cs | 26 +- YARG.Core/Song/Cache/CacheGroups/IniGroup.cs | 54 +- .../Song/Cache/CacheGroups/LockedList.cs | 18 - .../Song/Cache/CacheGroups/PackedCONGroup.cs | 132 +- .../Cache/CacheGroups/UnpackedCONGroup.cs | 52 +- .../Song/Cache/CacheGroups/UpdateGroup.cs | 85 +- .../Song/Cache/CacheGroups/UpgradeGroup.cs | 49 +- YARG.Core/Song/Cache/CacheHandler.Parallel.cs | 187 +-- .../Song/Cache/CacheHandler.Sequential.cs | 180 ++- YARG.Core/Song/Cache/CacheHandler.cs | 877 +++++++------ YARG.Core/Song/Cache/CacheNodes.cs | 1 - YARG.Core/Song/Cache/FileCollection.cs | 4 +- YARG.Core/Song/Cache/SongCategories.cs | 17 +- .../Entries/AvailableParts/AvailableParts.cs | 6 - .../Song/Entries/Ini/SongEntry.IniBase.cs | 362 +++--- YARG.Core/Song/Entries/Ini/SongEntry.Sng.cs | 158 ++- .../Song/Entries/Ini/SongEntry.UnpackedIni.cs | 168 +-- YARG.Core/Song/Entries/RBCON/RBAudio.cs | 24 +- .../Song/Entries/RBCON/RBCONDifficulties.cs | 5 - YARG.Core/Song/Entries/RBCON/RBMetadata.cs | 75 +- YARG.Core/Song/Entries/RBCON/RBProUpgrade.cs | 13 +- .../Entries/RBCON/SongEntry.PackedRBCON.cs | 200 +-- .../Song/Entries/RBCON/SongEntry.RBCON.cs | 1136 ++++++----------- .../Entries/RBCON/SongEntry.UnpackedRBCON.cs | 162 ++- YARG.Core/Song/Entries/SongEntry.Loading.cs | 4 - YARG.Core/Song/Entries/SongEntry.Scanning.cs | 24 +- YARG.Core/Song/Entries/SongEntry.Sorting.cs | 5 +- YARG.Core/Song/Entries/SongEntry.cs | 560 ++------ YARG.Core/Song/Entries/Types/HashWrapper.cs | 9 + YARG.Core/Song/Entries/Types/SongMetadata.cs | 179 ++- .../Midi/MidiEliteDrumsPreparser.cs | 5 +- .../Midi/MidiInstrumentPreparser.cs | 4 +- .../Preparsers/Midi/MidiVocalPreparser.cs | 3 +- 47 files changed, 2658 insertions(+), 2818 deletions(-) create mode 100644 YARG.Core/IO/DTA/DTAEntry.cs rename YARG.Core/IO/{ => DTA}/YARGDTAReader.cs (97%) create mode 100644 YARG.Core/Song/Cache/CONModification.cs delete mode 100644 YARG.Core/Song/Cache/CacheGroups/LockedList.cs diff --git a/TestConsole/CacheLoader.cs b/TestConsole/CacheLoader.cs index e0d4dc0d4..bc5367916 100644 --- a/TestConsole/CacheLoader.cs +++ b/TestConsole/CacheLoader.cs @@ -31,7 +31,7 @@ public static SongCache LoadCache() throw new Exception($"Please add at least one song directory to {SongDirsPath}"); var task = Task.Run(() => CacheHandler.RunScan( - fast: false, + tryQuickScan: false, SongCachePath, BadSongsPath, MULTITHREADING, diff --git a/YARG.Core/Extensions/StreamExtensions.cs b/YARG.Core/Extensions/StreamExtensions.cs index 4adcc594f..f734f5c51 100644 --- a/YARG.Core/Extensions/StreamExtensions.cs +++ b/YARG.Core/Extensions/StreamExtensions.cs @@ -33,7 +33,10 @@ public static TType Read(this Stream stream, Endianness endianness) public static bool ReadBoolean(this Stream stream) { byte b = (byte)stream.ReadByte(); - return Unsafe.As(ref b); + unsafe + { + return *(bool*)&b; + } } public static int Read7BitEncodedInt(this Stream stream) @@ -124,6 +127,59 @@ public static void Write(this Stream stream, TType value, Endianness endi } } + public static void Write(this Stream stream, bool value) + { + unsafe + { + stream.WriteByte(*(byte*) &value); + } + } + + public static void Write7BitEncodedInt(this Stream stream, int value) + { + // Write out an int 7 bits at a time. The high bit of the byte, + // when on, tells reader to continue reading more bytes. + uint v = (uint) value; // support negative numbers + while (v >= 0x80) + { + stream.WriteByte((byte) (v | 0x80)); + v >>= 7; + } + stream.WriteByte((byte) v); + } + + public static unsafe void Write(this Stream stream, string value) + { + if (value.Length == 0) + { + stream.WriteByte(0); + return; + } + + var buffer = stackalloc byte[value.Length * 4]; + fixed (char* chars = value) + { + int len = Encoding.UTF8.GetBytes(chars, value.Length, buffer, value.Length * 4); + stream.Write7BitEncodedInt(len); + stream.Write(new ReadOnlySpan(buffer, len)); + } + } + + public static void Write(this Stream stream, Color color) + { + stream.Write(color.ToArgb(), Endianness.Little); + } + + public static void Write(this Stream stream, Guid guid) + { + Span span = stackalloc byte[16]; + if (!guid.TryWriteBytes(span)) + { + throw new InvalidOperationException("Failed to write GUID bytes."); + } + stream.Write(span); + } + private static unsafe void CorrectByteOrder(byte* bytes, Endianness endianness) where TType : unmanaged, IComparable, IComparable, IConvertible, IEquatable, IFormattable { diff --git a/YARG.Core/IO/AbridgedFileInfo.cs b/YARG.Core/IO/AbridgedFileInfo.cs index 275bc72dd..7833fd527 100644 --- a/YARG.Core/IO/AbridgedFileInfo.cs +++ b/YARG.Core/IO/AbridgedFileInfo.cs @@ -55,10 +55,10 @@ public AbridgedFileInfo(string filename, in DateTime lastUpdatedTime) LastUpdatedTime = lastUpdatedTime; } - public void Serialize(BinaryWriter writer) + public void Serialize(MemoryStream stream) { - writer.Write(FullName); - writer.Write(LastUpdatedTime.ToBinary()); + stream.Write(FullName); + stream.Write(LastUpdatedTime.ToBinary(), Endianness.Little); } public bool Exists() diff --git a/YARG.Core/IO/ConHandler/CONFile.cs b/YARG.Core/IO/ConHandler/CONFile.cs index 01cf1fb3a..d51b09660 100644 --- a/YARG.Core/IO/ConHandler/CONFile.cs +++ b/YARG.Core/IO/ConHandler/CONFile.cs @@ -6,7 +6,7 @@ namespace YARG.Core.IO { - public readonly struct CONFile + public static class CONFile { private static readonly FourCC CON_TAG = new('C', 'O', 'N', ' '); private static readonly FourCC LIVE_TAG = new('L', 'I', 'V', 'E'); @@ -22,41 +22,32 @@ public readonly struct CONFile private const int BYTES_PER_BLOCK = 0x1000; private const int SIZEOF_FILELISTING = 0x40; - private readonly List _filenames; - private readonly Dictionary _listings; - - private CONFile(List filenames, Dictionary listings) - { - _filenames = filenames; - _listings = listings; - } - - public string GetFilename(int index) - { - return _filenames[index]; - } - - public bool TryGetListing(string name, out CONFileListing listing) + public static bool TryGetListing(this List listings, string name, out CONFileListing listing) { - return _listings.TryGetValue(name, out listing); + foreach (var file in listings) + { + if (file.Filename == name) + { + listing = file; + return true; + } + } + listing = null!; + return false; } - public static CONFile? TryParseListings(AbridgedFileInfo info) + public static List? TryParseListings(in AbridgedFileInfo info, FileStream filestream) { - using var stream = InitStream_Internal(info.FullName); - if (stream == null) - return null; - Span int32Buffer = stackalloc byte[BYTES_32BIT]; - if (stream.Read(int32Buffer) != BYTES_32BIT) + if (filestream.Read(int32Buffer) != BYTES_32BIT) return null; var tag = new FourCC(int32Buffer); if (tag != CON_TAG && tag != LIVE_TAG && tag != PIRS_TAG) return null; - stream.Seek(METADATA_POSITION, SeekOrigin.Begin); - if (stream.Read(int32Buffer) != BYTES_32BIT) + filestream.Seek(METADATA_POSITION, SeekOrigin.Begin); + if (filestream.Read(int32Buffer) != BYTES_32BIT) return null; byte shift = 0; @@ -66,45 +57,40 @@ public bool TryGetListing(string name, out CONFileListing listing) if ((entryID + 0xFFF & 0xF000) >> 0xC != 0xB) shift = 1; - stream.Seek(FILETABLEBLOCKCOUNT_POSITION, SeekOrigin.Begin); - if (stream.Read(int32Buffer[..BYTES_16BIT]) != BYTES_16BIT) + filestream.Seek(FILETABLEBLOCKCOUNT_POSITION, SeekOrigin.Begin); + if (filestream.Read(int32Buffer[..BYTES_16BIT]) != BYTES_16BIT) return null; int length = BYTES_PER_BLOCK * (int32Buffer[0] | int32Buffer[1] << 8); - stream.Seek(FILETABLEFIRSTBLOCK_POSITION, SeekOrigin.Begin); - if (stream.Read(int32Buffer[..BYTES_24BIT]) != BYTES_24BIT) + filestream.Seek(FILETABLEFIRSTBLOCK_POSITION, SeekOrigin.Begin); + if (filestream.Read(int32Buffer[..BYTES_24BIT]) != BYTES_24BIT) return null; int firstBlock = int32Buffer[0] << 16 | int32Buffer[1] << 8 | int32Buffer[2]; - try { - var filenames = new List(); - var listings = new Dictionary(); + var listings = new List(); - using var conStream = new CONFileStream(stream, true, length, firstBlock, shift); - Span listingBuffer = stackalloc byte[SIZEOF_FILELISTING]; - while (conStream.Read(listingBuffer) == SIZEOF_FILELISTING && listingBuffer[0] != 0) + using var listingBuffer = CONFileStream.LoadFile(filestream, true, length, firstBlock, shift); + unsafe { - short pathIndex = (short) (listingBuffer[0x32] << 8 | listingBuffer[0x33]); - if (pathIndex >= filenames.Count) + var endPtr = listingBuffer.Ptr + length; + for (var currPtr = listingBuffer.Ptr; currPtr + SIZEOF_FILELISTING <= endPtr && currPtr[0] != 0; currPtr += SIZEOF_FILELISTING) { - YargLogger.LogFormatError("Error while parsing {0} - Filelisting blocks constructed out of spec", info.FullName); - return null; - } - - string filename = pathIndex >= 0 ? filenames[pathIndex] + "/" : string.Empty; - filename += Encoding.UTF8.GetString(listingBuffer[..0x28]).TrimEnd('\0'); - filenames.Add(filename); - - var flags = (CONFileListingFlag) listingBuffer[0x28]; - if ((flags & CONFileListingFlag.Directory) == 0) - { - listings[filename] = new CONFileListing(info, filename, pathIndex, flags, shift, listingBuffer); + short pathIndex = (short) (currPtr[0x32] << 8 | currPtr[0x33]); + if (pathIndex >= listings.Count) + { + YargLogger.LogFormatError("Error while parsing {0} - Filelisting blocks constructed out of spec", info.FullName); + return null; + } + + string filename = pathIndex >= 0 ? listings[pathIndex].Filename + "/" : string.Empty; + filename += Encoding.UTF8.GetString(currPtr, 0x28).TrimEnd('\0'); + listings.Add(new CONFileListing(info, filename, pathIndex, shift, currPtr)); } } - return new CONFile(filenames, listings); + return listings; } catch (Exception ex) { @@ -112,18 +98,5 @@ public bool TryGetListing(string name, out CONFileListing listing) return null; } } - - private static FileStream? InitStream_Internal(string filename) - { - try - { - return new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read); - } - catch (Exception ex) - { - YargLogger.LogException(ex, $"Error loading {filename}"); - return null; - } - } } } diff --git a/YARG.Core/IO/ConHandler/CONFileListing.cs b/YARG.Core/IO/ConHandler/CONFileListing.cs index 27dc65d86..9d790a2ee 100644 --- a/YARG.Core/IO/ConHandler/CONFileListing.cs +++ b/YARG.Core/IO/ConHandler/CONFileListing.cs @@ -1,7 +1,5 @@ using System; -using System.Diagnostics; using System.IO; -using System.Text; using YARG.Core.Extensions; namespace YARG.Core.IO @@ -15,8 +13,6 @@ public enum CONFileListingFlag : byte public sealed class CONFileListing { - - private readonly int _shift; public readonly AbridgedFileInfo ConFile; @@ -28,7 +24,7 @@ public sealed class CONFileListing public readonly int Size; public readonly DateTime LastWrite; - public CONFileListing(AbridgedFileInfo conFile, string name, short pathIndex, CONFileListingFlag flags, int shift, ReadOnlySpan data) + public unsafe CONFileListing(AbridgedFileInfo conFile, string name, short pathIndex, int shift, byte* data) { _shift = shift; @@ -36,7 +32,7 @@ public CONFileListing(AbridgedFileInfo conFile, string name, short pathIndex, CO Filename = name; PathIndex = pathIndex; - Flags = flags; + Flags = (CONFileListingFlag) data[0x28]; NumBlocks = data[0x2B] << 16 | data[0x2A] << 8 | data[0x29]; FirstBlock = data[0x31] << 16 | data[0x30] << 8 | data[0x2F]; Size = data[0x34] << 24 | data[0x35] << 16 | data[0x36] << 8 | data[0x37]; diff --git a/YARG.Core/IO/DTA/DTAEntry.cs b/YARG.Core/IO/DTA/DTAEntry.cs new file mode 100644 index 000000000..a0c856950 --- /dev/null +++ b/YARG.Core/IO/DTA/DTAEntry.cs @@ -0,0 +1,310 @@ +using System.Text; +using YARG.Core.Song; + +namespace YARG.Core.IO +{ + public class DTAEntry + { + public string? Name; + public string? Artist; + public string? Album; + public string? Genre; + public string? Charter; + public string? Source; + public string? Playlist; + public int? YearAsNumber; + + public ulong? SongLength; + public uint? SongRating; // 1 = FF; 2 = SR; 3 = M; 4 = NR + + public long? PreviewStart; + public long? PreviewEnd; + public bool? IsMaster; + + public int? AlbumTrack; + + public string? SongID; + public uint? AnimTempo; + public string? DrumBank; + public string? VocalPercussionBank; + public uint? VocalSongScrollSpeed; + public bool? VocalGender; //true for male, false for female + //public bool HasAlbumArt; + //public bool IsFake; + public uint? VocalTonicNote; + public bool? SongTonality; // 0 = major, 1 = minor + public int? TuningOffsetCents; + public uint? VenueVersion; + + public string[]? Soloes; + public string[]? VideoVenues; + + public int[]? RealGuitarTuning; + public int[]? RealBassTuning; + + public RBAudio? Indices; + public int[]? CrowdChannels; + + public string? Location; + public float[]? Pans; + public float[]? Volumes; + public float[]? Cores; + public long? HopoThreshold; + public Encoding Encoding; + + public RBCONDifficulties Difficulties = RBCONDifficulties.Default; + + public DTAEntry(Encoding encoding) + { + Encoding = encoding; + } + + public DTAEntry(string nodename, in YARGTextContainer container) + { + Encoding = container.Encoding; + LoadData(nodename, container); + } + + public void LoadData(string nodename, YARGTextContainer container) + { + container.Encoding = Encoding; + while (YARGDTAReader.StartNode(ref container)) + { + string name = YARGDTAReader.GetNameOfNode(ref container, false); + switch (name) + { + case "name": Name = YARGDTAReader.ExtractText(ref container); break; + case "artist": Artist = YARGDTAReader.ExtractText(ref container); break; + case "master": IsMaster = YARGDTAReader.ExtractBoolean_FlippedDefault(ref container); break; + case "context": /*Context = YARGDTAReader.ExtractUInt32(ref container);*/ break; + case "song": + while (YARGDTAReader.StartNode(ref container)) + { + string descriptor = YARGDTAReader.GetNameOfNode(ref container, false); + switch (descriptor) + { + case "name": Location = YARGDTAReader.ExtractText(ref container); break; + case "tracks": + { + var indices = RBAudio.Empty; + while (YARGDTAReader.StartNode(ref container)) + { + while (YARGDTAReader.StartNode(ref container)) + { + switch (YARGDTAReader.GetNameOfNode(ref container, false)) + { + case "drum": indices.Drums = YARGDTAReader.ExtractArray_Int(ref container); break; + case "bass": indices.Bass = YARGDTAReader.ExtractArray_Int(ref container); break; + case "guitar": indices.Guitar = YARGDTAReader.ExtractArray_Int(ref container); break; + case "keys": indices.Keys = YARGDTAReader.ExtractArray_Int(ref container); break; + case "vocals": indices.Vocals = YARGDTAReader.ExtractArray_Int(ref container); break; + } + YARGDTAReader.EndNode(ref container); + } + YARGDTAReader.EndNode(ref container); + } + Indices = indices; + break; + } + case "crowd_channels": CrowdChannels = YARGDTAReader.ExtractArray_Int(ref container); break; + //case "vocal_parts": VocalParts = YARGDTAReader.ExtractUInt16(ref container); break; + case "pans": Pans = YARGDTAReader.ExtractArray_Float(ref container); break; + case "vols": Volumes = YARGDTAReader.ExtractArray_Float(ref container); break; + case "cores": Cores = YARGDTAReader.ExtractArray_Float(ref container); break; + case "hopo_threshold": HopoThreshold = YARGDTAReader.ExtractInt64(ref container); break; + } + YARGDTAReader.EndNode(ref container); + } + break; + case "song_vocals": while (YARGDTAReader.StartNode(ref container)) YARGDTAReader.EndNode(ref container); break; + case "song_scroll_speed": VocalSongScrollSpeed = YARGDTAReader.ExtractUInt32(ref container); break; + case "tuning_offset_cents": TuningOffsetCents = YARGDTAReader.ExtractInt32(ref container); break; + case "bank": VocalPercussionBank = YARGDTAReader.ExtractText(ref container); break; + case "anim_tempo": + { + string val = YARGDTAReader.ExtractText(ref container); + AnimTempo = val switch + { + "kTempoSlow" => 16, + "kTempoMedium" => 32, + "kTempoFast" => 64, + _ => uint.Parse(val) + }; + break; + } + case "preview": + PreviewStart = YARGDTAReader.ExtractInt64(ref container); + PreviewEnd = YARGDTAReader.ExtractInt64(ref container); + break; + case "rank": + while (YARGDTAReader.StartNode(ref container)) + { + string descriptor = YARGDTAReader.GetNameOfNode(ref container, false); + int diff = YARGDTAReader.ExtractInt32(ref container); + switch (descriptor) + { + case "drum": + case "drums": Difficulties.FourLaneDrums = (short) diff; break; + + case "guitar": Difficulties.FiveFretGuitar = (short) diff; break; + case "bass": Difficulties.FiveFretBass = (short) diff; break; + case "vocals": Difficulties.LeadVocals = (short) diff; break; + case "keys": Difficulties.Keys = (short) diff; break; + + case "realGuitar": + case "real_guitar": Difficulties.ProGuitar = (short) diff; break; + + case "realBass": + case "real_bass": Difficulties.ProBass = (short) diff; break; + + case "realKeys": + case "real_keys": Difficulties.ProKeys = (short) diff; break; + + case "realDrums": + case "real_drums": Difficulties.ProDrums = (short) diff; break; + + case "harmVocals": + case "vocal_harm": Difficulties.HarmonyVocals = (short) diff; break; + + case "band": Difficulties.Band = (short) diff; break; + } + YARGDTAReader.EndNode(ref container); + } + break; + case "solo": Soloes = YARGDTAReader.ExtractArray_String(ref container); break; + case "genre": Genre = YARGDTAReader.ExtractText(ref container); break; + case "decade": /*Decade = YARGDTAReader.ExtractText(ref container);*/ break; + case "vocal_gender": VocalGender = YARGDTAReader.ExtractText(ref container) == "male"; break; + case "format": /*Format = YARGDTAReader.ExtractUInt32(ref container);*/ break; + case "version": VenueVersion = YARGDTAReader.ExtractUInt32(ref container); break; + case "fake": /*IsFake = YARGDTAReader.ExtractText(ref container);*/ break; + case "downloaded": /*Downloaded = YARGDTAReader.ExtractText(ref container);*/ break; + case "game_origin": + { + string str = YARGDTAReader.ExtractText(ref container); + if ((str == "ugc" || str == "ugc_plus")) + { + if (!nodename.StartsWith("UGC_")) + Source = "customs"; + } + else if (str == "#ifdef") + { + string conditional = YARGDTAReader.ExtractText(ref container); + if (conditional == "CUSTOMSOURCE") + { + Source = YARGDTAReader.ExtractText(ref container); + } + else + { + Source = "customs"; + } + } + else + { + Source = str; + } + + //// if the source is any official RB game or its DLC, charter = Harmonix + //if (SongSources.GetSource(str).Type == SongSources.SourceType.RB) + //{ + // _charter = "Harmonix"; + //} + + //// if the source is meant for usage in TBRB, it's a master track + //// TODO: NEVER assume localized version contains "Beatles" + //if (SongSources.SourceToGameName(str).Contains("Beatles")) _isMaster = true; + break; + } + case "song_id": SongID = YARGDTAReader.ExtractText(ref container); break; + case "rating": SongRating = YARGDTAReader.ExtractUInt32(ref container); break; + case "short_version": /*ShortVersion = YARGDTAReader.ExtractUInt32(ref container);*/ break; + case "album_art": /*HasAlbumArt = YARGDTAReader.ExtractBoolean(ref container);*/ break; + case "year_released": + case "year_recorded": YearAsNumber = YARGDTAReader.ExtractInt32(ref container); break; + case "album_name": Album = YARGDTAReader.ExtractText(ref container); break; + case "album_track_number": AlbumTrack = YARGDTAReader.ExtractInt32(ref container); break; + case "pack_name": Playlist = YARGDTAReader.ExtractText(ref container); break; + case "base_points": /*BasePoints = YARGDTAReader.ExtractUInt32(ref container);*/ break; + case "band_fail_cue": /*BandFailCue = YARGDTAReader.ExtractText(ref container);*/ break; + case "drum_bank": DrumBank = YARGDTAReader.ExtractText(ref container); break; + case "song_length": SongLength = YARGDTAReader.ExtractUInt64(ref container); break; + case "sub_genre": /*Subgenre = YARGDTAReader.ExtractText(ref container);*/ break; + case "author": Charter = YARGDTAReader.ExtractText(ref container); break; + case "guide_pitch_volume": /*GuidePitchVolume = YARGDTAReader.ExtractFloat(ref container);*/ break; + case "encoding": + Encoding = YARGDTAReader.ExtractText(ref container).ToLower() switch + { + "latin1" => YARGTextReader.Latin1, + "utf-8" or + "utf8" => Encoding.UTF8, + _ => container.Encoding + }; + + var currEncoding = container.Encoding; + if (currEncoding != Encoding) + { + string Convert(string str) + { + byte[] bytes = currEncoding.GetBytes(str); + return Encoding.GetString(bytes); + } + + if (Name != null) + Name = Convert(Name); + if (Artist != null) + Artist = Convert(Artist); + if (Album != null) + Album = Convert(Album); + if (Genre != null) + Genre = Convert(Genre); + if (Charter != null) + Charter = Convert(Charter); + if (Source != null) + Source = Convert(Source); + + if (Playlist != null) + Playlist = Convert(Playlist); + container.Encoding = Encoding; + } + + break; + case "vocal_tonic_note": VocalTonicNote = YARGDTAReader.ExtractUInt32(ref container); break; + case "song_tonality": SongTonality = YARGDTAReader.ExtractBoolean(ref container); break; + case "alternate_path": /*AlternatePath = YARGDTAReader.ExtractBoolean(ref container);*/ break; + case "real_guitar_tuning": RealGuitarTuning = YARGDTAReader.ExtractArray_Int(ref container); break; + case "real_bass_tuning": RealBassTuning = YARGDTAReader.ExtractArray_Int(ref container); break; + case "video_venues": VideoVenues = YARGDTAReader.ExtractArray_String(ref container); break; + case "extra_authoring": + { + StringBuilder authors = new(); + foreach (string str in YARGDTAReader.ExtractArray_String(ref container)) + { + if (str != "disc_update") + { + if (authors.Length == 0 && Charter == SongMetadata.DEFAULT_CHARTER) + { + authors.Append(str); + } + else + { + if (authors.Length == 0) + authors.Append(Charter); + authors.Append(", " + str); + } + } + } + + if (authors.Length == 0) + { + authors.Append(Charter); + } + + Charter = authors.ToString(); + } + break; + } + YARGDTAReader.EndNode(ref container); + } + } + } +} diff --git a/YARG.Core/IO/YARGDTAReader.cs b/YARG.Core/IO/DTA/YARGDTAReader.cs similarity index 97% rename from YARG.Core/IO/YARGDTAReader.cs rename to YARG.Core/IO/DTA/YARGDTAReader.cs index de89755f7..5326eba14 100644 --- a/YARG.Core/IO/YARGDTAReader.cs +++ b/YARG.Core/IO/DTA/YARGDTAReader.cs @@ -1,30 +1,28 @@ using System; using System.Collections.Generic; -using System.IO; using System.Text; using YARG.Core.Extensions; using YARG.Core.Logging; namespace YARG.Core.IO { - public unsafe static class YARGDTAReader + public static unsafe class YARGDTAReader { - public static bool TryCreate(in FixedArray data, out YARGTextContainer container) + public static YARGTextContainer TryCreate(in FixedArray data) { if ((data[0] == 0xFF && data[1] == 0xFE) || (data[0] == 0xFE && data[1] == 0xFF)) { YargLogger.LogError("UTF-16 & UTF-32 are not supported for .dta files"); - container = default; - return false; + return YARGTextContainer.Null; } - container = new YARGTextContainer(in data, YARGTextReader.Latin1); + var container = new YARGTextContainer(in data, YARGTextReader.Latin1); if (data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF) { container.Position += 3; container.Encoding = Encoding.UTF8; } - return true; + return container; } public static char SkipWhitespace(ref YARGTextContainer container) diff --git a/YARG.Core/IO/FixedArray.cs b/YARG.Core/IO/FixedArray.cs index 3a9986e24..89c5dd904 100644 --- a/YARG.Core/IO/FixedArray.cs +++ b/YARG.Core/IO/FixedArray.cs @@ -27,10 +27,7 @@ public unsafe struct FixedArray : IDisposable /// /// A indisposable default instance with a null pointer /// - public static readonly FixedArray Null = new() - { - _disposed = true - }; + public static readonly FixedArray Null = new(null, 0); /// /// Loads all of the given file's data into a FixedArray buffer @@ -144,7 +141,7 @@ private FixedArray(T* ptr, long length) { Ptr = ptr; Length = length; - _disposed = false; + _disposed = ptr == null; } public readonly Span Slice(long offset, long count) diff --git a/YARG.Core/IO/Midi/YARGMidiFile.cs b/YARG.Core/IO/Midi/YARGMidiFile.cs index 2af64ab1e..1f7d83c57 100644 --- a/YARG.Core/IO/Midi/YARGMidiFile.cs +++ b/YARG.Core/IO/Midi/YARGMidiFile.cs @@ -13,9 +13,9 @@ public sealed class YARGMidiFile : IEnumerable private static readonly FourCC TRACK_TAG = new('M', 'T', 'r', 'k'); private readonly Stream _stream; - private readonly ushort _format; - private readonly ushort _numTracks; - private readonly ushort _tickRate; + public readonly ushort Format; + public readonly ushort NumTracks; + public readonly ushort Resolution; private ushort _trackNumber = 0; public ushort TrackNumber => _trackNumber; @@ -32,20 +32,22 @@ public YARGMidiFile(Stream stream) throw new Exception("Midi Header not of sufficient length"); long next = stream.Position + length; - _format = stream.Read(Endianness.Big); - _numTracks = stream.Read(Endianness.Big); - _tickRate = stream.Read(Endianness.Big); + Format = stream.Read(Endianness.Big); + NumTracks = stream.Read(Endianness.Big); + Resolution = stream.Read(Endianness.Big); stream.Position = next; } public YARGMidiTrack? LoadNextTrack() { - if (_trackNumber == _numTracks || _stream.Position == _stream.Length) + if (_trackNumber == NumTracks || _stream.Position == _stream.Length) return null; _trackNumber++; if (!TRACK_TAG.Matches(_stream)) + { throw new Exception($"Midi Track Tag 'MTrk' not found for Track '{_trackNumber}'"); + } try { diff --git a/YARG.Core/IO/SngHandler/SngFile.cs b/YARG.Core/IO/SngHandler/SngFile.cs index 318f25f04..20d9ed535 100644 --- a/YARG.Core/IO/SngHandler/SngFile.cs +++ b/YARG.Core/IO/SngHandler/SngFile.cs @@ -63,11 +63,11 @@ IEnumerator IEnumerable.GetEnumerator() private const int BYTES_16BIT = 2; private static readonly byte[] SNGPKG = { (byte)'S', (byte) 'N', (byte) 'G', (byte)'P', (byte)'K', (byte)'G' }; - public static SngFile? TryLoadFromFile(AbridgedFileInfo file) + public static SngFile? TryLoadFromFile(in AbridgedFileInfo file) { try { - using var filestream = File.OpenRead(file.FullName); + using var filestream = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 1); using var yargSongStream = YARGSongFileStream.TryLoad(filestream); if (yargSongStream != null) { diff --git a/YARG.Core/IO/TextReader/YARGTextContainer.cs b/YARG.Core/IO/TextReader/YARGTextContainer.cs index 25f5b7da2..dd69393f0 100644 --- a/YARG.Core/IO/TextReader/YARGTextContainer.cs +++ b/YARG.Core/IO/TextReader/YARGTextContainer.cs @@ -6,18 +6,17 @@ namespace YARG.Core.IO public unsafe struct YARGTextContainer where TChar : unmanaged, IConvertible { + public static readonly YARGTextContainer Null = new() + { + Encoding = null!, + Position = null, + }; + public readonly TChar* End; public Encoding Encoding; public TChar* Position; - public YARGTextContainer(in FixedArray buffer, Encoding encoding) - { - Position = buffer.Ptr; - End = buffer.Ptr + buffer.Length; - Encoding = encoding; - } - - public TChar CurrentValue + public readonly TChar CurrentValue { get { @@ -29,6 +28,15 @@ public TChar CurrentValue } } + public readonly bool IsActive => End != null; + + public YARGTextContainer(in FixedArray buffer, Encoding encoding) + { + Position = buffer.Ptr; + End = buffer.Ptr + buffer.Length; + Encoding = encoding; + } + public readonly bool IsCurrentCharacter(int cmp) { return Position->ToInt32(null).Equals(cmp); diff --git a/YARG.Core/Song/Cache/CONModification.cs b/YARG.Core/Song/Cache/CONModification.cs new file mode 100644 index 000000000..5c59e29f4 --- /dev/null +++ b/YARG.Core/Song/Cache/CONModification.cs @@ -0,0 +1,16 @@ +using YARG.Core.IO; + +namespace YARG.Core.Song.Cache +{ + public class CONModification + { + public bool Processed = false; + public AbridgedFileInfo? Midi; + public AbridgedFileInfo? Mogg; + public AbridgedFileInfo? Milo; + public AbridgedFileInfo? Image; + public DTAEntry? UpdateDTA; + public DTAEntry? UpgradeDTA; + public RBProUpgrade? UpgradeNode; + } +} diff --git a/YARG.Core/Song/Cache/CacheGroups/CacheGroup.cs b/YARG.Core/Song/Cache/CacheGroups/CacheGroup.cs index 841808693..dfda91002 100644 --- a/YARG.Core/Song/Cache/CacheGroups/CacheGroup.cs +++ b/YARG.Core/Song/Cache/CacheGroups/CacheGroup.cs @@ -1,36 +1,35 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; -using System.Text; +using YARG.Core.Extensions; namespace YARG.Core.Song.Cache { - public interface ICacheGroup - where TEntry : SongEntry + public interface ICacheGroup { public int Count { get; } - public ReadOnlyMemory SerializeEntries(Dictionary nodes); + public void SerializeEntries(MemoryStream stream, Dictionary nodes); public bool TryRemoveEntry(SongEntry entryToRemove); - public static void SerializeGroups(List groups, BinaryWriter writer, Dictionary nodes) - where TGroup : ICacheGroup + public static void SerializeGroups(List groups, FileStream fileStream, Dictionary nodes) + where TGroup : ICacheGroup { - var spans = new ReadOnlyMemory[groups.Count]; + var streams = new MemoryStream[groups.Count]; int length = 4; for (int i = 0; i < groups.Count; i++) { - spans[i] = groups[i].SerializeEntries(nodes); - length += sizeof(int) + spans[i].Length; + streams[i] = new MemoryStream(); + groups[i].SerializeEntries(streams[i], nodes); + length += sizeof(int) + (int) streams[i].Length; } - writer.Write(length); - writer.Write(groups.Count); - for (int i = 0; i < groups.Count; i++) + fileStream.Write(length, Endianness.Little); + fileStream.Write(streams.Length, Endianness.Little); + for (int i = 0; i < streams.Length; i++) { - var span = spans[i].Span; - writer.Write(span.Length); - writer.Write(span); + using var stream = streams[i]; + fileStream.Write((int) stream.Length, Endianness.Little); + fileStream.Write(stream.GetBuffer(), 0, (int) stream.Length); } } } diff --git a/YARG.Core/Song/Cache/CacheGroups/ConGroup.cs b/YARG.Core/Song/Cache/CacheGroups/ConGroup.cs index c7040a8b5..4d48fdbd9 100644 --- a/YARG.Core/Song/Cache/CacheGroups/ConGroup.cs +++ b/YARG.Core/Song/Cache/CacheGroups/ConGroup.cs @@ -1,39 +1,75 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; +using YARG.Core.Extensions; using YARG.Core.IO; namespace YARG.Core.Song.Cache { - public abstract class CONGroup : ICacheGroup + public abstract class CONGroup : ICacheGroup + where TEntry : RBCONEntry { - protected readonly Dictionary> entries = new(); - - private int _count; - public int Count { get { lock (entries) return _count; } } + protected readonly Dictionary> entries = new(); + public readonly string Location; + public readonly AbridgedFileInfo Info; public readonly string DefaultPlaylist; + public readonly FixedArray SongDTAData; - public abstract string Location { get; } + public int Count + { + get + { + int count = 0; + foreach (var node in entries.Values) + { + count += node.Count; + } + return count; + } + } - protected CONGroup(string defaultPlaylist) + protected CONGroup(in FixedArray songDTAData, string location, in AbridgedFileInfo info, string defaultPlaylist) { + SongDTAData = songDTAData; + Location = location; + Info = info; DefaultPlaylist = defaultPlaylist; } - public abstract void ReadEntry(string nodeName, int index, Dictionary, RBProUpgrade)> upgrades, UnmanagedMemoryStream stream, CategoryCacheStrings strings); - public abstract ReadOnlyMemory SerializeEntries(Dictionary nodes); + public abstract void ReadEntry(string nodeName, int index, RBProUpgrade upgrade, UnmanagedMemoryStream stream, CategoryCacheStrings strings); - public void AddEntry(string name, int index, RBCONEntry entry) + public void SerializeEntries(MemoryStream groupStream, Dictionary nodes) { - SortedDictionary dict; + groupStream.Write(Location); + groupStream.Write(Info.LastUpdatedTime.ToBinary(), Endianness.Little); + groupStream.Write(Count, Endianness.Little); + + using var entryStream = new MemoryStream(); + foreach (var entryList in entries) + { + foreach (var entry in entryList.Value) + { + groupStream.Write(entryList.Key); + groupStream.Write(entry.Key, Endianness.Little); + + entryStream.SetLength(0); + entry.Value.Serialize(entryStream, nodes[entry.Value]); + + groupStream.Write((int) entryStream.Length, Endianness.Little); + groupStream.Write(entryStream.GetBuffer(), 0, (int) entryStream.Length); + } + } + } + + public void AddEntry(string name, int index, TEntry entry) + { + SortedDictionary dict; lock (entries) { if (!entries.TryGetValue(name, out dict)) { - entries.Add(name, dict = new SortedDictionary()); + entries.Add(name, dict = new SortedDictionary()); } - ++_count; } lock (dict) @@ -48,8 +84,6 @@ public bool RemoveEntries(string name) { if (!entries.Remove(name, out var dict)) return false; - - _count -= dict.Count; } return true; } @@ -64,11 +98,10 @@ public void RemoveEntry(string name, int index) { entries.Remove(name); } - --_count; } } - public bool TryGetEntry(string name, int index, out RBCONEntry? entry) + public bool TryGetEntry(string name, int index, out TEntry? entry) { entry = null; lock (entries) @@ -92,37 +125,11 @@ public bool TryRemoveEntry(SongEntry entryToRemove) { entries.Remove(dict.Key); } - --_count; return true; } } } return false; } - - protected void Serialize(BinaryWriter writer, ref Dictionary nodes) - { - writer.Write(_count); - foreach (var entryList in entries) - { - foreach (var entry in entryList.Value) - { - writer.Write(entryList.Key); - writer.Write(entry.Key); - - byte[] data = SerializeEntry(entry.Value, nodes[entry.Value]); - writer.Write(data.Length); - writer.Write(data); - } - } - } - - private static byte[] SerializeEntry(RBCONEntry entry, CategoryCacheWriteNode node) - { - using MemoryStream ms = new(); - using BinaryWriter writer = new(ms); - entry.Serialize(writer, node); - return ms.ToArray(); - } } } diff --git a/YARG.Core/Song/Cache/CacheGroups/IModificationGroup.cs b/YARG.Core/Song/Cache/CacheGroups/IModificationGroup.cs index cd2b62660..4e2f2e118 100644 --- a/YARG.Core/Song/Cache/CacheGroups/IModificationGroup.cs +++ b/YARG.Core/Song/Cache/CacheGroups/IModificationGroup.cs @@ -1,32 +1,32 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; -using System.Text; +using YARG.Core.Extensions; namespace YARG.Core.Song.Cache { public interface IModificationGroup { - public ReadOnlyMemory SerializeModifications(); + public void SerializeModifications(MemoryStream stream); - public static void SerializeGroups(List groups, BinaryWriter writer) + public static void SerializeGroups(List groups, FileStream fileStream) where TGroup : IModificationGroup { - var spans = new ReadOnlyMemory[groups.Count]; + var streams = new MemoryStream[groups.Count]; int length = sizeof(int); for (int i = 0; i < groups.Count; i++) { - spans[i] = groups[i].SerializeModifications(); - length += sizeof(int) + spans[i].Length; + var stream = streams[i] = new MemoryStream(); + groups[i].SerializeModifications(stream); + length += sizeof(int) + (int) stream.Length; } - writer.Write(length); - writer.Write(groups.Count); + fileStream.Write(length, Endianness.Little); + fileStream.Write(streams.Length, Endianness.Little); for (int i = 0; i < groups.Count; i++) { - var span = spans[i].Span; - writer.Write(span.Length); - writer.Write(span); + using var stream = streams[i]; + fileStream.Write((int) stream.Length, Endianness.Little); + fileStream.Write(stream.GetBuffer(), 0, (int) stream.Length); } } } diff --git a/YARG.Core/Song/Cache/CacheGroups/IniGroup.cs b/YARG.Core/Song/Cache/CacheGroups/IniGroup.cs index 15ef32ff4..5dc3d03c6 100644 --- a/YARG.Core/Song/Cache/CacheGroups/IniGroup.cs +++ b/YARG.Core/Song/Cache/CacheGroups/IniGroup.cs @@ -1,19 +1,26 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; +using YARG.Core.Extensions; namespace YARG.Core.Song.Cache { - public sealed class IniGroup : ICacheGroup + public sealed class IniGroup : ICacheGroup { public readonly string Directory; public readonly Dictionary> entries = new(); - public readonly object iniLock = new(); - private int _count; - - public string Location => Directory; - public int Count => _count; + public int Count + { + get + { + int count = 0; + foreach (var node in entries.Values) + { + count += node.Count; + } + return count; + } + } public IniGroup(string directory) { @@ -24,13 +31,12 @@ public void AddEntry(IniSubEntry entry) { var hash = entry.Hash; List list; - lock (iniLock) + lock (entries) { if (!entries.TryGetValue(hash, out list)) { entries.Add(hash, list = new List()); } - ++_count; } lock (list) @@ -51,30 +57,38 @@ public bool TryRemoveEntry(SongEntry entryToRemove) { entries.Remove(entryToRemove.Hash); } - --_count; return true; } } return false; } - public ReadOnlyMemory SerializeEntries(Dictionary nodes) + public void SerializeEntries(MemoryStream groupStream, Dictionary nodes) { - using MemoryStream ms = new(); - using BinaryWriter writer = new(ms); + groupStream.Write(Directory); + groupStream.Write(Count, Endianness.Little); - writer.Write(Location); - writer.Write(_count); + using MemoryStream entryStream = new(); foreach (var shared in entries) { foreach (var entry in shared.Value) { - var buffer = entry.Serialize(nodes[entry], Location); - writer.Write(buffer.Length); - writer.Write(buffer); + entryStream.SetLength(0); + + // Validation block + entryStream.Write(entry.SubType == EntryType.Sng); + string relativePath = Path.GetRelativePath(Directory, entry.Location); + if (relativePath == ".") + { + relativePath = string.Empty; + } + entryStream.Write(relativePath); + entry.Serialize(entryStream, nodes[entry]); + + groupStream.Write((int) entryStream.Length, Endianness.Little); + groupStream.Write(entryStream.GetBuffer(), 0, (int) entryStream.Length); } } - return new ReadOnlyMemory(ms.GetBuffer(), 0, (int) ms.Length); } } } diff --git a/YARG.Core/Song/Cache/CacheGroups/LockedList.cs b/YARG.Core/Song/Cache/CacheGroups/LockedList.cs deleted file mode 100644 index d4b28cee8..000000000 --- a/YARG.Core/Song/Cache/CacheGroups/LockedList.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace YARG.Core.Song.Cache -{ - public class LockedList - { - public readonly object Lock = new(); - public readonly List Values = new(); - - public void Add(T value) - { - lock (Lock) - Values.Add(value); - } - } -} diff --git a/YARG.Core/Song/Cache/CacheGroups/PackedCONGroup.cs b/YARG.Core/Song/Cache/CacheGroups/PackedCONGroup.cs index accc76f53..d766632e9 100644 --- a/YARG.Core/Song/Cache/CacheGroups/PackedCONGroup.cs +++ b/YARG.Core/Song/Cache/CacheGroups/PackedCONGroup.cs @@ -1,129 +1,49 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; +using YARG.Core.Extensions; using YARG.Core.IO; -using YARG.Core.Logging; namespace YARG.Core.Song.Cache { - public sealed class PackedCONGroup : CONGroup, IUpgradeGroup + public sealed class PackedCONGroup : CONGroup, IModificationGroup { - public readonly AbridgedFileInfo Info; - public readonly CONFile ConFile; - public readonly CONFileListing? SongDTA; - public readonly CONFileListing? UpgradeDta; - public Stream? Stream; + public readonly List Listings; + public readonly Dictionary>> SongNodes = new(); + public readonly Dictionary Container, PackedRBProUpgrade Upgrade)> Upgrades = new(); + public readonly FixedArray UpgradeDTAData; + public FileStream Stream; - private FixedArray _songDTAData = FixedArray.Null; - private FixedArray _upgradeDTAData = FixedArray.Null; - - public override string Location => Info.FullName; - public Dictionary Upgrades { get; } = new(); - - public PackedCONGroup(CONFile conFile, AbridgedFileInfo info, string defaultPlaylist) - : base(defaultPlaylist) + public PackedCONGroup(List listings, FixedArray songData, FixedArray upgradeData, + Dictionary>> songNodes, Dictionary, PackedRBProUpgrade)> upgrades, + in AbridgedFileInfo info, string defaultPlaylist) + : base(in songData, info.FullName, in info, defaultPlaylist) { - const string SONGSFILEPATH = "songs/songs.dta"; - const string UPGRADESFILEPATH = "songs_upgrades/upgrades.dta"; - - Info = info; - ConFile = conFile; - conFile.TryGetListing(UPGRADESFILEPATH, out UpgradeDta); - conFile.TryGetListing(SONGSFILEPATH, out SongDTA); + Listings = listings; + SongNodes = songNodes; + Upgrades = upgrades; + UpgradeDTAData = upgradeData; + Stream = null!; } - public override void ReadEntry(string nodeName, int index, Dictionary, RBProUpgrade)> upgrades, UnmanagedMemoryStream stream, CategoryCacheStrings strings) + public override void ReadEntry(string nodeName, int index, RBProUpgrade upgrade, UnmanagedMemoryStream stream, CategoryCacheStrings strings) { - var song = PackedRBCONEntry.TryLoadFromCache(in ConFile, nodeName, upgrades, stream, strings); + var song = PackedRBCONEntry.TryLoadFromCache(Listings, nodeName, upgrade, stream, strings); if (song != null) { AddEntry(nodeName, index, song); } } - public override ReadOnlyMemory SerializeEntries(Dictionary nodes) - { - using MemoryStream ms = new(); - using BinaryWriter writer = new(ms); - - writer.Write(Location); - writer.Write(SongDTA!.LastWrite.ToBinary()); - Serialize(writer, ref nodes); - return new ReadOnlyMemory(ms.GetBuffer(), 0, (int)ms.Length); - } - - public void AddUpgrade(string name, RBProUpgrade upgrade) { lock (Upgrades) Upgrades[name] = upgrade; } - - public bool LoadUpgrades(out YARGTextContainer container) - { - if (UpgradeDta == null) - { - container = default; - return false; - } - - try - { - Stream = new FileStream(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 1); - _upgradeDTAData = UpgradeDta.LoadAllBytes(Stream); - return YARGDTAReader.TryCreate(_upgradeDTAData, out container); - } - catch (Exception ex) - { - YargLogger.LogException(ex, $"Error while loading {UpgradeDta.Filename}"); - container = default; - return false; - } - } - - public bool LoadSongs(out YARGTextContainer container) + public void SerializeModifications(MemoryStream stream) { - if (SongDTA == null) + stream.Write(Location); + stream.Write(Info.LastUpdatedTime.ToBinary(), Endianness.Little); + stream.Write(Upgrades.Count, Endianness.Little); + foreach (var node in Upgrades) { - container = default; - return false; + stream.Write(node.Key); + node.Value.Upgrade.WriteToCache(stream); } - - try - { - Stream ??= new FileStream(Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 1); - _songDTAData = SongDTA.LoadAllBytes(Stream); - return YARGDTAReader.TryCreate(_songDTAData, out container); - } - catch (Exception ex) - { - YargLogger.LogException(ex, $"Error while loading {SongDTA.Filename}"); - container = default; - return false; - } - } - - public ReadOnlyMemory SerializeModifications() - { - using MemoryStream ms = new(); - using BinaryWriter writer = new(ms); - - writer.Write(Location); - writer.Write(Info.LastUpdatedTime.ToBinary()); - writer.Write(UpgradeDta!.LastWrite.ToBinary()); - writer.Write(Upgrades.Count); - foreach (var upgrade in Upgrades) - { - writer.Write(upgrade.Key); - upgrade.Value.WriteToCache(writer); - } - return new ReadOnlyMemory(ms.GetBuffer(), 0, (int)ms.Length); - } - - public void DisposeStreamAndSongDTA() - { - Stream?.Dispose(); - _songDTAData.Dispose(); - } - - public void DisposeUpgradeDTA() - { - _upgradeDTAData.Dispose(); } } } diff --git a/YARG.Core/Song/Cache/CacheGroups/UnpackedCONGroup.cs b/YARG.Core/Song/Cache/CacheGroups/UnpackedCONGroup.cs index 737fe118a..244f6cf84 100644 --- a/YARG.Core/Song/Cache/CacheGroups/UnpackedCONGroup.cs +++ b/YARG.Core/Song/Cache/CacheGroups/UnpackedCONGroup.cs @@ -6,61 +6,23 @@ namespace YARG.Core.Song.Cache { - public sealed class UnpackedCONGroup : CONGroup, IDisposable + public sealed class UnpackedCONGroup : CONGroup { - public readonly AbridgedFileInfo DTA; - private FixedArray _fileData = FixedArray.Null; + public readonly Dictionary>> SongNodes = new(); - public override string Location { get; } - - public UnpackedCONGroup(string directory, FileInfo dta, string defaultPlaylist) - : base(defaultPlaylist) + public UnpackedCONGroup(in FixedArray data, Dictionary>> songNodes, string directory, in AbridgedFileInfo dta, string defaultPlaylist) + : base(in data, directory, in dta, defaultPlaylist) { - Location = directory; - DTA = new AbridgedFileInfo(dta); + SongNodes = songNodes; } - public bool LoadDTA(out YARGTextContainer container) + public override void ReadEntry(string nodeName, int index, RBProUpgrade upgrade, UnmanagedMemoryStream stream, CategoryCacheStrings strings) { - try - { - _fileData = FixedArray.Load(DTA.FullName); - return YARGDTAReader.TryCreate(_fileData, out container); - } - catch (Exception ex) - { - YargLogger.LogException(ex, $"Error while loading {DTA.FullName}"); - container = default; - return false; - } - } - - public override void ReadEntry(string nodeName, int index, Dictionary, RBProUpgrade)> upgrades, UnmanagedMemoryStream stream, CategoryCacheStrings strings) - { - var song = UnpackedRBCONEntry.TryLoadFromCache(Location, DTA, nodeName, upgrades, stream, strings); + var song = UnpackedRBCONEntry.TryLoadFromCache(Location, in Info, nodeName, upgrade, stream, strings); if (song != null) { AddEntry(nodeName, index, song); } } - - public override ReadOnlyMemory SerializeEntries(Dictionary nodes) - { - using MemoryStream ms = new(); - using BinaryWriter writer = new(ms); - - writer.Write(Location); - writer.Write(DTA.LastUpdatedTime.ToBinary()); - Serialize(writer, ref nodes); - return new ReadOnlyMemory(ms.GetBuffer(), 0, (int)ms.Length); - } - - public void Dispose() - { - if (_fileData.IsAllocated) - { - _fileData.Dispose(); - } - } } } diff --git a/YARG.Core/Song/Cache/CacheGroups/UpdateGroup.cs b/YARG.Core/Song/Cache/CacheGroups/UpdateGroup.cs index 578c11f6e..6ee432bec 100644 --- a/YARG.Core/Song/Cache/CacheGroups/UpdateGroup.cs +++ b/YARG.Core/Song/Cache/CacheGroups/UpdateGroup.cs @@ -6,87 +6,68 @@ namespace YARG.Core.Song.Cache { - public sealed class UpdateGroup : IModificationGroup, IDisposable + public sealed class UpdateGroup : IModificationGroup { - public readonly DirectoryInfo Directory; + public readonly string Directory; public readonly DateTime DTALastWrite; public readonly Dictionary Updates = new(); + public readonly FixedArray DTAData; - private readonly FixedArray _dtaData; - - public UpdateGroup(DirectoryInfo directory, DateTime dtaLastUpdate, in FixedArray dtaData) + public UpdateGroup(string directory, DateTime lastWrite, FixedArray data, Dictionary updates) { + DTAData = data; Directory = directory; - DTALastWrite = dtaLastUpdate; - _dtaData = dtaData; + DTALastWrite = lastWrite; + Updates = updates; } - public ReadOnlyMemory SerializeModifications() + public void SerializeModifications(MemoryStream stream) { - using MemoryStream ms = new(); - using BinaryWriter writer = new(ms); - - writer.Write(Directory.FullName); - writer.Write(DTALastWrite.ToBinary()); - writer.Write(Updates.Count); + stream.Write(Directory); + stream.Write(DTALastWrite.ToBinary(), Endianness.Little); + stream.Write(Updates.Count, Endianness.Little); foreach (var (name, update) in Updates) { - writer.Write(name); - update.Serialize(writer); + stream.Write(name); + update.Serialize(stream); } - return new ReadOnlyMemory(ms.GetBuffer(), 0, (int)ms.Length); - } - - public void Dispose() - { - _dtaData.Dispose(); } } public class SongUpdate { - private readonly List> _containers; - - public readonly string BaseDirectory; + public readonly List> Containers; public readonly AbridgedFileInfo? Midi; public readonly AbridgedFileInfo? Mogg; public readonly AbridgedFileInfo? Milo; public readonly AbridgedFileInfo? Image; - public YARGTextContainer[] Containers => _containers.ToArray(); - - internal SongUpdate(string directory, AbridgedFileInfo? midi, AbridgedFileInfo? mogg, AbridgedFileInfo? milo, AbridgedFileInfo? image) + internal SongUpdate(in AbridgedFileInfo? midi, in AbridgedFileInfo? mogg, in AbridgedFileInfo? milo, in AbridgedFileInfo? image) { - _containers = new(); - BaseDirectory = directory; + Containers = new(); Midi = midi; Mogg = mogg; Milo = milo; Image = image; } - public void Add(in YARGTextContainer container) + public void Serialize(MemoryStream stream) { - _containers.Add(container); - } - - public void Serialize(BinaryWriter writer) - { - WriteInfo(Midi, writer); - WriteInfo(Mogg, writer); - WriteInfo(Milo, writer); - WriteInfo(Image, writer); + WriteInfo(Midi, stream); + WriteInfo(Mogg, stream); + WriteInfo(Milo, stream); + WriteInfo(Image, stream); - static void WriteInfo(in AbridgedFileInfo? info, BinaryWriter writer) + static void WriteInfo(in AbridgedFileInfo? info, MemoryStream stream) { if (info != null) { - writer.Write(true); - writer.Write(info.Value.LastUpdatedTime.ToBinary()); + stream.Write(true); + stream.Write(info.Value.LastUpdatedTime.ToBinary(), Endianness.Little); } else { - writer.Write(false); + stream.Write(false); } } } @@ -117,19 +98,13 @@ public bool Validate(UnmanagedMemoryStream stream) static bool CheckInfo(in AbridgedFileInfo? info, UnmanagedMemoryStream stream) { - if (stream.ReadBoolean()) + if (!stream.ReadBoolean()) { - var lastWrite = DateTime.FromBinary(stream.Read(Endianness.Little)); - if (info == null || info.Value.LastUpdatedTime != lastWrite) - { - return false; - } + return info == null; } - else if (info != null) - { - return false; - } - return true; + + var lastWrite = DateTime.FromBinary(stream.Read(Endianness.Little)); + return info != null && info.Value.LastUpdatedTime == lastWrite; } } diff --git a/YARG.Core/Song/Cache/CacheGroups/UpgradeGroup.cs b/YARG.Core/Song/Cache/CacheGroups/UpgradeGroup.cs index 88c5efb79..af08e7b6f 100644 --- a/YARG.Core/Song/Cache/CacheGroups/UpgradeGroup.cs +++ b/YARG.Core/Song/Cache/CacheGroups/UpgradeGroup.cs @@ -1,49 +1,36 @@ using System; using System.Collections.Generic; using System.IO; +using YARG.Core.Extensions; using YARG.Core.IO; namespace YARG.Core.Song.Cache { - public interface IUpgradeGroup : IModificationGroup + public sealed class UpgradeGroup : IModificationGroup { - public Dictionary Upgrades { get; } - } - - public sealed class UpgradeGroup : IUpgradeGroup, IDisposable - { - private readonly string _directory; - private readonly DateTime _dtaLastUpdate; - private readonly FixedArray _dtaData; - - public Dictionary Upgrades { get; } = new(); + public readonly string Directory; + public readonly DateTime DTALastWrite; + public readonly Dictionary Container, UnpackedRBProUpgrade Upgrade)> Upgrades; + public readonly FixedArray DTAData; - public UpgradeGroup(string directory, DateTime dtaLastUpdate, in FixedArray dtaData) + public UpgradeGroup(string directory, DateTime lastWrite, in FixedArray data, Dictionary Node, UnpackedRBProUpgrade Upgrade)> upgrades) { - _directory = directory; - _dtaLastUpdate = dtaLastUpdate; - _dtaData = dtaData; + Directory = directory; + DTALastWrite = lastWrite; + Upgrades = upgrades; + DTAData = data; } - public ReadOnlyMemory SerializeModifications() + public void SerializeModifications(MemoryStream stream) { - using MemoryStream ms = new(); - using BinaryWriter writer = new(ms); - - writer.Write(_directory); - writer.Write(_dtaLastUpdate.ToBinary()); - writer.Write(Upgrades.Count); - foreach (var upgrade in Upgrades) + stream.Write(Directory); + stream.Write(DTALastWrite.ToBinary(), Endianness.Little); + stream.Write(Upgrades.Count, Endianness.Little); + foreach (var node in Upgrades) { - writer.Write(upgrade.Key); - upgrade.Value.WriteToCache(writer); + stream.Write(node.Key); + node.Value.Upgrade.WriteToCache(stream); } - return new ReadOnlyMemory(ms.GetBuffer(), 0, (int)ms.Length); - } - - public void Dispose() - { - _dtaData.Dispose(); } } } diff --git a/YARG.Core/Song/Cache/CacheHandler.Parallel.cs b/YARG.Core/Song/Cache/CacheHandler.Parallel.cs index 95e42dfd8..1f72302a3 100644 --- a/YARG.Core/Song/Cache/CacheHandler.Parallel.cs +++ b/YARG.Core/Song/Cache/CacheHandler.Parallel.cs @@ -30,11 +30,19 @@ protected override void FindNewEntries() var group = conGroups[i]; conActions[i] = () => { - if (group.LoadSongs(out var container)) + using var stream = group.Stream = new FileStream(group.Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 1); + Parallel.ForEach(group.SongNodes, node => { - ScanCONGroup(group, ref container, ScanPackedCONNode); - } - group.DisposeStreamAndSongDTA(); + for (int j = 0; j < node.Value.Count; ++j) + { + var container = node.Value[j]; + unsafe + { + ScanCONNode(group, node.Key, j, container, &PackedRBCONEntry.ProcessNewEntry); + } + } + }); + group.SongDTAData.Dispose(); }; } @@ -43,19 +51,22 @@ protected override void FindNewEntries() var group = extractedConGroups[i]; conActions[conGroups.Count + i] = () => { - if (group.LoadDTA(out var container)) + Parallel.ForEach(group.SongNodes, node => { - ScanCONGroup(group, ref container, ScanUnpackedCONNode); - } - group.Dispose(); + for (int j = 0; j < node.Value.Count; ++j) + { + var container = node.Value[j]; + unsafe + { + ScanCONNode(group, node.Key, j, container, &UnpackedRBCONEntry.ProcessNewEntry); + } + } + }); + group.SongDTAData.Dispose(); }; } Parallel.ForEach(conActions, action => action()); - foreach (var group in conGroups) - { - group.DisposeUpgradeDTA(); - } } protected override void TraverseDirectory(in FileCollection collection, IniGroup group, PlaylistTracker tracker) @@ -168,26 +179,6 @@ protected override void Deserialize_Quick(UnmanagedMemoryStream stream) } } - protected override void AddUpdate(string name, DateTime dtaLastWrite, SongUpdate update) - { - lock (updates) - { - if (!updates.TryGetValue(name, out var list)) - { - updates.Add(name, list = new()); - } - list.Add(dtaLastWrite, update); - } - } - - protected override void AddUpgrade(string name, in YARGTextContainer container, RBProUpgrade upgrade) - { - lock (upgrades) - { - upgrades[name] = new(container, upgrade); - } - } - protected override void AddPackedCONGroup(PackedCONGroup group) { lock (conGroups) @@ -228,6 +219,17 @@ protected override void AddCollectionToCache(in FileCollection collection) } } + protected override void AddCacheUpgrade(string name, RBProUpgrade upgrade) + { + lock (cacheUpgrades) + { + if (!cacheUpgrades.TryGetValue(name, out var currUpgrade) || currUpgrade.LastUpdatedTime < upgrade.LastUpdatedTime) + { + cacheUpgrades[name] = upgrade; + } + } + } + protected override void RemoveCONEntry(string shortname) { lock (conGroups) @@ -253,48 +255,26 @@ protected override void RemoveCONEntry(string shortname) } } - protected override bool CanAddUpgrade(string shortname, DateTime lastUpdated) + protected override CONModification GetModification(string name) { - lock (upgradeGroups) + CONModification modification; + lock (conModifications) { - return CanAddUpgrade(upgradeGroups, shortname, lastUpdated) ?? false; - } - } - - protected override bool CanAddUpgrade_CONInclusive(string shortname, DateTime lastUpdated) - { - lock (conGroups) - { - var result = CanAddUpgrade(conGroups, shortname, lastUpdated); - if (result != null) + if (!conModifications.TryGetValue(name, out modification)) { - return (bool) result; + conModifications.Add(name, modification = new CONModification()); } } - lock (upgradeGroups) - { - return CanAddUpgrade(upgradeGroups, shortname, lastUpdated) ?? false; - } - } - - protected override Dictionary> MapUpdateFiles(in FileCollection collection) - { - Dictionary> mapping = new(); - Parallel.ForEach(collection.SubDirectories, dir => + lock (modification) { - var infos = new Dictionary(); - foreach (var file in dir.Value.EnumerateFiles("*", SearchOption.AllDirectories)) + if (!modification.Processed) { - infos[file.Name] = file; + InitModification(modification, name); + modification.Processed = true; } - - lock (mapping) - { - mapping.Add(dir.Key, infos); - } - }); - return mapping; + } + return modification; } protected override bool FindOrMarkDirectory(string directory) @@ -337,33 +317,6 @@ protected override void AddInvalidSong(string name) } } - protected override void CleanupDuplicates() - { - Parallel.ForEach(duplicatesToRemove, entry => - { - lock (iniGroups) - { - if (TryRemove(iniGroups, entry)) - { - return; - } - } - - lock (conGroups) - { - if (TryRemove(conGroups, entry)) - { - return; - } - } - - lock (extractedConGroups) - { - TryRemove(extractedConGroups, entry); - } - }); - } - private sealed class ParallelExceptionTracker : Exception { private readonly object _lock = new object(); @@ -408,33 +361,6 @@ public override void GetObjectData(SerializationInfo info, StreamingContext cont } } - private void ScanCONGroup(TGroup group, ref YARGTextContainer container, Action> func) - where TGroup : CONGroup - { - try - { - var slices = new List<(string Name, int Index, YARGTextContainer Container)>(); - Dictionary indices = new(); - while (YARGDTAReader.StartNode(ref container)) - { - string name = YARGDTAReader.GetNameOfNode(ref container, true); - if (indices.TryGetValue(name, out int index)) - { - ++index; - } - indices[name] = index; - - slices.Add((name, index, container)); - YARGDTAReader.EndNode(ref container); - } - Parallel.ForEach(slices, slice => func(group, slice.Name, slice.Index, slice.Container)); - } - catch (Exception e) - { - YargLogger.LogException(e, $"Error while scanning CON group {group.Location}!"); - } - } - private readonly struct CacheEnumerable : IEnumerable { private readonly UnmanagedMemoryStream _stream; @@ -540,7 +466,7 @@ private void ReadPackedCONGroup(UnmanagedMemoryStream stream, CategoryCacheStrin var group = ReadCONGroupHeader(stream); if (group != null) { - ReadCONGroup(group, stream, strings, tracker); + ReadCONGroup(group, stream, strings, tracker); } } @@ -549,12 +475,13 @@ private void ReadUnpackedCONGroup(UnmanagedMemoryStream stream, CategoryCacheStr var group = ReadExtractedCONGroupHeader(stream); if (group != null) { - ReadCONGroup(group, stream, strings, tracker); + ReadCONGroup(group, stream, strings, tracker); } } - private void ReadCONGroup(TGroup group, UnmanagedMemoryStream stream, CategoryCacheStrings strings, ParallelExceptionTracker tracker) - where TGroup : CONGroup + private void ReadCONGroup(TGroup group, UnmanagedMemoryStream stream, CategoryCacheStrings strings, ParallelExceptionTracker tracker) + where TGroup : CONGroup + where TEntry : RBCONEntry { var enumerable = new CacheEnumerable<(string Name, int Index, UnmanagedMemoryStream Stream)?>(stream, tracker, () => { @@ -575,7 +502,8 @@ private void ReadCONGroup(TGroup group, UnmanagedMemoryStream stream, Ca // Error catching must be done per-thread try { - group.ReadEntry(value.Name, value.Index, upgrades, value.Stream, strings); + cacheUpgrades.TryGetValue(value.Name, out var upgrade); + group.ReadEntry(value.Name, value.Index, upgrade, value.Stream, strings); } catch (Exception ex) { @@ -608,10 +536,7 @@ private void QuickReadIniGroup(UnmanagedMemoryStream stream, CategoryCacheString private void QuickReadCONGroup(UnmanagedMemoryStream stream, CategoryCacheStrings strings, ParallelExceptionTracker tracker) { - var group = QuickReadCONGroupHeader(stream); - if (group == null) - return; - + var listings = QuickReadCONGroupHeader(stream); var enumerable = new CacheEnumerable<(string Name, UnmanagedMemoryStream Stream)>(stream, tracker, () => { string name = stream.ReadString(); @@ -626,7 +551,8 @@ private void QuickReadCONGroup(UnmanagedMemoryStream stream, CategoryCacheString { try { - AddEntry(PackedRBCONEntry.LoadFromCache_Quick(in group.ConFile, slice.Name, upgrades, slice.Stream, strings)); + cacheUpgrades.TryGetValue(slice.Name, out var upgrade); + AddEntry(PackedRBCONEntry.LoadFromCache_Quick(listings, slice.Name, upgrade, slice.Stream, strings)); } catch (Exception ex) { @@ -654,7 +580,8 @@ private void QuickReadExtractedCONGroup(UnmanagedMemoryStream stream, CategoryCa { try { - AddEntry(UnpackedRBCONEntry.LoadFromCache_Quick(directory, dta, slice.Name, upgrades, slice.Stream, strings)); + cacheUpgrades.TryGetValue(slice.Name, out var upgrade); + AddEntry(UnpackedRBCONEntry.LoadFromCache_Quick(directory, dta, upgrade, slice.Stream, strings)); } catch (Exception ex) { diff --git a/YARG.Core/Song/Cache/CacheHandler.Sequential.cs b/YARG.Core/Song/Cache/CacheHandler.Sequential.cs index b0416b2ed..2c1096939 100644 --- a/YARG.Core/Song/Cache/CacheHandler.Sequential.cs +++ b/YARG.Core/Song/Cache/CacheHandler.Sequential.cs @@ -23,25 +23,35 @@ protected override void FindNewEntries() foreach (var group in conGroups) { - if (group.LoadSongs(out var container)) + using var stream = group.Stream = new FileStream(group.Info.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 1); + foreach (var node in group.SongNodes) { - ScanCONGroup(group, ref container, ScanPackedCONNode); + for (int j = 0; j < node.Value.Count; ++j) + { + var container = node.Value[j]; + unsafe + { + ScanCONNode(group, node.Key, j, container, &PackedRBCONEntry.ProcessNewEntry); + } + } } - group.DisposeStreamAndSongDTA(); + group.SongDTAData.Dispose(); } foreach (var group in extractedConGroups) { - if (group.LoadDTA(out var container)) + foreach (var node in group.SongNodes) { - ScanCONGroup(group, ref container, ScanUnpackedCONNode); + for (int j = 0; j < node.Value.Count; ++j) + { + var container = node.Value[j]; + unsafe + { + ScanCONNode(group, node.Key, j, container, &UnpackedRBCONEntry.ProcessNewEntry); + } + } } - group.Dispose(); - } - - foreach (var group in conGroups) - { - group.DisposeUpgradeDTA(); + group.SongDTAData.Dispose(); } } @@ -105,20 +115,6 @@ protected override void Deserialize_Quick(UnmanagedMemoryStream stream) RunEntryTasks(stream, strings, QuickReadExtractedCONGroup); } - protected override void AddUpdate(string name, DateTime dtaLastWrite, SongUpdate update) - { - if (!updates.TryGetValue(name, out var list)) - { - updates.Add(name, list = new()); - } - list.Add(dtaLastWrite, update); - } - - protected override void AddUpgrade(string name, in YARGTextContainer container, RBProUpgrade upgrade) - { - upgrades[name] = new(container, upgrade); - } - protected override void AddPackedCONGroup(PackedCONGroup group) { conGroups.Add(group); @@ -144,6 +140,14 @@ protected override void AddCollectionToCache(in FileCollection collection) collectionCache.Add(collection.Directory.FullName, collection); } + protected override void AddCacheUpgrade(string name, RBProUpgrade upgrade) + { + if (!cacheUpgrades.TryGetValue(name, out var currUpgrade) || currUpgrade.LastUpdatedTime < upgrade.LastUpdatedTime) + { + cacheUpgrades[name] = upgrade; + } + } + protected override void RemoveCONEntry(string shortname) { foreach (var group in conGroups) @@ -163,34 +167,14 @@ protected override void RemoveCONEntry(string shortname) } } - protected override bool CanAddUpgrade(string shortname, DateTime lastUpdated) + protected override CONModification GetModification(string name) { - return CanAddUpgrade(upgradeGroups, shortname, lastUpdated) ?? false; - } - - protected override bool CanAddUpgrade_CONInclusive(string shortname, DateTime lastUpdated) - { - var result = CanAddUpgrade(conGroups, shortname, lastUpdated); - if (result != null) + if (!conModifications.TryGetValue(name, out var modification)) { - return (bool) result; + conModifications.Add(name, modification = new CONModification()); + InitModification(modification, name); } - return CanAddUpgrade(upgradeGroups, shortname, lastUpdated) ?? false; - } - - protected override Dictionary> MapUpdateFiles(in FileCollection collection) - { - Dictionary> mapping = new(); - foreach (var dir in collection.SubDirectories) - { - var infos = new Dictionary(); - foreach (var file in dir.Value.EnumerateFiles("*", SearchOption.AllDirectories)) - { - infos[file.Name] = file; - } - mapping[dir.Key] = infos; - } - return mapping; + return modification; } protected override PackedCONGroup? FindCONGroup(string filename) @@ -198,49 +182,6 @@ protected override Dictionary> MapUpdateFil return conGroups.Find(node => node.Location == filename); } - protected override void CleanupDuplicates() - { - foreach (var entry in duplicatesToRemove) - { - if (TryRemove(iniGroups, entry)) - { - continue; - } - - if (TryRemove(conGroups, entry)) - { - continue; - } - - TryRemove(extractedConGroups, entry); - } - } - - private void ScanCONGroup(TGroup group, ref YARGTextContainer container, Action> func) - where TGroup : CONGroup - { - try - { - Dictionary indices = new(); - while (YARGDTAReader.StartNode(ref container)) - { - string name = YARGDTAReader.GetNameOfNode(ref container, true); - if (indices.TryGetValue(name, out int index)) - { - ++index; - } - indices[name] = index; - - func(group, name, index, container); - YARGDTAReader.EndNode(ref container); - } - } - catch (Exception e) - { - YargLogger.LogException(e, $"Error while scanning CON group {group.Location}!"); - } - } - private void ReadIniGroup(UnmanagedMemoryStream stream, CategoryCacheStrings strings) { string directory = stream.ReadString(); @@ -262,18 +203,52 @@ private void ReadIniGroup(UnmanagedMemoryStream stream, CategoryCacheStrings str private void ReadPackedCONGroup(UnmanagedMemoryStream stream, CategoryCacheStrings strings) { var group = ReadCONGroupHeader(stream); - if (group != null) + if (group == null) + { + return; + } + + int count = stream.Read(Endianness.Little); + for (int i = 0; i < count; ++i) { - ReadCONGroup(stream, (string name, int index, UnmanagedMemoryStream slice) => group.ReadEntry(name, index, upgrades, slice, strings)); + string name = stream.ReadString(); + int index = stream.Read(Endianness.Little); + int length = stream.Read(Endianness.Little); + if (invalidSongsInCache.Contains(name)) + { + stream.Position += length; + continue; + } + + var entryStream = stream.Slice(length); + cacheUpgrades.TryGetValue(name, out var upgrade); + group.ReadEntry(name, index, upgrade, entryStream, strings); } } private void ReadUnpackedCONGroup(UnmanagedMemoryStream stream, CategoryCacheStrings strings) { var group = ReadExtractedCONGroupHeader(stream); - if (group != null) + if (group == null) { - ReadCONGroup(stream, (string name, int index, UnmanagedMemoryStream slice) => group.ReadEntry(name, index, upgrades, slice, strings)); + return; + } + + int count = stream.Read(Endianness.Little); + for (int i = 0; i < count; ++i) + { + string name = stream.ReadString(); + int index = stream.Read(Endianness.Little); + int length = stream.Read(Endianness.Little); + if (invalidSongsInCache.Contains(name)) + { + stream.Position += length; + continue; + } + + var entryStream = stream.Slice(length); + cacheUpgrades.TryGetValue(name, out var upgrade); + group.ReadEntry(name, index, upgrade, entryStream, strings); } } @@ -291,10 +266,7 @@ private void QuickReadIniGroup(UnmanagedMemoryStream stream, CategoryCacheString private void QuickReadCONGroup(UnmanagedMemoryStream stream, CategoryCacheStrings strings) { - var group = QuickReadCONGroupHeader(stream); - if (group == null) - return; - + var listings = QuickReadCONGroupHeader(stream); int count = stream.Read(Endianness.Little); for (int i = 0; i < count; ++i) { @@ -304,7 +276,8 @@ private void QuickReadCONGroup(UnmanagedMemoryStream stream, CategoryCacheString int length = stream.Read(Endianness.Little); var slice = stream.Slice(length); - AddEntry(PackedRBCONEntry.LoadFromCache_Quick(in group.ConFile, name, upgrades, slice, strings)); + cacheUpgrades.TryGetValue(name, out var upgrade); + AddEntry(PackedRBCONEntry.LoadFromCache_Quick(listings, name, upgrade, slice, strings)); } } @@ -322,7 +295,8 @@ private void QuickReadExtractedCONGroup(UnmanagedMemoryStream stream, CategoryCa int length = stream.Read(Endianness.Little); var slice = stream.Slice(length); - AddEntry(UnpackedRBCONEntry.LoadFromCache_Quick(directory, dta, name, upgrades, slice, strings)); + cacheUpgrades.TryGetValue(name, out var upgrade); + AddEntry(UnpackedRBCONEntry.LoadFromCache_Quick(directory, dta, upgrade, slice, strings)); } } diff --git a/YARG.Core/Song/Cache/CacheHandler.cs b/YARG.Core/Song/Cache/CacheHandler.cs index 23d2d00fa..ab989aeef 100644 --- a/YARG.Core/Song/Cache/CacheHandler.cs +++ b/YARG.Core/Song/Cache/CacheHandler.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using YARG.Core.Audio; using YARG.Core.Extensions; using YARG.Core.IO; @@ -34,12 +33,12 @@ public abstract class CacheHandler /// Format is YY_MM_DD_RR: Y = year, M = month, D = day, R = revision (reset across dates, only increment /// if multiple cache version changes happen in a single day). /// - public const int CACHE_VERSION = 24_09_24_01; + public const int CACHE_VERSION = 24_09_28_01; public static ScanProgressTracker Progress => _progress; private static ScanProgressTracker _progress; - public static SongCache RunScan(bool fast, string cacheLocation, string badSongsLocation, bool multithreading, bool allowDuplicates, bool fullDirectoryPlaylists, List baseDirectories) + public static SongCache RunScan(bool tryQuickScan, string cacheLocation, string badSongsLocation, bool multithreading, bool allowDuplicates, bool fullDirectoryPlaylists, List baseDirectories) { CacheHandler handler = multithreading ? new ParallelCacheHandler(baseDirectories, allowDuplicates, fullDirectoryPlaylists) @@ -48,23 +47,15 @@ public static SongCache RunScan(bool fast, string cacheLocation, string badSongs GlobalAudioHandler.LogMixerStatus = false; try { - if (!fast || !QuickScan(handler, cacheLocation)) - FullScan(handler, !fast, cacheLocation, badSongsLocation); + if (!tryQuickScan || !QuickScan(handler, cacheLocation)) + { + FullScan(handler, !tryQuickScan, cacheLocation, badSongsLocation); + } } catch (Exception ex) { YargLogger.LogException(ex, "Unknown error while running song scan!"); } - - foreach (var group in handler.upgradeGroups) - { - group.Dispose(); - } - - foreach (var group in handler.updateGroups) - { - group.Dispose(); - } GlobalAudioHandler.LogMixerStatus = true; return handler.cache; } @@ -121,6 +112,8 @@ private static void FullScan(CacheHandler handler, bool loadCache, string cacheL _progress.Stage = ScanStage.LoadingSongs; handler.FindNewEntries(); + handler.DisposeLeftoverData(); + _progress.Stage = ScanStage.CleaningDuplicates; handler.CleanupDuplicates(); @@ -165,9 +158,8 @@ private static void FullScan(CacheHandler handler, bool loadCache, string cacheL protected readonly List upgradeGroups = new(); protected readonly List conGroups = new(); protected readonly List extractedConGroups = new(); - - protected readonly Dictionary> updates = new(); - protected readonly Dictionary, RBProUpgrade)> upgrades = new(); + protected readonly Dictionary conModifications = new(); + protected readonly Dictionary cacheUpgrades = new(); protected readonly HashSet preScannedDirectories = new(); protected readonly HashSet preScannedFiles = new(); @@ -198,16 +190,12 @@ protected CacheHandler(List baseDirectories, bool allowDuplicates, bool } protected abstract void SortEntries(); - protected abstract void AddUpdate(string name, DateTime dtaLastWrite, SongUpdate update); - protected abstract void AddUpgrade(string name, in YARGTextContainer container, RBProUpgrade upgrade); protected abstract void AddPackedCONGroup(PackedCONGroup group); protected abstract void AddUnpackedCONGroup(UnpackedCONGroup group); protected abstract void AddUpdateGroup(UpdateGroup group); protected abstract void AddUpgradeGroup(UpgradeGroup group); protected abstract void RemoveCONEntry(string shortname); - protected abstract bool CanAddUpgrade(string shortname, DateTime lastUpdated); - protected abstract bool CanAddUpgrade_CONInclusive(string shortname, DateTime lastUpdated); - protected abstract Dictionary> MapUpdateFiles(in FileCollection collection); + protected abstract CONModification GetModification(string name); protected abstract void FindNewEntries(); protected abstract void TraverseDirectory(in FileCollection collection, IniGroup group, PlaylistTracker tracker); @@ -216,8 +204,7 @@ protected CacheHandler(List baseDirectories, bool allowDuplicates, bool protected abstract void Deserialize_Quick(UnmanagedMemoryStream stream); protected abstract void AddCollectionToCache(in FileCollection collection); protected abstract PackedCONGroup? FindCONGroup(string filename); - - protected abstract void CleanupDuplicates(); + protected abstract void AddCacheUpgrade(string name, RBProUpgrade upgrade); protected virtual bool FindOrMarkDirectory(string directory) { @@ -228,6 +215,7 @@ protected virtual bool FindOrMarkDirectory(string directory) _progress.NumScannedDirectories++; return true; } + protected virtual bool FindOrMarkFile(string file) { return preScannedFiles.Add(file); @@ -284,6 +272,53 @@ protected virtual void AddInvalidSong(string name) return null; } + private void DisposeLeftoverData() + { + foreach (var group in conGroups) + { + group.UpgradeDTAData.Dispose(); + } + + foreach (var group in upgradeGroups) + { + group.DTAData.Dispose(); + } + + foreach (var group in updateGroups) + { + group.DTAData.Dispose(); + } + } + + private void CleanupDuplicates() + { + foreach (var entry in duplicatesToRemove) + { + if (!TryRemove(iniGroups, entry) && !TryRemove(conGroups, entry)) + { + TryRemove(extractedConGroups, entry); + } + } + } + + private static bool TryRemove(List groups, SongEntry entry) + where TGroup : ICacheGroup + { + for (int i = 0; i < groups.Count; ++i) + { + var group = groups[i]; + if (group.TryRemoveEntry(entry)) + { + if (group.Count == 0) + { + groups.RemoveAt(i); + } + return true; + } + } + return false; + } + private void WriteBadSongs(string badSongsLocation) { using var stream = new FileStream(badSongsLocation, FileMode.Create, FileAccess.Write); @@ -325,15 +360,9 @@ private void WriteBadSongs(string badSongsLocation) case ScanResult.UnsupportedEncryption: writer.WriteLine("Mogg file uses unsupported encryption"); break; - case ScanResult.MissingMidi: + case ScanResult.MissingCONMidi: writer.WriteLine("Midi file queried for found missing"); break; - case ScanResult.MissingUpdateMidi: - writer.WriteLine("Update Midi file queried for found missing"); - break; - case ScanResult.MissingUpgradeMidi: - writer.WriteLine("Upgrade Midi file queried for found missing"); - break; case ScanResult.IniNotDownloaded: writer.WriteLine("Ini file not fully downloaded - try again once it completes"); break; @@ -366,211 +395,6 @@ private void WriteBadSongs(string badSongsLocation) writer.WriteLine(); } } - - private UpgradeGroup? CreateUpgradeGroup(in FileCollection collection, FileInfo dta, bool removeEntries) - { - var fileData = FixedArray.Null; - YARGTextContainer container; - try - { - fileData = FixedArray.Load(dta.FullName); - if (!YARGDTAReader.TryCreate(fileData, out container)) - { - fileData.Dispose(); - return null; - } - } - catch (Exception ex) - { - YargLogger.LogException(ex, $"Error while loading {dta.FullName}"); - fileData.Dispose(); - return null; - } - - var group = new UpgradeGroup(collection.Directory.FullName, dta.LastWriteTime, fileData); - try - { - while (YARGDTAReader.StartNode(ref container)) - { - string name = YARGDTAReader.GetNameOfNode(ref container, true); - if (collection.Subfiles.TryGetValue($"{name.ToLower()}_plus.mid", out var info) - && CanAddUpgrade(name, info.LastWriteTime)) - { - var abridged = new AbridgedFileInfo(info, false); - var upgrade = new UnpackedRBProUpgrade(abridged); - group.Upgrades[name] = upgrade; - AddUpgrade(name, container, upgrade); - - if (removeEntries) - { - RemoveCONEntry(name); - } - } - YARGDTAReader.EndNode(ref container); - } - } - catch (Exception ex) - { - YargLogger.LogException(ex, $"Error while scanning CON upgrade folder {collection.Directory.FullName}!"); - } - - if (group.Upgrades.Count == 0) - { - YargLogger.LogFormatWarning("{0} .dta file possibly malformed", collection.Directory.FullName); - return null; - } - return group; - } - - private UpdateGroup? CreateUpdateGroup(in FileCollection collection, FileInfo dta, bool removeEntries) - { - var fileData = FixedArray.Null; - YARGTextContainer container; - try - { - fileData = FixedArray.Load(dta.FullName); - if (!YARGDTAReader.TryCreate(fileData, out container)) - { - fileData.Dispose(); - return null; - } - } - catch (Exception ex) - { - YargLogger.LogException(ex, $"Error while loading {dta.FullName}"); - fileData.Dispose(); - return null; - } - - var group = new UpdateGroup(collection.Directory, dta.LastWriteTime, fileData); - try - { - var mapping = MapUpdateFiles(in collection); - while (YARGDTAReader.StartNode(ref container)) - { - string name = YARGDTAReader.GetNameOfNode(ref container, true); - if (!group.Updates.TryGetValue(name, out var update)) - { - AbridgedFileInfo? midi = null; - AbridgedFileInfo? mogg = null; - AbridgedFileInfo? milo = null; - AbridgedFileInfo? image = null; - - string subname = name.ToLowerInvariant(); - if (mapping.TryGetValue(subname, out var files)) - { - if (files.TryGetValue(subname + "_update.mid", out var file)) - { - midi = new AbridgedFileInfo(file, false); - } - if (files.TryGetValue(subname + "_update.mogg", out file)) - { - mogg = new AbridgedFileInfo(file, false); - } - if (files.TryGetValue(subname + ".milo_xbox", out file)) - { - milo = new AbridgedFileInfo(file, false); - } - if (files.TryGetValue(subname + "_keep.png_xbox", out file)) - { - image = new AbridgedFileInfo(file, false); - } - } - - group.Updates.Add(name, update = new SongUpdate(collection.Directory.FullName, midi, mogg, milo, image)); - AddUpdate(name, dta.LastWriteTime, update); - if (removeEntries) - { - RemoveCONEntry(name); - } - } - update.Add(in container); - YARGDTAReader.EndNode(ref container); - } - - } - catch (Exception ex) - { - YargLogger.LogException(ex, $"Error while scanning CON update folder {collection.Directory.FullName}!"); - } - - if (group.Updates.Count == 0) - { - YargLogger.LogFormatWarning("{0} .dta file possibly malformed", collection.Directory.FullName); - fileData.Dispose(); - return null; - } - return group; - } - - private bool TryParseUpgrades(string filename, PackedCONGroup group) - { - if (!group.LoadUpgrades(out var container)) - { - return false; - } - - try - { - while (YARGDTAReader.StartNode(ref container)) - { - string name = YARGDTAReader.GetNameOfNode(ref container, true); - if (group.ConFile.TryGetListing($"songs_upgrades/{name}_plus.mid", out var listing) - && CanAddUpgrade_CONInclusive(name, listing.LastWrite)) - { - var upgrade = new PackedRBProUpgrade(listing, listing.LastWrite); - group.Upgrades[name] = upgrade; - AddUpgrade(name, container, upgrade); - RemoveCONEntry(name); - } - - YARGDTAReader.EndNode(ref container); - } - } - catch (Exception ex) - { - YargLogger.LogException(ex, $"Error while scanning CON upgrades - {filename}!"); - } - return group.Upgrades.Count > 0; - } - - protected static bool? CanAddUpgrade(List groups, string shortname, DateTime lastUpdated) - where TGroup : IUpgradeGroup - { - foreach (var group in groups) - { - var upgrades = group.Upgrades; - if (upgrades.TryGetValue(shortname, out var currUpgrade)) - { - if (currUpgrade.LastUpdatedTime >= lastUpdated) - { - return false; - } - upgrades.Remove(shortname); - return true; - } - } - return null; - } - - protected static bool TryRemove(List groups, SongEntry entry) - where TGroup : ICacheGroup - where TEntry : SongEntry - { - for (int i = 0; i < groups.Count; ++i) - { - var group = groups[i]; - if (group.TryRemoveEntry(entry)) - { - if (group.Count == 0) - { - groups.RemoveAt(i); - } - return true; - } - } - return false; - } #endregion #region Scanning @@ -632,12 +456,15 @@ protected void ScanDirectory(DirectoryInfo directory, IniGroup group, PlaylistTr { if (collection.Subfiles.TryGetValue(SONGUPDATES_DTA, out var dta)) { - var updateGroup = CreateUpdateGroup(in collection, dta, true); + var updateGroup = CreateUpdateGroup(in collection, dta); if (updateGroup != null) { - AddUpdateGroup(updateGroup); - return; + foreach (var node in updateGroup.Updates) + { + RemoveCONEntry(node.Key); + } } + return; } break; } @@ -645,12 +472,15 @@ protected void ScanDirectory(DirectoryInfo directory, IniGroup group, PlaylistTr { if (collection.Subfiles.TryGetValue(SONGUPGRADES_DTA, out var dta)) { - var upgradeGroup = CreateUpgradeGroup(in collection, dta, true); + var upgradeGroup = CreateUpgradeGroup(in collection, dta); if (upgradeGroup != null) { - AddUpgradeGroup(upgradeGroup); - return; + foreach (var node in upgradeGroup.Upgrades) + { + RemoveCONEntry(node.Key); + } } + return; } break; } @@ -658,13 +488,13 @@ protected void ScanDirectory(DirectoryInfo directory, IniGroup group, PlaylistTr { if (collection.Subfiles.TryGetValue(SONGS_DTA, out var dta)) { - var exConGroup = new UnpackedCONGroup(directory.FullName, dta, tracker.Playlist); - AddUnpackedCONGroup(exConGroup); + var _ = CreateUnpackedCONGroup(directory.FullName, dta, tracker.Playlist); return; } break; } } + TraverseDirectory(collection, group, tracker.Append(directory.Name)); if (collection.ContainedDupes) { @@ -706,12 +536,13 @@ protected void ScanFile(FileInfo info, IniGroup group, in PlaylistTracker tracke } else { - var confile = CONFile.TryParseListings(abridged); - if (confile != null) + var conGroup = CreateCONGroup(in abridged, tracker.Playlist); + if (conGroup != null) { - var conGroup = new PackedCONGroup(confile.Value, abridged, tracker.Playlist); - TryParseUpgrades(info.FullName, conGroup); - AddPackedCONGroup(conGroup); + foreach (var node in conGroup.Upgrades) + { + RemoveCONEntry(node.Key); + } } } } @@ -727,20 +558,28 @@ protected void ScanFile(FileInfo info, IniGroup group, in PlaylistTracker tracke } } - protected void ScanPackedCONNode(PackedCONGroup group, string name, int index, YARGTextContainer node) + protected unsafe void ScanCONNode(TGroup group, string name, int index, in YARGTextContainer node, delegate* func) + where TGroup : CONGroup + where TEntry : RBCONEntry { if (group.TryGetEntry(name, index, out var entry)) { if (!AddEntry(entry!)) + { group.RemoveEntry(name, index); + } } else { - var song = PackedRBCONEntry.ProcessNewEntry(group, name, node, updates, upgrades); + var dtaEntry = new DTAEntry(name, in node); + var modification = GetModification(name); + var song = func(group, name, dtaEntry, modification); if (song.Item2 != null) { if (AddEntry(song.Item2)) + { group.AddEntry(name, index, song.Item2); + } } else { @@ -749,41 +588,62 @@ protected void ScanPackedCONNode(PackedCONGroup group, string name, int index, Y } } - protected void ScanUnpackedCONNode(UnpackedCONGroup group, string name, int index, YARGTextContainer node) + protected void InitModification(CONModification modification, string name) { - if (group.TryGetEntry(name, index, out var entry)) + var datetime = default(DateTime); + foreach (var group in updateGroups) { - if (!AddEntry(entry!)) - group.RemoveEntry(name, index); + if (group.Updates.TryGetValue(name, out var update)) + { + if (modification.UpdateDTA == null || datetime < group.DTALastWrite) + { + modification.UpdateDTA = new DTAEntry(update.Containers[0].Encoding); + foreach (var container in update.Containers) + { + modification.UpdateDTA.LoadData(name, container); + } + modification.Midi = update.Midi; + modification.Mogg = update.Mogg; + modification.Milo = update.Milo; + modification.Image = update.Image; + datetime = group.DTALastWrite; + } + } } - else + + foreach (var group in upgradeGroups) { - var song = UnpackedRBCONEntry.ProcessNewEntry(group, name, in node, updates, upgrades); - if (song.Item2 != null) + if (group.Upgrades.TryGetValue(name, out var node) && node.Upgrade != null) { - if (AddEntry(song.Item2)) - group.AddEntry(name, index, song.Item2); + if (modification.UpgradeDTA == null || datetime < group.DTALastWrite) + { + modification.UpgradeNode = node.Upgrade; + modification.UpgradeDTA = new DTAEntry(name, in node.Container); + datetime = group.DTALastWrite; + } } - else + } + + foreach (var group in conGroups) + { + if (group.Upgrades.TryGetValue(name, out var node) && node.Upgrade != null) { - AddToBadSongs(group.Location + $" - Node {name}", song.Item1); + if (modification.UpgradeDTA == null || datetime < group.Info.LastUpdatedTime) + { + modification.UpgradeNode = node.Upgrade; + modification.UpgradeDTA = new DTAEntry(name, in node.Container); + datetime = group.Info.LastUpdatedTime; + } } } } - private static readonly (string Name, ChartType Type)[] ChartTypes = - { - ("notes.mid", ChartType.Mid), - ("notes.midi", ChartType.Midi), - ("notes.chart", ChartType.Chart) - }; - private bool ScanIniEntry(in FileCollection collection, IniGroup group, string defaultPlaylist) { int i = collection.Subfiles.TryGetValue("song.ini", out var ini) ? 0 : 2; while (i < 3) { - if (!collection.Subfiles.TryGetValue(ChartTypes[i].Name, out var chart)) + if (!collection.Subfiles.TryGetValue(IniSubEntry.CHART_FILE_TYPES[i].Filename, out var chart)) { ++i; continue; @@ -797,8 +657,7 @@ private bool ScanIniEntry(in FileCollection collection, IniGroup group, string d try { - var node = new IniChartNode(chart, ChartTypes[i].Type); - var entry = UnpackedIniEntry.ProcessNewEntry(collection.Directory.FullName, in node, ini, defaultPlaylist); + var entry = UnpackedIniEntry.ProcessNewEntry(collection.Directory.FullName, chart, IniSubEntry.CHART_FILE_TYPES[i].Format, ini, defaultPlaylist); if (entry.Item2 == null) { AddToBadSongs(chart.FullName, entry.Item1); @@ -828,7 +687,7 @@ private void ScanSngFile(SngFile sngFile, IniGroup group, string defaultPlaylist int i = sngFile.Metadata.Count > 0 ? 0 : 2; while (i < 3) { - if (!sngFile.TryGetValue(ChartTypes[i].Name, out var chart)) + if (!sngFile.TryGetValue(IniSubEntry.CHART_FILE_TYPES[i].Filename, out var chart)) { ++i; continue; @@ -842,8 +701,7 @@ private void ScanSngFile(SngFile sngFile, IniGroup group, string defaultPlaylist try { - var node = new IniChartNode(chart, ChartTypes[i].Type); - var entry = SngEntry.ProcessNewEntry(sngFile, in node, defaultPlaylist); + var entry = SngEntry.ProcessNewEntry(sngFile, chart, IniSubEntry.CHART_FILE_TYPES[i].Format, defaultPlaylist); if (entry.Item2 == null) { AddToBadSongs(sngFile.Info.FullName, entry.Item1); @@ -905,20 +763,20 @@ private static FixedArray LoadCacheToMemory(string cacheLocation, bool ful private void Serialize(string cacheLocation) { - using var writer = new BinaryWriter(new FileStream(cacheLocation, FileMode.Create, FileAccess.Write)); + using var filestream = new FileStream(cacheLocation, FileMode.Create, FileAccess.Write); Dictionary nodes = new(); - writer.Write(CACHE_VERSION); - writer.Write(fullDirectoryPlaylists); + filestream.Write(CACHE_VERSION, Endianness.Little); + filestream.Write(fullDirectoryPlaylists); - CategoryWriter.WriteToCache(writer, cache.Titles, SongAttribute.Name, ref nodes); - CategoryWriter.WriteToCache(writer, cache.Artists, SongAttribute.Artist, ref nodes); - CategoryWriter.WriteToCache(writer, cache.Albums, SongAttribute.Album, ref nodes); - CategoryWriter.WriteToCache(writer, cache.Genres, SongAttribute.Genre, ref nodes); - CategoryWriter.WriteToCache(writer, cache.Years, SongAttribute.Year, ref nodes); - CategoryWriter.WriteToCache(writer, cache.Charters, SongAttribute.Charter, ref nodes); - CategoryWriter.WriteToCache(writer, cache.Playlists, SongAttribute.Playlist, ref nodes); - CategoryWriter.WriteToCache(writer, cache.Sources, SongAttribute.Source, ref nodes); + CategoryWriter.WriteToCache(filestream, cache.Titles, SongAttribute.Name, ref nodes); + CategoryWriter.WriteToCache(filestream, cache.Artists, SongAttribute.Artist, ref nodes); + CategoryWriter.WriteToCache(filestream, cache.Albums, SongAttribute.Album, ref nodes); + CategoryWriter.WriteToCache(filestream, cache.Genres, SongAttribute.Genre, ref nodes); + CategoryWriter.WriteToCache(filestream, cache.Years, SongAttribute.Year, ref nodes); + CategoryWriter.WriteToCache(filestream, cache.Charters, SongAttribute.Charter, ref nodes); + CategoryWriter.WriteToCache(filestream, cache.Playlists, SongAttribute.Playlist, ref nodes); + CategoryWriter.WriteToCache(filestream, cache.Sources, SongAttribute.Source, ref nodes); List upgradeCons = new(); List entryCons = new(); @@ -931,12 +789,12 @@ private void Serialize(string cacheLocation) entryCons.Add(group); } - ICacheGroup.SerializeGroups(iniGroups, writer, nodes); - IModificationGroup.SerializeGroups(updateGroups, writer); - IModificationGroup.SerializeGroups(upgradeGroups, writer); - IModificationGroup.SerializeGroups(upgradeCons, writer); - ICacheGroup.SerializeGroups(entryCons, writer, nodes); - ICacheGroup.SerializeGroups(extractedConGroups, writer, nodes); + ICacheGroup.SerializeGroups(iniGroups, filestream, nodes); + IModificationGroup.SerializeGroups(updateGroups, filestream); + IModificationGroup.SerializeGroups(upgradeGroups, filestream); + IModificationGroup.SerializeGroups(upgradeCons, filestream); + ICacheGroup.SerializeGroups(entryCons, filestream, nodes); + ICacheGroup.SerializeGroups(extractedConGroups, filestream, nodes); } protected void ReadIniEntry(IniGroup group, string directory, UnmanagedMemoryStream stream, CategoryCacheStrings strings) @@ -998,37 +856,35 @@ protected void ReadUpdateDirectory(UnmanagedMemoryStream stream) goto Invalidate; } - var group = CreateUpdateGroup(collection, dta, false); - if (group == null) - { - goto Invalidate; - } - - AddUpdateGroup(group); FindOrMarkDirectory(directory); - if (group.DTALastWrite != dtaLastWritten) - { - goto Invalidate; - } - - for (int i = 0; i < count; i++) + var group = CreateUpdateGroup(in collection, dta); + if (group != null && group.DTALastWrite == dtaLastWritten) { - string name = stream.ReadString(); - if (group.Updates.TryGetValue(name, out var update)) + var updates = new Dictionary(group.Updates); + for (int i = 0; i < count; i++) { - if (!update.Validate(stream)) + string name = stream.ReadString(); + if (updates.Remove(name, out var update)) + { + if (!update.Validate(stream)) + { + AddInvalidSong(name); + } + } + else { AddInvalidSong(name); + SongUpdate.SkipRead(stream); } } - else + + foreach (var leftover in updates.Keys) { - AddInvalidSong(name); - SongUpdate.SkipRead(stream); + AddInvalidSong(leftover); } + return; } - return; Invalidate: for (int i = 0; i < count; i++) @@ -1065,26 +921,12 @@ protected void ReadUpgradeDirectory(UnmanagedMemoryStream stream) FindOrMarkDirectory(directory); - var group = CreateUpgradeGroup(in collection, dta, false); - if (group == null) - { - goto Invalidate; - } - - AddUpgradeGroup(group); - if (dta.LastWriteTime != dtaLastWritten) - { - goto Invalidate; - } - - for (int i = 0; i < count; i++) + var group = CreateUpgradeGroup(in collection, dta); + if (group != null && dta.LastWriteTime == dtaLastWritten) { - string name = stream.ReadString(); - var lastUpdated = DateTime.FromBinary(stream.Read(Endianness.Little)); - if (!group.Upgrades.TryGetValue(name, out var upgrade) || upgrade!.LastUpdatedTime != lastUpdated) - AddInvalidSong(name); + ValidateUpgrades(group.Upgrades, count, stream); + return; } - return; Invalidate: for (int i = 0; i < count; i++) @@ -1098,38 +940,19 @@ protected void ReadUpgradeCON(UnmanagedMemoryStream stream) { string filename = stream.ReadString(); var conLastUpdated = DateTime.FromBinary(stream.Read(Endianness.Little)); - var dtaLastWritten = DateTime.FromBinary(stream.Read(Endianness.Little)); int count = stream.Read(Endianness.Little); var baseGroup = GetBaseIniGroup(filename); - if (baseGroup != null) + if (baseGroup == null) { - // Make playlist as the group is only made once - string playlist = ConstructPlaylist(filename, baseGroup.Directory); - var group = CreateCONGroup(filename, playlist); - if (group == null) - { - goto Invalidate; - } - - AddPackedCONGroup(group); + goto Invalidate; + } - if (TryParseUpgrades(filename, group) && group.UpgradeDta!.LastWrite == dtaLastWritten) - { - if (group.Info.LastUpdatedTime != conLastUpdated) - { - for (int i = 0; i < count; i++) - { - string name = stream.ReadString(); - var lastWrite = DateTime.FromBinary(stream.Read(Endianness.Little)); - if (group.Upgrades[name].LastUpdatedTime != lastWrite) - { - AddInvalidSong(name); - } - } - } - return; - } + var group = CreateCONGroup(filename, baseGroup.Directory); + if (group != null && group.Info.LastUpdatedTime == conLastUpdated) + { + ValidateUpgrades(group.Upgrades, count, stream); + return; } Invalidate: @@ -1140,44 +963,48 @@ protected void ReadUpgradeCON(UnmanagedMemoryStream stream) } } - protected PackedCONGroup? ReadCONGroupHeader(UnmanagedMemoryStream stream) + private void ValidateUpgrades(Dictionary Container, TUpgrade Upgrade)> groupUpgrades, int count, UnmanagedMemoryStream stream) + where TUpgrade : RBProUpgrade { - string filename = stream.ReadString(); - var baseGroup = GetBaseIniGroup(filename); - if (baseGroup == null) + var upgrades = new Dictionary(); + upgrades.EnsureCapacity(groupUpgrades.Count); + for (int i = 0; i < count; i++) { - return null; + string name = stream.ReadString(); + var lastUpdated = DateTime.FromBinary(stream.Read(Endianness.Little)); + upgrades.Add(name, lastUpdated); } - var dtaLastWrite = DateTime.FromBinary(stream.Read(Endianness.Little)); - var group = FindCONGroup(filename); - if (group == null) + foreach (var node in groupUpgrades) { - var info = new FileInfo(filename); - if (!info.Exists) + if (upgrades.Remove(node.Key, out var dateTime) && node.Value.Upgrade.LastUpdatedTime == dateTime) { - return null; + AddCacheUpgrade(node.Key, node.Value.Upgrade); } - - FindOrMarkFile(filename); - - var abridged = new AbridgedFileInfo(info); - var confile = CONFile.TryParseListings(abridged); - if (confile == null) + else { - return null; + AddInvalidSong(node.Key); } + } - string playlist = ConstructPlaylist(filename, baseGroup.Directory); - group = new PackedCONGroup(confile.Value, abridged, playlist); - AddPackedCONGroup(group); + foreach (var leftover in upgrades.Keys) + { + AddInvalidSong(leftover); } + } - if (group.SongDTA == null || group.SongDTA.LastWrite != dtaLastWrite) + protected PackedCONGroup? ReadCONGroupHeader(UnmanagedMemoryStream stream) + { + string filename = stream.ReadString(); + var baseGroup = GetBaseIniGroup(filename); + if (baseGroup == null) { return null; } - return group; + + var conLastUpdate = DateTime.FromBinary(stream.Read(Endianness.Little)); + var group = FindCONGroup(filename) ?? CreateCONGroup(filename, baseGroup.Directory); + return group != null && group.Info.LastUpdatedTime == conLastUpdate ? group : null; } protected UnpackedCONGroup? ReadExtractedCONGroupHeader(UnmanagedMemoryStream stream) @@ -1198,39 +1025,20 @@ protected void ReadUpgradeCON(UnmanagedMemoryStream stream) FindOrMarkDirectory(directory); string playlist = ConstructPlaylist(directory, baseGroup.Directory); - var group = new UnpackedCONGroup(directory, dtaInfo, playlist); - AddUnpackedCONGroup(group); - - if (dtaInfo.LastWriteTime != DateTime.FromBinary(stream.Read(Endianness.Little))) + var group = CreateUnpackedCONGroup(directory, dtaInfo, playlist); + if (group == null) { return null; } - return group; - } - - protected void ReadCONGroup(UnmanagedMemoryStream stream, Action func) - { - int count = stream.Read(Endianness.Little); - for (int i = 0; i < count; ++i) - { - string name = stream.ReadString(); - int index = stream.Read(Endianness.Little); - int length = stream.Read(Endianness.Little); - if (invalidSongsInCache.Contains(name)) - { - stream.Position += length; - continue; - } - var entryReader = stream.Slice(length); - func(name, index, entryReader); - } + var dtaLastWrite = DateTime.FromBinary(stream.Read(Endianness.Little)); + return dtaInfo.LastWriteTime == dtaLastWrite ? group : null; } protected void QuickReadUpgradeDirectory(UnmanagedMemoryStream stream) { string directory = stream.ReadString(); - var dtaLastUpdated = DateTime.FromBinary(stream.Read(Endianness.Little)); + stream.Position += sizeof(long); // Can skip the last update time int count = stream.Read(Endianness.Little); for (int i = 0; i < count; i++) @@ -1239,56 +1047,35 @@ protected void QuickReadUpgradeDirectory(UnmanagedMemoryStream stream) string filename = Path.Combine(directory, $"{name}_plus.mid"); var info = new AbridgedFileInfo(filename, stream); - AddUpgrade(name, default, new UnpackedRBProUpgrade(info)); + AddCacheUpgrade(name, new UnpackedRBProUpgrade(info)); } } protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) { - string filename = stream.ReadString(); - stream.Position += 2 * SIZEOF_DATETIME; + var listings = QuickReadCONGroupHeader(stream); int count = stream.Read(Endianness.Little); - - var group = CreateCONGroup(filename, string.Empty); - if (group != null) - { - AddPackedCONGroup(group); - } - for (int i = 0; i < count; i++) { string name = stream.ReadString(); var lastWrite = DateTime.FromBinary(stream.Read(Endianness.Little)); var listing = default(CONFileListing); - group?.ConFile.TryGetListing($"songs_upgrades/{name}_plus.mid", out listing); - - var upgrade = new PackedRBProUpgrade(listing, lastWrite); - AddUpgrade(name, default, upgrade); + listings?.TryGetListing($"songs_upgrades/{name}_plus.mid", out listing); + AddCacheUpgrade(name, new PackedRBProUpgrade(listing, lastWrite)); } } - protected PackedCONGroup? QuickReadCONGroupHeader(UnmanagedMemoryStream stream) + protected List? QuickReadCONGroupHeader(UnmanagedMemoryStream stream) { string filename = stream.ReadString(); - var dtaLastWrite = DateTime.FromBinary(stream.Read(Endianness.Little)); - - var group = FindCONGroup(filename); - if (group == null) - { - group = CreateCONGroup(filename, string.Empty); - if (group == null) - { - return null; - } - - AddPackedCONGroup(group); - } - - if (group.SongDTA == null || group.SongDTA.LastWrite != dtaLastWrite) + var info = new AbridgedFileInfo(filename, stream); + if (File.Exists(filename)) { return null; } - return group; + + using var filestream = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read, 1); + return CONFile.TryParseListings(in info, filestream); } private UnpackedIniEntry? ReadUnpackedIniEntry(string directory, UnmanagedMemoryStream stream, CategoryCacheStrings strings) @@ -1298,7 +1085,6 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) { return null; } - FindOrMarkDirectory(directory); return entry; } @@ -1314,21 +1100,223 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) return entry; } - private PackedCONGroup? CreateCONGroup(string filename, string defaultPlaylist) + private UpdateGroup? CreateUpdateGroup(in FileCollection collection, FileInfo dta) + { + UpdateGroup? group = null; + try + { + using var data = FixedArray.Load(dta.FullName); + var updates = new Dictionary(); + var container = YARGDTAReader.TryCreate(data); + while (YARGDTAReader.StartNode(ref container)) + { + string name = YARGDTAReader.GetNameOfNode(ref container, true); + if (!updates.TryGetValue(name, out var update)) + { + AbridgedFileInfo? midi = null; + AbridgedFileInfo? mogg = null; + AbridgedFileInfo? milo = null; + AbridgedFileInfo? image = null; + + string subname = name.ToLowerInvariant(); + if (collection.SubDirectories.TryGetValue(subname, out var directory)) + { + string midiName = subname + "_update.mid"; + string moggName = subname + "_update.mogg"; + string miloName = subname + ".milo_xbox"; + string imageName = subname + "_keep.png_xbox"; + foreach (var file in directory.EnumerateFiles("*", SearchOption.AllDirectories)) + { + string filename = file.Name; + if (filename == midiName) + { + midi = new AbridgedFileInfo(file, false); + } + else if (filename == moggName) + { + mogg = new AbridgedFileInfo(file, false); + } + else if (filename == miloName) + { + milo = new AbridgedFileInfo(file, false); + } + else if (filename == imageName) + { + image = new AbridgedFileInfo(file, false); + } + } + + } + updates.Add(name, update = new SongUpdate(in midi, in mogg, in milo, in image)); + } + update.Containers.Add(container); + YARGDTAReader.EndNode(ref container); + } + + if (updates.Count > 0) + { + group = new UpdateGroup(collection.Directory.FullName, dta.LastWriteTime, data.TransferOwnership(), updates); + AddUpdateGroup(group); + } + } + catch (Exception ex) + { + YargLogger.LogException(ex, $"Error while loading {dta.FullName}"); + } + return group; + } + + private UpgradeGroup? CreateUpgradeGroup(in FileCollection collection, FileInfo dta) + { + UpgradeGroup? group = null; + try + { + using var data = FixedArray.Load(dta.FullName); + var upgrades = new Dictionary Container, UnpackedRBProUpgrade Upgrade)>(); + var container = YARGDTAReader.TryCreate(data); + while (YARGDTAReader.StartNode(ref container)) + { + string name = YARGDTAReader.GetNameOfNode(ref container, true); + var upgrade = default(UnpackedRBProUpgrade); + if (collection.Subfiles.TryGetValue($"{name.ToLower()}_plus.mid", out var info)) + { + var abridged = new AbridgedFileInfo(info, false); + upgrade = new UnpackedRBProUpgrade(abridged); + upgrades[name] = (container, upgrade); + } + YARGDTAReader.EndNode(ref container); + } + + if (upgrades.Count > 0) + { + group = new UpgradeGroup(collection.Directory.FullName, dta.LastWriteTime, data.TransferOwnership(), upgrades); + AddUpgradeGroup(group); + } + } + catch (Exception ex) + { + YargLogger.LogException(ex, $"Error while loading {dta.FullName}"); + } + return group; + } + + private PackedCONGroup? CreateCONGroup(string filename, string baseDirectory) { var info = new FileInfo(filename); if (!info.Exists) + { return null; + } FindOrMarkFile(filename); + string playlist = ConstructPlaylist(filename, baseDirectory); var abridged = new AbridgedFileInfo(info); - var confile = CONFile.TryParseListings(abridged); - if (confile == null) + return CreateCONGroup(in abridged, playlist); + } + + private PackedCONGroup? CreateCONGroup(in AbridgedFileInfo info, string defaultPlaylist) + { + const string SONGSFILEPATH = "songs/songs.dta"; + const string UPGRADESFILEPATH = "songs_upgrades/upgrades.dta"; + PackedCONGroup? group = null; + // Holds the file that caused an error in some form + string errorFile = string.Empty; + try { - return null; + using var stream = new FileStream(info.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 1); + var listings = CONFile.TryParseListings(in info, stream); + if (listings == null) + { + return null; + } + + var songNodes = new Dictionary>>(); + // We call `using` to ensure the proper disposal of data if an error occurs + using var songDTAData = listings.TryGetListing(SONGSFILEPATH, out var songDTA) ? songDTA.LoadAllBytes(stream) : FixedArray.Null; + if (songDTAData.IsAllocated) + { + errorFile = SONGSFILEPATH; + var container = YARGDTAReader.TryCreate(songDTAData); + while (YARGDTAReader.StartNode(ref container)) + { + string name = YARGDTAReader.GetNameOfNode(ref container, true); + if (!songNodes.TryGetValue(name, out var list)) + { + songNodes.Add(name, list = new List>()); + } + list.Add(container); + YARGDTAReader.EndNode(ref container); + } + } + + var upgrades = new Dictionary Container, PackedRBProUpgrade Upgrade)>(); + // We call `using` to ensure the proper disposal of data if an error occurs + using var upgradeDTAData = listings.TryGetListing(UPGRADESFILEPATH, out var upgradeDta) ? upgradeDta.LoadAllBytes(stream) : FixedArray.Null; + if (upgradeDTAData.IsAllocated) + { + errorFile = UPGRADESFILEPATH; + var container = YARGDTAReader.TryCreate(upgradeDTAData); + while (YARGDTAReader.StartNode(ref container)) + { + string name = YARGDTAReader.GetNameOfNode(ref container, true); + if (listings.TryGetListing($"songs_upgrades/{name}_plus.mid", out var listing)) + { + var upgrade = new PackedRBProUpgrade(listing, listing.LastWrite); + upgrades[name] = (container, upgrade); + } + YARGDTAReader.EndNode(ref container); + } + } + + if (songNodes.Count > 0 || upgrades.Count > 0) + { + group = new PackedCONGroup(listings, songDTAData.TransferOwnership(), upgradeDTAData.TransferOwnership(), songNodes, upgrades, in info, defaultPlaylist); + AddPackedCONGroup(group); + } } - return new PackedCONGroup(confile.Value, abridged, defaultPlaylist); + catch (Exception ex) + { + YargLogger.LogException(ex, $"Error while loading {errorFile}"); + } + return group; + } + + private UnpackedCONGroup? CreateUnpackedCONGroup(string directory, FileInfo dta, string defaultPlaylist) + { + try + { + using var songDTAData = FixedArray.Load(dta.FullName); + + var songNodes = new Dictionary>>(); + if (songDTAData.IsAllocated) + { + var container = YARGDTAReader.TryCreate(songDTAData); + while (YARGDTAReader.StartNode(ref container)) + { + string name = YARGDTAReader.GetNameOfNode(ref container, true); + if (!songNodes.TryGetValue(name, out var list)) + { + songNodes.Add(name, list = new List>()); + } + list.Add(container); + YARGDTAReader.EndNode(ref container); + } + } + + if (songNodes.Count > 0) + { + var abridged = new AbridgedFileInfo(dta); + var group = new UnpackedCONGroup(songDTAData.TransferOwnership(), songNodes, directory, in abridged, defaultPlaylist); + AddUnpackedCONGroup(group); + return group; + } + } + catch (Exception ex) + { + YargLogger.LogException(ex, $"Error while loading {dta.FullName}"); + } + return null; } private string ConstructPlaylist(string filename, string baseDirectory) @@ -1352,6 +1340,11 @@ internal static class UnmanagedStreamSlicer { public static unsafe UnmanagedMemoryStream Slice(this UnmanagedMemoryStream stream, int length) { + if (stream.Position > stream.Length - length) + { + throw new EndOfStreamException(); + } + var newStream = new UnmanagedMemoryStream(stream.PositionPointer, length); stream.Position += length; return newStream; diff --git a/YARG.Core/Song/Cache/CacheNodes.cs b/YARG.Core/Song/Cache/CacheNodes.cs index e0b816d85..03036061f 100644 --- a/YARG.Core/Song/Cache/CacheNodes.cs +++ b/YARG.Core/Song/Cache/CacheNodes.cs @@ -2,7 +2,6 @@ using System.IO; using System.Threading.Tasks; using YARG.Core.Extensions; -using YARG.Core.IO; namespace YARG.Core.Song.Cache { diff --git a/YARG.Core/Song/Cache/FileCollection.cs b/YARG.Core/Song/Cache/FileCollection.cs index 02e61b63e..ccc32a2df 100644 --- a/YARG.Core/Song/Cache/FileCollection.cs +++ b/YARG.Core/Song/Cache/FileCollection.cs @@ -1,7 +1,5 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.IO; -using System.Text; namespace YARG.Core.Song.Cache { diff --git a/YARG.Core/Song/Cache/SongCategories.cs b/YARG.Core/Song/Cache/SongCategories.cs index 24eccc78e..784046344 100644 --- a/YARG.Core/Song/Cache/SongCategories.cs +++ b/YARG.Core/Song/Cache/SongCategories.cs @@ -1,7 +1,7 @@ using System; -using System.Collections; using System.Collections.Generic; using System.IO; +using YARG.Core.Extensions; namespace YARG.Core.Song.Cache { @@ -74,7 +74,7 @@ public string GetKey(SongEntry entry) public EntryComparer Comparer => _COMPARER; public DateTime GetKey(SongEntry entry) { - return entry.GetAddTime().Date; + return entry.GetAddDate(); } } @@ -190,7 +190,7 @@ public static void Add(SongEntry entry, SortedDictionary(BinaryWriter fileWriter, SortedDictionary> sections, SongAttribute attribute, ref Dictionary nodes) + public static void WriteToCache(FileStream filestream, SortedDictionary> sections, SongAttribute attribute, ref Dictionary nodes) { List strings = new(); foreach (var element in sections) @@ -238,13 +238,14 @@ public static void WriteToCache(BinaryWriter fileWriter, SortedDictionary< } using MemoryStream ms = new(); - using BinaryWriter writer = new(ms); - writer.Write(strings.Count); + ms.Write(strings.Count, Endianness.Little); foreach (string str in strings) - writer.Write(str); + { + ms.Write(str); + } - fileWriter.Write((int) ms.Length); - ms.WriteTo(fileWriter.BaseStream); + filestream.Write((int) ms.Length, Endianness.Little); + filestream.Write(ms.GetBuffer(), 0, (int)ms.Length); } } } diff --git a/YARG.Core/Song/Entries/AvailableParts/AvailableParts.cs b/YARG.Core/Song/Entries/AvailableParts/AvailableParts.cs index 6f009c5e2..1a7b30bbe 100644 --- a/YARG.Core/Song/Entries/AvailableParts/AvailableParts.cs +++ b/YARG.Core/Song/Entries/AvailableParts/AvailableParts.cs @@ -1,10 +1,4 @@ using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using YARG.Core.Chart; -using YARG.Core.IO; -using YARG.Core.Song.Preparsers; namespace YARG.Core.Song { diff --git a/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs b/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs index dfb19fddc..0ddd83c53 100644 --- a/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs +++ b/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs @@ -1,6 +1,4 @@ -using Melanchall.DryWetMidi.Core; -using MoonscraperChartEditor.Song.IO; -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -13,18 +11,6 @@ namespace YARG.Core.Song { - public readonly struct IniChartNode - { - public readonly T File; - public readonly ChartType Type; - - public IniChartNode(T file, ChartType type) - { - File = file; - Type = type; - } - } - public static class IniAudio { public static readonly string[] SupportedStems = { "song", "guitar", "bass", "rhythm", "keys", "vocals", "vocals_1", "vocals_2", "drums", "drums_1", "drums_2", "drums_3", "drums_4", "crowd", }; @@ -46,11 +32,11 @@ public static bool IsAudioFile(string file) public abstract class IniSubEntry : SongEntry { - public static readonly IniChartNode[] CHART_FILE_TYPES = + public static readonly (string Filename, ChartFormat Format)[] CHART_FILE_TYPES = { - new("notes.mid" , ChartType.Mid), - new("notes.midi" , ChartType.Midi), - new("notes.chart", ChartType.Chart), + ("notes.mid" , ChartFormat.Mid), + ("notes.midi" , ChartFormat.Midi), + ("notes.chart", ChartFormat.Chart), }; protected static readonly string[] ALBUMART_FILES; @@ -65,90 +51,64 @@ static IniSubEntry() } PREVIEW_FILES = new string[IniAudio.SupportedFormats.Length]; - for (int i = 0; i < PREVIEW_FILES.Length; i++ ) + for (int i = 0; i < PREVIEW_FILES.Length; i++) { PREVIEW_FILES[i] = "preview" + IniAudio.SupportedFormats[i]; } } - protected readonly string _background; - protected readonly string _video; - protected readonly string _cover; - public readonly bool Video_Loop; - - public abstract ChartType Type { get; } + public readonly string Background; + public readonly string Video; + public readonly string Cover; + + public override string Year { get; } + public override int YearAsNumber { get; } + public override bool LoopVideo { get; } - protected IniSubEntry(in AvailableParts parts, in HashWrapper hash, IniSection modifiers, string defaultPlaylist) - : base(in parts, in hash, modifiers, defaultPlaylist) + protected IniSubEntry(in SongMetadata metadata, in AvailableParts parts, in HashWrapper hash, in LoaderSettings settings, IniSection modifiers) + : base(in metadata, in parts, in hash, in settings) { - if (modifiers.TryGet("background", out _background)) + (Year, YearAsNumber) = ParseYear(Metadata.Year); + if (modifiers.TryGet("background", out Background)) { - string ext = Path.GetExtension(_background.Trim('\"')).ToLower(); - _background = IMAGE_EXTENSIONS.Contains(ext) ? _background.ToLowerInvariant() : string.Empty; + string ext = Path.GetExtension(Background.Trim('\"')).ToLower(); + Background = IMAGE_EXTENSIONS.Contains(ext) ? Background.ToLowerInvariant() : string.Empty; } - if (modifiers.TryGet("video", out _video)) + if (modifiers.TryGet("video", out Video)) { - string ext = Path.GetExtension(_video.Trim('\"')).ToLower(); - _video = VIDEO_EXTENSIONS.Contains(ext) ? _video.ToLowerInvariant() : string.Empty; + string ext = Path.GetExtension(Video.Trim('\"')).ToLower(); + Video = VIDEO_EXTENSIONS.Contains(ext) ? Video.ToLowerInvariant() : string.Empty; } - if (modifiers.TryGet("cover", out _cover)) + if (modifiers.TryGet("cover", out Cover)) { - string ext = Path.GetExtension(_cover.Trim('\"')).ToLower(); - _cover = IMAGE_EXTENSIONS.Contains(ext) ? _cover.ToLowerInvariant() : string.Empty; + string ext = Path.GetExtension(Cover.Trim('\"')).ToLower(); + Cover = IMAGE_EXTENSIONS.Contains(ext) ? Cover.ToLowerInvariant() : string.Empty; } - modifiers.TryGet("video_loop", out Video_Loop); + LoopVideo = modifiers.TryGet("video_loop", out bool loop) && loop; } protected IniSubEntry(UnmanagedMemoryStream stream, CategoryCacheStrings strings) : base(stream, strings) { - _background = stream.ReadString(); - _video = stream.ReadString(); - _cover = stream.ReadString(); - Video_Loop = stream.ReadBoolean(); - } - - protected abstract Stream? GetChartStream(); - - protected abstract void SerializeSubData(BinaryWriter writer); + YearAsNumber = stream.Read(Endianness.Little); + Background = stream.ReadString(); + Video = stream.ReadString(); + Cover = stream.ReadString(); + LoopVideo = stream.ReadBoolean(); - public ReadOnlySpan Serialize(CategoryCacheWriteNode node, string groupDirectory) - { - string relativePath = Path.GetRelativePath(groupDirectory, Location); - if (relativePath == ".") - relativePath = string.Empty; - - using MemoryStream ms = new(); - using BinaryWriter writer = new(ms); - - writer.Write(SubType == EntryType.Sng); - writer.Write(relativePath); - - SerializeSubData(writer); - SerializeMetadata(writer, node); - - writer.Write(_background); - writer.Write(_video); - writer.Write(_cover); - writer.Write(Video_Loop); - return new ReadOnlySpan(ms.GetBuffer(), 0, (int)ms.Length); + Year = YearAsNumber != int.MaxValue ? YearAsNumber.ToString() : Metadata.Year; } - public override SongChart? LoadChart() + public override void Serialize(MemoryStream stream, CategoryCacheWriteNode node) { - using var stream = GetChartStream(); - if (stream == null) - return null; - - if (Type != ChartType.Chart) - { - return SongChart.FromMidi(_parseSettings, MidFileLoader.LoadMidiFile(stream)); - } - - using var reader = new StreamReader(stream); - return SongChart.FromDotChart(_parseSettings, reader.ReadToEnd()); + base.Serialize(stream, node); + stream.Write(YearAsNumber, Endianness.Little); + stream.Write(Background); + stream.Write(Video); + stream.Write(Cover); + stream.Write(LoopVideo); } public override FixedArray LoadMiloData() @@ -156,7 +116,7 @@ public override FixedArray LoadMiloData() return FixedArray.Null; } - protected static (ScanResult Result, AvailableParts Parts) ScanIniChartFile(in FixedArray file, ChartType chartType, IniSection modifiers) + protected static (ScanResult Result, AvailableParts Parts, LoaderSettings Settings) ProcessChartFile(in FixedArray file, ChartFormat format, IniSection modifiers) { DrumPreparseHandler drums = new() { @@ -164,52 +124,200 @@ protected static (ScanResult Result, AvailableParts Parts) ScanIniChartFile(in F }; var parts = AvailableParts.Default; - if (chartType == ChartType.Chart) + var settings = default(LoaderSettings); + var results = default((ScanResult result, long resolution)); + if (format == ChartFormat.Chart) { if (YARGTextReader.IsUTF8(in file, out var byteContainer)) { - ParseDotChart(ref byteContainer, modifiers, ref parts, drums); + results = ParseDotChart(ref byteContainer, modifiers, ref parts, drums); } else { using var chars = YARGTextReader.ConvertToUTF16(in file, out var charContainer); if (chars.IsAllocated) { - ParseDotChart(ref charContainer, modifiers, ref parts, drums); + results = ParseDotChart(ref charContainer, modifiers, ref parts, drums); } else { using var ints = YARGTextReader.ConvertToUTF32(in file, out var intContainer); - ParseDotChart(ref intContainer, modifiers, ref parts, drums); + results = ParseDotChart(ref intContainer, modifiers, ref parts, drums); } } } else // if (chartType == ChartType.Mid || chartType == ChartType.Midi) // Uncomment for any future file type { - if (!ParseDotMidi(in file, modifiers, ref parts, drums)) - { - return (ScanResult.MultipleMidiTrackNames, parts); - } + results = ParseDotMidi(in file, modifiers, ref parts, drums); } - SetDrums(ref parts, drums); + if (results.result != ScanResult.Success) + { + return (results.result, parts, settings); + } + SetDrums(ref parts, drums); if (!CheckScanValidity(in parts)) - return (ScanResult.NoNotes, parts); + { + return (ScanResult.NoNotes, parts, settings); + } if (!modifiers.Contains("name")) - return (ScanResult.NoName, parts); + { + return (ScanResult.NoName, parts, settings); + } + + if (!modifiers.TryGet("hopo_frequency", out settings.HopoThreshold) || settings.HopoThreshold <= 0) + { + if (modifiers.TryGet("eighthnote_hopo", out bool eighthNoteHopo)) + { + settings.HopoThreshold = results.resolution / (eighthNoteHopo ? 2 : 3); + } + else if (modifiers.TryGet("hopofreq", out long hopoFreq)) + { + int denominator = hopoFreq switch + { + 0 => 24, + 1 => 16, + 2 => 12, + 3 => 8, + 4 => 6, + 5 => 4, + _ => throw new NotImplementedException($"Unhandled hopofreq value {hopoFreq}!") + }; + settings.HopoThreshold = 4 * results.resolution / denominator; + } + else + { + settings.HopoThreshold = results.resolution / 3; + } + + if (format == ChartFormat.Chart) + { + // With a 192 resolution, .chart has a HOPO threshold of 65 ticks, not 64, + // so we need to scale this factor to different resolutions (480 res = 162.5 threshold). + // Why?... idk, but I hate it. + const float DEFAULT_RESOLUTION = 192; + settings.HopoThreshold += (long) (results.resolution / DEFAULT_RESOLUTION); + } + } + + // .chart defaults to no cutting off sustains whatsoever if the ini does not define the value. + // Since a failed `TryGet` sets the value to zero, we would need no additional work outside .mid + if (!modifiers.TryGet("sustain_cutoff_threshold", out settings.SustainCutoffThreshold) && format != ChartFormat.Chart) + { + settings.SustainCutoffThreshold = results.resolution / 3; + } + + if (format == ChartFormat.Mid || format == ChartFormat.Midi) + { + if (!modifiers.TryGet("multiplier_note", out settings.OverdiveMidiNote) || settings.OverdiveMidiNote != 103) + { + settings.OverdiveMidiNote = 116; + } + } SetIntensities(modifiers, ref parts); - return (ScanResult.Success, parts); + return (ScanResult.Success, parts, settings); + } + + private static (string Parsed, int AsNumber) ParseYear(in string baseString) + { + string parsedString = baseString; + int number = int.MaxValue; + for (int i = 0; i <= baseString.Length - 4; ++i) + { + int pivot = i; + int tmpNumber = 0; + while (i < pivot + 4 && i < baseString.Length && char.IsDigit(baseString[i])) + { + tmpNumber = 10 * tmpNumber + baseString[i] - '0'; + ++i; + } + + if (i == pivot + 4) + { + parsedString = baseString[pivot..i]; + number = tmpNumber; + break; + } + } + return (parsedString, number); + } + + protected static bool TryGetRandomBackgroundImage(TEnumerable collection, out TValue? value) + where TEnumerable : IEnumerable> + { + // Choose a valid image background present in the folder at random + var images = new List(); + foreach (var format in SongEntry.IMAGE_EXTENSIONS) + { + var (_, image) = collection.FirstOrDefault(node => node.Key == "bg" + format); + if (image != null) + { + images.Add(image); + } + } + + foreach (var (shortname, image) in collection) + { + if (!shortname.StartsWith("background")) + { + continue; + } + + foreach (var format in SongEntry.IMAGE_EXTENSIONS) + { + if (shortname.EndsWith(format)) + { + images.Add(image); + break; + } + } + } + + if (images.Count == 0) + { + value = default!; + return false; + } + value = images[SongEntry.BACKROUND_RNG.Next(images.Count)]; + return true; + } + + protected static DrumsType ParseDrumsType(in AvailableParts parts) + { + if (parts.FourLaneDrums.SubTracks > 0) + { + return DrumsType.FourLane; + } + if (parts.FiveLaneDrums.SubTracks > 0) + { + return DrumsType.FiveLane; + } + return DrumsType.Unknown; } - private static void ParseDotChart(ref YARGTextContainer container, IniSection modifiers, ref AvailableParts parts, DrumPreparseHandler drums) + private static (ScanResult result, long resolution) ParseDotChart(ref YARGTextContainer container, IniSection modifiers, ref AvailableParts parts, DrumPreparseHandler drums) where TChar : unmanaged, IEquatable, IConvertible { + long resolution = 192; if (YARGChartFileReader.ValidateTrack(ref container, YARGChartFileReader.HEADERTRACK)) { var chartMods = YARGChartFileReader.ExtractModifiers(ref container); + if (chartMods.TryGetValue("Resolution", out var resolutions)) + { + unsafe + { + var mod = resolutions[0]; + resolution = mod.Buffer[0]; + if (resolution == 0) + { + return (ScanResult.ZeroResolution, 0); + } + } + chartMods.Remove("Resolution"); + } modifiers.Append(chartMods); } @@ -225,20 +333,25 @@ private static void ParseDotChart(ref YARGTextContainer container, } if (drums.Type == DrumsType.Unknown && drums.ValidatedDiffs > 0) + { drums.Type = DrumsType.FourLane; + } + return (ScanResult.Success, resolution); } - - private static bool ParseDotMidi(in FixedArray file, IniSection modifiers, ref AvailableParts parts, DrumPreparseHandler drums) + private static (ScanResult result, long resolution) ParseDotMidi(in FixedArray file, IniSection modifiers, ref AvailableParts parts, DrumPreparseHandler drums) { bool usePro = !modifiers.TryGet("pro_drums", out bool proDrums) || proDrums; if (drums.Type == DrumsType.Unknown) { if (usePro) + { drums.Type = DrumsType.UnknownPro; + } } else if (drums.Type == DrumsType.FourLane && usePro) + { drums.Type = DrumsType.ProDrums; - + } return ParseMidi(in file, drums, ref parts); } @@ -253,16 +366,16 @@ private static unsafe bool TraverseChartTrack(ref YARGTextContainer ChartPreparser.Traverse(ref container, difficulty, ref parts.FiveFretGuitar, &ChartPreparser.ValidateFiveFret), - Instrument.FiveFretBass => ChartPreparser.Traverse(ref container, difficulty, ref parts.FiveFretBass, &ChartPreparser.ValidateFiveFret), - Instrument.FiveFretRhythm => ChartPreparser.Traverse(ref container, difficulty, ref parts.FiveFretRhythm, &ChartPreparser.ValidateFiveFret), + Instrument.FiveFretGuitar => ChartPreparser.Traverse(ref container, difficulty, ref parts.FiveFretGuitar, &ChartPreparser.ValidateFiveFret), + Instrument.FiveFretBass => ChartPreparser.Traverse(ref container, difficulty, ref parts.FiveFretBass, &ChartPreparser.ValidateFiveFret), + Instrument.FiveFretRhythm => ChartPreparser.Traverse(ref container, difficulty, ref parts.FiveFretRhythm, &ChartPreparser.ValidateFiveFret), Instrument.FiveFretCoopGuitar => ChartPreparser.Traverse(ref container, difficulty, ref parts.FiveFretCoopGuitar, &ChartPreparser.ValidateFiveFret), - Instrument.SixFretGuitar => ChartPreparser.Traverse(ref container, difficulty, ref parts.SixFretGuitar, &ChartPreparser.ValidateSixFret), - Instrument.SixFretBass => ChartPreparser.Traverse(ref container, difficulty, ref parts.SixFretBass, &ChartPreparser.ValidateSixFret), - Instrument.SixFretRhythm => ChartPreparser.Traverse(ref container, difficulty, ref parts.SixFretRhythm, &ChartPreparser.ValidateSixFret), - Instrument.SixFretCoopGuitar => ChartPreparser.Traverse(ref container, difficulty, ref parts.SixFretCoopGuitar, &ChartPreparser.ValidateSixFret), - Instrument.Keys => ChartPreparser.Traverse(ref container, difficulty, ref parts.Keys, &ChartPreparser.ValidateFiveFret), - Instrument.FourLaneDrums => drums.ParseChart(ref container, difficulty), + Instrument.SixFretGuitar => ChartPreparser.Traverse(ref container, difficulty, ref parts.SixFretGuitar, &ChartPreparser.ValidateSixFret), + Instrument.SixFretBass => ChartPreparser.Traverse(ref container, difficulty, ref parts.SixFretBass, &ChartPreparser.ValidateSixFret), + Instrument.SixFretRhythm => ChartPreparser.Traverse(ref container, difficulty, ref parts.SixFretRhythm, &ChartPreparser.ValidateSixFret), + Instrument.SixFretCoopGuitar => ChartPreparser.Traverse(ref container, difficulty, ref parts.SixFretCoopGuitar, &ChartPreparser.ValidateSixFret), + Instrument.Keys => ChartPreparser.Traverse(ref container, difficulty, ref parts.Keys, &ChartPreparser.ValidateFiveFret), + Instrument.FourLaneDrums => drums.ParseChart(ref container, difficulty), _ => false, }; } @@ -417,44 +530,5 @@ private static void SetIntensities(IniSection modifiers, ref AvailableParts part } } } - - protected static bool TryGetRandomBackgroundImage(IEnumerable> collection, out T value) - { - // Choose a valid image background present in the folder at random - var images = new List(); - foreach (var format in IMAGE_EXTENSIONS) - { - var (_, image) = collection.FirstOrDefault(node => node.Key == "bg" + format); - if (image != null) - { - images.Add(image); - } - } - - foreach (var (shortname, image) in collection) - { - if (!shortname.StartsWith("background")) - { - continue; - } - - foreach (var format in IMAGE_EXTENSIONS) - { - if (shortname.EndsWith(format)) - { - images.Add(image); - break; - } - } - } - - if (images.Count == 0) - { - value = default!; - return false; - } - value = images[BACKROUND_RNG.Next(images.Count)]; - return true; - } } } diff --git a/YARG.Core/Song/Entries/Ini/SongEntry.Sng.cs b/YARG.Core/Song/Entries/Ini/SongEntry.Sng.cs index 9e57cbd8e..11e7ddd36 100644 --- a/YARG.Core/Song/Entries/Ini/SongEntry.Sng.cs +++ b/YARG.Core/Song/Entries/Ini/SongEntry.Sng.cs @@ -1,7 +1,9 @@ -using System; +using MoonscraperChartEditor.Song.IO; +using System; using System.IO; using System.Linq; using YARG.Core.Audio; +using YARG.Core.Chart; using YARG.Core.Extensions; using YARG.Core.IO; using YARG.Core.IO.Ini; @@ -15,37 +17,88 @@ public sealed class SngEntry : IniSubEntry { private readonly uint _version; private readonly AbridgedFileInfo _sngInfo; - private readonly string _chartName; + private readonly ChartFormat _chartFormat; public override string Location => _sngInfo.FullName; public override string DirectoryActual => Path.GetDirectoryName(_sngInfo.FullName); - public override ChartType Type { get; } - public override DateTime GetAddTime() => _sngInfo.LastUpdatedTime; - + public override DateTime GetAddDate() => _sngInfo.LastUpdatedTime.Date; public override EntryType SubType => EntryType.Sng; + public override ulong SongLengthMilliseconds { get; } + + private SngEntry(SngFile sngFile, ChartFormat format, IniSection modifiers, in AvailableParts parts, in HashWrapper hash, in SongMetadata metadata, in LoaderSettings settings) + : base(in metadata, in parts, in hash, in settings, modifiers) + { + _version = sngFile.Version; + _sngInfo = sngFile.Info; + _chartFormat = format; + if (!modifiers.TryGet("song_length", out ulong songLength)) + { + using var mixer = LoadAudio(0, 0); + if (mixer != null) + { + songLength = (ulong) (mixer.Length * SongMetadata.MILLISECOND_FACTOR); + } + } + SongLengthMilliseconds = songLength; + } - protected override void SerializeSubData(BinaryWriter writer) + private SngEntry(uint version, in AbridgedFileInfo sngInfo, ChartFormat format, UnmanagedMemoryStream stream, CategoryCacheStrings strings) + : base(stream, strings) { - writer.Write(_sngInfo.LastUpdatedTime.ToBinary()); - writer.Write(_version); - writer.Write((byte) Type); + _version = version; + _sngInfo = sngInfo; + _chartFormat = format; + SongLengthMilliseconds = stream.Read(Endianness.Little); } - protected override Stream? GetChartStream() + public override void Serialize(MemoryStream stream, CategoryCacheWriteNode node) + { + // Validation block + stream.Write(_sngInfo.LastUpdatedTime.ToBinary(), Endianness.Little); + stream.Write(_version, Endianness.Little); + stream.WriteByte((byte) _chartFormat); + + // Metadata block + base.Serialize(stream, node); + stream.Write(SongLengthMilliseconds, Endianness.Little); + } + + public override SongChart? LoadChart() { if (!_sngInfo.IsStillValid()) + { return null; + } - using var sngFile = SngFile.TryLoadFromFile(_sngInfo); + var sngFile = SngFile.TryLoadFromFile(_sngInfo); if (sngFile == null) + { return null; + } - return sngFile[_chartName].CreateStream(sngFile); + var parseSettings = new ParseSettings() + { + HopoThreshold = Settings.HopoThreshold, + SustainCutoffThreshold = Settings.SustainCutoffThreshold, + StarPowerNote = Settings.OverdiveMidiNote, + DrumsType = ParseDrumsType(in Parts), + ChordHopoCancellation = _chartFormat == ChartFormat.Chart + }; + + string file = CHART_FILE_TYPES[(int) _chartFormat].Filename; + using var stream = sngFile[file].CreateStream(sngFile); + if (_chartFormat == ChartFormat.Mid || _chartFormat == ChartFormat.Midi) + { + return SongChart.FromMidi(in parseSettings, MidFileLoader.LoadMidiFile(stream)); + } + + using var reader = new StreamReader(stream); + return SongChart.FromDotChart(in parseSettings, reader.ReadToEnd()); } public override StemMixer? LoadAudio(float speed, double volume, params SongStem[] ignoreStems) { - using var sngFile = SngFile.TryLoadFromFile(_sngInfo); + var sngFile = SngFile.TryLoadFromFile(_sngInfo); if (sngFile == null) { YargLogger.LogFormatError("Failed to load sng file {0}", _sngInfo.FullName); @@ -56,7 +109,7 @@ protected override void SerializeSubData(BinaryWriter writer) public override StemMixer? LoadPreviewAudio(float speed) { - using var sngFile = SngFile.TryLoadFromFile(_sngInfo); + var sngFile = SngFile.TryLoadFromFile(_sngInfo); if (sngFile == null) { YargLogger.LogFormatError("Failed to load sng file {0}", _sngInfo.FullName); @@ -85,18 +138,18 @@ protected override void SerializeSubData(BinaryWriter writer) public override YARGImage? LoadAlbumData() { - using var sngFile = SngFile.TryLoadFromFile(_sngInfo); + var sngFile = SngFile.TryLoadFromFile(_sngInfo); if (sngFile == null) return null; - if (!string.IsNullOrEmpty(_cover) && sngFile.TryGetValue(_video, out var cover)) + if (sngFile.TryGetValue(Cover, out var cover)) { var image = YARGImage.Load(in cover, sngFile); if (image != null) { return image; } - YargLogger.LogFormatError("SNG Image mapped to {0} failed to load", _video); + YargLogger.LogFormatError("SNG Image mapped to {0} failed to load", Cover); } foreach (string albumFile in ALBUMART_FILES) @@ -116,7 +169,7 @@ protected override void SerializeSubData(BinaryWriter writer) public override BackgroundResult? LoadBackground(BackgroundType options) { - using var sngFile = SngFile.TryLoadFromFile(_sngInfo); + var sngFile = SngFile.TryLoadFromFile(_sngInfo); if (sngFile == null) { return null; @@ -140,7 +193,7 @@ protected override void SerializeSubData(BinaryWriter writer) if ((options & BackgroundType.Video) > 0) { - if (!string.IsNullOrEmpty(_video) && sngFile.TryGetValue(_video, out var video)) + if (sngFile.TryGetValue(Video, out var video)) { var stream = video.CreateStream(sngFile); return new BackgroundResult(BackgroundType.Video, stream); @@ -172,8 +225,7 @@ protected override void SerializeSubData(BinaryWriter writer) if ((options & BackgroundType.Image) > 0) { - if ((!string.IsNullOrEmpty(_background) && sngFile.TryGetValue(_background, out var listing)) - || TryGetRandomBackgroundImage(sngFile, out listing)) + if (sngFile.TryGetValue(Background, out var listing) || TryGetRandomBackgroundImage(sngFile, out listing)) { var image = YARGImage.Load(in listing, sngFile); if (image != null) @@ -200,9 +252,14 @@ protected override void SerializeSubData(BinaryWriter writer) return null; } + public override FixedArray LoadMiloData() + { + return FixedArray.Null; + } + private StemMixer? CreateAudioMixer(float speed, double volume, SngFile sngFile, params SongStem[] ignoreStems) { - bool clampStemVolume = _metadata.Source.Str.ToLowerInvariant() == "yarg"; + bool clampStemVolume = Metadata.Source.Str.ToLowerInvariant() == "yarg"; var mixer = GlobalAudioHandler.CreateMixer(ToString(), speed, volume, clampStemVolume); if (mixer == null) { @@ -243,44 +300,18 @@ protected override void SerializeSubData(BinaryWriter writer) return mixer; } - private SngEntry(SngFile sngFile, in IniChartNode chart, in AvailableParts parts, in HashWrapper hash, IniSection modifiers, string defaultPlaylist) - : base(in parts, in hash, modifiers, defaultPlaylist) - { - _version = sngFile.Version; - _sngInfo = sngFile.Info; - _chartName = chart.File; - Type = chart.Type; - } - - private SngEntry(uint version, in AbridgedFileInfo sngInfo, in IniChartNode chart, UnmanagedMemoryStream stream, CategoryCacheStrings strings) - : base(stream, strings) + public static (ScanResult, SngEntry?) ProcessNewEntry(SngFile sng, in SngFileListing listing, ChartFormat format, string defaultPlaylist) { - _version = version; - _sngInfo = sngInfo; - _chartName = chart.File; - Type = chart.Type; - } - - public static (ScanResult, SngEntry?) ProcessNewEntry(SngFile sng, in IniChartNode chart, string defaultPlaylist) - { - using var file = chart.File.LoadAllBytes(sng); - var (result, parts) = ScanIniChartFile(in file, chart.Type, sng.Metadata); + using var file = listing.LoadAllBytes(sng); + var (result, parts, settings) = ProcessChartFile(file, format, sng.Metadata); if (result != ScanResult.Success) { return (result, null); } - var node = new IniChartNode(chart.File.Name, chart.Type); var hash = HashWrapper.Hash(file.ReadOnlySpan); - var entry = new SngEntry(sng, in node, in parts, in hash, sng.Metadata, defaultPlaylist); - if (!sng.Metadata.Contains("song_length")) - { - using var mixer = entry.LoadAudio(0, 0); - if (mixer != null) - { - entry.SongLengthSeconds = mixer.Length; - } - } + var metadata = new SongMetadata(sng.Metadata, defaultPlaylist); + var entry = new SngEntry(sng, format, sng.Metadata, in parts, in hash, in metadata, in settings); return (result, entry); } @@ -288,35 +319,28 @@ public static (ScanResult, SngEntry?) ProcessNewEntry(SngFile sng, in IniChartNo { var sngInfo = AbridgedFileInfo.TryParseInfo(filename, stream); if (sngInfo == null) + { return null; + } - using var sngFile = SngFile.TryLoadFromFile(sngInfo.Value); uint version = stream.Read(Endianness.Little); + var sngFile = SngFile.TryLoadFromFile(sngInfo.Value); if (sngFile == null || sngFile.Version != version) { // TODO: Implement Update-in-place functionality return null; } - byte chartTypeIndex = (byte)stream.ReadByte(); - if (chartTypeIndex >= CHART_FILE_TYPES.Length) - { - return null; - } - return new SngEntry(sngFile.Version, sngInfo.Value, CHART_FILE_TYPES[chartTypeIndex], stream, strings); + byte chartTypeIndex = (byte) stream.ReadByte(); + return chartTypeIndex < CHART_FILE_TYPES.Length ? new SngEntry(sngFile.Version, sngInfo.Value, (ChartFormat) chartTypeIndex, stream, strings) : null; } public static SngEntry? LoadFromCache_Quick(string filename, UnmanagedMemoryStream stream, CategoryCacheStrings strings) { var sngInfo = new AbridgedFileInfo(filename, stream); - uint version = stream.Read(Endianness.Little); - byte chartTypeIndex = (byte)stream.ReadByte(); - if (chartTypeIndex >= CHART_FILE_TYPES.Length) - { - return null; - } - return new SngEntry(version, sngInfo, CHART_FILE_TYPES[chartTypeIndex], stream, strings); + byte chartTypeIndex = (byte) stream.ReadByte(); + return chartTypeIndex < CHART_FILE_TYPES.Length ? new SngEntry(version, sngInfo, (ChartFormat) chartTypeIndex, stream, strings) : null; } } } diff --git a/YARG.Core/Song/Entries/Ini/SongEntry.UnpackedIni.cs b/YARG.Core/Song/Entries/Ini/SongEntry.UnpackedIni.cs index 7a9f3f1e2..804a9f49d 100644 --- a/YARG.Core/Song/Entries/Ini/SongEntry.UnpackedIni.cs +++ b/YARG.Core/Song/Entries/Ini/SongEntry.UnpackedIni.cs @@ -9,37 +9,102 @@ using System.Linq; using YARG.Core.Logging; using YARG.Core.Extensions; +using MoonscraperChartEditor.Song.IO; +using YARG.Core.Chart; namespace YARG.Core.Song { public sealed class UnpackedIniEntry : IniSubEntry { private readonly AbridgedFileInfo _chartFile; + private readonly ChartFormat _chartFormat; private readonly AbridgedFileInfo? _iniFile; public override string Location { get; } public override string DirectoryActual => Location; - public override ChartType Type { get; } - public override DateTime GetAddTime() => _chartFile.LastUpdatedTime; - + public override DateTime GetAddDate() => _chartFile.LastUpdatedTime.Date; public override EntryType SubType => EntryType.Ini; + public override ulong SongLengthMilliseconds { get; } + + private UnpackedIniEntry(string directory, FileInfo chartInfo, ChartFormat format, in AbridgedFileInfo? iniFile, IniSection modifiers, in AvailableParts parts, in HashWrapper hash, in SongMetadata metadata, in LoaderSettings settings) + : base(in metadata, in parts, in hash, in settings, modifiers) + { + Location = directory; + _chartFile = new AbridgedFileInfo(chartInfo); + _chartFormat = format; + _iniFile = iniFile; + + if (!modifiers.TryGet("song_length", out ulong songLength)) + { + using var mixer = LoadAudio(0, 0); + if (mixer != null) + { + songLength = (ulong) (mixer.Length * SongMetadata.MILLISECOND_FACTOR); + } + } + SongLengthMilliseconds = songLength; + } + + private UnpackedIniEntry(string directory, in AbridgedFileInfo chartInfo, ChartFormat format, in AbridgedFileInfo? iniFile, UnmanagedMemoryStream stream, CategoryCacheStrings strings) + : base(stream, strings) + { + Location = directory; + _chartFile = chartInfo; + _chartFormat = format; + _iniFile = iniFile; + SongLengthMilliseconds = stream.Read(Endianness.Little); + } - protected override void SerializeSubData(BinaryWriter writer) + public override void Serialize(MemoryStream stream, CategoryCacheWriteNode node) { - writer.Write((byte) Type); - writer.Write(_chartFile.LastUpdatedTime.ToBinary()); + // Validation block + stream.WriteByte((byte) _chartFormat); + stream.Write(_chartFile.LastUpdatedTime.ToBinary(), Endianness.Little); + stream.Write(_iniFile != null); if (_iniFile != null) { - writer.Write(true); - writer.Write(_iniFile.Value.LastUpdatedTime.ToBinary()); + stream.Write(_iniFile.Value.LastUpdatedTime.ToBinary(), Endianness.Little); } - else - writer.Write(false); + + // Metadata block + base.Serialize(stream, node); + stream.Write(SongLengthMilliseconds, Endianness.Little); + } + + public override SongChart? LoadChart() + { + if (!_chartFile.IsStillValid()) + { + return null; + } + + if (_iniFile != null ? !_iniFile.Value.IsStillValid() : File.Exists(Path.Combine(Location, "song.ini"))) + { + return null; + } + + var parseSettings = new ParseSettings() + { + HopoThreshold = Settings.HopoThreshold, + SustainCutoffThreshold = Settings.SustainCutoffThreshold, + StarPowerNote = Settings.OverdiveMidiNote, + DrumsType = ParseDrumsType(in Parts), + ChordHopoCancellation = _chartFormat == ChartFormat.Chart + }; + + using var stream = new FileStream(_chartFile.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 1); + if (_chartFormat == ChartFormat.Mid || _chartFormat == ChartFormat.Midi) + { + return SongChart.FromMidi(in parseSettings, MidFileLoader.LoadMidiFile(stream)); + } + + using var reader = new StreamReader(stream); + return SongChart.FromDotChart(in parseSettings, reader.ReadToEnd()); } public override StemMixer? LoadAudio(float speed, double volume, params SongStem[] ignoreStems) { - bool clampStemVolume = _metadata.Source.Str.ToLowerInvariant() == "yarg"; + bool clampStemVolume = Metadata.Source.Str.ToLowerInvariant() == "yarg"; var mixer = GlobalAudioHandler.CreateMixer(ToString(), speed, volume, clampStemVolume); if (mixer == null) { @@ -97,7 +162,7 @@ protected override void SerializeSubData(BinaryWriter writer) public override YARGImage? LoadAlbumData() { var subFiles = GetSubFiles(); - if (!string.IsNullOrEmpty(_cover) && subFiles.TryGetValue(_cover, out var cover)) + if (!string.IsNullOrEmpty(Cover) && subFiles.TryGetValue(Cover, out var cover)) { var image = YARGImage.Load(cover); if (image != null) @@ -136,7 +201,7 @@ protected override void SerializeSubData(BinaryWriter writer) if ((options & BackgroundType.Video) > 0) { - if (!string.IsNullOrEmpty(_video) && subFiles.TryGetValue(_video, out var video)) + if (subFiles.TryGetValue(Video, out var video)) { var stream = File.OpenRead(video.FullName); return new BackgroundResult(BackgroundType.Video, stream); @@ -157,10 +222,9 @@ protected override void SerializeSubData(BinaryWriter writer) if ((options & BackgroundType.Image) > 0) { - if ((!string.IsNullOrEmpty(_background) && subFiles.TryGetValue(_background, out var file)) - || TryGetRandomBackgroundImage(subFiles, out file)) + if (subFiles.TryGetValue(Background, out var file) || TryGetRandomBackgroundImage(subFiles, out file)) { - var image = YARGImage.Load(file); + var image = YARGImage.Load(file!); if (image != null) { return new BackgroundResult(image); @@ -170,22 +234,9 @@ protected override void SerializeSubData(BinaryWriter writer) return null; } - protected override Stream? GetChartStream() + public override FixedArray LoadMiloData() { - if (!_chartFile.IsStillValid()) - return null; - - if (_iniFile != null) - { - if (!_iniFile.Value.IsStillValid()) - return null; - } - else if (File.Exists(Path.Combine(Location, "song.ini"))) - { - return null; - } - - return new FileStream(_chartFile.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 1); + return FixedArray.Null; } private Dictionary GetSubFiles() @@ -202,25 +253,7 @@ private Dictionary GetSubFiles() return files; } - private UnpackedIniEntry(string directory, in IniChartNode chart, AbridgedFileInfo? iniFile, in AvailableParts parts, in HashWrapper hash, IniSection modifiers, string defaultPlaylist) - : base(in parts, in hash, modifiers, defaultPlaylist) - { - Location = directory; - Type = chart.Type; - _chartFile = chart.File; - _iniFile = iniFile; - } - - private UnpackedIniEntry(string directory, in IniChartNode chart, AbridgedFileInfo? iniFile, UnmanagedMemoryStream stream, CategoryCacheStrings strings) - : base(stream, strings) - { - Location = directory; - Type = chart.Type; - _chartFile = chart.File; - _iniFile = iniFile; - } - - public static (ScanResult, UnpackedIniEntry?) ProcessNewEntry(string chartDirectory, in IniChartNode chart, FileInfo? iniFile, string defaultPlaylist) + public static (ScanResult, UnpackedIniEntry?) ProcessNewEntry(string chartDirectory, FileInfo chartInfo, ChartFormat format, FileInfo? iniFile, string defaultPlaylist) { IniSection iniModifiers; AbridgedFileInfo? iniFileInfo = null; @@ -239,30 +272,21 @@ public static (ScanResult, UnpackedIniEntry?) ProcessNewEntry(string chartDirect iniModifiers = new(); } - if ((chart.File.Attributes & AbridgedFileInfo.RECALL_ON_DATA_ACCESS) > 0) + if ((chartInfo.Attributes & AbridgedFileInfo.RECALL_ON_DATA_ACCESS) > 0) { return (ScanResult.ChartNotDownloaded, null); } - using var file = FixedArray.Load(chart.File.FullName); - var (result, parts) = ScanIniChartFile(in file, chart.Type, iniModifiers); + using var file = FixedArray.Load(chartInfo.FullName); + var (result, parts, settings) = ProcessChartFile(file, format, iniModifiers); if (result != ScanResult.Success) { return (result, null); } - var abridged = new AbridgedFileInfo(chart.File); - var node = new IniChartNode(abridged, chart.Type); var hash = HashWrapper.Hash(file.ReadOnlySpan); - var entry = new UnpackedIniEntry(chartDirectory, in node, iniFileInfo, in parts, in hash, iniModifiers, defaultPlaylist); - if (!iniModifiers.Contains("song_length")) - { - using var mixer = entry.LoadAudio(0, 0); - if (mixer != null) - { - entry.SongLengthSeconds = mixer.Length; - } - } + var metadata = new SongMetadata(iniModifiers, defaultPlaylist); + var entry = new UnpackedIniEntry(chartDirectory, chartInfo, format, in iniFileInfo, iniModifiers, in parts, in hash, in metadata, in settings); return (result, entry); } @@ -275,7 +299,7 @@ public static (ScanResult, UnpackedIniEntry?) ProcessNewEntry(string chartDirect } var chart = CHART_FILE_TYPES[chartTypeIndex]; - var chartInfo = AbridgedFileInfo.TryParseInfo(Path.Combine(directory, chart.File), stream, true); + var chartInfo = AbridgedFileInfo.TryParseInfo(Path.Combine(directory, chart.Filename), stream); if (chartInfo == null) { return null; @@ -295,8 +319,7 @@ public static (ScanResult, UnpackedIniEntry?) ProcessNewEntry(string chartDirect { return null; } - var node = new IniChartNode(chartInfo.Value, chart.Type); - return new UnpackedIniEntry(directory, in node, iniInfo, stream, strings); + return new UnpackedIniEntry(directory, chartInfo.Value, chart.Format, in iniInfo, stream, strings); } public static UnpackedIniEntry? IniFromCache_Quick(string directory, UnmanagedMemoryStream stream, CategoryCacheStrings strings) @@ -308,14 +331,9 @@ public static (ScanResult, UnpackedIniEntry?) ProcessNewEntry(string chartDirect } var chart = CHART_FILE_TYPES[chartTypeIndex]; - var chartInfo = new AbridgedFileInfo(Path.Combine(directory, chart.File), stream); - AbridgedFileInfo? iniInfo = null; - if (stream.ReadBoolean()) - { - iniInfo = new AbridgedFileInfo(Path.Combine(directory, "song.ini"), stream); - } - var node = new IniChartNode(chartInfo, chart.Type); - return new UnpackedIniEntry(directory, in node, iniInfo, stream, strings); + var chartInfo = new AbridgedFileInfo(Path.Combine(directory, chart.Filename), stream); + AbridgedFileInfo? iniInfo = stream.ReadBoolean() ? new AbridgedFileInfo(Path.Combine(directory, "song.ini"), stream) : null; + return new UnpackedIniEntry(directory, chartInfo, chart.Format, in iniInfo, stream, strings); } } } diff --git a/YARG.Core/Song/Entries/RBCON/RBAudio.cs b/YARG.Core/Song/Entries/RBCON/RBAudio.cs index d483175d2..55dd22494 100644 --- a/YARG.Core/Song/Entries/RBCON/RBAudio.cs +++ b/YARG.Core/Song/Entries/RBCON/RBAudio.cs @@ -1,7 +1,5 @@ using System; -using System.Collections.Generic; using System.IO; -using System.Text; using YARG.Core.Extensions; namespace YARG.Core.Song @@ -39,26 +37,26 @@ public RBAudio(UnmanagedMemoryStream stream) Crowd = ReadArray(stream); } - public readonly void Serialize(BinaryWriter writer) + public readonly void Serialize(MemoryStream stream) { - WriteArray(Track, writer); - WriteArray(Drums, writer); - WriteArray(Bass, writer); - WriteArray(Guitar, writer); - WriteArray(Keys, writer); - WriteArray(Vocals, writer); - WriteArray(Crowd, writer); + WriteArray(Track, stream); + WriteArray(Drums, stream); + WriteArray(Bass, stream); + WriteArray(Guitar, stream); + WriteArray(Keys, stream); + WriteArray(Vocals, stream); + WriteArray(Crowd, stream); } - public static void WriteArray(in TType[] values, BinaryWriter writer) + public static void WriteArray(in TType[] values, MemoryStream stream) { - writer.Write(values.Length); + stream.Write(values.Length, Endianness.Little); unsafe { fixed (TType* ptr = values) { var span = new ReadOnlySpan(ptr, values.Length * sizeof(TType)); - writer.Write(span); + stream.Write(span); } } } diff --git a/YARG.Core/Song/Entries/RBCON/RBCONDifficulties.cs b/YARG.Core/Song/Entries/RBCON/RBCONDifficulties.cs index ce7a80166..75b82b53d 100644 --- a/YARG.Core/Song/Entries/RBCON/RBCONDifficulties.cs +++ b/YARG.Core/Song/Entries/RBCON/RBCONDifficulties.cs @@ -1,9 +1,4 @@ using System; -using System.Collections.Generic; -using System.IO; -using System.Text; -using YARG.Core.Extensions; -using YARG.Core.IO; namespace YARG.Core.Song { diff --git a/YARG.Core/Song/Entries/RBCON/RBMetadata.cs b/YARG.Core/Song/Entries/RBCON/RBMetadata.cs index 6397dd126..0430898dc 100644 --- a/YARG.Core/Song/Entries/RBCON/RBMetadata.cs +++ b/YARG.Core/Song/Entries/RBCON/RBMetadata.cs @@ -1,6 +1,6 @@ using System; -using System.Collections.Generic; -using System.Text; +using System.IO; +using YARG.Core.Extensions; namespace YARG.Core.Song { @@ -41,5 +41,76 @@ public struct RBMetadata public RBAudio Indices; public RBAudio Panning; + + public RBMetadata(UnmanagedMemoryStream stream) + { + AnimTempo = stream.Read(Endianness.Little); + SongID = stream.ReadString(); + VocalPercussionBank = stream.ReadString(); + VocalSongScrollSpeed = stream.Read(Endianness.Little); + VocalGender = stream.ReadBoolean(); + VocalTonicNote = stream.Read(Endianness.Little); + SongTonality = stream.ReadBoolean(); + TuningOffsetCents = stream.Read(Endianness.Little); + VenueVersion = stream.Read(Endianness.Little); + DrumBank = stream.ReadString(); + + RealGuitarTuning = RBAudio.ReadArray(stream); + RealBassTuning = RBAudio.ReadArray(stream); + + Indices = new RBAudio(stream); + Panning = new RBAudio(stream); + + Soloes = ReadStringArray(stream); + VideoVenues = ReadStringArray(stream); + } + + public readonly void Serialize(MemoryStream stream) + { + stream.Write(AnimTempo, Endianness.Little); + stream.Write(SongID); + stream.Write(VocalPercussionBank); + stream.Write(VocalSongScrollSpeed, Endianness.Little); + stream.Write(VocalGender); + stream.Write(VocalTonicNote, Endianness.Little); + stream.Write(SongTonality); + stream.Write(TuningOffsetCents, Endianness.Little); + stream.Write(VenueVersion, Endianness.Little); + stream.Write(DrumBank); + + RBAudio.WriteArray(in RealGuitarTuning, stream); + RBAudio.WriteArray(in RealBassTuning, stream); + + Indices.Serialize(stream); + Panning.Serialize(stream); + + stream.Write(Soloes.Length, Endianness.Little); + for (int i = 0; i < Soloes.Length; ++i) + { + stream.Write(Soloes[i]); + } + + stream.Write(VideoVenues.Length, Endianness.Little); + for (int i = 0; i < VideoVenues.Length; ++i) + { + stream.Write(VideoVenues[i]); + } + } + + private static string[] ReadStringArray(UnmanagedMemoryStream stream) + { + int length = stream.Read(Endianness.Little); + if (length == 0) + { + return Array.Empty(); + } + + var strings = new string[length]; + for (int i = 0; i < length; ++i) + { + strings[i] = stream.ReadString(); + } + return strings; + } } } diff --git a/YARG.Core/Song/Entries/RBCON/RBProUpgrade.cs b/YARG.Core/Song/Entries/RBCON/RBProUpgrade.cs index 9d5ea4ed6..351b60011 100644 --- a/YARG.Core/Song/Entries/RBCON/RBProUpgrade.cs +++ b/YARG.Core/Song/Entries/RBCON/RBProUpgrade.cs @@ -1,7 +1,6 @@ using System; -using System.Collections.Generic; using System.IO; -using System.Text; +using YARG.Core.Extensions; using YARG.Core.IO; namespace YARG.Core.Song @@ -9,7 +8,7 @@ namespace YARG.Core.Song public abstract class RBProUpgrade { public abstract DateTime LastUpdatedTime { get; } - public abstract void WriteToCache(BinaryWriter writer); + public abstract void WriteToCache(MemoryStream stream); public abstract Stream? GetUpgradeMidiStream(); public abstract FixedArray LoadUpgradeMidi(); } @@ -28,9 +27,9 @@ public PackedRBProUpgrade(CONFileListing? listing, DateTime lastWrite) _lastUpdatedTime = listing?.LastWrite ?? lastWrite; } - public override void WriteToCache(BinaryWriter writer) + public override void WriteToCache(MemoryStream stream) { - writer.Write(_lastUpdatedTime.ToBinary()); + stream.Write(_lastUpdatedTime.ToBinary(), Endianness.Little); } public override Stream? GetUpgradeMidiStream() @@ -63,9 +62,9 @@ public UnpackedRBProUpgrade(in AbridgedFileInfo info) _midi = info; } - public override void WriteToCache(BinaryWriter writer) + public override void WriteToCache(MemoryStream stream) { - writer.Write(_midi.LastUpdatedTime.ToBinary()); + stream.Write(_midi.LastUpdatedTime.ToBinary(), Endianness.Little); } public override Stream? GetUpgradeMidiStream() diff --git a/YARG.Core/Song/Entries/RBCON/SongEntry.PackedRBCON.cs b/YARG.Core/Song/Entries/RBCON/SongEntry.PackedRBCON.cs index 2b5533c3f..e1da2e71e 100644 --- a/YARG.Core/Song/Entries/RBCON/SongEntry.PackedRBCON.cs +++ b/YARG.Core/Song/Entries/RBCON/SongEntry.PackedRBCON.cs @@ -6,7 +6,6 @@ using YARG.Core.IO; using YARG.Core.Venue; using System.Linq; -using YARG.Core.Logging; namespace YARG.Core.Song { @@ -23,37 +22,57 @@ public sealed class PackedRBCONEntry : RBCONEntry public override string DirectoryActual => Path.GetDirectoryName(_midiListing?.ConFile.FullName); public override EntryType SubType => EntryType.CON; - public static (ScanResult, PackedRBCONEntry?) ProcessNewEntry(PackedCONGroup group, string nodename, in YARGTextContainer container, Dictionary> updates, Dictionary, RBProUpgrade)> upgrades) + public static (ScanResult, PackedRBCONEntry?) ProcessNewEntry(PackedCONGroup group, string nodename, DTAEntry node, CONModification modification) { - try + var (dtaResult, info) = ProcessDTAs(nodename, node, modification); + if (dtaResult != ScanResult.Success) { - var song = new PackedRBCONEntry(group, nodename, in container, updates, upgrades); - if (song._midiListing == null) - { - YargLogger.LogFormatError("Required midi file for {0} - {1} was not located", group.Info.FullName, item2: nodename); - return (ScanResult.MissingMidi, null); - } + return (dtaResult, null); + } - var result = song.ParseRBCONMidi(group.Stream); - if (result != ScanResult.Success) - { - return (result, null); - } - return (result, song); + group.Listings.TryGetListing(info.Location + ".mogg", out var moggListing); + if (!IsMoggValid(in modification.Mogg, moggListing, group.Stream)) + { + return (ScanResult.MoggError, null); } - catch (Exception ex) + + if (!group.Listings.TryGetListing(info.Location + ".mid", out var midiListing)) { - YargLogger.LogException(ex, null); - return (ScanResult.DTAError, null); + return (ScanResult.MissingCONMidi, null); } + + using var mainMidi = midiListing.LoadAllBytes(group.Stream); + var (midiResult, hash) = ParseRBCONMidi(in mainMidi, modification, ref info); + if (midiResult != ScanResult.Success) + { + return (midiResult, null); + } + + if (!info.Location!.StartsWith($"songs/{nodename}")) + { + nodename = midiListing.Filename.Split('/')[1]; + } + + string genPath = $"songs/{nodename}/gen/{nodename}"; + group.Listings.TryGetListing(genPath + ".milo_xbox", out var miloListing); + group.Listings.TryGetListing(genPath + "_keep.png_xbox", out var imgListing); + + if (info.Metadata.Playlist.Length == 0) + { + info.Metadata.Playlist = group.DefaultPlaylist; + } + + string psuedoDirectory = Path.Combine(group.Location, group.Listings[midiListing.PathIndex].Filename); + var entry = new PackedRBCONEntry(in info, modification, in hash, midiListing, moggListing, miloListing, imgListing, psuedoDirectory); + return (ScanResult.Success, entry); } - public static PackedRBCONEntry? TryLoadFromCache(in CONFile conFile, string nodename, Dictionary, RBProUpgrade Upgrade)> upgrades, UnmanagedMemoryStream stream, CategoryCacheStrings strings) + public static PackedRBCONEntry? TryLoadFromCache(List listings, string nodename, RBProUpgrade? upgrade, UnmanagedMemoryStream stream, CategoryCacheStrings strings) { var psuedoDirectory = stream.ReadString(); string midiFilename = stream.ReadString(); - if (!conFile.TryGetListing(midiFilename, out var midiListing)) + if (!listings.TryGetListing(midiFilename, out var midiListing)) { return null; } @@ -74,68 +93,75 @@ public static (ScanResult, PackedRBCONEntry?) ProcessNewEntry(PackedCONGroup gro } } - var upgrade = upgrades.TryGetValue(nodename, out var node) ? node.Upgrade : null; - - conFile.TryGetListing(Path.ChangeExtension(midiFilename, ".mogg"), out var moggListing); + listings.TryGetListing(Path.ChangeExtension(midiFilename, ".mogg"), out var moggListing); if (!midiFilename.StartsWith($"songs/{nodename}")) + { nodename = midiFilename.Split('/')[1]; + } string genPath = $"songs/{nodename}/gen/{nodename}"; - conFile.TryGetListing(genPath + ".milo_xbox", out var miloListing); - conFile.TryGetListing(genPath + "_keep.png_xbox", out var imgListing); + listings.TryGetListing(genPath + ".milo_xbox", out var miloListing); + listings.TryGetListing(genPath + "_keep.png_xbox", out var imgListing); return new PackedRBCONEntry(midiListing, lastMidiWrite, moggListing, miloListing, imgListing, psuedoDirectory, updateMidi, upgrade, stream, strings); } - public static PackedRBCONEntry LoadFromCache_Quick(in CONFile conFile, string nodename, Dictionary, RBProUpgrade Upgrade)> upgrades, UnmanagedMemoryStream stream, CategoryCacheStrings strings) + public static PackedRBCONEntry LoadFromCache_Quick(List? listings, string nodename, RBProUpgrade? upgrade, UnmanagedMemoryStream stream, CategoryCacheStrings strings) { var psuedoDirectory = stream.ReadString(); string midiFilename = stream.ReadString(); - conFile.TryGetListing(midiFilename, out var midiListing); + + var midiListing = default(CONFileListing); + listings?.TryGetListing(midiFilename, out midiListing); + var lastMidiWrite = DateTime.FromBinary(stream.Read(Endianness.Little)); AbridgedFileInfo? updateMidi = stream.ReadBoolean() ? new AbridgedFileInfo(stream) : null; - var upgrade = upgrades.TryGetValue(nodename, out var node) ? node.Upgrade : null; - conFile.TryGetListing(Path.ChangeExtension(midiFilename, ".mogg"), out var moggListing); + var moggListing = default(CONFileListing); + listings?.TryGetListing(Path.ChangeExtension(midiFilename, ".mogg"), out moggListing); if (!midiFilename.StartsWith($"songs/{nodename}")) + { nodename = midiFilename.Split('/')[1]; + } string genPath = $"songs/{nodename}/gen/{nodename}"; - conFile.TryGetListing(genPath + ".milo_xbox", out var miloListing); - conFile.TryGetListing(genPath + "_keep.png_xbox", out var imgListing); + + var miloListing = default(CONFileListing); + listings?.TryGetListing(genPath + ".milo_xbox", out miloListing); + + var imgListing = default(CONFileListing); + listings?.TryGetListing(genPath + "_keep.png_xbox", out imgListing); return new PackedRBCONEntry(midiListing, lastMidiWrite, moggListing, miloListing, imgListing, psuedoDirectory, updateMidi, upgrade, stream, strings); } - private PackedRBCONEntry(PackedCONGroup group, string nodename, in YARGTextContainer container, Dictionary> updates, Dictionary, RBProUpgrade)> upgrades) - : base() + private static bool IsMoggValid(in AbridgedFileInfo? info, CONFileListing? listing, FileStream stream) { - var results = Init(nodename, in container, updates, upgrades, group.DefaultPlaylist); - string midiPath = results.location + ".mid"; - if (!group.ConFile.TryGetListing(midiPath, out _midiListing)) + using var mogg = LoadUpdateMoggStream(in info); + if (mogg != null) { - Location = string.Empty; - return; + int version = mogg.Read(Endianness.Little); + return version == 0x0A || version == 0xf0; } + return listing != null && CONFileListing.GetMoggVersion(listing, stream) == 0x0A; + } - _lastMidiWrite = _midiListing.LastWrite; - - group.ConFile.TryGetListing(results.location + ".mogg", out _moggListing); - - if (!results.location.StartsWith($"songs/{nodename}")) - nodename = _midiListing.Filename.Split('/')[1]; - - string genPath = $"songs/{nodename}/gen/{nodename}"; - group.ConFile.TryGetListing(genPath + ".milo_xbox", out _miloListing); - group.ConFile.TryGetListing(genPath + "_keep.png_xbox", out _imgListing); + private PackedRBCONEntry(in ScanNode info, CONModification modification, in HashWrapper hash + , CONFileListing midiListing, CONFileListing? moggListing, CONFileListing? miloListing, CONFileListing? imgListing, string psuedoDirectory) + : base(in info, modification, in hash) + { + _midiListing = midiListing; + _lastMidiWrite = midiListing.LastWrite; - string midiDirectory = group.ConFile.GetFilename(_midiListing.PathIndex); - Location = Path.Combine(group.Location, midiDirectory); + _moggListing = moggListing; + _miloListing = miloListing; + _imgListing = imgListing; + Location = psuedoDirectory; } - private PackedRBCONEntry(CONFileListing? midi, DateTime midiLastWrite, CONFileListing? moggListing, CONFileListing? miloListing, CONFileListing? imgListing, string directory, + private PackedRBCONEntry(CONFileListing? midi, DateTime midiLastWrite, CONFileListing? moggListing, CONFileListing? miloListing, CONFileListing? imgListing, string psuedoDirectory, AbridgedFileInfo? updateMidi, RBProUpgrade? upgrade, UnmanagedMemoryStream stream, CategoryCacheStrings strings) : base(updateMidi, upgrade, stream, strings) { @@ -145,15 +171,31 @@ private PackedRBCONEntry(CONFileListing? midi, DateTime midiLastWrite, CONFileLi _imgListing = imgListing; _lastMidiWrite = midiLastWrite; - Location = directory; + Location = psuedoDirectory; } - public override void Serialize(BinaryWriter writer, CategoryCacheWriteNode node) + public override void Serialize(MemoryStream stream, CategoryCacheWriteNode node) { - writer.Write(Location); - writer.Write(_midiListing!.Filename); - writer.Write(_midiListing.LastWrite.ToBinary()); - base.Serialize(writer, node); + stream.Write(Location); + stream.Write(_midiListing!.Filename); + stream.Write(_midiListing.LastWrite.ToBinary(), Endianness.Little); + stream.Write(UpdateMidi != null); + UpdateMidi?.Serialize(stream); + base.Serialize(stream, node); + } + + public override YARGImage? LoadAlbumData() + { + var bytes = FixedArray.Null; + if (UpdateImage != null && UpdateImage.Value.Exists()) + { + bytes = FixedArray.Load(UpdateImage.Value.FullName); + } + else if (_imgListing != null) + { + bytes = _imgListing.LoadAllBytes(); + } + return bytes.IsAllocated ? new YARGImage(bytes) : null; } public override BackgroundResult? LoadBackground(BackgroundType options) @@ -236,57 +278,23 @@ public override void Serialize(BinaryWriter writer, CategoryCacheWriteNode node) public override FixedArray LoadMiloData() { - var bytes = base.LoadMiloData(); - if (bytes.IsAllocated) + if (UpdateMilo != null && UpdateMilo.Value.Exists()) { - return bytes; + return FixedArray.Load(UpdateMilo.Value.FullName); } return _miloListing != null ? _miloListing.LoadAllBytes() : FixedArray.Null; } protected override Stream? GetMidiStream() - { - if (_midiListing == null || !_midiListing.IsStillValid(_lastMidiWrite)) - return null; - return _midiListing.CreateStream(); - } - - protected override FixedArray LoadMidiFile(Stream? file) { return _midiListing != null && _midiListing.IsStillValid(_lastMidiWrite) - ? _midiListing.LoadAllBytes(file!) - : FixedArray.Null; - } - - protected override FixedArray LoadRawImageData() - { - var bytes = base.LoadRawImageData(); - if (bytes.IsAllocated) - { - return bytes; - } - return _imgListing != null ? _imgListing.LoadAllBytes() : FixedArray.Null; + ? _midiListing.CreateStream() + : null; } protected override Stream? GetMoggStream() { - var stream = base.GetMoggStream(); - if (stream != null) - { - return stream; - } - return _moggListing?.CreateStream(); - } - - protected override bool IsMoggValid(Stream? stream) - { - using var mogg = base.GetMoggStream(); - if (mogg != null) - { - int version = mogg.Read(Endianness.Little); - return version == 0x0A || version == 0xf0; - } - return _moggListing != null && CONFileListing.GetMoggVersion(_moggListing, stream!) == 0x0A; + return LoadUpdateMoggStream(in UpdateMogg) ?? _moggListing?.CreateStream(); } } } diff --git a/YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs b/YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs index 329ed5ed2..ff37ad82f 100644 --- a/YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs +++ b/YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text; using YARG.Core.Chart; using YARG.Core.Song.Cache; using YARG.Core.IO; @@ -16,59 +15,103 @@ namespace YARG.Core.Song { public abstract class RBCONEntry : SongEntry { - protected struct DTAResult - { - public static readonly DTAResult Empty = new() - { - pans = Array.Empty(), - volumes = Array.Empty(), - cores = Array.Empty(), - }; - - public bool alternatePath; - public bool discUpdate; - public string location; - public float[] pans; - public float[] volumes; - public float[] cores; - } - private const long NOTE_SNAP_THRESHOLD = 10; - private RBMetadata _rbMetadata; - private RBCONDifficulties _rbDifficulties; + public readonly RBMetadata RBMetadata; + public readonly RBCONDifficulties RBDifficulties; + + public readonly AbridgedFileInfo? UpdateMidi; + public readonly RBProUpgrade? Upgrade; - private AbridgedFileInfo? _updateMidi; - private RBProUpgrade? _upgrade; + public readonly AbridgedFileInfo? UpdateMogg; + public readonly AbridgedFileInfo? UpdateMilo; + public readonly AbridgedFileInfo? UpdateImage; - private AbridgedFileInfo? UpdateMogg; - private AbridgedFileInfo? UpdateMilo; - private AbridgedFileInfo? UpdateImage; + public string RBSongId => RBMetadata.SongID; + public int RBBandDiff => RBDifficulties.Band; - public string RBSongId => _rbMetadata.SongID; - public int RBBandDiff => _rbDifficulties.Band; + public override string Year { get; } + public override int YearAsNumber { get; } + public override ulong SongLengthMilliseconds { get; } + public override bool LoopVideo => false; protected abstract DateTime MidiLastUpdate { get; } - public override DateTime GetAddTime() + protected RBCONEntry(in ScanNode info, CONModification modification, in HashWrapper hash) + : base(in info.Metadata, in info.Parts, in hash, in info.Settings) + { + Year = info.Metadata.Year; + YearAsNumber = info.YearAsNumber; + SongLengthMilliseconds = info.SongLength; + RBMetadata = info.RBMetadata; + RBDifficulties = info.Difficulties; + UpdateMidi = modification.Midi; + UpdateMogg = modification.Mogg; + UpdateImage = modification.Image; + UpdateMilo = modification.Milo; + Upgrade = modification.UpgradeNode; + } + + protected RBCONEntry(AbridgedFileInfo? updateMidi, RBProUpgrade? upgrade, UnmanagedMemoryStream stream, CategoryCacheStrings strings) + : base(stream, strings) + { + UpdateMidi = updateMidi; + Upgrade = upgrade; + + YearAsNumber = stream.Read(Endianness.Little); + SongLengthMilliseconds = stream.Read(Endianness.Little); + + UpdateMogg = stream.ReadBoolean() ? new AbridgedFileInfo(stream.ReadString(), false) : null; + UpdateMilo = stream.ReadBoolean() ? new AbridgedFileInfo(stream.ReadString(), false) : null; + UpdateImage = stream.ReadBoolean() ? new AbridgedFileInfo(stream.ReadString(), false) : null; + unsafe + { + RBDifficulties = *(RBCONDifficulties*) stream.PositionPointer; + stream.Position += sizeof(RBCONDifficulties); + } + RBMetadata = new RBMetadata(stream); + + Year = YearAsNumber != int.MaxValue ? YearAsNumber.ToString() : Metadata.Year; + } + + public override void Serialize(MemoryStream stream, CategoryCacheWriteNode node) + { + base.Serialize(stream, node); + stream.Write(YearAsNumber, Endianness.Little); + stream.Write(SongLengthMilliseconds, Endianness.Little); + WriteUpdateInfo(UpdateMogg, stream); + WriteUpdateInfo(UpdateMilo, stream); + WriteUpdateInfo(UpdateImage, stream); + unsafe + { + fixed (RBCONDifficulties* ptr = &RBDifficulties) + { + var span = new ReadOnlySpan(ptr, sizeof(RBCONDifficulties)); + stream.Write(span); + } + } + RBMetadata.Serialize(stream); + } + + public override DateTime GetAddDate() { var lastUpdateTime = MidiLastUpdate; - if (_updateMidi != null) + if (UpdateMidi != null) { - if (_updateMidi.Value.LastUpdatedTime > lastUpdateTime) + if (UpdateMidi.Value.LastUpdatedTime > lastUpdateTime) { - lastUpdateTime = _updateMidi.Value.LastUpdatedTime; + lastUpdateTime = UpdateMidi.Value.LastUpdatedTime; } } - if (_upgrade != null) + if (Upgrade != null) { - if (_upgrade.LastUpdatedTime > lastUpdateTime) + if (Upgrade.LastUpdatedTime > lastUpdateTime) { - lastUpdateTime = _upgrade.LastUpdatedTime; + lastUpdateTime = Upgrade.LastUpdatedTime; } } - return lastUpdateTime; + return lastUpdateTime.Date; } public override SongChart? LoadChart() @@ -79,32 +122,46 @@ public override DateTime GetAddTime() using (var midiStream = GetMidiStream()) { if (midiStream == null) + { return null; + } midi = MidiFile.Read(midiStream, readingSettings); } // Merge update MIDI - if (_updateMidi != null) + if (UpdateMidi != null) { - if (!_updateMidi.Value.IsStillValid(false)) + if (!UpdateMidi.Value.IsStillValid(false)) + { return null; + } - using var midiStream = new FileStream(_updateMidi.Value.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); + using var midiStream = new FileStream(UpdateMidi.Value.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); var update = MidiFile.Read(midiStream, readingSettings); midi.Merge(update); } // Merge upgrade MIDI - if (_upgrade != null) + if (Upgrade != null) { - using var midiStream = _upgrade.GetUpgradeMidiStream(); + using var midiStream = Upgrade.GetUpgradeMidiStream(); if (midiStream == null) + { return null; + } var update = MidiFile.Read(midiStream, readingSettings); midi.Merge(update); } - return SongChart.FromMidi(_parseSettings, midi); + var parseSettings = new ParseSettings() + { + HopoThreshold = Settings.HopoThreshold, + SustainCutoffThreshold = Settings.SustainCutoffThreshold, + StarPowerNote = Settings.OverdiveMidiNote, + DrumsType = DrumsType.FourLane, + ChordHopoCancellation = false + }; + return SongChart.FromMidi(in parseSettings, midi); } public override StemMixer? LoadAudio(float speed, double volume, params SongStem[] ignoreStems) @@ -126,7 +183,7 @@ public override DateTime GetAddTime() int start = stream.Read(Endianness.Little); stream.Seek(start, SeekOrigin.Begin); - bool clampStemVolume = _metadata.Source.Str.ToLowerInvariant() == "yarg"; + bool clampStemVolume = Metadata.Source.Str.ToLowerInvariant() == "yarg"; var mixer = GlobalAudioHandler.CreateMixer(ToString(), stream, speed, volume, clampStemVolume); if (mixer == null) { @@ -136,58 +193,58 @@ public override DateTime GetAddTime() } - if (_rbMetadata.Indices.Drums.Length > 0 && !ignoreStems.Contains(SongStem.Drums)) + if (RBMetadata.Indices.Drums.Length > 0 && !ignoreStems.Contains(SongStem.Drums)) { - switch (_rbMetadata.Indices.Drums.Length) + switch (RBMetadata.Indices.Drums.Length) { //drum (0 1): stereo kit --> (0 1) case 1: case 2: - mixer.AddChannel(SongStem.Drums, _rbMetadata.Indices.Drums, _rbMetadata.Panning.Drums!); + mixer.AddChannel(SongStem.Drums, RBMetadata.Indices.Drums, RBMetadata.Panning.Drums!); break; //drum (0 1 2): mono kick, stereo snare/kit --> (0) (1 2) case 3: - mixer.AddChannel(SongStem.Drums1, _rbMetadata.Indices.Drums[0..1], _rbMetadata.Panning.Drums![0..2]); - mixer.AddChannel(SongStem.Drums2, _rbMetadata.Indices.Drums[1..3], _rbMetadata.Panning.Drums[2..6]); + mixer.AddChannel(SongStem.Drums1, RBMetadata.Indices.Drums[0..1], RBMetadata.Panning.Drums![0..2]); + mixer.AddChannel(SongStem.Drums2, RBMetadata.Indices.Drums[1..3], RBMetadata.Panning.Drums[2..6]); break; //drum (0 1 2 3): mono kick, mono snare, stereo kit --> (0) (1) (2 3) case 4: - mixer.AddChannel(SongStem.Drums1, _rbMetadata.Indices.Drums[0..1], _rbMetadata.Panning.Drums![0..2]); - mixer.AddChannel(SongStem.Drums2, _rbMetadata.Indices.Drums[1..2], _rbMetadata.Panning.Drums[2..4]); - mixer.AddChannel(SongStem.Drums3, _rbMetadata.Indices.Drums[2..4], _rbMetadata.Panning.Drums[4..8]); + mixer.AddChannel(SongStem.Drums1, RBMetadata.Indices.Drums[0..1], RBMetadata.Panning.Drums![0..2]); + mixer.AddChannel(SongStem.Drums2, RBMetadata.Indices.Drums[1..2], RBMetadata.Panning.Drums[2..4]); + mixer.AddChannel(SongStem.Drums3, RBMetadata.Indices.Drums[2..4], RBMetadata.Panning.Drums[4..8]); break; //drum (0 1 2 3 4): mono kick, stereo snare, stereo kit --> (0) (1 2) (3 4) case 5: - mixer.AddChannel(SongStem.Drums1, _rbMetadata.Indices.Drums[0..1], _rbMetadata.Panning.Drums![0..2]); - mixer.AddChannel(SongStem.Drums2, _rbMetadata.Indices.Drums[1..3], _rbMetadata.Panning.Drums[2..6]); - mixer.AddChannel(SongStem.Drums3, _rbMetadata.Indices.Drums[3..5], _rbMetadata.Panning.Drums[6..10]); + mixer.AddChannel(SongStem.Drums1, RBMetadata.Indices.Drums[0..1], RBMetadata.Panning.Drums![0..2]); + mixer.AddChannel(SongStem.Drums2, RBMetadata.Indices.Drums[1..3], RBMetadata.Panning.Drums[2..6]); + mixer.AddChannel(SongStem.Drums3, RBMetadata.Indices.Drums[3..5], RBMetadata.Panning.Drums[6..10]); break; //drum (0 1 2 3 4 5): stereo kick, stereo snare, stereo kit --> (0 1) (2 3) (4 5) case 6: - mixer.AddChannel(SongStem.Drums1, _rbMetadata.Indices.Drums[0..2], _rbMetadata.Panning.Drums![0..4]); - mixer.AddChannel(SongStem.Drums2, _rbMetadata.Indices.Drums[2..4], _rbMetadata.Panning.Drums[4..8]); - mixer.AddChannel(SongStem.Drums3, _rbMetadata.Indices.Drums[4..6], _rbMetadata.Panning.Drums[8..12]); + mixer.AddChannel(SongStem.Drums1, RBMetadata.Indices.Drums[0..2], RBMetadata.Panning.Drums![0..4]); + mixer.AddChannel(SongStem.Drums2, RBMetadata.Indices.Drums[2..4], RBMetadata.Panning.Drums[4..8]); + mixer.AddChannel(SongStem.Drums3, RBMetadata.Indices.Drums[4..6], RBMetadata.Panning.Drums[8..12]); break; } } - if (_rbMetadata.Indices.Bass.Length > 0 && !ignoreStems.Contains(SongStem.Bass)) - mixer.AddChannel(SongStem.Bass, _rbMetadata.Indices.Bass, _rbMetadata.Panning.Bass!); + if (RBMetadata.Indices.Bass.Length > 0 && !ignoreStems.Contains(SongStem.Bass)) + mixer.AddChannel(SongStem.Bass, RBMetadata.Indices.Bass, RBMetadata.Panning.Bass!); - if (_rbMetadata.Indices.Guitar.Length > 0 && !ignoreStems.Contains(SongStem.Guitar)) - mixer.AddChannel(SongStem.Guitar, _rbMetadata.Indices.Guitar, _rbMetadata.Panning.Guitar!); + if (RBMetadata.Indices.Guitar.Length > 0 && !ignoreStems.Contains(SongStem.Guitar)) + mixer.AddChannel(SongStem.Guitar, RBMetadata.Indices.Guitar, RBMetadata.Panning.Guitar!); - if (_rbMetadata.Indices.Keys.Length > 0 && !ignoreStems.Contains(SongStem.Keys)) - mixer.AddChannel(SongStem.Keys, _rbMetadata.Indices.Keys, _rbMetadata.Panning.Keys!); + if (RBMetadata.Indices.Keys.Length > 0 && !ignoreStems.Contains(SongStem.Keys)) + mixer.AddChannel(SongStem.Keys, RBMetadata.Indices.Keys, RBMetadata.Panning.Keys!); - if (_rbMetadata.Indices.Vocals.Length > 0 && !ignoreStems.Contains(SongStem.Vocals)) - mixer.AddChannel(SongStem.Vocals, _rbMetadata.Indices.Vocals, _rbMetadata.Panning.Vocals!); + if (RBMetadata.Indices.Vocals.Length > 0 && !ignoreStems.Contains(SongStem.Vocals)) + mixer.AddChannel(SongStem.Vocals, RBMetadata.Indices.Vocals, RBMetadata.Panning.Vocals!); - if (_rbMetadata.Indices.Track.Length > 0 && !ignoreStems.Contains(SongStem.Song)) - mixer.AddChannel(SongStem.Song, _rbMetadata.Indices.Track, _rbMetadata.Panning.Track!); + if (RBMetadata.Indices.Track.Length > 0 && !ignoreStems.Contains(SongStem.Song)) + mixer.AddChannel(SongStem.Song, RBMetadata.Indices.Track, RBMetadata.Panning.Track!); - if (_rbMetadata.Indices.Crowd.Length > 0 && !ignoreStems.Contains(SongStem.Crowd)) - mixer.AddChannel(SongStem.Crowd, _rbMetadata.Indices.Crowd, _rbMetadata.Panning.Crowd!); + if (RBMetadata.Indices.Crowd.Length > 0 && !ignoreStems.Contains(SongStem.Crowd)) + mixer.AddChannel(SongStem.Crowd, RBMetadata.Indices.Crowd, RBMetadata.Panning.Crowd!); if (mixer.Channels.Count == 0) { @@ -205,775 +262,410 @@ public override DateTime GetAddTime() return LoadAudio(speed, 0, SongStem.Crowd); } - public override YARGImage? LoadAlbumData() - { - var bytes = LoadRawImageData(); - return bytes.IsAllocated ? new YARGImage(bytes) : null; - } - - public override FixedArray LoadMiloData() - { - return UpdateMilo != null && UpdateMilo.Value.Exists() - ? FixedArray.Load(UpdateMilo.Value.FullName) - : FixedArray.Null; - } + protected abstract Stream? GetMidiStream(); + protected abstract Stream? GetMoggStream(); - public virtual void Serialize(BinaryWriter writer, CategoryCacheWriteNode node) + public struct ScanNode { - writer.Write(_updateMidi != null); - _updateMidi?.Serialize(writer); - - SerializeMetadata(writer, node); - - WriteUpdateInfo(UpdateMogg, writer); - WriteUpdateInfo(UpdateMilo, writer); - WriteUpdateInfo(UpdateImage, writer); - - writer.Write(_rbMetadata.AnimTempo); - writer.Write(_rbMetadata.SongID); - writer.Write(_rbMetadata.VocalPercussionBank); - writer.Write(_rbMetadata.VocalSongScrollSpeed); - writer.Write(_rbMetadata.VocalGender); - writer.Write(_rbMetadata.VocalTonicNote); - writer.Write(_rbMetadata.SongTonality); - writer.Write(_rbMetadata.TuningOffsetCents); - writer.Write(_rbMetadata.VenueVersion); - writer.Write(_rbMetadata.DrumBank); - - RBAudio.WriteArray(in _rbMetadata.RealGuitarTuning, writer); - RBAudio.WriteArray(in _rbMetadata.RealBassTuning, writer); - - _rbMetadata.Indices.Serialize(writer); - _rbMetadata.Panning.Serialize(writer); - - WriteStringArray(_rbMetadata.Soloes, writer); - WriteStringArray(_rbMetadata.VideoVenues, writer); - - unsafe - { - fixed (RBCONDifficulties* ptr = &_rbDifficulties) - { - var span = new ReadOnlySpan(ptr, sizeof(RBCONDifficulties)); - writer.Write(span); - } - } - } + public static readonly ScanNode Default = new() + { + Metadata = SongMetadata.Default, + RBMetadata = RBMetadata.Default, + Settings = LoaderSettings.Default, + Parts = AvailableParts.Default, + Difficulties = RBCONDifficulties.Default, + YearAsNumber = int.MaxValue, + }; - protected abstract bool IsMoggValid(Stream? file); - protected abstract FixedArray LoadMidiFile(Stream? file); - protected abstract Stream? GetMidiStream(); + public string? Location; + public SongMetadata Metadata; + public RBMetadata RBMetadata; + public LoaderSettings Settings; + public AvailableParts Parts; + public RBCONDifficulties Difficulties; - protected RBCONEntry() : base() - { - _rbMetadata = RBMetadata.Default; - _rbDifficulties = RBCONDifficulties.Default; - _parseSettings.DrumsType = DrumsType.FourLane; - _parseSettings.NoteSnapThreshold = NOTE_SNAP_THRESHOLD; + public int YearAsNumber; + public ulong SongLength; } - protected RBCONEntry(AbridgedFileInfo? updateMidi, RBProUpgrade? upgrade, UnmanagedMemoryStream stream, CategoryCacheStrings strings) - : base(stream, strings) + protected static (ScanResult Result, ScanNode Info) ProcessDTAs(string nodename, DTAEntry baseDTA, CONModification modification) { - _updateMidi = updateMidi; - _upgrade = upgrade; - - UpdateMogg = stream.ReadBoolean() ? new AbridgedFileInfo(stream.ReadString(), false) : null; - UpdateMilo = stream.ReadBoolean() ? new AbridgedFileInfo(stream.ReadString(), false) : null; - UpdateImage = stream.ReadBoolean() ? new AbridgedFileInfo(stream.ReadString(), false) : null; - - _rbMetadata.AnimTempo = stream.Read(Endianness.Little); - _rbMetadata.SongID = stream.ReadString(); - _rbMetadata.VocalPercussionBank = stream.ReadString(); - _rbMetadata.VocalSongScrollSpeed = stream.Read(Endianness.Little); - _rbMetadata.VocalGender = stream.ReadBoolean(); - _rbMetadata.VocalTonicNote = stream.Read(Endianness.Little); - _rbMetadata.SongTonality = stream.ReadBoolean(); - _rbMetadata.TuningOffsetCents = stream.Read(Endianness.Little); - _rbMetadata.VenueVersion = stream.Read(Endianness.Little); - _rbMetadata.DrumBank = stream.ReadString(); - - _rbMetadata.RealGuitarTuning = RBAudio.ReadArray(stream); - _rbMetadata.RealBassTuning = RBAudio.ReadArray(stream); - - _rbMetadata.Indices = new RBAudio(stream); - _rbMetadata.Panning = new RBAudio(stream); - - _rbMetadata.Soloes = ReadStringArray(stream); - _rbMetadata.VideoVenues = ReadStringArray(stream); - - unsafe - { - fixed (RBCONDifficulties* ptr = &_rbDifficulties) + float[]? volumes = null; + float[]? pans = null; + float[]? cores = null; + + var info = ScanNode.Default; + void ParseDTA(DTAEntry entry) + { + if (entry.Name != null) { info.Metadata.Name = entry.Name; } + if (entry.Artist != null) { info.Metadata.Artist = entry.Artist; } + if (entry.Charter != null) { info.Metadata.Charter = entry.Charter; } + if (entry.Genre != null) { info.Metadata.Genre = entry.Genre; } + if (entry.YearAsNumber != null) { - var span = new Span(ptr, sizeof(RBCONDifficulties)); - stream.Read(span); + info.YearAsNumber = entry.YearAsNumber.Value; + info.Metadata.Year = info.YearAsNumber.ToString(); + } + if (entry.Source != null) { info.Metadata.Source = entry.Source; } + if (entry.Playlist != null) { info.Metadata.Playlist = entry.Playlist; } + if (entry.SongLength != null) { info.SongLength = entry.SongLength.Value; } + if (entry.IsMaster != null) { info.Metadata.IsMaster = entry.IsMaster.Value; } + if (entry.AlbumTrack != null) { info.Metadata.AlbumTrack = entry.AlbumTrack.Value; } + if (entry.PreviewStart != null) + { + info.Metadata.PreviewStart = entry.PreviewStart.Value; + info.Metadata.PreviewEnd = entry.PreviewEnd!.Value; + } + if (entry.HopoThreshold != null) { info.Settings.HopoThreshold = entry.HopoThreshold.Value; } + if (entry.SongRating != null) { info.Metadata.SongRating = entry.SongRating.Value; } + if (entry.VocalPercussionBank != null) { info.RBMetadata.VocalPercussionBank = entry.VocalPercussionBank; } + if (entry.VocalGender != null) { info.RBMetadata.VocalGender = entry.VocalGender.Value; } + if (entry.VocalSongScrollSpeed != null) { info.RBMetadata.VocalSongScrollSpeed = entry.VocalSongScrollSpeed.Value; } + if (entry.VocalTonicNote != null) { info.RBMetadata.VocalTonicNote = entry.VocalTonicNote.Value; } + if (entry.VideoVenues != null) { info.RBMetadata.VideoVenues = entry.VideoVenues; } + if (entry.DrumBank != null) { info.RBMetadata.DrumBank = entry.DrumBank; } + if (entry.SongID != null) { info.RBMetadata.SongID = entry.SongID; } + if (entry.SongTonality != null) { info.RBMetadata.SongTonality = entry.SongTonality.Value; } + if (entry.Soloes != null) { info.RBMetadata.Soloes = entry.Soloes; } + if (entry.AnimTempo != null) { info.RBMetadata.AnimTempo = entry.AnimTempo.Value; } + if (entry.TuningOffsetCents != null) { info.RBMetadata.TuningOffsetCents = entry.TuningOffsetCents.Value; } + if (entry.RealGuitarTuning != null) { info.RBMetadata.RealGuitarTuning = entry.RealGuitarTuning; } + if (entry.RealBassTuning != null) { info.RBMetadata.RealBassTuning = entry.RealBassTuning; } + + if (entry.Cores != null) { cores = entry.Cores; } + if (entry.Volumes != null) { volumes = entry.Volumes; } + if (entry.Pans != null) { pans = entry.Pans; } + + if (entry.Location != null) { info.Location = entry.Location; } + + if (entry.Indices != null) + { + var crowd = info.RBMetadata.Indices.Crowd; + info.RBMetadata.Indices = entry.Indices.Value; + info.RBMetadata.Indices.Crowd = crowd; } - } - } - protected DTAResult Init(string nodeName, in YARGTextContainer container, Dictionary> updates, Dictionary, RBProUpgrade)> upgrades, string defaultPlaylist) - { - var dtaResults = ParseDTA(nodeName, container); - ApplyRBCONUpdates(ref dtaResults, nodeName, updates); - ApplyRBProUpgrade(nodeName, upgrades); + if (entry.CrowdChannels != null) { info.RBMetadata.Indices.Crowd = entry.CrowdChannels; } - if (dtaResults.pans.Length == 0 || dtaResults.volumes.Length == 0 || dtaResults.cores.Length == 0) - { - throw new Exception("Panning & Volume mappings not set from DTA"); + if (entry.Difficulties.Band >= 0) { info.Difficulties.Band = entry.Difficulties.Band; } + if (entry.Difficulties.FiveFretGuitar >= 0) { info.Difficulties.FiveFretGuitar = entry.Difficulties.FiveFretGuitar; } + if (entry.Difficulties.FiveFretBass >= 0) { info.Difficulties.FiveFretBass = entry.Difficulties.FiveFretBass; } + if (entry.Difficulties.FiveFretRhythm >= 0) { info.Difficulties.FiveFretRhythm = entry.Difficulties.FiveFretRhythm; } + if (entry.Difficulties.FiveFretCoop >= 0) { info.Difficulties.FiveFretCoop = entry.Difficulties.FiveFretCoop; } + if (entry.Difficulties.Keys >= 0) { info.Difficulties.Keys = entry.Difficulties.Keys; } + if (entry.Difficulties.FourLaneDrums >= 0) { info.Difficulties.FourLaneDrums = entry.Difficulties.FourLaneDrums; } + if (entry.Difficulties.ProDrums >= 0) { info.Difficulties.ProDrums = entry.Difficulties.ProDrums; } + if (entry.Difficulties.ProGuitar >= 0) { info.Difficulties.ProGuitar = entry.Difficulties.ProGuitar; } + if (entry.Difficulties.ProBass >= 0) { info.Difficulties.ProBass = entry.Difficulties.ProBass; } + if (entry.Difficulties.ProKeys >= 0) { info.Difficulties.ProKeys = entry.Difficulties.ProKeys; } + if (entry.Difficulties.LeadVocals >= 0) { info.Difficulties.LeadVocals = entry.Difficulties.LeadVocals; } + if (entry.Difficulties.HarmonyVocals >= 0) { info.Difficulties.HarmonyVocals = entry.Difficulties.HarmonyVocals; } } - FinalizeRBCONAudioValues(in dtaResults); - - if (_metadata.Playlist.Length == 0) - _metadata.Playlist = defaultPlaylist; - return dtaResults; - } - - - protected virtual FixedArray LoadRawImageData() - { - return UpdateImage != null && UpdateImage.Value.Exists() - ? FixedArray.Load(UpdateImage.Value.FullName) - : FixedArray.Null; - } - protected virtual Stream? GetMoggStream() - { - if (UpdateMogg == null) + ParseDTA(baseDTA); + if (modification.UpdateDTA != null) { - return null; + ParseDTA(modification.UpdateDTA); } - var mogg = UpdateMogg.Value; - if (!File.Exists(mogg.FullName)) + if (modification.UpgradeDTA != null) { - return null; + ParseDTA(modification.UpgradeDTA); } - if (mogg.FullName.EndsWith(".yarg_mogg")) + if (info.Metadata.Name.Length == 0) { - return new YargMoggReadStream(mogg.FullName); + return (ScanResult.NoName, info); } - return new FileStream(mogg.FullName, FileMode.Open, FileAccess.Read); - } - protected FixedArray LoadUpdateMidiFile() - { - return _updateMidi != null && _updateMidi.Value.IsStillValid(false) - ? FixedArray.Load(_updateMidi.Value.FullName) - : FixedArray.Null; - } - - protected ScanResult ParseRBCONMidi(Stream? file) - { - if (_metadata.Name.Length == 0) + if (info.Location == null || pans == null || volumes == null || cores == null) { - return ScanResult.NoName; + return (ScanResult.DTAError, info); } - if (!IsMoggValid(file)) + if (info.Difficulties.FourLaneDrums > -1) { - return ScanResult.MoggError; + SetRank(ref info.Parts.FourLaneDrums.Intensity, info.Difficulties.FourLaneDrums, DrumDiffMap); + if (info.Parts.ProDrums.Intensity == -1) + { + info.Parts.ProDrums.Intensity = info.Parts.FourLaneDrums.Intensity; + } } - - try + if (info.Difficulties.FiveFretGuitar > -1) { - using var chartFile = LoadMidiFile(file); - using var updateFile = LoadUpdateMidiFile(); - using var upgradeFile = _upgrade != null ? _upgrade.LoadUpgradeMidi() : FixedArray.Null; - - DrumPreparseHandler drumTracker = new() + SetRank(ref info.Parts.FiveFretGuitar.Intensity, info.Difficulties.FiveFretGuitar, GuitarDiffMap); + if (info.Parts.ProGuitar_17Fret.Intensity == -1) { - Type = DrumsType.ProDrums - }; - - long bufLength = 0; - if (_updateMidi != null) + info.Parts.ProGuitar_22Fret.Intensity = info.Parts.ProGuitar_17Fret.Intensity = info.Parts.FiveFretGuitar.Intensity; + } + } + if (info.Difficulties.FiveFretBass > -1) + { + SetRank(ref info.Parts.FiveFretBass.Intensity, info.Difficulties.FiveFretBass, GuitarDiffMap); + if (info.Parts.ProBass_17Fret.Intensity == -1) { - if (!updateFile.IsAllocated) - return ScanResult.MissingUpdateMidi; - - if (!ParseMidi(in updateFile, drumTracker, ref _parts)) - return ScanResult.MultipleMidiTrackNames_Update; - - bufLength += updateFile.Length; + info.Parts.ProBass_22Fret.Intensity = info.Parts.ProBass_17Fret.Intensity = info.Parts.FiveFretGuitar.Intensity; } - - if (_upgrade != null) + } + if (info.Difficulties.LeadVocals > -1) + { + SetRank(ref info.Parts.LeadVocals.Intensity, info.Difficulties.LeadVocals, GuitarDiffMap); + if (info.Parts.HarmonyVocals.Intensity == -1) { - if (!upgradeFile.IsAllocated) - return ScanResult.MissingUpgradeMidi; - - if (!ParseMidi(in upgradeFile, drumTracker, ref _parts)) - return ScanResult.MultipleMidiTrackNames_Upgrade; - - bufLength += upgradeFile.Length; + info.Parts.HarmonyVocals.Intensity = info.Parts.LeadVocals.Intensity; } - - if (!chartFile.IsAllocated) - return ScanResult.MissingMidi; - - if (!ParseMidi(in chartFile, drumTracker, ref _parts)) - return ScanResult.MultipleMidiTrackNames; - - bufLength += chartFile.Length; - - SetDrums(ref _parts, drumTracker); - if (!CheckScanValidity(in _parts)) + } + if (info.Difficulties.Keys > -1) + { + SetRank(ref info.Parts.Keys.Intensity, info.Difficulties.Keys, GuitarDiffMap); + if (info.Parts.ProKeys.Intensity == -1) { - return ScanResult.NoNotes; + info.Parts.ProKeys.Intensity = info.Parts.Keys.Intensity; } - - using var buffer = FixedArray.Alloc(bufLength); - unsafe + } + if (info.Difficulties.ProGuitar > -1) + { + SetRank(ref info.Parts.ProGuitar_17Fret.Intensity, info.Difficulties.ProGuitar, RealGuitarDiffMap); + info.Parts.ProGuitar_22Fret.Intensity = info.Parts.ProGuitar_17Fret.Intensity; + if (info.Parts.FiveFretGuitar.Intensity == -1) { - System.Runtime.CompilerServices.Unsafe.CopyBlock(buffer.Ptr, chartFile.Ptr, (uint) chartFile.Length); - - long offset = chartFile.Length; - if (updateFile.IsAllocated) - { - System.Runtime.CompilerServices.Unsafe.CopyBlock(buffer.Ptr + offset, updateFile.Ptr, (uint) updateFile.Length); - offset += updateFile.Length; - } - - if (upgradeFile.IsAllocated) - { - System.Runtime.CompilerServices.Unsafe.CopyBlock(buffer.Ptr + offset, upgradeFile.Ptr, (uint) upgradeFile.Length); - } + info.Parts.FiveFretGuitar.Intensity = info.Parts.ProGuitar_17Fret.Intensity; } - _hash = HashWrapper.Hash(buffer.ReadOnlySpan); - return ScanResult.Success; } - catch + if (info.Difficulties.ProBass > -1) { - return ScanResult.PossibleCorruption; + SetRank(ref info.Parts.ProBass_17Fret.Intensity, info.Difficulties.ProBass, RealGuitarDiffMap); + info.Parts.ProBass_22Fret.Intensity = info.Parts.ProBass_17Fret.Intensity; + if (info.Parts.FiveFretBass.Intensity == -1) + { + info.Parts.FiveFretBass.Intensity = info.Parts.ProBass_17Fret.Intensity; + } } - } - - private void ApplyRBCONUpdates(ref DTAResult mainResult, string nodeName, Dictionary> updates) - { - if (updates.TryGetValue(nodeName, out var updateList)) + if (info.Difficulties.ProKeys > -1) { - foreach (var update in updateList.Values) + SetRank(ref info.Parts.ProKeys.Intensity, info.Difficulties.ProKeys, RealKeysDiffMap); + if (info.Parts.Keys.Intensity == -1) { - try - { - var updateResults = ParseDTA(nodeName, update.Containers); - Update(update, nodeName, updateResults); - - if (updateResults.cores.Length > 0) - { - mainResult.cores = updateResults.cores; - } - - if (updateResults.volumes.Length > 0) - { - mainResult.volumes = updateResults.volumes; - } - - if (updateResults.pans.Length > 0) - { - mainResult.pans = updateResults.pans; - } - } - catch (Exception ex) - { - YargLogger.LogException(ex, $"Error processing CON Update {update.BaseDirectory} - {nodeName}!"); - } + info.Parts.Keys.Intensity = info.Parts.ProKeys.Intensity; } } - } - - private void ApplyRBProUpgrade(string nodeName, Dictionary Container, RBProUpgrade Upgrade)> upgrades) - { - if (upgrades.TryGetValue(nodeName, out var upgrade)) + if (info.Difficulties.ProDrums > -1) { - try + SetRank(ref info.Parts.ProDrums.Intensity, info.Difficulties.ProDrums, DrumDiffMap); + if (info.Parts.FourLaneDrums.Intensity == -1) { - ParseDTA(nodeName, upgrade.Container); - _upgrade = upgrade.Upgrade; + info.Parts.FourLaneDrums.Intensity = info.Parts.ProDrums.Intensity; } - catch (Exception ex) + } + if (info.Difficulties.HarmonyVocals > -1) + { + SetRank(ref info.Parts.HarmonyVocals.Intensity, info.Difficulties.HarmonyVocals, DrumDiffMap); + if (info.Parts.LeadVocals.Intensity == -1) { - YargLogger.LogException(ex, $"Error processing CON Upgrade {nodeName}!"); + info.Parts.LeadVocals.Intensity = info.Parts.HarmonyVocals.Intensity; } } - } + if (info.Difficulties.Band > -1) + { + SetRank(ref info.Parts.BandDifficulty.Intensity, info.Difficulties.Band, BandDiffMap); + info.Parts.BandDifficulty.SubTracks = 1; + } - private DTAResult ParseDTA(string nodeName, params YARGTextContainer[] containers) - { - var result = DTAResult.Empty; - for (int i = 0; i < containers.Length; ++i) + unsafe { - var container = containers[i]; - while (YARGDTAReader.StartNode(ref container)) + var usedIndices = stackalloc bool[pans.Length]; + float[] CalculateStemValues(int[] indices) { - string name = YARGDTAReader.GetNameOfNode(ref container, false); - switch (name) + float[] values = new float[2 * indices.Length]; + for (int i = 0; i < indices.Length; i++) { - case "name": _metadata.Name = YARGDTAReader.ExtractText(ref container); break; - case "artist": _metadata.Artist = YARGDTAReader.ExtractText(ref container); break; - case "master": _metadata.IsMaster = YARGDTAReader.ExtractBoolean_FlippedDefault(ref container); break; - case "context": /*Context = container.Read();*/ break; - case "song": SongLoop(ref result, ref container); break; - case "song_vocals": while (YARGDTAReader.StartNode(ref container)) YARGDTAReader.EndNode(ref container); break; - case "song_scroll_speed": _rbMetadata.VocalSongScrollSpeed = YARGDTAReader.ExtractUInt32(ref container); break; - case "tuning_offset_cents": _rbMetadata.TuningOffsetCents = YARGDTAReader.ExtractInt32(ref container); break; - case "bank": _rbMetadata.VocalPercussionBank = YARGDTAReader.ExtractText(ref container); break; - case "anim_tempo": - { - string val = YARGDTAReader.ExtractText(ref container); - _rbMetadata.AnimTempo = val switch - { - "kTempoSlow" => 16, - "kTempoMedium" => 32, - "kTempoFast" => 64, - _ => uint.Parse(val) - }; - break; - } - case "preview": - _metadata.PreviewStart = YARGDTAReader.ExtractInt64(ref container); - _metadata.PreviewEnd = YARGDTAReader.ExtractInt64(ref container); - break; - case "rank": DifficultyLoop(ref container); break; - case "solo": _rbMetadata.Soloes = YARGDTAReader.ExtractArray_String(ref container); break; - case "genre": _metadata.Genre = YARGDTAReader.ExtractText(ref container); break; - case "decade": /*Decade = container.ExtractText();*/ break; - case "vocal_gender": _rbMetadata.VocalGender = YARGDTAReader.ExtractText(ref container) == "male"; break; - case "format": /*Format = container.Read();*/ break; - case "version": _rbMetadata.VenueVersion = YARGDTAReader.ExtractUInt32(ref container); break; - case "fake": /*IsFake = container.ExtractText();*/ break; - case "downloaded": /*Downloaded = container.ExtractText();*/ break; - case "game_origin": - { - string str = YARGDTAReader.ExtractText(ref container); - if ((str == "ugc" || str == "ugc_plus")) - { - if (!nodeName.StartsWith("UGC_")) - _metadata.Source = "customs"; - } - else if (str == "#ifdef") - { - string conditional = YARGDTAReader.ExtractText(ref container); - if (conditional == "CUSTOMSOURCE") - { - _metadata.Source = YARGDTAReader.ExtractText(ref container); - } - else - { - _metadata.Source = "customs"; - } - } - else - { - _metadata.Source = str; - } - - //// if the source is any official RB game or its DLC, charter = Harmonix - //if (SongSources.GetSource(str).Type == SongSources.SourceType.RB) - //{ - // _charter = "Harmonix"; - //} - - //// if the source is meant for usage in TBRB, it's a master track - //// TODO: NEVER assume localized version contains "Beatles" - //if (SongSources.SourceToGameName(str).Contains("Beatles")) _isMaster = true; - break; - } - case "song_id": _rbMetadata.SongID = YARGDTAReader.ExtractText(ref container); break; - case "rating": _metadata.SongRating = YARGDTAReader.ExtractUInt32(ref container); break; - case "short_version": /*ShortVersion = container.Read();*/ break; - case "album_art": /*HasAlbumArt = container.ExtractBoolean();*/ break; - case "year_released": - case "year_recorded": YearAsNumber = YARGDTAReader.ExtractInt32(ref container); break; - case "album_name": _metadata.Album = YARGDTAReader.ExtractText(ref container); break; - case "album_track_number": _metadata.AlbumTrack = YARGDTAReader.ExtractInt32(ref container); break; - case "pack_name": _metadata.Playlist = YARGDTAReader.ExtractText(ref container); break; - case "base_points": /*BasePoints = container.Read();*/ break; - case "band_fail_cue": /*BandFailCue = container.ExtractText();*/ break; - case "drum_bank": _rbMetadata.DrumBank = YARGDTAReader.ExtractText(ref container); break; - case "song_length": _metadata.SongLength = YARGDTAReader.ExtractUInt64(ref container); break; - case "sub_genre": /*Subgenre = container.ExtractText();*/ break; - case "author": _metadata.Charter = YARGDTAReader.ExtractText(ref container); break; - case "guide_pitch_volume": /*GuidePitchVolume = container.ReadFloat();*/ break; - case "encoding": - var encoding = YARGDTAReader.ExtractText(ref container).ToLower() switch - { - "latin1" => YARGTextReader.Latin1, - "utf-8" or - "utf8" => Encoding.UTF8, - _ => container.Encoding - }; - - if (container.Encoding != encoding) - { - string Convert(string str) - { - byte[] bytes = container.Encoding.GetBytes(str); - return encoding.GetString(bytes); - } - - if (_metadata.Name != SongMetadata.DEFAULT_NAME) - _metadata.Name = Convert(_metadata.Name); - - if (_metadata.Artist != SongMetadata.DEFAULT_ARTIST) - _metadata.Artist = Convert(_metadata.Artist); - - if (_metadata.Album != SongMetadata.DEFAULT_ALBUM) - _metadata.Album = Convert(_metadata.Album); - - if (_metadata.Genre != SongMetadata.DEFAULT_GENRE) - _metadata.Genre = Convert(_metadata.Genre); - - if (_metadata.Charter != SongMetadata.DEFAULT_CHARTER) - _metadata.Charter = Convert(_metadata.Charter); - - if (_metadata.Source != SongMetadata.DEFAULT_SOURCE) - _metadata.Source = Convert(_metadata.Source); - - if (_metadata.Playlist.Str.Length != 0) - _metadata.Playlist = Convert(_metadata.Playlist); - container.Encoding = encoding; - } - - break; - case "vocal_tonic_note": _rbMetadata.VocalTonicNote = YARGDTAReader.ExtractUInt32(ref container); break; - case "song_tonality": _rbMetadata.SongTonality = YARGDTAReader.ExtractBoolean(ref container); break; - case "alternate_path": result.alternatePath = YARGDTAReader.ExtractBoolean(ref container); break; - case "real_guitar_tuning": _rbMetadata.RealGuitarTuning = YARGDTAReader.ExtractArray_Int(ref container); break; - case "real_bass_tuning": _rbMetadata.RealBassTuning = YARGDTAReader.ExtractArray_Int(ref container); break; - case "video_venues": _rbMetadata.VideoVenues = YARGDTAReader.ExtractArray_String(ref container); break; - case "extra_authoring": - { - StringBuilder authors = new(); - foreach (string str in YARGDTAReader.ExtractArray_String(ref container)) - { - if (str == "disc_update") - result.discUpdate = true; - else if (authors.Length == 0 && _metadata.Charter == SongMetadata.DEFAULT_CHARTER) - authors.Append(str); - else - { - if (authors.Length == 0) - authors.Append(_metadata.Charter); - authors.Append(", " + str); - } - } - - if (authors.Length == 0) - authors.Append(_metadata.Charter); - - _metadata.Charter = authors.ToString(); - } - break; + float theta = (pans[indices[i]] + 1) * ((float) Math.PI / 4); + float volRatio = (float) Math.Pow(10, volumes[indices[i]] / 20); + values[2 * i] = volRatio * (float) Math.Cos(theta); + values[2 * i + 1] = volRatio * (float) Math.Sin(theta); + usedIndices[indices[i]] = true; } - YARGDTAReader.EndNode(ref container); + return values; } - } - return result; - } - private void SongLoop(ref DTAResult result, ref YARGTextContainer container) - { - while (YARGDTAReader.StartNode(ref container)) - { - string descriptor = YARGDTAReader.GetNameOfNode(ref container, false); - switch (descriptor) + if (info.RBMetadata.Indices.Drums.Length > 0) { - case "name": result.location = YARGDTAReader.ExtractText(ref container); break; - case "tracks": TracksLoop(ref container); break; - case "crowd_channels": _rbMetadata.Indices.Crowd = YARGDTAReader.ExtractArray_Int(ref container); break; - //case "vocal_parts": VocalParts = container.Read(); break; - case "pans": result.pans = YARGDTAReader.ExtractArray_Float(ref container); break; - case "vols": result.volumes = YARGDTAReader.ExtractArray_Float(ref container); break; - case "cores": result.cores = YARGDTAReader.ExtractArray_Float(ref container); break; - case "hopo_threshold": _parseSettings.HopoThreshold = YARGDTAReader.ExtractInt64(ref container); break; + info.RBMetadata.Panning.Drums = CalculateStemValues(info.RBMetadata.Indices.Drums); } - YARGDTAReader.EndNode(ref container); - } - } - private void TracksLoop(ref YARGTextContainer container) - { - var crowd = _rbMetadata.Indices.Crowd; - _rbMetadata.Indices = RBAudio.Empty; - _rbMetadata.Indices.Crowd = crowd; - while (YARGDTAReader.StartNode(ref container)) - { - while (YARGDTAReader.StartNode(ref container)) + if (info.RBMetadata.Indices.Bass.Length > 0) { - switch (YARGDTAReader.GetNameOfNode(ref container, false)) - { - case "drum" : _rbMetadata.Indices.Drums = YARGDTAReader.ExtractArray_Int(ref container); break; - case "bass" : _rbMetadata.Indices.Bass = YARGDTAReader.ExtractArray_Int(ref container); break; - case "guitar": _rbMetadata.Indices.Guitar = YARGDTAReader.ExtractArray_Int(ref container); break; - case "keys" : _rbMetadata.Indices.Keys = YARGDTAReader.ExtractArray_Int(ref container); break; - case "vocals": _rbMetadata.Indices.Vocals = YARGDTAReader.ExtractArray_Int(ref container); break; - } - YARGDTAReader.EndNode(ref container); + info.RBMetadata.Panning.Bass = CalculateStemValues(info.RBMetadata.Indices.Bass); } - YARGDTAReader.EndNode(ref container); - } - } - private static readonly int[] BandDiffMap = { 163, 215, 243, 267, 292, 345 }; - private static readonly int[] GuitarDiffMap = { 139, 176, 221, 267, 333, 409 }; - private static readonly int[] BassDiffMap = { 135, 181, 228, 293, 364, 436 }; - private static readonly int[] DrumDiffMap = { 124, 151, 178, 242, 345, 448 }; - private static readonly int[] KeysDiffMap = { 153, 211, 269, 327, 385, 443 }; - private static readonly int[] VocalsDiffMap = { 132, 175, 218, 279, 353, 427 }; - private static readonly int[] RealGuitarDiffMap = { 150, 205, 264, 323, 382, 442 }; - private static readonly int[] RealBassDiffMap = { 150, 208, 267, 325, 384, 442 }; - private static readonly int[] RealDrumsDiffMap = { 124, 151, 178, 242, 345, 448 }; - private static readonly int[] RealKeysDiffMap = { 153, 211, 269, 327, 385, 443 }; - private static readonly int[] HarmonyDiffMap = { 132, 175, 218, 279, 353, 427 }; + if (info.RBMetadata.Indices.Guitar.Length > 0) + { + info.RBMetadata.Panning.Guitar = CalculateStemValues(info.RBMetadata.Indices.Guitar); + } - private void DifficultyLoop(ref YARGTextContainer container) - { - int diff; - while (YARGDTAReader.StartNode(ref container)) - { - string name = YARGDTAReader.GetNameOfNode(ref container, false); - diff = YARGDTAReader.ExtractInt32(ref container); - switch (name) - { - case "drum": - case "drums": - _rbDifficulties.FourLaneDrums = (short) diff; - SetRank(ref _parts.FourLaneDrums.Intensity, diff, DrumDiffMap); - if (_parts.ProDrums.Intensity == -1) - { - _parts.ProDrums.Intensity = _parts.FourLaneDrums.Intensity; - } - break; - case "guitar": - _rbDifficulties.FiveFretGuitar = (short) diff; - SetRank(ref _parts.FiveFretGuitar.Intensity, diff, GuitarDiffMap); - if (_parts.ProGuitar_17Fret.Intensity == -1) - { - _parts.ProGuitar_22Fret.Intensity = _parts.ProGuitar_17Fret.Intensity = _parts.FiveFretGuitar.Intensity; - } - break; - case "bass": - _rbDifficulties.FiveFretBass = (short) diff; - SetRank(ref _parts.FiveFretBass.Intensity, diff, BassDiffMap); - if (_parts.ProBass_17Fret.Intensity == -1) - { - _parts.ProBass_22Fret.Intensity = _parts.ProBass_17Fret.Intensity = _parts.FiveFretBass.Intensity; - } - break; - case "vocals": - _rbDifficulties.LeadVocals = (short) diff; - SetRank(ref _parts.LeadVocals.Intensity, diff, VocalsDiffMap); - if (_parts.HarmonyVocals.Intensity == -1) - { - _parts.HarmonyVocals.Intensity = _parts.LeadVocals.Intensity; - } - break; - case "keys": - _rbDifficulties.Keys = (short) diff; - SetRank(ref _parts.Keys.Intensity, diff, KeysDiffMap); - if (_parts.ProKeys.Intensity == -1) - { - _parts.ProKeys.Intensity = _parts.Keys.Intensity; - } - break; - case "realGuitar": - case "real_guitar": - _rbDifficulties.ProGuitar = (short) diff; - SetRank(ref _parts.ProGuitar_17Fret.Intensity, diff, RealGuitarDiffMap); - _parts.ProGuitar_22Fret.Intensity = _parts.ProGuitar_17Fret.Intensity; - if (_parts.FiveFretGuitar.Intensity == -1) - { - _parts.FiveFretGuitar.Intensity = _parts.ProGuitar_17Fret.Intensity; - } - break; - case "realBass": - case "real_bass": - _rbDifficulties.ProBass = (short) diff; - SetRank(ref _parts.ProBass_17Fret.Intensity, diff, RealBassDiffMap); - _parts.ProBass_22Fret.Intensity = _parts.ProBass_17Fret.Intensity; - if (_parts.FiveFretBass.Intensity == -1) - { - _parts.FiveFretBass.Intensity = _parts.ProBass_17Fret.Intensity; - } - break; - case "realKeys": - case "real_keys": - _rbDifficulties.ProKeys = (short) diff; - SetRank(ref _parts.ProKeys.Intensity, diff, RealKeysDiffMap); - if (_parts.Keys.Intensity == -1) - { - _parts.Keys.Intensity = _parts.ProKeys.Intensity; - } - break; - case "realDrums": - case "real_drums": - _rbDifficulties.ProDrums = (short) diff; - SetRank(ref _parts.ProDrums.Intensity, diff, RealDrumsDiffMap); - if (_parts.FourLaneDrums.Intensity == -1) - { - _parts.FourLaneDrums.Intensity = _parts.ProDrums.Intensity; - } - break; - case "harmVocals": - case "vocal_harm": - _rbDifficulties.HarmonyVocals = (short) diff; - SetRank(ref _parts.HarmonyVocals.Intensity, diff, HarmonyDiffMap); - if (_parts.LeadVocals.Intensity == -1) - { - _parts.LeadVocals.Intensity = _parts.HarmonyVocals.Intensity; - } - break; - case "band": - _rbDifficulties.Band = (short) diff; - SetRank(ref _parts.BandDifficulty.Intensity, diff, BandDiffMap); - _parts.BandDifficulty.SubTracks = 1; - break; + if (info.RBMetadata.Indices.Keys.Length > 0) + { + info.RBMetadata.Panning.Keys = CalculateStemValues(info.RBMetadata.Indices.Keys); } - YARGDTAReader.EndNode(ref container); - } - } - private static void SetRank(ref sbyte intensity, int rank, int[] values) - { - sbyte i = 0; - while (i < 6 && values[i] <= rank) - ++i; - intensity = i; - } + if (info.RBMetadata.Indices.Vocals.Length > 0) + { + info.RBMetadata.Panning.Vocals = CalculateStemValues(info.RBMetadata.Indices.Vocals); + } - private void Update(SongUpdate update, string nodename, in DTAResult results) - { - if (results.discUpdate) - { - if (update.Midi != null) + if (info.RBMetadata.Indices.Crowd.Length > 0) { - if (_updateMidi == null || update.Midi.Value.LastUpdatedTime > _updateMidi.Value.LastUpdatedTime) + info.RBMetadata.Panning.Crowd = CalculateStemValues(info.RBMetadata.Indices.Crowd); + } + + var leftover = new List(pans.Length); + for (int i = 0; i < pans.Length; i++) + { + if (!usedIndices[i]) { - _updateMidi = update.Midi; + leftover.Add(i); } } - else + + if (leftover.Count > 0) { - YargLogger.LogFormatWarning("Update midi expected in directory {0}", Path.Combine(update.BaseDirectory, nodename)); + info.RBMetadata.Indices.Track = leftover.ToArray(); + info.RBMetadata.Panning.Track = CalculateStemValues(info.RBMetadata.Indices.Track); } } - if (update.Mogg != null) + return (ScanResult.Success, info); + } + + protected static (ScanResult Result, HashWrapper Hash) ParseRBCONMidi(in FixedArray mainMidi, CONModification modification, ref ScanNode info) + { + try { - if (UpdateMogg == null || update.Mogg.Value.LastUpdatedTime > UpdateMogg.Value.LastUpdatedTime) + DrumPreparseHandler drumTracker = new() { - UpdateMogg = update.Mogg; - } - } + Type = DrumsType.ProDrums + }; - if (update.Milo != null) - { - if (UpdateMilo == null || update.Milo.Value.LastUpdatedTime > UpdateMilo.Value.LastUpdatedTime) + using var updateMidi = modification.Midi.HasValue ? FixedArray.Load(modification.Midi.Value.FullName) : FixedArray.Null; + using var upgradeMidi = modification.UpgradeNode != null ? modification.UpgradeNode.LoadUpgradeMidi() : FixedArray.Null; + if (modification.UpgradeNode != null && !upgradeMidi.IsAllocated) { - UpdateMilo = update.Milo; + throw new FileNotFoundException("Upgrade midi not located"); } - } - if (results.alternatePath) - { - if (update.Image != null) + long bufLength = mainMidi.Length; + if (updateMidi.IsAllocated) { - if (UpdateImage == null || update.Image.Value.LastUpdatedTime > UpdateImage.Value.LastUpdatedTime) + switch (ParseMidi(in updateMidi, drumTracker, ref info.Parts).Result) { - UpdateImage = update.Image; + case ScanResult.ZeroResolution: return (ScanResult.ZeroResolution_Update, default); + case ScanResult.MultipleMidiTrackNames: return (ScanResult.MultipleMidiTrackNames_Update, default); } + bufLength += updateMidi.Length; } - } - } - - private void FinalizeRBCONAudioValues(in DTAResult result) - { - HashSet pending = new(); - for (int i = 0; i < result.pans.Length; i++) - pending.Add(i); - if (_rbMetadata.Indices.Drums.Length > 0) - _rbMetadata.Panning.Drums = CalculateStemValues(_rbMetadata.Indices.Drums, in result, pending); + if (upgradeMidi.IsAllocated) + { + switch (ParseMidi(in upgradeMidi, drumTracker, ref info.Parts).Result) + { + case ScanResult.ZeroResolution: return (ScanResult.ZeroResolution_Upgrade, default); + case ScanResult.MultipleMidiTrackNames: return (ScanResult.MultipleMidiTrackNames_Upgrade, default); + } + bufLength += upgradeMidi.Length; + } - if (_rbMetadata.Indices.Bass.Length > 0) - _rbMetadata.Panning.Bass = CalculateStemValues(_rbMetadata.Indices.Bass, in result, pending); + var (result, resolution) = ParseMidi(in mainMidi, drumTracker, ref info.Parts); + if (result != ScanResult.Success) + { + return (result, default); + } - if (_rbMetadata.Indices.Guitar.Length > 0) - _rbMetadata.Panning.Guitar = CalculateStemValues(_rbMetadata.Indices.Guitar, in result, pending); + SetDrums(ref info.Parts, drumTracker); + if (!CheckScanValidity(in info.Parts)) + { + return (ScanResult.NoNotes, default); + } - if (_rbMetadata.Indices.Keys.Length > 0) - _rbMetadata.Panning.Keys = CalculateStemValues(_rbMetadata.Indices.Keys, in result, pending); + info.Settings.SustainCutoffThreshold = resolution / 3; + if (info.Settings.HopoThreshold == -1) + { + info.Settings.HopoThreshold = info.Settings.SustainCutoffThreshold; + } - if (_rbMetadata.Indices.Vocals.Length > 0) - _rbMetadata.Panning.Vocals = CalculateStemValues(_rbMetadata.Indices.Vocals, in result, pending); + using var buffer = FixedArray.Alloc(bufLength); + unsafe + { + System.Runtime.CompilerServices.Unsafe.CopyBlock(buffer.Ptr, mainMidi.Ptr, (uint) mainMidi.Length); - if (_rbMetadata.Indices.Crowd.Length > 0) - _rbMetadata.Panning.Crowd = CalculateStemValues(_rbMetadata.Indices.Crowd, in result, pending); + long offset = mainMidi.Length; + if (updateMidi.IsAllocated) + { + System.Runtime.CompilerServices.Unsafe.CopyBlock(buffer.Ptr + offset, updateMidi.Ptr, (uint) updateMidi.Length); + offset += updateMidi.Length; + } - if (pending.Count > 0) + if (upgradeMidi.IsAllocated) + { + System.Runtime.CompilerServices.Unsafe.CopyBlock(buffer.Ptr + offset, upgradeMidi.Ptr, (uint) upgradeMidi.Length); + } + } + return (ScanResult.Success, HashWrapper.Hash(buffer.ReadOnlySpan)); + } + catch (Exception ex) { - _rbMetadata.Indices.Track = pending.ToArray(); - _rbMetadata.Panning.Track = CalculateStemValues(_rbMetadata.Indices.Track, in result, pending); + YargLogger.LogException(ex); + return (ScanResult.PossibleCorruption, default); } } - private float[] CalculateStemValues(int[] indices, in DTAResult result, HashSet pending) + protected static Stream? LoadUpdateMoggStream(in AbridgedFileInfo? info) { - float[] values = new float[2 * indices.Length]; - for (int i = 0; i < indices.Length; i++) + if (info == null) { - int index = indices[i]; - float theta = (result.pans[index] + 1) * ((float) Math.PI / 4); - float volRatio = (float) Math.Pow(10, result.volumes[index] / 20); - values[2 * i] = volRatio * (float) Math.Cos(theta); - values[2 * i + 1] = volRatio * (float) Math.Sin(theta); - pending.Remove(index); + return null; } - return values; - } - private static AbridgedFileInfo? ReadUpdateInfo(UnmanagedMemoryStream stream) - { - if (!stream.ReadBoolean()) + var mogg = info.Value; + if (!File.Exists(mogg.FullName)) { return null; } - return new AbridgedFileInfo(stream.ReadString(), false); - } - private static string[] ReadStringArray(UnmanagedMemoryStream stream) - { - int length = stream.Read(Endianness.Little); - if (length == 0) + if (mogg.FullName.EndsWith(".yarg_mogg")) { - return Array.Empty(); + return new YargMoggReadStream(mogg.FullName); } - - var strings = new string[length]; - for (int i = 0; i < length; ++i) - strings[i] = stream.ReadString(); - return strings; + return new FileStream(mogg.FullName, FileMode.Open, FileAccess.Read); } - private static void WriteUpdateInfo(in AbridgedFileInfo? info, BinaryWriter writer) + private static readonly int[] BandDiffMap = { 163, 215, 243, 267, 292, 345 }; + private static readonly int[] GuitarDiffMap = { 139, 176, 221, 267, 333, 409 }; + private static readonly int[] BassDiffMap = { 135, 181, 228, 293, 364, 436 }; + private static readonly int[] DrumDiffMap = { 124, 151, 178, 242, 345, 448 }; + private static readonly int[] KeysDiffMap = { 153, 211, 269, 327, 385, 443 }; + private static readonly int[] VocalsDiffMap = { 132, 175, 218, 279, 353, 427 }; + private static readonly int[] RealGuitarDiffMap = { 150, 205, 264, 323, 382, 442 }; + private static readonly int[] RealBassDiffMap = { 150, 208, 267, 325, 384, 442 }; + private static readonly int[] RealDrumsDiffMap = { 124, 151, 178, 242, 345, 448 }; + private static readonly int[] RealKeysDiffMap = { 153, 211, 269, 327, 385, 443 }; + private static readonly int[] HarmonyDiffMap = { 132, 175, 218, 279, 353, 427 }; + + private static void SetRank(ref sbyte intensity, int rank, int[] values) { - if (info != null) + sbyte i = 0; + while (i < 6 && values[i] <= rank) { - writer.Write(true); - writer.Write(info.Value.FullName); + ++i; } - else - writer.Write(false); + intensity = i; } - private static void WriteStringArray(string[] strings, BinaryWriter writer) + private static void WriteUpdateInfo(in AbridgedFileInfo? info, MemoryStream stream) { - writer.Write(strings.Length); - for (int i = 0; i < strings.Length; ++i) + stream.Write(info != null); + if (info != null) { - writer.Write(strings[i]); + stream.Write(info.Value.FullName); } } } diff --git a/YARG.Core/Song/Entries/RBCON/SongEntry.UnpackedRBCON.cs b/YARG.Core/Song/Entries/RBCON/SongEntry.UnpackedRBCON.cs index fb64bbdd7..86868f8e9 100644 --- a/YARG.Core/Song/Entries/RBCON/SongEntry.UnpackedRBCON.cs +++ b/YARG.Core/Song/Entries/RBCON/SongEntry.UnpackedRBCON.cs @@ -1,10 +1,8 @@ using System; using System.IO; -using System.Collections.Generic; using YARG.Core.Extensions; using YARG.Core.Song.Cache; using YARG.Core.IO; -using YARG.Core.Logging; using YARG.Core.Venue; namespace YARG.Core.Song @@ -21,31 +19,48 @@ public sealed class UnpackedRBCONEntry : RBCONEntry public override string DirectoryActual => Location; public override EntryType SubType => EntryType.ExCON; - public static (ScanResult, UnpackedRBCONEntry?) ProcessNewEntry(UnpackedCONGroup group, string nodename, in YARGTextContainer container, Dictionary> updates, Dictionary, RBProUpgrade)> upgrades) + public static (ScanResult, UnpackedRBCONEntry?) ProcessNewEntry(UnpackedCONGroup group, string nodename, DTAEntry node, CONModification modification) { - try + var (dtaResult, info) = ProcessDTAs(nodename, node, modification); + if (dtaResult != ScanResult.Success) { - var song = new UnpackedRBCONEntry(group, nodename, in container, updates, upgrades); - if (song._midi == null) - { - return (ScanResult.MissingMidi, null); - } + return (dtaResult, null); + } - var result = song.ParseRBCONMidi(null); - if (result != ScanResult.Success) - { - return (result, null); - } - return (result, song); + if (!info.Location!.StartsWith("songs/" + nodename)) + { + nodename = info.Location!.Split('/')[1]; + } + + string directory = Path.Combine(group.Location, nodename); + if (!IsMoggValid(in modification.Mogg, directory, nodename)) + { + return (ScanResult.MoggError, null); + } + + var midiInfo = new FileInfo(Path.Combine(directory, nodename + ".mid")); + if (!midiInfo.Exists) + { + return (ScanResult.MissingCONMidi, null); + } + + using var mainMidi = FixedArray.Load(midiInfo.FullName); + var (midiResult, hash) = ParseRBCONMidi(in mainMidi, modification, ref info); + if (midiResult != ScanResult.Success) + { + return (midiResult, null); } - catch (Exception ex) + + if (info.Metadata.Playlist.Length == 0) { - YargLogger.LogException(ex, null); - return (ScanResult.DTAError, null); + info.Metadata.Playlist = group.DefaultPlaylist; } + + var entry = new UnpackedRBCONEntry(info, modification, in hash, directory, nodename, midiInfo, in group.Info); + return (ScanResult.Success, entry); } - public static UnpackedRBCONEntry? TryLoadFromCache(string directory, in AbridgedFileInfo dta, string nodename, Dictionary, RBProUpgrade Upgrade)> upgrades, UnmanagedMemoryStream stream, CategoryCacheStrings strings) + public static UnpackedRBCONEntry? TryLoadFromCache(string directory, in AbridgedFileInfo dta, string nodename, RBProUpgrade? upgrade, UnmanagedMemoryStream stream, CategoryCacheStrings strings) { string subname = stream.ReadString(); string songDirectory = Path.Combine(directory, subname); @@ -66,12 +81,10 @@ public static (ScanResult, UnpackedRBCONEntry?) ProcessNewEntry(UnpackedCONGroup return null; } } - - var upgrade = upgrades.TryGetValue(nodename, out var node) ? node.Upgrade : null; return new UnpackedRBCONEntry(midiInfo.Value, dta, songDirectory, subname, updateMidi, upgrade, stream, strings); } - public static UnpackedRBCONEntry LoadFromCache_Quick(string directory, in AbridgedFileInfo? dta, string nodename, Dictionary, RBProUpgrade Upgrade)> upgrades, UnmanagedMemoryStream stream, CategoryCacheStrings strings) + public static UnpackedRBCONEntry LoadFromCache_Quick(string directory, in AbridgedFileInfo? dta, RBProUpgrade? upgrade, UnmanagedMemoryStream stream, CategoryCacheStrings strings) { string subname = stream.ReadString(); string songDirectory = Path.Combine(directory, subname); @@ -80,11 +93,21 @@ public static UnpackedRBCONEntry LoadFromCache_Quick(string directory, in Abridg var midiInfo = new AbridgedFileInfo(midiPath, stream); AbridgedFileInfo? updateMidi = stream.ReadBoolean() ? new AbridgedFileInfo(stream) : null; - - var upgrade = upgrades.TryGetValue(nodename, out var node) ? node.Upgrade : null; return new UnpackedRBCONEntry(midiInfo, dta, songDirectory, subname, updateMidi, upgrade, stream, strings); } + private static bool IsMoggValid(in AbridgedFileInfo? info, string directory, string nodename) + { + using var stream = LoadMoggStream(in info, directory, nodename); + if (stream == null) + { + return false; + } + + int version = stream.Read(Endianness.Little); + return version == 0x0A || version == 0xf0; + } + private UnpackedRBCONEntry(AbridgedFileInfo midi, AbridgedFileInfo? dta, string directory, string nodename, AbridgedFileInfo? updateMidi, RBProUpgrade? upgrade, UnmanagedMemoryStream stream, CategoryCacheStrings strings) : base(updateMidi, upgrade, stream, strings) @@ -96,33 +119,35 @@ private UnpackedRBCONEntry(AbridgedFileInfo midi, AbridgedFileInfo? dta, string _nodename = nodename; } - private UnpackedRBCONEntry(UnpackedCONGroup group, string nodename, in YARGTextContainer container, Dictionary> updates, Dictionary, RBProUpgrade)> upgrades) - : base() + private UnpackedRBCONEntry(in ScanNode info, CONModification modification, in HashWrapper hash + , string directory, string nodename, FileInfo midiInfo, in AbridgedFileInfo dtaInfo) + : base(in info, modification, in hash) { - var results = Init(nodename, in container, updates, upgrades, group.DefaultPlaylist); - if (!results.location.StartsWith($"songs/" + nodename)) - nodename = results.location.Split('/')[1]; + Location = directory; _nodename = nodename; + _midi = new AbridgedFileInfo(midiInfo); + _dta = dtaInfo; + } - Location = Path.Combine(group.Location, nodename); - string midiPath = Path.Combine(Location, nodename + ".mid"); + public override void Serialize(MemoryStream stream, CategoryCacheWriteNode node) + { + stream.Write(_nodename); + stream.Write(_midi!.Value.LastUpdatedTime.ToBinary(), Endianness.Little); + stream.Write(UpdateMidi != null); + UpdateMidi?.Serialize(stream); + base.Serialize(stream, node); + } - FileInfo midiInfo = new(midiPath); - if (!midiInfo.Exists) + public override YARGImage? LoadAlbumData() + { + if (UpdateImage != null && UpdateImage.Value.Exists()) { - return; + var update = FixedArray.Load(UpdateImage.Value.FullName); + return new YARGImage(update); } - _midi = new AbridgedFileInfo(midiInfo); - _dta = group.DTA; - } - - public override void Serialize(BinaryWriter writer, CategoryCacheWriteNode node) - { - writer.Write(_nodename); - var info = _midi!.Value; - writer.Write(info.LastUpdatedTime.ToBinary()); - base.Serialize(writer, node); + string imgFilename = Path.Combine(Location, "gen", _nodename + "_keep.png_xbox"); + return File.Exists(imgFilename) ? new YARGImage(FixedArray.Load(imgFilename)) : null; } public override BackgroundResult? LoadBackground(BackgroundType options) @@ -179,10 +204,9 @@ public override void Serialize(BinaryWriter writer, CategoryCacheWriteNode node) public override FixedArray LoadMiloData() { - var bytes = base.LoadMiloData(); - if (bytes.IsAllocated) + if (UpdateMilo != null && UpdateMilo.Value.Exists()) { - return bytes; + return FixedArray.Load(UpdateMilo.Value.FullName); } string filename = Path.Combine(Location, "gen", _nodename + ".milo_xbox"); @@ -198,57 +222,27 @@ public override FixedArray LoadMiloData() return new FileStream(_midi.Value.FullName, FileMode.Open, FileAccess.Read, FileShare.Read); } - protected override FixedArray LoadMidiFile(Stream? file) + protected override Stream? GetMoggStream() { - return _dta != null && _dta.Value.IsStillValid() && _midi!.Value.IsStillValid() - ? FixedArray.Load(_midi.Value.FullName) - : FixedArray.Null; + return LoadMoggStream(in UpdateMogg, Location, _nodename); } - protected override FixedArray LoadRawImageData() + private static Stream? LoadMoggStream(in AbridgedFileInfo? updateMogg, string directory, string nodename) { - var bytes = base.LoadRawImageData(); - if (bytes.IsAllocated) - { - return bytes; - } - - string filename = Path.Combine(Location, "gen", _nodename + "_keep.png_xbox"); - return File.Exists(filename) ? FixedArray.Load(filename) : FixedArray.Null; - } - - protected override Stream? GetMoggStream() - { - var stream = base.GetMoggStream(); + var stream = LoadUpdateMoggStream(in updateMogg); if (stream != null) { return stream; } - string path = Path.Combine(Location, _nodename + ".yarg_mogg"); + string path = Path.Combine(directory, nodename + ".yarg_mogg"); if (File.Exists(path)) { return new YargMoggReadStream(path); } - path = Path.Combine(Location, _nodename + ".mogg"); - if (!File.Exists(path)) - { - return null; - } - return new FileStream(path, FileMode.Open, FileAccess.Read); - } - - protected override bool IsMoggValid(Stream? file) - { - using var stream = GetMoggStream(); - if (stream == null) - { - return false; - } - - int version = stream.Read(Endianness.Little); - return version == 0x0A || version == 0xf0; + path = Path.Combine(directory, nodename + ".mogg"); + return File.Exists(path) ? File.OpenRead(path) : null; } } } diff --git a/YARG.Core/Song/Entries/SongEntry.Loading.cs b/YARG.Core/Song/Entries/SongEntry.Loading.cs index c1787a25e..244185df9 100644 --- a/YARG.Core/Song/Entries/SongEntry.Loading.cs +++ b/YARG.Core/Song/Entries/SongEntry.Loading.cs @@ -1,12 +1,8 @@ using System; -using System.Collections.Generic; using System.IO; -using System.Threading; -using System.Threading.Tasks; using YARG.Core.Audio; using YARG.Core.Chart; using YARG.Core.IO; -using YARG.Core.Logging; using YARG.Core.Venue; namespace YARG.Core.Song diff --git a/YARG.Core/Song/Entries/SongEntry.Scanning.cs b/YARG.Core/Song/Entries/SongEntry.Scanning.cs index ea6e959e8..693ee5fbf 100644 --- a/YARG.Core/Song/Entries/SongEntry.Scanning.cs +++ b/YARG.Core/Song/Entries/SongEntry.Scanning.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; +using System.Text; using YARG.Core.Chart; using YARG.Core.IO; using YARG.Core.Song.Preparsers; @@ -10,9 +7,14 @@ namespace YARG.Core.Song { public abstract partial class SongEntry { - protected static bool ParseMidi(in FixedArray file, DrumPreparseHandler drums, ref AvailableParts parts) + protected static (ScanResult Result, long Resolution) ParseMidi(in FixedArray file, DrumPreparseHandler drums, ref AvailableParts parts) { var midiFile = new YARGMidiFile(file.ToStream()); + if (midiFile.Resolution == 0) + { + return (ScanResult.ZeroResolution, 0); + } + bool harm2 = false; bool harm3 = false; foreach (var track in midiFile) @@ -22,10 +24,14 @@ protected static bool ParseMidi(in FixedArray file, DrumPreparseHandler dr var trackname = track.FindTrackName(Encoding.ASCII); if (trackname == null) - return false; + { + return (ScanResult.MultipleMidiTrackNames, 0); + } if (!YARGMidiTrack.TRACKNAMES.TryGetValue(trackname, out var type)) + { continue; + } switch (type) { @@ -73,18 +79,22 @@ protected static bool ParseMidi(in FixedArray file, DrumPreparseHandler dr parts.HarmonyVocals.SetSubtrack(2); } } - return true; + return (ScanResult.Success, midiFile.Resolution); } protected static void SetDrums(ref AvailableParts parts, DrumPreparseHandler drumTracker) { if (drumTracker.Type == DrumsType.FiveLane) + { parts.FiveLaneDrums.Difficulties = drumTracker.ValidatedDiffs; + } else { parts.FourLaneDrums.Difficulties = drumTracker.ValidatedDiffs; if (drumTracker.Type == DrumsType.ProDrums) + { parts.ProDrums.Difficulties = drumTracker.ValidatedDiffs; + } } } diff --git a/YARG.Core/Song/Entries/SongEntry.Sorting.cs b/YARG.Core/Song/Entries/SongEntry.Sorting.cs index 06806b1fe..8d33c0534 100644 --- a/YARG.Core/Song/Entries/SongEntry.Sorting.cs +++ b/YARG.Core/Song/Entries/SongEntry.Sorting.cs @@ -33,10 +33,7 @@ public int CompareTo(SongEntry other) return strCmp; } - public virtual DateTime GetAddTime() - { - return DateTime.MinValue; - } + public abstract DateTime GetAddDate(); public bool IsPreferedOver(SongEntry other) { diff --git a/YARG.Core/Song/Entries/SongEntry.cs b/YARG.Core/Song/Entries/SongEntry.cs index f70c8f9bf..37b60144c 100644 --- a/YARG.Core/Song/Entries/SongEntry.cs +++ b/YARG.Core/Song/Entries/SongEntry.cs @@ -1,8 +1,6 @@ using System; using System.IO; -using YARG.Core.Chart; using YARG.Core.Extensions; -using YARG.Core.IO.Ini; using YARG.Core.Song.Cache; namespace YARG.Core.Song @@ -20,12 +18,14 @@ public enum ScanResult DTAError, MoggError, UnsupportedEncryption, - MissingMidi, - MissingUpdateMidi, - MissingUpgradeMidi, + MissingCONMidi, PossibleCorruption, FailedSngLoad, + ZeroResolution, + ZeroResolution_Update, + ZeroResolution_Upgrade, + NoAudio, PathTooLong, MultipleMidiTrackNames, @@ -38,7 +38,7 @@ public enum ScanResult /// /// The type of chart file to read. /// - public enum ChartType + public enum ChartFormat { Mid, Midi, @@ -53,6 +53,20 @@ public enum EntryType CON, } + public struct LoaderSettings + { + public static readonly LoaderSettings Default = new() + { + HopoThreshold = -1, + SustainCutoffThreshold = -1, + OverdiveMidiNote = 116 + }; + + public long HopoThreshold; + public long SustainCutoffThreshold; + public int OverdiveMidiNote; + } + /// /// The metadata for a song. /// @@ -71,7 +85,6 @@ public enum EntryType [Serializable] public abstract partial class SongEntry { - public const double MILLISECOND_FACTOR = 1000.0; protected static readonly string[] BACKGROUND_FILENAMES = { "bg", "background", "video" @@ -91,150 +104,89 @@ public abstract partial class SongEntry protected static readonly string YARGROUND_FULLNAME = "bg.yarground"; protected static readonly Random BACKROUND_RNG = new(); - private string _parsedYear; - private int _intYear; - - protected SongMetadata _metadata; - protected AvailableParts _parts; - protected ParseSettings _parseSettings; - protected HashWrapper _hash; + public readonly SongMetadata Metadata; + public readonly AvailableParts Parts; + public readonly HashWrapper Hash; + public readonly LoaderSettings Settings; public abstract string Location { get; } public abstract string DirectoryActual { get; } - public abstract EntryType SubType { get; } + public abstract string Year { get; } + public abstract int YearAsNumber { get; } + public abstract ulong SongLengthMilliseconds { get; } + public abstract bool LoopVideo { get; } - public SortString Name => _metadata.Name; - public SortString Artist => _metadata.Artist; - public SortString Album => _metadata.Album; - public SortString Genre => _metadata.Genre; - public SortString Charter => _metadata.Charter; - public SortString Source => _metadata.Source; - public SortString Playlist => _metadata.Playlist; + public SortString Name => Metadata.Name; + public SortString Artist => Metadata.Artist; + public SortString Album => Metadata.Album; + public SortString Genre => Metadata.Genre; + public SortString Charter => Metadata.Charter; + public SortString Source => Metadata.Source; + public SortString Playlist => Metadata.Playlist; - public string Year => _parsedYear; + public string UnmodifiedYear => Metadata.Year; - public string UnmodifiedYear => _metadata.Year; + public bool IsMaster => Metadata.IsMaster; - public int YearAsNumber - { - get => _intYear; - set - { - _intYear = value; - _parsedYear = _metadata.Year = value.ToString(); - } - } + public int AlbumTrack => Metadata.AlbumTrack; - public bool IsMaster => _metadata.IsMaster; + public int PlaylistTrack => Metadata.PlaylistTrack; - public int AlbumTrack => _metadata.AlbumTrack; + public string LoadingPhrase => Metadata.LoadingPhrase; + + public string CreditWrittenBy => Metadata.CreditWrittenBy; + + public string CreditPerformedBy => Metadata.CreditPerformedBy; + + public string CreditCourtesyOf => Metadata.CreditCourtesyOf; + + public string CreditAlbumCover => Metadata.CreditAlbumCover; + + public string CreditLicense => Metadata.CreditLicense; - public int PlaylistTrack => _metadata.PlaylistTrack; + public long SongOffsetMilliseconds => Metadata.SongOffset; - public string LoadingPhrase => _metadata.LoadingPhrase; + public long PreviewStartMilliseconds => Metadata.PreviewStart; - public string CreditWrittenBy => _metadata.CreditWrittenBy; - public string CreditPerformedBy => _metadata.CreditPerformedBy; - public string CreditCourtesyOf => _metadata.CreditCourtesyOf; - public string CreditAlbumCover => _metadata.CreditAlbumCover; - public string CreditLicense => _metadata.CreditLicense; + public long PreviewEndMilliseconds => Metadata.PreviewEnd; - public ulong SongLengthMilliseconds - { - get => _metadata.SongLength; - set => _metadata.SongLength = value; - } + public long VideoStartTimeMilliseconds => Metadata.VideoStartTime; - public long SongOffsetMilliseconds - { - get => _metadata.SongOffset; - set => _metadata.SongOffset = value; - } + public long VideoEndTimeMilliseconds => Metadata.VideoEndTime; - public double SongLengthSeconds - { - get => _metadata.SongLength / MILLISECOND_FACTOR; - set => _metadata.SongLength = (ulong) (value * MILLISECOND_FACTOR); - } - - public double SongOffsetSeconds - { - get => _metadata.SongOffset / MILLISECOND_FACTOR; - set => _metadata.SongOffset = (long) (value * MILLISECOND_FACTOR); - } + public double SongLengthSeconds => SongLengthMilliseconds / SongMetadata.MILLISECOND_FACTOR; - public long PreviewStartMilliseconds - { - get => _metadata.PreviewStart; - set => _metadata.PreviewStart = value; - } + public double SongOffsetSeconds => SongOffsetMilliseconds / SongMetadata.MILLISECOND_FACTOR; - public long PreviewEndMilliseconds - { - get => _metadata.PreviewEnd; - set => _metadata.PreviewEnd = value; - } + public double PreviewStartSeconds => PreviewStartMilliseconds / SongMetadata.MILLISECOND_FACTOR; - public double PreviewStartSeconds - { - get => _metadata.PreviewStart / MILLISECOND_FACTOR; - set => _metadata.PreviewStart = (long) (value * MILLISECOND_FACTOR); - } + public double PreviewEndSeconds => PreviewEndMilliseconds / SongMetadata.MILLISECOND_FACTOR; - public double PreviewEndSeconds - { - get => _metadata.PreviewEnd / MILLISECOND_FACTOR; - set => _metadata.PreviewEnd = (long) (value * MILLISECOND_FACTOR); - } + public double VideoStartTimeSeconds => VideoStartTimeMilliseconds / SongMetadata.MILLISECOND_FACTOR; - public long VideoStartTimeMilliseconds - { - get => _metadata.VideoStartTime; - set => _metadata.VideoStartTime = value; - } - - public long VideoEndTimeMilliseconds - { - get => _metadata.VideoEndTime; - set => _metadata.VideoEndTime = value; - } - - public double VideoStartTimeSeconds - { - get => _metadata.VideoStartTime / MILLISECOND_FACTOR; - set => _metadata.VideoStartTime = (long) (value * MILLISECOND_FACTOR); - } - - public double VideoEndTimeSeconds - { - get => _metadata.VideoEndTime >= 0 ? _metadata.VideoEndTime / MILLISECOND_FACTOR : -1; - set => _metadata.VideoEndTime = value >= 0 ? (long) (value * MILLISECOND_FACTOR) : -1; - } - - public HashWrapper Hash => _hash; + public double VideoEndTimeSeconds => VideoEndTimeMilliseconds >= 0 ? VideoEndTimeMilliseconds / SongMetadata.MILLISECOND_FACTOR : -1; public int VocalsCount { get { - if (_parts.HarmonyVocals[2]) + if (Parts.HarmonyVocals[2]) { return 3; } - if (_parts.HarmonyVocals[1]) + if (Parts.HarmonyVocals[1]) { return 2; } - return _parts.HarmonyVocals[0] || _parts.LeadVocals[0] ? 1 : 0; + return Parts.HarmonyVocals[0] || Parts.LeadVocals[0] ? 1 : 0; } } + public sbyte BandDifficulty => Parts.BandDifficulty.Intensity; - public sbyte BandDifficulty => _parts.BandDifficulty.Intensity; - - public override string ToString() { return _metadata.Artist + " | " + _metadata.Name; } + public override string ToString() { return Artist + " | " + Name; } public PartValues this[Instrument instrument] { @@ -242,35 +194,35 @@ public PartValues this[Instrument instrument] { return instrument switch { - Instrument.FiveFretGuitar => _parts.FiveFretGuitar, - Instrument.FiveFretBass => _parts.FiveFretBass, - Instrument.FiveFretRhythm => _parts.FiveFretRhythm, - Instrument.FiveFretCoopGuitar => _parts.FiveFretCoopGuitar, - Instrument.Keys => _parts.Keys, + Instrument.FiveFretGuitar => Parts.FiveFretGuitar, + Instrument.FiveFretBass => Parts.FiveFretBass, + Instrument.FiveFretRhythm => Parts.FiveFretRhythm, + Instrument.FiveFretCoopGuitar => Parts.FiveFretCoopGuitar, + Instrument.Keys => Parts.Keys, - Instrument.SixFretGuitar => _parts.SixFretGuitar, - Instrument.SixFretBass => _parts.SixFretBass, - Instrument.SixFretRhythm => _parts.SixFretRhythm, - Instrument.SixFretCoopGuitar => _parts.SixFretCoopGuitar, + Instrument.SixFretGuitar => Parts.SixFretGuitar, + Instrument.SixFretBass => Parts.SixFretBass, + Instrument.SixFretRhythm => Parts.SixFretRhythm, + Instrument.SixFretCoopGuitar => Parts.SixFretCoopGuitar, - Instrument.FourLaneDrums => _parts.FourLaneDrums, - Instrument.FiveLaneDrums => _parts.FiveLaneDrums, - Instrument.ProDrums => _parts.ProDrums, + Instrument.FourLaneDrums => Parts.FourLaneDrums, + Instrument.FiveLaneDrums => Parts.FiveLaneDrums, + Instrument.ProDrums => Parts.ProDrums, - Instrument.EliteDrums => _parts.EliteDrums, + Instrument.EliteDrums => Parts.EliteDrums, - Instrument.ProGuitar_17Fret => _parts.ProGuitar_17Fret, - Instrument.ProGuitar_22Fret => _parts.ProGuitar_22Fret, - Instrument.ProBass_17Fret => _parts.ProBass_17Fret, - Instrument.ProBass_22Fret => _parts.ProBass_22Fret, + Instrument.ProGuitar_17Fret => Parts.ProGuitar_17Fret, + Instrument.ProGuitar_22Fret => Parts.ProGuitar_22Fret, + Instrument.ProBass_17Fret => Parts.ProBass_17Fret, + Instrument.ProBass_22Fret => Parts.ProBass_22Fret, - Instrument.ProKeys => _parts.ProKeys, + Instrument.ProKeys => Parts.ProKeys, // Instrument.Dj => DJ, - Instrument.Vocals => _parts.LeadVocals, - Instrument.Harmony => _parts.HarmonyVocals, - Instrument.Band => _parts.BandDifficulty, + Instrument.Vocals => Parts.LeadVocals, + Instrument.Harmony => Parts.HarmonyVocals, + Instrument.Band => Parts.BandDifficulty, _ => throw new NotImplementedException($"Unhandled instrument {instrument}!") }; @@ -281,328 +233,76 @@ public bool HasInstrument(Instrument instrument) { return instrument switch { - Instrument.FiveFretGuitar => _parts.FiveFretGuitar.SubTracks > 0, - Instrument.FiveFretBass => _parts.FiveFretBass.SubTracks > 0, - Instrument.FiveFretRhythm => _parts.FiveFretRhythm.SubTracks > 0, - Instrument.FiveFretCoopGuitar => _parts.FiveFretCoopGuitar.SubTracks > 0, - Instrument.Keys => _parts.Keys.SubTracks > 0, + Instrument.FiveFretGuitar => Parts.FiveFretGuitar.SubTracks > 0, + Instrument.FiveFretBass => Parts.FiveFretBass.SubTracks > 0, + Instrument.FiveFretRhythm => Parts.FiveFretRhythm.SubTracks > 0, + Instrument.FiveFretCoopGuitar => Parts.FiveFretCoopGuitar.SubTracks > 0, + Instrument.Keys => Parts.Keys.SubTracks > 0, - Instrument.SixFretGuitar => _parts.SixFretGuitar.SubTracks > 0, - Instrument.SixFretBass => _parts.SixFretBass.SubTracks > 0, - Instrument.SixFretRhythm => _parts.SixFretRhythm.SubTracks > 0, - Instrument.SixFretCoopGuitar => _parts.SixFretCoopGuitar.SubTracks > 0, + Instrument.SixFretGuitar => Parts.SixFretGuitar.SubTracks > 0, + Instrument.SixFretBass => Parts.SixFretBass.SubTracks > 0, + Instrument.SixFretRhythm => Parts.SixFretRhythm.SubTracks > 0, + Instrument.SixFretCoopGuitar => Parts.SixFretCoopGuitar.SubTracks > 0, - Instrument.FourLaneDrums => _parts.FourLaneDrums.SubTracks > 0, - Instrument.FiveLaneDrums => _parts.FiveLaneDrums.SubTracks > 0, - Instrument.ProDrums => _parts.ProDrums.SubTracks > 0, + Instrument.FourLaneDrums => Parts.FourLaneDrums.SubTracks > 0, + Instrument.FiveLaneDrums => Parts.FiveLaneDrums.SubTracks > 0, + Instrument.ProDrums => Parts.ProDrums.SubTracks > 0, - Instrument.EliteDrums => _parts.EliteDrums.SubTracks > 0, + Instrument.EliteDrums => Parts.EliteDrums.SubTracks > 0, - Instrument.ProGuitar_17Fret => _parts.ProGuitar_17Fret.SubTracks > 0, - Instrument.ProGuitar_22Fret => _parts.ProGuitar_22Fret.SubTracks > 0, - Instrument.ProBass_17Fret => _parts.ProBass_17Fret.SubTracks > 0, - Instrument.ProBass_22Fret => _parts.ProBass_22Fret.SubTracks > 0, + Instrument.ProGuitar_17Fret => Parts.ProGuitar_17Fret.SubTracks > 0, + Instrument.ProGuitar_22Fret => Parts.ProGuitar_22Fret.SubTracks > 0, + Instrument.ProBass_17Fret => Parts.ProBass_17Fret.SubTracks > 0, + Instrument.ProBass_22Fret => Parts.ProBass_22Fret.SubTracks > 0, - Instrument.ProKeys => _parts.ProKeys.SubTracks > 0, + Instrument.ProKeys => Parts.ProKeys.SubTracks > 0, - // Instrument.Dj => _parts.DJ.SubTracks > 0, + // Instrument.Dj => Parts.DJ.SubTracks > 0, - Instrument.Vocals => _parts.LeadVocals.SubTracks > 0, - Instrument.Harmony => _parts.HarmonyVocals.SubTracks > 0, - Instrument.Band => _parts.BandDifficulty.SubTracks > 0, + Instrument.Vocals => Parts.LeadVocals.SubTracks > 0, + Instrument.Harmony => Parts.HarmonyVocals.SubTracks > 0, + Instrument.Band => Parts.BandDifficulty.SubTracks > 0, _ => false }; } - protected SongEntry() - { - _metadata = SongMetadata.Default; - _parts = AvailableParts.Default; - _parseSettings = ParseSettings.Default; - _parsedYear = SongMetadata.DEFAULT_YEAR; - _intYear = int.MaxValue; - } - - private static readonly SortString DEFAULT_NAME_SORT = SortString.Convert(SongMetadata.DEFAULT_NAME); - private static readonly SortString DEFAULT_ARTIST_SORT = SortString.Convert(SongMetadata.DEFAULT_ARTIST); - private static readonly SortString DEFAULT_ALBUM_SORT = SortString.Convert(SongMetadata.DEFAULT_ALBUM); - private static readonly SortString DEFAULT_GENRE_SORT = SortString.Convert(SongMetadata.DEFAULT_GENRE); - private static readonly SortString DEFAULT_CHARTER_SORT = SortString.Convert(SongMetadata.DEFAULT_CHARTER); - private static readonly SortString DEFAULT_SOURCE_SORT = SortString.Convert(SongMetadata.DEFAULT_SOURCE); - - protected SongEntry(in AvailableParts parts, in HashWrapper hash, IniSection modifiers, in string defaultPlaylist) + protected SongEntry(in SongMetadata metadata, in AvailableParts parts, in HashWrapper hash, in LoaderSettings settings) { - _parts = parts; - _hash = hash; - - modifiers.TryGet("name", out _metadata.Name, DEFAULT_NAME_SORT); - modifiers.TryGet("artist", out _metadata.Artist, DEFAULT_ARTIST_SORT); - modifiers.TryGet("album", out _metadata.Album, DEFAULT_ALBUM_SORT); - modifiers.TryGet("genre", out _metadata.Genre, DEFAULT_GENRE_SORT); - - if (!modifiers.TryGet("year", out _metadata.Year)) - { - // Capitalization = from .chart - if (modifiers.TryGet("Year", out _metadata.Year)) - { - if (_metadata.Year.StartsWith(", ")) - { - _metadata.Year = _metadata.Year[2..]; - } - else if (_metadata.Year.StartsWith(',')) - { - _metadata.Year = _metadata.Year[1..]; - } - } - else - { - _metadata.Year = SongMetadata.DEFAULT_YEAR; - } - } - - _intYear = ParseYear(in _metadata.Year, out _parsedYear) - ? int.Parse(_parsedYear) - : int.MaxValue; - - if (!modifiers.TryGet("charter", out _metadata.Charter, DEFAULT_CHARTER_SORT)) - { - modifiers.TryGet("frets", out _metadata.Charter, DEFAULT_CHARTER_SORT); - } - - modifiers.TryGet("icon", out _metadata.Source, DEFAULT_SOURCE_SORT); - modifiers.TryGet("playlist", out _metadata.Playlist, defaultPlaylist); - - modifiers.TryGet("loading_phrase", out _metadata.LoadingPhrase); - - modifiers.TryGet("credit_written_by", out _metadata.CreditWrittenBy); - modifiers.TryGet("credit_performed_by", out _metadata.CreditPerformedBy); - modifiers.TryGet("credit_courtesy_of", out _metadata.CreditCourtesyOf); - modifiers.TryGet("credit_album_cover", out _metadata.CreditAlbumCover); - modifiers.TryGet("credit_license", out _metadata.CreditLicense); - - if (!modifiers.TryGet("playlist_track", out _metadata.PlaylistTrack)) - { - _metadata.PlaylistTrack = -1; - } - - if (!modifiers.TryGet("album_track", out _metadata.AlbumTrack)) - { - _metadata.AlbumTrack = -1; - } - - modifiers.TryGet("song_length", out _metadata.SongLength); - modifiers.TryGet("rating", out _metadata.SongRating); - - modifiers.TryGet("video_start_time", out _metadata.VideoStartTime); - if (!modifiers.TryGet("video_end_time", out _metadata.VideoEndTime)) - { - _metadata.VideoEndTime = -1; - } - - if (!modifiers.TryGet("preview", out _metadata.PreviewStart, out _metadata.PreviewEnd)) - { - if (!modifiers.TryGet("preview_start_time", out _metadata.PreviewStart)) - { - // Capitlization = from .chart - if (modifiers.TryGet("PreviewStart", out double previewStartSeconds)) - { - _metadata.PreviewStart = (long) (previewStartSeconds * MILLISECOND_FACTOR); - } - else - { - _metadata.PreviewStart = -1; - } - } - - if (!modifiers.TryGet("preview_end_time", out _metadata.PreviewEnd)) - { - // Capitlization = from .chart - if (modifiers.TryGet("PreviewEnd", out double previewEndSeconds)) - { - _metadata.PreviewEnd = (long) (previewEndSeconds * MILLISECOND_FACTOR); - } - else - { - _metadata.PreviewEnd = -1; - } - } - } - - if (!modifiers.TryGet("delay", out _metadata.SongOffset) || _metadata.SongOffset == 0) - { - // Capitlization = from .chart - if (modifiers.TryGet("Offset", out double songOffsetSeconds)) - { - _metadata.SongOffset = (long) (songOffsetSeconds * MILLISECOND_FACTOR); - } - } - - if (parts.FourLaneDrums.SubTracks > 0) - { - _parseSettings.DrumsType = DrumsType.FourLane; - } - else if (parts.FiveLaneDrums.SubTracks > 0) - { - _parseSettings.DrumsType = DrumsType.FiveLane; - } - else - { - _parseSettings.DrumsType = DrumsType.Unknown; - } - - if (!modifiers.TryGet("hopo_frequency", out _parseSettings.HopoThreshold)) - { - _parseSettings.HopoThreshold = -1; - } - - if (!modifiers.TryGet("hopofreq", out _parseSettings.HopoFreq_FoF)) - { - _parseSettings.HopoFreq_FoF = -1; - } - - modifiers.TryGet("eighthnote_hopo", out _parseSettings.EighthNoteHopo); - - if (!modifiers.TryGet("sustain_cutoff_threshold", out _parseSettings.SustainCutoffThreshold)) - { - _parseSettings.SustainCutoffThreshold = -1; - } - - if (!modifiers.TryGet("multiplier_note", out _parseSettings.StarPowerNote)) - { - _parseSettings.StarPowerNote = -1; - } - - _metadata.IsMaster = !modifiers.TryGet("tags", out string tag) || tag.ToLower() != "cover"; + Metadata = metadata; + Parts = parts; + Hash = hash; + Settings = settings; } - protected SongEntry(UnmanagedMemoryStream stream, in CategoryCacheStrings strings) + protected SongEntry(UnmanagedMemoryStream stream, CategoryCacheStrings strings) { - _metadata.Name = strings.titles[stream.Read(Endianness.Little)]; - _metadata.Artist = strings.artists[stream.Read(Endianness.Little)]; - _metadata.Album = strings.albums[stream.Read(Endianness.Little)]; - _metadata.Genre = strings.genres[stream.Read(Endianness.Little)]; - - _metadata.Year = strings.years[stream.Read(Endianness.Little)]; - _metadata.Charter = strings.charters[stream.Read(Endianness.Little)]; - _metadata.Playlist = strings.playlists[stream.Read(Endianness.Little)]; - _metadata.Source = strings.sources[stream.Read(Endianness.Little)]; - - _metadata.IsMaster = stream.ReadBoolean(); - - _metadata.AlbumTrack = stream.Read(Endianness.Little); - _metadata.PlaylistTrack = stream.Read(Endianness.Little); - - _metadata.SongLength = stream.Read(Endianness.Little); - _metadata.SongOffset = stream.Read(Endianness.Little); - _metadata.SongRating = stream.Read(Endianness.Little); - - _metadata.PreviewStart = stream.Read(Endianness.Little); - _metadata.PreviewEnd = stream.Read(Endianness.Little); - - _metadata.VideoStartTime = stream.Read(Endianness.Little); - _metadata.VideoEndTime = stream.Read(Endianness.Little); - - _metadata.LoadingPhrase = stream.ReadString(); - - _metadata.CreditWrittenBy = stream.ReadString(); - _metadata.CreditPerformedBy = stream.ReadString(); - _metadata.CreditCourtesyOf = stream.ReadString(); - _metadata.CreditAlbumCover = stream.ReadString(); - _metadata.CreditLicense = stream.ReadString(); - - _parseSettings.HopoThreshold = stream.Read(Endianness.Little); - _parseSettings.HopoFreq_FoF = stream.Read(Endianness.Little); - _parseSettings.EighthNoteHopo = stream.ReadBoolean(); - _parseSettings.SustainCutoffThreshold = stream.Read(Endianness.Little); - _parseSettings.NoteSnapThreshold = stream.Read(Endianness.Little); - _parseSettings.StarPowerNote = stream.Read(Endianness.Little); - _parseSettings.DrumsType = (DrumsType) stream.Read(Endianness.Little); - + Hash = HashWrapper.Deserialize(stream); unsafe { - fixed (AvailableParts* ptr = &_parts) - { - stream.Read(new Span(ptr, sizeof(AvailableParts))); - } + Parts = *(AvailableParts*) stream.PositionPointer; + stream.Position += sizeof(AvailableParts); } - _hash = HashWrapper.Deserialize(stream); - - _intYear = ParseYear(in _metadata.Year, out _parsedYear) - ? int.Parse(_parsedYear) - : int.MaxValue; + Metadata = new SongMetadata(stream, strings); + Settings.HopoThreshold = stream.Read(Endianness.Little); + Settings.SustainCutoffThreshold = stream.Read(Endianness.Little); + Settings.OverdiveMidiNote = stream.Read(Endianness.Little); } - protected void SerializeMetadata(in BinaryWriter writer, in CategoryCacheWriteNode node) + public virtual void Serialize(MemoryStream stream, CategoryCacheWriteNode node) { - writer.Write(node.title); - writer.Write(node.artist); - writer.Write(node.album); - writer.Write(node.genre); - writer.Write(node.year); - writer.Write(node.charter); - writer.Write(node.playlist); - writer.Write(node.source); - - writer.Write(_metadata.IsMaster); - - writer.Write(_metadata.AlbumTrack); - writer.Write(_metadata.PlaylistTrack); - - writer.Write(_metadata.SongLength); - writer.Write(_metadata.SongOffset); - writer.Write(_metadata.SongRating); - - writer.Write(_metadata.PreviewStart); - writer.Write(_metadata.PreviewEnd); - - writer.Write(_metadata.VideoStartTime); - writer.Write(_metadata.VideoEndTime); - - writer.Write(_metadata.LoadingPhrase); - - writer.Write(_metadata.CreditWrittenBy); - writer.Write(_metadata.CreditPerformedBy); - writer.Write(_metadata.CreditCourtesyOf); - writer.Write(_metadata.CreditAlbumCover); - writer.Write(_metadata.CreditLicense); - - writer.Write(_parseSettings.HopoThreshold); - writer.Write(_parseSettings.HopoFreq_FoF); - writer.Write(_parseSettings.EighthNoteHopo); - writer.Write(_parseSettings.SustainCutoffThreshold); - writer.Write(_parseSettings.NoteSnapThreshold); - writer.Write(_parseSettings.StarPowerNote); - writer.Write((int) _parseSettings.DrumsType); - + Hash.Serialize(stream); unsafe { - fixed (AvailableParts* ptr = &_parts) - { - writer.Write(new Span(ptr, sizeof(AvailableParts))); - } - } - _hash.Serialize(writer); - } - - private static bool ParseYear(in string baseString, out string parsedString) - { - for (int i = 0; i <= baseString.Length - 4; ++i) - { - int pivot = i; - while (i < pivot + 4 && i < baseString.Length && char.IsDigit(baseString[i])) - { - ++i; - } - - if (i == pivot + 4) + fixed (AvailableParts* ptr = &Parts) { - parsedString = baseString[pivot..i]; - return true; + stream.Write(new Span(ptr, sizeof(AvailableParts))); } } - parsedString = baseString; - return false; + Metadata.Serialize(stream, node); + stream.Write(Settings.HopoThreshold, Endianness.Little); + stream.Write(Settings.SustainCutoffThreshold, Endianness.Little); + stream.Write(Settings.OverdiveMidiNote, Endianness.Little); } } } \ No newline at end of file diff --git a/YARG.Core/Song/Entries/Types/HashWrapper.cs b/YARG.Core/Song/Entries/Types/HashWrapper.cs index 41cad34c8..325e793e4 100644 --- a/YARG.Core/Song/Entries/Types/HashWrapper.cs +++ b/YARG.Core/Song/Entries/Types/HashWrapper.cs @@ -94,6 +94,15 @@ public readonly void Serialize(BinaryWriter writer) } } + public readonly void Serialize(Stream stream) + { + fixed (int* values = _hash) + { + var bytes = new Span(values, HASH_SIZE_IN_BYTES); + stream.Write(bytes); + } + } + public readonly int CompareTo(HashWrapper other) { for (int i = 0; i < HASH_SIZE_IN_INTS; ++i) diff --git a/YARG.Core/Song/Entries/Types/SongMetadata.cs b/YARG.Core/Song/Entries/Types/SongMetadata.cs index 0979a4788..131d5dd31 100644 --- a/YARG.Core/Song/Entries/Types/SongMetadata.cs +++ b/YARG.Core/Song/Entries/Types/SongMetadata.cs @@ -1,7 +1,13 @@ -namespace YARG.Core.Song +using System.IO; +using YARG.Core.Extensions; +using YARG.Core.IO.Ini; +using YARG.Core.Song.Cache; + +namespace YARG.Core.Song { public struct SongMetadata { + public const double MILLISECOND_FACTOR = 1000.0; public static readonly SortString DEFAULT_NAME = "Unknown Name"; public static readonly SortString DEFAULT_ARTIST = "Unknown Artist"; public static readonly SortString DEFAULT_ALBUM = "Unknown Album"; @@ -29,7 +35,6 @@ public struct SongMetadata CreditAlbumCover = string.Empty, CreditLicense = string.Empty, Year = DEFAULT_YEAR, - SongLength = 0, SongOffset = 0, PreviewStart = -1, PreviewEnd = -1, @@ -47,7 +52,6 @@ public struct SongMetadata public string Year; - public ulong SongLength; public long SongOffset; public uint SongRating; // 1 = FF; 2 = SR; 3 = M; 4 = NR @@ -69,5 +73,174 @@ public struct SongMetadata public string CreditCourtesyOf; public string CreditAlbumCover; public string CreditLicense; + + public SongMetadata(IniSection modifiers, string defaultPlaylist) + { + modifiers.TryGet("name", out Name, DEFAULT_NAME); + modifiers.TryGet("artist", out Artist, DEFAULT_ARTIST); + modifiers.TryGet("album", out Album, DEFAULT_ALBUM); + modifiers.TryGet("genre", out Genre, DEFAULT_GENRE); + + if (!modifiers.TryGet("year", out Year)) + { + if (modifiers.TryGet("Year", out Year)) + { + if (Year.StartsWith(", ")) + { + Year = Year[2..]; + } + else if (Year.StartsWith(',')) + { + Year = Year[1..]; + } + } + else + { + Year = DEFAULT_YEAR; + } + } + + if (!modifiers.TryGet("charter", out Charter, DEFAULT_CHARTER)) + { + modifiers.TryGet("frets", out Charter, DEFAULT_CHARTER); + } + + modifiers.TryGet("icon", out Source, DEFAULT_SOURCE); + modifiers.TryGet("playlist", out Playlist, defaultPlaylist); + + modifiers.TryGet("loading_phrase", out LoadingPhrase); + + modifiers.TryGet("credit_written_by", out CreditWrittenBy); + modifiers.TryGet("credit_performed_by", out CreditPerformedBy); + modifiers.TryGet("credit_courtesy_of", out CreditCourtesyOf); + modifiers.TryGet("credit_album_cover", out CreditAlbumCover); + modifiers.TryGet("credit_license", out CreditLicense); + + if (!modifiers.TryGet("playlist_track", out PlaylistTrack)) + { + PlaylistTrack = -1; + } + + if (!modifiers.TryGet("album_track", out AlbumTrack)) + { + AlbumTrack = -1; + } + + modifiers.TryGet("rating", out SongRating); + + modifiers.TryGet("video_start_time", out VideoStartTime); + if (!modifiers.TryGet("video_end_time", out VideoEndTime)) + { + VideoEndTime = -1; + } + + if (!modifiers.TryGet("preview", out PreviewStart, out PreviewEnd)) + { + if (!modifiers.TryGet("preview_start_time", out PreviewStart)) + { + // Capitlization = from .chart + if (modifiers.TryGet("PreviewStart", out double previewStartSeconds)) + { + PreviewStart = (long) (previewStartSeconds * MILLISECOND_FACTOR); + } + else + { + PreviewStart = -1; + } + } + + if (!modifiers.TryGet("preview_end_time", out PreviewEnd)) + { + // Capitlization = from .chart + if (modifiers.TryGet("PreviewEnd", out double previewEndSeconds)) + { + PreviewEnd = (long) (previewEndSeconds * MILLISECOND_FACTOR); + } + else + { + PreviewEnd = -1; + } + } + } + + if (!modifiers.TryGet("delay", out SongOffset) || SongOffset == 0) + { + if (modifiers.TryGet("Offset", out double songOffsetSeconds)) + { + SongOffset = (long) (songOffsetSeconds * MILLISECOND_FACTOR); + } + } + + IsMaster = !modifiers.TryGet("tags", out string tag) || tag.ToLower() != "cover"; + } + + public SongMetadata(UnmanagedMemoryStream stream, CategoryCacheStrings strings) + { + Name = strings.titles[stream.Read(Endianness.Little)]; + Artist = strings.artists[stream.Read(Endianness.Little)]; + Album = strings.albums[stream.Read(Endianness.Little)]; + Genre = strings.genres[stream.Read(Endianness.Little)]; + + Year = strings.years[stream.Read(Endianness.Little)]; + Charter = strings.charters[stream.Read(Endianness.Little)]; + Playlist = strings.playlists[stream.Read(Endianness.Little)]; + Source = strings.sources[stream.Read(Endianness.Little)]; + + IsMaster = stream.ReadBoolean(); + + AlbumTrack = stream.Read(Endianness.Little); + PlaylistTrack = stream.Read(Endianness.Little); + + SongOffset = stream.Read(Endianness.Little); + SongRating = stream.Read(Endianness.Little); + + PreviewStart = stream.Read(Endianness.Little); + PreviewEnd = stream.Read(Endianness.Little); + + VideoStartTime = stream.Read(Endianness.Little); + VideoEndTime = stream.Read(Endianness.Little); + + LoadingPhrase = stream.ReadString(); + + CreditWrittenBy = stream.ReadString(); + CreditPerformedBy = stream.ReadString(); + CreditCourtesyOf = stream.ReadString(); + CreditAlbumCover = stream.ReadString(); + CreditLicense = stream.ReadString(); + } + + public readonly void Serialize(MemoryStream stream, CategoryCacheWriteNode node) + { + stream.Write(node.title, Endianness.Little); + stream.Write(node.artist, Endianness.Little); + stream.Write(node.album, Endianness.Little); + stream.Write(node.genre, Endianness.Little); + stream.Write(node.year, Endianness.Little); + stream.Write(node.charter, Endianness.Little); + stream.Write(node.playlist, Endianness.Little); + stream.Write(node.source, Endianness.Little); + + stream.Write(IsMaster); + + stream.Write(AlbumTrack, Endianness.Little); + stream.Write(PlaylistTrack, Endianness.Little); + + stream.Write(SongOffset, Endianness.Little); + stream.Write(SongRating, Endianness.Little); + + stream.Write(PreviewStart, Endianness.Little); + stream.Write(PreviewEnd, Endianness.Little); + + stream.Write(VideoStartTime, Endianness.Little); + stream.Write(VideoEndTime, Endianness.Little); + + stream.Write(LoadingPhrase); + + stream.Write(CreditWrittenBy); + stream.Write(CreditPerformedBy); + stream.Write(CreditCourtesyOf); + stream.Write(CreditAlbumCover); + stream.Write(CreditLicense); + } } } diff --git a/YARG.Core/Song/Preparsers/Midi/MidiEliteDrumsPreparser.cs b/YARG.Core/Song/Preparsers/Midi/MidiEliteDrumsPreparser.cs index bbeb7f788..4251a5c56 100644 --- a/YARG.Core/Song/Preparsers/Midi/MidiEliteDrumsPreparser.cs +++ b/YARG.Core/Song/Preparsers/Midi/MidiEliteDrumsPreparser.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; -using YARG.Core.IO; +using YARG.Core.IO; namespace YARG.Core.Song { diff --git a/YARG.Core/Song/Preparsers/Midi/MidiInstrumentPreparser.cs b/YARG.Core/Song/Preparsers/Midi/MidiInstrumentPreparser.cs index c6205a4eb..0c0d72a00 100644 --- a/YARG.Core/Song/Preparsers/Midi/MidiInstrumentPreparser.cs +++ b/YARG.Core/Song/Preparsers/Midi/MidiInstrumentPreparser.cs @@ -1,6 +1,4 @@ -using YARG.Core.IO; - -namespace YARG.Core.Song +namespace YARG.Core.Song { public static class MidiPreparser_Constants { diff --git a/YARG.Core/Song/Preparsers/Midi/MidiVocalPreparser.cs b/YARG.Core/Song/Preparsers/Midi/MidiVocalPreparser.cs index f4d4c91dd..14de5c96d 100644 --- a/YARG.Core/Song/Preparsers/Midi/MidiVocalPreparser.cs +++ b/YARG.Core/Song/Preparsers/Midi/MidiVocalPreparser.cs @@ -1,5 +1,4 @@ -using System; -using YARG.Core.IO; +using YARG.Core.IO; namespace YARG.Core.Song { From 3d43b7d69eca1553315384af383c6d7f35d17181 Mon Sep 17 00:00:00 2001 From: sonicfind Date: Sun, 29 Sep 2024 16:29:29 -0500 Subject: [PATCH 02/12] Add documentation to CacheHandler.cs --- YARG.Core/Song/Cache/CacheHandler.cs | 387 ++++++++++++++++++++++++++- 1 file changed, 386 insertions(+), 1 deletion(-) diff --git a/YARG.Core/Song/Cache/CacheHandler.cs b/YARG.Core/Song/Cache/CacheHandler.cs index ab989aeef..41e2217bf 100644 --- a/YARG.Core/Song/Cache/CacheHandler.cs +++ b/YARG.Core/Song/Cache/CacheHandler.cs @@ -44,11 +44,17 @@ public static SongCache RunScan(bool tryQuickScan, string cacheLocation, string ? new ParallelCacheHandler(baseDirectories, allowDuplicates, fullDirectoryPlaylists) : new SequentialCacheHandler(baseDirectories, allowDuplicates, fullDirectoryPlaylists); + // Some ini entry items won't come with the song length defined in the .ini file. + // In those instances, we'll need to attempt to load the audio files that accompany the chart + // to evaluate the length directly. + // This toggle simply keeps those generated mixers from spamming the logs on creation. GlobalAudioHandler.LogMixerStatus = false; try { + // Quick scans only fail if they parse zero entries (which could be the result of a few things) if (!tryQuickScan || !QuickScan(handler, cacheLocation)) { + // If a quick scan failed, there's no point to re-reading it in the full scan FullScan(handler, !tryQuickScan, cacheLocation, badSongsLocation); } } @@ -60,6 +66,13 @@ public static SongCache RunScan(bool tryQuickScan, string cacheLocation, string return handler.cache; } + /// + /// Reads the entries from a cache file - performing very few validation checks on the entries contained within + /// for the sole purpose of speeding through to gameplay. + /// + /// A parallel or sequential handler + /// File path of the cache + /// Whether the scan sucessfully parsed entries private static bool QuickScan(CacheHandler handler, string cacheLocation) { try @@ -90,6 +103,18 @@ private static bool QuickScan(CacheHandler handler, string cacheLocation) return true; } + /// + /// Runs a full scan process for a user's library. + /// Firstly, it attempts to read entries from a cache file - performing all validation checks necessary + /// to ensure that the player can immediately play whatever comes off the cache. + /// Secondly, we traverse the user's filesystem starting from their provided base directory nodes for any entries + /// that were not found from the cache or required re-evaluating. + /// Finally, we write the results of the scan back to a cache file and, if necessary, a badsongs.txt file containing the failures. + /// + /// A parallel or sequential handler + /// A flag communicating whether to perform the cache read (false only from failed quick scans) + /// File path of the cache + /// File path of the badsongs.txt private static void FullScan(CacheHandler handler, bool loadCache, string cacheLocation, string badSongsLocation) { if (loadCache) @@ -112,6 +137,8 @@ private static void FullScan(CacheHandler handler, bool loadCache, string cacheL _progress.Stage = ScanStage.LoadingSongs; handler.FindNewEntries(); + // CON, Upgrade, and Update groups hold onto the DTA data in memory. + // Once all entries are processed, they are no longer useful to us, so we dispose of them here. handler.DisposeLeftoverData(); _progress.Stage = ScanStage.CleaningDuplicates; @@ -189,23 +216,96 @@ protected CacheHandler(List baseDirectories, bool allowDuplicates, bool } } + /// + /// Sorts entries + /// protected abstract void SortEntries(); + + /// + /// Adds a instance to the shared list of packed con groups. + /// protected abstract void AddPackedCONGroup(PackedCONGroup group); + + /// + /// Adds a instance to the shared list of unpacked con groups. + /// protected abstract void AddUnpackedCONGroup(UnpackedCONGroup group); + + /// + /// Adds a instance to the shared list of update groups. + /// protected abstract void AddUpdateGroup(UpdateGroup group); + + /// + /// Adds a instance to the shared list of upgrade groups. + /// protected abstract void AddUpgradeGroup(UpgradeGroup group); + + /// + /// Removes all the entries present in all packed and unpacked con groups that have a matching DTA node name + /// protected abstract void RemoveCONEntry(string shortname); + + /// + /// Grabs or constructs a node containing all the updates or upgrades that can applied to any DTA entries + /// that have a name matching the one provided. + /// + /// The name of the DTA node for the entry + /// The node with the update and upgrade information protected abstract CONModification GetModification(string name); + /// + /// Performs the traversal of the filesystem in search of new entries to add to a user's library + /// protected abstract void FindNewEntries(); + + /// + /// Splits a collection into the tasks that traverse subdirectories or scan subfiles during the "New Entries" process + /// + /// The collection containing the sub directories and/or files + /// The group aligning to one of the base directories provided by the user + /// A tracker used to apply provide entries with default playlists protected abstract void TraverseDirectory(in FileCollection collection, IniGroup group, PlaylistTracker tracker); + /// + /// Deserializes a cache file into the separate song entries with all validation checks + /// + /// The stream containging the cache file data protected abstract void Deserialize(UnmanagedMemoryStream stream); + + /// + /// Deserializes a cache file into the separate song entries with minimal validations + /// + /// The stream containging the cache file data protected abstract void Deserialize_Quick(UnmanagedMemoryStream stream); + + /// + /// Adds a collection constructed during the full deserialization to a cache for use during + /// the full scan "New Entries" step. This skips the need to re-process a directory's list of files + /// where applicable. + /// protected abstract void AddCollectionToCache(in FileCollection collection); + + /// + /// Returns a CON group if the upgradeCON deserialization step already generated a group with the same filename + /// + /// The file path of the CON to potentially load (if not already) + /// A pre-loaded CON group on success find; otherwise protected abstract PackedCONGroup? FindCONGroup(string filename); + + /// + /// Upgrade nodes need to exist before CON entries can be processed. We therefore place all valid upgrades from cache + /// into a list for quick access. + /// + /// The DTA node name for the upgrade + /// The upgrade node to add protected abstract void AddCacheUpgrade(string name, RBProUpgrade upgrade); + /// + /// Attempts to mark a directory as "processed" + /// + /// The directory to mark + /// if the directory was not previously marked protected virtual bool FindOrMarkDirectory(string directory) { if (!preScannedDirectories.Add(directory)) @@ -216,16 +316,34 @@ protected virtual bool FindOrMarkDirectory(string directory) return true; } + /// + /// Attempts to mark a file as "processed" + /// + /// The file to mark + /// if the file was not previously marked protected virtual bool FindOrMarkFile(string file) { return preScannedFiles.Add(file); } + + /// + /// Adds an instance of a bad song + /// + /// The file that produced the error + /// The error produced protected virtual void AddToBadSongs(string filePath, ScanResult err) { badSongs.Add(filePath, err); _progress.BadSongCount++; } + /// + /// Attempts to add a new entry to current list. If duplicates are allowed, this will always return true. + /// If they are disallowed, then this will only succeed if the entry is not a duplicate or if it + /// takes precedence over the entry currently in its place (based on a variety of factors) + /// + /// The entry to add + /// Whether the song was accepted into the list protected virtual bool AddEntry(SongEntry entry) { var hash = entry.Hash; @@ -253,11 +371,20 @@ protected virtual bool AddEntry(SongEntry entry) return true; } + /// + /// Marks a CON song with the DTA name as invalid for addition from the cache + /// + /// The DTA name to mark protected virtual void AddInvalidSong(string name) { invalidSongsInCache.Add(name); } + /// + /// Grabs the iniGroup that parents the provided path, if one exists + /// + /// The absolute file path + /// The applicable group if found; otherwise protected IniGroup? GetBaseIniGroup(string path) { foreach (var group in iniGroups) @@ -272,6 +399,10 @@ protected virtual void AddInvalidSong(string name) return null; } + /// + /// Disposes all DTA FixedArray data present in upgrade and update nodes. + /// The songDTA arrays will already have been disposed of before reaching this point. + /// private void DisposeLeftoverData() { foreach (var group in conGroups) @@ -290,6 +421,9 @@ private void DisposeLeftoverData() } } + /// + /// Goes through all the groups that contain song entries to remove specific instances of duplicates + /// private void CleanupDuplicates() { foreach (var entry in duplicatesToRemove) @@ -319,6 +453,10 @@ private static bool TryRemove(List groups, SongEntry entry) return false; } + /// + /// Writes all bad song instances to a badsongs.txt file for the user + /// + /// The path for the file private void WriteBadSongs(string badSongsLocation) { using var stream = new FileStream(badSongsLocation, FileMode.Create, FileAccess.Write); @@ -402,6 +540,8 @@ private void WriteBadSongs(string badSongsLocation) protected readonly struct PlaylistTracker { private readonly bool _fullDirectoryFlag; + // We use `null` as the default state to grant two levels of subdirectories before + // supplying directories as the actual playlist (null -> empty -> directory) private readonly string? _playlist; public string Playlist => !string.IsNullOrEmpty(_playlist) ? _playlist : "Unknown Playlist"; @@ -427,6 +567,15 @@ public PlaylistTracker Append(string directory) protected const string SONGUPDATES_DTA = "songs_updates.dta"; protected const string SONGUPGRADES_DTA = "upgrades.dta"; + /// + /// Checks for the presence of files pertaining to an unpacked ini entry or whether the directory + /// is to be used for CON updates, upgrades, or extracted CON song entries. + /// If none of those, this will further traverse through any of the subdirectories present in this directory + /// and process all the subfiles for potential CONs or SNGs. + /// + /// The directory instance to load and scan through + /// The group aligning to one of the base directories provided by the user + /// A tracker used to apply provide entries with default playlists protected void ScanDirectory(DirectoryInfo directory, IniGroup group, PlaylistTracker tracker) { try @@ -436,13 +585,19 @@ protected void ScanDirectory(DirectoryInfo directory, IniGroup group, PlaylistTr return; } + // An update, upgrade, or unpacked con group might've failed during the cache load. + // In certain conditions, the collections that we would otherwise use for those would instead be in this cache if (!collectionCache.TryGetValue(directory.FullName, out var collection)) { collection = new FileCollection(directory); } + // If we discover any combo of valid unpacked ini entry files in this directory, + // we will traverse none of the subdirectories present in this scope if (ScanIniEntry(in collection, group, tracker.Playlist)) { + // However, the presence subdirectories could mean that the user didn't properly + // organize their collection. So as a service, we warn them in the badsongs.txt. if (collection.SubDirectories.Count > 0) { AddToBadSongs(directory.FullName, ScanResult.LooseChart_Warning); @@ -452,6 +607,7 @@ protected void ScanDirectory(DirectoryInfo directory, IniGroup group, PlaylistTr switch (directory.Name) { + // FOR ALL OF THE CASES: a missing dta file means that we will treat the folder like any other subdirectory case "songs_updates": { if (collection.Subfiles.TryGetValue(SONGUPDATES_DTA, out var dta)) @@ -459,6 +615,7 @@ protected void ScanDirectory(DirectoryInfo directory, IniGroup group, PlaylistTr var updateGroup = CreateUpdateGroup(in collection, dta); if (updateGroup != null) { + // Ensures any con entries pulled from cache are removed for re-evaluation foreach (var node in updateGroup.Updates) { RemoveCONEntry(node.Key); @@ -475,6 +632,7 @@ protected void ScanDirectory(DirectoryInfo directory, IniGroup group, PlaylistTr var upgradeGroup = CreateUpgradeGroup(in collection, dta); if (upgradeGroup != null) { + // Ensures any con entries pulled from cache are removed for re-evaluation foreach (var node in upgradeGroup.Upgrades) { RemoveCONEntry(node.Key); @@ -496,6 +654,7 @@ protected void ScanDirectory(DirectoryInfo directory, IniGroup group, PlaylistTr } TraverseDirectory(collection, group, tracker.Append(directory.Name)); + // Only possible on UNIX-based systems where file names are case-sensitive if (collection.ContainedDupes) { AddToBadSongs(collection.Directory.FullName, ScanResult.DuplicateFilesFound); @@ -512,6 +671,12 @@ protected void ScanDirectory(DirectoryInfo directory, IniGroup group, PlaylistTr } } + /// + /// Attempts to process the provided file as either a CON or SNG + /// + /// The info for provided file + /// The group aligning to one of the base directories provided by the user + /// A tracker used to apply provide entries with default playlists protected void ScanFile(FileInfo info, IniGroup group, in PlaylistTracker tracker) { string filename = info.FullName; @@ -539,6 +704,7 @@ protected void ScanFile(FileInfo info, IniGroup group, in PlaylistTracker tracke var conGroup = CreateCONGroup(in abridged, tracker.Playlist); if (conGroup != null) { + // Ensures any con entries pulled from cache are removed for re-evaluation foreach (var node in conGroup.Upgrades) { RemoveCONEntry(node.Key); @@ -558,6 +724,16 @@ protected void ScanFile(FileInfo info, IniGroup group, in PlaylistTracker tracke } } + /// + /// A templated helper function used for scanning a new CON entry to the list + /// + /// The group type (shocker) + /// The entry type for the group (again... shocker) + /// The group that contains or will contain the entry + /// The DTA node name for the entry + /// The index for the specific node (for CON packs that contain songs that share the same DTA name FOR SOME FUCKING REASON) + /// The raw byte data for the entry's base DTA information + /// The function used to convert the DTA info and modifications to the desired entry type protected unsafe void ScanCONNode(TGroup group, string name, int index, in YARGTextContainer node, delegate* func) where TGroup : CONGroup where TEntry : RBCONEntry @@ -588,8 +764,17 @@ protected unsafe void ScanCONNode(TGroup group, string name, int } } + /// + /// Loads the updates and upgrades that apply to con entries that share the same DTA node name + /// + /// The modification node to initialize + /// The DTA name of the node protected void InitModification(CONModification modification, string name) { + // To put the behavior simply: different folders mapping to the same nodes + // for like modification types is no bueno. Only the one with the most recent DTA + // write time will get utilized for entries with the current DTA name. + var datetime = default(DateTime); foreach (var group in updateGroups) { @@ -638,6 +823,14 @@ protected void InitModification(CONModification modification, string name) } } + /// + /// Searches for a ".ini" and any .mid or .chart file to possibly extract as a song entry. + /// If found, even if we can't extract an entry from them, we should perform no further directory traversal. + /// + /// The collection containing the subfiles to search from + /// The group aligning to one of the base directories provided by the user + /// The default directory-based playlist to use for any successful entry + /// Whether files pertaining to an unpacked ini entry were discovered private bool ScanIniEntry(in FileCollection collection, IniGroup group, string defaultPlaylist) { int i = collection.Subfiles.TryGetValue("song.ini", out var ini) ? 0 : 2; @@ -649,6 +842,10 @@ private bool ScanIniEntry(in FileCollection collection, IniGroup group, string d continue; } + // Can't play a song without any audio can you? + // + // Note though that this is purely a pre-add check. + // We will not invalidate an entry from cache if the user removes the audio after the fact. if (!collection.ContainsAudio()) { AddToBadSongs(chart.FullName, ScanResult.NoAudio); @@ -682,6 +879,12 @@ private bool ScanIniEntry(in FileCollection collection, IniGroup group, string d return false; } + /// + /// Searches for any .mid or .chart file to possibly extract as a song entry. + /// + /// The sngfile to search through + /// The group aligning to one of the base directories provided by the user + /// The default directory-based playlist to use for any successful entry private void ScanSngFile(SngFile sngFile, IniGroup group, string defaultPlaylist) { int i = sngFile.Metadata.Count > 0 ? 0 : 2; @@ -737,6 +940,13 @@ private void ScanSngFile(SngFile sngFile, IniGroup group, string defaultPlaylist /// private const int MIN_CACHEFILESIZE = 93; + /// + /// Attempts to laod the cache file's data into a FixedArray. This will fail if an error is thrown, + /// the cache is outdated, or if the the "full playlist" toggle mismatches. + /// + /// File location for the cache + /// Toggle for the display style of directory-based playlists + /// A FixedArray instance pointing to a buffer of the cache file's data, or .Null if invalid private static FixedArray LoadCacheToMemory(string cacheLocation, bool fullDirectoryPlaylists) { FileInfo info = new(cacheLocation); @@ -761,6 +971,10 @@ private static FixedArray LoadCacheToMemory(string cacheLocation, bool ful return FixedArray.ReadRemainder(stream); } + /// + /// Serializes the cache to a file, duhhhhhhh + /// + /// Location to save to private void Serialize(string cacheLocation) { using var filestream = new FileStream(cacheLocation, FileMode.Create, FileAccess.Write); @@ -797,8 +1011,17 @@ private void Serialize(string cacheLocation) ICacheGroup.SerializeGroups(extractedConGroups, filestream, nodes); } + /// + /// Reads a ini-based entry from the cache, with all validation steps + /// + /// Group mapping to the *user's* base directory + /// String of the base directory *written in the cache* + /// Stream containing the entry data + /// Container of the main metadata arrays protected void ReadIniEntry(IniGroup group, string directory, UnmanagedMemoryStream stream, CategoryCacheStrings strings) { + // An ini entry can be either unpacked (.ini) or packed (.sng). + // This boolean variable in the cache communicates to us which type it is. bool isSngEntry = stream.ReadBoolean(); string fullname = Path.Combine(directory, stream.ReadString()); @@ -806,14 +1029,24 @@ protected void ReadIniEntry(IniGroup group, string directory, UnmanagedMemoryStr ? ReadSngEntry(fullname, stream, strings) : ReadUnpackedIniEntry(fullname, stream, strings); + // If the "duplicates" toggle is set to false, regardless of what's within the cache, + // we will only accept non-duplicates if (entry != null && AddEntry(entry)) { group.AddEntry(entry); } } + /// + /// Reads a ini-based entry from the cache, with very few validation steps + /// + /// String of the base directory *written in the cache* + /// Stream containing the entry data + /// Container of the main metadata arrays protected void QuickReadIniEntry(string directory, UnmanagedMemoryStream stream, CategoryCacheStrings strings) { + // An ini entry can be either unpacked (.ini) or packed (.sng). + // This boolean variable in the cache communicates to us which type it is. bool isSngEntry = stream.ReadBoolean(); string fullname = Path.Combine(directory, stream.ReadString()); @@ -823,6 +1056,8 @@ protected void QuickReadIniEntry(string directory, UnmanagedMemoryStream stream, if (entry != null) { + // If the "duplicates" toggle is set to false, regardless of what's within the cache, + // we will only accept non-duplicates AddEntry(entry); } else @@ -831,6 +1066,12 @@ protected void QuickReadIniEntry(string directory, UnmanagedMemoryStream stream, } } + /// + /// Reads a section of the cache containing a list of updates to apply from a specific directory, + /// performing validations on each update node. If an update node from the cache is invalidated, it will mark + /// any RBCON entry nodes that share its DTA name as invalid, forcing re-evaluation. + /// + /// The stream containing the list of updates protected void ReadUpdateDirectory(UnmanagedMemoryStream stream) { string directory = stream.ReadString(); @@ -852,21 +1093,27 @@ protected void ReadUpdateDirectory(UnmanagedMemoryStream stream) var collection = new FileCollection(dirInfo); if (!collection.Subfiles.TryGetValue(SONGUPDATES_DTA, out var dta)) { + // We don't *mark* the directory to allow the "New Entries" process + // to access this collection collectionCache.Add(directory, collection); goto Invalidate; } FindOrMarkDirectory(directory); + // Will add the update group to the shared list on success var group = CreateUpdateGroup(in collection, dta); if (group != null && group.DTALastWrite == dtaLastWritten) { + // We need to compare what we have on the filesystem against what's written one by one var updates = new Dictionary(group.Updates); for (int i = 0; i < count; i++) { string name = stream.ReadString(); + // `Remove` returns true if the node was present if (updates.Remove(name, out var update)) { + // Validates midi, mogg, image, and milo write dates if (!update.Validate(stream)) { AddInvalidSong(name); @@ -879,6 +1126,7 @@ protected void ReadUpdateDirectory(UnmanagedMemoryStream stream) } } + // Anything left in the dictionary may require invalidation of cached entries foreach (var leftover in updates.Keys) { AddInvalidSong(leftover); @@ -894,6 +1142,12 @@ protected void ReadUpdateDirectory(UnmanagedMemoryStream stream) } } + /// + /// Reads a section of the cache containing a list of upgrades to apply from a specific directory, + /// performing validations on each upgrade node. If an upgrade node from the cache is invalidated, it will mark + /// any RBCON entry nodes that share its DTA name as invalid, forcing re-evaluation. + /// + /// The stream containing the list of upgrades protected void ReadUpgradeDirectory(UnmanagedMemoryStream stream) { string directory = stream.ReadString(); @@ -915,12 +1169,15 @@ protected void ReadUpgradeDirectory(UnmanagedMemoryStream stream) var collection = new FileCollection(dirInfo); if (!collection.Subfiles.TryGetValue(SONGUPGRADES_DTA, out var dta)) { + // We don't *mark* the directory to allow the "New Entries" process + // to access this collection collectionCache.Add(directory, collection); goto Invalidate; } FindOrMarkDirectory(directory); + // Will add the upgrade group to the shared list on success var group = CreateUpgradeGroup(in collection, dta); if (group != null && dta.LastWriteTime == dtaLastWritten) { @@ -936,6 +1193,12 @@ protected void ReadUpgradeDirectory(UnmanagedMemoryStream stream) } } + /// + /// Reads a section of the cache containing a list of upgrades to apply from a packed CON file, + /// performing validations on each upgrade node. If an upgrade node from the cache is invalidated, it will mark + /// any RBCON entry nodes that share its DTA name as invalid, forcing re-evaluation. + /// + /// The stream containing the list of upgrades protected void ReadUpgradeCON(UnmanagedMemoryStream stream) { string filename = stream.ReadString(); @@ -948,6 +1211,7 @@ protected void ReadUpgradeCON(UnmanagedMemoryStream stream) goto Invalidate; } + // Will add the packed CON group to the shared list on success var group = CreateCONGroup(filename, baseGroup.Directory); if (group != null && group.Info.LastUpdatedTime == conLastUpdated) { @@ -963,9 +1227,18 @@ protected void ReadUpgradeCON(UnmanagedMemoryStream stream) } } + /// + /// Helper function that runs validation on all the upgrade nodes present within either an upgrade directory or + /// upgrade CON section of a cache file. + /// + /// Type of upgrade node (packed or unpacked) + /// Dictionary containing the upgrade nodes freshly loaded from the source directory or CON + /// The number of upgrade nodes present in the cache file + /// Stream containing the upgrade node entries private void ValidateUpgrades(Dictionary Container, TUpgrade Upgrade)> groupUpgrades, int count, UnmanagedMemoryStream stream) where TUpgrade : RBProUpgrade { + // All we need to compare are the last update times var upgrades = new Dictionary(); upgrades.EnsureCapacity(groupUpgrades.Count); for (int i = 0; i < count; i++) @@ -977,8 +1250,11 @@ private void ValidateUpgrades(Dictionary(Dictionary + /// Attempts to load a PackedCONGroup instance from the validation data present in the cache. + /// Even if a group is generated however, if the last-update time in the cache does not match what we + /// receive from the filesystem, this function will not return that instance. Therefore, the caller can not parse + /// any of the accompanying CON entries that follow the validation data. + /// + /// Stream containing the CON validation information + /// The packed CON group on success; otherwise protected PackedCONGroup? ReadCONGroupHeader(UnmanagedMemoryStream stream) { string filename = stream.ReadString(); @@ -1007,6 +1292,14 @@ private void ValidateUpgrades(Dictionary + /// Attempts to load a UnpackedCONGroup instance from the validation data present in the cache. + /// Even if a group is generated however, if the last-update time of the songs.dta file in the cache does not match what we + /// receive from the filesystem, this function will not return that instance. Therefore, the caller can not parse + /// any of the accompanying CON entries that follow the validation data. + /// + /// Stream containing the song.dta validation information + /// The unpacked CON group on success; otherwise protected UnpackedCONGroup? ReadExtractedCONGroupHeader(UnmanagedMemoryStream stream) { string directory = stream.ReadString(); @@ -1035,6 +1328,10 @@ private void ValidateUpgrades(Dictionary + /// Loads all the upgrade nodes present in the cache from an "upgrades folder" section + /// + /// Stream containing the data for a folder's upgrade nodes protected void QuickReadUpgradeDirectory(UnmanagedMemoryStream stream) { string directory = stream.ReadString(); @@ -1047,10 +1344,16 @@ protected void QuickReadUpgradeDirectory(UnmanagedMemoryStream stream) string filename = Path.Combine(directory, $"{name}_plus.mid"); var info = new AbridgedFileInfo(filename, stream); + // Upgrade nodes need to exist before adding CON entries, so we must have a separate list for + // all upgrade nodes processed from cache AddCacheUpgrade(name, new UnpackedRBProUpgrade(info)); } } + /// + /// Loads all the upgrade nodes present in the cache from an "upgrade CON" section. + /// + /// Stream containing the data for a CON's upgrade nodes protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) { var listings = QuickReadCONGroupHeader(stream); @@ -1061,10 +1364,19 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) var lastWrite = DateTime.FromBinary(stream.Read(Endianness.Little)); var listing = default(CONFileListing); listings?.TryGetListing($"songs_upgrades/{name}_plus.mid", out listing); + // Upgrade nodes need to exist before adding CON entries, so we must have a separate list for + // all upgrade nodes processed from cache AddCacheUpgrade(name, new PackedRBProUpgrade(listing, lastWrite)); } } + /// + /// Attempts to load a list of CONFileListings from a CON file off the filesystem. + /// The listings that get loaded will utilize the last write information from the cache file, + /// rather than the last write info off the filesystem. + /// + /// Stream containing the CON validation information + /// A list of CONFileListings on success; otherwise protected List? QuickReadCONGroupHeader(UnmanagedMemoryStream stream) { string filename = stream.ReadString(); @@ -1078,6 +1390,14 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) return CONFile.TryParseListings(in info, filestream); } + /// + /// Attempts to read an unpacked .ini entry from the cache - with all validation checks. + /// If successful, we mark the directory as "already processed" so that the "New Entries" step doesn't attempt to access it. + /// + /// The directory potentially containing an entry + /// The stream containing the entry's information + /// Container will the basic metadata arrays + /// An unpacked ini entry on success; otherwise private UnpackedIniEntry? ReadUnpackedIniEntry(string directory, UnmanagedMemoryStream stream, CategoryCacheStrings strings) { var entry = UnpackedIniEntry.TryLoadFromCache(directory, stream, strings); @@ -1089,6 +1409,14 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) return entry; } + /// + /// Attempts to read an unpacked .sng entry from the cache - with all validation checks. + /// If successful, we mark the file as "already processed" so that the "New Entries" step doesn't attempt to access it. + /// + /// The file path to a potential entry + /// The stream containing the entry's information + /// Container will the basic metadata arrays + /// An packed sng entry on success; otherwise private SngEntry? ReadSngEntry(string fullname, UnmanagedMemoryStream stream, CategoryCacheStrings strings) { var entry = SngEntry.TryLoadFromCache(fullname, stream, strings); @@ -1100,11 +1428,18 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) return entry; } + /// + /// Creates an UpdateGroup... self-explanatory + /// + /// The collection of subdirectories and subfiles to locate updates from + /// The file info for the main DTA + /// An UpdateGroup instance on success; otherwise private UpdateGroup? CreateUpdateGroup(in FileCollection collection, FileInfo dta) { UpdateGroup? group = null; try { + // We call `using` to ensure the proper disposal of data if an error occurs using var data = FixedArray.Load(dta.FullName); var updates = new Dictionary(); var container = YARGDTAReader.TryCreate(data); @@ -1113,6 +1448,7 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) string name = YARGDTAReader.GetNameOfNode(ref container, true); if (!updates.TryGetValue(name, out var update)) { + // We only need to check for the files one time per-DTA name AbridgedFileInfo? midi = null; AbridgedFileInfo? mogg = null; AbridgedFileInfo? milo = null; @@ -1125,6 +1461,8 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) string moggName = subname + "_update.mogg"; string miloName = subname + ".milo_xbox"; string imageName = subname + "_keep.png_xbox"; + // Enumerating through the available files through the DirectoryInfo instance + // provides a speed boost over manual `File.Exists` checks foreach (var file in directory.EnumerateFiles("*", SearchOption.AllDirectories)) { string filename = file.Name; @@ -1145,16 +1483,19 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) image = new AbridgedFileInfo(file, false); } } - } updates.Add(name, update = new SongUpdate(in midi, in mogg, in milo, in image)); } + // Updates may contain multiple entries for the same DTA name, so we must collect them all under + // the same node. However, we won't actually load the information unless later stages require it. update.Containers.Add(container); YARGDTAReader.EndNode(ref container); } if (updates.Count > 0) { + // We transfer ownership of the FixedArray data to give the group responsibility over the disposal. + // Otherwise, the `using` call would dispose of the data after the scope of this function. group = new UpdateGroup(collection.Directory.FullName, dta.LastWriteTime, data.TransferOwnership(), updates); AddUpdateGroup(group); } @@ -1166,11 +1507,18 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) return group; } + /// + /// Creates an UpgradeGroup... self-explanatory + /// + /// The collection of subdirectories and subfiles to locate upgrades from + /// The file info for the main DTA + /// An UpgradeGroup instance on success; otherwise private UpgradeGroup? CreateUpgradeGroup(in FileCollection collection, FileInfo dta) { UpgradeGroup? group = null; try { + // We call `using` to ensure the proper disposal of data if an error occurs using var data = FixedArray.Load(dta.FullName); var upgrades = new Dictionary Container, UnpackedRBProUpgrade Upgrade)>(); var container = YARGDTAReader.TryCreate(data); @@ -1178,6 +1526,7 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) { string name = YARGDTAReader.GetNameOfNode(ref container, true); var upgrade = default(UnpackedRBProUpgrade); + // If there is no upgrade file accompanying the DTA node, there's no point in adding the upgrade if (collection.Subfiles.TryGetValue($"{name.ToLower()}_plus.mid", out var info)) { var abridged = new AbridgedFileInfo(info, false); @@ -1189,6 +1538,8 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) if (upgrades.Count > 0) { + // We transfer ownership of the FixedArray data to give the group responsibility over the disposal. + // Otherwise, the `using` call would dispose of the data after the scope of this function. group = new UpgradeGroup(collection.Directory.FullName, dta.LastWriteTime, data.TransferOwnership(), upgrades); AddUpgradeGroup(group); } @@ -1200,6 +1551,12 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) return group; } + /// + /// Attempts to create a PackedCONGroup from the file at the provided path. + /// + /// The path for the file + /// One of the base directories provided by the user + /// A PackedCONGroup instance on success; otherwise private PackedCONGroup? CreateCONGroup(string filename, string baseDirectory) { var info = new FileInfo(filename); @@ -1215,6 +1572,12 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) return CreateCONGroup(in abridged, playlist); } + /// + /// Attempts to create a PackedCONGroup with the provided fileinfo. + /// + /// The file info for the possible CONFile + /// The playlist to use for any entries generated from the CON (if it is one) + /// A PackedCONGroup instance on success; otherwise private PackedCONGroup? CreateCONGroup(in AbridgedFileInfo info, string defaultPlaylist) { const string SONGSFILEPATH = "songs/songs.dta"; @@ -1271,6 +1634,8 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) if (songNodes.Count > 0 || upgrades.Count > 0) { + // We transfer ownership of the FixedArray data to give the group responsibility over the disposal. + // Otherwise, the `using` calls would dispose of the data after the scope of this function. group = new PackedCONGroup(listings, songDTAData.TransferOwnership(), upgradeDTAData.TransferOwnership(), songNodes, upgrades, in info, defaultPlaylist); AddPackedCONGroup(group); } @@ -1282,6 +1647,13 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) return group; } + /// + /// Attempts to create an UnpackedCONGroup from the file at the provided path. + /// + /// The directory containing the list of entry subdirectories and the main DTA + /// The info for the main DTA file + /// The playlist to use for any entries generated from the CON (if it is one) + /// An UnpackedCONGroup instance on success; otherwise private UnpackedCONGroup? CreateUnpackedCONGroup(string directory, FileInfo dta, string defaultPlaylist) { try @@ -1319,6 +1691,12 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) return null; } + /// + /// Constructs a directory-based playlist based on the provided file name + /// + /// The path for the current file + /// One of the base directories provided by the user + /// The default playlist to potentially use private string ConstructPlaylist(string filename, string baseDirectory) { string directory = Path.GetDirectoryName(filename); @@ -1338,6 +1716,13 @@ private string ConstructPlaylist(string filename, string baseDirectory) internal static class UnmanagedStreamSlicer { + /// + /// Splice a section of the unmanaged stream into a separate instance. + /// Useful for parallelization and for potentially catching "end of stream" errors. + /// + /// The base stream to slice + /// The amount of the data to slice out from the base stream + /// The new stream containing the slice public static unsafe UnmanagedMemoryStream Slice(this UnmanagedMemoryStream stream, int length) { if (stream.Position > stream.Length - length) From 8b9cf3204ff0cd116d43339ac6995e7927d6e16c Mon Sep 17 00:00:00 2001 From: sonicfind Date: Wed, 2 Oct 2024 22:12:43 -0500 Subject: [PATCH 03/12] Simplify `.Remove("Resolution")` call --- YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs b/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs index 0ddd83c53..edf620c40 100644 --- a/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs +++ b/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs @@ -305,7 +305,7 @@ private static (ScanResult result, long resolution) ParseDotChart(ref YAR if (YARGChartFileReader.ValidateTrack(ref container, YARGChartFileReader.HEADERTRACK)) { var chartMods = YARGChartFileReader.ExtractModifiers(ref container); - if (chartMods.TryGetValue("Resolution", out var resolutions)) + if (chartMods.Remove("Resolution", out var resolutions)) { unsafe { @@ -316,7 +316,6 @@ private static (ScanResult result, long resolution) ParseDotChart(ref YAR return (ScanResult.ZeroResolution, 0); } } - chartMods.Remove("Resolution"); } modifiers.Append(chartMods); } From 05aa5d9a6389c58f78c35cf8b5daacb9af1c69f7 Mon Sep 17 00:00:00 2001 From: sonicfind Date: Wed, 2 Oct 2024 23:41:14 -0500 Subject: [PATCH 04/12] Split Modifiers between .Ini & .Chart Realized that `Resolution` wasn't actually being parsed, but placing it in the same big dictionary made no sense. + Changes (and fixes) the Debug-only validation pattern --- YARG.Core/IO/Ini/SongIniHandler.cs | 49 ++++++++++---------- YARG.Core/IO/YARGChartFileReader.cs | 17 ++++++- YARG.Core/Song/Cache/CacheHandler.cs | 2 +- YARG.Core/Song/Entries/Types/SongMetadata.cs | 8 ++-- 4 files changed, 46 insertions(+), 30 deletions(-) diff --git a/YARG.Core/IO/Ini/SongIniHandler.cs b/YARG.Core/IO/Ini/SongIniHandler.cs index 87b94d8c3..5b1bd530c 100644 --- a/YARG.Core/IO/Ini/SongIniHandler.cs +++ b/YARG.Core/IO/Ini/SongIniHandler.cs @@ -21,19 +21,20 @@ public static IniSection ReadSongIniFile(string iniPath) public static readonly Dictionary SONG_INI_MODIFIERS; #if DEBUG - private static readonly Dictionary _validations; + private static readonly Dictionary _types; + private static readonly Dictionary _outputs; public static void ThrowIfNot(string key) { - if (!SONG_INI_MODIFIERS.TryGetValue(key, out var mod)) + if (!_outputs.TryGetValue(key, out var modifierType)) { throw new ArgumentException($"Dev: {key} is not a valid modifier!"); } var typename = typeof(T).Name; - var type = _validations[typename]; - if (type != mod.Type - && (type == ModifierType.SortString) != (mod.Type == ModifierType.SortString_Chart) - && (type == ModifierType.String) != (mod.Type == ModifierType.String_Chart)) + var type = _types[typename]; + if (type != modifierType + && (type == ModifierType.SortString) != (modifierType == ModifierType.SortString_Chart) + && (type == ModifierType.String) != (modifierType == ModifierType.String_Chart)) { throw new ArgumentException($"Dev: Modifier {key} is not of type {typename}"); } @@ -45,10 +46,8 @@ static SongIniHandler() SONG_INI_MODIFIERS = new() { { "album", new("album", ModifierType.SortString) }, - { "Album", new("album", ModifierType.SortString_Chart) }, { "album_track", new("album_track", ModifierType.Int32) }, { "artist", new("artist", ModifierType.SortString) }, - { "Artist", new("artist", ModifierType.SortString_Chart) }, { "background", new("background", ModifierType.String) }, //{ "banner_link_a", new("banner_link_a", ModifierType.String) }, @@ -58,13 +57,16 @@ static SongIniHandler() //{ "cassettecolor", new("cassettecolor", ModifierType.UInt32) }, { "charter", new("charter", ModifierType.SortString) }, - { "Charter", new("charter", ModifierType.SortString_Chart) }, { "count", new("count", ModifierType.UInt32) }, { "cover", new("cover", ModifierType.String) }, + { "credit_written_by", new("credit_written_by", ModifierType.String) }, + { "credit_performed_by", new("credit_performed_by", ModifierType.String) }, + { "credit_courtesy_of", new("credit_courtesy_of", ModifierType.String) }, + { "credit_album_cover", new("credit_album_cover", ModifierType.String) }, + { "credit_license", new("credit_license", ModifierType.String) }, { "dance_type", new("dance_type", ModifierType.UInt32) }, { "delay", new("delay", ModifierType.Int64) }, - { "Difficulty", new("diff_band", ModifierType.Int32) }, { "diff_band", new("diff_band", ModifierType.Int32) }, { "diff_bass", new("diff_bass", ModifierType.Int32) }, { "diff_bass_real", new("diff_bass_real", ModifierType.Int32) }, @@ -99,7 +101,6 @@ static SongIniHandler() { "frets", new("frets", ModifierType.SortString) }, { "genre", new("genre", ModifierType.SortString) }, - { "Genre", new("genre", ModifierType.SortString_Chart) }, { "guitar_type", new("guitar_type", ModifierType.UInt32) }, { "hopo_frequency", new("hopo_frequency", ModifierType.Int64) }, @@ -115,27 +116,16 @@ static SongIniHandler() { "loading_phrase", new("loading_phrase", ModifierType.String) }, { "lyrics", new("lyrics", ModifierType.Bool) }, - { "credit_written_by", new("credit_written_by", ModifierType.String) }, - { "credit_performed_by", new("credit_performed_by", ModifierType.String) }, - { "credit_courtesy_of", new("credit_courtesy_of", ModifierType.String) }, - { "credit_album_cover", new("credit_album_cover", ModifierType.String) }, - { "credit_license", new("credit_license", ModifierType.String) }, - { "modchart", new("modchart", ModifierType.Bool) }, { "multiplier_note", new("multiplier_note", ModifierType.Int32) }, { "name", new("name", ModifierType.SortString) }, - { "Name", new("name", ModifierType.SortString_Chart) }, - - { "Offset", new("Offset", ModifierType.Double) }, { "playlist", new("playlist", ModifierType.SortString) }, { "playlist_track", new("playlist_track", ModifierType.Int32) }, { "preview", new("preview", ModifierType.Int64Array) }, { "preview_end_time", new("preview_end_time", ModifierType.Int64) }, - { "PreviewEnd", new("PreviewEnd", ModifierType.Double) }, { "preview_start_time", new("preview_start_time", ModifierType.Int64) }, - { "PreviewStart", new("PreviewStart", ModifierType.Double) }, { "pro_drum", new("pro_drums", ModifierType.Bool) }, { "pro_drums", new("pro_drums", ModifierType.Bool) }, @@ -178,7 +168,6 @@ static SongIniHandler() { "vocal_gender", new("vocal_gender", ModifierType.UInt32) }, { "year", new("year", ModifierType.String) }, - { "Year", new("Year", ModifierType.String) }, }; SONG_INI_DICTIONARY = new() @@ -187,7 +176,7 @@ static SongIniHandler() }; #if DEBUG - _validations = new() + _types = new() { { nameof(SortString), ModifierType.SortString }, { typeof(string).Name, ModifierType.String }, @@ -202,6 +191,18 @@ static SongIniHandler() { typeof(double).Name, ModifierType.Double }, { typeof(long[]).Name, ModifierType.Int64Array }, }; + + _outputs = new Dictionary(); + _outputs.EnsureCapacity(SONG_INI_MODIFIERS.Count + YARGChartFileReader.CHART_MODIFIERS.Count); + foreach (var node in SONG_INI_MODIFIERS.Values) + { + _outputs.TryAdd(node.OutputName, node.Type); + } + + foreach (var node in YARGChartFileReader.CHART_MODIFIERS.Values) + { + _outputs.TryAdd(node.OutputName, node.Type); + } #endif } } diff --git a/YARG.Core/IO/YARGChartFileReader.cs b/YARG.Core/IO/YARGChartFileReader.cs index 7c7d7959a..72518d286 100644 --- a/YARG.Core/IO/YARGChartFileReader.cs +++ b/YARG.Core/IO/YARGChartFileReader.cs @@ -217,6 +217,21 @@ public static unsafe bool TryParseEvent(ref YARGTextContainer cont return true; } + public static readonly Dictionary CHART_MODIFIERS = new() + { + { "Album", new("album", ModifierType.SortString_Chart) }, + { "Artist", new("artist", ModifierType.SortString_Chart) }, + { "Charter", new("charter", ModifierType.SortString_Chart) }, + { "Difficulty", new("diff_band", ModifierType.Int32) }, + { "Genre", new("genre", ModifierType.SortString_Chart) }, + { "Name", new("name", ModifierType.SortString_Chart) }, + { "Offset", new("delay_chart", ModifierType.Double) }, + { "PreviewEnd", new("previewEnd_chart", ModifierType.Double) }, + { "PreviewStart", new("previewStart_chart", ModifierType.Double) }, + { "Resolution", new("Resolution", ModifierType.Int64) }, + { "Year", new("year_chart", ModifierType.String_Chart) }, + }; + public unsafe static Dictionary> ExtractModifiers(ref YARGTextContainer container) where TChar : unmanaged, IEquatable, IConvertible { @@ -224,7 +239,7 @@ public unsafe static Dictionary> ExtractModifiers - public const int CACHE_VERSION = 24_09_28_01; + public const int CACHE_VERSION = 24_10_02_01; public static ScanProgressTracker Progress => _progress; private static ScanProgressTracker _progress; diff --git a/YARG.Core/Song/Entries/Types/SongMetadata.cs b/YARG.Core/Song/Entries/Types/SongMetadata.cs index 131d5dd31..8ab95f285 100644 --- a/YARG.Core/Song/Entries/Types/SongMetadata.cs +++ b/YARG.Core/Song/Entries/Types/SongMetadata.cs @@ -83,7 +83,7 @@ public SongMetadata(IniSection modifiers, string defaultPlaylist) if (!modifiers.TryGet("year", out Year)) { - if (modifiers.TryGet("Year", out Year)) + if (modifiers.TryGet("year_chart", out Year)) { if (Year.StartsWith(", ")) { @@ -139,7 +139,7 @@ public SongMetadata(IniSection modifiers, string defaultPlaylist) if (!modifiers.TryGet("preview_start_time", out PreviewStart)) { // Capitlization = from .chart - if (modifiers.TryGet("PreviewStart", out double previewStartSeconds)) + if (modifiers.TryGet("previewStart_chart", out double previewStartSeconds)) { PreviewStart = (long) (previewStartSeconds * MILLISECOND_FACTOR); } @@ -152,7 +152,7 @@ public SongMetadata(IniSection modifiers, string defaultPlaylist) if (!modifiers.TryGet("preview_end_time", out PreviewEnd)) { // Capitlization = from .chart - if (modifiers.TryGet("PreviewEnd", out double previewEndSeconds)) + if (modifiers.TryGet("previewEnd_chart", out double previewEndSeconds)) { PreviewEnd = (long) (previewEndSeconds * MILLISECOND_FACTOR); } @@ -165,7 +165,7 @@ public SongMetadata(IniSection modifiers, string defaultPlaylist) if (!modifiers.TryGet("delay", out SongOffset) || SongOffset == 0) { - if (modifiers.TryGet("Offset", out double songOffsetSeconds)) + if (modifiers.TryGet("delay_chart", out double songOffsetSeconds)) { SongOffset = (long) (songOffsetSeconds * MILLISECOND_FACTOR); } From 51d2418f86ff8f304919d90b48caa63543ee9ad7 Mon Sep 17 00:00:00 2001 From: sonicfind Date: Wed, 2 Oct 2024 23:45:18 -0500 Subject: [PATCH 05/12] Accounting for invalid negative resolutions in .chart + Changes `ZeroResolution` to `InvalidResolution` --- YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs | 4 ++-- YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs | 4 ++-- YARG.Core/Song/Entries/SongEntry.Scanning.cs | 2 +- YARG.Core/Song/Entries/SongEntry.cs | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs b/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs index edf620c40..7d8903cae 100644 --- a/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs +++ b/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs @@ -311,9 +311,9 @@ private static (ScanResult result, long resolution) ParseDotChart(ref YAR { var mod = resolutions[0]; resolution = mod.Buffer[0]; - if (resolution == 0) + if (resolution < 1) { - return (ScanResult.ZeroResolution, 0); + return (ScanResult.InvalidResolution, 0); } } } diff --git a/YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs b/YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs index ff37ad82f..23da5e2b8 100644 --- a/YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs +++ b/YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs @@ -558,7 +558,7 @@ protected static (ScanResult Result, HashWrapper Hash) ParseRBCONMidi(in FixedAr { switch (ParseMidi(in updateMidi, drumTracker, ref info.Parts).Result) { - case ScanResult.ZeroResolution: return (ScanResult.ZeroResolution_Update, default); + case ScanResult.InvalidResolution: return (ScanResult.InvalidResolution_Update, default); case ScanResult.MultipleMidiTrackNames: return (ScanResult.MultipleMidiTrackNames_Update, default); } bufLength += updateMidi.Length; @@ -568,7 +568,7 @@ protected static (ScanResult Result, HashWrapper Hash) ParseRBCONMidi(in FixedAr { switch (ParseMidi(in upgradeMidi, drumTracker, ref info.Parts).Result) { - case ScanResult.ZeroResolution: return (ScanResult.ZeroResolution_Upgrade, default); + case ScanResult.InvalidResolution: return (ScanResult.InvalidResolution_Upgrade, default); case ScanResult.MultipleMidiTrackNames: return (ScanResult.MultipleMidiTrackNames_Upgrade, default); } bufLength += upgradeMidi.Length; diff --git a/YARG.Core/Song/Entries/SongEntry.Scanning.cs b/YARG.Core/Song/Entries/SongEntry.Scanning.cs index 693ee5fbf..81d51c09c 100644 --- a/YARG.Core/Song/Entries/SongEntry.Scanning.cs +++ b/YARG.Core/Song/Entries/SongEntry.Scanning.cs @@ -12,7 +12,7 @@ protected static (ScanResult Result, long Resolution) ParseMidi(in FixedArray Date: Wed, 2 Oct 2024 23:45:22 -0500 Subject: [PATCH 06/12] Add missing Badsongs text for InvalidResolution --- YARG.Core/Song/Cache/CacheHandler.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/YARG.Core/Song/Cache/CacheHandler.cs b/YARG.Core/Song/Cache/CacheHandler.cs index cce9a21cd..90d99fbb5 100644 --- a/YARG.Core/Song/Cache/CacheHandler.cs +++ b/YARG.Core/Song/Cache/CacheHandler.cs @@ -529,6 +529,15 @@ private void WriteBadSongs(string badSongsLocation) writer.WriteLine("Loose chart files halted all traversal into the subdirectories at this location."); writer.WriteLine("To fix, if desired, place the loose chart files in a separate dedicated folder."); break; + case ScanResult.InvalidResolution: + writer.WriteLine("This chart uses an invalid resolution (or possibly contains it in an improper format, if .chart)"); + break; + case ScanResult.InvalidResolution_Update: + writer.WriteLine("The midi chart update file applicable with this chart has an invalid resolution of zero"); + break; + case ScanResult.InvalidResolution_Upgrade: + writer.WriteLine("The midi pro guitar upgrade file applicable with this chart has an invalid resolution of zero"); + break; } writer.WriteLine(); } From 53c6025dc173d9f7c88c4c402074a8bee245db4c Mon Sep 17 00:00:00 2001 From: sonicfind Date: Thu, 3 Oct 2024 23:04:42 -0500 Subject: [PATCH 07/12] Fix ChordHopoCancellation --- YARG.Core/Song/Entries/Ini/SongEntry.Sng.cs | 2 +- YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/YARG.Core/Song/Entries/Ini/SongEntry.Sng.cs b/YARG.Core/Song/Entries/Ini/SongEntry.Sng.cs index 11e7ddd36..87b1f0002 100644 --- a/YARG.Core/Song/Entries/Ini/SongEntry.Sng.cs +++ b/YARG.Core/Song/Entries/Ini/SongEntry.Sng.cs @@ -82,7 +82,7 @@ public override void Serialize(MemoryStream stream, CategoryCacheWriteNode node) SustainCutoffThreshold = Settings.SustainCutoffThreshold, StarPowerNote = Settings.OverdiveMidiNote, DrumsType = ParseDrumsType(in Parts), - ChordHopoCancellation = _chartFormat == ChartFormat.Chart + ChordHopoCancellation = _chartFormat != ChartFormat.Chart }; string file = CHART_FILE_TYPES[(int) _chartFormat].Filename; diff --git a/YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs b/YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs index 23da5e2b8..b789fa50a 100644 --- a/YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs +++ b/YARG.Core/Song/Entries/RBCON/SongEntry.RBCON.cs @@ -159,7 +159,7 @@ public override DateTime GetAddDate() SustainCutoffThreshold = Settings.SustainCutoffThreshold, StarPowerNote = Settings.OverdiveMidiNote, DrumsType = DrumsType.FourLane, - ChordHopoCancellation = false + ChordHopoCancellation = true }; return SongChart.FromMidi(in parseSettings, midi); } From 0173ce17a57ad87d4278b55b6e16db4a2e0df64d Mon Sep 17 00:00:00 2001 From: sonicfind Date: Fri, 4 Oct 2024 00:33:06 -0500 Subject: [PATCH 08/12] Slight "sustain_cutoff_threshold" comment reword --- YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs b/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs index 7d8903cae..251bdf5d2 100644 --- a/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs +++ b/YARG.Core/Song/Entries/Ini/SongEntry.IniBase.cs @@ -202,8 +202,8 @@ protected static (ScanResult Result, AvailableParts Parts, LoaderSettings Settin } } - // .chart defaults to no cutting off sustains whatsoever if the ini does not define the value. - // Since a failed `TryGet` sets the value to zero, we would need no additional work outside .mid + // .chart defaults to no sustain cutoff whatsoever if the ini does not define the value. + // Since a failed `TryGet` sets the value to zero, we would need no additional work unless it's .mid if (!modifiers.TryGet("sustain_cutoff_threshold", out settings.SustainCutoffThreshold) && format != ChartFormat.Chart) { settings.SustainCutoffThreshold = results.resolution / 3; From 32e2ffb7ef812dd0b5cd8418754e66b381529da9 Mon Sep 17 00:00:00 2001 From: sonicfind Date: Fri, 4 Oct 2024 17:04:52 -0500 Subject: [PATCH 09/12] Add `Default_Midi` And `Default_Chart` ParseSettings constants --- .../Parsing/DotChartParsingBenchmarks.cs | 3 +-- .../Parsing/MidiParsingBenchmarks.cs | 4 +--- .../Engine/DrumEngineTester.cs | 6 ++--- YARG.Core/Chart/ParsingProperties.cs | 23 +++++++++++++++---- .../IO/Chart/ChartReader.cs | 4 ++-- .../IO/Midi/MidReader.cs | 6 ++--- 6 files changed, 28 insertions(+), 18 deletions(-) diff --git a/YARG.Core.Benchmarks/Parsing/DotChartParsingBenchmarks.cs b/YARG.Core.Benchmarks/Parsing/DotChartParsingBenchmarks.cs index 590fa7de3..e53306e09 100644 --- a/YARG.Core.Benchmarks/Parsing/DotChartParsingBenchmarks.cs +++ b/YARG.Core.Benchmarks/Parsing/DotChartParsingBenchmarks.cs @@ -9,7 +9,6 @@ namespace YARG.Core.Benchmarks // [SimpleJob(RunStrategy.ColdStart, targetCount: 25, invocationCount: 1)] public class DotChartParsingBenchmarks { - private ParseSettings settings = ParseSettings.Default; private string chartText; [GlobalSetup] @@ -25,7 +24,7 @@ public void Initialize() [Benchmark] public SongChart ChartParsing() { - return SongChart.FromDotChart(settings, chartText); + return SongChart.FromDotChart(in ParseSettings.Default_Chart, chartText); } } } diff --git a/YARG.Core.Benchmarks/Parsing/MidiParsingBenchmarks.cs b/YARG.Core.Benchmarks/Parsing/MidiParsingBenchmarks.cs index 578712db4..869e83171 100644 --- a/YARG.Core.Benchmarks/Parsing/MidiParsingBenchmarks.cs +++ b/YARG.Core.Benchmarks/Parsing/MidiParsingBenchmarks.cs @@ -9,8 +9,6 @@ namespace YARG.Core.Benchmarks // [SimpleJob(RunStrategy.ColdStart, targetCount: 25, invocationCount: 1)] public class MidiParsingBenchmarks { - - private ParseSettings settings = ParseSettings.Default; private MidiFile midi; [GlobalSetup] @@ -26,7 +24,7 @@ public void Initialize() [Benchmark] public SongChart ChartParsing() { - return SongChart.FromMidi(settings, midi); + return SongChart.FromMidi(in ParseSettings.Default_Midi, midi); } } } diff --git a/YARG.Core.UnitTests/Engine/DrumEngineTester.cs b/YARG.Core.UnitTests/Engine/DrumEngineTester.cs index 20b3b7a1b..94d316c91 100644 --- a/YARG.Core.UnitTests/Engine/DrumEngineTester.cs +++ b/YARG.Core.UnitTests/Engine/DrumEngineTester.cs @@ -20,8 +20,6 @@ public class DrumEngineTester private string? _chartsDirectory; - private readonly ParseSettings _settings = ParseSettings.Default; - [SetUp] public void Setup() { @@ -37,7 +35,7 @@ public void DrumSoloThatEndsInChord_ShouldWorkCorrectly() { var chartPath = Path.Combine(_chartsDirectory!, "drawntotheflame.mid"); var midi = MidiFile.Read(chartPath); - var chart = SongChart.FromMidi(_settings, midi); + var chart = SongChart.FromMidi(in ParseSettings.Default_Midi, midi); var notes = chart.ProDrums.GetDifficulty(Difficulty.Expert); var engine = new YargDrumsEngine(notes, chart.SyncTrack, _engineParams, true); @@ -56,7 +54,7 @@ public void DrumTrackWithKickDrumRemoved_ShouldWorkCorrectly() { var chartPath = Path.Combine(_chartsDirectory!, "drawntotheflame.mid"); var midi = MidiFile.Read(chartPath); - var chart = SongChart.FromMidi(_settings, midi); + var chart = SongChart.FromMidi(in ParseSettings.Default_Midi, midi); var notes = chart.ProDrums.GetDifficulty(Difficulty.Expert); notes.RemoveKickDrumNotes(); diff --git a/YARG.Core/Chart/ParsingProperties.cs b/YARG.Core/Chart/ParsingProperties.cs index 85a6da9ae..bc6cfb414 100644 --- a/YARG.Core/Chart/ParsingProperties.cs +++ b/YARG.Core/Chart/ParsingProperties.cs @@ -25,16 +25,31 @@ public struct ParseSettings public static readonly ParseSettings Default = new() { DrumsType = DrumsType.Unknown, + HopoThreshold = SETTING_DEFAULT, + SustainCutoffThreshold = SETTING_DEFAULT, + ChordHopoCancellation = false, + StarPowerNote = SETTING_DEFAULT, + NoteSnapThreshold = 0, + }; + public static readonly ParseSettings Default_Chart = new() + { + DrumsType = DrumsType.Unknown, HopoThreshold = SETTING_DEFAULT, - HopoFreq_FoF = SETTING_DEFAULT, - EighthNoteHopo = false, + SustainCutoffThreshold = 0, ChordHopoCancellation = false, + StarPowerNote = SETTING_DEFAULT, + NoteSnapThreshold = 0, + }; + public static readonly ParseSettings Default_Midi = new() + { + DrumsType = DrumsType.Unknown, + HopoThreshold = SETTING_DEFAULT, SustainCutoffThreshold = SETTING_DEFAULT, + ChordHopoCancellation = false, + StarPowerNote = 116, NoteSnapThreshold = 0, - - StarPowerNote = SETTING_DEFAULT, }; /// diff --git a/YARG.Core/MoonscraperChartParser/IO/Chart/ChartReader.cs b/YARG.Core/MoonscraperChartParser/IO/Chart/ChartReader.cs index 0d9cdbc26..185b88011 100644 --- a/YARG.Core/MoonscraperChartParser/IO/Chart/ChartReader.cs +++ b/YARG.Core/MoonscraperChartParser/IO/Chart/ChartReader.cs @@ -78,13 +78,13 @@ private static ReadOnlySpan public static MoonSong ReadFromFile(string filepath) { - var settings = ParseSettings.Default; + var settings = ParseSettings.Default_Chart; return ReadFromFile(ref settings, filepath); } public static MoonSong ReadFromText(ReadOnlySpan chartText) { - var settings = ParseSettings.Default; + var settings = ParseSettings.Default_Chart; return ReadFromText(ref settings, chartText); } diff --git a/YARG.Core/MoonscraperChartParser/IO/Midi/MidReader.cs b/YARG.Core/MoonscraperChartParser/IO/Midi/MidReader.cs index 142c8feeb..a89f61a7a 100644 --- a/YARG.Core/MoonscraperChartParser/IO/Midi/MidReader.cs +++ b/YARG.Core/MoonscraperChartParser/IO/Midi/MidReader.cs @@ -71,19 +71,19 @@ private struct EventProcessParams public static MoonSong ReadMidi(string path) { - var settings = ParseSettings.Default; + var settings = ParseSettings.Default_Midi; return ReadMidi(ref settings, path); } public static MoonSong ReadMidi(Stream stream) { - var settings = ParseSettings.Default; + var settings = ParseSettings.Default_Midi; return ReadMidi(ref settings, stream); } public static MoonSong ReadMidi(MidiFile midi) { - var settings = ParseSettings.Default; + var settings = ParseSettings.Default_Midi; return ReadMidi(ref settings, midi); } From afa54ab8b3908d26f69b71369e018403ec2ff2a3 Mon Sep 17 00:00:00 2001 From: sonicfind Date: Fri, 4 Oct 2024 18:34:27 -0500 Subject: [PATCH 10/12] ParseSettings and Moonscraper cleanup Since the scanning process now pre-calculates HopoThreshold and SustainCutoffThreshold, we need to adjust the reader code so that it doesn't re-calculate the thresholds to incorrect values when called from a song entry. + Remove `HopoFreq_FoF` & `EighthNoteHopo` ParseSettings variables + Remove `GetHopoThreshold`-like functions + Convert MoonSong resolution to uint --- .../Parsing/ParseBehaviorTests.Midi.cs | 4 +- .../Loaders/MoonSong/MoonSongLoader.Lyrics.cs | 4 +- .../Chart/Loaders/MoonSong/MoonSongLoader.cs | 9 +-- YARG.Core/Chart/ParsingProperties.cs | 55 ------------------- .../IO/Chart/ChartIOHelper.cs | 12 ---- .../IO/Chart/ChartReader.cs | 32 +++++------ .../IO/Midi/MidIOHelper.cs | 7 +-- .../IO/Midi/MidReader.cs | 37 ++++--------- YARG.Core/MoonscraperChartParser/MoonSong.cs | 8 +-- 9 files changed, 38 insertions(+), 130 deletions(-) diff --git a/YARG.Core.UnitTests/Parsing/ParseBehaviorTests.Midi.cs b/YARG.Core.UnitTests/Parsing/ParseBehaviorTests.Midi.cs index 300ca712a..d5e8b9297 100644 --- a/YARG.Core.UnitTests/Parsing/ParseBehaviorTests.Midi.cs +++ b/YARG.Core.UnitTests/Parsing/ParseBehaviorTests.Midi.cs @@ -20,7 +20,7 @@ namespace YARG.Core.UnitTests.Parsing public class MidiParseBehaviorTests { private const uint SUSTAIN_CUTOFF_THRESHOLD = RESOLUTION / 3; - private static readonly uint HopoThreshold = (uint)GetHopoThreshold(ParseSettings.Default, RESOLUTION); + private const uint HOPO_THRESHOLD = (RESOLUTION / 3) + 1; private static readonly Dictionary InstrumentToNameLookup = new() { @@ -450,7 +450,7 @@ private static void GenerateNotesForDifficulty(MidiEventList events, if ((canForceStrum || canForceHopo) && (flags & Flags.Forced) != 0) { MoonNoteType type; - if (canForceHopo && lastStartDelta >= HopoThreshold) + if (canForceHopo && lastStartDelta >= HOPO_THRESHOLD) { type = MoonNoteType.Hopo; // Apply additional flag to match the parsed data diff --git a/YARG.Core/Chart/Loaders/MoonSong/MoonSongLoader.Lyrics.cs b/YARG.Core/Chart/Loaders/MoonSong/MoonSongLoader.Lyrics.cs index d607d58b1..1694546c2 100644 --- a/YARG.Core/Chart/Loaders/MoonSong/MoonSongLoader.Lyrics.cs +++ b/YARG.Core/Chart/Loaders/MoonSong/MoonSongLoader.Lyrics.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using MoonscraperChartEditor.Song; @@ -69,7 +69,7 @@ public void AddPhraseEvent(string text, uint tick) public LyricsTrack LoadLyrics() { var converter = new LyricConverter(_moonSong); - var maxTick = _moonSong.Charts.Max(x => x.events.LastOrDefault()?.tick + (uint)_moonSong.resolution ?? 0); + var maxTick = _moonSong.Charts.Max(x => x.events.LastOrDefault()?.tick + _moonSong.resolution ?? 0); TextEvents.ConvertToPhrases(_moonSong.events, converter, maxTick); return new LyricsTrack(converter.Phrases); } diff --git a/YARG.Core/Chart/Loaders/MoonSong/MoonSongLoader.cs b/YARG.Core/Chart/Loaders/MoonSong/MoonSongLoader.cs index ca0b5ef28..39ee25d77 100644 --- a/YARG.Core/Chart/Loaders/MoonSong/MoonSongLoader.cs +++ b/YARG.Core/Chart/Loaders/MoonSong/MoonSongLoader.cs @@ -30,11 +30,8 @@ private delegate TNote CreateNoteDelegate(MoonNote moonNote, CurrentPhras private MoonSong.MoonInstrument _currentMoonInstrument; private MoonSong.Difficulty _currentMoonDifficulty; - public MoonSongLoader(MoonSong song, ParseSettings settings) + public MoonSongLoader(MoonSong song, in ParseSettings settings) { - if (settings.NoteSnapThreshold < 0) - settings.NoteSnapThreshold = 0; - _moonSong = song; _settings = settings; } @@ -333,7 +330,7 @@ private static bool IsEventInPhrase(MoonObject songObj, MoonPhrase phrase) if (phrase.length == 0) return songObj.tick == phrase.tick; - return songObj.tick >= phrase.tick && songObj.tick < (phrase.tick + phrase.length); + return phrase.tick <= songObj.tick && songObj.tick < (phrase.tick + phrase.length); } private static bool IsNoteClosestToEndOfPhrase(MoonSong song, MoonNote note, MoonPhrase phrase) @@ -366,7 +363,7 @@ private static bool IsNoteClosestToEndOfPhrase(MoonSong song, MoonNote note, Moo if (previousNote is null) { // This is the first note in the chart, check by distance - float tickThreshold = song.resolution / 3; // 1/12th note + uint tickThreshold = song.resolution / 3; // 1/12th note return Math.Abs((int) note.tick - endTick) < tickThreshold; } else if (note.tick >= endTick && previousNote.tick < endTick) diff --git a/YARG.Core/Chart/ParsingProperties.cs b/YARG.Core/Chart/ParsingProperties.cs index bc6cfb414..362d56b6d 100644 --- a/YARG.Core/Chart/ParsingProperties.cs +++ b/YARG.Core/Chart/ParsingProperties.cs @@ -72,23 +72,6 @@ public struct ParseSettings /// public long HopoThreshold; - /// - /// The FoF HOPO threshold setting number to use. - /// - /// - /// Uses the hopofreq tag from song.ini files.
- /// 0 -> 1/24th note, 1 -> 1/16th note, 2 -> 1/12th note, 3 -> 1/8th note, 4 -> 1/6th note, 5 -> 1/4th note. - /// - public int HopoFreq_FoF; - - /// - /// Set the HOPO threshold to a 1/8th note instead of a 1/12th note. - /// - /// - /// Uses the eighthnote_hopo tag from song.ini files. - /// - public bool EighthNoteHopo; - /// /// Skip marking single notes after chords as HOPOs /// if the single note shares a fret with the chord. @@ -120,43 +103,5 @@ public struct ParseSettings /// Defaults to 116. /// public int StarPowerNote; - - /// - /// Calculates the HOPO threshold to use from the various HOPO settings. - /// - public readonly float GetHopoThreshold(float resolution) - { - // Prefer in this order: - // 1. hopo_threshold - // 2. eighthnote_hopo - // 3. hopofreq - - if (HopoThreshold > 0) - { - return HopoThreshold; - } - else if (EighthNoteHopo) - { - return resolution / 2; - } - else if (HopoFreq_FoF >= 0) - { - int denominator = HopoFreq_FoF switch - { - 0 => 24, - 1 => 16, - 2 => 12, - 3 => 8, - 4 => 6, - 5 => 4, - _ => throw new NotImplementedException($"Unhandled hopofreq value {HopoFreq_FoF}!") - }; - return (resolution * 4) / denominator; - } - else - { - return resolution / 3; - } - } } } \ No newline at end of file diff --git a/YARG.Core/MoonscraperChartParser/IO/Chart/ChartIOHelper.cs b/YARG.Core/MoonscraperChartParser/IO/Chart/ChartIOHelper.cs index c3fca34c3..c52dae9f7 100644 --- a/YARG.Core/MoonscraperChartParser/IO/Chart/ChartIOHelper.cs +++ b/YARG.Core/MoonscraperChartParser/IO/Chart/ChartIOHelper.cs @@ -96,17 +96,5 @@ internal static class ChartIOHelper { "GHLRhythm", MoonSong.MoonInstrument.GHLiveRhythm }, { "GHLCoop", MoonSong.MoonInstrument.GHLiveCoop }, }; - - public static float GetHopoThreshold(in ParseSettings settings, float resolution) - { - // With a 192 resolution, .chart has a HOPO threshold of 65 ticks, not 64, - // so we need to scale this factor to different resolutions (480 res = 162.5 threshold) - // This extra tick is meant for some slight leniency; .mid has it too, but it's applied - // after factoring in the resolution there, not before. - const float DEFAULT_RESOLUTION = 192; - const float THRESHOLD_LENIENCY_FACTOR = 1 / DEFAULT_RESOLUTION; - - return settings.GetHopoThreshold(resolution) + THRESHOLD_LENIENCY_FACTOR * resolution; - } } } diff --git a/YARG.Core/MoonscraperChartParser/IO/Chart/ChartReader.cs b/YARG.Core/MoonscraperChartParser/IO/Chart/ChartReader.cs index 185b88011..2cba6960b 100644 --- a/YARG.Core/MoonscraperChartParser/IO/Chart/ChartReader.cs +++ b/YARG.Core/MoonscraperChartParser/IO/Chart/ChartReader.cs @@ -107,6 +107,7 @@ public static MoonSong ReadFromFile(ref ParseSettings settings, string filepath) } } + private const uint DEFAULT_RESOLUTION = 192; public static MoonSong ReadFromText(ref ParseSettings settings, ReadOnlySpan chartText) { int textIndex = 0; @@ -124,7 +125,15 @@ static void ExpectSection(ReadOnlySpan chartText, ref int textIndex, // Check for the [Song] section first explicitly, need the Resolution property up-front ExpectSection(chartText, ref textIndex, ChartIOHelper.SECTION_SONG, out var sectionBody); var song = SubmitDataSong(sectionBody); - ValidateAndApplySettings(song, ref settings); + + // With a 192 resolution, .chart has a HOPO threshold of 65 ticks, not 64, + // so we need to scale this factor to different resolutions (480 res = 162.5 threshold) + // This extra tick is meant for some slight leniency; .mid has it too, but it's applied + // after factoring in the resolution there, not before. + const uint THRESHOLD_AT_DEFAULT = 65; + song.hopoThreshold = settings.HopoThreshold > ParseSettings.SETTING_DEFAULT + ? (uint) settings.HopoThreshold + : (song.resolution * THRESHOLD_AT_DEFAULT) / DEFAULT_RESOLUTION; // Check for [SyncTrack] next, we need it for time conversions ExpectSection(chartText, ref textIndex, ChartIOHelper.SECTION_SYNC_TRACK, out sectionBody); @@ -255,6 +264,7 @@ private static void SubmitChartData(ref ParseSettings settings, MoonSong song, R private static MoonSong SubmitDataSong(AsciiTrimSplitter sectionLines) { + uint resolution = DEFAULT_RESOLUTION; foreach (var line in sectionLines) { var key = line.SplitOnceTrimmed('=', out var value); @@ -262,25 +272,11 @@ private static MoonSong SubmitDataSong(AsciiTrimSplitter sectionLines) if (key.Equals("Resolution", StringComparison.Ordinal)) { - uint resolution = (uint)FastInt32Parse(value); - return new MoonSong(resolution); + resolution = (uint)FastInt32Parse(value); + break; } } - - throw new InvalidDataException("No resolution was found in the chart data!"); - } - - private static void ValidateAndApplySettings(MoonSong song, ref ParseSettings settings) - { - // Apply HOPO threshold settings - song.hopoThreshold = ChartIOHelper.GetHopoThreshold(in settings, song.resolution); - - // Sustain cutoff threshold is not verified, sustains are not cut off by default in .chart - // SP note is not verified, as it is only relevant for .mid - // Note snap threshold is not verified, as the parser doesn't use it - - // Chord HOPO cancellation does not apply in .chart - settings.ChordHopoCancellation = false; + return new MoonSong(resolution); } private static void SubmitDataSync(MoonSong song, AsciiTrimSplitter sectionLines) diff --git a/YARG.Core/MoonscraperChartParser/IO/Midi/MidIOHelper.cs b/YARG.Core/MoonscraperChartParser/IO/Midi/MidIOHelper.cs index 7449242af..4c375d8f2 100644 --- a/YARG.Core/MoonscraperChartParser/IO/Midi/MidIOHelper.cs +++ b/YARG.Core/MoonscraperChartParser/IO/Midi/MidIOHelper.cs @@ -1,4 +1,4 @@ -// Copyright (c) 2016-2020 Alexander Ong +// Copyright (c) 2016-2020 Alexander Ong // See LICENSE in project root for license information. using System.Collections.Generic; @@ -283,10 +283,5 @@ public static bool IsTextEvent(MidiEvent trackEvent, [NotNullWhen(true)] out Bas return false; } - - public static float GetHopoThreshold(ParseSettings settings, float resolution) - { - return settings.GetHopoThreshold(resolution) + 1; // +1 for a small bit of leniency - } } } diff --git a/YARG.Core/MoonscraperChartParser/IO/Midi/MidReader.cs b/YARG.Core/MoonscraperChartParser/IO/Midi/MidReader.cs index a89f61a7a..5798c8f47 100644 --- a/YARG.Core/MoonscraperChartParser/IO/Midi/MidReader.cs +++ b/YARG.Core/MoonscraperChartParser/IO/Midi/MidReader.cs @@ -108,7 +108,18 @@ public static MoonSong ReadMidi(ref ParseSettings settings, MidiFile midi) var song = new MoonSong((uint)ticks.TicksPerQuarterNote); // Apply settings - ValidateAndApplySettings(song, ref settings); + song.hopoThreshold = settings.HopoThreshold > ParseSettings.SETTING_DEFAULT + ? (uint)settings.HopoThreshold + : (song.resolution / 3); + + if (settings.SustainCutoffThreshold <= ParseSettings.SETTING_DEFAULT) + { + settings.SustainCutoffThreshold = song.resolution / 3; + } + + // +1 for a small bit of leniency + song.hopoThreshold++; + settings.SustainCutoffThreshold++; // Read all bpm data in first. This will also allow song.TimeToTick to function properly. ReadSync(midi.GetTempoMap(), song); @@ -195,30 +206,6 @@ static void ReadProKeys(ref ParseSettings settings, TrackChunk track, MoonSong s } } - private static void ValidateAndApplySettings(MoonSong song, ref ParseSettings settings) - { - // Apply HOPO threshold settings - song.hopoThreshold = MidIOHelper.GetHopoThreshold(settings, song.resolution); - - // Verify sustain cutoff threshold - if (settings.SustainCutoffThreshold < 0) - { - // Default to 1/12th step + 1 - settings.SustainCutoffThreshold = (long) (song.resolution / 3) + 1; - } - else - { - // Limit minimum cutoff to 1 tick, non-sustain notes created by charting programs are 1 tick - settings.SustainCutoffThreshold = Math.Max(settings.SustainCutoffThreshold, 1); - } - - // SP note is not verified, as it being set is checked for by SP fixups - // Note snap threshold is also not verified, as the parser doesn't use it - - // Enable chord HOPO cancellation - settings.ChordHopoCancellation = true; - } - private static void ReadSync(TempoMap tempoMap, MoonSong song) { YargLogger.LogTrace("Reading sync track"); diff --git a/YARG.Core/MoonscraperChartParser/MoonSong.cs b/YARG.Core/MoonscraperChartParser/MoonSong.cs index c7e179c7a..7115b6229 100644 --- a/YARG.Core/MoonscraperChartParser/MoonSong.cs +++ b/YARG.Core/MoonscraperChartParser/MoonSong.cs @@ -11,8 +11,8 @@ namespace MoonscraperChartEditor.Song { internal class MoonSong { - public float resolution => syncTrack.Resolution; - public float hopoThreshold; + public uint resolution => syncTrack.Resolution; + public uint hopoThreshold; // Charts private readonly MoonChart[] charts; @@ -171,9 +171,9 @@ public bool Remove(MoonVenue venueEvent) return MoonObjectHelper.Remove(venueEvent, venue); } - public float ResolutionScaleRatio(float targetResoltion) + public float ResolutionScaleRatio(uint targetResoltion) { - return targetResoltion / resolution; + return (float)targetResoltion / resolution; } public static MoonChart.GameMode InstrumentToChartGameMode(MoonInstrument instrument) From b72fcc9bda63604e14ca22874124e2f39a97a0eb Mon Sep 17 00:00:00 2001 From: sonicfind Date: Sat, 5 Oct 2024 13:14:50 -0500 Subject: [PATCH 11/12] Switch to bitmasks from boolean arrays for .mid preparsers --- .../Song/Preparsers/DrumPreparseHandler.cs | 27 ++++++++----------- .../Midi/MidiEliteDrumsPreparser.cs | 14 +++++----- .../Preparsers/Midi/MidiFiveFretPreparser.cs | 14 +++++----- .../Preparsers/Midi/MidiProGuitarPreparser.cs | 14 +++++----- .../Preparsers/Midi/MidiProKeysPreparser.cs | 7 ++--- .../Preparsers/Midi/MidiSixFretPreparser.cs | 14 +++++----- 6 files changed, 43 insertions(+), 47 deletions(-) diff --git a/YARG.Core/Song/Preparsers/DrumPreparseHandler.cs b/YARG.Core/Song/Preparsers/DrumPreparseHandler.cs index 8a86e10e2..0ed5563db 100644 --- a/YARG.Core/Song/Preparsers/DrumPreparseHandler.cs +++ b/YARG.Core/Song/Preparsers/DrumPreparseHandler.cs @@ -90,16 +90,12 @@ public unsafe void ParseMidi(YARGMidiTrack track) const int MAX_NUMPADS = 7; const int DRUMNOTE_MAX = 101; const int DOUBLE_KICK_NOTE = 95; - const int EXPERT_INDEX = 3; - const int EXPERT_PLUS_INDEX = 4; - const int DOUBLE_KICK_OFFSET = EXPERT_INDEX * MAX_NUMPADS + 1; + const int DOUBLE_KICK_MASK = 1 << (3 * MAX_NUMPADS + 1); const int FIVE_LANE_INDEX = 6; const int YELLOW_FLAG = 110; const int GREEN_FLAG = 112; - // +1 for Expert+ - var difficulties = stackalloc bool[MidiPreparser_Constants.NUM_DIFFICULTIES + 1]; - var statuses = stackalloc bool[MidiPreparser_Constants.NUM_DIFFICULTIES * MAX_NUMPADS]; + int statusBitMask = 0; var note = default(MidiNote); while (track.ParseEvent()) { @@ -112,7 +108,7 @@ public unsafe void ParseMidi(YARGMidiTrack track) // Must be checked first as it still resides in the normal note range window if (note.value == DOUBLE_KICK_NOTE) { - if (difficulties[EXPERT_PLUS_INDEX]) + if ((_validations & DifficultyMask.ExpertPlus) > 0) { continue; } @@ -120,22 +116,21 @@ public unsafe void ParseMidi(YARGMidiTrack track) // Note Ons with no velocity equates to a note Off by spec if (track.Type == MidiEventType.Note_On && note.velocity > 0) { - statuses[DOUBLE_KICK_OFFSET] = true; + statusBitMask |= DOUBLE_KICK_MASK; } // NoteOff here - else if (statuses[DOUBLE_KICK_OFFSET]) + else if ((statusBitMask & DOUBLE_KICK_MASK) > 0) { _validations |= DifficultyMask.Expert | DifficultyMask.ExpertPlus; - difficulties[EXPERT_INDEX] = true; - difficulties[EXPERT_PLUS_INDEX] = true; } } else if (MidiPreparser_Constants.DEFAULT_NOTE_MIN <= note.value && note.value <= DRUMNOTE_MAX) { int noteOffset = note.value - MidiPreparser_Constants.DEFAULT_NOTE_MIN; int diffIndex = MidiPreparser_Constants.DIFF_INDICES[noteOffset]; + var diffMask = (DifficultyMask) (1 << (diffIndex + 1)); // Necessary to account for potential five lane - if (difficulties[diffIndex] && Type != DrumsType.Unknown && Type != DrumsType.UnknownPro) + if ((_validations & diffMask) > 0 && Type != DrumsType.Unknown && Type != DrumsType.UnknownPro) { continue; } @@ -148,20 +143,20 @@ public unsafe void ParseMidi(YARGMidiTrack track) continue; } + int statusMask = 1 << (diffIndex * MAX_NUMPADS + laneIndex); // Note Ons with no velocity equates to a note Off by spec if (track.Type == MidiEventType.Note_On && note.velocity > 0) { - statuses[diffIndex * MAX_NUMPADS + laneIndex] = true; + statusBitMask |= statusMask; if (laneIndex == FIVE_LANE_INDEX) { Type = DrumsType.FiveLane; } } // NoteOff here - else if (statuses[diffIndex * MAX_NUMPADS + laneIndex]) + else if ((statusBitMask & statusMask) > 0) { - _validations |= (DifficultyMask) (1 << (diffIndex + 1)); - difficulties[diffIndex] = true; + _validations |= diffMask; } } else if (YELLOW_FLAG <= note.value && note.value <= GREEN_FLAG && Type != DrumsType.FiveLane) diff --git a/YARG.Core/Song/Preparsers/Midi/MidiEliteDrumsPreparser.cs b/YARG.Core/Song/Preparsers/Midi/MidiEliteDrumsPreparser.cs index 4251a5c56..53f59bf00 100644 --- a/YARG.Core/Song/Preparsers/Midi/MidiEliteDrumsPreparser.cs +++ b/YARG.Core/Song/Preparsers/Midi/MidiEliteDrumsPreparser.cs @@ -11,8 +11,7 @@ public static class Midi_EliteDrums_Preparser public static unsafe DifficultyMask Parse(YARGMidiTrack track) { var validations = default(DifficultyMask); - var difficulties = stackalloc bool[MidiPreparser_Constants.NUM_DIFFICULTIES]; - var statuses = stackalloc bool[MidiPreparser_Constants.NUM_DIFFICULTIES * NUM_LANES]; + long statusBitMask = 0; var note = default(MidiNote); while (track.ParseEvent()) @@ -28,21 +27,22 @@ public static unsafe DifficultyMask Parse(YARGMidiTrack track) int diffIndex = MidiPreparser_Constants.EXTENDED_DIFF_INDICES[note.value]; int laneIndex = MidiPreparser_Constants.EXTENDED_LANE_INDICES[note.value]; - if (difficulties[diffIndex] || laneIndex >= NUM_LANES) + var diffMask = (DifficultyMask)(1 << (diffIndex + 1)); + if ((validations & diffMask) > 0 || laneIndex >= NUM_LANES) { continue; } + long statusMask = 1L << (diffIndex * NUM_LANES + laneIndex); // Note Ons with no velocity equates to a note Off by spec if (track.Type == MidiEventType.Note_On && note.velocity > 0) { - statuses[diffIndex * NUM_LANES + laneIndex] = true; + statusBitMask |= statusMask; } // Note off here - else if (statuses[diffIndex * NUM_LANES + laneIndex]) + else if ((statusBitMask & statusMask) > 0) { - validations |= (DifficultyMask) (1 << (diffIndex + 1)); - difficulties[diffIndex] = true; + validations |= diffMask; if (validations == MidiPreparser_Constants.ALL_DIFFICULTIES) { break; diff --git a/YARG.Core/Song/Preparsers/Midi/MidiFiveFretPreparser.cs b/YARG.Core/Song/Preparsers/Midi/MidiFiveFretPreparser.cs index 5fb990b27..7b2f0d1bb 100644 --- a/YARG.Core/Song/Preparsers/Midi/MidiFiveFretPreparser.cs +++ b/YARG.Core/Song/Preparsers/Midi/MidiFiveFretPreparser.cs @@ -25,8 +25,7 @@ public static unsafe DifficultyMask Parse(YARGMidiTrack track) { ReadOnlySpan SYSEXTAG = stackalloc byte[] { (byte) 'P', (byte) 'S', (byte) '\0', }; var validations = default(DifficultyMask); - var difficulties = stackalloc bool[MidiPreparser_Constants.NUM_DIFFICULTIES]; - var statuses = stackalloc bool[MidiPreparser_Constants.NUM_DIFFICULTIES * NUM_LANES]; + int statusBitMask = 0; // Zero is reserved for open notes. Open notes apply in two situations: // 1. The 13s will swap to zeroes when the ENHANCED_OPENS toggle occurs @@ -55,21 +54,22 @@ public static unsafe DifficultyMask Parse(YARGMidiTrack track) int noteOffset = note.value - FIVEFRET_MIN; int diffIndex = MidiPreparser_Constants.DIFF_INDICES[noteOffset]; int laneIndex = indices[noteOffset]; - if (difficulties[diffIndex] || laneIndex >= NUM_LANES) + var diffMask = (DifficultyMask) (1 << (diffIndex + 1)); + if ((validations & diffMask) > 0 || laneIndex >= NUM_LANES) { continue; } + int statusMask = 1 << (diffIndex * NUM_LANES + laneIndex); // Note Ons with no velocity equates to a note Off by spec if (track.Type == MidiEventType.Note_On && note.velocity > 0) { - statuses[diffIndex * NUM_LANES + laneIndex] = true; + statusBitMask |= statusMask; } // Note off here - else if (statuses[diffIndex * NUM_LANES + laneIndex]) + else if ((statusBitMask & statusMask) > 0) { - validations |= (DifficultyMask) (1 << (diffIndex + 1)); - difficulties[diffIndex] = true; + validations |= diffMask; if (validations == MidiPreparser_Constants.ALL_DIFFICULTIES) { break; diff --git a/YARG.Core/Song/Preparsers/Midi/MidiProGuitarPreparser.cs b/YARG.Core/Song/Preparsers/Midi/MidiProGuitarPreparser.cs index 93737a13d..4a66b4a2f 100644 --- a/YARG.Core/Song/Preparsers/Midi/MidiProGuitarPreparser.cs +++ b/YARG.Core/Song/Preparsers/Midi/MidiProGuitarPreparser.cs @@ -26,8 +26,7 @@ public static DifficultyMask Parse_22Fret(YARGMidiTrack track) private static unsafe DifficultyMask Parse(YARGMidiTrack track, int maxVelocity) { var validations = default(DifficultyMask); - var difficulties = stackalloc bool[MidiPreparser_Constants.NUM_DIFFICULTIES]; - var statuses = stackalloc bool[MidiPreparser_Constants.NUM_DIFFICULTIES * NUM_STRINGS]; + int statusBitMask = 0; var note = default(MidiNote); while (track.ParseEvent()) @@ -43,25 +42,26 @@ private static unsafe DifficultyMask Parse(YARGMidiTrack track, int maxVelocity) int noteOffset = note.value - PROGUITAR_MIN; int diffIndex = MidiPreparser_Constants.EXTENDED_DIFF_INDICES[noteOffset]; int laneIndex = MidiPreparser_Constants.EXTENDED_LANE_INDICES[noteOffset]; + var diffMask = (DifficultyMask) (1 << (diffIndex + 1)); // Ghost notes aren't played - if (difficulties[diffIndex] || laneIndex >= NUM_STRINGS || track.Channel == ARPEGGIO_CHANNEL) + if ((validations & diffMask) > 0 || laneIndex >= NUM_STRINGS || track.Channel == ARPEGGIO_CHANNEL) { continue; } + int statusMask = 1 << (diffIndex * NUM_STRINGS + laneIndex); // Note Ons with no velocity equates to a note Off by spec if (track.Type == MidiEventType.Note_On && note.velocity > 0) { if (MIN_VELOCITY <= note.velocity && note.velocity <= maxVelocity) { - statuses[diffIndex * NUM_STRINGS + laneIndex] = true; + statusBitMask |= statusMask; } } // Note off here - else if (statuses[diffIndex * NUM_STRINGS + laneIndex]) + else if ((statusBitMask & statusMask) > 0) { - validations |= (DifficultyMask) (1 << (diffIndex + 1)); - difficulties[diffIndex] = true; + validations |= diffMask; if (validations == MidiPreparser_Constants.ALL_DIFFICULTIES) { break; diff --git a/YARG.Core/Song/Preparsers/Midi/MidiProKeysPreparser.cs b/YARG.Core/Song/Preparsers/Midi/MidiProKeysPreparser.cs index ff48c5caa..0cfb69a16 100644 --- a/YARG.Core/Song/Preparsers/Midi/MidiProKeysPreparser.cs +++ b/YARG.Core/Song/Preparsers/Midi/MidiProKeysPreparser.cs @@ -10,7 +10,7 @@ public static class Midi_ProKeys_Preparser public static unsafe bool Parse(YARGMidiTrack track) { - var statuses = stackalloc bool[NOTES_IN_DIFFICULTY]; + int statusBitMask = 0; var note = default(MidiNote); while (track.ParseEvent()) { @@ -19,11 +19,12 @@ public static unsafe bool Parse(YARGMidiTrack track) track.ExtractMidiNote(ref note); if (PROKEYS_MIN <= note.value && note.value <= PROKEYS_MAX) { + int statusMask = 1 << (note.value - PROKEYS_MIN); if (track.Type == MidiEventType.Note_On && note.velocity > 0) { - statuses[note.value - PROKEYS_MIN] = true; + statusBitMask |= statusMask; } - else if (statuses[note.value - PROKEYS_MIN]) + else if ((statusBitMask & statusMask) > 0) { return true; } diff --git a/YARG.Core/Song/Preparsers/Midi/MidiSixFretPreparser.cs b/YARG.Core/Song/Preparsers/Midi/MidiSixFretPreparser.cs index d2b23273d..1b2d52f49 100644 --- a/YARG.Core/Song/Preparsers/Midi/MidiSixFretPreparser.cs +++ b/YARG.Core/Song/Preparsers/Midi/MidiSixFretPreparser.cs @@ -21,8 +21,7 @@ public static class Midi_SixFret_Preparser public static unsafe DifficultyMask Parse(YARGMidiTrack track) { var validations = default(DifficultyMask); - var difficulties = stackalloc bool[MidiPreparser_Constants.NUM_DIFFICULTIES]; - var statuses = stackalloc bool[MidiPreparser_Constants.NUM_DIFFICULTIES * NUM_LANES]; + int statusBitMask = 0; var note = default(MidiNote); while (track.ParseEvent()) @@ -38,21 +37,22 @@ public static unsafe DifficultyMask Parse(YARGMidiTrack track) int noteOffset = note.value - SIXFRET_MIN; int diffIndex = MidiPreparser_Constants.DIFF_INDICES[noteOffset]; int laneIndex = INDICES[noteOffset]; - if (difficulties[diffIndex] || laneIndex >= NUM_LANES) + var diffMask = (DifficultyMask) (1 << (diffIndex + 1)); + if ((validations & diffMask) > 0 || laneIndex >= NUM_LANES) { continue; } + int statusMask = 1 << (diffIndex * NUM_LANES + laneIndex); // Note Ons with no velocity equates to a note Off by spec if (track.Type == MidiEventType.Note_On && note.velocity > 0) { - statuses[diffIndex * NUM_LANES + laneIndex] = true; + statusBitMask |= statusMask; } // Note off here - else if (statuses[diffIndex * NUM_LANES + laneIndex]) + else if ((statusBitMask & statusMask) > 0) { - validations |= (DifficultyMask) (1 << (diffIndex + 1)); - difficulties[diffIndex] = true; + validations |= diffMask; if (validations == MidiPreparser_Constants.ALL_DIFFICULTIES) { break; From 5cf202b8917bbfdb63d079ffa07128d93cc4e9e6 Mon Sep 17 00:00:00 2001 From: sonicfind Date: Mon, 7 Oct 2024 01:17:49 -0500 Subject: [PATCH 12/12] Fix CONs on quick scan --- YARG.Core/Song/Cache/CacheHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/YARG.Core/Song/Cache/CacheHandler.cs b/YARG.Core/Song/Cache/CacheHandler.cs index 90d99fbb5..930d64db6 100644 --- a/YARG.Core/Song/Cache/CacheHandler.cs +++ b/YARG.Core/Song/Cache/CacheHandler.cs @@ -1390,7 +1390,7 @@ protected void QuickReadUpgradeCON(UnmanagedMemoryStream stream) { string filename = stream.ReadString(); var info = new AbridgedFileInfo(filename, stream); - if (File.Exists(filename)) + if (!File.Exists(filename)) { return null; }