From a6de1ac3dac495539d12f3e0fd9cfd8a3e90e41e Mon Sep 17 00:00:00 2001 From: Bayu Satiyo Date: Fri, 28 Jul 2023 16:11:01 +0700 Subject: [PATCH] Add support for compressed build (LZ4/LZ4HC) Signed-off-by: Bayu Satiyo --- .vscode/launch.json | 7 +- Program.cs | 284 ++++++++++++++++++++++++++++++-------------- USSR.csproj | 1 + UnityWebData.cs | 146 +++++++++++++++++++++++ Utility.cs | 59 +++++++++ 5 files changed, 408 insertions(+), 89 deletions(-) create mode 100644 UnityWebData.cs create mode 100644 Utility.cs diff --git a/.vscode/launch.json b/.vscode/launch.json index abe7689..3d542fe 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -14,8 +14,9 @@ "args": [], "cwd": "${workspaceFolder}", // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console - "console": "internalConsole", - "stopAtEntry": false + "console": "integratedTerminal", + "stopAtEntry": false, + "requireExactSource": false }, { "name": ".NET Core Attach", @@ -23,4 +24,4 @@ "request": "attach" } ] -} \ No newline at end of file +} diff --git a/Program.cs b/Program.cs index f6b918a..b48bc76 100644 --- a/Program.cs +++ b/Program.cs @@ -2,14 +2,22 @@ using AssetsTools.NET; using AssetsTools.NET.Extra; -namespace kiraio.USSR +namespace Kiraio.USSR { public class Program { const string VERSION = "1.0.0"; const string ASSET_CLASS_DB = "classdata.tpk"; - public static void Main(string[] args) + public enum LoadTypes + { + Asset, + Bundle + } + + static LoadTypes loadTypes; + + static void Main(string[] args) { if (args.Length < 1) { @@ -31,6 +39,7 @@ public static void Main(string[] args) string? ussrPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); string? assetClassTypesPackage = Path.Combine(ussrPath, ASSET_CLASS_DB); + if (!File.Exists(assetClassTypesPackage)) { Console.WriteLine($"Asset class types package not found: {assetClassTypesPackage}"); @@ -43,35 +52,51 @@ public static void Main(string[] args) AssetsManager? assetsManager = new(); assetsManager.LoadClassPackage(assetClassTypesPackage); - // Target string? execPath = args[0]; string? execExtension = Path.GetExtension(execPath); string? rootPath = Path.GetDirectoryName(execPath); string? targetFile; + List temporaryFiles = new(); + Console.WriteLine("Checking for supported platforms..."); switch (execExtension) { case ".exe": case ".x86": case ".x86_64": case ".dmg": + Console.WriteLine("Supported."); + string? dataPath = Path.Combine( rootPath, $"{Path.GetFileNameWithoutExtension(execPath)}_Data" ); + + Console.WriteLine("Checking for globalgamemanagers..."); + + // Default compression targetFile = Path.Combine(dataPath, "globalgamemanagers"); + loadTypes = LoadTypes.Asset; - // Use compression + // LZMA/LZ4 compression if (!File.Exists(targetFile)) + { + Console.WriteLine( + "globalgamemanagers not found. Checking for data.unity3d instead..." + ); + targetFile = Path.Combine(dataPath, "data.unity3d"); + loadTypes = LoadTypes.Bundle; + } break; case ".html": - targetFile = Path.Combine(rootPath, "Build", "WebGL.data"); // TODO: Process WebGL.data - break; + // targetFile = Path.Combine(rootPath, "Build", "WebGL.data"); + // break; + return; default: - Console.WriteLine("Sorry, unsupported platform."); + Console.WriteLine("Sorry, unsupported platform :("); Console.ReadLine(); return; } @@ -80,68 +105,116 @@ public static void Main(string[] args) if (!File.Exists(targetFile)) { Console.WriteLine($"{targetFile} doesn't exists!"); + Console.WriteLine("The file is moved or deleted."); Console.ReadLine(); return; } // Make temporary copy - string? inspectedFile = CloneFile( - targetFile, - backupDestinationPath: $"{targetFile}.temp" - ); + string? inspectedFile = Utility.CloneFile(targetFile, $"{targetFile}.temp"); + temporaryFiles.Add(inspectedFile); - Console.WriteLine("Loading asset file and it's dependencies..."); - // Load target file and it's dependencies - // Loading the dependencies is required to check unity logo asset - AssetsFileInstance? assetFileInstance = assetsManager.LoadAssetsFile( - inspectedFile, - true - ); + AssetsFileInstance? assetFileInstance = null; + BundleFileInstance? bundleFileInstance = null; + FileStream? bundleStream = null; + + try + { + switch (loadTypes) + { + // globalgamemanagers + case LoadTypes.Asset: + // Load target file and it's dependencies + // Loading the dependencies is required to check unity logo asset + Console.WriteLine("Loading asset file and it's dependencies..."); + assetFileInstance = assetsManager.LoadAssetsFile(inspectedFile, true); + break; + // data.unity3d + case LoadTypes.Bundle: + Console.WriteLine("Loading asset bundle file..."); + bundleFileInstance = assetsManager.LoadBundleFile(inspectedFile, false); + + string? bundleStreamFile = $"{targetFile}.stream"; + bundleStream = File.Open(bundleStreamFile, FileMode.Create); + + bundleFileInstance.file = BundleHelper.UnpackBundleToStream( + bundleFileInstance.file, + bundleStream + ); + + // Add to cleanup chores + temporaryFiles.Add(bundleStreamFile); + + Console.WriteLine("Loading asset file and it's dependencies..."); + assetFileInstance = assetsManager.LoadAssetsFileFromBundle( + bundleFileInstance, + 0, + true + ); + + break; + } + } + catch (Exception ex) + { + Console.WriteLine($"Error loading asset file. {ex.Message}"); + Console.ReadLine(); + return; + } - AssetsFile assetFile = assetFileInstance.file; + AssetBundleFile? bundleFile = bundleFileInstance?.file; + AssetsFile? assetFile = assetFileInstance?.file; Console.WriteLine("Loading asset class types database..."); - assetsManager.LoadClassDatabaseFromPackage(assetFile.Metadata.UnityVersion); + assetsManager.LoadClassDatabaseFromPackage(assetFile?.Metadata.UnityVersion); - List? buildSettingsInfo = assetFile.GetAssetsOfType( + List? buildSettingsInfo = assetFile?.GetAssetsOfType( AssetClassID.BuildSettings ); - // Get base field + // Get BuildSettings base field AssetTypeValueField? buildSettingsBase = assetsManager.GetBaseField( assetFileInstance, - buildSettingsInfo[0] + buildSettingsInfo?[0] ); - List? playerSettingsInfo = assetFile.GetAssetsOfType( + List? playerSettingsInfo = assetFile?.GetAssetsOfType( AssetClassID.PlayerSettings ); - // Get base field + // Get PlayerSettings base field AssetTypeValueField? playerSettingsBase = assetsManager.GetBaseField( assetFileInstance, - playerSettingsInfo[0] + playerSettingsInfo?[0] ); + // Get m_SplashScreenLogos field as array AssetTypeValueField? splashScreenLogos = playerSettingsBase[ "m_SplashScreenLogos.Array" ]; - // Get necessary fields + // Get required fields to remove the splash screen bool isProVersion = buildSettingsBase["hasPROVersion"].AsBool; bool showUnityLogo = playerSettingsBase["m_ShowUnitySplashLogo"].AsBool; if (isProVersion && !showUnityLogo) { Console.WriteLine( - "Unity splash screen didn't exist or already removed. Nothing to do." + "Unity splash screen logo didn't exist or already removed. Nothing to do." ); - assetsManager.UnloadAssetsFile(inspectedFile); - File.Delete(inspectedFile); + + bundleStream?.Close(); + assetsManager.UnloadAll(true); + Utility.CleanUp(temporaryFiles); + Console.ReadLine(); return; } - // Backup target file - Console.WriteLine("Backup original file..."); - CloneFile(targetFile, backupDestinationPath: $"{targetFile}.bak"); + // Backup original file + string? backupOriginalFile = $"{targetFile}.bak"; + if (!File.Exists(backupOriginalFile)) + { + Console.WriteLine("Backup original file..."); + Utility.CloneFile(targetFile, backupOriginalFile); + } Console.WriteLine("Removing Unity splash screen..."); @@ -154,51 +227,117 @@ public static void Main(string[] args) foreach (AssetTypeValueField data in splashScreenLogos) { - AssetTypeValueField? logoPointer = data["logo"]; + // Get the Sprite asset + AssetTypeValueField? logoPointer = data?["logo"]; + // Get the external asset AssetExternal logoExtInfo = assetsManager.GetExtAsset( assetFileInstance, logoPointer ); - AssetTypeValueField? logoBase = logoExtInfo.baseField; - string? logoName = logoBase["m_Name"].AsString; - // If it's Unity splash screen logo - if (logoName.Contains("UnitySplash-cube")) + // IDK why AssetsTools won't load "UnitySplash-cube" + // external asset while in Bundle file. So, we can + // check it's name then remove it. + // So, we break it into 2 types of file load. + switch (loadTypes) { - unityLogo = data; - break; + case LoadTypes.Asset: + // Get the base field + AssetTypeValueField? logoBase = logoExtInfo.baseField; + string? logoName = logoBase["m_Name"].AsString; + + // If it's Unity splash screen logo + if (logoName.Contains("UnitySplash-cube")) + unityLogo = data; + + break; + case LoadTypes.Bundle: + // After some testing, I realize only Unity + // splash screen logo external asset that + // won't load. So, we can use it to mark + // that this is the Unity splash screen logo + if (logoExtInfo.baseField == null) + unityLogo = data; + break; } } - // Remove Unity splash screen logo to make sure we completely remove Unity splash screen. Only our logo remained. - splashScreenLogos.Children.Remove(unityLogo); + // Remove Unity splash screen logo to completely remove + // Unity splash screen logo. Only our logo remained. + if (unityLogo != null) + splashScreenLogos?.Children.Remove(unityLogo); - Console.WriteLine( - $"hasPROVersion: {buildSettingsBase["hasPROVersion"].AsBool} | m_ShowUnitySplashLogo: {playerSettingsBase["m_ShowUnitySplashLogo"].AsBool} | UnitySplash-cube: {splashScreenLogos.Children.Contains(unityLogo)}" - ); + Console.WriteLine("Done."); // Store modified base fields - List? replacers = + List? assetsReplacers = new() { new AssetsReplacerFromMemory( assetFile, - buildSettingsInfo[0], + buildSettingsInfo?[0], buildSettingsBase ), new AssetsReplacerFromMemory( assetFile, - playerSettingsInfo[0], + playerSettingsInfo?[0], playerSettingsBase ) }; + List bundleReplacers = + new() + { + new BundleReplacerFromAssets( + assetFileInstance?.name, + null, + assetFile, + assetsReplacers + ) + }; + + FileStream? uncompressedBundleStream = null; + try { // Write modified asset file to disk - using AssetsFileWriter writer = new(targetFile); Console.WriteLine("Writing changes to disk..."); - assetFile.Write(writer, 0, replacers); + + switch (loadTypes) + { + case LoadTypes.Asset: + using (AssetsFileWriter writer = new(targetFile)) + { + assetFile?.Write(writer, 0, assetsReplacers); + } + break; + case LoadTypes.Bundle: + string uncompressedBundleFile = $"{targetFile}.uncompressed"; + temporaryFiles.Add(uncompressedBundleFile); + + using (AssetsFileWriter writer = new(uncompressedBundleFile)) + { + bundleFile?.Write(writer, bundleReplacers); + } + + uncompressedBundleStream = File.OpenRead(uncompressedBundleFile); + + AssetBundleFile? uncompressedBundle = new(); + uncompressedBundle.Read(new AssetsFileReader(uncompressedBundleStream)); + + using (AssetsFileReader reader = new(uncompressedBundleStream)) + { + Console.WriteLine("Compressing asset bundle file..."); + + using AssetsFileWriter writer = new(targetFile); + uncompressedBundle.Pack( + uncompressedBundle.Reader, + writer, + AssetBundleCompressionType.LZ4 + ); + } + break; + } } catch (Exception ex) { @@ -209,47 +348,20 @@ public static void Main(string[] args) return; } + // Cleanup temporary files + Console.WriteLine("Cleaning up temporary files..."); + + bundleStream?.Close(); + uncompressedBundleStream?.Close(); + assetsManager.UnloadAllBundleFiles(); + assetsManager.UnloadAllAssetsFiles(true); + Utility.CleanUp(temporaryFiles); + Console.WriteLine("Successfully removed Unity splash screen. Enjoy :) \n"); Console.WriteLine( "Don't forget to visit USSR repo: https://github.com/kiraio-moe/USSR and give it a star!" ); Console.ReadLine(); } - - /// - /// Clone a file. - /// - /// - /// - /// Cloned file path - static string CloneFile(string sourceFilePath, string backupDestinationPath) - { - try - { - // Check if the source file exists - if (!File.Exists(sourceFilePath)) - { - return new FileNotFoundException( - "Backup source file does not exist!" - ).ToString(); - } - - // Create the backup destination directory if it doesn't exist - string? backupDir = Path.GetDirectoryName(backupDestinationPath); - if (Directory.Exists(backupDir)) - { - Directory.CreateDirectory(backupDir); - } - - // Copy the source file to the backup destination - File.Copy(sourceFilePath, backupDestinationPath, true); - } - catch (Exception ex) - { - Console.WriteLine($"An error occurred during the backup process: {ex.Message}"); - } - - return backupDestinationPath; - } } } diff --git a/USSR.csproj b/USSR.csproj index db8d1fe..13ffb49 100644 --- a/USSR.csproj +++ b/USSR.csproj @@ -9,6 +9,7 @@ + diff --git a/UnityWebData.cs b/UnityWebData.cs new file mode 100644 index 0000000..9618986 --- /dev/null +++ b/UnityWebData.cs @@ -0,0 +1,146 @@ +// This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild + +namespace Kaitai +{ + public partial class UnityWebData : KaitaiStruct + { + public static UnityWebData FromFile(string fileName) + { + return new UnityWebData(new KaitaiStream(fileName)); + } + + public UnityWebData( + KaitaiStream p__io, + KaitaiStruct p__parent = null, + UnityWebData p__root = null + ) : base(p__io) + { + m_parent = p__parent; + m_root = p__root ?? this; + _read(); + } + + private void _read() + { + _magic = System.Text.Encoding + .GetEncoding("utf-8") + .GetString(m_io.ReadBytesTerm(0, false, true, true)); + _beginOffset = m_io.ReadU4le(); + _files = new List(); + { + var i = 0; + FileEntry M_; + do + { + M_ = new FileEntry(m_io, this, m_root); + _files.Add(M_); + i++; + } while (!(M_Io.Pos == BeginOffset)); + } + } + + public partial class FileEntry : KaitaiStruct + { + public static FileEntry FromFile(string fileName) + { + return new FileEntry(new KaitaiStream(fileName)); + } + + public FileEntry( + KaitaiStream p__io, + UnityWebData p__parent = null, + UnityWebData p__root = null + ) : base(p__io) + { + m_parent = p__parent; + m_root = p__root; + f_data = false; + _read(); + } + + private void _read() + { + _fileOffset = m_io.ReadU4le(); + _fileSize = m_io.ReadU4le(); + _filenameSize = m_io.ReadU4le(); + _filename = System.Text.Encoding + .GetEncoding("utf-8") + .GetString(m_io.ReadBytes(FilenameSize)); + } + + private bool f_data; + private byte[] _data; + public byte[] Data + { + get + { + if (f_data) + return _data; + KaitaiStream io = M_Root.M_Io; + long _pos = io.Pos; + io.Seek(FileOffset); + _data = io.ReadBytes(FileSize); + io.Seek(_pos); + f_data = true; + return _data; + } + } + private uint _fileOffset; + private uint _fileSize; + private uint _filenameSize; + private string _filename; + private UnityWebData m_root; + private UnityWebData m_parent; + public uint FileOffset + { + get { return _fileOffset; } + } + public uint FileSize + { + get { return _fileSize; } + } + public uint FilenameSize + { + get { return _filenameSize; } + } + public string Filename + { + get { return _filename; } + } + public UnityWebData M_Root + { + get { return m_root; } + } + public UnityWebData M_Parent + { + get { return m_parent; } + } + } + + private string _magic; + private uint _beginOffset; + private List _files; + private UnityWebData m_root; + private KaitaiStruct m_parent; + public string Magic + { + get { return _magic; } + } + public uint BeginOffset + { + get { return _beginOffset; } + } + public List Files + { + get { return _files; } + } + public UnityWebData M_Root + { + get { return m_root; } + } + public KaitaiStruct M_Parent + { + get { return m_parent; } + } + } +} diff --git a/Utility.cs b/Utility.cs new file mode 100644 index 0000000..c6b05c3 --- /dev/null +++ b/Utility.cs @@ -0,0 +1,59 @@ +namespace Kiraio.USSR +{ + public class Utility + { + /// + /// Clone a file. + /// + /// + /// + /// Cloned file path + public static string CloneFile(string sourceFilePath, string backupDestinationPath) + { + try + { + // Check if the source file exists + if (!File.Exists(sourceFilePath)) + { + return new FileNotFoundException( + "Backup source file does not exist!" + ).ToString(); + } + + // Create the backup destination directory if it doesn't exist + string? backupDir = Path.GetDirectoryName(backupDestinationPath); + if (Directory.Exists(backupDir)) + { + Directory.CreateDirectory(backupDir); + } + + // Copy the source file to the backup destination + File.Copy(sourceFilePath, backupDestinationPath, true); + } + catch (Exception ex) + { + Console.WriteLine($"An error occurred during the backup process: {ex.Message}"); + } + + return backupDestinationPath; + } + + /// + /// Delete unnecessary . + /// + /// + public static void CleanUp(List? files) + { + if (files.Count < 1) + return; + + foreach (string file in files) + { + if (File.Exists(file)) + { + File.Delete(file); + } + } + } + } +}