diff --git a/.gitignore b/.gitignore index 7398c67..6011239 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ config.toml data/ .idea *.user +.vs/ +launchSettings.json \ No newline at end of file diff --git a/UnityDataMiner/DownloadableAssetCollection.cs b/UnityDataMiner/DownloadableAssetCollection.cs new file mode 100644 index 0000000..0a6a019 --- /dev/null +++ b/UnityDataMiner/DownloadableAssetCollection.cs @@ -0,0 +1,64 @@ +using AssetRipper.Primitives; +using Serilog; +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; + +namespace UnityDataMiner +{ + internal class DownloadableAssetCollection + { + // url -> path + private readonly Dictionary assets = new(); + + // returns actual asset path + public string AddAsset(string url, string destPath) + { + if (!assets.TryGetValue(url, out var realPath)) + { + assets.Add(url, realPath = destPath); + } + + return realPath; + } + + public async Task DownloadAssetsAsync(Func downloadFunc, UnityVersion version, + SemaphoreSlim? downloadLock = null, CancellationToken cancellationToken = default) + { + while (true) + { + try + { + if (downloadLock is not null) + { + await downloadLock.WaitAsync(cancellationToken); + } + try + { + foreach (var (url, dest) in assets) + { + await downloadFunc(url, dest, cancellationToken); + } + } + finally + { + if (!cancellationToken.IsCancellationRequested && downloadLock is not null) + { + downloadLock.Release(); + } + } + + break; + } + catch (IOException e) when (e.InnerException is SocketException { SocketErrorCode: SocketError.ConnectionReset }) + { + Log.Warning("Failed to download {Version}, waiting 5 seconds before retrying...", version); + await Task.Delay(5000, cancellationToken); + } + } + } + } +} diff --git a/UnityDataMiner/Unstrip.cs b/UnityDataMiner/EuUnstrip.cs similarity index 100% rename from UnityDataMiner/Unstrip.cs rename to UnityDataMiner/EuUnstrip.cs diff --git a/UnityDataMiner/JobPlanner.cs b/UnityDataMiner/JobPlanner.cs new file mode 100644 index 0000000..dc53935 --- /dev/null +++ b/UnityDataMiner/JobPlanner.cs @@ -0,0 +1,336 @@ +using Serilog; +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.CommandLine; +using System.Diagnostics; +using System.Linq; +using System.Threading; + +namespace UnityDataMiner +{ + internal class JobPlanner + { + public static JobPlan? Plan(ImmutableArray jobs, UnityBuild build, CancellationToken cancellationToken) + { + Log.Information("[{Version}] Planning mine...", build.Version); + + var needJobs = new List(); + var jobDeps = new List>(); + + // first, go through the provided jobs, and filter to the ones that can and should run + foreach (var job in jobs) + { + if (!job.CanRunFor(build)) + { + Log.Debug("[{Version}] Skipping {Job} because it's not applicable", build.Version, job.Name); + continue; + } + + if (!job.ShouldRunFor(build)) + { + Log.Debug("[{Version}] Skipping {Job} because it does not need to run", build.Version, job.Name); + continue; + } + + var depOptions = job.GetDependencies(build); + needJobs.Add(job); + jobDeps.Add(depOptions); + } + + Debug.Assert(needJobs.Count == jobDeps.Count); + + if (needJobs.Count == 0) + { + Log.Information("[{Version}] No applicable jobs; no work needs to be done.", build.Version); + return null; + } + + Log.Information("[{Version}] {Jobs} jobs to run", build.Version, needJobs.Count); + + var planTree = new PlanTree(build, needJobs, jobDeps); + + return planTree.ComputeFullJobPlan(cancellationToken); + } + + private sealed class PlanTree(UnityBuild build, List jobs, List> jobDeps) + { + private readonly UnityBuild build = build; + + private readonly List jobs = jobs; + private readonly List> jobDeps = jobDeps; + + private readonly PriorityQueue queue = new(); + + private void EnqueueCandidateMoves(ImmutableHashSet curPlan, int jobIndex) + { + foreach (var depSet in jobDeps[jobIndex]) + { + EnqueueForDepSet(curPlan, depSet, 0, jobIndex); + } + } + + // TODO: it would be nice to figure out some early-outs that can enable us to skip large branches of the tree + private void EnqueueForDepSet(ImmutableHashSet plan, MinerDependencyOption depSet, int depIndex, int jobIndex) + { + // enclose in look to try to tail-call opt this explicitly + do + { + if (depIndex == depSet.NeededPackages.Length) + { + // no more deps to enqueue, enqueue this plan + var newPlan = new InProgressPlan(plan, jobIndex + 1); + queue.Enqueue(newPlan, newPlan.PlanWeight); // TODO: compute weight as we go? + return; + } + + var dep = depSet.NeededPackages[depIndex]; + + // check whether this dep is already satisfied + foreach (var planned in plan) + { + if (dep.Matches(planned.Package)) + { + goto NextDep; + } + } + + // try to find suitable packages + if (!build.IsMonolithic) + { + ReadOnlySpan<(EditorOS selectedOs, UnityBuildInfo? buildInfo)> candidates = dep.OS switch + { + EditorOS.Windows => [(EditorOS.Windows, build.WindowsInfo)], + EditorOS.Linux => [(EditorOS.Linux, build.LinuxInfo)], + EditorOS.MacOS => [(EditorOS.MacOS, build.MacOsInfo)], + + EditorOS.Any => [ + (EditorOS.Windows, build.WindowsInfo), + (EditorOS.Linux, build.LinuxInfo), + (EditorOS.MacOS, build.MacOsInfo), + ], + + _ => throw new NotSupportedException(), + }; + + if (candidates is [(var selectedOs, { } info)]) + { + // we have a single viable candidate, use it and tail-recurse + if (GetPlannedPackageForBuild(info, dep.Kind, selectedOs) is not { } package) + { + // the single candidate couldn't be fully resolved; try the fallback + } + else + { + plan = plan.Add(package); + goto NextDep; + } + } + else + { + // we have multiple candidates, add them all recursively. If none we viable, try the fallback. + var anyViable = false; + foreach (var t in candidates) + { + (selectedOs, info) = t; + + if (GetPlannedPackageForBuild(info, dep.Kind, selectedOs) is not { } package) + { + continue; + } + + anyViable = true; + EnqueueForDepSet(plan.Add(package), depSet, depIndex + 1, jobIndex); + } + + if (anyViable) + { + // we've already added stuff for this tree, we're done at this level + return; + } + } + } + + // always fall out to this path + + // if we're looking at a monolithic build, always try for an Editor component, Windows mode. + var resultPackage = new UnityPackage(UnityPackageKind.Editor, EditorOS.Windows); + if (!dep.Matches(resultPackage)) goto NoMatch; // dep doesn't meet criteria, can't continue + + var editorDownloadPrefix = build.IsLegacyDownload ? "UnitySetup-" : "UnitySetup64-"; + plan = plan.Add(new PlannedPackage(resultPackage, editorDownloadPrefix + build.ShortVersion + ".exe")); + goto NextDep; + + NoMatch: + // if we found no match, but that's allowed, go to the next dep anyway + if (dep.AllowMissing) goto NextDep; + return; + + NextDep: + depIndex++; + } + while (true); + } + + public JobPlan? ComputeFullJobPlan(CancellationToken cancellationToken) + { + EnqueueCandidateMoves(ImmutableHashSet.Empty, 0); + + while (queue.TryDequeue(out var plan, out _)) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (plan.NextJob == jobDeps.Count) + { + // we've found an optimal layout + + // first, make sure that this is actually a valid layout (and figure out which dep set we're using for each) + var matchingDepOptions = new List(); + foreach (var deps in jobDeps) + { + (MinerDependencyOption depOpt, int skipped)? option = null; + foreach (var depSet in deps) + { + var skipped = 0; + foreach (var dep in depSet.NeededPackages) + { + var isSatisfied = false; + foreach (var planned in plan.Packages) + { + if (dep.Matches(planned.Package)) + { + isSatisfied = true; + break; + } + } + + if (!isSatisfied) + { + if (dep.AllowMissing) + { + skipped++; + continue; + } + else + { + goto NextSet; + } + } + } + + // make sure we pick the dep set with the least skipped deps + if (option is null || skipped < option.Value.skipped) + { + option = (depSet, skipped); + } + + NextSet:; + } + + if (option is not (var set, _)) + { + // this job doesn't have its deps satisfied; skip + goto NextIter; + } + else + { + // this job has deps satisfied + matchingDepOptions.Add(set); + } + } + + // this is a valid configuration, we're done! build the final plan and exit + return BuildCompletePlan(jobs, matchingDepOptions, plan.Packages); + } + else + { + // this is an incomplete job plan, add candidates for the current job in the plan + EnqueueCandidateMoves(plan.Packages, plan.NextJob); + } + + NextIter:; + } + + Log.Warning("[{Version}] Could not compute viable plan", build.Version); + return null; + } + } + + private static PlannedPackage? GetPlannedPackageForBuild(UnityBuildInfo? info, UnityPackageKind kind, EditorOS selectedOs) + { + if (info is null) return null; + + // Don't get support packages from the same OS as the target of the support package + if (kind is UnityPackageKind.WindowsMonoSupport && selectedOs is EditorOS.Windows) return null; + if (kind is UnityPackageKind.LinuxMonoSupport && selectedOs is EditorOS.Linux) return null; + if (kind is UnityPackageKind.MacMonoSupport && selectedOs is EditorOS.MacOS) return null; + + var module = kind switch + { + UnityPackageKind.Editor => info.Unity, + UnityPackageKind.Android => info.Android, + UnityPackageKind.WindowsMonoSupport => info.WindowsMono, + UnityPackageKind.LinuxMonoSupport => info.LinuxMono, + UnityPackageKind.MacMonoSupport => info.MacMono, + + _ => throw new ArgumentOutOfRangeException(nameof(kind)), + }; + + if (module is null) + { + return null; + } + + return new PlannedPackage(new(kind, selectedOs), module.Url); + } + + public readonly record struct PlannedPackage(UnityPackage Package, string Url); + + private sealed record InProgressPlan( + ImmutableHashSet Packages, + int NextJob + ) + { + public int PlanWeight => Packages.Sum(p => p.Package.HeuristicSize); + } + + private static JobPlan BuildCompletePlan(List jobs, List selectedDepSet, ImmutableHashSet planned) + { + Debug.Assert(jobs.Count == selectedDepSet.Count); + + var allPackages = planned.ToImmutableArray(); + var jobsBuilder = ImmutableArray.CreateBuilder<(ImmutableArray need, MinerJob job)>(jobs.Count); + + for (var i = 0; i < jobs.Count; i++) + { + var job = jobs[i]; + var depSet = selectedDepSet[i]; + + // build up this list in order + var needBuilder = ImmutableArray.CreateBuilder(depSet.NeededPackages.Length); + foreach (var dep in depSet.NeededPackages) + { + for (var j = 0; j < allPackages.Length; j++) + { + var pkg = allPackages[j]; + if (dep.Matches(pkg.Package)) + { + needBuilder.Add(j); + break; + } + } + } + + jobsBuilder.Add((needBuilder.DrainToImmutable(), job)); + } + + return new(allPackages, jobsBuilder.DrainToImmutable()); + } + + public sealed record JobPlan( + ImmutableArray Packages, + ImmutableArray<(ImmutableArray NeedsPackages, MinerJob Job)> Jobs + ); + + } +} diff --git a/UnityDataMiner/Jobs/AndroidMinerJob.cs b/UnityDataMiner/Jobs/AndroidMinerJob.cs new file mode 100644 index 0000000..64720db --- /dev/null +++ b/UnityDataMiner/Jobs/AndroidMinerJob.cs @@ -0,0 +1,96 @@ +using AssetRipper.Primitives; +using Serilog; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace UnityDataMiner.Jobs +{ + internal sealed class AndroidMinerJob : MinerJob + { + public override string Name => "Android binaries"; + + public override bool CanRunFor(UnityBuild build) + => !build.Version.IsMonolithic() + && (build.LinuxInfo is not null || build.MacOsInfo is not null); + + public override bool ShouldRunFor(UnityBuild build) + => !Directory.Exists(build.AndroidPath); + + public override ImmutableArray GetDependencies(UnityBuild build) + => [ + new([new(UnityPackageKind.Android, EditorOS.Linux)]), // Prefer the Linux package + new([new(UnityPackageKind.Android, EditorOS.MacOS)]), + ]; + + public override async Task ExtractFromAssets(UnityBuild build, string tmpDir, + ImmutableArray fulfilledDependency, ImmutableArray packagePaths, + CancellationToken cancellationToken) + { + Debug.Assert(packagePaths.Length is 1); + Debug.Assert(fulfilledDependency is [{ Kind: UnityPackageKind.Android }]); + + var packagePath = packagePaths[0]; + var packageOs = fulfilledDependency[0].OS; + + Log.Information("[{Version}] Extracting android binaries", build.Version); + using var stopwatch = new AutoStopwatch(); + var archiveDirectory = + Path.Combine(tmpDir, Path.GetFileNameWithoutExtension(packagePath)!); + + const string libs = "Variations/il2cpp/Release/Libs"; + const string symbols = "Variations/il2cpp/Release/Symbols"; + await build.ExtractAsync(packagePath, archiveDirectory, + [$"./{libs}/*/libunity.so", $"./{symbols}/*/libunity.sym.so"], + cancellationToken, flat: false); + + var androidDirectory = Path.Combine(tmpDir, "android"); + + Directory.CreateDirectory(androidDirectory); + + IEnumerable directories = Directory.GetDirectories(Path.Combine(archiveDirectory, libs)); + + var hasSymbols = build.Version > new UnityVersion(5, 3, 5, UnityVersionType.Final, 1); + + if (hasSymbols) + { + directories = + directories.Concat(Directory.GetDirectories(Path.Combine(archiveDirectory, symbols))); + } + + foreach (var directory in directories) + { + var directoryInfo = + Directory.CreateDirectory(Path.Combine(androidDirectory, Path.GetFileName(directory))); + foreach (var file in Directory.GetFiles(directory)) + { + File.Copy(file, Path.Combine(directoryInfo.FullName, Path.GetFileName(file)), true); + } + } + + if (hasSymbols) + { + foreach (var directory in Directory.GetDirectories(androidDirectory)) + { + await EuUnstrip.UnstripAsync(Path.Combine(directory, "libunity.so"), + Path.Combine(directory, "libunity.sym.so"), cancellationToken); + } + } + + Directory.CreateDirectory(build.AndroidPath); + + foreach (var directory in Directory.GetDirectories(androidDirectory)) + { + ZipFile.CreateFromDirectory(directory, + Path.Combine(build.AndroidPath, Path.GetFileName(directory) + ".zip")); + } + + Log.Information("[{Version}] Extracted android binaries in {Time}", build.Version, stopwatch.Elapsed); + } + } +} diff --git a/UnityDataMiner/Jobs/CorlibMinerJob.cs b/UnityDataMiner/Jobs/CorlibMinerJob.cs new file mode 100644 index 0000000..1939f91 --- /dev/null +++ b/UnityDataMiner/Jobs/CorlibMinerJob.cs @@ -0,0 +1,63 @@ +using AsmResolver.PE.Tls; +using Serilog; +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace UnityDataMiner.Jobs +{ + internal sealed class CorlibMinerJob : MinerJob + { + public override string Name => "corlibs"; + + public override bool CanRunFor(UnityBuild build) => true; + + public override bool ShouldRunFor(UnityBuild build) => !File.Exists(build.CorlibZipPath); + + public override ImmutableArray GetDependencies(UnityBuild build) + => [new([new(UnityPackageKind.Editor, EditorOS.Any)])]; + + public override async Task ExtractFromAssets(UnityBuild build, string tmpDir, ImmutableArray chosenPackages, ImmutableArray packagePaths, CancellationToken cancellationToken) + { + Debug.Assert(packagePaths.Length is 1); + Debug.Assert(chosenPackages is [{ Kind: UnityPackageKind.Editor }]); + + var packagePath = packagePaths[0]; + var packageOs = chosenPackages[0].OS; + + var corlibDirectory = Path.Combine(tmpDir, "corlib"); + + Log.Information("[{Version}] Extracting corlibs", build.Version); + using var stopwatch = new AutoStopwatch(); + + // TODO: Maybe grab both 2.0 and 4.5 DLLs for < 2018 monos + var corlibPath = (build.IsLegacyDownload, packageOs) switch + { + (true, _) => "Data/Mono/lib/mono/2.0", + (_, EditorOS.MacOS) => "./Unity/Unity.app/Contents/MonoBleedingEdge/lib/mono/4.5", + _ => "Editor/Data/MonoBleedingEdge/lib/mono/4.5", + }; + + await build.ExtractAsync(packagePath, corlibDirectory, [$"{corlibPath}/*.dll"], cancellationToken); + + if (!Directory.Exists(corlibDirectory) || + Directory.GetFiles(corlibDirectory, "*.dll").Length <= 0) + { + throw new Exception("Corlibs directory is empty"); + } + + File.Delete(build.CorlibZipPath); + ZipFile.CreateFromDirectory(corlibDirectory, build.CorlibZipPath); + + Log.Information("[{Version}] Extracted corlibs in {Time}", build.Version, stopwatch.Elapsed); + } + + } +} diff --git a/UnityDataMiner/Jobs/LibIl2CppSourceMinerJob.cs b/UnityDataMiner/Jobs/LibIl2CppSourceMinerJob.cs new file mode 100644 index 0000000..6d2ae98 --- /dev/null +++ b/UnityDataMiner/Jobs/LibIl2CppSourceMinerJob.cs @@ -0,0 +1,67 @@ +using AsmResolver.PE.Tls; +using Serilog; +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace UnityDataMiner.Jobs +{ + internal sealed class LibIl2CppSourceMinerJob : MinerJob + { + public override string Name => "IL2CPP sources"; + + public override bool CanRunFor(UnityBuild build) + => build.HasLibIl2Cpp; + + public override bool ShouldRunFor(UnityBuild build) + => !File.Exists(build.LibIl2CppSourceZipPath); + + public override ImmutableArray GetDependencies(UnityBuild build) + => [new([new(UnityPackageKind.Editor, EditorOS.Any)])]; + + public override async Task ExtractFromAssets(UnityBuild build, string tmpDir, + ImmutableArray chosenPackages, ImmutableArray packagePaths, + CancellationToken cancellationToken) + { + Debug.Assert(packagePaths.Length is 1); + Debug.Assert(chosenPackages is [{ Kind: UnityPackageKind.Editor }]); + + var packagePath = packagePaths[0]; + var packageOs = chosenPackages[0].OS; + + var libil2cppSourceDirectory = Path.Combine(tmpDir, "libil2cpp-source"); + + Log.Information("[{Version}] Extracting libil2cpp source code", build.Version); + using var stopwatch = new AutoStopwatch(); + + // TODO: find out if the path changes in different versions + var libil2cppSourcePath = packageOs switch + { + EditorOS.MacOS => "./Unity/Unity.app/Contents/il2cpp/libil2cpp", + _ => "Editor/Data/il2cpp/libil2cpp", + }; + + await build.ExtractAsync(packagePath, libil2cppSourceDirectory, + [$"{libil2cppSourcePath}/**"], cancellationToken, false); + + var zipDir = Path.Combine(libil2cppSourceDirectory, libil2cppSourcePath); + if (!Directory.Exists(zipDir) || Directory.GetFiles(zipDir).Length <= 0) + { + throw new Exception("LibIl2Cpp source code directory is empty"); + } + + File.Delete(build.LibIl2CppSourceZipPath); + ZipFile.CreateFromDirectory(zipDir, build.LibIl2CppSourceZipPath); + + Log.Information("[{Version}] Extracted libil2cpp source code in {Time}", build.Version, + stopwatch.Elapsed); + } + } +} diff --git a/UnityDataMiner/Jobs/MonoMinerJob.cs b/UnityDataMiner/Jobs/MonoMinerJob.cs new file mode 100644 index 0000000..43c6e87 --- /dev/null +++ b/UnityDataMiner/Jobs/MonoMinerJob.cs @@ -0,0 +1,272 @@ +using Serilog; +using System; +using System.Collections.Immutable; +using System.IO; +using System.IO.Compression; +using System.Threading; +using System.Threading.Tasks; + +namespace UnityDataMiner.Jobs +{ + internal sealed class MonoMinerJob : MinerJob + { + public override string Name => "Mono runtime"; + + public override bool CanRunFor(UnityBuild build) => true; + + public override bool ShouldRunFor(UnityBuild build) + => !Directory.Exists(build.MonoPath); + + public override bool RunIncrementally => true; + + public override ImmutableArray GetDependencies(UnityBuild build) + => [ + // first, prefer getting just a component for each + new MinerDependencyOption([ + new(UnityPackageKind.WindowsMonoSupport, EditorOS.Any), + new(UnityPackageKind.LinuxMonoSupport, EditorOS.Any, AllowMissing: true), + new(UnityPackageKind.MacMonoSupport, EditorOS.Any), + ]), + + // then, the editor package for the host + the support packages for the others + ..(ReadOnlySpan)( + build.IsMonolithic + ? [new([new(UnityPackageKind.Editor, EditorOS.Any)])] // in monolithic builds, we need the editor always + : build.HasModularPlayer + ? [ // in non-monolithic builds (that are modular with the OS support package in the main editor), we also want to allow editor + support packages + new([ + new(UnityPackageKind.Editor, EditorOS.Windows), + new(UnityPackageKind.LinuxMonoSupport, EditorOS.Any, AllowMissing: true), + new(UnityPackageKind.MacMonoSupport, EditorOS.Any), + ]), + new([ + new(UnityPackageKind.WindowsMonoSupport, EditorOS.Any), + new(UnityPackageKind.Editor, EditorOS.Linux), + new(UnityPackageKind.MacMonoSupport, EditorOS.Any), + ]), + new([ + new(UnityPackageKind.WindowsMonoSupport, EditorOS.Any), + new(UnityPackageKind.LinuxMonoSupport, EditorOS.Any, AllowMissing: true), + new(UnityPackageKind.Editor, EditorOS.MacOS), + ]), + ] + : [] // if this build is in the short window where the editor is modular, but doesn't include the platform's runtime in the editor bundle, + // we need all of the support bundles as well, which is the previous dep + ), + ]; + + public override async Task ExtractFromAssets(UnityBuild build, string tmpDir, + ImmutableArray chosenPackages, ImmutableArray packagePaths, + CancellationToken cancellationToken) + { + var monoBaseDir = Path.Combine(tmpDir, "mono"); + Directory.CreateDirectory(monoBaseDir); + + Log.Information("[{Version}] Packaging Mono binaries", build.Version); + using var stopwatch = new AutoStopwatch(); + + for (var i = 0; i < chosenPackages.Length; i++) + { + var (packageKind, packageOs, _) = chosenPackages[i]; + var packagePath = packagePaths[i]; + + if (build.IsMonolithic) + { + // TODO: + throw new NotImplementedException(); + } + else + { + var monoTargetOs = packageKind switch + { + UnityPackageKind.Editor => packageOs, + UnityPackageKind.WindowsMonoSupport => EditorOS.Windows, + UnityPackageKind.LinuxMonoSupport => EditorOS.Linux, + UnityPackageKind.MacMonoSupport => EditorOS.MacOS, + _ => throw new NotImplementedException(), + }; + + var targetOsName = monoTargetOs switch + { + EditorOS.Any => "any", + EditorOS.Windows => "win", + EditorOS.Linux => "linux", + EditorOS.MacOS => "mac", + _ => throw new NotImplementedException(), + }; + + Log.Information("[{Version}] Processing {TargetOS}", build.Version, monoTargetOs); + + var thisPkgDir = Path.Combine(monoBaseDir, monoTargetOs.ToString()); + if (Directory.Exists(thisPkgDir)) + { + Directory.Delete(thisPkgDir, true); + } + Directory.CreateDirectory(thisPkgDir); + + var playerdirSuffix = (build.Version.Major, monoTargetOs) switch + { + ( >= 2021, _) => "_player_nondevelopment_mono", + (_, EditorOS.Linux) => "_withgfx_nondevelopment_mono", + _ => "_nondevelopment_mono" + }; + + var prefix = "**/"; + var extractPath = monoTargetOs switch + { + // TODO: editor package paths + + EditorOS.Windows => $"Variations/*{playerdirSuffix}/**", + EditorOS.Linux => $"Variations/*{playerdirSuffix}/**", + EditorOS.MacOS => $"Variations/*{playerdirSuffix}/UnityPlayer.app/Contents/**", + + _ => throw new NotImplementedException() + }; + + if (packageKind is UnityPackageKind.Editor) + { + var editorPrefixPath = monoTargetOs switch + { + EditorOS.Windows => throw new NotImplementedException(), + EditorOS.Linux => "Editor/Data/PlaybackEngines/LinuxStandaloneSupport/", + EditorOS.MacOS => "Unity/Unity.app/Contents/PlaybackEngines/MacStandaloneSupport/", + + _ => throw new NotSupportedException(), + }; + extractPath = editorPrefixPath + extractPath; + } + + if (monoTargetOs is not EditorOS.MacOS || packageKind is not UnityPackageKind.Editor) + { + // Windows and Linux layouts are relatively convenient + await build.ExtractAsync(packagePath, thisPkgDir, [prefix + extractPath], cancellationToken, flat: false); + var variationsDir = GetVariationsDir(build, thisPkgDir); + if (variationsDir is null) continue; // can't do anything without a known variations dir + + foreach (var playerDir in Directory.EnumerateDirectories(variationsDir)) + { + var arch = Path.GetFileName(playerDir).Replace(playerdirSuffix, ""); + if (!arch.Contains(targetOsName)) + { + arch = targetOsName + "_" + arch; + } + + // search subdirs so we can find multiple Mono builds + foreach (var mono in Directory.EnumerateDirectories(playerDir, "Mono*", SearchOption.AllDirectories)) + { + var monoName = Path.GetFileName(mono); + var targetRuntimeDir = Path.Combine(mono, "runtime"); + + // If there's a dir that's not named "etc", rename that to "runtime" + var foundRuntimeDir = false; + foreach (var subdir in Directory.EnumerateDirectories(mono)) + { + if (Path.GetFileName(subdir) != "etc") + { + var hasMonoBin = false; + foreach (var file in Directory.EnumerateFiles(subdir, "*mono*")) + { + if (Path.GetFileName(file).Contains("mono")) + { + hasMonoBin = true; + break; + } + } + + if (!hasMonoBin) + { + // this dir doesn't actually have any Mono binaries, skip it + continue; + } + + Directory.Move(subdir, targetRuntimeDir); + foundRuntimeDir = true; + break; + } + } + + if (!foundRuntimeDir && monoTargetOs is EditorOS.MacOS) + { + // on MacOS, the non-mac distribution is even more screwey than usual. The binaries are in /UnityPlayer.app/Contents/Frameworks//MonoEmbedRuntime/osx/ + var binariesDir = Path.Combine(playerDir, "UnityPlayer.app/Contents/Frameworks", monoName, "MonoEmbedRuntime/osx/"); + if (!Directory.Exists(binariesDir)) + { + Log.Warning("[{Version}] Could not find MacOS binaries ({TestDir})", build.Version, binariesDir); + continue; // no use trying to do anything useful here + } + Directory.Move(binariesDir, targetRuntimeDir); + } + + // then we can create the zip file + ZipFile.CreateFromDirectory(mono, Path.Combine(monoBaseDir, $"{arch}_{monoName}.zip")); + } + } + } + else + { + // MacOS has a more annoying layout. The config is entirely separate from the runtime. + await build.ExtractAsync(packagePath, thisPkgDir, [prefix + extractPath, "./Mono*/**"], cancellationToken, flat: false); + var variationsDir = GetVariationsDir(build, thisPkgDir); + if (variationsDir is null) continue; // can't do anything without a known variations dir + + // our outer loop is for the config + foreach (var monoConfigDir in Directory.EnumerateDirectories(thisPkgDir, "Mono*")) + { + var monoName = Path.GetFileName(monoConfigDir); + var runtimeDir = Path.Combine(monoConfigDir, "runtime"); + + foreach (var playerDir in Directory.EnumerateDirectories(variationsDir)) + { + var arch = Path.GetFileName(playerDir).Replace(playerdirSuffix, ""); + if (!arch.Contains(targetOsName)) + { + arch = targetOsName + "_" + arch; + } + + if (Directory.Exists(runtimeDir)) + { + Directory.Delete(runtimeDir, true); + } + + Directory.Move(Path.Combine(playerDir, "UnityPlayer.app", "Contents", "Frameworks"), runtimeDir); + ZipFile.CreateFromDirectory(monoConfigDir, Path.Combine(monoBaseDir, $"{arch}_{monoName}.zip")); + } + } + } + } + } + + Directory.CreateDirectory(build.MonoPath); + foreach (var zip in Directory.EnumerateFiles(monoBaseDir, "*.zip")) + { + File.Move(zip, Path.Combine(build.MonoPath, Path.GetFileName(zip))); + } + + Log.Information("[{Version}] Mono binaries packaged in {Time}", build.Version, stopwatch.Elapsed); + + static string? GetVariationsDir(UnityBuild build, string thisPkgDir) + { + string? variationsDir = null; + foreach (var candidate in Directory.EnumerateDirectories(thisPkgDir, "Variations", SearchOption.AllDirectories)) + { + if (Path.GetFileName(candidate) is "Variations") + { + variationsDir = candidate; + break; + } + } + + if (variationsDir is null) + { + Log.Error("[{Version}] Could not find variations dir in selectively extracted package", build.Version); + } + else + { + Log.Debug("[{Version}] Found variations dir {Dir}", build.Version, variationsDir); + } + + return variationsDir; + } + } + } +} diff --git a/UnityDataMiner/Jobs/UnityLibsMinerJob.cs b/UnityDataMiner/Jobs/UnityLibsMinerJob.cs new file mode 100644 index 0000000..22b71c4 --- /dev/null +++ b/UnityDataMiner/Jobs/UnityLibsMinerJob.cs @@ -0,0 +1,139 @@ +using AssetRipper.Primitives; +using BepInEx.AssemblyPublicizer; +using NuGet.Frameworks; +using NuGet.Packaging; +using Serilog; +using System; +using System.Collections.Immutable; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace UnityDataMiner.Jobs +{ + internal sealed class UnityLibsMinerJob : MinerJob + { + public override string Name => "Unity libs + NuGet packages"; + + public override bool CanRunFor(UnityBuild build) => true; + + public override bool ShouldRunFor(UnityBuild build) + => !File.Exists(build.UnityLibsZipFilePath) || !File.Exists(build.NuGetPackagePath); + + public override ImmutableArray GetDependencies(UnityBuild build) + => [ + new([new(UnityPackageKind.Editor, EditorOS.Any)]) + ]; + + public override async Task ExtractFromAssets(UnityBuild build, string tmpDir, + ImmutableArray chosenPackages, ImmutableArray packagePaths, + CancellationToken cancellationToken) + { + Debug.Assert(chosenPackages is [{ Kind: UnityPackageKind.Editor }]); + Debug.Assert(packagePaths.Length is 1); + + var packageOs = chosenPackages[0].OS; + var packagePath = packagePaths[0]; + + var managedDirectory = Path.Combine(tmpDir, "managed"); + Directory.CreateDirectory(managedDirectory); + + Log.Information("[{Version}] Extracting Unity libraries", build.Version); + using var stopwatch = new AutoStopwatch(); + + // select the correct path in the archive + string managedPath; + if (build.IsMonolithic) + { + Debug.Assert(packageOs is EditorOS.Windows); + if (build.IsLegacyDownload) + { + managedPath = build.Version is { Major: 4, Minor: >= 5 } + ? "Data/PlaybackEngines/windowsstandalonesupport/Variations/win64_nondevelopment/Data/Managed" + : "Data/PlaybackEngines/windows64standaloneplayer/Managed"; + } + else + { + managedPath = "Editor/Data/PlaybackEngines/windowsstandalonesupport/Variations/win64_nondevelopment_mono/Data/Managed"; + } + } + else + { + managedPath = packageOs switch + { + EditorOS.Windows + => "./Variations/win64_nondevelopment_mono/Data/Managed", + EditorOS.MacOS + => "./Unity/Unity.app/Contents/PlaybackEngines/MacStandaloneSupport/Variations/macosx64_nondevelopment_mono/Data/Managed", + EditorOS.Linux when build.Version >= new UnityVersion(2021, 2) + => "Editor/Data/PlaybackEngines/LinuxStandaloneSupport/Variations/linux64_player_nondevelopment_mono/Data/Managed", + EditorOS.Linux + => "Editor/Data/PlaybackEngines/LinuxStandaloneSupport/Variations/linux64_withgfx_nondevelopment_mono/Data/Managed", + + _ => throw new NotSupportedException(), + }; + } + + // extract the binaries from the archive + await build.ExtractAsync(packagePath, managedDirectory, [managedPath + "/*.dll"], cancellationToken, flat: true); + + Log.Information("[{Version}] Packaging Unity libraries into zip", build.Version); + var tmpZip = Path.Combine(tmpDir, "libs.zip"); + ZipFile.CreateFromDirectory(managedDirectory, tmpZip); + + if (File.Exists(build.UnityLibsZipFilePath)) + { + File.Delete(build.UnityLibsZipFilePath); + } + File.Move(tmpZip, build.UnityLibsZipFilePath); + + + Log.Information("[{Version}] Creating NuGet package for Unity libraries", build.Version); + + // first, publicize and strip the assemblies + foreach (var file in Directory.EnumerateFiles(managedDirectory, "*.dll")) + { + AssemblyPublicizer.Publicize(file, file, new() { Strip = true }); + } + + // now create the package + var frameworkTargets = new[] { "net35", "net45", "netstandard2.0" }; + + var meta = new ManifestMetadata + { + Id = "UnityEngine.Modules", + Authors = ["Unity"], + Version = build.NuGetVersion, + Description = "UnityEngine modules", + DevelopmentDependency = true, + DependencyGroups = frameworkTargets.Select(d => + new PackageDependencyGroup(NuGetFramework.Parse(d), [])) + }; + + var builder = new PackageBuilder(true); + builder.PopulateFiles(managedDirectory, frameworkTargets.Select(d => new ManifestFile + { + Source = "*.dll", + Target = $"lib/{d}" + })); + builder.Populate(meta); + + var tmpPkg = Path.Combine(tmpDir, "libs.nupkg"); + using (var fs = File.Create(tmpPkg)) + { + builder.Save(fs); + } + + if (File.Exists(build.NuGetPackagePath)) + { + File.Delete(build.NuGetPackagePath); + } + File.Move(tmpPkg, build.NuGetPackagePath); + + Log.Information("[{Version}] Extracted and packaged Unity libraries in {Time}", build.Version, stopwatch.Elapsed); + } + } +} diff --git a/UnityDataMiner/MineCommand.cs b/UnityDataMiner/MineCommand.cs index 7843952..db7d0d8 100644 --- a/UnityDataMiner/MineCommand.cs +++ b/UnityDataMiner/MineCommand.cs @@ -1,227 +1,239 @@ -using System; -using System.Collections.Generic; -using System.CommandLine; -using System.CommandLine.Invocation; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using AssetRipper.Primitives; -using LibGit2Sharp; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace UnityDataMiner; - -public partial class MineCommand : RootCommand -{ - [GeneratedRegex(@"unityhub:\/\/(?[\d.\w]+)\/(?[0-9a-f]+)")] - private static partial Regex UnityHubLinkRegex(); - - public MineCommand() - { - Add(new Argument("version") - { - Arity = ArgumentArity.ZeroOrOne, - }); - Add(new Option("--repository", () => new DirectoryInfo(Directory.GetCurrentDirectory()))); - } - - public new class Handler : ICommandHandler - { - private readonly ILogger _logger; - private readonly MinerOptions _minerOptions; - private readonly IHttpClientFactory _clientFactory; - - public string? Version { get; init; } - public DirectoryInfo Repository { get; init; } - - public Handler(ILogger logger, IOptions minerOptions, IHttpClientFactory clientFactory) - { - _logger = logger; - _minerOptions = minerOptions.Value; - _clientFactory = clientFactory; - } - - public async Task InvokeAsync(InvocationContext context) - { - await SevenZip.EnsureInstalled(); - await EuUnstrip.EnsureInstalled(); - - var token = context.GetCancellationToken(); - - Directory.CreateDirectory(Path.Combine(Repository.FullName, "libraries")); - Directory.CreateDirectory(Path.Combine(Repository.FullName, "packages")); - Directory.CreateDirectory(Path.Combine(Repository.FullName, "corlibs")); - Directory.CreateDirectory(Path.Combine(Repository.FullName, "libil2cpp-source")); - Directory.CreateDirectory(Path.Combine(Repository.FullName, "android")); - Directory.CreateDirectory(Path.Combine(Repository.FullName, "versions")); - - var unityVersions = await FetchUnityVersionsAsync(Repository.FullName); - - await Parallel.ForEachAsync(unityVersions.Where(x => x.Version.Major >= 5 && x.NeedsInfoFetch), token, async (unityVersion, cancellationToken) => - { - try - { - await unityVersion.FetchInfoAsync(cancellationToken); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - } - catch (Exception e) - { - _logger.LogError(e, "Failed to fetch info for {Version}", unityVersion.Version); - } - }); - - var toRun = string.IsNullOrEmpty(Version) - ? unityVersions.Where(unityVersion => unityVersion.IsRunNeeded).ToArray() - : new[] { unityVersions.Single(x => x.ShortVersion == Version) }; - - _logger.LogInformation("Mining {Count} unity versions", toRun.Length); - - await Parallel.ForEachAsync(toRun, token, async (unityVersion, cancellationToken) => - { - try - { - await unityVersion.MineAsync(cancellationToken); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - } - catch (Exception e) - { - _logger.LogError(e, "Failed to download {Version}", unityVersion.Version); - } - }); - - if (!string.IsNullOrEmpty(_minerOptions.NuGetSource) && !string.IsNullOrEmpty(_minerOptions.NuGetSourceKey)) - await Task.WhenAll(toRun.Select(unityVersion => Task.Run(async () => - { - try - { - await unityVersion.UploadNuGetPackageAsync(_minerOptions.NuGetSource, _minerOptions.NuGetSourceKey); - } - catch (Exception e) - { - _logger.LogError(e, "Failed to download {Version}", unityVersion.Version); - } - }, token))); - else - _logger.LogInformation("Skipping pushing NuGet packages (no package config specified)"); - - UpdateGitRepository(Repository.FullName, unityVersions); - - return 0; - } - - private async Task> FetchUnityVersionsAsync(string repositoryPath) - { - var unityVersions = new Dictionary(); - await FetchUnityVersionsAsync(repositoryPath, unityVersions, "releases/editor/archive"); - await FetchUnityVersionsAsync(repositoryPath, unityVersions, "releases/editor/releases.xml"); - await FetchUnityVersionsAsync(repositoryPath, unityVersions, "releases/editor/beta/latest.xml"); - await FetchUnityVersionsAsync(repositoryPath, unityVersions, "releases/editor/lts-releases.xml"); - await FillMissingUnityVersionsAsync(repositoryPath, unityVersions); - return unityVersions.Values.ToList(); - } - - private async Task FetchUnityVersionsAsync(string repositoryPath, Dictionary unityVersions, string path) - { - var httpClient = _clientFactory.CreateClient("unity"); - var text = await httpClient.GetStringAsync(path); - - var count = 0; - - foreach (Match match in UnityHubLinkRegex().Matches(text)) - { - var unityVersion = UnityVersion.Parse(match.Groups["version"].Value); - - if (!unityVersions.ContainsKey(unityVersion)) - { - unityVersions.Add(unityVersion, new UnityBuild(repositoryPath, match.Groups["id"].Value, unityVersion)); - count++; - } - } - - _logger.LogInformation("Found {Count} new unity versions in {Path}", count, path); - } - - private async Task FillMissingUnityVersionsAsync(string repositoryPath, Dictionary unityVersions) - { - var versionsCachePath = Path.Combine(repositoryPath, "versions"); - if (!Directory.Exists(versionsCachePath)) return; - - var count = 0; - - await Parallel.ForEachAsync(Directory.GetDirectories(versionsCachePath), async (versionCachePath, token) => - { - var unityBuildInfo = UnityBuildInfo.Parse(await File.ReadAllTextAsync(Path.Combine(versionCachePath, "win.ini"), token)); - if (unityBuildInfo.Unity.Title == "Unity 5") return; - - var unityVersion = UnityVersion.Parse(unityBuildInfo.Unity.Title.Replace("Unity ", "")); - - if (!unityVersions.ContainsKey(unityVersion)) - { - lock (unityVersions) - { - unityVersions.Add(unityVersion, new UnityBuild(repositoryPath, Path.GetFileName(versionCachePath), unityVersion)); - count++; - } - } - }); - - _logger.LogInformation("Filled {Count} unity versions", count); - } - - private void UpdateGitRepository(string repositoryPath, List unityVersions) - { - _logger.LogInformation("Initialising the git repository"); - - var repository = new Repository(LibGit2Sharp.Repository.Init(repositoryPath)); - - _logger.LogInformation("Staging"); - - Commands.Stage(repository, Path.Combine(repositoryPath, "*")); - - _logger.LogInformation("Comparing"); - - var currentCommit = repository.Head.Tip; - var changes = repository.Diff.Compare(currentCommit?.Tree, DiffTargets.WorkingDirectory); - - if (currentCommit == null || changes.Any()) - { - var author = new Signature("UnityDataMiner", "UnityDataMiner@bepinex.dev", DateTimeOffset.Now); - - _logger.LogInformation("Committing"); - - var commit = repository.Commit("Automatically mined", author, author); - - _logger.LogInformation("Committed {Sha}", commit.Sha); - - // Shell out to git for SSH support - var pushProcess = Process.Start(new ProcessStartInfo("git", "push") - { - WorkingDirectory = repositoryPath, - RedirectStandardError = true, - }) ?? throw new Exception("Failed to start git push"); - pushProcess.WaitForExit(); - - if (pushProcess.ExitCode == 0) - { - _logger.LogInformation("Pushed!"); - } - else - { - _logger.LogError("Failed to push\n{Error}", pushProcess.StandardError.ReadToEnd()); - } - } - else - { - _logger.LogInformation("No git changes found"); - } - } - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using AssetRipper.Primitives; +using LibGit2Sharp; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using UnityDataMiner.Jobs; + +namespace UnityDataMiner; + +public partial class MineCommand : RootCommand +{ + [GeneratedRegex(@"unityhub:\/\/(?[\d.\w]+)\/(?[0-9a-f]+)")] + private static partial Regex UnityHubLinkRegex(); + + public MineCommand() + { + Add(new Argument("versions") + { + Arity = ArgumentArity.ZeroOrMore, + }); + Add(new Option("--repository", () => new DirectoryInfo(Directory.GetCurrentDirectory()))); + } + + public new class Handler : ICommandHandler + { + private readonly ILogger _logger; + private readonly MinerOptions _minerOptions; + private readonly IHttpClientFactory _clientFactory; + + public string[]? Versions { get; init; } + public DirectoryInfo Repository { get; init; } + + public Handler(ILogger logger, IOptions minerOptions, IHttpClientFactory clientFactory) + { + _logger = logger; + _minerOptions = minerOptions.Value; + _clientFactory = clientFactory; + } + + public async Task InvokeAsync(InvocationContext context) + { + await SevenZip.EnsureInstalled(); + await EuUnstrip.EnsureInstalled(); + + var token = context.GetCancellationToken(); + + Directory.CreateDirectory(Path.Combine(Repository.FullName, "libraries")); + Directory.CreateDirectory(Path.Combine(Repository.FullName, "packages")); + Directory.CreateDirectory(Path.Combine(Repository.FullName, "corlibs")); + Directory.CreateDirectory(Path.Combine(Repository.FullName, "mono")); + Directory.CreateDirectory(Path.Combine(Repository.FullName, "libil2cpp-source")); + Directory.CreateDirectory(Path.Combine(Repository.FullName, "android")); + Directory.CreateDirectory(Path.Combine(Repository.FullName, "versions")); + + var unityVersions = await FetchUnityVersionsAsync(Repository.FullName); + + await Parallel.ForEachAsync(unityVersions.Where(x => x.Version.Major >= 5 && x.NeedsInfoFetch), token, async (unityVersion, cancellationToken) => + { + try + { + await unityVersion.FetchInfoAsync(cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (Exception e) + { + _logger.LogError(e, "Failed to fetch info for {Version}", unityVersion.Version); + } + }); + + var toRun = Versions is null or [] + ? unityVersions.ToArray() + : unityVersions.Where(v => Versions.Contains(v.ShortVersion)).ToArray(); + + var jobs = ImmutableArray.Create([ + new AndroidMinerJob(), + new CorlibMinerJob(), + new LibIl2CppSourceMinerJob(), + new MonoMinerJob(), + new UnityLibsMinerJob(), + ]); + + _logger.LogInformation("Mining {Count} unity versions", toRun.Length); + + await Parallel.ForEachAsync(toRun, token, async (unityVersion, cancellationToken) => + { + try + { + //await unityVersion.MineAsync(cancellationToken); + await unityVersion.ExecuteJobsAsync(jobs, cancellationToken); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + } + catch (Exception e) + { + _logger.LogError(e, "Failed to download {Version}", unityVersion.Version); + } + }); + + if (!string.IsNullOrEmpty(_minerOptions.NuGetSource) && !string.IsNullOrEmpty(_minerOptions.NuGetSourceKey)) + await Task.WhenAll(toRun.Select(unityVersion => Task.Run(async () => + { + try + { + await unityVersion.UploadNuGetPackageAsync(_minerOptions.NuGetSource, _minerOptions.NuGetSourceKey); + } + catch (Exception e) + { + _logger.LogError(e, "Failed to download {Version}", unityVersion.Version); + } + }, token))); + else + _logger.LogInformation("Skipping pushing NuGet packages (no package config specified)"); + + UpdateGitRepository(Repository.FullName, unityVersions); + + return 0; + } + + private async Task> FetchUnityVersionsAsync(string repositoryPath) + { + var unityVersions = new Dictionary(); + await FetchUnityVersionsAsync(repositoryPath, unityVersions, "releases/editor/archive"); + await FetchUnityVersionsAsync(repositoryPath, unityVersions, "releases/editor/releases.xml"); + await FetchUnityVersionsAsync(repositoryPath, unityVersions, "releases/editor/beta/latest.xml"); + await FetchUnityVersionsAsync(repositoryPath, unityVersions, "releases/editor/lts-releases.xml"); + await FillMissingUnityVersionsAsync(repositoryPath, unityVersions); + return unityVersions.Values.ToList(); + } + + private async Task FetchUnityVersionsAsync(string repositoryPath, Dictionary unityVersions, string path) + { + var httpClient = _clientFactory.CreateClient("unity"); + var text = await httpClient.GetStringAsync(path); + + var count = 0; + + foreach (Match match in UnityHubLinkRegex().Matches(text)) + { + var unityVersion = UnityVersion.Parse(match.Groups["version"].Value); + + if (!unityVersions.ContainsKey(unityVersion)) + { + unityVersions.Add(unityVersion, new UnityBuild(repositoryPath, match.Groups["id"].Value, unityVersion)); + count++; + } + } + + _logger.LogInformation("Found {Count} new unity versions in {Path}", count, path); + } + + private async Task FillMissingUnityVersionsAsync(string repositoryPath, Dictionary unityVersions) + { + var versionsCachePath = Path.Combine(repositoryPath, "versions"); + if (!Directory.Exists(versionsCachePath)) return; + + var count = 0; + + await Parallel.ForEachAsync(Directory.GetDirectories(versionsCachePath), async (versionCachePath, token) => + { + var unityBuildInfo = UnityBuildInfo.Parse(await File.ReadAllTextAsync(Path.Combine(versionCachePath, "win.ini"), token)); + if (unityBuildInfo.Unity.Title == "Unity 5") return; + + var unityVersion = UnityVersion.Parse(unityBuildInfo.Unity.Title.Replace("Unity ", "")); + + if (!unityVersions.ContainsKey(unityVersion)) + { + lock (unityVersions) + { + unityVersions.Add(unityVersion, new UnityBuild(repositoryPath, Path.GetFileName(versionCachePath), unityVersion)); + count++; + } + } + }); + + _logger.LogInformation("Filled {Count} unity versions", count); + } + + private void UpdateGitRepository(string repositoryPath, List unityVersions) + { + _logger.LogInformation("Initialising the git repository"); + + var repository = new Repository(LibGit2Sharp.Repository.Init(repositoryPath)); + + _logger.LogInformation("Staging"); + + Commands.Stage(repository, Path.Combine(repositoryPath, "*")); + + _logger.LogInformation("Comparing"); + + var currentCommit = repository.Head.Tip; + var changes = repository.Diff.Compare(currentCommit?.Tree, DiffTargets.WorkingDirectory); + + if (currentCommit == null || changes.Any()) + { + var author = new Signature("UnityDataMiner", "UnityDataMiner@bepinex.dev", DateTimeOffset.Now); + + _logger.LogInformation("Committing"); + + var commit = repository.Commit("Automatically mined", author, author); + + _logger.LogInformation("Committed {Sha}", commit.Sha); + + // Shell out to git for SSH support + var pushProcess = Process.Start(new ProcessStartInfo("git", "push") + { + WorkingDirectory = repositoryPath, + RedirectStandardError = true, + }) ?? throw new Exception("Failed to start git push"); + pushProcess.WaitForExit(); + + if (pushProcess.ExitCode == 0) + { + _logger.LogInformation("Pushed!"); + } + else + { + _logger.LogError("Failed to push\n{Error}", pushProcess.StandardError.ReadToEnd()); + } + } + else + { + _logger.LogInformation("No git changes found"); + } + } + } +} diff --git a/UnityDataMiner/MinerJob.cs b/UnityDataMiner/MinerJob.cs new file mode 100644 index 0000000..e747a71 --- /dev/null +++ b/UnityDataMiner/MinerJob.cs @@ -0,0 +1,84 @@ +using System.Collections.Immutable; +using System.Threading; +using System.Threading.Tasks; + +namespace UnityDataMiner +{ + public abstract class MinerJob + { + public abstract string Name { get; } + + // answers whether this job can act for this build + public abstract bool CanRunFor(UnityBuild build); + // answers whether this job should run, i.e. whether it is out-of-date + public abstract bool ShouldRunFor(UnityBuild build); + + // returns a set of options, i.e. OR'd, in order of preference. + public abstract ImmutableArray GetDependencies(UnityBuild build); + + // if true, each dependency will be passed in as it is downloaded + // note: the order guarantee that is normally provided in ExtractFromAssets does not apply in this case + public virtual bool RunIncrementally => false; + + // chosenPackages is in the same order as the dependency, but with the Any OS replaced with the actually selected OS + public abstract Task ExtractFromAssets(UnityBuild build, string tmpDir, + ImmutableArray chosenPackages, ImmutableArray packagePaths, + CancellationToken cancellationToken); + } + + // TODO: we may be able to replace the heuristic size with actual size, but that doesn't really + // enable us to encode preference + + // AllowMissing means that if this package isn't available for the given Unity version, + // the dependency group this is a part of is still otherwise valid. + public readonly record struct UnityPackage(UnityPackageKind Kind, EditorOS OS, bool AllowMissing = false) + { + public int HeuristicSize => Kind.GetRelativePackageSize() + (OS switch + { + EditorOS.Any => 0, + EditorOS.Windows => 2, + EditorOS.Linux => 0, + EditorOS.MacOS => 1, + _ => throw new System.NotImplementedException(), + }); + + public bool Matches(UnityPackage package) + => Kind == package.Kind + && (OS == package.OS || OS is EditorOS.Any || package.OS is EditorOS.Any); + } + + // requires ALL packages to do its job + public readonly record struct MinerDependencyOption(ImmutableArray NeededPackages); + + public enum UnityPackageKind + { + Editor, + Android, + WindowsMonoSupport, + LinuxMonoSupport, + MacMonoSupport + } + + public static class UnityPackageKindExtensions + { + // We use this during planning as a heuristic to minimize the overall download size. + public static int GetRelativePackageSize(this UnityPackageKind kind) + => kind switch + { + UnityPackageKind.Editor => 10, + UnityPackageKind.Android => 1, + UnityPackageKind.WindowsMonoSupport => 1, + UnityPackageKind.LinuxMonoSupport => 1, + UnityPackageKind.MacMonoSupport => 1, + _ => 0, + }; + } + + public enum EditorOS + { + Any, + Windows, + Linux, + MacOS, + } +} diff --git a/UnityDataMiner/Program.cs b/UnityDataMiner/Program.cs index ced6c7f..80f9d63 100644 --- a/UnityDataMiner/Program.cs +++ b/UnityDataMiner/Program.cs @@ -1,64 +1,65 @@ -using System; -using System.CommandLine.Builder; -using System.CommandLine.Hosting; -using System.CommandLine.Parsing; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Polly; -using Serilog; -using Tommy.Extensions.Configuration; - -namespace UnityDataMiner -{ - public class MinerOptions - { - public string? NuGetSource { get; set; } - public string? NuGetSourceKey { get; set; } - } - - internal static class Program - { - private static readonly MinerOptions Options = new(); - - public static async Task Main(string[] args) - { - Log.Logger = new LoggerConfiguration() - .WriteTo.Console() - .CreateBootstrapLogger(); - - try - { - return await new CommandLineBuilder(new MineCommand()) - .UseHost(builder => - { - builder - .UseConsoleLifetime(opts => opts.SuppressStatusMessages = true) - .ConfigureAppConfiguration(configuration => configuration.AddTomlFile("config.toml", true)) - .ConfigureServices(services => - { - services.AddOptions().BindConfiguration("MinerOptions"); - - services.AddHttpClient("unity", client => - { - client.BaseAddress = new Uri("https://unity.com/"); - }).AddTransientHttpErrorPolicy(policy => policy.WaitAndRetryAsync(5, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)))); - }) - .UseCommandHandler() - .UseSerilog((context, services, loggerConfiguration) => loggerConfiguration - .ReadFrom.Configuration(context.Configuration) - .Enrich.FromLogContext() - .WriteTo.Console()); - }) - .UseDefaults() - .UseExceptionHandler((ex, _) => Log.Fatal(ex, "Exception, cannot continue!"), -1) - .Build() - .InvokeAsync(args); - } - finally - { - await Log.CloseAndFlushAsync(); - } - } - } -} +using System; +using System.CommandLine.Builder; +using System.CommandLine.Hosting; +using System.CommandLine.Parsing; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Polly; +using Serilog; +using Tommy.Extensions.Configuration; + +namespace UnityDataMiner +{ + public class MinerOptions + { + public string? NuGetSource { get; set; } + public string? NuGetSourceKey { get; set; } + } + + internal static class Program + { + private static readonly MinerOptions Options = new(); + + public static async Task Main(string[] args) + { + Log.Logger = new LoggerConfiguration() + .WriteTo.Console() + .CreateBootstrapLogger(); + + try + { + return await new CommandLineBuilder(new MineCommand()) + .UseHost(builder => + { + builder + .UseConsoleLifetime(opts => opts.SuppressStatusMessages = true) + .ConfigureAppConfiguration(configuration => configuration.AddTomlFile("config.toml", true)) + .ConfigureServices(services => + { + services.AddOptions().BindConfiguration("MinerOptions"); + + services.AddHttpClient("unity", client => + { + client.BaseAddress = new Uri("https://unity.com/"); + }).AddTransientHttpErrorPolicy(policy => policy.WaitAndRetryAsync(5, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)))); + }) + .UseCommandHandler() + .UseSerilog((context, services, loggerConfiguration) => loggerConfiguration + .MinimumLevel.Debug() + .ReadFrom.Configuration(context.Configuration) + .Enrich.FromLogContext() + .WriteTo.Console()); + }) + .UseDefaults() + .UseExceptionHandler((ex, _) => Log.Fatal(ex, "Exception, cannot continue!"), -1) + .Build() + .InvokeAsync(args); + } + finally + { + await Log.CloseAndFlushAsync(); + } + } + } +} diff --git a/UnityDataMiner/SevenZip.cs b/UnityDataMiner/SevenZip.cs index 49034b8..b48de7a 100644 --- a/UnityDataMiner/SevenZip.cs +++ b/UnityDataMiner/SevenZip.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; using NuGet.Packaging; +using Serilog; namespace UnityDataMiner; @@ -43,7 +44,7 @@ public static async Task ExtractAsync(string archivePath, string outputDirectory processStartInfo.ArgumentList.AddRange(fileFilter); } - var process = Process.Start(processStartInfo) ?? throw new SevenZipException("Couldn't start 7z process"); + using var process = Process.Start(processStartInfo) ?? throw new SevenZipException("Couldn't start 7z process"); await process.WaitForExitAsync(cancellationToken); @@ -51,6 +52,16 @@ public static async Task ExtractAsync(string archivePath, string outputDirectory { throw new SevenZipException("7z returned " + process.ExitCode + "\n" + (await process.StandardError.ReadToEndAsync()).Trim()); } + else + { +#if DEBUG && false + string? line; + while ((line = process.StandardOutput.ReadLine()) is not null) + { + Log.Debug("7z stdout: {OutputLine}", line); + } +#endif + } } } diff --git a/UnityDataMiner/UnityBuild.cs b/UnityDataMiner/UnityBuild.cs new file mode 100644 index 0000000..0da3179 --- /dev/null +++ b/UnityDataMiner/UnityBuild.cs @@ -0,0 +1,959 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Sockets; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using AssetRipper.Primitives; +using BepInEx.AssemblyPublicizer; +using NuGet.Common; +using NuGet.Frameworks; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Versioning; +using Serilog; + +namespace UnityDataMiner +{ + public class UnityBuild + { + public string? Id { get; } + + public UnityVersion Version { get; } + + public string ShortVersion { get; } + + public NuGetVersion NuGetVersion { get; } + + public string UnityLibsZipFilePath { get; } + + public string MonoPath { get; } + + public string AndroidPath { get; } + + public string NuGetPackagePath { get; } + + public string InfoCacheDir { get; } + + public string CorlibZipPath { get; } + + public string LibIl2CppSourceZipPath { get; } + + public bool IsRunNeeded => !File.Exists(UnityLibsZipFilePath) || !File.Exists(NuGetPackagePath) || + !File.Exists(CorlibZipPath) || !Directory.Exists(MonoPath) || + (HasLibIl2Cpp && !File.Exists(LibIl2CppSourceZipPath)) || + (!Version.IsMonolithic() && !Directory.Exists(AndroidPath)); + + public string BaseDownloadUrl => Version.GetDownloadUrl() + (Id == null ? string.Empty : $"{Id}/"); + + public UnityBuildInfo? WindowsInfo { get; private set; } + public UnityBuildInfo? LinuxInfo { get; private set; } + public UnityBuildInfo? MacOsInfo { get; private set; } + + public UnityBuild(string repositoryPath, string? id, UnityVersion version) + { + Id = id; + Version = version; + + if (Version.Major >= 5 && Id == null) + { + throw new Exception("Hash cannot be null after 5.x"); + } + + ShortVersion = Version.ToStringWithoutType(); + NuGetVersion = Version.Type == UnityVersionType.Final + ? new NuGetVersion(Version.Major, Version.Minor, Version.Build) + : new NuGetVersion(Version.Major, Version.Minor, Version.Build, Version.Type switch + { + UnityVersionType.Alpha => "alpha", + UnityVersionType.Beta => "beta", + UnityVersionType.China => "china", + UnityVersionType.Final => "final", + UnityVersionType.Patch => "patch", + UnityVersionType.Experimental => "experimental", + _ => throw new ArgumentOutOfRangeException(nameof(Version.Type), Version.Type, + "Invalid Version.Type for " + Version), + } + "." + Version.TypeNumber); + + var versionName = Version.Type == UnityVersionType.Final ? ShortVersion : Version.ToString(); + var zipName = $"{versionName}.zip"; + UnityLibsZipFilePath = Path.Combine(repositoryPath, "libraries", zipName); + MonoPath = Path.Combine(repositoryPath, "mono", versionName); + AndroidPath = Path.Combine(repositoryPath, "android", versionName); + CorlibZipPath = Path.Combine(repositoryPath, "corlibs", zipName); + LibIl2CppSourceZipPath = Path.Combine(repositoryPath, "libil2cpp-source", zipName); + NuGetPackagePath = Path.Combine(repositoryPath, "packages", $"{NuGetVersion}.nupkg"); + InfoCacheDir = Path.Combine(repositoryPath, "versions", $"{id}"); + + WindowsInfo = ReadInfo("win"); + + if (Version >= _firstLinuxVersion) + { + LinuxInfo = ReadInfo("linux"); + } + + MacOsInfo = ReadInfo("osx"); + } + + private static readonly HttpClient _httpClient = new(); + private static readonly SemaphoreSlim _downloadLock = new(2, 2); + + private static readonly UnityVersion _firstLinuxVersion = new(2018, 1, 5); + + // First modular version where own native player is included in the default installer + private static readonly UnityVersion _firstMergedModularVersion = new(5, 4); + private static readonly UnityVersion _firstLibIl2CppVersion = new(5, 0, 2); + + // TODO: Might need to define more DLLs? This should be enough for basic unhollowing. + private static readonly string[] _importantCorlibs = + { + "Microsoft.CSharp", + "Mono.Posix", + "Mono.Security", + "mscorlib", + "Facades/netstandard", + "System.Configuration", + "System.Core", + "System.Data", + "System", + "System.Net.Http", + "System.Numerics", + "System.Runtime.Serialization", + "System.Security", + "System.Xml", + "System.Xml.Linq", + }; + + public bool HasLinuxEditor => LinuxInfo is not null; + public bool HasModularPlayer => Version >= _firstMergedModularVersion; + public bool IsMonolithic => Version.IsMonolithic(); + public bool HasLibIl2Cpp => Version >= _firstLibIl2CppVersion; + public bool IsLegacyDownload => Id == null || Version.Major < 5; + + public bool NeedsInfoFetch { get; private set; } + + private UnityBuildInfo? ReadInfo(string variation) + { + if (!Directory.Exists(InfoCacheDir)) + { + NeedsInfoFetch = true; + return null; + } + + var path = Path.Combine(InfoCacheDir, $"{variation}.ini"); + try + { + var variationIni = File.ReadAllText(path); + var info = UnityBuildInfo.Parse(variationIni); + if (info.Unity.Version != null && !info.Unity.Version.Equals(Version)) + { + throw new Exception(); + } + + return info; + } + catch (Exception) + { + NeedsInfoFetch = true; + return null; + } + } + + public async Task FetchInfoAsync(CancellationToken cancellationToken) + { + if (!NeedsInfoFetch) + { + return; + } + + async Task FetchVariation(string variation) + { + var variationUrl = BaseDownloadUrl + $"unity-{Version}-{variation}.ini"; + string variationIni; + try + { + Log.Information("Fetching {Variation} info for {Version} from {Url}", variation, Version, variationUrl); + variationIni = await _httpClient.GetStringAsync(variationUrl, cancellationToken); + } + catch (HttpRequestException hre) when (hre.StatusCode == HttpStatusCode.Forbidden) + { + Log.Warning("Could not fetch {Variation} info for {Version} from {Url}. Got 'Access forbidden'", + variation, Version, variationUrl); + return null; + } + + var info = UnityBuildInfo.Parse(variationIni); + if (info.Unity.Version != null && !info.Unity.Version.Equals(Version)) + { + throw new Exception( + $"Build info version is invalid (expected {Version}, got {info.Unity.Version})"); + } + + Directory.CreateDirectory(InfoCacheDir); + await File.WriteAllTextAsync(Path.Combine(InfoCacheDir, $"{variation}.ini"), variationIni, + cancellationToken); + return info; + } + + WindowsInfo = await FetchVariation("win"); + MacOsInfo = await FetchVariation("osx"); + LinuxInfo = await FetchVariation("linux"); + NeedsInfoFetch = false; + } + + private string GetDownloadFile() + { + var isLegacyDownload = Id == null || Version.Major < 5; + var editorDownloadPrefix = isLegacyDownload ? "UnitySetup-" : "UnitySetup64-"; + + if (LinuxInfo != null) + { + return LinuxInfo.Unity.Url; + } + + if (MacOsInfo != null && HasModularPlayer) + { + return MacOsInfo.Unity.Url; + } + + if (WindowsInfo != null) + { + return WindowsInfo.Unity.Url; + } + + return $"{editorDownloadPrefix}{ShortVersion}.exe"; + } + + + public async Task MineAsync(CancellationToken cancellationToken) + { + var isLegacyDownload = Id == null || Version.Major < 5; + + var downloadFile = GetDownloadFile(); + var monoDownloadFile = GetDownloadFile(); + + var monoDownloadUrl = BaseDownloadUrl + downloadFile; + var corlibDownloadUrl = ""; + // For specific versions, the installer has no players at all + // So for corlib, download both the installer and the support module + if (!IsMonolithic && !HasModularPlayer) + { + corlibDownloadUrl = monoDownloadUrl; + monoDownloadUrl = BaseDownloadUrl + monoDownloadFile; + } + + var androidDownloadUrl = + (LinuxInfo == null && MacOsInfo == null) || + Version.IsMonolithic() // TODO make monolithic handling better + ? null + : BaseDownloadUrl + (LinuxInfo ?? MacOsInfo)!.Android!.Url; + + var tmpDirectory = Path.Combine(Path.GetTempPath(), "UnityDataMiner", Version.ToString()); + Directory.CreateDirectory(tmpDirectory); + + var managedDirectory = Path.Combine(tmpDirectory, "managed"); + var corlibDirectory = Path.Combine(tmpDirectory, "corlib"); + var libil2cppSourceDirectory = Path.Combine(tmpDirectory, "libil2cpp-source"); + var androidDirectory = Path.Combine(tmpDirectory, "android"); + + var monoArchivePath = Path.Combine(tmpDirectory, Path.GetFileName(monoDownloadUrl)); + var corlibArchivePath = !IsMonolithic && !HasModularPlayer + ? Path.Combine(tmpDirectory, Path.GetFileName(corlibDownloadUrl)) + : monoArchivePath; + var androidArchivePath = androidDownloadUrl == null + ? null + : Path.Combine(tmpDirectory, Path.GetFileName(androidDownloadUrl)); + var libil2cppSourceArchivePath = Path.Combine(tmpDirectory, Path.GetFileName(downloadFile)); + + var assetCollection = new DownloadableAssetCollection(); + + // main downloads + monoArchivePath = assetCollection.AddAsset(monoDownloadUrl, monoArchivePath); + corlibArchivePath = !string.IsNullOrEmpty(corlibDownloadUrl) + ? assetCollection.AddAsset(corlibDownloadUrl, corlibArchivePath) + : monoArchivePath; + androidArchivePath = androidDownloadUrl is not null + ? assetCollection.AddAsset(androidDownloadUrl, androidArchivePath!) + : androidArchivePath; + + // extra Mono pack downloads + // Note: we want to avoid Windows downloads, because 7z can't extract some of those. + // We also want to prefer downloading specific platform support bundles because those are significantly smaller. + var monoWinBuild = LinuxInfo?.WindowsMono ?? MacOsInfo?.WindowsMono; + var monoLinuxBuild = MacOsInfo?.LinuxMono ?? WindowsInfo?.LinuxMono; + var monoMacBuild = LinuxInfo?.MacMono ?? WindowsInfo?.MacMono; + + string? AddMonoBuildDownload(UnityBuildInfo.Module? module, string osName) + { + if (module is null) + { + // TODO: try to fall back to the main pack if unavailable? + Log.Warning("[{Version}] Could not get URL for Mono pack for {OS}", Version, osName); + return null; + } + + return assetCollection.AddAsset(BaseDownloadUrl + module.Url, + Path.Combine(tmpDirectory, $"{osName}-{Path.GetFileName(module.Url)}")); + } + + var monoWinArchive = AddMonoBuildDownload(monoWinBuild, "windows"); + var monoLinuxArchive = AddMonoBuildDownload(monoLinuxBuild, "linux"); + var monoMacArchive = AddMonoBuildDownload(monoMacBuild, "macos"); + + try + { + await assetCollection.DownloadAssetsAsync(DownloadAsync, Version, null, cancellationToken); + + // process android + if (androidDownloadUrl != null && !Directory.Exists(AndroidPath)) + { + Log.Information("[{Version}] Extracting android binaries", Version); + using var stopwatch = new AutoStopwatch(); + + var archiveDirectory = + Path.Combine(tmpDirectory, Path.GetFileNameWithoutExtension(androidArchivePath)!); + + const string libs = "Variations/il2cpp/Release/Libs"; + const string symbols = "Variations/il2cpp/Release/Symbols"; + + await ExtractAsync(androidArchivePath!, archiveDirectory, + new[] { $"./{libs}/*/libunity.so", $"./{symbols}/*/libunity.sym.so" }, cancellationToken, false); + + Directory.CreateDirectory(androidDirectory); + + IEnumerable directories = Directory.GetDirectories(Path.Combine(archiveDirectory, libs)); + + var hasSymbols = Version > new UnityVersion(5, 3, 5, UnityVersionType.Final, 1); + + if (hasSymbols) + { + directories = + directories.Concat(Directory.GetDirectories(Path.Combine(archiveDirectory, symbols))); + } + + foreach (var directory in directories) + { + var directoryInfo = + Directory.CreateDirectory(Path.Combine(androidDirectory, Path.GetFileName(directory))); + foreach (var file in Directory.GetFiles(directory)) + { + File.Copy(file, Path.Combine(directoryInfo.FullName, Path.GetFileName(file)), true); + } + } + + if (hasSymbols) + { + foreach (var directory in Directory.GetDirectories(androidDirectory)) + { + await EuUnstrip.UnstripAsync(Path.Combine(directory, "libunity.so"), + Path.Combine(directory, "libunity.sym.so"), cancellationToken); + } + } + + Directory.CreateDirectory(AndroidPath); + + foreach (var directory in Directory.GetDirectories(androidDirectory)) + { + ZipFile.CreateFromDirectory(directory, + Path.Combine(AndroidPath, Path.GetFileName(directory) + ".zip")); + } + + Log.Information("[{Version}] Extracted android binaries in {Time}", Version, stopwatch.Elapsed); + } + + // process libil2cpp + if (!File.Exists(LibIl2CppSourceZipPath)) + { + Log.Information("[{Version}] Extracting libil2cpp source code", Version); + using (var stopwatch = new AutoStopwatch()) + { + // TODO: find out if the path changes in different versions + var libil2cppSourcePath = HasLinuxEditor switch + { + true => "Editor/Data/il2cpp/libil2cpp", + false when HasModularPlayer => "./Unity/Unity.app/Contents/il2cpp/libil2cpp", + false => "Editor/Data/il2cpp/libil2cpp", + }; + await ExtractAsync(libil2cppSourceArchivePath, libil2cppSourceDirectory, + new[] { $"{libil2cppSourcePath}/**" }, cancellationToken, false); + var zipDir = Path.Combine(libil2cppSourceDirectory, libil2cppSourcePath); + if (!Directory.Exists(zipDir) || Directory.GetFiles(zipDir).Length <= 0) + { + throw new Exception("LibIl2Cpp source code directory is empty"); + } + + File.Delete(LibIl2CppSourceZipPath); + ZipFile.CreateFromDirectory(zipDir, LibIl2CppSourceZipPath); + + Log.Information("[{Version}] Extracted libil2cpp source code in {Time}", Version, + stopwatch.Elapsed); + } + } + + async Task ExtractManagedDir() + { + bool Exists() => Directory.Exists(managedDirectory) && + Directory.GetFiles(managedDirectory, "*.dll").Length > 0; + + if (Exists()) + { + return; + } + + // TODO: Clean up this massive mess + var monoPath = (Version.IsMonolithic(), isLegacyDownload) switch + { + (true, true) when Version.Major == 4 && Version.Minor >= 5 => + "Data/PlaybackEngines/windowsstandalonesupport/Variations/win64_nondevelopment/Data/Managed", + (true, true) => "Data/PlaybackEngines/windows64standaloneplayer/Managed", + (true, false) => + "Editor/Data/PlaybackEngines/windowsstandalonesupport/Variations/win64_nondevelopment_mono/Data/Managed", + (false, true) => throw new Exception( + "Release can't be both legacy and modular at the same time"), + (false, false) when HasLinuxEditor => + $"Editor/Data/PlaybackEngines/LinuxStandaloneSupport/Variations/linux64{(Version >= new UnityVersion(2021, 2) ? "_player" : "_withgfx")}_nondevelopment_mono/Data/Managed", + (false, false) when !HasLinuxEditor && HasModularPlayer => + $"./Unity/Unity.app/Contents/PlaybackEngines/MacStandaloneSupport/Variations/macosx64_nondevelopment_mono/Data/Managed", + (false, false) => "./Variations/win64_nondevelopment_mono/Data/Managed", + }; + + await ExtractAsync(monoArchivePath, managedDirectory, new[] { $"{monoPath}/*.dll" }, + cancellationToken); + + if (!Exists()) + { + throw new Exception("Managed directory is empty"); + } + } + + // process unity libs + if (!File.Exists(UnityLibsZipFilePath)) + { + Log.Information("[{Version}] Extracting mono libraries", Version); + using (var stopwatch = new AutoStopwatch()) + { + await ExtractManagedDir(); + ZipFile.CreateFromDirectory(managedDirectory, UnityLibsZipFilePath); + Log.Information("[{Version}] Extracted mono libraries in {Time}", Version, stopwatch.Elapsed); + } + } + + // process corlibs + if (!File.Exists(CorlibZipPath)) + { + using (var stopwatch = new AutoStopwatch()) + { + // TODO: Maybe grab both 2.0 and 4.5 DLLs for < 2018 monos + var corlibPath = isLegacyDownload switch + { + true => "Data/Mono/lib/mono/2.0", + false when HasLinuxEditor || !HasModularPlayer => + "Editor/Data/MonoBleedingEdge/lib/mono/4.5", + false => "./Unity/Unity.app/Contents/MonoBleedingEdge/lib/mono/4.5", + }; + + await ExtractAsync(corlibArchivePath, corlibDirectory, + _importantCorlibs.Select(s => $"{corlibPath}/{s}.dll").ToArray(), cancellationToken); + + if (!Directory.Exists(corlibDirectory) || + Directory.GetFiles(corlibDirectory, "*.dll").Length <= 0) + { + throw new Exception("Corlibs directory is empty"); + } + + File.Delete(CorlibZipPath); + ZipFile.CreateFromDirectory(corlibDirectory, CorlibZipPath); + + Log.Information("[{Version}] Extracted corlibs in {Time}", Version, stopwatch.Elapsed); + } + } + + // generate nuget package + if (!File.Exists(NuGetPackagePath)) + { + Log.Information("[{Version}] Creating NuGet package for mono libraries", Version); + using (var stopwatch = new AutoStopwatch()) + { + await ExtractManagedDir(); + CreateNuGetPackage(managedDirectory); + Log.Information("[{Version}] Created NuGet package for mono libraries in {Time}", Version, + stopwatch.Elapsed); + } + } + + // process Mono builds + if (!Directory.Exists(MonoPath)) + { + Log.Information("[{Version}] Packaging Mono binaries", Version); + using var stopwatch = new AutoStopwatch(); + + var monoBaseDir = Path.Combine(tmpDirectory, "mono"); + var winBaseDir = Path.Combine(monoBaseDir, "win"); + var linuxBaseDir = Path.Combine(monoBaseDir, "linux"); + var macBaseDir = Path.Combine(monoBaseDir, "mac"); + + // first, Windows + if (monoWinArchive is not null) + { + Log.Information("[{Version}] Processing Windows", Version); + + // extract all of the Mono variants + await ExtractAsync(monoWinArchive, winBaseDir, + ["./Variations/*_player_nondevelopment_mono/Mono*/**"], + cancellationToken, flat: false); + + // Windows is nice and easy, we just want to pack up all the subdirs of *_player_nondevelopment_mono + // They contain fully self-contained Mono installs minus the corelib, which we extract above. + // The actual executable binaries are in /EmbedRuntime + foreach (var playerDir in Directory.EnumerateDirectories(Path.Combine(winBaseDir, "Variations"))) + { + var arch = Path.GetFileName(playerDir).Replace("_player_nondevelopment_mono", ""); + if (!arch.Contains("win")) + { + arch = "win_" + arch; + } + + foreach (var monoDir in Directory.EnumerateDirectories(playerDir)) + { + var monoName = Path.GetFileName(monoDir); + + // rename EmbedRuntime to just runtime for consistency across platforms + var runtimeDir = Path.Combine(monoDir, "runtime"); + if (Directory.Exists(runtimeDir)) + { + Directory.Delete(runtimeDir, true); + } + Directory.Move(Path.Combine(monoDir, "EmbedRuntime"), runtimeDir); + ZipFile.CreateFromDirectory(monoDir, Path.Combine(monoBaseDir, $"{arch}_{monoName}.zip")); + } + } + } + + // next, Linux + if (monoLinuxArchive is not null) + { + Log.Information("[{Version}] Processing Linux", Version); + + await ExtractAsync(monoLinuxArchive, linuxBaseDir, + ["./Variations/*_player_nondevelopment_mono/Data/Mono*/**"], + cancellationToken, flat: false); + + // Linux is mostly similar to Windows, except that the runtime binaries are in x86_64 instead of EmbedRuntime + // Presumably, if non-x64 support is added, the runtime files would end up in folders named for the arch, but + // Unity doesn't support any of those right now, so who knows. + foreach (var playerDir in Directory.EnumerateDirectories(Path.Combine(linuxBaseDir, "Variations"))) + { + var arch = Path.GetFileName(playerDir).Replace("_player_nondevelopment_mono", ""); + if (!arch.Contains("linux")) + { + arch = "linux_" + arch; + } + + foreach (var monoDir in Directory.EnumerateDirectories(Path.Combine(playerDir, "Data"))) + { + var monoName = Path.GetFileName(monoDir); + + // rename the runtime directory for consistency + + var runtimeDir = Path.Combine(monoDir, "runtime"); + if (Directory.Exists(runtimeDir)) + { + Directory.Delete(runtimeDir, true); + } + Directory.Move(Path.Combine(monoDir, "x86_64"), runtimeDir); + ZipFile.CreateFromDirectory(monoDir, Path.Combine(monoBaseDir, $"{arch}_{monoName}.zip")); + } + } + } + + // finally, MacOS + if (monoMacArchive is not null) + { + Log.Information("[{Version}] Processing MacOS", Version); + + await ExtractAsync(monoMacArchive, macBaseDir, + [ + "./Mono*/**", // this contains the configuration + "./Variations/*_player_nondevelopment_mono/UnityPlayer.app/Contents/Frameworks/lib*" // this contains the actual runtime + // note: we filter to the lib prefix to avoid extracting UnityPlayer.dylib + ], + cancellationToken, flat: false); + + // MacOS is the messiest. There's only one copy of the config files, but several of the runtime, in a very unusual structure. + // We'll just have to cope though. + foreach (var monoConfigDir in Directory.EnumerateDirectories(macBaseDir, "Mono*")) + { + var monoName = Path.GetFileName(monoConfigDir); + var runtimeDir = Path.Combine(monoConfigDir, "runtime"); + + foreach (var playerDir in Directory.EnumerateDirectories(Path.Combine(macBaseDir, "Variations"))) + { + var arch = Path.GetFileName(playerDir).Replace("_player_nondevelopment_mono", ""); + if (!arch.Contains("macos")) + { + arch = "macos_" + arch; + } + + if (Directory.Exists(runtimeDir)) + { + Directory.Delete(runtimeDir, true); + } + + Directory.Move(Path.Combine(playerDir, "UnityPlayer.app", "Contents", "Frameworks"), runtimeDir); + ZipFile.CreateFromDirectory(monoConfigDir, Path.Combine(monoBaseDir, $"{arch}_{monoName}.zip")); + } + } + } + + // we've created all of the zip files, move them into place + Directory.CreateDirectory(MonoPath); + foreach (var zip in Directory.EnumerateFiles(monoBaseDir, "*.zip")) + { + File.Move(zip, Path.Combine(MonoPath, Path.GetFileName(zip))); + } + + Log.Information("[{Version}] Mono binaries packaged in {Time}", Version, + stopwatch.Elapsed); + } + } + finally + { + Directory.Delete(tmpDirectory, true); + } + } + + public Task ExecuteJobsAsync(ImmutableArray jobs, CancellationToken cancellationToken) + { + var plan = JobPlanner.Plan(jobs, this, cancellationToken); + if (plan is null) return Task.CompletedTask; // if we failed to find a plan for a problematic reason, it's already been logged + return ExecutePlan(plan, cancellationToken); + } + + private async Task ExecutePlan(JobPlanner.JobPlan plan, CancellationToken cancellationToken) + { + var jobsToStart = plan.Jobs.ToBuilder(); + + var tmpDirectory = Path.Combine(Path.GetTempPath(), "UnityDataMiner", Version.ToString()); + Directory.CreateDirectory(tmpDirectory); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + + try + { + var targetPaths = new string[plan.Packages.Length]; + var downloadTasks = new Task?[plan.Packages.Length]; + + for (var i = 0; i < plan.Packages.Length; i++) + { + var package = plan.Packages[i]; + var targetName = Path.GetFileName(package.Url); + var arcPath = Path.Combine(tmpDirectory, i + "-" + targetName); + targetPaths[i] = arcPath; + // start all download tasks simultaneously to try to fully saturate our allowed parallel downlads + downloadTasks[i] = DownloadAndPreExtractAsync(BaseDownloadUrl + package.Url, arcPath, cts.Token); + } + + var jobTasks = new List(jobsToStart.Count); + var downloadedPackages = new HashSet(plan.Packages.Length); + + // our main download processing loop + while (downloadTasks.Any(t => t is not null)) + { + var completed = await Task.WhenAny(downloadTasks.Where(t => t is not null)!); + + var i = Array.IndexOf(downloadTasks, completed); + + switch (completed.Status) + { + default: + case TaskStatus.Created: + case TaskStatus.WaitingForActivation: + case TaskStatus.WaitingToRun: + case TaskStatus.Running: + case TaskStatus.WaitingForChildrenToComplete: + // none of these are completion states, it should never happen + continue; + + case TaskStatus.Canceled: + // download was cancelled, which means that our own CT was set. Wait for everything, and (inevitably) bail. + // First, lets cancel everything though. Want to bail as quickly as possible. + await cts.CancelAsync(); + await Task.WhenAll(jobTasks.Concat(downloadTasks.Where(t => t is not null))!); + break; + + case TaskStatus.Faulted: + // task faulted; check the exception and maybe retry, otherwise do the same as when cancelled + if (completed.Exception is { + InnerExceptions: [ IOException { + InnerException: SocketException { SocketErrorCode: SocketError.ConnectionReset } + }] + }) + { + // we want to retry + static async Task RetryDownload(UnityBuild @this, string url, string packagePath, CancellationToken cancellationToken) + { + Log.Warning("[{Version}] Failed to download {Url}, waiting 5 seconds before retrying...", @this.Version, url); + await Task.Delay(5000, cancellationToken); + await @this.DownloadAndPreExtractAsync(@this.BaseDownloadUrl + url, packagePath, cancellationToken); + } + + downloadTasks[i] = RetryDownload(this, plan.Packages[i].Url, targetPaths[i], cts.Token); + } + else + { + // unknown exception, bail out + goto case TaskStatus.Canceled; + } + break; + + case TaskStatus.RanToCompletion: + // the download completed. Lets await it to connect up the invocation, then start any jobs that can now start. + downloadTasks[i] = null; // clear the task so we don't hit it again + _ = downloadedPackages.Add(i); + await completed; + + var incrNo = 0; + for (var j = 0; j < jobsToStart.Count; j++) + { + var (needs, job) = jobsToStart[j]; + + if (job.RunIncrementally) + { + // incremental job + if (!needs.Contains(i)) + { + // this package isn't one that this job cares about + continue; + } + + // this job cares about this package; invoke for it + + // create its temp dir + var localTmpDir = Path.Combine(tmpDirectory, $"i-{i}-{incrNo++}"); + Directory.CreateDirectory(localTmpDir); + + var incPackage = plan.Packages[i]; + var incTargetPath = targetPaths[i]; + + Log.Debug("[{Version}] Starting job {Job} incrementally for {Asset}", Version, job.Name, Path.GetFileName(incPackage.Url)); + + // start the job + jobTasks.Add( + job.ExtractFromAssets(this, localTmpDir, + [incPackage.Package], + [incTargetPath], + cts.Token)); + } + else + { + // normal, non-incremental job + if (!needs.All(downloadedPackages.Contains)) + { + // not all of this jobs requirements are downloaded, keep checking + continue; + } + + // we have everything we need, set up the arrays we need to pass to the job + // first, remove it from our list though + jobsToStart.RemoveAt(j--); + + var packages = new UnityPackage[needs.Length]; + var archivePaths = new string[needs.Length]; + + for (var k = 0; k < needs.Length; k++) + { + var jobIndex = needs[k]; + packages[k] = plan.Packages[jobIndex].Package; + archivePaths[k] = targetPaths[jobIndex]; + } + + // give it its own temp dir + var localTmpDir = Path.Combine(tmpDirectory, jobsToStart.Count.ToString()); + Directory.CreateDirectory(localTmpDir); + + Log.Debug("[{Version}] Starting job {Job} (local dir: {RemainingJobs})", Version, job.Name, jobsToStart.Count); + + // and start the job + jobTasks.Add( + job.ExtractFromAssets(this, localTmpDir, + ImmutableCollectionsMarshal.AsImmutableArray(packages), + ImmutableCollectionsMarshal.AsImmutableArray(archivePaths), + cts.Token)); + } + } + break; + } + } + + // we've now downloaded all of our assets, and are just waiting for the jobs to complete + await Task.WhenAll(jobTasks); + } + finally + { + Directory.Delete(tmpDirectory, true); + } + } + + private async Task DownloadAndPreExtractAsync(string downloadUrl, string archivePath, CancellationToken cancellationToken) + { + await DownloadAsync(downloadUrl, archivePath, cancellationToken); + // run a pre-extraction on the archive, if it's one that needs it, to avoid data races later + Log.Information("[{Version}] Pre-extracting {Archive}", Version, Path.GetFileName(archivePath)); + await ExtractAsync(archivePath, "", [], cancellationToken, firstStageOnly: true); + Log.Information("[{Version}] Pre-extract complete", Version); + } + + public async Task DownloadAsync(string downloadUrl, string archivePath, CancellationToken cancellationToken) + { + if (File.Exists(archivePath)) + { + Log.Information("[{Version}] Skipping download because {File} exists", Version, archivePath); + } + else + { + using var stopwatch = new AutoStopwatch(); + try + { + await _downloadLock.WaitAsync(cancellationToken); + + Log.Information("[{Version}] Downloading {Url}", Version, downloadUrl); + stopwatch.Restart(); + + await using (var stream = await _httpClient.GetStreamAsync(downloadUrl, cancellationToken)) + await using (var fileStream = File.OpenWrite(archivePath + ".part")) + { + await stream.CopyToAsync(fileStream, cancellationToken); + } + } + finally + { + if (!cancellationToken.IsCancellationRequested) + { + _downloadLock.Release(); + } + } + + // do move outside lock in case it takes a lot of time + File.Move(archivePath + ".part", archivePath); + + Log.Information("[{Version}] Downloaded {Url} in {Time}", Version, downloadUrl, stopwatch.Elapsed); + } + } + + // TODO: it's probably a good idea to lock around extraction for the double-extract cases + public async Task ExtractAsync(string archivePath, string destinationDirectory, string[] filter, + CancellationToken cancellationToken, bool flat = true, bool firstStageOnly = false) + { + var archiveDirectory = Path.Combine(Path.GetDirectoryName(archivePath)!, + Path.GetFileNameWithoutExtension(archivePath)); + var extension = Path.GetExtension(archivePath); + + switch (extension) + { + case ".pkg": + { + const string payloadName = "Payload~"; + var payloadPath = Path.Combine(archiveDirectory, payloadName); + if (!File.Exists(payloadPath)) + { + await SevenZip.ExtractAsync(archivePath, archiveDirectory, [payloadName], true, + cancellationToken); + } + if (!firstStageOnly) + { + await SevenZip.ExtractAsync(payloadPath, destinationDirectory, + filter, flat, cancellationToken); + } + + break; + } + + case ".exe": + { + if (!firstStageOnly) + { + await SevenZip.ExtractAsync(archivePath, destinationDirectory, filter, flat, cancellationToken); + } + + break; + } + + case ".xz": + { + string payloadName = Path.GetFileNameWithoutExtension(archivePath); + var payloadPath = Path.Combine(archiveDirectory, payloadName); + if (!File.Exists(payloadPath)) + { + await SevenZip.ExtractAsync(archivePath, archiveDirectory, [payloadName], true, + cancellationToken); + } + if (!firstStageOnly) + { + await SevenZip.ExtractAsync(payloadPath, destinationDirectory, + filter, flat, cancellationToken); + } + + break; + } + + default: + throw new ArgumentOutOfRangeException(nameof(extension), extension, "Unrecognized archive type"); + } + } + + private void CreateNuGetPackage(string pkgDir) + { + foreach (var file in Directory.EnumerateFiles(pkgDir, "*.dll")) + AssemblyPublicizer.Publicize(file, file, new AssemblyPublicizerOptions { Strip = true }); + + var deps = new[] { "net35", "net45", "netstandard2.0" }; + + var meta = new ManifestMetadata + { + Id = "UnityEngine.Modules", + Authors = new[] { "Unity" }, + Version = NuGetVersion, + Description = "UnityEngine modules", + DevelopmentDependency = true, + DependencyGroups = deps.Select(d => + new PackageDependencyGroup(NuGetFramework.Parse(d), Array.Empty())) + }; + + var builder = new PackageBuilder(true); + builder.PopulateFiles(pkgDir, deps.Select(d => new ManifestFile + { + Source = "*.dll", + Target = $"lib/{d}" + })); + builder.Populate(meta); + using var fs = File.Create(NuGetPackagePath); + builder.Save(fs); + } + + public async Task UploadNuGetPackageAsync(string sourceUrl, string apikey) + { + Log.Information("[{Version}] Pushing NuGet package", Version); + var repo = Repository.Factory.GetCoreV3(sourceUrl); + var updateResource = await repo.GetResourceAsync(); + await updateResource.Push(new[] { NuGetPackagePath }, + null, + 2 * 60, + false, + s => apikey, + s => null, + false, + true, + null, + NullLogger.Instance); + Log.Information("[{Version}] Pushed NuGet package", Version); + } + } +} diff --git a/UnityDataMiner/UnityReleaseInfo.cs b/UnityDataMiner/UnityBuildInfo.cs similarity index 77% rename from UnityDataMiner/UnityReleaseInfo.cs rename to UnityDataMiner/UnityBuildInfo.cs index 9ccf602..e9e15e4 100644 --- a/UnityDataMiner/UnityReleaseInfo.cs +++ b/UnityDataMiner/UnityBuildInfo.cs @@ -11,6 +11,9 @@ public record Module(string Title, string Url, UnityVersion? Version); public Module Unity => Components["Unity"]; public Module? WindowsMono => Components.TryGetValue("Windows-Mono", out var result) || Components.TryGetValue("Windows", out result) ? result : null; + public Module? MacMono => Components.TryGetValue("Mac-Mono", out var result) || Components.TryGetValue("Mac", out result) ? result : null; + public Module? LinuxMono => Components.TryGetValue("Linux-Mono", out var result) || Components.TryGetValue("Linux", out result) ? result : null; + public Module? Android => Components.TryGetValue("Android", out var result) ? result : null; public static UnityBuildInfo Parse(string ini) diff --git a/UnityDataMiner/UnityDataMiner.csproj b/UnityDataMiner/UnityDataMiner.csproj index 1a0b06a..57781b1 100644 --- a/UnityDataMiner/UnityDataMiner.csproj +++ b/UnityDataMiner/UnityDataMiner.csproj @@ -8,22 +8,23 @@ - - - - - + + + + + + - - + + - + - + diff --git a/UnityDataMiner/UnityRelease.cs b/UnityDataMiner/UnityRelease.cs deleted file mode 100644 index c7d1260..0000000 --- a/UnityDataMiner/UnityRelease.cs +++ /dev/null @@ -1,601 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Sockets; -using System.Threading; -using System.Threading.Tasks; -using AssetRipper.Primitives; -using BepInEx.AssemblyPublicizer; -using NuGet.Common; -using NuGet.Frameworks; -using NuGet.Packaging; -using NuGet.Packaging.Core; -using NuGet.Protocol; -using NuGet.Protocol.Core.Types; -using NuGet.Versioning; -using Serilog; - -namespace UnityDataMiner -{ - public class UnityBuild - { - public string? Id { get; } - - public UnityVersion Version { get; } - - public string ShortVersion { get; } - - public NuGetVersion NuGetVersion { get; } - - public string ZipFilePath { get; } - - public string AndroidPath { get; } - - public string NuGetPackagePath { get; } - - public string InfoCacheDir { get; } - - public string CorlibZipPath { get; } - - public string LibIl2CppSourceZipPath { get; } - - public bool IsRunNeeded => !File.Exists(ZipFilePath) || !File.Exists(NuGetPackagePath) || - !File.Exists(CorlibZipPath) || - (HasLibIl2Cpp && !File.Exists(LibIl2CppSourceZipPath)) || - !Version.IsMonolithic() && !Directory.Exists(AndroidPath); - - public string BaseDownloadUrl => Version.GetDownloadUrl() + (Id == null ? string.Empty : $"{Id}/"); - - public UnityBuildInfo? WindowsInfo { get; private set; } - public UnityBuildInfo? LinuxInfo { get; private set; } - public UnityBuildInfo? MacOsInfo { get; private set; } - - public UnityBuild(string repositoryPath, string? id, UnityVersion version) - { - Id = id; - Version = version; - - if (Version.Major >= 5 && Id == null) - { - throw new Exception("Hash cannot be null after 5.x"); - } - - ShortVersion = Version.ToStringWithoutType(); - NuGetVersion = Version.Type == UnityVersionType.Final - ? new NuGetVersion(Version.Major, Version.Minor, Version.Build) - : new NuGetVersion(Version.Major, Version.Minor, Version.Build, Version.Type switch - { - UnityVersionType.Alpha => "alpha", - UnityVersionType.Beta => "beta", - UnityVersionType.China => "china", - UnityVersionType.Final => "final", - UnityVersionType.Patch => "patch", - UnityVersionType.Experimental => "experimental", - _ => throw new ArgumentOutOfRangeException(nameof(Version.Type), Version.Type, - "Invalid Version.Type for " + Version), - } + "." + Version.TypeNumber); - - var versionName = Version.Type == UnityVersionType.Final ? ShortVersion : Version.ToString(); - var zipName = $"{versionName}.zip"; - ZipFilePath = Path.Combine(repositoryPath, "libraries", zipName); - AndroidPath = Path.Combine(repositoryPath, "android", versionName); - CorlibZipPath = Path.Combine(repositoryPath, "corlibs", zipName); - LibIl2CppSourceZipPath = Path.Combine(repositoryPath, "libil2cpp-source", zipName); - NuGetPackagePath = Path.Combine(repositoryPath, "packages", $"{NuGetVersion}.nupkg"); - InfoCacheDir = Path.Combine(repositoryPath, "versions", $"{id}"); - - WindowsInfo = ReadInfo("win"); - - if (Version >= _firstLinuxVersion) - { - LinuxInfo = ReadInfo("linux"); - } - - MacOsInfo = ReadInfo("osx"); - } - - private static readonly HttpClient _httpClient = new(); - private static readonly SemaphoreSlim _downloadLock = new(2, 2); - - private static readonly UnityVersion _firstLinuxVersion = new(2018, 1, 5); - - // First modular version where own native player is included in the default installer - private static readonly UnityVersion _firstMergedModularVersion = new(5, 4); - private static readonly UnityVersion _firstLibIl2CppVersion = new(5, 0, 2); - - // TODO: Might need to define more DLLs? This should be enough for basic unhollowing. - private static readonly string[] _importantCorlibs = - { - "Microsoft.CSharp", - "Mono.Posix", - "Mono.Security", - "mscorlib", - "Facades/netstandard", - "System.Configuration", - "System.Core", - "System.Data", - "System", - "System.Net.Http", - "System.Numerics", - "System.Runtime.Serialization", - "System.Security", - "System.Xml", - "System.Xml.Linq", - }; - - private bool HasLinuxEditor => LinuxInfo is not null; - private bool HasModularPlayer => Version >= _firstMergedModularVersion; - private bool IsMonolithic => Version.IsMonolithic(); - private bool HasLibIl2Cpp => Version >= _firstLibIl2CppVersion; - - public bool NeedsInfoFetch { get; private set; } - - private UnityBuildInfo? ReadInfo(string variation) - { - if (!Directory.Exists(InfoCacheDir)) - { - NeedsInfoFetch = true; - return null; - } - - var path = Path.Combine(InfoCacheDir, $"{variation}.ini"); - try - { - var variationIni = File.ReadAllText(path); - var info = UnityBuildInfo.Parse(variationIni); - if (info.Unity.Version != null && !info.Unity.Version.Equals(Version)) - { - throw new Exception(); - } - - return info; - } - catch (Exception) - { - NeedsInfoFetch = true; - return null; - } - } - - public async Task FetchInfoAsync(CancellationToken cancellationToken) - { - if (!NeedsInfoFetch) - { - return; - } - - async Task FetchVariation(string variation) - { - var variationUrl = BaseDownloadUrl + $"unity-{Version}-{variation}.ini"; - string variationIni; - try - { - Log.Information("Fetching {Variation} info for {Version} from {Url}", variation, Version, variationUrl); - variationIni = await _httpClient.GetStringAsync(variationUrl, cancellationToken); - } - catch (HttpRequestException hre) when (hre.StatusCode == HttpStatusCode.Forbidden) - { - Log.Warning("Could not fetch {Variation} info for {Version} from {Url}. Got 'Access forbidden'", - variation, Version, variationUrl); - return null; - } - - var info = UnityBuildInfo.Parse(variationIni); - if (info.Unity.Version != null && !info.Unity.Version.Equals(Version)) - { - throw new Exception( - $"Build info version is invalid (expected {Version}, got {info.Unity.Version})"); - } - - Directory.CreateDirectory(InfoCacheDir); - await File.WriteAllTextAsync(Path.Combine(InfoCacheDir, $"{variation}.ini"), variationIni, - cancellationToken); - return info; - } - - WindowsInfo = await FetchVariation("win"); - MacOsInfo = await FetchVariation("osx"); - LinuxInfo = await FetchVariation("linux"); - NeedsInfoFetch = false; - } - - private string GetDownloadFile() - { - var isLegacyDownload = Id == null || Version.Major < 5; - var editorDownloadPrefix = isLegacyDownload ? "UnitySetup-" : "UnitySetup64-"; - - if (LinuxInfo != null) - { - return LinuxInfo.Unity.Url; - } - - if (MacOsInfo != null && HasModularPlayer) - { - return MacOsInfo.Unity.Url; - } - - if (WindowsInfo != null) - { - return WindowsInfo.Unity.Url; - } - - return $"{editorDownloadPrefix}{ShortVersion}.exe"; - } - - public async Task MineAsync(CancellationToken cancellationToken) - { - var isLegacyDownload = Id == null || Version.Major < 5; - - var downloadFile = GetDownloadFile(); - var monoDownloadFile = GetDownloadFile(); - - var monoDownloadUrl = BaseDownloadUrl + downloadFile; - var corlibDownloadUrl = ""; - // For specific versions, the installer has no players at all - // So for corlib, download both the installer and the support module - if (!IsMonolithic && !HasModularPlayer) - { - corlibDownloadUrl = monoDownloadUrl; - monoDownloadUrl = BaseDownloadUrl + monoDownloadFile; - } - - var androidDownloadUrl = - (LinuxInfo == null && MacOsInfo == null) || - Version.IsMonolithic() // TODO make monolithic handling better - ? null - : BaseDownloadUrl + (LinuxInfo ?? MacOsInfo)!.Android!.Url; - - var tmpDirectory = Path.Combine(Path.GetTempPath(), "UnityDataMiner", Version.ToString()); - Directory.CreateDirectory(tmpDirectory); - - var managedDirectory = Path.Combine(tmpDirectory, "managed"); - var corlibDirectory = Path.Combine(tmpDirectory, "corlib"); - var libil2cppSourceDirectory = Path.Combine(tmpDirectory, "libil2cpp-source"); - var androidDirectory = Path.Combine(tmpDirectory, "android"); - - var monoArchivePath = Path.Combine(tmpDirectory, Path.GetFileName(monoDownloadUrl)); - var corlibArchivePath = !IsMonolithic && !HasModularPlayer - ? Path.Combine(tmpDirectory, Path.GetFileName(corlibDownloadUrl)) - : monoArchivePath; - var androidArchivePath = androidDownloadUrl == null - ? null - : Path.Combine(tmpDirectory, Path.GetFileName(androidDownloadUrl)); - var libil2cppSourceArchivePath = Path.Combine(tmpDirectory, Path.GetFileName(downloadFile)); - - try - { - while (true) - { - try - { - await _downloadLock.WaitAsync(cancellationToken); - try - { - await DownloadAsync(monoDownloadUrl, monoArchivePath, cancellationToken); - - if (corlibDownloadUrl is not "") - { - await DownloadAsync(corlibDownloadUrl, corlibArchivePath, cancellationToken); - } - - if (androidDownloadUrl != null) - { - await DownloadAsync(androidDownloadUrl, androidArchivePath!, cancellationToken); - } - } - finally - { - if (!cancellationToken.IsCancellationRequested) - { - _downloadLock.Release(); - } - } - - break; - } - catch (IOException e) when (e.InnerException is SocketException - { - SocketErrorCode: SocketError.ConnectionReset - }) - { - Log.Warning("Failed to download {Version}, waiting 5 seconds before retrying...", Version); - await Task.Delay(5000, cancellationToken); - } - } - - if (androidDownloadUrl != null && !Directory.Exists(AndroidPath)) - { - Log.Information("[{Version}] Extracting android binaries", Version); - using var stopwatch = new AutoStopwatch(); - - var archiveDirectory = - Path.Combine(tmpDirectory, Path.GetFileNameWithoutExtension(androidArchivePath)!); - - const string libs = "Variations/il2cpp/Release/Libs"; - const string symbols = "Variations/il2cpp/Release/Symbols"; - - await ExtractAsync(androidArchivePath!, archiveDirectory, - new[] { $"./{libs}/*/libunity.so", $"./{symbols}/*/libunity.sym.so" }, cancellationToken, false); - - Directory.CreateDirectory(androidDirectory); - - IEnumerable directories = Directory.GetDirectories(Path.Combine(archiveDirectory, libs)); - - var hasSymbols = Version > new UnityVersion(5, 3, 5, UnityVersionType.Final, 1); - - if (hasSymbols) - { - directories = - directories.Concat(Directory.GetDirectories(Path.Combine(archiveDirectory, symbols))); - } - - foreach (var directory in directories) - { - var directoryInfo = - Directory.CreateDirectory(Path.Combine(androidDirectory, Path.GetFileName(directory))); - foreach (var file in Directory.GetFiles(directory)) - { - File.Copy(file, Path.Combine(directoryInfo.FullName, Path.GetFileName(file)), true); - } - } - - if (hasSymbols) - { - foreach (var directory in Directory.GetDirectories(androidDirectory)) - { - await EuUnstrip.UnstripAsync(Path.Combine(directory, "libunity.so"), - Path.Combine(directory, "libunity.sym.so"), cancellationToken); - } - } - - Directory.CreateDirectory(AndroidPath); - - foreach (var directory in Directory.GetDirectories(androidDirectory)) - { - ZipFile.CreateFromDirectory(directory, - Path.Combine(AndroidPath, Path.GetFileName(directory) + ".zip")); - } - - Log.Information("[{Version}] Extracted android binaries in {Time}", Version, stopwatch.Elapsed); - } - - if (!File.Exists(ZipFilePath)) - { - Log.Information("[{Version}] Extracting libil2cpp source code", Version); - using (var stopwatch = new AutoStopwatch()) - { - // TODO: find out if the path changes in different versions - var libil2cppSourcePath = HasLinuxEditor switch - { - true => "Editor/Data/il2cpp/libil2cpp", - false when HasModularPlayer => "./Unity/Unity.app/Contents/il2cpp/libil2cpp", - false => "Editor/Data/il2cpp/libil2cpp", - }; - await ExtractAsync(libil2cppSourceArchivePath, libil2cppSourceDirectory, - new[] { $"{libil2cppSourcePath}/**" }, cancellationToken, false); - var zipDir = Path.Combine(libil2cppSourceDirectory, libil2cppSourcePath); - if (!Directory.Exists(zipDir) || Directory.GetFiles(zipDir).Length <= 0) - { - throw new Exception("LibIl2Cpp source code directory is empty"); - } - - File.Delete(LibIl2CppSourceZipPath); - ZipFile.CreateFromDirectory(zipDir, LibIl2CppSourceZipPath); - - Log.Information("[{Version}] Extracted libil2cpp source code in {Time}", Version, - stopwatch.Elapsed); - } - } - - async Task ExtractManagedDir() - { - bool Exists() => Directory.Exists(managedDirectory) && - Directory.GetFiles(managedDirectory, "*.dll").Length > 0; - - if (Exists()) - { - return; - } - - // TODO: Clean up this massive mess - var monoPath = (Version.IsMonolithic(), isLegacyDownload) switch - { - (true, true) when Version.Major == 4 && Version.Minor >= 5 => - "Data/PlaybackEngines/windowsstandalonesupport/Variations/win64_nondevelopment/Data/Managed", - (true, true) => "Data/PlaybackEngines/windows64standaloneplayer/Managed", - (true, false) => - "Editor/Data/PlaybackEngines/windowsstandalonesupport/Variations/win64_nondevelopment_mono/Data/Managed", - (false, true) => throw new Exception( - "Release can't be both legacy and modular at the same time"), - (false, false) when HasLinuxEditor => - $"Editor/Data/PlaybackEngines/LinuxStandaloneSupport/Variations/linux64{(Version >= new UnityVersion(2021, 2) ? "_player" : "_withgfx")}_nondevelopment_mono/Data/Managed", - (false, false) when !HasLinuxEditor && HasModularPlayer => - $"./Unity/Unity.app/Contents/PlaybackEngines/MacStandaloneSupport/Variations/macosx64_nondevelopment_mono/Data/Managed", - (false, false) => "./Variations/win64_nondevelopment_mono/Data/Managed", - }; - - await ExtractAsync(monoArchivePath, managedDirectory, new[] { $"{monoPath}/*.dll" }, - cancellationToken); - - if (!Exists()) - { - throw new Exception("Managed directory is empty"); - } - } - - if (!File.Exists(ZipFilePath)) - { - Log.Information("[{Version}] Extracting mono libraries", Version); - using (var stopwatch = new AutoStopwatch()) - { - await ExtractManagedDir(); - ZipFile.CreateFromDirectory(managedDirectory, ZipFilePath); - Log.Information("[{Version}] Extracted mono libraries in {Time}", Version, stopwatch.Elapsed); - } - } - - if (!File.Exists(CorlibZipPath)) - { - using (var stopwatch = new AutoStopwatch()) - { - // TODO: Maybe grab both 2.0 and 4.5 DLLs for < 2018 monos - var corlibPath = isLegacyDownload switch - { - true => "Data/Mono/lib/mono/2.0", - false when HasLinuxEditor || !HasModularPlayer => - "Editor/Data/MonoBleedingEdge/lib/mono/4.5", - false => "./Unity/Unity.app/Contents/MonoBleedingEdge/lib/mono/4.5", - }; - - await ExtractAsync(corlibArchivePath, corlibDirectory, - _importantCorlibs.Select(s => $"{corlibPath}/{s}.dll").ToArray(), cancellationToken); - - if (!Directory.Exists(corlibDirectory) || - Directory.GetFiles(corlibDirectory, "*.dll").Length <= 0) - { - throw new Exception("Corlibs directory is empty"); - } - - File.Delete(CorlibZipPath); - ZipFile.CreateFromDirectory(corlibDirectory, CorlibZipPath); - - Log.Information("[{Version}] Extracted corlibs in {Time}", Version, stopwatch.Elapsed); - } - } - - if (!File.Exists(NuGetPackagePath)) - { - Log.Information("[{Version}] Creating NuGet package for mono libraries", Version); - using (var stopwatch = new AutoStopwatch()) - { - await ExtractManagedDir(); - CreateNuGetPackage(managedDirectory); - Log.Information("[{Version}] Created NuGet package for mono libraries in {Time}", Version, - stopwatch.Elapsed); - } - } - } - finally - { - Directory.Delete(tmpDirectory, true); - } - } - - private async Task DownloadAsync(string downloadUrl, string archivePath, CancellationToken cancellationToken) - { - if (File.Exists(archivePath)) - { - Log.Information("[{Version}] Skipping download because {File} exists", Version, archivePath); - } - else - { - Log.Information("[{Version}] Downloading {Url}", Version, downloadUrl); - using var stopwatch = new AutoStopwatch(); - - await using (var stream = await _httpClient.GetStreamAsync(downloadUrl, cancellationToken)) - await using (var fileStream = File.OpenWrite(archivePath + ".part")) - { - await stream.CopyToAsync(fileStream, cancellationToken); - } - - File.Move(archivePath + ".part", archivePath); - - Log.Information("[{Version}] Downloaded {Url} in {Time}", Version, downloadUrl, stopwatch.Elapsed); - } - } - - private async Task ExtractAsync(string archivePath, string destinationDirectory, string[] filter, - CancellationToken cancellationToken, bool flat = true) - { - var archiveDirectory = Path.Combine(Path.GetDirectoryName(archivePath)!, - Path.GetFileNameWithoutExtension(archivePath)); - var extension = Path.GetExtension(archivePath); - - switch (extension) - { - case ".pkg": - { - const string payloadName = "Payload~"; - await SevenZip.ExtractAsync(archivePath, archiveDirectory, new[] { payloadName }, true, - cancellationToken); - await SevenZip.ExtractAsync(Path.Combine(archiveDirectory, payloadName), destinationDirectory, - filter, flat, cancellationToken); - - break; - } - - case ".exe": - { - await SevenZip.ExtractAsync(archivePath, destinationDirectory, filter, flat, cancellationToken); - - break; - } - - case ".xz": - { - string payloadName = Path.GetFileNameWithoutExtension(archivePath); - await SevenZip.ExtractAsync(archivePath, archiveDirectory, new[] { payloadName }, true, - cancellationToken); - await SevenZip.ExtractAsync(Path.Combine(archiveDirectory, payloadName), destinationDirectory, - filter, flat, cancellationToken); - - break; - } - - default: - throw new ArgumentOutOfRangeException(nameof(extension), extension, "Unrecognized archive type"); - } - } - - private void CreateNuGetPackage(string pkgDir) - { - foreach (var file in Directory.EnumerateFiles(pkgDir, "*.dll")) - AssemblyPublicizer.Publicize(file, file, new AssemblyPublicizerOptions { Strip = true }); - - var deps = new[] { "net35", "net45", "netstandard2.0" }; - - var meta = new ManifestMetadata - { - Id = "UnityEngine.Modules", - Authors = new[] { "Unity" }, - Version = NuGetVersion, - Description = "UnityEngine modules", - DevelopmentDependency = true, - DependencyGroups = deps.Select(d => - new PackageDependencyGroup(NuGetFramework.Parse(d), Array.Empty())) - }; - - var builder = new PackageBuilder(true); - builder.PopulateFiles(pkgDir, deps.Select(d => new ManifestFile - { - Source = "*.dll", - Target = $"lib/{d}" - })); - builder.Populate(meta); - using var fs = File.Create(NuGetPackagePath); - builder.Save(fs); - } - - public async Task UploadNuGetPackageAsync(string sourceUrl, string apikey) - { - Log.Information("[{Version}] Pushing NuGet package", Version); - var repo = Repository.Factory.GetCoreV3(sourceUrl); - var updateResource = await repo.GetResourceAsync(); - await updateResource.Push(new[] { NuGetPackagePath }, - null, - 2 * 60, - false, - s => apikey, - s => null, - false, - true, - null, - NullLogger.Instance); - Log.Information("[{Version}] Pushed NuGet package", Version); - } - } -}