From 024c638e4a10c8492c366dc2eea93064c13a33c0 Mon Sep 17 00:00:00 2001 From: Kouji Matsui Date: Fri, 3 Jan 2025 20:56:45 +0900 Subject: [PATCH 1/7] Refactored. --- chibiar/chibiar.core.Tests/ArchiverTestRunner.cs | 5 +++-- chibiar/chibiar.core.Tests/ArchiverTests.cs | 4 ++-- chibiar/chibiar.core/Archiver.cs | 2 +- chibias/chibias.core/Assembler.cs | 2 +- chibild/chibild.core.Tests/LinkerTestRunner.cs | 5 +++-- chibild/chibild.core/CilLinker.cs | 2 +- .../Generating/ArchivedObjectInputFragment.cs | 3 ++- chibild/chibild.core/Generating/CodeGenerator_Emit.cs | 2 +- chibild/chibild.core/Internal/NetCoreWriter.cs | 3 ++- toolchain.common/Archiving/ArchiverUtilities.cs | 7 ++++--- toolchain.common/IO/StreamUtilities.cs | 9 +++++++++ 11 files changed, 29 insertions(+), 15 deletions(-) diff --git a/chibiar/chibiar.core.Tests/ArchiverTestRunner.cs b/chibiar/chibiar.core.Tests/ArchiverTestRunner.cs index 4edab8b..af60369 100644 --- a/chibiar/chibiar.core.Tests/ArchiverTestRunner.cs +++ b/chibiar/chibiar.core.Tests/ArchiverTestRunner.cs @@ -12,6 +12,7 @@ using System.Runtime.CompilerServices; using System.Text; using System.Threading.Tasks; +using chibicc.toolchain.IO; using chibicc.toolchain.Logging; using DiffEngine; @@ -46,8 +47,8 @@ public static async Task RunAsync( var logPath = Path.Combine(basePath, "log.txt"); using var logfs = new FileStream( logPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None); - var logtw = new StreamWriter( - logfs, Encoding.UTF8); + var logtw = StreamUtilities.CreateTextWriter( + logfs); var logger = new TextWriterLogger( LogLevels.Debug, logtw); diff --git a/chibiar/chibiar.core.Tests/ArchiverTests.cs b/chibiar/chibiar.core.Tests/ArchiverTests.cs index 7f1b7ab..809160f 100644 --- a/chibiar/chibiar.core.Tests/ArchiverTests.cs +++ b/chibiar/chibiar.core.Tests/ArchiverTests.cs @@ -16,7 +16,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; - +using chibicc.toolchain.IO; using static VerifyNUnit.Verifier; using static chibiar.ArchiverTestRunner; @@ -31,7 +31,7 @@ private static async Task VerifySymbolTableAsync(ZipArchive zip) var symTableEntry = zip.GetEntry(ArchiverUtilities.SymbolTableFileName)!; using var afs = symTableEntry.Open(); - var tr = new StreamReader(afs, Encoding.UTF8, true); + var tr = StreamUtilities.CreateTextReader(afs); var actual = await tr.ReadToEndAsync(); await Verify(actual); diff --git a/chibiar/chibiar.core/Archiver.cs b/chibiar/chibiar.core/Archiver.cs index ba4a0ab..1c9f95e 100644 --- a/chibiar/chibiar.core/Archiver.cs +++ b/chibiar/chibiar.core/Archiver.cs @@ -230,7 +230,7 @@ internal bool AddOrUpdate( bool isCreateSymbolTable, bool isDryrun) { - var outputArchiveFilePath = archiveFilePath + $"_{Guid.NewGuid():N}"; + var outputArchiveFilePath = $"{archiveFilePath}_{Guid.NewGuid():N}"; try { diff --git a/chibias/chibias.core/Assembler.cs b/chibias/chibias.core/Assembler.cs index cae043e..d282952 100644 --- a/chibias/chibias.core/Assembler.cs +++ b/chibias/chibias.core/Assembler.cs @@ -46,7 +46,7 @@ public bool Assemble( // Checking only parsing. using (var inputStream = StreamUtilities.OpenStream(sourceFilePath, false)) { - var tr = new StreamReader(inputStream, Encoding.UTF8, true); + var tr = StreamUtilities.CreateTextReader(inputStream); var parser = new CilParser(this.logger); var _ = parser.Parse( diff --git a/chibild/chibild.core.Tests/LinkerTestRunner.cs b/chibild/chibild.core.Tests/LinkerTestRunner.cs index 7fd64cc..fe94765 100644 --- a/chibild/chibild.core.Tests/LinkerTestRunner.cs +++ b/chibild/chibild.core.Tests/LinkerTestRunner.cs @@ -15,6 +15,7 @@ using System.IO; using System.Linq; using System.Text; +using chibicc.toolchain.IO; using DiffEngine; namespace chibild; @@ -55,8 +56,8 @@ public static string RunCore( using (var logfs = new FileStream( logPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None)) { - var logtw = new StreamWriter( - logfs, Encoding.UTF8); + var logtw = StreamUtilities.CreateTextWriter( + logfs); var logger = new TextWriterLogger( LogLevels.Debug, logtw); diff --git a/chibild/chibild.core/CilLinker.cs b/chibild/chibild.core/CilLinker.cs index f160445..6fbfe4f 100644 --- a/chibild/chibild.core/CilLinker.cs +++ b/chibild/chibild.core/CilLinker.cs @@ -110,7 +110,7 @@ private bool TryLoadInputReferences( using (var fs = ObjectStreamUtilities.OpenObjectStream( Path.Combine(baseInputPath, relativePath), false)) { - var tr = new StreamReader(fs, Encoding.UTF8, true); + var tr = StreamUtilities.CreateTextReader(fs); if (!ObjectFileInputFragment.TryLoad( this.logger, baseInputPath, diff --git a/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs b/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs index dc3a9c2..1b8b843 100644 --- a/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs +++ b/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs @@ -19,6 +19,7 @@ using System.Linq; using System.Text; using System.Threading; +using chibicc.toolchain.IO; namespace chibild.Generating; @@ -192,7 +193,7 @@ public LoadObjectResults LoadObjectIfRequired( } using var _s = stream; - var tr = new StreamReader(stream, Encoding.UTF8, true); + var tr = StreamUtilities.CreateTextReader(stream); var parser = new CilParser(logger); var declarations = parser.Parse( diff --git a/chibild/chibild.core/Generating/CodeGenerator_Emit.cs b/chibild/chibild.core/Generating/CodeGenerator_Emit.cs index cc10310..12de7d1 100644 --- a/chibild/chibild.core/Generating/CodeGenerator_Emit.cs +++ b/chibild/chibild.core/Generating/CodeGenerator_Emit.cs @@ -679,7 +679,7 @@ private bool TryLoadAndConsumeCAbiStartUpObjectIfRequired( objectPath, false); - var tr = new StreamReader(fs, Encoding.UTF8, true); + var tr = StreamUtilities.CreateTextReader(fs); if (!this.TryLoadAndConsumeAdhocObject( inputFragments, diff --git a/chibild/chibild.core/Internal/NetCoreWriter.cs b/chibild/chibild.core/Internal/NetCoreWriter.cs index eb7c87d..1fa7524 100644 --- a/chibild/chibild.core/Internal/NetCoreWriter.cs +++ b/chibild/chibild.core/Internal/NetCoreWriter.cs @@ -14,13 +14,14 @@ using System.IO; using System.Runtime.InteropServices; using System.Text; +using chibicc.toolchain.IO; namespace chibild.Internal; internal static class NetCoreWriter { private static readonly string runtimeConfigJsonTemplate = - new StreamReader(typeof(CilLinker).Assembly.GetManifestResourceStream( + StreamUtilities.CreateTextReader(typeof(CilLinker).Assembly.GetManifestResourceStream( "chibild.Internal.runtimeconfig.json")!). ReadToEnd(); diff --git a/toolchain.common/Archiving/ArchiverUtilities.cs b/toolchain.common/Archiving/ArchiverUtilities.cs index e567c56..1cfa8e5 100644 --- a/toolchain.common/Archiving/ArchiverUtilities.cs +++ b/toolchain.common/Archiving/ArchiverUtilities.cs @@ -18,6 +18,7 @@ using System.IO.Compression; using System.Linq; using System.Text; +using chibicc.toolchain.IO; namespace chibicc.toolchain.Archiving; @@ -35,7 +36,7 @@ private enum ObjectSymbolStates private static IEnumerable InternalEnumerateSymbolsFromObjectFile( Stream objectFileStream) { - var tr = new StreamReader(objectFileStream, Encoding.UTF8, true); + var tr = StreamUtilities.CreateTextReader(objectFileStream); var state = ObjectSymbolStates.Idle; string? currentDirective = null; @@ -141,7 +142,7 @@ public static void WriteSymbolTable( Stream symbolTableStream, SymbolList[] symbolLists) { - var tw = new StreamWriter(symbolTableStream, Encoding.UTF8); + var tw = StreamUtilities.CreateTextWriter(symbolTableStream); foreach (var symbolList in symbolLists) { @@ -167,7 +168,7 @@ public static IEnumerable EnumerateSymbolTable( if (archive.GetEntry(SymbolTableFileName) is { } entry) { using var stream = entry.Open(); - var tr = new StreamReader(stream, Encoding.UTF8, true); + var tr = StreamUtilities.CreateTextReader(stream); Token? currentObjectName = null; var symbols = new List(); diff --git a/toolchain.common/IO/StreamUtilities.cs b/toolchain.common/IO/StreamUtilities.cs index 2d48faa..3c5c4df 100644 --- a/toolchain.common/IO/StreamUtilities.cs +++ b/toolchain.common/IO/StreamUtilities.cs @@ -9,11 +9,14 @@ using System; using System.IO; +using System.Text; namespace chibicc.toolchain.IO; public static class StreamUtilities { + private static readonly Encoding utf8 = new UTF8Encoding(false); + public static Stream OpenStream(string path, bool writable) => (path == "-") ? (writable ? Console.OpenStandardOutput() : Console.OpenStandardInput()) : @@ -23,4 +26,10 @@ public static Stream OpenStream(string path, bool writable) => writable ? FileAccess.ReadWrite : FileAccess.Read, FileShare.Read, 1024 * 1024); + + public static TextReader CreateTextReader(Stream stream) => + new StreamReader(stream, utf8, true); + + public static TextWriter CreateTextWriter(Stream stream) => + new StreamWriter(stream, utf8); } From ac8f1f9210ac2f57c743eee5c61e72a91c8e22d3 Mon Sep 17 00:00:00 2001 From: Kouji Matsui Date: Sun, 5 Jan 2025 01:00:32 +0900 Subject: [PATCH 2/7] Improved archiver. --- .../ArchiverTests.Update.verified.txt | 1449 +++++++---------- chibiar/chibiar.core.Tests/ArchiverTests.cs | 87 +- chibiar/chibiar.core/Archiver.cs | 328 +--- .../Generating/ArchivedObjectInputFragment.cs | 2 +- chibild/chibild.core/Internal/Utilities.cs | 13 - .../Archiving/ArchiveObjectStream.cs | 62 - toolchain.common/Archiving/ArchiveReader.cs | 77 + .../Archiving/ArchiverUtilities.cs | 414 ++++- toolchain.common/IO/LittleEndianReader.cs | 116 ++ toolchain.common/IO/LittleEndianWriter.cs | 72 + toolchain.common/IO/NullStream.cs | 52 + toolchain.common/IO/RangeStream.cs | 86 + toolchain.common/IO/StreamUtilities.cs | 7 +- toolchain.common/Internal/CommonUtilities.cs | 16 + 14 files changed, 1440 insertions(+), 1341 deletions(-) delete mode 100644 toolchain.common/Archiving/ArchiveObjectStream.cs create mode 100644 toolchain.common/Archiving/ArchiveReader.cs create mode 100644 toolchain.common/IO/LittleEndianReader.cs create mode 100644 toolchain.common/IO/LittleEndianWriter.cs create mode 100644 toolchain.common/IO/NullStream.cs create mode 100644 toolchain.common/IO/RangeStream.cs diff --git a/chibiar/chibiar.core.Tests/ArchiverTests.Update.verified.txt b/chibiar/chibiar.core.Tests/ArchiverTests.Update.verified.txt index e472d36..be6e214 100644 --- a/chibiar/chibiar.core.Tests/ArchiverTests.Update.verified.txt +++ b/chibiar/chibiar.core.Tests/ArchiverTests.Update.verified.txt @@ -1632,826 +1632,531 @@ structure public Type 22 structure public Obj 21 .object codegen.o - global file _const_$_L928$__file__d___s__c_ - global file _L928$__file__d___s__c_ - global file _const_$_L925$_codegen - global file _L925$_codegen - global file _const_$_L924$_codegen - global file _L924$_codegen - global file _const_$_L923$___ret - global file _L923$___ret - global file _const_$_L922$__L_return_ - global file _L922$__L_return_ - global file _const_$_L921$____local__s - global file _L921$____local__s - global file _const_$_L920$____local__s__s - global file _L920$____local__s__s - global file _const_$_L917$____s - global file _L917$____s - global file _const_$_L916$_C_type___va_arglist - global file _L916$_C_type___va_arglist - global file _const_$_L915$__ - global file _L915$__ - global file _const_$_L914$___s__s - global file _L914$___s__s - global file _const_$_L913$__s__s - global file _L913$__s__s - global file _const_$_L910$___s_ - global file _L910$___s_ - global file _const_$_L909$__function_public - global file _L909$__function_public - global file _const_$_L908$__function_file - global file _L908$__function_file - global file _const_$_L905$_emit_text - global file _L905$_emit_text - global file _const_$_L904$_emit_text - global file _L904$_emit_text - global file _const_$_L903$___ret - global file _L903$___ret - global file _const_$_L902$__initializer_interna - global file _L902$__initializer_interna - global file _const_$_L901$___ret - global file _L901$___ret - global file _const_$_L900$__initializer_file - global file _L900$__initializer_file - global file _const_$_L899$__global_public__s___ - global file _L899$__global_public__s___ - global file _const_$_L898$__global_file__s___s - global file _L898$__global_file__s___s - global file _const_$_L897$__global_public___tls - global file _L897$__global_public___tls - global file _const_$_L896$__global_file___tls_s - global file _L896$__global_file___tls_s - global file _const_$_L895$___ - global file _L895$___ - global file _const_$_L894$__0x_hhx - global file _L894$__0x_hhx - global file _const_$_L891$__global_internal__s_ - global file _L891$__global_internal__s_ - global file _const_$_L890$__global_file__s__con - global file _L890$__global_file__s__con - global file _const_$_L887$_emit_data - global file _L887$_emit_data - global file _const_$_L886$_emit_data - global file _L886$_emit_data - global file _const_$_L885$___ret - global file _L885$___ret - global file _const_$_L884$___stobj__s - global file _L884$___stobj__s - global file _const_$_L883$___ldobj__s - global file _L883$___ldobj__s - global file _const_$_L882$___ldsflda__const___s - global file _L882$___ldsflda__const___s - global file _const_$_L881$___call___get_tls_val - global file _L881$___call___get_tls_val - global file _const_$_L880$___ldsfld___tls__s__ - global file _L880$___ldsfld___tls__s__ - global file _const_$_L879$__function_internal_v - global file _L879$__function_internal_v - global file _const_$_L878$__function_file_void_ - global file _L878$__function_file_void_ - global file _const_$_L877$_emit_tls_data_init - global file _L877$_emit_tls_data_init - global file _const_$_L876$_emit_tls_data_init - global file _L876$_emit_tls_data_init - global file _const_$_L875$___stobj__s - global file _L875$___stobj__s - global file _const_$_L874$___ldobj__s - global file _L874$___ldobj__s - global file _const_$_L873$___ldsflda__const___s - global file _L873$___ldsflda__const___s - global file _const_$_L872$___ldsfld__s - global file _L872$___ldsfld__s - global file _const_$_L871$_emit_non_tls_data_in - global file _L871$_emit_non_tls_data_in - global file _const_$_L870$_emit_non_tls_data_in - global file _L870$_emit_non_tls_data_in - global file _const_$_L869$__L_alloc__d_ - global file _L869$__L_alloc__d_ - global file _const_$_L868$___stsfld__s - global file _L868$___stsfld__s - global file _const_$_L867$___call_calloc - global file _L867$___call_calloc - global file _const_$_L866$___conv_u - global file _L866$___conv_u - global file _const_$_L865$___ldc_i8__ld - global file _L865$___ldc_i8__ld - global file _const_$_L864$___conv_u - global file _L864$___conv_u - global file _const_$_L863$___ldc_i4_1 - global file _L863$___ldc_i4_1 - global file _const_$_L862$___stsfld___tls__s__ - global file _L862$___stsfld___tls__s__ - global file _const_$_L861$___call___alloc_tls_s - global file _L861$___call___alloc_tls_s - global file _const_$_L860$___ldftn___tls_init__ - global file _L860$___ldftn___tls_init__ - global file _const_$_L859$___conv_u - global file _L859$___conv_u - global file _const_$_L858$___ldc_i8__ld - global file _L858$___ldc_i8__ld - global file _const_$_L857$___brfalse_s__L_alloc - global file _L857$___brfalse_s__L_alloc - global file _const_$_L856$___ceq - global file _L856$___ceq - global file _const_$_L855$___conv_u - global file _L855$___conv_u - global file _const_$_L854$___ldc_i4_0 - global file _L854$___ldc_i4_0 - global file _const_$_L853$___ldsfld__s - global file _L853$___ldsfld__s - global file _const_$_L852$___ldsfld___tls__s__ - global file _L852$___ldsfld___tls__s__ - global file _const_$_L851$_emit_data_alloc - global file _L851$_emit_data_alloc - global file _const_$_L850$_emit_data_alloc - global file _L850$_emit_data_alloc - global file _const_$_L843$_assign_lvar_offsets - global file _L843$_assign_lvar_offsets - global file _const_$_L842$_assign_lvar_offsets - global file _L842$_assign_lvar_offsets - global file _const_$_L841$___internal_uint8__ld - global file _L841$___internal_uint8__ld - global file _const_$_L840$_Could_not_adjust_on_ - global file _L840$_Could_not_adjust_on_ - global file _const_$_L839$___public__s__s_0 - global file _L839$___public__s__s_0 - global file _const_$_L836$__structure__s__s_exp - global file _L836$__structure__s__s_exp - global file _const_$_L834$_codegen_c - global file _L834$_codegen_c - global file _const_$_L833$_internal_error_at__s - global file _L833$_internal_error_at__s - global file _const_$_L832$___internal_uint8__ld - global file _L832$___internal_uint8__ld - global file _const_$_L831$_Could_not_adjust_on_ - global file _L831$_Could_not_adjust_on_ - global file _const_$_L830$_codegen_c - global file _L830$_codegen_c - global file _const_$_L829$_internal_error_at__s - global file _L829$_internal_error_at__s - global file _const_$_L828$___public__s__s - global file _L828$___public__s__s - global file _const_$_L827$___internal_uint8__ld - global file _L827$___internal_uint8__ld - global file _const_$_L826$_Could_not_adjust_on_ - global file _L826$_Could_not_adjust_on_ - global file _const_$_L825$___public__s__s - global file _L825$___public__s__s - global file _const_$_L824$___public_uint8___bit - global file _L824$___public_uint8___bit - global file _const_$_L823$___public_uint8___bit - global file _L823$___public_uint8___bit - global file _const_$_L818$__structure__s__s - global file _L818$__structure__s__s - global file _const_$_L816$____s__d - global file _L816$____s__d - global file _const_$_L813$__enumeration__s_int3 - global file _L813$__enumeration__s_int3 - global file _const_$_L810$_file - global file _L810$_file - global file _const_$_L809$_public - global file _L809$_public - global file _const_$_L806$_emit_type - global file _L806$_emit_type - global file _const_$_L805$_emit_type - global file _L805$_emit_type - global file _const_$_L804$___member_d - global file _L804$___member_d - global file _const_$_L803$_get_member_name - global file _L803$_get_member_name - global file _const_$_L802$_get_member_name - global file _L802$_get_member_name - global file _const_$_L801$_get_var_interior - global file _L801$_get_var_interior - global file _const_$_L800$_get_var_interior - global file _L800$_get_var_interior - global file _const_$_L798$_void - global file _L798$_void - global file _const_$_L796$__s_ - global file _L796$__s_ - global file _const_$_L793$_safe_to_cil_typename - global file _L793$_safe_to_cil_typename - global file _const_$_L792$_safe_to_cil_typename - global file _L792$_safe_to_cil_typename - global file _const_$_L785$_aggregate_types - global file _L785$_aggregate_types - global file _const_$_L784$_aggregate_types - global file _L784$_aggregate_types - global file _const_$_L783$_invalid_statement - global file _L783$_invalid_statement - global file _const_$_L782$____s - global file _L782$____s - global file _const_$_L779$___br__L_return - global file _L779$___br__L_return - global file _const_$_L777$__s_ - global file _L777$__s_ - global file _const_$_L775$___br__s - global file _L775$___br__s - global file _const_$_L770$__s_ - global file _L770$__s_ - global file _const_$_L768$__s_ - global file _L768$__s_ - global file _const_$_L767$___br__s - global file _L767$___br__s - global file _const_$_L766$___br__s - global file _L766$___br__s - global file _const_$_L765$___beq__s - global file _L765$___beq__s - global file _const_$_L764$___ldc_i4__d - global file _L764$___ldc_i4__d - global file _const_$_L762$___conv_i - global file _L762$___conv_i - global file _const_$_L761$___ldc_i8__ld - global file _L761$___ldc_i8__ld - global file _const_$_L759$___ldc_i8__ld - global file _L759$___ldc_i8__ld - global file _const_$_L753$__s_ - global file _L753$__s_ - global file _const_$_L752$___brfalse__L_begin__ - global file _L752$___brfalse__L_begin__ - global file _const_$_L751$__s_ - global file _L751$__s_ - global file _const_$_L750$__L_begin__d_ - global file _L750$__L_begin__d_ - global file _const_$_L748$__s_ - global file _L748$__s_ - global file _const_$_L747$___br__L_begin__d - global file _L747$___br__L_begin__d - global file _const_$_L746$__s_ - global file _L746$__s_ - global file _const_$_L745$___brtrue__s - global file _L745$___brtrue__s - global file _const_$_L744$__L_begin__d_ - global file _L744$__L_begin__d_ - global file _const_$_L743$_codegen_c - global file _L743$_codegen_c - global file _const_$_L742$_internal_error_at__s - global file _L742$_internal_error_at__s - global file _const_$_L740$__L_end__d_ - global file _L740$__L_end__d_ - global file _const_$_L739$__L_else__d_ - global file _L739$__L_else__d_ - global file _const_$_L738$___br__L_end__d - global file _L738$___br__L_end__d - global file _const_$_L737$___brtrue__L_else__d - global file _L737$___brtrue__L_else__d - global file _const_$_L734$_gen_stmt - global file _L734$_gen_stmt - global file _const_$_L733$_gen_stmt - global file _L733$_gen_stmt - global file _const_$_L732$_codegen_c - global file _L732$_codegen_c - global file _const_$_L731$_internal_error_at__s - global file _L731$_internal_error_at__s - global file _const_$_L730$___ldobj__s - global file _L730$___ldobj__s - global file _const_$_L729$___localloc - global file _L729$___localloc - global file _const_$_L728$___sizeof__s - global file _L728$___sizeof__s - global file _const_$_L724$___conv_i - global file _L724$___conv_i - global file _const_$_L723$___ldc_i4_0 - global file _L723$___ldc_i4_0 - global file _const_$_L718$___conv_r8 - global file _L718$___conv_r8 - global file _const_$_L717$___ldc_i4_0 - global file _L717$___ldc_i4_0 - global file _const_$_L715$___conv_r4 - global file _L715$___conv_r4 - global file _const_$_L714$___ldc_i4_0 - global file _L714$___ldc_i4_0 - global file _const_$_L712$___conv_i8 - global file _L712$___conv_i8 - global file _const_$_L711$___ldc_i4_0 - global file _L711$___ldc_i4_0 - global file _const_$_L709$___ldc_i4_0 - global file _L709$___ldc_i4_0 - global file _const_$_L702$_gen_dummy_value - global file _L702$_gen_dummy_value - global file _const_$_L701$_gen_dummy_value - global file _L701$_gen_dummy_value - global file _const_$_L700$_invalid_expression - global file _L700$_invalid_expression - global file _const_$_L699$___shr - global file _L699$___shr - global file _const_$_L698$___shr_un - global file _L698$___shr_un - global file _const_$_L696$___shl - global file _L696$___shl - global file _const_$_L694$___ceq - global file _L694$___ceq - global file _const_$_L693$___ldc_i4_0 - global file _L693$___ldc_i4_0 - global file _const_$_L692$___cgt - global file _L692$___cgt - global file _const_$_L691$___cgt_un - global file _L691$___cgt_un - global file _const_$_L689$___clt - global file _L689$___clt - global file _const_$_L688$___clt_un - global file _L688$___clt_un - global file _const_$_L686$___ceq - global file _L686$___ceq - global file _const_$_L685$___ldc_i4_0 - global file _L685$___ldc_i4_0 - global file _const_$_L684$___ceq - global file _L684$___ceq - global file _const_$_L682$___ceq - global file _L682$___ceq - global file _const_$_L680$___xor - global file _L680$___xor - global file _const_$_L678$___or - global file _L678$___or - global file _const_$_L676$___and - global file _L676$___and - global file _const_$_L674$___rem - global file _L674$___rem - global file _const_$_L673$___rem_un - global file _L673$___rem_un - global file _const_$_L671$___div - global file _L671$___div - global file _const_$_L670$___div_un - global file _L670$___div_un - global file _const_$_L668$___mul - global file _L668$___mul - global file _const_$_L666$___sub - global file _L666$___sub - global file _const_$_L664$___add - global file _L664$___add - global file _const_$_L660$__L_end__d_ - global file _L660$__L_end__d_ - global file _const_$_L659$___brfalse__L_end__d - global file _L659$___brfalse__L_end__d - global file _const_$_L658$__L_end__d_ - global file _L658$__L_end__d_ - global file _const_$_L657$___ldc_i4_1 - global file _L657$___ldc_i4_1 - global file _const_$_L656$__L_true__d_ - global file _L656$__L_true__d_ - global file _const_$_L655$___br_s__L_end__d - global file _L655$___br_s__L_end__d - global file _const_$_L654$___ldc_i4_0 - global file _L654$___ldc_i4_0 - global file _const_$_L653$___brfalse_s__L_true_ - global file _L653$___brfalse_s__L_true_ - global file _const_$_L652$___brfalse__L_true__d - global file _L652$___brfalse__L_true__d - global file _const_$_L650$__L_end__d_ - global file _L650$__L_end__d_ - global file _const_$_L649$___brtrue__L_end__d - global file _L649$___brtrue__L_end__d - global file _const_$_L648$__L_end__d_ - global file _L648$__L_end__d_ - global file _const_$_L647$___ldc_i4_0 - global file _L647$___ldc_i4_0 - global file _const_$_L646$__L_false__d_ - global file _L646$__L_false__d_ - global file _const_$_L645$___br_s__L_end__d - global file _L645$___br_s__L_end__d - global file _const_$_L644$___ldc_i4_1 - global file _L644$___ldc_i4_1 - global file _const_$_L643$___brtrue_s__L_false_ - global file _L643$___brtrue_s__L_false_ - global file _const_$_L642$___brtrue__L_false__d - global file _L642$___brtrue__L_false__d - global file _const_$_L640$___not - global file _L640$___not - global file _const_$_L637$__L_end__d_ - global file _L637$__L_end__d_ - global file _const_$_L636$__L_else__d_ - global file _L636$__L_else__d_ - global file _const_$_L635$___br__L_end__d - global file _L635$___br__L_end__d - global file _const_$_L634$___brtrue__L_else__d - global file _L634$___brtrue__L_else__d - global file _const_$_L632$___initobj__s - global file _L632$___initobj__s - global file _const_$_L625$___dup - global file _L625$___dup - global file _const_$_L624$___ldarg__d - global file _L624$___ldarg__d - global file _const_$_L623$___starg__d - global file _L623$___starg__d - global file _const_$_L621$___ldloc__d - global file _L621$___ldloc__d - global file _const_$_L620$___stloc__d - global file _L620$___stloc__d - global file _const_$_L612$___ldloc__s - global file _L612$___ldloc__s - global file _const_$_L611$___or - global file _L611$___or - global file _const_$_L610$___shl - global file _L610$___shl - global file _const_$_L609$___and - global file _L609$___and - global file _const_$_L608$___ldloc__s - global file _L608$___ldloc__s - global file _const_$_L607$___stloc__s - global file _L607$___stloc__s - global file _const_$_L606$___conv_i8 - global file _L606$___conv_i8 - global file _const_$_L605$___and - global file _L605$___and - global file _const_$_L604$___not - global file _L604$___not - global file _const_$_L603$___shl - global file _L603$___shl - global file _const_$_L602$___dup - global file _L602$___dup - global file _const_$_L597$___shr - global file _L597$___shr - global file _const_$_L596$___shr_un - global file _L596$___shr_un - global file _const_$_L595$___shl - global file _L595$___shl - global file _const_$_L594$___sub - global file _L594$___sub - global file _const_$_L592$___ldarg__d - global file _L592$___ldarg__d - global file _const_$_L590$___ldloc__d - global file _L590$___ldloc__d - global file _const_$_L581$___neg - global file _L581$___neg - global file _const_$_L579$_codegen_c - global file _L579$_codegen_c - global file _const_$_L578$_internal_error_at__s - global file _L578$_internal_error_at__s - global file _const_$_L577$___ldc_r8__f - global file _L577$___ldc_r8__f - global file _const_$_L575$___ldc_r4__f - global file _L575$___ldc_r4__f - global file _const_$_L561$_gen_expr - global file _L561$_gen_expr - global file _const_$_L560$_gen_expr - global file _L560$_gen_expr - global file _const_$_L559$____location__d__d__d - global file _L559$____location__d__d__d - global file _const_$_L558$_gen_location - global file _L558$_gen_location - global file _const_$_L557$_gen_location - global file _L557$_gen_location - global file _const_$_L556$_codegen_c - global file _L556$_codegen_c - global file _const_$_L555$_internal_error_at__s - global file _L555$_internal_error_at__s - global file _const_$_L554$___conv_i - global file _L554$___conv_i - global file _const_$_L553$___ldc_i4__ld - global file _L553$___ldc_i4__ld - global file _const_$_L552$___ldc_i4_m1 - global file _L552$___ldc_i4_m1 - global file _const_$_L551$___ldc_i4_s__ld - global file _L551$___ldc_i4_s__ld - global file _const_$_L550$___ldc_i4__ld - global file _L550$___ldc_i4__ld - global file _const_$_L549$___ldc_i8__ld - global file _L549$___ldc_i8__ld - global file _const_$_L546$___conv_i8 - global file _L546$___conv_i8 - global file _const_$_L545$___ldc_i4__ld - global file _L545$___ldc_i4__ld - global file _const_$_L544$___conv_i8 - global file _L544$___conv_i8 - global file _const_$_L543$___ldc_i4_m1 - global file _L543$___ldc_i4_m1 - global file _const_$_L542$___conv_i8 - global file _L542$___conv_i8 - global file _const_$_L541$___ldc_i4_s__ld - global file _L541$___ldc_i4_s__ld - global file _const_$_L540$___conv_i8 - global file _L540$___conv_i8 - global file _const_$_L539$___ldc_i4__ld - global file _L539$___ldc_i4__ld - global file _const_$_L538$___ldc_i8__ld - global file _L538$___ldc_i8__ld - global file _const_$_L536$___ldc_i4__d - global file _L536$___ldc_i4__d - global file _const_$_L535$___ldc_i4_m1 - global file _L535$___ldc_i4_m1 - global file _const_$_L534$___ldc_i4_s__d - global file _L534$___ldc_i4_s__d - global file _const_$_L533$___ldc_i4__d - global file _L533$___ldc_i4__d - global file _const_$_L528$___ldc_i4__d - global file _L528$___ldc_i4__d - global file _const_$_L525$_gen_const_integer - global file _L525$_gen_const_integer - global file _const_$_L524$_gen_const_integer - global file _L524$_gen_const_integer - global file _const_$_L523$___conv_u - global file _L523$___conv_u - global file _const_$_L522$___sizeof__s - global file _L522$___sizeof__s - global file _const_$_L521$_codegen_c - global file _L521$_codegen_c - global file _const_$_L520$_internal_error_at__s - global file _L520$_internal_error_at__s - global file _const_$_L519$___conv_u - global file _L519$___conv_u - global file _const_$_L518$___ldc_i4_0 - global file _L518$___ldc_i4_0 - global file _const_$_L514$_gen_sizeof - global file _L514$_gen_sizeof - global file _const_$_L513$_gen_sizeof - global file _L513$_gen_sizeof - global file _const_$_L512$___conv_r8 - global file _L512$___conv_r8 - global file _const_$_L510$___conv_r4 - global file _L510$___conv_r4 - global file _const_$_L508$___conv_i - global file _L508$___conv_i - global file _const_$_L507$___conv_u - global file _L507$___conv_u - global file _const_$_L504$___conv_i2 - global file _L504$___conv_i2 - global file _const_$_L503$___conv_u2 - global file _L503$___conv_u2 - global file _const_$_L501$___conv_i1 - global file _L501$___conv_i1 - global file _const_$_L500$___conv_u1 - global file _L500$___conv_u1 - global file _const_$_L498$___ceq - global file _L498$___ceq - global file _const_$_L497$___ldc_i4_0 - global file _L497$___ldc_i4_0 - global file _const_$_L496$___ceq - global file _L496$___ceq - global file _const_$_L495$___ldc_i4_0 - global file _L495$___ldc_i4_0 - global file _const_$_L494$___conv_u1 - global file _L494$___conv_u1 - global file _const_$_L491$___pop - global file _L491$___pop - global file _const_$_L490$___calli__s - global file _L490$___calli__s - global file _const_$_L489$___call__s - global file _L489$___call__s - global file _const_$_L486$___call___va_arglist_ - global file _L486$___call___va_arglist_ - global file _const_$_L485$___box__s - global file _L485$___box__s - global file _const_$_L484$___box_nint - global file _L484$___box_nint - global file _const_$_L483$___dup - global file _L483$___dup - global file _const_$_L480$___call___va_arglist_ - global file _L480$___call___va_arglist_ - global file _const_$_L479$___ldc_i4__d - global file _L479$___ldc_i4__d - global file _const_$_L472$___call___va_copy - global file _L472$___call___va_copy - global file _const_$_L471$_invalid_argument - global file _L471$_invalid_argument - global file _const_$_L470$___builtin_va_copy - global file _L470$___builtin_va_copy - global file _const_$_L469$___call___va_arg - global file _L469$___call___va_arg - global file _const_$_L468$_invalid_argument - global file _L468$_invalid_argument - global file _const_$_L467$_invalid_argument - global file _L467$_invalid_argument - global file _const_$_L466$_invalid_argument - global file _L466$_invalid_argument - global file _const_$_L465$_invalid_argument - global file _L465$_invalid_argument - global file _const_$_L464$___builtin_va_arg - global file _L464$___builtin_va_arg - global file _const_$_L463$___call___va_start - global file _L463$___call___va_start - global file _const_$_L462$___ldarg_s__d - global file _L462$___ldarg_s__d - global file _const_$_L461$_invalid_argument - global file _L461$_invalid_argument - global file _const_$_L460$___builtin_va_start - global file _L460$___builtin_va_start - global file _const_$_L459$___break - global file _L459$___break - global file _const_$_L458$_invalid_argument - global file _L458$_invalid_argument - global file _const_$_L457$___builtin_trap - global file _L457$___builtin_trap - global file _const_$_L456$___localloc - global file _L456$___localloc - global file _const_$_L455$_could_not_use_alloca - global file _L455$_could_not_use_alloca - global file _const_$_L454$_invalid_argument - global file _L454$_invalid_argument - global file _const_$_L453$_alloca - global file _L453$_alloca - global file _const_$_L452$_gen_funcall - global file _L452$_gen_funcall - global file _const_$_L451$_gen_funcall - global file _L451$_gen_funcall - global file _const_$_L450$___conv_r8 - global file _L450$___conv_r8 - global file _const_$_L449$____s - global file _L449$____s - global file _const_$_L448$___ceq - global file _L448$___ceq - global file _const_$_L447$___ldc_i4_0 - global file _L447$___ldc_i4_0 - global file _const_$_L446$_cast - global file _L446$_cast - global file _const_$_L445$_cast - global file _L445$_cast - global file cast_table - global file convr8 - global file convr4 - global file convrun - global file convnuint - global file convnint - global file convu8 - global file convi8 - global file convu4 - global file convi4 - global file convu2 - global file convi2 - global file convu1 - global file convi1 - global file _const_$_L433$_getTypeId - global file _L433$_getTypeId - global file _const_$_L432$_getTypeId - global file _L432$_getTypeId - global file _const_$_L431$___ceq - global file _L431$___ceq - global file _const_$_L430$___conv_r8 - global file _L430$___conv_r8 - global file _const_$_L428$___conv_r4 - global file _L428$___conv_r4 - global file _const_$_L426$___conv_i - global file _L426$___conv_i - global file _const_$_L423$___conv_i8 - global file _L423$___conv_i8 - global file _const_$_L420$___ldc_i4_0 - global file _L420$___ldc_i4_0 - global file _const_$_L419$_cmp_zero - global file _L419$_cmp_zero - global file _const_$_L418$_cmp_zero - global file _L418$_cmp_zero - global file _const_$_L417$_codegen_c - global file _L417$_codegen_c - global file _const_$_L416$_internal_error_at__s - global file _L416$_internal_error_at__s - global file _const_$_L415$___stind_r8 - global file _L415$___stind_r8 - global file _const_$_L413$___stind_r4 - global file _L413$___stind_r4 - global file _const_$_L411$___stind_i8 - global file _L411$___stind_i8 - global file _const_$_L409$___stind_i4 - global file _L409$___stind_i4 - global file _const_$_L406$___stind_i2 - global file _L406$___stind_i2 - global file _const_$_L404$___stind_i1 - global file _L404$___stind_i1 - global file _const_$_L401$___stind_i - global file _L401$___stind_i - global file _const_$_L398$___stobj__s - global file _L398$___stobj__s - global file _const_$_L393$_store - global file _L393$_store - global file _const_$_L392$_store - global file _L392$_store - global file _const_$_L391$_codegen_c - global file _L391$_codegen_c - global file _const_$_L390$_internal_error_at__s - global file _L390$_internal_error_at__s - global file _const_$_L389$___ldind__s4 - global file _L389$___ldind__s4 - global file _const_$_L387$___ldind__s2 - global file _L387$___ldind__s2 - global file _const_$_L385$___ldind__s1 - global file _L385$___ldind__s1 - global file _const_$_L382$_i - global file _L382$_i - global file _const_$_L381$_u - global file _L381$_u - global file _const_$_L380$___ldind_r8 - global file _L380$___ldind_r8 - global file _const_$_L378$___ldind_r4 - global file _L378$___ldind_r4 - global file _const_$_L376$___ldind_i - global file _L376$___ldind_i - global file _const_$_L373$___ldind_i8 - global file _L373$___ldind_i8 - global file _const_$_L371$___ldind_i4 - global file _L371$___ldind_i4 - global file _const_$_L369$___ldind_u1 - global file _L369$___ldind_u1 - global file _const_$_L367$___ldobj__s - global file _L367$___ldobj__s - global file _const_$_L360$_load - global file _L360$_load - global file _const_$_L359$_load - global file _L359$_load - global file _const_$_L358$___ldloc___ptr_d_ - global file _L358$___ldloc___ptr_d_ - global file _const_$_L357$___stloc___ptr_d_ - global file _L357$___stloc___ptr_d_ - global file _const_$_L356$____local__s____ptr_d - global file _L356$____local__s____ptr_d - global file _const_$_L355$_gen_make_ptr - global file _L355$_gen_make_ptr - global file _const_$_L354$_gen_make_ptr - global file _L354$_gen_make_ptr - global file _const_$_L353$_not_an_lvalue - global file _L353$_not_an_lvalue - global file _const_$_L352$___ldloca___retval_d_ - global file _L352$___ldloca___retval_d_ - global file _const_$_L351$___stloc___retval_d_ - global file _L351$___stloc___retval_d_ - global file _const_$_L350$____local__s___retval - global file _L350$____local__s___retval - global file _const_$_L348$___add - global file _L348$___add - global file _const_$_L344$_codegen_c - global file _L344$_codegen_c - global file _const_$_L343$_internal_error_at__s - global file _L343$_internal_error_at__s - global file _const_$_L342$___ldarga__d - global file _L342$___ldarga__d - global file _const_$_L340$___ldloca__d - global file _L340$___ldloca__d - global file _const_$_L338$___ldsfld__s - global file _L338$___ldsfld__s - global file _const_$_L337$___call___get_tls_val - global file _L337$___call___get_tls_val - global file _const_$_L336$___ldsfld___tls__s__ - global file _L336$___ldsfld___tls__s__ - global file _const_$_L333$___ldftn__s - global file _L333$___ldftn__s - global file _const_$_L328$_gen_addr - global file _L328$_gen_addr - global file _const_$_L327$_gen_addr - global file _L327$_gen_addr - global file _const_$_L326$____local__s__s - global file _L326$____local__s__s - global file _const_$_L325$___temp_d_ - global file _L325$___temp_d_ - global file _const_$_L324$_gen_make_temp - global file _L324$_gen_make_temp - global file _const_$_L323$_gen_make_temp - global file _L323$_gen_make_temp - global file _const_$_L322$__s__s_ - global file _L322$__s__s_ - global file _const_$_L321$__s_C_type___va_argli - global file _L321$__s_C_type___va_argli - global file _const_$_L320$_C_type___va_arglist - global file _L320$_C_type___va_arglist - global file _const_$_L319$__s__s - global file _L319$__s__s - global file _const_$_L316$_ - global file _L316$_ - global file _const_$_L315$_get_cil_callsite - global file _L315$_get_cil_callsite - global file _const_$_L314$_get_cil_callsite - global file _L314$_get_cil_callsite - global file _const_$_L313$_make_public_type - global file _L313$_make_public_type - global file _const_$_L312$_make_public_type - global file _L312$_make_public_type - global file _const_$_L301$_make_public_type_rec - global file _L301$_make_public_type_rec - global file _const_$_L300$_make_public_type_rec - global file _L300$_make_public_type_rec - global file _const_$_L299$_aggregate_type - global file _L299$_aggregate_type - global file _const_$_L298$_aggregate_type - global file _L298$_aggregate_type - global file _const_$_L286$_aggregate_type_recur - global file _L286$_aggregate_type_recur - global file _const_$_L285$_aggregate_type_recur - global file _L285$_aggregate_type_recur - global file _const_$_L282$_add_using_type - global file _L282$_add_using_type - global file _const_$_L281$_add_using_type - global file _L281$_add_using_type - global file _L280$_slvar - global file _const_$_L279$_count - global file _L279$_count - global file _const_$_L278$_count - global file _L278$_count - global file _const_$_L277$_print - global file _L277$_print - global file _const_$_L276$_print - global file _L276$_print - global file _const_$_L275$__ - global file _L275$__ - global file _const_$_L274$_println - global file _L274$_println - global file _const_$_L273$_println - global file _L273$_println - global file lvar_offset - global file using_type + global file _L590$_slvar + global file _const_$_L589$____ + global file _L589$____ + global file _const_$_L588$_tokenize_file + global file _L588$_tokenize_file + global file _const_$_L587$_tokenize_file + global file _L587$_tokenize_file + global file _const_$_L586$__U + global file _L586$__U + global file _const_$_L585$__u + global file _L585$__u + global file _const_$_L582$_convert_universal_ch + global file _L582$_convert_universal_ch + global file _const_$_L581$_convert_universal_ch + global file _L581$_convert_universal_ch + global file _const_$_L578$_read_universal_char + global file _L578$_read_universal_char + global file _const_$_L577$_read_universal_char + global file _L577$_read_universal_char + global file _const_$_L570$_remove_backslash_new + global file _L570$_remove_backslash_new + global file _const_$_L569$_remove_backslash_new + global file _L569$_remove_backslash_new + global file _const_$_L566$_canonicalize_newline + global file _L566$_canonicalize_newline + global file _const_$_L565$_canonicalize_newline + global file _L565$_canonicalize_newline + global file _const_$_L564$_new_file + global file _L564$_new_file + global file _const_$_L563$_new_file + global file _L563$_new_file + global file _const_$_L562$_get_input_files + global file _L562$_get_input_files + global file _const_$_L561$_get_input_files + global file _L561$_get_input_files + global file _const_$_L558$_r + global file _L558$_r + global file _const_$_L557$__ + global file _L557$__ + global file _const_$_L556$_read_file + global file _L556$_read_file + global file _const_$_L555$_read_file + global file _L555$_read_file + global file _const_$_L554$_invalid_token + global file _L554$_invalid_token + global file _const_$_L553$_U_ + global file _L553$_U_ + global file _const_$_L552$_L_ + global file _L552$_L_ + global file _const_$_L551$_u_ + global file _L551$_u_ + global file _const_$_L550$_U_ + global file _L550$_U_ + global file _const_$_L549$_L_ + global file _L549$_L_ + global file _const_$_L548$_u_ + global file _L548$_u_ + global file _const_$_L547$_u8_ + global file _L547$_u8_ + global file _const_$_L546$___ + global file _L546$___ + global file _const_$_L545$_eEpP + global file _L545$_eEpP + global file _const_$_L542$_unclosed_block_comme + global file _L542$_unclosed_block_comme + global file _const_$_L541$___ + global file _L541$___ + global file _const_$_L540$___ + global file _L540$___ + global file _const_$_L537$___ + global file _L537$___ + global file _const_$_L534$_tokenize + global file _L534$_tokenize + global file _const_$_L533$_tokenize + global file _L533$_tokenize + global file _const_$_L532$_tokenize_string_lite + global file _L532$_tokenize_string_lite + global file _const_$_L531$_tokenize_string_lite + global file _L531$_tokenize_string_lite + global file _const_$_L528$_add_line_numbers + global file _L528$_add_line_numbers + global file _const_$_L527$_add_line_numbers + global file _L527$_add_line_numbers + global file _const_$_L524$_convert_pp_tokens + global file _L524$_convert_pp_tokens + global file _const_$_L523$_convert_pp_tokens + global file _L523$_convert_pp_tokens + global file _const_$_L522$_invalid_numeric_cons + global file _L522$_invalid_numeric_cons + global file _const_$_L521$_convert_pp_number + global file _L521$_convert_pp_number + global file _const_$_L520$_convert_pp_number + global file _L520$_convert_pp_number + global file _const_$_L519$_ll + global file _L519$_ll + global file _const_$_L518$_LL + global file _L518$_LL + global file _const_$_L517$_ul + global file _L517$_ul + global file _const_$_L516$_lu + global file _L516$_lu + global file _const_$_L515$_ull + global file _L515$_ull + global file _const_$_L514$_uLL + global file _L514$_uLL + global file _const_$_L513$_Ull + global file _L513$_Ull + global file _const_$_L512$_ULL + global file _L512$_ULL + global file _const_$_L511$_llu + global file _L511$_llu + global file _const_$_L510$_llU + global file _L510$_llU + global file _const_$_L509$_LLu + global file _L509$_LLu + global file _const_$_L508$_LLU + global file _L508$_LLU + global file _const_$_L507$_0b + global file _L507$_0b + global file _const_$_L506$_0x + global file _L506$_0x + global file _const_$_L505$_convert_pp_int + global file _L505$_convert_pp_int + global file _const_$_L504$_convert_pp_int + global file _L504$_convert_pp_int + global file _const_$_L503$_unclosed_char_litera + global file _L503$_unclosed_char_litera + global file _const_$_L502$_unclosed_char_litera + global file _L502$_unclosed_char_litera + global file _const_$_L501$_read_char_literal + global file _L501$_read_char_literal + global file _const_$_L500$_read_char_literal + global file _L500$_read_char_literal + global file _const_$_L497$_read_utf32_string_li + global file _L497$_read_utf32_string_li + global file _const_$_L496$_read_utf32_string_li + global file _L496$_read_utf32_string_li + global file _const_$_L493$_read_utf16_string_li + global file _L493$_read_utf16_string_li + global file _const_$_L492$_read_utf16_string_li + global file _L492$_read_utf16_string_li + global file _const_$_L489$_read_string_literal + global file _L489$_read_string_literal + global file _const_$_L488$_read_string_literal + global file _L488$_read_string_literal + global file _const_$_L487$_unclosed_string_lite + global file _L487$_unclosed_string_lite + global file _const_$_L484$_string_literal_end + global file _L484$_string_literal_end + global file _const_$_L483$_string_literal_end + global file _L483$_string_literal_end + global file _const_$_L470$_invalid_hex_escape_s + global file _L470$_invalid_hex_escape_s + global file _const_$_L469$_read_escaped_char + global file _L469$_read_escaped_char + global file _const_$_L468$_read_escaped_char + global file _L468$_read_escaped_char + global file _const_$_L465$___thread + global file _L465$___thread + global file _const_$_L464$__Thread_local + global file _L464$__Thread_local + global file _const_$_L463$_asm + global file _L463$_asm + global file _const_$_L462$_typeof + global file _L462$_typeof + global file _const_$_L461$_double + global file _L461$_double + global file _const_$_L460$_float + global file _L460$_float + global file _const_$_L459$__Noreturn + global file _L459$__Noreturn + global file _const_$_L458$___restrict__ + global file _L458$___restrict__ + global file _const_$_L457$___restrict + global file _L457$___restrict + global file _const_$_L456$_restrict + global file _L456$_restrict + global file _const_$_L455$_register + global file _L455$_register + global file _const_$_L454$_auto + global file _L454$_auto + global file _const_$_L453$_volatile + global file _L453$_volatile + global file _const_$_L452$_const + global file _L452$_const + global file _const_$_L451$___builtin_nuint + global file _L451$___builtin_nuint + global file _const_$_L450$___builtin_nint + global file _L450$___builtin_nint + global file _const_$_L449$_unsigned + global file _L449$_unsigned + global file _const_$_L448$_signed + global file _L448$_signed + global file _const_$_L447$___builtin_va_list + global file _L447$___builtin_va_list + global file _const_$_L446$_do + global file _L446$_do + global file _const_$_L445$__Alignas + global file _L445$__Alignas + global file _const_$_L444$__Alignof + global file _L444$__Alignof + global file _const_$_L443$_extern + global file _L443$_extern + global file _const_$_L442$_default + global file _L442$_default + global file _const_$_L441$_case + global file _L441$_case + global file _const_$_L440$_switch + global file _L440$_switch + global file _const_$_L439$_continue + global file _L439$_continue + global file _const_$_L438$_break + global file _L438$_break + global file _const_$_L437$_goto + global file _L437$_goto + global file _const_$_L436$_static + global file _L436$_static + global file _const_$_L435$_enum + global file _L435$_enum + global file _const_$_L434$__Bool + global file _L434$__Bool + global file _const_$_L433$_typedef + global file _L433$_typedef + global file _const_$_L432$_void + global file _L432$_void + global file _const_$_L431$_long + global file _L431$_long + global file _const_$_L430$_short + global file _L430$_short + global file _const_$_L429$_union + global file _L429$_union + global file _const_$_L428$_struct + global file _L428$_struct + global file _const_$_L427$_char + global file _L427$_char + global file _const_$_L426$_sizeof + global file _L426$_sizeof + global file _const_$_L425$_int + global file _L425$_int + global file _const_$_L424$_while + global file _L424$_while + global file _const_$_L423$_for + global file _L423$_for + global file _const_$_L422$_else + global file _L422$_else + global file _const_$_L421$_if + global file _L421$_if + global file _const_$_L420$_return + global file _L420$_return + global file _const_$_L419$___thread + global file _L419$___thread + global file _const_$_L418$__Thread_local + global file _L418$__Thread_local + global file _const_$_L417$_asm + global file _L417$_asm + global file _const_$_L416$_typeof + global file _L416$_typeof + global file _const_$_L415$_double + global file _L415$_double + global file _const_$_L414$_float + global file _L414$_float + global file _const_$_L413$__Noreturn + global file _L413$__Noreturn + global file _const_$_L412$___restrict__ + global file _L412$___restrict__ + global file _const_$_L411$___restrict + global file _L411$___restrict + global file _const_$_L410$_restrict + global file _L410$_restrict + global file _const_$_L409$_register + global file _L409$_register + global file _const_$_L408$_auto + global file _L408$_auto + global file _const_$_L407$_volatile + global file _L407$_volatile + global file _const_$_L406$_const + global file _L406$_const + global file _const_$_L405$___builtin_nuint + global file _L405$___builtin_nuint + global file _const_$_L404$___builtin_nint + global file _L404$___builtin_nint + global file _const_$_L403$_unsigned + global file _L403$_unsigned + global file _const_$_L402$_signed + global file _L402$_signed + global file _const_$_L401$___builtin_va_list + global file _L401$___builtin_va_list + global file _const_$_L400$_do + global file _L400$_do + global file _const_$_L399$__Alignas + global file _L399$__Alignas + global file _const_$_L398$__Alignof + global file _L398$__Alignof + global file _const_$_L397$_extern + global file _L397$_extern + global file _const_$_L396$_default + global file _L396$_default + global file _const_$_L395$_case + global file _L395$_case + global file _const_$_L394$_switch + global file _L394$_switch + global file _const_$_L393$_continue + global file _L393$_continue + global file _const_$_L392$_break + global file _L392$_break + global file _const_$_L391$_goto + global file _L391$_goto + global file _const_$_L390$_static + global file _L390$_static + global file _const_$_L389$_enum + global file _L389$_enum + global file _const_$_L388$__Bool + global file _L388$__Bool + global file _const_$_L387$_typedef + global file _L387$_typedef + global file _const_$_L386$_void + global file _L386$_void + global file _const_$_L385$_long + global file _L385$_long + global file _const_$_L384$_short + global file _L384$_short + global file _const_$_L383$_union + global file _L383$_union + global file _const_$_L382$_struct + global file _L382$_struct + global file _const_$_L381$_char + global file _L381$_char + global file _const_$_L380$_sizeof + global file _L380$_sizeof + global file _const_$_L379$_int + global file _L379$_int + global file _const_$_L378$_while + global file _L378$_while + global file _const_$_L377$_for + global file _L377$_for + global file _const_$_L376$_else + global file _L376$_else + global file _const_$_L375$_if + global file _L375$_if + global file _const_$_L374$_return + global file _L374$_return + global file _L373$_slvar + global file _L372$_slvar + global file _const_$_L371$_is_keyword + global file _L371$_is_keyword + global file _const_$_L370$_is_keyword + global file _L370$_is_keyword + global file _const_$_L367$___ + global file _L367$___ + global file _const_$_L366$___ + global file _L366$___ + global file _const_$_L365$___ + global file _L365$___ + global file _const_$_L364$___ + global file _L364$___ + global file _const_$_L363$___ + global file _L363$___ + global file _const_$_L362$___ + global file _L362$___ + global file _const_$_L361$___ + global file _L361$___ + global file _const_$_L360$___ + global file _L360$___ + global file _const_$_L359$___ + global file _L359$___ + global file _const_$_L358$___ + global file _L358$___ + global file _const_$_L357$___ + global file _L357$___ + global file _const_$_L356$___ + global file _L356$___ + global file _const_$_L355$___ + global file _L355$___ + global file _const_$_L354$___ + global file _L354$___ + global file _const_$_L353$___ + global file _L353$___ + global file _const_$_L352$___ + global file _L352$___ + global file _const_$_L351$___ + global file _L351$___ + global file _const_$_L350$___ + global file _L350$___ + global file _const_$_L349$___ + global file _L349$___ + global file _const_$_L348$___ + global file _L348$___ + global file _const_$_L347$____ + global file _L347$____ + global file _const_$_L346$____ + global file _L346$____ + global file _const_$_L345$____ + global file _L345$____ + global file _const_$_L344$___ + global file _L344$___ + global file _const_$_L343$___ + global file _L343$___ + global file _const_$_L342$___ + global file _L342$___ + global file _const_$_L341$___ + global file _L341$___ + global file _const_$_L340$___ + global file _L340$___ + global file _const_$_L339$___ + global file _L339$___ + global file _const_$_L338$___ + global file _L338$___ + global file _const_$_L337$___ + global file _L337$___ + global file _const_$_L336$___ + global file _L336$___ + global file _const_$_L335$___ + global file _L335$___ + global file _const_$_L334$___ + global file _L334$___ + global file _const_$_L333$___ + global file _L333$___ + global file _const_$_L332$___ + global file _L332$___ + global file _const_$_L331$___ + global file _L331$___ + global file _const_$_L330$___ + global file _L330$___ + global file _const_$_L329$___ + global file _L329$___ + global file _const_$_L328$___ + global file _L328$___ + global file _const_$_L327$___ + global file _L327$___ + global file _const_$_L326$___ + global file _L326$___ + global file _const_$_L325$___ + global file _L325$___ + global file _const_$_L324$____ + global file _L324$____ + global file _const_$_L323$____ + global file _L323$____ + global file _const_$_L322$____ + global file _L322$____ + global file _L321$_slvar + global file _const_$_L320$_read_punct + global file _L320$_read_punct + global file _const_$_L319$_read_punct + global file _L319$_read_punct + global file _const_$_L318$_from_hex + global file _L318$_from_hex + global file _const_$_L317$_from_hex + global file _L317$_from_hex + global file _const_$_L314$_read_ident + global file _L314$_read_ident + global file _const_$_L313$_read_ident + global file _L313$_read_ident + global file _const_$_L312$_startswith + global file _L312$_startswith + global file _const_$_L311$_startswith + global file _L311$_startswith + global file _const_$_L310$_new_token + global file _L310$_new_token + global file _const_$_L309$_new_token + global file _L309$_new_token + global file _const_$_L308$_consume + global file _L308$_consume + global file _const_$_L307$_consume + global file _L307$_consume + global file _const_$_L306$_expected___s_ + global file _L306$_expected___s_ + global file _const_$_L305$_skip + global file _L305$_skip + global file _const_$_L304$_skip + global file _L304$_skip + global file _const_$_L303$_equal + global file _L303$_equal + global file _const_$_L302$_equal + global file _L302$_equal + global file _const_$_L301$_get_string + global file _L301$_get_string + global file _const_$_L300$_get_string + global file _L300$_get_string + global file _const_$_L299$_warn_tok + global file _L299$_warn_tok + global file _const_$_L298$_warn_tok + global file _L298$_warn_tok + global file _const_$_L297$_error_tok + global file _L297$_error_tok + global file _const_$_L296$_error_tok + global file _L296$_error_tok + global file _const_$_L293$_error_at + global file _L293$_error_at + global file _const_$_L292$_error_at + global file _L292$_error_at + global file _const_$_L291$__ + global file _L291$__ + global file _const_$_L290$___ + global file _L290$___ + global file _const_$_L289$_ + global file _L289$_ + global file _const_$_L288$___s + global file _L288$___s + global file _const_$_L287$____s_ + global file _L287$____s_ + global file _const_$_L286$__s__d__ + global file _L286$__s__d__ + global file _const_$_L281$_verror_at + global file _L281$_verror_at + global file _const_$_L280$_verror_at + global file _L280$_verror_at + global file _const_$_L279$__ + global file _L279$__ + global file _const_$_L278$_error + global file _L278$_error + global file _const_$_L277$_error + global file _L277$_error + global file has_space + global file at_bol + global file input_files + global file current_file + global file _L276$_disp + global file _L275$_disp + global file _L274$_disp + global file _L273$_disp global file _L272$_disp global file _L271$_disp global file _L270$_disp global file _L269$_disp global file _L268$_disp - global file current_fn - global file output_file global file _L267$_disp global file _L266$_disp global file _L265$_disp @@ -2724,66 +2429,64 @@ global file _L2$_disp global file _L1$_disp global file _L0$_disp - function public codegen - function file emit_text - function file emit_data - function file emit_tls_data_init - function file emit_non_tls_data_init - function file emit_data_alloc - function file assign_lvar_offsets - function file emit_type - function file get_member_name - function file get_var_interior - function file safe_to_cil_typename - function file aggregate_types - function file gen_stmt - function file gen_dummy_value - function file gen_expr - function file gen_location - function file gen_const_integer - function file gen_sizeof - function file gen_funcall - function file cast - function file getTypeId - function file cmp_zero - function file store - function file load - function file gen_make_ptr - function file gen_addr - function file gen_make_temp - function file get_cil_callsite - function file make_public_type - function file make_public_type_recursive - function file aggregate_type - function file aggregate_type_recursive - function file add_using_type - function file count - function file print - function file println + function public tokenize_file + function file convert_universal_chars + function file read_universal_char + function file remove_backslash_newline + function file canonicalize_newline + function public new_file + function public get_input_files + function file read_file + function public tokenize + function public tokenize_string_literal + function file add_line_numbers + function public convert_pp_tokens + function file convert_pp_number + function file convert_pp_int + function file read_char_literal + function file read_utf32_string_literal + function file read_utf16_string_literal + function file read_string_literal + function file string_literal_end + function file read_escaped_char + function file is_keyword + function file read_punct + function file from_hex + function file read_ident + function file startswith + function file new_token + function public consume + function public skip + function public equal + function public get_string + function public warn_tok + function public error_tok + function public error_at + function file verror_at + function public error function file WIFSIGNALED function file WIFEXITED structure public glob_t 4 - structure public tm 9 + structure public _tm_$20 9 structure public timespec 2 structure public stat 3 - enumeration public MemoryModel 3 - structure public UsingType 2 - enumeration public TypeId 12 - enumeration public AfterStmt 3 - structure public FILE 0 - structure public StringArray 3 - enumeration public ObjKind 4 - structure public EnumMember 3 - structure public VarScope 7 - structure public Scope 3 - structure public TagScope 4 - structure public Member 11 - structure public Hideset 0 - structure public File 5 - enumeration public TokenKind 7 - structure public Token 17 - enumeration public NodeKind 44 - structure public Node 31 - enumeration public TypeKind 16 - structure public Type 22 - structure public Obj 21 + structure public _tag_$19 5 + structure public _tag_$18 3 + enumeration public _MemoryModel_$17 3 + enumeration public _TokenKind_$16 7 + structure public _HashEntry_$15 3 + structure public _HashMap_$14 3 + structure public _FILE_$13 0 + structure public _Hideset_$12 0 + structure public _File_$11 6 + structure public _tag_$10 3 + structure public _StringArray_$9 3 + enumeration public _ObjKind_$8 4 + structure public _tag_$7 21 + structure public _tag_$6 11 + enumeration public _NodeKind_$5 44 + structure public _tag_$4 33 + enumeration public _TypeKind_$3 16 + structure public _tag_$2 23 + enumeration public _TokenKind_$1 7 + structure public _tag_$0 17 diff --git a/chibiar/chibiar.core.Tests/ArchiverTests.cs b/chibiar/chibiar.core.Tests/ArchiverTests.cs index 809160f..53285ed 100644 --- a/chibiar/chibiar.core.Tests/ArchiverTests.cs +++ b/chibiar/chibiar.core.Tests/ArchiverTests.cs @@ -8,15 +8,11 @@ ///////////////////////////////////////////////////////////////////////////////////// using chibicc.toolchain.Archiving; +using chibicc.toolchain.IO; using chibicc.toolchain.Logging; using NUnit.Framework; -using System; using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Text; using System.Threading.Tasks; -using chibicc.toolchain.IO; using static VerifyNUnit.Verifier; using static chibiar.ArchiverTestRunner; @@ -26,17 +22,18 @@ namespace chibiar; [Parallelizable(ParallelScope.All)] public sealed class ArchiverTests { - private static async Task VerifySymbolTableAsync(ZipArchive zip) + private static async Task VerifySymbolTableAsync(ArchiveReader archiveReader) { - var symTableEntry = zip.GetEntry(ArchiverUtilities.SymbolTableFileName)!; - using var afs = symTableEntry.Open(); + Assert.That( + archiveReader.TryOpenObjectStream(ArchiverUtilities.SymbolTableFileName, true, out var afs), + Is.True); - var tr = StreamUtilities.CreateTextReader(afs); + using var tr = StreamUtilities.CreateTextReader(afs); var actual = await tr.ReadToEndAsync(); await Verify(actual); } - + [Test] public Task ArchiveOne() { @@ -45,56 +42,52 @@ public Task ArchiveOne() logger.Information($"Test runner BasePath={basePath}"); var archivePath = Path.Combine(basePath, "output.a"); - + var archiver = new Archiver(logger); var actual = archiver.AddOrUpdate( archivePath, - new[] - { - Path.Combine(ArtifactsBasePath, "parse.o"), - }, + [ Path.Combine(ArtifactsBasePath, "parse.o"), ], true, false); - + Assert.That(actual, Is.True); - using var zip = ZipFile.OpenRead(archivePath); - + var archiveReader = new ArchiveReader(archivePath); + Assert.That( - zip.Entries.Select(e => e.Name), - Is.EqualTo(new[] { "parse.o", ArchiverUtilities.SymbolTableFileName })); + archiveReader.ObjectNames, + Is.EqualTo(new[] { ArchiverUtilities.SymbolTableFileName, "parse.o", })); - await VerifySymbolTableAsync(zip); + await VerifySymbolTableAsync(archiveReader); }); } - + [Test] public Task ArchiveTwo() { return RunAsync(async (basePath, logger) => { var archivePath = Path.Combine(basePath, "output.a"); - + var archiver = new Archiver(logger); var actual = archiver.AddOrUpdate( archivePath, - new[] - { + [ Path.Combine(ArtifactsBasePath, "parse.o"), Path.Combine(ArtifactsBasePath, "codegen.o"), - }, + ], true, false); - + Assert.That(actual, Is.True); - using var zip = ZipFile.OpenRead(archivePath); - + var archiveReader = new ArchiveReader(archivePath); + Assert.That( - zip.Entries.Select(e => e.Name), - Is.EqualTo(new[] { "parse.o", "codegen.o", ArchiverUtilities.SymbolTableFileName })); + archiveReader.ObjectNames, + Is.EqualTo(new[] { ArchiverUtilities.SymbolTableFileName, "parse.o", "codegen.o" })); - await VerifySymbolTableAsync(zip); + await VerifySymbolTableAsync(archiveReader); }); } @@ -104,45 +97,41 @@ public Task Update() return RunAsync(async (basePath, logger) => { var archivePath = Path.Combine(basePath, "output.a"); - + var newCodegenPath = Path.Combine(basePath, "codegen.o"); - + // It's made for dummy updated codegen.o. File.Copy( Path.Combine(ArtifactsBasePath, "tokenize.o"), newCodegenPath); - + var archiver = new Archiver(logger); var actual1 = archiver.AddOrUpdate( archivePath, - new[] - { + [ Path.Combine(ArtifactsBasePath, "parse.o"), Path.Combine(ArtifactsBasePath, "codegen.o"), - }, + ], true, false); - + Assert.That(actual1, Is.True); var actual2 = archiver.AddOrUpdate( archivePath, - new[] - { - newCodegenPath, - }, + [ newCodegenPath, ], true, false); - + Assert.That(actual2, Is.False); - using var zip = ZipFile.OpenRead(archivePath); - + var archiveReader = new ArchiveReader(archivePath); + Assert.That( - zip.Entries.Select(e => e.Name), - Is.EqualTo(new[] { "parse.o", "codegen.o", ArchiverUtilities.SymbolTableFileName })); + archiveReader.ObjectNames, + Is.EqualTo(new[] { ArchiverUtilities.SymbolTableFileName, "parse.o", "codegen.o", })); - await VerifySymbolTableAsync(zip); + await VerifySymbolTableAsync(archiveReader); }); } } diff --git a/chibiar/chibiar.core/Archiver.cs b/chibiar/chibiar.core/Archiver.cs index 1c9f95e..44e59ef 100644 --- a/chibiar/chibiar.core/Archiver.cs +++ b/chibiar/chibiar.core/Archiver.cs @@ -7,18 +7,15 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using chibiar.Cli; +using chibicc.toolchain.Archiving; +using chibicc.toolchain.IO; +using chibicc.toolchain.Logging; using System; using System.Collections.Generic; using System.IO; -using System.IO.Compression; using System.Linq; -using System.Text; using System.Threading.Tasks; -using chibiar.Cli; -using chibicc.toolchain.Archiving; -using chibicc.toolchain.Internal; -using chibicc.toolchain.IO; -using chibicc.toolchain.Logging; namespace chibiar; @@ -29,244 +26,51 @@ public sealed class Archiver public Archiver(ILogger logger) => this.logger = logger; - private static bool IsSourceFile(string path) => - Path.GetExtension(path) is not ".o"; - - private static Stream OpenObjectStreamToCompressed(string objectFilePath) - { - var ofs = StreamUtilities.OpenStream(objectFilePath, false); - - return IsSourceFile(objectFilePath) ? - new GZipStream(ofs, CompressionLevel.Optimal) : ofs; - } - - private static string GetObjectName(string objectFilePath) => - IsSourceFile(objectFilePath) ? - (Path.GetFileNameWithoutExtension(objectFilePath) + ".o") : - Path.GetFileName(objectFilePath); - - private static bool TryAddOrUpdateObjectFiles( - string archiveFilePath, - string outputArchiveFilePath, - string[] objectFilePaths, - bool isDryrun, - out string[] outputObjectNames) - { - if (!isDryrun) - { - var objectNames = objectFilePaths. - Select(GetObjectName). - ToArray(); - - using var outputArchive = ZipFile.Open( - outputArchiveFilePath, - ZipArchiveMode.Create, - Encoding.UTF8); - - using var readArchive = File.Exists(archiveFilePath) ? - ZipFile.Open( - archiveFilePath, - ZipArchiveMode.Read, - Encoding.UTF8) : - null; - - var outputObjectNameList = new List(); - var isReplaced = new bool[objectFilePaths.Length]; - - foreach (var readEntry in - readArchive?.Entries.AsEnumerable() ?? - CommonUtilities.Empty()) - { - if (readEntry.Name != ArchiverUtilities.SymbolTableFileName) - { - outputObjectNameList.Add(readEntry.Name); - - var index = Array.IndexOf(objectNames, readEntry.Name); - if (index >= 0) - { - var objectFilePath = objectFilePaths[index]; - var objectLastWriteTime = File.GetLastWriteTime(objectFilePath); - - if (objectLastWriteTime > readEntry.LastWriteTime) - { - var objectName = objectNames[index]; - - var outputEntry = outputArchive.CreateEntry( - objectName, CompressionLevel.NoCompression); - - outputEntry.LastWriteTime = objectLastWriteTime; - - using var outputStream = outputEntry.Open(); - using var readStream = OpenObjectStreamToCompressed(objectFilePath); - - readStream.CopyTo(outputStream); - outputStream.Flush(); - - isReplaced[index] = true; - continue; - } - } - - if (readEntry.Name != ArchiverUtilities.SymbolTableFileName) - { - var outputEntry = outputArchive.CreateEntry( - readEntry.Name, CompressionLevel.NoCompression); - - outputEntry.LastWriteTime = readEntry.LastWriteTime; - - using var outputStream = outputEntry.Open(); - using var readStream = readEntry.Open(); - - readStream.CopyTo(outputStream); - outputStream.Flush(); - } - } - } - - for (var index = 0; index < objectFilePaths.Length; index++) - { - if (!isReplaced[index]) - { - var objectFilePath = objectFilePaths[index]; - var objectLastWriteTime = File.GetLastWriteTime(objectFilePath); - - var objectName = objectNames[index]; - outputObjectNameList.Add(objectName); - - var outputEntry = outputArchive.CreateEntry( - objectName, CompressionLevel.NoCompression); - - outputEntry.LastWriteTime = objectLastWriteTime; - - using var outputStream = outputEntry.Open(); - using var readStream = OpenObjectStreamToCompressed(objectFilePath); - - readStream.CopyTo(outputStream); - outputStream.Flush(); - } - } - - outputObjectNames = outputObjectNameList. - ToArray(); - return readArchive == null; - } - else - { - outputObjectNames = objectFilePaths. - Select(GetObjectName). - ToArray(); - return false; - } - } - - private static SymbolList[] GetSymbolLists( - string archiveFilePath, - string[] objectNames, - bool isDryrun) - { - var symbolLists = new SymbolList[objectNames.Length]!; - - if (!isDryrun || File.Exists(archiveFilePath)) - { - Parallel.ForEach(objectNames, - new() { MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount - 1) }, - (objectName, _, index) => - { - if (ArchiverUtilities.TryOpenArchivedObject( - archiveFilePath, - objectName, - true, - out var stream)) - { - using var _s = stream; - - var symbols = ArchiverUtilities.EnumerateSymbolsFromObjectFile(stream). - ToArray(); - - symbolLists[index] = new(objectName, symbols); - } - }); - } - else - { - for (var index = 0; index < symbolLists.Length; index++) - { - symbolLists[index] = new(objectNames[index], CommonUtilities.Empty()); - } - } - - return symbolLists; - } - - private static void AddSymbolTable( - string archiveFilePath, - SymbolList[] symbolLists, - bool isDryrun) - { - if (!isDryrun) - { - using var archive = ZipFile.Open( - archiveFilePath, - ZipArchiveMode.Update, - Encoding.UTF8); - - var symbolTableEntry = archive.CreateEntry( - ArchiverUtilities.SymbolTableFileName, - CompressionLevel.Optimal); - - using var outputStream = symbolTableEntry.Open(); - - ArchiverUtilities.WriteSymbolTable(outputStream, symbolLists); - } - else - { - ArchiverUtilities.WriteSymbolTable(new MemoryStream(), symbolLists); - } - } - internal bool AddOrUpdate( string archiveFilePath, string[] objectFilePaths, - bool isCreateSymbolTable, + bool writeSymbolTable, bool isDryrun) { - var outputArchiveFilePath = $"{archiveFilePath}_{Guid.NewGuid():N}"; + var outputArchiveFilePath = Path.Combine( + Path.GetTempPath(), + $"chibiar_{Path.GetFileNameWithoutExtension(archiveFilePath)}_{Guid.NewGuid():N}{Path.GetExtension(archiveFilePath)}"); try { - if (TryAddOrUpdateObjectFiles( + var isExistArchiveFile = File.Exists(archiveFilePath); + + var symbolListEntries = ArchiverUtilities.GetCombinedSymbolListEntries( archiveFilePath, - outputArchiveFilePath, - objectFilePaths, - isDryrun, - out var outputObjectNames)) + objectFilePaths); + + using (var outputArchiveFileStream = isDryrun ? + new NullStream() : + StreamUtilities.OpenStream(outputArchiveFilePath, true)) { - var symbolLists = GetSymbolLists( - outputArchiveFilePath, - outputObjectNames, - isDryrun); + ArchiverUtilities.WriteArchive( + outputArchiveFileStream, + symbolListEntries, + archiveFilePath, + writeSymbolTable); + } - if (isCreateSymbolTable) - { - AddSymbolTable( - outputArchiveFilePath, - symbolLists, - isDryrun); - } - + if (!isDryrun) + { File.Delete(archiveFilePath); File.Move(outputArchiveFilePath, archiveFilePath); - - return true; } + + return !isExistArchiveFile; } catch { - File.Delete(outputArchiveFilePath); + if (!isDryrun) + { + File.Delete(outputArchiveFilePath); + } throw; } - - return false; } internal void Extract( @@ -274,30 +78,34 @@ internal void Extract( string[] objectNames, bool isDryrun) { - if (!isDryrun || File.Exists(archiveFilePath)) + var hashedObjectNames = new HashSet(objectNames); + var descriptors = ArchiverUtilities.LoadArchivedObjectItemDescriptors( + archiveFilePath, aod => hashedObjectNames.Contains(aod.ObjectName) ? aod : null). + ToArray(); + + Parallel.ForEach(descriptors, + descriptor => + { + var aod = (ArchivedObjectItemDescriptor)descriptor; + + using var archiveFileStream = StreamUtilities.OpenStream(archiveFilePath, false); + + archiveFileStream.Position = aod.Position; + var objectStream = new RangedStream( + archiveFileStream, aod.Length, false); + + using var outputObjectFileStream = isDryrun ? + new NullStream() : + StreamUtilities.OpenStream(aod.ObjectName, true); + objectStream.CopyTo(outputObjectFileStream); + outputObjectFileStream.Flush(); + }); + + foreach (var exceptName in descriptors. + Select(aod => aod!.ObjectName). + Except(objectNames)) { - Parallel.ForEach(objectNames, - new() { MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount - 1) }, - objectName => - { - if (ArchiverUtilities.TryOpenArchivedObject( - archiveFilePath, - objectName, - false, - out var inputStream)) - { - using var outputStream = isDryrun ? - new MemoryStream() : - StreamUtilities.OpenStream(objectName, true); - - inputStream.CopyTo(outputStream); - outputStream.Flush(); - } - else - { - this.logger.Error($"Object is not found: {objectName}"); - } - }); + this.logger.Error($"Object is not found: {exceptName}"); } } @@ -305,24 +113,14 @@ internal void List( string archiveFilePath, string[] objectNames) { - if (objectNames.Length >= 1) - { - var existObjectNames = new HashSet( - ArchiverUtilities.EnumerateArchivedObjectNames(archiveFilePath)); + var hashedObjectNames = new HashSet(objectNames); + var descriptors = ArchiverUtilities.LoadArchivedObjectItemDescriptors( + archiveFilePath, aod => hashedObjectNames.Contains(aod.ObjectName) ? aod : null). + ToArray(); - foreach (var objectName in objectNames. - Where(existObjectNames.Contains)) - { - Console.WriteLine(objectName); - } - } - else + foreach (var descriptor in descriptors) { - foreach (var objectName in - ArchiverUtilities.EnumerateArchivedObjectNames(archiveFilePath)) - { - Console.WriteLine(objectName); - } + Console.WriteLine(descriptor.ObjectName); } } diff --git a/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs b/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs index 1b8b843..de5243e 100644 --- a/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs +++ b/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs @@ -238,7 +238,7 @@ public static ArchivedObjectInputFragment[] Load( { logger.Information($"Loading symbol table: {relativePath}"); - var symbolLists = ArchiverUtilities.EnumerateSymbolTable( + var symbolLists = ArchiverUtilities.EnumerateSymbolListFromArchive( Path.Combine(baseInputPath, relativePath)); return symbolLists.Select(symbolList => diff --git a/chibild/chibild.core/Internal/Utilities.cs b/chibild/chibild.core/Internal/Utilities.cs index f457e51..9ac0f37 100644 --- a/chibild/chibild.core/Internal/Utilities.cs +++ b/chibild/chibild.core/Internal/Utilities.cs @@ -56,19 +56,6 @@ public static bool TryAdd( } #endif -#if NET45 || NET461 - public static IEnumerable Prepend( - this IEnumerable enumerable, - T value) - { - yield return value; - foreach (var item in enumerable) - { - yield return item; - } - } -#endif - #if NET45 || NET461 || NETSTANDARD2_0 public static HashSet ToHashSet(this IEnumerable enumerable) => new(enumerable); diff --git a/toolchain.common/Archiving/ArchiveObjectStream.cs b/toolchain.common/Archiving/ArchiveObjectStream.cs deleted file mode 100644 index 6707664..0000000 --- a/toolchain.common/Archiving/ArchiveObjectStream.cs +++ /dev/null @@ -1,62 +0,0 @@ -///////////////////////////////////////////////////////////////////////////////////// -// -// chibicc-toolchain - The specialized backend toolchain for chibicc-cil -// Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) -// -// Licensed under MIT: https://opensource.org/licenses/MIT -// -///////////////////////////////////////////////////////////////////////////////////// - -using System; -using System.IO; -using System.IO.Compression; - -namespace chibicc.toolchain.Archiving; - -internal sealed class ArchiveObjectStream : Stream -{ - // Automatic closer for parent ZipArchive. - - private readonly ZipArchive archive; - private readonly Stream parent; - - public ArchiveObjectStream(ZipArchive archive, Stream parent) - { - this.archive = archive; - this.parent = parent; - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - this.parent.Dispose(); - this.archive.Dispose(); - } - - public override bool CanRead => true; - public override bool CanSeek => false; - public override bool CanWrite => false; - public override long Length => throw new NotImplementedException(); - - public override long Position - { - get => throw new NotImplementedException(); - set => throw new NotImplementedException(); - } - - public override void Flush() - { - } - - public override int Read(byte[] buffer, int offset, int count) => - this.parent.Read(buffer, offset, count); - - public override long Seek(long offset, SeekOrigin origin) => - throw new System.NotImplementedException(); - - public override void SetLength(long value) => - throw new System.NotImplementedException(); - - public override void Write(byte[] buffer, int offset, int count) => - throw new System.NotImplementedException(); -} diff --git a/toolchain.common/Archiving/ArchiveReader.cs b/toolchain.common/Archiving/ArchiveReader.cs new file mode 100644 index 0000000..7126fab --- /dev/null +++ b/toolchain.common/Archiving/ArchiveReader.cs @@ -0,0 +1,77 @@ +///////////////////////////////////////////////////////////////////////////////////// +// +// chibicc-toolchain - The specialized backend toolchain for chibicc-cil +// Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) +// +// Licensed under MIT: https://opensource.org/licenses/MIT +// +///////////////////////////////////////////////////////////////////////////////////// + +using chibicc.toolchain.IO; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; + +namespace chibicc.toolchain.Archiving; + +public sealed class ArchiveReader +{ + private readonly string archiveFilePath; + private readonly Dictionary aods; + + public ArchiveReader( + string archiveFilePath) + { + this.archiveFilePath = archiveFilePath; + var descriptors = ArchiverUtilities.LoadArchivedObjectItemDescriptors( + this.archiveFilePath, aod => aod); + this.ObjectNames = descriptors. + Select(d => d.ObjectName). + ToArray(); + this.aods = descriptors. + ToDictionary(d => d.ObjectName, d => (ArchivedObjectItemDescriptor)d); + } + + public ArchiveReader( + string archiveFilePath, + IEnumerable objectNames) + { + this.archiveFilePath = archiveFilePath; + var hashedObjectNames = new HashSet(objectNames); + var descriptors = ArchiverUtilities.LoadArchivedObjectItemDescriptors( + this.archiveFilePath, aod => hashedObjectNames.Contains(aod.ObjectName) ? aod : null); + this.ObjectNames = descriptors. + Select(d => d.ObjectName). + ToArray(); + this.aods = descriptors. + ToDictionary(d => d.ObjectName, d => (ArchivedObjectItemDescriptor)d); + } + + public IReadOnlyList ObjectNames { get; } + + public bool TryOpenObjectStream( + string objectName, + bool decodeBody, + out Stream stream) + { + if (!this.aods.TryGetValue(objectName, out var aod)) + { + stream = null!; + return false; + } + + var archiveFileStream = StreamUtilities.OpenStream( + this.archiveFilePath, + false); + archiveFileStream.Position = aod.Position; + var rangedStream = new RangedStream( + archiveFileStream, + aod.Length, + false); + stream = decodeBody ? + new GZipStream(rangedStream, CompressionMode.Decompress) : + rangedStream; + return true; + } +} diff --git a/toolchain.common/Archiving/ArchiverUtilities.cs b/toolchain.common/Archiving/ArchiverUtilities.cs index 1cfa8e5..1621c77 100644 --- a/toolchain.common/Archiving/ArchiverUtilities.cs +++ b/toolchain.common/Archiving/ArchiverUtilities.cs @@ -7,25 +7,74 @@ // ///////////////////////////////////////////////////////////////////////////////////// -using System; using chibicc.toolchain.Internal; +using chibicc.toolchain.IO; using chibicc.toolchain.Parsing; using chibicc.toolchain.Tokenizing; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.IO; using System.IO.Compression; using System.Linq; -using System.Text; -using chibicc.toolchain.IO; +using System.Threading.Tasks; namespace chibicc.toolchain.Archiving; +public interface IObjectItemDescriptor +{ + int Length { get; } + string ObjectName { get; } +} + +public sealed class ArchivedObjectItemDescriptor : IObjectItemDescriptor +{ + public long Position { get; private set; } + public int Length { get; } + public string ObjectName { get; } + + internal ArchivedObjectItemDescriptor(long relativePosition, int length, string objectName) + { + this.Position = relativePosition; + this.Length = length; + this.ObjectName = objectName; + } + + internal void AdjustPosition(long basePosition) => + this.Position += basePosition; +} + +public sealed class ObjectFileDescriptor : IObjectItemDescriptor +{ + public int Length { get; } + public string Path { get; } + public string ObjectName => + ArchiverUtilities.GetObjectName(this.Path); + + internal ObjectFileDescriptor(int length, string path) + { + this.Length = length; + this.Path = path; + } +} + +public record struct SymbolListEntry( + IObjectItemDescriptor Descriptor, + SymbolList SymbolList); + public static class ArchiverUtilities { public static readonly string SymbolTableFileName = "__.SYMDEF"; + public static bool IsSourceFile(string path) => + Path.GetExtension(path) is not ".o"; + + public static string GetObjectName(string objectFilePath) => + IsSourceFile(objectFilePath) ? + (Path.GetFileNameWithoutExtension(objectFilePath) + ".o") : + Path.GetFileName(objectFilePath); + private enum ObjectSymbolStates { Idle, @@ -33,7 +82,7 @@ private enum ObjectSymbolStates Structure, } - private static IEnumerable InternalEnumerateSymbolsFromObjectFile( + private static IEnumerable InternalEnumerateSymbolsFromObjectFileStream( Stream objectFileStream) { var tr = StreamUtilities.CreateTextReader(objectFileStream); @@ -133,47 +182,306 @@ tokens[0] is (TokenTypes.Identity, _)) } } - public static IEnumerable EnumerateSymbolsFromObjectFile( + private static IEnumerable EnumerateSymbolsFromObjectFileStream( Stream objectFileStream) => - InternalEnumerateSymbolsFromObjectFile(objectFileStream). + InternalEnumerateSymbolsFromObjectFileStream(objectFileStream). Distinct(); + + //////////////////////////////////////////////////////////////////// + + private static Stream ToCompressedObjectStream( + Stream objectFileStream, + string referenceObjectFilePath) + { + if (!IsSourceFile(referenceObjectFilePath)) + { + return objectFileStream; + } + var gzs = new GZipStream(objectFileStream, CompressionLevel.Optimal); + var ms = new MemoryStream(); + gzs.CopyTo(ms); + objectFileStream.Dispose(); + ms.Position = 0; + return ms; + } + + private static Stream OpenCompressedObjectStream(string objectFilePath) + { + var ofs = StreamUtilities.OpenStream(objectFilePath, false); + return ToCompressedObjectStream(ofs, objectFilePath); + } + + private static Stream ToDecompressedObjectStream( + Stream objectFileStream, + string referenceObjectFilePath) + { + if (IsSourceFile(referenceObjectFilePath)) + { + return objectFileStream; + } + var gzs = new GZipStream(objectFileStream, CompressionMode.Decompress); + var ms = new MemoryStream(); + gzs.CopyTo(ms); + objectFileStream.Dispose(); + ms.Position = 0; + return ms; + } + + private static Stream OpenDecompressedObjectStream(string objectFilePath) + { + var ofs = StreamUtilities.OpenStream(objectFilePath, false); + return ToDecompressedObjectStream(ofs, objectFilePath); + } + + //////////////////////////////////////////////////////////////////// + + private static IEnumerable EnumerateArchivedObjectItemDescriptors( + Stream archiveFileStream) + { + var ler = new LittleEndianReader(archiveFileStream); + var relativePosition = 0; + while (true) + { + if (!ler.TryReadString(256, out var objectName)) + { + throw new FormatException( + $"Invalid object naming: Position={ler.Position?.ToString() ?? "(Unknown)"}"); + } + if (objectName.Length == 0) + { + // Detected termination. + break; + } + if (!ler.TryReadInt32(out var length)) + { + throw new FormatException( + $"Invalid object length: Position={ler.Position?.ToString() ?? "(Unknown)"}"); + } + yield return new(relativePosition, length, objectName); + relativePosition += length; + } + } + + public static IObjectItemDescriptor[] LoadArchivedObjectItemDescriptors( + string archiveFilePath, + Func selector) + { + using var archiveFileStream = StreamUtilities.OpenStream(archiveFilePath, false); + var descriptors = EnumerateArchivedObjectItemDescriptors(archiveFileStream). + Select(aod => selector(aod)!). + Where(d => d != null). + ToArray(); + var archiveFileBasePosition = archiveFileStream.Position; + foreach (var descriptor in descriptors) + { + if (descriptor is ArchivedObjectItemDescriptor aod) + { + aod.AdjustPosition(archiveFileBasePosition); + } + } + return descriptors; + } + + public static SymbolListEntry[] GetCombinedSymbolListEntries( + string archiveFilePath, + string[] objectFilePaths) + { + var archivedObjectItems = CommonUtilities.Empty(); + + if (File.Exists(archiveFilePath)) + { + var hashedObjectNames = objectFilePaths.ToDictionary(GetObjectName); + archivedObjectItems = LoadArchivedObjectItemDescriptors( + archiveFilePath, + aod => hashedObjectNames.TryGetValue(aod.ObjectName, out var path) ? + new ObjectFileDescriptor(-1, path) : // Will update object file + aod.ObjectName == SymbolTableFileName ? + null : // Will ignore symbol table + aod); // Archived object file + } + + var willAddObjectFileItems = objectFilePaths. + Except(archivedObjectItems. + OfType(). + Select(d => d.Path)). + Select(path => new ObjectFileDescriptor(-1, path)). + ToArray(); + + var symbolListEntries = new SymbolListEntry[ + archivedObjectItems.Length + willAddObjectFileItems.Length]; - public static void WriteSymbolTable( + Parallel.ForEach( + archivedObjectItems.Concat(willAddObjectFileItems), + (entry, _, index) => + { + switch (entry) + { + case ObjectFileDescriptor ofd: + using (var objectRawStream = StreamUtilities.OpenStream(ofd.Path, false)) + { + var length = objectRawStream.Length; + var objectStream = ToDecompressedObjectStream(objectRawStream, ofd.Path); + var symbols = EnumerateSymbolsFromObjectFileStream(objectStream). + ToArray(); + symbolListEntries[index] = new( + new ObjectFileDescriptor((int)length, ofd.Path), + new(ofd.ObjectName, symbols)); + } + break; + case ArchivedObjectItemDescriptor aod: + using (var archiveFileStream = StreamUtilities.OpenStream(archiveFilePath!, false)) + { + archiveFileStream.Position = aod.Position; + var objectStream = ToDecompressedObjectStream( + new RangedStream(archiveFileStream, aod.Length, false), + aod.ObjectName); + var symbols = EnumerateSymbolsFromObjectFileStream(objectStream). + ToArray(); + symbolListEntries[index] = new(aod, new(aod.ObjectName, symbols)); + } + break; + } + }); + + return symbolListEntries; + } + + //////////////////////////////////////////////////////////////////// + + private static void WriteSymbolTable( Stream symbolTableStream, - SymbolList[] symbolLists) + IEnumerable symbolLists) { var tw = StreamUtilities.CreateTextWriter(symbolTableStream); - foreach (var symbolList in symbolLists) { - tw.WriteLine($".object {symbolList.ObjectName}"); - + tw.WriteLine( + $".object {symbolList.ObjectName}"); foreach (var symbol in symbolList.Symbols.Distinct()) { - tw.WriteLine($" {symbol.Directive} {symbol.Scope} {symbol.Name}{(symbol.MemberCount is { } mc ? $" {mc}" : "")}"); + tw.WriteLine( + $" {symbol.Directive} {symbol.Scope} {symbol.Name}{(symbol.MemberCount is { } mc ? $" {mc}" : "")}"); } } - tw.Flush(); } - public static IEnumerable EnumerateSymbolTable( + private static byte[] CreateSymbolTableImage( + IEnumerable symbolListEntries) + { + var symbolTableStream = new MemoryStream(); + using (var compressedStream = new GZipStream(symbolTableStream, CompressionLevel.Fastest)) + { + WriteSymbolTable( + compressedStream, + symbolListEntries.Select(entry => entry.SymbolList)); + compressedStream.Flush(); + } + return symbolTableStream.ToArray(); + } + + //////////////////////////////////////////////////////////////////// + + private static void WriteObjectItemDescriptors( + Stream outputArchiveFileStream, + IEnumerable descriptors) + { + var lew = new LittleEndianWriter(outputArchiveFileStream); + foreach (var descriptor in descriptors) + { + lew.Write(descriptor.ObjectName); + lew.Write(descriptor.Length); + } + lew.Write(string.Empty); // Termination + } + + private static void WriteObjectItemBodies( + Stream outputArchiveFileStream, + string readArchiveFilePath, + IEnumerable descriptors) + { + foreach (var descriptor in descriptors) + { + switch (descriptor) + { + case ObjectFileDescriptor ofd: + using (var objectFileStream = OpenCompressedObjectStream(ofd.Path)) + { + objectFileStream.CopyTo(outputArchiveFileStream); + } + break; + case ArchivedObjectItemDescriptor aod: + using (var archiveFileStream = StreamUtilities.OpenStream(readArchiveFilePath, false)) + { + archiveFileStream.Position = aod.Position; + var objectStream = new RangedStream(archiveFileStream, aod.Length, false); + objectStream.CopyTo(outputArchiveFileStream); + } + break; + } + } + } + + private sealed class SymbolTableDescriptor(int length) : IObjectItemDescriptor + { + public int Length => + length; + public string ObjectName => + SymbolTableFileName; + } + + public static void WriteArchive( + Stream outputArchiveFileStream, + SymbolListEntry[] symbolListEntries, + string readArchiveFilePath, + bool writeSymbolTable) + { + if (writeSymbolTable) + { + var symbolTableStream = new MemoryStream( + CreateSymbolTableImage(symbolListEntries)); + + WriteObjectItemDescriptors( + outputArchiveFileStream, + symbolListEntries. + Select(entry => entry.Descriptor). + Prepend(new SymbolTableDescriptor((int)symbolTableStream.Length))); + + symbolTableStream.CopyTo(outputArchiveFileStream); + } + else + { + WriteObjectItemDescriptors( + outputArchiveFileStream, + symbolListEntries.Select(entry => entry.Descriptor)); + } + + WriteObjectItemBodies( + outputArchiveFileStream, + readArchiveFilePath, + symbolListEntries.Select(entry => entry.Descriptor)); + + outputArchiveFileStream.Flush(); + } + + //////////////////////////////////////////////////////////////////// + + public static IEnumerable EnumerateSymbolListFromArchive( string archiveFilePath) { - using var archive = ZipFile.Open( - archiveFilePath, - ZipArchiveMode.Read, - Encoding.UTF8); + var archiveReader = new ArchiveReader( + archiveFilePath, [ SymbolTableFileName ]); - if (archive.GetEntry(SymbolTableFileName) is { } entry) + if (archiveReader.TryOpenObjectStream( + SymbolTableFileName, true, out var symbolTableStream)) { - using var stream = entry.Open(); - var tr = StreamUtilities.CreateTextReader(stream); + var tr = StreamUtilities.CreateTextReader(symbolTableStream); Token? currentObjectName = null; var symbols = new List(); - - foreach (var tokens in CilTokenizer.TokenizeAll("", SymbolTableFileName, tr). + + foreach (var tokens in CilTokenizer.TokenizeAll( + "", SymbolTableFileName, tr). Where(tokens => tokens.Length >= 2)) { switch (tokens[0]) @@ -194,12 +502,10 @@ when tokens[1] is (TokenTypes.Identity, _): // enumeration public enumbaz 3 // structure public structhoge 5 case (TokenTypes.Identity, var directive) - when tokens.Length >= 3 && - tokens[1] is (TokenTypes.Identity, var scope) && - CommonUtilities.TryParseEnum(scope, out _) && - tokens[2] is (TokenTypes.Identity, var name): - if (tokens.Length >= 4 && - tokens[3] is (TokenTypes.Identity, var mc) && + when tokens is [_, (TokenTypes.Identity, var scope), _, ..] && + CommonUtilities.TryParseEnum(scope, out _) && + tokens[2] is (TokenTypes.Identity, var name): + if (tokens is [_, _, _, (TokenTypes.Identity, var mc), ..] && int.TryParse(mc, NumberStyles.Integer, CultureInfo.InvariantCulture, out var memberCount) && memberCount >= 0) { @@ -212,7 +518,6 @@ tokens[3] is (TokenTypes.Identity, var mc) && break; } } - if (currentObjectName is var (_, con2)) { yield return new( @@ -222,54 +527,15 @@ tokens[3] is (TokenTypes.Identity, var mc) && } } - public static IEnumerable EnumerateArchivedObjectNames( - string archiveFilePath) - { - using var archive = ZipFile.Open( - archiveFilePath, - ZipArchiveMode.Read, - Encoding.UTF8); - - foreach (var entry in archive.Entries. - Where(entry => entry.Name != SymbolTableFileName)) - { - yield return entry.Name; - } - } - public static bool TryOpenArchivedObject( string archiveFilePath, string objectName, - bool is_decoded_body, + bool decodedBody, out Stream stream) { - var archive = ZipFile.Open( - archiveFilePath, - ZipArchiveMode.Read, - Encoding.UTF8); - - try - { - if (archive.GetEntry(objectName) is { } entry) - { - var ofs = is_decoded_body ? - new GZipStream(entry.Open(), CompressionMode.Decompress) : - entry.Open(); - - stream = new ArchiveObjectStream(archive, ofs); - return true; - } - else - { - archive.Dispose(); - stream = null!; - return false; - } - } - catch - { - archive.Dispose(); - throw; - } + var archiveReader = new ArchiveReader( + archiveFilePath, [ objectName ]); + return archiveReader.TryOpenObjectStream( + objectName, decodedBody, out stream); } } diff --git a/toolchain.common/IO/LittleEndianReader.cs b/toolchain.common/IO/LittleEndianReader.cs new file mode 100644 index 0000000..f98ce33 --- /dev/null +++ b/toolchain.common/IO/LittleEndianReader.cs @@ -0,0 +1,116 @@ +///////////////////////////////////////////////////////////////////////////////////// +// +// chibicc-toolchain - The specialized backend toolchain for chibicc-cil +// Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) +// +// Licensed under MIT: https://opensource.org/licenses/MIT +// +///////////////////////////////////////////////////////////////////////////////////// + +using chibicc.toolchain.Internal; +using System.Collections.Generic; +using System.IO; + +namespace chibicc.toolchain.IO; + +internal sealed class LittleEndianReader +{ + private readonly Stream parent; + + public LittleEndianReader(Stream parent) => + this.parent = parent; + + public long? Position => + this.parent.CanSeek ? this.parent.Position : null; + + private int ReadByte() => + this.parent.ReadByte(); + + public bool TryReadByte(out byte value) + { + var ch = this.ReadByte(); + if (ch == -1) + { + value = 0; + return false; + } + value = (byte)ch; + return true; + } + + public bool TryReadInt16(out short value) + { + ushort v = 0; + for (var i = 0; i < sizeof(short); i++) + { + var ch = this.ReadByte(); + if (ch == -1) + { + value = 0; + return false; + } + v |= (ushort)(((ushort)ch) << (i * 8)); + } + value = (short)v; + return true; + } + + public bool TryReadInt32(out int value) + { + uint v = 0; + for (var i = 0; i < sizeof(int); i++) + { + var ch = this.ReadByte(); + if (ch == -1) + { + value = 0; + return false; + } + v |= (uint)(((uint)ch) << (i * 8)); + } + value = (int)v; + return true; + } + + public bool TryReadInt64(out long value) + { + ulong v = 0; + for (var i = 0; i < sizeof(long); i++) + { + var ch = this.ReadByte(); + if (ch == -1) + { + value = 0; + return false; + } + v |= (ulong)(((ulong)ch) << (i * 8)); + } + value = (long)v; + return true; + } + + public bool TryReadString(int maxLength, out string value) + { + if (maxLength >= 1) + { + var buffer = new List(maxLength); + while (buffer.Count < maxLength) + { + var ch = this.parent.ReadByte(); + if (ch == -1) + { + value = null!; + return false; + } + if (ch == 0) + { + value = CommonUtilities.UTF8.GetString(buffer.ToArray()); + return true; + } + buffer.Add((byte)ch); + } + } + value = null!; + return false; + } +} diff --git a/toolchain.common/IO/LittleEndianWriter.cs b/toolchain.common/IO/LittleEndianWriter.cs new file mode 100644 index 0000000..a7c6eb5 --- /dev/null +++ b/toolchain.common/IO/LittleEndianWriter.cs @@ -0,0 +1,72 @@ +///////////////////////////////////////////////////////////////////////////////////// +// +// chibicc-toolchain - The specialized backend toolchain for chibicc-cil +// Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) +// +// Licensed under MIT: https://opensource.org/licenses/MIT +// +///////////////////////////////////////////////////////////////////////////////////// + +using chibicc.toolchain.Internal; +using System.IO; + +namespace chibicc.toolchain.IO; + +internal sealed class LittleEndianWriter +{ + private readonly Stream parent; + private readonly byte[] buffer = new byte[8]; + + public LittleEndianWriter(Stream parent) => + this.parent = parent; + + public void Write(byte value) + { + this.buffer[0] = value; + this.parent.Write(this.buffer, 0, sizeof(byte)); + } + + public void Write(short value) + { + for (var i = 0; i < sizeof(short); i++) + { + this.buffer[i] = (byte)value; + value >>= 8; + } + this.parent.Write(this.buffer, 0, sizeof(short)); + } + + public void Write(int value) + { + for (var i = 0; i < sizeof(int); i++) + { + this.buffer[i] = (byte)value; + value >>= 8; + } + this.parent.Write(this.buffer, 0, sizeof(int)); + } + + public void Write(long value) + { + for (var i = 0; i < sizeof(long); i++) + { + this.buffer[i] = (byte)value; + value >>= 8; + } + this.parent.Write(this.buffer, 0, sizeof(long)); + } + + public void Write(string value) + { + var bytes = CommonUtilities.UTF8.GetBytes(value); + this.parent.Write(bytes, 0, bytes.Length); + this.buffer[0] = 0; + this.parent.Write(this.buffer, 0, sizeof(byte)); + } + + public void Write(byte[] buffer, int offset, int length) => + this.parent.Write(buffer, offset, length); + + public void Flush() => + this.parent.Flush(); +} diff --git a/toolchain.common/IO/NullStream.cs b/toolchain.common/IO/NullStream.cs new file mode 100644 index 0000000..0936c0d --- /dev/null +++ b/toolchain.common/IO/NullStream.cs @@ -0,0 +1,52 @@ +///////////////////////////////////////////////////////////////////////////////////// +// +// chibicc-toolchain - The specialized backend toolchain for chibicc-cil +// Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) +// +// Licensed under MIT: https://opensource.org/licenses/MIT +// +///////////////////////////////////////////////////////////////////////////////////// + +using System; +using System.IO; + +namespace chibicc.toolchain.IO; + +internal sealed class NullStream : Stream +{ + public NullStream() + { + } + + public override bool CanRead => + true; + public override bool CanSeek => + false; + public override bool CanWrite => + true; + + public override long Length => + throw new InvalidOperationException(); + + public override long Position + { + get => throw new InvalidOperationException(); + set => throw new InvalidOperationException(); + } + + public override int Read(byte[] buffer, int offset, int count) => + 0; + + public override void Write(byte[] buffer, int offset, int count) + { + } + + public override void Flush() + { + } + + public override long Seek(long offset, SeekOrigin origin) => + throw new InvalidOperationException(); + public override void SetLength(long value) => + throw new InvalidOperationException(); +} diff --git a/toolchain.common/IO/RangeStream.cs b/toolchain.common/IO/RangeStream.cs new file mode 100644 index 0000000..2ed03b0 --- /dev/null +++ b/toolchain.common/IO/RangeStream.cs @@ -0,0 +1,86 @@ +///////////////////////////////////////////////////////////////////////////////////// +// +// chibicc-toolchain - The specialized backend toolchain for chibicc-cil +// Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) +// +// Licensed under MIT: https://opensource.org/licenses/MIT +// +///////////////////////////////////////////////////////////////////////////////////// + +using System; +using System.IO; +using System.Threading; + +namespace chibicc.toolchain.IO; + +internal sealed class RangedStream : Stream +{ + private readonly bool leaveOpen; + private Stream parent; + private long position; + + public RangedStream(Stream parent, long length, bool leaveOpen) + { + this.parent = parent; + this.Length = length; + this.leaveOpen = leaveOpen; + } + + public override bool CanRead => + true; + public override bool CanSeek => + false; + public override bool CanWrite => + false; + + public override long Length { get; } + public override long Position + { + get => this.position; + set => throw new InvalidOperationException(); + } + + public override void Close() + { + if (!this.leaveOpen) + { + if (Interlocked.Exchange(ref this.parent, null!) is { } parent) + { + parent.Dispose(); + } + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + this.Close(); + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + var remains = (int)Math.Min(count, this.Length - this.position); + if (remains == 0) + { + return 0; + } + + var read = this.parent.Read(buffer, offset, remains); + + this.position += read; + return read; + } + + public override void Flush() + { + } + + public override long Seek(long offset, SeekOrigin origin) => + throw new InvalidOperationException(); + public override void SetLength(long value) => + throw new InvalidOperationException(); + public override void Write(byte[] buffer, int offset, int count) => + throw new InvalidOperationException(); +} diff --git a/toolchain.common/IO/StreamUtilities.cs b/toolchain.common/IO/StreamUtilities.cs index 3c5c4df..9b7a4c5 100644 --- a/toolchain.common/IO/StreamUtilities.cs +++ b/toolchain.common/IO/StreamUtilities.cs @@ -10,13 +10,12 @@ using System; using System.IO; using System.Text; +using chibicc.toolchain.Internal; namespace chibicc.toolchain.IO; public static class StreamUtilities { - private static readonly Encoding utf8 = new UTF8Encoding(false); - public static Stream OpenStream(string path, bool writable) => (path == "-") ? (writable ? Console.OpenStandardOutput() : Console.OpenStandardInput()) : @@ -28,8 +27,8 @@ public static Stream OpenStream(string path, bool writable) => 1024 * 1024); public static TextReader CreateTextReader(Stream stream) => - new StreamReader(stream, utf8, true); + new StreamReader(stream, CommonUtilities.UTF8, true); public static TextWriter CreateTextWriter(Stream stream) => - new StreamWriter(stream, utf8); + new StreamWriter(stream, CommonUtilities.UTF8); } diff --git a/toolchain.common/Internal/CommonUtilities.cs b/toolchain.common/Internal/CommonUtilities.cs index 13f238b..9caf6cc 100644 --- a/toolchain.common/Internal/CommonUtilities.cs +++ b/toolchain.common/Internal/CommonUtilities.cs @@ -10,6 +10,7 @@ using System; using System.Globalization; using System.IO; +using System.Text; #if NET45 || NET461 using System.Collections.Generic; @@ -52,6 +53,8 @@ internal static class CommonUtilities { private static readonly IFormatProvider invariantCulture = CultureInfo.InvariantCulture; + public static readonly Encoding UTF8 = new UTF8Encoding(false); + public static readonly bool IsInWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; @@ -73,6 +76,19 @@ public static T[] Empty() => Array.Empty(); #endif +#if NET40 || NET45 || NET461 + public static IEnumerable Prepend( + this IEnumerable enumerable, + T value) + { + yield return value; + foreach (var item in enumerable) + { + yield return item; + } + } +#endif + public static bool TryParseUInt8( string word, out byte value) => From 71e43779994ec6e695d62292d2032d841bc18209 Mon Sep 17 00:00:00 2001 From: Kouji Matsui Date: Sun, 5 Jan 2025 12:16:10 +0900 Subject: [PATCH 3/7] Refactored. --- chibiar/chibiar.core/Archiver.cs | 56 ++-- .../chibiar.core/Archiving/ArchiveWriter.cs | 222 ++++++++++++++ .../Archiving/ArchiverUtilities.cs | 271 +++++------------- 3 files changed, 314 insertions(+), 235 deletions(-) create mode 100644 chibiar/chibiar.core/Archiving/ArchiveWriter.cs diff --git a/chibiar/chibiar.core/Archiver.cs b/chibiar/chibiar.core/Archiver.cs index 44e59ef..bb33182 100644 --- a/chibiar/chibiar.core/Archiver.cs +++ b/chibiar/chibiar.core/Archiver.cs @@ -7,12 +7,12 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using chibiar.Archiving; using chibiar.Cli; using chibicc.toolchain.Archiving; using chibicc.toolchain.IO; using chibicc.toolchain.Logging; using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -40,7 +40,7 @@ internal bool AddOrUpdate( { var isExistArchiveFile = File.Exists(archiveFilePath); - var symbolListEntries = ArchiverUtilities.GetCombinedSymbolListEntries( + var symbolListEntries = ArchiveWriter.GetCombinedSymbolListEntries( archiveFilePath, objectFilePaths); @@ -48,11 +48,13 @@ internal bool AddOrUpdate( new NullStream() : StreamUtilities.OpenStream(outputArchiveFilePath, true)) { - ArchiverUtilities.WriteArchive( + ArchiveWriter.WriteArchive( outputArchiveFileStream, symbolListEntries, archiveFilePath, writeSymbolTable); + + outputArchiveFileStream.Flush(); } if (!isDryrun) @@ -78,34 +80,35 @@ internal void Extract( string[] objectNames, bool isDryrun) { - var hashedObjectNames = new HashSet(objectNames); - var descriptors = ArchiverUtilities.LoadArchivedObjectItemDescriptors( - archiveFilePath, aod => hashedObjectNames.Contains(aod.ObjectName) ? aod : null). - ToArray(); + var archiveReader = new ArchiveReader(archiveFilePath, objectNames); + var read = objectNames.ToDictionary(objectName => objectName, _ => false); - Parallel.ForEach(descriptors, - descriptor => + Parallel.ForEach( + archiveReader.ObjectNames, + objectName => { - var aod = (ArchivedObjectItemDescriptor)descriptor; - - using var archiveFileStream = StreamUtilities.OpenStream(archiveFilePath, false); - - archiveFileStream.Position = aod.Position; - var objectStream = new RangedStream( - archiveFileStream, aod.Length, false); - + if (!archiveReader.TryOpenObjectStream(objectName, false, out var objectStream)) + { + throw new ArgumentException( + $"Could not extract an object: Path={archiveFilePath}, Name={objectName}"); + } + + using var _ = objectStream; using var outputObjectFileStream = isDryrun ? new NullStream() : - StreamUtilities.OpenStream(aod.ObjectName, true); + StreamUtilities.OpenStream(objectName, true); objectStream.CopyTo(outputObjectFileStream); outputObjectFileStream.Flush(); + + lock (read) + { + read[objectName] = true; + } }); - foreach (var exceptName in descriptors. - Select(aod => aod!.ObjectName). - Except(objectNames)) + foreach (var entry in read.Where(entry => !entry.Value)) { - this.logger.Error($"Object is not found: {exceptName}"); + this.logger.Error($"Object is not found: {entry.Key}"); } } @@ -113,14 +116,11 @@ internal void List( string archiveFilePath, string[] objectNames) { - var hashedObjectNames = new HashSet(objectNames); - var descriptors = ArchiverUtilities.LoadArchivedObjectItemDescriptors( - archiveFilePath, aod => hashedObjectNames.Contains(aod.ObjectName) ? aod : null). - ToArray(); + var archiveReader = new ArchiveReader(archiveFilePath, objectNames); - foreach (var descriptor in descriptors) + foreach (var objectItem in archiveReader.ObjectNames) { - Console.WriteLine(descriptor.ObjectName); + Console.WriteLine(objectItem); } } diff --git a/chibiar/chibiar.core/Archiving/ArchiveWriter.cs b/chibiar/chibiar.core/Archiving/ArchiveWriter.cs new file mode 100644 index 0000000..ba6cb01 --- /dev/null +++ b/chibiar/chibiar.core/Archiving/ArchiveWriter.cs @@ -0,0 +1,222 @@ +///////////////////////////////////////////////////////////////////////////////////// +// +// chibicc-toolchain - The specialized backend toolchain for chibicc-cil +// Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) +// +// Licensed under MIT: https://opensource.org/licenses/MIT +// +///////////////////////////////////////////////////////////////////////////////////// + +using chibicc.toolchain.Archiving; +using chibicc.toolchain.Internal; +using chibicc.toolchain.IO; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; + +namespace chibiar.Archiving; + +public static class ArchiveWriter +{ + private sealed class WillUpdateObjectFileDescriptor : IObjectFileDescriptor + { + public int Length => + throw new InvalidOperationException(); + public string ObjectName => + ArchiverUtilities.GetObjectName(this.Path); + public string Path { get; } + + public WillUpdateObjectFileDescriptor(string path) => + this.Path = path; + } + + public static SymbolListEntry[] GetCombinedSymbolListEntries( + string archiveFilePath, + string[] objectFilePaths) + { + var archivedObjectItems = CommonUtilities.Empty(); + + if (File.Exists(archiveFilePath)) + { + var hashedObjectNames = objectFilePaths.ToDictionary(ArchiverUtilities.GetObjectName); + archivedObjectItems = ArchiverUtilities.LoadArchivedObjectItemDescriptors( + archiveFilePath, + aod => hashedObjectNames.TryGetValue(aod.ObjectName, out var path) ? + new WillUpdateObjectFileDescriptor(path) : // Will update object file + aod.ObjectName == ArchiverUtilities.SymbolTableFileName ? + null : // Will ignore symbol table + aod); // Archived object file + } + + // Aggregate and extract to add new object files + var willAddNewObjectFileItems = objectFilePaths. + Except(archivedObjectItems. + OfType(). + Select(d => d.Path)). + Select(path => new WillUpdateObjectFileDescriptor(path)). + ToArray(); + + var symbolListEntries = new SymbolListEntry[ + archivedObjectItems.Length + willAddNewObjectFileItems.Length]; + + Parallel.ForEach( + archivedObjectItems.Concat(willAddNewObjectFileItems), + (entry, _, index) => + { + switch (entry) + { + case IObjectFileDescriptor ofd: + using (var objectRawStream = StreamUtilities.OpenStream(ofd.Path, false)) + { + var length = objectRawStream.Length; + var objectStream = ArchiverUtilities.ToDecompressedObjectStream(objectRawStream, ofd.Path); + var symbols = ArchiverUtilities.EnumerateSymbolsFromObjectFileStream(objectStream). + ToArray(); + symbolListEntries[index] = new( + new ObjectFileDescriptor((int)length, ofd.Path), + new(ofd.ObjectName, symbols)); + } + break; + case ArchivedObjectItemDescriptor aod: + using (var archiveFileStream = StreamUtilities.OpenStream(archiveFilePath!, false)) + { + archiveFileStream.Position = aod.Position; + var objectStream = ArchiverUtilities.ToDecompressedObjectStream( + new RangedStream(archiveFileStream, aod.Length, false), + aod.ObjectName); + var symbols = ArchiverUtilities.EnumerateSymbolsFromObjectFileStream(objectStream). + ToArray(); + symbolListEntries[index] = new(aod, new(aod.ObjectName, symbols)); + } + break; + } + }); + + return symbolListEntries; + } + + //////////////////////////////////////////////////////////////////// + + private static void WriteSymbolTable( + Stream symbolTableStream, + IEnumerable symbolLists) + { + var tw = StreamUtilities.CreateTextWriter(symbolTableStream); + foreach (var symbolList in symbolLists) + { + tw.WriteLine( + $".object {symbolList.ObjectName}"); + foreach (var symbol in symbolList.Symbols.Distinct()) + { + tw.WriteLine( + $" {symbol.Directive} {symbol.Scope} {symbol.Name}{(symbol.MemberCount is { } mc ? $" {mc}" : "")}"); + } + } + tw.Flush(); + } + + private static byte[] CreateSymbolTableImage( + IEnumerable symbolListEntries) + { + var symbolTableStream = new MemoryStream(); + using (var compressedStream = new GZipStream(symbolTableStream, CompressionLevel.Fastest)) + { + WriteSymbolTable( + compressedStream, + symbolListEntries.Select(entry => entry.SymbolList)); + compressedStream.Flush(); + } + return symbolTableStream.ToArray(); + } + + //////////////////////////////////////////////////////////////////// + + private static void WriteObjectItemDescriptors( + Stream outputArchiveFileStream, + IEnumerable descriptors) + { + var lew = new LittleEndianWriter(outputArchiveFileStream); + foreach (var descriptor in descriptors) + { + lew.Write(descriptor.ObjectName); + lew.Write(descriptor.Length); + } + lew.Write(string.Empty); // Termination + } + + private static void WriteObjectItemBodies( + Stream outputArchiveFileStream, + string readArchiveFilePath, + IEnumerable descriptors) + { + foreach (var descriptor in descriptors) + { + switch (descriptor) + { + case ObjectFileDescriptor ofd: + using (var objectFileStream = ArchiverUtilities.OpenCompressedObjectStream(ofd.Path)) + { + objectFileStream.CopyTo(outputArchiveFileStream); + } + break; + case ArchivedObjectItemDescriptor aod: + using (var archiveFileStream = StreamUtilities.OpenStream(readArchiveFilePath, false)) + { + archiveFileStream.Position = aod.Position; + var objectStream = new RangedStream(archiveFileStream, aod.Length, false); + objectStream.CopyTo(outputArchiveFileStream); + } + break; + } + } + } + + private sealed class SymbolTableDescriptor(int length) : IObjectItemDescriptor + { + public int Length => + length; + public string ObjectName => + ArchiverUtilities.SymbolTableFileName; + } + + public static void WriteArchive( + Stream outputArchiveFileStream, + SymbolListEntry[] symbolListEntries, + string readArchiveFilePath, + bool writeSymbolTable) + { + if (writeSymbolTable) + { + // Create symbol table image (and get that length). + var symbolTableImage = CreateSymbolTableImage(symbolListEntries); + + // Write the descriptors to the head of the archive file. + WriteObjectItemDescriptors( + outputArchiveFileStream, + symbolListEntries. + Select(entry => entry.Descriptor). + // Insert the symbol table descriptor to the front. + Prepend(new SymbolTableDescriptor(symbolTableImage.Length))); + + // Write the symbol table image next to the descriptors. + outputArchiveFileStream.Write( + symbolTableImage, 0, symbolTableImage.Length); + } + else + { + // Write the descriptors to the head of the archive file. + WriteObjectItemDescriptors( + outputArchiveFileStream, + symbolListEntries.Select(entry => entry.Descriptor)); + } + + // Write all required object items. + WriteObjectItemBodies( + outputArchiveFileStream, + readArchiveFilePath, + symbolListEntries.Select(entry => entry.Descriptor)); + } +} diff --git a/toolchain.common/Archiving/ArchiverUtilities.cs b/toolchain.common/Archiving/ArchiverUtilities.cs index 1621c77..fb8ff4a 100644 --- a/toolchain.common/Archiving/ArchiverUtilities.cs +++ b/toolchain.common/Archiving/ArchiverUtilities.cs @@ -45,7 +45,12 @@ internal void AdjustPosition(long basePosition) => this.Position += basePosition; } -public sealed class ObjectFileDescriptor : IObjectItemDescriptor +internal interface IObjectFileDescriptor : IObjectItemDescriptor +{ + string Path { get; } +} + +public sealed class ObjectFileDescriptor : IObjectFileDescriptor { public int Length { get; } public string Path { get; } @@ -63,15 +68,27 @@ public record struct SymbolListEntry( IObjectItemDescriptor Descriptor, SymbolList SymbolList); +public enum FileTypes +{ + Other, + ObjectFile, + SourceFile, +} + public static class ArchiverUtilities { public static readonly string SymbolTableFileName = "__.SYMDEF"; - public static bool IsSourceFile(string path) => - Path.GetExtension(path) is not ".o"; + public static FileTypes GetFileType(string path) => + Path.GetExtension(path) switch + { + ".o" => FileTypes.ObjectFile, + ".s" => FileTypes.SourceFile, + _ => FileTypes.Other, + }; public static string GetObjectName(string objectFilePath) => - IsSourceFile(objectFilePath) ? + GetFileType(objectFilePath) == FileTypes.SourceFile ? (Path.GetFileNameWithoutExtension(objectFilePath) + ".o") : Path.GetFileName(objectFilePath); @@ -182,52 +199,74 @@ tokens[0] is (TokenTypes.Identity, _)) } } - private static IEnumerable EnumerateSymbolsFromObjectFileStream( + public static IEnumerable EnumerateSymbolsFromObjectFileStream( Stream objectFileStream) => InternalEnumerateSymbolsFromObjectFileStream(objectFileStream). Distinct(); //////////////////////////////////////////////////////////////////// - private static Stream ToCompressedObjectStream( + public static Stream ToCompressedObjectStream( Stream objectFileStream, string referenceObjectFilePath) { - if (!IsSourceFile(referenceObjectFilePath)) + if (GetFileType(referenceObjectFilePath) == FileTypes.ObjectFile) { - return objectFileStream; + if (objectFileStream.CanSeek) + { + return objectFileStream; + } + var ms = new MemoryStream(); + objectFileStream.CopyTo(ms); + objectFileStream.Dispose(); + ms.Position = 0; + return ms; + } + else + { + var gzs = new GZipStream(objectFileStream, CompressionLevel.Optimal); + var ms = new MemoryStream(); + gzs.CopyTo(ms); + objectFileStream.Dispose(); + ms.Position = 0; + return ms; } - var gzs = new GZipStream(objectFileStream, CompressionLevel.Optimal); - var ms = new MemoryStream(); - gzs.CopyTo(ms); - objectFileStream.Dispose(); - ms.Position = 0; - return ms; } - private static Stream OpenCompressedObjectStream(string objectFilePath) + public static Stream OpenCompressedObjectStream(string objectFilePath) { var ofs = StreamUtilities.OpenStream(objectFilePath, false); return ToCompressedObjectStream(ofs, objectFilePath); } - private static Stream ToDecompressedObjectStream( + public static Stream ToDecompressedObjectStream( Stream objectFileStream, string referenceObjectFilePath) { - if (IsSourceFile(referenceObjectFilePath)) + if (GetFileType(referenceObjectFilePath) != FileTypes.ObjectFile) + { + if (objectFileStream.CanSeek) + { + return objectFileStream; + } + var ms = new MemoryStream(); + objectFileStream.CopyTo(ms); + objectFileStream.Dispose(); + ms.Position = 0; + return ms; + } + else { - return objectFileStream; + var gzs = new GZipStream(objectFileStream, CompressionMode.Decompress); + var ms = new MemoryStream(); + gzs.CopyTo(ms); + objectFileStream.Dispose(); + ms.Position = 0; + return ms; } - var gzs = new GZipStream(objectFileStream, CompressionMode.Decompress); - var ms = new MemoryStream(); - gzs.CopyTo(ms); - objectFileStream.Dispose(); - ms.Position = 0; - return ms; } - private static Stream OpenDecompressedObjectStream(string objectFilePath) + public static Stream OpenDecompressedObjectStream(string objectFilePath) { var ofs = StreamUtilities.OpenStream(objectFilePath, false); return ToDecompressedObjectStream(ofs, objectFilePath); @@ -282,188 +321,6 @@ public static IObjectItemDescriptor[] LoadArchivedObjectItemDescriptors( return descriptors; } - public static SymbolListEntry[] GetCombinedSymbolListEntries( - string archiveFilePath, - string[] objectFilePaths) - { - var archivedObjectItems = CommonUtilities.Empty(); - - if (File.Exists(archiveFilePath)) - { - var hashedObjectNames = objectFilePaths.ToDictionary(GetObjectName); - archivedObjectItems = LoadArchivedObjectItemDescriptors( - archiveFilePath, - aod => hashedObjectNames.TryGetValue(aod.ObjectName, out var path) ? - new ObjectFileDescriptor(-1, path) : // Will update object file - aod.ObjectName == SymbolTableFileName ? - null : // Will ignore symbol table - aod); // Archived object file - } - - var willAddObjectFileItems = objectFilePaths. - Except(archivedObjectItems. - OfType(). - Select(d => d.Path)). - Select(path => new ObjectFileDescriptor(-1, path)). - ToArray(); - - var symbolListEntries = new SymbolListEntry[ - archivedObjectItems.Length + willAddObjectFileItems.Length]; - - Parallel.ForEach( - archivedObjectItems.Concat(willAddObjectFileItems), - (entry, _, index) => - { - switch (entry) - { - case ObjectFileDescriptor ofd: - using (var objectRawStream = StreamUtilities.OpenStream(ofd.Path, false)) - { - var length = objectRawStream.Length; - var objectStream = ToDecompressedObjectStream(objectRawStream, ofd.Path); - var symbols = EnumerateSymbolsFromObjectFileStream(objectStream). - ToArray(); - symbolListEntries[index] = new( - new ObjectFileDescriptor((int)length, ofd.Path), - new(ofd.ObjectName, symbols)); - } - break; - case ArchivedObjectItemDescriptor aod: - using (var archiveFileStream = StreamUtilities.OpenStream(archiveFilePath!, false)) - { - archiveFileStream.Position = aod.Position; - var objectStream = ToDecompressedObjectStream( - new RangedStream(archiveFileStream, aod.Length, false), - aod.ObjectName); - var symbols = EnumerateSymbolsFromObjectFileStream(objectStream). - ToArray(); - symbolListEntries[index] = new(aod, new(aod.ObjectName, symbols)); - } - break; - } - }); - - return symbolListEntries; - } - - //////////////////////////////////////////////////////////////////// - - private static void WriteSymbolTable( - Stream symbolTableStream, - IEnumerable symbolLists) - { - var tw = StreamUtilities.CreateTextWriter(symbolTableStream); - foreach (var symbolList in symbolLists) - { - tw.WriteLine( - $".object {symbolList.ObjectName}"); - foreach (var symbol in symbolList.Symbols.Distinct()) - { - tw.WriteLine( - $" {symbol.Directive} {symbol.Scope} {symbol.Name}{(symbol.MemberCount is { } mc ? $" {mc}" : "")}"); - } - } - tw.Flush(); - } - - private static byte[] CreateSymbolTableImage( - IEnumerable symbolListEntries) - { - var symbolTableStream = new MemoryStream(); - using (var compressedStream = new GZipStream(symbolTableStream, CompressionLevel.Fastest)) - { - WriteSymbolTable( - compressedStream, - symbolListEntries.Select(entry => entry.SymbolList)); - compressedStream.Flush(); - } - return symbolTableStream.ToArray(); - } - - //////////////////////////////////////////////////////////////////// - - private static void WriteObjectItemDescriptors( - Stream outputArchiveFileStream, - IEnumerable descriptors) - { - var lew = new LittleEndianWriter(outputArchiveFileStream); - foreach (var descriptor in descriptors) - { - lew.Write(descriptor.ObjectName); - lew.Write(descriptor.Length); - } - lew.Write(string.Empty); // Termination - } - - private static void WriteObjectItemBodies( - Stream outputArchiveFileStream, - string readArchiveFilePath, - IEnumerable descriptors) - { - foreach (var descriptor in descriptors) - { - switch (descriptor) - { - case ObjectFileDescriptor ofd: - using (var objectFileStream = OpenCompressedObjectStream(ofd.Path)) - { - objectFileStream.CopyTo(outputArchiveFileStream); - } - break; - case ArchivedObjectItemDescriptor aod: - using (var archiveFileStream = StreamUtilities.OpenStream(readArchiveFilePath, false)) - { - archiveFileStream.Position = aod.Position; - var objectStream = new RangedStream(archiveFileStream, aod.Length, false); - objectStream.CopyTo(outputArchiveFileStream); - } - break; - } - } - } - - private sealed class SymbolTableDescriptor(int length) : IObjectItemDescriptor - { - public int Length => - length; - public string ObjectName => - SymbolTableFileName; - } - - public static void WriteArchive( - Stream outputArchiveFileStream, - SymbolListEntry[] symbolListEntries, - string readArchiveFilePath, - bool writeSymbolTable) - { - if (writeSymbolTable) - { - var symbolTableStream = new MemoryStream( - CreateSymbolTableImage(symbolListEntries)); - - WriteObjectItemDescriptors( - outputArchiveFileStream, - symbolListEntries. - Select(entry => entry.Descriptor). - Prepend(new SymbolTableDescriptor((int)symbolTableStream.Length))); - - symbolTableStream.CopyTo(outputArchiveFileStream); - } - else - { - WriteObjectItemDescriptors( - outputArchiveFileStream, - symbolListEntries.Select(entry => entry.Descriptor)); - } - - WriteObjectItemBodies( - outputArchiveFileStream, - readArchiveFilePath, - symbolListEntries.Select(entry => entry.Descriptor)); - - outputArchiveFileStream.Flush(); - } - //////////////////////////////////////////////////////////////////// public static IEnumerable EnumerateSymbolListFromArchive( From 31d2273fc32f847ce6f86861b4ea31d711ed858e Mon Sep 17 00:00:00 2001 From: Kouji Matsui Date: Sun, 5 Jan 2025 12:53:46 +0900 Subject: [PATCH 4/7] Changed to use ArchiveReader on the linker. --- .../Generating/ArchivedObjectInputFragment.cs | 17 +++++++++++------ toolchain.common/Archiving/ArchiverUtilities.cs | 17 +---------------- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs b/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs index de5243e..c754b1e 100644 --- a/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs +++ b/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs @@ -40,6 +40,7 @@ private enum RequiredStates Loaded, } + private readonly ArchiveReader archiveReader; private readonly string archivedObjectName; private readonly Dictionary typeSymbols; @@ -58,12 +59,14 @@ private enum RequiredStates private ArchivedObjectInputFragment( string baseInputPath, string relativePath, + ArchiveReader archiveReader, string archivedObjectName, Dictionary typeSymbols, Dictionary variableSymbols, Dictionary functionSymbols) : base(baseInputPath, relativePath) { + this.archiveReader = archiveReader; this.archivedObjectName = archivedObjectName; this.ObjectName = Path.GetFileNameWithoutExtension(this.archivedObjectName); this.ObjectPath = $"{this.archivedObjectName}@{base.ObjectPath}"; @@ -181,19 +184,18 @@ public LoadObjectResults LoadObjectIfRequired( { logger.Information($"Loading: {this.ObjectPath}"); - if (!ArchiverUtilities.TryOpenArchivedObject( - Path.Combine(this.BaseInputPath, this.RelativePath), + if (!this.archiveReader.TryOpenObjectStream( this.archivedObjectName, true, - out var stream)) + out var objectStream)) { logger.Error( $"Unable find an object on archive: ObjectName={this.archivedObjectName}, ArchiveFile={this.RelativePath}"); return LoadObjectResults.CaughtError; } - using var _s = stream; - var tr = StreamUtilities.CreateTextReader(stream); + using var _ = objectStream; + var tr = StreamUtilities.CreateTextReader(objectStream); var parser = new CilParser(logger); var declarations = parser.Parse( @@ -238,8 +240,10 @@ public static ArchivedObjectInputFragment[] Load( { logger.Information($"Loading symbol table: {relativePath}"); - var symbolLists = ArchiverUtilities.EnumerateSymbolListFromArchive( + var archiveReader = new ArchiveReader( Path.Combine(baseInputPath, relativePath)); + + var symbolLists = archiveReader.EnumerateSymbolListFromArchive(); return symbolLists.Select(symbolList => { @@ -271,6 +275,7 @@ public static ArchivedObjectInputFragment[] Load( return new ArchivedObjectInputFragment( baseInputPath, relativePath, + archiveReader, symbolList.ObjectName, symbols.TryGetValue("type", out var types) ? types : empty, symbols.TryGetValue("variable", out var variableNames) ? variableNames : empty, diff --git a/toolchain.common/Archiving/ArchiverUtilities.cs b/toolchain.common/Archiving/ArchiverUtilities.cs index fb8ff4a..c51a370 100644 --- a/toolchain.common/Archiving/ArchiverUtilities.cs +++ b/toolchain.common/Archiving/ArchiverUtilities.cs @@ -324,11 +324,8 @@ public static IObjectItemDescriptor[] LoadArchivedObjectItemDescriptors( //////////////////////////////////////////////////////////////////// public static IEnumerable EnumerateSymbolListFromArchive( - string archiveFilePath) + this ArchiveReader archiveReader) { - var archiveReader = new ArchiveReader( - archiveFilePath, [ SymbolTableFileName ]); - if (archiveReader.TryOpenObjectStream( SymbolTableFileName, true, out var symbolTableStream)) { @@ -383,16 +380,4 @@ tokens[2] is (TokenTypes.Identity, var name): } } } - - public static bool TryOpenArchivedObject( - string archiveFilePath, - string objectName, - bool decodedBody, - out Stream stream) - { - var archiveReader = new ArchiveReader( - archiveFilePath, [ objectName ]); - return archiveReader.TryOpenObjectStream( - objectName, decodedBody, out stream); - } } From 643e68914cbc81b20b49505ed52a9c495c6eec17 Mon Sep 17 00:00:00 2001 From: Kouji Matsui Date: Sun, 5 Jan 2025 13:03:04 +0900 Subject: [PATCH 5/7] Added archive identity. --- chibiar/chibiar.core/Archiving/ArchiveWriter.cs | 9 ++++++++- toolchain.common/Archiving/ArchiveReader.cs | 2 ++ toolchain.common/Archiving/ArchiverUtilities.cs | 8 ++++++++ toolchain.common/IO/LittleEndianReader.cs | 2 +- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/chibiar/chibiar.core/Archiving/ArchiveWriter.cs b/chibiar/chibiar.core/Archiving/ArchiveWriter.cs index ba6cb01..a7d5c4f 100644 --- a/chibiar/chibiar.core/Archiving/ArchiveWriter.cs +++ b/chibiar/chibiar.core/Archiving/ArchiveWriter.cs @@ -139,12 +139,19 @@ private static void WriteObjectItemDescriptors( IEnumerable descriptors) { var lew = new LittleEndianWriter(outputArchiveFileStream); + + // Head of the identity. + lew.Write(ArchiverUtilities.ArchiveIdentity); + + // Descriptors. foreach (var descriptor in descriptors) { lew.Write(descriptor.ObjectName); lew.Write(descriptor.Length); } - lew.Write(string.Empty); // Termination + + // Descriptor termination. + lew.Write(string.Empty); } private static void WriteObjectItemBodies( diff --git a/toolchain.common/Archiving/ArchiveReader.cs b/toolchain.common/Archiving/ArchiveReader.cs index 7126fab..ce25266 100644 --- a/toolchain.common/Archiving/ArchiveReader.cs +++ b/toolchain.common/Archiving/ArchiveReader.cs @@ -24,6 +24,7 @@ public ArchiveReader( string archiveFilePath) { this.archiveFilePath = archiveFilePath; + var descriptors = ArchiverUtilities.LoadArchivedObjectItemDescriptors( this.archiveFilePath, aod => aod); this.ObjectNames = descriptors. @@ -38,6 +39,7 @@ public ArchiveReader( IEnumerable objectNames) { this.archiveFilePath = archiveFilePath; + var hashedObjectNames = new HashSet(objectNames); var descriptors = ArchiverUtilities.LoadArchivedObjectItemDescriptors( this.archiveFilePath, aod => hashedObjectNames.Contains(aod.ObjectName) ? aod : null); diff --git a/toolchain.common/Archiving/ArchiverUtilities.cs b/toolchain.common/Archiving/ArchiverUtilities.cs index c51a370..5b7dbde 100644 --- a/toolchain.common/Archiving/ArchiverUtilities.cs +++ b/toolchain.common/Archiving/ArchiverUtilities.cs @@ -77,6 +77,7 @@ public enum FileTypes public static class ArchiverUtilities { + public static readonly string ArchiveIdentity = "chibias1"; public static readonly string SymbolTableFileName = "__.SYMDEF"; public static FileTypes GetFileType(string path) => @@ -278,6 +279,13 @@ private static IEnumerable EnumerateArchivedObject Stream archiveFileStream) { var ler = new LittleEndianReader(archiveFileStream); + if (!ler.TryReadString(ArchiveIdentity.Length, out var identity) || + identity != ArchiveIdentity) + { + throw new FormatException( + "Invalid archive identity."); + } + var relativePosition = 0; while (true) { diff --git a/toolchain.common/IO/LittleEndianReader.cs b/toolchain.common/IO/LittleEndianReader.cs index f98ce33..a4acc87 100644 --- a/toolchain.common/IO/LittleEndianReader.cs +++ b/toolchain.common/IO/LittleEndianReader.cs @@ -94,7 +94,7 @@ public bool TryReadString(int maxLength, out string value) if (maxLength >= 1) { var buffer = new List(maxLength); - while (buffer.Count < maxLength) + while (buffer.Count <= maxLength) { var ch = this.parent.ReadByte(); if (ch == -1) From f1539c6f241fd60c841f98a54453d9b00af69305 Mon Sep 17 00:00:00 2001 From: Kouji Matsui Date: Sun, 5 Jan 2025 13:08:32 +0900 Subject: [PATCH 6/7] Updated readme. --- README.ja.md | 2 -- README.md | 2 -- 2 files changed, 4 deletions(-) diff --git a/README.ja.md b/README.ja.md index 633e768..9cb6ff1 100644 --- a/README.ja.md +++ b/README.ja.md @@ -62,8 +62,6 @@ CILソースコードの実質的なアセンブル処理は、chibildが行い * chibiasは、パーサーによる構文チェックを行いますが、シンボルの妥当性など細かい検査を行いません。 * chibiasが出力するオブジェクトファイルは、実際には入力となるCILソースファイルをgzipで圧縮しただけです。 このことは、実際にオブジェクトファイルを `gzip -d` することで確かめることができます。 -* 同様に、chibiarが出力するアーカイブファイルは、実際にはシンボルテーブルファイルを含めたzipファイルです。 - 全く同様に、`unzip` コマンドで確かめることができます。 このような、chibias, chibild, chibiarの奇妙な挙動は実装の制約によるものですが、 ツールチェイン使用者にとって見れば、POSIXで想定されるas,ld,arなどのツールチェインと対比させて使用方法を理解できるという強みがあります。 diff --git a/README.md b/README.md index ce39751..45d9c3f 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,6 @@ and chibild performs the actual assembly process of the CIL source code. * chibias performs syntax checking by the parser, but does not perform detailed checking such as symbol validity. * The '*.o' file output by chibias is actually just a compression with gzip format of the input CIL source file. This can be verified by actually `gzip -d` the object file. -* Similarly, the '*.a' file output by chibiar is actually a zip file format including the symbol table file. - Exactly the same can be verified with the `unzip` command. The strange internal behavior of chibias, chibild and chibiar is due to implementation limitations. For toolchain users, the advantage is that they can contrast their usage with From bede0aaecde64d91fad7a7058e690f9080a67b63 Mon Sep 17 00:00:00 2001 From: Kouji Matsui Date: Mon, 6 Jan 2025 20:38:51 +0900 Subject: [PATCH 7/7] Added more debug logs. --- .../chibiar.core.Tests/ArchiverTestRunner.cs | 4 +- chibiar/chibiar.core/Archiver.cs | 15 +++++ .../chibiar.core/Archiving/ArchiveWriter.cs | 30 ++++++++- chibiar/chibiar.core/Cli/CliOptions.cs | 4 ++ chibiar/chibiar/Program.cs | 1 + chibias/chibias.core/Assembler.cs | 22 ++++--- chibias/chibias.core/Cli/CliOptions.cs | 4 ++ chibias/chibias/Program.cs | 1 + .../chibild.core.Tests/LinkerTestRunner.cs | 2 +- chibild/chibild.core/CilLinker.cs | 10 +++ chibild/chibild.core/Cli/CliOptions.cs | 4 ++ .../Generating/ArchivedObjectInputFragment.cs | 14 ++++- .../Generating/AssemblyInputFragment.cs | 4 ++ .../chibild.core/Generating/CodeGenerator.cs | 35 ++++++++--- .../Generating/CodeGenerator_Consumer.cs | 20 +++--- .../Generating/CodeGenerator_Emit.cs | 62 +++++++++++++------ .../Generating/ObjectFileInputFragment.cs | 6 ++ chibild/chibild/Program.cs | 1 + toolchain.common/Logging/ILogger.cs | 4 +- toolchain.common/Logging/LoggerBase.cs | 27 +++++--- toolchain.common/Logging/LoggerExtension.cs | 51 +++++++++++++++ toolchain.common/Logging/TextWriterLogger.cs | 4 +- 22 files changed, 262 insertions(+), 63 deletions(-) diff --git a/chibiar/chibiar.core.Tests/ArchiverTestRunner.cs b/chibiar/chibiar.core.Tests/ArchiverTestRunner.cs index af60369..e3139ee 100644 --- a/chibiar/chibiar.core.Tests/ArchiverTestRunner.cs +++ b/chibiar/chibiar.core.Tests/ArchiverTestRunner.cs @@ -50,7 +50,9 @@ public static async Task RunAsync( var logtw = StreamUtilities.CreateTextWriter( logfs); var logger = new TextWriterLogger( - LogLevels.Debug, logtw); + "chibiar", + LogLevels.Debug, + logtw); try { diff --git a/chibiar/chibiar.core/Archiver.cs b/chibiar/chibiar.core/Archiver.cs index bb33182..45a0916 100644 --- a/chibiar/chibiar.core/Archiver.cs +++ b/chibiar/chibiar.core/Archiver.cs @@ -13,6 +13,7 @@ using chibicc.toolchain.IO; using chibicc.toolchain.Logging; using System; +using System.Diagnostics; using System.IO; using System.Linq; using System.Threading.Tasks; @@ -32,6 +33,8 @@ internal bool AddOrUpdate( bool writeSymbolTable, bool isDryrun) { + using var scope = this.logger.BeginScope(LogLevels.Debug); + var outputArchiveFilePath = Path.Combine( Path.GetTempPath(), $"chibiar_{Path.GetFileNameWithoutExtension(archiveFilePath)}_{Guid.NewGuid():N}{Path.GetExtension(archiveFilePath)}"); @@ -41,14 +44,18 @@ internal bool AddOrUpdate( var isExistArchiveFile = File.Exists(archiveFilePath); var symbolListEntries = ArchiveWriter.GetCombinedSymbolListEntries( + this.logger, archiveFilePath, objectFilePaths); + scope.Debug("Step 1"); + using (var outputArchiveFileStream = isDryrun ? new NullStream() : StreamUtilities.OpenStream(outputArchiveFilePath, true)) { ArchiveWriter.WriteArchive( + this.logger, outputArchiveFileStream, symbolListEntries, archiveFilePath, @@ -57,12 +64,16 @@ internal bool AddOrUpdate( outputArchiveFileStream.Flush(); } + scope.Debug("Step 2"); + if (!isDryrun) { File.Delete(archiveFilePath); File.Move(outputArchiveFilePath, archiveFilePath); } + scope.Debug("Step 3"); + return !isExistArchiveFile; } catch @@ -80,6 +91,8 @@ internal void Extract( string[] objectNames, bool isDryrun) { + using var scope = this.logger.BeginScope(LogLevels.Debug); + var archiveReader = new ArchiveReader(archiveFilePath, objectNames); var read = objectNames.ToDictionary(objectName => objectName, _ => false); @@ -104,6 +117,8 @@ internal void Extract( { read[objectName] = true; } + + scope.Debug($"Extracted: {objectName}"); }); foreach (var entry in read.Where(entry => !entry.Value)) diff --git a/chibiar/chibiar.core/Archiving/ArchiveWriter.cs b/chibiar/chibiar.core/Archiving/ArchiveWriter.cs index a7d5c4f..6f2580b 100644 --- a/chibiar/chibiar.core/Archiving/ArchiveWriter.cs +++ b/chibiar/chibiar.core/Archiving/ArchiveWriter.cs @@ -10,8 +10,10 @@ using chibicc.toolchain.Archiving; using chibicc.toolchain.Internal; using chibicc.toolchain.IO; +using chibicc.toolchain.Logging; using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; @@ -34,9 +36,12 @@ public WillUpdateObjectFileDescriptor(string path) => } public static SymbolListEntry[] GetCombinedSymbolListEntries( + ILogger logger, string archiveFilePath, string[] objectFilePaths) { + using var scope = logger.BeginScope(LogLevels.Debug); + var archivedObjectItems = CommonUtilities.Empty(); if (File.Exists(archiveFilePath)) @@ -50,7 +55,9 @@ public static SymbolListEntry[] GetCombinedSymbolListEntries( null : // Will ignore symbol table aod); // Archived object file } - + + scope.Debug("Step 1"); + // Aggregate and extract to add new object files var willAddNewObjectFileItems = objectFilePaths. Except(archivedObjectItems. @@ -93,8 +100,12 @@ public static SymbolListEntry[] GetCombinedSymbolListEntries( } break; } + + scope.Debug($"Step 2: {entry.ObjectName}"); }); + scope.Debug($"Step 3"); + return symbolListEntries; } @@ -155,10 +166,13 @@ private static void WriteObjectItemDescriptors( } private static void WriteObjectItemBodies( + ILogger logger, Stream outputArchiveFileStream, string readArchiveFilePath, IEnumerable descriptors) { + using var scope = logger.BeginScope(LogLevels.Debug); + foreach (var descriptor in descriptors) { switch (descriptor) @@ -178,7 +192,11 @@ private static void WriteObjectItemBodies( } break; } + + scope.Debug($"Step 1: {descriptor.ObjectName}"); } + + scope.Debug($"Step 2"); } private sealed class SymbolTableDescriptor(int length) : IObjectItemDescriptor @@ -190,16 +208,21 @@ private sealed class SymbolTableDescriptor(int length) : IObjectItemDescriptor } public static void WriteArchive( + ILogger logger, Stream outputArchiveFileStream, SymbolListEntry[] symbolListEntries, string readArchiveFilePath, bool writeSymbolTable) { + using var scope = logger.BeginScope(LogLevels.Debug); + if (writeSymbolTable) { // Create symbol table image (and get that length). var symbolTableImage = CreateSymbolTableImage(symbolListEntries); + scope.Debug("Step 1"); + // Write the descriptors to the head of the archive file. WriteObjectItemDescriptors( outputArchiveFileStream, @@ -220,10 +243,15 @@ public static void WriteArchive( symbolListEntries.Select(entry => entry.Descriptor)); } + scope.Debug("Step 2"); + // Write all required object items. WriteObjectItemBodies( + logger, outputArchiveFileStream, readArchiveFilePath, symbolListEntries.Select(entry => entry.Descriptor)); + + scope.Debug("Step 3"); } } diff --git a/chibiar/chibiar.core/Cli/CliOptions.cs b/chibiar/chibiar.core/Cli/CliOptions.cs index 07353d0..ef97159 100644 --- a/chibiar/chibiar.core/Cli/CliOptions.cs +++ b/chibiar/chibiar.core/Cli/CliOptions.cs @@ -30,7 +30,11 @@ public sealed class CliOptions public bool IsSilent = false; public bool IsCreateSymbolTable = true; public bool IsDryRun = false; +#if DEBUG + public LogLevels LogLevel = LogLevels.Trace; +#else public LogLevels LogLevel = LogLevels.Warning; +#endif public bool ShowHelp = false; public readonly List ObjectNames = new(); diff --git a/chibiar/chibiar/Program.cs b/chibiar/chibiar/Program.cs index 3fca868..4d486a4 100644 --- a/chibiar/chibiar/Program.cs +++ b/chibiar/chibiar/Program.cs @@ -39,6 +39,7 @@ public static int Main(string[] args) } using var logger = new TextWriterLogger( + "chibiar", options.LogLevel, Console.Out); diff --git a/chibias/chibias.core/Assembler.cs b/chibias/chibias.core/Assembler.cs index d282952..f7dbdf8 100644 --- a/chibias/chibias.core/Assembler.cs +++ b/chibias/chibias.core/Assembler.cs @@ -7,17 +7,17 @@ // ///////////////////////////////////////////////////////////////////////////////////// -using System; +using chibicc.toolchain.Internal; using chibicc.toolchain.IO; using chibicc.toolchain.Logging; using chibicc.toolchain.Parsing; using chibicc.toolchain.Tokenizing; +using System; using System.IO; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; -using chibicc.toolchain.Internal; namespace chibias; @@ -33,6 +33,8 @@ public bool Assemble( string sourceFilePath, bool isDryrun) { + using var scope = this.logger.BeginScope(LogLevels.Debug); + var outputTemporaryFilePath = Path.Combine( CommonUtilities.GetDirectoryPath(outputObjectFilePath), @@ -59,23 +61,25 @@ public bool Assemble( Interlocked.Increment(ref count); } } + + scope.Debug("Checked"); }, () => { // Convert source code to object file. using var outputStream = isDryrun ? - null : ObjectStreamUtilities.OpenObjectStream(outputTemporaryFilePath, true); + new NullStream() : + ObjectStreamUtilities.OpenObjectStream(outputTemporaryFilePath, true); - if (outputStream != null) + using (var inputStream = StreamUtilities.OpenStream(sourceFilePath, false)) { - using (var inputStream = StreamUtilities.OpenStream(sourceFilePath, false)) - { - inputStream.CopyTo(outputStream); - outputStream.Flush(); - } + inputStream.CopyTo(outputStream); + outputStream.Flush(); } Interlocked.Increment(ref count); + + scope.Debug("Converted"); }, }; diff --git a/chibias/chibias.core/Cli/CliOptions.cs b/chibias/chibias.core/Cli/CliOptions.cs index cca755f..8cab617 100644 --- a/chibias/chibias.core/Cli/CliOptions.cs +++ b/chibias/chibias.core/Cli/CliOptions.cs @@ -18,7 +18,11 @@ public sealed class CliOptions public string OutputObjectFilePath = null!; public bool IsLinked = true; public bool IsDryRun = false; +#if DEBUG + public LogLevels LogLevel = LogLevels.Trace; +#else public LogLevels LogLevel = LogLevels.Warning; +#endif public bool ShowHelp = false; public string? SourceFilePath; diff --git a/chibias/chibias/Program.cs b/chibias/chibias/Program.cs index cfad8df..1927d8e 100644 --- a/chibias/chibias/Program.cs +++ b/chibias/chibias/Program.cs @@ -38,6 +38,7 @@ public static int Main(string[] args) } using var logger = new TextWriterLogger( + "chibias", options.LogLevel, Console.Out); diff --git a/chibild/chibild.core.Tests/LinkerTestRunner.cs b/chibild/chibild.core.Tests/LinkerTestRunner.cs index fe94765..c880f9a 100644 --- a/chibild/chibild.core.Tests/LinkerTestRunner.cs +++ b/chibild/chibild.core.Tests/LinkerTestRunner.cs @@ -59,7 +59,7 @@ public static string RunCore( var logtw = StreamUtilities.CreateTextWriter( logfs); var logger = new TextWriterLogger( - LogLevels.Debug, logtw); + "chibild", LogLevels.Debug, logtw); logger.Information($"Test runner BasePath={basePath}"); diff --git a/chibild/chibild.core/CilLinker.cs b/chibild/chibild.core/CilLinker.cs index 6fbfe4f..ac33b8d 100644 --- a/chibild/chibild.core/CilLinker.cs +++ b/chibild/chibild.core/CilLinker.cs @@ -287,9 +287,12 @@ public bool Link( string baseInputPath, params InputReference[] inputReferences) { + using var scope = this.logger.BeginScope(LogLevels.Debug); + if (!inputReferences.OfType(). Any()) { + scope.Debug("No inputs"); return false; } @@ -323,8 +326,11 @@ injectToAssemblyPath is { } injectPath ? produceDebuggingInformation, out var loadedFragments)) { + scope.Debug("No input references"); return false; } + + scope.Debug($"Loaded inputs: Fragments={loadedFragments.Length}"); ////////////////////////////////////////////////////////////// @@ -370,6 +376,8 @@ injectToAssemblyPath is { } injectPath ? } var targetModule = primaryAssembly.MainModule; + + scope.Debug("Initialized primary assembly"); ////////////////////////////////////////////////////////////// @@ -382,6 +390,7 @@ injectToAssemblyPath is { } injectPath ? loadedFragments, produceDebuggingInformation)) { + scope.Debug("Caught any errors [1]"); return false; } @@ -392,6 +401,7 @@ injectToAssemblyPath is { } injectPath ? options.CreationOptions, options.PrependExecutionSearchPaths)) { + scope.Debug("Caught any errors [2]"); return false; } diff --git a/chibild/chibild.core/Cli/CliOptions.cs b/chibild/chibild.core/Cli/CliOptions.cs index 1d6f4ae..f986c45 100644 --- a/chibild/chibild.core/Cli/CliOptions.cs +++ b/chibild/chibild.core/Cli/CliOptions.cs @@ -38,7 +38,11 @@ public sealed class CliOptions public string OutputAssemblyPath = null!; public readonly LinkerOptions LinkerOptions = new(); public string? InjectToAssemblyPath; +#if DEBUG + public LogLevels LogLevel = LogLevels.Trace; +#else public LogLevels LogLevel = LogLevels.Warning; +#endif public bool ShowHelp = false; public string BaseInputPath = Directory.GetCurrentDirectory(); public InputReference[] InputReferences = CommonUtilities.Empty(); diff --git a/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs b/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs index c754b1e..dbf1f19 100644 --- a/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs +++ b/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs @@ -15,6 +15,7 @@ using chibicc.toolchain.Internal; using chibild.Internal; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Text; @@ -240,14 +241,17 @@ public static ArchivedObjectInputFragment[] Load( { logger.Information($"Loading symbol table: {relativePath}"); + using var scope = logger.BeginScope(LogLevels.Debug); + var archiveReader = new ArchiveReader( Path.Combine(baseInputPath, relativePath)); var symbolLists = archiveReader.EnumerateSymbolListFromArchive(); - - return symbolLists.Select(symbolList => + + var fragments = symbolLists.Select(symbolList => { var symbols = symbolList.Symbols. + AsParallel(). GroupBy(symbol => { switch (symbol.Directive) @@ -272,6 +276,8 @@ public static ArchivedObjectInputFragment[] Load( var empty = new Dictionary(); + scope.Debug($"Examined a symbol list: {relativePath}, {symbolList.ObjectName}"); + return new ArchivedObjectInputFragment( baseInputPath, relativePath, @@ -282,5 +288,9 @@ public static ArchivedObjectInputFragment[] Load( symbols.TryGetValue("function", out var functionNames) ? functionNames : empty); }). ToArray(); + + scope.Debug($"Loaded symbol table: {relativePath}"); + + return fragments; } } diff --git a/chibild/chibild.core/Generating/AssemblyInputFragment.cs b/chibild/chibild.core/Generating/AssemblyInputFragment.cs index bdd885e..ae81a59 100644 --- a/chibild/chibild.core/Generating/AssemblyInputFragment.cs +++ b/chibild/chibild.core/Generating/AssemblyInputFragment.cs @@ -307,6 +307,8 @@ public static AssemblyInputFragment Load( logger.Information($"Loading assembly: {relativePath}"); + using var scope = logger.BeginScope(LogLevels.Debug); + var assembly = assemblyResolver.ReadAssemblyFrom( Path.Combine(baseInputPath, relativePath)); @@ -403,6 +405,8 @@ static IEnumerable IterateTypesDescendants(TypeDefinition type) // Sorted descending longer parameters. g => g.OrderByDescending(method => method.Parameters.Count).ToArray()); + scope.Debug($"Loaded assembly: {relativePath}"); + return new( baseInputPath, relativePath, diff --git a/chibild/chibild.core/Generating/CodeGenerator.cs b/chibild/chibild.core/Generating/CodeGenerator.cs index 4cefbfa..fcad41a 100644 --- a/chibild/chibild.core/Generating/CodeGenerator.cs +++ b/chibild/chibild.core/Generating/CodeGenerator.cs @@ -13,6 +13,7 @@ using Mono.Cecil.Cil; using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading.Tasks; @@ -62,18 +63,14 @@ private void OutputWarning(Token token, string message) $"{token.RelativePath}:{token.Line + 1}:{token.StartColumn + 1}: {message}"); } - private void OutputTrace(Token token, string message) - { - this.logger.Trace( - $"{token.RelativePath}:{token.Line + 1}:{token.StartColumn + 1}: {message}"); - } - ////////////////////////////////////////////////////////////// private void ConsumeFragment( ObjectInputFragment currentFragment, InputFragment[] inputFragments) { + using var scope = this.logger.BeginScope(LogLevels.Debug); + var context = new LookupContext( this.targetModule, currentFragment, @@ -84,26 +81,37 @@ private void ConsumeFragment( { this.ConsumeGlobalVariable(context, variable); } + scope.Debug($"[1]: {currentFragment.ObjectName}"); + foreach (var constant in currentFragment.GlobalConstants) { this.ConsumeGlobalConstant(context, constant); } + scope.Debug($"[2]: {currentFragment.ObjectName}"); + foreach (var function in currentFragment.Functions) { this.ConsumeFunction(context, function); } + scope.Debug($"[3]: {currentFragment.ObjectName}"); + foreach (var initializer in currentFragment.Initializers) { this.ConsumeInitializer(context, initializer); } + scope.Debug($"[4]: {currentFragment.ObjectName}"); + foreach (var enumeration in currentFragment.Enumerations) { this.ConsumeEnumeration(context, enumeration); } + scope.Debug($"[5]: {currentFragment.ObjectName}"); + foreach (var structure in currentFragment.Structures) { this.ConsumeStructure(context, structure); } + scope.Debug($"[6]: {currentFragment.ObjectName}"); #else Parallel.Invoke( () => @@ -112,6 +120,7 @@ private void ConsumeFragment( { this.ConsumeGlobalVariable(context, variable); } + scope.Debug($"[1]: {currentFragment.ObjectName}"); }, () => { @@ -119,6 +128,7 @@ private void ConsumeFragment( { this.ConsumeGlobalConstant(context, constant); } + scope.Debug($"[2]: {currentFragment.ObjectName}"); }, () => { @@ -126,6 +136,7 @@ private void ConsumeFragment( { this.ConsumeFunction(context, function); } + scope.Debug($"[3]: {currentFragment.ObjectName}"); }, () => { @@ -133,6 +144,7 @@ private void ConsumeFragment( { this.ConsumeInitializer(context, initializer); } + scope.Debug($"[4]: {currentFragment.ObjectName}"); }, () => { @@ -140,6 +152,7 @@ private void ConsumeFragment( { this.ConsumeEnumeration(context, enumeration); } + scope.Debug($"[5]: {currentFragment.ObjectName}"); }, () => { @@ -147,6 +160,7 @@ private void ConsumeFragment( { this.ConsumeStructure(context, structure); } + scope.Debug($"[6]: {currentFragment.ObjectName}"); }); #endif } @@ -219,6 +233,9 @@ public bool ConsumeInputs( InputFragment[] inputFragments, bool isLocationOriginSource) { + using var scope = this.logger.BeginScope(LogLevels.Debug); + scope.Debug($"InputFragments={inputFragments.Length}"); + //////////////////////////////////// // Step 1. Consume all objects. @@ -246,13 +263,17 @@ public bool ConsumeInputs( return false; } + scope.Debug("Step 2"); + //////////////////////////////////// // Step 2. Consume scheduled object referring in archives. - + this.ConsumeArchivedObject( inputFragments, isLocationOriginSource); + scope.Debug("Finished"); + return !this.caughtError; } } diff --git a/chibild/chibild.core/Generating/CodeGenerator_Consumer.cs b/chibild/chibild.core/Generating/CodeGenerator_Consumer.cs index 26c8fd6..19c5c80 100644 --- a/chibild/chibild.core/Generating/CodeGenerator_Consumer.cs +++ b/chibild/chibild.core/Generating/CodeGenerator_Consumer.cs @@ -50,9 +50,8 @@ private void ConsumeGlobalVariable( out var declaredFragment) && declaredFragment != context.CurrentFragment) { - this.OutputTrace( - globalVariable.Name.Token, - $"Declaration ignored, because already declared in: {declaredFragment.ObjectPath}"); + this.logger.Debug( + $"{globalVariable.Name.Token}: Declaration '{globalVariable.Name}' ignored, because already declared in: {declaredFragment.ObjectPath}"); return; } @@ -113,9 +112,8 @@ private void ConsumeGlobalConstant( out var declaredFragment) && declaredFragment != context.CurrentFragment) { - this.OutputTrace( - globalConstant.Name.Token, - $"Declaration ignored, because already declared in: {declaredFragment.ObjectPath}"); + this.logger.Debug( + $"{globalConstant.Name.Token}: Declaration '{globalConstant.Name}' ignored, because already declared in: {declaredFragment.ObjectPath}"); return; } @@ -853,9 +851,8 @@ private void ConsumeEnumeration( out var declaredFragment) && declaredFragment != context.CurrentFragment) { - this.OutputTrace( - enumeration.Name.Token, - $"Declaration ignored, because already declared in: {declaredFragment.ObjectPath}"); + this.logger.Debug( + $"{enumeration.Name.Token}: Declaration '{enumeration.Name}' ignored, because already declared in: {declaredFragment.ObjectPath}"); return; } @@ -947,9 +944,8 @@ private void ConsumeStructure( out var declaredFragment) && declaredFragment != context.CurrentFragment) { - this.OutputTrace( - structure.Name.Token, - $"Declaration ignored, because already declared in: {declaredFragment.ObjectPath}"); + this.logger.Debug( + $"{structure.Name.Token}: Declaration '{structure.Name}' ignored, because already declared in: {declaredFragment.ObjectPath}"); return; } diff --git a/chibild/chibild.core/Generating/CodeGenerator_Emit.cs b/chibild/chibild.core/Generating/CodeGenerator_Emit.cs index 12de7d1..296a69e 100644 --- a/chibild/chibild.core/Generating/CodeGenerator_Emit.cs +++ b/chibild/chibild.core/Generating/CodeGenerator_Emit.cs @@ -349,6 +349,8 @@ public Mono.Collections.Generic.Collection GetDataInitializerBody() private void EmitMembers( ObjectInputFragment fragment) { + using var scope = this.logger.BeginScope(LogLevels.Debug); + var holder = new TypeDefinitionsHolder( fragment, this.targetModule); @@ -364,7 +366,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted enumerations [file]: {count}"); + scope.Debug($"Total emitted enumerations [file]: {count}"); count = 0; foreach (var structure in fragment.GetDeclaredStructures(true)) @@ -375,7 +377,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted structures [file]: {count}"); + scope.Debug($"Total emitted structures [file]: {count}"); count = 0; foreach (var function in fragment.GetDeclaredFunctions(true)) @@ -386,7 +388,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted functions [file]: {count}"); + scope.Debug($"Total emitted functions [file]: {count}"); count = 0; foreach (var variable in fragment.GetDeclaredVariables(true)) @@ -397,7 +399,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted variables [file]: {count}"); + scope.Debug($"Total emitted variables [file]: {count}"); count = 0; foreach (var constant in fragment.GetDeclaredConstants(true)) @@ -408,7 +410,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted constants [file]: {count}"); + scope.Debug($"Total emitted constants [file]: {count}"); var fileScopedInitializerNames = (holder.GetFileScopedTypeIfAvailable()?. Methods. @@ -438,7 +440,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted initializers [file]: {count}"); + scope.Debug($"Total emitted initializers [file]: {count}"); /////////////////////////////////////////////////// @@ -449,7 +451,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted enumerations [public/internal]: {count}"); + scope.Debug($"Total emitted enumerations [public/internal]: {count}"); count = 0; foreach (var structure in fragment.GetDeclaredStructures(false)) @@ -458,7 +460,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted structures [public/internal]: {count}"); + scope.Debug($"Total emitted structures [public/internal]: {count}"); count = 0; foreach (var function in fragment.GetDeclaredFunctions(false)) @@ -469,7 +471,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted functions [public/internal]: {count}"); + scope.Debug($"Total emitted functions [public/internal]: {count}"); count = 0; foreach (var variable in fragment.GetDeclaredVariables(false)) @@ -480,7 +482,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted variables [public/internal]: {count}"); + scope.Debug($"Total emitted variables [public/internal]: {count}"); count = 0; foreach (var constant in fragment.GetDeclaredConstants(false)) @@ -491,7 +493,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted constants [public/internal]: {count}"); + scope.Debug($"Total emitted constants [public/internal]: {count}"); var initializerNames = (holder.GetDataTypeIfAvailable()?. Methods. @@ -521,7 +523,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted initializers [internal]: {count}"); + scope.Debug($"Total emitted initializers [internal]: {count}"); /////////////////////////////////////////////////// @@ -537,7 +539,7 @@ private void EmitMembers( count++; } - this.logger.Trace($"Total emitted module functions: {count}"); + scope.Debug($"Total emitted module functions: {count}"); } /////////////////////////////////////////////////////////////////////////////// @@ -545,6 +547,8 @@ private void EmitMembers( private void OptimizeMethods( ObjectInputFragment fragment) { + using var scope = this.logger.BeginScope(LogLevels.Debug); + var count = 0; foreach (var function in fragment.GetDeclaredFunctions(true)) { @@ -552,7 +556,7 @@ private void OptimizeMethods( count++; } - this.logger.Trace($"Total optimized methods [file]: {count}"); + scope.Debug($"Total optimized methods [file]: {count}"); count = 0; foreach (var function in fragment.GetDeclaredFunctions(false)) @@ -561,7 +565,7 @@ private void OptimizeMethods( count++; } - this.logger.Trace($"Total optimized methods [public/internal]: {count}"); + scope.Debug($"Total optimized methods [public/internal]: {count}"); count = 0; foreach (var initializer in fragment.GetDeclaredInitializer(true)) @@ -570,7 +574,7 @@ private void OptimizeMethods( count++; } - this.logger.Trace($"Total optimized methods [file initializer]: {count}"); + scope.Debug($"Total optimized methods [file initializer]: {count}"); count = 0; foreach (var initializer in fragment.GetDeclaredInitializer(false)) @@ -579,7 +583,7 @@ private void OptimizeMethods( count++; } - this.logger.Trace($"Total optimized methods [public/internal initializer]: {count}"); + scope.Debug($"Total optimized methods [public/internal initializer]: {count}"); } /////////////////////////////////////////////////////////////////////////////// @@ -783,6 +787,8 @@ private void InsertPrependExecutionPath( private void InvokeDelayedLookingUps() { + using var scope = this.logger.BeginScope(LogLevels.Debug); + // Apply all delayed looking up (types). var count = 0; while (this.delayLookingUpEntries1.Count >= 1) @@ -792,7 +798,7 @@ private void InvokeDelayedLookingUps() count++; } - this.logger.Trace($"Total delayed looking up [1]: {count}"); + scope.Debug($"Total delayed looking up [1]: {count}"); // Apply all delayed looking up (not types). count = 0; @@ -803,7 +809,7 @@ private void InvokeDelayedLookingUps() count++; } - this.logger.Trace($"Total delayed looking up [2]: {count}"); + scope.Debug($"Total delayed looking up [2]: {count}"); // Assert reverse dependencies are nothing. Debug.Assert(this.delayLookingUpEntries1.Count == 0); @@ -816,6 +822,8 @@ public bool Emit( LinkerCreationOptions? creationOptions, string[] prependExecutionSearchPaths) { + using var scope = this.logger.BeginScope(LogLevels.Debug); + // Try add fundamental attributes. if (creationOptions != null) { @@ -836,6 +844,8 @@ public bool Emit( this.AddPointerVisualizerAttributes( inputFragments); } + + scope.Debug("Step 2"); // Combine all object fragments into target module. foreach (var fragment in inputFragments. @@ -843,9 +853,13 @@ public bool Emit( { this.EmitMembers(fragment); } + + scope.Debug("Step 3"); // Invoke delayed looking ups. this.InvokeDelayedLookingUps(); + + scope.Debug("Step 4"); // Load CABI main object. if (creationOptions is { } co && @@ -864,6 +878,8 @@ public bool Emit( this.InvokeDelayedLookingUps(); } } + + scope.Debug("Step 5"); // Assign entry point. if (creationOptions is { } co2 && @@ -872,6 +888,8 @@ public bool Emit( this.AssignEntryPoint( co2.EntryPointSymbol); } + + scope.Debug("Step 6"); // Insert prepend search path. if (prependExecutionSearchPaths.Length >= 1 && @@ -882,6 +900,8 @@ public bool Emit( prependExecutionSearchPaths, targetMethod); } + + scope.Debug("Step 7"); // Apply method optimization for all object fragments. if (applyOptimization) @@ -892,6 +912,8 @@ public bool Emit( this.OptimizeMethods(fragment); } } + + scope.Debug("Step 8"); // Apply all delayed debugger information. if (this.produceDebuggingInformation) @@ -907,6 +929,8 @@ public bool Emit( Debug.Assert(this.delayDebuggingInsertionEntries.Count == 0); // (Completed all CIL implementations in this place.) + + scope.Debug("Finished"); /////////////////////////////////////////////// diff --git a/chibild/chibild.core/Generating/ObjectFileInputFragment.cs b/chibild/chibild.core/Generating/ObjectFileInputFragment.cs index 7e1c753..f6c1874 100644 --- a/chibild/chibild.core/Generating/ObjectFileInputFragment.cs +++ b/chibild/chibild.core/Generating/ObjectFileInputFragment.cs @@ -12,6 +12,7 @@ using chibicc.toolchain.Tokenizing; using chibild.Internal; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; @@ -144,6 +145,8 @@ public static bool TryLoad( { logger.Information($"Loading: {relativePath}"); + using var scope = logger.BeginScope(LogLevels.Debug); + var parser = new CilParser(logger); var declarations = parser.Parse( CilTokenizer.TokenizeAll(baseInputPath, relativePath, tr), @@ -180,6 +183,9 @@ public static bool TryLoad( initializers, enumerations, structures); + + scope.Debug($"Loaded: {relativePath}"); + return !parser.CaughtError; } } diff --git a/chibild/chibild/Program.cs b/chibild/chibild/Program.cs index b339882..2dce9f0 100644 --- a/chibild/chibild/Program.cs +++ b/chibild/chibild/Program.cs @@ -40,6 +40,7 @@ public static int Main(string[] args) } using var logger = new TextWriterLogger( + "chibild", options.LogLevel, Console.Out); diff --git a/toolchain.common/Logging/ILogger.cs b/toolchain.common/Logging/ILogger.cs index 9b2508b..e091cbd 100644 --- a/toolchain.common/Logging/ILogger.cs +++ b/toolchain.common/Logging/ILogger.cs @@ -13,8 +13,8 @@ namespace chibicc.toolchain.Logging; public enum LogLevels { - Debug = 1, - Trace, + Trace = 1, + Debug, Information, Warning, Error, diff --git a/toolchain.common/Logging/LoggerBase.cs b/toolchain.common/Logging/LoggerBase.cs index 72c92c9..43062e4 100644 --- a/toolchain.common/Logging/LoggerBase.cs +++ b/toolchain.common/Logging/LoggerBase.cs @@ -8,15 +8,26 @@ ///////////////////////////////////////////////////////////////////////////////////// using System; +using System.Diagnostics; +using System.Threading; namespace chibicc.toolchain.Logging; public abstract class LoggerBase : ILogger { + private static readonly int processId = Process.GetCurrentProcess().Id; + private static int sequenceId; + + private readonly string prefix; + public readonly LogLevels BaseLevel; - protected LoggerBase(LogLevels baseLevel) => + protected LoggerBase(string prefix, LogLevels baseLevel) + { this.BaseLevel = baseLevel; + var id = Interlocked.Increment(ref sequenceId); + this.prefix = $"{prefix} [{processId}{(id <= 1 ? string.Empty : $",{id}")}]:"; + } public void OutputLog( LogLevels logLevel, string? message, Exception? ex) @@ -27,23 +38,25 @@ public void OutputLog( } } + private static string GetLogLevelString(LogLevels logLevel) => + logLevel != LogLevels.Information ? + $" {logLevel.ToString().ToLowerInvariant()}:" : + string.Empty; + protected virtual string? ToString( LogLevels logLevel, string? message, Exception? ex) { - static string GetLogLevelString(LogLevels logLevel) => - logLevel != LogLevels.Information ? $" {logLevel.ToString().ToLowerInvariant()}:" : ""; - if (message is { } && ex is { }) { - return $"chibild:{GetLogLevelString(logLevel)} {message}, {ex}"; + return $"{prefix}{GetLogLevelString(logLevel)} {message}, {ex}"; } else if (message is { }) { - return $"chibild:{GetLogLevelString(logLevel)} {message}"; + return $"{prefix}{GetLogLevelString(logLevel)} {message}"; } else if (ex is { }) { - return $"chibild:{GetLogLevelString(logLevel)} {ex}"; + return $"{prefix}{GetLogLevelString(logLevel)} {ex}"; } else { diff --git a/toolchain.common/Logging/LoggerExtension.cs b/toolchain.common/Logging/LoggerExtension.cs index 51538bc..b907aa0 100644 --- a/toolchain.common/Logging/LoggerExtension.cs +++ b/toolchain.common/Logging/LoggerExtension.cs @@ -8,9 +8,15 @@ ///////////////////////////////////////////////////////////////////////////////////// using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; namespace chibicc.toolchain.Logging; +public interface IDisposableLogger : ILogger, IDisposable +{ +} + public static class LoggerExtension { public static void Debug(this ILogger logger, string message) => @@ -35,4 +41,49 @@ public static void Error(this ILogger logger, Exception ex) => logger.OutputLog(LogLevels.Error, null, ex); public static void Error(this ILogger logger, Exception ex, string message) => logger.OutputLog(LogLevels.Error, message, ex); + + public static IDisposableLogger BeginScope( + this ILogger logger, + LogLevels logLevel, + [CallerMemberName] string memberName = null!) + { + logger.OutputLog(logLevel, $"{memberName}: Started.", null); + return new DisposableLogger(logger, logLevel, memberName); + } + + private sealed class DisposableLogger : IDisposableLogger + { + private readonly ILogger parent; + private readonly LogLevels logLevel; + private readonly string memberName; + private readonly Stopwatch sw; + + public DisposableLogger( + ILogger parent, LogLevels logLevel, string memberName) + { + this.parent = parent; + this.logLevel = logLevel; + this.memberName = memberName; + this.sw = Stopwatch.StartNew(); + } + + public void Dispose() + { + if (this.sw.IsRunning) + { + this.sw.Stop(); + this.parent.OutputLog( + this.logLevel, + $"{this.memberName}: Exited, Elapsed={this.sw.Elapsed}", + null); + } + } + + public void OutputLog( + LogLevels logLevel, string? message, Exception? ex) => + this.parent.OutputLog( + logLevel, + $"{this.memberName}: {message}, Elapsed={this.sw.Elapsed}", + ex); + } } diff --git a/toolchain.common/Logging/TextWriterLogger.cs b/toolchain.common/Logging/TextWriterLogger.cs index a93b742..15d7ae3 100644 --- a/toolchain.common/Logging/TextWriterLogger.cs +++ b/toolchain.common/Logging/TextWriterLogger.cs @@ -16,8 +16,8 @@ public sealed class TextWriterLogger : LoggerBase, IDisposable { public readonly TextWriter Writer; - public TextWriterLogger(LogLevels baseLevel, TextWriter tw) : - base(baseLevel) => + public TextWriterLogger(string prefix, LogLevels baseLevel, TextWriter tw) : + base(prefix, baseLevel) => this.Writer = tw; public void Dispose() =>