Use a stream when preprocessing a file

Rather than read & allocate all the text in every file
OpenDreamProject · Nov 3, 2023 · 189adce · 189adce
1 parent 22cf6a9
commit 189adce
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 10 deletions.
diff --git a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessor.cs b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessor.cs
@@ -253,10 +253,8 @@ public void IncludeFile(string includeDir, string file, Location? includedFrom =
 
         public void PreprocessFile(string includeDir, string file) {
             string filePath = Path.Combine(includeDir, file).Replace('\\', Path.DirectorySeparatorChar);
-            string source = File.ReadAllText(filePath);
-            source += '\n';
 
-            _lexerStack.Push(new DMPreprocessorLexer(includeDir, file.Replace('\\', Path.DirectorySeparatorChar), source));
+            _lexerStack.Push(new DMPreprocessorLexer(includeDir, filePath));
         }
 
         private bool VerifyDirectiveUsage(Token token) {

diff --git a/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs b/DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs
@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using System.Runtime.CompilerServices;
 using System.Text;
 using OpenDreamShared.Compiler;
@@ -13,14 +14,25 @@ internal sealed class DMPreprocessorLexer {
         public readonly string IncludeDirectory;
         public readonly string File;
 
-        private readonly string _source;
-        private int _currentIndex, _currentLine = 1, _currentColumn = 1;
+        private readonly StreamReader _source;
+        private char _current;
+        private int _currentLine = 1, _currentColumn;
         private readonly Queue<Token> _pendingTokenQueue = new(); // TODO: Possible to remove this?
 
         public DMPreprocessorLexer(string includeDirectory, string file, string source) {
             IncludeDirectory = includeDirectory;
             File = file;
-            _source = source;
+
+            _source = new StreamReader(new MemoryStream(Encoding.ASCII.GetBytes(source)), Encoding.ASCII);
+            Advance();
+        }
+
+        public DMPreprocessorLexer(string includeDirectory, string file) {
+            IncludeDirectory = includeDirectory;
+            File = file;
+
+            _source = new StreamReader(Path.Combine(includeDirectory, file), Encoding.ASCII);
+            Advance();
         }
 
         public Token NextToken(bool ignoreWhitespace = false) {
@@ -579,19 +591,26 @@ private bool HandleLineEnd() {
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private char GetCurrent() {
-            return AtEndOfSource() ? '\0' : _source[_currentIndex];
+            return _current;
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private char Advance() {
-            _currentColumn++;
-            _currentIndex++;
+            int value = _source.Read();
+
+            if (value == -1) {
+                _current = '\0';
+            }  else {
+                _currentColumn++;
+                _current = (char)value;
+            }
+
             return GetCurrent();
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private bool AtEndOfSource() {
-            return _currentIndex >= _source.Length;
+            return _current == '\0';
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]