Skip to content

Commit

Permalink
Reduce compiler allocations (#1530)
Browse files Browse the repository at this point in the history 
* Make tokens a struct instead of a class

* Cache a StringBuilder in `DMPreprocessorLexer`
It's not necessary to be creating a new one for every token that uses it
Can't do this in `LexString()` unfortunately because it has a nested `NextToken()` call

* Use a file-scoped namespace in `DMVisitorExpression.cs`

* Turn `DMVisitorExpression` into `DMExpressionBuilder`
Not a DMASTVisitor
Static class, doesn't need to be instantiated/allocated every time it's used

* Fix some inferred paths

* Make the cached StringBuilder `readonly`

* Cache a StringBuilder in `DMLexer`

* Use a file-scoped namespace in `DMLexer.cs`
  • Loading branch information
@wixoaGit
wixoaGit authored Nov 8, 2023
1 parent ebcb4b1 commit 03fcfcb
Show file tree
Hide file tree
Showing 11 changed files with 1,265 additions and 1,469 deletions.
593 changes: 298 additions & 295 deletions DMCompiler/Compiler/DM/DMLexer.cs
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion DMCompiler/Compiler/DMPreprocessor/DMMacro.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public bool HasParameters() {
/// <returns>A list of tokens replacing the identifier</returns>
/// <exception cref="ArgumentException">Thrown if no parameters were given but are required</exception>
// TODO: Convert this to an IEnumerator<Token>? Could cut down on allocations.
public virtual List<Token> Expand(Token replacing, List<List<Token>> parameters) {
public virtual List<Token> Expand(Token replacing, List<List<Token>>? parameters) {
// If this macro has no parameters then we can just return our list of tokens
if (!HasParameters())
return _tokens;
Expand Down
20 changes: 10 additions & 10 deletions DMCompiler/Compiler/DMPreprocessor/DMPreprocessor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using DMCompiler.Compiler.DM;
Expand Down Expand Up @@ -473,11 +474,11 @@ private List<Token> GetLineOfTokens()
/// <returns>true if the Token ended up meaning a macro sequence.</returns>
private bool TryMacro(Token token) {
DebugTools.Assert(token.Type == TokenType.DM_Preproc_Identifier); // Check this before passing anything to this function.
if (!_defines.TryGetValue(token.Text, out DMMacro macro)) {
if (!_defines.TryGetValue(token.Text, out DMMacro? macro)) {
return false;
}

List<List<Token>> parameters = null;
List<List<Token>>? parameters = null;
if (macro.HasParameters() && !TryGetMacroParameters(out parameters)) {
return false;
}
Expand Down Expand Up @@ -598,7 +599,7 @@ private void PushToken(Token token) {
/// If whitespace may be important later, use <see cref="CheckForTokenIgnoringWhitespace(TokenType, out Token)"/>.
/// </remarks>
private Token GetNextToken(bool ignoreWhitespace = false) {
if (_unprocessedTokens.TryPop(out Token? nextToken)) {
if (_unprocessedTokens.TryPop(out Token nextToken)) {
if (ignoreWhitespace && nextToken.Type == TokenType.DM_Preproc_Whitespace) { // This doesn't need to be a loop since whitespace tokens should never occur next to each other
nextToken = GetNextToken(true);
}
Expand Down Expand Up @@ -683,7 +684,7 @@ private bool Check(TokenType tokenType) {
/// <summary>
/// The alternative to <see cref="GetNextToken(bool)"/> if you don't know whether you'll consume the whitespace or not.
/// </summary>
private bool CheckForTokenIgnoringWhitespace(TokenType type, out Token result) {
private bool CheckForTokenIgnoringWhitespace(TokenType type, [NotNullWhen(true)] out Token? result) {
Token firstToken = GetNextToken();
if (firstToken.Type == TokenType.DM_Preproc_Whitespace) { // This doesn't need to be a loop since whitespace tokens should never occur next to each other
Token secondToken = GetNextToken();
Expand All @@ -695,12 +696,10 @@ private bool CheckForTokenIgnoringWhitespace(TokenType type, out Token result) {
}
result = secondToken;
return true;
}
else if (firstToken.Type == type) {
} else if (firstToken.Type == type) {
result = firstToken;
return true;
}
else {
} else {
PushToken(firstToken);
result = null;
return false;
Expand Down Expand Up @@ -746,11 +745,12 @@ private bool SkipIfBody(bool calledByElseDirective = false) {
return false;
}

private bool TryGetMacroParameters(out List<List<Token>> parameters) {
private bool TryGetMacroParameters(out List<List<Token>>? parameters) {
if (!CheckForTokenIgnoringWhitespace(TokenType.DM_Preproc_Punctuator_LeftParenthesis, out var leftParenToken)) {
parameters = null;
return false;
}

parameters = new();
List<Token> currentParameter = new();

Expand Down Expand Up @@ -797,7 +797,7 @@ private bool TryGetMacroParameters(out List<List<Token>> parameters) {

parameters.Add(currentParameter);
if (parameterToken.Type != TokenType.DM_Preproc_Punctuator_RightParenthesis) {
DMCompiler.Emit(WarningCode.BadDirective, leftParenToken.Location, "Missing ')' in macro call");
DMCompiler.Emit(WarningCode.BadDirective, leftParenToken.Value.Location, "Missing ')' in macro call");

return false;
}
Expand Down
63 changes: 33 additions & 30 deletions DMCompiler/Compiler/DMPreprocessor/DMPreprocessorLexer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ namespace DMCompiler.Compiler.DMPreprocessor;
/// taking in raw text and outputting vague tokens descriptive enough for the preprocessor to run on them.
/// </summary>
internal sealed class DMPreprocessorLexer {
private static readonly StringBuilder TokenTextBuilder = new();

public readonly string IncludeDirectory;
public readonly string File;

Expand Down Expand Up @@ -288,15 +290,15 @@ public Token NextToken(bool ignoreWhitespace = false) {
}
case '@': { //Raw string
char delimiter = Advance();
StringBuilder textBuilder = new StringBuilder();

textBuilder.Append('@');
textBuilder.Append(delimiter);
TokenTextBuilder.Clear();
TokenTextBuilder.Append('@');
TokenTextBuilder.Append(delimiter);

bool isLong = false;
c = Advance();
if (delimiter == '{') {
textBuilder.Append(c);
TokenTextBuilder.Append(c);

if (c == '"') isLong = true;
}
Expand All @@ -309,7 +311,7 @@ public Token NextToken(bool ignoreWhitespace = false) {
if(nextCharCanTerm && c == '}')
break;
else {
textBuilder.Append(c);
TokenTextBuilder.Append(c);
nextCharCanTerm = false;
}

Expand All @@ -319,16 +321,16 @@ public Token NextToken(bool ignoreWhitespace = false) {
} while (!AtEndOfSource());
} else {
while (c != delimiter && !AtLineEnd() && !AtEndOfSource()) {
textBuilder.Append(c);
TokenTextBuilder.Append(c);
c = Advance();
}
}

textBuilder.Append(c);
TokenTextBuilder.Append(c);
if (!HandleLineEnd())
Advance();

string text = textBuilder.ToString();
string text = TokenTextBuilder.ToString();
string value = isLong ? text.Substring(3, text.Length - 5) : text.Substring(2, text.Length - 3);
return CreateToken(TokenType.DM_Preproc_ConstantString, text, value);
}
Expand All @@ -348,45 +350,48 @@ public Token NextToken(bool ignoreWhitespace = false) {
Advance();
}

StringBuilder textBuilder = new StringBuilder();
TokenTextBuilder.Clear();
while (char.IsAsciiLetter(GetCurrent()) || GetCurrent() == '_') {
textBuilder.Append(GetCurrent());
TokenTextBuilder.Append(GetCurrent());
Advance();
}

string text = textBuilder.ToString();
string text = TokenTextBuilder.ToString();
if (text == string.Empty) {
return NextToken(ignoreWhitespace); // Skip this token
} else if (isConcat) {
return CreateToken(TokenType.DM_Preproc_TokenConcat, $"##{text}", text);
}

if (TryMacroKeyword(text, out var macroKeyword))
return macroKeyword;
return macroKeyword.Value;

string macroAttempt = text.ToLower();
if (TryMacroKeyword(macroAttempt, out var attemptKeyword)) { // if they mis-capitalized the keyword
DMCompiler.Emit(WarningCode.MiscapitalizedDirective, attemptKeyword.Location,
DMCompiler.Emit(WarningCode.MiscapitalizedDirective, attemptKeyword.Value.Location,
$"#{text} is not a valid macro keyword. Did you mean '#{macroAttempt}'?");
}

return CreateToken(TokenType.DM_Preproc_ParameterStringify, $"#{text}", text);
}
default: {
if (char.IsAsciiLetter(c) || c == '_') {
StringBuilder textBuilder = new StringBuilder(char.ToString(c));
TokenTextBuilder.Clear();
TokenTextBuilder.Append(c);
while ((char.IsAsciiLetterOrDigit(Advance()) || GetCurrent() == '_') && !AtEndOfSource())
textBuilder.Append(GetCurrent());
TokenTextBuilder.Append(GetCurrent());

return CreateToken(TokenType.DM_Preproc_Identifier, textBuilder.ToString());
return CreateToken(TokenType.DM_Preproc_Identifier, TokenTextBuilder.ToString());
} else if (char.IsAsciiDigit(c)) {
StringBuilder textBuilder = new StringBuilder(char.ToString(c));
bool error = false;

TokenTextBuilder.Clear();
TokenTextBuilder.Append(c);

while (!AtEndOfSource()) {
char next = Advance();
if ((c == 'e' || c == 'E') && (next == '-' || next == '+')) { //1e-10 or 1e+10
textBuilder.Append(next);
TokenTextBuilder.Append(next);
next = Advance();
} else if (c == '#' && next == 'I') { //1.#INF and 1.#IND
if (Advance() != 'N' || Advance() != 'F' && GetCurrent() != 'D') {
Expand All @@ -395,22 +400,22 @@ public Token NextToken(bool ignoreWhitespace = false) {
break;
}

textBuilder.Append("IN");
textBuilder.Append(GetCurrent());
TokenTextBuilder.Append("IN");
TokenTextBuilder.Append(GetCurrent());
next = Advance();
}

c = next;
if (char.IsAsciiHexDigit(c) || c == '.' || c == 'x' || c == '#' || c == 'e' || c == 'E' || c == 'p' || c == 'P') {
textBuilder.Append(c);
TokenTextBuilder.Append(c);
} else {
break;
}
}

return error
? CreateToken(TokenType.Error, string.Empty, "Invalid number")
: CreateToken(TokenType.DM_Preproc_Number, textBuilder.ToString());
: CreateToken(TokenType.DM_Preproc_Number, TokenTextBuilder.ToString());
}

Advance();
Expand All @@ -424,25 +429,23 @@ private bool TryMacroKeyword(string text, [NotNullWhen(true)] out Token? token)
switch (text) {
case "warn":
case "warning": {
StringBuilder message = new StringBuilder();

TokenTextBuilder.Clear();
while (!AtEndOfSource() && !AtLineEnd()) {
message.Append(GetCurrent());
TokenTextBuilder.Append(GetCurrent());
Advance();
}

token = CreateToken(TokenType.DM_Preproc_Warning, "#warn" + message);
token = CreateToken(TokenType.DM_Preproc_Warning, "#warn" + TokenTextBuilder);
break;
}
case "error": {
StringBuilder message = new StringBuilder();

TokenTextBuilder.Clear();
while (!AtEndOfSource() && !AtLineEnd()) {
message.Append(GetCurrent());
TokenTextBuilder.Append(GetCurrent());
Advance();
}

token = CreateToken(TokenType.DM_Preproc_Error, "#error" + message);
token = CreateToken(TokenType.DM_Preproc_Error, "#error" + TokenTextBuilder);
break;
}
case "include": token = CreateToken(TokenType.DM_Preproc_Include, "#include"); break;
Expand Down
Loading

0 comments on commit 03fcfcb

Please sign in to comment.