Skip to content

Commit

Permalink
Refactored Tool a bit to have a method to get the results as a string…
Browse files Browse the repository at this point in the history 
… via API + some other refactorings

Signed-off-by: KOLANICH <[email protected]>
  • Loading branch information
@KOLANICH
KOLANICH committed Mar 29, 2023
1 parent 6f2c050 commit 58f7c38
Show file tree
Hide file tree
Showing 9 changed files with 593 additions and 79 deletions.
80 changes: 61 additions & 19 deletions tool/src/org/antlr/v4/Tool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
import org.antlr.v4.automata.ParserATNFactory;
import org.antlr.v4.codegen.CodeGenPipeline;
import org.antlr.v4.codegen.CodeGenerator;
import org.antlr.v4.codegen.InMemoryCodeGenerator;
import org.antlr.v4.codegen.inMemoryResult.InMemoryCodeGenResult;
import org.antlr.v4.codegen.inMemoryResult.InMemoryFile;
import org.antlr.v4.codegen.inMemoryResult.DataFiles;
import org.antlr.v4.misc.Graph;
import org.antlr.v4.parse.ANTLRParser;
import org.antlr.v4.parse.GrammarASTAdaptor;
Expand Down Expand Up @@ -320,13 +324,23 @@ else if (errMgr.getNumErrors() == 0) {
}
}

public void process(Grammar g, boolean gencode) {
process(g, gencode, null);
}

public InMemoryCodeGenResult processInMemory(Grammar g, boolean gencode) {
InMemoryCodeGenResult res = new InMemoryCodeGenResult();
process(g, gencode, res);
return res;
}

/** To process a grammar, we load all of its imported grammars into
subordinate grammar objects. Then we merge the imported rules
into the root grammar. If a root grammar is a combined grammar,
we have to extract the implicit lexer. Once all this is done, we
process the lexer first, if present, and then the parser grammar
*/
public void process(Grammar g, boolean gencode) {
public void process(Grammar g, boolean gencode, InMemoryCodeGenResult res) {
g.loadImportedGrammars();

GrammarTransformPipeline transform = new GrammarTransformPipeline(g, this);
Expand All @@ -348,50 +362,67 @@ public void process(Grammar g, boolean gencode) {
lexerg.originalGrammar = g;
g.implicitLexer = lexerg;
lexerg.implicitLexerOwner = g;
processNonCombinedGrammar(lexerg, gencode);
processNonCombinedGrammar(lexerg, gencode, res);
// System.out.println("lexer tokens="+lexerg.tokenNameToTypeMap);
// System.out.println("lexer strings="+lexerg.stringLiteralToTypeMap);
}
}
if ( g.implicitLexer!=null ) g.importVocab(g.implicitLexer);
// System.out.println("tokens="+g.tokenNameToTypeMap);
// System.out.println("strings="+g.stringLiteralToTypeMap);
processNonCombinedGrammar(g, gencode);
processNonCombinedGrammar(g, gencode, res);
}

public void processNonCombinedGrammar(Grammar g, boolean gencode) {
if ( g.ast==null || g.ast.hasErrors ) return;
public boolean prepareProcessNonCombinedGrammar(Grammar g, boolean gencode) {
if ( g.ast==null || g.ast.hasErrors ) return false;

boolean ruleFail = checkForRuleIssues(g);
if ( ruleFail ) return;
if ( ruleFail ) return false;

int prevErrors = errMgr.getNumErrors();
// MAKE SURE GRAMMAR IS SEMANTICALLY CORRECT (FILL IN GRAMMAR OBJECT)
SemanticPipeline sem = new SemanticPipeline(g);
sem.process();

if ( errMgr.getNumErrors()>prevErrors ) return;
if ( errMgr.getNumErrors()>prevErrors ) return false;

CodeGenerator codeGenerator = CodeGenerator.create(g);
if (codeGenerator == null) {
return;
return false;
}

if ( errMgr.getNumErrors()>prevErrors ) return false;

// BUILD ATN FROM AST
ATNFactory factory;
if ( g.isLexer() ) factory = new LexerATNFactory((LexerGrammar)g, codeGenerator);
else factory = new ParserATNFactory(g);
g.atn = factory.createATN();

if ( generate_ATN_dot ) generateATNs(g);
return true;
}

if (gencode && g.tool.getNumErrors()==0 ) {
String interpFile = generateInterpreterData(g);
try (Writer fw = getOutputFileWriter(g, getInterpFileName(g))) {
fw.write(interpFile);
}
catch (IOException ioe) {
errMgr.toolError(ErrorType.CANNOT_WRITE_FILE, ioe);
public void processNonCombinedGrammar(Grammar g, boolean gencode) {
processNonCombinedGrammar(g, gencode, null);
}

public void processNonCombinedGrammar(Grammar g, boolean gencode, InMemoryCodeGenResult res) {
if (!prepareProcessNonCombinedGrammar(g, gencode)) return;
int prevErrors = errMgr.getNumErrors();

if (gencode && g.tool.getNumErrors()==0 ){
if(res != null){
DataFiles selectedDFiles = (g.isLexer()?res.lexerData:res.mainData);
selectedDFiles.interp = new InMemoryFile(getInterpFileName(g), generateInterpreterData(g));
}else{
String interpFile = generateInterpreterData(g);
try (Writer fw = getOutputFileWriter(g, g.name + ".interp")) {
fw.write(interpFile);
}
catch (IOException ioe) {
errMgr.toolError(ErrorType.CANNOT_WRITE_FILE, ioe);
}
}
}

Expand All @@ -405,11 +436,23 @@ public void processNonCombinedGrammar(Grammar g, boolean gencode) {

// GENERATE CODE
if ( gencode ) {
CodeGenerator codeGenerator = null;
if(res != null){
codeGenerator = InMemoryCodeGenerator.create(g, res);
} else {
codeGenerator = CodeGenerator.create(g);
}
CodeGenPipeline gen = new CodeGenPipeline(g, codeGenerator);
gen.process();
gen.process(res);
}
}

public InMemoryCodeGenResult processNonCombinedGrammarInMemory(Grammar g) {
InMemoryCodeGenResult res = new InMemoryCodeGenResult();
processNonCombinedGrammar(g, true, res);
return res;
}

// Don't inline: used in https://github.com/antlr/antlr4/issues/3874
protected static String getInterpFileName(Grammar g){
return g.name + ".interp";
Expand Down Expand Up @@ -691,7 +734,7 @@ public void generateATNs(Grammar g) {
writeDOTFile(g, r, dot);
}
}
catch (IOException ioe) {
catch (IOException ioe) {
errMgr.toolError(ErrorType.CANNOT_WRITE_FILE, ioe);
}
}
Expand Down Expand Up @@ -745,7 +788,6 @@ public static String generateInterpreterData(Grammar g) {

content.append("atn:\n");
content.append(serializedATN.toString());

return content.toString();
}

Expand All @@ -767,7 +809,7 @@ public static String generateInterpreterData(Grammar g) {
* If outputDirectory==null then write a String.
*/
public Writer getOutputFileWriter(Grammar g, String fileName) throws IOException {
if (outputDirectory == null) {
if (outputDirectory == null || fileName == null) {
return new StringWriter();
}
// output directory is a function of where the grammar file lives
Expand Down
138 changes: 79 additions & 59 deletions tool/src/org/antlr/v4/codegen/CodeGenPipeline.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,100 +10,120 @@
import org.antlr.v4.tool.Rule;
import org.stringtemplate.v4.ST;
import org.stringtemplate.v4.gui.STViz;
import org.antlr.v4.codegen.inMemoryResult.InMemoryCodeGenResult;
import org.antlr.v4.codegen.inMemoryResult.InMemoryFile;

public class CodeGenPipeline {
final Grammar g;
final CodeGenerator gen;
int errorCount;

public CodeGenPipeline(Grammar g, CodeGenerator gen) {
this.g = g;
this.gen = gen;
}

public void process() {
// all templates are generated in memory to report the most complete
// error information possible, but actually writing output files stops
// after the first error is reported
int errorCount = g.tool.errMgr.getNumErrors();

if ( g.isLexer() ) {
if (gen.getTarget().needsHeader()) {
ST lexer = gen.generateLexer(true); // Header file if needed.
if (g.tool.errMgr.getNumErrors() == errorCount) {
writeRecognizer(lexer, gen, true);
}
public void processLexer(CodeGenerator gen, Grammar g){
if (gen.getTarget().needsHeader()) {
ST lexer = gen.generateLexer(true); // Header file if needed.
if (g.tool.errMgr.getNumErrors() == errorCount) {
writeRecognizer(lexer, gen, true);
}
ST lexer = gen.generateLexer(false);
}
ST lexer = gen.generateLexer(false);
if (g.tool.errMgr.getNumErrors() == errorCount) {
writeRecognizer(lexer, gen, false);
}
}

public void processParser(CodeGenerator gen, Grammar g){
if (gen.getTarget().needsHeader()) {
ST parser = gen.generateParser(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
writeRecognizer(lexer, gen, false);
writeRecognizer(parser, gen, true);
}
}
else {
ST parser = gen.generateParser(false);
if (g.tool.errMgr.getNumErrors() == errorCount) {
writeRecognizer(parser, gen, false);
}

if ( g.tool.gen_listener ) {
if (gen.getTarget().needsHeader()) {
ST parser = gen.generateParser(true);
ST listener = gen.generateListener(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
writeRecognizer(parser, gen, true);
gen.writeListener(listener, true);
}
}
ST parser = gen.generateParser(false);
ST listener = gen.generateListener(false);
if (g.tool.errMgr.getNumErrors() == errorCount) {
writeRecognizer(parser, gen, false);
gen.writeListener(listener, false);
}

if ( g.tool.gen_listener ) {
if (gen.getTarget().needsHeader()) {
ST listener = gen.generateListener(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeListener(listener, true);
}
}
ST listener = gen.generateListener(false);
if (gen.getTarget().needsHeader()) {
ST baseListener = gen.generateBaseListener(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeListener(listener, false);
}

if (gen.getTarget().needsHeader()) {
ST baseListener = gen.generateBaseListener(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeBaseListener(baseListener, true);
}
}
if (gen.getTarget().wantsBaseListener()) {
ST baseListener = gen.generateBaseListener(false);
if ( g.tool.errMgr.getNumErrors()==errorCount ) {
gen.writeBaseListener(baseListener, false);
}
gen.writeBaseListener(baseListener, true);
}
}
if ( g.tool.gen_visitor ) {
if (gen.getTarget().needsHeader()) {
ST visitor = gen.generateVisitor(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeVisitor(visitor, true);
}
if (gen.getTarget().wantsBaseListener()) {
ST baseListener = gen.generateBaseListener(false);
if ( g.tool.errMgr.getNumErrors()==errorCount ) {
gen.writeBaseListener(baseListener, false);
}
ST visitor = gen.generateVisitor(false);
}
}
if ( g.tool.gen_visitor ) {
if (gen.getTarget().needsHeader()) {
ST visitor = gen.generateVisitor(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeVisitor(visitor, false);
gen.writeVisitor(visitor, true);
}
}
ST visitor = gen.generateVisitor(false);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeVisitor(visitor, false);
}

if (gen.getTarget().needsHeader()) {
ST baseVisitor = gen.generateBaseVisitor(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeBaseVisitor(baseVisitor, true);
}
if (gen.getTarget().needsHeader()) {
ST baseVisitor = gen.generateBaseVisitor(true);
if (g.tool.errMgr.getNumErrors() == errorCount) {
gen.writeBaseVisitor(baseVisitor, true);
}
if (gen.getTarget().wantsBaseVisitor()) {
ST baseVisitor = gen.generateBaseVisitor(false);
if ( g.tool.errMgr.getNumErrors()==errorCount ) {
gen.writeBaseVisitor(baseVisitor, false);
}
}
if (gen.getTarget().wantsBaseVisitor()) {
ST baseVisitor = gen.generateBaseVisitor(false);
if ( g.tool.errMgr.getNumErrors()==errorCount ) {
gen.writeBaseVisitor(baseVisitor, false);
}
}
}
}

public void processWithCodeGen(CodeGenerator gen){
// all templates are generated in memory to report the most complete
// error information possible, but actually writing output files stops
// after the first error is reported
errorCount = g.tool.errMgr.getNumErrors();

if ( g.isLexer() ) {
processLexer(gen, g);
}
else {
processParser(gen, g);
}
}

public void process(InMemoryCodeGenResult res) {
if ( g.getLanguage() == null ) return;

//CodeGenerator gen = new CodeGenerator(null, g); // TODO
gen.result = res;
processWithCodeGen(gen);
gen.writeVocabFile();
}


protected void writeRecognizer(ST template, CodeGenerator gen, boolean header) {
if ( g.tool.launch_ST_inspector ) {
STViz viz = template.inspect();
Expand Down
6 changes: 5 additions & 1 deletion tool/src/org/antlr/v4/codegen/CodeGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@

import org.antlr.v4.Tool;
import org.antlr.v4.codegen.model.OutputModelObject;
import org.antlr.v4.codegen.InMemoryCodeGenerator;
import org.antlr.v4.codegen.inMemoryResult.InMemoryCodeGenResult;
import org.antlr.v4.codegen.inMemoryResult.InMemoryFile;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.tool.ErrorType;
import org.antlr.v4.tool.Grammar;
Expand Down Expand Up @@ -41,6 +44,8 @@ public class CodeGenerator {

public int lineWidth = 72;

public InMemoryCodeGenResult result = null;

public static CodeGenerator create(Grammar g) {
return create(g.tool, g, g.getLanguage());
}
Expand Down Expand Up @@ -220,5 +225,4 @@ public String getHeaderFileName() {
String recognizerName = g.getRecognizerName();
return recognizerName+extST.render();
}

}
Loading

0 comments on commit 58f7c38

Please sign in to comment.