fix(parser): handle flags, prefixes, and suffixes in include files (#42)

In the toolchain, the parser holds the flags, prefixes and suffixes
and the assembler uses them when finalizing the assembly. The parsers
of include files, however, aren't known to the assembler. In addition,
the flags, prefixes, and suffixes in include files must be scoped to the
include file.

This commit adds a new function to post process include file parsing.
The included content is enclosed in a new assemble block and flags,
prefixes, and suffixes are perpended / appended to the content.

* Improve include file parsing and regular expressions

Only wrap include file output in an assemble block if it is necessary,
otherwise we might modify semantics

regex/definitions.go

@@ -14,7 +14,7 @@ var IncludeRegex = regexp.MustCompile(`^##!>\s*include\s*(.*)$`)
 var DefinitionRegex = regexp.MustCompile(`^##!>\s*define\s+([a-zA-Z0-9-_]+)\s+(.*)$`)
 
 // CommentRegex matches a comment line (##!, no other directives)
-var CommentRegex = regexp.MustCompile(`^##![^^$+><=]`)
+var CommentRegex = regexp.MustCompile(`^##!(?:[^^$+><=]|$)`)
 
 // FlagsRegex matches a flags line (##!+ <value>).
 // The value is captured in group 1.

regex/operators/assembler_test.go

@@ -95,6 +95,7 @@ func (s *definitionsTestSuite) TearDownSuite() {
 
 func (s *fileFormatTestSuite) TestPreprocessIgnoresSimpleComments() {
 	contents := `##!line1
+##!
 ##! line2
 ##!\tline3
 `

regex/parser/parser.go

@@ -14,6 +14,7 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
 
 	"github.com/imdario/mergo"
@@ -118,7 +119,7 @@ func (p *Parser) Parse(formatOnly bool) (*bytes.Buffer, int) {
 		case include:
 			if !formatOnly {
 				// go read the included file and paste text here
-				content, _ := includeFile(p.ctx, parsedLine.result[includePatternName])
+				content, _ := includeFile(p, parsedLine.result[includePatternName])
 				text = content.String()
 			}
 		case flags:
@@ -197,7 +198,7 @@ func (p *Parser) parseLine(line string) ParsedLine {
 }
 
 // includeFile does just a new call to the Parser on the named file. It will use the context to find files that have relative filenames.
-func includeFile(ctx *processors.Context, includeName string) (*bytes.Buffer, int) {
+func includeFile(rootParser *Parser, includeName string) (*bytes.Buffer, int) {
 	filename := includeName
 	logger.Trace().Msgf("reading include file: %v", filename)
 	if path.Ext(filename) != ".data" {
@@ -207,14 +208,56 @@ func includeFile(ctx *processors.Context, includeName string) (*bytes.Buffer, in
 	// check if filename has an absolute path
 	// if it is relative, use the context to get the parent directory where we should search for the file.
 	if !filepath.IsAbs(filename) {
-		filename = filepath.Join(ctx.RootContext().IncludeDir(), filename)
+		filename = filepath.Join(rootParser.ctx.RootContext().IncludeDir(), filename)
 	}
 	readFile, err := os.Open(filename)
 	if err != nil {
 		logger.Fatal().Msgf("cannot open file for inclusion: %v", err.Error())
 	}
-	newP := NewParser(ctx, bufio.NewReader(readFile))
-	return newP.Parse(false)
+	newP := NewParser(rootParser.ctx, bufio.NewReader(readFile))
+	out, _ := newP.Parse(false)
+	newOut := mergeFlagsPrefixesSuffixes(rootParser, newP, out)
+	logger.Trace().Msg(newOut.String())
+	return newOut, newOut.Len()
+}
+
+// Merge flags, prefixes, and suffixes from include files into another parser.
+// All of these need to be treated as local to the source parser.
+func mergeFlagsPrefixesSuffixes(target *Parser, source *Parser, out *bytes.Buffer) *bytes.Buffer {
+	logger.Trace().Msg("merging flags, prefixes, suffixes from included file")
+	// IMPORTANT: don't write the assemble block at all if there are no flags, prefixes, or
+	// suffixes. Enclosing the output in an assemble block can change the semantics, for example,
+	// when the included content is processed by the cmdline processor in the including file.
+	if len(source.Flags) == 0 && len(source.Prefixes) == 0 && len(source.Suffixes) == 0 {
+		return out
+	}
+
+	newOut := new(bytes.Buffer)
+	newOut.WriteString("##!> assemble\n")
+
+	if len(source.Flags) > 0 {
+		flags := make([]string, 0, len(source.Flags))
+		for flag := range source.Flags {
+			flags = append(flags, string(flag))
+		}
+		sort.Strings(flags)
+		newOut.WriteString("(?" + strings.Join(flags, "") + ")")
+		newOut.WriteString("\n##!=>\n")
+	}
+	for _, prefix := range source.Prefixes {
+		newOut.WriteString(prefix)
+		newOut.WriteString("\n##!=>\n")
+	}
+	if _, err := out.WriteTo(newOut); err != nil {
+		logger.Fatal().Err(err).Msg("failed to copy output to new buffer")
+	}
+	for _, suffix := range source.Suffixes {
+		newOut.WriteString("\n##!=>\n")
+		newOut.WriteString(suffix)
+		newOut.WriteString("\n##!=>\n")
+	}
+	newOut.WriteString("##!<\n")
+	return newOut
 }
 
 func expandDefinitions(src *bytes.Buffer, variables map[string]string) *bytes.Buffer {

regex/parser/parser_test.go

@@ -97,7 +97,7 @@ type parserIncludeTestSuite struct {
 	includeFile *os.File
 }
 
-func (s *parserIncludeTestSuite) SetupSuite() {
+func (s *parserIncludeTestSuite) SetupTest() {
 	var err error
 	s.tempDir, err = os.MkdirTemp("", "include-tests")
 	s.NoError(err)
@@ -109,28 +109,112 @@ func (s *parserIncludeTestSuite) SetupSuite() {
 	s.ctx = processors.NewContext(s.tempDir)
 	s.includeFile, err = os.Create(path.Join(s.includeDir, "test.data"))
 	s.NoError(err, "couldn't create %s file", s.includeFile.Name())
-
-	n, err := s.includeFile.WriteString("This data comes from the included file.\n")
-	s.NoError(err, "writing temp include file failed")
-
-	s.Equal(len("This data comes from the included file.\n"), n)
-	s.reader = strings.NewReader(fmt.Sprintf("##!> include %s\n##! This is a comment line.\n", s.includeFile.Name()))
 }
 
-func (s *parserIncludeTestSuite) TearDownSuite() {
+func (s *parserIncludeTestSuite) TearDownTest() {
 	s.NoError(s.includeFile.Close())
 	s.NoError(os.RemoveAll(s.tempDir))
 }
 
 func (s *parserIncludeTestSuite) TestParserInclude_FromFile() {
+	s.writeDataFile("This data comes from the include file.\n", "##!This is a comment\n")
 	parser := NewParser(s.ctx, s.reader)
 	actual, n := parser.Parse(false)
-	expected := bytes.NewBufferString("This data comes from the included file.\n")
+	expected := bytes.NewBufferString("This data comes from the include file.\n")
 
 	s.Equal(expected.String(), actual.String())
 	s.Equal(expected.Len(), n)
 }
 
+func (s *parserIncludeTestSuite) TestParserInclude_Flags() {
+	s.writeDataFile(`##!+si
+included regex`, "data regex")
+	parser := NewParser(s.ctx, s.reader)
+	actual, _ := parser.Parse(false)
+	expected := bytes.NewBufferString(`##!> assemble
+(?is)
+##!=>
+included regex
+##!<
+data regex
+`)
+
+	s.Equal(expected.String(), actual.String())
+}
+
+func (s *parserIncludeTestSuite) TestParserInclude_Prefixes() {
+	s.writeDataFile(`##!^ prefix1
+##!^ prefix2
+included regex`, "data regex")
+	parser := NewParser(s.ctx, s.reader)
+	actual, _ := parser.Parse(false)
+	expected := bytes.NewBufferString(`##!> assemble
+prefix1
+##!=>
+prefix2
+##!=>
+included regex
+##!<
+data regex
+`)
+	s.Equal(expected.String(), actual.String())
+}
+
+func (s *parserIncludeTestSuite) TestParserInclude_Suffixes() {
+	s.writeDataFile(`##!$ suffix1
+##!$ suffix2
+included regex`, "data regex")
+	parser := NewParser(s.ctx, s.reader)
+	actual, _ := parser.Parse(false)
+	expected := bytes.NewBufferString(`##!> assemble
+included regex
+
+##!=>
+suffix1
+##!=>
+
+##!=>
+suffix2
+##!=>
+##!<
+data regex
+`)
+
+	s.Equal(expected.String(), actual.String())
+}
+
+func (s *parserIncludeTestSuite) TestParserInclude_FlagsPrefixesSuffixes() {
+	s.writeDataFile(`##!$ suffix1
+##!$ suffix2
+##!^ prefix1
+##!^ prefix2
+##!+ si
+included regex`, "data regex")
+	parser := NewParser(s.ctx, s.reader)
+	actual, _ := parser.Parse(false)
+	expected := bytes.NewBufferString(`##!> assemble
+(?is)
+##!=>
+prefix1
+##!=>
+prefix2
+##!=>
+included regex
+
+##!=>
+suffix1
+##!=>
+
+##!=>
+suffix2
+##!=>
+##!<
+data regex
+`)
+
+	s.Equal(expected.String(), actual.String())
+}
+
 // Test Suite to perform multiple inclusions
 type parserMultiIncludeTestSuite struct {
 	suite.Suite
@@ -175,7 +259,11 @@ func (s *parserMultiIncludeTestSuite) TearDownSuite() {
 func (s *parserMultiIncludeTestSuite) TestParserMultiInclude_FromMultiFile() {
 	parser := NewParser(s.ctx, s.reader)
 	actual, n := parser.Parse(false)
-	expected := bytes.NewBufferString("This is comment 3.\nThis is comment 2.\nThis is comment 1.\nThis is comment 0.\n")
+	expected := bytes.NewBufferString(
+		"This is comment 3.\n" +
+			"This is comment 2.\n" +
+			"This is comment 1.\n" +
+			"This is comment 0.\n")
 
 	s.Equal(expected.String(), actual.String())
 	s.Equal(expected.Len(), n)
@@ -286,3 +374,10 @@ func (s *parserIncludeWithDefinitions) TestParser_DanglingDefinitions() {
 
 	s.Contains(out.String(), "no match found for definition: {{hallo}}")
 }
+
+func (s *parserIncludeTestSuite) writeDataFile(includeContents string, dataContents string) {
+	_, err := s.includeFile.WriteString(includeContents)
+	s.NoError(err, "writing temp include file failed")
+
+	s.reader = strings.NewReader(fmt.Sprintf("##!> include %s\n%s", s.includeFile.Name(), dataContents))
+}