Merge pull request #91 from coreruleset/improve-include-handling

feat: improve include handling

.github/workflows/lint.yml

@@ -11,6 +11,6 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
-          go-version: v1.20.x
+          go-version: v1.21.x
           cache: true
       - run: go run mage.go lint

.github/workflows/regression.yml

@@ -19,7 +19,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.19.x, 1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     steps:

go.mod

@@ -1,49 +1,50 @@
 module github.com/coreruleset/crs-toolchain
 
-go 1.20
+go 1.21
 
 require (
-	github.com/imdario/mergo v0.3.15
+	github.com/imdario/mergo v0.3.16
 	github.com/magefile/mage v1.15.0
 	github.com/spf13/cobra v1.7.0
 )
 
 require (
-	github.com/creativeprojects/go-selfupdate v1.1.0
+	github.com/creativeprojects/go-selfupdate v1.1.1
 	github.com/google/uuid v1.3.0
 	github.com/itchyny/rassemble-go v0.1.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	code.gitea.io/sdk/gitea v0.15.1 // indirect
+	code.gitea.io/sdk/gitea v0.16.0 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davidmz/go-pageant v1.0.2 // indirect
+	github.com/go-fed/httpsig v1.1.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-github/v30 v30.1.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/kr/pretty v0.2.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
-	github.com/xanzy/go-gitlab v0.83.0 // indirect
-	golang.org/x/crypto v0.9.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/oauth2 v0.8.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
+	github.com/xanzy/go-gitlab v0.92.3 // indirect
+	golang.org/x/crypto v0.13.0 // indirect
+	golang.org/x/oauth2 v0.12.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 )
 
 require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/rs/zerolog v1.29.1
+	github.com/rs/zerolog v1.31.0
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.8.2
 )

regex/parser/include_except_builder.go

@@ -5,6 +5,8 @@ package parser
 
 import (
 	"bufio"
+	"bytes"
+	"regexp"
 	"sort"
 	"strings"
 )
@@ -30,34 +32,66 @@ func (h inclusionLineSlice) Swap(i, j int) {
 	h[i], h[j] = h[j], h[i]
 }
 
-func buildIncludeExceptString(parser *Parser, parsedLine ParsedLine) string {
-	includeFileName := parsedLine.result[0]
-	excludeFileName := parsedLine.result[1]
+func buildIncludeString(parser *Parser, parsedLine ParsedLine) (string, error) {
+	content, _ := parseFile(parser, parsedLine.includeFileName, nil)
+	return replaceSuffixes(content, parsedLine.suffixReplacements)
+}
 
+func buildIncludeExceptString(parser *Parser, parsedLine ParsedLine) (string, error) {
 	// 1. build a map with lines as keys for fast access;
-	//    store the line itself and its position in the value (a inclusionLine) for later
+	//    store the line itself and its position in the value (an inclusionLine) for later
 	// 2. remove exclusions from the map
 	// 3. put the inclusionLines back into an array, still out of order
 	// 4. build the resulting string by sorting the array and joining the lines
-	includeMap, definitions := buildinclusionLineMap(parser, includeFileName)
-	removeExclusions(parser, excludeFileName, includeMap, definitions)
+	includeMap, definitions := buildinclusionLineMap(parser, parsedLine.includeFileName)
+	removeExclusions(parser, parsedLine.excludeFileNames, includeMap, definitions)
 
 	inclusionLines := make(inclusionLineSlice, 0, len(includeMap))
 	for _, value := range includeMap {
 		inclusionLines = append(inclusionLines, value)
 	}
 
-	return stringFromInclusionLines(inclusionLines)
+	contentWithoutExclusions := stringFromInclusionLines(inclusionLines)
+	return replaceSuffixes(bytes.NewBufferString(contentWithoutExclusions), parsedLine.suffixReplacements)
 }
 
-func removeExclusions(parser *Parser, excludeFileName string, includeMap map[string]inclusionLine, definitions map[string]string) {
-	excludeContent, _ := parseFile(parser, excludeFileName, definitions)
-	scanner := bufio.NewScanner(excludeContent)
+func replaceSuffixes(inputLines *bytes.Buffer, suffixReplacements map[string]string) (string, error) {
+	if suffixReplacements == nil {
+		return inputLines.String(), nil
+	}
+
+	var sb strings.Builder
+	scanner := bufio.NewScanner(inputLines)
 	scanner.Split(bufio.ScanLines)
+	skipRegex := regexp.MustCompile(`^(?:##!|\s*$)`)
 	for scanner.Scan() {
-		exclusion := scanner.Text()
-		delete(includeMap, exclusion)
-		logger.Debug().Msgf("Excluded entry from include file: %s", exclusion)
+		entry := scanner.Text()
+		if !skipRegex.MatchString(entry) {
+			for match, replacement := range suffixReplacements {
+				var found bool
+				entry, found = strings.CutSuffix(entry, match)
+				if found {
+					entry += replacement
+				}
+			}
+		}
+		sb.WriteString(entry)
+		sb.WriteRune('\n')
+	}
+	return sb.String(), nil
+}
+
+func removeExclusions(parser *Parser, excludeFileNames []string, includeMap map[string]inclusionLine, definitions map[string]string) {
+	for _, fileName := range excludeFileNames {
+		logger.Debug().Msgf("Processing exclusions from %s", fileName)
+		excludeContent, _ := parseFile(parser, fileName, definitions)
+		scanner := bufio.NewScanner(excludeContent)
+		scanner.Split(bufio.ScanLines)
+		for scanner.Scan() {
+			exclusion := scanner.Text()
+			delete(includeMap, exclusion)
+			logger.Debug().Msgf("Excluded entry from include file: %s", exclusion)
+		}
 	}
 }
 

regex/parser/include_except_test.go

@@ -182,3 +182,78 @@ func (s *parserIncludeExceptTestSuite) TestIncludeExcept_DontPanicWhenExclusions
 
 	s.Equal("include1\n", actual.String())
 }
+
+func (s *parserIncludeExceptTestSuite) TestIncludeExcept_SuffixReplacements() {
+	includePath := s.writeFile(`no suffix1
+suffix with@
+suffix with~
+no suffix 2`,
+		s.includeDir)
+	excludePath := s.writeFile("no suffix1", s.excludeDir)
+	assemblyPath := s.writeFile(
+		fmt.Sprintf(
+			"##!> include-except %s %s -- %s %s %s %s", includePath, excludePath,
+			"@", `[\s><]`,
+			"~", `[^\s]`),
+		s.assemblyDir)
+
+	assemblyFile, err := os.Open(assemblyPath)
+	s.Require().NoError(err)
+	defer assemblyFile.Close()
+
+	parser := NewParser(s.ctx, assemblyFile)
+	actual, _ := parser.Parse(false)
+	expected := `suffix with[\s><]
+suffix with[^\s]
+no suffix 2
+`
+
+	s.Equal(expected, actual.String())
+}
+
+func (s *parserIncludeExceptTestSuite) TestIncludeExcept_MultipleExcludes() {
+	includePath := s.writeFile(`\s*include1
+leave me alone`, s.includeDir)
+	excludePath1 := s.writeFile(`\s*include1
+include2`, s.excludeDir)
+	excludePath2 := s.writeFile(`[a-c]include4+
+a*b|include3`, s.excludeDir)
+	assemblyPath := s.writeFile(fmt.Sprintf("##!> include-except %s %s %s", includePath, excludePath1, excludePath2), s.assemblyDir)
+
+	assemblyFile, err := os.Open(assemblyPath)
+	s.Require().NoError(err)
+	defer assemblyFile.Close()
+
+	parser := NewParser(s.ctx, assemblyFile)
+	actual, _ := parser.Parse(false)
+
+	s.Equal("leave me alone\n", actual.String())
+}
+
+func (s *parserIncludeExceptTestSuite) TestIncludeExcept_SuffixReplacements_WithMultipleExclusions() {
+	includePath := s.writeFile(`no suffix1
+suffix with@
+suffix with~
+no suffix 2`,
+		s.includeDir)
+	excludePath1 := s.writeFile("no suffix1", s.excludeDir)
+	excludePath2 := s.writeFile("no suffix 2", s.excludeDir)
+	assemblyPath := s.writeFile(
+		fmt.Sprintf(
+			"##!> include-except %s %s %s -- %s %s %s %s", includePath, excludePath1, excludePath2,
+			"@", `[\s><]`,
+			"~", `[^\s]`),
+		s.assemblyDir)
+
+	assemblyFile, err := os.Open(assemblyPath)
+	s.Require().NoError(err)
+	defer assemblyFile.Close()
+
+	parser := NewParser(s.ctx, assemblyFile)
+	actual, _ := parser.Parse(false)
+	expected := `suffix with[\s><]
+suffix with[^\s]
+`
+
+	s.Equal(expected, actual.String())
+}

regex/parser/include_test.go

@@ -132,3 +132,47 @@ data regex
 
 	s.Equal(expected.String(), actual.String())
 }
+
+func (s *parserIncludeTestSuite) TestParserInclude_SuffixReplacements() {
+	_, err := s.includeFile.WriteString(`no suffix1
+suffix with@
+suffix with~
+no suffix 2`)
+	s.Require().NoError(err, "writing temp include file failed")
+
+	s.reader = strings.NewReader(fmt.Sprintf(
+		"##!> include %s -- %s %s %s %s", s.includeFile.Name(),
+		"@", `[\s><]`,
+		"~", `[^\s]`))
+	parser := NewParser(s.ctx, s.reader)
+	actual, _ := parser.Parse(false)
+	expected := bytes.NewBufferString(`no suffix1
+suffix with[\s><]
+suffix with[^\s]
+no suffix 2
+`)
+
+	s.Equal(expected.String(), actual.String())
+}
+
+func (s *parserIncludeTestSuite) TestParserInclude_SuffixReplacements_WithBacSpacing() {
+	_, err := s.includeFile.WriteString(`no suffix1
+suffix with@
+suffix with~
+no suffix 2`)
+	s.Require().NoError(err, "writing temp include file failed")
+
+	s.reader = strings.NewReader(fmt.Sprintf(
+		"##!> include %s--   %s %s %s %s", s.includeFile.Name(),
+		"  @", `[\s><]  `,
+		"~   ", `   [^\s]`))
+	parser := NewParser(s.ctx, s.reader)
+	actual, _ := parser.Parse(false)
+	expected := bytes.NewBufferString(`no suffix1
+suffix with[\s><]
+suffix with[^\s]
+no suffix 2
+`)
+
+	s.Equal(expected.String(), actual.String())
+}