Add support for ignoring files via .cpeignore

Introduced the ability to ignore files during code map generation, type resolution, and system message building by loading patterns from a .cpeignore file. Integrated the glob library for pattern matching and updated the related functions to consider these ignore rules.

codemap/output.go

@@ -2,6 +2,7 @@ package codemap
 
 import (
 	"fmt"
+	"github.com/spachava753/cpe/ignore"
 	sitter "github.com/tree-sitter/go-tree-sitter"
 	"io/fs"
 	"path/filepath"
@@ -31,13 +32,13 @@ type FileCodeMap struct {
 }
 
 // GenerateOutput creates the code map output for each file using AST
-func GenerateOutput(fsys fs.FS, maxLiteralLen int) ([]FileCodeMap, error) {
+func GenerateOutput(fsys fs.FS, maxLiteralLen int, ignoreRules *ignore.IgnoreRules) ([]FileCodeMap, error) {
 	var filePaths []string
 	err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
-		if !d.IsDir() && isSourceCode(path) {
+		if !d.IsDir() && isSourceCode(path) && !ignoreRules.ShouldIgnore(path) {
 			filePaths = append(filePaths, path)
 		}
 		return nil

go.mod

@@ -5,6 +5,7 @@ go 1.23
 toolchain go1.23.2
 
 require (
+	github.com/gobwas/glob v0.2.3
 	github.com/google/generative-ai-go v0.18.0
 	github.com/googleapis/gax-go/v2 v2.13.0
 	github.com/pkoukk/tiktoken-go v0.1.7

go.sum

@@ -31,6 +31,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=

ignore/ignore.go

@@ -0,0 +1,72 @@
+package ignore
+
+import (
+	"bufio"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gobwas/glob"
+)
+
+type IgnoreRules struct {
+	patterns []glob.Glob
+}
+
+func NewIgnoreRules() *IgnoreRules {
+	return &IgnoreRules{
+		patterns: []glob.Glob{},
+	}
+}
+
+func (ir *IgnoreRules) LoadIgnoreFile(startDir string) error {
+	ignoreFile := findIgnoreFile(startDir)
+	if ignoreFile == "" {
+		return nil // No .cpeignore file found, which is okay
+	}
+
+	file, err := os.Open(ignoreFile)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		pattern := strings.TrimSpace(scanner.Text())
+		if pattern != "" && !strings.HasPrefix(pattern, "#") {
+			g, err := glob.Compile(pattern)
+			if err != nil {
+				return err
+			}
+			ir.patterns = append(ir.patterns, g)
+		}
+	}
+
+	return scanner.Err()
+}
+
+func (ir *IgnoreRules) ShouldIgnore(path string) bool {
+	for _, pattern := range ir.patterns {
+		if pattern.Match(path) {
+			return true
+		}
+	}
+	return false
+}
+
+func findIgnoreFile(startDir string) string {
+	dir := startDir
+	for {
+		ignoreFile := filepath.Join(dir, ".cpeignore")
+		if _, err := os.Stat(ignoreFile); err == nil {
+			return ignoreFile
+		}
+		parent := filepath.Dir(dir)
+		if parent == dir {
+			break
+		}
+		dir = parent
+	}
+	return ""
+}

main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/spachava753/cpe/codemap"
 	"github.com/spachava753/cpe/extract"
 	"github.com/spachava753/cpe/fileops"
+	"github.com/spachava753/cpe/ignore"
 	"github.com/spachava753/cpe/llm"
 	"github.com/spachava753/cpe/tiktokenloader"
 	"github.com/spachava753/cpe/typeresolver"
@@ -24,8 +25,8 @@ func logTimeElapsed(start time.Time, operation string) {
 	fmt.Printf("Time elapsed for %s: %v\n", operation, elapsed)
 }
 
-func generateCodeMapOutput(maxLiteralLen int) (string, error) {
-	fileCodeMaps, err := codemap.GenerateOutput(os.DirFS("."), maxLiteralLen)
+func generateCodeMapOutput(maxLiteralLen int, ignoreRules *ignore.IgnoreRules) (string, error) {
+	fileCodeMaps, err := codemap.GenerateOutput(os.DirFS("."), maxLiteralLen, ignoreRules)
 	if err != nil {
 		return "", fmt.Errorf("error generating code map output: %w", err)
 	}
@@ -91,13 +92,13 @@ func performCodeMapAnalysis(provider llm.LLMProvider, genConfig llm.GenConfig, c
 	return nil, fmt.Errorf("no files selected for analysis")
 }
 
-func buildSystemMessageWithSelectedFiles(allFiles []string) (string, error) {
+func buildSystemMessageWithSelectedFiles(allFiles []string, ignoreRules *ignore.IgnoreRules) (string, error) {
 	var systemMessage strings.Builder
 	systemMessage.WriteString(CodeAnalysisModificationPrompt)
 
 	// Use the current directory for resolveTypeFiles
 	currentDir := "."
-	resolvedFiles, err := typeresolver.ResolveTypeAndFunctionFiles(allFiles, os.DirFS(currentDir))
+	resolvedFiles, err := typeresolver.ResolveTypeAndFunctionFiles(allFiles, os.DirFS(currentDir), ignoreRules)
 	if err != nil {
 		return "", fmt.Errorf("error resolving type files: %w", err)
 	}
@@ -178,6 +179,13 @@ func main() {
 
 	flags := ParseFlags()
 
+	// Initialize ignore rules
+	ignoreRules := ignore.NewIgnoreRules()
+	if err := ignoreRules.LoadIgnoreFile("."); err != nil {
+		fmt.Printf("Error loading .cpeignore file: %v\n", err)
+		return
+	}
+
 	if flags.Version {
 		fmt.Printf("cpe version %s\n", version)
 		return
@@ -244,7 +252,7 @@ func main() {
 		// Generate low-fidelity code map output
 		codeMapStart := time.Now()
 		maxLiteralLen := 100 // You can adjust this value or make it configurable
-		codeMapOutput, err := generateCodeMapOutput(maxLiteralLen)
+		codeMapOutput, err := generateCodeMapOutput(maxLiteralLen, ignoreRules)
 		logTimeElapsed(codeMapStart, "generateCodeMapOutput")
 		if err != nil {
 			fmt.Printf("Error generating code map output: %v\n", err)
@@ -270,7 +278,7 @@ func main() {
 		allFiles := append(selectedFiles, includeFiles...)
 		// Build system message with all files
 		buildMessageStart := time.Now()
-		systemMessage, err = buildSystemMessageWithSelectedFiles(allFiles)
+		systemMessage, err = buildSystemMessageWithSelectedFiles(allFiles, ignoreRules)
 		logTimeElapsed(buildMessageStart, "buildSystemMessageWithSelectedFiles")
 		if err != nil {
 			fmt.Println("Error building system message:", err)

typeresolver/resolver.go

@@ -2,6 +2,7 @@ package typeresolver
 
 import (
 	"fmt"
+	"github.com/spachava753/cpe/ignore"
 	"io/fs"
 	"strings"
 
@@ -10,7 +11,7 @@ import (
 )
 
 // ResolveTypeAndFunctionFiles resolves all type and function definitions used in the given files
-func ResolveTypeAndFunctionFiles(selectedFiles []string, sourceFS fs.FS) (map[string]bool, error) {
+func ResolveTypeAndFunctionFiles(selectedFiles []string, sourceFS fs.FS, ignoreRules *ignore.IgnoreRules) (map[string]bool, error) {
 	typeDefinitions := make(map[string]map[string]string)     // package.type -> file
 	functionDefinitions := make(map[string]map[string]string) // package.function -> file
 	usages := make(map[string]bool)
@@ -75,7 +76,7 @@ func ResolveTypeAndFunctionFiles(selectedFiles []string, sourceFS fs.FS) (map[st
 		if err != nil {
 			return err
 		}
-		if !d.IsDir() && strings.HasSuffix(path, ".go") {
+		if !d.IsDir() && strings.HasSuffix(path, ".go") && !ignoreRules.ShouldIgnore(path) {
 			content, readErr := fs.ReadFile(sourceFS, path)
 			if readErr != nil {
 				return fmt.Errorf("error reading file %s: %w", path, readErr)