diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b6ebeba..d4af5a6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -18,6 +18,17 @@ updates: schedule: interval: "weekly" target-branch: "dev" + commit-message: + prefix: "chore" + include: "scope" + labels: + - "Type: Maintenance" + - + package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + target-branch: "dev" commit-message: prefix: "chore" include: "scope" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fec6d63..25af8db 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,17 +29,18 @@ jobs: with: go-version: '>=1.23' - - name: Checkout the repository + name: Code Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - - name: Go modules hygine + - + name: Go Module Management run: | - go clean -modcache - go mod tidy + make go-mod-clean + make go-mod-tidy working-directory: . - - name: Go build - run: go build -v . - working-directory: ./cmd/xurlfind3r \ No newline at end of file + name: Go Build + run: | + make go-build + working-directory: . diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 5c29437..d23028c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -29,7 +29,7 @@ jobs: security-events: write steps: - - name: Checkout the repository + name: Code Checkout uses: actions/checkout@v4 with: fetch-depth: 0 @@ -38,9 +38,17 @@ jobs: uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} - - - name: Autobuild - uses: github/codeql-action/autobuild@v3 + - + name: Go Module Management + run: | + make go-mod-clean + make go-mod-tidy + working-directory: . + - + name: Go Build + run: | + make go-build + working-directory: . - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 \ No newline at end of file diff --git a/.github/workflows/dockerhub-push.yaml b/.github/workflows/dockerhub-push.yaml index 0371fb7..a33a229 100644 --- a/.github/workflows/dockerhub-push.yaml +++ b/.github/workflows/dockerhub-push.yaml @@ -9,7 +9,7 @@ on: jobs: push: - name: DockerHub Push + name: DockerHub Push runs-on: ubuntu-latest permissions: packages: write @@ -18,24 +18,23 @@ jobs: id-token: write steps: - - name: Checkout + name: Code Checkout uses: actions/checkout@v4 - + with: + fetch-depth: 0 - - name: Get Github tag + name: Get Github Tag id: meta run: | curl --silent "https://api.github.com/repos/hueristiq/xurlfind3r/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT - - name: Log in to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push Docker image + name: Build and Push Docker Image uses: docker/build-push-action@v6 with: context: . diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 34d36e6..a7a8ecb 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,14 +30,14 @@ jobs: go-version: '>=1.23' cache: false - - name: Checkout the repository + name: Code Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Run golangci-lint + name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.61.0 + version: v1.62.2 args: --timeout 5m working-directory: . \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad2bf28..02c482a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,12 +18,12 @@ jobs: with: go-version: '>=1.23' - - name: Checkout the repository + name: Code Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Run GoReleaser + name: GoReleaser uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser @@ -33,4 +33,4 @@ jobs: env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" - DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" \ No newline at end of file + DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" diff --git a/Dockerfile b/Dockerfile index d73b5c1..f3bb368 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ FROM golang:1.23.1-alpine3.20 AS build-stage RUN < <| |_| | | | | _| | | | | (_| |___) | | /_/\_\\__,_|_| |_|_| |_|_| |_|\__,_|____/|_| @@ -161,11 +159,11 @@ USAGE: xurlfind3r [OPTIONS] CONFIGURATION: - -c, --configuration string configuration file (default: $HOME/.config/xurlfind3r/config.yaml) + -c, --configuration string configuration file path (default: $HOME/.config/xurlfind3r/config.yaml) INPUT: -d, --domain string[] target domain - -l, --list string target domains' list file path + -l, --list string target domains list file path TIP: For multiple input domains use comma(,) separated value with `-d`, specify multiple `-d`, load from file with `-l` or load from stdin. @@ -174,40 +172,44 @@ SCOPE: --include-subdomains bool match subdomain's URLs SOURCES: - --sources bool list supported sources - -u, --use-sources string[] comma(,) separated sources to use + --sources bool list available sources -e, --exclude-sources string[] comma(,) separated sources to exclude + -u, --use-sources string[] comma(,) separated sources to use FILTER & MATCH: -f, --filter string regex to filter URLs -m, --match string regex to match URLs OUTPUT: - --no-color bool disable colored output + --json bool output URLs in JSONL format + --monochrome bool stdout monochrome output -o, --output string output URLs file path -O, --output-directory string output URLs directory path - -s, --silent bool display output subdomains only - -v, --verbose bool display verbose output + -s, --silent bool stdout URLs only output + -v, --verbose bool stdout verbose output -pflag: help requested ``` -### Examples +- Basic + +```bash +xurlfind3r -d hackerone.com +``` -#### Basic +- Include Subdomains ```bash xurlfind3r -d hackerone.com --include-subdomains ``` -#### Filter Regex +- Filter Regex ```bash # filter images xurlfind3r -d hackerone.com --include-subdomains -f '`^https?://[^/]*?/.*\.(jpg|jpeg|png|gif|bmp)(\?[^\s]*)?$`' ``` -#### Match Regex +- Match Regex ```bash # match js URLs diff --git a/cmd/xurlfind3r/main.go b/cmd/xurlfind3r/main.go index 42fb175..49f42d0 100644 --- a/cmd/xurlfind3r/main.go +++ b/cmd/xurlfind3r/main.go @@ -3,6 +3,7 @@ package main import ( "bufio" "fmt" + "io" "log" "os" "path/filepath" @@ -14,6 +15,8 @@ import ( "github.com/hueristiq/hqgolog/formatter" "github.com/hueristiq/hqgolog/levels" "github.com/hueristiq/xurlfind3r/internal/configuration" + "github.com/hueristiq/xurlfind3r/internal/input" + "github.com/hueristiq/xurlfind3r/internal/output" "github.com/hueristiq/xurlfind3r/pkg/xurlfind3r" "github.com/hueristiq/xurlfind3r/pkg/xurlfind3r/sources" "github.com/logrusorgru/aurora/v3" @@ -25,38 +28,46 @@ var ( au aurora.Aurora configurationFilePath string - domains []string - domainsListFilePath string - includeSubdomains bool - listSources bool - sourcesToUse []string - sourcesToExclude []string - filterPattern string - matchPattern string - monochrome bool - output string - outputDirectory string - silent bool - verbose bool + + inputDomains []string + inputDomainsListFilePath string + + includeSubdomains bool + + listSources bool + sourcesToExclude []string + sourcesToUse []string + + filterPattern string + matchPattern string + + outputInJSONL bool + monochrome bool + outputFilePath string + outputDirectoryPath string + silent bool + verbose bool ) func init() { pflag.StringVarP(&configurationFilePath, "configuration", "c", configuration.DefaultConfigurationFilePath, "") - pflag.StringSliceVarP(&domains, "domain", "d", []string{}, "") - pflag.StringVarP(&domainsListFilePath, "list", "l", "", "") + pflag.StringSliceVarP(&inputDomains, "domain", "d", []string{}, "") + pflag.StringVarP(&inputDomainsListFilePath, "list", "l", "", "") pflag.BoolVar(&includeSubdomains, "include-subdomains", false, "") + pflag.BoolVar(&listSources, "sources", false, "") - pflag.StringSliceVarP(&sourcesToUse, "use-sources", "u", []string{}, "") pflag.StringSliceVarP(&sourcesToExclude, "exclude-sources", "e", []string{}, "") + pflag.StringSliceVarP(&sourcesToUse, "use-sources", "u", []string{}, "") pflag.StringVarP(&filterPattern, "filter", "f", "", "") pflag.StringVarP(&matchPattern, "match", "m", "", "") - pflag.BoolVar(&monochrome, "no-color", false, "") - pflag.StringVarP(&output, "output", "o", "", "") - pflag.StringVarP(&outputDirectory, "output-directory", "O", "", "") + pflag.BoolVar(&outputInJSONL, "json", false, "") + pflag.BoolVar(&monochrome, "monochrome", false, "") + pflag.StringVarP(&outputFilePath, "output", "o", "", "") + pflag.StringVarP(&outputDirectoryPath, "output-directory", "O", "", "") pflag.BoolVarP(&silent, "silent", "s", false, "") pflag.BoolVarP(&verbose, "verbose", "v", false, "") @@ -69,11 +80,11 @@ func init() { h += "\nCONFIGURATION:\n" defaultConfigurationFilePath := strings.ReplaceAll(configuration.DefaultConfigurationFilePath, configuration.UserDotConfigDirectoryPath, "$HOME/.config") - h += fmt.Sprintf(" -c, --configuration string configuration file (default: %s)\n", defaultConfigurationFilePath) + h += fmt.Sprintf(" -c, --configuration string configuration file path (default: %s)\n", defaultConfigurationFilePath) h += "\nINPUT:\n" h += " -d, --domain string[] target domain\n" - h += " -l, --list string target domains' list file path\n" + h += " -l, --list string target domains list file path\n" h += "\nTIP: For multiple input domains use comma(,) separated value with `-d`,\n" h += " specify multiple `-d`, load from file with `-l` or load from stdin.\n" @@ -82,20 +93,21 @@ func init() { h += " --include-subdomains bool match subdomain's URLs\n" h += "\nSOURCES:\n" - h += " --sources bool list supported sources\n" - h += " -u, --use-sources string[] comma(,) separated sources to use\n" + h += " --sources bool list available sources\n" h += " -e, --exclude-sources string[] comma(,) separated sources to exclude\n" + h += " -u, --use-sources string[] comma(,) separated sources to use\n" h += "\nFILTER & MATCH:\n" h += " -f, --filter string regex to filter URLs\n" h += " -m, --match string regex to match URLs\n" h += "\nOUTPUT:\n" - h += " --no-color bool disable colored output\n" + h += " --json bool output URLs in JSONL format\n" + h += " --monochrome bool stdout monochrome output\n" h += " -o, --output string output URLs file path\n" h += " -O, --output-directory string output URLs directory path\n" - h += " -s, --silent bool display output subdomains only\n" - h += " -v, --verbose bool display verbose output\n" + h += " -s, --silent bool stdout URLs only output\n" + h += " -v, --verbose bool stdout verbose output\n" fmt.Fprintln(os.Stderr, h) } @@ -167,10 +179,10 @@ func main() { os.Exit(0) } - if domainsListFilePath != "" { + if inputDomainsListFilePath != "" { var file *os.File - file, err = os.Open(domainsListFilePath) + file, err = os.Open(inputDomainsListFilePath) if err != nil { hqgolog.Fatal().Msg(err.Error()) } @@ -181,23 +193,25 @@ func main() { domain := scanner.Text() if domain != "" { - domains = append(domains, domain) + inputDomains = append(inputDomains, domain) } } if err = scanner.Err(); err != nil { hqgolog.Fatal().Msg(err.Error()) } + + file.Close() } - if hasStdin() { + if input.HasStdin() { scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { domain := scanner.Text() if domain != "" { - domains = append(domains, domain) + inputDomains = append(inputDomains, domain) } } @@ -220,99 +234,63 @@ func main() { hqgolog.Fatal().Msg(err.Error()) } - var consolidatedWriter *bufio.Writer - - if output != "" { - directory := filepath.Dir(output) + outputWritter := output.NewWritter() - mkdir(directory) - - var consolidatedFile *os.File - - consolidatedFile, err = os.OpenFile(output, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) - if err != nil { - hqgolog.Fatal().Msg(err.Error()) - } - - defer consolidatedFile.Close() - - consolidatedWriter = bufio.NewWriter(consolidatedFile) + if outputInJSONL { + outputWritter.SetFormatToJSONL() } - if outputDirectory != "" { - mkdir(outputDirectory) - } + for index := range inputDomains { + domain := inputDomains[index] - for _, domain := range domains { if !silent { hqgolog.Print().Msg("") hqgolog.Info().Msgf("Finding URLs for %v...", au.Underline(domain).Bold()) hqgolog.Print().Msg("") } - results := finder.Find(domain) + writers := []io.Writer{ + os.Stdout, + } - switch { - case output != "": - outputURLs(consolidatedWriter, results) - case outputDirectory != "": - var domainFile *os.File + var file *os.File - domainFile, err = os.OpenFile(filepath.Join(outputDirectory, domain+".txt"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + switch { + case outputFilePath != "": + file, err = outputWritter.CreateFile(outputFilePath) if err != nil { - hqgolog.Fatal().Msg(err.Error()) + hqgolog.Error().Msg(err.Error()) } - domainWriter := bufio.NewWriter(domainFile) - - outputURLs(domainWriter, results) - default: - outputURLs(nil, results) - } - } -} - -func hasStdin() bool { - stat, err := os.Stdin.Stat() - if err != nil { - return false - } - - isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 - isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 - - return isPipedFromChrDev || isPipedFromFIFO -} + writers = append(writers, file) + case outputDirectoryPath != "": + file, err = outputWritter.CreateFile(filepath.Join(outputDirectoryPath, domain)) + if err != nil { + hqgolog.Error().Msg(err.Error()) + } -func mkdir(path string) { - if _, err := os.Stat(path); os.IsNotExist(err) { - if err = os.MkdirAll(path, os.ModePerm); err != nil { - hqgolog.Fatal().Msg(err.Error()) + writers = append(writers, file) } - } -} -func outputURLs(writer *bufio.Writer, URLs chan sources.Result) { - for URL := range URLs { - switch URL.Type { - case sources.ResultError: - if verbose { - hqgolog.Error().Msgf("%s: %s\n", URL.Source, URL.Error) - } - case sources.ResultURL: - if verbose { - hqgolog.Print().Msgf("[%s] %s", au.BrightBlue(URL.Source), URL.Value) - } else { - hqgolog.Print().Msg(URL.Value) - } - - if writer != nil { - fmt.Fprintln(writer, URL.Value) + results := finder.Find(domain) - if err := writer.Flush(); err != nil { - hqgolog.Fatal().Msg(err.Error()) + for result := range results { + for index := range writers { + writer := writers[index] + + switch result.Type { + case sources.ResultError: + if verbose { + hqgolog.Error().Msgf("%s: %s\n", result.Source, result.Error) + } + case sources.ResultURL: + if err := outputWritter.Write(writer, domain, result); err != nil { + hqgolog.Error().Msg(err.Error()) + } } } } + + file.Close() } } diff --git a/internal/configuration/configuration.go b/internal/configuration/configuration.go index a1d725c..4d6ab10 100644 --- a/internal/configuration/configuration.go +++ b/internal/configuration/configuration.go @@ -11,28 +11,12 @@ import ( "gopkg.in/yaml.v3" ) -// Configuration represents the core utility settings. -// It is structured to support extensibility and ease of management. -// -// Fields: -// - Version: Specifies the configuration schema's version, aiding compatibility checks. -// - Sources: Lists source configurations for external integrations or data sources. -// - Keys: Holds API keys for various services to be utilized by the utility. type Configuration struct { Version string `yaml:"version"` Sources []string `yaml:"sources"` Keys sources.Keys `yaml:"keys"` } -// Write persists the current configuration into a YAML file. -// It ensures that the target directory exists and handles file creation -// with the correct permissions. -// -// Parameters: -// - path string: The file system path where the configuration will be stored. -// -// Returns: -// - err error: Returns an error if writing the configuration fails, otherwise nil. func (configuration *Configuration) Write(path string) (err error) { var file *os.File @@ -62,14 +46,11 @@ func (configuration *Configuration) Write(path string) (err error) { } const ( - // NAME is the utility identifier used in configuration and branding. - NAME = "xurlfind3r" - // VERSION specifies the current version of the utility. + NAME = "xurlfind3r" VERSION = "0.6.0" ) var ( - // BANNER provides a visually formatted utility banner for display. BANNER = aurora.Sprintf( aurora.BrightBlue(` _ __ _ _ _____ @@ -80,8 +61,6 @@ __ ___ _ _ __| |/ _(_)_ __ __| |___ / _ __ %s`).Bold(), aurora.BrightRed("v"+VERSION).Bold(), ) - // UserDotConfigDirectoryPath returns the user's configuration directory path, - // ensuring the utility can save configuration files in a standard location. UserDotConfigDirectoryPath = func() (userDotConfig string) { var err error @@ -92,11 +71,8 @@ __ ___ _ _ __| |/ _(_)_ __ __| |___ / _ __ return }() - // DefaultConfigurationFilePath defines the default location for the utility's configuration file. DefaultConfigurationFilePath = filepath.Join(UserDotConfigDirectoryPath, NAME, "config.yaml") - // DefaultConfiguration provides a pre-configured instance with default values. - // It includes default sources and empty API keys for services. - DefaultConfiguration = Configuration{ + DefaultConfiguration = Configuration{ Version: VERSION, Sources: sources.List, Keys: sources.Keys{ @@ -108,15 +84,6 @@ __ ___ _ _ __| |/ _(_)_ __ __| |___ / _ __ } ) -// CreateOrUpdate ensures a configuration file exists at the specified path. -// If the file does not exist, it creates one using default settings. -// If the file exists but is outdated or missing settings, it updates the file. -// -// Parameters: -// - path string: The file path where the configuration will be checked or updated. -// -// Returns: -// - err error: Returns an error if the process fails, otherwise nil. func CreateUpdate(path string) (err error) { var cfg Configuration @@ -154,15 +121,6 @@ func CreateUpdate(path string) (err error) { return } -// Read loads a YAML configuration file from the specified path. -// It initializes a Configuration struct with the values found in the file. -// -// Parameters: -// - path string: The file path from which the configuration will be loaded. -// -// Returns: -// - cfg Configuration: A pointer to the loaded Configuration instance. -// - err error: Returns an error if reading the file or parsing the YAML fails. func Read(path string) (configuration Configuration, err error) { var file *os.File diff --git a/internal/configuration/doc.go b/internal/configuration/doc.go deleted file mode 100644 index cd256f6..0000000 --- a/internal/configuration/doc.go +++ /dev/null @@ -1,5 +0,0 @@ -// Package configuration provides utilities to manage the utility's configuration. -// This includes loading, writing, and maintaining the configuration file while ensuring -// compatibility and default settings. The package ensures seamless management of settings -// and configurations required by the utility. -package configuration diff --git a/internal/input/input.go b/internal/input/input.go new file mode 100644 index 0000000..04e6dba --- /dev/null +++ b/internal/input/input.go @@ -0,0 +1,15 @@ +package input + +import "os" + +func HasStdin() bool { + stat, err := os.Stdin.Stat() + if err != nil { + return false + } + + isPipedFromChrDev := (stat.Mode() & os.ModeCharDevice) == 0 + isPipedFromFIFO := (stat.Mode() & os.ModeNamedPipe) != 0 + + return isPipedFromChrDev || isPipedFromFIFO +} diff --git a/internal/output/output.go b/internal/output/output.go new file mode 100644 index 0000000..8c7a84b --- /dev/null +++ b/internal/output/output.go @@ -0,0 +1,129 @@ +package output + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/hueristiq/xurlfind3r/pkg/xurlfind3r/sources" +) + +type Writer struct { + format format +} + +func (w *Writer) SetFormatToJSONL() { + w.format = formatJSONL +} + +func (w *Writer) CreateFile(path string) (file *os.File, err error) { + if path == "" { + err = ErrNoFilePathSpecified + + return + } + + extension := filepath.Ext(path) + + if w.format == formatTXT && extension != ".txt" { + path += ".txt" + } + + if w.format == formatJSONL && extension != ".json" { + path += ".json" + } + + directory := filepath.Dir(path) + + if directory != "" { + if _, err = os.Stat(directory); os.IsNotExist(err) { + err = os.MkdirAll(directory, os.ModePerm) + if err != nil { + return + } + } + } + + file, err = os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return + } + + return +} + +func (w *Writer) Write(writer io.Writer, domain string, result sources.Result) (err error) { + if w.format == formatTXT { + err = w.writeTXT(writer, result) + } else if w.format == formatJSONL { + err = w.writeJSON(writer, domain, result) + } + + return +} + +func (w *Writer) writeTXT(writer io.Writer, result sources.Result) (err error) { + bw := bufio.NewWriter(writer) + + fmt.Fprintln(bw, result.Value) + + if err = bw.Flush(); err != nil { + return + } + + return +} + +func (w *Writer) writeJSON(writer io.Writer, domain string, result sources.Result) (err error) { + data := resultForJSONL{ + Domain: domain, + URL: result.Value, + Source: result.Source, + } + + var dataJSONBytes []byte + + dataJSONBytes, err = json.Marshal(data) + if err != nil { + return + } + + dataJSONString := string(dataJSONBytes) + + bw := bufio.NewWriter(writer) + + fmt.Fprintln(bw, dataJSONString) + + if err = bw.Flush(); err != nil { + return + } + + return +} + +type format string + +type resultForJSONL struct { + Domain string `json:"domain"` + URL string `json:"url"` + Source string `json:"source"` +} + +const ( + formatJSONL format = "JSON" + formatTXT format = "TXT" +) + +var ErrNoFilePathSpecified = errors.New("no file path specified") + +func NewWritter() (writter *Writer) { + writter = &Writer{ + format: formatTXT, + } + + return +}