From 0db2332c27559130684b1f34ef67d77d84b5aff5 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sat, 26 Oct 2024 21:05:53 +0530 Subject: [PATCH 01/23] feat: added initial live DAST server implementation --- cmd/nuclei/main.go | 10 ++ go.mod | 5 +- go.sum | 6 ++ internal/runner/runner.go | 20 ++++ internal/server/dedupe.go | 122 ++++++++++++++++++++++++ internal/server/exec.go | 90 ++++++++++++++++++ internal/server/requests_worker.go | 63 +++++++++++++ internal/server/scope/scope.go | 77 ++++++++++++++++ internal/server/scope/scope_test.go | 26 ++++++ internal/server/server.go | 138 ++++++++++++++++++++++++++++ internal/server/server_test.go | 24 +++++ pkg/types/types.go | 10 ++ 12 files changed, 589 insertions(+), 2 deletions(-) create mode 100644 internal/server/dedupe.go create mode 100644 internal/server/exec.go create mode 100644 internal/server/requests_worker.go create mode 100644 internal/server/scope/scope.go create mode 100644 internal/server/scope/scope_test.go create mode 100644 internal/server/server.go create mode 100644 internal/server/server_test.go diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 5e95a8b1b2..ef1bd6d996 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -185,6 +185,11 @@ func main() { go func() { for range c { gologger.Info().Msgf("CTRL+C pressed: Exiting\n") + if options.DASTServer { + nucleiRunner.Close() + os.Exit(1) + } + gologger.Info().Msgf("Attempting graceful shutdown...") if options.EnableCloudUpload { gologger.Info().Msgf("Uploading scan results to cloud...") @@ -354,9 +359,14 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.FuzzingMode, "fuzzing-mode", "fm", "", "overrides fuzzing mode set in template (multiple, single)"), flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"), flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"), + flagSet.BoolVarP(&options.DASTServer, "dast-server", "dts", false, "enable dast server mode (live fuzzing)"), + flagSet.StringVarP(&options.DASTServerToken, "dast-server-token", "dtst", "", "dast server token (optional)"), + flagSet.StringVarP(&options.DASTServerAddress, "dast-server-address", "dtsa", "localhost:9055", "dast server address"), flagSet.BoolVarP(&options.DisplayFuzzPoints, "display-fuzz-points", "dfp", false, "display fuzz points in the output for debugging"), flagSet.IntVar(&options.FuzzParamFrequency, "fuzz-param-frequency", 10, "frequency of uninteresting parameters for fuzzing before skipping"), flagSet.StringVarP(&options.FuzzAggressionLevel, "fuzz-aggression", "fa", "low", "fuzzing aggression level controls payload count for fuzz (low, medium, high)"), + flagSet.StringSliceVarP(&options.Scope, "fuzz-scope", "cs", nil, "in scope url regex to be followed by fuzzer", goflags.FileCommaSeparatedStringSliceOptions), + flagSet.StringSliceVarP(&options.OutOfScope, "fuzz-out-scope", "cos", nil, "out of scope url regex to be excluded by fuzzer", goflags.FileCommaSeparatedStringSliceOptions), ) flagSet.CreateGroup("uncover", "Uncover", diff --git a/go.mod b/go.mod index e4a71aec7d..e527ff451a 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/invopop/yaml v0.3.1 github.com/kitabisa/go-ci v1.0.3 - github.com/labstack/echo/v4 v4.10.2 + github.com/labstack/echo/v4 v4.12.0 github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.22 @@ -102,6 +102,7 @@ require ( github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 + github.com/sourcegraph/conc v0.3.0 github.com/stretchr/testify v1.9.0 github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 github.com/yassinebenaid/godump v0.10.0 @@ -347,7 +348,7 @@ require ( github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/labstack/gommon v0.4.0 // indirect + github.com/labstack/gommon v0.4.2 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect diff --git a/go.sum b/go.sum index 2223302430..254a456d85 100644 --- a/go.sum +++ b/go.sum @@ -669,8 +669,12 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M= github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k= +github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0= +github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM= github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= +github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= +github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa h1:KQKuQDgA3DZX6C396lt3WDYB9Um1gLITLbvficVbqXk= @@ -997,6 +1001,8 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 516bd7ca50..e0452e66e3 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -14,6 +14,7 @@ import ( "time" "github.com/projectdiscovery/nuclei/v3/internal/pdcp" + "github.com/projectdiscovery/nuclei/v3/internal/server" "github.com/projectdiscovery/nuclei/v3/pkg/authprovider" "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/frequency" "github.com/projectdiscovery/nuclei/v3/pkg/input/provider" @@ -436,6 +437,25 @@ func (r *Runner) setupPDCPUpload(writer output.Writer) output.Writer { // RunEnumeration sets up the input layer for giving input nuclei. // binary and runs the actual enumeration func (r *Runner) RunEnumeration() error { + // If the user has asked for DAST server mode, run the live + // DAST fuzzing server. + if r.options.DASTServer { + dastServer, err := server.New(&server.Options{ + Address: r.options.DASTServerAddress, + Concurrency: r.options.BulkSize, + Templates: r.options.Templates, + OutputWriter: r.output, + Verbose: r.options.Verbose, + Token: r.options.DASTServerToken, + InScope: r.options.Scope, + OutScope: r.options.OutOfScope, + }) + if err != nil { + return err + } + return dastServer.Start() + } + // If user asked for new templates to be executed, collect the list from the templates' directory. if r.options.NewTemplates { if arr := config.DefaultConfig.GetNewAdditions(); len(arr) > 0 { diff --git a/internal/server/dedupe.go b/internal/server/dedupe.go new file mode 100644 index 0000000000..f5c5b775bf --- /dev/null +++ b/internal/server/dedupe.go @@ -0,0 +1,122 @@ +package server + +import ( + "crypto/sha256" + "encoding/hex" + "net/url" + "sort" + "strings" + "sync" + + "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + mapsutil "github.com/projectdiscovery/utils/maps" +) + +var dynamicHeaders = map[string]bool{ + "date": true, + "if-modified-since": true, + "if-unmodified-since": true, + "cache-control": true, + "if-none-match": true, + "if-match": true, + "authorization": true, + "cookie": true, + "x-csrf-token": true, + "content-length": true, + "content-md5": true, + "host": true, + "x-request-id": true, + "x-correlation-id": true, + "user-agent": true, + "referer": true, +} + +type requestDeduplicator struct { + hashes map[string]struct{} + lock *sync.RWMutex +} + +func newRequestDeduplicator() *requestDeduplicator { + return &requestDeduplicator{ + hashes: make(map[string]struct{}), + lock: &sync.RWMutex{}, + } +} + +func (r *requestDeduplicator) isDuplicate(req *types.RequestResponse) bool { + hash, err := hashRequest(req) + if err != nil { + return false + } + + r.lock.RLock() + _, ok := r.hashes[hash] + r.lock.RUnlock() + if ok { + return true + } + + r.lock.Lock() + r.hashes[hash] = struct{}{} + r.lock.Unlock() + return false +} + +func hashRequest(req *types.RequestResponse) (string, error) { + normalizedURL, err := normalizeURL(req.URL.URL) + if err != nil { + return "", err + } + + var hashContent strings.Builder + hashContent.WriteString(req.Request.Method) + hashContent.WriteString(normalizedURL) + + headers := sortedNonDynamicHeaders(req.Request.Headers) + for _, header := range headers { + hashContent.WriteString(header.Key) + hashContent.WriteString(header.Value) + } + + if len(req.Request.Body) > 0 { + hashContent.Write([]byte(req.Request.Body)) + } + + // Calculate the SHA256 hash + hash := sha256.Sum256([]byte(hashContent.String())) + return hex.EncodeToString(hash[:]), nil +} + +func normalizeURL(u *url.URL) (string, error) { + query := u.Query() + sortedQuery := make(url.Values) + for k, v := range query { + sort.Strings(v) + sortedQuery[k] = v + } + u.RawQuery = sortedQuery.Encode() + + if u.Path == "" { + u.Path = "/" + } + return u.String(), nil +} + +type header struct { + Key string + Value string +} + +func sortedNonDynamicHeaders(headers mapsutil.OrderedMap[string, string]) []header { + var result []header + headers.Iterate(func(k, v string) bool { + if !dynamicHeaders[strings.ToLower(k)] { + result = append(result, header{Key: k, Value: v}) + } + return true + }) + sort.Slice(result, func(i, j int) bool { + return result[i].Key < result[j].Key + }) + return result +} diff --git a/internal/server/exec.go b/internal/server/exec.go new file mode 100644 index 0000000000..7ee72b1071 --- /dev/null +++ b/internal/server/exec.go @@ -0,0 +1,90 @@ +package server + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + + "github.com/projectdiscovery/nuclei/v3/pkg/output" + "gopkg.in/yaml.v2" +) + +// proxifyRequest is a request for proxify +type proxifyRequest struct { + URL string `json:"url"` + Request struct { + Header map[string]string `json:"header"` + Body string `json:"body"` + Raw string `json:"raw"` + } `json:"request"` +} + +func runNucleiWithFuzzingInput(target PostReuestsHandlerRequest, templates []string) ([]output.ResultEvent, error) { + cmd := exec.Command("nuclei") + + tempFile, err := os.CreateTemp("", "nuclei-fuzz-*.yaml") + if err != nil { + return nil, fmt.Errorf("error creating temp file: %s", err) + } + defer os.Remove(tempFile.Name()) + + payload := proxifyRequest{ + URL: target.URL, + Request: struct { + Header map[string]string `json:"header"` + Body string `json:"body"` + Raw string `json:"raw"` + }{ + Raw: target.RawHTTP, + }, + } + + marshalledYaml, err := yaml.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("error marshalling yaml: %s", err) + } + + if _, err := tempFile.Write(marshalledYaml); err != nil { + return nil, fmt.Errorf("error writing to temp file: %s", err) + } + + argsArray := []string{ + "-duc", + "-dast", + "-silent", + "-no-color", + "-jsonl", + } + for _, template := range templates { + argsArray = append(argsArray, "-t", template) + } + argsArray = append(argsArray, "-l", tempFile.Name()) + argsArray = append(argsArray, "-im=yaml") + cmd.Args = append(cmd.Args, argsArray...) + + data, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("error running nuclei: %w", err) + } + + var nucleiResult []output.ResultEvent + decoder := json.NewDecoder(bytes.NewReader(data)) + for { + var result output.ResultEvent + if err := decoder.Decode(&result); err != nil { + if err == io.EOF { + break + } + return nil, fmt.Errorf("error decoding nuclei output: %w", err) + } + // Filter results with a valid template-id + if result.TemplateID != "" { + nucleiResult = append(nucleiResult, result) + } + } + + return nucleiResult, nil +} diff --git a/internal/server/requests_worker.go b/internal/server/requests_worker.go new file mode 100644 index 0000000000..8616578fe7 --- /dev/null +++ b/internal/server/requests_worker.go @@ -0,0 +1,63 @@ +package server + +import ( + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/input/types" +) + +func (s *DASTServer) setupWorkers() { + go s.tasksConsumer() +} + +func (s *DASTServer) tasksConsumer() { + for req := range s.fuzzRequests { + parsedReq, err := parseRawRequest(req) + if err != nil { + gologger.Warning().Msgf("Could not parse raw request: %s\n", err) + continue + } + + inScope, err := s.scopeManager.Validate(parsedReq.URL.URL, "") + if err != nil { + gologger.Warning().Msgf("Could not validate scope: %s\n", err) + continue + } + if !inScope { + gologger.Warning().Msgf("Request is out of scope: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String()) + continue + } + + if s.deduplicator.isDuplicate(parsedReq) { + gologger.Warning().Msgf("Duplicate request detected: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String()) + continue + } + + gologger.Verbose().Msgf("Fuzzing request: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String()) + s.tasksPool.Go(func() { + s.fuzzRequest(req) + }) + } +} + +func (s *DASTServer) fuzzRequest(req PostReuestsHandlerRequest) { + results, err := runNucleiWithFuzzingInput(req, s.options.Templates) + if err != nil { + gologger.Warning().Msgf("Could not run nuclei: %s\n", err) + return + } + + for _, result := range results { + if err := s.options.OutputWriter.Write(&result); err != nil { + gologger.Error().Msgf("Could not write result: %s\n", err) + } + } +} + +func parseRawRequest(req PostReuestsHandlerRequest) (*types.RequestResponse, error) { + parsedReq, err := types.ParseRawRequestWithURL(req.RawHTTP, req.URL) + if err != nil { + return nil, errors.Wrap(err, "could not parse raw HTTP") + } + return parsedReq, nil +} diff --git a/internal/server/scope/scope.go b/internal/server/scope/scope.go new file mode 100644 index 0000000000..63dd01fd6c --- /dev/null +++ b/internal/server/scope/scope.go @@ -0,0 +1,77 @@ +// From Katana +package scope + +import ( + "fmt" + "net/url" + "regexp" +) + +// Manager manages scope for crawling process +type Manager struct { + inScope []*regexp.Regexp + outOfScope []*regexp.Regexp + noScope bool +} + +// NewManager returns a new scope manager for crawling +func NewManager(inScope, outOfScope []string) (*Manager, error) { + manager := &Manager{} + + for _, regex := range inScope { + if compiled, err := regexp.Compile(regex); err != nil { + return nil, fmt.Errorf("could not compile regex %s: %s", regex, err) + } else { + manager.inScope = append(manager.inScope, compiled) + } + } + for _, regex := range outOfScope { + if compiled, err := regexp.Compile(regex); err != nil { + return nil, fmt.Errorf("could not compile regex %s: %s", regex, err) + } else { + manager.outOfScope = append(manager.outOfScope, compiled) + } + } + if len(manager.inScope) == 0 && len(manager.outOfScope) == 0 { + manager.noScope = true + } + return manager, nil +} + +// Validate returns true if the URL matches scope rules +func (m *Manager) Validate(URL *url.URL, rootHostname string) (bool, error) { + if m.noScope { + return true, nil + } + + urlStr := URL.String() + + urlValidated, err := m.validateURL(urlStr) + if err != nil { + return false, err + } + if urlValidated { + return true, nil + } + return false, nil +} + +func (m *Manager) validateURL(URL string) (bool, error) { + for _, item := range m.outOfScope { + if item.MatchString(URL) { + return false, nil + } + } + if len(m.inScope) == 0 { + return true, nil + } + + var inScopeMatched bool + for _, item := range m.inScope { + if item.MatchString(URL) { + inScopeMatched = true + break + } + } + return inScopeMatched, nil +} diff --git a/internal/server/scope/scope_test.go b/internal/server/scope/scope_test.go new file mode 100644 index 0000000000..a612cf4f3d --- /dev/null +++ b/internal/server/scope/scope_test.go @@ -0,0 +1,26 @@ +package scope + +import ( + "testing" + + urlutil "github.com/projectdiscovery/utils/url" + "github.com/stretchr/testify/require" +) + +func TestManagerValidate(t *testing.T) { + t.Run("url", func(t *testing.T) { + manager, err := NewManager([]string{`example`}, []string{`logout\.php`}) + require.NoError(t, err, "could not create scope manager") + + parsed, _ := urlutil.Parse("https://test.com/index.php/example") + validated, err := manager.Validate(parsed.URL, "test.com") + require.NoError(t, err, "could not validate url") + require.True(t, validated, "could not get correct in-scope validation") + + parsed, _ = urlutil.Parse("https://test.com/logout.php") + validated, err = manager.Validate(parsed.URL, "another.com") + require.NoError(t, err, "could not validate url") + require.False(t, validated, "could not get correct out-scope validation") + }) + +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000000..c13d7c7ce7 --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,138 @@ +package server + +import ( + "fmt" + "strings" + "time" + + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/internal/server/scope" + "github.com/projectdiscovery/nuclei/v3/pkg/output" + "github.com/sourcegraph/conc/pool" +) + +// DASTServer is a server that performs execution of fuzzing templates +// on user input passed to the API. +type DASTServer struct { + echo *echo.Echo + options *Options + tasksPool *pool.Pool + deduplicator *requestDeduplicator + scopeManager *scope.Manager + fuzzRequests chan PostReuestsHandlerRequest +} + +// Options contains the configuration options for the server. +type Options struct { + // Address is the address to bind the server to + Address string + // Token is the token to use for authentication (optional) + Token string + // Concurrency is the concurrency level to use for the targets + Concurrency int + // Templates is the list of templates to use for fuzzing + Templates []string + // Verbose is a flag that controls verbose output + Verbose bool + + // Scope fields for fuzzer + InScope []string + OutScope []string + + OutputWriter output.Writer +} + +// New creates a new instance of the DAST server. +func New(options *Options) (*DASTServer, error) { + bufferSize := options.Concurrency * 100 + + server := &DASTServer{ + options: options, + tasksPool: pool.New().WithMaxGoroutines(options.Concurrency), + deduplicator: newRequestDeduplicator(), + fuzzRequests: make(chan PostReuestsHandlerRequest, bufferSize), + } + server.setupHandlers() + server.setupWorkers() + + scopeManager, err := scope.NewManager( + options.InScope, + options.OutScope, + ) + if err != nil { + return nil, err + } + server.scopeManager = scopeManager + + var builder strings.Builder + builder.WriteString(fmt.Sprintf("Using %d parallel tasks with %d buffer", options.Concurrency, bufferSize)) + if options.Token != "" { + builder.WriteString(" (with token)") + } + gologger.Info().Msgf(builder.String()) + gologger.Info().Msgf("Connection URL: %s", server.buildConnectionURL()) + + return server, nil +} + +func (s *DASTServer) buildConnectionURL() string { + url := fmt.Sprintf("http://%s/requests", s.options.Address) + if s.options.Token != "" { + url += "?token=" + s.options.Token + } + return url +} + +func (s *DASTServer) setupHandlers() { + e := echo.New() + e.Use(middleware.Recover()) + if s.options.Verbose { + e.Use(middleware.Logger()) + } + e.Use(middleware.CORS()) + + if s.options.Token != "" { + e.Use(middleware.KeyAuthWithConfig(middleware.KeyAuthConfig{ + KeyLookup: "query:token", + Validator: func(key string, c echo.Context) (bool, error) { + return key == s.options.Token, nil + }, + })) + } + + e.HideBanner = true + // POST /requests - Queue a request for fuzzing + e.POST("/requests", s.handleRequest) + s.echo = e +} + +func (s *DASTServer) Start() error { + return s.echo.Start(s.options.Address) +} + +// PostReuestsHandlerRequest is the request body for the /requests POST handler. +type PostReuestsHandlerRequest struct { + RawHTTP string `json:"raw_http"` + URL string `json:"url"` +} + +func (s *DASTServer) handleRequest(c echo.Context) error { + var req PostReuestsHandlerRequest + if err := c.Bind(&req); err != nil { + return err + } + + // Validate the request + if req.RawHTTP == "" || req.URL == "" { + return c.JSON(400, map[string]string{"error": "missing required fields"}) + } + + select { + case s.fuzzRequests <- req: + return c.NoContent(200) + case timeout := <-time.After(5 * time.Second): + return c.JSON(429, map[string]string{"error": fmt.Sprintf("server busy, try again after %v", timeout)}) + } +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 0000000000..d5e0981517 --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,24 @@ +package server + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_parseRawRequest(t *testing.T) { + parsed, err := parseRawRequest(PostReuestsHandlerRequest{ + URL: "http://example.com/testpath", + RawHTTP: "GET /testpath HTTP/1.1\nHost: example.com\nUser-Agent: Mozilla/5.0\n\n", + }) + require.NoError(t, err) + require.Equal(t, "http://example.com/testpath", parsed.URL.String()) + + // Example POST request + parsed, err = parseRawRequest(PostReuestsHandlerRequest{ + URL: "http://example.com", + RawHTTP: "POST /testpath HTTP/1.1\nHost: example.com\nUser-Agent: Mozilla/5.0\nContent-Length: 5\n\nhello", + }) + require.NoError(t, err) + require.Equal(t, "hello", parsed.Request.Body) +} diff --git a/pkg/types/types.go b/pkg/types/types.go index f6e7ab4470..8f9d74a50e 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -409,6 +409,16 @@ type Options struct { ProbeConcurrency int // Dast only runs DAST templates DAST bool + // DASTServer is the flag to start nuclei as a DAST server + DASTServer bool + // DASTServerToken is the token optional for the dast server + DASTServerToken string + // DASTServerAddress is the address for the dast server + DASTServerAddress string + // Scope contains a list of regexes for in-scope URLS + Scope goflags.StringSlice + // OutOfScope contains a list of regexes for out-scope URLS + OutOfScope goflags.StringSlice // HttpApiEndpoint is the experimental http api endpoint HttpApiEndpoint string // ListTemplateProfiles lists all available template profiles From 64ef60eb0aaf5535840285731640841f7f44f810 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Wed, 30 Oct 2024 15:48:24 +0530 Subject: [PATCH 02/23] feat: more logging + misc additions --- internal/server/exec.go | 31 +++++++++++++++++++++++++----- internal/server/requests_worker.go | 2 +- internal/server/server.go | 11 +++++++++++ 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/internal/server/exec.go b/internal/server/exec.go index 7ee72b1071..cc53992d7e 100644 --- a/internal/server/exec.go +++ b/internal/server/exec.go @@ -22,7 +22,7 @@ type proxifyRequest struct { } `json:"request"` } -func runNucleiWithFuzzingInput(target PostReuestsHandlerRequest, templates []string) ([]output.ResultEvent, error) { +func (s *DASTServer) runNucleiWithFuzzingInput(target PostReuestsHandlerRequest, templates []string) ([]output.ResultEvent, error) { cmd := exec.Command("nuclei") tempFile, err := os.CreateTemp("", "nuclei-fuzz-*.yaml") @@ -54,7 +54,6 @@ func runNucleiWithFuzzingInput(target PostReuestsHandlerRequest, templates []str argsArray := []string{ "-duc", "-dast", - "-silent", "-no-color", "-jsonl", } @@ -63,15 +62,34 @@ func runNucleiWithFuzzingInput(target PostReuestsHandlerRequest, templates []str } argsArray = append(argsArray, "-l", tempFile.Name()) argsArray = append(argsArray, "-im=yaml") + + var stderrBuf bytes.Buffer + if s.options.Verbose { + cmd.Stderr = &stderrBuf + argsArray = append(argsArray, "-v") + } else { + argsArray = append(argsArray, "-silent") + } cmd.Args = append(cmd.Args, argsArray...) - data, err := cmd.Output() + stdoutPipe, err := cmd.StdoutPipe() if err != nil { - return nil, fmt.Errorf("error running nuclei: %w", err) + return nil, fmt.Errorf("error creating stdout pipe: %s", err) + } + + errWithStderr := func(err error) error { + if s.options.Verbose { + return fmt.Errorf("error running nuclei: %s\n%s", err, stderrBuf.String()) + } + return fmt.Errorf("error starting nuclei: %s", err) + } + + if err := cmd.Start(); err != nil { + return nil, errWithStderr(err) } var nucleiResult []output.ResultEvent - decoder := json.NewDecoder(bytes.NewReader(data)) + decoder := json.NewDecoder(stdoutPipe) for { var result output.ResultEvent if err := decoder.Decode(&result); err != nil { @@ -86,5 +104,8 @@ func runNucleiWithFuzzingInput(target PostReuestsHandlerRequest, templates []str } } + if err := cmd.Wait(); err != nil { + return nil, errWithStderr(err) + } return nucleiResult, nil } diff --git a/internal/server/requests_worker.go b/internal/server/requests_worker.go index 8616578fe7..3f254576b5 100644 --- a/internal/server/requests_worker.go +++ b/internal/server/requests_worker.go @@ -41,7 +41,7 @@ func (s *DASTServer) tasksConsumer() { } func (s *DASTServer) fuzzRequest(req PostReuestsHandlerRequest) { - results, err := runNucleiWithFuzzingInput(req, s.options.Templates) + results, err := s.runNucleiWithFuzzingInput(req, s.options.Templates) if err != nil { gologger.Warning().Msgf("Could not run nuclei: %s\n", err) return diff --git a/internal/server/server.go b/internal/server/server.go index c13d7c7ce7..d3ab8e50cb 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,6 +1,7 @@ package server import ( + "encoding/json" "fmt" "strings" "time" @@ -48,6 +49,11 @@ type Options struct { func New(options *Options) (*DASTServer, error) { bufferSize := options.Concurrency * 100 + // If the user has specified no templates, use the default ones + // for DAST only. + if len(options.Templates) == 0 { + options.Templates = []string{"dast/"} + } server := &DASTServer{ options: options, tasksPool: pool.New().WithMaxGoroutines(options.Concurrency), @@ -129,6 +135,11 @@ func (s *DASTServer) handleRequest(c echo.Context) error { return c.JSON(400, map[string]string{"error": "missing required fields"}) } + if s.options.Verbose { + marshalIndented, _ := json.MarshalIndent(req, "", " ") + gologger.Verbose().Msgf("Received request: %s", marshalIndented) + } + select { case s.fuzzRequests <- req: return c.NoContent(200) From efd8ab93428a4119ad4d4e496a81a940cc1decf6 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Thu, 7 Nov 2024 21:07:54 +0530 Subject: [PATCH 03/23] feat: auth file support enhancements for more complex scenarios + misc --- internal/runner/lazy.go | 9 +- pkg/authprovider/authx/cookies_auth.go | 3 + pkg/authprovider/authx/dynamic.go | 153 ++++++++++++++++--------- pkg/authprovider/authx/strategy.go | 12 +- pkg/authprovider/file.go | 8 +- 5 files changed, 122 insertions(+), 63 deletions(-) diff --git a/internal/runner/lazy.go b/internal/runner/lazy.go index 900850b673..b0d3341d46 100644 --- a/internal/runner/lazy.go +++ b/internal/runner/lazy.go @@ -92,8 +92,13 @@ func GetLazyAuthFetchCallback(opts *AuthLazyFetchOptions) authx.LazyFetchSecret } // dynamic values for k, v := range e.OperatorsResult.DynamicValues { - if len(v) > 0 { - data[k] = v[0] + // Iterate through all the values and choose the + // largest value as the extracted value + for _, value := range v { + oldVal, ok := data[k] + if !ok || len(value) > len(oldVal.(string)) { + data[k] = value + } } } // named extractors diff --git a/pkg/authprovider/authx/cookies_auth.go b/pkg/authprovider/authx/cookies_auth.go index 7f3e756a71..9df56fb1b1 100644 --- a/pkg/authprovider/authx/cookies_auth.go +++ b/pkg/authprovider/authx/cookies_auth.go @@ -33,6 +33,9 @@ func (s *CookiesAuthStrategy) Apply(req *http.Request) { // ApplyOnRR applies the cookies auth strategy to the retryable request func (s *CookiesAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { + // Before adding new cookies, remove existing cookies + req.Header.Del("Cookie") + for _, cookie := range s.Data.Cookies { c := &http.Cookie{ Name: cookie.Key, diff --git a/pkg/authprovider/authx/dynamic.go b/pkg/authprovider/authx/dynamic.go index 0e210cf5e7..f61fc5d31c 100644 --- a/pkg/authprovider/authx/dynamic.go +++ b/pkg/authprovider/authx/dynamic.go @@ -9,6 +9,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/replacer" errorutil "github.com/projectdiscovery/utils/errors" + sliceutil "github.com/projectdiscovery/utils/slice" ) type LazyFetchSecret func(d *Dynamic) error @@ -22,7 +23,8 @@ var ( // ex: username and password are dynamic secrets, the actual secret is the token obtained // after authenticating with the username and password type Dynamic struct { - Secret `yaml:",inline"` // this is a static secret that will be generated after the dynamic secret is resolved + *Secret `yaml:",inline"` // this is a static secret that will be generated after the dynamic secret is resolved + Secrets []*Secret `yaml:"secrets"` TemplatePath string `json:"template" yaml:"template"` Variables []KV `json:"variables" yaml:"variables"` Input string `json:"input" yaml:"input"` // (optional) target for the dynamic secret @@ -33,6 +35,22 @@ type Dynamic struct { error error `json:"-" yaml:"-"` // error if any } +func (d *Dynamic) GetDomainAndDomainRegex() ([]string, []string) { + var domains []string + var domainRegex []string + for _, secret := range d.Secrets { + domains = append(domains, secret.Domains...) + domainRegex = append(domainRegex, secret.DomainsRegex...) + } + if d.Secret != nil { + domains = append(domains, d.Secret.Domains...) + domainRegex = append(domainRegex, d.Secret.DomainsRegex...) + } + uniqueDomains := sliceutil.Dedupe(domains) + uniqueDomainRegex := sliceutil.Dedupe(domainRegex) + return uniqueDomains, uniqueDomainRegex +} + func (d *Dynamic) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(data, &d); err != nil { return err @@ -41,7 +59,7 @@ func (d *Dynamic) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(data, &s); err != nil { return err } - d.Secret = s + d.Secret = &s return nil } @@ -54,9 +72,18 @@ func (d *Dynamic) Validate() error { if len(d.Variables) == 0 { return errorutil.New("variables are required for dynamic secret") } - d.skipCookieParse = true // skip cookie parsing in dynamic secrets during validation - if err := d.Secret.Validate(); err != nil { - return err + + if d.Secret != nil { + d.Secret.skipCookieParse = true // skip cookie parsing in dynamic secrets during validation + if err := d.Secret.Validate(); err != nil { + return err + } + } + for _, secret := range d.Secrets { + secret.skipCookieParse = true + if err := secret.Validate(); err != nil { + return err + } } return nil } @@ -74,76 +101,98 @@ func (d *Dynamic) SetLazyFetchCallback(callback LazyFetchSecret) { return fmt.Errorf("no extracted values found for dynamic secret") } - // evaluate headers - for i, header := range d.Headers { - if strings.Contains(header.Value, "{{") { - header.Value = replacer.Replace(header.Value, d.Extracted) + if d.Secret != nil { + if err := d.applyValuesToSecret(d.Secret); err != nil { + return err } - if strings.Contains(header.Key, "{{") { - header.Key = replacer.Replace(header.Key, d.Extracted) - } - d.Headers[i] = header } - // evaluate cookies - for i, cookie := range d.Cookies { - if strings.Contains(cookie.Value, "{{") { - cookie.Value = replacer.Replace(cookie.Value, d.Extracted) - } - if strings.Contains(cookie.Key, "{{") { - cookie.Key = replacer.Replace(cookie.Key, d.Extracted) + for _, secret := range d.Secrets { + if err := d.applyValuesToSecret(secret); err != nil { + return err } - if strings.Contains(cookie.Raw, "{{") { - cookie.Raw = replacer.Replace(cookie.Raw, d.Extracted) - } - d.Cookies[i] = cookie } + return nil + } +} - // evaluate query params - for i, query := range d.Params { - if strings.Contains(query.Value, "{{") { - query.Value = replacer.Replace(query.Value, d.Extracted) - } - if strings.Contains(query.Key, "{{") { - query.Key = replacer.Replace(query.Key, d.Extracted) - } - d.Params[i] = query +func (d *Dynamic) applyValuesToSecret(secret *Secret) error { + // evaluate headers + for i, header := range secret.Headers { + if strings.Contains(header.Value, "{{") { + header.Value = replacer.Replace(header.Value, d.Extracted) + } + if strings.Contains(header.Key, "{{") { + header.Key = replacer.Replace(header.Key, d.Extracted) } + secret.Headers[i] = header + } - // check username, password and token - if strings.Contains(d.Username, "{{") { - d.Username = replacer.Replace(d.Username, d.Extracted) + // evaluate cookies + for i, cookie := range secret.Cookies { + if strings.Contains(cookie.Value, "{{") { + cookie.Value = replacer.Replace(cookie.Value, d.Extracted) } - if strings.Contains(d.Password, "{{") { - d.Password = replacer.Replace(d.Password, d.Extracted) + if strings.Contains(cookie.Key, "{{") { + cookie.Key = replacer.Replace(cookie.Key, d.Extracted) } - if strings.Contains(d.Token, "{{") { - d.Token = replacer.Replace(d.Token, d.Extracted) + if strings.Contains(cookie.Raw, "{{") { + cookie.Raw = replacer.Replace(cookie.Raw, d.Extracted) } + secret.Cookies[i] = cookie + } + + // evaluate query params + for i, query := range secret.Params { + if strings.Contains(query.Value, "{{") { + query.Value = replacer.Replace(query.Value, d.Extracted) + } + if strings.Contains(query.Key, "{{") { + query.Key = replacer.Replace(query.Key, d.Extracted) + } + secret.Params[i] = query + } - // now attempt to parse the cookies - d.skipCookieParse = false - for i, cookie := range d.Cookies { - if cookie.Raw != "" { - if err := cookie.Parse(); err != nil { - return fmt.Errorf("[%s] invalid raw cookie in cookiesAuth: %s", d.TemplatePath, err) - } - d.Cookies[i] = cookie + // check username, password and token + if strings.Contains(secret.Username, "{{") { + secret.Username = replacer.Replace(secret.Username, d.Extracted) + } + if strings.Contains(secret.Password, "{{") { + secret.Password = replacer.Replace(secret.Password, d.Extracted) + } + if strings.Contains(secret.Token, "{{") { + secret.Token = replacer.Replace(secret.Token, d.Extracted) + } + + // now attempt to parse the cookies + secret.skipCookieParse = false + for i, cookie := range secret.Cookies { + if cookie.Raw != "" { + if err := cookie.Parse(); err != nil { + return fmt.Errorf("[%s] invalid raw cookie in cookiesAuth: %s", d.TemplatePath, err) } + secret.Cookies[i] = cookie } - return nil } + return nil } -// GetStrategy returns the auth strategy for the dynamic secret -func (d *Dynamic) GetStrategy() AuthStrategy { +// GetStrategy returns the auth strategies for the dynamic secret +func (d *Dynamic) GetStrategies() []AuthStrategy { if !d.fetched { _ = d.Fetch(true) } if d.error != nil { return nil } - return d.Secret.GetStrategy() + var strategies []AuthStrategy + if d.Secret != nil { + strategies = append(strategies, d.Secret.GetStrategy()) + } + for _, secret := range d.Secrets { + strategies = append(strategies, secret.GetStrategy()) + } + return strategies } // Fetch fetches the dynamic secret diff --git a/pkg/authprovider/authx/strategy.go b/pkg/authprovider/authx/strategy.go index 8204083989..775862954c 100644 --- a/pkg/authprovider/authx/strategy.go +++ b/pkg/authprovider/authx/strategy.go @@ -24,16 +24,16 @@ type DynamicAuthStrategy struct { // Apply applies the strategy to the request func (d *DynamicAuthStrategy) Apply(req *http.Request) { - strategy := d.Dynamic.GetStrategy() - if strategy != nil { - strategy.Apply(req) + strategy := d.Dynamic.GetStrategies() + for _, s := range strategy { + s.Apply(req) } } // ApplyOnRR applies the strategy to the retryable request func (d *DynamicAuthStrategy) ApplyOnRR(req *retryablehttp.Request) { - strategy := d.Dynamic.GetStrategy() - if strategy != nil { - strategy.ApplyOnRR(req) + strategy := d.Dynamic.GetStrategies() + for _, s := range strategy { + s.ApplyOnRR(req) } } diff --git a/pkg/authprovider/file.go b/pkg/authprovider/file.go index 3a32a94fe4..64cfcb8793 100644 --- a/pkg/authprovider/file.go +++ b/pkg/authprovider/file.go @@ -85,8 +85,10 @@ func (f *FileAuthProvider) init() { } } for _, dynamic := range f.store.Dynamic { - if len(dynamic.DomainsRegex) > 0 { - for _, domain := range dynamic.DomainsRegex { + domain, domainsRegex := dynamic.GetDomainAndDomainRegex() + + if len(domainsRegex) > 0 { + for _, domain := range domainsRegex { if f.compiled == nil { f.compiled = make(map[*regexp.Regexp][]authx.AuthStrategy) } @@ -101,7 +103,7 @@ func (f *FileAuthProvider) init() { } } } - for _, domain := range dynamic.Domains { + for _, domain := range domain { if f.domains == nil { f.domains = make(map[string][]authx.AuthStrategy) } From ae870e6319ef58c29247f8e572526bdff4a200c5 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Thu, 21 Nov 2024 01:26:40 +0530 Subject: [PATCH 04/23] feat: added io.Reader support to input providers for http --- pkg/input/formats/burp/burp.go | 12 ++----- pkg/input/formats/burp/burp_test.go | 9 ++++-- pkg/input/formats/formats.go | 3 +- pkg/input/formats/json/json.go | 11 ++----- pkg/input/formats/json/json_test.go | 9 ++++-- pkg/input/formats/openapi/openapi.go | 6 ++-- pkg/input/formats/openapi/openapi_test.go | 9 ++++-- pkg/input/formats/swagger/swagger.go | 21 ++++-------- pkg/input/formats/swagger/swagger_test.go | 9 ++++-- pkg/input/formats/yaml/multidoc.go | 11 ++----- pkg/input/formats/yaml/multidoc_test.go | 9 ++++-- pkg/input/provider/http/multiformat.go | 39 ++++++++++++++++++----- 12 files changed, 86 insertions(+), 62 deletions(-) diff --git a/pkg/input/formats/burp/burp.go b/pkg/input/formats/burp/burp.go index 6ad5f548b5..9b2a362dfe 100644 --- a/pkg/input/formats/burp/burp.go +++ b/pkg/input/formats/burp/burp.go @@ -2,7 +2,7 @@ package burp import ( "encoding/base64" - "os" + "io" "strings" "github.com/pkg/errors" @@ -35,14 +35,8 @@ func (j *BurpFormat) SetOptions(options formats.InputFormatOptions) { // Parse parses the input and calls the provided callback // function for each RawRequest it discovers. -func (j *BurpFormat) Parse(input string, resultsCb formats.ParseReqRespCallback) error { - file, err := os.Open(input) - if err != nil { - return errors.Wrap(err, "could not open data file") - } - defer file.Close() - - items, err := burpxml.Parse(file, true) +func (j *BurpFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallback, filePath string) error { + items, err := burpxml.Parse(input, true) if err != nil { return errors.Wrap(err, "could not decode burp xml schema") } diff --git a/pkg/input/formats/burp/burp_test.go b/pkg/input/formats/burp/burp_test.go index 330218a9e5..97e80c534f 100644 --- a/pkg/input/formats/burp/burp_test.go +++ b/pkg/input/formats/burp/burp_test.go @@ -1,6 +1,7 @@ package burp import ( + "os" "testing" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" @@ -14,10 +15,14 @@ func TestBurpParse(t *testing.T) { var gotMethodsToURLs []string - err := format.Parse(proxifyInputFile, func(request *types.RequestResponse) bool { + file, err := os.Open(proxifyInputFile) + require.Nilf(t, err, "error opening proxify input file: %v", err) + defer file.Close() + + err = format.Parse(file, func(request *types.RequestResponse) bool { gotMethodsToURLs = append(gotMethodsToURLs, request.URL.String()) return false - }) + }, proxifyInputFile) if err != nil { t.Fatal(err) } diff --git a/pkg/input/formats/formats.go b/pkg/input/formats/formats.go index af2b4569c6..03c65d3fea 100644 --- a/pkg/input/formats/formats.go +++ b/pkg/input/formats/formats.go @@ -2,6 +2,7 @@ package formats import ( "errors" + "io" "os" "strings" @@ -35,7 +36,7 @@ type Format interface { Name() string // Parse parses the input and calls the provided callback // function for each RawRequest it discovers. - Parse(input string, resultsCb ParseReqRespCallback) error + Parse(input io.Reader, resultsCb ParseReqRespCallback, filePath string) error // SetOptions sets the options for the input format SetOptions(options InputFormatOptions) } diff --git a/pkg/input/formats/json/json.go b/pkg/input/formats/json/json.go index 69e628c684..38c2117fcb 100644 --- a/pkg/input/formats/json/json.go +++ b/pkg/input/formats/json/json.go @@ -3,7 +3,6 @@ package json import ( "encoding/json" "io" - "os" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" @@ -46,14 +45,8 @@ func (j *JSONFormat) SetOptions(options formats.InputFormatOptions) { // Parse parses the input and calls the provided callback // function for each RawRequest it discovers. -func (j *JSONFormat) Parse(input string, resultsCb formats.ParseReqRespCallback) error { - file, err := os.Open(input) - if err != nil { - return errors.Wrap(err, "could not open json file") - } - defer file.Close() - - decoder := json.NewDecoder(file) +func (j *JSONFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallback, filePath string) error { + decoder := json.NewDecoder(input) for { var request proxifyRequest err := decoder.Decode(&request) diff --git a/pkg/input/formats/json/json_test.go b/pkg/input/formats/json/json_test.go index b72bf4c197..a6734f083e 100644 --- a/pkg/input/formats/json/json_test.go +++ b/pkg/input/formats/json/json_test.go @@ -1,6 +1,7 @@ package json import ( + "os" "testing" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" @@ -41,11 +42,15 @@ func TestJSONFormatterParse(t *testing.T) { proxifyInputFile := "../testdata/ginandjuice.proxify.json" + file, err := os.Open(proxifyInputFile) + require.Nilf(t, err, "error opening proxify input file: %v", err) + defer file.Close() + var urls []string - err := format.Parse(proxifyInputFile, func(request *types.RequestResponse) bool { + err = format.Parse(file, func(request *types.RequestResponse) bool { urls = append(urls, request.URL.String()) return false - }) + }, proxifyInputFile) if err != nil { t.Fatal(err) } diff --git a/pkg/input/formats/openapi/openapi.go b/pkg/input/formats/openapi/openapi.go index afbe379fd2..c2086636b4 100644 --- a/pkg/input/formats/openapi/openapi.go +++ b/pkg/input/formats/openapi/openapi.go @@ -1,6 +1,8 @@ package openapi import ( + "io" + "github.com/getkin/kin-openapi/openapi3" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" @@ -29,9 +31,9 @@ func (j *OpenAPIFormat) SetOptions(options formats.InputFormatOptions) { // Parse parses the input and calls the provided callback // function for each RawRequest it discovers. -func (j *OpenAPIFormat) Parse(input string, resultsCb formats.ParseReqRespCallback) error { +func (j *OpenAPIFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallback, filePath string) error { loader := openapi3.NewLoader() - schema, err := loader.LoadFromFile(input) + schema, err := loader.LoadFromIoReader(input) if err != nil { return errors.Wrap(err, "could not decode openapi 3.0 schema") } diff --git a/pkg/input/formats/openapi/openapi_test.go b/pkg/input/formats/openapi/openapi_test.go index f48385a808..c202bdcbee 100644 --- a/pkg/input/formats/openapi/openapi_test.go +++ b/pkg/input/formats/openapi/openapi_test.go @@ -1,6 +1,7 @@ package openapi import ( + "os" "strings" "testing" @@ -41,11 +42,15 @@ func TestOpenAPIParser(t *testing.T) { gotMethodsToURLs := make(map[string][]string) - err := format.Parse(proxifyInputFile, func(rr *types.RequestResponse) bool { + file, err := os.Open(proxifyInputFile) + require.Nilf(t, err, "error opening proxify input file: %v", err) + defer file.Close() + + err = format.Parse(file, func(rr *types.RequestResponse) bool { gotMethodsToURLs[rr.Request.Method] = append(gotMethodsToURLs[rr.Request.Method], strings.Replace(rr.URL.String(), baseURL, "{{baseUrl}}", 1)) return false - }) + }, proxifyInputFile) if err != nil { t.Fatal(err) } diff --git a/pkg/input/formats/swagger/swagger.go b/pkg/input/formats/swagger/swagger.go index 30a7564ecc..5410b24fc0 100644 --- a/pkg/input/formats/swagger/swagger.go +++ b/pkg/input/formats/swagger/swagger.go @@ -3,15 +3,14 @@ package swagger import ( "encoding/json" "io" - "os" "path" "github.com/getkin/kin-openapi/openapi2" "github.com/getkin/kin-openapi/openapi3" + "github.com/invopop/yaml" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/openapi" - "github.com/invopop/yaml" "github.com/getkin/kin-openapi/openapi2conv" ) @@ -39,24 +38,18 @@ func (j *SwaggerFormat) SetOptions(options formats.InputFormatOptions) { // Parse parses the input and calls the provided callback // function for each RawRequest it discovers. -func (j *SwaggerFormat) Parse(input string, resultsCb formats.ParseReqRespCallback) error { - file, err := os.Open(input) - if err != nil { - return errors.Wrap(err, "could not open data file") - } - defer file.Close() - +func (j *SwaggerFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallback, filePath string) error { schemav2 := &openapi2.T{} - ext := path.Ext(input) - + ext := path.Ext(filePath) + var err error if ext == ".yaml" || ext == ".yml" { - data, err_data := io.ReadAll(file) - if err_data != nil { + data, err := io.ReadAll(input) + if err != nil { return errors.Wrap(err, "could not read data file") } err = yaml.Unmarshal(data, schemav2) } else { - err = json.NewDecoder(file).Decode(schemav2) + err = json.NewDecoder(input).Decode(schemav2) } if err != nil { return errors.Wrap(err, "could not decode openapi 2.0 schema") diff --git a/pkg/input/formats/swagger/swagger_test.go b/pkg/input/formats/swagger/swagger_test.go index 065ae78f63..caed82a13b 100644 --- a/pkg/input/formats/swagger/swagger_test.go +++ b/pkg/input/formats/swagger/swagger_test.go @@ -1,6 +1,7 @@ package swagger import ( + "os" "testing" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" @@ -14,10 +15,14 @@ func TestSwaggerAPIParser(t *testing.T) { var gotMethodsToURLs []string - err := format.Parse(proxifyInputFile, func(request *types.RequestResponse) bool { + file, err := os.Open(proxifyInputFile) + require.Nilf(t, err, "error opening proxify input file: %v", err) + defer file.Close() + + err = format.Parse(file, func(request *types.RequestResponse) bool { gotMethodsToURLs = append(gotMethodsToURLs, request.URL.String()) return false - }) + }, proxifyInputFile) if err != nil { t.Fatal(err) } diff --git a/pkg/input/formats/yaml/multidoc.go b/pkg/input/formats/yaml/multidoc.go index dc258408c1..6d75e0334a 100644 --- a/pkg/input/formats/yaml/multidoc.go +++ b/pkg/input/formats/yaml/multidoc.go @@ -2,7 +2,6 @@ package yaml import ( "io" - "os" "strings" "github.com/pkg/errors" @@ -46,14 +45,8 @@ func (j *YamlMultiDocFormat) SetOptions(options formats.InputFormatOptions) { // Parse parses the input and calls the provided callback // function for each RawRequest it discovers. -func (j *YamlMultiDocFormat) Parse(input string, resultsCb formats.ParseReqRespCallback) error { - file, err := os.Open(input) - if err != nil { - return errors.Wrap(err, "could not open json file") - } - defer file.Close() - - decoder := YamlUtil.NewDecoder(file) +func (j *YamlMultiDocFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallback, filePath string) error { + decoder := YamlUtil.NewDecoder(input) for { var request proxifyRequest err := decoder.Decode(&request) diff --git a/pkg/input/formats/yaml/multidoc_test.go b/pkg/input/formats/yaml/multidoc_test.go index 6275eae593..0b91e774a3 100644 --- a/pkg/input/formats/yaml/multidoc_test.go +++ b/pkg/input/formats/yaml/multidoc_test.go @@ -1,6 +1,7 @@ package yaml import ( + "os" "testing" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" @@ -17,11 +18,15 @@ func TestYamlFormatterParse(t *testing.T) { "https://ginandjuice.shop/users/3", } + file, err := os.Open(proxifyInputFile) + require.Nilf(t, err, "error opening proxify input file: %v", err) + defer file.Close() + var urls []string - err := format.Parse(proxifyInputFile, func(request *types.RequestResponse) bool { + err = format.Parse(file, func(request *types.RequestResponse) bool { urls = append(urls, request.URL.String()) return false - }) + }, proxifyInputFile) require.Nilf(t, err, "error parsing yaml file: %v", err) require.Len(t, urls, len(expectedUrls), "invalid number of urls") require.ElementsMatch(t, urls, expectedUrls, "invalid urls") diff --git a/pkg/input/provider/http/multiformat.go b/pkg/input/provider/http/multiformat.go index d58970fec5..5218dc005a 100644 --- a/pkg/input/provider/http/multiformat.go +++ b/pkg/input/provider/http/multiformat.go @@ -1,6 +1,8 @@ package http import ( + "io" + "os" "strings" "github.com/pkg/errors" @@ -23,14 +25,18 @@ type HttpMultiFormatOptions struct { InputFile string // InputMode is the mode of input InputMode string + + // optional input reader + InputContents string } // HttpInputProvider implements an input provider for nuclei that loads // inputs from multiple formats like burp, openapi, postman,proxify, etc. type HttpInputProvider struct { - format formats.Format - inputFile string - count int64 + format formats.Format + inputReader io.Reader + inputFile string + count int64 } // NewHttpInputProvider creates a new input provider for nuclei from a file @@ -48,14 +54,31 @@ func NewHttpInputProvider(opts *HttpMultiFormatOptions) (*HttpInputProvider, err // Do a first pass over the input to identify any errors // and get the count of the input file as well count := int64(0) - parseErr := format.Parse(opts.InputFile, func(request *types.RequestResponse) bool { + var inputFile *os.File + var inputReader io.Reader + if opts.InputFile != "" { + file, err := os.Open(opts.InputFile) + if err != nil { + return nil, errors.Wrap(err, "could not open input file") + } + inputFile = file + inputReader = file + } else { + inputReader = strings.NewReader(opts.InputContents) + } + defer func() { + if inputFile != nil { + inputFile.Close() + } + }() + parseErr := format.Parse(inputReader, func(request *types.RequestResponse) bool { count++ return false - }) + }, opts.InputFile) if parseErr != nil { return nil, errors.Wrap(parseErr, "could not parse input file") } - return &HttpInputProvider{format: format, inputFile: opts.InputFile, count: count}, nil + return &HttpInputProvider{format: format, inputReader: inputReader, inputFile: opts.InputFile, count: count}, nil } // Count returns the number of items for input provider @@ -65,12 +88,12 @@ func (i *HttpInputProvider) Count() int64 { // Iterate over all inputs in order func (i *HttpInputProvider) Iterate(callback func(value *contextargs.MetaInput) bool) { - err := i.format.Parse(i.inputFile, func(request *types.RequestResponse) bool { + err := i.format.Parse(i.inputReader, func(request *types.RequestResponse) bool { metaInput := contextargs.NewMetaInput() metaInput.ReqResp = request metaInput.Input = request.URL.String() return callback(metaInput) - }) + }, i.inputFile) if err != nil { gologger.Warning().Msgf("Could not parse input file while iterating: %s\n", err) } From 50d0952ec661fad481df04263568e55819ba5356 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Thu, 21 Nov 2024 02:46:35 +0530 Subject: [PATCH 05/23] feat: added stats db to fuzzing + use sdk for dast server + misc --- cmd/nuclei/main.go | 6 + go.mod | 4 +- go.sum | 19 +-- internal/runner/runner.go | 47 +++++- internal/server/exec.go | 111 -------------- internal/server/nuclei_sdk.go | 197 +++++++++++++++++++++++++ internal/server/requests_worker.go | 19 ++- internal/server/server.go | 37 ++++- pkg/fuzz/stats/db.go | 189 ++++++++++++++++++++++++ pkg/fuzz/stats/db_test.go | 44 ++++++ pkg/fuzz/stats/schema.sql | 60 ++++++++ pkg/fuzz/stats/stats.go | 55 +++++++ pkg/input/provider/http/multiformat.go | 21 ++- pkg/protocols/http/request.go | 13 ++ pkg/protocols/protocols.go | 3 + pkg/types/types.go | 2 + 16 files changed, 676 insertions(+), 151 deletions(-) delete mode 100644 internal/server/exec.go create mode 100644 internal/server/nuclei_sdk.go create mode 100644 pkg/fuzz/stats/db.go create mode 100644 pkg/fuzz/stats/db_test.go create mode 100644 pkg/fuzz/stats/schema.sql create mode 100644 pkg/fuzz/stats/stats.go diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index ef1bd6d996..645ecb2c11 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/copernicium-112/namegenerator" _pdcp "github.com/projectdiscovery/nuclei/v3/internal/pdcp" "github.com/projectdiscovery/utils/auth/pdcp" "github.com/projectdiscovery/utils/env" @@ -220,6 +221,10 @@ func main() { } } +var ( + nameGenerator = namegenerator.NewNameGenerator(time.Now().UnixNano()) +) + func readConfig() *goflags.FlagSet { // when true updates nuclei binary to latest version @@ -360,6 +365,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"), flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"), flagSet.BoolVarP(&options.DASTServer, "dast-server", "dts", false, "enable dast server mode (live fuzzing)"), + flagSet.StringVarP(&options.DASTScanName, "dast-scan-report", "dtr", "", "write dast scan report to file"), flagSet.StringVarP(&options.DASTServerToken, "dast-server-token", "dtst", "", "dast server token (optional)"), flagSet.StringVarP(&options.DASTServerAddress, "dast-server-address", "dtsa", "localhost:9055", "dast server address"), flagSet.BoolVarP(&options.DisplayFuzzPoints, "display-fuzz-points", "dfp", false, "display fuzz points in the output for debugging"), diff --git a/go.mod b/go.mod index 1c9d6bc1b7..0da92bc734 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( github.com/DataDog/gostackparse v0.6.0 github.com/Masterminds/semver/v3 v3.2.1 github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 + github.com/alitto/pond v1.9.2 github.com/antchfx/xmlquery v1.3.17 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/aws/aws-sdk-go-v2 v1.19.0 @@ -61,6 +62,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/charmbracelet/glamour v0.8.0 github.com/clbanning/mxj/v2 v2.7.0 + github.com/copernicium-112/namegenerator v0.0.0-20230403095523-b8a39e9024ce github.com/ditashi/jsbeautifier-go v0.0.0-20141206144643-2520a8026a9c github.com/docker/go-units v0.5.0 github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 @@ -76,7 +78,7 @@ require ( github.com/labstack/echo/v4 v4.12.0 github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa github.com/lib/pq v1.10.9 - github.com/mattn/go-sqlite3 v1.14.22 + github.com/mattn/go-sqlite3 v1.14.24 github.com/mholt/archiver v3.1.1+incompatible github.com/microsoft/go-mssqldb v1.6.0 github.com/ory/dockertest/v3 v3.10.0 diff --git a/go.sum b/go.sum index 9c426501e1..e83c94cbb7 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,8 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA= github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= +github.com/alitto/pond v1.9.2 h1:9Qb75z/scEZVCoSU+osVmQ0I0JOeLfdTDafrbcJ8CLs= +github.com/alitto/pond v1.9.2/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= @@ -259,6 +261,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/copernicium-112/namegenerator v0.0.0-20230403095523-b8a39e9024ce h1:RYy81PJmnYFxSgtpEzUz9ebM5bBQPbYL0kgtVgQh5HQ= +github.com/copernicium-112/namegenerator v0.0.0-20230403095523-b8a39e9024ce/go.mod h1:ix5lu9xtBLUk6EyGj+NFmXNkjGnJM1ep+zHmCruA/gI= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -667,12 +671,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M= -github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k= github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+kI0= github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM= -github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= -github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= @@ -701,12 +701,10 @@ github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJv github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -715,8 +713,8 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mholt/acmez v1.2.0 h1:1hhLxSgY5FvH5HCnGUuwbKY2VQVo8IU7rxXKSnZ7F30= @@ -1078,7 +1076,6 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= @@ -1391,10 +1388,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1633,7 +1627,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo= diff --git a/internal/runner/runner.go b/internal/runner/runner.go index a20c4d8959..e4e78374a7 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -41,6 +41,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v3/pkg/core" "github.com/projectdiscovery/nuclei/v3/pkg/external/customtemplates" + fuzzStats "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats" "github.com/projectdiscovery/nuclei/v3/pkg/input" parsers "github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow" "github.com/projectdiscovery/nuclei/v3/pkg/output" @@ -443,15 +444,30 @@ func (r *Runner) RunEnumeration() error { // If the user has asked for DAST server mode, run the live // DAST fuzzing server. if r.options.DASTServer { + execurOpts := &server.NucleiExecutorOptions{ + Options: r.options, + Output: r.output, + Progress: r.progress, + Catalog: r.catalog, + IssuesClient: r.issuesClient, + RateLimiter: r.rateLimiter, + Interactsh: r.interactsh, + ProjectFile: r.projectFile, + Browser: r.browser, + Colorizer: r.colorizer, + Parser: r.parser, + TemporaryDirectory: r.tmpDir, + } dastServer, err := server.New(&server.Options{ - Address: r.options.DASTServerAddress, - Concurrency: r.options.BulkSize, - Templates: r.options.Templates, - OutputWriter: r.output, - Verbose: r.options.Verbose, - Token: r.options.DASTServerToken, - InScope: r.options.Scope, - OutScope: r.options.OutOfScope, + Address: r.options.DASTServerAddress, + Concurrency: r.options.BulkSize, + Templates: r.options.Templates, + OutputWriter: r.output, + Verbose: r.options.Verbose, + Token: r.options.DASTServerToken, + InScope: r.options.Scope, + OutScope: r.options.OutOfScope, + NucleiExecutorOptions: execurOpts, }) if err != nil { return err @@ -501,6 +517,13 @@ func (r *Runner) RunEnumeration() error { FuzzParamsFrequency: fuzzFreqCache, GlobalMatchers: globalmatchers.New(), } + if r.options.DASTScanName != "" { + var err error + executorOpts.FuzzStatsDB, err = fuzzStats.NewTracker(r.options.DASTScanName) + if err != nil { + return errors.Wrap(err, "could not create fuzz stats db") + } + } if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) { // Go StdLib style experimental/debug feature switch @@ -653,6 +676,14 @@ func (r *Runner) RunEnumeration() error { return err } + if executorOpts.FuzzStatsDB != nil { + err = executorOpts.FuzzStatsDB.GenerateReport("report.html") + if err != nil { + gologger.Error().Msgf("Failed to generate fuzzing report: %v", err) + } + executorOpts.FuzzStatsDB.Close() + + } if r.interactsh != nil { matched := r.interactsh.Close() if matched { diff --git a/internal/server/exec.go b/internal/server/exec.go deleted file mode 100644 index cc53992d7e..0000000000 --- a/internal/server/exec.go +++ /dev/null @@ -1,111 +0,0 @@ -package server - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "os" - "os/exec" - - "github.com/projectdiscovery/nuclei/v3/pkg/output" - "gopkg.in/yaml.v2" -) - -// proxifyRequest is a request for proxify -type proxifyRequest struct { - URL string `json:"url"` - Request struct { - Header map[string]string `json:"header"` - Body string `json:"body"` - Raw string `json:"raw"` - } `json:"request"` -} - -func (s *DASTServer) runNucleiWithFuzzingInput(target PostReuestsHandlerRequest, templates []string) ([]output.ResultEvent, error) { - cmd := exec.Command("nuclei") - - tempFile, err := os.CreateTemp("", "nuclei-fuzz-*.yaml") - if err != nil { - return nil, fmt.Errorf("error creating temp file: %s", err) - } - defer os.Remove(tempFile.Name()) - - payload := proxifyRequest{ - URL: target.URL, - Request: struct { - Header map[string]string `json:"header"` - Body string `json:"body"` - Raw string `json:"raw"` - }{ - Raw: target.RawHTTP, - }, - } - - marshalledYaml, err := yaml.Marshal(payload) - if err != nil { - return nil, fmt.Errorf("error marshalling yaml: %s", err) - } - - if _, err := tempFile.Write(marshalledYaml); err != nil { - return nil, fmt.Errorf("error writing to temp file: %s", err) - } - - argsArray := []string{ - "-duc", - "-dast", - "-no-color", - "-jsonl", - } - for _, template := range templates { - argsArray = append(argsArray, "-t", template) - } - argsArray = append(argsArray, "-l", tempFile.Name()) - argsArray = append(argsArray, "-im=yaml") - - var stderrBuf bytes.Buffer - if s.options.Verbose { - cmd.Stderr = &stderrBuf - argsArray = append(argsArray, "-v") - } else { - argsArray = append(argsArray, "-silent") - } - cmd.Args = append(cmd.Args, argsArray...) - - stdoutPipe, err := cmd.StdoutPipe() - if err != nil { - return nil, fmt.Errorf("error creating stdout pipe: %s", err) - } - - errWithStderr := func(err error) error { - if s.options.Verbose { - return fmt.Errorf("error running nuclei: %s\n%s", err, stderrBuf.String()) - } - return fmt.Errorf("error starting nuclei: %s", err) - } - - if err := cmd.Start(); err != nil { - return nil, errWithStderr(err) - } - - var nucleiResult []output.ResultEvent - decoder := json.NewDecoder(stdoutPipe) - for { - var result output.ResultEvent - if err := decoder.Decode(&result); err != nil { - if err == io.EOF { - break - } - return nil, fmt.Errorf("error decoding nuclei output: %w", err) - } - // Filter results with a valid template-id - if result.TemplateID != "" { - nucleiResult = append(nucleiResult, result) - } - } - - if err := cmd.Wait(); err != nil { - return nil, errWithStderr(err) - } - return nucleiResult, nil -} diff --git a/internal/server/nuclei_sdk.go b/internal/server/nuclei_sdk.go new file mode 100644 index 0000000000..7a5108a1a0 --- /dev/null +++ b/internal/server/nuclei_sdk.go @@ -0,0 +1,197 @@ +package server + +import ( + "context" + "fmt" + _ "net/http/pprof" + "strings" + + "github.com/logrusorgru/aurora" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/frequency" + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" + "github.com/projectdiscovery/nuclei/v3/pkg/input/provider/http" + "github.com/projectdiscovery/nuclei/v3/pkg/projectfile" + "gopkg.in/yaml.v3" + + "github.com/pkg/errors" + "github.com/projectdiscovery/ratelimit" + + "github.com/projectdiscovery/nuclei/v3/pkg/catalog" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" + "github.com/projectdiscovery/nuclei/v3/pkg/core" + fuzzStats "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats" + "github.com/projectdiscovery/nuclei/v3/pkg/input" + "github.com/projectdiscovery/nuclei/v3/pkg/loader/parser" + parsers "github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow" + "github.com/projectdiscovery/nuclei/v3/pkg/output" + "github.com/projectdiscovery/nuclei/v3/pkg/progress" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/globalmatchers" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/hosterrorscache" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/excludematchers" + browserEngine "github.com/projectdiscovery/nuclei/v3/pkg/protocols/headless/engine" + "github.com/projectdiscovery/nuclei/v3/pkg/reporting" + "github.com/projectdiscovery/nuclei/v3/pkg/templates" + "github.com/projectdiscovery/nuclei/v3/pkg/types" +) + +type nucleiExecutor struct { + engine *core.Engine + store *loader.Store + options *NucleiExecutorOptions + executorOpts protocols.ExecutorOptions +} + +type NucleiExecutorOptions struct { + Options *types.Options + Output output.Writer + Progress progress.Progress + Catalog catalog.Catalog + IssuesClient reporting.Client + RateLimiter *ratelimit.Limiter + Interactsh *interactsh.Client + ProjectFile *projectfile.ProjectFile + Browser *browserEngine.Browser + Colorizer aurora.Aurora + Parser parser.Parser + TemporaryDirectory string +} + +func newNucleiExecutor(opts *NucleiExecutorOptions) (*nucleiExecutor, error) { + fuzzFreqCache := frequency.New(frequency.DefaultMaxTrackCount, opts.Options.FuzzParamFrequency) + resumeCfg := types.NewResumeCfg() + + // Create the executor options which will be used throughout the execution + // stage by the nuclei engine modules. + executorOpts := protocols.ExecutorOptions{ + Output: opts.Output, + Options: opts.Options, + Progress: opts.Progress, + Catalog: opts.Catalog, + IssuesClient: opts.IssuesClient, + RateLimiter: opts.RateLimiter, + Interactsh: opts.Interactsh, + ProjectFile: opts.ProjectFile, + Browser: opts.Browser, + Colorizer: opts.Colorizer, + ResumeCfg: resumeCfg, + ExcludeMatchers: excludematchers.New(opts.Options.ExcludeMatchers), + InputHelper: input.NewHelper(), + TemporaryDirectory: opts.TemporaryDirectory, + Parser: opts.Parser, + FuzzParamsFrequency: fuzzFreqCache, + GlobalMatchers: globalmatchers.New(), + } + if opts.Options.DASTScanName != "" { + var err error + executorOpts.FuzzStatsDB, err = fuzzStats.NewTracker(opts.Options.DASTScanName) + if err != nil { + return nil, errors.Wrap(err, "could not create fuzz stats db") + } + } + + if opts.Options.ShouldUseHostError() { + maxHostError := opts.Options.MaxHostError + if maxHostError == 30 { + maxHostError = 100 // auto adjust for fuzzings + } + if opts.Options.TemplateThreads > maxHostError { + gologger.Info().Msgf("Adjusting max-host-error to the concurrency value: %d", opts.Options.TemplateThreads) + + maxHostError = opts.Options.TemplateThreads + } + + cache := hosterrorscache.New(maxHostError, hosterrorscache.DefaultMaxHostsCount, opts.Options.TrackError) + cache.SetVerbose(opts.Options.Verbose) + + executorOpts.HostErrorsCache = cache + } + + executorEngine := core.New(opts.Options) + executorEngine.SetExecuterOptions(executorOpts) + + workflowLoader, err := parsers.NewLoader(&executorOpts) + if err != nil { + return nil, errors.Wrap(err, "Could not create loadeopts.") + } + executorOpts.WorkflowLoader = workflowLoader + + // If using input-file flags, only load http fuzzing based templates. + loaderConfig := loader.NewConfig(opts.Options, opts.Catalog, executorOpts) + if !strings.EqualFold(opts.Options.InputFileMode, "list") || opts.Options.DAST || opts.Options.DASTServer { + // if input type is not list (implicitly enable fuzzing) + opts.Options.DAST = true + } + store, err := loader.New(loaderConfig) + if err != nil { + return nil, errors.Wrap(err, "Could not create loadeopts.") + } + store.Load() + + return &nucleiExecutor{ + engine: executorEngine, + store: store, + options: opts, + executorOpts: executorOpts, + }, nil +} + +func (n *nucleiExecutor) ExecuteScan(target PostReuestsHandlerRequest) error { + finalTemplates := []*templates.Template{} + finalTemplates = append(finalTemplates, n.store.Templates()...) + finalTemplates = append(finalTemplates, n.store.Workflows()...) + + if len(finalTemplates) == 0 { + return errors.New("no templates provided for scan") + } + + payload := proxifyRequest{ + URL: target.URL, + Request: struct { + Header map[string]string `json:"header"` + Body string `json:"body"` + Raw string `json:"raw"` + }{ + Raw: target.RawHTTP, + }, + } + + marshalledYaml, err := yaml.Marshal(payload) + if err != nil { + return fmt.Errorf("error marshalling yaml: %s", err) + } + + inputProvider, err := http.NewHttpInputProvider(&http.HttpMultiFormatOptions{ + InputContents: string(marshalledYaml), + InputMode: "yaml", + Options: formats.InputFormatOptions{ + Variables: make(map[string]interface{}), + }, + }) + if err != nil { + return errors.Wrap(err, "could not create input provider") + } + _ = n.engine.ExecuteScanWithOpts(context.Background(), finalTemplates, inputProvider, true) + return nil +} + +func (n *nucleiExecutor) Close() { + var err error + if n.executorOpts.FuzzStatsDB != nil { + err = n.executorOpts.FuzzStatsDB.GenerateReport("report.html") + if err != nil { + gologger.Error().Msgf("Failed to generate fuzzing report: %v", err) + } + n.executorOpts.FuzzStatsDB.Close() + + } + if n.options.Interactsh != nil { + _ = n.options.Interactsh.Close() + } + if n.executorOpts.InputHelper != nil { + _ = n.executorOpts.InputHelper.Close() + } + +} diff --git a/internal/server/requests_worker.go b/internal/server/requests_worker.go index 3f254576b5..d0c509463f 100644 --- a/internal/server/requests_worker.go +++ b/internal/server/requests_worker.go @@ -34,24 +34,29 @@ func (s *DASTServer) tasksConsumer() { } gologger.Verbose().Msgf("Fuzzing request: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String()) - s.tasksPool.Go(func() { + s.tasksPool.Submit(func() { s.fuzzRequest(req) }) } } func (s *DASTServer) fuzzRequest(req PostReuestsHandlerRequest) { - results, err := s.runNucleiWithFuzzingInput(req, s.options.Templates) + err := s.nucleiExecutor.ExecuteScan(req) if err != nil { gologger.Warning().Msgf("Could not run nuclei: %s\n", err) return } + // results, err := s.runNucleiWithFuzzingInput(req, s.options.Templates) + // if err != nil { + // gologger.Warning().Msgf("Could not run nuclei: %s\n", err) + // return + // } - for _, result := range results { - if err := s.options.OutputWriter.Write(&result); err != nil { - gologger.Error().Msgf("Could not write result: %s\n", err) - } - } + // for _, result := range results { + // if err := s.options.OutputWriter.Write(&result); err != nil { + // gologger.Error().Msgf("Could not write result: %s\n", err) + // } + // } } func parseRawRequest(req PostReuestsHandlerRequest) (*types.RequestResponse, error) { diff --git a/internal/server/server.go b/internal/server/server.go index d3ab8e50cb..be799c1af8 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -6,12 +6,13 @@ import ( "strings" "time" + "github.com/alitto/pond" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v3/internal/server/scope" "github.com/projectdiscovery/nuclei/v3/pkg/output" - "github.com/sourcegraph/conc/pool" ) // DASTServer is a server that performs execution of fuzzing templates @@ -19,10 +20,12 @@ import ( type DASTServer struct { echo *echo.Echo options *Options - tasksPool *pool.Pool + tasksPool *pond.WorkerPool deduplicator *requestDeduplicator scopeManager *scope.Manager fuzzRequests chan PostReuestsHandlerRequest + + nucleiExecutor *nucleiExecutor } // Options contains the configuration options for the server. @@ -43,10 +46,15 @@ type Options struct { OutScope []string OutputWriter output.Writer + + NucleiExecutorOptions *NucleiExecutorOptions } // New creates a new instance of the DAST server. func New(options *Options) (*DASTServer, error) { + gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) + options.NucleiExecutorOptions.Options.Verbose = true + bufferSize := options.Concurrency * 100 // If the user has specified no templates, use the default ones @@ -56,13 +64,19 @@ func New(options *Options) (*DASTServer, error) { } server := &DASTServer{ options: options, - tasksPool: pool.New().WithMaxGoroutines(options.Concurrency), + tasksPool: pond.New(10000, options.Concurrency), deduplicator: newRequestDeduplicator(), fuzzRequests: make(chan PostReuestsHandlerRequest, bufferSize), } server.setupHandlers() server.setupWorkers() + executor, err := newNucleiExecutor(options.NucleiExecutorOptions) + if err != nil { + return nil, err + } + server.nucleiExecutor = executor + scopeManager, err := scope.NewManager( options.InScope, options.OutScope, @@ -77,7 +91,7 @@ func New(options *Options) (*DASTServer, error) { if options.Token != "" { builder.WriteString(" (with token)") } - gologger.Info().Msgf(builder.String()) + gologger.Info().Msgf("%s", builder.String()) gologger.Info().Msgf("Connection URL: %s", server.buildConnectionURL()) return server, nil @@ -105,15 +119,30 @@ func (s *DASTServer) setupHandlers() { Validator: func(key string, c echo.Context) (bool, error) { return key == s.options.Token, nil }, + Skipper: func(c echo.Context) bool { + return c.Path() == "/stats" + }, })) } e.HideBanner = true // POST /requests - Queue a request for fuzzing e.POST("/requests", s.handleRequest) + e.GET("/stats", s.handleStats) + + // Serve a Web Server to visualize the stats in a live HTML report + e.GET("/ui", func(c echo.Context) error { + return c.File("internal/server/ui/index.html") + }) s.echo = e } +func (s *DASTServer) handleStats(c echo.Context) error { + return c.JSON(200, map[string]interface{}{ + "queued_requests": len(s.fuzzRequests), + }) +} + func (s *DASTServer) Start() error { return s.echo.Start(s.options.Address) } diff --git a/pkg/fuzz/stats/db.go b/pkg/fuzz/stats/db.go new file mode 100644 index 0000000000..e84db9ac45 --- /dev/null +++ b/pkg/fuzz/stats/db.go @@ -0,0 +1,189 @@ +package stats + +import ( + "database/sql" + _ "embed" + "fmt" + "os" + "sync" + + _ "github.com/mattn/go-sqlite3" + "github.com/pkg/errors" +) + +type StatsDatabase interface { + Close() + + InsertRecord(event FuzzingEvent) error +} + +var ( + //go:embed schema.sql + dbSchemaCreateStatement string +) + +type sqliteStatsDatabase struct { + db *sql.DB + scanName string + + siteIDCache map[string]int + templateIDCache map[string]int + componentIDCache map[string]int + cacheMutex *sync.Mutex +} + +func newSqliteStatsDatabase(scanName string) (*sqliteStatsDatabase, error) { + filename := fmt.Sprintf("%s.stats.db", scanName) + + connectionString := fmt.Sprintf("./%s?_journal_mode=WAL&_synchronous=NORMAL", filename) + db, err := sql.Open("sqlite3", connectionString) + if err != nil { + return nil, errors.Wrap(err, "could not open database") + } + + _, err = db.Exec(dbSchemaCreateStatement) + if err != nil { + return nil, errors.Wrap(err, "could not create schema") + } + + database := &sqliteStatsDatabase{ + scanName: scanName, + db: db, + siteIDCache: make(map[string]int), + templateIDCache: make(map[string]int), + componentIDCache: make(map[string]int), + cacheMutex: &sync.Mutex{}, + } + return database, nil +} + +func (s *sqliteStatsDatabase) Close() { + // Disable WAL mode and switch back to DELETE mode + _ = s.db.Close() + os.Remove(fmt.Sprintf("%s.stats.db-wal", s.scanName)) + os.Remove(fmt.Sprintf("%s.stats.db-shm", s.scanName)) +} + +func (s *sqliteStatsDatabase) InsertRecord(event FuzzingEvent) error { + tx, err := s.db.Begin() + if err != nil { + return errors.Wrap(err, "could not begin transaction") + } + + defer func() { + if err != nil { + tx.Rollback() + } + }() + + siteID, err := s.getSiteID(tx, event.SiteName) + if err != nil { + return errors.Wrap(err, "could not get site_id") + } + + templateID, err := s.getTemplateID(tx, event.TemplateID) + if err != nil { + return errors.Wrap(err, "could not get template_id") + } + + componentID, err := s.getComponentID(tx, siteID, event.ComponentType, event.ComponentName) + if err != nil { + return errors.Wrap(err, "could not get component_id") + } + + err = s.insertFuzzingResult(tx, componentID, templateID, event.PayloadSent, event.StatusCode) + if err != nil { + return errors.Wrap(err, "could not insert fuzzing result") + } + + if err := tx.Commit(); err != nil { + return errors.Wrap(err, "could not commit transaction") + } + + return nil +} + +func (s *sqliteStatsDatabase) getSiteID(tx *sql.Tx, siteName string) (int, error) { + var siteID int + + s.cacheMutex.Lock() + if id, ok := s.siteIDCache[siteName]; ok { + s.cacheMutex.Unlock() + return id, nil + } + s.cacheMutex.Unlock() + + err := tx.QueryRow( + `INSERT OR IGNORE INTO sites (site_name) + VALUES (?) RETURNING site_id + `, siteName).Scan(&siteID) + if err != nil { + return 0, err + } + + // Cache the site_id + s.cacheMutex.Lock() + s.siteIDCache[siteName] = siteID + s.cacheMutex.Unlock() + + return siteID, nil +} + +func (s *sqliteStatsDatabase) getTemplateID(tx *sql.Tx, templateName string) (int, error) { + var templateID int + + s.cacheMutex.Lock() + if id, ok := s.templateIDCache[templateName]; ok { + s.cacheMutex.Unlock() + return id, nil + } + s.cacheMutex.Unlock() + + err := tx.QueryRow(` + INSERT OR IGNORE INTO templates (template_name) + VALUES (?) RETURNING template_id + `, templateName).Scan(&templateID) + if err != nil { + return 0, err + } + + s.cacheMutex.Lock() + s.templateIDCache[templateName] = templateID + s.cacheMutex.Unlock() + + return templateID, nil +} + +func (s *sqliteStatsDatabase) getComponentID(tx *sql.Tx, siteID int, componentType, componentName string) (int, error) { + key := fmt.Sprintf("%d:%s:%s", siteID, componentType, componentName) + var componentID int + + s.cacheMutex.Lock() + if id, ok := s.componentIDCache[key]; ok { + s.cacheMutex.Unlock() + return id, nil + } + s.cacheMutex.Unlock() + + err := tx.QueryRow(` + INSERT OR IGNORE INTO components (site_id, component_type, component_name) + VALUES (?, ?, ?) RETURNING component_id + `, siteID, componentType, componentName).Scan(&componentID) + if err != nil { + return 0, err + } + + s.cacheMutex.Lock() + s.componentIDCache[key] = componentID + s.cacheMutex.Unlock() + + return componentID, nil +} + +func (s *sqliteStatsDatabase) insertFuzzingResult(tx *sql.Tx, componentID, templateID int, payloadSent string, statusCode int) error { + _, err := tx.Exec(` + INSERT INTO fuzzing_results (component_id, template_id, payload_sent, status_code_received) + VALUES (?, ?, ?, ?) + `, componentID, templateID, payloadSent, statusCode) + return err +} diff --git a/pkg/fuzz/stats/db_test.go b/pkg/fuzz/stats/db_test.go new file mode 100644 index 0000000000..7e73a21754 --- /dev/null +++ b/pkg/fuzz/stats/db_test.go @@ -0,0 +1,44 @@ +package stats + +import ( + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_NewStatsDatabase(t *testing.T) { + db, err := newSqliteStatsDatabase("test") + require.NoError(t, err) + + err = db.InsertRecord(FuzzingEvent{ + URL: "http://localhost:8080/login", + SiteName: "localhost:8080", + TemplateID: "apache-struts2-001", + ComponentType: "path", + ComponentName: "/login", + PayloadSent: "/login'\"><", + StatusCode: 401, + }) + require.NoError(t, err) + + var siteName string + err = db.db.QueryRow("SELECT template_name FROM templates WHERE template_id = 1").Scan(&siteName) + require.NoError(t, err) + require.Equal(t, "apache-struts2-001", siteName) + + err = db.InsertRecord(FuzzingEvent{ + URL: "http://localhost:8080/login", + SiteName: "localhost:8080", + TemplateID: "apache-struts2-001", + ComponentType: "path", + ComponentName: "/login", + PayloadSent: "/login'\"><", + StatusCode: 401, + }) + require.NoError(t, err) + + db.Close() + + os.Remove("test.stats.db") +} diff --git a/pkg/fuzz/stats/schema.sql b/pkg/fuzz/stats/schema.sql new file mode 100644 index 0000000000..2ff4cf216b --- /dev/null +++ b/pkg/fuzz/stats/schema.sql @@ -0,0 +1,60 @@ +CREATE TABLE IF NOT EXISTS sites ( + site_id INTEGER PRIMARY KEY AUTOINCREMENT, + site_name TEXT UNIQUE NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_sites_site_name ON sites(site_name); + +CREATE TABLE IF NOT EXISTS components ( + component_id INTEGER PRIMARY KEY AUTOINCREMENT, + site_id INTEGER NOT NULL, + component_type TEXT NOT NULL CHECK (component_type IN ('path', 'query', 'header', 'body', 'cookie')), + component_name TEXT NOT NULL, + last_fuzzed DATETIME, + total_fuzz_count INTEGER DEFAULT 0, + FOREIGN KEY (site_id) REFERENCES sites(site_id), + UNIQUE (site_id, component_type, component_name) +); +CREATE INDEX IF NOT EXISTS idx_components_site_type_name ON components (site_id, component_type, component_name); + + +CREATE TABLE IF NOT EXISTS templates ( + template_id INTEGER PRIMARY KEY AUTOINCREMENT, + template_name TEXT UNIQUE NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_templates_template_name ON templates(template_name); + +CREATE TABLE IF NOT EXISTS component_templates ( + component_id INTEGER NOT NULL, + template_id INTEGER NOT NULL, + times_applied INTEGER DEFAULT 0, + PRIMARY KEY (component_id, template_id), + FOREIGN KEY (component_id) REFERENCES components(component_id), + FOREIGN KEY (template_id) REFERENCES templates(template_id) +); + +CREATE TABLE IF NOT EXISTS fuzzing_results ( + result_id INTEGER PRIMARY KEY AUTOINCREMENT, + component_id INTEGER NOT NULL, + template_id INTEGER NOT NULL, + payload_sent TEXT NOT NULL, + status_code_received INTEGER NOT NULL, + timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (component_id) REFERENCES components(component_id), + FOREIGN KEY (template_id) REFERENCES templates(template_id) +); +CREATE INDEX IF NOT EXISTS idx_FuzzingResults_comp_temp_time ON fuzzing_results (component_id, template_id, timestamp); + +-- Trigger to update stats when a new fuzzing result is inserted +CREATE TRIGGER IF NOT EXISTS update_component_stats +AFTER INSERT ON fuzzing_results +BEGIN + UPDATE components + SET last_fuzzed = NEW.timestamp, + total_fuzz_count = total_fuzz_count + 1 + WHERE component_id = NEW.component_id; + + INSERT INTO component_templates (component_id, template_id, times_applied) + VALUES (NEW.component_id, NEW.template_id, 1) + ON CONFLICT(component_id, template_id) DO UPDATE SET + times_applied = times_applied + 1; +END; diff --git a/pkg/fuzz/stats/stats.go b/pkg/fuzz/stats/stats.go new file mode 100644 index 0000000000..d29484d225 --- /dev/null +++ b/pkg/fuzz/stats/stats.go @@ -0,0 +1,55 @@ +// Package stats implements a statistics recording module for +// nuclei fuzzing. +package stats + +import ( + "net/url" + + "github.com/pkg/errors" +) + +// Tracker is a stats tracker module for fuzzing server +type Tracker struct { + database StatsDatabase +} + +// NewTracker creates a new tracker instance +func NewTracker(scanName string) (*Tracker, error) { + db, err := newSqliteStatsDatabase(scanName) + if err != nil { + return nil, errors.Wrap(err, "could not create new tracker") + } + + tracker := &Tracker{ + database: db, + } + return tracker, nil +} + +// Close closes the tracker +func (t *Tracker) Close() { + t.database.Close() +} + +// FuzzingEvent is a fuzzing event +type FuzzingEvent struct { + URL string + ComponentType string + ComponentName string + TemplateID string + PayloadSent string + StatusCode int + SiteName string +} + +func (t *Tracker) RecordEvent(event FuzzingEvent) { + parsed, err := url.Parse(event.URL) + if err != nil { + return + } + + // Site is the host:port combo + event.SiteName = parsed.Host + + t.database.InsertRecord(event) +} diff --git a/pkg/input/provider/http/multiformat.go b/pkg/input/provider/http/multiformat.go index 5218dc005a..f4bb656143 100644 --- a/pkg/input/provider/http/multiformat.go +++ b/pkg/input/provider/http/multiformat.go @@ -1,6 +1,7 @@ package http import ( + "bytes" "io" "os" "strings" @@ -33,10 +34,10 @@ type HttpMultiFormatOptions struct { // HttpInputProvider implements an input provider for nuclei that loads // inputs from multiple formats like burp, openapi, postman,proxify, etc. type HttpInputProvider struct { - format formats.Format - inputReader io.Reader - inputFile string - count int64 + format formats.Format + inputData []byte + inputFile string + count int64 } // NewHttpInputProvider creates a new input provider for nuclei from a file @@ -71,14 +72,20 @@ func NewHttpInputProvider(opts *HttpMultiFormatOptions) (*HttpInputProvider, err inputFile.Close() } }() - parseErr := format.Parse(inputReader, func(request *types.RequestResponse) bool { + + data, err := io.ReadAll(inputReader) + if err != nil { + return nil, errors.Wrap(err, "could not read input file") + } + + parseErr := format.Parse(bytes.NewReader(data), func(request *types.RequestResponse) bool { count++ return false }, opts.InputFile) if parseErr != nil { return nil, errors.Wrap(parseErr, "could not parse input file") } - return &HttpInputProvider{format: format, inputReader: inputReader, inputFile: opts.InputFile, count: count}, nil + return &HttpInputProvider{format: format, inputData: data, inputFile: opts.InputFile, count: count}, nil } // Count returns the number of items for input provider @@ -88,7 +95,7 @@ func (i *HttpInputProvider) Count() int64 { // Iterate over all inputs in order func (i *HttpInputProvider) Iterate(callback func(value *contextargs.MetaInput) bool) { - err := i.format.Parse(i.inputReader, func(request *types.RequestResponse) bool { + err := i.format.Parse(bytes.NewReader(i.inputData), func(request *types.RequestResponse) bool { metaInput := contextargs.NewMetaInput() metaInput.ReqResp = request metaInput.Input = request.URL.String() diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 4ce9e57f55..6cde77e30e 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -20,6 +20,7 @@ import ( "github.com/projectdiscovery/fastdialer/fastdialer" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/analyzers" + fuzzStats "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats" "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" @@ -979,6 +980,18 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ finalEvent[k] = v } + if request.options.FuzzStatsDB != nil && generatedRequest.fuzzGeneratedRequest.Request != nil { + request.options.FuzzStatsDB.RecordEvent(fuzzStats.FuzzingEvent{ + URL: matchedURL, + SiteName: hostname, + TemplateID: request.options.TemplateID, + ComponentType: generatedRequest.fuzzGeneratedRequest.Component.Name(), + ComponentName: generatedRequest.fuzzGeneratedRequest.Key, + PayloadSent: generatedRequest.fuzzGeneratedRequest.Value, + StatusCode: respChain.Response().StatusCode, + }) + } + // Add to history the current request number metadata if asked by the user. if request.NeedsRequestCondition() { for k, v := range outputEvent { diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 9ead70321d..7b7f71d485 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -14,6 +14,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/authprovider" "github.com/projectdiscovery/nuclei/v3/pkg/catalog" "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/frequency" + "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats" "github.com/projectdiscovery/nuclei/v3/pkg/input" "github.com/projectdiscovery/nuclei/v3/pkg/js/compiler" "github.com/projectdiscovery/nuclei/v3/pkg/loader/parser" @@ -99,6 +100,8 @@ type ExecutorOptions struct { InputHelper *input.Helper // FuzzParamsFrequency is a cache for parameter frequency FuzzParamsFrequency *frequency.Tracker + // FuzzStatsDB is a database for fuzzing stats + FuzzStatsDB *stats.Tracker Operators []*operators.Operators // only used by offlinehttp module diff --git a/pkg/types/types.go b/pkg/types/types.go index 8f9d74a50e..4ca2b28af6 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -415,6 +415,8 @@ type Options struct { DASTServerToken string // DASTServerAddress is the address for the dast server DASTServerAddress string + // DASTScanName is the name of the scan to use for the dast report + DASTScanName string // Scope contains a list of regexes for in-scope URLS Scope goflags.StringSlice // OutOfScope contains a list of regexes for out-scope URLS From 03959515336b5efbedac382e6b697e4f34047554 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 22 Nov 2024 00:32:39 +0530 Subject: [PATCH 06/23] feat: more additions and enhancements --- go.mod | 3 +- go.sum | 6 ++- internal/runner/runner.go | 6 --- internal/server/nuclei_sdk.go | 16 +++--- internal/server/requests_worker.go | 82 ++++++++++++----------------- internal/server/scope/extensions.go | 33 ++++++++++++ internal/server/server.go | 46 ++++++---------- internal/server/server_test.go | 24 --------- pkg/catalog/loader/loader.go | 3 +- pkg/fuzz/component/component.go | 2 +- pkg/fuzz/component/cookie.go | 48 ----------------- pkg/fuzz/stats/db.go | 69 ++++++++++++++++++------ pkg/fuzz/stats/db_test.go | 10 ++-- pkg/fuzz/stats/schema.sql | 6 ++- pkg/fuzz/stats/stats.go | 17 ++++-- pkg/protocols/http/request.go | 37 ++++++++----- 16 files changed, 206 insertions(+), 202 deletions(-) create mode 100644 internal/server/scope/extensions.go delete mode 100644 internal/server/server_test.go diff --git a/go.mod b/go.mod index 0da92bc734..a0c377428b 100644 --- a/go.mod +++ b/go.mod @@ -72,6 +72,7 @@ require ( github.com/go-ldap/ldap/v3 v3.4.5 github.com/go-pg/pg v8.0.7+incompatible github.com/go-sql-driver/mysql v1.7.1 + github.com/gorilla/mux v1.8.1 github.com/h2non/filetype v1.1.3 github.com/invopop/yaml v0.3.1 github.com/kitabisa/go-ci v1.0.3 @@ -104,7 +105,6 @@ require ( github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 - github.com/sourcegraph/conc v0.3.0 github.com/stretchr/testify v1.9.0 github.com/tarunKoyalwar/goleak v0.0.0-20240429141123-0efa90dbdcf9 github.com/yassinebenaid/godump v0.10.0 @@ -138,6 +138,7 @@ require ( github.com/bits-and-blooms/bloom/v3 v3.5.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect + github.com/ccojocar/randdetect v0.0.0-20241118085251-1581dcdbf207 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/charmbracelet/lipgloss v0.13.0 // indirect diff --git a/go.sum b/go.sum index e83c94cbb7..f0b109466f 100644 --- a/go.sum +++ b/go.sum @@ -216,6 +216,8 @@ github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZX github.com/caddyserver/certmagic v0.19.2 h1:HZd1AKLx4592MalEGQS39DKs2ZOAJCEM/xYPMQ2/ui0= github.com/caddyserver/certmagic v0.19.2/go.mod h1:fsL01NomQ6N+kE2j37ZCnig2MFosG+MIO4ztnmG/zz8= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/ccojocar/randdetect v0.0.0-20241118085251-1581dcdbf207 h1:ZXvIckmW4Ky9CYRXGzf3kdnivvpUOUiEdDb5afC0VKk= +github.com/ccojocar/randdetect v0.0.0-20241118085251-1581dcdbf207/go.mod h1:bR+6Ytp4l03qh4oOxwjzR/ld5ssouHtjIOdTKb8fox0= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -529,6 +531,8 @@ github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -999,8 +1003,6 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= diff --git a/internal/runner/runner.go b/internal/runner/runner.go index e4e78374a7..45310d4c80 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -460,7 +460,6 @@ func (r *Runner) RunEnumeration() error { } dastServer, err := server.New(&server.Options{ Address: r.options.DASTServerAddress, - Concurrency: r.options.BulkSize, Templates: r.options.Templates, OutputWriter: r.output, Verbose: r.options.Verbose, @@ -677,12 +676,7 @@ func (r *Runner) RunEnumeration() error { } if executorOpts.FuzzStatsDB != nil { - err = executorOpts.FuzzStatsDB.GenerateReport("report.html") - if err != nil { - gologger.Error().Msgf("Failed to generate fuzzing report: %v", err) - } executorOpts.FuzzStatsDB.Close() - } if r.interactsh != nil { matched := r.interactsh.Close() diff --git a/internal/server/nuclei_sdk.go b/internal/server/nuclei_sdk.go index 7a5108a1a0..37966afd0e 100644 --- a/internal/server/nuclei_sdk.go +++ b/internal/server/nuclei_sdk.go @@ -138,6 +138,16 @@ func newNucleiExecutor(opts *NucleiExecutorOptions) (*nucleiExecutor, error) { }, nil } +// proxifyRequest is a request for proxify +type proxifyRequest struct { + URL string `json:"url"` + Request struct { + Header map[string]string `json:"header"` + Body string `json:"body"` + Raw string `json:"raw"` + } `json:"request"` +} + func (n *nucleiExecutor) ExecuteScan(target PostReuestsHandlerRequest) error { finalTemplates := []*templates.Template{} finalTemplates = append(finalTemplates, n.store.Templates()...) @@ -178,14 +188,8 @@ func (n *nucleiExecutor) ExecuteScan(target PostReuestsHandlerRequest) error { } func (n *nucleiExecutor) Close() { - var err error if n.executorOpts.FuzzStatsDB != nil { - err = n.executorOpts.FuzzStatsDB.GenerateReport("report.html") - if err != nil { - gologger.Error().Msgf("Failed to generate fuzzing report: %v", err) - } n.executorOpts.FuzzStatsDB.Close() - } if n.options.Interactsh != nil { _ = n.options.Interactsh.Close() diff --git a/internal/server/requests_worker.go b/internal/server/requests_worker.go index d0c509463f..8058015a42 100644 --- a/internal/server/requests_worker.go +++ b/internal/server/requests_worker.go @@ -1,43 +1,50 @@ package server import ( - "github.com/pkg/errors" + "path" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/internal/server/scope" "github.com/projectdiscovery/nuclei/v3/pkg/input/types" ) -func (s *DASTServer) setupWorkers() { - go s.tasksConsumer() -} +func (s *DASTServer) consumeTaskRequest(req PostReuestsHandlerRequest) { + parsedReq, err := types.ParseRawRequestWithURL(req.RawHTTP, req.URL) + if err != nil { + gologger.Warning().Msgf("Could not parse raw request: %s\n", err) + return + } -func (s *DASTServer) tasksConsumer() { - for req := range s.fuzzRequests { - parsedReq, err := parseRawRequest(req) - if err != nil { - gologger.Warning().Msgf("Could not parse raw request: %s\n", err) - continue - } + if parsedReq.URL.Scheme != "http" && parsedReq.URL.Scheme != "https" { + return + } - inScope, err := s.scopeManager.Validate(parsedReq.URL.URL, "") - if err != nil { - gologger.Warning().Msgf("Could not validate scope: %s\n", err) - continue - } - if !inScope { - gologger.Warning().Msgf("Request is out of scope: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String()) - continue - } + // Check filenames and don't allow non-interesting files + extension := path.Base(parsedReq.URL.Path) + if extension != "/" && extension != "" && scope.IsUninterestingPath(extension) { + gologger.Warning().Msgf("Uninteresting path: %s\n", parsedReq.URL.Path) + return + } - if s.deduplicator.isDuplicate(parsedReq) { - gologger.Warning().Msgf("Duplicate request detected: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String()) - continue - } + inScope, err := s.scopeManager.Validate(parsedReq.URL.URL, "") + if err != nil { + gologger.Warning().Msgf("Could not validate scope: %s\n", err) + return + } + if !inScope { + gologger.Warning().Msgf("Request is out of scope: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String()) + return + } - gologger.Verbose().Msgf("Fuzzing request: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String()) - s.tasksPool.Submit(func() { - s.fuzzRequest(req) - }) + if s.deduplicator.isDuplicate(parsedReq) { + gologger.Warning().Msgf("Duplicate request detected: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String()) + return } + + gologger.Verbose().Msgf("Fuzzing request: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String()) + + // Fuzz the request finally + s.fuzzRequest(req) } func (s *DASTServer) fuzzRequest(req PostReuestsHandlerRequest) { @@ -46,23 +53,4 @@ func (s *DASTServer) fuzzRequest(req PostReuestsHandlerRequest) { gologger.Warning().Msgf("Could not run nuclei: %s\n", err) return } - // results, err := s.runNucleiWithFuzzingInput(req, s.options.Templates) - // if err != nil { - // gologger.Warning().Msgf("Could not run nuclei: %s\n", err) - // return - // } - - // for _, result := range results { - // if err := s.options.OutputWriter.Write(&result); err != nil { - // gologger.Error().Msgf("Could not write result: %s\n", err) - // } - // } -} - -func parseRawRequest(req PostReuestsHandlerRequest) (*types.RequestResponse, error) { - parsedReq, err := types.ParseRawRequestWithURL(req.RawHTTP, req.URL) - if err != nil { - return nil, errors.Wrap(err, "could not parse raw HTTP") - } - return parsedReq, nil } diff --git a/internal/server/scope/extensions.go b/internal/server/scope/extensions.go new file mode 100644 index 0000000000..f7e5929189 --- /dev/null +++ b/internal/server/scope/extensions.go @@ -0,0 +1,33 @@ +package scope + +import "path" + +func IsUninterestingPath(uriPath string) bool { + extension := path.Ext(uriPath) + if _, ok := excludedExtensions[extension]; ok { + return true + } + return false +} + +var excludedExtensions = map[string]struct{}{ + ".jpg": {}, ".jpeg": {}, ".png": {}, ".gif": {}, ".bmp": {}, ".tiff": {}, ".ico": {}, + ".mp4": {}, ".avi": {}, ".mov": {}, ".wmv": {}, ".flv": {}, ".mkv": {}, ".webm": {}, + ".mp3": {}, ".wav": {}, ".aac": {}, ".flac": {}, ".ogg": {}, ".wma": {}, + ".zip": {}, ".rar": {}, ".7z": {}, ".tar": {}, ".gz": {}, ".bz2": {}, + ".exe": {}, ".bin": {}, ".iso": {}, ".img": {}, + ".doc": {}, ".docx": {}, ".xls": {}, ".xlsx": {}, ".ppt": {}, ".pptx": {}, + ".pdf": {}, ".psd": {}, ".ai": {}, ".eps": {}, ".indd": {}, + ".swf": {}, ".fla": {}, ".css": {}, ".scss": {}, ".less": {}, + ".js": {}, ".ts": {}, ".jsx": {}, ".tsx": {}, + ".xml": {}, ".json": {}, ".yaml": {}, ".yml": {}, + ".csv": {}, ".txt": {}, ".log": {}, ".md": {}, + ".ttf": {}, ".otf": {}, ".woff": {}, ".woff2": {}, ".eot": {}, + ".svg": {}, ".svgz": {}, ".webp": {}, ".tif": {}, + ".mpg": {}, ".mpeg": {}, ".weba": {}, + ".m4a": {}, ".m4v": {}, ".3gp": {}, ".3g2": {}, + ".ogv": {}, ".ogm": {}, ".oga": {}, ".ogx": {}, + ".srt": {}, ".min.js": {}, ".min.css": {}, ".js.map": {}, + ".min.js.map": {}, ".chunk.css.map": {}, ".hub.js.map": {}, + ".hub.css.map": {}, ".map": {}, +} diff --git a/internal/server/server.go b/internal/server/server.go index be799c1af8..953e418743 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,18 +1,16 @@ package server import ( - "encoding/json" "fmt" "strings" - "time" "github.com/alitto/pond" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "github.com/projectdiscovery/gologger" - "github.com/projectdiscovery/gologger/levels" "github.com/projectdiscovery/nuclei/v3/internal/server/scope" "github.com/projectdiscovery/nuclei/v3/pkg/output" + "github.com/projectdiscovery/utils/env" ) // DASTServer is a server that performs execution of fuzzing templates @@ -23,7 +21,6 @@ type DASTServer struct { tasksPool *pond.WorkerPool deduplicator *requestDeduplicator scopeManager *scope.Manager - fuzzRequests chan PostReuestsHandlerRequest nucleiExecutor *nucleiExecutor } @@ -34,8 +31,6 @@ type Options struct { Address string // Token is the token to use for authentication (optional) Token string - // Concurrency is the concurrency level to use for the targets - Concurrency int // Templates is the list of templates to use for fuzzing Templates []string // Verbose is a flag that controls verbose output @@ -52,24 +47,26 @@ type Options struct { // New creates a new instance of the DAST server. func New(options *Options) (*DASTServer, error) { - gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) - options.NucleiExecutorOptions.Options.Verbose = true - - bufferSize := options.Concurrency * 100 - // If the user has specified no templates, use the default ones // for DAST only. if len(options.Templates) == 0 { options.Templates = []string{"dast/"} } + // Disable bulk mode and single threaded execution + // by auto adjusting in case of default values + if options.NucleiExecutorOptions.Options.BulkSize == 25 && options.NucleiExecutorOptions.Options.TemplateThreads == 25 { + options.NucleiExecutorOptions.Options.BulkSize = 1 + options.NucleiExecutorOptions.Options.TemplateThreads = 1 + } + maxWorkers := env.GetEnvOrDefault[int]("FUZZ_MAX_WORKERS", 1) + bufferSize := env.GetEnvOrDefault[int]("FUZZ_BUFFER_SIZE", 10000) + server := &DASTServer{ options: options, - tasksPool: pond.New(10000, options.Concurrency), + tasksPool: pond.New(maxWorkers, bufferSize), deduplicator: newRequestDeduplicator(), - fuzzRequests: make(chan PostReuestsHandlerRequest, bufferSize), } server.setupHandlers() - server.setupWorkers() executor, err := newNucleiExecutor(options.NucleiExecutorOptions) if err != nil { @@ -87,7 +84,7 @@ func New(options *Options) (*DASTServer, error) { server.scopeManager = scopeManager var builder strings.Builder - builder.WriteString(fmt.Sprintf("Using %d parallel tasks with %d buffer", options.Concurrency, bufferSize)) + builder.WriteString(fmt.Sprintf("Using %d parallel tasks with %d buffer", maxWorkers, bufferSize)) if options.Token != "" { builder.WriteString(" (with token)") } @@ -138,9 +135,7 @@ func (s *DASTServer) setupHandlers() { } func (s *DASTServer) handleStats(c echo.Context) error { - return c.JSON(200, map[string]interface{}{ - "queued_requests": len(s.fuzzRequests), - }) + return c.JSON(200, map[string]interface{}{}) } func (s *DASTServer) Start() error { @@ -164,15 +159,8 @@ func (s *DASTServer) handleRequest(c echo.Context) error { return c.JSON(400, map[string]string{"error": "missing required fields"}) } - if s.options.Verbose { - marshalIndented, _ := json.MarshalIndent(req, "", " ") - gologger.Verbose().Msgf("Received request: %s", marshalIndented) - } - - select { - case s.fuzzRequests <- req: - return c.NoContent(200) - case timeout := <-time.After(5 * time.Second): - return c.JSON(429, map[string]string{"error": fmt.Sprintf("server busy, try again after %v", timeout)}) - } + s.tasksPool.Submit(func() { + s.consumeTaskRequest(req) + }) + return c.NoContent(200) } diff --git a/internal/server/server_test.go b/internal/server/server_test.go deleted file mode 100644 index d5e0981517..0000000000 --- a/internal/server/server_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package server - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func Test_parseRawRequest(t *testing.T) { - parsed, err := parseRawRequest(PostReuestsHandlerRequest{ - URL: "http://example.com/testpath", - RawHTTP: "GET /testpath HTTP/1.1\nHost: example.com\nUser-Agent: Mozilla/5.0\n\n", - }) - require.NoError(t, err) - require.Equal(t, "http://example.com/testpath", parsed.URL.String()) - - // Example POST request - parsed, err = parseRawRequest(PostReuestsHandlerRequest{ - URL: "http://example.com", - RawHTTP: "POST /testpath HTTP/1.1\nHost: example.com\nUser-Agent: Mozilla/5.0\nContent-Length: 5\n\nhello", - }) - require.NoError(t, err) - require.Equal(t, "hello", parsed.Request.Body) -} diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index 06d0f8b6d1..dd0d439df1 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -498,7 +498,8 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ // Skip DAST filter when loading auth templates if store.ID() != AuthStoreId && store.config.ExecutorOptions.Options.DAST { // check if the template is a DAST template - if parsed.IsFuzzing() { + // also allow global matchers template to be loaded + if parsed.IsFuzzing() || parsed.Options.GlobalMatchers != nil && parsed.Options.GlobalMatchers.HasMatchers() { loadTemplate(parsed) } } else if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless { diff --git a/pkg/fuzz/component/component.go b/pkg/fuzz/component/component.go index a15ac2856a..c3500048b1 100644 --- a/pkg/fuzz/component/component.go +++ b/pkg/fuzz/component/component.go @@ -67,8 +67,8 @@ const ( var Components = []string{ RequestBodyComponent, RequestQueryComponent, - RequestPathComponent, RequestHeaderComponent, + RequestPathComponent, RequestCookieComponent, } diff --git a/pkg/fuzz/component/cookie.go b/pkg/fuzz/component/cookie.go index 77667c7479..295dfce9f5 100644 --- a/pkg/fuzz/component/cookie.go +++ b/pkg/fuzz/component/cookie.go @@ -52,10 +52,6 @@ func (c *Cookie) Parse(req *retryablehttp.Request) (bool, error) { // Iterate iterates through the component func (c *Cookie) Iterate(callback func(key string, value interface{}) error) (err error) { c.value.parsed.Iterate(func(key string, value any) bool { - // Skip ignored cookies - if _, ok := defaultIgnoredCookieKeys[key]; ok { - return ok - } if errx := callback(key, value); errx != nil { err = errx return false @@ -106,47 +102,3 @@ func (c *Cookie) Clone() Component { req: c.req.Clone(context.Background()), } } - -// A list of cookies that are essential to the request and -// must not be fuzzed. -var defaultIgnoredCookieKeys = map[string]struct{}{ - "awsELB": {}, - "AWSALB": {}, - "AWSALBCORS": {}, - "__utma": {}, - "__utmb": {}, - "__utmc": {}, - "__utmt": {}, - "__utmz": {}, - "_ga": {}, - "_gat": {}, - "_gid": {}, - "_gcl_au": {}, - "_fbp": {}, - "fr": {}, - "__hstc": {}, - "hubspotutk": {}, - "__hssc": {}, - "__hssrc": {}, - "mp_mixpanel__c": {}, - "JSESSIONID": {}, - "NREUM": {}, - "_pk_id": {}, - "_pk_ref": {}, - "_pk_ses": {}, - "_pk_cvar": {}, - "_pk_hsr": {}, - "_hjIncludedInSample": {}, - "__cfduid": {}, - "cf_use_ob": {}, - "cf_ob_info": {}, - "intercom-session": {}, - "optimizelyEndUserId": {}, - "optimizelySegments": {}, - "optimizelyBuckets": {}, - "optimizelyPendingLogEvents": {}, - "YSC": {}, - "VISITOR_INFO1_LIVE": {}, - "PREF": {}, - "GPS": {}, -} diff --git a/pkg/fuzz/stats/db.go b/pkg/fuzz/stats/db.go index e84db9ac45..cef72ccc1f 100644 --- a/pkg/fuzz/stats/db.go +++ b/pkg/fuzz/stats/db.go @@ -14,7 +14,8 @@ import ( type StatsDatabase interface { Close() - InsertRecord(event FuzzingEvent) error + InsertComponent(event FuzzingEvent) error + InsertMatchedRecord(event FuzzingEvent) error } var ( @@ -32,7 +33,7 @@ type sqliteStatsDatabase struct { cacheMutex *sync.Mutex } -func newSqliteStatsDatabase(scanName string) (*sqliteStatsDatabase, error) { +func NewSqliteStatsDatabase(scanName string) (*sqliteStatsDatabase, error) { filename := fmt.Sprintf("%s.stats.db", scanName) connectionString := fmt.Sprintf("./%s?_journal_mode=WAL&_synchronous=NORMAL", filename) @@ -64,7 +65,45 @@ func (s *sqliteStatsDatabase) Close() { os.Remove(fmt.Sprintf("%s.stats.db-shm", s.scanName)) } -func (s *sqliteStatsDatabase) InsertRecord(event FuzzingEvent) error { +func (s *sqliteStatsDatabase) DB() *sql.DB { + return s.db +} + +func (s *sqliteStatsDatabase) InsertComponent(event FuzzingEvent) error { + tx, err := s.db.Begin() + if err != nil { + return errors.Wrap(err, "could not begin transaction") + } + + defer func() { + if err != nil { + tx.Rollback() + } + }() + + siteID, err := s.getSiteID(tx, event.SiteName) + if err != nil { + return errors.Wrap(err, "could not get site_id") + } + + _, err = s.getTemplateID(tx, event.TemplateID) + if err != nil { + return errors.Wrap(err, "could not get template_id") + } + + _, err = s.getComponentID(tx, siteID, event.ComponentType, event.ComponentName, event.URL) + if err != nil { + return errors.Wrap(err, "could not get component_id") + } + + if err := tx.Commit(); err != nil { + return errors.Wrap(err, "could not commit transaction") + } + + return nil +} + +func (s *sqliteStatsDatabase) InsertMatchedRecord(event FuzzingEvent) error { tx, err := s.db.Begin() if err != nil { return errors.Wrap(err, "could not begin transaction") @@ -86,12 +125,12 @@ func (s *sqliteStatsDatabase) InsertRecord(event FuzzingEvent) error { return errors.Wrap(err, "could not get template_id") } - componentID, err := s.getComponentID(tx, siteID, event.ComponentType, event.ComponentName) + componentID, err := s.getComponentID(tx, siteID, event.ComponentType, event.ComponentName, event.URL) if err != nil { return errors.Wrap(err, "could not get component_id") } - err = s.insertFuzzingResult(tx, componentID, templateID, event.PayloadSent, event.StatusCode) + err = s.insertFuzzingResult(tx, componentID, templateID, event.PayloadSent, event.StatusCode, event.Matched) if err != nil { return errors.Wrap(err, "could not insert fuzzing result") } @@ -153,9 +192,8 @@ func (s *sqliteStatsDatabase) getTemplateID(tx *sql.Tx, templateName string) (in return templateID, nil } - -func (s *sqliteStatsDatabase) getComponentID(tx *sql.Tx, siteID int, componentType, componentName string) (int, error) { - key := fmt.Sprintf("%d:%s:%s", siteID, componentType, componentName) +func (s *sqliteStatsDatabase) getComponentID(tx *sql.Tx, siteID int, componentType, componentName, url string) (int, error) { + key := fmt.Sprintf("%d:%s:%s:%s", siteID, componentType, componentName, url) var componentID int s.cacheMutex.Lock() @@ -166,9 +204,10 @@ func (s *sqliteStatsDatabase) getComponentID(tx *sql.Tx, siteID int, componentTy s.cacheMutex.Unlock() err := tx.QueryRow(` - INSERT OR IGNORE INTO components (site_id, component_type, component_name) - VALUES (?, ?, ?) RETURNING component_id - `, siteID, componentType, componentName).Scan(&componentID) + INSERT OR IGNORE INTO components (site_id, component_type, component_name, url) + VALUES (?, ?, ?, ?) + RETURNING component_id + `, siteID, componentType, componentName, url).Scan(&componentID) if err != nil { return 0, err } @@ -180,10 +219,10 @@ func (s *sqliteStatsDatabase) getComponentID(tx *sql.Tx, siteID int, componentTy return componentID, nil } -func (s *sqliteStatsDatabase) insertFuzzingResult(tx *sql.Tx, componentID, templateID int, payloadSent string, statusCode int) error { +func (s *sqliteStatsDatabase) insertFuzzingResult(tx *sql.Tx, componentID, templateID int, payloadSent string, statusCode int, matched bool) error { _, err := tx.Exec(` - INSERT INTO fuzzing_results (component_id, template_id, payload_sent, status_code_received) - VALUES (?, ?, ?, ?) - `, componentID, templateID, payloadSent, statusCode) + INSERT INTO fuzzing_results (component_id, template_id, payload_sent, status_code_received, matched) + VALUES (?, ?, ?, ?, ?) + `, componentID, templateID, payloadSent, statusCode, matched) return err } diff --git a/pkg/fuzz/stats/db_test.go b/pkg/fuzz/stats/db_test.go index 7e73a21754..92de173fe8 100644 --- a/pkg/fuzz/stats/db_test.go +++ b/pkg/fuzz/stats/db_test.go @@ -1,17 +1,16 @@ package stats import ( - "os" "testing" "github.com/stretchr/testify/require" ) func Test_NewStatsDatabase(t *testing.T) { - db, err := newSqliteStatsDatabase("test") + db, err := NewSqliteStatsDatabase("test") require.NoError(t, err) - err = db.InsertRecord(FuzzingEvent{ + err = db.InsertComponent(FuzzingEvent{ URL: "http://localhost:8080/login", SiteName: "localhost:8080", TemplateID: "apache-struts2-001", @@ -27,7 +26,7 @@ func Test_NewStatsDatabase(t *testing.T) { require.NoError(t, err) require.Equal(t, "apache-struts2-001", siteName) - err = db.InsertRecord(FuzzingEvent{ + err = db.InsertMatchedRecord(FuzzingEvent{ URL: "http://localhost:8080/login", SiteName: "localhost:8080", TemplateID: "apache-struts2-001", @@ -35,10 +34,11 @@ func Test_NewStatsDatabase(t *testing.T) { ComponentName: "/login", PayloadSent: "/login'\"><", StatusCode: 401, + Matched: true, }) require.NoError(t, err) db.Close() - os.Remove("test.stats.db") + //os.Remove("test.stats.db") } diff --git a/pkg/fuzz/stats/schema.sql b/pkg/fuzz/stats/schema.sql index 2ff4cf216b..233fa11da0 100644 --- a/pkg/fuzz/stats/schema.sql +++ b/pkg/fuzz/stats/schema.sql @@ -10,11 +10,12 @@ CREATE TABLE IF NOT EXISTS components ( component_type TEXT NOT NULL CHECK (component_type IN ('path', 'query', 'header', 'body', 'cookie')), component_name TEXT NOT NULL, last_fuzzed DATETIME, + url TEXT NOT NULL, total_fuzz_count INTEGER DEFAULT 0, FOREIGN KEY (site_id) REFERENCES sites(site_id), - UNIQUE (site_id, component_type, component_name) + UNIQUE (site_id, component_type, component_name, url) ); -CREATE INDEX IF NOT EXISTS idx_components_site_type_name ON components (site_id, component_type, component_name); +CREATE INDEX IF NOT EXISTS idx_components_site_type_name ON components (site_id, component_type, component_name, url); CREATE TABLE IF NOT EXISTS templates ( @@ -38,6 +39,7 @@ CREATE TABLE IF NOT EXISTS fuzzing_results ( template_id INTEGER NOT NULL, payload_sent TEXT NOT NULL, status_code_received INTEGER NOT NULL, + matched BOOLEAN DEFAULT FALSE NOT NULL, timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (component_id) REFERENCES components(component_id), FOREIGN KEY (template_id) REFERENCES templates(template_id) diff --git a/pkg/fuzz/stats/stats.go b/pkg/fuzz/stats/stats.go index d29484d225..5ea6aa70cb 100644 --- a/pkg/fuzz/stats/stats.go +++ b/pkg/fuzz/stats/stats.go @@ -15,7 +15,7 @@ type Tracker struct { // NewTracker creates a new tracker instance func NewTracker(scanName string) (*Tracker, error) { - db, err := newSqliteStatsDatabase(scanName) + db, err := NewSqliteStatsDatabase(scanName) if err != nil { return nil, errors.Wrap(err, "could not create new tracker") } @@ -39,10 +39,11 @@ type FuzzingEvent struct { TemplateID string PayloadSent string StatusCode int + Matched bool SiteName string } -func (t *Tracker) RecordEvent(event FuzzingEvent) { +func (t *Tracker) RecordResultEvent(event FuzzingEvent) { parsed, err := url.Parse(event.URL) if err != nil { return @@ -50,6 +51,16 @@ func (t *Tracker) RecordEvent(event FuzzingEvent) { // Site is the host:port combo event.SiteName = parsed.Host + t.database.InsertMatchedRecord(event) +} + +func (t *Tracker) RecordComponentEvent(event FuzzingEvent) { + parsed, err := url.Parse(event.URL) + if err != nil { + return + } - t.database.InsertRecord(event) + // Site is the host:port combo + event.SiteName = parsed.Host + t.database.InsertComponent(event) } diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 6cde77e30e..333f7edfe9 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -930,6 +930,18 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ } } + if request.options.FuzzStatsDB != nil && generatedRequest.fuzzGeneratedRequest.Request != nil { + request.options.FuzzStatsDB.RecordComponentEvent(fuzzStats.FuzzingEvent{ + URL: input.MetaInput.Target(), + SiteName: hostname, + TemplateID: request.options.TemplateID, + ComponentType: generatedRequest.fuzzGeneratedRequest.Component.Name(), + ComponentName: generatedRequest.fuzzGeneratedRequest.Parameter, + PayloadSent: generatedRequest.fuzzGeneratedRequest.Value, + StatusCode: respChain.Response().StatusCode, + }) + } + finalEvent := make(output.InternalEvent) if request.Analyzer != nil { @@ -980,18 +992,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ finalEvent[k] = v } - if request.options.FuzzStatsDB != nil && generatedRequest.fuzzGeneratedRequest.Request != nil { - request.options.FuzzStatsDB.RecordEvent(fuzzStats.FuzzingEvent{ - URL: matchedURL, - SiteName: hostname, - TemplateID: request.options.TemplateID, - ComponentType: generatedRequest.fuzzGeneratedRequest.Component.Name(), - ComponentName: generatedRequest.fuzzGeneratedRequest.Key, - PayloadSent: generatedRequest.fuzzGeneratedRequest.Value, - StatusCode: respChain.Response().StatusCode, - }) - } - // Add to history the current request number metadata if asked by the user. if request.NeedsRequestCondition() { for k, v := range outputEvent { @@ -1035,6 +1035,19 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ callback(event) + if request.options.FuzzStatsDB != nil && generatedRequest.fuzzGeneratedRequest.Request != nil { + request.options.FuzzStatsDB.RecordResultEvent(fuzzStats.FuzzingEvent{ + URL: input.MetaInput.Target(), + SiteName: hostname, + TemplateID: request.options.TemplateID, + ComponentType: generatedRequest.fuzzGeneratedRequest.Component.Name(), + ComponentName: generatedRequest.fuzzGeneratedRequest.Parameter, + PayloadSent: generatedRequest.fuzzGeneratedRequest.Value, + StatusCode: respChain.Response().StatusCode, + Matched: event.HasResults(), + }) + } + // Skip further responses if we have stop-at-first-match and a match if (request.options.Options.StopAtFirstMatch || request.options.StopAtFirstMatch || request.StopAtFirstMatch) && event.HasResults() { return nil From 7c27c226d7aeda894cdb97a138c3e2043fbc9c6e Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 22 Nov 2024 16:00:38 +0530 Subject: [PATCH 07/23] misc changes to live server --- internal/runner/runner.go | 5 +++++ internal/server/server.go | 7 +++++++ pkg/fuzz/stats/db.go | 37 ++++++++++++++++++++++++++++++----- pkg/fuzz/stats/schema.sql | 11 ++++++++++- pkg/fuzz/stats/stats.go | 37 ++++++++++++++++++++++++----------- pkg/protocols/http/request.go | 2 ++ 6 files changed, 82 insertions(+), 17 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 45310d4c80..d8cdd3cbb1 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -98,6 +98,7 @@ type Runner struct { tmpDir string parser parser.Parser httpApiEndpoint *httpapi.Server + dastServer *server.DASTServer } const pprofServerAddress = "127.0.0.1:8086" @@ -364,6 +365,9 @@ func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions, // Close releases all the resources and cleans up func (r *Runner) Close() { + if r.dastServer != nil { + r.dastServer.Close() + } // dump hosterrors cache if r.hostErrors != nil { r.hostErrors.Close() @@ -471,6 +475,7 @@ func (r *Runner) RunEnumeration() error { if err != nil { return err } + r.dastServer = dastServer return dastServer.Start() } diff --git a/internal/server/server.go b/internal/server/server.go index 953e418743..c7d709daef 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -3,6 +3,7 @@ package server import ( "fmt" "strings" + "time" "github.com/alitto/pond" "github.com/labstack/echo/v4" @@ -94,6 +95,12 @@ func New(options *Options) (*DASTServer, error) { return server, nil } +func (s *DASTServer) Close() { + s.nucleiExecutor.Close() + s.echo.Close() + s.tasksPool.StopAndWaitFor(1 * time.Minute) +} + func (s *DASTServer) buildConnectionURL() string { url := fmt.Sprintf("http://%s/requests", s.options.Address) if s.options.Token != "" { diff --git a/pkg/fuzz/stats/db.go b/pkg/fuzz/stats/db.go index cef72ccc1f..d53a16e397 100644 --- a/pkg/fuzz/stats/db.go +++ b/pkg/fuzz/stats/db.go @@ -130,7 +130,13 @@ func (s *sqliteStatsDatabase) InsertMatchedRecord(event FuzzingEvent) error { return errors.Wrap(err, "could not get component_id") } - err = s.insertFuzzingResult(tx, componentID, templateID, event.PayloadSent, event.StatusCode, event.Matched) + requestID, err := s.insertFuzzingRequestResponse(tx, event.RawRequest, event.RawResponse) + if err != nil { + fmt.Printf("could not insert fuzzing request response: %s\n", err) + return errors.Wrap(err, "could not insert fuzzing request response") + } + + err = s.insertFuzzingResult(tx, componentID, templateID, event.PayloadSent, event.StatusCode, event.Matched, requestID) if err != nil { return errors.Wrap(err, "could not insert fuzzing result") } @@ -219,10 +225,31 @@ func (s *sqliteStatsDatabase) getComponentID(tx *sql.Tx, siteID int, componentTy return componentID, nil } -func (s *sqliteStatsDatabase) insertFuzzingResult(tx *sql.Tx, componentID, templateID int, payloadSent string, statusCode int, matched bool) error { +const responseSaveSize = 2 * 1024 + +func (s *sqliteStatsDatabase) insertFuzzingRequestResponse(tx *sql.Tx, rawRequest, rawResponse string) (int, error) { + var requestID int + + // Only ingest 2kb of response + if len(rawResponse) > responseSaveSize { + rawResponse = rawResponse[:responseSaveSize] + } + + err := tx.QueryRow(` + INSERT INTO fuzzing_request_response (raw_request, raw_response) + VALUES (?, ?) RETURNING request_id + `, rawRequest, rawResponse).Scan(&requestID) + if err != nil { + return 0, err + } + + return requestID, nil +} + +func (s *sqliteStatsDatabase) insertFuzzingResult(tx *sql.Tx, componentID, templateID int, payloadSent string, statusCode int, matched bool, requestID int) error { _, err := tx.Exec(` - INSERT INTO fuzzing_results (component_id, template_id, payload_sent, status_code_received, matched) - VALUES (?, ?, ?, ?, ?) - `, componentID, templateID, payloadSent, statusCode, matched) + INSERT INTO fuzzing_results (component_id, template_id, payload_sent, status_code_received, matched, request_id) + VALUES (?, ?, ?, ?, ?, ?) + `, componentID, templateID, payloadSent, statusCode, matched, requestID) return err } diff --git a/pkg/fuzz/stats/schema.sql b/pkg/fuzz/stats/schema.sql index 233fa11da0..e4d5aa7e2a 100644 --- a/pkg/fuzz/stats/schema.sql +++ b/pkg/fuzz/stats/schema.sql @@ -40,9 +40,11 @@ CREATE TABLE IF NOT EXISTS fuzzing_results ( payload_sent TEXT NOT NULL, status_code_received INTEGER NOT NULL, matched BOOLEAN DEFAULT FALSE NOT NULL, + request_id INTEGER NOT NULL, timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, FOREIGN KEY (component_id) REFERENCES components(component_id), - FOREIGN KEY (template_id) REFERENCES templates(template_id) + FOREIGN KEY (template_id) REFERENCES templates(template_id), + FOREIGN KEY (request_id) REFERENCES fuzzing_request_response(request_id) ); CREATE INDEX IF NOT EXISTS idx_FuzzingResults_comp_temp_time ON fuzzing_results (component_id, template_id, timestamp); @@ -60,3 +62,10 @@ BEGIN ON CONFLICT(component_id, template_id) DO UPDATE SET times_applied = times_applied + 1; END; + +-- +CREATE TABLE IF NOT EXISTS fuzzing_request_response ( + request_id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_request TEXT NOT NULL, + raw_response TEXT NOT NULL +); \ No newline at end of file diff --git a/pkg/fuzz/stats/stats.go b/pkg/fuzz/stats/stats.go index 5ea6aa70cb..7ab375a2a4 100644 --- a/pkg/fuzz/stats/stats.go +++ b/pkg/fuzz/stats/stats.go @@ -3,9 +3,11 @@ package stats import ( + "fmt" "net/url" "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" ) // Tracker is a stats tracker module for fuzzing server @@ -28,6 +30,11 @@ func NewTracker(scanName string) (*Tracker, error) { // Close closes the tracker func (t *Tracker) Close() { + _, err := t.database.(*sqliteStatsDatabase).db.Exec("VACUUM") + if err != nil { + gologger.Error().Msgf("could not truncate wal: %s", err) + } + t.database.Close() } @@ -41,26 +48,34 @@ type FuzzingEvent struct { StatusCode int Matched bool SiteName string + RawRequest string + RawResponse string } func (t *Tracker) RecordResultEvent(event FuzzingEvent) { - parsed, err := url.Parse(event.URL) - if err != nil { - return - } - - // Site is the host:port combo - event.SiteName = parsed.Host + event.SiteName = getCorrectSiteName(event.URL) t.database.InsertMatchedRecord(event) } func (t *Tracker) RecordComponentEvent(event FuzzingEvent) { - parsed, err := url.Parse(event.URL) + event.SiteName = getCorrectSiteName(event.URL) + t.database.InsertComponent(event) +} + +func getCorrectSiteName(originalURL string) string { + parsed, err := url.Parse(originalURL) if err != nil { - return + return "" } // Site is the host:port combo - event.SiteName = parsed.Host - t.database.InsertComponent(event) + siteName := parsed.Host + if parsed.Port() == "" { + if parsed.Scheme == "https" { + siteName = fmt.Sprintf("%s:443", siteName) + } else if parsed.Scheme == "http" { + siteName = fmt.Sprintf("%s:80", siteName) + } + } + return siteName } diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 333f7edfe9..5e54f46f72 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -1045,6 +1045,8 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ PayloadSent: generatedRequest.fuzzGeneratedRequest.Value, StatusCode: respChain.Response().StatusCode, Matched: event.HasResults(), + RawRequest: string(dumpedRequest), + RawResponse: respChain.FullResponse().String(), }) } From 090cadbc8519c85405ddfacdabddad074ab8383e Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 26 Nov 2024 16:39:22 +0530 Subject: [PATCH 08/23] misc --- pkg/fuzz/component/cookie.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/fuzz/component/cookie.go b/pkg/fuzz/component/cookie.go index 295dfce9f5..25f29e794a 100644 --- a/pkg/fuzz/component/cookie.go +++ b/pkg/fuzz/component/cookie.go @@ -81,6 +81,7 @@ func (c *Cookie) Delete(key string) error { // Rebuild returns a new request with the // component rebuilt func (c *Cookie) Rebuild() (*retryablehttp.Request, error) { + // TODO: Fix cookie duplication with auth-file cloned := c.req.Clone(context.Background()) cloned.Header.Del("Cookie") From 727ff908d7f01d5614e4fb5611d56538f37f4ee6 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sat, 30 Nov 2024 02:53:26 +0530 Subject: [PATCH 09/23] use utils pprof server --- go.mod | 15 ++++++++------- go.sum | 35 +++++++++++++++++++++++------------ internal/runner/runner.go | 18 +++++------------- 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index a0c377428b..137c8b6f58 100644 --- a/go.mod +++ b/go.mod @@ -20,12 +20,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.1.1 - github.com/projectdiscovery/fastdialer v0.2.9 - github.com/projectdiscovery/hmap v0.0.67 + github.com/projectdiscovery/fastdialer v0.2.10 + github.com/projectdiscovery/hmap v0.0.68 github.com/projectdiscovery/interactsh v1.2.0 github.com/projectdiscovery/rawhttp v0.1.74 - github.com/projectdiscovery/retryabledns v1.0.85 - github.com/projectdiscovery/retryablehttp-go v1.0.86 + github.com/projectdiscovery/retryabledns v1.0.86 + github.com/projectdiscovery/retryablehttp-go v1.0.88 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -59,6 +59,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.13.27 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.72 github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 + github.com/ccojocar/randdetect v0.0.0-20241118085251-1581dcdbf207 github.com/cespare/xxhash v1.1.0 github.com/charmbracelet/glamour v0.8.0 github.com/clbanning/mxj/v2 v2.7.0 @@ -88,7 +89,7 @@ require ( github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb github.com/projectdiscovery/goflags v0.1.65 - github.com/projectdiscovery/gologger v1.1.31 + github.com/projectdiscovery/gologger v1.1.33 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.3 github.com/projectdiscovery/httpx v1.6.9 @@ -100,7 +101,7 @@ require ( github.com/projectdiscovery/tlsx v1.1.8 github.com/projectdiscovery/uncover v1.0.9 github.com/projectdiscovery/useragent v0.0.78 - github.com/projectdiscovery/utils v0.2.18 + github.com/projectdiscovery/utils v0.2.22-0.20241129171309-2f4ef522155e github.com/projectdiscovery/wappalyzergo v0.2.2 github.com/redis/go-redis/v9 v9.1.0 github.com/seh-msft/burpxml v1.0.1 @@ -138,7 +139,6 @@ require ( github.com/bits-and-blooms/bloom/v3 v3.5.0 // indirect github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect - github.com/ccojocar/randdetect v0.0.0-20241118085251-1581dcdbf207 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/charmbracelet/lipgloss v0.13.0 // indirect @@ -156,6 +156,7 @@ require ( github.com/docker/docker v24.0.9+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/fatih/color v1.16.0 // indirect + github.com/felixge/fgprof v0.9.5 // indirect github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gaissmai/bart v0.9.5 // indirect diff --git a/go.sum b/go.sum index f0b109466f..f5c8acd9f6 100644 --- a/go.sum +++ b/go.sum @@ -241,12 +241,18 @@ github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME= github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -331,6 +337,8 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= +github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -585,6 +593,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= +github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= @@ -679,6 +688,7 @@ github.com/labstack/echo/v4 v4.12.0 h1:IKpw49IMryVB2p1a4dzwlhP1O2Tf2E0Ir/450lH+k github.com/labstack/echo/v4 v4.12.0/go.mod h1:UP9Cr2DJXbOK3Kr9ONYzNowSh7HP0aG0ShAyycHSJvM= github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/leslie-qiwa/flat v0.0.0-20230424180412-f9d1cf014baa h1:KQKuQDgA3DZX6C396lt3WDYB9Um1gLITLbvficVbqXk= @@ -810,6 +820,7 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4= github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= @@ -853,8 +864,8 @@ github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB7 github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= github.com/projectdiscovery/dsl v0.3.3 h1:4Ij5S86cHlb6xFrS7+5zAiJPeBt5h970XBTHqeTkpyU= github.com/projectdiscovery/dsl v0.3.3/go.mod h1:DAjSeaogLM9f0Ves2zDc/vbJrfcv+kEmS51p0dLLaPI= -github.com/projectdiscovery/fastdialer v0.2.9 h1:vDCqxVMCyUu3oVEizEK1K8K+CCcLkVDW3X2HfiWaVFA= -github.com/projectdiscovery/fastdialer v0.2.9/go.mod h1:mYv5QaNBDDSHlZO9DI0niRMw+G5hUzwIhs8QixSElUI= +github.com/projectdiscovery/fastdialer v0.2.10 h1:5iciZXMPdynbk/9iuqkJT1gqMXwzgEpFSWdoj/5CHCo= +github.com/projectdiscovery/fastdialer v0.2.10/go.mod h1:21rwXMecVsPVdSvON8Up761/GgxC4OSc9Rvx5LNH5fY= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= @@ -863,14 +874,14 @@ github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG90 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY= github.com/projectdiscovery/goflags v0.1.65 h1:rjoj+5lP/FDzgeM0WILUTX9AOOnw0J0LXtl8P1SVeGE= github.com/projectdiscovery/goflags v0.1.65/go.mod h1:cg6+yrLlaekP1hnefBc/UXbH1YGWa0fuzEW9iS1aG4g= -github.com/projectdiscovery/gologger v1.1.31 h1:FlZi1RsDoRtOkj9+a1PhcOmwD3NdRpDyjp/0/fmpQ/s= -github.com/projectdiscovery/gologger v1.1.31/go.mod h1:zVbkxOmWuh1GEyr6dviEPNwH/GMWdnJrMUSOJbRmDqI= +github.com/projectdiscovery/gologger v1.1.33 h1:wQxaQ8p/0Rx89lowBp0PnY2QSWiqf9QW1vGYAllsVJ4= +github.com/projectdiscovery/gologger v1.1.33/go.mod h1:P/WwqKstshQATJxN39V0KJ9ZuiGLOizmSqHIYrrz1T4= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= github.com/projectdiscovery/gostruct v0.0.2/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= github.com/projectdiscovery/gozero v0.0.3 h1:tsYkrSvWw4WdIUJyisd4MB1vRiw1X57TuVVk3p8Z3G8= github.com/projectdiscovery/gozero v0.0.3/go.mod h1:MpJ37Dsh94gy2EKqaemdeh+CzduGVB2SDfhr6Upsjew= -github.com/projectdiscovery/hmap v0.0.67 h1:PG09AyXH6mchdZCdxAS7WkZz0xxsOsIxJOmEixEmnzI= -github.com/projectdiscovery/hmap v0.0.67/go.mod h1:WxK8i2J+wcdimIXCgpYzfj9gKxCqRqOM4KENDRzGgAA= +github.com/projectdiscovery/hmap v0.0.68 h1:/z1Cz2wKYedTJc97UNzBBgdm744xkXi6j7125b7toqg= +github.com/projectdiscovery/hmap v0.0.68/go.mod h1:B37g7giW6i7+X1pJAeG0NPoKFpFJ7M26a18gfwfLeEc= github.com/projectdiscovery/httpx v1.6.9 h1:ihyFclesLjvQpiJpRIlAYeebapyIbOI/arDAvvy1ES8= github.com/projectdiscovery/httpx v1.6.9/go.mod h1:zQtX5CtcDYXzIRWne1ztCVtqG0sXCnx84tFwfMHoB8Q= github.com/projectdiscovery/interactsh v1.2.0 h1:Al6jHiR+Usl9egYJDLJaWNHOcH8Rugk8gWMasc8Cmw8= @@ -891,10 +902,10 @@ github.com/projectdiscovery/rawhttp v0.1.74 h1:ahE23GwPyFDBSofmo92MuW439P4x20GBY github.com/projectdiscovery/rawhttp v0.1.74/go.mod h1:xEqBY17CHgGmMfuLOWYntjFQ9crb4PG1xoNgexcAq4g= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.85 h1:9aLPWu0bcmtK8bPm/JJyfts28hgWf74UPsSG0KMXrqo= -github.com/projectdiscovery/retryabledns v1.0.85/go.mod h1:cZe0rydjby+ns2oIY7JmywHvtkwWxPzp3PuQz1rV50E= -github.com/projectdiscovery/retryablehttp-go v1.0.86 h1:r/rqVrT/fSMe6/syIq1FGd8do/vt+Kgca9pFegyHG88= -github.com/projectdiscovery/retryablehttp-go v1.0.86/go.mod h1:upk8ItKt9hayUp6Z7E60tH314BAnIUQ5y4KS4x9R90g= +github.com/projectdiscovery/retryabledns v1.0.86 h1:8YMJGJ94lFBKKN3t7NOzJfbGsZoh9qNpi49xdfJcZVc= +github.com/projectdiscovery/retryabledns v1.0.86/go.mod h1:5PhXvlLkEFmlYOt9i4wiKA1eONLrNiZ6DQE88Ph9rgU= +github.com/projectdiscovery/retryablehttp-go v1.0.88 h1:uR6T+i8Sy1isfG1KClhhsXnOqkOR6E8MAvuyOFq3T10= +github.com/projectdiscovery/retryablehttp-go v1.0.88/go.mod h1:ktjiIKyej+plUeK9vksqRf3wGicqY3E1rW84V/O7p0M= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -905,8 +916,8 @@ github.com/projectdiscovery/uncover v1.0.9 h1:s5RbkD/V4r8QcPkys4gTTqMuRSgXq0Jpre github.com/projectdiscovery/uncover v1.0.9/go.mod h1:2PUF3SpB5QNIJ8epaB2xbRzkPaxEAWRDm3Ir2ijt81U= github.com/projectdiscovery/useragent v0.0.78 h1:YpgiY3qXpzygFA88SWVseAyWeV9ZKrIpDkfOY+mQ/UY= github.com/projectdiscovery/useragent v0.0.78/go.mod h1:SQgk2DZu1qCvYqBRYWs2sjenXqLEDnRw65wJJoolwZ4= -github.com/projectdiscovery/utils v0.2.18 h1:uV5JIYKIq8gXdu9wrCeUq3yqPiSCokTrKuLuZwXMSSw= -github.com/projectdiscovery/utils v0.2.18/go.mod h1:gcKxBTK1eNF+K8vzD62sMMVFf1eJoTgEiS81mp7CQjI= +github.com/projectdiscovery/utils v0.2.22-0.20241129171309-2f4ef522155e h1:+nJvs27gwt+MrZni3Z/B2cGczbRL7X673PdD3RqBS4w= +github.com/projectdiscovery/utils v0.2.22-0.20241129171309-2f4ef522155e/go.mod h1:k2XlmfaYO4k6T4vAyUa3Kn/0BxPTIlNiBFpM6nVCbz0= github.com/projectdiscovery/wappalyzergo v0.2.2 h1:AQT6+oo++HOcseTFSTa2en08vWv5miE/NgnJlqL1lCQ= github.com/projectdiscovery/wappalyzergo v0.2.2/go.mod h1:k3aujwFsLcB24ppzwNE0lYpV3tednKGJVTbk4JgrhmI= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 60a423c931..44002a7bfc 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -4,8 +4,6 @@ import ( "context" "encoding/json" "fmt" - "net/http" - _ "net/http/pprof" "os" "path/filepath" "reflect" @@ -26,6 +24,7 @@ import ( "github.com/projectdiscovery/utils/env" fileutil "github.com/projectdiscovery/utils/file" permissionutil "github.com/projectdiscovery/utils/permission" + pprofutil "github.com/projectdiscovery/utils/pprof" updateutils "github.com/projectdiscovery/utils/update" "github.com/logrusorgru/aurora" @@ -90,7 +89,7 @@ type Runner struct { rateLimiter *ratelimit.Limiter hostErrors hosterrorscache.CacheInterface resumeCfg *types.ResumeCfg - pprofServer *http.Server + pprofServer *pprofutil.PprofServer pdcpUploadErrMsg string inputProvider provider.InputProvider fuzzFrequencyCache *frequency.Tracker @@ -219,15 +218,8 @@ func New(options *types.Options) (*Runner, error) { templates.SeverityColorizer = colorizer.New(runner.colorizer) if options.EnablePprof { - server := &http.Server{ - Addr: pprofServerAddress, - Handler: http.DefaultServeMux, - } - gologger.Info().Msgf("Listening pprof debug server on: %s", pprofServerAddress) - runner.pprofServer = server - go func() { - _ = server.ListenAndServe() - }() + runner.pprofServer = pprofutil.NewPprofServer() + runner.pprofServer.Start() } if options.HttpApiEndpoint != "" { @@ -386,7 +378,7 @@ func (r *Runner) Close() { } protocolinit.Close() if r.pprofServer != nil { - _ = r.pprofServer.Shutdown(context.Background()) + r.pprofServer.Stop() } if r.rateLimiter != nil { r.rateLimiter.Stop() From a10530693f15bce30853bfa316184419dfa905cb Mon Sep 17 00:00:00 2001 From: Ice3man Date: Sun, 1 Dec 2024 19:16:47 +0530 Subject: [PATCH 10/23] feat: added simpler stats tracking system --- cmd/nuclei/main.go | 2 +- internal/runner/runner.go | 59 +++-- internal/server/nuclei_sdk.go | 11 +- internal/server/requests_worker.go | 12 +- internal/server/server.go | 155 ++++++++++-- internal/server/templates/index.html | 340 +++++++++++++++++++++++++++ pkg/fuzz/execute.go | 13 + pkg/fuzz/stats/db.go | 244 +------------------ pkg/fuzz/stats/db_test.go | 22 +- pkg/fuzz/stats/schema.sql | 71 ------ pkg/fuzz/stats/simple.go | 164 +++++++++++++ pkg/fuzz/stats/stats.go | 44 +++- pkg/output/output.go | 9 +- pkg/protocols/http/request.go | 14 +- pkg/types/types.go | 4 +- 15 files changed, 756 insertions(+), 408 deletions(-) create mode 100644 internal/server/templates/index.html delete mode 100644 pkg/fuzz/stats/schema.sql create mode 100644 pkg/fuzz/stats/simple.go diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 180c713d4e..9c3f8a1add 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -367,7 +367,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"), flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"), flagSet.BoolVarP(&options.DASTServer, "dast-server", "dts", false, "enable dast server mode (live fuzzing)"), - flagSet.StringVarP(&options.DASTScanName, "dast-scan-report", "dtr", "", "write dast scan report to file"), + flagSet.BoolVarP(&options.DASTReport, "dast-report", "drg", false, "write dast scan report to file"), flagSet.StringVarP(&options.DASTServerToken, "dast-server-token", "dtst", "", "dast server token (optional)"), flagSet.StringVarP(&options.DASTServerAddress, "dast-server-address", "dtsa", "localhost:9055", "dast server address"), flagSet.BoolVarP(&options.DisplayFuzzPoints, "display-fuzz-points", "dfp", false, "display fuzz points in the output for debugging"), diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 44002a7bfc..f7258ada40 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -97,6 +97,7 @@ type Runner struct { tmpDir string parser parser.Parser httpApiEndpoint *httpapi.Server + fuzzStats *fuzzStats.Tracker dastServer *server.DASTServer } @@ -244,14 +245,6 @@ func New(options *types.Options) (*Runner, error) { } runner.inputProvider = inputProvider - // Create the output file if asked - outputWriter, err := output.NewStandardWriter(options) - if err != nil { - return nil, errors.Wrap(err, "could not create output file") - } - // setup a proxy writer to automatically upload results to PDCP - runner.output = runner.setupPDCPUpload(outputWriter) - if options.JSONL && options.EnableProgressBar { options.StatsJSON = true } @@ -344,6 +337,42 @@ func New(options *types.Options) (*Runner, error) { runner.tmpDir = tmpDir } + if options.DASTReport || options.DASTServer { + var err error + runner.fuzzStats, err = fuzzStats.NewTracker() + if err != nil { + return nil, errors.Wrap(err, "could not create fuzz stats db") + } + if !options.DASTServer { + dastServer, err := server.NewStatsServer(runner.fuzzStats) + if err != nil { + return nil, errors.Wrap(err, "could not create dast server") + } + runner.dastServer = dastServer + } + } + + // Create the output file if asked + outputWriter, err := output.NewStandardWriter(options) + if err != nil { + return nil, errors.Wrap(err, "could not create output file") + } + if runner.fuzzStats != nil { + outputWriter.RequestHook = func(request *output.JSONLogRequest) { + if request.Error == "none" || request.Error == "" { + return + } + runner.fuzzStats.RecordErrorEvent(fuzzStats.ErrorEvent{ + TemplateID: request.Template, + URL: request.Input, + Error: request.Error, + }) + } + } + + // setup a proxy writer to automatically upload results to PDCP + runner.output = runner.setupPDCPUpload(outputWriter) + return runner, nil } @@ -453,6 +482,7 @@ func (r *Runner) RunEnumeration() error { Colorizer: r.colorizer, Parser: r.parser, TemporaryDirectory: r.tmpDir, + FuzzStatsDB: r.fuzzStats, } dastServer, err := server.New(&server.Options{ Address: r.options.DASTServerAddress, @@ -513,13 +543,6 @@ func (r *Runner) RunEnumeration() error { FuzzParamsFrequency: fuzzFreqCache, GlobalMatchers: globalmatchers.New(), } - if r.options.DASTScanName != "" { - var err error - executorOpts.FuzzStatsDB, err = fuzzStats.NewTracker(r.options.DASTScanName) - if err != nil { - return errors.Wrap(err, "could not create fuzz stats db") - } - } if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) { // Go StdLib style experimental/debug feature switch @@ -663,6 +686,12 @@ func (r *Runner) RunEnumeration() error { Retries: r.options.Retries, }, "") + if r.dastServer != nil { + if err := r.dastServer.Start(); err != nil { + r.dastServer.Start() + } + } + enumeration := false var results *atomic.Bool results, err = r.runStandardEnumeration(executorOpts, store, executorEngine) diff --git a/internal/server/nuclei_sdk.go b/internal/server/nuclei_sdk.go index 37966afd0e..9ff531b163 100644 --- a/internal/server/nuclei_sdk.go +++ b/internal/server/nuclei_sdk.go @@ -9,6 +9,7 @@ import ( "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/frequency" + "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" "github.com/projectdiscovery/nuclei/v3/pkg/input/provider/http" "github.com/projectdiscovery/nuclei/v3/pkg/projectfile" @@ -20,7 +21,6 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/catalog" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader" "github.com/projectdiscovery/nuclei/v3/pkg/core" - fuzzStats "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats" "github.com/projectdiscovery/nuclei/v3/pkg/input" "github.com/projectdiscovery/nuclei/v3/pkg/loader/parser" parsers "github.com/projectdiscovery/nuclei/v3/pkg/loader/workflow" @@ -54,6 +54,7 @@ type NucleiExecutorOptions struct { Interactsh *interactsh.Client ProjectFile *projectfile.ProjectFile Browser *browserEngine.Browser + FuzzStatsDB *stats.Tracker Colorizer aurora.Aurora Parser parser.Parser TemporaryDirectory string @@ -83,13 +84,7 @@ func newNucleiExecutor(opts *NucleiExecutorOptions) (*nucleiExecutor, error) { Parser: opts.Parser, FuzzParamsFrequency: fuzzFreqCache, GlobalMatchers: globalmatchers.New(), - } - if opts.Options.DASTScanName != "" { - var err error - executorOpts.FuzzStatsDB, err = fuzzStats.NewTracker(opts.Options.DASTScanName) - if err != nil { - return nil, errors.Wrap(err, "could not create fuzz stats db") - } + FuzzStatsDB: opts.FuzzStatsDB, } if opts.Options.ShouldUseHostError() { diff --git a/internal/server/requests_worker.go b/internal/server/requests_worker.go index 8058015a42..3c976b2e82 100644 --- a/internal/server/requests_worker.go +++ b/internal/server/requests_worker.go @@ -9,6 +9,8 @@ import ( ) func (s *DASTServer) consumeTaskRequest(req PostReuestsHandlerRequest) { + defer s.endpointsInQueue.Add(-1) + parsedReq, err := types.ParseRawRequestWithURL(req.RawHTTP, req.URL) if err != nil { gologger.Warning().Msgf("Could not parse raw request: %s\n", err) @@ -16,6 +18,7 @@ func (s *DASTServer) consumeTaskRequest(req PostReuestsHandlerRequest) { } if parsedReq.URL.Scheme != "http" && parsedReq.URL.Scheme != "https" { + gologger.Warning().Msgf("Invalid scheme: %s\n", parsedReq.URL.Scheme) return } @@ -43,12 +46,11 @@ func (s *DASTServer) consumeTaskRequest(req PostReuestsHandlerRequest) { gologger.Verbose().Msgf("Fuzzing request: %s %s\n", parsedReq.Request.Method, parsedReq.URL.String()) - // Fuzz the request finally - s.fuzzRequest(req) -} + s.endpointsBeingTested.Add(1) + defer s.endpointsBeingTested.Add(-1) -func (s *DASTServer) fuzzRequest(req PostReuestsHandlerRequest) { - err := s.nucleiExecutor.ExecuteScan(req) + // Fuzz the request finally + err = s.nucleiExecutor.ExecuteScan(req) if err != nil { gologger.Warning().Msgf("Could not run nuclei: %s\n", err) return diff --git a/internal/server/server.go b/internal/server/server.go index c7d709daef..5fdbf9d6d1 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1,8 +1,12 @@ package server import ( + _ "embed" "fmt" + "html/template" + "net/http" "strings" + "sync/atomic" "time" "github.com/alitto/pond" @@ -10,7 +14,10 @@ import ( "github.com/labstack/echo/v4/middleware" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/internal/server/scope" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" + "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats" "github.com/projectdiscovery/nuclei/v3/pkg/output" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/utils/env" ) @@ -22,6 +29,11 @@ type DASTServer struct { tasksPool *pond.WorkerPool deduplicator *requestDeduplicator scopeManager *scope.Manager + startTime time.Time + + // metrics + endpointsInQueue atomic.Int64 + endpointsBeingTested atomic.Int64 nucleiExecutor *nucleiExecutor } @@ -66,8 +78,9 @@ func New(options *Options) (*DASTServer, error) { options: options, tasksPool: pond.New(maxWorkers, bufferSize), deduplicator: newRequestDeduplicator(), + startTime: time.Now(), } - server.setupHandlers() + server.setupHandlers(false) executor, err := newNucleiExecutor(options.NucleiExecutorOptions) if err != nil { @@ -91,6 +104,21 @@ func New(options *Options) (*DASTServer, error) { } gologger.Info().Msgf("%s", builder.String()) gologger.Info().Msgf("Connection URL: %s", server.buildConnectionURL()) + gologger.Info().Msgf("Stats UI URL: %s", server.buildStatsURL()) + + return server, nil +} + +func NewStatsServer(fuzzStatsDB *stats.Tracker) (*DASTServer, error) { + server := &DASTServer{ + nucleiExecutor: &nucleiExecutor{ + executorOpts: protocols.ExecutorOptions{ + FuzzStatsDB: fuzzStatsDB, + }, + }, + } + server.setupHandlers(true) + gologger.Info().Msgf("Stats UI URL: %s", server.buildStatsURL()) return server, nil } @@ -109,11 +137,24 @@ func (s *DASTServer) buildConnectionURL() string { return url } -func (s *DASTServer) setupHandlers() { +func (s *DASTServer) buildStatsURL() string { + url := fmt.Sprintf("http://%s/stats", s.options.Address) + if s.options.Token != "" { + url += "?token=" + s.options.Token + } + return url +} + +func (s *DASTServer) setupHandlers(onlyStats bool) { e := echo.New() e.Use(middleware.Recover()) if s.options.Verbose { - e.Use(middleware.Logger()) + cfg := middleware.DefaultLoggerConfig + cfg.Skipper = func(c echo.Context) bool { + // Skip /stats and /stats.json + return c.Request().URL.Path == "/stats" || c.Request().URL.Path == "/stats.json" + } + e.Use(middleware.LoggerWithConfig(cfg)) } e.Use(middleware.CORS()) @@ -123,30 +164,25 @@ func (s *DASTServer) setupHandlers() { Validator: func(key string, c echo.Context) (bool, error) { return key == s.options.Token, nil }, - Skipper: func(c echo.Context) bool { - return c.Path() == "/stats" - }, })) } e.HideBanner = true // POST /requests - Queue a request for fuzzing - e.POST("/requests", s.handleRequest) + if !onlyStats { + e.POST("/requests", s.handleRequest) + } e.GET("/stats", s.handleStats) + e.GET("/stats.json", s.handleStatsJSON) - // Serve a Web Server to visualize the stats in a live HTML report - e.GET("/ui", func(c echo.Context) error { - return c.File("internal/server/ui/index.html") - }) s.echo = e } -func (s *DASTServer) handleStats(c echo.Context) error { - return c.JSON(200, map[string]interface{}{}) -} - func (s *DASTServer) Start() error { - return s.echo.Start(s.options.Address) + if err := s.echo.Start(s.options.Address); err != nil && err != http.ErrServerClosed { + return err + } + return nil } // PostReuestsHandlerRequest is the request body for the /requests POST handler. @@ -158,16 +194,103 @@ type PostReuestsHandlerRequest struct { func (s *DASTServer) handleRequest(c echo.Context) error { var req PostReuestsHandlerRequest if err := c.Bind(&req); err != nil { + fmt.Printf("Error binding request: %s\n", err) return err } // Validate the request if req.RawHTTP == "" || req.URL == "" { + fmt.Printf("Missing required fields\n") return c.JSON(400, map[string]string{"error": "missing required fields"}) } + s.endpointsInQueue.Add(1) s.tasksPool.Submit(func() { s.consumeTaskRequest(req) }) return c.NoContent(200) } + +type StatsResponse struct { + DASTServerInfo DASTServerInfo `json:"dast_server_info"` + DASTScanStatistics DASTScanStatistics `json:"dast_scan_statistics"` + DASTScanStatusStatistics map[string]int64 `json:"dast_scan_status_statistics"` + DASTScanSeverityBreakdown map[string]int64 `json:"dast_scan_severity_breakdown"` + DASTScanErrorStatistics map[string]int64 `json:"dast_scan_error_statistics"` + DASTScanStartTime time.Time `json:"dast_scan_start_time"` +} + +type DASTServerInfo struct { + NucleiVersion string `json:"nuclei_version"` + NucleiTemplateVersion string `json:"nuclei_template_version"` + NucleiDastServerAPI string `json:"nuclei_dast_server_api"` + ServerAuthEnabled bool `json:"sever_auth_enabled"` +} + +type DASTScanStatistics struct { + EndpointsInQueue int64 `json:"endpoints_in_queue"` + EndpointsBeingTested int64 `json:"endpoints_being_tested"` + TotalTemplatesLoaded int64 `json:"total_dast_templates_loaded"` + TotalTemplatesTested int64 `json:"total_dast_templates_tested"` + TotalMatchedResults int64 `json:"total_matched_results"` + TotalComponentsTested int64 `json:"total_components_tested"` + TotalEndpointsTested int64 `json:"total_endpoints_tested"` + TotalFuzzedRequests int64 `json:"total_fuzzed_requests"` + TotalErroredRequests int64 `json:"total_errored_requests"` +} + +func (s *DASTServer) getStats() (StatsResponse, error) { + cfg := config.DefaultConfig + + resp := StatsResponse{ + DASTServerInfo: DASTServerInfo{ + NucleiVersion: config.Version, + NucleiTemplateVersion: cfg.TemplateVersion, + NucleiDastServerAPI: s.buildConnectionURL(), + ServerAuthEnabled: s.options.Token != "", + }, + DASTScanStartTime: s.startTime, + DASTScanStatistics: DASTScanStatistics{ + EndpointsInQueue: s.endpointsInQueue.Load(), + EndpointsBeingTested: s.endpointsBeingTested.Load(), + TotalTemplatesLoaded: int64(len(s.nucleiExecutor.store.Templates())), + }, + } + if s.nucleiExecutor.executorOpts.FuzzStatsDB != nil { + fuzzStats := s.nucleiExecutor.executorOpts.FuzzStatsDB.GetStats() + resp.DASTScanSeverityBreakdown = fuzzStats.SeverityCounts + resp.DASTScanStatusStatistics = fuzzStats.StatusCodes + resp.DASTScanStatistics.TotalMatchedResults = fuzzStats.TotalMatchedResults + resp.DASTScanStatistics.TotalComponentsTested = fuzzStats.TotalComponentsTested + resp.DASTScanStatistics.TotalEndpointsTested = fuzzStats.TotalEndpointsTested + resp.DASTScanStatistics.TotalFuzzedRequests = fuzzStats.TotalFuzzedRequests + resp.DASTScanStatistics.TotalTemplatesTested = fuzzStats.TotalTemplatesTested + resp.DASTScanStatistics.TotalErroredRequests = fuzzStats.TotalErroredRequests + resp.DASTScanErrorStatistics = fuzzStats.ErrorGroupedStats + } + return resp, nil +} + +//go:embed templates/index.html +var indexTemplate string + +func (s *DASTServer) handleStats(c echo.Context) error { + stats, err := s.getStats() + if err != nil { + return c.JSON(500, map[string]string{"error": err.Error()}) + } + + tmpl, err := template.New("index").Parse(indexTemplate) + if err != nil { + return c.JSON(500, map[string]string{"error": err.Error()}) + } + return tmpl.Execute(c.Response().Writer, stats) +} + +func (s *DASTServer) handleStatsJSON(c echo.Context) error { + resp, err := s.getStats() + if err != nil { + return c.JSON(500, map[string]string{"error": err.Error()}) + } + return c.JSONPretty(200, resp, " ") +} diff --git a/internal/server/templates/index.html b/internal/server/templates/index.html new file mode 100644 index 0000000000..52b07bf17d --- /dev/null +++ b/internal/server/templates/index.html @@ -0,0 +1,340 @@ + + + + + DAST Scan Report + + + + +
+ + +
+ +
+
+ __ _ + ____ __ _______/ /__ (_) + / __ \/ / / / ___/ / _ \/ / + / / / / /_/ / /__/ / __/ / +/_/ /_/\__,_/\___/_/\___/_/ {{.DASTServerInfo.NucleiVersion}} + + projectdiscovery.io + +Dynamic Application Security Testing (DAST) Server Stats +
+
[+] Scan started at: {{.DASTScanStartTime}}
+
+ +
+
[*] Server Configuration
+
Nuclei Version{{.DASTServerInfo.NucleiVersion}}
+
Template Version{{.DASTServerInfo.NucleiTemplateVersion}}
+
DAST Server API{{.DASTServerInfo.NucleiDastServerAPI}}
+
Auth Status{{if .DASTServerInfo.ServerAuthEnabled}}ENABLED{{else}}DISABLED{{end}}
+
+ +
+
[+] Scan Progress
+
Total Results{{.DASTScanStatistics.TotalMatchedResults}} findings
+
Endpoints In Queue{{.DASTScanStatistics.EndpointsInQueue}}
+
Currently Testing{{.DASTScanStatistics.EndpointsBeingTested}}
+
Components Tested{{.DASTScanStatistics.TotalComponentsTested}}
+
Endpoints Tested{{.DASTScanStatistics.TotalEndpointsTested}}
+
Templates Loaded{{.DASTScanStatistics.TotalTemplatesLoaded}}
+
Templates Tested{{.DASTScanStatistics.TotalTemplatesTested}}
+
Total Requests{{.DASTScanStatistics.TotalFuzzedRequests}}
+
Total Errors{{.DASTScanStatistics.TotalErroredRequests}}
+
+ +
+
[!] Security Findings
+
+
+
Critical
+
{{index .DASTScanSeverityBreakdown "critical"}} findings
+
+
+
High
+
{{index .DASTScanSeverityBreakdown "high"}} findings
+
+
+
Medium
+
{{index .DASTScanSeverityBreakdown "medium"}} findings
+
+
+
Low
+
{{index .DASTScanSeverityBreakdown "low"}} findings
+
+
+
Info
+
{{index .DASTScanSeverityBreakdown "info"}} findings
+
+
+
+ +
+
[-] Status Codes Breakdown
+ +
Response Codes
+ {{range $status, $count := .DASTScanStatusStatistics}} +
  {{$status}}{{$count}} times
+ {{end}} +
+ +
+
[-] Error Breakdown
+
+ {{range $error, $count := .DASTScanErrorStatistics}} +
+
{{$error}}
+
{{$count}} times
+
+ {{end}} +
+
+ + + + \ No newline at end of file diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go index f97f3149ba..ea5ad410d1 100644 --- a/pkg/fuzz/execute.go +++ b/pkg/fuzz/execute.go @@ -10,6 +10,7 @@ import ( "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/component" + fuzzStats "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/stats" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" @@ -122,6 +123,18 @@ func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) { return nil }) } + + if rule.options.FuzzStatsDB != nil { + component.Iterate(func(key string, value interface{}) error { + rule.options.FuzzStatsDB.RecordComponentEvent(fuzzStats.ComponentEvent{ + URL: input.Input.MetaInput.Target(), + ComponentType: componentName, + ComponentName: fmt.Sprintf("%v", value), + }) + return nil + }) + } + finalComponentList = append(finalComponentList, component) } if len(displayDebugFuzzPoints) > 0 { diff --git a/pkg/fuzz/stats/db.go b/pkg/fuzz/stats/db.go index d53a16e397..d5caf9a75d 100644 --- a/pkg/fuzz/stats/db.go +++ b/pkg/fuzz/stats/db.go @@ -1,255 +1,15 @@ package stats import ( - "database/sql" _ "embed" - "fmt" - "os" - "sync" _ "github.com/mattn/go-sqlite3" - "github.com/pkg/errors" ) type StatsDatabase interface { Close() - InsertComponent(event FuzzingEvent) error + InsertComponent(event ComponentEvent) error InsertMatchedRecord(event FuzzingEvent) error -} - -var ( - //go:embed schema.sql - dbSchemaCreateStatement string -) - -type sqliteStatsDatabase struct { - db *sql.DB - scanName string - - siteIDCache map[string]int - templateIDCache map[string]int - componentIDCache map[string]int - cacheMutex *sync.Mutex -} - -func NewSqliteStatsDatabase(scanName string) (*sqliteStatsDatabase, error) { - filename := fmt.Sprintf("%s.stats.db", scanName) - - connectionString := fmt.Sprintf("./%s?_journal_mode=WAL&_synchronous=NORMAL", filename) - db, err := sql.Open("sqlite3", connectionString) - if err != nil { - return nil, errors.Wrap(err, "could not open database") - } - - _, err = db.Exec(dbSchemaCreateStatement) - if err != nil { - return nil, errors.Wrap(err, "could not create schema") - } - - database := &sqliteStatsDatabase{ - scanName: scanName, - db: db, - siteIDCache: make(map[string]int), - templateIDCache: make(map[string]int), - componentIDCache: make(map[string]int), - cacheMutex: &sync.Mutex{}, - } - return database, nil -} - -func (s *sqliteStatsDatabase) Close() { - // Disable WAL mode and switch back to DELETE mode - _ = s.db.Close() - os.Remove(fmt.Sprintf("%s.stats.db-wal", s.scanName)) - os.Remove(fmt.Sprintf("%s.stats.db-shm", s.scanName)) -} - -func (s *sqliteStatsDatabase) DB() *sql.DB { - return s.db -} - -func (s *sqliteStatsDatabase) InsertComponent(event FuzzingEvent) error { - tx, err := s.db.Begin() - if err != nil { - return errors.Wrap(err, "could not begin transaction") - } - - defer func() { - if err != nil { - tx.Rollback() - } - }() - - siteID, err := s.getSiteID(tx, event.SiteName) - if err != nil { - return errors.Wrap(err, "could not get site_id") - } - - _, err = s.getTemplateID(tx, event.TemplateID) - if err != nil { - return errors.Wrap(err, "could not get template_id") - } - - _, err = s.getComponentID(tx, siteID, event.ComponentType, event.ComponentName, event.URL) - if err != nil { - return errors.Wrap(err, "could not get component_id") - } - - if err := tx.Commit(); err != nil { - return errors.Wrap(err, "could not commit transaction") - } - - return nil -} - -func (s *sqliteStatsDatabase) InsertMatchedRecord(event FuzzingEvent) error { - tx, err := s.db.Begin() - if err != nil { - return errors.Wrap(err, "could not begin transaction") - } - - defer func() { - if err != nil { - tx.Rollback() - } - }() - - siteID, err := s.getSiteID(tx, event.SiteName) - if err != nil { - return errors.Wrap(err, "could not get site_id") - } - - templateID, err := s.getTemplateID(tx, event.TemplateID) - if err != nil { - return errors.Wrap(err, "could not get template_id") - } - - componentID, err := s.getComponentID(tx, siteID, event.ComponentType, event.ComponentName, event.URL) - if err != nil { - return errors.Wrap(err, "could not get component_id") - } - - requestID, err := s.insertFuzzingRequestResponse(tx, event.RawRequest, event.RawResponse) - if err != nil { - fmt.Printf("could not insert fuzzing request response: %s\n", err) - return errors.Wrap(err, "could not insert fuzzing request response") - } - - err = s.insertFuzzingResult(tx, componentID, templateID, event.PayloadSent, event.StatusCode, event.Matched, requestID) - if err != nil { - return errors.Wrap(err, "could not insert fuzzing result") - } - - if err := tx.Commit(); err != nil { - return errors.Wrap(err, "could not commit transaction") - } - - return nil -} - -func (s *sqliteStatsDatabase) getSiteID(tx *sql.Tx, siteName string) (int, error) { - var siteID int - - s.cacheMutex.Lock() - if id, ok := s.siteIDCache[siteName]; ok { - s.cacheMutex.Unlock() - return id, nil - } - s.cacheMutex.Unlock() - - err := tx.QueryRow( - `INSERT OR IGNORE INTO sites (site_name) - VALUES (?) RETURNING site_id - `, siteName).Scan(&siteID) - if err != nil { - return 0, err - } - - // Cache the site_id - s.cacheMutex.Lock() - s.siteIDCache[siteName] = siteID - s.cacheMutex.Unlock() - - return siteID, nil -} - -func (s *sqliteStatsDatabase) getTemplateID(tx *sql.Tx, templateName string) (int, error) { - var templateID int - - s.cacheMutex.Lock() - if id, ok := s.templateIDCache[templateName]; ok { - s.cacheMutex.Unlock() - return id, nil - } - s.cacheMutex.Unlock() - - err := tx.QueryRow(` - INSERT OR IGNORE INTO templates (template_name) - VALUES (?) RETURNING template_id - `, templateName).Scan(&templateID) - if err != nil { - return 0, err - } - - s.cacheMutex.Lock() - s.templateIDCache[templateName] = templateID - s.cacheMutex.Unlock() - - return templateID, nil -} -func (s *sqliteStatsDatabase) getComponentID(tx *sql.Tx, siteID int, componentType, componentName, url string) (int, error) { - key := fmt.Sprintf("%d:%s:%s:%s", siteID, componentType, componentName, url) - var componentID int - - s.cacheMutex.Lock() - if id, ok := s.componentIDCache[key]; ok { - s.cacheMutex.Unlock() - return id, nil - } - s.cacheMutex.Unlock() - - err := tx.QueryRow(` - INSERT OR IGNORE INTO components (site_id, component_type, component_name, url) - VALUES (?, ?, ?, ?) - RETURNING component_id - `, siteID, componentType, componentName, url).Scan(&componentID) - if err != nil { - return 0, err - } - - s.cacheMutex.Lock() - s.componentIDCache[key] = componentID - s.cacheMutex.Unlock() - - return componentID, nil -} - -const responseSaveSize = 2 * 1024 - -func (s *sqliteStatsDatabase) insertFuzzingRequestResponse(tx *sql.Tx, rawRequest, rawResponse string) (int, error) { - var requestID int - - // Only ingest 2kb of response - if len(rawResponse) > responseSaveSize { - rawResponse = rawResponse[:responseSaveSize] - } - - err := tx.QueryRow(` - INSERT INTO fuzzing_request_response (raw_request, raw_response) - VALUES (?, ?) RETURNING request_id - `, rawRequest, rawResponse).Scan(&requestID) - if err != nil { - return 0, err - } - - return requestID, nil -} - -func (s *sqliteStatsDatabase) insertFuzzingResult(tx *sql.Tx, componentID, templateID int, payloadSent string, statusCode int, matched bool, requestID int) error { - _, err := tx.Exec(` - INSERT INTO fuzzing_results (component_id, template_id, payload_sent, status_code_received, matched, request_id) - VALUES (?, ?, ?, ?, ?, ?) - `, componentID, templateID, payloadSent, statusCode, matched, requestID) - return err + InsertError(event ErrorEvent) error } diff --git a/pkg/fuzz/stats/db_test.go b/pkg/fuzz/stats/db_test.go index 92de173fe8..e8a5c1e313 100644 --- a/pkg/fuzz/stats/db_test.go +++ b/pkg/fuzz/stats/db_test.go @@ -7,38 +7,18 @@ import ( ) func Test_NewStatsDatabase(t *testing.T) { - db, err := NewSqliteStatsDatabase("test") + db, err := NewSimpleStats() require.NoError(t, err) - err = db.InsertComponent(FuzzingEvent{ - URL: "http://localhost:8080/login", - SiteName: "localhost:8080", - TemplateID: "apache-struts2-001", - ComponentType: "path", - ComponentName: "/login", - PayloadSent: "/login'\"><", - StatusCode: 401, - }) - require.NoError(t, err) - - var siteName string - err = db.db.QueryRow("SELECT template_name FROM templates WHERE template_id = 1").Scan(&siteName) - require.NoError(t, err) - require.Equal(t, "apache-struts2-001", siteName) - err = db.InsertMatchedRecord(FuzzingEvent{ URL: "http://localhost:8080/login", - SiteName: "localhost:8080", TemplateID: "apache-struts2-001", ComponentType: "path", ComponentName: "/login", PayloadSent: "/login'\"><", StatusCode: 401, - Matched: true, }) require.NoError(t, err) - db.Close() - //os.Remove("test.stats.db") } diff --git a/pkg/fuzz/stats/schema.sql b/pkg/fuzz/stats/schema.sql deleted file mode 100644 index e4d5aa7e2a..0000000000 --- a/pkg/fuzz/stats/schema.sql +++ /dev/null @@ -1,71 +0,0 @@ -CREATE TABLE IF NOT EXISTS sites ( - site_id INTEGER PRIMARY KEY AUTOINCREMENT, - site_name TEXT UNIQUE NOT NULL -); -CREATE INDEX IF NOT EXISTS idx_sites_site_name ON sites(site_name); - -CREATE TABLE IF NOT EXISTS components ( - component_id INTEGER PRIMARY KEY AUTOINCREMENT, - site_id INTEGER NOT NULL, - component_type TEXT NOT NULL CHECK (component_type IN ('path', 'query', 'header', 'body', 'cookie')), - component_name TEXT NOT NULL, - last_fuzzed DATETIME, - url TEXT NOT NULL, - total_fuzz_count INTEGER DEFAULT 0, - FOREIGN KEY (site_id) REFERENCES sites(site_id), - UNIQUE (site_id, component_type, component_name, url) -); -CREATE INDEX IF NOT EXISTS idx_components_site_type_name ON components (site_id, component_type, component_name, url); - - -CREATE TABLE IF NOT EXISTS templates ( - template_id INTEGER PRIMARY KEY AUTOINCREMENT, - template_name TEXT UNIQUE NOT NULL -); -CREATE INDEX IF NOT EXISTS idx_templates_template_name ON templates(template_name); - -CREATE TABLE IF NOT EXISTS component_templates ( - component_id INTEGER NOT NULL, - template_id INTEGER NOT NULL, - times_applied INTEGER DEFAULT 0, - PRIMARY KEY (component_id, template_id), - FOREIGN KEY (component_id) REFERENCES components(component_id), - FOREIGN KEY (template_id) REFERENCES templates(template_id) -); - -CREATE TABLE IF NOT EXISTS fuzzing_results ( - result_id INTEGER PRIMARY KEY AUTOINCREMENT, - component_id INTEGER NOT NULL, - template_id INTEGER NOT NULL, - payload_sent TEXT NOT NULL, - status_code_received INTEGER NOT NULL, - matched BOOLEAN DEFAULT FALSE NOT NULL, - request_id INTEGER NOT NULL, - timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, - FOREIGN KEY (component_id) REFERENCES components(component_id), - FOREIGN KEY (template_id) REFERENCES templates(template_id), - FOREIGN KEY (request_id) REFERENCES fuzzing_request_response(request_id) -); -CREATE INDEX IF NOT EXISTS idx_FuzzingResults_comp_temp_time ON fuzzing_results (component_id, template_id, timestamp); - --- Trigger to update stats when a new fuzzing result is inserted -CREATE TRIGGER IF NOT EXISTS update_component_stats -AFTER INSERT ON fuzzing_results -BEGIN - UPDATE components - SET last_fuzzed = NEW.timestamp, - total_fuzz_count = total_fuzz_count + 1 - WHERE component_id = NEW.component_id; - - INSERT INTO component_templates (component_id, template_id, times_applied) - VALUES (NEW.component_id, NEW.template_id, 1) - ON CONFLICT(component_id, template_id) DO UPDATE SET - times_applied = times_applied + 1; -END; - --- -CREATE TABLE IF NOT EXISTS fuzzing_request_response ( - request_id INTEGER PRIMARY KEY AUTOINCREMENT, - raw_request TEXT NOT NULL, - raw_response TEXT NOT NULL -); \ No newline at end of file diff --git a/pkg/fuzz/stats/simple.go b/pkg/fuzz/stats/simple.go new file mode 100644 index 0000000000..4a93aaaa42 --- /dev/null +++ b/pkg/fuzz/stats/simple.go @@ -0,0 +1,164 @@ +package stats + +import ( + "fmt" + "net/http" + "net/url" + "strings" + "sync" + "sync/atomic" +) + +type simpleStats struct { + totalComponentsTested atomic.Int64 + totalEndpointsTested atomic.Int64 + totalFuzzedRequests atomic.Int64 + totalMatchedResults atomic.Int64 + totalTemplatesTested atomic.Int64 + totalErroredRequests atomic.Int64 + + statusCodes sync.Map + severityCounts sync.Map + + componentsUniqueMap sync.Map + endpointsUniqueMap sync.Map + templatesUniqueMap sync.Map + errorGroupedStats sync.Map +} + +func NewSimpleStats() (*simpleStats, error) { + return &simpleStats{ + totalComponentsTested: atomic.Int64{}, + totalEndpointsTested: atomic.Int64{}, + totalMatchedResults: atomic.Int64{}, + totalFuzzedRequests: atomic.Int64{}, + totalTemplatesTested: atomic.Int64{}, + totalErroredRequests: atomic.Int64{}, + statusCodes: sync.Map{}, + severityCounts: sync.Map{}, + componentsUniqueMap: sync.Map{}, + endpointsUniqueMap: sync.Map{}, + templatesUniqueMap: sync.Map{}, + errorGroupedStats: sync.Map{}, + }, nil +} + +func (s *simpleStats) Close() {} + +func (s *simpleStats) InsertComponent(event ComponentEvent) error { + componentKey := fmt.Sprintf("%s_%s", event.ComponentName, event.ComponentType) + if _, ok := s.componentsUniqueMap.Load(componentKey); !ok { + s.componentsUniqueMap.Store(componentKey, true) + s.totalComponentsTested.Add(1) + } + + parsedURL, err := url.Parse(event.URL) + if err != nil { + return err + } + + endpointsKey := fmt.Sprintf("%s_%s", event.siteName, parsedURL.Path) + if _, ok := s.endpointsUniqueMap.Load(endpointsKey); !ok { + s.endpointsUniqueMap.Store(endpointsKey, true) + s.totalEndpointsTested.Add(1) + } + + return nil +} + +func (s *simpleStats) InsertMatchedRecord(event FuzzingEvent) error { + s.totalFuzzedRequests.Add(1) + + s.incrementStatusCode(event.StatusCode) + if event.Matched { + s.totalMatchedResults.Add(1) + + s.incrementSeverityCount(event.Severity) + } + + if _, ok := s.templatesUniqueMap.Load(event.TemplateID); !ok { + s.templatesUniqueMap.Store(event.TemplateID, true) + s.totalTemplatesTested.Add(1) + } + return nil +} + +func (s *simpleStats) InsertError(event ErrorEvent) error { + s.totalErroredRequests.Add(1) + + value, _ := s.errorGroupedStats.LoadOrStore(event.Error, &atomic.Int64{}) + if counter, ok := value.(*atomic.Int64); ok { + counter.Add(1) + } + return nil +} + +type SimpleStatsResponse struct { + TotalMatchedResults int64 + TotalComponentsTested int64 + TotalEndpointsTested int64 + TotalFuzzedRequests int64 + TotalTemplatesTested int64 + TotalErroredRequests int64 + StatusCodes map[string]int64 + SeverityCounts map[string]int64 + ErrorGroupedStats map[string]int64 +} + +func (s *simpleStats) GetStatistics() SimpleStatsResponse { + statusStats := make(map[string]int64) + s.statusCodes.Range(func(key, value interface{}) bool { + if count, ok := value.(*atomic.Int64); ok { + statusStats[formatStatusCode(key.(int))] = count.Load() + } + return true + }) + + severityStats := make(map[string]int64) + s.severityCounts.Range(func(key, value interface{}) bool { + if count, ok := value.(*atomic.Int64); ok { + severityStats[key.(string)] = count.Load() + } + return true + }) + + errorStats := make(map[string]int64) + s.errorGroupedStats.Range(func(key, value interface{}) bool { + if count, ok := value.(*atomic.Int64); ok { + errorStats[key.(string)] = count.Load() + } + return true + }) + + return SimpleStatsResponse{ + TotalMatchedResults: s.totalMatchedResults.Load(), + StatusCodes: statusStats, + SeverityCounts: severityStats, + TotalComponentsTested: s.totalComponentsTested.Load(), + TotalEndpointsTested: s.totalEndpointsTested.Load(), + TotalFuzzedRequests: s.totalFuzzedRequests.Load(), + TotalTemplatesTested: s.totalTemplatesTested.Load(), + TotalErroredRequests: s.totalErroredRequests.Load(), + ErrorGroupedStats: errorStats, + } +} + +func (s *simpleStats) incrementStatusCode(statusCode int) { + value, _ := s.statusCodes.LoadOrStore(statusCode, &atomic.Int64{}) + if counter, ok := value.(*atomic.Int64); ok { + counter.Add(1) + } +} + +func (s *simpleStats) incrementSeverityCount(severity string) { + value, _ := s.severityCounts.LoadOrStore(severity, &atomic.Int64{}) + if counter, ok := value.(*atomic.Int64); ok { + counter.Add(1) + } +} + +func formatStatusCode(code int) string { + escapedText := strings.ToTitle(strings.ReplaceAll(http.StatusText(code), " ", "_")) + formatted := fmt.Sprintf("%d_%s", code, escapedText) + return formatted +} diff --git a/pkg/fuzz/stats/stats.go b/pkg/fuzz/stats/stats.go index 7ab375a2a4..a2b0dcc056 100644 --- a/pkg/fuzz/stats/stats.go +++ b/pkg/fuzz/stats/stats.go @@ -7,17 +7,16 @@ import ( "net/url" "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" ) // Tracker is a stats tracker module for fuzzing server type Tracker struct { - database StatsDatabase + database *simpleStats } // NewTracker creates a new tracker instance -func NewTracker(scanName string) (*Tracker, error) { - db, err := NewSqliteStatsDatabase(scanName) +func NewTracker() (*Tracker, error) { + db, err := NewSimpleStats() if err != nil { return nil, errors.Wrap(err, "could not create new tracker") } @@ -28,13 +27,12 @@ func NewTracker(scanName string) (*Tracker, error) { return tracker, nil } +func (t *Tracker) GetStats() SimpleStatsResponse { + return t.database.GetStatistics() +} + // Close closes the tracker func (t *Tracker) Close() { - _, err := t.database.(*sqliteStatsDatabase).db.Exec("VACUUM") - if err != nil { - gologger.Error().Msgf("could not truncate wal: %s", err) - } - t.database.Close() } @@ -47,21 +45,41 @@ type FuzzingEvent struct { PayloadSent string StatusCode int Matched bool - SiteName string RawRequest string RawResponse string + Severity string + + siteName string } func (t *Tracker) RecordResultEvent(event FuzzingEvent) { - event.SiteName = getCorrectSiteName(event.URL) + event.siteName = getCorrectSiteName(event.URL) t.database.InsertMatchedRecord(event) } -func (t *Tracker) RecordComponentEvent(event FuzzingEvent) { - event.SiteName = getCorrectSiteName(event.URL) +type ComponentEvent struct { + URL string + ComponentType string + ComponentName string + + siteName string +} + +func (t *Tracker) RecordComponentEvent(event ComponentEvent) { + event.siteName = getCorrectSiteName(event.URL) t.database.InsertComponent(event) } +type ErrorEvent struct { + TemplateID string + URL string + Error string +} + +func (t *Tracker) RecordErrorEvent(event ErrorEvent) { + t.database.InsertError(event) +} + func getCorrectSiteName(originalURL string) string { parsed, err := url.Parse(originalURL) if err != nil { diff --git a/pkg/output/output.go b/pkg/output/output.go index 84201c0d73..af72d3c188 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -73,6 +73,8 @@ type StandardWriter struct { DisableStdout bool AddNewLinesOutputFile bool // by default this is only done for stdout KeysToRedact []string + + RequestHook func(*JSONLogRequest) } var decolorizerRegex = regexp.MustCompile(`\x1B\[[0-9;]*[a-zA-Z]`) @@ -348,7 +350,7 @@ type JSONLogRequest struct { // Request writes a log the requests trace log func (w *StandardWriter) Request(templatePath, input, requestType string, requestErr error) { - if w.traceFile == nil && w.errorFile == nil { + if w.traceFile == nil && w.errorFile == nil && w.RequestHook == nil { return } request := &JSONLogRequest{ @@ -397,6 +399,11 @@ func (w *StandardWriter) Request(templatePath, input, requestType string, reques if val := errkit.GetAttrValue(requestErr, "address"); val.Any() != nil { request.Address = val.String() } + + if w.RequestHook != nil { + w.RequestHook(request) + } + data, err := jsoniter.Marshal(request) if err != nil { return diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 5e54f46f72..ff616ed87b 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -930,18 +930,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ } } - if request.options.FuzzStatsDB != nil && generatedRequest.fuzzGeneratedRequest.Request != nil { - request.options.FuzzStatsDB.RecordComponentEvent(fuzzStats.FuzzingEvent{ - URL: input.MetaInput.Target(), - SiteName: hostname, - TemplateID: request.options.TemplateID, - ComponentType: generatedRequest.fuzzGeneratedRequest.Component.Name(), - ComponentName: generatedRequest.fuzzGeneratedRequest.Parameter, - PayloadSent: generatedRequest.fuzzGeneratedRequest.Value, - StatusCode: respChain.Response().StatusCode, - }) - } - finalEvent := make(output.InternalEvent) if request.Analyzer != nil { @@ -1038,7 +1026,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if request.options.FuzzStatsDB != nil && generatedRequest.fuzzGeneratedRequest.Request != nil { request.options.FuzzStatsDB.RecordResultEvent(fuzzStats.FuzzingEvent{ URL: input.MetaInput.Target(), - SiteName: hostname, TemplateID: request.options.TemplateID, ComponentType: generatedRequest.fuzzGeneratedRequest.Component.Name(), ComponentName: generatedRequest.fuzzGeneratedRequest.Parameter, @@ -1047,6 +1034,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ Matched: event.HasResults(), RawRequest: string(dumpedRequest), RawResponse: respChain.FullResponse().String(), + Severity: request.options.TemplateInfo.SeverityHolder.Severity.String(), }) } diff --git a/pkg/types/types.go b/pkg/types/types.go index fa3365c0b4..8fbf070f8f 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -419,8 +419,8 @@ type Options struct { DASTServerToken string // DASTServerAddress is the address for the dast server DASTServerAddress string - // DASTScanName is the name of the scan to use for the dast report - DASTScanName string + // DASTReport enables dast report server & final report generation + DASTReport bool // Scope contains a list of regexes for in-scope URLS Scope goflags.StringSlice // OutOfScope contains a list of regexes for out-scope URLS From 38f25f5263885d7fb52af525caf7a7e3d9b444aa Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 2 Dec 2024 17:10:36 +0530 Subject: [PATCH 11/23] feat: fixed analyzer timeout issue + missing case fix --- pkg/fuzz/analyzers/time/time_delay.go | 4 ++-- pkg/protocols/http/httpclientpool/clientpool.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/fuzz/analyzers/time/time_delay.go b/pkg/fuzz/analyzers/time/time_delay.go index d3b684ceff..75d438667f 100644 --- a/pkg/fuzz/analyzers/time/time_delay.go +++ b/pkg/fuzz/analyzers/time/time_delay.go @@ -166,6 +166,6 @@ func (o *simpleLinearRegression) Predict(x float64) float64 { func (o *simpleLinearRegression) IsWithinConfidence(correlationErrorRange float64, expectedSlope float64, slopeErrorRange float64, ) bool { - return o.correlation > 1.0-correlationErrorRange && - math.Abs(expectedSlope-o.slope) < slopeErrorRange + return o.correlation > 1.0-correlationErrorRange + //math.Abs(expectedSlope-o.slope) < slopeErrorRange } diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index 94ed61ff3f..e4dc6eb332 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -111,6 +111,7 @@ func (c *Configuration) Clone() *Configuration { if c.Connection != nil { cloneConnection := &ConnectionConfiguration{ DisableKeepAlive: c.Connection.DisableKeepAlive, + CustomMaxTimeout: c.Connection.CustomMaxTimeout, } if c.Connection.HasCookieJar() { cookiejar := *c.Connection.GetCookieJar() From fda616533d3a4bc8446264dae48977b0c35bfcad Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 2 Dec 2024 19:14:41 +0530 Subject: [PATCH 12/23] misc changes fix --- internal/runner/runner.go | 72 +++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index f7258ada40..68dc2710f4 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -284,6 +284,42 @@ func New(options *types.Options) (*Runner, error) { } runner.resumeCfg = resumeCfg + if options.DASTReport || options.DASTServer { + var err error + runner.fuzzStats, err = fuzzStats.NewTracker() + if err != nil { + return nil, errors.Wrap(err, "could not create fuzz stats db") + } + if !options.DASTServer { + dastServer, err := server.NewStatsServer(runner.fuzzStats) + if err != nil { + return nil, errors.Wrap(err, "could not create dast server") + } + runner.dastServer = dastServer + } + } + + // Create the output file if asked + outputWriter, err := output.NewStandardWriter(options) + if err != nil { + return nil, errors.Wrap(err, "could not create output file") + } + if runner.fuzzStats != nil { + outputWriter.RequestHook = func(request *output.JSONLogRequest) { + if request.Error == "none" || request.Error == "" { + return + } + runner.fuzzStats.RecordErrorEvent(fuzzStats.ErrorEvent{ + TemplateID: request.Template, + URL: request.Input, + Error: request.Error, + }) + } + } + + // setup a proxy writer to automatically upload results to PDCP + runner.output = runner.setupPDCPUpload(outputWriter) + opts := interactsh.DefaultOptions(runner.output, runner.issuesClient, runner.progress) opts.Debug = runner.options.Debug opts.NoColor = runner.options.NoColor @@ -337,42 +373,6 @@ func New(options *types.Options) (*Runner, error) { runner.tmpDir = tmpDir } - if options.DASTReport || options.DASTServer { - var err error - runner.fuzzStats, err = fuzzStats.NewTracker() - if err != nil { - return nil, errors.Wrap(err, "could not create fuzz stats db") - } - if !options.DASTServer { - dastServer, err := server.NewStatsServer(runner.fuzzStats) - if err != nil { - return nil, errors.Wrap(err, "could not create dast server") - } - runner.dastServer = dastServer - } - } - - // Create the output file if asked - outputWriter, err := output.NewStandardWriter(options) - if err != nil { - return nil, errors.Wrap(err, "could not create output file") - } - if runner.fuzzStats != nil { - outputWriter.RequestHook = func(request *output.JSONLogRequest) { - if request.Error == "none" || request.Error == "" { - return - } - runner.fuzzStats.RecordErrorEvent(fuzzStats.ErrorEvent{ - TemplateID: request.Template, - URL: request.Input, - Error: request.Error, - }) - } - } - - // setup a proxy writer to automatically upload results to PDCP - runner.output = runner.setupPDCPUpload(outputWriter) - return runner, nil } From 7f55f01349d6fa69810adda9efb30cec07c82dd1 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Tue, 10 Dec 2024 12:18:49 +0530 Subject: [PATCH 13/23] feat: changed the logics a bit + misc changes and additions --- go.mod | 4 +- go.sum | 10 +- pkg/fuzz/analyzers/analyzers.go | 11 +- pkg/fuzz/analyzers/time/analyzer.go | 21 +- pkg/fuzz/analyzers/time/time_delay.go | 90 ++-- pkg/fuzz/analyzers/time/time_delay_test.go | 554 +++++++++++++++++---- pkg/protocols/http/http.go | 4 +- 7 files changed, 539 insertions(+), 155 deletions(-) diff --git a/go.mod b/go.mod index 4f054ca982..b6e347a5f4 100644 --- a/go.mod +++ b/go.mod @@ -20,11 +20,11 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.1.1 - github.com/projectdiscovery/fastdialer v0.2.10 + github.com/projectdiscovery/fastdialer v0.2.13 github.com/projectdiscovery/hmap v0.0.69 github.com/projectdiscovery/interactsh v1.2.0 github.com/projectdiscovery/rawhttp v0.1.76 - github.com/projectdiscovery/retryabledns v1.0.86 + github.com/projectdiscovery/retryabledns v1.0.87 github.com/projectdiscovery/retryablehttp-go v1.0.88 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 diff --git a/go.sum b/go.sum index 8edaf6a970..4c5ffd818c 100644 --- a/go.sum +++ b/go.sum @@ -864,8 +864,10 @@ github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB7 github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= github.com/projectdiscovery/dsl v0.3.3 h1:4Ij5S86cHlb6xFrS7+5zAiJPeBt5h970XBTHqeTkpyU= github.com/projectdiscovery/dsl v0.3.3/go.mod h1:DAjSeaogLM9f0Ves2zDc/vbJrfcv+kEmS51p0dLLaPI= -github.com/projectdiscovery/fastdialer v0.2.10 h1:5iciZXMPdynbk/9iuqkJT1gqMXwzgEpFSWdoj/5CHCo= -github.com/projectdiscovery/fastdialer v0.2.10/go.mod h1:21rwXMecVsPVdSvON8Up761/GgxC4OSc9Rvx5LNH5fY= +github.com/projectdiscovery/fastdialer v0.2.12-0.20241205195710-bb4879dd1d39 h1:NfDFJnc0r33XDYJLvBjm7kV1pc6RhDhLco/W2j459Wo= +github.com/projectdiscovery/fastdialer v0.2.12-0.20241205195710-bb4879dd1d39/go.mod h1:R1lMBMgp1orUO39tOe9kujDbEO2iQNQZgDM/2TqIRf8= +github.com/projectdiscovery/fastdialer v0.2.13 h1:5XzSv0hwITzRAMwyoJ9GFZSTVtaI4jmwER968TbDLbI= +github.com/projectdiscovery/fastdialer v0.2.13/go.mod h1:T1EaYHbWmCnVHSYz12nAjnHMNFEfGMLLw37cb0k1X3A= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.7 h1:Q6uXo/j8SaV/GlAHkEYQi8WQoPXyJWxyspx+aFmz9Qk= @@ -902,8 +904,8 @@ github.com/projectdiscovery/rawhttp v0.1.76 h1:O2IoYSyG7unH5oW8r8j3539koCNkimyzc github.com/projectdiscovery/rawhttp v0.1.76/go.mod h1:ZxvbdkRV2PBoCbJxHh9B0P0nC5gVG3p1Z5uiua3iC5I= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= -github.com/projectdiscovery/retryabledns v1.0.86 h1:8YMJGJ94lFBKKN3t7NOzJfbGsZoh9qNpi49xdfJcZVc= -github.com/projectdiscovery/retryabledns v1.0.86/go.mod h1:5PhXvlLkEFmlYOt9i4wiKA1eONLrNiZ6DQE88Ph9rgU= +github.com/projectdiscovery/retryabledns v1.0.87 h1:MPEXVKdu89FEW23xIMpBzzvdegvtcAs7osSqHimBVOs= +github.com/projectdiscovery/retryabledns v1.0.87/go.mod h1:snDTjRcmBj+iveber/o0jC0iLEkM6c0Sdo2IXe2O+fE= github.com/projectdiscovery/retryablehttp-go v1.0.88 h1:uR6T+i8Sy1isfG1KClhhsXnOqkOR6E8MAvuyOFq3T10= github.com/projectdiscovery/retryablehttp-go v1.0.88/go.mod h1:ktjiIKyej+plUeK9vksqRf3wGicqY3E1rW84V/O7p0M= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= diff --git a/pkg/fuzz/analyzers/analyzers.go b/pkg/fuzz/analyzers/analyzers.go index 8eedb6b71b..6266e8bb01 100644 --- a/pkg/fuzz/analyzers/analyzers.go +++ b/pkg/fuzz/analyzers/analyzers.go @@ -81,18 +81,11 @@ func ApplyPayloadTransformations(value string) string { } const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -const ( - letterIdxBits = 6 // 6 bits to represent a letter index - letterIdxMask = 1< 0 && varY > 0 { + o.correlation = covXY / (math.Sqrt(varX) * math.Sqrt(varY)) + } else { + o.correlation = 0.0 } } @@ -164,8 +178,12 @@ func (o *simpleLinearRegression) Predict(x float64) float64 { return o.slope*x + o.intercept } -func (o *simpleLinearRegression) IsWithinConfidence(correlationErrorRange float64, expectedSlope float64, slopeErrorRange float64, -) bool { +func (o *simpleLinearRegression) IsWithinConfidence(correlationErrorRange float64, expectedSlope float64, slopeErrorRange float64) bool { + // For now, just check correlation as originally done: + // You might later reintroduce slope checks: + // return math.Abs(expectedSlope-o.slope) < slopeErrorRange && o.correlation > 1.0 - correlationErrorRange + if o.count < 2 { + return true + } return o.correlation > 1.0-correlationErrorRange - //math.Abs(expectedSlope-o.slope) < slopeErrorRange } diff --git a/pkg/fuzz/analyzers/time/time_delay_test.go b/pkg/fuzz/analyzers/time/time_delay_test.go index 8a71243595..451766912f 100644 --- a/pkg/fuzz/analyzers/time/time_delay_test.go +++ b/pkg/fuzz/analyzers/time/time_delay_test.go @@ -3,141 +3,507 @@ package time import ( - "math" "math/rand" + "reflect" "testing" "time" - - "github.com/stretchr/testify/require" ) -const ( - correlationErrorRange = float64(0.1) - slopeErrorRange = float64(0.2) -) +// This test suite verifies the timing dependency detection algorithm by testing various scenarios: +// +// Test Categories: +// 1. Perfect Linear Cases +// - TestPerfectLinear: Basic case with slope=1, no noise +// - TestPerfectLinearSlopeOne_NoNoise: Similar to above but with different parameters +// - TestPerfectLinearSlopeTwo_NoNoise: Tests detection of slope=2 relationship +// +// 2. Noisy Cases +// - TestLinearWithNoise: Verifies detection works with moderate noise (±0.2s) +// - TestNoisyLinear: Similar but with different noise parameters +// - TestHighNoiseConcealsSlope: Verifies detection fails with extreme noise (±5s) +// +// 3. No Correlation Cases +// - TestNoCorrelation: Basic case where delay has no effect +// - TestNoCorrelationHighBaseline: High baseline (~15s) masks any delay effect +// - TestNegativeSlopeScenario: Verifies detection rejects negative correlations +// +// 4. Edge Cases +// - TestMinimalData: Tests behavior with minimal data points (2 requests) +// - TestLargeNumberOfRequests: Tests stability with many data points (20 requests) +// - TestChangingBaseline: Tests detection with shifting baseline mid-test +// - TestHighBaselineLowSlope: Tests detection of subtle correlations (slope=0.5) +// +// ZAP Test Cases: +// +// 1. Alternating Sequence Tests +// - TestAlternatingSequences: Verifies correct alternation between high and low delays +// +// 2. Non-Injectable Cases +// - TestNonInjectableQuickFail: Tests quick failure when response time < requested delay +// - TestSlowNonInjectableCase: Tests early termination with consistently high response times +// - TestRealWorldNonInjectableCase: Tests behavior with real-world response patterns +// +// 3. Error Tolerance Tests +// - TestSmallErrorDependence: Verifies detection works with small random variations +// +// Key Parameters Tested: +// - requestsLimit: Number of requests to make (2-20) +// - highSleepTimeSeconds: Maximum delay to test (typically 5s) +// - correlationErrorRange: Acceptable deviation from perfect correlation (0.05-0.3) +// - slopeErrorRange: Acceptable deviation from expected slope (0.1-1.5) +// +// The test suite uses various mock senders (perfectLinearSender, noCorrelationSender, etc.) +// to simulate different timing behaviors and verify the detection algorithm works correctly +// across a wide range of scenarios. -var rng = rand.New(rand.NewSource(time.Now().UnixNano())) +// Mock request sender that simulates a perfect linear relationship: +// Observed delay = baseline + requested_delay +func perfectLinearSender(baseline float64) func(delay int) (float64, error) { + return func(delay int) (float64, error) { + // simulate some processing time + time.Sleep(10 * time.Millisecond) // just a small artificial sleep to mimic network + return baseline + float64(delay), nil + } +} -func Test_should_generate_alternating_sequences(t *testing.T) { - var generatedDelays []float64 - reqSender := func(delay int) (float64, error) { - generatedDelays = append(generatedDelays, float64(delay)) - return float64(delay), nil +// Mock request sender that simulates no correlation: +// The response time is random around a certain constant baseline, ignoring requested delay. +func noCorrelationSender(baseline, noiseAmplitude float64) func(int) (float64, error) { + return func(delay int) (float64, error) { + time.Sleep(10 * time.Millisecond) + noise := 0.0 + if noiseAmplitude > 0 { + noise = (rand.Float64()*2 - 1) * noiseAmplitude + } + return baseline + noise, nil } - matched, _, err := checkTimingDependency(4, 15, correlationErrorRange, slopeErrorRange, reqSender) - require.NoError(t, err) - require.True(t, matched) - require.EqualValues(t, []float64{15, 1, 15, 1}, generatedDelays) } -func Test_should_giveup_non_injectable(t *testing.T) { - var timesCalled int - reqSender := func(delay int) (float64, error) { - timesCalled++ - return 0.5, nil +// Mock request sender that simulates partial linearity but with some noise. +func noisyLinearSender(baseline float64) func(delay int) (float64, error) { + return func(delay int) (float64, error) { + time.Sleep(10 * time.Millisecond) + // Add some noise (±0.2s) to a linear relationship + noise := 0.2 + return baseline + float64(delay) + noise, nil } - matched, _, err := checkTimingDependency(4, 15, correlationErrorRange, slopeErrorRange, reqSender) - require.NoError(t, err) - require.False(t, matched) - require.Equal(t, 1, timesCalled) } -func Test_should_giveup_slow_non_injectable(t *testing.T) { - var timesCalled int - reqSender := func(delay int) (float64, error) { - timesCalled++ - return 10 + rng.Float64()*0.5, nil +func TestPerfectLinear(t *testing.T) { + // Expect near-perfect correlation and slope ~ 1.0 + requestsLimit := 6 // 3 pairs: enough data for stable regression + highSleepTimeSeconds := 5 + corrErrRange := 0.1 + slopeErrRange := 0.2 + + sender := perfectLinearSender(5.0) // baseline 5s, observed = 5s + requested_delay + match, reason, err := checkTimingDependency( + requestsLimit, + highSleepTimeSeconds, + corrErrRange, + slopeErrRange, + sender, + ) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if !match { + t.Fatalf("Expected a match but got none. Reason: %s", reason) } - matched, _, err := checkTimingDependency(4, 15, correlationErrorRange, slopeErrorRange, reqSender) - require.NoError(t, err) - require.False(t, matched) - require.LessOrEqual(t, timesCalled, 3) } -func Test_should_giveup_slow_non_injectable_realworld(t *testing.T) { - var timesCalled int - var iteration = 0 - counts := []float64{21, 11, 21, 11} - reqSender := func(delay int) (float64, error) { - timesCalled++ - iteration++ - return counts[iteration-1], nil +func TestNoCorrelation(t *testing.T) { + // Expect no match because requested delay doesn't influence observed delay + requestsLimit := 6 + highSleepTimeSeconds := 5 + corrErrRange := 0.1 + slopeErrRange := 0.5 + + sender := noCorrelationSender(8.0, 0.1) + match, reason, err := checkTimingDependency( + requestsLimit, + highSleepTimeSeconds, + corrErrRange, + slopeErrRange, + sender, + ) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if match { + t.Fatalf("Expected no match but got one. Reason: %s", reason) } - matched, _, err := checkTimingDependency(4, 15, correlationErrorRange, slopeErrorRange, reqSender) - require.NoError(t, err) - require.False(t, matched) - require.LessOrEqual(t, timesCalled, 4) } -func Test_should_detect_dependence_with_small_error(t *testing.T) { - reqSender := func(delay int) (float64, error) { - return float64(delay) + rng.Float64()*0.5, nil +func TestNoisyLinear(t *testing.T) { + // Even with some noise, it should detect a strong positive correlation if + // we allow a slightly bigger margin for slope/correlation. + requestsLimit := 10 // More requests to average out noise + highSleepTimeSeconds := 5 + corrErrRange := 0.2 // allow some lower correlation due to noise + slopeErrRange := 0.5 // slope may deviate slightly + + sender := noisyLinearSender(2.0) // baseline 2s, observed ~ 2s + requested_delay ±0.2 + match, reason, err := checkTimingDependency( + requestsLimit, + highSleepTimeSeconds, + corrErrRange, + slopeErrRange, + sender, + ) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + // We expect a match since it's still roughly linear. The slope should be close to 1. + if !match { + t.Fatalf("Expected a match in noisy linear test but got none. Reason: %s", reason) } - matched, reason, err := checkTimingDependency(4, 15, correlationErrorRange, slopeErrorRange, reqSender) - require.NoError(t, err) - require.True(t, matched) - require.NotEmpty(t, reason) } -func Test_LinearRegression_Numerical_stability(t *testing.T) { - variables := [][]float64{ - {1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {1, 1}, {2, 2}, {2, 2}, {2, 2}, +func TestMinimalData(t *testing.T) { + // With too few requests, correlation might not be stable. + // Here, we send only 2 requests (1 pair) and see if the logic handles it gracefully. + requestsLimit := 2 + highSleepTimeSeconds := 5 + corrErrRange := 0.3 + slopeErrRange := 0.5 + + // Perfect linear sender again + sender := perfectLinearSender(5.0) + match, reason, err := checkTimingDependency( + requestsLimit, + highSleepTimeSeconds, + corrErrRange, + slopeErrRange, + sender, + ) + if err != nil { + t.Fatalf("Unexpected error: %v", err) } - slope := float64(1) - correlation := float64(1) + if !match { + t.Fatalf("Expected match but got none. Reason: %s", reason) + } +} + +// Utility functions to generate different behaviors + +// linearSender returns a sender that calculates observed delay as: +// observed = baseline + slope * requested_delay + noise +func linearSender(baseline, slope, noiseAmplitude float64) func(int) (float64, error) { + return func(delay int) (float64, error) { + time.Sleep(10 * time.Millisecond) + noise := 0.0 + if noiseAmplitude > 0 { + noise = (rand.Float64()*2 - 1) * noiseAmplitude // random noise in [-noiseAmplitude, noiseAmplitude] + } + return baseline + slope*float64(delay) + noise, nil + } +} + +// changingBaselineSender simulates a baseline that changes after half the requests are done. +func changingBaselineSender(initialBaseline, newBaseline, slope, noiseAmplitude float64, switchAfter int, counter *int) func(int) (float64, error) { + return func(delay int) (float64, error) { + time.Sleep(10 * time.Millisecond) + base := initialBaseline + if *counter >= switchAfter { + base = newBaseline + } + *counter++ + noise := 0.0 + if noiseAmplitude > 0 { + noise = (rand.Float64()*2 - 1) * noiseAmplitude + } + return base + slope*float64(delay) + noise, nil + } +} - regression := newSimpleLinearRegression() - for _, v := range variables { - regression.AddPoint(v[0], v[1]) +// negativeSlopeSender just for completeness - higher delay = less observed time +func negativeSlopeSender(baseline float64) func(int) (float64, error) { + return func(delay int) (float64, error) { + time.Sleep(10 * time.Millisecond) + return baseline - float64(delay)*2.0, nil } - require.True(t, almostEqual(regression.slope, slope)) - require.True(t, almostEqual(regression.correlation, correlation)) } -func Test_LinearRegression_exact_verify(t *testing.T) { - variables := [][]float64{ - {1, 1}, {2, 3}, +// We assume you have an imported checkTimingDependency function. Adjust imports as needed. +// func checkTimingDependency(...) (bool, string, error) { ... } + +func TestPerfectLinearSlopeOne_NoNoise(t *testing.T) { + match, reason, err := checkTimingDependency( + 10, // requestsLimit + 5, // highSleepTimeSeconds + 0.1, // correlationErrorRange + 0.2, // slopeErrorRange (allowing slope between 0.8 and 1.2) + linearSender(2.0, 1.0, 0.0), + ) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if !match { + t.Fatalf("Expected a match for perfect linear slope=1. Reason: %s", reason) } - slope := float64(2) - correlation := float64(1) +} - regression := newSimpleLinearRegression() - for _, v := range variables { - regression.AddPoint(v[0], v[1]) +func TestPerfectLinearSlopeTwo_NoNoise(t *testing.T) { + // slope=2 means observed = baseline + 2*requested_delay + match, reason, err := checkTimingDependency( + 10, + 5, + 0.1, // correlation must still be good + 1.5, // allow slope in range (0.5 to 2.5), we should be close to 2.0 anyway + linearSender(1.0, 2.0, 0.0), + ) + if err != nil { + t.Fatalf("Error: %v", err) + } + if !match { + t.Fatalf("Expected a match for slope=2. Reason: %s", reason) } - require.True(t, almostEqual(regression.slope, slope)) - require.True(t, almostEqual(regression.correlation, correlation)) } -func Test_LinearRegression_known_verify(t *testing.T) { - variables := [][]float64{ - {1, 1.348520581}, {2, 2.524046187}, {3, 3.276944688}, {4, 4.735374498}, {5, 5.150291657}, +func TestLinearWithNoise(t *testing.T) { + // slope=1 but with noise ±0.2 seconds + match, reason, err := checkTimingDependency( + 12, + 5, + 0.2, // correlationErrorRange relaxed to account for noise + 0.5, // slopeErrorRange also relaxed + linearSender(5.0, 1.0, 0.2), + ) + if err != nil { + t.Fatalf("Error: %v", err) } - slope := float64(0.981487046) - correlation := float64(0.979228906) + if !match { + t.Fatalf("Expected a match for noisy linear data. Reason: %s", reason) + } +} - regression := newSimpleLinearRegression() - for _, v := range variables { - regression.AddPoint(v[0], v[1]) +func TestNoCorrelationHighBaseline(t *testing.T) { + // baseline ~15s, requested delays won't matter + match, reason, err := checkTimingDependency( + 10, + 5, + 0.1, // correlation should be near zero, so no match expected + 0.5, + noCorrelationSender(15.0, 0.1), + ) + if err != nil { + t.Fatalf("Error: %v", err) + } + if match { + t.Fatalf("Expected no match for no correlation scenario. Got: %s", reason) } - require.True(t, almostEqual(regression.slope, slope)) - require.True(t, almostEqual(regression.correlation, correlation)) } -func Test_LinearRegression_nonlinear_verify(t *testing.T) { - variables := [][]float64{ - {1, 2}, {2, 4}, {3, 8}, {4, 16}, {5, 32}, +func TestNegativeSlopeScenario(t *testing.T) { + // Increasing delay decreases observed time + match, reason, err := checkTimingDependency( + 10, + 5, + 0.2, + 0.5, + negativeSlopeSender(10.0), + ) + if err != nil { + t.Fatalf("Error: %v", err) } + if match { + t.Fatalf("Expected no match in negative slope scenario. Reason: %s", reason) + } +} - regression := newSimpleLinearRegression() - for _, v := range variables { - regression.AddPoint(v[0], v[1]) +func TestChangingBaseline(t *testing.T) { + counter := 0 + // baseline = 2s initially, then after 5 requests it changes to 5s. + // slope=1 means observed = baseline + requested_delay. + // Even with changing baseline, a strong correlation should still appear if slope is consistent. + match, reason, err := checkTimingDependency( + 12, + 5, + 0.2, + 0.5, + changingBaselineSender(2.0, 5.0, 1.0, 0.1, 6, &counter), + ) + if err != nil { + t.Fatalf("Error: %v", err) + } + // Still should see a linear relationship overall (requests with delay=5 should be consistently ~delay more than delay=1). + if !match { + t.Fatalf("Expected a match despite baseline changes. Reason: %s", reason) } - require.Less(t, regression.correlation, 0.9) } -const float64EqualityThreshold = 1e-8 +func TestLargeNumberOfRequests(t *testing.T) { + // 20 requests, slope=1.0, no noise. Should be very stable and produce a very high correlation. + match, reason, err := checkTimingDependency( + 20, + 5, + 0.05, // very strict correlation requirement + 0.1, // very strict slope range + linearSender(1.0, 1.0, 0.0), + ) + if err != nil { + t.Fatalf("Error: %v", err) + } + if !match { + t.Fatalf("Expected a strong match with many requests and perfect linearity. Reason: %s", reason) + } +} -func almostEqual(a, b float64) bool { - return math.Abs(a-b) <= float64EqualityThreshold +func TestHighBaselineLowSlope(t *testing.T) { + // baseline=10s, slope=0.5 means each requested second only adds 0.5s to observed time. + // If our thresholds are strict, this should still show correlation, but slope <1. + match, reason, err := checkTimingDependency( + 10, + 5, + 0.2, + 0.6, // expecting slope around 0.5, allow range ~0.4 to 0.6 + linearSender(10.0, 0.5, 0.0), + ) + if err != nil { + t.Fatalf("Error: %v", err) + } + if !match { + t.Fatalf("Expected a match for slope=0.5 linear scenario. Reason: %s", reason) + } +} + +func TestHighNoiseConcealsSlope(t *testing.T) { + // slope=1, but noise=5 seconds is huge and might conceal the correlation. + // With large noise, the test may fail to detect correlation. + match, reason, err := checkTimingDependency( + 12, + 5, + 0.1, // still strict + 0.2, // still strict + linearSender(5.0, 1.0, 5.0), + ) + if err != nil { + t.Fatalf("Error: %v", err) + } + // Expect no match because the noise level is too high to establish a reliable correlation. + if match { + t.Fatalf("Expected no match due to extreme noise. Reason: %s", reason) + } +} + +func TestAlternatingSequences(t *testing.T) { + var generatedDelays []float64 + reqSender := func(delay int) (float64, error) { + generatedDelays = append(generatedDelays, float64(delay)) + return float64(delay), nil + } + match, reason, err := checkTimingDependency( + 4, // requestsLimit + 15, // highSleepTimeSeconds + 0.1, // correlationErrorRange + 0.2, // slopeErrorRange + reqSender, + ) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if !match { + t.Fatalf("Expected a match but got none. Reason: %s", reason) + } + // Verify alternating sequence of delays + expectedDelays := []float64{15, 1, 15, 1} + if !reflect.DeepEqual(generatedDelays, expectedDelays) { + t.Fatalf("Expected delays %v but got %v", expectedDelays, generatedDelays) + } +} + +func TestNonInjectableQuickFail(t *testing.T) { + var timesCalled int + reqSender := func(delay int) (float64, error) { + timesCalled++ + return 0.5, nil // Return value less than delay + } + match, _, err := checkTimingDependency( + 4, // requestsLimit + 15, // highSleepTimeSeconds + 0.1, // correlationErrorRange + 0.2, // slopeErrorRange + reqSender, + ) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if match { + t.Fatal("Expected no match for non-injectable case") + } + if timesCalled != 1 { + t.Fatalf("Expected quick fail after 1 call, got %d calls", timesCalled) + } +} + +func TestSlowNonInjectableCase(t *testing.T) { + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + var timesCalled int + reqSender := func(delay int) (float64, error) { + timesCalled++ + return 10 + rng.Float64()*0.5, nil + } + match, _, err := checkTimingDependency( + 4, // requestsLimit + 15, // highSleepTimeSeconds + 0.1, // correlationErrorRange + 0.2, // slopeErrorRange + reqSender, + ) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if match { + t.Fatal("Expected no match for slow non-injectable case") + } + if timesCalled > 3 { + t.Fatalf("Expected early termination (≤3 calls), got %d calls", timesCalled) + } +} + +func TestRealWorldNonInjectableCase(t *testing.T) { + var iteration int + counts := []float64{11, 21, 11, 21, 11} + reqSender := func(delay int) (float64, error) { + iteration++ + return counts[iteration-1], nil + } + match, _, err := checkTimingDependency( + 4, // requestsLimit + 15, // highSleepTimeSeconds + 0.1, // correlationErrorRange + 0.2, // slopeErrorRange + reqSender, + ) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if match { + t.Fatal("Expected no match for real-world non-injectable case") + } + if iteration > 4 { + t.Fatalf("Expected ≤4 iterations, got %d", iteration) + } +} + +func TestSmallErrorDependence(t *testing.T) { + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + reqSender := func(delay int) (float64, error) { + return float64(delay) + rng.Float64()*0.5, nil + } + match, reason, err := checkTimingDependency( + 4, // requestsLimit + 15, // highSleepTimeSeconds + 0.1, // correlationErrorRange + 0.2, // slopeErrorRange + reqSender, + ) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if !match { + t.Fatalf("Expected match for small error case. Reason: %s", reason) + } } diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index 2243cab9e9..31f85bf44a 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -320,8 +320,8 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { timeoutVal = 5 } - // Add 3x buffer to the timeout - customTimeout = int(math.Ceil(float64(timeoutVal) * 3)) + // Add 5x buffer to the timeout + customTimeout = int(math.Ceil(float64(timeoutVal) * 5)) } if customTimeout > 0 { connectionConfiguration.Connection.CustomMaxTimeout = time.Duration(customTimeout) * time.Second From ad5ec969bd1763f597a88b2a8a8433aee53aa987 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 13 Dec 2024 11:53:30 +0530 Subject: [PATCH 14/23] feat: re-added slope checks + misc --- pkg/fuzz/analyzers/time/time_delay.go | 10 ++++++++-- pkg/fuzz/analyzers/time/time_delay_test.go | 10 ++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/fuzz/analyzers/time/time_delay.go b/pkg/fuzz/analyzers/time/time_delay.go index 806db146c2..0280d51d1e 100644 --- a/pkg/fuzz/analyzers/time/time_delay.go +++ b/pkg/fuzz/analyzers/time/time_delay.go @@ -199,10 +199,16 @@ func (o *simpleLinearRegression) Predict(x float64) float64 { } func (o *simpleLinearRegression) IsWithinConfidence(correlationErrorRange float64, expectedSlope float64, slopeErrorRange float64) bool { - // For now, just check correlation as originally done: - // return math.Abs(expectedSlope-o.slope) < slopeErrorRange && o.correlation > 1.0 - correlationErrorRange if o.count < 2 { return true } + // Check if slope is within error range of expected slope + // Also consider cases where slope is approximately 2x of expected slope + // as this can happen with time-based responses + slopeDiff := math.Abs(expectedSlope - o.slope) + slope2xDiff := math.Abs(expectedSlope*2 - o.slope) + if slopeDiff > slopeErrorRange && slope2xDiff > slopeErrorRange { + return false + } return o.correlation > 1.0-correlationErrorRange } diff --git a/pkg/fuzz/analyzers/time/time_delay_test.go b/pkg/fuzz/analyzers/time/time_delay_test.go index ef043dabe6..3d6d32503b 100644 --- a/pkg/fuzz/analyzers/time/time_delay_test.go +++ b/pkg/fuzz/analyzers/time/time_delay_test.go @@ -31,7 +31,7 @@ import ( // - TestMinimalData: Tests behavior with minimal data points (2 requests) // - TestLargeNumberOfRequests: Tests stability with many data points (20 requests) // - TestChangingBaseline: Tests detection with shifting baseline mid-test -// - TestHighBaselineLowSlope: Tests detection of subtle correlations (slope=0.5) +// - TestHighBaselineLowSlope: Tests detection of subtle correlations (slope=0.85) // // ZAP Test Cases: // @@ -311,14 +311,12 @@ func TestLargeNumberOfRequests(t *testing.T) { } func TestHighBaselineLowSlope(t *testing.T) { - // baseline=10s, slope=0.5 means each requested second only adds 0.5s to observed time. - // If our thresholds are strict, this should still show correlation, but slope <1. match, reason, err := checkTimingDependency( 10, 5, 0.2, - 0.6, // expecting slope around 0.5, allow range ~0.4 to 0.6 - linearSender(10.0, 0.5, 0.0), + 0.2, // expecting slope around 0.5, allow range ~0.4 to 0.6 + linearSender(10.0, 0.85, 0.0), ) if err != nil { t.Fatalf("Error: %v", err) @@ -367,7 +365,7 @@ func TestAlternatingSequences(t *testing.T) { t.Fatalf("Expected a match but got none. Reason: %s", reason) } // Verify alternating sequence of delays - expectedDelays := []float64{15, 4, 15, 4} + expectedDelays := []float64{15, 3, 15, 3} if !reflect.DeepEqual(generatedDelays, expectedDelays) { t.Fatalf("Expected delays %v but got %v", expectedDelays, generatedDelays) } From 70bd93ab73b0911afabc952dc72cbb8a513a7c68 Mon Sep 17 00:00:00 2001 From: Ice3man Date: Fri, 13 Dec 2024 13:19:52 +0530 Subject: [PATCH 15/23] feat: added baseline measurements for time based checks --- pkg/fuzz/analyzers/time/analyzer.go | 29 ++++++++++++- pkg/fuzz/analyzers/time/time_delay.go | 15 +++++-- pkg/fuzz/analyzers/time/time_delay_test.go | 50 ++++++++++++++++++---- 3 files changed, 82 insertions(+), 12 deletions(-) diff --git a/pkg/fuzz/analyzers/time/analyzer.go b/pkg/fuzz/analyzers/time/analyzer.go index fa6c0a66fe..cdf40e6bf6 100644 --- a/pkg/fuzz/analyzers/time/analyzer.go +++ b/pkg/fuzz/analyzers/time/analyzer.go @@ -15,7 +15,8 @@ import ( ) // Analyzer is a time delay analyzer for the fuzzer -type Analyzer struct{} +type Analyzer struct { +} const ( DefaultSleepDuration = int(7) @@ -131,11 +132,18 @@ func (a *Analyzer) Analyze(options *analyzers.Options) (bool, string, error) { return timeTaken, nil } + // Check the baseline delay of the request by doing two requests + baselineDelay, err := getBaselineDelay(reqSender) + if err != nil { + return false, "", err + } + matched, matchReason, err := checkTimingDependency( requestsLimit, sleepDuration, timeCorrelationErrorRange, timeSlopeErrorRange, + baselineDelay, reqSender, ) if err != nil { @@ -147,6 +155,25 @@ func (a *Analyzer) Analyze(options *analyzers.Options) (bool, string, error) { return false, "", nil } +func getBaselineDelay(reqSender timeDelayRequestSender) (float64, error) { + var delays []float64 + // Use zero or a very small delay to measure baseline + for i := 0; i < 3; i++ { + delay, err := reqSender(0) + if err != nil { + return 0, errors.Wrap(err, "could not get baseline delay") + } + delays = append(delays, delay) + } + + var total float64 + for _, d := range delays { + total += d + } + avg := total / float64(len(delays)) + return avg, nil +} + // doHTTPRequestWithTimeTracing does a http request with time tracing func doHTTPRequestWithTimeTracing(req *retryablehttp.Request, httpclient *retryablehttp.Client) (float64, error) { var serverTime time.Duration diff --git a/pkg/fuzz/analyzers/time/time_delay.go b/pkg/fuzz/analyzers/time/time_delay.go index 0280d51d1e..3dd8590409 100644 --- a/pkg/fuzz/analyzers/time/time_delay.go +++ b/pkg/fuzz/analyzers/time/time_delay.go @@ -47,6 +47,7 @@ func checkTimingDependency( highSleepTimeSeconds int, correlationErrorRange float64, slopeErrorRange float64, + baselineDelay float64, requestSender timeDelayRequestSender, ) (bool, string, error) { if requestsLimit < 2 { @@ -62,25 +63,32 @@ func checkTimingDependency( break } - isCorrelationPossible, delayRecieved, err := sendRequestAndTestConfidence(regression, highSleepTimeSeconds, requestSender) + isCorrelationPossible, delayRecieved, err := sendRequestAndTestConfidence(regression, highSleepTimeSeconds, requestSender, baselineDelay) if err != nil { return false, "", err } if !isCorrelationPossible { return false, "", nil } + // Check the delay is greater than baseline by seconds requested + if delayRecieved < baselineDelay+float64(highSleepTimeSeconds)*0.8 { + return false, "", nil + } requestsSent = append(requestsSent, requstsSentMetadata{ delay: highSleepTimeSeconds, delayReceived: delayRecieved, }) - isCorrelationPossibleSecond, delayRecievedSecond, err := sendRequestAndTestConfidence(regression, int(DefaultLowSleepTimeSeconds), requestSender) + isCorrelationPossibleSecond, delayRecievedSecond, err := sendRequestAndTestConfidence(regression, int(DefaultLowSleepTimeSeconds), requestSender, baselineDelay) if err != nil { return false, "", err } if !isCorrelationPossibleSecond { return false, "", nil } + if delayRecievedSecond < baselineDelay+float64(DefaultLowSleepTimeSeconds)*0.8 { + return false, "", nil + } requestsLeft = requestsLeft - 2 requestsSent = append(requestsSent, requstsSentMetadata{ @@ -111,6 +119,7 @@ func sendRequestAndTestConfidence( regression *simpleLinearRegression, delay int, requestSender timeDelayRequestSender, + baselineDelay float64, ) (bool, float64, error) { delayReceived, err := requestSender(delay) if err != nil { @@ -121,7 +130,7 @@ func sendRequestAndTestConfidence( return false, 0, nil } - regression.AddPoint(float64(delay), delayReceived) + regression.AddPoint(float64(delay), delayReceived-baselineDelay) if !regression.IsWithinConfidence(0.3, 1.0, 0.5) { return false, delayReceived, nil diff --git a/pkg/fuzz/analyzers/time/time_delay_test.go b/pkg/fuzz/analyzers/time/time_delay_test.go index 3d6d32503b..91b2ba657a 100644 --- a/pkg/fuzz/analyzers/time/time_delay_test.go +++ b/pkg/fuzz/analyzers/time/time_delay_test.go @@ -95,6 +95,7 @@ func TestPerfectLinear(t *testing.T) { highSleepTimeSeconds := 5 corrErrRange := 0.1 slopeErrRange := 0.2 + baseline := 5.0 sender := perfectLinearSender(5.0) // baseline 5s, observed = 5s + requested_delay match, reason, err := checkTimingDependency( @@ -102,6 +103,7 @@ func TestPerfectLinear(t *testing.T) { highSleepTimeSeconds, corrErrRange, slopeErrRange, + baseline, sender, ) if err != nil { @@ -118,6 +120,7 @@ func TestNoCorrelation(t *testing.T) { highSleepTimeSeconds := 5 corrErrRange := 0.1 slopeErrRange := 0.5 + baseline := 8.0 sender := noCorrelationSender(8.0, 0.1) match, reason, err := checkTimingDependency( @@ -125,6 +128,7 @@ func TestNoCorrelation(t *testing.T) { highSleepTimeSeconds, corrErrRange, slopeErrRange, + baseline, sender, ) if err != nil { @@ -142,6 +146,7 @@ func TestNoisyLinear(t *testing.T) { highSleepTimeSeconds := 5 corrErrRange := 0.2 // allow some lower correlation due to noise slopeErrRange := 0.5 // slope may deviate slightly + baseline := 2.0 sender := noisyLinearSender(2.0) // baseline 2s, observed ~ 2s + requested_delay ±0.2 match, reason, err := checkTimingDependency( @@ -149,6 +154,7 @@ func TestNoisyLinear(t *testing.T) { highSleepTimeSeconds, corrErrRange, slopeErrRange, + baseline, sender, ) if err != nil { @@ -168,6 +174,7 @@ func TestMinimalData(t *testing.T) { highSleepTimeSeconds := 5 corrErrRange := 0.3 slopeErrRange := 0.5 + baseline := 5.0 // Perfect linear sender again sender := perfectLinearSender(5.0) @@ -176,6 +183,7 @@ func TestMinimalData(t *testing.T) { highSleepTimeSeconds, corrErrRange, slopeErrRange, + baseline, sender, ) if err != nil { @@ -210,12 +218,14 @@ func negativeSlopeSender(baseline float64) func(int) (float64, error) { } func TestPerfectLinearSlopeOne_NoNoise(t *testing.T) { + baseline := 2.0 match, reason, err := checkTimingDependency( 10, // requestsLimit 5, // highSleepTimeSeconds 0.1, // correlationErrorRange 0.2, // slopeErrorRange (allowing slope between 0.8 and 1.2) - linearSender(2.0, 1.0, 0.0), + baseline, + linearSender(baseline, 1.0, 0.0), ) if err != nil { t.Fatalf("Unexpected error: %v", err) @@ -226,13 +236,15 @@ func TestPerfectLinearSlopeOne_NoNoise(t *testing.T) { } func TestPerfectLinearSlopeTwo_NoNoise(t *testing.T) { + baseline := 2.0 // slope=2 means observed = baseline + 2*requested_delay match, reason, err := checkTimingDependency( 10, 5, 0.1, // correlation must still be good 1.5, // allow slope in range (0.5 to 2.5), we should be close to 2.0 anyway - linearSender(1.0, 2.0, 0.0), + baseline, + linearSender(baseline, 2.0, 0.0), ) if err != nil { t.Fatalf("Error: %v", err) @@ -243,13 +255,15 @@ func TestPerfectLinearSlopeTwo_NoNoise(t *testing.T) { } func TestLinearWithNoise(t *testing.T) { + baseline := 5.0 // slope=1 but with noise ±0.2 seconds match, reason, err := checkTimingDependency( 12, 5, 0.2, // correlationErrorRange relaxed to account for noise 0.5, // slopeErrorRange also relaxed - linearSender(5.0, 1.0, 0.2), + baseline, + linearSender(baseline, 1.0, 0.2), ) if err != nil { t.Fatalf("Error: %v", err) @@ -260,13 +274,15 @@ func TestLinearWithNoise(t *testing.T) { } func TestNoCorrelationHighBaseline(t *testing.T) { + baseline := 15.0 // baseline ~15s, requested delays won't matter match, reason, err := checkTimingDependency( 10, 5, 0.1, // correlation should be near zero, so no match expected 0.5, - noCorrelationSender(15.0, 0.1), + baseline, + noCorrelationSender(baseline, 0.1), ) if err != nil { t.Fatalf("Error: %v", err) @@ -277,13 +293,15 @@ func TestNoCorrelationHighBaseline(t *testing.T) { } func TestNegativeSlopeScenario(t *testing.T) { + baseline := 10.0 // Increasing delay decreases observed time match, reason, err := checkTimingDependency( 10, 5, 0.2, 0.5, - negativeSlopeSender(10.0), + baseline, + negativeSlopeSender(baseline), ) if err != nil { t.Fatalf("Error: %v", err) @@ -294,13 +312,15 @@ func TestNegativeSlopeScenario(t *testing.T) { } func TestLargeNumberOfRequests(t *testing.T) { + baseline := 1.0 // 20 requests, slope=1.0, no noise. Should be very stable and produce a very high correlation. match, reason, err := checkTimingDependency( 20, 5, 0.05, // very strict correlation requirement 0.1, // very strict slope range - linearSender(1.0, 1.0, 0.0), + baseline, + linearSender(baseline, 1.0, 0.0), ) if err != nil { t.Fatalf("Error: %v", err) @@ -311,12 +331,14 @@ func TestLargeNumberOfRequests(t *testing.T) { } func TestHighBaselineLowSlope(t *testing.T) { + baseline := 15.0 match, reason, err := checkTimingDependency( 10, 5, 0.2, 0.2, // expecting slope around 0.5, allow range ~0.4 to 0.6 - linearSender(10.0, 0.85, 0.0), + baseline, + linearSender(baseline, 0.85, 0.0), ) if err != nil { t.Fatalf("Error: %v", err) @@ -327,6 +349,7 @@ func TestHighBaselineLowSlope(t *testing.T) { } func TestHighNoiseConcealsSlope(t *testing.T) { + baseline := 5.0 // slope=1, but noise=5 seconds is huge and might conceal the correlation. // With large noise, the test may fail to detect correlation. match, reason, err := checkTimingDependency( @@ -334,7 +357,8 @@ func TestHighNoiseConcealsSlope(t *testing.T) { 5, 0.1, // still strict 0.2, // still strict - linearSender(5.0, 1.0, 5.0), + baseline, + linearSender(baseline, 1.0, 5.0), ) if err != nil { t.Fatalf("Error: %v", err) @@ -346,6 +370,7 @@ func TestHighNoiseConcealsSlope(t *testing.T) { } func TestAlternatingSequences(t *testing.T) { + baseline := 0.0 var generatedDelays []float64 reqSender := func(delay int) (float64, error) { generatedDelays = append(generatedDelays, float64(delay)) @@ -356,6 +381,7 @@ func TestAlternatingSequences(t *testing.T) { 15, // highSleepTimeSeconds 0.1, // correlationErrorRange 0.2, // slopeErrorRange + baseline, reqSender, ) if err != nil { @@ -372,6 +398,7 @@ func TestAlternatingSequences(t *testing.T) { } func TestNonInjectableQuickFail(t *testing.T) { + baseline := 0.5 var timesCalled int reqSender := func(delay int) (float64, error) { timesCalled++ @@ -382,6 +409,7 @@ func TestNonInjectableQuickFail(t *testing.T) { 15, // highSleepTimeSeconds 0.1, // correlationErrorRange 0.2, // slopeErrorRange + baseline, reqSender, ) if err != nil { @@ -396,6 +424,7 @@ func TestNonInjectableQuickFail(t *testing.T) { } func TestSlowNonInjectableCase(t *testing.T) { + baseline := 10.0 rng := rand.New(rand.NewSource(time.Now().UnixNano())) var timesCalled int reqSender := func(delay int) (float64, error) { @@ -407,6 +436,7 @@ func TestSlowNonInjectableCase(t *testing.T) { 15, // highSleepTimeSeconds 0.1, // correlationErrorRange 0.2, // slopeErrorRange + baseline, reqSender, ) if err != nil { @@ -421,6 +451,7 @@ func TestSlowNonInjectableCase(t *testing.T) { } func TestRealWorldNonInjectableCase(t *testing.T) { + baseline := 0.0 var iteration int counts := []float64{11, 21, 11, 21, 11} reqSender := func(delay int) (float64, error) { @@ -432,6 +463,7 @@ func TestRealWorldNonInjectableCase(t *testing.T) { 15, // highSleepTimeSeconds 0.1, // correlationErrorRange 0.2, // slopeErrorRange + baseline, reqSender, ) if err != nil { @@ -446,6 +478,7 @@ func TestRealWorldNonInjectableCase(t *testing.T) { } func TestSmallErrorDependence(t *testing.T) { + baseline := 0.0 rng := rand.New(rand.NewSource(time.Now().UnixNano())) reqSender := func(delay int) (float64, error) { return float64(delay) + rng.Float64()*0.5, nil @@ -455,6 +488,7 @@ func TestSmallErrorDependence(t *testing.T) { 15, // highSleepTimeSeconds 0.1, // correlationErrorRange 0.2, // slopeErrorRange + baseline, reqSender, ) if err != nil { From 98646e6673867479d6759be43f2d2fcf883126ab Mon Sep 17 00:00:00 2001 From: Dwi Siswanto <25837540+dwisiswant0@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:08:00 +0700 Subject: [PATCH 16/23] chore(server): fix typos Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- internal/server/nuclei_sdk.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/nuclei_sdk.go b/internal/server/nuclei_sdk.go index 9ff531b163..d3cedc5748 100644 --- a/internal/server/nuclei_sdk.go +++ b/internal/server/nuclei_sdk.go @@ -109,7 +109,7 @@ func newNucleiExecutor(opts *NucleiExecutorOptions) (*nucleiExecutor, error) { workflowLoader, err := parsers.NewLoader(&executorOpts) if err != nil { - return nil, errors.Wrap(err, "Could not create loadeopts.") + return nil, errors.Wrap(err, "Could not create loader options.") } executorOpts.WorkflowLoader = workflowLoader From f31e963560029e251d44009ee2843df9982843d5 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto <25837540+dwisiswant0@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:21:44 +0700 Subject: [PATCH 17/23] fix(templates): potential DOM XSS Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- internal/server/templates/index.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/server/templates/index.html b/internal/server/templates/index.html index 52b07bf17d..720c32718e 100644 --- a/internal/server/templates/index.html +++ b/internal/server/templates/index.html @@ -333,7 +333,9 @@ }); function toggleJSON() { - window.location.href = window.location.href + '.json'; + const url = new URL(window.location.href); + url.pathname = url.pathname + '.json'; + window.location.href = url.toString(); } From 1d2a1dcd34f5f19db37a5403ac09e99e9fd4e450 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto <25837540+dwisiswant0@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:30:45 +0700 Subject: [PATCH 18/23] fix(authx): potential NIL deref Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/authprovider/authx/strategy.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/authprovider/authx/strategy.go b/pkg/authprovider/authx/strategy.go index 775862954c..54ff8e81c4 100644 --- a/pkg/authprovider/authx/strategy.go +++ b/pkg/authprovider/authx/strategy.go @@ -24,8 +24,14 @@ type DynamicAuthStrategy struct { // Apply applies the strategy to the request func (d *DynamicAuthStrategy) Apply(req *http.Request) { - strategy := d.Dynamic.GetStrategies() - for _, s := range strategy { + strategies := d.Dynamic.GetStrategies() + if strategies == nil { + return + } + for _, s := range strategies { + if s == nil { + continue + } s.Apply(req) } } From 9f4b89ac8434c376463bdbc0145d475f03ebc97f Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 16 Dec 2024 21:10:01 +0530 Subject: [PATCH 19/23] feat: misc review changes --- cmd/nuclei/main.go | 2 +- go.sum | 2 -- internal/runner/options.go | 13 ++++++++++ internal/runner/runner.go | 10 ++++---- internal/server/nuclei_sdk.go | 7 ++++-- internal/server/requests_worker.go | 4 ++-- internal/server/scope/scope.go | 2 +- internal/server/scope/scope_test.go | 4 ++-- internal/server/server.go | 33 +++++++++++++------------- internal/server/templates/index.html | 2 +- pkg/core/workpool.go | 6 +++++ pkg/fuzz/analyzers/time/time_delay.go | 13 ++++++---- pkg/input/provider/http/multiformat.go | 7 ++++++ pkg/output/output.go | 10 ++++---- 14 files changed, 75 insertions(+), 40 deletions(-) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 64ca1c3241..9a6ffa8cb7 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -368,7 +368,7 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"), flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"), flagSet.BoolVarP(&options.DASTServer, "dast-server", "dts", false, "enable dast server mode (live fuzzing)"), - flagSet.BoolVarP(&options.DASTReport, "dast-report", "drg", false, "write dast scan report to file"), + flagSet.BoolVarP(&options.DASTReport, "dast-report", "dtr", false, "write dast scan report to file"), flagSet.StringVarP(&options.DASTServerToken, "dast-server-token", "dtst", "", "dast server token (optional)"), flagSet.StringVarP(&options.DASTServerAddress, "dast-server-address", "dtsa", "localhost:9055", "dast server address"), flagSet.BoolVarP(&options.DisplayFuzzPoints, "display-fuzz-points", "dfp", false, "display fuzz points in the output for debugging"), diff --git a/go.sum b/go.sum index e93737c69f..8117f33eeb 100644 --- a/go.sum +++ b/go.sum @@ -864,8 +864,6 @@ github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB7 github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0= github.com/projectdiscovery/dsl v0.3.3 h1:4Ij5S86cHlb6xFrS7+5zAiJPeBt5h970XBTHqeTkpyU= github.com/projectdiscovery/dsl v0.3.3/go.mod h1:DAjSeaogLM9f0Ves2zDc/vbJrfcv+kEmS51p0dLLaPI= -github.com/projectdiscovery/fastdialer v0.2.11 h1:DTx2vJ3tytv34wDe+Oh72L7v9pZWhzNGFJgwheN0n1Q= -github.com/projectdiscovery/fastdialer v0.2.11/go.mod h1:jjDMLl+hnKoSSP82eWPxn8U+KivlWqf/o3pSz4n4dik= github.com/projectdiscovery/fastdialer v0.2.13 h1:5XzSv0hwITzRAMwyoJ9GFZSTVtaI4jmwER968TbDLbI= github.com/projectdiscovery/fastdialer v0.2.13/go.mod h1:T1EaYHbWmCnVHSYz12nAjnHMNFEfGMLLw37cb0k1X3A= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= diff --git a/internal/runner/options.go b/internal/runner/options.go index e36c248a64..65cd4ae790 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -171,6 +171,11 @@ func ValidateOptions(options *types.Options) error { if options.Validate { validateTemplatePaths(config.DefaultConfig.TemplatesDirectory, options.Templates, options.Workflows) } + if options.DAST { + if err := validateDASTOptions(options); err != nil { + return err + } + } // Verify if any of the client certificate options were set since it requires all three to work properly if options.HasClientCertificates() { @@ -274,6 +279,14 @@ func validateMissingGitLabOptions(options *types.Options) []string { return missing } +func validateDASTOptions(options *types.Options) error { + // Ensure the DAST server token meets minimum length requirement + if len(options.DASTServerToken) > 0 && len(options.DASTServerToken) < 16 { + return fmt.Errorf("DAST server token must be at least 16 characters long") + } + return nil +} + func createReportingOptions(options *types.Options) (*reporting.Options, error) { var reportingOptions = &reporting.Options{} if options.ReportingConfig != "" { diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 68dc2710f4..e56e1b5575 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -305,7 +305,7 @@ func New(options *types.Options) (*Runner, error) { return nil, errors.Wrap(err, "could not create output file") } if runner.fuzzStats != nil { - outputWriter.RequestHook = func(request *output.JSONLogRequest) { + outputWriter.JSONLogRequestHook = func(request *output.JSONLogRequest) { if request.Error == "none" || request.Error == "" { return } @@ -687,9 +687,11 @@ func (r *Runner) RunEnumeration() error { }, "") if r.dastServer != nil { - if err := r.dastServer.Start(); err != nil { - r.dastServer.Start() - } + go func() { + if err := r.dastServer.Start(); err != nil { + gologger.Error().Msgf("could not start dast server: %v", err) + } + }() } enumeration := false diff --git a/internal/server/nuclei_sdk.go b/internal/server/nuclei_sdk.go index d3cedc5748..aad3377437 100644 --- a/internal/server/nuclei_sdk.go +++ b/internal/server/nuclei_sdk.go @@ -121,7 +121,7 @@ func newNucleiExecutor(opts *NucleiExecutorOptions) (*nucleiExecutor, error) { } store, err := loader.New(loaderConfig) if err != nil { - return nil, errors.Wrap(err, "Could not create loadeopts.") + return nil, errors.Wrap(err, "Could not create loader options.") } store.Load() @@ -143,7 +143,7 @@ type proxifyRequest struct { } `json:"request"` } -func (n *nucleiExecutor) ExecuteScan(target PostReuestsHandlerRequest) error { +func (n *nucleiExecutor) ExecuteScan(target PostRequestsHandlerRequest) error { finalTemplates := []*templates.Template{} finalTemplates = append(finalTemplates, n.store.Templates()...) finalTemplates = append(finalTemplates, n.store.Workflows()...) @@ -178,6 +178,9 @@ func (n *nucleiExecutor) ExecuteScan(target PostReuestsHandlerRequest) error { if err != nil { return errors.Wrap(err, "could not create input provider") } + + // We don't care about the result as its a boolean + // stating whether we got matches or not _ = n.engine.ExecuteScanWithOpts(context.Background(), finalTemplates, inputProvider, true) return nil } diff --git a/internal/server/requests_worker.go b/internal/server/requests_worker.go index 3c976b2e82..e811a005ac 100644 --- a/internal/server/requests_worker.go +++ b/internal/server/requests_worker.go @@ -8,7 +8,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/input/types" ) -func (s *DASTServer) consumeTaskRequest(req PostReuestsHandlerRequest) { +func (s *DASTServer) consumeTaskRequest(req PostRequestsHandlerRequest) { defer s.endpointsInQueue.Add(-1) parsedReq, err := types.ParseRawRequestWithURL(req.RawHTTP, req.URL) @@ -29,7 +29,7 @@ func (s *DASTServer) consumeTaskRequest(req PostReuestsHandlerRequest) { return } - inScope, err := s.scopeManager.Validate(parsedReq.URL.URL, "") + inScope, err := s.scopeManager.Validate(parsedReq.URL.URL) if err != nil { gologger.Warning().Msgf("Could not validate scope: %s\n", err) return diff --git a/internal/server/scope/scope.go b/internal/server/scope/scope.go index 63dd01fd6c..31c74a76de 100644 --- a/internal/server/scope/scope.go +++ b/internal/server/scope/scope.go @@ -39,7 +39,7 @@ func NewManager(inScope, outOfScope []string) (*Manager, error) { } // Validate returns true if the URL matches scope rules -func (m *Manager) Validate(URL *url.URL, rootHostname string) (bool, error) { +func (m *Manager) Validate(URL *url.URL) (bool, error) { if m.noScope { return true, nil } diff --git a/internal/server/scope/scope_test.go b/internal/server/scope/scope_test.go index a612cf4f3d..d2256363db 100644 --- a/internal/server/scope/scope_test.go +++ b/internal/server/scope/scope_test.go @@ -13,12 +13,12 @@ func TestManagerValidate(t *testing.T) { require.NoError(t, err, "could not create scope manager") parsed, _ := urlutil.Parse("https://test.com/index.php/example") - validated, err := manager.Validate(parsed.URL, "test.com") + validated, err := manager.Validate(parsed.URL) require.NoError(t, err, "could not validate url") require.True(t, validated, "could not get correct in-scope validation") parsed, _ = urlutil.Parse("https://test.com/logout.php") - validated, err = manager.Validate(parsed.URL, "another.com") + validated, err = manager.Validate(parsed.URL) require.NoError(t, err, "could not validate url") require.False(t, validated, "could not get correct out-scope validation") }) diff --git a/internal/server/server.go b/internal/server/server.go index 5fdbf9d6d1..987967a4d4 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -5,6 +5,7 @@ import ( "fmt" "html/template" "net/http" + "net/url" "strings" "sync/atomic" "time" @@ -103,8 +104,8 @@ func New(options *Options) (*DASTServer, error) { builder.WriteString(" (with token)") } gologger.Info().Msgf("%s", builder.String()) - gologger.Info().Msgf("Connection URL: %s", server.buildConnectionURL()) - gologger.Info().Msgf("Stats UI URL: %s", server.buildStatsURL()) + gologger.Info().Msgf("Connection URL: %s", server.buildURL("/requests")) + gologger.Info().Msgf("Stats UI URL: %s", server.buildURL("/stats")) return server, nil } @@ -118,7 +119,7 @@ func NewStatsServer(fuzzStatsDB *stats.Tracker) (*DASTServer, error) { }, } server.setupHandlers(true) - gologger.Info().Msgf("Stats UI URL: %s", server.buildStatsURL()) + gologger.Info().Msgf("Stats UI URL: %s", server.buildURL("/stats")) return server, nil } @@ -129,20 +130,20 @@ func (s *DASTServer) Close() { s.tasksPool.StopAndWaitFor(1 * time.Minute) } -func (s *DASTServer) buildConnectionURL() string { - url := fmt.Sprintf("http://%s/requests", s.options.Address) +func (s *DASTServer) buildURL(endpoint string) string { + values := make(url.Values) if s.options.Token != "" { - url += "?token=" + s.options.Token + values.Set("token", s.options.Token) } - return url -} -func (s *DASTServer) buildStatsURL() string { - url := fmt.Sprintf("http://%s/stats", s.options.Address) - if s.options.Token != "" { - url += "?token=" + s.options.Token + // Use url.URL struct to safely construct the URL + u := &url.URL{ + Scheme: "http", + Host: s.options.Address, + Path: endpoint, + RawQuery: values.Encode(), } - return url + return u.String() } func (s *DASTServer) setupHandlers(onlyStats bool) { @@ -186,13 +187,13 @@ func (s *DASTServer) Start() error { } // PostReuestsHandlerRequest is the request body for the /requests POST handler. -type PostReuestsHandlerRequest struct { +type PostRequestsHandlerRequest struct { RawHTTP string `json:"raw_http"` URL string `json:"url"` } func (s *DASTServer) handleRequest(c echo.Context) error { - var req PostReuestsHandlerRequest + var req PostRequestsHandlerRequest if err := c.Bind(&req); err != nil { fmt.Printf("Error binding request: %s\n", err) return err @@ -246,7 +247,7 @@ func (s *DASTServer) getStats() (StatsResponse, error) { DASTServerInfo: DASTServerInfo{ NucleiVersion: config.Version, NucleiTemplateVersion: cfg.TemplateVersion, - NucleiDastServerAPI: s.buildConnectionURL(), + NucleiDastServerAPI: s.buildURL("/requests"), ServerAuthEnabled: s.options.Token != "", }, DASTScanStartTime: s.startTime, diff --git a/internal/server/templates/index.html b/internal/server/templates/index.html index 720c32718e..fa3488e9b4 100644 --- a/internal/server/templates/index.html +++ b/internal/server/templates/index.html @@ -3,7 +3,7 @@ DAST Scan Report - +