diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index a942c5b118..2763c5c628 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -472,18 +472,16 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa const drainReqSize = int64(8 * 1024) // executeRequest executes the actual generated request and returns error if occurred -func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, callback protocols.OutputEventCallback, requestCount int) (err error) { - var event *output.InternalWrappedEvent - defer func() { - if event != nil { - if event.InternalEvent == nil { - event.InternalEvent = make(map[string]interface{}) - event.InternalEvent["template-id"] = request.options.TemplateID - } - // add the request URL pattern to the event - event.InternalEvent[ReqURLPatternKey] = generatedRequest.requestURLPattern - } - }() +func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) { + + // wrap one more callback for validation and fixing event + callback := func(event *output.InternalWrappedEvent) { + // validateNFixEvent performs necessary validation on generated event + // and attempts to fix it , this includes things like making sure + // `template-id` is set , `request-url-pattern` is set etc + request.validateNFixEvent(input, generatedRequest, err, event) + processEvent(event) + } request.setCustomHeaders(generatedRequest) @@ -692,7 +690,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if len(generatedRequest.interactshURLs) > 0 { // according to logic we only need to trigger a callback if interactsh was used // and request failed in hope that later on oast interaction will be received - event = &output.InternalWrappedEvent{} + event := &output.InternalWrappedEvent{} if request.CompiledOperators != nil && request.CompiledOperators.HasDSL() { event.InternalEvent = outputEvent } @@ -807,7 +805,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ // prune signature internal values if any request.pruneSignatureInternalValues(generatedRequest.meta) - event = eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) if hasInteractMatchers { @@ -843,6 +841,37 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ return errx } +// validateNFixEvent validates and fixes the event +// it adds any missing template-id and request-url-pattern +func (request *Request) validateNFixEvent(input *contextargs.Context, gr *generatedRequest, err error, event *output.InternalWrappedEvent) { + if event != nil { + if event.InternalEvent == nil { + event.InternalEvent = make(map[string]interface{}) + event.InternalEvent["template-id"] = request.options.TemplateID + } + // add the request URL pattern to the event + event.InternalEvent[ReqURLPatternKey] = gr.requestURLPattern + if event.InternalEvent["host"] == nil { + event.InternalEvent["host"] = input.MetaInput.Input + } + if event.InternalEvent["template-id"] == nil { + event.InternalEvent["template-id"] = request.options.TemplateID + } + if event.InternalEvent["type"] == nil { + event.InternalEvent["type"] = request.Type().String() + } + if event.InternalEvent["template-path"] == nil { + event.InternalEvent["template-path"] = request.options.TemplatePath + } + if event.InternalEvent["template-info"] == nil { + event.InternalEvent["template-info"] = request.options.TemplateInfo + } + if err != nil { + event.InternalEvent["error"] = err.Error() + } + } +} + // handleSignature of the http request func (request *Request) handleSignature(generatedRequest *generatedRequest) error { switch request.Signature.Value { diff --git a/pkg/reporting/exporters/sarif/sarif.go b/pkg/reporting/exporters/sarif/sarif.go index d2fe63bd4b..67a438d80c 100644 --- a/pkg/reporting/exporters/sarif/sarif.go +++ b/pkg/reporting/exporters/sarif/sarif.go @@ -53,8 +53,8 @@ func (exporter *Exporter) addToolDetails() { FullDescription: &sarif.MultiformatMessageString{ Text: "Fast and customizable vulnerability scanner based on simple YAML based DSL", }, - FullName: "Nuclei v" + config.Version, - SemanticVersion: "v" + config.Version, + FullName: "Nuclei " + config.Version, + SemanticVersion: config.Version, DownloadURI: "https://github.com/projectdiscovery/nuclei/releases", Rules: exporter.rules, } diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go index e404a3d28c..b1e0b56e2d 100644 --- a/pkg/templates/cluster.go +++ b/pkg/templates/cluster.go @@ -59,6 +59,14 @@ func Cluster(list []*Template) [][]*Template { final = append(final, []*Template{template}) continue } + + // it is not possible to cluster flow and multiprotocol due to dependent execution + if template.Flow != "" || template.Options.IsMultiProtocol { + _ = skip.Set(key, struct{}{}) + final = append(final, []*Template{template}) + continue + } + _ = skip.Set(key, struct{}{}) var templateType types.ProtocolType @@ -81,6 +89,13 @@ func Cluster(list []*Template) [][]*Template { continue } + // it is not possible to cluster flow and multiprotocol due to dependent execution + if other.Flow != "" || other.Options.IsMultiProtocol { + _ = skip.Set(otherKey, struct{}{}) + final = append(final, []*Template{other}) + continue + } + switch templateType { case types.DNSProtocol: if len(other.RequestsDNS) != 1 { diff --git a/pkg/templates/cluster_test.go b/pkg/templates/cluster_test.go index bfd10fd427..c8e68b3e07 100644 --- a/pkg/templates/cluster_test.go +++ b/pkg/templates/cluster_test.go @@ -3,15 +3,25 @@ package templates import ( "testing" + "github.com/projectdiscovery/nuclei/v3/pkg/model" + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http" + "github.com/projectdiscovery/nuclei/v3/pkg/testutils" "github.com/stretchr/testify/require" ) func TestClusterTemplates(t *testing.T) { + // state of whether template is flow or multiprotocol is stored in executerOptions i.e why we need to pass it + execOptions := testutils.NewMockExecuterOptions(testutils.DefaultOptions, &testutils.TemplateInfo{ + ID: "templateID", + Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, + }) t.Run("http-cluster-get", func(t *testing.T) { tp1 := &Template{Path: "first.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}} tp2 := &Template{Path: "second.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}} + tp1.Options = execOptions + tp2.Options = execOptions tpls := []*Template{tp1, tp2} // cluster 0 expected := []*Template{tp1, tp2} @@ -21,6 +31,8 @@ func TestClusterTemplates(t *testing.T) { t.Run("no-http-cluster", func(t *testing.T) { tp1 := &Template{Path: "first.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}/random"}}}} tp2 := &Template{Path: "second.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}/another"}}}} + tp1.Options = execOptions + tp2.Options = execOptions tpls := []*Template{tp1, tp2} expected := [][]*Template{{tp1}, {tp2}} got := Cluster(tpls) @@ -29,6 +41,8 @@ func TestClusterTemplates(t *testing.T) { t.Run("dns-cluster", func(t *testing.T) { tp1 := &Template{Path: "first.yaml", RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}} tp2 := &Template{Path: "second.yaml", RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}} + tp1.Options = execOptions + tp2.Options = execOptions tpls := []*Template{tp1, tp2} // cluster 0 expected := []*Template{tp1, tp2} diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index d5c2aaeede..e434f2173c 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -92,7 +92,11 @@ func (e *TemplateExecuter) Requests() int { // Execute executes the protocol group and returns true or false if results were found. func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { - results := &atomic.Bool{} + // executed contains status of execution if it was successfully executed or not + // doesn't matter if it was matched or not + executed := &atomic.Bool{} + // matched in this case means something was exported / written to output + matched := &atomic.Bool{} defer func() { // it is essential to remove template context of `Scan i.e template x input pair` // since it is of no use after scan is completed (regardless of success or failure) @@ -101,11 +105,11 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { var lastMatcherEvent *output.InternalWrappedEvent writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { - if !results.Load() && matcherStatus { + if !matched.Load() && matcherStatus { if err := e.options.Output.WriteFailure(event); err != nil { gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) } - results.CompareAndSwap(false, true) + executed.CompareAndSwap(false, true) } } @@ -133,11 +137,11 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { // If no results were found, and also interactsh is not being used // in that case we can skip it, otherwise we've to show failure in // case of matcher-status flag. - if !event.HasOperatorResult() && !event.UsesInteractsh { + if !event.HasOperatorResult() && event.InternalEvent != nil { lastMatcherEvent = event } else { if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { - results.CompareAndSwap(false, true) + matched.Store(true) } else { lastMatcherEvent = event } @@ -152,7 +156,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { // so in compile step earlier we compile it to validate javascript syntax and other things // and while executing we create new instance of flow executor everytime if e.options.Flow != "" { - flowexec, err := flow.NewFlowExecutor(e.requests, ctx, e.options, results, e.program) + flowexec, err := flow.NewFlowExecutor(e.requests, ctx, e.options, executed, e.program) if err != nil { ctx.LogError(err) return false, fmt.Errorf("could not create flow executor: %s", err) @@ -169,7 +173,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { if lastMatcherEvent != nil { writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) } - return results.Load(), errx + return executed.Load() || matched.Load(), errx } // ExecuteWithResults executes the protocol requests and returns results instead of writing them.