diff --git a/.gitignore b/.gitignore index 57d2daa008..65815d8db6 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,6 @@ pkg/protocols/headless/engine/.cache **/*-cache /fuzzplayground integration_tests/fuzzplayground +/dsl.md + diff --git a/Makefile b/Makefile index f6950b31a7..dadbe6205d 100644 --- a/Makefile +++ b/Makefile @@ -47,4 +47,7 @@ fuzzplayground: memogen: $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "memogen" cmd/memogen/memogen.go ./memogen -src pkg/js/libs -tpl cmd/memogen/function.tpl - +dsl-docs: + rm -f dsl.md scrapefuncs 2>/dev/null + $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "scrapefuncs" pkg/js/devtools/scrapefuncs/main.go + ./scrapefuncs -out dsl.md diff --git a/README.md b/README.md index e231221c2e..07bce472b5 100644 --- a/README.md +++ b/README.md @@ -221,7 +221,8 @@ INTERACTSH: FUZZING: -ft, -fuzzing-type string overrides fuzzing type set in template (replace, prefix, postfix, infix) -fm, -fuzzing-mode string overrides fuzzing mode set in template (multiple, single) - -fuzz enable loading fuzzing templates + -fuzz enable loading fuzzing templates (Deprecated: use -dast instead) + -dast only run DAST templates UNCOVER: -uc, -uncover enable uncover engine diff --git a/README_CN.md b/README_CN.md index c2d31c1a8f..095f9c109f 100644 --- a/README_CN.md +++ b/README_CN.md @@ -57,7 +57,7 @@ Nuclei使用零误报的定制模板向目标发送请求，同时可以对主 # 安装Nuclei -Nuclei需要**go1**才能安装成功。执行下列命令安装最新版本的Nuclei +Nuclei需要 **go1.21** 才能安装成功。执行下列命令安装最新版本的Nuclei ```sh go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest @@ -396,8 +396,8 @@ Nuclei通过增加手动、自动的过程，极大地改变了安全评估的 Nuclei构建很简单，通过数百名安全研究员的社区模板，Nuclei可以随时扫描来了解安全威胁。Nuclei通常用来用于复测，以确定漏洞是否被修复。 -- **CI/CD：**工程师已经支持了CI/CD，可以通过Nuclei使用定制模板来监控模拟环境和生产环境 -- **周期性扫描：**使用Nuclei创建新发现的漏洞模板，通过Nuclei可以周期性扫描消除漏洞 +- **CI/CD：** 工程师已经支持了CI/CD，可以通过Nuclei使用定制模板来监控模拟环境和生产环境 +- **周期性扫描：** 使用Nuclei创建新发现的漏洞模板，通过Nuclei可以周期性扫描消除漏洞 我们有个[讨论组](https://github.com/projectdiscovery/nuclei-templates/discussions/693)，黑客提交自己的模板后可以获得赏金，这可以减少资产的漏洞，并且减少重复。如果你想实行该计划，可以[联系我](mailto:contact@projectdiscovery.io)。我们非常乐意提供帮助，或者在[讨论组](https://github.com/projectdiscovery/nuclei-templates/discussions/693)中发布相关信息。 diff --git a/SYNTAX-REFERENCE.md b/SYNTAX-REFERENCE.md index 2433ae2138..9222e34aad 100755 --- a/SYNTAX-REFERENCE.md +++ b/SYNTAX-REFERENCE.md @@ -379,6 +379,7 @@ Stop execution once first match is found
Signature is the request signature method +WARNING: 'signature' will be deprecated and will be removed in a future release. Prefer using 'code' protocol for writing cloud checks Valid values: @@ -1613,12 +1614,12 @@ DisablePathAutomerge disables merging target url path with raw request path
-filters []matchers.Matcher +pre-condition []matchers.Matcher
-Filter is matcher-like field to check if fuzzing should be performed on this request or not +Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not
@@ -1626,12 +1627,12 @@ Filter is matcher-like field to check if fuzzing should be performed on this req
-filters-condition string +pre-condition-operator string
-Filter condition is the condition to apply on the filter (AND/OR). Default is OR +FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR
@@ -1981,7 +1982,7 @@ Matcher is used to match a part in the output from a protocol. Appears in: -- http.Request.filters +- http.Request.pre-condition @@ -4156,6 +4157,19 @@ Engine type
+pre-condition string + +
+
+ +PreCondition is a condition which is evaluated before sending the request. + +
+ +
+ +
+ args []string
diff --git a/cmd/docgen/docgen.go b/cmd/docgen/docgen.go index dc0ac71430..c589b98b0e 100644 --- a/cmd/docgen/docgen.go +++ b/cmd/docgen/docgen.go @@ -8,7 +8,7 @@ import ( "regexp" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" "github.com/projectdiscovery/nuclei/v3/pkg/templates" ) @@ -32,11 +32,7 @@ func main() { } // Generate jsonschema - r := &jsonschema.Reflector{ - PreferYAMLSchema: true, - YAMLEmbeddedStructs: true, - FullyQualifyTypeNames: true, - } + r := &jsonschema.Reflector{} jsonschemaData := r.Reflect(&templates.Template{}) var buf bytes.Buffer diff --git a/cmd/integration-test/code.go b/cmd/integration-test/code.go index 320eadf2f5..94c28ea001 100644 --- a/cmd/integration-test/code.go +++ b/cmd/integration-test/code.go @@ -23,6 +23,7 @@ var codeTestCases = []TestCaseInfo{ {Path: "protocols/code/py-nosig.yaml", TestCase: &codePyNoSig{}, DisableOn: isCodeDisabled}, {Path: "protocols/code/py-interactsh.yaml", TestCase: &codeSnippet{}, DisableOn: isCodeDisabled}, {Path: "protocols/code/ps1-snippet.yaml", TestCase: &codeSnippet{}, DisableOn: func() bool { return !osutils.IsWindows() || isCodeDisabled() }}, + {Path: "protocols/code/pre-condition.yaml", TestCase: &codePreCondition{}, DisableOn: isCodeDisabled}, } const ( @@ -94,6 +95,22 @@ func (h *codeSnippet) Execute(filePath string) error { return expectResultsCount(results, 1) } +type codePreCondition struct{} + +// Execute executes a test case and returns an error if occurred +func (h *codePreCondition) Execute(filePath string) error { + results, err := testutils.RunNucleiArgsWithEnvAndGetResults(debug, getEnvValues(), "-t", filePath, "-u", "input", "-code") + if err != nil { + return err + } + if osutils.IsLinux() { + return expectResultsCount(results, 1) + } else { + return expectResultsCount(results, 0) + + } +} + type codeFile struct{} // Execute executes a test case and returns an error if occurred diff --git a/cmd/integration-test/fuzz.go b/cmd/integration-test/fuzz.go index be2ac16169..f5e71774d0 100644 --- a/cmd/integration-test/fuzz.go +++ b/cmd/integration-test/fuzz.go @@ -21,8 +21,6 @@ var fuzzingTestCases = []TestCaseInfo{ {Path: "fuzz/fuzz-type.yaml", TestCase: &fuzzTypeOverride{}}, {Path: "fuzz/fuzz-query.yaml", TestCase: &httpFuzzQuery{}}, {Path: "fuzz/fuzz-headless.yaml", TestCase: &HeadlessFuzzingQuery{}}, - {Path: "fuzz/fuzz-header-basic.yaml", TestCase: &FuzzHeaderBasic{}}, - {Path: "fuzz/fuzz-header-multiple.yaml", TestCase: &FuzzHeaderMultiple{}}, // for fuzzing we should prioritize adding test case related backend // logic in fuzz playground server instead of adding them here {Path: "fuzz/fuzz-query-num-replace.yaml", TestCase: &genericFuzzTestCase{expectedResults: 2}}, @@ -176,52 +174,3 @@ func (h *HeadlessFuzzingQuery) Execute(filePath string) error { } return expectResultsCount(got, 2) } - -type FuzzHeaderBasic struct{} - -// Execute executes a test case and returns an error if occurred -func (h *FuzzHeaderBasic) Execute(filePath string) error { - router := httprouter.New() - router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - host := r.Header.Get("Origin") - // redirect to different domain - w.WriteHeader(http.StatusOK) - fmt.Fprintf(w, "Click Here") - }) - ts := httptest.NewTLSServer(router) - defer ts.Close() - - got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-fuzz") - if err != nil { - return err - } - return expectResultsCount(got, 1) -} - -type FuzzHeaderMultiple struct{} - -// Execute executes a test case and returns an error if occurred -func (h *FuzzHeaderMultiple) Execute(filePath string) error { - router := httprouter.New() - router.GET("/", func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - host1 := r.Header.Get("Origin") - host2 := r.Header.Get("X-Forwared-For") - - fmt.Printf("host1: %s, host2: %s\n", host1, host2) - if host1 == host2 && host2 == "secret.local" { - w.WriteHeader(http.StatusOK) - fmt.Fprintf(w, "welcome! to secret admin panel") - return - } - // redirect to different domain - w.WriteHeader(http.StatusForbidden) - }) - ts := httptest.NewTLSServer(router) - defer ts.Close() - - got, err := testutils.RunNucleiTemplateAndGetResults(filePath, ts.URL, debug, "-fuzz") - if err != nil { - return err - } - return expectResultsCount(got, 1) -} diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index 41e1910a86..b4f957e692 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -82,6 +82,8 @@ var httpTestcases = []TestCaseInfo{ {Path: "protocols/http/multi-request.yaml", TestCase: &httpMultiRequest{}}, {Path: "protocols/http/http-matcher-extractor-dy-extractor.yaml", TestCase: &httpMatcherExtractorDynamicExtractor{}}, {Path: "protocols/http/multi-http-var-sharing.yaml", TestCase: &httpMultiVarSharing{}}, + {Path: "protocols/http/raw-path-single-slash.yaml", TestCase: &httpRawPathSingleSlash{}}, + {Path: "protocols/http/raw-unsafe-path-single-slash.yaml", TestCase: &httpRawUnsafePathSingleSlash{}}, } type httpMultiVarSharing struct{} @@ -1560,3 +1562,53 @@ func (h *httpMultiRequest) Execute(filePath string) error { return expectResultsCount(results, 1) } + +type httpRawPathSingleSlash struct{} + +func (h *httpRawPathSingleSlash) Execute(filepath string) error { + expectedPath := "/index.php" + results, err := testutils.RunNucleiBinaryAndGetCombinedOutput(debug, []string{"-t", filepath, "-u", "scanme.sh/index.php", "-debug-req"}) + if err != nil { + return err + } + + var actual string + for _, v := range strings.Split(results, "\n") { + if strings.Contains(v, "GET") { + parts := strings.Fields(v) + if len(parts) == 3 { + actual = parts[1] + } + } + } + + if actual != expectedPath { + return fmt.Errorf("expected: %v\n\nactual: %v", expectedPath, actual) + } + return nil +} + +type httpRawUnsafePathSingleSlash struct{} + +func (h *httpRawUnsafePathSingleSlash) Execute(filepath string) error { + expectedPath := "/index.php" + results, err := testutils.RunNucleiBinaryAndGetCombinedOutput(debug, []string{"-t", filepath, "-u", "scanme.sh/index.php", "-debug-req"}) + if err != nil { + return err + } + + var actual string + for _, v := range strings.Split(results, "\n") { + if strings.Contains(v, "GET") { + parts := strings.Fields(v) + if len(parts) == 3 { + actual = parts[1] + } + } + } + + if actual != expectedPath { + return fmt.Errorf("expected: %v\n\nactual: %v", expectedPath, actual) + } + return nil +} diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index 731865d7e6..3caec21cbd 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -186,6 +186,7 @@ func readConfig() *goflags.FlagSet { // when true updates nuclei binary to latest version var updateNucleiBinary bool var pdcpauth string + var fuzzFlag bool flagSet := goflags.NewFlagSet() flagSet.CaseSensitive = true @@ -313,7 +314,8 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.CreateGroup("fuzzing", "Fuzzing", flagSet.StringVarP(&options.FuzzingType, "fuzzing-type", "ft", "", "overrides fuzzing type set in template (replace, prefix, postfix, infix)"), flagSet.StringVarP(&options.FuzzingMode, "fuzzing-mode", "fm", "", "overrides fuzzing mode set in template (multiple, single)"), - flagSet.BoolVar(&options.FuzzTemplates, "fuzz", false, "enable loading fuzzing templates"), + flagSet.BoolVar(&fuzzFlag, "fuzz", false, "enable loading fuzzing templates (Deprecated: use -dast instead)"), + flagSet.BoolVar(&options.DAST, "dast", false, "enable / run dast (fuzz) nuclei templates"), ) flagSet.CreateGroup("uncover", "Uncover", @@ -436,6 +438,12 @@ Additional documentation is available at: https://docs.nuclei.sh/getting-started goflags.DisableAutoConfigMigration = true _ = flagSet.Parse() + // when fuzz flag is enabled, set the dast flag to true + if fuzzFlag { + // backwards compatibility for fuzz flag + options.DAST = true + } + // api key hierarchy: cli flag > env var > .pdcp/credential file if pdcpauth == "true" { runner.AuthWithPDCP() diff --git a/go.mod b/go.mod index c0e779ec79..7eeaf57386 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.21 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible - github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 github.com/andygrunwald/go-jira v1.16.0 github.com/antchfx/htmlquery v1.3.0 github.com/bluele/gcache v0.0.2 @@ -12,6 +11,7 @@ require ( github.com/go-rod/rod v0.114.0 github.com/gobwas/ws v1.2.1 github.com/google/go-github v17.0.0+incompatible + github.com/invopop/jsonschema v0.12.0 github.com/itchyny/gojq v0.12.13 github.com/json-iterator/go v1.1.12 github.com/julienschmidt/httprouter v1.3.0 @@ -20,12 +20,12 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/projectdiscovery/clistats v0.0.20 - github.com/projectdiscovery/fastdialer v0.0.62 + github.com/projectdiscovery/fastdialer v0.0.64 github.com/projectdiscovery/hmap v0.0.41 github.com/projectdiscovery/interactsh v1.1.9 - github.com/projectdiscovery/rawhttp v0.1.40 + github.com/projectdiscovery/rawhttp v0.1.41 github.com/projectdiscovery/retryabledns v1.0.58 - github.com/projectdiscovery/retryablehttp-go v1.0.51 + github.com/projectdiscovery/retryablehttp-go v1.0.54 github.com/projectdiscovery/yamldoc-go v1.0.4 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.5.0 @@ -78,26 +78,25 @@ require ( github.com/mholt/archiver v3.1.1+incompatible github.com/ory/dockertest/v3 v3.10.0 github.com/praetorian-inc/fingerprintx v1.1.9 - github.com/projectdiscovery/dsl v0.0.46 + github.com/projectdiscovery/dsl v0.0.50 github.com/projectdiscovery/fasttemplate v0.0.2 github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb - github.com/projectdiscovery/goflags v0.1.42 + github.com/projectdiscovery/goflags v0.1.46 github.com/projectdiscovery/gologger v1.1.12 github.com/projectdiscovery/gostruct v0.0.2 github.com/projectdiscovery/gozero v0.0.2 github.com/projectdiscovery/httpx v1.6.0 github.com/projectdiscovery/mapcidr v1.1.16 github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 - github.com/projectdiscovery/ratelimit v0.0.27 + github.com/projectdiscovery/ratelimit v0.0.35 github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 github.com/projectdiscovery/sarif v0.0.1 github.com/projectdiscovery/tlsx v1.1.6 github.com/projectdiscovery/uncover v1.0.7 github.com/projectdiscovery/useragent v0.0.40 - github.com/projectdiscovery/utils v0.0.84-0.20240313184656-e3ec80f4dd42 - github.com/projectdiscovery/wappalyzergo v0.0.112 + github.com/projectdiscovery/utils v0.0.87 + github.com/projectdiscovery/wappalyzergo v0.0.116 github.com/redis/go-redis/v9 v9.1.0 - github.com/sashabaranov/go-openai v1.15.3 github.com/seh-msft/burpxml v1.0.1 github.com/stretchr/testify v1.9.0 github.com/zmap/zgrab2 v0.1.8-0.20230806160807-97ba87c0e706 @@ -123,8 +122,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.8.0 // indirect github.com/bits-and-blooms/bloom/v3 v3.5.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect @@ -139,7 +140,7 @@ require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dlclark/regexp2 v1.11.0 // indirect github.com/docker/cli v24.0.5+incompatible // indirect - github.com/docker/docker v24.0.7+incompatible // indirect + github.com/docker/docker v24.0.9+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/free5gc/util v1.0.5-0.20230511064842-2e120956883b // indirect @@ -201,8 +202,9 @@ require ( github.com/projectdiscovery/ldapserver v1.0.2-0.20240219154113-dcc758ebc0cb // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect github.com/projectdiscovery/stringsutil v0.0.2 // indirect - github.com/quic-go/quic-go v0.40.1 // indirect + github.com/quic-go/quic-go v0.42.0 // indirect github.com/refraction-networking/utls v1.6.1 // indirect + github.com/sashabaranov/go-openai v1.15.3 // indirect github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -218,6 +220,7 @@ require ( github.com/tim-ywliu/nested-logrus-formatter v1.3.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect @@ -225,6 +228,7 @@ require ( github.com/ysmood/got v0.34.1 // indirect github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect + github.com/zcalusic/sysinfo v1.0.2 // indirect github.com/zeebo/blake3 v0.2.3 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/sync v0.6.0 // indirect @@ -263,7 +267,6 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.2 // indirect github.com/hdm/jarm-go v0.0.7 // indirect - github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/leodido/go-urn v1.2.4 // indirect @@ -301,7 +304,7 @@ require ( golang.org/x/exp v0.0.0-20240119083558-1b970713d09a golang.org/x/mod v0.14.0 // indirect golang.org/x/sys v0.17.0 // indirect - golang.org/x/time v0.3.0 // indirect + golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.17.0 google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/go.sum b/go.sum index 955016dec5..cefd9ae445 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,6 @@ github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= -github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725 h1:NjwIgLQlD46o79bheVG4SCdRnnOz4XtgUN1WABX5DLA= -github.com/alecthomas/jsonschema v0.0.0-20211022214203-8b29eab41725/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/alecthomas/kingpin/v2 v2.3.1/go.mod h1:oYL5vtsvEHZGHxU7DMp32Dvx+qL+ptGn6lWaot2vCNE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= @@ -180,6 +178,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -196,6 +196,8 @@ github.com/bsm/ginkgo/v2 v2.9.5 h1:rtVBYPs3+TC5iLUVOis1B9tjLTup7Cj5IfzosKtvTJ0= github.com/bsm/ginkgo/v2 v2.9.5/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= @@ -275,8 +277,8 @@ github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/cli v24.0.5+incompatible h1:WeBimjvS0eKdH4Ygx+ihVq1Q++xg36M/rMi4aXAvodc= github.com/docker/cli v24.0.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= -github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0= +github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -558,8 +560,6 @@ github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk= -github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -567,6 +567,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= +github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/itchyny/gojq v0.12.13 h1:IxyYlHYIlspQHHTE0f3cJF0NKDMfajxViuhBLnHd/QU= @@ -827,18 +829,18 @@ github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQO github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/clistats v0.0.20 h1:5jO5SLiRJ7f0nDV0ndBNmBeesbROouPooH+DGMgoWq4= github.com/projectdiscovery/clistats v0.0.20/go.mod h1:GJ2av0KnOvK0AISQnP8hyDclYIji1LVkx2l0pwnzAu4= -github.com/projectdiscovery/dsl v0.0.46 h1:zBNNzSBA1aakGY44w6KhnjJQx/zO+oW2Wx7TR8xZm/A= -github.com/projectdiscovery/dsl v0.0.46/go.mod h1:eoZEJFCIT5emI00xj8HTQwHz4YwwGAD+grL3G7CDlfs= -github.com/projectdiscovery/fastdialer v0.0.62 h1:Wyba2hD6ZF3S04MgCn380mC+1RXJ+dq14Yq8u2yk7ps= -github.com/projectdiscovery/fastdialer v0.0.62/go.mod h1:2baj2TRXTw+hHbKTW9IZR4dhpxCGJkq5AKL1ge5gis8= +github.com/projectdiscovery/dsl v0.0.50 h1:4SuAwTS9l6o1tqlIC/79+EcUwTM6CjaU7MpY/nDlFaM= +github.com/projectdiscovery/dsl v0.0.50/go.mod h1:6g740l4tH4d2j9UYtIchtxudb0Dphkq4o+VatpR4M6g= +github.com/projectdiscovery/fastdialer v0.0.64 h1:xivkA4g14nwQElOVsxPkGMWsdcYPcp7DPhVjvI6yQkw= +github.com/projectdiscovery/fastdialer v0.0.64/go.mod h1:S/7PAQRmVDYRaU7u4xXD0qA5a48NAZq2JcpcVoEVrlo= github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA= github.com/projectdiscovery/fasttemplate v0.0.2/go.mod h1:XYWWVMxnItd+r0GbjA1GCsUopMw1/XusuQxdyAIHMCw= github.com/projectdiscovery/freeport v0.0.5 h1:jnd3Oqsl4S8n0KuFkE5Hm8WGDP24ITBvmyw5pFTHS8Q= github.com/projectdiscovery/freeport v0.0.5/go.mod h1:PY0bxSJ34HVy67LHIeF3uIutiCSDwOqKD8ruBkdiCwE= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb h1:rutG906Drtbpz4DwU5mhGIeOhRcktDH4cGQitGUMAsg= github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb/go.mod h1:FLjF1DmZ+POoGEiIQdWuYVwS++C/GwpX8YaCsTSm1RY= -github.com/projectdiscovery/goflags v0.1.42 h1:C3CleYUODv5Jdn4+FTCUpzm3eOXHl+GiLHbXSw5iNsQ= -github.com/projectdiscovery/goflags v0.1.42/go.mod h1:UuS8nOpeYbVebPVWeTvcpILESC8daIwRVJ14UEt7L7c= +github.com/projectdiscovery/goflags v0.1.46 h1:JlYvFxJcimKJGWYbygiFBN052MWrbls/kKiwOKpLzEE= +github.com/projectdiscovery/goflags v0.1.46/go.mod h1:X7A6ELNgczyOyEy2gyNC/tJTuhtwQk6ZLyzsnDVlZkw= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= github.com/projectdiscovery/gostruct v0.0.2 h1:s8gP8ApugGM4go1pA+sVlPDXaWqNP5BBDDSv7VEdG1M= @@ -861,16 +863,16 @@ github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5 h1:L/e8z8yw github.com/projectdiscovery/n3iwf v0.0.0-20230523120440-b8cd232ff1f5/go.mod h1:pGW2ncnTxTxHtP9wzcIJAB+3/NMp6IiuQWd2NK7K+oc= github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= -github.com/projectdiscovery/ratelimit v0.0.27 h1:McTgnl8CtaEPmPtb9JG7EfgaQ1Rhu0pHa0Kf5Kld6Xs= -github.com/projectdiscovery/ratelimit v0.0.27/go.mod h1:5suG3x1d5+UV4xe2RBE/QCvQkz8CaxPvdwztjab3GzM= -github.com/projectdiscovery/rawhttp v0.1.40 h1:HSWABDP2gSeyYCskQbcjVXrtBD9aiOPf+qZiemQKlXo= -github.com/projectdiscovery/rawhttp v0.1.40/go.mod h1:mvX/3WTDta3hvAd4BqJT7muGAqYzAocoDeUgBN2aDLc= +github.com/projectdiscovery/ratelimit v0.0.35 h1:epEzFATOcXZ4tssV4Hax5Op9lrbUnQMEGMV5PoUpTKc= +github.com/projectdiscovery/ratelimit v0.0.35/go.mod h1:mPqa8UpV5I7eAN5/ZcsjLiXMhjtVvZRrHtpBRsTPuyA= +github.com/projectdiscovery/rawhttp v0.1.41 h1:0n6CohOf0Aq7dsXv+ozznhlYr4ANDKLwvPmdzTet3qU= +github.com/projectdiscovery/rawhttp v0.1.41/go.mod h1:TyVfCwNbAsQSwrMOKu8o1g80AO3t1OnlJx+flgSV/CQ= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917 h1:m03X4gBVSorSzvmm0bFa7gDV4QNSOWPL/fgZ4kTXBxk= github.com/projectdiscovery/rdap v0.9.1-0.20221108103045-9865884d1917/go.mod h1:JxXtZC9e195awe7EynrcnBJmFoad/BNDzW9mzFkK8Sg= github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= -github.com/projectdiscovery/retryablehttp-go v1.0.51 h1:8XMrNC8JrwvySESe2d+XWF9bq4unWqD4PUPEC4Cai8s= -github.com/projectdiscovery/retryablehttp-go v1.0.51/go.mod h1:6cdh/acYHpeYWg7+Iblh4xBRb87bC118L4G4mpvCMuA= +github.com/projectdiscovery/retryablehttp-go v1.0.54 h1:lUmQA3obq3Ya3xU1vouKf+hVjbLFKzJCK6FcNKPZ8vQ= +github.com/projectdiscovery/retryablehttp-go v1.0.54/go.mod h1:J+pg00bYLEgWOZJISi16icHUDbsnkjnA1PmSa2kSMYs= github.com/projectdiscovery/sarif v0.0.1 h1:C2Tyj0SGOKbCLgHrx83vaE6YkzXEVrMXYRGLkKCr/us= github.com/projectdiscovery/sarif v0.0.1/go.mod h1:cEYlDu8amcPf6b9dSakcz2nNnJsoz4aR6peERwV+wuQ= github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= @@ -881,10 +883,10 @@ github.com/projectdiscovery/uncover v1.0.7 h1:ut+2lTuvmftmveqF5RTjMWAgyLj8ltPQC7 github.com/projectdiscovery/uncover v1.0.7/go.mod h1:HFXgm1sRPuoN0D4oATljPIdmbo/EEh1wVuxQqo/dwFE= github.com/projectdiscovery/useragent v0.0.40 h1:1LUhReSGPkhqsM5n40OOC9dIoNqMGs1dyGFJcOmg2Fo= github.com/projectdiscovery/useragent v0.0.40/go.mod h1:EvK1x3s948Gtqb/XOahXcauyejCL/rSgy5d1IAvsKT4= -github.com/projectdiscovery/utils v0.0.84-0.20240313184656-e3ec80f4dd42 h1:l22rSOP8i6HXu1QfAtIot8NvmJgUmBHEn6Mih7s8Gak= -github.com/projectdiscovery/utils v0.0.84-0.20240313184656-e3ec80f4dd42/go.mod h1:VsoXXTuNAAziuodKWakLyurVXaV4tNTJU4Eo8umyr3Q= -github.com/projectdiscovery/wappalyzergo v0.0.112 h1:QPpp5jmj1lqLd5mFdFKQ9VvcYhQNqyU9Mr+IB0US2zA= -github.com/projectdiscovery/wappalyzergo v0.0.112/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= +github.com/projectdiscovery/utils v0.0.87 h1:9+RiTEhpUB/vk6XJUVpysNWJ2aCTD7WuyoyAcNnbIzk= +github.com/projectdiscovery/utils v0.0.87/go.mod h1:jGK450sL9AVDTjaPwEs9za8NVeEC9xE97IWNoK138kI= +github.com/projectdiscovery/wappalyzergo v0.0.116 h1:xy+mBpwbYo/0PSzmJOQ/RXHomEh0D3nDBcbCxsW69m8= +github.com/projectdiscovery/wappalyzergo v0.0.116/go.mod h1:hc/o+fgM8KtdpFesjfBTmHTwsR+yBd+4kYZW/DGy/x8= github.com/projectdiscovery/yamldoc-go v1.0.4 h1:eZoESapnMw6WAHiVgRwNqvbJEfNHEH148uthhFbG5jE= github.com/projectdiscovery/yamldoc-go v1.0.4/go.mod h1:8PIPRcUD55UbtQdcfFR1hpIGRWG0P7alClXNGt1TBik= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -919,8 +921,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1Q= -github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c= +github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= +github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY= github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c= @@ -996,7 +998,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -1063,6 +1064,8 @@ github.com/weppos/publicsuffix-go v0.30.0/go.mod h1:kBi8zwYnR0zrbm8RcuN1o9Fzgpnn github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 h1:h2JizvZl9aIj6za9S5AyrkU+OzIS4CetQthH/ejO+lg= github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222/go.mod h1:s41lQh6dIsDWIC1OWh7ChWJXLH0zkJ9KHZVqA7vHyuQ= github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220927085643-dc0d00c92642/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/xanzy/go-gitlab v0.84.0 h1:PdpCaskQSgcVDsx21c6ikf8Rfyo7SNtFAJwP9PrbCFE= github.com/xanzy/go-gitlab v0.84.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= @@ -1106,6 +1109,8 @@ github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18W github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zcalusic/sysinfo v1.0.2 h1:nwTTo2a+WQ0NXwo0BGRojOJvJ/5XKvQih+2RrtWqfxc= +github.com/zcalusic/sysinfo v1.0.2/go.mod h1:kluzTYflRWo6/tXVMJPdEjShsbPpsFRyy+p1mBQPC30= github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg= @@ -1409,8 +1414,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/integration_tests/flow/iterate-one-value-flow.yaml b/integration_tests/flow/iterate-one-value-flow.yaml index d45eaa586b..9ab6633c0c 100644 --- a/integration_tests/flow/iterate-one-value-flow.yaml +++ b/integration_tests/flow/iterate-one-value-flow.yaml @@ -4,10 +4,13 @@ info: name: Test Flow Iterate One Value Flow author: pdteam severity: info + description: | + If length of template.extracted variable is not know, i.e it could be an array of 1 or more values, then iterate function + should be used to iterate over values because nuclei by default converts array to string if it has only 1 value. flow: | http(1) - for(let value of template.extracted){ + for(let value of iterate(template.extracted)){ set("value", value) http(2) } diff --git a/integration_tests/fuzz/fuzz-body-generic-sqli.yaml b/integration_tests/fuzz/fuzz-body-generic-sqli.yaml index 83ca496220..6ae25a7ac6 100644 --- a/integration_tests/fuzz/fuzz-body-generic-sqli.yaml +++ b/integration_tests/fuzz/fuzz-body-generic-sqli.yaml @@ -10,7 +10,7 @@ info: and performs fuzzing on the value of every key http: - - filters: + - pre-condition: - type: dsl dsl: - method != "GET" diff --git a/integration_tests/fuzz/fuzz-body-json-sqli.yaml b/integration_tests/fuzz/fuzz-body-json-sqli.yaml index 187ce1b46c..5dfeca451d 100644 --- a/integration_tests/fuzz/fuzz-body-json-sqli.yaml +++ b/integration_tests/fuzz/fuzz-body-json-sqli.yaml @@ -10,7 +10,7 @@ info: Note: this is example template, and payloads/matchers need to be modified appropriately. http: - - filters: + - pre-condition: - type: dsl dsl: - method != "GET" diff --git a/integration_tests/fuzz/fuzz-body-multipart-form-sqli.yaml b/integration_tests/fuzz/fuzz-body-multipart-form-sqli.yaml index dcca2e18d5..c7e8fc9d21 100644 --- a/integration_tests/fuzz/fuzz-body-multipart-form-sqli.yaml +++ b/integration_tests/fuzz/fuzz-body-multipart-form-sqli.yaml @@ -10,7 +10,7 @@ info: Note: this is example template, and payloads/matchers need to be modified appropriately. http: - - filters: + - pre-condition: - type: dsl dsl: - method != "GET" diff --git a/integration_tests/fuzz/fuzz-body-params-sqli.yaml b/integration_tests/fuzz/fuzz-body-params-sqli.yaml index 2fdd174261..b090feb210 100644 --- a/integration_tests/fuzz/fuzz-body-params-sqli.yaml +++ b/integration_tests/fuzz/fuzz-body-params-sqli.yaml @@ -10,7 +10,7 @@ info: Note: this is example template, and payloads/matchers need to be modified appropriately. http: - - filters: + - pre-condition: - type: dsl dsl: - method != "GET" diff --git a/integration_tests/fuzz/fuzz-body-xml-sqli.yaml b/integration_tests/fuzz/fuzz-body-xml-sqli.yaml index 8ac62842d5..ae77efd3ea 100644 --- a/integration_tests/fuzz/fuzz-body-xml-sqli.yaml +++ b/integration_tests/fuzz/fuzz-body-xml-sqli.yaml @@ -10,7 +10,7 @@ info: Note: this is example template, and payloads/matchers need to be modified appropriately. http: - - filters: + - pre-condition: - type: dsl dsl: - method != "GET" diff --git a/integration_tests/fuzz/fuzz-cookie-error-sqli.yaml b/integration_tests/fuzz/fuzz-cookie-error-sqli.yaml index 86bb2a1639..3e457800aa 100644 --- a/integration_tests/fuzz/fuzz-cookie-error-sqli.yaml +++ b/integration_tests/fuzz/fuzz-cookie-error-sqli.yaml @@ -9,7 +9,7 @@ info: Note: this is example template, and payloads/matchers need to be modified appropriately. http: - - filters: + - pre-condition: - type: dsl dsl: - 'method == "GET"' diff --git a/integration_tests/fuzz/fuzz-header-basic.yaml b/integration_tests/fuzz/fuzz-header-basic.yaml deleted file mode 100644 index 10d2928c32..0000000000 --- a/integration_tests/fuzz/fuzz-header-basic.yaml +++ /dev/null @@ -1,49 +0,0 @@ -id: fuzz-header-basic - -info: - name: fuzz header basic - author: pdteam - severity: info - description: | - In this template we check for any reflection when fuzzing Origin header - -variables: - first: "{{rand_int(10000, 99999)}}" - -http: - - raw: - - | - GET /?x=aaa&y=bbb HTTP/1.1 - Host: {{Hostname}} - Origin: https://example.com - X-Fuzz-Header: 1337 - Cookie: z=aaa; bb=aaa - User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) - Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 - Accept-Language: en-US,en;q=0.9 - Connection: close - - payloads: - reflection: - - "'\"><{{first}}" - - fuzzing: - - part: header - type: replace - mode: single - keys: ["Origin"] - fuzz: - - "{{reflection}}" - - stop-at-first-match: true - matchers-condition: and - matchers: - - type: word - part: body - words: - - "{{reflection}}" - - - type: word - part: header - words: - - "text/html" \ No newline at end of file diff --git a/integration_tests/fuzz/fuzz-header-multiple.yaml b/integration_tests/fuzz/fuzz-header-multiple.yaml deleted file mode 100644 index 0a535b5776..0000000000 --- a/integration_tests/fuzz/fuzz-header-multiple.yaml +++ /dev/null @@ -1,41 +0,0 @@ -id: fuzz-header-multiple - -info: - name: fuzz header multiple - author: pdteam - severity: info - description: | - In this template we fuzz multiple headers with single payload - -http: - - raw: - - | - GET /?x=aaa&y=bbb HTTP/1.1 - Host: {{Hostname}} - Origin: https://example.com - X-Forwared-For: 1337 - Cookie: z=aaa; bb=aaa - User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) - Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 - Accept-Language: en-US,en;q=0.9 - Connection: close - - payloads: - reflection: - - "secret.local" - - fuzzing: - - part: header - type: replace - mode: multiple - keys: ["Origin", "X-Forwared-For"] - fuzz: - - "{{reflection}}" - - stop-at-first-match: true - matchers-condition: and - matchers: - - type: word - part: body - words: - - "admin" \ No newline at end of file diff --git a/integration_tests/fuzz/fuzz-host-header-injection.yaml b/integration_tests/fuzz/fuzz-host-header-injection.yaml index cda22235e1..551398de2c 100644 --- a/integration_tests/fuzz/fuzz-host-header-injection.yaml +++ b/integration_tests/fuzz/fuzz-host-header-injection.yaml @@ -10,7 +10,7 @@ variables: domain: "oast.fun" http: - - filters: + - pre-condition: - type: dsl dsl: - 'method == "GET"' diff --git a/integration_tests/fuzz/fuzz-path-sqli.yaml b/integration_tests/fuzz/fuzz-path-sqli.yaml index e034dec3ff..e23098d931 100644 --- a/integration_tests/fuzz/fuzz-path-sqli.yaml +++ b/integration_tests/fuzz/fuzz-path-sqli.yaml @@ -11,7 +11,7 @@ info: Note: this is example template, and payloads/matchers need to be modified appropriately. http: - - filters: + - pre-condition: - type: dsl dsl: - 'method == "GET"' diff --git a/integration_tests/fuzz/fuzz-query-num-replace.yaml b/integration_tests/fuzz/fuzz-query-num-replace.yaml index 90f3a3934f..dc62833613 100644 --- a/integration_tests/fuzz/fuzz-query-num-replace.yaml +++ b/integration_tests/fuzz/fuzz-query-num-replace.yaml @@ -7,7 +7,7 @@ info: description: Query Value Fuzzing using Fuzzing Rules http: - - filters: + - pre-condition: - type: dsl dsl: - 'len(query) > 0' @@ -16,7 +16,7 @@ http: part: path words: - /blog/post - filters-condition: and + pre-condition-operator: and payloads: nums: diff --git a/integration_tests/generic/auth/certificate/http-get.yaml b/integration_tests/generic/auth/certificate/http-get.yaml index c52c577a13..93181f74bd 100644 --- a/integration_tests/generic/auth/certificate/http-get.yaml +++ b/integration_tests/generic/auth/certificate/http-get.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/library/test.yaml b/integration_tests/library/test.yaml index d616531965..286c0aa844 100644 --- a/integration_tests/library/test.yaml +++ b/integration_tests/library/test.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/loader/excluded-template.yaml b/integration_tests/loader/excluded-template.yaml index 6c25a6314a..9840a4cb88 100644 --- a/integration_tests/loader/excluded-template.yaml +++ b/integration_tests/loader/excluded-template.yaml @@ -6,7 +6,7 @@ info: severity: info tags: fuzz -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/loader/get-headers.yaml b/integration_tests/loader/get-headers.yaml index bae3670525..8d3ad45959 100644 --- a/integration_tests/loader/get-headers.yaml +++ b/integration_tests/loader/get-headers.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/loader/get.yaml b/integration_tests/loader/get.yaml index c7e07e8cf4..854652c98d 100644 --- a/integration_tests/loader/get.yaml +++ b/integration_tests/loader/get.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/code/pre-condition.yaml b/integration_tests/protocols/code/pre-condition.yaml new file mode 100644 index 0000000000..a61b4f90d8 --- /dev/null +++ b/integration_tests/protocols/code/pre-condition.yaml @@ -0,0 +1,26 @@ +id: pre-condition-code + +info: + name: example code template + author: pdteam + severity: info + + +self-contained: true + +variables: + OAST: "{{interactsh-url}}" + +code: + - pre-condition: IsLinux() + engine: + - sh + - bash + source: | + echo "$OAST" | base64 + + matchers: + - type: dsl + dsl: + - true +# digest: 4a0a00473045022100c7215ce9f11e6a51c193bb54643a05cdd1cde18a3abb6c9983c5c7524d3ff03002203d93581c81d3ad5db463570cbbd2bdee529328d32a5b00e037610c211e448cef:4a3eb6b4988d95847d4203be25ed1d46 \ No newline at end of file diff --git a/integration_tests/protocols/http/annotation-timeout.yaml b/integration_tests/protocols/http/annotation-timeout.yaml index ddb0b3e6a5..2b5b624abf 100644 --- a/integration_tests/protocols/http/annotation-timeout.yaml +++ b/integration_tests/protocols/http/annotation-timeout.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | @timeout: 5s diff --git a/integration_tests/protocols/http/cl-body-with-header.yaml b/integration_tests/protocols/http/cl-body-with-header.yaml index 21b9fbe2a7..48833de1e2 100644 --- a/integration_tests/protocols/http/cl-body-with-header.yaml +++ b/integration_tests/protocols/http/cl-body-with-header.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/cl-body-without-header.yaml b/integration_tests/protocols/http/cl-body-without-header.yaml index d75e481fc1..365d7cff9a 100644 --- a/integration_tests/protocols/http/cl-body-without-header.yaml +++ b/integration_tests/protocols/http/cl-body-without-header.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/cli-with-constants.yaml b/integration_tests/protocols/http/cli-with-constants.yaml index 3d04d06e3f..846b435daf 100644 --- a/integration_tests/protocols/http/cli-with-constants.yaml +++ b/integration_tests/protocols/http/cli-with-constants.yaml @@ -8,7 +8,7 @@ info: constants: test: test-in-template -requests: +http: - method: GET path: - "{{BaseURL}}?p={{test}}" diff --git a/integration_tests/protocols/http/custom-attack-type.yaml b/integration_tests/protocols/http/custom-attack-type.yaml index 18d6d00377..1b344815f5 100644 --- a/integration_tests/protocols/http/custom-attack-type.yaml +++ b/integration_tests/protocols/http/custom-attack-type.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}/?a={{test}}&b={{values}}" diff --git a/integration_tests/protocols/http/default-matcher-condition.yaml b/integration_tests/protocols/http/default-matcher-condition.yaml index ea9d3b09b1..d5fc7c36a4 100644 --- a/integration_tests/protocols/http/default-matcher-condition.yaml +++ b/integration_tests/protocols/http/default-matcher-condition.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | GET /?action=curltest&url={{interactsh-url}} HTTP/1.1 diff --git a/integration_tests/protocols/http/disable-redirects.yaml b/integration_tests/protocols/http/disable-redirects.yaml index ec6616bb9b..09a7d62508 100644 --- a/integration_tests/protocols/http/disable-redirects.yaml +++ b/integration_tests/protocols/http/disable-redirects.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/dsl-functions.yaml b/integration_tests/protocols/http/dsl-functions.yaml index 5afce178ed..f1b52f45e5 100644 --- a/integration_tests/protocols/http/dsl-functions.yaml +++ b/integration_tests/protocols/http/dsl-functions.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: # Note for the integration test: dsl expression should not contain commas - | diff --git a/integration_tests/protocols/http/dsl-matcher-variable.yaml b/integration_tests/protocols/http/dsl-matcher-variable.yaml index ecbe5f9e5e..32d55d8d03 100644 --- a/integration_tests/protocols/http/dsl-matcher-variable.yaml +++ b/integration_tests/protocols/http/dsl-matcher-variable.yaml @@ -5,7 +5,7 @@ info: author: pd-team severity: info -requests: +http: - path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/get-all-ips.yaml b/integration_tests/protocols/http/get-all-ips.yaml index 1dba45b119..090bd888b6 100644 --- a/integration_tests/protocols/http/get-all-ips.yaml +++ b/integration_tests/protocols/http/get-all-ips.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/get-case-insensitive.yaml b/integration_tests/protocols/http/get-case-insensitive.yaml index e8c4054b75..fa48d63f87 100644 --- a/integration_tests/protocols/http/get-case-insensitive.yaml +++ b/integration_tests/protocols/http/get-case-insensitive.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/get-headers.yaml b/integration_tests/protocols/http/get-headers.yaml index bae3670525..8d3ad45959 100644 --- a/integration_tests/protocols/http/get-headers.yaml +++ b/integration_tests/protocols/http/get-headers.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/get-host-redirects.yaml b/integration_tests/protocols/http/get-host-redirects.yaml index 1a438929a7..77cb9137ac 100644 --- a/integration_tests/protocols/http/get-host-redirects.yaml +++ b/integration_tests/protocols/http/get-host-redirects.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/get-override-sni.yaml b/integration_tests/protocols/http/get-override-sni.yaml index 61239ee5ce..c5e250edb7 100644 --- a/integration_tests/protocols/http/get-override-sni.yaml +++ b/integration_tests/protocols/http/get-override-sni.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | @tls-sni:request.host diff --git a/integration_tests/protocols/http/get-query-string.yaml b/integration_tests/protocols/http/get-query-string.yaml index 129c080d78..e6f39b78ee 100644 --- a/integration_tests/protocols/http/get-query-string.yaml +++ b/integration_tests/protocols/http/get-query-string.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}?test=nuclei" diff --git a/integration_tests/protocols/http/get-redirects-chain-headers.yaml b/integration_tests/protocols/http/get-redirects-chain-headers.yaml index 5120730185..4ded474ea8 100644 --- a/integration_tests/protocols/http/get-redirects-chain-headers.yaml +++ b/integration_tests/protocols/http/get-redirects-chain-headers.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/get-redirects.yaml b/integration_tests/protocols/http/get-redirects.yaml index e6bf0c44b0..3710f765ac 100644 --- a/integration_tests/protocols/http/get-redirects.yaml +++ b/integration_tests/protocols/http/get-redirects.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/get-sni-unsafe.yaml b/integration_tests/protocols/http/get-sni-unsafe.yaml index 16a923e406..f001c3a61e 100644 --- a/integration_tests/protocols/http/get-sni-unsafe.yaml +++ b/integration_tests/protocols/http/get-sni-unsafe.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - |+ GET / HTTP/1.1 diff --git a/integration_tests/protocols/http/get-sni.yaml b/integration_tests/protocols/http/get-sni.yaml index 8adae0e0d9..e00fcf1b08 100644 --- a/integration_tests/protocols/http/get-sni.yaml +++ b/integration_tests/protocols/http/get-sni.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/get-without-scheme.yaml b/integration_tests/protocols/http/get-without-scheme.yaml index 020800aa72..2603288055 100644 --- a/integration_tests/protocols/http/get-without-scheme.yaml +++ b/integration_tests/protocols/http/get-without-scheme.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/get.yaml b/integration_tests/protocols/http/get.yaml index c7e07e8cf4..854652c98d 100644 --- a/integration_tests/protocols/http/get.yaml +++ b/integration_tests/protocols/http/get.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/http-preprocessor.yaml b/integration_tests/protocols/http/http-preprocessor.yaml index b856f3ccb1..795e86150b 100644 --- a/integration_tests/protocols/http/http-preprocessor.yaml +++ b/integration_tests/protocols/http/http-preprocessor.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | GET /?test={{randstr}} HTTP/1.1 diff --git a/integration_tests/protocols/http/interactsh-requests-mc-and.yaml b/integration_tests/protocols/http/interactsh-requests-mc-and.yaml index e9eea1be61..ea9f037e94 100644 --- a/integration_tests/protocols/http/interactsh-requests-mc-and.yaml +++ b/integration_tests/protocols/http/interactsh-requests-mc-and.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | GET /api/geoping/{{interactsh-url}} HTTP/1.1 diff --git a/integration_tests/protocols/http/interactsh-stop-at-first-match.yaml b/integration_tests/protocols/http/interactsh-stop-at-first-match.yaml index a200ecdac9..bd3d54e504 100644 --- a/integration_tests/protocols/http/interactsh-stop-at-first-match.yaml +++ b/integration_tests/protocols/http/interactsh-stop-at-first-match.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}/?a=1" diff --git a/integration_tests/protocols/http/interactsh.yaml b/integration_tests/protocols/http/interactsh.yaml index cd1892aee2..e34a4e6501 100644 --- a/integration_tests/protocols/http/interactsh.yaml +++ b/integration_tests/protocols/http/interactsh.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/post-body.yaml b/integration_tests/protocols/http/post-body.yaml index 7eb36ca48a..752d99d13a 100644 --- a/integration_tests/protocols/http/post-body.yaml +++ b/integration_tests/protocols/http/post-body.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: POST path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/post-json-body.yaml b/integration_tests/protocols/http/post-json-body.yaml index b4c6c3178f..8e341f6cbd 100644 --- a/integration_tests/protocols/http/post-json-body.yaml +++ b/integration_tests/protocols/http/post-json-body.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: POST path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/post-multipart-body.yaml b/integration_tests/protocols/http/post-multipart-body.yaml index 73bd5131a3..7eb9360225 100644 --- a/integration_tests/protocols/http/post-multipart-body.yaml +++ b/integration_tests/protocols/http/post-multipart-body.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: POST path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/race-multiple.yaml b/integration_tests/protocols/http/race-multiple.yaml index 89d4db9d37..17dc3da858 100644 --- a/integration_tests/protocols/http/race-multiple.yaml +++ b/integration_tests/protocols/http/race-multiple.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | GET / HTTP/1.1 diff --git a/integration_tests/protocols/http/race-simple.yaml b/integration_tests/protocols/http/race-simple.yaml index 39a7fb9136..3eca99fc43 100644 --- a/integration_tests/protocols/http/race-simple.yaml +++ b/integration_tests/protocols/http/race-simple.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | GET / HTTP/1.1 diff --git a/integration_tests/protocols/http/raw-cookie-reuse.yaml b/integration_tests/protocols/http/raw-cookie-reuse.yaml index 009431ca4c..61d18b9fa4 100644 --- a/integration_tests/protocols/http/raw-cookie-reuse.yaml +++ b/integration_tests/protocols/http/raw-cookie-reuse.yaml @@ -4,7 +4,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | POST / HTTP/1.1 diff --git a/integration_tests/protocols/http/raw-dynamic-extractor.yaml b/integration_tests/protocols/http/raw-dynamic-extractor.yaml index 4e10205d83..4753af625f 100644 --- a/integration_tests/protocols/http/raw-dynamic-extractor.yaml +++ b/integration_tests/protocols/http/raw-dynamic-extractor.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | POST / HTTP/1.1 diff --git a/integration_tests/protocols/http/raw-get-query.yaml b/integration_tests/protocols/http/raw-get-query.yaml index 71ff50c262..55279e207e 100644 --- a/integration_tests/protocols/http/raw-get-query.yaml +++ b/integration_tests/protocols/http/raw-get-query.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | GET ?test=nuclei HTTP/1.1 diff --git a/integration_tests/protocols/http/raw-get.yaml b/integration_tests/protocols/http/raw-get.yaml index 572a6ef384..0d98d81bb5 100644 --- a/integration_tests/protocols/http/raw-get.yaml +++ b/integration_tests/protocols/http/raw-get.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | GET / HTTP/1.1 diff --git a/integration_tests/protocols/http/raw-path-single-slash.yaml b/integration_tests/protocols/http/raw-path-single-slash.yaml new file mode 100644 index 0000000000..4ea491f4be --- /dev/null +++ b/integration_tests/protocols/http/raw-path-single-slash.yaml @@ -0,0 +1,13 @@ +id: raw-path-single-slash + +info: + name: Test RAW HTTP Template with single slash + author: pdteam + severity: info + +requests: + - raw: + - | + GET / HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} \ No newline at end of file diff --git a/integration_tests/protocols/http/raw-path-trailing-slash.yaml b/integration_tests/protocols/http/raw-path-trailing-slash.yaml index 34496ccfff..022f6f343e 100644 --- a/integration_tests/protocols/http/raw-path-trailing-slash.yaml +++ b/integration_tests/protocols/http/raw-path-trailing-slash.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | GET /test/..;/..;/ HTTP/1.1 diff --git a/integration_tests/protocols/http/raw-payload.yaml b/integration_tests/protocols/http/raw-payload.yaml index 77b20b89bb..6601e8e16e 100644 --- a/integration_tests/protocols/http/raw-payload.yaml +++ b/integration_tests/protocols/http/raw-payload.yaml @@ -4,7 +4,7 @@ info: author: pdteam severity: info -requests: +http: - payloads: username: - test diff --git a/integration_tests/protocols/http/raw-post-body.yaml b/integration_tests/protocols/http/raw-post-body.yaml index 68bc865133..b7047086bd 100644 --- a/integration_tests/protocols/http/raw-post-body.yaml +++ b/integration_tests/protocols/http/raw-post-body.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | POST / HTTP/1.1 diff --git a/integration_tests/protocols/http/raw-unsafe-path-single-slash.yaml b/integration_tests/protocols/http/raw-unsafe-path-single-slash.yaml new file mode 100644 index 0000000000..a356d18e80 --- /dev/null +++ b/integration_tests/protocols/http/raw-unsafe-path-single-slash.yaml @@ -0,0 +1,15 @@ +id: raw-unsafe-path-single-slash + +info: + name: Test RAW Unsafe HTTP Template with single slash + author: pdteam + severity: info + +requests: + - raw: + - |+ + GET / HTTP/1.1 + Host: {{Hostname}} + Origin: {{BaseURL}} + + unsafe: true \ No newline at end of file diff --git a/integration_tests/protocols/http/raw-unsafe-path.yaml b/integration_tests/protocols/http/raw-unsafe-path.yaml index a10721eb37..eaf03b5a4a 100644 --- a/integration_tests/protocols/http/raw-unsafe-path.yaml +++ b/integration_tests/protocols/http/raw-unsafe-path.yaml @@ -20,7 +20,7 @@ info: # Test all unsafe URL Handling Edgecases -requests: +http: - raw: # relative path without leading slash - |+ diff --git a/integration_tests/protocols/http/raw-unsafe-request.yaml b/integration_tests/protocols/http/raw-unsafe-request.yaml index e7c45c983b..0f896c52b1 100644 --- a/integration_tests/protocols/http/raw-unsafe-request.yaml +++ b/integration_tests/protocols/http/raw-unsafe-request.yaml @@ -5,7 +5,7 @@ info: author: pd-team severity: info -requests: +http: - raw: - |+ GET / HTTP/1.1 diff --git a/integration_tests/protocols/http/raw-unsafe-with-params.yaml b/integration_tests/protocols/http/raw-unsafe-with-params.yaml index 65ebde2e04..a22d54e121 100644 --- a/integration_tests/protocols/http/raw-unsafe-with-params.yaml +++ b/integration_tests/protocols/http/raw-unsafe-with-params.yaml @@ -7,7 +7,7 @@ info: # this test is used to check automerge of params in both unsafe & safe requests # key1=value1 is added from inputURL -requests: +http: - raw: - |+ GET /?key2=value2 HTTP/1.1 diff --git a/integration_tests/protocols/http/raw-with-params.yaml b/integration_tests/protocols/http/raw-with-params.yaml index a13a0c4ca5..f56051fecf 100644 --- a/integration_tests/protocols/http/raw-with-params.yaml +++ b/integration_tests/protocols/http/raw-with-params.yaml @@ -7,7 +7,7 @@ info: # this test is used to check automerge of params in both unsafe & safe requests # key1=value1 is added from inputURL -requests: +http: - raw: - | GET /?key2=value2 HTTP/1.1 diff --git a/integration_tests/protocols/http/redirect-match-url.yaml b/integration_tests/protocols/http/redirect-match-url.yaml index d24f96d69e..0a9814d77b 100644 --- a/integration_tests/protocols/http/redirect-match-url.yaml +++ b/integration_tests/protocols/http/redirect-match-url.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/protocols/http/request-condition-new.yaml b/integration_tests/protocols/http/request-condition-new.yaml index 1a7fd992e4..9d0fdf33fb 100644 --- a/integration_tests/protocols/http/request-condition-new.yaml +++ b/integration_tests/protocols/http/request-condition-new.yaml @@ -5,7 +5,7 @@ info: author: pd-team severity: info -requests: +http: - method: GET id: first path: diff --git a/integration_tests/protocols/http/request-condition.yaml b/integration_tests/protocols/http/request-condition.yaml index 62e9b1c039..e0601f7997 100644 --- a/integration_tests/protocols/http/request-condition.yaml +++ b/integration_tests/protocols/http/request-condition.yaml @@ -5,7 +5,7 @@ info: author: pd-team severity: info -requests: +http: - method: GET path: - "{{BaseURL}}/200" diff --git a/integration_tests/protocols/http/self-contained-file-input.yaml b/integration_tests/protocols/http/self-contained-file-input.yaml index a5ac4f78e1..3b8a826913 100644 --- a/integration_tests/protocols/http/self-contained-file-input.yaml +++ b/integration_tests/protocols/http/self-contained-file-input.yaml @@ -6,7 +6,7 @@ info: severity: info self-contained: true -requests: +http: - method: GET path: - "http://127.0.0.1:5431/{{test}}" diff --git a/integration_tests/protocols/http/self-contained-with-params.yaml b/integration_tests/protocols/http/self-contained-with-params.yaml index 73665c772c..aad16ee42d 100644 --- a/integration_tests/protocols/http/self-contained-with-params.yaml +++ b/integration_tests/protocols/http/self-contained-with-params.yaml @@ -6,7 +6,7 @@ info: severity: info self-contained: true -requests: +http: - raw: - | GET http://127.0.0.1:5431/?something=here&key=value HTTP/1.1 diff --git a/integration_tests/protocols/http/self-contained-with-path.yaml b/integration_tests/protocols/http/self-contained-with-path.yaml index 1196cd67e6..4ab6dc707d 100644 --- a/integration_tests/protocols/http/self-contained-with-path.yaml +++ b/integration_tests/protocols/http/self-contained-with-path.yaml @@ -6,7 +6,7 @@ info: severity: info self-contained: true -requests: +http: - raw: - | GET / HTTP/1.1 diff --git a/integration_tests/protocols/http/self-contained.yaml b/integration_tests/protocols/http/self-contained.yaml index b3c9aa9939..b25b71a213 100644 --- a/integration_tests/protocols/http/self-contained.yaml +++ b/integration_tests/protocols/http/self-contained.yaml @@ -6,7 +6,7 @@ info: severity: info self-contained: true -requests: +http: - raw: - | GET http://127.0.0.1:5431/ HTTP/1.1 diff --git a/integration_tests/protocols/http/stop-at-first-match-with-extractors.yaml b/integration_tests/protocols/http/stop-at-first-match-with-extractors.yaml index b22e9f4ec9..70d007eb70 100644 --- a/integration_tests/protocols/http/stop-at-first-match-with-extractors.yaml +++ b/integration_tests/protocols/http/stop-at-first-match-with-extractors.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}?a=1" diff --git a/integration_tests/protocols/http/stop-at-first-match.yaml b/integration_tests/protocols/http/stop-at-first-match.yaml index a5a06da78a..1b79339f54 100644 --- a/integration_tests/protocols/http/stop-at-first-match.yaml +++ b/integration_tests/protocols/http/stop-at-first-match.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}?a=1" diff --git a/integration_tests/protocols/http/variable-dsl-function.yaml b/integration_tests/protocols/http/variable-dsl-function.yaml index d6759f34ac..21bc77c23f 100644 --- a/integration_tests/protocols/http/variable-dsl-function.yaml +++ b/integration_tests/protocols/http/variable-dsl-function.yaml @@ -9,7 +9,7 @@ variables: a1: "{{to_lower(rand_base(5))}}" -requests: +http: - method: GET path: - "{{BaseURL}}/?x={{a1}}" diff --git a/integration_tests/protocols/http/variables.yaml b/integration_tests/protocols/http/variables.yaml index 59562b7fdc..d088cfabcb 100644 --- a/integration_tests/protocols/http/variables.yaml +++ b/integration_tests/protocols/http/variables.yaml @@ -9,7 +9,7 @@ variables: a1: "value" a2: "{{base64('{{Host}}')}}" -requests: +http: - raw: - | GET / HTTP/1.1 diff --git a/integration_tests/protocols/offlinehttp/offline-allowed-paths.yaml b/integration_tests/protocols/offlinehttp/offline-allowed-paths.yaml index de861008a9..3d4d298995 100644 --- a/integration_tests/protocols/offlinehttp/offline-allowed-paths.yaml +++ b/integration_tests/protocols/offlinehttp/offline-allowed-paths.yaml @@ -6,7 +6,7 @@ info: severity: info description: offline-allowed-paths -requests: +http: - path: - "{{BaseURL}}" - "{{BaseURL}}/" diff --git a/integration_tests/protocols/offlinehttp/offline-raw.yaml b/integration_tests/protocols/offlinehttp/offline-raw.yaml index c166b94f52..e71c857c3b 100644 --- a/integration_tests/protocols/offlinehttp/offline-raw.yaml +++ b/integration_tests/protocols/offlinehttp/offline-raw.yaml @@ -4,7 +4,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | GET / HTTP/1.1 diff --git a/integration_tests/protocols/offlinehttp/rfc-req-resp.yaml b/integration_tests/protocols/offlinehttp/rfc-req-resp.yaml index 582af906a1..5d65de11bb 100644 --- a/integration_tests/protocols/offlinehttp/rfc-req-resp.yaml +++ b/integration_tests/protocols/offlinehttp/rfc-req-resp.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - method: GET path: - "{{BaseURL}}" diff --git a/integration_tests/workflow/dns-value-share-template-3.yaml b/integration_tests/workflow/dns-value-share-template-3.yaml index 7765a95f0e..2d626a5525 100644 --- a/integration_tests/workflow/dns-value-share-template-3.yaml +++ b/integration_tests/workflow/dns-value-share-template-3.yaml @@ -5,7 +5,7 @@ info: author: pdteam severity: info -requests: +http: - raw: - | GET / HTTP/1.1 diff --git a/internal/pdcp/writer.go b/internal/pdcp/writer.go index fa0fb17251..bd0a7d4712 100644 --- a/internal/pdcp/writer.go +++ b/internal/pdcp/writer.go @@ -13,10 +13,12 @@ import ( "time" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/retryablehttp-go" pdcpauth "github.com/projectdiscovery/utils/auth/pdcp" errorutil "github.com/projectdiscovery/utils/errors" + updateutils "github.com/projectdiscovery/utils/update" urlutil "github.com/projectdiscovery/utils/url" ) @@ -217,6 +219,8 @@ func (u *UploadWriter) getRequest(bin []byte) (*retryablehttp.Request, error) { if err != nil { return nil, errorutil.NewWithErr(err).Msgf("could not create cloud upload request") } + // add pdtm meta params + req.URL.RawQuery = updateutils.GetpdtmParams(config.Version) req.Header.Set(pdcpauth.ApiKeyHeaderName, u.creds.APIKey) req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Accept", "application/json") diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 608b48688b..3c6aaf0b53 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -374,7 +374,7 @@ func (r *Runner) setupPDCPUpload(writer output.Writer) output.Writer { r.options.EnableCloudUpload = true } if !(r.options.EnableCloudUpload || EnableCloudUpload) { - r.pdcpUploadErrMsg = fmt.Sprintf("[%v] Scan results upload to cloud is disabled.", aurora.BrightYellow("WRN")) + r.pdcpUploadErrMsg = fmt.Sprintf("[%v] Scan results upload to cloud is disabled.", r.colorizer.BrightYellow("WRN")) return writer } color := aurora.NewAurora(!r.options.NoColor) @@ -439,6 +439,11 @@ func (r *Runner) RunEnumeration() error { Parser: r.parser, } + if env.GetEnvOrDefault("NUCLEI_ARGS", "") == "req_url_pattern=true" { + // Go StdLib style experimental/debug feature switch + executorOpts.ExportReqURLPattern = true + } + if len(r.options.SecretsFile) > 0 && !r.options.Validate { authTmplStore, err := GetAuthTmplStore(*r.options, r.catalog, executorOpts) if err != nil { @@ -475,10 +480,9 @@ func (r *Runner) RunEnumeration() error { // If using input-file flags, only load http fuzzing based templates. loaderConfig := loader.NewConfig(r.options, r.catalog, executorOpts) - if !strings.EqualFold(r.options.InputFileMode, "list") || r.options.FuzzTemplates { + if !strings.EqualFold(r.options.InputFileMode, "list") || r.options.DAST { // if input type is not list (implicitly enable fuzzing) - r.options.FuzzTemplates = true - loaderConfig.OnlyLoadHTTPFuzzing = true + r.options.DAST = true } store, err := loader.New(loaderConfig) if err != nil { @@ -640,22 +644,31 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { stats.Display(templates.SyntaxWarningStats) stats.Display(templates.SyntaxErrorStats) stats.Display(templates.RuntimeWarningsStats) - if r.options.Verbose { + tmplCount := len(store.Templates()) + workflowCount := len(store.Workflows()) + if r.options.Verbose || (tmplCount == 0 && workflowCount == 0) { // only print these stats in verbose mode - stats.DisplayAsWarning(templates.HeadlessFlagWarningStats) - stats.DisplayAsWarning(templates.CodeFlagWarningStats) - stats.DisplayAsWarning(templates.TemplatesExecutedStats) - stats.DisplayAsWarning(templates.HeadlessFlagWarningStats) - stats.DisplayAsWarning(templates.CodeFlagWarningStats) - stats.DisplayAsWarning(templates.FuzzFlagWarningStats) - stats.DisplayAsWarning(templates.TemplatesExecutedStats) + stats.ForceDisplayWarning(templates.ExcludedHeadlessTmplStats) + stats.ForceDisplayWarning(templates.ExcludedCodeTmplStats) + stats.ForceDisplayWarning(templates.ExludedDastTmplStats) + stats.ForceDisplayWarning(templates.TemplatesExcludedStats) } - stats.DisplayAsWarning(templates.UnsignedCodeWarning) + if tmplCount == 0 && workflowCount == 0 { + // if dast flag is used print explicit warning + if r.options.DAST { + gologger.DefaultLogger.Print().Msgf("[%v] No DAST templates found", aurora.BrightYellow("WRN")) + } + stats.ForceDisplayWarning(templates.SkippedCodeTmplTamperedStats) + } else { + stats.DisplayAsWarning(templates.SkippedCodeTmplTamperedStats) + } stats.ForceDisplayWarning(templates.SkippedUnsignedStats) + stats.ForceDisplayWarning(templates.SkippedRequestSignatureStats) cfg := config.DefaultConfig + updateutils.Aurora = r.colorizer gologger.Info().Msgf("Current nuclei version: %v %v", config.Version, updateutils.GetVersionDescription(config.Version, cfg.LatestNucleiVersion)) gologger.Info().Msgf("Current nuclei-templates version: %v %v", cfg.TemplateVersion, updateutils.GetVersionDescription(cfg.TemplateVersion, cfg.LatestNucleiTemplatesVersion)) if !HideAutoSaveMsg { @@ -666,25 +679,22 @@ func (r *Runner) displayExecutionInfo(store *loader.Store) { } } - if len(store.Templates()) > 0 { - gologger.Info().Msgf("New templates added in latest release: %d", len(config.DefaultConfig.GetNewAdditions())) - gologger.Info().Msgf("Templates loaded for current scan: %d", len(store.Templates())) - } - if len(store.Workflows()) > 0 { - gologger.Info().Msgf("Workflows loaded for current scan: %d", len(store.Workflows())) - } - for k, v := range templates.SignatureStats { - value := v.Load() - if k == templates.Unsigned && value > 0 { - // adjust skipped unsigned templates via code or -dut flag - value = value - uint64(stats.GetValue(templates.SkippedUnsignedStats)) - value = value - uint64(stats.GetValue(templates.CodeFlagWarningStats)) + if tmplCount > 0 || workflowCount > 0 { + if len(store.Templates()) > 0 { + gologger.Info().Msgf("New templates added in latest release: %d", len(config.DefaultConfig.GetNewAdditions())) + gologger.Info().Msgf("Templates loaded for current scan: %d", len(store.Templates())) + } + if len(store.Workflows()) > 0 { + gologger.Info().Msgf("Workflows loaded for current scan: %d", len(store.Workflows())) } - if value > 0 { - if k != templates.Unsigned { - gologger.Info().Msgf("Executing %d signed templates from %s", value, k) - } else if !r.options.Silent && !config.DefaultConfig.HideTemplateSigWarning { - gologger.Print().Msgf("[%v] Loaded %d unsigned templates for scan. Use with caution.", aurora.BrightYellow("WRN"), value) + for k, v := range templates.SignatureStats { + value := v.Load() + if value > 0 { + if k == templates.Unsigned && !r.options.Silent && !config.DefaultConfig.HideTemplateSigWarning { + gologger.Print().Msgf("[%v] Loading %d unsigned templates for scan. Use with caution.", r.colorizer.BrightYellow("WRN"), value) + } else { + gologger.Info().Msgf("Executing %d signed templates from %s", value, k) + } } } } diff --git a/lib/config.go b/lib/config.go index 58fab00a56..52a8e2328a 100644 --- a/lib/config.go +++ b/lib/config.go @@ -376,10 +376,18 @@ func LoadSecretsFromFile(files []string, prefetch bool) NucleiSDKOptions { } } -// EnableFuzzTemplates allows enabling template fuzzing -func EnableFuzzTemplates() NucleiSDKOptions { +// DASTMode only run DAST templates +func DASTMode() NucleiSDKOptions { return func(e *NucleiEngine) error { - e.opts.FuzzTemplates = true + e.opts.DAST = true + return nil + } +} + +// SignedTemplatesOnly only run signed templates and disabled loading all unsigned templates +func SignedTemplatesOnly() NucleiSDKOptions { + return func(e *NucleiEngine) error { + e.opts.DisableUnsignedTemplates = true return nil } } diff --git a/nuclei-jsonschema.json b/nuclei-jsonschema.json index b161393bcb..0f4c2a8f65 100644 --- a/nuclei-jsonschema.json +++ b/nuclei-jsonschema.json @@ -1,111 +1,29 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/templates.Template", - "definitions": { - "fuzz.Rule": { - "properties": { - "type": { - "enum": [ - "replace", - "prefix", - "postfix", - "infix", - "replace-regex" - ], - "type": "string", - "title": "type of rule", - "description": "Type of fuzzing rule to perform" - }, - "part": { - "enum": [ - "query", - "header", - "path", - "body", - "cookie", - "request" - ], - "type": "string", - "title": "part of rule", - "description": "Part of request rule to fuzz" - }, - "mode": { - "enum": [ - "single", - "multiple" - ], - "type": "string", - "title": "mode of rule", - "description": "Mode of request rule to fuzz" - }, - "keys": { - "items": { - "type": "string" - }, - "type": "array", - "title": "keys of parameters to fuzz", - "description": "Keys of parameters to fuzz" - }, - "keys-regex": { - "items": { - "type": "string" - }, - "type": "array", - "title": "keys regex to fuzz", - "description": "Regex of parameter keys to fuzz" - }, - "values": { - "items": { - "type": "string" - }, - "type": "array", - "title": "values regex to fuzz", - "description": "Regex of parameter values to fuzz" - }, - "fuzz": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/fuzz.SliceOrMapSlice", - "title": "payloads of fuzz rule", - "description": "Payloads to perform fuzzing substitutions with" - }, - "replace-regex": { - "type": "string", - "title": "replace regex of rule", - "description": "Regex for regex-replace rule type" - } - }, - "additionalProperties": false, - "type": "object" - }, - "fuzz.SliceOrMapSlice": { - "required": [ - "Value", - "KV" - ], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://template", + "$ref": "#/$defs/Template", + "$defs": { + "AttackTypeHolder": { "properties": { "Value": { - "items": { - "type": "string" - }, - "type": "array" - }, - "KV": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/github.com/projectdiscovery/utils/maps.OrderedMap[string,string]" + "type": "integer" } }, "additionalProperties": false, - "type": "object" + "type": "object", + "required": [ + "Value" + ] }, - "model.Classification": { + "Classification": { "properties": { "cve-id": { - "$ref": "#/definitions/stringslice.StringSlice", + "$ref": "#/$defs/StringOrSlice", "title": "cve ids for the template", "description": "CVE IDs for the template" }, "cwe-id": { - "$ref": "#/definitions/stringslice.StringSlice", + "$ref": "#/$defs/StringOrSlice", "title": "cwe ids for the template", "description": "CWE IDs for the template" }, @@ -120,17 +38,26 @@ "cvss-score": { "type": "number", "title": "cvss score for the template", - "description": "CVSS Score for the template" + "description": "CVSS Score for the template", + "examples": [ + 9.8 + ] }, "epss-score": { "type": "number", "title": "epss score for the template", - "description": "EPSS Score for the template" + "description": "EPSS Score for the template", + "examples": [ + 0.42509 + ] }, "epss-percentile": { "type": "number", "title": "epss percentile for the template", - "description": "EPSS Percentile for the template" + "description": "EPSS Percentile for the template", + "examples": [ + 0.42509 + ] }, "cpe": { "type": "string", @@ -144,128 +71,7 @@ "additionalProperties": false, "type": "object" }, - "model.Info": { - "properties": { - "name": { - "type": "string", - "title": "name of the template", - "description": "Name is a short summary of what the template does", - "examples": [ - "Nagios Default Credentials Check" - ] - }, - "author": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/stringslice.StringSlice", - "title": "author of the template", - "description": "Author is the author of the template" - }, - "tags": { - "$ref": "#/definitions/stringslice.StringSlice", - "title": "tags of the template", - "description": "Any tags for the template" - }, - "description": { - "type": "string", - "title": "description of the template", - "description": "In-depth explanation on what the template does", - "examples": [ - "Bower is a package manager which stores package information in the bower.json file" - ] - }, - "impact": { - "type": "string", - "title": "impact of the template", - "description": "In-depth explanation on the impact of the issue found by the template", - "examples": [ - "Successful exploitation of this vulnerability could allow an attacker to execute arbitrary SQL queries" - ] - }, - "reference": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/stringslice.RawStringSlice", - "title": "references for the template", - "description": "Links relevant to the template" - }, - "severity": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/severity.Holder" - }, - "metadata": { - "patternProperties": { - ".*": { - "additionalProperties": true - } - }, - "type": "object", - "title": "additional metadata for the template", - "description": "Additional metadata fields for the template" - }, - "classification": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/model.Classification", - "title": "classification info for the template", - "description": "Classification information for the template" - }, - "remediation": { - "type": "string", - "title": "remediation steps for the template", - "description": "In-depth explanation on how to fix the issues found by the template", - "examples": [ - "Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties" - ] - } - }, - "additionalProperties": false, - "type": "object" - }, - "severity.Holder": { - "enum": [ - "info", - "low", - "medium", - "high", - "critical", - "unknown" - ], - "type": "string", - "title": "severity of the template", - "description": "Seriousness of the implications of the template" - }, - "stringslice.RawStringSlice": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "array" - } - ] - }, - "stringslice.StringSlice": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "array" - } - ] - }, - "userAgent.UserAgentHolder": { - "enum": [ - "off", - "default", - "custom" - ], - "type": "string", - "title": "userAgent for the headless", - "description": "userAgent for the headless http request" - }, - "extractors.Extractor": { - "required": [ - "type" - ], + "Extractor": { "properties": { "name": { "type": "string", @@ -273,8 +79,7 @@ "description": "Name of the extractor" }, "type": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/extractors.ExtractorTypeHolder" + "$ref": "#/$defs/ExtractorTypeHolder" }, "regex": { "items": { @@ -343,611 +148,272 @@ } }, "additionalProperties": false, - "type": "object" + "type": "object", + "required": [ + "type" + ] }, - "extractors.ExtractorTypeHolder": { - "enum": [ - "regex", - "kval", - "xpath", - "json", - "dsl" - ], - "type": "string", - "title": "type of the extractor", - "description": "Type of the extractor" + "ExtractorTypeHolder": { + "properties": { + "ExtractorType": { + "type": "integer" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "ExtractorType" + ] }, - "matchers.Matcher": { + "HTTPMethodTypeHolder": { + "properties": { + "MethodType": { + "type": "integer" + } + }, + "additionalProperties": false, + "type": "object", "required": [ - "type" + "MethodType" + ] + }, + "Holder": { + "type": "string", + "enum": [ + "info", + "low", + "medium", + "high", + "critical", + "unknown" ], + "title": "severity of the template", + "description": "Seriousness of the implications of the template" + }, + "Info": { "properties": { - "type": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/matchers.MatcherTypeHolder", - "title": "type of matcher", - "description": "Type of the matcher" + "name": { + "type": "string", + "title": "name of the template", + "description": "Name is a short summary of what the template does", + "examples": [ + "Nagios Default Credentials Check" + ] }, - "condition": { - "enum": [ - "and", - "or" + "author": { + "$ref": "#/$defs/StringOrSlice", + "oneOf": [ + { + "type": "string", + "examples": [ + "pdteam" + ] + }, + { + "type": "array", + "examples": [ + "pdteam,mr.robot" + ] + } ], - "type": "string", - "title": "condition between matcher variables", - "description": "Condition between the matcher variables" + "title": "author of the template", + "description": "Author is the author of the template" }, - "part": { - "type": "string", - "title": "part of response to match", - "description": "Part of response to match data from" + "tags": { + "$ref": "#/$defs/StringOrSlice", + "title": "tags of the template", + "description": "Any tags for the template" }, - "negative": { - "type": "boolean", - "title": "negative specifies if match reversed", - "description": "Negative specifies if the match should be reversed. It will only match if the condition is not true" - }, - "name": { - "type": "string", - "title": "name of the matcher", - "description": "Name of the matcher" - }, - "status": { - "items": { - "type": "integer" - }, - "type": "array", - "title": "status to match", - "description": "Status to match for the response" - }, - "size": { - "items": { - "type": "integer" - }, - "type": "array", - "title": "acceptable size for response", - "description": "Size is the acceptable size for the response" - }, - "words": { - "items": { - "type": "string" - }, - "type": "array", - "title": "words to match in response", - "description": " Words contains word patterns required to be present in the response part" - }, - "regex": { - "items": { - "type": "string" - }, - "type": "array", - "title": "regex to match in response", - "description": "Regex contains regex patterns required to be present in the response part" - }, - "binary": { - "items": { - "type": "string" - }, - "type": "array", - "title": "binary patterns to match in response", - "description": "Binary are the binary patterns required to be present in the response part" - }, - "dsl": { - "items": { - "type": "string" - }, - "type": "array", - "title": "dsl expressions to match in response", - "description": "DSL are the dsl expressions that will be evaluated as part of nuclei matching rules" - }, - "xpath": { - "items": { - "type": "string" - }, - "type": "array", - "title": "xpath queries to match in response", - "description": "xpath are the XPath queries that will be evaluated against the response part of nuclei matching rules" - }, - "encoding": { - "enum": [ - "hex" - ], + "description": { "type": "string", - "title": "encoding for word field", - "description": "Optional encoding for the word fields" - }, - "case-insensitive": { - "type": "boolean", - "title": "use case insensitive match", - "description": "use case insensitive match" - }, - "match-all": { - "type": "boolean", - "title": "match all values", - "description": "match all matcher values ignoring condition" - }, - "internal": { - "type": "boolean", - "title": "hide matcher from output", - "description": "hide matcher from output" - } - }, - "additionalProperties": false, - "type": "object" - }, - "matchers.MatcherTypeHolder": { - "enum": [ - "word", - "regex", - "binary", - "status", - "size", - "dsl", - "xpath" - ], - "type": "string", - "title": "type of the matcher", - "description": "Type of the matcher" - }, - "code.Request": { - "properties": { - "matchers": { - "items": { - "$ref": "#/definitions/matchers.Matcher" - }, - "type": "array", - "title": "matchers to run on response", - "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" - }, - "extractors": { - "items": { - "$ref": "#/definitions/extractors.Extractor" - }, - "type": "array", - "title": "extractors to run on response", - "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" + "title": "description of the template", + "description": "In-depth explanation on what the template does", + "examples": [ + "Bower is a package manager which stores package information in the bower.json file" + ] }, - "matchers-condition": { - "enum": [ - "and", - "or" - ], + "impact": { "type": "string", - "title": "condition between the matchers", - "description": "Conditions between the matchers" + "title": "impact of the template", + "description": "In-depth explanation on the impact of the issue found by the template", + "examples": [ + "Successful exploitation of this vulnerability could allow an attacker to execute arbitrary SQL queries" + ] }, - "id": { - "type": "string", - "title": "id of the request", - "description": "ID is the optional ID of the Request" + "reference": { + "$ref": "#/$defs/StringOrSlice", + "title": "references for the template", + "description": "Links relevant to the template" }, - "engine": { - "items": { - "type": "string" - }, - "type": "array", - "title": "engine", - "description": "Engine" + "severity": { + "$ref": "#/$defs/Holder" }, - "args": { - "items": { - "type": "string" - }, - "type": "array", - "title": "args", - "description": "Args" + "metadata": { + "type": "object", + "title": "additional metadata for the template", + "description": "Additional metadata fields for the template" }, - "pattern": { - "type": "string", - "title": "pattern", - "description": "Pattern" + "classification": { + "$ref": "#/$defs/Classification", + "type": "object", + "title": "classification info for the template", + "description": "Classification information for the template" }, - "source": { + "remediation": { "type": "string", - "title": "source file/snippet", - "description": "Source snippet" + "title": "remediation steps for the template", + "description": "In-depth explanation on how to fix the issues found by the template", + "examples": [ + "Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties" + ] } }, "additionalProperties": false, - "type": "object" - }, - "generators.AttackTypeHolder": { - "enum": [ - "batteringram", - "pitchfork", - "clusterbomb" - ], - "type": "string", - "title": "type of the attack", - "description": "Type of the attack" - }, - "variables.Variable": { - "additionalProperties": true, "type": "object", - "title": "variables for the request", - "description": "Additional variables for the request" - }, - "dns.DNSRequestTypeHolder": { - "enum": [ - "A", - "NS", - "DS", - "CNAME", - "SOA", - "PTR", - "MX", - "TXT", - "AAAA", - "CAA", - "TLSA", - "ANY" - ], - "type": "string", - "title": "type of DNS request to make", - "description": "Type is the type of DNS request to make" + "required": [ + "name", + "author" + ] }, - "dns.Request": { + "Matcher": { "properties": { - "matchers": { - "items": { - "$ref": "#/definitions/matchers.Matcher" - }, - "type": "array", - "title": "matchers to run on response", - "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" - }, - "extractors": { - "items": { - "$ref": "#/definitions/extractors.Extractor" - }, - "type": "array", - "title": "extractors to run on response", - "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" + "type": { + "$ref": "#/$defs/MatcherTypeHolder", + "title": "type of matcher", + "description": "Type of the matcher" }, - "matchers-condition": { + "condition": { + "type": "string", "enum": [ "and", "or" ], - "type": "string", - "title": "condition between the matchers", - "description": "Conditions between the matchers" - }, - "id": { - "type": "string", - "title": "id of the dns request", - "description": "ID is the optional ID of the DNS Request" - }, - "name": { - "type": "string", - "title": "hostname to make dns request for", - "description": "Name is the Hostname to make DNS request for" - }, - "type": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/dns.DNSRequestTypeHolder", - "title": "type of dns request to make", - "description": "Type is the type of DNS request to make" + "title": "condition between matcher variables", + "description": "Condition between the matcher variables" }, - "class": { - "enum": [ - "inet", - "csnet", - "chaos", - "hesiod", - "none", - "any" - ], + "part": { "type": "string", - "title": "class of DNS request", - "description": "Class is the class of the DNS request" - }, - "retries": { - "type": "integer", - "title": "retries for dns request", - "description": "Retries is the number of retries for the DNS request" - }, - "trace": { - "type": "boolean", - "title": "trace operation", - "description": "Trace performs a trace operation for the target." - }, - "trace-max-recursion": { - "type": "integer", - "title": "trace-max-recursion level for dns request", - "description": "TraceMaxRecursion is the number of max recursion allowed for trace operations" - }, - "attack": { - "$ref": "#/definitions/generators.AttackTypeHolder", - "title": "attack is the payload combination", - "description": "Attack is the type of payload combinations to perform" - }, - "payloads": { - "patternProperties": { - ".*": { - "additionalProperties": true - } - }, - "type": "object", - "title": "payloads for the network request", - "description": "Payloads contains any payloads for the current request" - }, - "threads": { - "type": "integer", - "title": "threads for sending requests", - "description": "Threads specifies number of threads to use sending requests. This enables Connection Pooling" + "title": "part of response to match", + "description": "Part of response to match data from" }, - "recursion": { + "negative": { "type": "boolean", - "title": "recurse all servers", - "description": "Recursion determines if resolver should recurse all records to get fresh results" - }, - "resolvers": { - "items": { - "type": "string" - }, - "type": "array", - "title": "Resolvers", - "description": "Define resolvers to use within the template" - } - }, - "additionalProperties": false, - "type": "object" - }, - "file.Request": { - "properties": { - "matchers": { - "items": { - "$ref": "#/definitions/matchers.Matcher" - }, - "type": "array", - "title": "matchers to run on response", - "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" - }, - "extractors": { - "items": { - "$ref": "#/definitions/extractors.Extractor" - }, - "type": "array", - "title": "extractors to run on response", - "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" + "title": "negative specifies if match reversed", + "description": "Negative specifies if the match should be reversed. It will only match if the condition is not true" }, - "matchers-condition": { - "enum": [ - "and", - "or" - ], + "name": { "type": "string", - "title": "condition between the matchers", - "description": "Conditions between the matchers" + "title": "name of the matcher", + "description": "Name of the matcher" }, - "extensions": { + "status": { "items": { - "type": "string" + "type": "integer" }, "type": "array", - "title": "extensions to match", - "description": "List of extensions to perform matching on" + "title": "status to match", + "description": "Status to match for the response" }, - "denylist": { + "size": { "items": { - "type": "string" - }, - "type": "array", - "title": "denylist", - "description": "List of files" - }, - "id": { - "type": "string", - "title": "id of the request", - "description": "ID is the optional ID for the request" - }, - "max-size": { - "type": "string", - "title": "max size data to run request on", - "description": "Maximum size of the file to run request on" - }, - "archive": { - "type": "boolean", - "title": "enable archives", - "description": "Process compressed archives without unpacking" - }, - "mime-type": { - "type": "boolean", - "title": "enable filtering by mime-type", - "description": "Filter files by mime-type" - }, - "no-recursive": { - "type": "boolean", - "title": "do not perform recursion", - "description": "Specifies whether to not do recursive checks if folders are provided" - } - }, - "additionalProperties": false, - "type": "object" - }, - "headless.Request": { - "properties": { - "id": { - "type": "string", - "title": "id of the request", - "description": "Optional ID of the headless request" - }, - "attack": { - "$ref": "#/definitions/generators.AttackTypeHolder", - "title": "attack is the payload combination", - "description": "Attack is the type of payload combinations to perform" - }, - "payloads": { - "patternProperties": { - ".*": { - "additionalProperties": true - } + "type": "integer" }, - "type": "object", - "title": "payloads for the headless request", - "description": "Payloads contains any payloads for the current request" + "type": "array", + "title": "acceptable size for response", + "description": "Size is the acceptable size for the response" }, - "steps": { + "words": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/engine.Action" + "type": "string" }, "type": "array", - "title": "list of actions for headless request", - "description": "List of actions to run for headless request" - }, - "user_agent": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/userAgent.UserAgentHolder", - "title": "user agent for the headless request", - "description": "User agent for the headless request" + "title": "words to match in response", + "description": " Words contains word patterns required to be present in the response part" }, - "custom_user_agent": { - "type": "string", - "title": "custom user agent for the headless request", - "description": "Custom user agent for the headless request" + "regex": { + "items": { + "type": "string" + }, + "type": "array", + "title": "regex to match in response", + "description": "Regex contains regex patterns required to be present in the response part" }, - "stop-at-first-match": { - "type": "boolean", - "title": "stop at first match", - "description": "Stop the execution after a match is found" + "binary": { + "items": { + "type": "string" + }, + "type": "array", + "title": "binary patterns to match in response", + "description": "Binary are the binary patterns required to be present in the response part" }, - "matchers": { + "dsl": { "items": { - "$ref": "#/definitions/matchers.Matcher" + "type": "string" }, "type": "array", - "title": "matchers to run on response", - "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" + "title": "dsl expressions to match in response", + "description": "DSL are the dsl expressions that will be evaluated as part of nuclei matching rules" }, - "extractors": { + "xpath": { "items": { - "$ref": "#/definitions/extractors.Extractor" + "type": "string" }, "type": "array", - "title": "extractors to run on response", - "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" + "title": "xpath queries to match in response", + "description": "xpath are the XPath queries that will be evaluated against the response part of nuclei matching rules" }, - "matchers-condition": { + "encoding": { + "type": "string", "enum": [ - "and", - "or" + "hex" ], - "type": "string", - "title": "condition between the matchers", - "description": "Conditions between the matchers" + "title": "encoding for word field", + "description": "Optional encoding for the word fields" }, - "fuzzing": { - "items": { - "$ref": "#/definitions/fuzz.Rule" - }, - "type": "array", - "title": "fuzzin rules for http fuzzing", - "description": "Fuzzing describes rule schema to fuzz headless requests" + "case-insensitive": { + "type": "boolean", + "title": "use case insensitive match", + "description": "use case insensitive match" }, - "cookie-reuse": { + "match-all": { "type": "boolean", - "title": "optional cookie reuse enable", - "description": "Optional setting that enables cookie reuse" + "title": "match all values", + "description": "match all matcher values ignoring condition" }, - "disable-cookie": { + "internal": { "type": "boolean", - "title": "optional disable cookie reuse", - "description": "Optional setting that disables cookie reuse" + "title": "hide matcher from output", + "description": "hide matcher from output" } }, "additionalProperties": false, - "type": "object" - }, - "engine.Action": { + "type": "object", "required": [ - "action" - ], + "type" + ] + }, + "MatcherTypeHolder": { "properties": { - "args": { - "patternProperties": { - ".*": { - "type": "string" - } - }, - "type": "object", - "title": "arguments for headless action", - "description": "Args contain arguments for the headless action" - }, - "name": { - "type": "string", - "title": "name for headless action", - "description": "Name is the name assigned to the headless action" - }, - "description": { - "type": "string", - "title": "description for headless action", - "description": "Description of the headless action" - }, - "action": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/engine.ActionTypeHolder", - "title": "action to perform", - "description": "Type of actions to perform" + "MatcherType": { + "type": "integer" } }, "additionalProperties": false, - "type": "object" - }, - "engine.ActionTypeHolder": { - "enum": [ - "navigate", - "script", - "click", - "rightclick", - "text", - "screenshot", - "time", - "select", - "files", - "waitload", - "getresource", - "extract", - "setmethod", - "addheader", - "setheader", - "deleteheader", - "setbody", - "waitevent", - "keyboard", - "debug", - "sleep", - "waitvisible" - ], - "type": "string", - "title": "action to perform", - "description": "Type of actions to perform" + "type": "object", + "required": [ + "MatcherType" + ] }, - "http.HTTPMethodTypeHolder": { - "enum": [ - "GET", - "HEAD", - "POST", - "PUT", - "DELETE", - "CONNECT", - "OPTIONS", - "TRACE", - "PATCH", - "PURGE", - "DEBUG" - ], - "type": "string", - "title": "method is the HTTP request method", - "description": "Method is the HTTP Request Method" + "OrderedMap[string,string]": { + "properties": {}, + "additionalProperties": false, + "type": "object" }, - "http.Request": { + "Request": { "properties": { "matchers": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/matchers.Matcher" + "$ref": "#/$defs/Matcher" }, "type": "array", "title": "matchers to run on response", @@ -955,19 +421,18 @@ }, "extractors": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/extractors.Extractor" + "$ref": "#/$defs/Extractor" }, "type": "array", "title": "extractors to run on response", "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" }, "matchers-condition": { + "type": "string", "enum": [ "and", "or" ], - "type": "string", "title": "condition between the matchers", "description": "Conditions between the matchers" }, @@ -997,14 +462,12 @@ "description": "Optional name for the HTTP Request" }, "attack": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/generators.AttackTypeHolder", + "$ref": "#/$defs/AttackTypeHolder", "title": "attack is the payload combination", "description": "Attack is the type of payload combinations to perform" }, "method": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/http.HTTPMethodTypeHolder", + "$ref": "#/$defs/HTTPMethodTypeHolder", "title": "method is the http request method", "description": "Method is the HTTP Request Method" }, @@ -1014,20 +477,13 @@ "description": "Body is an optional parameter which contains HTTP Request body" }, "payloads": { - "patternProperties": { - ".*": { - "additionalProperties": true - } - }, "type": "object", "title": "payloads for the http request", "description": "Payloads contains any payloads for the current request" }, "headers": { - "patternProperties": { - ".*": { - "type": "string" - } + "additionalProperties": { + "type": "string" }, "type": "object", "title": "headers to send with the http request", @@ -1065,8 +521,7 @@ }, "fuzzing": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/fuzz.Rule" + "$ref": "#/$defs/Rule" }, "type": "array", "title": "fuzzin rules for http fuzzing", @@ -1076,8 +531,7 @@ "type": "boolean" }, "signature": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/http.SignatureTypeHolder", + "$ref": "#/$defs/SignatureTypeHolder", "title": "signature is the http request signature method", "description": "Signature is the HTTP Request signature Method" }, @@ -1136,524 +590,167 @@ "title": "skip variable checks", "description": "Skips the check for unresolved variables in request" }, - "iterate-all": { - "type": "boolean", - "title": "iterate all the values", - "description": "Iterates all the values extracted from internal extractors" - }, - "digest-username": { - "type": "string", - "title": "specifies the username for digest authentication", - "description": "Optional parameter which specifies the username for digest auth" - }, - "digest-password": { - "type": "string", - "title": "specifies the password for digest authentication", - "description": "Optional parameter which specifies the password for digest auth" - }, - "disable-path-automerge": { - "type": "boolean", - "title": "disable auto merging of path", - "description": "Disable merging target url path with raw request path" - }, - "filters": { - "items": { - "$ref": "#/definitions/matchers.Matcher" - }, - "type": "array", - "title": "filter for fuzzing", - "description": "Filter is matcher-like field to check if fuzzing should be performed on this request or not" - }, - "filters-condition": { - "enum": [ - "and", - "or" - ], - "type": "string", - "title": "condition between the filters", - "description": "Conditions between the filters" - } - }, - "additionalProperties": false, - "type": "object" - }, - "http.SignatureTypeHolder": { - "enum": [ - "AWS" - ], - "type": "string", - "title": "type of the signature", - "description": "Type of the signature" - }, - "javascript.Request": { - "properties": { - "matchers": { - "items": { - "$ref": "#/definitions/matchers.Matcher" - }, - "type": "array", - "title": "matchers to run on response", - "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" - }, - "extractors": { - "items": { - "$ref": "#/definitions/extractors.Extractor" - }, - "type": "array", - "title": "extractors to run on response", - "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" - }, - "matchers-condition": { - "enum": [ - "and", - "or" - ], - "type": "string", - "title": "condition between the matchers", - "description": "Conditions between the matchers" - }, - "id": { - "type": "string", - "title": "id of the request", - "description": "ID is the optional ID of the Request" - }, - "init": { - "type": "string", - "title": "init javascript code", - "description": "Init is the javascript code to execute after compiling template" - }, - "pre-condition": { - "type": "string", - "title": "pre-condition for the request", - "description": "PreCondition is a condition which is evaluated before sending the request" - }, - "args": { - "patternProperties": { - ".*": { - "additionalProperties": true - } - }, - "type": "object" - }, - "code": { - "type": "string", - "title": "code to execute in javascript", - "description": "Executes inline javascript code for the request" - }, - "timeout": { - "type": "integer", - "title": "timeout for javascript execution", - "description": "Timeout in seconds is optional timeout for entire javascript script execution" - }, - "stop-at-first-match": { - "type": "boolean", - "title": "stop at first match", - "description": "Stop the execution after a match is found" - }, - "attack": { - "$ref": "#/definitions/generators.AttackTypeHolder", - "title": "attack is the payload combination", - "description": "Attack is the type of payload combinations to perform" - }, - "threads": { - "type": "integer", - "title": "threads for sending requests", - "description": "Threads specifies number of threads to use sending requests. This enables Connection Pooling" - }, - "payloads": { - "patternProperties": { - ".*": { - "additionalProperties": true - } - }, - "type": "object", - "title": "payloads for the webosocket request", - "description": "Payloads contains any payloads for the current request" - } - }, - "additionalProperties": false, - "type": "object" - }, - "network.Input": { - "properties": { - "data": { - "type": "string", - "title": "data to send as input", - "description": "Data is the data to send as the input" - }, - "type": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/network.NetworkInputTypeHolder", - "title": "type is the type of input data", - "description": "Type of input specified in data field" - }, - "read": { - "type": "integer", - "title": "bytes to read from socket", - "description": "Number of bytes to read from socket" - }, - "name": { - "type": "string", - "title": "optional name for data read", - "description": "Optional name of the data read to provide matching on" - } - }, - "additionalProperties": false, - "type": "object" - }, - "network.NetworkInputTypeHolder": { - "enum": [ - "hex", - "text" - ], - "type": "string", - "title": "type is the type of input data", - "description": "description=Type of input specified in data field" - }, - "network.Request": { - "properties": { - "id": { - "type": "string", - "title": "id of the request", - "description": "ID of the network request" - }, - "host": { - "items": { - "type": "string" - }, - "type": "array", - "title": "host to send requests to", - "description": "Host to send network requests to" - }, - "attack": { - "$ref": "#/definitions/generators.AttackTypeHolder", - "title": "attack is the payload combination", - "description": "Attack is the type of payload combinations to perform" - }, - "payloads": { - "patternProperties": { - ".*": { - "additionalProperties": true - } - }, - "type": "object", - "title": "payloads for the network request", - "description": "Payloads contains any payloads for the current request" - }, - "threads": { - "type": "integer", - "title": "threads for sending requests", - "description": "Threads specifies number of threads to use sending requests. This enables Connection Pooling" - }, - "inputs": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/network.Input" - }, - "type": "array", - "title": "inputs for the network request", - "description": "Inputs contains any input/output for the current request" + "iterate-all": { + "type": "boolean", + "title": "iterate all the values", + "description": "Iterates all the values extracted from internal extractors" }, - "port": { + "digest-username": { "type": "string", - "title": "port to send requests to", - "description": "Port to send network requests to" + "title": "specifies the username for digest authentication", + "description": "Optional parameter which specifies the username for digest auth" }, - "exclude-ports": { + "digest-password": { "type": "string", - "title": "exclude ports from being scanned", - "description": "Exclude ports from being scanned" - }, - "read-size": { - "type": "integer", - "title": "size of network response to read", - "description": "Size of response to read at the end. Default is 1024 bytes" + "title": "specifies the password for digest authentication", + "description": "Optional parameter which specifies the password for digest auth" }, - "read-all": { + "disable-path-automerge": { "type": "boolean", - "title": "read all response stream", - "description": "Read all response stream till the server stops sending" - }, - "matchers": { - "items": { - "$ref": "#/definitions/matchers.Matcher" - }, - "type": "array", - "title": "matchers to run on response", - "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" + "title": "disable auto merging of path", + "description": "Disable merging target url path with raw request path" }, - "extractors": { + "pre-condition": { "items": { - "$ref": "#/definitions/extractors.Extractor" + "$ref": "#/$defs/Matcher" }, "type": "array", - "title": "extractors to run on response", - "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" + "title": "pre-condition for fuzzing/dast", + "description": "PreCondition is matcher-like field to check if fuzzing should be performed on this request or not" }, - "matchers-condition": { + "pre-condition-operator": { + "type": "string", "enum": [ "and", "or" ], - "type": "string", - "title": "condition between the matchers", - "description": "Conditions between the matchers" + "title": "condition between the filters", + "description": "Operator to use between multiple per-conditions" } }, "additionalProperties": false, "type": "object" }, - "ssl.Request": { + "Rule": { "properties": { - "matchers": { - "items": { - "$ref": "#/definitions/matchers.Matcher" - }, - "type": "array", - "title": "matchers to run on response", - "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" - }, - "extractors": { - "items": { - "$ref": "#/definitions/extractors.Extractor" - }, - "type": "array", - "title": "extractors to run on response", - "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" - }, - "matchers-condition": { + "type": { + "type": "string", "enum": [ - "and", - "or" + "replace", + "prefix", + "postfix", + "infix", + "replace-regex" ], - "type": "string", - "title": "condition between the matchers", - "description": "Conditions between the matchers" - }, - "id": { - "type": "string", - "title": "id of the request", - "description": "ID of the request" + "title": "type of rule", + "description": "Type of fuzzing rule to perform" }, - "address": { + "part": { "type": "string", - "title": "address for the ssl request", - "description": "Address contains address for the request" - }, - "min_version": { "enum": [ - "sslv3", - "tls10", - "tls11", - "tls12", - "tls13" + "query", + "header", + "path", + "body", + "cookie", + "request" ], - "type": "string", - "title": "Min. TLS version", - "description": "Minimum tls version - automatic if not specified." + "title": "part of rule", + "description": "Part of request rule to fuzz" }, - "max_version": { + "mode": { + "type": "string", "enum": [ - "sslv3", - "tls10", - "tls11", - "tls12", - "tls13" + "single", + "multiple" ], - "type": "string", - "title": "Max. TLS version", - "description": "Max tls version - automatic if not specified." + "title": "mode of rule", + "description": "Mode of request rule to fuzz" }, - "cipher_suites": { + "keys": { "items": { "type": "string" }, - "type": "array" - }, - "scan_mode": { - "enum": [ - "ctls", - "ztls", - "auto" - ], - "type": "string", - "title": "Scan Mode", - "description": "Scan Mode - auto if not specified." - }, - "tls_version_enum": { - "type": "boolean", - "title": "Enumerate Versions", - "description": "Enumerate Version - false if not specified" + "type": "array", + "title": "keys of parameters to fuzz", + "description": "Keys of parameters to fuzz" }, - "tls_cipher_enum": { - "type": "boolean", - "title": "Enumerate Ciphers", - "description": "Enumerate Ciphers - false if not specified" + "keys-regex": { + "items": { + "type": "string" + }, + "type": "array", + "title": "keys regex to fuzz", + "description": "Regex of parameter keys to fuzz" }, - "tls_cipher_types": { + "values": { "items": { - "enum": [ - "weak", - "secure", - "insecure", - "all" - ], "type": "string" }, "type": "array", - "title": "TLS Cipher Types", - "description": "TLS Cipher Types to enumerate" + "title": "values regex to fuzz", + "description": "Regex of parameter values to fuzz" + }, + "fuzz": { + "$ref": "#/$defs/SliceOrMapSlice", + "title": "payloads of fuzz rule", + "description": "Payloads to perform fuzzing substitutions with" + }, + "replace-regex": { + "type": "string", + "title": "replace regex of rule", + "description": "Regex for regex-replace rule type" } }, "additionalProperties": false, "type": "object" }, - "websocket.Input": { + "SignatureTypeHolder": { "properties": { - "data": { - "type": "string", - "title": "data to send as input", - "description": "Data is the data to send as the input" - }, - "name": { - "type": "string", - "title": "optional name for data read", - "description": "Optional name of the data read to provide matching on" + "Value": { + "type": "integer" } }, "additionalProperties": false, - "type": "object" + "type": "object", + "required": [ + "Value" + ] }, - "websocket.Request": { + "SliceOrMapSlice": { "properties": { - "matchers": { - "items": { - "$ref": "#/definitions/matchers.Matcher" - }, - "type": "array", - "title": "matchers to run on response", - "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" - }, - "extractors": { - "items": { - "$ref": "#/definitions/extractors.Extractor" - }, - "type": "array", - "title": "extractors to run on response", - "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" - }, - "matchers-condition": { - "enum": [ - "and", - "or" - ], - "type": "string", - "title": "condition between the matchers", - "description": "Conditions between the matchers" - }, - "id": { - "type": "string", - "title": "id of the request", - "description": "ID of the network request" - }, - "address": { - "type": "string", - "title": "address for the websocket request", - "description": "Address contains address for the request" - }, - "inputs": { + "Value": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/websocket.Input" - }, - "type": "array", - "title": "inputs for the websocket request", - "description": "Inputs contains any input/output for the current request" - }, - "headers": { - "patternProperties": { - ".*": { - "type": "string" - } + "type": "string" }, - "type": "object", - "title": "headers contains the request headers", - "description": "Headers contains headers for the request" - }, - "attack": { - "$ref": "#/definitions/generators.AttackTypeHolder", - "title": "attack is the payload combination", - "description": "Attack is the type of payload combinations to perform" + "type": "array" }, - "payloads": { - "patternProperties": { - ".*": { - "additionalProperties": true - } - }, - "type": "object", - "title": "payloads for the websocket request", - "description": "Payloads contains any payloads for the current request" + "KV": { + "$ref": "#/$defs/OrderedMap[string,string]" } }, "additionalProperties": false, - "type": "object" + "type": "object", + "required": [ + "Value", + "KV" + ] }, - "whois.Request": { - "properties": { - "matchers": { - "items": { - "$ref": "#/definitions/matchers.Matcher" - }, - "type": "array", - "title": "matchers to run on response", - "description": "Detection mechanism to identify whether the request was successful by doing pattern matching" - }, - "extractors": { - "items": { - "$ref": "#/definitions/extractors.Extractor" - }, - "type": "array", - "title": "extractors to run on response", - "description": "Extractors contains the extraction mechanism for the request to identify and extract parts of the response" - }, - "matchers-condition": { - "enum": [ - "and", - "or" - ], - "type": "string", - "title": "condition between the matchers", - "description": "Conditions between the matchers" - }, - "id": { - "type": "string", - "title": "id of the request", - "description": "ID of the network request" - }, - "query": { - "type": "string", - "title": "query for the WHOIS request", - "description": "Query contains query for the request" + "StringOrSlice": { + "oneOf": [ + { + "type": "string" }, - "server": { - "type": "string", - "title": "server url to execute the WHOIS request on", - "description": "Server contains the server url to execute the WHOIS request on" + { + "type": "array" } - }, - "additionalProperties": false, - "type": "object" + ] }, - "templates.Template": { - "required": [ - "id", - "info" - ], + "Template": { "properties": { "id": { - "pattern": "^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$", "type": "string", + "pattern": "^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$", "title": "id of the template", "description": "The Unique ID for the template", "examples": [ @@ -1661,20 +758,22 @@ ] }, "info": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/model.Info", + "$ref": "#/$defs/Info", + "type": "object", "title": "info for the template", "description": "Info contains metadata for the template" }, "flow": { "type": "string", "title": "template execution flow in js", - "description": "Flow contains js code which defines how the template should be executed" + "description": "Flow contains js code which defines how the template should be executed", + "examples": [ + "'flow: http(0) \u0026\u0026 http(1)'" + ] }, "requests": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/http.Request" + "$ref": "#/$defs/Request" }, "type": "array", "title": "http requests to make", @@ -1682,7 +781,7 @@ }, "http": { "items": { - "$ref": "#/definitions/http.Request" + "$ref": "#/$defs/Request" }, "type": "array", "title": "http requests to make", @@ -1690,8 +789,7 @@ }, "dns": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/dns.Request" + "$ref": "#/$defs/Request" }, "type": "array", "title": "dns requests to make", @@ -1699,8 +797,7 @@ }, "file": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/file.Request" + "$ref": "#/$defs/Request" }, "type": "array", "title": "file requests to make", @@ -1708,8 +805,7 @@ }, "network": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/network.Request" + "$ref": "#/$defs/Request" }, "type": "array", "title": "network requests to make", @@ -1717,7 +813,7 @@ }, "tcp": { "items": { - "$ref": "#/definitions/network.Request" + "$ref": "#/$defs/Request" }, "type": "array", "title": "network(tcp) requests to make", @@ -1725,8 +821,7 @@ }, "headless": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/headless.Request" + "$ref": "#/$defs/Request" }, "type": "array", "title": "headless requests to make", @@ -1734,8 +829,7 @@ }, "ssl": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/ssl.Request" + "$ref": "#/$defs/Request" }, "type": "array", "title": "ssl requests to make", @@ -1743,8 +837,7 @@ }, "websocket": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/websocket.Request" + "$ref": "#/$defs/Request" }, "type": "array", "title": "websocket requests to make", @@ -1752,8 +845,7 @@ }, "whois": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/whois.Request" + "$ref": "#/$defs/Request" }, "type": "array", "title": "whois requests to make", @@ -1761,8 +853,7 @@ }, "code": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/code.Request" + "$ref": "#/$defs/Request" }, "type": "array", "title": "code snippets to make", @@ -1770,8 +861,7 @@ }, "javascript": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/javascript.Request" + "$ref": "#/$defs/Request" }, "type": "array", "title": "javascript requests to make", @@ -1779,8 +869,7 @@ }, "workflows": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/workflows.WorkflowTemplate" + "$ref": "#/$defs/WorkflowTemplate" }, "type": "array", "title": "list of workflows to execute", @@ -1797,59 +886,35 @@ "description": "Stop at first match for the template" }, "signature": { - "$ref": "#/definitions/http.SignatureTypeHolder", + "$ref": "#/$defs/SignatureTypeHolder", "title": "signature is the http request signature method", "description": "Signature is the HTTP Request signature Method" }, "variables": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/variables.Variable", + "$ref": "#/$defs/Variable", + "type": "object", "title": "variables for the http request", "description": "Variables contains any variables for the current request" }, "constants": { - "patternProperties": { - ".*": { - "additionalProperties": true - } - }, "type": "object", "title": "constant for the template", "description": "constants contains any constant for the template" } }, "additionalProperties": false, - "type": "object" + "type": "object", + "required": [ + "id", + "info" + ] }, - "workflows.Matcher": { - "properties": { - "name": { - "$ref": "#/definitions/stringslice.StringSlice", - "title": "name of items to match", - "description": "Name of items to match" - }, - "condition": { - "enum": [ - "and", - "or" - ], - "type": "string", - "title": "condition between names", - "description": "Condition between the names" - }, - "subtemplates": { - "items": { - "$ref": "#/definitions/workflows.WorkflowTemplate" - }, - "type": "array", - "title": "templates to run after match", - "description": "Templates to run after match" - } - }, + "Variable": { + "properties": {}, "additionalProperties": false, "type": "object" }, - "workflows.WorkflowTemplate": { + "WorkflowTemplate": { "properties": { "template": { "type": "string", @@ -1857,14 +922,13 @@ "description": "Template or directory to execute as part of workflow" }, "tags": { - "$ref": "#/definitions/stringslice.StringSlice", + "$ref": "#/$defs/StringOrSlice", "title": "tags to execute", "description": "Tags to run template based on" }, "matchers": { "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "$ref": "#/definitions/workflows.Matcher" + "$ref": "#/$defs/Matcher" }, "type": "array", "title": "name based template result matchers", @@ -1872,7 +936,7 @@ }, "subtemplates": { "items": { - "$ref": "#/definitions/workflows.WorkflowTemplate" + "$ref": "#/$defs/WorkflowTemplate" }, "type": "array", "title": "subtemplate based result matchers", @@ -1881,11 +945,6 @@ }, "additionalProperties": false, "type": "object" - }, - "github.com/projectdiscovery/utils/maps.OrderedMap[string,string]": { - "properties": {}, - "additionalProperties": false, - "type": "object" } } } diff --git a/pkg/catalog/config/constants.go b/pkg/catalog/config/constants.go index 287371e96f..9c1bacdca9 100644 --- a/pkg/catalog/config/constants.go +++ b/pkg/catalog/config/constants.go @@ -31,7 +31,7 @@ const ( CLIConfigFileName = "config.yaml" ReportingConfigFilename = "reporting-config.yaml" // Version is the current version of nuclei - Version = `v3.2.2` + Version = `v3.2.3` // Directory Names of custom templates CustomS3TemplatesDirName = "s3" CustomGitHubTemplatesDirName = "github" diff --git a/pkg/catalog/loader/loader.go b/pkg/catalog/loader/loader.go index 1884fa338e..711008f2f0 100644 --- a/pkg/catalog/loader/loader.go +++ b/pkg/catalog/loader/loader.go @@ -13,7 +13,6 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/catalog" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" - cfg "github.com/projectdiscovery/nuclei/v3/pkg/catalog/config" "github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader/filter" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" @@ -61,8 +60,6 @@ type Config struct { Catalog catalog.Catalog ExecutorOptions protocols.ExecutorOptions - - OnlyLoadHTTPFuzzing bool } // Store is a storage for loaded nuclei templates @@ -389,6 +386,21 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ templatePathMap := store.pathFilter.Match(includedTemplates) loadedTemplates := make([]*templates.Template, 0, len(templatePathMap)) + + loadTemplate := func(tmpl *templates.Template) { + loadedTemplates = append(loadedTemplates, tmpl) + // increment signed/unsigned counters + if tmpl.Verified { + if tmpl.TemplateVerifier == "" { + templates.SignatureStats[templates.PDVerifier].Add(1) + } else { + templates.SignatureStats[tmpl.TemplateVerifier].Add(1) + } + } else { + templates.SignatureStats[templates.Unsigned].Add(1) + } + } + for templatePath := range templatePathMap { loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, tags, store.config.Catalog) if loaded || store.pathFilter.MatchIncluded(templatePath) { @@ -405,42 +417,51 @@ func (store *Store) LoadTemplatesWithTags(templatesList, tags []string) []*templ stats.Increment(templates.SkippedUnsignedStats) continue } - if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless { + // if template has request signature like aws then only signed and verified templates are allowed + if parsed.UsesRequestSignature() && !parsed.Verified { + stats.Increment(templates.SkippedRequestSignatureStats) + continue + } + // DAST only templates + if store.config.ExecutorOptions.Options.DAST { + // check if the template is a DAST template + if parsed.IsFuzzing() { + loadTemplate(parsed) + } + } else if len(parsed.RequestsHeadless) > 0 && !store.config.ExecutorOptions.Options.Headless { // donot include headless template in final list if headless flag is not set - stats.Increment(templates.HeadlessFlagWarningStats) + stats.Increment(templates.ExcludedHeadlessTmplStats) if config.DefaultConfig.LogAllEvents { gologger.Print().Msgf("[%v] Headless flag is required for headless template '%s'.\n", aurora.Yellow("WRN").String(), templatePath) } } else if len(parsed.RequestsCode) > 0 && !store.config.ExecutorOptions.Options.EnableCodeTemplates { // donot include 'Code' protocol custom template in final list if code flag is not set - stats.Increment(templates.CodeFlagWarningStats) + stats.Increment(templates.ExcludedCodeTmplStats) if config.DefaultConfig.LogAllEvents { gologger.Print().Msgf("[%v] Code flag is required for code protocol template '%s'.\n", aurora.Yellow("WRN").String(), templatePath) } } else if len(parsed.RequestsCode) > 0 && !parsed.Verified && len(parsed.Workflows) == 0 { // donot include unverified 'Code' protocol custom template in final list - stats.Increment(templates.UnsignedCodeWarning) + stats.Increment(templates.SkippedCodeTmplTamperedStats) // these will be skipped so increment skip counter stats.Increment(templates.SkippedUnsignedStats) if config.DefaultConfig.LogAllEvents { gologger.Print().Msgf("[%v] Tampered/Unsigned template at %v.\n", aurora.Yellow("WRN").String(), templatePath) } - } else if parsed.IsFuzzing() && !store.config.ExecutorOptions.Options.FuzzTemplates { - stats.Increment(templates.FuzzFlagWarningStats) + } else if parsed.IsFuzzing() && !store.config.ExecutorOptions.Options.DAST { + stats.Increment(templates.ExludedDastTmplStats) if config.DefaultConfig.LogAllEvents { - gologger.Print().Msgf("[%v] Fuzz flag is required for fuzzing template '%s'.\n", aurora.Yellow("WRN").String(), templatePath) + gologger.Print().Msgf("[%v] -dast flag is required for DAST template '%s'.\n", aurora.Yellow("WRN").String(), templatePath) } - } else if store.config.OnlyLoadHTTPFuzzing && !parsed.IsFuzzing() { - gologger.Warning().Msgf("Non-Fuzzing template '%s' can only be run on list input mode targets\n", templatePath) } else { - loadedTemplates = append(loadedTemplates, parsed) + loadTemplate(parsed) } } } if err != nil { if strings.Contains(err.Error(), templates.ErrExcluded.Error()) { - stats.Increment(templates.TemplatesExecutedStats) - if cfg.DefaultConfig.LogAllEvents { + stats.Increment(templates.TemplatesExcludedStats) + if config.DefaultConfig.LogAllEvents { gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error()) } continue diff --git a/pkg/fuzz/component/body.go b/pkg/fuzz/component/body.go index 79d8d6a881..9d5bbe66c8 100644 --- a/pkg/fuzz/component/body.go +++ b/pkg/fuzz/component/body.go @@ -55,16 +55,17 @@ func (b *Body) Parse(req *retryablehttp.Request) (bool, error) { } b.value = NewValue(dataStr) - if b.value.Parsed() != nil { + tmp := b.value.Parsed() + if !tmp.IsNIL() { return true, nil } switch { - case strings.Contains(contentType, "application/json") && b.value.Parsed() == nil: + case strings.Contains(contentType, "application/json") && tmp.IsNIL(): return b.parseBody(dataformat.JSONDataFormat, req) - case strings.Contains(contentType, "application/xml") && b.value.Parsed() == nil: + case strings.Contains(contentType, "application/xml") && tmp.IsNIL(): return b.parseBody(dataformat.XMLDataFormat, req) - case strings.Contains(contentType, "multipart/form-data") && b.value.Parsed() == nil: + case strings.Contains(contentType, "multipart/form-data") && tmp.IsNIL(): return b.parseBody(dataformat.MultiPartFormDataFormat, req) } parsed, err := b.parseBody(dataformat.FormDataFormat, req) @@ -93,16 +94,18 @@ func (b *Body) parseBody(decoderName string, req *retryablehttp.Request) (bool, } // Iterate iterates through the component -func (b *Body) Iterate(callback func(key string, value interface{}) error) error { - for key, value := range b.value.Parsed() { +func (b *Body) Iterate(callback func(key string, value interface{}) error) (errx error) { + b.value.parsed.Iterate(func(key string, value any) bool { if strings.HasPrefix(key, "#_") { - continue + return true } if err := callback(key, value); err != nil { - return err + errx = err + return false } - } - return nil + return true + }) + return } // SetValue sets a value in the component @@ -138,3 +141,10 @@ func (b *Body) Rebuild() (*retryablehttp.Request, error) { cloned.Header.Set("Content-Length", strconv.Itoa(len(encoded))) return cloned, nil } + +func (b *Body) Clone() Component { + return &Body{ + value: b.value.Clone(), + req: b.req.Clone(context.Background()), + } +} diff --git a/pkg/fuzz/component/body_test.go b/pkg/fuzz/component/body_test.go index 42e7e405d3..1cfcac8352 100644 --- a/pkg/fuzz/component/body_test.go +++ b/pkg/fuzz/component/body_test.go @@ -4,11 +4,11 @@ import ( "bytes" "io" "mime/multipart" - "net/url" "strings" "testing" "github.com/projectdiscovery/retryablehttp-go" + urlutil "github.com/projectdiscovery/utils/url" "github.com/stretchr/testify/require" ) @@ -80,7 +80,7 @@ func TestBodyXMLComponent(t *testing.T) { } func TestBodyFormComponent(t *testing.T) { - formData := url.Values{} + formData := urlutil.NewOrderedParams() formData.Set("key1", "value1") formData.Set("key2", "value2") diff --git a/pkg/fuzz/component/component.go b/pkg/fuzz/component/component.go index 5a8279c441..a15ac2856a 100644 --- a/pkg/fuzz/component/component.go +++ b/pkg/fuzz/component/component.go @@ -46,6 +46,8 @@ type Component interface { // Rebuild returns a new request with the // component rebuilt Rebuild() (*retryablehttp.Request, error) + // Clones current state of this component + Clone() Component } const ( diff --git a/pkg/fuzz/component/cookie.go b/pkg/fuzz/component/cookie.go index 7269284ccc..77667c7479 100644 --- a/pkg/fuzz/component/cookie.go +++ b/pkg/fuzz/component/cookie.go @@ -2,9 +2,12 @@ package component import ( "context" + "fmt" "net/http" + "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat" "github.com/projectdiscovery/retryablehttp-go" + mapsutil "github.com/projectdiscovery/utils/maps" ) // Cookie is a component for a request cookie @@ -35,29 +38,31 @@ func (c *Cookie) Parse(req *retryablehttp.Request) (bool, error) { c.req = req c.value = NewValue("") - parsedCookies := make(map[string]interface{}) + parsedCookies := mapsutil.NewOrderedMap[string, any]() for _, cookie := range req.Cookies() { - parsedCookies[cookie.Name] = cookie.Value + parsedCookies.Set(cookie.Name, cookie.Value) } - if len(parsedCookies) == 0 { + if parsedCookies.Len() == 0 { return false, nil } - c.value.SetParsed(parsedCookies, "") + c.value.SetParsed(dataformat.KVOrderedMap(&parsedCookies), "") return true, nil } // Iterate iterates through the component -func (c *Cookie) Iterate(callback func(key string, value interface{}) error) error { - for key, value := range c.value.Parsed() { +func (c *Cookie) Iterate(callback func(key string, value interface{}) error) (err error) { + c.value.parsed.Iterate(func(key string, value any) bool { // Skip ignored cookies if _, ok := defaultIgnoredCookieKeys[key]; ok { - continue + return ok } - if err := callback(key, value); err != nil { - return err + if errx := callback(key, value); errx != nil { + err = errx + return false } - } - return nil + return true + }) + return } // SetValue sets a value in the component @@ -83,16 +88,25 @@ func (c *Cookie) Rebuild() (*retryablehttp.Request, error) { cloned := c.req.Clone(context.Background()) cloned.Header.Del("Cookie") - for key, value := range c.value.Parsed() { + c.value.parsed.Iterate(func(key string, value any) bool { cookie := &http.Cookie{ Name: key, - Value: value.(string), // Assume the value is always a string for cookies + Value: fmt.Sprint(value), // Assume the value is always a string for cookies } cloned.AddCookie(cookie) - } + return true + }) return cloned, nil } +// Clone clones current state of this component +func (c *Cookie) Clone() Component { + return &Cookie{ + value: c.value.Clone(), + req: c.req.Clone(context.Background()), + } +} + // A list of cookies that are essential to the request and // must not be fuzzed. var defaultIgnoredCookieKeys = map[string]struct{}{ diff --git a/pkg/fuzz/component/headers.go b/pkg/fuzz/component/headers.go index 60ac980480..fa0f5e1126 100644 --- a/pkg/fuzz/component/headers.go +++ b/pkg/fuzz/component/headers.go @@ -4,6 +4,7 @@ import ( "context" "strings" + "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat" "github.com/projectdiscovery/retryablehttp-go" ) @@ -40,22 +41,24 @@ func (q *Header) Parse(req *retryablehttp.Request) (bool, error) { } parsedHeaders[key] = value } - q.value.SetParsed(parsedHeaders, "") + q.value.SetParsed(dataformat.KVMap(parsedHeaders), "") return true, nil } // Iterate iterates through the component -func (q *Header) Iterate(callback func(key string, value interface{}) error) error { - for key, value := range q.value.Parsed() { +func (q *Header) Iterate(callback func(key string, value interface{}) error) (errx error) { + q.value.parsed.Iterate(func(key string, value any) bool { // Skip ignored headers if _, ok := defaultIgnoredHeaderKeys[key]; ok { - continue + return ok } if err := callback(key, value); err != nil { - return err + errx = err + return false } - } - return nil + return true + }) + return } // SetValue sets a value in the component @@ -79,25 +82,34 @@ func (q *Header) Delete(key string) error { // component rebuilt func (q *Header) Rebuild() (*retryablehttp.Request, error) { cloned := q.req.Clone(context.Background()) - for key, value := range q.value.parsed { + q.value.parsed.Iterate(func(key string, value any) bool { if strings.EqualFold(key, "Host") { - cloned.Host = value.(string) + return true + } + if vx, ok := IsTypedSlice(value); ok { + // convert to []interface{} + value = vx } - switch v := value.(type) { - case []interface{}: + if v, ok := value.([]interface{}); ok { for _, vv := range v { - if cloned.Header[key] == nil { - cloned.Header[key] = make([]string, 0) - } - cloned.Header[key] = append(cloned.Header[key], vv.(string)) + cloned.Header.Add(key, vv.(string)) } - case string: - cloned.Header[key] = []string{v} + return true } - } + cloned.Header.Set(key, value.(string)) + return true + }) return cloned, nil } +// Clones current state of this component +func (q *Header) Clone() Component { + return &Header{ + value: q.value.Clone(), + req: q.req.Clone(context.Background()), + } +} + // A list of headers that are essential to the request and // must not be fuzzed. var defaultIgnoredHeaderKeys = map[string]struct{}{ diff --git a/pkg/fuzz/component/path.go b/pkg/fuzz/component/path.go index c1d31cee69..9d2e758025 100644 --- a/pkg/fuzz/component/path.go +++ b/pkg/fuzz/component/path.go @@ -42,13 +42,15 @@ func (q *Path) Parse(req *retryablehttp.Request) (bool, error) { } // Iterate iterates through the component -func (q *Path) Iterate(callback func(key string, value interface{}) error) error { - for key, value := range q.value.Parsed() { - if err := callback(key, value); err != nil { - return err +func (q *Path) Iterate(callback func(key string, value interface{}) error) (err error) { + q.value.parsed.Iterate(func(key string, value any) bool { + if errx := callback(key, value); errx != nil { + err = errx + return false } - } - return nil + return true + }) + return } // SetValue sets a value in the component @@ -81,3 +83,11 @@ func (q *Path) Rebuild() (*retryablehttp.Request, error) { } return cloned, nil } + +// Clones current state to a new component +func (q *Path) Clone() Component { + return &Path{ + value: q.value.Clone(), + req: q.req.Clone(context.Background()), + } +} diff --git a/pkg/fuzz/component/query.go b/pkg/fuzz/component/query.go index 0952c97986..3dc07e6cf1 100644 --- a/pkg/fuzz/component/query.go +++ b/pkg/fuzz/component/query.go @@ -47,13 +47,15 @@ func (q *Query) Parse(req *retryablehttp.Request) (bool, error) { } // Iterate iterates through the component -func (q *Query) Iterate(callback func(key string, value interface{}) error) error { - for key, value := range q.value.Parsed() { +func (q *Query) Iterate(callback func(key string, value interface{}) error) (errx error) { + q.value.parsed.Iterate(func(key string, value interface{}) bool { if err := callback(key, value); err != nil { - return err + errx = err + return false } - } - return nil + return true + }) + return } // SetValue sets a value in the component @@ -90,3 +92,11 @@ func (q *Query) Rebuild() (*retryablehttp.Request, error) { cloned.Update() return cloned, nil } + +// Clones current state to a new component +func (q *Query) Clone() Component { + return &Query{ + value: q.value.Clone(), + req: q.req.Clone(context.Background()), + } +} diff --git a/pkg/fuzz/component/value.go b/pkg/fuzz/component/value.go index c6d7a9603a..ad2044cf1b 100644 --- a/pkg/fuzz/component/value.go +++ b/pkg/fuzz/component/value.go @@ -1,9 +1,11 @@ package component import ( + "reflect" "strconv" "github.com/leslie-qiwa/flat" + "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/fuzz/dataformat" ) @@ -15,7 +17,7 @@ import ( // all the data values that are used in a request. type Value struct { data string - parsed map[string]interface{} + parsed dataformat.KV dataFormat string } @@ -34,40 +36,58 @@ func NewValue(data string) *Value { return v } +// Clones current state of this value +func (v *Value) Clone() *Value { + return &Value{ + data: v.data, + parsed: v.parsed.Clone(), + dataFormat: v.dataFormat, + } +} + // String returns the string representation of the value func (v *Value) String() string { return v.data } // Parsed returns the parsed value -func (v *Value) Parsed() map[string]interface{} { +func (v *Value) Parsed() dataformat.KV { return v.parsed } // SetParsed sets the parsed value map -func (v *Value) SetParsed(parsed map[string]interface{}, dataFormat string) { +func (v *Value) SetParsed(data dataformat.KV, dataFormat string) { + v.dataFormat = dataFormat + if data.OrderedMap != nil { + v.parsed = data + return + } + parsed := data.Map flattened, err := flat.Flatten(parsed, flatOpts) if err == nil { - v.parsed = flattened + v.parsed = dataformat.KVMap(flattened) } else { - v.parsed = parsed + v.parsed = dataformat.KVMap(parsed) } - v.dataFormat = dataFormat } // SetParsedValue sets the parsed value for a key // in the parsed map func (v *Value) SetParsedValue(key string, value string) bool { - origValue, ok := v.parsed[key] - if !ok { - v.parsed[key] = value + origValue := v.parsed.Get(key) + if origValue == nil { + v.parsed.Set(key, value) return true } // If the value is a list, append to it // otherwise replace it switch v := origValue.(type) { case []interface{}: - origValue = append(v, value) + // update last value + if len(v) > 0 { + v[len(v)-1] = value + } + origValue = v case string: origValue = value case int, int32, int64, float32, float64: @@ -82,35 +102,49 @@ func (v *Value) SetParsedValue(key string, value string) bool { return false } origValue = parsed - case nil: - origValue = value default: - gologger.Error().Msgf("unknown type %T for value %s", v, v) + // explicitly check for typed slice + if val, ok := IsTypedSlice(v); ok { + if len(val) > 0 { + val[len(val)-1] = value + } + origValue = val + } else { + // make it default warning instead of error + gologger.DefaultLogger.Print().Msgf("[%v] unknown type %T for value %s", aurora.BrightYellow("WARN"), v, v) + } } - v.parsed[key] = origValue + v.parsed.Set(key, origValue) return true } // Delete removes a key from the parsed value func (v *Value) Delete(key string) bool { - if _, ok := v.parsed[key]; !ok { - return false - } - delete(v.parsed, key) - return true + return v.parsed.Delete(key) } // Encode encodes the value into a string // using the dataformat and encoding func (v *Value) Encode() (string, error) { toEncodeStr := v.data + if v.parsed.OrderedMap != nil { + // flattening orderedmap not supported + if v.dataFormat != "" { + dataformatStr, err := dataformat.Encode(v.parsed, v.dataFormat) + if err != nil { + return "", err + } + toEncodeStr = dataformatStr + } + return toEncodeStr, nil + } - nested, err := flat.Unflatten(v.parsed, flatOpts) + nested, err := flat.Unflatten(v.parsed.Map, flatOpts) if err != nil { return "", err } if v.dataFormat != "" { - dataformatStr, err := dataformat.Encode(nested, v.dataFormat) + dataformatStr, err := dataformat.Encode(dataformat.KVMap(nested), v.dataFormat) if err != nil { return "", err } @@ -118,3 +152,18 @@ func (v *Value) Encode() (string, error) { } return toEncodeStr, nil } + +// In go, []int, []string are not implictily converted to []interface{} +// when using type assertion and they need to be handled separately. +func IsTypedSlice(v interface{}) ([]interface{}, bool) { + if reflect.ValueOf(v).Kind() == reflect.Slice { + // iterate and convert to []interface{} + slice := reflect.ValueOf(v) + interfaceSlice := make([]interface{}, slice.Len()) + for i := 0; i < slice.Len(); i++ { + interfaceSlice[i] = slice.Index(i).Interface() + } + return interfaceSlice, true + } + return nil, false +} diff --git a/pkg/fuzz/component/value_test.go b/pkg/fuzz/component/value_test.go index bafd02775c..e732643863 100644 --- a/pkg/fuzz/component/value_test.go +++ b/pkg/fuzz/component/value_test.go @@ -37,3 +37,17 @@ func TestFlatMap_FlattenUnflatten(t *testing.T) { } require.Equal(t, data, nested, "unexpected data") } + +func TestAnySlice(t *testing.T) { + data := []any{} + data = append(data, []int{1, 2, 3}) + data = append(data, []string{"foo", "bar"}) + data = append(data, []bool{true, false}) + data = append(data, []float64{1.1, 2.2, 3.3}) + + for _, d := range data { + val, ok := IsTypedSlice(d) + require.True(t, ok, "expected slice") + require.True(t, val != nil, "expected value but got nil") + } +} diff --git a/pkg/fuzz/dataformat/dataformat.go b/pkg/fuzz/dataformat/dataformat.go index a07abc28f4..9d3cdc3061 100644 --- a/pkg/fuzz/dataformat/dataformat.go +++ b/pkg/fuzz/dataformat/dataformat.go @@ -8,6 +8,12 @@ import ( // dataformats is a list of dataformats var dataformats map[string]DataFormat +const ( + // DefaultKey is the key i.e used when given + // data is not of k-v type + DefaultKey = "value" +) + func init() { dataformats = make(map[string]DataFormat) @@ -49,9 +55,9 @@ type DataFormat interface { // Name returns the name of the encoder Name() string // Encode encodes the data into a format - Encode(data map[string]interface{}) (string, error) + Encode(data KV) (string, error) // Decode decodes the data from a format - Decode(input string) (map[string]interface{}, error) + Decode(input string) (KV, error) } // Decoded is a decoded data format @@ -59,7 +65,7 @@ type Decoded struct { // DataFormat is the data format DataFormat string // Data is the decoded data - Data map[string]interface{} + Data KV } // Decode decodes the data from a format @@ -81,7 +87,7 @@ func Decode(data string) (*Decoded, error) { } // Encode encodes the data into a format -func Encode(data map[string]interface{}, dataformat string) (string, error) { +func Encode(data KV, dataformat string) (string, error) { if dataformat == "" { return "", errors.New("dataformat is required") } diff --git a/pkg/fuzz/dataformat/dataformat_test.go b/pkg/fuzz/dataformat/dataformat_test.go index 7595f9b962..c51a8f6f26 100644 --- a/pkg/fuzz/dataformat/dataformat_test.go +++ b/pkg/fuzz/dataformat/dataformat_test.go @@ -14,7 +14,7 @@ func TestDataformatDecodeEncode_JSON(t *testing.T) { if decoded.DataFormat != "json" { t.Fatal("unexpected data format") } - if decoded.Data["foo"] != "bar" { + if decoded.Data.Get("foo") != "bar" { t.Fatal("unexpected data") } @@ -37,11 +37,19 @@ func TestDataformatDecodeEncode_XML(t *testing.T) { if decoded.DataFormat != "xml" { t.Fatal("unexpected data format") } - if decoded.Data["foo"].(map[string]interface{})["#text"] != "bar" { - t.Fatal("unexpected data") + fooValue := decoded.Data.Get("foo") + if fooValue == nil { + t.Fatal("key 'foo' not found") } - if decoded.Data["foo"].(map[string]interface{})["-attr"] != "baz" { - t.Fatal("unexpected data") + fooMap, ok := fooValue.(map[string]interface{}) + if !ok { + t.Fatal("type assertion to map[string]interface{} failed") + } + if fooMap["#text"] != "bar" { + t.Fatal("unexpected data for '#text'") + } + if fooMap["-attr"] != "baz" { + t.Fatal("unexpected data for '-attr'") } encoded, err := Encode(decoded.Data, decoded.DataFormat) diff --git a/pkg/fuzz/dataformat/form.go b/pkg/fuzz/dataformat/form.go index 088a62fe11..ab335299f7 100644 --- a/pkg/fuzz/dataformat/form.go +++ b/pkg/fuzz/dataformat/form.go @@ -1,9 +1,34 @@ package dataformat import ( - "net/url" + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/projectdiscovery/gologger" + mapsutil "github.com/projectdiscovery/utils/maps" + urlutil "github.com/projectdiscovery/utils/url" +) + +const ( + normalizedRegex = `_(\d+)$` ) +var ( + reNormalized = regexp.MustCompile(normalizedRegex) +) + +// == Handling Duplicate Query Parameters / Form Data == +// Nuclei supports fuzzing duplicate query parameters by internally normalizing +// them and denormalizing them back when creating request this normalization +// can be leveraged to specify custom fuzzing behaviour in template as well +// if a query like `?foo=bar&foo=baz&foo=fuzzz` is provided, it will be normalized to +// foo_1=bar , foo_2=baz , foo=fuzzz (i.e last value is given original key which is usual behaviour in HTTP and its implementations) +// this way this change does not break any existing rules in template given by keys-regex or keys +// At same time if user wants to specify 2nd or 1st duplicate value in template, they can use foo_1 or foo_2 in keys-regex or keys +// Note: By default all duplicate query parameters are fuzzed + type Form struct{} var ( @@ -21,38 +46,94 @@ func (f *Form) IsType(data string) bool { } // Encode encodes the data into Form format -func (f *Form) Encode(data map[string]interface{}) (string, error) { - query := url.Values{} - for key, value := range data { - switch v := value.(type) { - case []interface{}: - for _, val := range v { - query.Add(key, val.(string)) +func (f *Form) Encode(data KV) (string, error) { + params := urlutil.NewOrderedParams() + + data.Iterate(func(key string, value any) bool { + params.Add(key, fmt.Sprint(value)) + return true + }) + + normalized := map[string]map[string]string{} + // Normalize the data + for _, origKey := range data.OrderedMap.GetKeys() { + // here origKey is base key without _1, _2 etc. + if origKey != "" && !reNormalized.MatchString(origKey) { + params.Iterate(func(key string, value []string) bool { + if strings.HasPrefix(key, origKey) && reNormalized.MatchString(key) { + m := map[string]string{} + if normalized[origKey] != nil { + m = normalized[origKey] + } + if len(value) == 1 { + m[key] = value[0] + } else { + m[key] = "" + } + normalized[origKey] = m + params.Del(key) + } + return true + }) + } + } + + if len(normalized) > 0 { + for k, v := range normalized { + maxIndex := -1 + for key := range v { + matches := reNormalized.FindStringSubmatch(key) + if len(matches) == 2 { + dataIdx, err := strconv.Atoi(matches[1]) + if err != nil { + gologger.Verbose().Msgf("error converting normalized index(%v) to integer: %v", matches[1], err) + continue + } + if dataIdx > maxIndex { + maxIndex = dataIdx + } + } + } + data := make([]string, maxIndex+1) // Ensure the slice is large enough + for key, value := range v { + matches := reNormalized.FindStringSubmatch(key) + if len(matches) == 2 { + dataIdx, _ := strconv.Atoi(matches[1]) // Error already checked above + data[dataIdx-1] = value // Use dataIdx-1 since slice is 0-indexed + } } - case string: - query.Set(key, v) + data[maxIndex] = fmt.Sprint(params.Get(k)) // Use maxIndex which is the last index + // remove existing + params.Del(k) + params.Add(k, data...) } } - encoded := query.Encode() + + encoded := params.Encode() return encoded, nil } // Decode decodes the data from Form format -func (f *Form) Decode(data string) (map[string]interface{}, error) { - parsed, err := url.ParseQuery(data) - if err != nil { - return nil, err - } +func (f *Form) Decode(data string) (KV, error) { + ordered_params := urlutil.NewOrderedParams() + ordered_params.Merge(data) - values := make(map[string]interface{}) - for key, value := range parsed { + values := mapsutil.NewOrderedMap[string, any]() + ordered_params.Iterate(func(key string, value []string) bool { if len(value) == 1 { - values[key] = value[0] + values.Set(key, value[0]) } else { - values[key] = value + // in case of multiple query params in form data + // last value is considered and previous values are exposed with _1, _2, _3 etc. + // note that last value will not be included in _1, _2, _3 etc. + for i := 0; i < len(value)-1; i++ { + values.Set(key+"_"+strconv.Itoa(i+1), value[i]) + } + values.Set(key, value[len(value)-1]) } - } - return values, nil + return true + }) + return KVOrderedMap(&values), nil } // Name returns the name of the encoder diff --git a/pkg/fuzz/dataformat/json.go b/pkg/fuzz/dataformat/json.go index 3979ed5e65..99dd0430ec 100644 --- a/pkg/fuzz/dataformat/json.go +++ b/pkg/fuzz/dataformat/json.go @@ -30,16 +30,16 @@ func (j *JSON) IsType(data string) bool { } // Encode encodes the data into JSON format -func (j *JSON) Encode(data map[string]interface{}) (string, error) { - encoded, err := jsoniter.Marshal(data) +func (j *JSON) Encode(data KV) (string, error) { + encoded, err := jsoniter.Marshal(data.Map) return string(encoded), err } // Decode decodes the data from JSON format -func (j *JSON) Decode(data string) (map[string]interface{}, error) { +func (j *JSON) Decode(data string) (KV, error) { var decoded map[string]interface{} err := jsoniter.Unmarshal([]byte(data), &decoded) - return decoded, err + return KVMap(decoded), err } // Name returns the name of the encoder diff --git a/pkg/fuzz/dataformat/kv.go b/pkg/fuzz/dataformat/kv.go new file mode 100644 index 0000000000..72bd0da62e --- /dev/null +++ b/pkg/fuzz/dataformat/kv.go @@ -0,0 +1,126 @@ +package dataformat + +import ( + mapsutil "github.com/projectdiscovery/utils/maps" + "golang.org/x/exp/maps" +) + +// KV is a key-value struct +// that is implemented or used by fuzzing package +// to represent a key-value pair +// sometimes order or key-value pair is important (query params) +// so we use ordered map to represent the data +// if it's not important/significant (ex: json,xml) we use map +// this also allows us to iteratively implement ordered map +type KV struct { + Map map[string]interface{} + OrderedMap *mapsutil.OrderedMap[string, any] +} + +// Clones the current state of the KV struct +func (kv *KV) Clone() KV { + newKV := KV{} + if kv.OrderedMap == nil { + newKV.Map = maps.Clone(kv.Map) + return newKV + } + clonedOrderedMap := kv.OrderedMap.Clone() + newKV.OrderedMap = &clonedOrderedMap + return newKV +} + +// IsNIL returns true if the KV struct is nil +func (kv *KV) IsNIL() bool { + return kv.Map == nil && kv.OrderedMap == nil +} + +// IsOrderedMap returns true if the KV struct is an ordered map +func (kv *KV) IsOrderedMap() bool { + return kv.OrderedMap != nil +} + +// Set sets a value in the KV struct +func (kv *KV) Set(key string, value any) { + if kv.OrderedMap != nil { + kv.OrderedMap.Set(key, value) + return + } + if kv.Map == nil { + kv.Map = make(map[string]interface{}) + } + kv.Map[key] = value +} + +// Get gets a value from the KV struct +func (kv *KV) Get(key string) interface{} { + if kv.OrderedMap != nil { + value, ok := kv.OrderedMap.Get(key) + if !ok { + return nil + } + return value + } + return kv.Map[key] +} + +// Iterate iterates over the KV struct in insertion order +func (kv *KV) Iterate(f func(key string, value any) bool) { + if kv.OrderedMap != nil { + kv.OrderedMap.Iterate(func(key string, value any) bool { + return f(key, value) + }) + return + } + for key, value := range kv.Map { + if !f(key, value) { + break + } + } +} + +// Delete deletes a key from the KV struct +func (kv *KV) Delete(key string) bool { + if kv.OrderedMap != nil { + _, ok := kv.OrderedMap.Get(key) + if !ok { + return false + } + kv.OrderedMap.Delete(key) + return true + } + _, ok := kv.Map[key] + if !ok { + return false + } + delete(kv.Map, key) + return true +} + +// KVMap returns a new KV struct with the given map +func KVMap(data map[string]interface{}) KV { + return KV{Map: data} +} + +// KVOrderedMap returns a new KV struct with the given ordered map +func KVOrderedMap(data *mapsutil.OrderedMap[string, any]) KV { + return KV{OrderedMap: data} +} + +// ToMap converts the ordered map to a map +func ToMap(m *mapsutil.OrderedMap[string, any]) map[string]interface{} { + data := make(map[string]interface{}) + m.Iterate(func(key string, value any) bool { + data[key] = value + return true + }) + return data +} + +// ToOrderedMap converts the map to an ordered map +func ToOrderedMap(data map[string]interface{}) *mapsutil.OrderedMap[string, any] { + m := mapsutil.NewOrderedMap[string, any]() + for key, value := range data { + m.Set(key, value) + } + return &m +} diff --git a/pkg/fuzz/dataformat/multipart.go b/pkg/fuzz/dataformat/multipart.go index 658f77b321..d7e40af10c 100644 --- a/pkg/fuzz/dataformat/multipart.go +++ b/pkg/fuzz/dataformat/multipart.go @@ -6,6 +6,8 @@ import ( "io" "mime" "mime/multipart" + + mapsutil "github.com/projectdiscovery/utils/maps" ) type MultiPartForm struct { @@ -28,23 +30,30 @@ func (m *MultiPartForm) IsType(data string) bool { } // Encode encodes the data into MultiPartForm format -func (m *MultiPartForm) Encode(data map[string]interface{}) (string, error) { +func (m *MultiPartForm) Encode(data KV) (string, error) { var b bytes.Buffer w := multipart.NewWriter(&b) if err := w.SetBoundary(m.boundary); err != nil { return "", err } - for key, value := range data { + var Itererr error + data.Iterate(func(key string, value any) bool { var fw io.Writer var err error // Add field if fw, err = w.CreateFormField(key); err != nil { - return "", err + Itererr = err + return false } if _, err = fw.Write([]byte(value.(string))); err != nil { - return "", err + Itererr = err + return false } + return true + }) + if Itererr != nil { + return "", Itererr } w.Close() @@ -65,7 +74,7 @@ func (m *MultiPartForm) ParseBoundary(contentType string) error { } // Decode decodes the data from MultiPartForm format -func (m *MultiPartForm) Decode(data string) (map[string]interface{}, error) { +func (m *MultiPartForm) Decode(data string) (KV, error) { // Create a buffer from the string data b := bytes.NewBufferString(data) // The boundary parameter should be extracted from the Content-Type header of the HTTP request @@ -75,18 +84,18 @@ func (m *MultiPartForm) Decode(data string) (map[string]interface{}, error) { form, err := r.ReadForm(32 << 20) // 32MB is the max memory used to parse the form if err != nil { - return nil, err + return KV{}, err } defer func() { _ = form.RemoveAll() }() - result := make(map[string]interface{}) + result := mapsutil.NewOrderedMap[string, any]() for key, values := range form.Value { if len(values) > 1 { - result[key] = values + result.Set(key, values) } else { - result[key] = values[0] + result.Set(key, values[0]) } } for key, files := range form.File { @@ -94,20 +103,19 @@ func (m *MultiPartForm) Decode(data string) (map[string]interface{}, error) { for _, fileHeader := range files { file, err := fileHeader.Open() if err != nil { - return nil, err + return KV{}, err } defer file.Close() buffer := new(bytes.Buffer) if _, err := buffer.ReadFrom(file); err != nil { - return nil, err + return KV{}, err } fileContents = append(fileContents, buffer.String()) } - result[key] = fileContents + result.Set(key, fileContents) } - - return result, nil + return KVOrderedMap(&result), nil } // Name returns the name of the encoder diff --git a/pkg/fuzz/dataformat/raw.go b/pkg/fuzz/dataformat/raw.go index 70431528fb..1c2d5fa77e 100644 --- a/pkg/fuzz/dataformat/raw.go +++ b/pkg/fuzz/dataformat/raw.go @@ -17,15 +17,15 @@ func (r *Raw) IsType(data string) bool { } // Encode encodes the data into Raw format -func (r *Raw) Encode(data map[string]interface{}) (string, error) { - return data["value"].(string), nil +func (r *Raw) Encode(data KV) (string, error) { + return data.Get("value").(string), nil } // Decode decodes the data from Raw format -func (r *Raw) Decode(data string) (map[string]interface{}, error) { - return map[string]interface{}{ +func (r *Raw) Decode(data string) (KV, error) { + return KVMap(map[string]interface{}{ "value": data, - }, nil + }), nil } // Name returns the name of the encoder diff --git a/pkg/fuzz/dataformat/xml.go b/pkg/fuzz/dataformat/xml.go index 0609031ba8..2fb605bbde 100644 --- a/pkg/fuzz/dataformat/xml.go +++ b/pkg/fuzz/dataformat/xml.go @@ -22,13 +22,13 @@ func (x *XML) IsType(data string) bool { } // Encode encodes the data into XML format -func (x *XML) Encode(data map[string]interface{}) (string, error) { +func (x *XML) Encode(data KV) (string, error) { var header string - if value, ok := data["#_xml_header"]; ok && value != nil { + if value := data.Get("#_xml_header"); value != nil { header = value.(string) - delete(data, "#_xml_header") + data.Delete("#_xml_header") } - marshalled, err := mxj.Map(data).Xml() + marshalled, err := mxj.Map(data.Map).Xml() if err != nil { return "", err } @@ -41,7 +41,7 @@ func (x *XML) Encode(data map[string]interface{}) (string, error) { var xmlHeader = regexp.MustCompile(`\<\?(.*)\?\>`) // Decode decodes the data from XML format -func (x *XML) Decode(data string) (map[string]interface{}, error) { +func (x *XML) Decode(data string) (KV, error) { var prefixStr string prefix := xmlHeader.FindAllStringSubmatch(data, -1) if len(prefix) > 0 { @@ -50,10 +50,10 @@ func (x *XML) Decode(data string) (map[string]interface{}, error) { decoded, err := mxj.NewMapXml([]byte(data)) if err != nil { - return nil, err + return KV{}, err } decoded["#_xml_header"] = prefixStr - return decoded, nil + return KVMap(decoded), nil } // Name returns the name of the encoder diff --git a/pkg/fuzz/execute.go b/pkg/fuzz/execute.go index 3a86ef7e33..6e1a7a6a23 100644 --- a/pkg/fuzz/execute.go +++ b/pkg/fuzz/execute.go @@ -64,11 +64,6 @@ type GeneratedRequest struct { // Input is not thread safe and should not be shared between concurrent // goroutines. func (rule *Rule) Execute(input *ExecuteRuleInput) (err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("got panic while executing rule: %v", r) - } - }() if !rule.isInputURLValid(input.Input) { return ErrRuleNotApplicable.Msgf("invalid input url: %v", input.Input.MetaInput.Input) } diff --git a/pkg/fuzz/parts.go b/pkg/fuzz/parts.go index 9b24446bb7..2796a7dc96 100644 --- a/pkg/fuzz/parts.go +++ b/pkg/fuzz/parts.go @@ -34,12 +34,13 @@ func (rule *Rule) checkRuleApplicableOnComponent(component component.Component) // executePartComponent executes this rule on a given component and payload func (rule *Rule) executePartComponent(input *ExecuteRuleInput, payload ValueOrKeyValue, ruleComponent component.Component) error { + // Note: component needs to be cloned because they contain values copied by reference if payload.IsKV() { // for kv fuzzing - return rule.executePartComponentOnKV(input, payload, ruleComponent) + return rule.executePartComponentOnKV(input, payload, ruleComponent.Clone()) } else { // for value only fuzzing - return rule.executePartComponentOnValues(input, payload.Value, ruleComponent) + return rule.executePartComponentOnValues(input, payload.Value, ruleComponent.Clone()) } } diff --git a/pkg/input/types/http.go b/pkg/input/types/http.go index 62b2524976..a407d6a066 100644 --- a/pkg/input/types/http.go +++ b/pkg/input/types/http.go @@ -207,11 +207,6 @@ func (hr *HttpResponse) Clone() *HttpResponse { // and returns the request and response object // Note: it currently does not parse response and is meant to be added manually since its a optional field func ParseRawRequest(raw string) (rr *RequestResponse, err error) { - defer func() { - if r := recover(); r != nil { - err = fmt.Errorf("panic: %v", r) - } - }() protoReader := textproto.NewReader(bufio.NewReader(strings.NewReader(raw))) methodLine, err := protoReader.ReadLine() if err != nil { diff --git a/pkg/js/devtools/scrapefuncs/main.go b/pkg/js/devtools/scrapefuncs/main.go index 04c62f91f1..3ecbe6dea7 100644 --- a/pkg/js/devtools/scrapefuncs/main.go +++ b/pkg/js/devtools/scrapefuncs/main.go @@ -1,113 +1,92 @@ package main import ( - "bytes" - "context" "flag" "fmt" "go/ast" "go/parser" "go/token" - "log" "os" + "path/filepath" + "sort" "strings" - filutil "github.com/projectdiscovery/utils/file" - "github.com/sashabaranov/go-openai" + mapsutil "github.com/projectdiscovery/utils/maps" + "golang.org/x/exp/maps" ) -var sysprompt = ` -data present after ---raw data--- contains raw data extracted by a parser and contains information about function ---- example --- -Name: log -Signatures: "log(msg string)" -Signatures: "log(msg map[string]interface{})" -Description: log prints given input to stdout with [JS] prefix for debugging purposes ---- end example --- -Here Name is name of function , signature[s] is actual function declaration and description is description of function -using this data for every such function generate a abstract implementation of function in javascript along with jsdoc annotations ---- example expected output--- -/** - * log prints given input to stdout with [JS] prefix for debugging purposes - * log(msg string) - * log(msg map[string]interface{}) - * @function - * @param {string} msg - The message to print. - */ -function log(msg) { - // implemented in go -}; ---- instructions --- -ACT as helpful coding assistant and do the same for all functions present in data -` - -const userPrompt = ` ----raw data--- -{{source}} ----new javascript--- -` - var ( - dir string - key string - keyfile string - out string + dir string + out string ) +type DSLHelperFunc struct { + Name string + Description string + Signatures []string +} + +var pkg2NameMapping = map[string]string{ + "code": "Code Protocol", + "javascript": "JavaScript Protocol", + "global": "Javascript Runtime", + "compiler": "Javascript Runtime", + "flow": "Template Flow", +} + +var preferredOrder = []string{"Javascript Runtime", "Template Flow", "Code Protocol", "JavaScript Protocol"} + func main() { - flag.StringVar(&dir, "dir", "pkg/js/global", "directory to process") - flag.StringVar(&key, "key", "", "openai api key") - flag.StringVar(&keyfile, "keyfile", "", "openai api key file") - flag.StringVar(&out, "out", "", "output js file with declarations of all global functions") + flag.StringVar(&dir, "dir", "pkg/", "directory to process") + flag.StringVar(&out, "out", "", "output markdown file with helper file declarations") flag.Parse() - finalKey := "" - if key != "" { - key = finalKey + dirList := []string{} + + if err := filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error { + if d.IsDir() { + dirList = append(dirList, path) + } + return nil + }); err != nil { + panic(err) } - if keyfile != "" && filutil.FileExists(keyfile) { - data, err := os.ReadFile(keyfile) + pkgs := map[string]*ast.Package{} + + for _, dir := range dirList { + fset := token.NewFileSet() + pkgss, err := parser.ParseDir(fset, dir, nil, 0) if err != nil { - log.Fatal(err) + fmt.Println(err) + return } - finalKey = string(data) - } - if key := os.Getenv("OPENAI_API_KEY"); key != "" { - finalKey = key + pkgs = mapsutil.Merge(pkgs, pkgss) } - if finalKey == "" { - log.Fatal("openai api key is not set") - } - llm := openai.NewClient(finalKey) - var buff bytes.Buffer - - fset := token.NewFileSet() - pkgs, err := parser.ParseDir(fset, dir, nil, 0) - if err != nil { - fmt.Println(err) - return - } + dslHelpers := map[string][]DSLHelperFunc{} for _, pkg := range pkgs { - for _, file := range pkg.Files { + for fname, file := range pkg.Files { ast.Inspect(file, func(n ast.Node) bool { switch x := n.(type) { case *ast.CallExpr: if sel, ok := x.Fun.(*ast.SelectorExpr); ok { if sel.Sel.Name == "RegisterFuncWithSignature" { + hf := DSLHelperFunc{} for _, arg := range x.Args { if kv, ok := arg.(*ast.CompositeLit); ok { for _, elt := range kv.Elts { if kv, ok := elt.(*ast.KeyValueExpr); ok { key := kv.Key.(*ast.Ident).Name switch key { - case "Name", "Description": - buff.WriteString(fmt.Sprintf("%s: %s\n", key, strings.Trim(kv.Value.(*ast.BasicLit).Value, `"`))) + case "Name": + hf.Name = strings.Trim(kv.Value.(*ast.BasicLit).Value, `"`) + case "Description": + hf.Description = strings.Trim(kv.Value.(*ast.BasicLit).Value, `"`) case "Signatures": if comp, ok := kv.Value.(*ast.CompositeLit); ok { for _, signature := range comp.Elts { - buff.WriteString(fmt.Sprintf("%s: %s\n", key, signature.(*ast.BasicLit).Value)) + hf.Signatures = append(hf.Signatures, strings.Trim(signature.(*ast.BasicLit).Value, `"`)) } } } @@ -115,7 +94,17 @@ func main() { } } } - buff.WriteString("\n") + if hf.Name != "" { + identifier := pkg2NameMapping[pkg.Name] + if identifier == "" { + identifier = pkg.Name + " (" + filepath.Dir(fname) + ")" + } + + if dslHelpers[identifier] == nil { + dslHelpers[identifier] = []DSLHelperFunc{} + } + dslHelpers[identifier] = append(dslHelpers[identifier], hf) + } } } } @@ -124,32 +113,54 @@ func main() { } } - fmt.Printf("[+] Scraped %d functions\n\n", strings.Count(buff.String(), "Name:")) - fmt.Println(buff.String()) - - fmt.Printf("[+] Generating jsdoc for all functions\n\n") - resp, err := llm.CreateChatCompletion(context.TODO(), openai.ChatCompletionRequest{ - Model: "gpt-4", - Messages: []openai.ChatCompletionMessage{ - {Role: "system", Content: sysprompt}, - {Role: "user", Content: strings.ReplaceAll(userPrompt, "{{source}}", buff.String())}, - }, - Temperature: 0.1, - }) - if err != nil { - fmt.Println(err) - return - } - if len(resp.Choices) == 0 { - fmt.Println("no choices returned") - return + // DSL Helper functions stats + for pkg, funcs := range dslHelpers { + fmt.Printf("Found %d DSL Helper functions in package %s\n", len(funcs), pkg) } - data := resp.Choices[0].Message.Content - - fmt.Println(data) + // Generate Markdown tables with ## as package name if out != "" { - if err := os.WriteFile(out, []byte(data), 0600); err != nil { + var sb strings.Builder + sb.WriteString(`--- +title: "Javascript Helper Functions" +description: "Available JS Helper Functions that can be used in global js runtime & protocol specific helpers." +icon: "function" +iconType: "solid" +--- + + +`) + + actualKeys := maps.Keys(dslHelpers) + sort.Slice(actualKeys, func(i, j int) bool { + for _, preferredKey := range preferredOrder { + if actualKeys[i] == preferredKey { + return true + } + if actualKeys[j] == preferredKey { + return false + } + } + return actualKeys[i] < actualKeys[j] + }) + + for _, v := range actualKeys { + pkg := v + funcs := dslHelpers[pkg] + sb.WriteString("## " + pkg + "\n\n") + sb.WriteString("| Name | Description | Signatures |\n") + sb.WriteString("|------|-------------|------------|\n") + for _, f := range funcs { + sigSlice := []string{} + for _, sig := range f.Signatures { + sigSlice = append(sigSlice, "`"+sig+"`") + } + sb.WriteString(fmt.Sprintf("| %s | %s | %s |\n", f.Name, f.Description, strings.Join(sigSlice, ", "))) + } + sb.WriteString("\n") + } + + if err := os.WriteFile(out, []byte(sb.String()), 0644); err != nil { fmt.Println(err) return } diff --git a/pkg/js/global/helpers.go b/pkg/js/global/helpers.go index 1dae2af0f2..5510d7ae33 100644 --- a/pkg/js/global/helpers.go +++ b/pkg/js/global/helpers.go @@ -38,3 +38,40 @@ func registerAdditionalHelpers(runtime *goja.Runtime) { }, }) } + +func init() { + // these are dummy functions we use trigger documentation generation + // actual definations are in exports.js + _ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{ + Name: "to_json", + Signatures: []string{ + "to_json(any) object", + }, + Description: "Converts a given object to JSON", + }) + + _ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{ + Name: "dump_json", + Signatures: []string{ + "dump_json(any)", + }, + Description: "Prints a given object as JSON in console", + }) + + _ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{ + Name: "to_array", + Signatures: []string{ + "to_array(any) array", + }, + Description: "Sets/Updates objects prototype to array to enable Array.XXX functions", + }) + + _ = gojs.RegisterFuncWithSignature(nil, gojs.FuncOpts{ + Name: "hex_to_ascii", + Signatures: []string{ + "hex_to_ascii(string) string", + }, + Description: "Converts a given hex string to ascii", + }) + +} diff --git a/pkg/js/gojs/set.go b/pkg/js/gojs/set.go index 38e45d8d72..9703a3c6e8 100644 --- a/pkg/js/gojs/set.go +++ b/pkg/js/gojs/set.go @@ -7,6 +7,7 @@ import ( var ( ErrInvalidFuncOpts = errorutil.NewWithFmt("invalid function options: %v") + ErrNilRuntime = errorutil.New("runtime is nil") ) type FuncOpts struct { @@ -23,6 +24,9 @@ func (f *FuncOpts) valid() bool { // RegisterFunc registers a function with given name, signatures and description func RegisterFuncWithSignature(runtime *goja.Runtime, opts FuncOpts) error { + if runtime == nil { + return ErrNilRuntime + } if !opts.valid() { return ErrInvalidFuncOpts.Msgf("name: %s, signatures: %v, description: %s", opts.Name, opts.Signatures, opts.Description) } diff --git a/pkg/model/model.go b/pkg/model/model.go index 7fc8c5be53..d4aec3c071 100644 --- a/pkg/model/model.go +++ b/pkg/model/model.go @@ -1,10 +1,22 @@ package model import ( + "github.com/invopop/jsonschema" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice" ) +type schemaMetadata struct { + PropName string + PropType string + Example []interface{} + OneOf []*schemaMetadata +} + +var infoSchemaMetadata = []schemaMetadata{ + {PropName: "author", OneOf: []*schemaMetadata{{PropType: "string", Example: []interface{}{`pdteam`}}, {PropType: "array", Example: []interface{}{`pdteam,mr.robot`}}}}, +} + // Info contains metadata information about a template type Info struct { // description: | @@ -13,14 +25,14 @@ type Info struct { // examples: // - value: "\"bower.json file disclosure\"" // - value: "\"Nagios Default Credentials Check\"" - Name string `json:"name,omitempty" yaml:"name,omitempty" jsonschema:"title=name of the template,description=Name is a short summary of what the template does,example=Nagios Default Credentials Check"` + Name string `json:"name,omitempty" yaml:"name,omitempty" jsonschema:"title=name of the template,description=Name is a short summary of what the template does,type=string,required,example=Nagios Default Credentials Check"` // description: | // Author of the template. // // Multiple values can also be specified separated by commas. // examples: // - value: "\"\"" - Authors stringslice.StringSlice `json:"author,omitempty" yaml:"author,omitempty" jsonschema:"title=author of the template,description=Author is the author of the template,example=username"` + Authors stringslice.StringSlice `json:"author,omitempty" yaml:"author,omitempty" jsonschema:"title=author of the template,description=Author is the author of the template,required,example=username"` // description: | // Any tags for the template. // @@ -38,7 +50,7 @@ type Info struct { // examples: // - value: "\"Bower is a package manager which stores package information in the bower.json file\"" // - value: "\"Subversion ALM for the enterprise before 8.8.2 allows reflected XSS at multiple locations\"" - Description string `json:"description,omitempty" yaml:"description,omitempty" jsonschema:"title=description of the template,description=In-depth explanation on what the template does,example=Bower is a package manager which stores package information in the bower.json file"` + Description string `json:"description,omitempty" yaml:"description,omitempty" jsonschema:"title=description of the template,description=In-depth explanation on what the template does,type=string,example=Bower is a package manager which stores package information in the bower.json file"` // description: | // Impact of the template. // @@ -47,7 +59,7 @@ type Info struct { // examples: // - value: "\"Successful exploitation of this vulnerability could allow an attacker to execute arbitrary SQL queries, potentially leading to unauthorized access, data leakage, or data manipulation.\"" // - value: "\"Successful exploitation of this vulnerability could allow an attacker to execute arbitrary script code in the context of the victim's browser, potentially leading to session hijacking, defacement, or theft of sensitive information.\"" - Impact string `json:"impact,omitempty" yaml:"impact,omitempty" jsonschema:"title=impact of the template,description=In-depth explanation on the impact of the issue found by the template,example=Successful exploitation of this vulnerability could allow an attacker to execute arbitrary SQL queries, potentially leading to unauthorized access, data leakage, or data manipulation."` + Impact string `json:"impact,omitempty" yaml:"impact,omitempty" jsonschema:"title=impact of the template,description=In-depth explanation on the impact of the issue found by the template,example=Successful exploitation of this vulnerability could allow an attacker to execute arbitrary SQL queries, potentially leading to unauthorized access, data leakage, or data manipulation.,type=string"` // description: | // References for the template. // @@ -66,11 +78,11 @@ type Info struct { // examples: // - value: > // map[string]string{"customField1":"customValue1"} - Metadata map[string]interface{} `json:"metadata,omitempty" yaml:"metadata,omitempty" jsonschema:"title=additional metadata for the template,description=Additional metadata fields for the template"` + Metadata map[string]interface{} `json:"metadata,omitempty" yaml:"metadata,omitempty" jsonschema:"title=additional metadata for the template,description=Additional metadata fields for the template,type=object"` // description: | // Classification contains classification information about the template. - Classification *Classification `json:"classification,omitempty" yaml:"classification,omitempty" jsonschema:"title=classification info for the template,description=Classification information for the template"` + Classification *Classification `json:"classification,omitempty" yaml:"classification,omitempty" jsonschema:"title=classification info for the template,description=Classification information for the template,type=object"` // description: | // Remediation steps for the template. @@ -79,7 +91,30 @@ type Info struct { // // examples: // - value: "\"Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties\"" - Remediation string `json:"remediation,omitempty" yaml:"remediation,omitempty" jsonschema:"title=remediation steps for the template,description=In-depth explanation on how to fix the issues found by the template,example=Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties"` + Remediation string `json:"remediation,omitempty" yaml:"remediation,omitempty" jsonschema:"title=remediation steps for the template,description=In-depth explanation on how to fix the issues found by the template,example=Change the default administrative username and password of Apache ActiveMQ by editing the file jetty-realm.properties,type=string"` +} + +// JSONSchemaProperty returns the JSON schema property for the Info object. +func (i Info) JSONSchemaExtend(base *jsonschema.Schema) { + // since we are re-using a stringslice and rawStringSlice everywhere, we can extend/edit the schema here + // thus allowing us to add examples, descriptions, etc. to the properties + for _, metadata := range infoSchemaMetadata { + if prop, ok := base.Properties.Get(metadata.PropName); ok { + if len(metadata.OneOf) > 0 { + for _, oneOf := range metadata.OneOf { + prop.OneOf = append(prop.OneOf, &jsonschema.Schema{ + Type: oneOf.PropType, + Examples: oneOf.Example, + }) + } + } else { + if metadata.PropType != "" { + prop.Type = metadata.PropType + } + prop.Examples = []interface{}{metadata.Example} + } + } + } } // Classification contains the vulnerability classification data for a template. diff --git a/pkg/model/types/severity/severity.go b/pkg/model/types/severity/severity.go index c186ec44bb..611c9721df 100644 --- a/pkg/model/types/severity/severity.go +++ b/pkg/model/types/severity/severity.go @@ -4,7 +4,7 @@ import ( "encoding/json" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" "github.com/pkg/errors" ) @@ -71,16 +71,18 @@ type Holder struct { Severity Severity `mapping:"true"` } -func (severityHolder Holder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ +// Implement a jsonschema for the severity holder +func (severityHolder Holder) JSONSchema() *jsonschema.Schema { + enums := []interface{}{} + for _, severity := range GetSupportedSeverities() { + enums = append(enums, severity.String()) + } + return &jsonschema.Schema{ Type: "string", Title: "severity of the template", Description: "Seriousness of the implications of the template", + Enum: enums, } - for _, severity := range GetSupportedSeverities() { - gotType.Enum = append(gotType.Enum, severity.String()) - } - return gotType } func (severityHolder *Holder) UnmarshalYAML(unmarshal func(interface{}) error) error { diff --git a/pkg/model/types/stringslice/stringslice.go b/pkg/model/types/stringslice/stringslice.go index 10f19fb757..290687f6d2 100644 --- a/pkg/model/types/stringslice/stringslice.go +++ b/pkg/model/types/stringslice/stringslice.go @@ -5,26 +5,38 @@ import ( "fmt" "strings" - "github.com/alecthomas/jsonschema" - + "github.com/invopop/jsonschema" "github.com/projectdiscovery/nuclei/v3/pkg/utils" ) +type StringOrSlice string + +func (StringOrSlice) JSONSchema() *jsonschema.Schema { + return &jsonschema.Schema{ + OneOf: []*jsonschema.Schema{ + { + Type: "string", + }, + { + Type: "array", + }, + }, + } +} + // StringSlice represents a single (in-lined) or multiple string value(s). // The unmarshaller does not automatically convert in-lined strings to []string, hence the interface{} type is required. type StringSlice struct { Value interface{} } -func New(value interface{}) StringSlice { - return StringSlice{Value: value} +// Implement alias for stringslice and reuse it everywhere +func (stringSlice StringSlice) JSONSchemaAlias() any { + return StringOrSlice("") } -func (stringSlice StringSlice) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ - OneOf: []*jsonschema.Type{{Type: "string"}, {Type: "array"}}, - } - return gotType +func New(value interface{}) StringSlice { + return StringSlice{Value: value} } func (stringSlice *StringSlice) IsEmpty() bool { diff --git a/pkg/model/types/stringslice/stringslice_raw.go b/pkg/model/types/stringslice/stringslice_raw.go index 494731ca5c..7d9e470bfa 100644 --- a/pkg/model/types/stringslice/stringslice_raw.go +++ b/pkg/model/types/stringslice/stringslice_raw.go @@ -11,3 +11,7 @@ func NewRawStringSlice(value interface{}) *RawStringSlice { func (rawStringSlice *RawStringSlice) Normalize(value string) string { return value } + +func (rawStringSlice RawStringSlice) JSONSchemaAlias() any { + return StringOrSlice("") +} diff --git a/pkg/model/types/userAgent/user_agent.go b/pkg/model/types/userAgent/user_agent.go index 30173ca2b5..4a187d06a1 100644 --- a/pkg/model/types/userAgent/user_agent.go +++ b/pkg/model/types/userAgent/user_agent.go @@ -4,7 +4,7 @@ import ( "encoding/json" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" "github.com/pkg/errors" ) @@ -61,8 +61,8 @@ type UserAgentHolder struct { Value UserAgent `mapping:"true"` } -func (userAgentHolder UserAgentHolder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ +func (userAgentHolder UserAgentHolder) JSONSchemaType() *jsonschema.Schema { + gotType := &jsonschema.Schema{ Type: "string", Title: "userAgent for the headless", Description: "userAgent for the headless http request", diff --git a/pkg/operators/extractors/extractor_types.go b/pkg/operators/extractors/extractor_types.go index f2ee89d787..79fa191f8b 100644 --- a/pkg/operators/extractors/extractor_types.go +++ b/pkg/operators/extractors/extractor_types.go @@ -5,7 +5,7 @@ import ( "errors" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" ) // ExtractorType is the type of the extractor specified @@ -72,8 +72,8 @@ type ExtractorTypeHolder struct { ExtractorType ExtractorType `mapping:"true"` } -func (holder ExtractorTypeHolder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ +func (holder ExtractorTypeHolder) JSONSchemaType() *jsonschema.Schema { + gotType := &jsonschema.Schema{ Type: "string", Title: "type of the extractor", Description: "Type of the extractor", diff --git a/pkg/operators/matchers/matchers_types.go b/pkg/operators/matchers/matchers_types.go index 9c872192a8..5bcbc67628 100644 --- a/pkg/operators/matchers/matchers_types.go +++ b/pkg/operators/matchers/matchers_types.go @@ -5,7 +5,7 @@ import ( "errors" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" ) // MatcherType is the type of the matcher specified @@ -82,8 +82,8 @@ func (t MatcherTypeHolder) String() string { return t.MatcherType.String() } -func (holder MatcherTypeHolder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ +func (holder MatcherTypeHolder) JSONSchemaType() *jsonschema.Schema { + gotType := &jsonschema.Schema{ Type: "string", Title: "type of the matcher", Description: "Type of the matcher", diff --git a/pkg/output/format_screen.go b/pkg/output/format_screen.go index d468981de0..2d6310df3f 100644 --- a/pkg/output/format_screen.go +++ b/pkg/output/format_screen.go @@ -59,8 +59,12 @@ func (w *StandardWriter) formatScreen(output *ResultEvent) []byte { for i, item := range output.ExtractedResults { // trim trailing space + // quote non-ascii and non printable characters and then + // unquote quotes (`"`) for readability item = strings.TrimSpace(item) - item = strings.ReplaceAll(item, "\n", "\\n") // only replace newlines + item = strconv.QuoteToASCII(item) + item = strings.ReplaceAll(item, `\"`, `"`) + builder.WriteString(w.aurora.BrightCyan(item).String()) if i != len(output.ExtractedResults)-1 { diff --git a/pkg/output/output.go b/pkg/output/output.go index 6899d6cf15..4a85c32480 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -171,6 +171,9 @@ type ResultEvent struct { // IssueTrackers is the metadata for issue trackers IssueTrackers map[string]IssueTrackerMetadata `json:"issue_trackers,omitempty"` + // ReqURLPattern when enabled contains base URL pattern that was used to generate the request + // must be enabled by setting protocols.ExecuterOptions.ExportReqURLPattern to true + ReqURLPattern string `json:"req_url_pattern,omitempty"` FileToIndexPosition map[string]int `json:"-"` Error string `json:"error,omitempty"` diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index 57eb9b8891..5193344ab6 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -8,11 +8,15 @@ import ( "strings" "time" + "github.com/alecthomas/chroma/quick" + "github.com/ditashi/jsbeautifier-go/jsbeautifier" + "github.com/dop251/goja" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gozero" gozerotypes "github.com/projectdiscovery/gozero/types" + "github.com/projectdiscovery/nuclei/v3/pkg/js/compiler" "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/operators/extractors" "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" @@ -52,6 +56,9 @@ type Request struct { // Engine type Engine []string `yaml:"engine,omitempty" jsonschema:"title=engine,description=Engine"` // description: | + // PreCondition is a condition which is evaluated before sending the request. + PreCondition string `yaml:"pre-condition,omitempty" json:"pre-condition,omitempty" jsonschema:"title=pre-condition for the request,description=PreCondition is a condition which is evaluated before sending the request"` + // description: | // Engine Arguments Args []string `yaml:"args,omitempty" jsonschema:"title=args,description=Args"` // description: | @@ -61,9 +68,10 @@ type Request struct { // Source File/Snippet Source string `yaml:"source,omitempty" jsonschema:"title=source file/snippet,description=Source snippet"` - options *protocols.ExecutorOptions - gozero *gozero.Gozero - src *gozero.Source + options *protocols.ExecutorOptions `yaml:"-" json:"-"` + preConditionCompiled *goja.Program `yaml:"-" json:"-"` + gozero *gozero.Gozero `yaml:"-" json:"-"` + src *gozero.Source `yaml:"-" json:"-"` } // Compile compiles the request generators preparing any requests possible. @@ -110,6 +118,15 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } request.CompiledOperators = compiled } + + // compile pre-condition if any + if request.PreCondition != "" { + preConditionCompiled, err := compiler.WrapScriptNCompile(request.PreCondition, false) + if err != nil { + return errorutil.NewWithTag(request.TemplateID, "could not compile pre-condition: %s", err) + } + request.preConditionCompiled = preConditionCompiled + } return nil } @@ -130,11 +147,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa return err } defer func() { - // catch any panics just in case - if r := recover(); r != nil { - gologger.Error().Msgf("[%s] Panic occurred in code protocol: %s\n", request.options.TemplateID, r) - err = fmt.Errorf("panic occurred: %s", r) - } if err := metaSrc.Cleanup(); err != nil { gologger.Warning().Msgf("%s\n", err) } @@ -160,7 +172,45 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa allvars[name] = v metaSrc.AddVariable(gozerotypes.Variable{Name: name, Value: v}) } + + // set timeout using multiplier timeout := TimeoutMultiplier * request.options.Options.Timeout + + if request.PreCondition != "" { + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Debug().Msgf("[%s] Executing Precondition for Code request\n", request.TemplateID) + var highlightFormatter = "terminal256" + if request.options.Options.NoColor { + highlightFormatter = "text" + } + var buff bytes.Buffer + _ = quick.Highlight(&buff, beautifyJavascript(request.PreCondition), "javascript", highlightFormatter, "monokai") + prettyPrint(request.TemplateID, buff.String()) + } + + args := compiler.NewExecuteArgs() + args.TemplateCtx = allvars + + result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, args, + &compiler.ExecuteOptions{ + Timeout: timeout, + Source: &request.PreCondition, + Callback: registerPreConditionFunctions, + Cleanup: cleanUpPreConditionFunctions, + }) + if err != nil { + return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err) + } + if !result.GetSuccess() || types.ToString(result["error"]) != "" { + gologger.Warning().Msgf("[%s] Precondition for request %s was not satisfied\n", request.TemplateID, request.PreCondition) + request.options.Progress.IncrementFailedRequestsBy(1) + return nil + } + if request.options.Options.Debug || request.options.Options.DebugRequests { + gologger.Debug().Msgf("[%s] Precondition for request was satisfied\n", request.TemplateID) + } + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second) defer cancel() // Note: we use contextutil despite the fact that gozero accepts context as argument @@ -336,3 +386,23 @@ func interpretEnvVars(source string, vars map[string]interface{}) string { } return source } + +func beautifyJavascript(code string) string { + opts := jsbeautifier.DefaultOptions() + beautified, err := jsbeautifier.Beautify(&code, opts) + if err != nil { + return code + } + return beautified +} + +func prettyPrint(templateId string, buff string) { + lines := strings.Split(buff, "\n") + final := []string{} + for _, v := range lines { + if v != "" { + final = append(final, "\t"+v) + } + } + gologger.Debug().Msgf(" [%v] Pre-condition Code:\n\n%v\n\n", templateId, strings.Join(final, "\n")) +} diff --git a/pkg/protocols/code/helpers.go b/pkg/protocols/code/helpers.go new file mode 100644 index 0000000000..f67144e796 --- /dev/null +++ b/pkg/protocols/code/helpers.go @@ -0,0 +1,272 @@ +package code + +import ( + goruntime "runtime" + + "github.com/dop251/goja" + "github.com/projectdiscovery/nuclei/v3/pkg/js/gojs" + osutils "github.com/projectdiscovery/utils/os" +) + +// registerPreConditionFunctions registers the pre-condition functions +func registerPreConditionFunctions(runtime *goja.Runtime) error { + // Note: the only reason we are not using forloop to generate these functions is because + // 'scrapefuncs' uses this function to find all dsl helper functions and document them. + + // === OS === + err := gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "OS", + Signatures: []string{ + "OS() string", + }, + Description: "OS returns the current OS", + FuncDecl: func() string { + return goruntime.GOOS + }, + }) + if err != nil { + return err + } + + // IsLinux checks if the current OS is Linux + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsLinux", + Signatures: []string{ + "IsLinux() bool", + }, + Description: "IsLinux checks if the current OS is Linux", + FuncDecl: func() bool { + return osutils.IsLinux() + }, + }) + if err != nil { + return err + } + + // IsWindows checks if the current OS is Windows + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsWindows", + Signatures: []string{ + "IsWindows() bool", + }, + Description: "IsWindows checks if the current OS is Windows", + FuncDecl: func() bool { + return osutils.IsWindows() + }, + }) + if err != nil { + return err + } + + // IsOSX checks if the current OS is OSX + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsOSX", + Signatures: []string{ + "IsOSX() bool", + }, + Description: "IsOSX checks if the current OS is OSX", + + FuncDecl: func() bool { + return osutils.IsOSX() + }, + }) + if err != nil { + return err + } + + // IsAndroid checks if the current OS is Android + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsAndroid", + Signatures: []string{ + "IsAndroid() bool", + }, + Description: "IsAndroid checks if the current OS is Android", + FuncDecl: func() bool { + return osutils.IsAndroid() + }, + }) + if err != nil { + return err + } + + // IsIOS checks if the current OS is IOS + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsIOS", + Signatures: []string{ + "IsIOS() bool", + }, + Description: "IsIOS checks if the current OS is IOS", + FuncDecl: func() bool { + return osutils.IsIOS() + }, + }) + if err != nil { + return err + } + + // IsJS checks if the current OS is JS + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsJS", + Signatures: []string{ + "IsJS() bool", + }, + Description: "IsJS checks if the current OS is JS", + FuncDecl: func() bool { + return osutils.IsJS() + }, + }) + if err != nil { + return err + } + + // IsFreeBSD checks if the current OS is FreeBSD + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsFreeBSD", + Signatures: []string{ + "IsFreeBSD() bool", + }, + Description: "IsFreeBSD checks if the current OS is FreeBSD", + FuncDecl: func() bool { + return osutils.IsFreeBSD() + }, + }) + if err != nil { + return err + } + + // IsOpenBSD checks if the current OS is OpenBSD + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsOpenBSD", + Signatures: []string{ + "IsOpenBSD() bool", + }, + Description: "IsOpenBSD checks if the current OS is OpenBSD", + FuncDecl: func() bool { + return osutils.IsOpenBSD() + }, + }) + if err != nil { + return err + } + + // IsSolaris checks if the current OS is Solaris + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsSolaris", + Signatures: []string{ + "IsSolaris() bool", + }, + Description: "IsSolaris checks if the current OS is Solaris", + FuncDecl: func() bool { + return osutils.IsSolaris() + }, + }) + if err != nil { + return err + } + + // === Arch === + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "Arch", + Signatures: []string{ + "Arch() string", + }, + Description: "Arch returns the current architecture", + FuncDecl: func() string { + return goruntime.GOARCH + }, + }) + if err != nil { + return err + } + + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "Is386", + Signatures: []string{ + "Is386() bool", + }, + Description: "Is386 checks if the current architecture is 386", + FuncDecl: func() bool { + return osutils.Is386() + }, + }) + if err != nil { + return err + } + + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsAmd64", + Signatures: []string{ + "IsAmd64() bool", + }, + Description: "IsAmd64 checks if the current architecture is Amd64", + FuncDecl: func() bool { + return osutils.IsAmd64() + }, + }) + if err != nil { + return err + } + + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsARM", + Signatures: []string{ + "IsARM() bool", + }, + Description: "IsArm checks if the current architecture is Arm", + FuncDecl: func() bool { + return osutils.IsARM() + }, + }) + if err != nil { + return err + } + + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsARM64", + Signatures: []string{ + "IsARM64() bool", + }, + Description: "IsArm64 checks if the current architecture is Arm64", + FuncDecl: func() bool { + return osutils.IsARM64() + }, + }) + if err != nil { + return err + } + + err = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "IsWasm", + Signatures: []string{ + "IsWasm() bool", + }, + Description: "IsWasm checks if the current architecture is Wasm", + FuncDecl: func() bool { + return osutils.IsWasm() + }, + }) + if err != nil { + return err + } + + return nil +} + +func cleanUpPreConditionFunctions(runtime *goja.Runtime) { + _ = runtime.GlobalObject().Delete("OS") + _ = runtime.GlobalObject().Delete("IsLinux") + _ = runtime.GlobalObject().Delete("IsWindows") + _ = runtime.GlobalObject().Delete("IsOSX") + _ = runtime.GlobalObject().Delete("IsAndroid") + _ = runtime.GlobalObject().Delete("IsIOS") + _ = runtime.GlobalObject().Delete("IsJS") + _ = runtime.GlobalObject().Delete("IsFreeBSD") + _ = runtime.GlobalObject().Delete("IsOpenBSD") + _ = runtime.GlobalObject().Delete("IsSolaris") + _ = runtime.GlobalObject().Delete("Arch") + _ = runtime.GlobalObject().Delete("Is386") + _ = runtime.GlobalObject().Delete("IsAmd64") + _ = runtime.GlobalObject().Delete("IsARM") + _ = runtime.GlobalObject().Delete("IsARM64") + _ = runtime.GlobalObject().Delete("IsWasm") +} diff --git a/pkg/protocols/common/generators/attack_types.go b/pkg/protocols/common/generators/attack_types.go index 662a72399f..c0ad882f81 100644 --- a/pkg/protocols/common/generators/attack_types.go +++ b/pkg/protocols/common/generators/attack_types.go @@ -4,7 +4,7 @@ import ( "encoding/json" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" "github.com/pkg/errors" ) @@ -61,8 +61,8 @@ type AttackTypeHolder struct { Value AttackType `mapping:"true"` } -func (holder AttackTypeHolder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ +func (holder AttackTypeHolder) JSONSchemaType() *jsonschema.Schema { + gotType := &jsonschema.Schema{ Type: "string", Title: "type of the attack", Description: "Type of the attack", diff --git a/pkg/protocols/common/interactsh/interactsh.go b/pkg/protocols/common/interactsh/interactsh.go index b808359856..c156164805 100644 --- a/pkg/protocols/common/interactsh/interactsh.go +++ b/pkg/protocols/common/interactsh/interactsh.go @@ -153,11 +153,6 @@ func requestShouldStopAtFirstMatch(request *RequestData) bool { func (c *Client) processInteractionForRequest(interaction *server.Interaction, data *RequestData) bool { var result *operators.Result var matched bool - defer func() { - if r := recover(); r != nil { - gologger.Error().Msgf("panic occurred while processing interaction with result=%v matched=%v err=%v", result, matched, r) - } - }() data.Event.Lock() data.Event.InternalEvent["interactsh_protocol"] = interaction.Protocol if strings.EqualFold(interaction.Protocol, "dns") { diff --git a/pkg/protocols/common/variables/variables.go b/pkg/protocols/common/variables/variables.go index f26df59383..131098f9c9 100644 --- a/pkg/protocols/common/variables/variables.go +++ b/pkg/protocols/common/variables/variables.go @@ -4,7 +4,7 @@ import ( "encoding/json" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/expressions" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" @@ -21,12 +21,12 @@ type Variable struct { utils.InsertionOrderedStringMap `yaml:"-" json:"-"` } -func (variables Variable) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ +func (variables Variable) JSONSchemaType() *jsonschema.Schema { + gotType := &jsonschema.Schema{ Type: "object", Title: "variables for the request", Description: "Additional variables for the request", - AdditionalProperties: []byte("true"), + AdditionalProperties: &jsonschema.Schema{}, } return gotType } @@ -135,6 +135,12 @@ func (variables *Variable) checkForLazyEval() bool { return } } + // this is a hotfix and not the best way to do it + // will be refactored once we move scan state to scanContext (see: https://github.com/projectdiscovery/nuclei/issues/4631) + if strings.Contains(types.ToString(value), "interactsh-url") { + variables.LazyEval = true + return + } }) return variables.LazyEval } diff --git a/pkg/protocols/dns/dns_types.go b/pkg/protocols/dns/dns_types.go index 7f1f94ab34..1c90c5836d 100644 --- a/pkg/protocols/dns/dns_types.go +++ b/pkg/protocols/dns/dns_types.go @@ -5,7 +5,7 @@ import ( "errors" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" ) // DNSRequestType is the type of the method specified @@ -92,8 +92,8 @@ func (holder DNSRequestTypeHolder) String() string { return holder.DNSRequestType.String() } -func (holder DNSRequestTypeHolder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ +func (holder DNSRequestTypeHolder) JSONSchemaType() *jsonschema.Schema { + gotType := &jsonschema.Schema{ Type: "string", Title: "type of DNS request to make", Description: "Type is the type of DNS request to make", diff --git a/pkg/protocols/headless/engine/action_types.go b/pkg/protocols/headless/engine/action_types.go index d39cc0010d..52873e45de 100644 --- a/pkg/protocols/headless/engine/action_types.go +++ b/pkg/protocols/headless/engine/action_types.go @@ -5,7 +5,7 @@ import ( "errors" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" ) // ActionType defines the action type for a browser action @@ -171,8 +171,8 @@ type ActionTypeHolder struct { func (holder ActionTypeHolder) String() string { return holder.ActionType.String() } -func (holder ActionTypeHolder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ +func (holder ActionTypeHolder) JSONSchemaType() *jsonschema.Schema { + gotType := &jsonschema.Schema{ Type: "string", Title: "action to perform", Description: "Type of actions to perform", diff --git a/pkg/protocols/http/build_request.go b/pkg/protocols/http/build_request.go index 7408925a2b..bae9a37af0 100644 --- a/pkg/protocols/http/build_request.go +++ b/pkg/protocols/http/build_request.go @@ -32,6 +32,10 @@ import ( urlutil "github.com/projectdiscovery/utils/url" ) +const ( + ReqURLPatternKey = "req_url_pattern" +) + // ErrEvalExpression var ( ErrEvalExpression = errorutil.NewWithTag("expr", "could not evaluate helper expressions") @@ -48,6 +52,36 @@ type generatedRequest struct { dynamicValues map[string]interface{} interactshURLs []string customCancelFunction context.CancelFunc + // requestURLPattern tracks unmodified request url pattern without values ( it is used for constant vuln_hash) + // ex: {{BaseURL}}/api/exp?param={{randstr}} + requestURLPattern string +} + +// setReqURLPattern sets the url request pattern for the generated request +func (gr *generatedRequest) setReqURLPattern(reqURLPattern string) { + data := strings.Split(reqURLPattern, "\n") + if len(data) > 1 { + reqURLPattern = strings.TrimSpace(data[0]) + // this is raw request (if it has 3 parts after strings.Fields then its valid only use 2nd part) + parts := strings.Fields(reqURLPattern) + if len(parts) >= 3 { + // remove first and last and use all in between + parts = parts[1 : len(parts)-1] + reqURLPattern = strings.Join(parts, " ") + } + } else { + reqURLPattern = strings.TrimSpace(reqURLPattern) + } + + // now urlRequestPattern is generated replace preprocessor values with actual placeholders + // that were used (these are called generated 'constants' and contains {{}} in var name) + for k, v := range gr.original.options.Constants { + if strings.HasPrefix(k, "{{") && strings.HasSuffix(k, "}}") { + // this takes care of all preprocessors ( currently we have randstr and its variations) + reqURLPattern = strings.ReplaceAll(reqURLPattern, fmt.Sprint(v), k) + } + } + gr.requestURLPattern = reqURLPattern } // ApplyAuth applies the auth provider to the generated request @@ -96,7 +130,13 @@ func (r *requestGenerator) Total() int { // Make creates a http request for the provided input. // It returns ErrNoMoreRequests as error when all the requests have been exhausted. -func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, error) { +func (r *requestGenerator) Make(ctx context.Context, input *contextargs.Context, reqData string, payloads, dynamicValues map[string]interface{}) (gr *generatedRequest, err error) { + origReqData := reqData + defer func() { + if gr != nil { + gr.setReqURLPattern(origReqData) + } + }() // value of `reqData` depends on the type of request specified in template // 1. If request is raw request = reqData contains raw request (i.e http request dump) // 2. If request is Normal ( simply put not a raw request) (Ex: with placeholders `path`) = reqData contains relative path diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index 5ed5b63964..51e5a36dfc 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -211,13 +211,12 @@ type Request struct { // DisablePathAutomerge disables merging target url path with raw request path DisablePathAutomerge bool `yaml:"disable-path-automerge,omitempty" json:"disable-path-automerge,omitempty" jsonschema:"title=disable auto merging of path,description=Disable merging target url path with raw request path"` // description: | - // Filter is matcher-like field to check if fuzzing should be performed on this request or not - FuzzingFilter []*matchers.Matcher `yaml:"filters,omitempty" json:"filter,omitempty" jsonschema:"title=filter for fuzzing,description=Filter is matcher-like field to check if fuzzing should be performed on this request or not"` + // Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not + FuzzPreCondition []*matchers.Matcher `yaml:"pre-condition,omitempty" json:"pre-condition,omitempty" jsonschema:"title=pre-condition for fuzzing/dast,description=PreCondition is matcher-like field to check if fuzzing should be performed on this request or not"` // description: | - // Filter condition is the condition to apply on the filter (AND/OR). Default is OR - FuzzingFilterCondition string `yaml:"filters-condition,omitempty" json:"filter-condition,omitempty" jsonschema:"title=condition between the filters,description=Conditions between the filters,enum=and,enum=or"` - // cached variables that may be used along with request. - fuzzingFilterCondition matchers.ConditionType `yaml:"-" json:"-"` + // FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR + FuzzPreConditionOperator string `yaml:"pre-condition-operator,omitempty" json:"pre-condition-operator,omitempty" jsonschema:"title=condition between the filters,description=Operator to use between multiple per-conditions,enum=and,enum=or"` + fuzzPreConditionOperator matchers.ConditionType `yaml:"-" json:"-"` } // Options returns executer options for http request @@ -326,13 +325,13 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { // === fuzzing filters ===== // - if request.FuzzingFilterCondition != "" { - request.fuzzingFilterCondition = matchers.ConditionTypes[request.FuzzingFilterCondition] + if request.FuzzPreConditionOperator != "" { + request.fuzzPreConditionOperator = matchers.ConditionTypes[request.FuzzPreConditionOperator] } else { - request.fuzzingFilterCondition = matchers.ORCondition + request.fuzzPreConditionOperator = matchers.ORCondition } - for _, filter := range request.FuzzingFilter { + for _, filter := range request.FuzzPreCondition { if err := filter.CompileMatchers(); err != nil { return errors.Wrap(err, "could not compile matcher") } diff --git a/pkg/protocols/http/http_method_types.go b/pkg/protocols/http/http_method_types.go index 571446ab7e..9ef78911b0 100644 --- a/pkg/protocols/http/http_method_types.go +++ b/pkg/protocols/http/http_method_types.go @@ -5,7 +5,7 @@ import ( "errors" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" ) // HTTPMethodType is the type of the method specified @@ -89,8 +89,8 @@ func (holder HTTPMethodTypeHolder) String() string { return holder.MethodType.String() } -func (holder HTTPMethodTypeHolder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ +func (holder HTTPMethodTypeHolder) JSONSchemaType() *jsonschema.Schema { + gotType := &jsonschema.Schema{ Type: "string", Title: "method is the HTTP request method", Description: "Method is the HTTP Request Method", diff --git a/pkg/protocols/http/raw/raw.go b/pkg/protocols/http/raw/raw.go index a1eb544a20..f6c427ad96 100644 --- a/pkg/protocols/http/raw/raw.go +++ b/pkg/protocols/http/raw/raw.go @@ -82,6 +82,13 @@ func Parse(request string, inputURL *urlutil.URL, unsafe, disablePathAutomerge b } } } else { + // Edgecase if raw request is + // GET / HTTP/1.1 + //use case: https://github.com/projectdiscovery/nuclei/issues/4921 + if rawrequest.Path == "/" && cloned.Path != "" { + rawrequest.Path = "" + } + if disablePathAutomerge { cloned.Path = "" } @@ -97,6 +104,13 @@ func Parse(request string, inputURL *urlutil.URL, unsafe, disablePathAutomerge b default: cloned := inputURL.Clone() cloned.Params.IncludeEquals = true + // Edgecase if raw request is + // GET / HTTP/1.1 + //use case: https://github.com/projectdiscovery/nuclei/issues/4921 + if rawrequest.Path == "/" { + rawrequest.Path = "" + } + if disablePathAutomerge { cloned.Path = "" } @@ -283,7 +297,6 @@ func (r *Request) ApplyAuthStrategy(strategy authx.AuthStrategy) { gologger.Error().Msgf("auth strategy failed to parse url: %s got %v", r.FullURL, err) return } - _ = parsed for _, p := range s.Data.Params { parsed.Params.Add(p.Key, p.Value) } diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 0202f20e87..2763c5c628 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -51,6 +51,8 @@ const ( var ( MaxBodyRead = int64(1 << 22) // 4MB using shift operator + // ErrMissingVars is error occured when variables are missing + ErrMissingVars = errors.New("stop execution due to unresolved variables") ) // Type returns the type of the protocol request @@ -135,11 +137,6 @@ func (request *Request) executeRaceRequest(input *contextargs.Context, previous // execute http request go func(httpRequest *generatedRequest) { defer spmHandler.Release() - defer func() { - if r := recover(); r != nil { - gologger.Verbose().Msgf("[%s] Recovered from panic: %v\n", request.options.TemplateID, r) - } - }() if spmHandler.FoundFirstMatch() { // stop sending more requests condition is met return @@ -216,11 +213,6 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV spmHandler.Acquire() go func(httpRequest *generatedRequest) { defer spmHandler.Release() - defer func() { - if r := recover(); r != nil { - gologger.Verbose().Msgf("[%s] Recovered from panic: %v\n", request.options.TemplateID, r) - } - }() if spmHandler.FoundFirstMatch() { return } @@ -317,11 +309,6 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu spmHandler.Acquire() go func(httpRequest *generatedRequest) { defer spmHandler.Release() - defer func() { - if r := recover(); r != nil { - gologger.Verbose().Msgf("[%s] Recovered from panic: %v\n", request.options.TemplateID, r) - } - }() if spmHandler.FoundFirstMatch() { // skip if first match is found return @@ -437,7 +424,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa }, generator.currentIndex) // If a variable is unresolved, skip all further requests - if errors.Is(execReqErr, errStopExecution) { + if errors.Is(execReqErr, ErrMissingVars) { return true, nil } if execReqErr != nil { @@ -484,38 +471,17 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa const drainReqSize = int64(8 * 1024) -var errStopExecution = errors.New("stop execution due to unresolved variables") - // executeRequest executes the actual generated request and returns error if occurred -func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, callback protocols.OutputEventCallback, requestCount int) error { - outputEvent := output.InternalEvent{} - var event *output.InternalWrappedEvent - // event should never be nil as per existing logic - defer func() { - if event == nil { - event := &output.InternalWrappedEvent{ - InternalEvent: map[string]interface{}{ - "template-id": request.options.TemplateID, - "host": input.MetaInput.Input, - }, - } - if request.CompiledOperators != nil && request.CompiledOperators.HasDSL() { - event.InternalEvent = outputEvent - } - callback(event) - return - } - if event.InternalEvent == nil { - event.InternalEvent = outputEvent - } - // make sure templateId is never nil - if event.InternalEvent["template-id"] == nil { - event.InternalEvent["template-id"] = request.options.TemplateID - } - if event.InternalEvent["host"] == nil { - event.InternalEvent["host"] = input.MetaInput.Input - } - }() +func (request *Request) executeRequest(input *contextargs.Context, generatedRequest *generatedRequest, previousEvent output.InternalEvent, hasInteractMatchers bool, processEvent protocols.OutputEventCallback, requestCount int) (err error) { + + // wrap one more callback for validation and fixing event + callback := func(event *output.InternalWrappedEvent) { + // validateNFixEvent performs necessary validation on generated event + // and attempts to fix it , this includes things like making sure + // `template-id` is set , `request-url-pattern` is set etc + request.validateNFixEvent(input, generatedRequest, err, event) + processEvent(event) + } request.setCustomHeaders(generatedRequest) @@ -527,12 +493,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ finalMap["ip"] = input.MetaInput.CustomIP } - // we should never evaluate all variables of a template - // for payloadName, payloadValue := range generatedRequest.dynamicValues { - // if data, err := expressions.Evaluate(types.ToString(payloadValue), finalMap); err == nil { - // generatedRequest.dynamicValues[payloadName] = data - // } - // } for payloadName, payloadValue := range generatedRequest.meta { if data, err := expressions.Evaluate(types.ToString(payloadValue), finalMap); err == nil { generatedRequest.meta[payloadName] = data @@ -543,7 +503,6 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ resp *http.Response fromCache bool dumpedRequest []byte - err error ) // Dump request for variables checks @@ -586,12 +545,12 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ if ignoreList := GetVariablesNamesSkipList(generatedRequest.original.Signature.Value); ignoreList != nil { if varErr := expressions.ContainsVariablesWithIgnoreList(ignoreList, dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr) - return errStopExecution + return ErrMissingVars } } else { // Check if are there any unresolved variables. If yes, skip unless overridden by user. if varErr := expressions.ContainsUnresolvedVariables(dumpedRequestString); varErr != nil && !request.SkipVariablesCheck { gologger.Warning().Msgf("[%s] Could not make http request for %s: %v\n", request.options.TemplateID, input.MetaInput.Input, varErr) - return errStopExecution + return ErrMissingVars } } } @@ -717,7 +676,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ // In case of interactsh markers and request times out, still send // a callback event so in case we receive an interaction, correlation is possible. // Also, to log failed use-cases. - outputEvent = request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, convUtil.String(dumpedRequest), "", "", "", 0, generatedRequest.meta) + outputEvent := request.responseToDSLMap(&http.Response{}, input.MetaInput.Input, formedURL, convUtil.String(dumpedRequest), "", "", "", 0, generatedRequest.meta) if i := strings.LastIndex(hostname, ":"); i != -1 { hostname = hostname[:i] } @@ -727,6 +686,16 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ } else { outputEvent["ip"] = httpclientpool.Dialer.GetDialedIP(hostname) } + + if len(generatedRequest.interactshURLs) > 0 { + // according to logic we only need to trigger a callback if interactsh was used + // and request failed in hope that later on oast interaction will be received + event := &output.InternalWrappedEvent{} + if request.CompiledOperators != nil && request.CompiledOperators.HasDSL() { + event.InternalEvent = outputEvent + } + callback(event) + } return err } @@ -798,7 +767,7 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ } finalEvent := make(output.InternalEvent) - outputEvent = request.responseToDSLMap(respChain.Response(), input.MetaInput.Input, matchedURL, convUtil.String(dumpedRequest), respChain.FullResponse().String(), respChain.Body().String(), respChain.Headers().String(), duration, generatedRequest.meta) + outputEvent := request.responseToDSLMap(respChain.Response(), input.MetaInput.Input, matchedURL, convUtil.String(dumpedRequest), respChain.FullResponse().String(), respChain.Body().String(), respChain.Headers().String(), duration, generatedRequest.meta) // add response fields to template context and merge templatectx variables to output event request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent) if request.options.HasTemplateCtx(input.MetaInput) { @@ -836,13 +805,21 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ // prune signature internal values if any request.pruneSignatureInternalValues(generatedRequest.meta) - event = eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { + event := eventcreator.CreateEventWithAdditionalOptions(request, generators.MergeMaps(generatedRequest.dynamicValues, finalEvent), request.options.Options.Debug || request.options.Options.DebugResponse, func(internalWrappedEvent *output.InternalWrappedEvent) { internalWrappedEvent.OperatorsResult.PayloadValues = generatedRequest.meta }) if hasInteractMatchers { event.UsesInteractsh = true } + // if requrlpattern is enabled, only then it is reflected in result event else it is empty string + // consult @Ice3man543 before changing this logic (context: vuln_hash) + if request.options.ExportReqURLPattern { + for _, v := range event.Results { + v.ReqURLPattern = generatedRequest.requestURLPattern + } + } + responseContentType := respChain.Response().Header.Get("Content-Type") isResponseTruncated := request.MaxSize > 0 && respChain.Body().Len() >= request.MaxSize dumpResponse(event, request, respChain.FullResponse().Bytes(), formedURL, responseContentType, isResponseTruncated, input.MetaInput.Input) @@ -864,6 +841,37 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ return errx } +// validateNFixEvent validates and fixes the event +// it adds any missing template-id and request-url-pattern +func (request *Request) validateNFixEvent(input *contextargs.Context, gr *generatedRequest, err error, event *output.InternalWrappedEvent) { + if event != nil { + if event.InternalEvent == nil { + event.InternalEvent = make(map[string]interface{}) + event.InternalEvent["template-id"] = request.options.TemplateID + } + // add the request URL pattern to the event + event.InternalEvent[ReqURLPatternKey] = gr.requestURLPattern + if event.InternalEvent["host"] == nil { + event.InternalEvent["host"] = input.MetaInput.Input + } + if event.InternalEvent["template-id"] == nil { + event.InternalEvent["template-id"] = request.options.TemplateID + } + if event.InternalEvent["type"] == nil { + event.InternalEvent["type"] = request.Type().String() + } + if event.InternalEvent["template-path"] == nil { + event.InternalEvent["template-path"] = request.options.TemplatePath + } + if event.InternalEvent["template-info"] == nil { + event.InternalEvent["template-info"] = request.options.TemplateInfo + } + if err != nil { + event.InternalEvent["error"] = err.Error() + } + } +} + // handleSignature of the http request func (request *Request) handleSignature(generatedRequest *generatedRequest) error { switch request.Signature.Value { diff --git a/pkg/protocols/http/request_fuzz.go b/pkg/protocols/http/request_fuzz.go index b9f22131e4..d6a29c181c 100644 --- a/pkg/protocols/http/request_fuzz.go +++ b/pkg/protocols/http/request_fuzz.go @@ -8,6 +8,7 @@ package http import ( "context" "fmt" + "net/http" "strings" "github.com/pkg/errors" @@ -23,13 +24,14 @@ import ( protocolutils "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" "github.com/projectdiscovery/retryablehttp-go" + urlutil "github.com/projectdiscovery/utils/url" ) // executeFuzzingRule executes fuzzing request for a URL // TODO: // 1. use SPMHandler and rewrite stop at first match logic here // 2. use scanContext instead of contextargs.Context -func (request *Request) executeFuzzingRule(input *contextargs.Context, _ output.InternalEvent, callback protocols.OutputEventCallback) error { +func (request *Request) executeFuzzingRule(input *contextargs.Context, previous output.InternalEvent, callback protocols.OutputEventCallback) error { // methdology: // to check applicablity of rule, we first try to execute it with one value // if it is applicable, we execute all requests @@ -46,29 +48,20 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, _ output. return nil } - // Iterate through all requests for template and queue them for fuzzing - generator := request.newGenerator(true) - - // this will generate next value along with request it is meant to be used with - currRequest, payloads, result := generator.nextValue() - if !result && input.MetaInput.ReqResp == nil { - // this case is only true if input is not a full http request - return fmt.Errorf("no values to generate requests") + if input.MetaInput.Input == "" && input.MetaInput.ReqResp == nil { + return errors.New("empty input provided for fuzzing") } - // if it is a full http request obtained from target file + // ==== fuzzing when full HTTP request is provided ===== + if input.MetaInput.ReqResp != nil { - // Note: in case of full http request, we only need to build it once - // and then reuse it for all requests and completely abandon the request - // returned by generator - _ = currRequest - generated, err := input.MetaInput.ReqResp.BuildRequest() + baseRequest, err := input.MetaInput.ReqResp.BuildRequest() if err != nil { return errors.Wrap(err, "fuzz: could not build request obtained from target file") } - input.MetaInput.Input = generated.URL.String() + input.MetaInput.Input = baseRequest.URL.String() // execute with one value first to checks its applicability - err = request.executePayloadUsingRules(input, payloads, generated, callback) + err = request.executeAllFuzzingRules(input, previous, baseRequest, callback) if err != nil { // in case of any error, return it if fuzz.IsErrRuleNotApplicable(err) { @@ -76,39 +69,28 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, _ output. gologger.Verbose().Msgf("[%s] fuzz: %s\n", request.options.TemplateID, err) return nil } - if errors.Is(err, errStopExecution) { + if errors.Is(err, ErrMissingVars) { return err } - gologger.Verbose().Msgf("[%s] fuzz: inital payload request execution failed : %s\n", request.options.TemplateID, err) - } - - // if it is applicable, execute all requests - for { - _, payloads, result := generator.nextValue() - if !result { - break - } - err = request.executePayloadUsingRules(input, payloads, generated, callback) - if err != nil { - // continue to next request since this is payload specific - gologger.Verbose().Msgf("[%s] fuzz: payload request execution failed : %s\n", request.options.TemplateID, err) - continue - } + gologger.Verbose().Msgf("[%s] fuzz: payload request execution failed : %s\n", request.options.TemplateID, err) } return nil } // ==== fuzzing when only URL is provided ===== - generated, err := generator.Make(context.Background(), input, currRequest, payloads, nil) + // we need to use this url instead of input + inputx := input.Clone() + parsed, err := urlutil.ParseAbsoluteURL(input.MetaInput.Input, true) + if err != nil { + return errors.Wrap(err, "fuzz: could not parse input url") + } + baseRequest, err := retryablehttp.NewRequestFromURL(http.MethodGet, parsed, nil) if err != nil { return errors.Wrap(err, "fuzz: could not build request from url") } - // we need to use this url instead of input - inputx := input.Clone() - inputx.MetaInput.Input = generated.request.URL.String() // execute with one value first to checks its applicability - err = request.executePayloadUsingRules(inputx, generated.dynamicValues, generated.request, callback) + err = request.executeAllFuzzingRules(inputx, previous, baseRequest, callback) if err != nil { // in case of any error, return it if fuzz.IsErrRuleNotApplicable(err) { @@ -116,37 +98,16 @@ func (request *Request) executeFuzzingRule(input *contextargs.Context, _ output. gologger.Verbose().Msgf("[%s] fuzz: rule not applicable : %s\n", request.options.TemplateID, err) return nil } - if errors.Is(err, errStopExecution) { + if errors.Is(err, ErrMissingVars) { return err } - gologger.Verbose().Msgf("[%s] fuzz: inital payload request execution failed : %s\n", request.options.TemplateID, err) - } - - // continue to next request since this is payload specific - for { - currRequest, payloads, result = generator.nextValue() - if !result { - break - } - generated, err := generator.Make(context.Background(), input, currRequest, payloads, nil) - if err != nil { - return errors.Wrap(err, "fuzz: could not build request from url") - } - // we need to use this url instead of input - inputx := input.Clone() - inputx.MetaInput.Input = generated.request.URL.String() - // execute with one value first to checks its applicability - err = request.executePayloadUsingRules(inputx, generated.dynamicValues, generated.request, callback) - if err != nil { - gologger.Verbose().Msgf("[%s] fuzz: payload request execution failed : %s\n", request.options.TemplateID, err) - continue - } + gologger.Verbose().Msgf("[%s] fuzz: payload request execution failed : %s\n", request.options.TemplateID, err) } return nil } -// executePayloadUsingRules executes a payload using rules with given payload i.e values -func (request *Request) executePayloadUsingRules(input *contextargs.Context, values map[string]interface{}, baseRequest *retryablehttp.Request, callback protocols.OutputEventCallback) error { +// executeAllFuzzingRules executes all fuzzing rules defined in template for a given base request +func (request *Request) executeAllFuzzingRules(input *contextargs.Context, values map[string]interface{}, baseRequest *retryablehttp.Request, callback protocols.OutputEventCallback) error { applicable := false for _, rule := range request.Fuzzing { err := rule.Execute(&fuzz.ExecuteRuleInput{ @@ -156,7 +117,7 @@ func (request *Request) executePayloadUsingRules(input *contextargs.Context, val return request.executeGeneratedFuzzingRequest(gr, input, callback) }, Values: values, - BaseRequest: baseRequest, + BaseRequest: baseRequest.Clone(context.TODO()), }) if err == nil { applicable = true @@ -212,7 +173,7 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, } }, 0) // If a variable is unresolved, skip all further requests - if errors.Is(requestErr, errStopExecution) { + if errors.Is(requestErr, ErrMissingVars) { return false } if requestErr != nil { @@ -233,11 +194,11 @@ func (request *Request) executeGeneratedFuzzingRequest(gr fuzz.GeneratedRequest, // ShouldFuzzTarget checks if given target should be fuzzed or not using `filter` field in template func (request *Request) ShouldFuzzTarget(input *contextargs.Context) bool { - if len(request.FuzzingFilter) == 0 { + if len(request.FuzzPreCondition) == 0 { return true } status := []bool{} - for index, filter := range request.FuzzingFilter { + for index, filter := range request.FuzzPreCondition { isMatch, _ := request.Match(request.filterDataMap(input), filter) status = append(status, isMatch) if request.options.Options.MatcherStatus { @@ -248,7 +209,7 @@ func (request *Request) ShouldFuzzTarget(input *contextargs.Context) bool { return true } var matched bool - if request.fuzzingFilterCondition == matchers.ANDCondition { + if request.fuzzPreConditionOperator == matchers.ANDCondition { matched = operators.EvalBoolSlice(status, true) } else { matched = operators.EvalBoolSlice(status, false) @@ -295,6 +256,9 @@ func (request *Request) filterDataMap(input *contextargs.Context) map[string]int return true }) m["header"] = sb.String() + } else { + // add default method value + m["method"] = http.MethodGet } // dump if svd is enabled diff --git a/pkg/protocols/http/request_test.go b/pkg/protocols/http/request_test.go index 3d9697125c..cd8e860b07 100644 --- a/pkg/protocols/http/request_test.go +++ b/pkg/protocols/http/request_test.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers" "github.com/projectdiscovery/nuclei/v3/pkg/output" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/interactsh" "github.com/projectdiscovery/nuclei/v3/pkg/testutils" ) @@ -184,3 +185,74 @@ func TestDisableTE(t *testing.T) { require.NotNil(t, finalEvent, "could not get event output from request") require.Equal(t, 2, matchCount, "could not get correct match count") } + +// consult @Ice3man543 before making any breaking changes to this test (context: vuln_hash) +func TestReqURLPattern(t *testing.T) { + options := testutils.DefaultOptions + + // assume this was a preprocessor + // {{randstr}} => 2eNU2kbrOcUDzhnUL1RGvSo1it7 + testutils.Init(options) + templateID := "testing-http" + request := &Request{ + ID: templateID, + Raw: []string{ + `GET /{{rand_char("abc")}}/{{interactsh-url}}/123?query={{rand_int(1, 10)}}&data=2eNU2kbrOcUDzhnUL1RGvSo1it7 HTTP/1.1 + Host: {{Hostname}} + User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0 + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + Accept-Language: en-US,en;q=0.5 + `, + }, + Operators: operators.Operators{ + Matchers: []*matchers.Matcher{{ + Type: matchers.MatcherTypeHolder{MatcherType: matchers.DSLMatcher}, + DSL: []string{"true"}, + }}, + }, + IterateAll: true, + } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // always return 200 + w.WriteHeader(200) + _, _ = w.Write([]byte(`match`)) + })) + defer ts.Close() + + executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{ + ID: templateID, + Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, + }) + client, _ := interactsh.New(interactsh.DefaultOptions(executerOpts.Output, nil, executerOpts.Progress)) + executerOpts.Interactsh = client + defer client.Close() + executerOpts.ExportReqURLPattern = true + + // this is how generated constants are added to template + // generated constants are preprocessors that are executed while loading once + executerOpts.Constants = map[string]interface{}{ + "{{randstr}}": "2eNU2kbrOcUDzhnUL1RGvSo1it7", + } + + err := request.Compile(executerOpts) + require.Nil(t, err, "could not compile network request") + + var finalEvent *output.InternalWrappedEvent + var matchCount int + t.Run("test", func(t *testing.T) { + metadata := make(output.InternalEvent) + previous := make(output.InternalEvent) + ctxArgs := contextargs.NewWithInput(ts.URL) + err := request.ExecuteWithResults(ctxArgs, metadata, previous, func(event *output.InternalWrappedEvent) { + if event.OperatorsResult != nil && event.OperatorsResult.Matched { + matchCount++ + } + finalEvent = event + }) + require.Nil(t, err, "could not execute network request") + }) + require.NotNil(t, finalEvent, "could not get event output from request") + require.Equal(t, 1, matchCount, "could not get correct match count") + require.NotEmpty(t, finalEvent.Results[0].ReqURLPattern, "could not get req url pattern") + require.Equal(t, `/{{rand_char("abc")}}/{{interactsh-url}}/123?query={{rand_int(1, 10)}}&data={{randstr}}`, finalEvent.Results[0].ReqURLPattern) +} diff --git a/pkg/protocols/http/signature.go b/pkg/protocols/http/signature.go index b8d491712d..c28ecf6827 100644 --- a/pkg/protocols/http/signature.go +++ b/pkg/protocols/http/signature.go @@ -4,7 +4,7 @@ import ( "encoding/json" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" "github.com/pkg/errors" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/signer" @@ -51,8 +51,8 @@ type SignatureTypeHolder struct { Value SignatureType } -func (holder SignatureTypeHolder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ +func (holder SignatureTypeHolder) JSONSchemaType() *jsonschema.Schema { + gotType := &jsonschema.Schema{ Type: "string", Title: "type of the signature", Description: "Type of the signature", diff --git a/pkg/protocols/network/network_input_types.go b/pkg/protocols/network/network_input_types.go index 9180757c9f..e8b294eaab 100644 --- a/pkg/protocols/network/network_input_types.go +++ b/pkg/protocols/network/network_input_types.go @@ -5,7 +5,7 @@ import ( "errors" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" ) // NetworkInputType is the type of the network input specified @@ -66,8 +66,8 @@ func (holder NetworkInputTypeHolder) String() string { return holder.NetworkInputType.String() } -func (holder NetworkInputTypeHolder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ +func (holder NetworkInputTypeHolder) JSONSchemaType() *jsonschema.Schema { + gotType := &jsonschema.Schema{ Type: "string", Title: "type is the type of input data", Description: "description=Type of input specified in data field", diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 5cdc2a50d2..f9d9064200 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -123,6 +123,9 @@ type ExecutorOptions struct { //TemporaryDirectory is the directory to store temporary files TemporaryDirectory string Parser parser.Parser + // ExportReqURLPattern exports the request URL pattern + // in ResultEvent it contains the exact url pattern (ex: {{BaseURL}}/{{randstr}}/xyz) used in the request + ExportReqURLPattern bool } // GetThreadsForPayloadRequests returns the number of threads to use as default for diff --git a/pkg/reporting/exporters/sarif/sarif.go b/pkg/reporting/exporters/sarif/sarif.go index d2fe63bd4b..67a438d80c 100644 --- a/pkg/reporting/exporters/sarif/sarif.go +++ b/pkg/reporting/exporters/sarif/sarif.go @@ -53,8 +53,8 @@ func (exporter *Exporter) addToolDetails() { FullDescription: &sarif.MultiformatMessageString{ Text: "Fast and customizable vulnerability scanner based on simple YAML based DSL", }, - FullName: "Nuclei v" + config.Version, - SemanticVersion: "v" + config.Version, + FullName: "Nuclei " + config.Version, + SemanticVersion: config.Version, DownloadURI: "https://github.com/projectdiscovery/nuclei/releases", Rules: exporter.rules, } diff --git a/pkg/templates/cluster.go b/pkg/templates/cluster.go index e404a3d28c..b1e0b56e2d 100644 --- a/pkg/templates/cluster.go +++ b/pkg/templates/cluster.go @@ -59,6 +59,14 @@ func Cluster(list []*Template) [][]*Template { final = append(final, []*Template{template}) continue } + + // it is not possible to cluster flow and multiprotocol due to dependent execution + if template.Flow != "" || template.Options.IsMultiProtocol { + _ = skip.Set(key, struct{}{}) + final = append(final, []*Template{template}) + continue + } + _ = skip.Set(key, struct{}{}) var templateType types.ProtocolType @@ -81,6 +89,13 @@ func Cluster(list []*Template) [][]*Template { continue } + // it is not possible to cluster flow and multiprotocol due to dependent execution + if other.Flow != "" || other.Options.IsMultiProtocol { + _ = skip.Set(otherKey, struct{}{}) + final = append(final, []*Template{other}) + continue + } + switch templateType { case types.DNSProtocol: if len(other.RequestsDNS) != 1 { diff --git a/pkg/templates/cluster_test.go b/pkg/templates/cluster_test.go index bfd10fd427..c8e68b3e07 100644 --- a/pkg/templates/cluster_test.go +++ b/pkg/templates/cluster_test.go @@ -3,15 +3,25 @@ package templates import ( "testing" + "github.com/projectdiscovery/nuclei/v3/pkg/model" + "github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/dns" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/http" + "github.com/projectdiscovery/nuclei/v3/pkg/testutils" "github.com/stretchr/testify/require" ) func TestClusterTemplates(t *testing.T) { + // state of whether template is flow or multiprotocol is stored in executerOptions i.e why we need to pass it + execOptions := testutils.NewMockExecuterOptions(testutils.DefaultOptions, &testutils.TemplateInfo{ + ID: "templateID", + Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"}, + }) t.Run("http-cluster-get", func(t *testing.T) { tp1 := &Template{Path: "first.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}} tp2 := &Template{Path: "second.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}"}}}} + tp1.Options = execOptions + tp2.Options = execOptions tpls := []*Template{tp1, tp2} // cluster 0 expected := []*Template{tp1, tp2} @@ -21,6 +31,8 @@ func TestClusterTemplates(t *testing.T) { t.Run("no-http-cluster", func(t *testing.T) { tp1 := &Template{Path: "first.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}/random"}}}} tp2 := &Template{Path: "second.yaml", RequestsHTTP: []*http.Request{{Path: []string{"{{BaseURL}}/another"}}}} + tp1.Options = execOptions + tp2.Options = execOptions tpls := []*Template{tp1, tp2} expected := [][]*Template{{tp1}, {tp2}} got := Cluster(tpls) @@ -29,6 +41,8 @@ func TestClusterTemplates(t *testing.T) { t.Run("dns-cluster", func(t *testing.T) { tp1 := &Template{Path: "first.yaml", RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}} tp2 := &Template{Path: "second.yaml", RequestsDNS: []*dns.Request{{Name: "{{Hostname}}"}}} + tp1.Options = execOptions + tp2.Options = execOptions tpls := []*Template{tp1, tp2} // cluster 0 expected := []*Template{tp1, tp2} diff --git a/pkg/templates/compile.go b/pkg/templates/compile.go index f831485d89..b821fcc675 100644 --- a/pkg/templates/compile.go +++ b/pkg/templates/compile.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/nuclei/v3/pkg/js/compiler" "github.com/projectdiscovery/nuclei/v3/pkg/operators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/offlinehttp" "github.com/projectdiscovery/nuclei/v3/pkg/templates/signer" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec" @@ -34,7 +35,8 @@ var ( ) const ( - Unsigned = "unsigned" + Unsigned = "unsigned" + PDVerifier = "projectdiscovery/nuclei-templates" ) func init() { @@ -271,7 +273,6 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option if config.DefaultConfig.LogAllEvents { gologger.DefaultLogger.Print().Msgf("[%v] Template %s is not signed or tampered\n", aurora.Yellow("WRN").String(), template.ID) } - SignatureStats[Unsigned].Add(1) } return template, nil } @@ -292,17 +293,24 @@ func ParseTemplateFromReader(reader io.Reader, preprocessor Preprocessor, option if config.DefaultConfig.LogAllEvents { gologger.DefaultLogger.Print().Msgf("[%v] Template %s is not signed or tampered\n", aurora.Yellow("WRN").String(), template.ID) } - SignatureStats[Unsigned].Add(1) } + generatedConstants := map[string]interface{}{} // ==== execute preprocessors ====== for _, v := range allPreprocessors { - data = v.Process(data) + var replaced map[string]interface{} + data, replaced = v.ProcessNReturnData(data) + // preprocess kind of act like a constant and are generated while loading + // and stay constant for the template lifecycle + generatedConstants = generators.MergeMaps(generatedConstants, replaced) } reParsed, err := parseTemplate(data, options) if err != nil { return nil, err } + // add generated constants to constants map and executer options + reParsed.Constants = generators.MergeMaps(reParsed.Constants, generatedConstants) + reParsed.Options.Constants = reParsed.Constants reParsed.Verified = isVerified return reParsed, nil } @@ -390,7 +398,7 @@ func parseTemplate(data []byte, options protocols.ExecutorOptions) (*Template, e for _, verifier = range signer.DefaultTemplateVerifiers { template.Verified, _ = verifier.Verify(data, template) if template.Verified { - SignatureStats[verifier.Identifier()].Add(1) + template.TemplateVerifier = verifier.Identifier() break } } diff --git a/pkg/templates/parser_stats.go b/pkg/templates/parser_stats.go index 473439b6f5..290601032d 100644 --- a/pkg/templates/parser_stats.go +++ b/pkg/templates/parser_stats.go @@ -1,13 +1,14 @@ package templates const ( - SyntaxWarningStats = "syntax-warnings" - SyntaxErrorStats = "syntax-errors" - RuntimeWarningsStats = "runtime-warnings" - UnsignedCodeWarning = "unsigned-warnings" - HeadlessFlagWarningStats = "headless-flag-missing-warnings" - TemplatesExecutedStats = "templates-executed" - CodeFlagWarningStats = "code-flag-missing-warnings" - FuzzFlagWarningStats = "fuzz-flag-missing-warnings" - SkippedUnsignedStats = "skipped-unsigned-stats" // tracks loading of unsigned templates + SyntaxWarningStats = "syntax-warnings" + SyntaxErrorStats = "syntax-errors" + RuntimeWarningsStats = "runtime-warnings" + SkippedCodeTmplTamperedStats = "unsigned-warnings" + ExcludedHeadlessTmplStats = "headless-flag-missing-warnings" + TemplatesExcludedStats = "templates-executed" + ExcludedCodeTmplStats = "code-flag-missing-warnings" + ExludedDastTmplStats = "fuzz-flag-missing-warnings" + SkippedUnsignedStats = "skipped-unsigned-stats" // tracks loading of unsigned templates + SkippedRequestSignatureStats = "skipped-request-signature-stats" ) diff --git a/pkg/templates/preprocessors.go b/pkg/templates/preprocessors.go index d7e5864d6b..054a0bc41e 100644 --- a/pkg/templates/preprocessors.go +++ b/pkg/templates/preprocessors.go @@ -10,7 +10,7 @@ import ( type Preprocessor interface { // Process processes the data and returns the processed data. - Process(data []byte) []byte + ProcessNReturnData(data []byte) ([]byte, map[string]interface{}) // Exists check if the preprocessor exists in the data. Exists(data []byte) bool } @@ -39,10 +39,10 @@ var _ Preprocessor = &randStrPreprocessor{} type randStrPreprocessor struct{} -// Process processes the data and returns the processed data. -func (r *randStrPreprocessor) Process(data []byte) []byte { +// ProcessNReturnData processes the data and returns the key-value pairs of generated/replaced data. +func (r *randStrPreprocessor) ProcessNReturnData(data []byte) ([]byte, map[string]interface{}) { foundMap := make(map[string]struct{}) - + dataMap := make(map[string]interface{}) for _, expression := range preprocessorRegex.FindAllStringSubmatch(string(data), -1) { if len(expression) != 2 { continue @@ -57,10 +57,12 @@ func (r *randStrPreprocessor) Process(data []byte) []byte { } foundMap[value] = struct{}{} if strings.EqualFold(value, "randstr") || strings.HasPrefix(value, "randstr_") { - data = bytes.ReplaceAll(data, []byte(expression[0]), []byte(ksuid.New().String())) + randStr := ksuid.New().String() + data = bytes.ReplaceAll(data, []byte(expression[0]), []byte(randStr)) + dataMap[expression[0]] = randStr } } - return data + return data, dataMap } // Exists check if the preprocessor exists in the data. diff --git a/pkg/templates/stats.go b/pkg/templates/stats.go index 25a61e6e06..fe3d4ca339 100644 --- a/pkg/templates/stats.go +++ b/pkg/templates/stats.go @@ -6,10 +6,11 @@ func init() { stats.NewEntry(SyntaxWarningStats, "Found %d templates with syntax warning (use -validate flag for further examination)") stats.NewEntry(SyntaxErrorStats, "Found %d templates with syntax error (use -validate flag for further examination)") stats.NewEntry(RuntimeWarningsStats, "Found %d templates with runtime error (use -validate flag for further examination)") - stats.NewEntry(UnsignedCodeWarning, "Found %d unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them)") - stats.NewEntry(HeadlessFlagWarningStats, "Excluded %d headless template[s] (disabled as default), use -headless option to run headless templates.") - stats.NewEntry(CodeFlagWarningStats, "Excluded %d code template[s] (disabled as default), use -code option to run code templates.") - stats.NewEntry(TemplatesExecutedStats, "Excluded %d template[s] with known weak matchers / tags excluded from default run using .nuclei-ignore") - stats.NewEntry(FuzzFlagWarningStats, "Excluded %d fuzz template[s] (disabled as default), use -fuzz option to run fuzz templates.") + stats.NewEntry(SkippedCodeTmplTamperedStats, "Found %d unsigned or tampered code template (carefully examine before using it & use -sign flag to sign them)") + stats.NewEntry(ExcludedHeadlessTmplStats, "Excluded %d headless template[s] (disabled as default), use -headless option to run headless templates.") + stats.NewEntry(ExcludedCodeTmplStats, "Excluded %d code template[s] (disabled as default), use -code option to run code templates.") + stats.NewEntry(TemplatesExcludedStats, "Excluded %d template[s] with known weak matchers / tags excluded from default run using .nuclei-ignore") + stats.NewEntry(ExludedDastTmplStats, "Excluded %d dast template[s] (disabled as default), use -dast option to run dast templates.") stats.NewEntry(SkippedUnsignedStats, "Skipping %d unsigned template[s]") + stats.NewEntry(SkippedRequestSignatureStats, "Skipping %d templates, HTTP Request signatures can only be used in Signed & Verified templates.") } diff --git a/pkg/templates/templates.go b/pkg/templates/templates.go index bd81d1451b..6403c2767a 100644 --- a/pkg/templates/templates.go +++ b/pkg/templates/templates.go @@ -45,12 +45,12 @@ type Template struct { // examples: // - name: ID Example // value: "\"CVE-2021-19520\"" - ID string `yaml:"id" json:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"` + ID string `yaml:"id" json:"id" jsonschema:"title=id of the template,description=The Unique ID for the template,required,example=cve-2021-19520,pattern=^([a-zA-Z0-9]+[-_])*[a-zA-Z0-9]+$"` // description: | // Info contains metadata information about the template. // examples: // - value: exampleInfoStructure - Info model.Info `yaml:"info" json:"info" jsonschema:"title=info for the template,description=Info contains metadata for the template"` + Info model.Info `yaml:"info" json:"info" jsonschema:"title=info for the template,description=Info contains metadata for the template,required,type=object"` // description: | // Flow contains the execution flow for the template. // examples: @@ -62,13 +62,13 @@ type Template struct { // http(1) // } // - Flow string `yaml:"flow,omitempty" json:"flow,omitempty" jsonschema:"title=template execution flow in js,description=Flow contains js code which defines how the template should be executed"` + Flow string `yaml:"flow,omitempty" json:"flow,omitempty" jsonschema:"title=template execution flow in js,description=Flow contains js code which defines how the template should be executed,type=string,example='flow: http(0) && http(1)'"` // description: | // Requests contains the http request to make in the template. // WARNING: 'requests' will be deprecated and will be removed in a future release. Please use 'http' instead. // examples: // - value: exampleNormalHTTPRequest - RequestsHTTP []*http.Request `yaml:"requests,omitempty" json:"requests,omitempty" jsonschema:"title=http requests to make,description=HTTP requests to make for the template"` + RequestsHTTP []*http.Request `yaml:"requests,omitempty" json:"requests,omitempty" jsonschema:"title=http requests to make,description=HTTP requests to make for the template,deprecated=true"` // description: | // HTTP contains the http request to make in the template. // examples: @@ -91,7 +91,7 @@ type Template struct { // WARNING: 'network' will be deprecated and will be removed in a future release. Please use 'tcp' instead. // examples: // - value: exampleNormalNetworkRequest - RequestsNetwork []*network.Request `yaml:"network,omitempty" json:"network,omitempty" jsonschema:"title=network requests to make,description=Network requests to make for the template"` + RequestsNetwork []*network.Request `yaml:"network,omitempty" json:"network,omitempty" jsonschema:"title=network requests to make,description=Network requests to make for the template,deprecated=true"` // description: | // TCP contains the network request to make in the template // examples: @@ -132,17 +132,18 @@ type Template struct { // description: | // Signature is the request signature method + // WARNING: 'signature' will be deprecated and will be removed in a future release. Prefer using 'code' protocol for writing cloud checks // values: // - "AWS" - Signature http.SignatureTypeHolder `yaml:"signature,omitempty" json:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS"` + Signature http.SignatureTypeHolder `yaml:"signature,omitempty" json:"signature,omitempty" jsonschema:"title=signature is the http request signature method,description=Signature is the HTTP Request signature Method,enum=AWS,deprecated=true"` // description: | // Variables contains any variables for the current request. - Variables variables.Variable `yaml:"variables,omitempty" json:"variables,omitempty" jsonschema:"title=variables for the http request,description=Variables contains any variables for the current request"` + Variables variables.Variable `yaml:"variables,omitempty" json:"variables,omitempty" jsonschema:"title=variables for the http request,description=Variables contains any variables for the current request,type=object"` // description: | // Constants contains any scalar constant for the current template - Constants map[string]interface{} `yaml:"constants,omitempty" json:"constants,omitempty" jsonschema:"title=constant for the template,description=constants contains any constant for the template"` + Constants map[string]interface{} `yaml:"constants,omitempty" json:"constants,omitempty" jsonschema:"title=constant for the template,description=constants contains any constant for the template,type=object"` // TotalRequests is the total number of requests for the template. TotalRequests int `yaml:"-" json:"-"` @@ -153,7 +154,8 @@ type Template struct { // Verified defines if the template signature is digitally verified Verified bool `yaml:"-" json:"-"` - + // TemplateVerifier is identifier verifier used to verify the template (default nuclei-templates have projectdiscovery/nuclei-templates) + TemplateVerifier string `yaml:"-" json:"-"` // RequestsQueue contains all template requests in order (both protocol & request order) RequestsQueue []protocols.Request `yaml:"-" json:"-"` @@ -214,6 +216,11 @@ func (template *Template) IsFuzzing() bool { return false } +// UsesRequestSignature returns true if the template uses a request signature like AWS +func (template *Template) UsesRequestSignature() bool { + return template.Signature.Value.String() != "" +} + // HasCodeProtocol returns true if the template has a code protocol section func (template *Template) HasCodeProtocol() bool { return len(template.RequestsCode) > 0 diff --git a/pkg/templates/templates_doc.go b/pkg/templates/templates_doc.go index cfa78e3ef5..dbd865c4e7 100644 --- a/pkg/templates/templates_doc.go +++ b/pkg/templates/templates_doc.go @@ -148,7 +148,7 @@ func init() { TemplateDoc.Fields[17].Name = "signature" TemplateDoc.Fields[17].Type = "http.SignatureTypeHolder" TemplateDoc.Fields[17].Note = "" - TemplateDoc.Fields[17].Description = "Signature is the request signature method" + TemplateDoc.Fields[17].Description = "Signature is the request signature method\nWARNING: 'signature' will be deprecated and will be removed in a future release. Prefer using 'code' protocol for writing cloud checks" TemplateDoc.Fields[17].Comments[encoder.LineComment] = "Signature is the request signature method" TemplateDoc.Fields[17].Values = []string{ "AWS", @@ -653,16 +653,16 @@ func init() { HTTPRequestDoc.Fields[32].Note = "" HTTPRequestDoc.Fields[32].Description = "DisablePathAutomerge disables merging target url path with raw request path" HTTPRequestDoc.Fields[32].Comments[encoder.LineComment] = "DisablePathAutomerge disables merging target url path with raw request path" - HTTPRequestDoc.Fields[33].Name = "filters" + HTTPRequestDoc.Fields[33].Name = "pre-condition" HTTPRequestDoc.Fields[33].Type = "[]matchers.Matcher" HTTPRequestDoc.Fields[33].Note = "" - HTTPRequestDoc.Fields[33].Description = "Filter is matcher-like field to check if fuzzing should be performed on this request or not" - HTTPRequestDoc.Fields[33].Comments[encoder.LineComment] = "Filter is matcher-like field to check if fuzzing should be performed on this request or not" - HTTPRequestDoc.Fields[34].Name = "filters-condition" + HTTPRequestDoc.Fields[33].Description = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not" + HTTPRequestDoc.Fields[33].Comments[encoder.LineComment] = "Fuzz PreCondition is matcher-like field to check if fuzzing should be performed on this request or not" + HTTPRequestDoc.Fields[34].Name = "pre-condition-operator" HTTPRequestDoc.Fields[34].Type = "string" HTTPRequestDoc.Fields[34].Note = "" - HTTPRequestDoc.Fields[34].Description = "Filter condition is the condition to apply on the filter (AND/OR). Default is OR" - HTTPRequestDoc.Fields[34].Comments[encoder.LineComment] = "Filter condition is the condition to apply on the filter (AND/OR). Default is OR" + HTTPRequestDoc.Fields[34].Description = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR" + HTTPRequestDoc.Fields[34].Comments[encoder.LineComment] = "FuzzPreConditionOperator is the operator between multiple PreConditions for fuzzing Default is OR" GENERATORSAttackTypeHolderDoc.Type = "generators.AttackTypeHolder" GENERATORSAttackTypeHolderDoc.Comments[encoder.LineComment] = " AttackTypeHolder is used to hold internal type of the protocol" @@ -836,7 +836,7 @@ func init() { MATCHERSMatcherDoc.AppearsIn = []encoder.Appearance{ { TypeName: "http.Request", - FieldName: "filters", + FieldName: "pre-condition", }, } MATCHERSMatcherDoc.Fields = make([]encoder.Doc, 16) @@ -1851,7 +1851,7 @@ func init() { Value: "Matched is the input which was matched upon", }, } - CODERequestDoc.Fields = make([]encoder.Doc, 5) + CODERequestDoc.Fields = make([]encoder.Doc, 6) CODERequestDoc.Fields[0].Name = "id" CODERequestDoc.Fields[0].Type = "string" CODERequestDoc.Fields[0].Note = "" @@ -1862,21 +1862,26 @@ func init() { CODERequestDoc.Fields[1].Note = "" CODERequestDoc.Fields[1].Description = "Engine type" CODERequestDoc.Fields[1].Comments[encoder.LineComment] = "Engine type" - CODERequestDoc.Fields[2].Name = "args" - CODERequestDoc.Fields[2].Type = "[]string" + CODERequestDoc.Fields[2].Name = "pre-condition" + CODERequestDoc.Fields[2].Type = "string" CODERequestDoc.Fields[2].Note = "" - CODERequestDoc.Fields[2].Description = "Engine Arguments" - CODERequestDoc.Fields[2].Comments[encoder.LineComment] = "Engine Arguments" - CODERequestDoc.Fields[3].Name = "pattern" - CODERequestDoc.Fields[3].Type = "string" + CODERequestDoc.Fields[2].Description = "PreCondition is a condition which is evaluated before sending the request." + CODERequestDoc.Fields[2].Comments[encoder.LineComment] = "PreCondition is a condition which is evaluated before sending the request." + CODERequestDoc.Fields[3].Name = "args" + CODERequestDoc.Fields[3].Type = "[]string" CODERequestDoc.Fields[3].Note = "" - CODERequestDoc.Fields[3].Description = "Pattern preferred for file name" - CODERequestDoc.Fields[3].Comments[encoder.LineComment] = "Pattern preferred for file name" - CODERequestDoc.Fields[4].Name = "source" + CODERequestDoc.Fields[3].Description = "Engine Arguments" + CODERequestDoc.Fields[3].Comments[encoder.LineComment] = "Engine Arguments" + CODERequestDoc.Fields[4].Name = "pattern" CODERequestDoc.Fields[4].Type = "string" CODERequestDoc.Fields[4].Note = "" - CODERequestDoc.Fields[4].Description = "Source File/Snippet" - CODERequestDoc.Fields[4].Comments[encoder.LineComment] = "Source File/Snippet" + CODERequestDoc.Fields[4].Description = "Pattern preferred for file name" + CODERequestDoc.Fields[4].Comments[encoder.LineComment] = "Pattern preferred for file name" + CODERequestDoc.Fields[5].Name = "source" + CODERequestDoc.Fields[5].Type = "string" + CODERequestDoc.Fields[5].Note = "" + CODERequestDoc.Fields[5].Description = "Source File/Snippet" + CODERequestDoc.Fields[5].Comments[encoder.LineComment] = "Source File/Snippet" JAVASCRIPTRequestDoc.Type = "javascript.Request" JAVASCRIPTRequestDoc.Comments[encoder.LineComment] = " Request is a request for the javascript protocol" diff --git a/pkg/templates/types/types.go b/pkg/templates/types/types.go index b8987ec322..f1195dc74d 100644 --- a/pkg/templates/types/types.go +++ b/pkg/templates/types/types.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "github.com/alecthomas/jsonschema" + "github.com/invopop/jsonschema" "github.com/pkg/errors" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/nuclei/v3/pkg/model/types/stringslice" @@ -92,8 +92,8 @@ type TypeHolder struct { ProtocolType ProtocolType `mapping:"true"` } -func (holder TypeHolder) JSONSchemaType() *jsonschema.Type { - gotType := &jsonschema.Type{ +func (holder TypeHolder) JSONSchemaType() *jsonschema.Schema { + gotType := &jsonschema.Schema{ Type: "string", Title: "type of the protocol", Description: "Type of the protocol", diff --git a/pkg/templates/workflows.go b/pkg/templates/workflows.go index c402ce7610..bafb7aaf31 100644 --- a/pkg/templates/workflows.go +++ b/pkg/templates/workflows.go @@ -86,6 +86,10 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr stats.Increment(SkippedUnsignedStats) continue } + if template.UsesRequestSignature() && !template.Verified { + stats.Increment(SkippedRequestSignatureStats) + continue + } if len(template.RequestsCode) > 0 { if !options.Options.EnableCodeTemplates { @@ -97,6 +101,17 @@ func parseWorkflowTemplate(workflow *workflows.WorkflowTemplate, preprocessor Pr continue } } + + // increment signed/unsigned counters + if template.Verified { + if template.TemplateVerifier == "" { + SignatureStats[PDVerifier].Add(1) + } else { + SignatureStats[template.TemplateVerifier].Add(1) + } + } else { + SignatureStats[Unsigned].Add(1) + } workflowTemplates = append(workflowTemplates, template) } diff --git a/pkg/tmplexec/exec.go b/pkg/tmplexec/exec.go index 3e0e8fdded..e434f2173c 100644 --- a/pkg/tmplexec/exec.go +++ b/pkg/tmplexec/exec.go @@ -3,7 +3,6 @@ package tmplexec import ( "errors" "fmt" - "runtime/debug" "strings" "sync/atomic" @@ -93,28 +92,24 @@ func (e *TemplateExecuter) Requests() int { // Execute executes the protocol group and returns true or false if results were found. func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { - results := &atomic.Bool{} + // executed contains status of execution if it was successfully executed or not + // doesn't matter if it was matched or not + executed := &atomic.Bool{} + // matched in this case means something was exported / written to output + matched := &atomic.Bool{} defer func() { // it is essential to remove template context of `Scan i.e template x input pair` // since it is of no use after scan is completed (regardless of success or failure) e.options.RemoveTemplateCtx(ctx.Input.MetaInput) }() - defer func() { - // try catching unknown panics - if r := recover(); r != nil { - stacktrace := debug.Stack() - ctx.LogError(fmt.Errorf("panic: %v\n%s", r, stacktrace)) - gologger.Verbose().Msgf("panic: %v\n%s", r, stacktrace) - } - }() var lastMatcherEvent *output.InternalWrappedEvent writeFailureCallback := func(event *output.InternalWrappedEvent, matcherStatus bool) { - if !results.Load() && matcherStatus { + if !matched.Load() && matcherStatus { if err := e.options.Output.WriteFailure(event); err != nil { gologger.Warning().Msgf("Could not write failure event to output: %s\n", err) } - results.CompareAndSwap(false, true) + executed.CompareAndSwap(false, true) } } @@ -142,11 +137,11 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { // If no results were found, and also interactsh is not being used // in that case we can skip it, otherwise we've to show failure in // case of matcher-status flag. - if !event.HasOperatorResult() && !event.UsesInteractsh { + if !event.HasOperatorResult() && event.InternalEvent != nil { lastMatcherEvent = event } else { if writer.WriteResult(event, e.options.Output, e.options.Progress, e.options.IssuesClient) { - results.CompareAndSwap(false, true) + matched.Store(true) } else { lastMatcherEvent = event } @@ -161,7 +156,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { // so in compile step earlier we compile it to validate javascript syntax and other things // and while executing we create new instance of flow executor everytime if e.options.Flow != "" { - flowexec, err := flow.NewFlowExecutor(e.requests, ctx, e.options, results, e.program) + flowexec, err := flow.NewFlowExecutor(e.requests, ctx, e.options, executed, e.program) if err != nil { ctx.LogError(err) return false, fmt.Errorf("could not create flow executor: %s", err) @@ -178,7 +173,7 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) { if lastMatcherEvent != nil { writeFailureCallback(lastMatcherEvent, e.options.Options.MatcherStatus) } - return results.Load(), errx + return executed.Load() || matched.Load(), errx } // ExecuteWithResults executes the protocol requests and returns results instead of writing them. diff --git a/pkg/tmplexec/flow/flow_executor.go b/pkg/tmplexec/flow/flow_executor.go index 33821ac005..31aa9f0f4d 100644 --- a/pkg/tmplexec/flow/flow_executor.go +++ b/pkg/tmplexec/flow/flow_executor.go @@ -8,7 +8,6 @@ import ( "sync/atomic" "github.com/dop251/goja" - "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" "github.com/projectdiscovery/nuclei/v3/pkg/scan" @@ -175,13 +174,6 @@ func (f *FlowExecutor) Compile() error { // ExecuteWithResults executes the flow and returns results func (f *FlowExecutor) ExecuteWithResults(ctx *scan.ScanContext) error { - defer func() { - if e := recover(); e != nil { - f.ctx.LogError(fmt.Errorf("panic occurred while executing target %v with flow: %v", ctx.Input.MetaInput.Input, e)) - gologger.Error().Label(f.options.TemplateID).Msgf("panic occurred while executing target %v with flow: %v", ctx.Input.MetaInput.Input, e) - } - }() - f.ctx.Input = ctx.Input // -----Load all types of variables----- // add all input args to template context diff --git a/pkg/tmplexec/flow/flow_internal.go b/pkg/tmplexec/flow/flow_internal.go index b7f589d375..6e82bd960e 100644 --- a/pkg/tmplexec/flow/flow_internal.go +++ b/pkg/tmplexec/flow/flow_internal.go @@ -96,10 +96,12 @@ func (f *FlowExecutor) protocolResultCallback(req protocols.Request, matcherStat if len(v) == 1 { // add it to flatten keys list so it will be flattened to a string later f.flattenKeys = append(f.flattenKeys, k) + // flatten and convert it to string + f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(k, v[0]) + } else { + // keep it as slice + f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(k, v) } - // always preserve extracted value type - f.options.GetTemplateCtx(f.ctx.Input.MetaInput).Set(k, v) - } } } else if !result.HasOperatorResult() && !hasOperators(req.GetCompiledOperators()) { diff --git a/pkg/tmplexec/flow/vm.go b/pkg/tmplexec/flow/vm.go index 41d6c39255..2e22bd8e12 100644 --- a/pkg/tmplexec/flow/vm.go +++ b/pkg/tmplexec/flow/vm.go @@ -7,6 +7,7 @@ import ( "github.com/dop251/goja" "github.com/logrusorgru/aurora" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/js/gojs" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/utils/vardump" "github.com/projectdiscovery/nuclei/v3/pkg/tmplexec/flow/builtin" @@ -49,46 +50,68 @@ var gojapool = &sync.Pool{ } func registerBuiltins(runtime *goja.Runtime) { - _ = runtime.Set("log", func(call goja.FunctionCall) goja.Value { - // TODO: verify string interpolation and handle multiple args - arg := call.Argument(0).Export() - switch value := arg.(type) { - case string: - gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) - case map[string]interface{}: - gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value)) - default: - gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) - } - return call.Argument(0) // return the same value - }) - _ = runtime.Set("iterate", func(call goja.FunctionCall) goja.Value { - allVars := []any{} - for _, v := range call.Arguments { - if v.Export() == nil { - continue + _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "log", + Description: "Logs a given object/message to stdout (only for debugging purposes)", + Signatures: []string{ + "log(obj any) any", + }, + FuncDecl: func(call goja.FunctionCall) goja.Value { + arg := call.Argument(0).Export() + switch value := arg.(type) { + case string: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) + case map[string]interface{}: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), vardump.DumpVariables(value)) + default: + gologger.DefaultLogger.Print().Msgf("[%v] %v", aurora.BrightCyan("JS"), value) } - if v.ExportType().Kind() == reflect.Slice { - // convert []datatype to []interface{} - // since it cannot be type asserted to []interface{} directly - rfValue := reflect.ValueOf(v.Export()) - for i := 0; i < rfValue.Len(); i++ { - allVars = append(allVars, rfValue.Index(i).Interface()) + return call.Argument(0) // return the same value + }, + }) + + _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "iterate", + Description: "Normalizes and Iterates over all arguments (can be a string,array,null etc) and returns an array of objects\nNote: If the object type is unknown(i.e could be a string or array) iterate should be used and it will always return an array of strings", + Signatures: []string{ + "iterate(...any) []any", + }, + FuncDecl: func(call goja.FunctionCall) goja.Value { + allVars := []any{} + for _, v := range call.Arguments { + if v.Export() == nil { + continue + } + if v.ExportType().Kind() == reflect.Slice { + // convert []datatype to []interface{} + // since it cannot be type asserted to []interface{} directly + rfValue := reflect.ValueOf(v.Export()) + for i := 0; i < rfValue.Len(); i++ { + allVars = append(allVars, rfValue.Index(i).Interface()) + } + } else { + allVars = append(allVars, v.Export()) } - } else { - allVars = append(allVars, v.Export()) } - } - return runtime.ToValue(allVars) + return runtime.ToValue(allVars) + }, }) - _ = runtime.Set("Dedupe", func(call goja.ConstructorCall) *goja.Object { - d := builtin.NewDedupe(runtime) - obj := call.This - // register these methods - _ = obj.Set("Add", d.Add) - _ = obj.Set("Values", d.Values) - return nil + _ = gojs.RegisterFuncWithSignature(runtime, gojs.FuncOpts{ + Name: "Dedupe", + Description: "De-duplicates given values and returns a new array of unique values", + Signatures: []string{ + "new Dedupe()", + }, + FuncDecl: func(call goja.ConstructorCall) *goja.Object { + d := builtin.NewDedupe(runtime) + obj := call.This + // register these methods + _ = obj.Set("Add", d.Add) + _ = obj.Set("Values", d.Values) + return nil + }, }) + } diff --git a/pkg/types/types.go b/pkg/types/types.go index 45a8788a8e..ce78380901 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -372,9 +372,6 @@ type Options struct { ScanID string // JsConcurrency is the number of concurrent js routines to run JsConcurrency int - // Fuzz enabled execution of fuzzing templates - // Note: when Fuzz is enabled other templates will not be executed - FuzzTemplates bool // SecretsFile is file containing secrets for nuclei SecretsFile goflags.StringSlice // PreFetchSecrets pre-fetches the secrets from the auth provider @@ -385,6 +382,8 @@ type Options struct { SkipFormatValidation bool // PayloadConcurrency is the number of concurrent payloads to run per template PayloadConcurrency int + // Dast only runs DAST templates + DAST bool } // ShouldLoadResume resume file