diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2a3ac831..d0b098d1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,18 +6,6 @@ version: 2 updates: - # Maintain dependencies for GitHub Actions - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - target-branch: "dev" - commit-message: - prefix: "chore" - include: "scope" - labels: - - "Type: Maintenance" - # Maintain dependencies for go modules - package-ecosystem: "gomod" directory: "/" @@ -29,15 +17,29 @@ updates: include: "scope" labels: - "Type: Maintenance" + allow: + - dependency-name: "github.com/projectdiscovery/*" - # Maintain dependencies for docker - - package-ecosystem: "docker" - directory: "/" - schedule: - interval: "weekly" - target-branch: "dev" - commit-message: - prefix: "chore" - include: "scope" - labels: - - "Type: Maintenance" \ No newline at end of file +# # Maintain dependencies for GitHub Actions +# - package-ecosystem: "github-actions" +# directory: "/" +# schedule: +# interval: "weekly" +# target-branch: "dev" +# commit-message: +# prefix: "chore" +# include: "scope" +# labels: +# - "Type: Maintenance" +# +# # Maintain dependencies for docker +# - package-ecosystem: "docker" +# directory: "/" +# schedule: +# interval: "weekly" +# target-branch: "dev" +# commit-message: +# prefix: "chore" +# include: "scope" +# labels: +# - "Type: Maintenance" \ No newline at end of file diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 00000000..4b578d58 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,18 @@ +changelog: + exclude: + authors: + - app/dependabot + - dependabot + categories: + - title: ๐ŸŽ‰ New Features + labels: + - "Type: Enhancement" + - title: ๐Ÿž Bug Fixes + labels: + - "Type: Bug" + - title: ๐Ÿ”จ Maintenance + labels: + - "Type: Maintenance" + - title: Other Changes + labels: + - "*" \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 26981367..3c74a0a1 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -14,8 +14,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest-16-cores, windows-latest-8-cores, macOS-12] - go-version: [1.19.x, 1.20.x] + os: [ubuntu-latest, windows-latest, macOS-latest] + go-version: [1.20.x] steps: - name: Set up Go uses: actions/setup-go@v4 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3d5dc36f..90c913a9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -12,7 +12,7 @@ on: jobs: analyze: name: Analyze - runs-on: ubuntu-latest-16-cores + runs-on: ubuntu-latest permissions: actions: read contents: read diff --git a/.github/workflows/dep-auto-merge.yml b/.github/workflows/dep-auto-merge.yml new file mode 100644 index 00000000..84b26e1f --- /dev/null +++ b/.github/workflows/dep-auto-merge.yml @@ -0,0 +1,26 @@ +name: ๐Ÿค– dep auto merge + +on: + pull_request: + branches: + - dev + workflow_dispatch: + +permissions: + pull-requests: write + issues: write + repository-projects: write + +jobs: + automerge: + runs-on: ubuntu-latest + if: github.actor == 'dependabot[bot]' + steps: + - uses: actions/checkout@v3 + with: + token: ${{ secrets.DEPENDABOT_PAT }} + + - uses: ahmadnassri/action-dependabot-auto-merge@v2 + with: + github-token: ${{ secrets.DEPENDABOT_PAT }} + target: all \ No newline at end of file diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml new file mode 100644 index 00000000..85a79940 --- /dev/null +++ b/.github/workflows/functional-test.yml @@ -0,0 +1,30 @@ +name: ๐Ÿงช Functional Test + +on: + pull_request: + paths: + - '**.go' + - '**.mod' + workflow_dispatch: + +jobs: + functional: + name: Functional Test + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + steps: + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.20.x + + - name: Check out code + uses: actions/checkout@v3 + + - name: Functional Tests + run: | + chmod +x run.sh + bash run.sh ${{ matrix.os }} + working-directory: cmd/functional-test diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 6718a4b1..b2204cf1 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -11,7 +11,7 @@ on: jobs: lint: name: Lint Test - runs-on: ubuntu-latest-16-cores + runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v3 @@ -19,11 +19,11 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.20.x cache: true - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.4.0 + uses: golangci/golangci-lint-action@v3.6.0 with: version: latest args: --timeout 5m diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 12382a7d..de59a3d6 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -7,8 +7,8 @@ on: workflow_dispatch: jobs: - release: - runs-on: ubuntu-latest-16-cores + build-mac: + runs-on: macos-latest steps: - name: "Check out code" uses: actions/checkout@v3 @@ -18,13 +18,63 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.20.x cache: true - name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v4 with: - args: "release --rm-dist" + args: "release -f .goreleaser/mac.yml --clean" + version: latest + workdir: . + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + build-windows: + runs-on: windows-latest-8-cores + steps: + - name: "Check out code" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: "Set up Go" + uses: actions/setup-go@v4 + with: + go-version: 1.20.x + cache: true + + - name: "Create release on GitHub" + uses: goreleaser/goreleaser-action@v4 + with: + args: "release -f .goreleaser/windows.yml --clean" + version: latest + workdir: . + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + build-linux: + runs-on: ubuntu-latest-16-cores + steps: + - name: "Check out code" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: "Set up Go" + uses: actions/setup-go@v4 + with: + go-version: 1.20.x + cache: true + + # todo: musl compatible? + - name: Install Dependences + run: sudo apt install gcc-aarch64-linux-gnu + + - name: "Create release on GitHub" + uses: goreleaser/goreleaser-action@v4 + with: + args: "release -f .goreleaser/linux.yml --clean" version: latest workdir: . env: diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml new file mode 100644 index 00000000..027440d0 --- /dev/null +++ b/.github/workflows/release-test.yml @@ -0,0 +1,80 @@ +name: ๐Ÿ”จ Release Test + +on: + pull_request: + paths: + - '**.yml' + - '**.go' + - '**.mod' + workflow_dispatch: + +jobs: + release-test-mac: + runs-on: macos-latest + steps: + - name: "Check out code" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.20.x + + - name: release test + uses: goreleaser/goreleaser-action@v4 + with: + args: "release -f .goreleaser/mac.yml --clean --snapshot" + version: latest + workdir: . + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + release-test-linux: + runs-on: ubuntu-latest-16-cores + steps: + - name: "Check out code" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.20.x + + # todo: musl compatible? + - name: Install Dependences + run: sudo apt install gcc-aarch64-linux-gnu + + - name: release test + uses: goreleaser/goreleaser-action@v4 + with: + args: "release -f .goreleaser/linux.yml --clean --snapshot" + version: latest + workdir: . + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + release-test-windows: + runs-on: windows-latest-8-cores + steps: + - name: "Check out code" + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.20.x + + - name: release test + uses: goreleaser/goreleaser-action@v4 + with: + args: "release -f .goreleaser/windows.yml --clean --snapshot" + version: latest + workdir: . + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file diff --git a/.github/workflows/security-crawl-maze-score.yaml b/.github/workflows/security-crawl-maze-score.yaml index fabbd608..bdf87ba8 100644 --- a/.github/workflows/security-crawl-maze-score.yaml +++ b/.github/workflows/security-crawl-maze-score.yaml @@ -11,7 +11,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.20.x - name: Check out code uses: actions/checkout@v3 @@ -21,11 +21,11 @@ jobs: working-directory: cmd/katana/ - name: Run Katana Standard - run: ./katana -u https://security-crawl-maze.app/ -kf all -jc -d 10 -o output_standard.txt -cos node_modules + run: ./katana -u https://security-crawl-maze.app/ -kf all -jc -jsluice -d 10 -o output_standard.txt -cos node_modules working-directory: cmd/katana - name: Run Katana Headless - run: ./katana -u https://security-crawl-maze.app/ -kf all -jc -d 10 -headless -o output_headless.txt -cos node_modules + run: ./katana -u https://security-crawl-maze.app/ -kf all -jc -jsluice -d 10 -headless -o output_headless.txt -cos node_modules working-directory: cmd/katana - name: Run Score diff --git a/.gitignore b/.gitignore index cfb9d110..1aae5fa9 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ cmd/katana/katana katana *.exe katana_*/ -katana_*/ \ No newline at end of file +katana_*/ +dist/ \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml deleted file mode 100644 index 3702bc48..00000000 --- a/.goreleaser.yml +++ /dev/null @@ -1,46 +0,0 @@ -before: - hooks: - - go mod tidy - -builds: -- env: - - CGO_ENABLED=0 - goos: - - windows - - linux - - darwin - goarch: - - amd64 - - 386 - - arm - - arm64 - - ignore: - - goos: darwin - goarch: '386' - - goos: windows - goarch: 'arm' - - goos: windows - goarch: 'arm64' - - binary: '{{ .ProjectName }}' - main: cmd/katana/main.go - -archives: -- format: zip - replacements: - darwin: macOS - -checksum: - algorithm: sha256 - -announce: - slack: - enabled: true - channel: '#release' - username: GoReleaser - message_template: '{{ .ProjectName }} {{ .Tag }} is out! Check it out at {{ .ReleaseURL }}' - - discord: - enabled: true - message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' \ No newline at end of file diff --git a/.goreleaser/linux.yml b/.goreleaser/linux.yml new file mode 100644 index 00000000..1a68315e --- /dev/null +++ b/.goreleaser/linux.yml @@ -0,0 +1,48 @@ +env: + - GO111MODULE=on +before: + hooks: + - go mod tidy +project_name: katana +builds: + - id: katana-linux-generic + ldflags: + - -s -w + binary: katana + env: + - CGO_ENABLED=1 + main: ./cmd/katana/main.go + goos: + - linux + goarch: + - amd64 + + - id: katana-linux-arm + ldflags: + - -s -w + binary: katana + env: + - CGO_ENABLED=1 + - CC=aarch64-linux-gnu-gcc + main: ./cmd/katana/main.go + goos: + - linux + goarch: + - arm64 + +archives: +- format: zip + +checksum: + name_template: "{{ .ProjectName }}-linux-checksums.txt" + +announce: + slack: + enabled: true + channel: '#release' + username: GoReleaser + message_template: 'New Release: {{ .ProjectName }} {{.Tag}} is published! Check it out at {{ .ReleaseURL }}' + + discord: + enabled: true + message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' \ No newline at end of file diff --git a/.goreleaser/mac.yml b/.goreleaser/mac.yml new file mode 100644 index 00000000..53b240a5 --- /dev/null +++ b/.goreleaser/mac.yml @@ -0,0 +1,28 @@ +env: + - GO111MODULE=on +before: + hooks: + - go mod tidy +project_name: katana +builds: + - id: katana-darwin + ldflags: + - -s -w + binary: katana + env: + - CGO_ENABLED=1 + main: ./cmd/katana/main.go + goos: + - darwin + goarch: + - amd64 + - arm64 + - 386 + - arm + +archives: +- format: zip + name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' + +checksum: + name_template: "{{ .ProjectName }}-mac-checksums.txt" diff --git a/.goreleaser/windows.yml b/.goreleaser/windows.yml new file mode 100644 index 00000000..f2b1448d --- /dev/null +++ b/.goreleaser/windows.yml @@ -0,0 +1,24 @@ +env: + - GO111MODULE=on +before: + hooks: + - go mod tidy +project_name: katana +builds: + - id: katana-windows + ldflags: + - -s -w + binary: katana + env: + - CGO_ENABLED=1 + main: ./cmd/katana/main.go + goos: + - windows + goarch: + - amd64 + +archives: +- format: zip + +checksum: + name_template: "{{ .ProjectName }}-windows-checksums.txt" diff --git a/Dockerfile b/Dockerfile index e516766f..ff55876d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM golang:1.20.3-alpine AS builder -RUN apk add --no-cache git +FROM golang:1.20.6-alpine AS builder +RUN apk add --no-cache git gcc musl-dev WORKDIR /app COPY . /app RUN go mod download RUN go build ./cmd/katana -FROM alpine:3.17.3 +FROM alpine:3.18.2 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates chromium COPY --from=builder /app/katana /usr/local/bin/ diff --git a/README.md b/README.md index 17e327c1..679c7ef0 100644 --- a/README.md +++ b/README.md @@ -120,19 +120,22 @@ CONFIGURATION: -r, -resolvers string[] list of custom resolver (file or comma separated) -d, -depth int maximum depth to crawl (default 3) -jc, -js-crawl enable endpoint parsing / crawling in javascript file - -ct, -crawl-duration int maximum duration to crawl the target for + -jsl, -jsluice enable jsluice parsing in javascript file (memory intensive) + -ct, -crawl-duration value maximum duration to crawl the target for (s, m, h, d) (default s) -kf, -known-files string enable crawling of known files (all,robotstxt,sitemapxml) -mrs, -max-response-size int maximum response size to read (default 9223372036854775807) -timeout int time to wait for request in seconds (default 10) -aff, -automatic-form-fill enable automatic form filling (experimental) + -fx, -form-extraction extract form, input, textarea & select elements in jsonl output -retry int number of times to retry the request (default 1) -proxy string http/socks5 proxy to use - -H, -headers string[] custom header/cookie to include in request + -H, -headers string[] custom header/cookie to include in all http request in header:value format (file) -config string path to the katana configuration file -fc, -form-config string path to custom form configuration file -flc, -field-config string path to custom field configuration file -s, -strategy string Visit strategy (depth-first, breadth-first) (default "depth-first") -iqp, -ignore-query-params Ignore crawling same path with different query-param values + -tlsi, -tls-impersonate enable experimental client hello (ja3) tls randomization DEBUG: -health-check, -hc run diagnostic check up @@ -147,11 +150,13 @@ HEADLESS: -cdd, -chrome-data-dir string path to store chrome browser data -scp, -system-chrome-path string use specified chrome browser for headless crawling -noi, -no-incognito start headless chrome without incognito mode + -cwu, -chrome-ws-url string use chrome browser instance launched elsewhere with the debugger listening at this URL + -xhr, -xhr-extraction extract xhr request url,method in jsonl output SCOPE: -cs, -crawl-scope string[] in scope url regex to be followed by crawler -cos, -crawl-out-scope string[] out of scope url regex to be excluded by crawler - -fs, -field-scope string pre-defined scope field (dn,rdn,fqdn) (default "rdn") + -fs, -field-scope string pre-defined scope field (dn,rdn,fqdn) or custom regex (e.g., '(company-staging.io|company.com)') (default "rdn") -ns, -no-scope disables host based default scope -do, -display-out-scope display external endpoint from scoped crawling @@ -162,6 +167,8 @@ FILTER: -sf, -store-field string field to store in per-host output (url,path,fqdn,rdn,rurl,qurl,qpath,file,ufile,key,value,kv,dir,udir) -em, -extension-match string[] match output for given extension (eg, -em php,html,js) -ef, -extension-filter string[] filter output for given extension (eg, -ef png,css) + -mdc, -match-condition string match response with dsl based condition + -fdc, -filter-condition string filter response with dsl based condition RATE-LIMIT: -c, -concurrency int number of concurrent fetchers to use (default 10) @@ -178,7 +185,9 @@ OUTPUT: -o, -output string file to write output to -sr, -store-response store http requests/responses -srd, -store-response-dir string store http requests/responses to custom directory - -j, -json write output in JSONL(ines) format + -or, -omit-raw omit raw requests/responses from jsonl output + -ob, -omit-body omit response body from jsonl output + -j, -jsonl write output in jsonl format -nc, -no-color disable output content coloring (ANSI escape codes) -silent display output only -v, -verbose display verbose output @@ -307,6 +316,8 @@ HEADLESS: -cdd, -chrome-data-dir string path to store chrome browser data -scp, -system-chrome-path string use specified chrome browser for headless crawling -noi, -no-incognito start headless chrome without incognito mode + -cwu, -chrome-ws-url string use chrome browser instance launched elsewhere with the debugger listening at this URL + -xhr, -xhr-extraction extract xhr requests ``` *`-no-sandbox`* @@ -486,6 +497,38 @@ Automatic form filling is experimental feature. katana -u https://tesla.com -aff ``` +## Authenticated Crawling + +Authenticated crawling involves including custom headers or cookies in HTTP requests to access protected resources. These headers provide authentication or authorization information, allowing you to crawl authenticated content / endpoint. You can specify headers directly in the command line or provide them as a file with katana to perfrom authenticated crawling. + +> **Note**: User needs to be manually perform the authentication and export the session cookie / header to file to use with katana. + +*`-headers`* +---- + +Option to add a custom header or cookie to the request. +> Syntax of [headers](https://datatracker.ietf.org/doc/html/rfc7230#section-3.2) in the HTTP specification + +Here is an example of adding a cookie to the request: +``` +katana -u https://tesla.com -H 'Cookie: usrsess=AmljNrESo' +``` + +It is also possible to supply headers or cookies as a file. For example: + +``` +$ cat cookie.txt + +Cookie: PHPSESSIONID=XXXXXXXXX +X-API-KEY: XXXXX +TOKEN=XX +``` + +``` +katana -u https://tesla.com -H cookie.txt +``` + + There are more options to configure when needed, here is all the config related CLI options - ```console @@ -501,6 +544,7 @@ CONFIGURATION: -mrs, -max-response-size int maximum response size to read (default 9223372036854775807) -timeout int time to wait for request in seconds (default 10) -aff, -automatic-form-fill enable automatic form filling (experimental) + -fx, -form-extraction enable extraction of form, input, textarea & select elements -retry int number of times to retry the request (default 1) -proxy string http/socks5 proxy to use -H, -headers string[] custom header/cookie to include in request @@ -510,6 +554,41 @@ CONFIGURATION: -s, -strategy string Visit strategy (depth-first, breadth-first) (default "depth-first") ``` +### Connecting to Active Browser Session + +Katana can also connect to active browser session where user is already logged in and authenticated. and use it for crawling. The only requirement for this is to start browser with remote debugging enabled. + +Here is an example of starting chrome browser with remote debugging enabled and using it with katana - + +**step 1) First Locate path of chrome executable** + +| Operating System | Chromium Executable Location | Google Chrome Executable Location | +|------------------|------------------------------|-----------------------------------| +| Windows (64-bit) | `C:\Program Files (x86)\Google\Chromium\Application\chrome.exe` | `C:\Program Files (x86)\Google\Chrome\Application\chrome.exe` | +| Windows (32-bit) | `C:\Program Files\Google\Chromium\Application\chrome.exe` | `C:\Program Files\Google\Chrome\Application\chrome.exe` | +| macOS | `/Applications/Chromium.app/Contents/MacOS/Chromium` | `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome` | +| Linux | `/usr/bin/chromium` | `/usr/bin/google-chrome` | + +**step 2) Start chrome with remote debugging enabled and it will return websocker url. For example, on MacOS, you can start chrome with remote debugging enabled using following command** - + +```console +$ /Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --remote-debugging-port=9222 + + +DevTools listening on ws://127.0.0.1:9222/devtools/browser/c5316c9c-19d6-42dc-847a-41d1aeebf7d6 +``` + +> Now login to the website you want to crawl and keep the browser open. + +**step 3) Now use the websocket url with katana to connect to the active browser session and crawl the website** + +```console +katana -headless -u https://tesla.com -cwu ws://127.0.0.1:9222/devtools/browser/c5316c9c-19d6-42dc-847a-41d1aeebf7d6 -no-incognito +``` + +> **Note**: you can use `-cdd` option to specify custom chrome data directory to store browser data and cookies but that does not save session data if cookie is set to `Session` only or expires after certain time. + + ## Filters *`-field`* @@ -622,6 +701,7 @@ The `-store-field` option can be useful for collecting information to build a ta - Finding commonly used files - Identifying related or unknown subdomains +### Katana Filters *`-extension-match`* --- @@ -656,6 +736,28 @@ The `-filter-regex` or `-fr` flag allows you to filter output URLs using regular katana -u https://tesla.com -fr 'https://www\.tesla\.com/*' -silent ``` +### Advance Filtering + +Katana supports DSL-based expressions for advanced matching and filtering capabilities: + +- To match endpoints with a 200 status code: +```shell +katana -u https://www.hackerone.com -mdc 'status_code == 200' +``` +- To match endpoints that contain "default" and have a status code other than 403: +```shell +katana -u https://www.hackerone.com -mdc 'contains(endpoint, "default") && status_code != 403' +``` +- To match endpoints with PHP technologies: +```shell +katana -u https://www.hackerone.com -mdc 'contains(to_lower(technologies), "php")' +``` +- To filter out endpoints running on Cloudflare: +```shell +katana -u https://www.hackerone.com -fdc 'contains(to_lower(technologies), "cloudflare")' +``` +DSL functions can be applied to any keys in the jsonl output. For more information on available DSL functions, please visit the [dsl project](https://github.com/projectdiscovery/dsl). + Here are additional filter options - ```console @@ -669,8 +771,11 @@ FILTER: -sf, -store-field string field to store in per-host output (url,path,fqdn,rdn,rurl,qurl,qpath,file,ufile,key,value,kv,dir,udir) -em, -extension-match string[] match output for given extension (eg, -em php,html,js) -ef, -extension-filter string[] filter output for given extension (eg, -ef png,css) + -mdc, -match-condition string match response with dsl based condition + -fdc, -filter-condition string filter response with dsl based condition ``` + ## Rate Limit It's easy to get blocked / banned while crawling if not following target websites limits, katana comes with multiple option to tune the crawl to go as fast / slow we want. @@ -744,11 +849,11 @@ By default, katana outputs the crawled endpoints in plain text format. The resul katana -u https://example.com -no-scope -output example_endpoints.txt ``` -*`-json`* +*`-jsonl`* --- ```console -katana -u https://example.com -json | jq . +katana -u https://example.com -jsonl | jq . ``` ```json @@ -827,7 +932,7 @@ OUTPUT: `katana` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Using the options you can create `crawlerOptions` and so standard or hybrid `crawler`. `crawler.Crawl` method should be called to crawl the input. -``` +```go package main import ( diff --git a/cmd/functional-test/main.go b/cmd/functional-test/main.go new file mode 100644 index 00000000..7e76e1e9 --- /dev/null +++ b/cmd/functional-test/main.go @@ -0,0 +1,58 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "strings" + + "github.com/logrusorgru/aurora" + "github.com/pkg/errors" + "github.com/projectdiscovery/katana/internal/testutils" +) + +var ( + debug = os.Getenv("DEBUG") == "true" + success = aurora.Green("[โœ“]").String() + failed = aurora.Red("[โœ˜]").String() + errored = false + devKatanaBinary = flag.String("dev", "", "Dev Branch Katana Binary") +) + +func main() { + flag.Parse() + if err := runFunctionalTests(); err != nil { + log.Fatalf("Could not run functional tests: %s\n", err) + } + if errored { + os.Exit(1) + } +} + +func runFunctionalTests() error { + for _, testcase := range testutils.TestCases { + if err := runIndividualTestCase(testcase); err != nil { + errored = true + fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, testcase.Name, err) + } else { + fmt.Printf("%s Test \"%s\" passed!\n", success, testcase.Name) + } + } + return nil +} + +func runIndividualTestCase(testcase testutils.TestCase) error { + argsParts := strings.Fields(testcase.Args) + devOutput, err := testutils.RunKatanaBinaryAndGetResults(testcase.Target, *devKatanaBinary, debug, argsParts) + if err != nil { + return errors.Wrap(err, "could not run Katana dev test") + } + if testcase.CompareFunc != nil { + return testcase.CompareFunc(testcase.Target, devOutput) + } + if !testutils.CompareOutput(devOutput, testcase.Expected) { + return errors.Errorf("expected output %s, got %s", testcase.Expected, devOutput) + } + return nil +} diff --git a/cmd/functional-test/run.sh b/cmd/functional-test/run.sh new file mode 100644 index 00000000..af44e4f5 --- /dev/null +++ b/cmd/functional-test/run.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# reading os type from arguments +CURRENT_OS=$1 + +if [ "${CURRENT_OS}" == "windows-latest" ];then + extension=.exe +fi + +echo "::group::Building functional-test binary" +go build -o functional-test$extension +echo "::endgroup::" + +echo "::group::Building katana binary from current branch" +go build -o katana_dev$extension ../katana +echo "::endgroup::" + + +echo 'Starting katana functional test' +./functional-test$extension -dev ./katana_dev$extension diff --git a/cmd/integration-test/filters.go b/cmd/integration-test/filters.go new file mode 100644 index 00000000..78961a3d --- /dev/null +++ b/cmd/integration-test/filters.go @@ -0,0 +1,77 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "strings" +) + +var filtersTestcases = map[string]TestCase{ + "match condition": &matchConditionIntegrationTest{}, + "filter condition": &filterConditionIntegrationTest{}, +} + +type matchConditionIntegrationTest struct{} + +// Execute executes a test case and returns an error if occurred +// Execute the docs at ../README.md if the code stops working for integration. +func (h *matchConditionIntegrationTest) Execute() error { + results, _ := RunKatanaAndGetResults(false, + "-u", "scanme.sh", + "-match-condition", "status_code == 200 && contains(body, 'ok')", + ) + + if len(results) != 1 { + return fmt.Errorf("match condition failed") + } + return nil +} + +type filterConditionIntegrationTest struct{} + +// Execute executes a test case and returns an error if occurred +// Execute the docs at ../README.md if the code stops working for integration. +func (h *filterConditionIntegrationTest) Execute() error { + results, _ := RunKatanaAndGetResults(false, + "-u", "scanme.sh", + "-filter-condition", "status_code == 200 && contains(body, 'ok')", + ) + + if len(results) != 0 { + return fmt.Errorf("filter condition failed") + } + return nil +} + +// ExtraArgs +var ExtraDebugArgs = []string{} + +func RunKatanaAndGetResults(debug bool, extra ...string) ([]string, error) { + cmd := exec.Command("./katana") + extra = append(extra, ExtraDebugArgs...) + cmd.Args = append(cmd.Args, extra...) + cmd.Args = append(cmd.Args, "-duc") // disable auto updates + if debug { + cmd.Args = append(cmd.Args, "-debug") + cmd.Stderr = os.Stderr + fmt.Println(cmd.String()) + } else { + cmd.Args = append(cmd.Args, "-silent") + } + data, err := cmd.Output() + if debug { + fmt.Println(string(data)) + } + if len(data) < 1 && err != nil { + return nil, fmt.Errorf("%v: %v", err.Error(), string(data)) + } + var parts []string + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts, nil +} diff --git a/cmd/integration-test/integration-test.go b/cmd/integration-test/integration-test.go index 3c724505..28051783 100644 --- a/cmd/integration-test/integration-test.go +++ b/cmd/integration-test/integration-test.go @@ -22,7 +22,8 @@ var ( failed = aurora.Red("[โœ˜]").String() tests = map[string]map[string]TestCase{ - "code": libraryTestcases, + "code": libraryTestcases, + "filters": filtersTestcases, } ) diff --git a/cmd/katana/main.go b/cmd/katana/main.go index 2f34261f..0490d553 100644 --- a/cmd/katana/main.go +++ b/cmd/katana/main.go @@ -5,8 +5,10 @@ import ( "math" "os" "os/signal" + "path/filepath" "strings" "syscall" + "time" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/gologger" @@ -14,6 +16,8 @@ import ( "github.com/projectdiscovery/katana/pkg/output" "github.com/projectdiscovery/katana/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" + fileutil "github.com/projectdiscovery/utils/file" + "github.com/rs/xid" ) var ( @@ -42,12 +46,20 @@ func main() { defer katanaRunner.Close() // close handler + resumeFilename := defaultResumeFilename() go func() { c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) for range c { gologger.DefaultLogger.Info().Msg("- Ctrl+C pressed in Terminal") katanaRunner.Close() + + gologger.Info().Msgf("Creating resume file: %s\n", resumeFilename) + err := katanaRunner.SaveState(resumeFilename) + if err != nil { + gologger.Error().Msgf("Couldn't create resume file: %s\n", err) + } + os.Exit(0) } }() @@ -55,6 +67,12 @@ func main() { if err := katanaRunner.ExecuteCrawling(); err != nil { gologger.Fatal().Msgf("could not execute crawling: %s", err) } + + // on successful execution remove the resume file in case it exists + if fileutil.FileExists(resumeFilename) { + os.Remove(resumeFilename) + } + } func readFlags() (*goflags.FlagSet, error) { @@ -64,25 +82,30 @@ pipelines offering both headless and non-headless crawling.`) flagSet.CreateGroup("input", "Input", flagSet.StringSliceVarP(&options.URLs, "list", "u", nil, "target url / list to crawl", goflags.FileCommaSeparatedStringSliceOptions), + flagSet.StringVar(&options.Resume, "resume", "", "resume scan using resume.cfg"), ) flagSet.CreateGroup("config", "Configuration", flagSet.StringSliceVarP(&options.Resolvers, "resolvers", "r", nil, "list of custom resolver (file or comma separated)", goflags.FileCommaSeparatedStringSliceOptions), flagSet.IntVarP(&options.MaxDepth, "depth", "d", 3, "maximum depth to crawl"), flagSet.BoolVarP(&options.ScrapeJSResponses, "js-crawl", "jc", false, "enable endpoint parsing / crawling in javascript file"), - flagSet.IntVarP(&options.CrawlDuration, "crawl-duration", "ct", 0, "maximum duration to crawl the target for"), + flagSet.BoolVarP(&options.ScrapeJSLuiceResponses, "jsluice", "jsl", false, "enable jsluice parsing in javascript file (memory intensive)"), + flagSet.DurationVarP(&options.CrawlDuration, "crawl-duration", "ct", 0, "maximum duration to crawl the target for (s, m, h, d) (default s)"), flagSet.StringVarP(&options.KnownFiles, "known-files", "kf", "", "enable crawling of known files (all,robotstxt,sitemapxml)"), flagSet.IntVarP(&options.BodyReadSize, "max-response-size", "mrs", math.MaxInt, "maximum response size to read"), flagSet.IntVar(&options.Timeout, "timeout", 10, "time to wait for request in seconds"), flagSet.BoolVarP(&options.AutomaticFormFill, "automatic-form-fill", "aff", false, "enable automatic form filling (experimental)"), + flagSet.BoolVarP(&options.FormExtraction, "form-extraction", "fx", false, "extract form, input, textarea & select elements in jsonl output"), flagSet.IntVar(&options.Retries, "retry", 1, "number of times to retry the request"), flagSet.StringVar(&options.Proxy, "proxy", "", "http/socks5 proxy to use"), - flagSet.StringSliceVarP(&options.CustomHeaders, "headers", "H", nil, "custom header/cookie to include in request", goflags.StringSliceOptions), + flagSet.StringSliceVarP(&options.CustomHeaders, "headers", "H", nil, "custom header/cookie to include in all http request in header:value format (file)", goflags.FileStringSliceOptions), flagSet.StringVar(&cfgFile, "config", "", "path to the katana configuration file"), flagSet.StringVarP(&options.FormConfig, "form-config", "fc", "", "path to custom form configuration file"), flagSet.StringVarP(&options.FieldConfig, "field-config", "flc", "", "path to custom field configuration file"), flagSet.StringVarP(&options.Strategy, "strategy", "s", "depth-first", "Visit strategy (depth-first, breadth-first)"), flagSet.BoolVarP(&options.IgnoreQueryParams, "ignore-query-params", "iqp", false, "Ignore crawling same path with different query-param values"), + flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"), + flagSet.BoolVarP(&options.DisableRedirects, "disable-redirects", "dr", false, "disable following redirects (default false)"), ) flagSet.CreateGroup("debug", "Debug", @@ -99,12 +122,14 @@ pipelines offering both headless and non-headless crawling.`) flagSet.StringVarP(&options.ChromeDataDir, "chrome-data-dir", "cdd", "", "path to store chrome browser data"), flagSet.StringVarP(&options.SystemChromePath, "system-chrome-path", "scp", "", "use specified chrome browser for headless crawling"), flagSet.BoolVarP(&options.HeadlessNoIncognito, "no-incognito", "noi", false, "start headless chrome without incognito mode"), + flagSet.StringVarP(&options.ChromeWSUrl, "chrome-ws-url", "cwu", "", "use chrome browser instance launched elsewhere with the debugger listening at this URL"), + flagSet.BoolVarP(&options.XhrExtraction, "xhr-extraction", "xhr", false, "extract xhr request url,method in jsonl output"), ) flagSet.CreateGroup("scope", "Scope", flagSet.StringSliceVarP(&options.Scope, "crawl-scope", "cs", nil, "in scope url regex to be followed by crawler", goflags.FileCommaSeparatedStringSliceOptions), flagSet.StringSliceVarP(&options.OutOfScope, "crawl-out-scope", "cos", nil, "out of scope url regex to be excluded by crawler", goflags.FileCommaSeparatedStringSliceOptions), - flagSet.StringVarP(&options.FieldScope, "field-scope", "fs", "rdn", "pre-defined scope field (dn,rdn,fqdn)"), + flagSet.StringVarP(&options.FieldScope, "field-scope", "fs", "rdn", "pre-defined scope field (dn,rdn,fqdn) or custom regex (e.g., '(company-staging.io|company.com)')"), flagSet.BoolVarP(&options.NoScope, "no-scope", "ns", false, "disables host based default scope"), flagSet.BoolVarP(&options.DisplayOutScope, "display-out-scope", "do", false, "display external endpoint from scoped crawling"), ) @@ -117,6 +142,8 @@ pipelines offering both headless and non-headless crawling.`) flagSet.StringVarP(&options.StoreFields, "store-field", "sf", "", fmt.Sprintf("field to store in per-host output (%s)", availableFields)), flagSet.StringSliceVarP(&options.ExtensionsMatch, "extension-match", "em", nil, "match output for given extension (eg, -em php,html,js)", goflags.CommaSeparatedStringSliceOptions), flagSet.StringSliceVarP(&options.ExtensionFilter, "extension-filter", "ef", nil, "filter output for given extension (eg, -ef png,css)", goflags.CommaSeparatedStringSliceOptions), + flagSet.StringVarP(&options.OutputMatchCondition, "match-condition", "mdc", "", "match response with dsl based condition"), + flagSet.StringVarP(&options.OutputFilterCondition, "filter-condition", "fdc", "", "filter response with dsl based condition"), ) flagSet.CreateGroup("ratelimit", "Rate-Limit", @@ -136,7 +163,9 @@ pipelines offering both headless and non-headless crawling.`) flagSet.StringVarP(&options.OutputFile, "output", "o", "", "file to write output to"), flagSet.BoolVarP(&options.StoreResponse, "store-response", "sr", false, "store http requests/responses"), flagSet.StringVarP(&options.StoreResponseDir, "store-response-dir", "srd", "", "store http requests/responses to custom directory"), - flagSet.BoolVarP(&options.JSON, "json", "j", false, "write output in JSONL(ines) format"), + flagSet.BoolVarP(&options.OmitRaw, "omit-raw", "or", false, "omit raw requests/responses from jsonl output"), + flagSet.BoolVarP(&options.OmitBody, "omit-body", "ob", false, "omit response body from jsonl output"), + flagSet.BoolVarP(&options.JSON, "jsonl", "j", false, "write output in jsonl format"), flagSet.BoolVarP(&options.NoColors, "no-color", "nc", false, "disable output content coloring (ANSI escape codes)"), flagSet.BoolVar(&options.Silent, "silent", false, "display output only"), flagSet.BoolVarP(&options.Verbose, "verbose", "v", false, "display verbose output"), @@ -153,6 +182,7 @@ pipelines offering both headless and non-headless crawling.`) return nil, errorutil.NewWithErr(err).Msgf("could not read config file") } } + cleanupOldResumeFiles() return flagSet, nil } @@ -162,3 +192,26 @@ func init() { errorutil.ShowStackTrace = true } } + +func defaultResumeFilename() string { + homedir, err := os.UserHomeDir() + if err != nil { + gologger.Fatal().Msgf("could not get home directory: %s", err) + } + configDir := filepath.Join(homedir, ".config", "katana") + return filepath.Join(configDir, fmt.Sprintf("resume-%s.cfg", xid.New().String())) +} + +// cleanupOldResumeFiles cleans up resume files older than 10 days. +func cleanupOldResumeFiles() { + homedir, err := os.UserHomeDir() + if err != nil { + gologger.Fatal().Msgf("could not get home directory: %s", err) + } + root := filepath.Join(homedir, ".config", "katana") + filter := fileutil.FileFilters{ + OlderThan: 24 * time.Hour * 10, // cleanup on the 10th day + Prefix: "resume-", + } + _ = fileutil.DeleteFilesOlderThan(root, filter) +} diff --git a/go.mod b/go.mod index 1a159263..86100d8e 100644 --- a/go.mod +++ b/go.mod @@ -1,68 +1,91 @@ module github.com/projectdiscovery/katana -go 1.19 +go 1.20 require ( + github.com/BishopFox/jsluice v0.0.0-20230623145428-f10429e1016a github.com/PuerkitoBio/goquery v1.8.1 - github.com/go-rod/rod v0.112.8 + github.com/go-rod/rod v0.114.1 github.com/json-iterator/go v1.1.12 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3 - github.com/projectdiscovery/fastdialer v0.0.24 - github.com/projectdiscovery/goflags v0.1.8 - github.com/projectdiscovery/gologger v1.1.8 - github.com/projectdiscovery/hmap v0.0.11 - github.com/projectdiscovery/ratelimit v0.0.6 - github.com/projectdiscovery/retryablehttp-go v1.0.14 - github.com/projectdiscovery/utils v0.0.19 - github.com/projectdiscovery/wappalyzergo v0.0.90 + github.com/mitchellh/mapstructure v1.5.0 + github.com/pkg/errors v0.9.1 + github.com/projectdiscovery/dsl v0.0.29 + github.com/projectdiscovery/fastdialer v0.0.43 + github.com/projectdiscovery/goflags v0.1.26 + github.com/projectdiscovery/gologger v1.1.11 + github.com/projectdiscovery/hmap v0.0.23 + github.com/projectdiscovery/ratelimit v0.0.13 + github.com/projectdiscovery/retryablehttp-go v1.0.34 + github.com/projectdiscovery/utils v0.0.60 + github.com/projectdiscovery/wappalyzergo v0.0.109 github.com/remeh/sizedwaitgroup v1.0.0 - github.com/rs/xid v1.4.0 - github.com/shirou/gopsutil/v3 v3.23.3 - github.com/stretchr/testify v1.8.2 + github.com/rs/xid v1.5.0 + github.com/shirou/gopsutil/v3 v3.23.7 + github.com/stretchr/testify v1.8.4 go.uber.org/multierr v1.11.0 - golang.org/x/net v0.9.0 + golang.org/x/net v0.17.0 gopkg.in/yaml.v3 v3.0.1 ) require ( aead.dev/minisign v0.2.0 // indirect - github.com/Masterminds/semver/v3 v3.2.0 // indirect + github.com/Knetic/govaluate v3.0.0+incompatible // indirect + github.com/Masterminds/semver/v3 v3.2.1 // indirect + github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/alecthomas/chroma v0.10.0 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/charmbracelet/glamour v0.6.0 // indirect - github.com/cheggaaa/pb/v3 v3.1.2 // indirect + github.com/cheggaaa/pb/v3 v3.1.4 // indirect + github.com/cloudflare/circl v1.3.3 // indirect + github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/dlclark/regexp2 v1.8.1 // indirect - github.com/fatih/color v1.14.1 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/fatih/color v1.15.0 // indirect + github.com/gaukas/godicttls v0.0.4 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hdm/jarm-go v0.0.7 // indirect + github.com/kataras/jwt v0.1.8 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/minio/selfupdate v0.6.0 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.1 // indirect - github.com/nxadm/tail v1.4.8 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/projectdiscovery/blackrock v0.0.1 // indirect + github.com/projectdiscovery/gostruct v0.0.1 // indirect + github.com/projectdiscovery/mapcidr v1.1.14 // indirect + github.com/quic-go/quic-go v0.37.4 // indirect + github.com/refraction-networking/utls v1.5.4 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/shoenig/go-m1cpu v0.1.4 // indirect - github.com/tidwall/btree v1.4.3 // indirect - github.com/tidwall/buntdb v1.2.10 // indirect - github.com/tidwall/gjson v1.14.3 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/sashabaranov/go-openai v1.14.2 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/smacker/go-tree-sitter v0.0.0-20220628134258-ac06e95cfa11 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/tidwall/btree v1.6.0 // indirect + github.com/tidwall/buntdb v1.3.0 // indirect + github.com/tidwall/gjson v1.14.4 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect + github.com/ysmood/fetchup v0.2.3 // indirect + github.com/ysmood/got v0.34.1 // indirect github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect - golang.org/x/oauth2 v0.5.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.31.0 // indirect ) require ( @@ -80,40 +103,38 @@ require ( github.com/gorilla/css v1.0.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mholt/archiver v3.1.1+incompatible // indirect - github.com/microcosm-cc/bluemonday v1.0.23 // indirect - github.com/miekg/dns v1.1.52 // indirect + github.com/microcosm-cc/bluemonday v1.0.25 // indirect + github.com/miekg/dns v1.1.56 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/nwaples/rardecode v1.1.0 // indirect - github.com/pierrec/lz4 v2.6.0+incompatible // indirect + github.com/nwaples/rardecode v1.1.3 // indirect + github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/projectdiscovery/iputil v0.0.2 // indirect - github.com/projectdiscovery/networkpolicy v0.0.4 // indirect - github.com/projectdiscovery/retryabledns v1.0.21 // indirect - github.com/projectdiscovery/stringsutil v0.0.2 // indirect + github.com/projectdiscovery/networkpolicy v0.0.6 // indirect + github.com/projectdiscovery/retryabledns v1.0.40 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect - github.com/ulikunitz/xz v0.5.8 // indirect + github.com/ulikunitz/xz v0.5.11 // indirect github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect - github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 // indirect + github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/ysmood/goob v0.4.0 // indirect github.com/ysmood/gson v0.7.3 // indirect github.com/ysmood/leakless v0.8.0 // indirect - github.com/yusufpapurcu/wmi v1.2.2 // indirect - github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect - github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect + github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect - golang.org/x/crypto v0.7.0 // indirect - golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 // indirect - golang.org/x/mod v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/tools v0.13.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index fb8c5c19..feb1f6ce 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,27 @@ aead.dev/minisign v0.2.0 h1:kAWrq/hBRu4AARY6AlciO83xhNnW9UaC8YipS2uhLPk= aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= -github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= -github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +github.com/BishopFox/jsluice v0.0.0-20230623145428-f10429e1016a h1:1ASr2ZThan+AgONnY/9gcky6douB0+LkUrh9t/SCTBg= +github.com/BishopFox/jsluice v0.0.0-20230623145428-f10429e1016a/go.mod h1:/B9Tq6APMaiaPAA6mKbwSKAOq0kz4CjkiffnBjn5Fbo= +github.com/Knetic/govaluate v3.0.0+incompatible h1:7o6+MAPhYTCF0+fdvoz1xDedhRb4f6s9Tn1Tt7/WTEg= +github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057 h1:KFac3SiGbId8ub47e7kd2PLZeACxc1LkiiNoDOFRClE= +github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057/go.mod h1:iLB2pivrPICvLOuROKmlqURtFIEsoJZaMidQfCG1+D4= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mod h1:jD2+mU+E2SZUuAOHZvZj4xP4frlOo+N/YrXDvASFhkE= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -21,18 +31,21 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/bits-and-blooms/bitset v1.3.1 h1:y+qrlmq3XsWi+xZqSaueaE8ry8Y127iMxlMfqcK8p0g= -github.com/bits-and-blooms/bloom/v3 v3.3.1 h1:K2+A19bXT8gJR5mU7y+1yW6hsKfNCjcP2uNfLFKncjQ= +github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= +github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= -github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ= -github.com/cheggaaa/pb/v3 v3.1.2/go.mod h1:SNjnd0yKcW+kw0brSusraeDd5Bf1zBfxAzTL2ss3yQ4= +github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= +github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= +github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= @@ -41,56 +54,67 @@ github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnm github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= +github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-rod/rod v0.112.8 h1:lYFnHv/lFyjW/Ye0IhyKLeHw/zfhHbSTqawoCi2z/nI= -github.com/go-rod/rod v0.112.8/go.mod h1:ElViL9ABbcshNQw93+11FrYRH92RRhMKleuILo6+5V0= +github.com/go-rod/rod v0.114.1 h1:osBWr88guzTXAIzwJWVmGZe3/utT9+lqKjkGSBsYMxw= +github.com/go-rod/rod v0.114.1/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= +github.com/google/go-github/v50 v50.1.0/go.mod h1:Ev4Tre8QoKiolvbpOSG3FIi4Mlon3S2Nt9W5JYqKiwA= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= +github.com/hdm/jarm-go v0.0.7 h1:Eq0geenHrBSYuKrdVhrBdMMzOmA+CAMLzN2WrF3eL6A= +github.com/hdm/jarm-go v0.0.7/go.mod h1:kinGoS0+Sdn1Rr54OtanET5E5n7AlD6T6CrJAKDjJSQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/kataras/jwt v0.1.8 h1:u71baOsYD22HWeSOg32tCHbczPjdCk7V4MMeJqTtmGk= +github.com/kataras/jwt v0.1.8/go.mod h1:Q5j2IkcIHnfwy+oNY3TVWuEBJNw0ADgCcXK9CaZwV4o= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= @@ -102,8 +126,8 @@ github.com/lukasbob/srcset v0.0.0-20190730101422-86b742e617f3/go.mod h1:j16TYl5p github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= @@ -111,86 +135,106 @@ github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= -github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY= -github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4= -github.com/miekg/dns v1.1.52 h1:Bmlc/qsNNULOe6bpXcUTsuOajd0DzRHwup6D9k1An0c= -github.com/miekg/dns v1.1.52/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= +github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= +github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= +github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE= +github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY= github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= +github.com/mreiferson/go-httpclient v0.0.0-20201222173833-5e475fde3a4d/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc= github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= -github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= -github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= +github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= -github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/projectdiscovery/fastdialer v0.0.24 h1:yEyYALCmDQpPYWttZ4uo9AJseqt4mYWcyx3s9WYzqW8= -github.com/projectdiscovery/fastdialer v0.0.24/go.mod h1:X7zZy3BGdGoprR6CftHKeJyV86a3OjSAlJcNU7FL26E= -github.com/projectdiscovery/goflags v0.1.8 h1:Urhm2Isq2BdRt8h4h062lHKYXO65RHRjGTDSkUwex/g= -github.com/projectdiscovery/goflags v0.1.8/go.mod h1:Yxi9tclgwGczzDU65ntrwaIql5cXeTvW5j2WxFuF+Jk= -github.com/projectdiscovery/gologger v1.1.8 h1:CFlCzGlqAhPqWIrAXBt1OVh5jkMs1qgoR/z4xhdzLNE= -github.com/projectdiscovery/gologger v1.1.8/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= -github.com/projectdiscovery/hmap v0.0.11 h1:nA3qCFzWPcOw27T8PII5IWI3ZP0ys7TGCi2nLSnHXVA= -github.com/projectdiscovery/hmap v0.0.11/go.mod h1:5sbLn2OHexvpVupStNOhusWO9jLCyEm5jcHwWB2nOkI= -github.com/projectdiscovery/iputil v0.0.2 h1:f6IGnZF4RImJLysPSPG3D84jyTH34q3lihCFeP+eZzI= -github.com/projectdiscovery/iputil v0.0.2/go.mod h1:J3Pcz1q51pi4/JL871mQztg0KOzyWDPxnPLOYJm2pVQ= -github.com/projectdiscovery/networkpolicy v0.0.4 h1:zcGjEqZbyECZEdyCy1jVuwOS7Ww1mzgCefQU75XqdJA= -github.com/projectdiscovery/networkpolicy v0.0.4/go.mod h1:DIXwKs3sQyfCoWHKRLQiRrEorSQW4Zrh4ftu7oDVK6w= -github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= -github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= -github.com/projectdiscovery/retryabledns v1.0.21 h1:vOpPQR1q8Z824uoA8JXCI/RyvDAssPeD68Onz9hP/ds= -github.com/projectdiscovery/retryabledns v1.0.21/go.mod h1:6oTPKMRlKZ7lIIEzTH723K6RvNRjmm6fe9br4Dom3UI= -github.com/projectdiscovery/retryablehttp-go v1.0.14 h1:PJaHQtHWE00xrmryZwhtma2b72GbkfA9gJM0yIR9ENY= -github.com/projectdiscovery/retryablehttp-go v1.0.14/go.mod h1:L5HwtGSvc0E3dNVtVqPACWOmr21Bbop2ZhpbCPYEeYU= -github.com/projectdiscovery/stringsutil v0.0.2 h1:uzmw3IVLJSMW1kEg8eCStG/cGbYYZAja8BH3LqqJXMA= -github.com/projectdiscovery/stringsutil v0.0.2/go.mod h1:EJ3w6bC5fBYjVou6ryzodQq37D5c6qbAYQpGmAy+DC0= -github.com/projectdiscovery/utils v0.0.19 h1:5m70xVhBq86h5KsOMVWSO0H+PTmkwlOGAekWcHsaTxI= -github.com/projectdiscovery/utils v0.0.19/go.mod h1:Cu216AlQ7rAYa8aDBqB2OgNfu5p24Uj+tG9RxV8Wbfs= -github.com/projectdiscovery/wappalyzergo v0.0.90 h1:u3W4Uck04sfci1L3RI06Uu1bcL506UwGwW5vtdCf4IE= -github.com/projectdiscovery/wappalyzergo v0.0.90/go.mod h1:HvYuW0Be4JCjVds/+XAEaMSqRG9yrI97UmZq0TPk6A0= +github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= +github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= +github.com/projectdiscovery/dsl v0.0.29 h1:6ne4vWL06vk0sb6lI3iTcDw2leoWa9WUkBCE+h9o2ao= +github.com/projectdiscovery/dsl v0.0.29/go.mod h1:ocF2NbK8nUQPexa5kF/1dBUG56lpsM8S/91gmqOgvmA= +github.com/projectdiscovery/fastdialer v0.0.43 h1:tL1Ivcx4XVJ5hjPA0OoXy8P3TB5paPTnRjkOm/DSLNs= +github.com/projectdiscovery/fastdialer v0.0.43/go.mod h1:pPYldB5+HwlRNB69vHOUBx/Uk0NBcSw1y4z3UZyPOiU= +github.com/projectdiscovery/goflags v0.1.26 h1:o3bFCN5CIYJoLUl/uyKHjXZKkITaBrGPVJ8hg2KErKE= +github.com/projectdiscovery/goflags v0.1.26/go.mod h1:SgLaQOTHspoYYCyhJEmWQZxbeAFGUfMKyultrRsAFYo= +github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= +github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= +github.com/projectdiscovery/gostruct v0.0.1 h1:1KvR6Pn4mDbQqoLEQzhRfHpbreLno2R9xqRCCt5tgmU= +github.com/projectdiscovery/gostruct v0.0.1/go.mod h1:H86peL4HKwMXcQQtEa6lmC8FuD9XFt6gkNR0B/Mu5PE= +github.com/projectdiscovery/hmap v0.0.23 h1:tV/5gQuabE2nqDMS55vrd3HQYdwTuRJAm49nGu3DVl4= +github.com/projectdiscovery/hmap v0.0.23/go.mod h1:DYt1/UjEPA4vw6sk3PY8UB34ZnvXrDC3PQ+LBpkNlOA= +github.com/projectdiscovery/mapcidr v1.1.14 h1:5UP2JCYm+vGr71p3mufqiU0oUXOoGJ72VBKWgBvl0WI= +github.com/projectdiscovery/mapcidr v1.1.14/go.mod h1:q/aK7RR4niZeFCoHhOcUPi+L4KvV0VSIYruRHER70W8= +github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o= +github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= +github.com/projectdiscovery/ratelimit v0.0.13 h1:wB1tbxsMx7yJC0jtTBc6sR5n0SDCASiVD3RRffPTIFU= +github.com/projectdiscovery/ratelimit v0.0.13/go.mod h1:Zj1Hf9aYzD8FjPwMNbLAEkDJNi5ro9EtSus2e8/9Abs= +github.com/projectdiscovery/retryabledns v1.0.40 h1:BasP4PSL9oRGghJNiRNQGd5EYM/gySoZQH0i7WKFEsg= +github.com/projectdiscovery/retryabledns v1.0.40/go.mod h1:BI0AsUmaWF8k/AjEhDGRekMwqda6IDLI43misQuTdYQ= +github.com/projectdiscovery/retryablehttp-go v1.0.34 h1:WbRc4uC3zojCQc0iQvh5Bmb/0W/0TOxEKdmDTaKkaQs= +github.com/projectdiscovery/retryablehttp-go v1.0.34/go.mod h1:tZ3+g7wye1AWuU8utoO02joOB+Jbty8968yEXFCnLPk= +github.com/projectdiscovery/utils v0.0.60 h1:aIPFjFbOHplk12yy5dLWiuGK9A+sxBeqkMAqzrOrB2M= +github.com/projectdiscovery/utils v0.0.60/go.mod h1:TVnAjN6sPBwzxqrJO7WPLWlb6OK4RRl39iZStF8/8mw= +github.com/projectdiscovery/wappalyzergo v0.0.109 h1:BERfwTRn1dvB1tbhyc5m67R8VkC9zbVuPsEq4VEm07k= +github.com/projectdiscovery/wappalyzergo v0.0.109/go.mod h1:4Z3DKhi75zIPMuA+qSDDWxZvnhL4qTLmDx4dxNMu7MA= +github.com/quic-go/quic-go v0.37.4 h1:ke8B73yMCWGq9MfrCCAw0Uzdm7GaViC3i39dsIdDlH4= +github.com/quic-go/quic-go v0.37.4/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= +github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= +github.com/refraction-networking/utls v1.5.4/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= -github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE= -github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU= -github.com/shoenig/go-m1cpu v0.1.4 h1:SZPIgRM2sEF9NJy50mRHu9PKGwxyyTTJIWvCtgVbozs= -github.com/shoenig/go-m1cpu v0.1.4/go.mod h1:Wwvst4LR89UxjeFtLRMrpgRiyY4xPsejnVZym39dbAQ= -github.com/shoenig/test v0.6.3 h1:GVXWJFk9PiOjN0KoJ7VrJGH6uLPnqxR7/fe3HUPfE0c= -github.com/shoenig/test v0.6.3/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/sashabaranov/go-openai v1.14.2 h1:5DPTtR9JBjKPJS008/A409I5ntFhUPPGCmaAihcPRyo= +github.com/sashabaranov/go-openai v1.14.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= +github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smacker/go-tree-sitter v0.0.0-20220628134258-ac06e95cfa11 h1:l4ch+twh4vEZ5VDPyiqC/6h8BhGWHiDxdFRN4M/ZAck= +github.com/smacker/go-tree-sitter v0.0.0-20220628134258-ac06e95cfa11/go.mod h1:q99oHDsbP0xRwmn7Vmob8gbSMNyvJ83OauXPSuHQuKE= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -200,27 +244,28 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= -github.com/tidwall/btree v1.4.3 h1:Lf5U/66bk0ftNppOBjVoy/AIPBrLMkheBp4NnSNiYOo= -github.com/tidwall/btree v1.4.3/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= -github.com/tidwall/buntdb v1.2.10 h1:U/ebfkmYPBnyiNZIirUiWFcxA/mgzjbKlyPynFsPtyM= -github.com/tidwall/buntdb v1.2.10/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= +github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= +github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= +github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= +github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= -github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= @@ -230,23 +275,25 @@ github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7Am github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= -github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= +github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs= -github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= -github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37 h1:oRCu5zb6sklsDvy5sOz3dFqGg5vAEYBBD2MAYhNThCQ= -github.com/weppos/publicsuffix-go v0.15.1-0.20220724114530-e087fba66a37/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA= -github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= +github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= +github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db h1:/WcxBne+5CbtbgWd/sV2wbravmr4sT7y52ifQaCgoLs= +github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= +github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ= +github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns= github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18= -github.com/ysmood/got v0.32.0 h1:aAHdQgfgMb/lo4v+OekM+SSqEJYFI035h5YYvLXsVyU= -github.com/ysmood/got v0.32.0/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= +github.com/ysmood/gop v0.0.2 h1:VuWweTmXK+zedLqYufJdh3PlxDNBOfFHjIZlPT2T5nw= +github.com/ysmood/gop v0.0.2/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk= +github.com/ysmood/got v0.34.1 h1:IrV2uWLs45VXNvZqhJ6g2nIhY+pgIG1CUoOcqfXFl1s= +github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM= github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY= github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= @@ -260,107 +307,141 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= -github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 h1:kKCF7VX/wTmdg2ZjEaqlq99Bjsoiz7vH6sFniF/vI4M= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= +github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= +github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= -github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be h1:M5QjuCbUeNZsup53dlJkI/cx6pVdnDOPzyy+XppoowY= -github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be/go.mod h1:bRZdjnJaHWVXKEwrfAZMd0gfRjZGNhTbZwzp07s0Abw= +github.com/zmap/zcertificate v0.0.1/go.mod h1:q0dlN54Jm4NVSSuzisusQY0hqDWvu92C+TWveAxiVWk= +github.com/zmap/zcrypto v0.0.0-20201128221613-3719af1573cf/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= +github.com/zmap/zcrypto v0.0.0-20201211161100-e54a5822fb7e/go.mod h1:aPM7r+JOkfL+9qSB4KbYjtoEzJqUK50EXkkJabeNJDQ= +github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 h1:YOQ1vXEwE4Rnj+uQ/3oCuJk5wgVsvUyW+glsndwYuyA= +github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968/go.mod h1:xIuOvYCZX21S5Z9bK1BMrertTGX/F8hgAPw7ERJRNS0= +github.com/zmap/zlint/v3 v3.0.0/go.mod h1:paGwFySdHIBEMJ61YjoqT4h7Ge+fdYG4sUQhnTb1lJ8= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0 h1:LGJsf5LRplCck6jUCH3dBL2dmycNruWNF5xugkSlfXw= -golang.org/x/exp v0.0.0-20230310171629-522b1b587ee0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= +golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -368,10 +449,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/runner/banner.go b/internal/runner/banner.go index f3ab6102..0bbeb4e4 100644 --- a/internal/runner/banner.go +++ b/internal/runner/banner.go @@ -12,7 +12,7 @@ var banner = (` /_/\_\\_,_/\__/\_,_/_//_/\_,_/ `) -var version = "v1.0.2-dev" +var version = "v1.0.4" // showBanner is used to show the banner to the user func showBanner() { diff --git a/internal/runner/executer.go b/internal/runner/executer.go index e3d4033c..ce4245d9 100644 --- a/internal/runner/executer.go +++ b/internal/runner/executer.go @@ -15,6 +15,9 @@ func (r *Runner) ExecuteCrawling() error { if len(inputs) == 0 { return errorutil.New("no input provided for crawling") } + for _, input := range inputs { + _ = r.state.InFlightUrls.Set(addSchemeIfNotExists(input), struct{}{}) + } defer r.crawler.Close() @@ -28,6 +31,7 @@ func (r *Runner) ExecuteCrawling() error { if err := r.crawler.Crawl(input); err != nil { gologger.Warning().Msgf("Could not crawl %s: %s", input, err) } + r.state.InFlightUrls.Delete(input) }(input) } wg.Wait() diff --git a/internal/runner/options.go b/internal/runner/options.go index 432f1a53..5f003e53 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -14,12 +14,13 @@ import ( "github.com/projectdiscovery/katana/pkg/utils" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" + logutil "github.com/projectdiscovery/utils/log" "gopkg.in/yaml.v3" ) // validateOptions validates the provided options for crawler func validateOptions(options *types.Options) error { - if options.MaxDepth <= 0 && options.CrawlDuration <= 0 { + if options.MaxDepth <= 0 && options.CrawlDuration.Seconds() <= 0 { return errorutil.New("either max-depth or crawl-duration must be specified") } if len(options.URLs) == 0 && !fileutil.HasStdin() { @@ -112,7 +113,7 @@ func configureOutput(options *types.Options) { gologger.DefaultLogger.SetMaxLevel(levels.LevelInfo) } - // logutil.DisableDefaultLogger() + logutil.DisableDefaultLogger() } func initExampleFormFillConfig() error { @@ -125,6 +126,9 @@ func initExampleFormFillConfig() error { if fileutil.FileExists(defaultConfig) { return nil } + if err := os.MkdirAll(filepath.Dir(defaultConfig), 0775); err != nil { + return err + } exampleConfig, err := os.Create(defaultConfig) if err != nil { return errorutil.NewWithErr(err).Msgf("could not get home directory") diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 01f0bf83..5c6be197 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -1,6 +1,9 @@ package runner import ( + "encoding/json" + "os" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/katana/pkg/engine" "github.com/projectdiscovery/katana/pkg/engine/hybrid" @@ -9,8 +12,9 @@ import ( "github.com/projectdiscovery/katana/pkg/types" errorutil "github.com/projectdiscovery/utils/errors" fileutil "github.com/projectdiscovery/utils/file" - "go.uber.org/multierr" + mapsutil "github.com/projectdiscovery/utils/maps" updateutils "github.com/projectdiscovery/utils/update" + "go.uber.org/multierr" ) // Runner creates the required resources for crawling @@ -20,10 +24,32 @@ type Runner struct { stdin bool crawler engine.Engine options *types.Options + state *RunnerState +} + +type RunnerState struct { + InFlightUrls *mapsutil.SyncLockMap[string, struct{}] } // New returns a new crawl runner structure func New(options *types.Options) (*Runner, error) { + // create the resume configuration structure + if options.ShouldResume() { + gologger.Info().Msg("Resuming from save checkpoint") + + file, err := os.ReadFile(options.Resume) + if err != nil { + return nil, err + } + + runnerState := &RunnerState{} + err = json.Unmarshal(file, runnerState) + if err != nil { + return nil, err + } + options.URLs = mapsutil.GetKeys(runnerState.InFlightUrls.GetAll()) + } + configureOutput(options) showBanner() @@ -33,7 +59,7 @@ func New(options *types.Options) (*Runner, error) { } if !options.DisableUpdateCheck { - latestVersion, err := updateutils.GetVersionCheckCallback("katana")() + latestVersion, err := updateutils.GetToolVersionCallback("katana", version)() if err != nil { if options.Verbose { gologger.Error().Msgf("katana version check failed: %v", err.Error()) @@ -72,7 +98,7 @@ func New(options *types.Options) (*Runner, error) { if err != nil { return nil, errorutil.NewWithErr(err).Msgf("could not create standard crawler") } - runner := &Runner{options: options, stdin: fileutil.HasStdin(), crawlerOptions: crawlerOptions, crawler: crawler} + runner := &Runner{options: options, stdin: fileutil.HasStdin(), crawlerOptions: crawlerOptions, crawler: crawler, state: &RunnerState{InFlightUrls: mapsutil.NewSyncLockMap[string, struct{}]()}} return runner, nil } @@ -84,3 +110,9 @@ func (r *Runner) Close() error { r.crawlerOptions.Close(), ) } + +func (r *Runner) SaveState(resumeFilename string) error { + runnerState := r.state + data, _ := json.Marshal(runnerState) + return os.WriteFile(resumeFilename, data, os.ModePerm) +} diff --git a/internal/testutils/helper.go b/internal/testutils/helper.go new file mode 100644 index 00000000..3f20b9d1 --- /dev/null +++ b/internal/testutils/helper.go @@ -0,0 +1,13 @@ +package testutils + +func CompareOutput(input, expected []string) bool { + if len(input) != len(expected) { + return false + } + for i, v := range input { + if v != expected[i] { + return false + } + } + return true +} diff --git a/internal/testutils/integration.go b/internal/testutils/integration.go new file mode 100644 index 00000000..b77a22c8 --- /dev/null +++ b/internal/testutils/integration.go @@ -0,0 +1,27 @@ +package testutils + +import ( + "fmt" + "os/exec" + "strings" +) + +func RunKatanaBinaryAndGetResults(target string, katanaBinary string, debug bool, args []string) ([]string, error) { + cmd := exec.Command("bash", "-c") + cmdLine := fmt.Sprintf(`echo %s | %s `, target, katanaBinary) + cmdLine += strings.Join(args, " ") + + cmd.Args = append(cmd.Args, cmdLine) + data, err := cmd.Output() + if err != nil { + return nil, err + } + parts := []string{} + items := strings.Split(string(data), "\n") + for _, i := range items { + if i != "" { + parts = append(parts, i) + } + } + return parts, nil +} diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go new file mode 100644 index 00000000..8c85fbdc --- /dev/null +++ b/internal/testutils/testutils.go @@ -0,0 +1,32 @@ +package testutils + +import ( + "strings" + + errorutils "github.com/projectdiscovery/utils/errors" +) + +type TestCase struct { + Name string + Target string + Args string + Expected []string + CompareFunc func(target string, got []string) error +} + +var TestCases = []TestCase{ + { + Name: "Headless Browser Without Incognito", + Target: "https://www.hackerone.com/", + Expected: nil, + Args: "-headless -no-incognito -depth 2 -silent", + CompareFunc: func(target string, got []string) error { + for _, res := range got { + if strings.Contains(res, target) { + return nil + } + } + return errorutils.New("expected %v target in output, but got %v ", target, strings.Join(got, "\n")) + }, + }, +} diff --git a/pkg/engine/common/base.go b/pkg/engine/common/base.go index 1664b3e5..2c11ca35 100644 --- a/pkg/engine/common/base.go +++ b/pkg/engine/common/base.go @@ -127,20 +127,21 @@ type CrawlSession struct { func (s *Shared) NewCrawlSessionWithURL(URL string) (*CrawlSession, error) { ctx, cancel := context.WithCancel(context.Background()) - if s.Options.Options.CrawlDuration > 0 { + if s.Options.Options.CrawlDuration.Seconds() > 0 { //nolint - ctx, cancel = context.WithTimeout(ctx, time.Duration(s.Options.Options.CrawlDuration)*time.Second) + ctx, cancel = context.WithTimeout(ctx, s.Options.Options.CrawlDuration) } parsed, err := urlutil.Parse(URL) if err != nil { - //nolint + cancel() return nil, errorutil.New("could not parse root URL").Wrap(err) } hostname := parsed.Hostname() queue, err := queue.New(s.Options.Options.Strategy, s.Options.Options.Timeout) if err != nil { + cancel() return nil, err } queue.Push(&navigation.Request{Method: http.MethodGet, URL: URL, Depth: 0}, 0) @@ -170,6 +171,7 @@ func (s *Shared) NewCrawlSessionWithURL(URL string) (*CrawlSession, error) { s.Enqueue(queue, navigationRequests...) }) if err != nil { + cancel() return nil, errorutil.New("could not create http client").Wrap(err) } crawlSession := &CrawlSession{ @@ -206,10 +208,6 @@ func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error { gologger.Debug().Msgf("`%v` not in scope. skipping", req.URL) continue } - if !s.Options.ValidatePath(req.URL) { - gologger.Debug().Msgf("skipping url with blacklisted extension %v", req.URL) - continue - } wg.Add() // gologger.Debug().Msgf("Visting: %v", req.URL) // not sure if this is needed @@ -241,6 +239,9 @@ func (s *Shared) Do(crawlSession *CrawlSession, doRequest DoRequestFunc) error { if resp.Resp == nil || resp.Reader == nil { return } + if s.Options.Options.DisableRedirects && resp.IsRedirect() { + return + } navigationRequests := parser.ParseResponse(resp) s.Enqueue(crawlSession.Queue, navigationRequests...) diff --git a/pkg/engine/common/http.go b/pkg/engine/common/http.go index d1bc4180..87b25ddc 100644 --- a/pkg/engine/common/http.go +++ b/pkg/engine/common/http.go @@ -1,17 +1,20 @@ package common import ( + "context" "crypto/tls" + "net" "net/http" "net/url" "time" "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" "github.com/projectdiscovery/katana/pkg/navigation" "github.com/projectdiscovery/katana/pkg/types" "github.com/projectdiscovery/retryablehttp-go" errorutil "github.com/projectdiscovery/utils/errors" - proxyutil "github.com/projectdiscovery/utils/http/proxy" + proxyutil "github.com/projectdiscovery/utils/proxy" ) type RedirectCallback func(resp *http.Response, depth int) @@ -22,7 +25,13 @@ func BuildHttpClient(dialer *fastdialer.Dialer, options *types.Options, redirect retryablehttpOptions := retryablehttp.DefaultOptionsSingle retryablehttpOptions.RetryMax = options.Retries transport := &http.Transport{ - DialContext: dialer.Dial, + DialContext: dialer.Dial, + DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + if options.TlsImpersonate { + return dialer.DialTLSWithConfigImpersonate(ctx, network, addr, &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}, impersonate.Random, nil) + } + return dialer.DialTLS(ctx, network, addr) + }, MaxIdleConns: 100, MaxIdleConnsPerHost: 10, MaxConnsPerHost: 100, @@ -45,6 +54,9 @@ func BuildHttpClient(dialer *fastdialer.Dialer, options *types.Options, redirect Transport: transport, Timeout: time.Duration(options.Timeout) * time.Second, CheckRedirect: func(req *http.Request, via []*http.Request) error { + if options.DisableRedirects { + return http.ErrUseLastResponse + } if len(via) == 10 { return errorutil.New("stopped after 10 redirects") } diff --git a/pkg/engine/hybrid/crawl.go b/pkg/engine/hybrid/crawl.go index ef35fe0e..125b626a 100644 --- a/pkg/engine/hybrid/crawl.go +++ b/pkg/engine/hybrid/crawl.go @@ -5,11 +5,13 @@ import ( "io" "net/http" "net/http/httputil" + "net/url" "strings" "time" "github.com/PuerkitoBio/goquery" + "github.com/go-rod/rod" "github.com/go-rod/rod/lib/proto" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/katana/pkg/engine/common" @@ -35,12 +37,15 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re return nil, errorutil.NewWithTag("hybrid", "could not create target").Wrap(err) } defer page.Close() + c.addHeadersToPage(page) pageRouter := NewHijack(page) pageRouter.SetPattern(&proto.FetchRequestPattern{ URLPattern: "*", RequestStage: proto.FetchRequestStageResponse, }) + + xhrRequests := []navigation.Request{} go pageRouter.Start(func(e *proto.FetchRequestPaused) error { URL, _ := urlutil.Parse(e.Request.URL) body, _ := FetchGetResponseBody(page, e) @@ -64,6 +69,13 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re if err != nil { return errorutil.NewWithErr(err).Msgf("could not create http request") } + // Note: headers are originally sent using `c.addHeadersToPage` below changes are done so that + // headers are reflected in request dump + if httpreq != nil { + for k, v := range c.Headers { + httpreq.Header.Set(k, v) + } + } httpresp := &http.Response{ Proto: "HTTP/1.1", ProtoMajor: 1, @@ -98,6 +110,25 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re Raw: string(rawBytesResponse), } + requestHeaders := make(map[string][]string) + for name, value := range e.Request.Headers { + requestHeaders[name] = []string{value.Str()} + } + + if e.ResourceType == "XHR" && c.Options.Options.XhrExtraction { + xhr := navigation.Request{ + URL: httpreq.URL.String(), + Method: httpreq.Method, + Body: e.Request.PostData, + } + if len(httpreq.Header) > 0 { + xhr.Headers = utils.FlattenHeaders(httpreq.Header) + } else { + xhr.Headers = utils.FlattenHeaders(requestHeaders) + } + xhrRequests = append(xhrRequests, xhr) + } + // trim trailing / normalizedheadlessURL := strings.TrimSuffix(e.Request.URL, "/") matchOriginalURL := stringsutil.EqualFoldAny(request.URL, e.Request.URL, normalizedheadlessURL) @@ -109,6 +140,11 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re // process the raw response navigationRequests := parser.ParseResponse(resp) c.Enqueue(s.Queue, navigationRequests...) + + // do not continue following the request if it's a redirect and redirects are disabled + if c.Options.Options.DisableRedirects && resp.IsRedirect() { + return nil + } return FetchContinueRequest(page, e) })() //nolint defer func() { @@ -157,6 +193,10 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re if err != nil { return nil, errorutil.NewWithTag("hybrid", "url could not be parsed").Wrap(err) } + + if response.Resp == nil { + return nil, errorutil.NewWithTag("hybrid", "response is nil").Wrap(err) + } response.Resp.Request.URL = parsed.URL // Create a copy of intrapolated shadow DOM elements and parse them separately @@ -170,14 +210,36 @@ func (c *Crawler) navigateRequest(s *common.CrawlSession, request *navigation.Re } response.Body = body + response.Reader.Url, _ = url.Parse(request.URL) + if c.Options.Options.FormExtraction { + response.Forms = append(response.Forms, utils.ParseFormFields(response.Reader)...) + } response.Reader, err = goquery.NewDocumentFromReader(strings.NewReader(response.Body)) if err != nil { return nil, errorutil.NewWithTag("hybrid", "could not parse html").Wrap(err) } + + response.XhrRequests = xhrRequests + return response, nil } +func (c *Crawler) addHeadersToPage(page *rod.Page) { + if len(c.Headers) == 0 { + return + } + var arr []string + for k, v := range c.Headers { + arr = append(arr, k, v) + } + // ignore cleanup callback + _, err := page.SetExtraHeaders(arr) + if err != nil { + gologger.Error().Msgf("headless: could not set extra headers: %v", err) + } +} + // traverseDOMNode performs traversal of node completely building a pseudo-HTML // from it including the Shadow DOM, Pseudo elements and other children. // diff --git a/pkg/engine/hybrid/hybrid.go b/pkg/engine/hybrid/hybrid.go index 6c6b4396..89392ad3 100644 --- a/pkg/engine/hybrid/hybrid.go +++ b/pkg/engine/hybrid/hybrid.go @@ -41,60 +41,40 @@ func New(options *types.CrawlerOptions) (*Crawler, error) { previousPIDs := findChromeProcesses() - chromeLauncher := launcher.New(). - Leakless(false). - Set("disable-gpu", "true"). - Set("ignore-certificate-errors", "true"). - Set("ignore-certificate-errors", "1"). - Set("disable-crash-reporter", "true"). - Set("disable-notifications", "true"). - Set("hide-scrollbars", "true"). - Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)). - Set("mute-audio", "true"). - Delete("use-mock-keychain"). - UserDataDir(dataStore) + var launcherURL string + var chromeLauncher *launcher.Launcher - if options.Options.UseInstalledChrome { - if chromePath, hasChrome := launcher.LookPath(); hasChrome { - chromeLauncher.Bin(chromePath) - } else { - return nil, errorutil.NewWithTag("hybrid", "the chrome browser is not installed").WithLevel(errorutil.Fatal) - } - } - if options.Options.SystemChromePath != "" { - chromeLauncher.Bin(options.Options.SystemChromePath) - } - - if options.Options.ShowBrowser { - chromeLauncher = chromeLauncher.Headless(false) + if options.Options.ChromeWSUrl != "" { + launcherURL = options.Options.ChromeWSUrl } else { - chromeLauncher = chromeLauncher.Headless(true) - } - - if options.Options.HeadlessNoSandbox { - chromeLauncher.Set("no-sandbox", "true") - } - - if options.Options.Proxy != "" && options.Options.Headless { - proxyURL, err := urlutil.Parse(options.Options.Proxy) + // create new chrome launcher instance + chromeLauncher, err = buildChromeLauncher(options, dataStore) if err != nil { return nil, err } - chromeLauncher.Set("proxy-server", proxyURL.String()) - } - - for k, v := range options.Options.ParseHeadlessOptionalArguments() { - chromeLauncher.Set(flags.Flag(k), v) - } - launcherURL, err := chromeLauncher.Launch() - if err != nil { - return nil, err + // launch chrome headless process + launcherURL, err = chromeLauncher.Launch() + if err != nil { + return nil, err + } } browser := rod.New().ControlURL(launcherURL) if browserErr := browser.Connect(); browserErr != nil { - return nil, browserErr + return nil, errorutil.NewWithErr(browserErr).Msgf("failed to connect to chrome instance at %s", launcherURL) + } + + // create a new browser instance (default to incognito mode) + if !options.Options.HeadlessNoIncognito { + incognito, err := browser.Incognito() + if err != nil { + if chromeLauncher != nil { + chromeLauncher.Kill() + } + return nil, errorutil.NewWithErr(err).Msgf("failed to create incognito browser") + } + browser = incognito } shared, err := common.NewShared(options) @@ -114,8 +94,10 @@ func New(options *types.CrawlerOptions) (*Crawler, error) { // Close closes the crawler process func (c *Crawler) Close() error { - if err := c.browser.Close(); err != nil { - return err + if c.Options.Options.ChromeWSUrl == "" { + if err := c.browser.Close(); err != nil { + return err + } } if c.Options.Options.ChromeDataDir == "" { if err := os.RemoveAll(c.tempDir); err != nil { @@ -128,30 +110,68 @@ func (c *Crawler) Close() error { // Crawl crawls a URL with the specified options func (c *Crawler) Crawl(rootURL string) error { crawlSession, err := c.NewCrawlSessionWithURL(rootURL) + crawlSession.Browser = c.browser if err != nil { return errorutil.NewWithErr(err).WithTag("hybrid") } defer crawlSession.CancelFunc() - // create a new browser instance (default to incognito mode) - if c.Options.Options.HeadlessNoIncognito { - if err := c.browser.Connect(); err != nil { - return err + gologger.Info().Msgf("Started headless crawling for => %v", rootURL) + if err := c.Do(crawlSession, c.navigateRequest); err != nil { + return errorutil.NewWithErr(err).WithTag("standard") + } + return nil +} + +// buildChromeLauncher builds a new chrome launcher instance +func buildChromeLauncher(options *types.CrawlerOptions, dataStore string) (*launcher.Launcher, error) { + chromeLauncher := launcher.New(). + Leakless(false). + Set("disable-gpu", "true"). + Set("ignore-certificate-errors", "true"). + Set("ignore-certificate-errors", "1"). + Set("disable-crash-reporter", "true"). + Set("disable-notifications", "true"). + Set("hide-scrollbars", "true"). + Set("window-size", fmt.Sprintf("%d,%d", 1080, 1920)). + Set("mute-audio", "true"). + Delete("use-mock-keychain"). + UserDataDir(dataStore) + + if options.Options.UseInstalledChrome { + if chromePath, hasChrome := launcher.LookPath(); hasChrome { + chromeLauncher.Bin(chromePath) + } else { + return nil, errorutil.NewWithTag("hybrid", "the chrome browser is not installed").WithLevel(errorutil.Fatal) } - crawlSession.Browser = c.browser + } + if options.Options.SystemChromePath != "" { + chromeLauncher.Bin(options.Options.SystemChromePath) + } + + if options.Options.ShowBrowser { + chromeLauncher = chromeLauncher.Headless(false) } else { - var err error - crawlSession.Browser, err = c.browser.Incognito() + chromeLauncher = chromeLauncher.Headless(true) + } + + if options.Options.HeadlessNoSandbox { + chromeLauncher.Set("no-sandbox", "true") + } + + if options.Options.Proxy != "" && options.Options.Headless { + proxyURL, err := urlutil.Parse(options.Options.Proxy) if err != nil { - return err + return nil, err } + chromeLauncher.Set("proxy-server", proxyURL.String()) } - gologger.Info().Msgf("Started headless crawling for => %v", rootURL) - if err := c.Do(crawlSession, c.navigateRequest); err != nil { - return errorutil.NewWithErr(err).WithTag("standard") + for k, v := range options.Options.ParseHeadlessOptionalArguments() { + chromeLauncher.Set(flags.Flag(k), v) } - return nil + + return chromeLauncher, nil } // killChromeProcesses any and all new chrome processes started after diff --git a/pkg/engine/parser/parser.go b/pkg/engine/parser/parser.go index 106d76da..f9fd0dd3 100644 --- a/pkg/engine/parser/parser.go +++ b/pkg/engine/parser/parser.go @@ -2,6 +2,7 @@ package parser import ( "encoding/json" + "fmt" "mime/multipart" "strings" @@ -37,7 +38,6 @@ var responseParsers = []responseParser{ // Header based parsers {headerParser, headerContentLocationParser}, {headerParser, headerLinkParser}, - {headerParser, headerLocationParser}, {headerParser, headerRefreshParser}, // Body based parsers @@ -75,11 +75,18 @@ func InitWithOptions(options *types.Options) { if options.AutomaticFormFill { responseParsers = append(responseParsers, responseParser{bodyParser, bodyFormTagParser}) } + if options.ScrapeJSLuiceResponses { + responseParsers = append(responseParsers, responseParser{bodyParser, scriptContentJsluiceParser}) + responseParsers = append(responseParsers, responseParser{contentParser, scriptJSFileJsluiceParser}) + } if options.ScrapeJSResponses { responseParsers = append(responseParsers, responseParser{bodyParser, scriptContentRegexParser}) responseParsers = append(responseParsers, responseParser{contentParser, scriptJSFileRegexParser}) responseParsers = append(responseParsers, responseParser{contentParser, bodyScrapeEndpointsParser}) } + if !options.DisableRedirects { + responseParsers = append(responseParsers, responseParser{headerParser, headerLocationParser}) + } } // parseResponse runs the response parsers on the navigation response @@ -532,7 +539,7 @@ func bodyFormTagParser(resp *navigation.Response) (navigationRequests []*navigat isMultipartForm := strings.HasPrefix(encType, "multipart/") - queryValuesWriter := make(urlutil.Params) + queryValuesWriter := urlutil.NewOrderedParams() var sb strings.Builder var multipartWriter *multipart.Writer @@ -550,16 +557,17 @@ func bodyFormTagParser(resp *navigation.Response) (navigationRequests []*navigat }) dataMap := utils.FormInputFillSuggestions(formInputs) - for key, value := range dataMap { + dataMap.Iterate(func(key, value string) bool { if key == "" || value == "" { - continue + return true } if isMultipartForm { _ = multipartWriter.WriteField(key, value) } else { queryValuesWriter.Set(key, value) } - } + return true + }) // Guess content-type var contentType string @@ -581,7 +589,7 @@ func bodyFormTagParser(resp *navigation.Response) (navigationRequests []*navigat } switch method { case "GET": - parsed.Params.Merge(queryValuesWriter) + parsed.Params.Merge(queryValuesWriter.Encode()) req.URL = parsed.String() case "POST": if multipartWriter != nil { @@ -623,6 +631,7 @@ func scriptContentRegexParser(resp *navigation.Response) (navigationRequests []* if text == "" { return } + endpoints := utils.ExtractRelativeEndpoints(text) for _, item := range endpoints { navigationRequests = append(navigationRequests, navigation.NewNavigationRequestURLFromResponse(item, resp.Resp.Request.URL.String(), "script", "text", resp)) @@ -631,6 +640,22 @@ func scriptContentRegexParser(resp *navigation.Response) (navigationRequests []* return } +// scriptContentJsluiceParser parses script content endpoints using jsluice from response +func scriptContentJsluiceParser(resp *navigation.Response) (navigationRequests []*navigation.Request) { + resp.Reader.Find("script").Each(func(i int, item *goquery.Selection) { + text := item.Text() + if text == "" { + return + } + + endpointItems := utils.ExtractJsluiceEndpoints(text) + for _, item := range endpointItems { + navigationRequests = append(navigationRequests, navigation.NewNavigationRequestURLFromResponse(item.Endpoint, resp.Resp.Request.URL.String(), "script", fmt.Sprintf("jsluice-%s", item.Type), resp)) + } + }) + return +} + // scriptJSFileRegexParser parses relative endpoints from js file pages func scriptJSFileRegexParser(resp *navigation.Response) (navigationRequests []*navigation.Request) { // Only process javascript file based on path or content type @@ -639,8 +664,8 @@ func scriptJSFileRegexParser(resp *navigation.Response) (navigationRequests []*n if !(strings.HasSuffix(resp.Resp.Request.URL.Path, ".js") || strings.HasSuffix(resp.Resp.Request.URL.Path, ".css") || strings.Contains(contentType, "/javascript")) { return } - endpoints := utils.ExtractRelativeEndpoints(string(resp.Body)) - for _, item := range endpoints { + endpointsItems := utils.ExtractRelativeEndpoints(string(resp.Body)) + for _, item := range endpointsItems { navigationRequests = append(navigationRequests, navigation.NewNavigationRequestURLFromResponse(item, resp.Resp.Request.URL.String(), "js", "regex", resp)) } @@ -654,6 +679,26 @@ func scriptJSFileRegexParser(resp *navigation.Response) (navigationRequests []*n return } +// scriptJSFileJsluiceParser parses endpoints using jsluice from js file pages +func scriptJSFileJsluiceParser(resp *navigation.Response) (navigationRequests []*navigation.Request) { + // Only process javascript file based on path or content type + // CSS, JS are supported for relative endpoint extraction. + contentType := resp.Resp.Header.Get("Content-Type") + if !(strings.HasSuffix(resp.Resp.Request.URL.Path, ".js") || strings.HasSuffix(resp.Resp.Request.URL.Path, ".css") || strings.Contains(contentType, "/javascript")) { + return + } + // Skip common js libraries + if utils.IsPathCommonJSLibraryFile(resp.Resp.Request.URL.Path) { + return + } + + endpointsItems := utils.ExtractJsluiceEndpoints(string(resp.Body)) + for _, item := range endpointsItems { + navigationRequests = append(navigationRequests, navigation.NewNavigationRequestURLFromResponse(item.Endpoint, resp.Resp.Request.URL.String(), "js", fmt.Sprintf("jsluice-%s", item.Type), resp)) + } + return +} + // bodyScrapeEndpointsParser parses scraped URLs from HTML body func bodyScrapeEndpointsParser(resp *navigation.Response) (navigationRequests []*navigation.Request) { if strings.HasSuffix(resp.Resp.Request.URL.Path, ".map") { diff --git a/pkg/engine/standard/crawl.go b/pkg/engine/standard/crawl.go index e0ad546b..1281d45b 100644 --- a/pkg/engine/standard/crawl.go +++ b/pkg/engine/standard/crawl.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/http/httputil" + "net/url" "strings" "github.com/PuerkitoBio/goquery" @@ -81,8 +82,12 @@ func (c *Crawler) makeRequest(s *common.CrawlSession, request *navigation.Reques response.Body = string(data) response.Resp = resp response.Reader, err = goquery.NewDocumentFromReader(bytes.NewReader(data)) + response.Reader.Url, _ = url.Parse(request.URL) response.StatusCode = resp.StatusCode response.Headers = utils.FlattenHeaders(resp.Header) + if c.Options.Options.FormExtraction { + response.Forms = append(response.Forms, utils.ParseFormFields(response.Reader)...) + } resp.ContentLength = int64(len(data)) diff --git a/pkg/navigation/response.go b/pkg/navigation/response.go index 526a24c5..cb6f0ef2 100644 --- a/pkg/navigation/response.go +++ b/pkg/navigation/response.go @@ -10,6 +10,13 @@ import ( type Headers map[string]string +type Form struct { + Method string `json:"method,omitempty"` + Action string `json:"action,omitempty"` + Enctype string `json:"enctype,omitempty"` + Parameters []string `json:"parameters,omitempty"` +} + func (h *Headers) MarshalJSON() ([]byte, error) { hCopy := make(Headers) for k, v := range *h { @@ -21,15 +28,18 @@ func (h *Headers) MarshalJSON() ([]byte, error) { // Response is a response generated from crawler navigation type Response struct { - Resp *http.Response `json:"-"` - Depth int `json:"-"` - Reader *goquery.Document `json:"-"` - StatusCode int `json:"status_code,omitempty"` - Headers Headers `json:"headers,omitempty"` - Body string `json:"body,omitempty"` - RootHostname string `json:"-"` - Technologies []string `json:"technologies,omitempty"` - Raw string `json:"raw,omitempty"` + Resp *http.Response `json:"-"` + Depth int `json:"-"` + Reader *goquery.Document `json:"-"` + StatusCode int `json:"status_code,omitempty"` + Headers Headers `json:"headers,omitempty"` + Body string `json:"body,omitempty"` + RootHostname string `json:"-"` + Technologies []string `json:"technologies,omitempty"` + Raw string `json:"raw,omitempty"` + Forms []Form `json:"forms,omitempty"` + XhrRequests []Request `json:"xhr_requests,omitempty"` + StoredResponsePath string `json:"stored_response_path,omitempty"` } func (n Response) AbsoluteURL(path string) string { @@ -48,3 +58,7 @@ func (n Response) AbsoluteURL(path string) string { final := absURL.String() return final } + +func (n Response) IsRedirect() bool { + return n.StatusCode >= 300 && n.StatusCode <= 399 +} diff --git a/pkg/output/fields.go b/pkg/output/fields.go index e22dc7b4..72eefe56 100644 --- a/pkg/output/fields.go +++ b/pkg/output/fields.go @@ -102,19 +102,17 @@ func formatField(output *Result, fields string) []fieldOutput { return svalue } - queryLen := len(parsed.Query()) - queryBoth := make([]string, 0, queryLen) - queryKeys := make([]string, 0, queryLen) - queryValues := make([]string, 0, queryLen) - if queryLen > 0 { - for k, v := range parsed.Query() { - for _, value := range v { - queryBoth = append(queryBoth, strings.Join([]string{k, value}, "=")) - } - queryKeys = append(queryKeys, k) - queryValues = append(queryValues, v...) + queryBoth := []string{} + queryKeys := []string{} + queryValues := []string{} + parsed.Query().Iterate(func(k string, v []string) bool { + for _, value := range v { + queryBoth = append(queryBoth, strings.Join([]string{k, value}, "=")) } - } + queryKeys = append(queryKeys, k) + queryValues = append(queryValues, v...) + return true + }) for _, f := range stringsutil.SplitAny(fields, ",") { switch f { case "url": diff --git a/pkg/output/options.go b/pkg/output/options.go index 7bd50d83..03e1bb37 100644 --- a/pkg/output/options.go +++ b/pkg/output/options.go @@ -1,19 +1,28 @@ package output -import "regexp" +import ( + "regexp" + + "github.com/projectdiscovery/katana/pkg/utils/extensions" +) // Options contains the configuration options for output writer type Options struct { - Colors bool - JSON bool - Verbose bool - StoreResponse bool - OutputFile string - Fields string - StoreFields string - StoreResponseDir string - FieldConfig string - ErrorLogFile string - MatchRegex []*regexp.Regexp - FilterRegex []*regexp.Regexp + Colors bool + JSON bool + Verbose bool + StoreResponse bool + OmitRaw bool + OmitBody bool + OutputFile string + Fields string + StoreFields string + StoreResponseDir string + FieldConfig string + ErrorLogFile string + MatchRegex []*regexp.Regexp + FilterRegex []*regexp.Regexp + ExtensionValidator *extensions.Validator + OutputMatchCondition string + OutputFilterCondition string } diff --git a/pkg/output/output.go b/pkg/output/output.go index 98b967a8..43b1bca5 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -1,6 +1,7 @@ package output import ( + "errors" "fmt" "os" "path/filepath" @@ -10,7 +11,10 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/logrusorgru/aurora" + "github.com/mitchellh/mapstructure" + "github.com/projectdiscovery/dsl" "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/katana/pkg/utils/extensions" errorutil "github.com/projectdiscovery/utils/errors" ) @@ -35,32 +39,42 @@ type Writer interface { // StandardWriter is an standard output writer structure type StandardWriter struct { - storeFields []string - fields string - json bool - verbose bool - aurora aurora.Aurora - outputFile *fileWriter - outputMutex *sync.Mutex - storeResponse bool - storeResponseDir string - errorFile *fileWriter - matchRegex []*regexp.Regexp - filterRegex []*regexp.Regexp + storeFields []string + fields string + json bool + verbose bool + aurora aurora.Aurora + outputFile *fileWriter + outputMutex *sync.Mutex + storeResponse bool + storeResponseDir string + omitRaw bool + omitBody bool + errorFile *fileWriter + matchRegex []*regexp.Regexp + filterRegex []*regexp.Regexp + extensionValidator *extensions.Validator + outputMatchCondition string + outputFilterCondition string } // New returns a new output writer instance func New(options Options) (Writer, error) { writer := &StandardWriter{ - fields: options.Fields, - json: options.JSON, - verbose: options.Verbose, - aurora: aurora.NewAurora(options.Colors), - outputMutex: &sync.Mutex{}, - storeResponse: options.StoreResponse, - storeResponseDir: options.StoreResponseDir, - matchRegex: options.MatchRegex, - filterRegex: options.FilterRegex, + fields: options.Fields, + json: options.JSON, + verbose: options.Verbose, + aurora: aurora.NewAurora(options.Colors), + outputMutex: &sync.Mutex{}, + storeResponse: options.StoreResponse, + storeResponseDir: options.StoreResponseDir, + omitRaw: options.OmitRaw, + omitBody: options.OmitBody, + matchRegex: options.MatchRegex, + filterRegex: options.FilterRegex, + extensionValidator: options.ExtensionValidator, + outputMatchCondition: options.OutputMatchCondition, + outputFilterCondition: options.OutputFilterCondition, } // if fieldConfig empty get the default file if options.FieldConfig == "" { @@ -128,6 +142,11 @@ func (w *StandardWriter) Write(result *Result) error { if len(w.storeFields) > 0 { storeFields(result, w.storeFields) } + + if !w.extensionValidator.ValidatePath(result.Request.URL) { + return nil + } + if !w.matchOutput(result) { return nil } @@ -137,6 +156,36 @@ func (w *StandardWriter) Write(result *Result) error { var data []byte var err error + if w.storeResponse && result.HasResponse() { + if fileName, fileWriter, err := getResponseFile(w.storeResponseDir, result.Response.Resp.Request.URL.String()); err == nil { + if absPath, err := filepath.Abs(fileName); err == nil { + fileName = absPath + } + result.Response.StoredResponsePath = fileName + data, err := w.formatResult(result) + if err != nil { + return errorutil.NewWithTag("output", "could not store response").Wrap(err) + } + if err := updateIndex(w.storeResponseDir, result); err != nil { + return errorutil.NewWithTag("output", "could not store response").Wrap(err) + } + if err := fileWriter.Write(data); err != nil { + return errorutil.NewWithTag("output", "could not store response").Wrap(err) + } + fileWriter.Close() + } + } + + if w.omitRaw { + result.Request.Raw = "" + if result.Response != nil { + result.Response.Raw = "" + } + } + if w.omitBody && result.HasResponse() { + result.Response.Body = "" + } + if w.json { data, err = w.formatJSON(result) } else { @@ -162,22 +211,6 @@ func (w *StandardWriter) Write(result *Result) error { } } - if w.storeResponse && result.HasResponse() { - if file, err := getResponseFile(w.storeResponseDir, result.Response.Resp.Request.URL.String()); err == nil { - data, err := w.formatResult(result) - if err != nil { - return errorutil.NewWithTag("output", "could not store response").Wrap(err) - } - if err := updateIndex(w.storeResponseDir, result); err != nil { - return errorutil.NewWithTag("output", "could not store response").Wrap(err) - } - if err := file.Write(data); err != nil { - return errorutil.NewWithTag("output", "could not store response").Wrap(err) - } - file.Close() - } - } - return nil } @@ -219,26 +252,104 @@ func (w *StandardWriter) Close() error { // matchOutput checks if the event matches the output regex func (w *StandardWriter) matchOutput(event *Result) bool { - if w.matchRegex == nil { + if w.matchRegex == nil && w.outputMatchCondition == "" { return true } + for _, regex := range w.matchRegex { if regex.MatchString(event.Request.URL) { return true } } + + if w.outputMatchCondition != "" { + return evalDslExpr(event, w.outputMatchCondition) + } + return false } // filterOutput returns true if the event should be filtered out func (w *StandardWriter) filterOutput(event *Result) bool { - if w.filterRegex == nil { + if w.filterRegex == nil && w.outputFilterCondition == "" { return false } + for _, regex := range w.filterRegex { if regex.MatchString(event.Request.URL) { return true } } + + if w.outputFilterCondition != "" { + return evalDslExpr(event, w.outputFilterCondition) + } + + return false +} + +func evalDslExpr(result *Result, dslExpr string) bool { + resultMap, err := resultToMap(*result) + if err != nil { + gologger.Warning().Msgf("Could not map result: %s\n", err) + return false + } + + res, err := dsl.EvalExpr(dslExpr, resultMap) + if err != nil && !ignoreErr(err) { + gologger.Error().Msgf("Could not evaluate DSL expression: %s\n", err) + return false + } + return res == true +} + +func resultToMap(result Result) (map[string]interface{}, error) { + resultMap := make(map[string]any) + config := &mapstructure.DecoderConfig{ + TagName: "json", + Result: &resultMap, + } + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return nil, fmt.Errorf("error creating decoder: %v", err) + } + err = decoder.Decode(result) + if err != nil { + return nil, fmt.Errorf("error decoding: %v", err) + } + return flatten(resultMap), nil +} + +// mapsutil.Flatten w/o separator +func flatten(m map[string]any) map[string]any { + o := make(map[string]any) + for k, v := range m { + switch child := v.(type) { + case map[string]any: + nm := flatten(child) + for nk, nv := range nm { + o[nk] = nv + } + default: + o[k] = v + } + } + return o +} + +var ( + // showDSLErr controls whether to show hidden DSL errors or not + showDSLErr = strings.EqualFold(os.Getenv("SHOW_DSL_ERRORS"), "true") +) + +// ignoreErr checks if the error is to be ignored or not +// Reference: https://github.com/projectdiscovery/katana/pull/537 +func ignoreErr(err error) bool { + if showDSLErr { + return false + } + if errors.Is(err, dsl.ErrParsingArg) || strings.Contains(err.Error(), "No parameter") { + return true + } return false } diff --git a/pkg/output/responses.go b/pkg/output/responses.go index f5d1e3d0..63174a31 100644 --- a/pkg/output/responses.go +++ b/pkg/output/responses.go @@ -44,17 +44,18 @@ func createHostDir(storeResponseFolder, domain string) string { return filepath.Join(storeResponseFolder, domain) } -func getResponseFile(storeResponseFolder, URL string) (*fileWriter, error) { +func getResponseFile(storeResponseFolder, URL string) (string, *fileWriter, error) { domain, err := getResponseHost(URL) if err != nil { - return nil, err + return "", nil, err } - output, err := newFileOutputWriter(getResponseFileName(storeResponseFolder, domain, URL)) + fileName := getResponseFileName(storeResponseFolder, domain, URL) + output, err := newFileOutputWriter(fileName) if err != nil { - return nil, errorutil.NewWithTag("output", "could not create output file").Wrap(err) + return "", nil, errorutil.NewWithTag("output", "could not create output file").Wrap(err) } - return output, nil + return fileName, output, nil } func getResponseFileName(storeResponseFolder, domain, URL string) string { diff --git a/pkg/types/crawler_options.go b/pkg/types/crawler_options.go index 90a0fca5..9451890f 100644 --- a/pkg/types/crawler_options.go +++ b/pkg/types/crawler_options.go @@ -58,18 +58,23 @@ func NewCrawlerOptions(options *Options) (*CrawlerOptions, error) { } outputOptions := output.Options{ - Colors: !options.NoColors, - JSON: options.JSON, - Verbose: options.Verbose, - StoreResponse: options.StoreResponse, - OutputFile: options.OutputFile, - Fields: options.Fields, - StoreFields: options.StoreFields, - StoreResponseDir: options.StoreResponseDir, - FieldConfig: options.FieldConfig, - ErrorLogFile: options.ErrorLogFile, - MatchRegex: options.MatchRegex, - FilterRegex: options.FilterRegex, + Colors: !options.NoColors, + JSON: options.JSON, + Verbose: options.Verbose, + StoreResponse: options.StoreResponse, + OutputFile: options.OutputFile, + Fields: options.Fields, + StoreFields: options.StoreFields, + StoreResponseDir: options.StoreResponseDir, + OmitRaw: options.OmitRaw, + OmitBody: options.OmitBody, + FieldConfig: options.FieldConfig, + ErrorLogFile: options.ErrorLogFile, + MatchRegex: options.MatchRegex, + FilterRegex: options.FilterRegex, + ExtensionValidator: extensionsValidator, + OutputMatchCondition: options.OutputMatchCondition, + OutputFilterCondition: options.OutputFilterCondition, } outputWriter, err := output.New(outputOptions) if err != nil { diff --git a/pkg/types/options.go b/pkg/types/options.go index 7b55ba8e..5e4b8607 100644 --- a/pkg/types/options.go +++ b/pkg/types/options.go @@ -3,9 +3,11 @@ package types import ( "regexp" "strings" + "time" "github.com/projectdiscovery/goflags" "github.com/projectdiscovery/katana/pkg/output" + fileutil "github.com/projectdiscovery/utils/file" ) // OnResultCallback (output.Result) @@ -14,6 +16,8 @@ type OnResultCallback func(output.Result) type Options struct { // URLs contains a list of URLs for crawling URLs goflags.StringSlice + // Resume the scan from the state stored in the resume config file + Resume string // Scope contains a list of regexes for in-scope URLS Scope goflags.StringSlice // OutOfScope contains a list of regexes for out-scope URLS @@ -26,6 +30,10 @@ type Options struct { ExtensionsMatch goflags.StringSlice // ExtensionFilter contains additional items for filter list ExtensionFilter goflags.StringSlice + // OutputMatchCondition is the condition to match output + OutputMatchCondition string + // OutputFilterCondition is the condition to filter output + OutputFilterCondition string // MaxDepth is the maximum depth to crawl MaxDepth int // BodyReadSize is the maximum size of response body to read @@ -33,7 +41,7 @@ type Options struct { // Timeout is the time to wait for request in seconds Timeout int // CrawlDuration is the duration in seconds to crawl target from - CrawlDuration int + CrawlDuration time.Duration // Delay is the delay between each crawl requests in seconds Delay int // RateLimit is the maximum number of requests to send per second @@ -76,12 +84,16 @@ type Options struct { Version bool // ScrapeJSResponses enables scraping of relative endpoints from javascript ScrapeJSResponses bool + // ScrapeJSLuiceResponses enables scraping of endpoints from javascript using jsluice + ScrapeJSLuiceResponses bool // CustomHeaders is a list of custom headers to add to request CustomHeaders goflags.StringSlice // Headless enables headless scraping Headless bool // AutomaticFormFill enables optional automatic form filling and submission AutomaticFormFill bool + // FormExtraction enables extraction of form, input, textarea & select elements + FormExtraction bool // UseInstalledChrome skips chrome install and use local instance UseInstalledChrome bool // ShowBrowser specifies whether the show the browser in headless mode @@ -92,16 +104,24 @@ type Options struct { HeadlessNoSandbox bool // SystemChromePath : Specify the chrome binary path for headless crawling SystemChromePath string + // ChromeWSUrl : Specify the Chrome debugger websocket url for a running Chrome instance to attach to + ChromeWSUrl string // OnResult allows callback function on a result OnResult OnResultCallback // StoreResponse specifies if katana should store http requests/responses StoreResponse bool // StoreResponseDir specifies if katana should use a custom directory to store http requests/responses StoreResponseDir string + // OmitRaw omits raw requests/responses from the output + OmitRaw bool + // OmitBody omits the response body from the output + OmitBody bool // ChromeDataDir : Specify the --user-data-dir to chrome binary to preserve sessions ChromeDataDir string // HeadlessNoIncognito specifies if chrome should be started without incognito mode HeadlessNoIncognito bool + // XhrExtraction extract xhr requests + XhrExtraction bool // HealthCheck determines if a self-healthcheck should be performed HealthCheck bool // ErrorLogFile specifies a file to write with the errors of all requests @@ -122,6 +142,10 @@ type Options struct { IgnoreQueryParams bool // Debug Debug bool + // TlsImpersonate enables experimental tls ClientHello randomization for standard crawler + TlsImpersonate bool + //DisableRedirects disables the following of redirects + DisableRedirects bool } func (options *Options) ParseCustomHeaders() map[string]string { @@ -135,15 +159,30 @@ func (options *Options) ParseCustomHeaders() map[string]string { } func (options *Options) ParseHeadlessOptionalArguments() map[string]string { - optionalArguments := make(map[string]string) + var ( + lastKey string + optionalArguments = make(map[string]string) + ) for _, v := range options.HeadlessOptionalArguments { + if v == "" { + continue + } if argParts := strings.SplitN(v, "=", 2); len(argParts) >= 2 { key := strings.TrimSpace(argParts[0]) value := strings.TrimSpace(argParts[1]) if key != "" && value != "" { optionalArguments[key] = value + lastKey = key } + } else if !strings.HasPrefix(v, "--") { + optionalArguments[lastKey] += "," + v + } else { + optionalArguments[v] = "" } } return optionalArguments } + +func (options *Options) ShouldResume() bool { + return options.Resume != "" && fileutil.FileExists(options.Resume) +} diff --git a/pkg/types/options_test.go b/pkg/types/options_test.go index 82ac4fb4..3f73bea3 100644 --- a/pkg/types/options_test.go +++ b/pkg/types/options_test.go @@ -85,6 +85,21 @@ func TestParseHeadlessOptionalArguments(t *testing.T) { input: "a=b,a=b", want: map[string]string{"a": "b"}, }, + { + name: "values with dash with boolean flag at the end", + input: "--a=a/b,c/d--z--n--m/a,--c=k,--h", + want: map[string]string{"--a": "a/b,c/d--z--n--m/a", "--c": "k", "--h": ""}, + }, + { + name: "values with dash boolean flag at the beginning", + input: "--h,--a=a/b,c/d--z--n--m/a,--c=k", + want: map[string]string{"--h": "", "--a": "a/b,c/d--z--n--m/a", "--c": "k"}, + }, + { + name: "values with dash boolean flag in the middle", + input: "--a=a/b,c/d--z--n--m/a,--h,--c=k", + want: map[string]string{"--a": "a/b,c/d--z--n--m/a", "--h": "", "--c": "k"}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/utils/formfields.go b/pkg/utils/formfields.go new file mode 100644 index 00000000..4c488fe5 --- /dev/null +++ b/pkg/utils/formfields.go @@ -0,0 +1,86 @@ +package utils + +import ( + "strings" + + "github.com/projectdiscovery/katana/pkg/navigation" + + "github.com/PuerkitoBio/goquery" + "github.com/projectdiscovery/utils/generic" + urlutil "github.com/projectdiscovery/utils/url" +) + +// parses form, input, textarea & select elements +func ParseFormFields(document *goquery.Document) []navigation.Form { + var forms []navigation.Form + + document.Find("form").Each(func(i int, formElem *goquery.Selection) { + form := navigation.Form{} + + action, _ := formElem.Attr("action") + method, _ := formElem.Attr("method") + enctype, _ := formElem.Attr("enctype") + + if method == "" { + method = "GET" + } + + if enctype == "" && method != "GET" { + enctype = "application/x-www-form-urlencoded" + } + + if action != "" { + actionUrl, err := urlutil.ParseURL(action, true) + if err != nil { + return + } + + // donot modify absolute urls and windows paths + if actionUrl.IsAbs() || strings.HasPrefix(action, "//") || strings.HasPrefix(action, "\\") { + // keep absolute urls as is + _ = action + } else if document.Url != nil { + // concatenate relative urls with base url + // clone base url + cloned, err := urlutil.ParseURL(document.Url.String(), true) + if err != nil { + return + } + + if strings.HasPrefix(action, "/") { + // relative path + //
=> https://example.com/root_rel + _ = cloned.UpdateRelPath(action, true) + action = cloned.String() + } else { + //
=> https://example.com/path/path_rel + if err := cloned.MergePath(action, false); err != nil { + return + } + action = cloned.String() + } + } + } else { + action = document.Url.String() + } + + form.Method = strings.ToUpper(method) + form.Action = action + form.Enctype = enctype + + formElem.Find("input, textarea, select").Each(func(i int, inputElem *goquery.Selection) { + name, ok := inputElem.Attr("name") + if !ok { + return + } + + form.Parameters = append(form.Parameters, name) + }) + + if !generic.EqualsAll("", form.Action, form.Method, form.Enctype) || len(form.Parameters) > 0 { + forms = append(forms, form) + } + }) + + return forms +} diff --git a/pkg/utils/formfields_test.go b/pkg/utils/formfields_test.go new file mode 100644 index 00000000..7edb5b2c --- /dev/null +++ b/pkg/utils/formfields_test.go @@ -0,0 +1,58 @@ +package utils + +import ( + "net/url" + "strings" + "testing" + + "github.com/PuerkitoBio/goquery" + "github.com/stretchr/testify/require" +) + +var htmlFormExample = ` + + HTML Form Test + + +
+
+ + + +
+
+
+
+
+
+
+ +` + +func TestParseFormFields(t *testing.T) { + document, err := goquery.NewDocumentFromReader(strings.NewReader(htmlFormExample)) + require.NoError(t, err, "could not read document") + + document.Url, _ = url.Parse("https://example.com/path") + forms := ParseFormFields(document) + + require.Equal(t, "https://example.com/test", forms[0].Action) + require.Equal(t, "POST", forms[0].Method) + require.Equal(t, "POST", forms[1].Method) + require.Equal(t, "https://abs.example.com", forms[1].Action) + require.Equal(t, "GET", forms[2].Method) + require.Equal(t, "//prel.example.com", forms[2].Action) + require.Equal(t, "GET", forms[3].Method) + require.Equal(t, "\\\\unc.example.com", forms[3].Action) + require.Equal(t, "GET", forms[4].Method) + require.Equal(t, "https://example.com/root_rel", forms[4].Action) + require.Equal(t, "GET", forms[5].Method) + require.Equal(t, "https://example.com/path/rel_path", forms[5].Action) + require.Equal(t, "GET", forms[6].Method) + require.Equal(t, "https://example.com/path", forms[6].Action) + require.Contains(t, forms[0].Parameters, "firstname") + require.Contains(t, forms[0].Parameters, "textarea1") + require.Contains(t, forms[0].Parameters, "select1") + require.Equal(t, 3, len(forms[0].Parameters), "found more or less parameters than where present") + require.Equal(t, 7, len(forms), "found more or less forms than where present") +} diff --git a/pkg/utils/formfill.go b/pkg/utils/formfill.go index 25559c92..194f8e8e 100644 --- a/pkg/utils/formfill.go +++ b/pkg/utils/formfill.go @@ -5,6 +5,7 @@ import ( "strconv" "github.com/PuerkitoBio/goquery" + mapsutil "github.com/projectdiscovery/utils/maps" "github.com/rs/xid" ) @@ -37,69 +38,69 @@ type FormInput struct { Type string Name string Value string - Attributes map[string]string + Attributes mapsutil.OrderedMap[string, string] } // FormInputFillSuggestions returns a list of form filling suggestions // for inputs returning the specified recommended values. -func FormInputFillSuggestions(inputs []FormInput) map[string]string { - data := make(map[string]string) +func FormInputFillSuggestions(inputs []FormInput) mapsutil.OrderedMap[string, string] { + data := mapsutil.NewOrderedMap[string, string]() // Fill checkboxes and radioboxes first or default values first for _, input := range inputs { switch input.Type { case "radio": // Use a single radio name per value - if _, ok := data[input.Name]; !ok { - data[input.Name] = input.Value + if !data.Has(input.Name) { + data.Set(input.Name, input.Value) } case "checkbox": - data[input.Name] = input.Value + data.Set(input.Name, input.Value) default: // If there is a value, use it for the input. Else // infer the values based on input types. if input.Value != "" { - data[input.Name] = input.Value + data.Set(input.Name, input.Value) } } } + // getIntWithdefault returns the integer value of the key or default value + getIntWithdefault := func(input *FormInput, key string, defaultValue int) int { + if value, ok := input.Attributes.Get(key); ok { + if intValue, err := strconv.Atoi(value); err == nil { + return intValue + } + } + return defaultValue + } + // Fill rest of the inputs based on their types or name and ids for _, input := range inputs { if input.Value != "" { continue } - switch input.Type { case "email": - data[input.Name] = FormData.Email + data.Set(input.Name, FormData.Email) case "color": - data[input.Name] = FormData.Color + data.Set(input.Name, FormData.Color) case "number", "range": - var err error - var max, min, step, val int - - if min, err = strconv.Atoi(input.Attributes["min"]); err != nil { - min = 1 - } - if max, err = strconv.Atoi(input.Attributes["max"]); err != nil { - max = 10 - } - if step, err = strconv.Atoi(input.Attributes["step"]); err != nil { - step = 1 - } - val = min + step + min := getIntWithdefault(&input, "min", 1) + max := getIntWithdefault(&input, "max", 10) + step := getIntWithdefault(&input, "step", 1) + val := min + step if val > max { val = max - step } - data[input.Name] = strconv.Itoa(val) + data.Set(input.Name, strconv.Itoa(val)) case "password": - data[input.Name] = FormData.Password + data.Set(input.Name, FormData.Password) case "tel": - data[input.Name] = FormData.Password + data.Set(input.Name, FormData.Password) default: - data[input.Name] = FormData.Placeholder + data.Set(input.Name, FormData.Placeholder) } } return data @@ -108,7 +109,7 @@ func FormInputFillSuggestions(inputs []FormInput) map[string]string { // ConvertGoquerySelectionToFormInput converts goquery selection to form input func ConvertGoquerySelectionToFormInput(item *goquery.Selection) FormInput { attrs := item.Nodes[0].Attr - input := FormInput{Attributes: make(map[string]string)} + input := FormInput{Attributes: mapsutil.NewOrderedMap[string, string]()} for _, attribute := range attrs { switch attribute.Key { @@ -119,7 +120,7 @@ func ConvertGoquerySelectionToFormInput(item *goquery.Selection) FormInput { case "type": input.Type = attribute.Val default: - input.Attributes[attribute.Key] = attribute.Val + input.Attributes.Set(attribute.Key, attribute.Val) } } return input diff --git a/pkg/utils/formfill_test.go b/pkg/utils/formfill_test.go index b9bcd88d..268f654c 100644 --- a/pkg/utils/formfill_test.go +++ b/pkg/utils/formfill_test.go @@ -54,13 +54,13 @@ func TestFormInputFillSuggestions(t *testing.T) { }) dataMap := FormInputFillSuggestions(formInputs) - for key, value := range dataMap { + dataMap.Iterate(func(key, value string) bool { if key == "" || value == "" { - continue + return true } queryValuesWriter.Set(key, value) - } - + return true + }) value := queryValuesWriter.Encode() require.Equal(t, "Startdate=katana&color=red&firstname=katana&num=51&password=katana&sport1=cricket&sport2=tennis&sport3=football&telephone=katanaP%40assw0rd1&upclick=%23a52a2a", value, "could not get correct encoded form") }) diff --git a/pkg/utils/jsluice.go b/pkg/utils/jsluice.go new file mode 100644 index 00000000..d33d8e66 --- /dev/null +++ b/pkg/utils/jsluice.go @@ -0,0 +1,50 @@ +package utils + +import ( + "regexp" + + "github.com/BishopFox/jsluice" +) + +var ( + // CommonJSLibraryFileRegex is a regex to match common js library files. + CommonJSLibraryFileRegex = `(?:amplify|quantserve|slideshow|jquery|modernizr|polyfill|vendor|modules|gtm|underscor|tween|retina|selectivizr|cufon|underscore|angular|swf|sha1|freestyle|jquery|bootstrap|modernizr|d3|backbone|videojs|google_analytics|material|redux|knockout|datepicker|datetimepicker|ember|react|ng|angular|fusion|analytics|lib|libs|vendor|vendors|node_modules)([-._][\w\d]*)*\.js$` + commonJSLibraryFileRegexCompiled = regexp.MustCompile(CommonJSLibraryFileRegex) +) + +// IsPathCommonJSLibraryFile checks if a given path is a common js library file. +func IsPathCommonJSLibraryFile(path string) bool { + return commonJSLibraryFileRegexCompiled.MatchString(path) +} + +type JSLuiceEndpoint struct { + Endpoint string + Type string +} + +// ExtractJsluiceEndpoints extracts jsluice endpoints from a given string. +// +// We use tomnomnom and bishopfox's jsluice to extract endpoints from javascript +// files. +// +// We apply several optimizations before running jsluice: +// - We skip common js library files. +// - We skip lines that are too long and contain a lot of characters. +func ExtractJsluiceEndpoints(data string) []JSLuiceEndpoint { + analyzer := jsluice.NewAnalyzer([]byte(data)) + + // TODO: add new user url matchers + // analyzer.AddURLMatcher(matcher) + + var endpoints []JSLuiceEndpoint + foundURLs := analyzer.GetURLs() + + for _, url := range foundURLs { + url := url + endpoints = append(endpoints, JSLuiceEndpoint{ + Endpoint: url.URL, + Type: url.Type, + }) + } + return endpoints +} diff --git a/pkg/utils/jsluice_test.go b/pkg/utils/jsluice_test.go new file mode 100644 index 00000000..519ceb3b --- /dev/null +++ b/pkg/utils/jsluice_test.go @@ -0,0 +1,36 @@ +package utils + +import "testing" + +func TestIsPathCommonJSLibraryFile(t *testing.T) { + type args struct { + path string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "jquery.js", + args: args{ + path: "jquery.js", + }, + want: true, + }, + { + name: "app.js", + args: args{ + path: "app.js", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsPathCommonJSLibraryFile(tt.args.path); got != tt.want { + t.Errorf("IsPathCommonJSLibraryFile() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/utils/regex.go b/pkg/utils/regex.go index 282fb08e..61ead03f 100644 --- a/pkg/utils/regex.go +++ b/pkg/utils/regex.go @@ -7,13 +7,14 @@ import ( var ( BodyA0 = `(?:` BodyB0 = `(` - BodyC0 = `(?:[\.]{1,2}/[A-Za-z0-9-_/\\?&@\.?=%]+)` - BodyC1 = `|(https?://[A-Za-z0-9_\-\.]+([\.]{0,2})?\/[A-Za-z0-9-_/\\?&@\.?=%]+)` - BodyC2 = `|(/[A-Za-z0-9-_/\\?&@\.%]+\.(aspx?|action|cfm|cgi|do|pl|css|x?html?|js(p|on)?|pdf|php5?|py|rss))` + BodyC0 = `(?:[\.]{1,2}/[A-Za-z0-9\-_/\\?&@\.?=%]+)` + BodyC1 = `|(https?://[A-Za-z0-9_\-\.]+([\.]{0,2})?\/[A-Za-z0-9\-_/\\?&@\.?=%]+)` + BodyC2 = `|(/[A-Za-z0-9\-_/\\?&@\.%]+\.(aspx?|action|cfm|cgi|do|pl|css|x?html?|js(p|on)?|pdf|php5?|py|rss))` + BodyC3 = `|([A-Za-z0-9\-_?&@\.%]+/[A-Za-z0-9/\\\-_?&@\.%]+\.(aspx?|action|cfm|cgi|do|pl|css|x?html?|js(p|on)?|pdf|php5?|py|rss))` BodyB1 = `)` BodyA1 = `)` // pageBodyRegex extracts endpoints from page body - pageBodyRegex = regexp.MustCompile(BodyA0 + BodyB0 + BodyC0 + BodyC1 + BodyC2 + BodyB1 + BodyA1) + pageBodyRegex = regexp.MustCompile(BodyA0 + BodyB0 + BodyC0 + BodyC1 + BodyC2 + BodyC3 + BodyB1 + BodyA1) JsA0 = `(?:"|'|\s)` JsB0 = `(` diff --git a/pkg/utils/scope/scope.go b/pkg/utils/scope/scope.go index 0487b13a..7fcc1463 100644 --- a/pkg/utils/scope/scope.go +++ b/pkg/utils/scope/scope.go @@ -12,10 +12,11 @@ import ( // Manager manages scope for crawling process type Manager struct { - inScope []*regexp.Regexp - outOfScope []*regexp.Regexp - noScope bool - fieldScope dnsScopeField + inScope []*regexp.Regexp + outOfScope []*regexp.Regexp + noScope bool + fieldScope dnsScopeField + fieldScopePattern *regexp.Regexp } type dnsScopeField int @@ -24,6 +25,7 @@ const ( dnDnsScopeField dnsScopeField = iota + 1 rdnDnsScopeField fqdnDNSScopeField + customDNSScopeField ) var stringToDNSScopeField = map[string]dnsScopeField{ @@ -39,7 +41,12 @@ func NewManager(inScope, outOfScope []string, fieldScope string, noScope bool) ( } if scopeValue, ok := stringToDNSScopeField[fieldScope]; !ok { - return nil, fmt.Errorf("invalid dns scope field specified: %s", fieldScope) + manager.fieldScope = customDNSScopeField + if compiled, err := regexp.Compile(fieldScope); err != nil { + return nil, fmt.Errorf("could not compile regex %s: %s", fieldScope, err) + } else { + manager.fieldScopePattern = compiled + } } else { manager.fieldScope = scopeValue } @@ -108,7 +115,12 @@ func (m *Manager) validateURL(URL string) (bool, error) { func (m *Manager) validateDNS(hostname, rootHostname string) (bool, error) { parsed := net.ParseIP(hostname) - + if m.fieldScope == customDNSScopeField { + // If we have a custom regex, we need to match it against the full hostname + if m.fieldScopePattern.MatchString(hostname) { + return true, nil + } + } if m.fieldScope == fqdnDNSScopeField || parsed != nil { matched := strings.EqualFold(hostname, rootHostname) return matched, nil diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 8b6ec0e3..80c2d90c 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -83,9 +83,10 @@ func ReplaceAllQueryParam(reqUrl, val string) string { return reqUrl } params := u.Query() - for k := range params { - params.Set(k, "") - } + params.Iterate(func(key string, value []string) bool { + params.Set(key, "") + return true + }) u.RawQuery = params.Encode() return u.String() }