diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index a10c59651..79346cbf5 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.20.x + go-version: 1.21.x - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index dbe1ce1ad..da4f58218 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.20.x + go-version: 1.21.x - name: Run golangci-lint uses: golangci/golangci-lint-action@v3.6.0 with: diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index f0936c4dd..960e3f480 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -18,7 +18,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v4 with: - go-version: 1.20.x + go-version: 1.21.x - name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v3 diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index 0e3e9ee9d..d5b099447 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.20.x + go-version: 1.21.x - name: release test uses: goreleaser/goreleaser-action@v4 diff --git a/README.md b/README.md index cb252e6d5..749098226 100644 --- a/README.md +++ b/README.md @@ -89,8 +89,8 @@ OUTPUT: -oI, -ip include host IP in output (-active only) CONFIGURATION: - -config string flag config file (default "$HOME/.config/subfinder/config.yaml") - -pc, -provider-config string provider config file (default "$HOME/.config/subfinder/provider-config.yaml") + -config string flag config file (default "$CONFIG/subfinder/config.yaml") + -pc, -provider-config string provider config file (default "$CONFIG/subfinder/provider-config.yaml") -r string[] comma separated list of resolvers to use -rL, -rlist string file containing list of resolvers to use -nW, -active display active subdomains only @@ -117,140 +117,15 @@ OPTIMIZATION: go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest ``` -## Post Installation Instructions - -`subfinder` can be used right after the installation, however the following services require configuring API keys to work: - -[BeVigil](https://bevigil.com/osint-api), [BinaryEdge](https://binaryedge.io), [BufferOver](https://tls.bufferover.run), [C99](https://api.c99.nl/), [Censys](https://censys.io), [CertSpotter](https://sslmate.com/certspotter/api/), [Chaos](https://chaos.projectdiscovery.io), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [FullHunt](https://fullhunt.io), [GitHub](https://github.com), [Intelx](https://intelx.io), [PassiveTotal](http://passivetotal.org), [quake](https://quake.360.cn), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [ThreatBook](https://x.threatbook.cn/en), [VirusTotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [ZoomEye](https://www.zoomeye.org), [ZoomEye API](https://api.zoomeye.org), [dnsrepo](https://dnsrepo.noc.org), [Hunter](https://hunter.qianxin.com/), [Facebook](https://developers.facebook.com), [BuiltWith](https://api.builtwith.com/domain-api) - -You can also use the `subfinder -ls` command to display all the available sources. - -These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys -can be specified for each of these services from which one of them will be used for enumeration. - -Composite keys for sources like, `Censys`, `PassiveTotal`, `Fofa`, `Intellix` and `360quake`, need to be separated with a colon (`:`). - -An example provider config file: - -```yaml -binaryedge: - - 0bf8919b-aab9-42e4-9574-d3b639324597 - - ac244e2f-b635-4581-878a-33f4e79a2c13 -censys: - - ac244e2f-b635-4581-878a-33f4e79a2c13:dd510d6e-1b6e-4655-83f6-f347b363def9 -certspotter: [] -passivetotal: - - sample-email@user.com:sample_password -redhuntlabs: - - ENDPOINT:API_TOKEN - - https://reconapi.redhuntlabs.com/community/v1/domains/subdomains:joEPzJJp2AuOCw7teAj63HYrPGnsxuPQ -securitytrails: [] -shodan: - - AAAAClP1bJJSRMEYJazgwhJKrggRwKA -github: - - ghp_lkyJGU3jv1xmwk4SDXavrLDJ4dl2pSJMzj4X - - ghp_gkUuhkIYdQPj13ifH4KA3cXRn8JD2lqir2d4 -zoomeyeapi: - - 4f73021d-ff95-4f53-937f-83d6db719eec -quake: - - 0cb9030c-0a40-48a3-b8c4-fca28e466ba3 -facebook: - - APP_ID:APP_SECRET -intelx: - - HOST:API_KEY - - 2.intelx.io:s4324-b98b-41b2-220e8-3320f6a1284d -``` - -Note: RedHunt Labs's [Attack Surface Recon API](https://devportal.redhuntlabs.com/) has different API endpoints depending on the user's subscription. Make sure to add the appropriate endpoint before running any scans. - -# Running Subfinder - -To run the tool on a target, just use the following command. - -```console -subfinder -d hackerone.com - - __ _____ __ - _______ __/ /_ / __(_)___ ____/ /__ _____ - / ___/ / / / __ \/ /_/ / __ \/ __ / _ \/ ___/ - (__ ) /_/ / /_/ / __/ / / / / /_/ / __/ / -/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.4.9 - - projectdiscovery.io - -Use with caution. You are responsible for your actions -Developers assume no liability and are not responsible for any misuse or damage. -By using subfinder, you also agree to the terms of the APIs used. - -[INF] Enumerating subdomains for hackerone.com - -www.hackerone.com -support.hackerone.com -links.hackerone.com -api.hackerone.com -o1.email.hackerone.com -go.hackerone.com -3d.hackerone.com -resources.hackerone.com -a.ns.hackerone.com -b.ns.hackerone.com -mta-sts.hackerone.com -docs.hackerone.com -mta-sts.forwarding.hackerone.com -gslink.hackerone.com -hackerone.com -info.hackerone.com -mta-sts.managed.hackerone.com -events.hackerone.com - -[INF] Found 18 subdomains for hackerone.com in 3 seconds 672 milliseconds -``` - -The subdomains discovered can be piped to other tools too. For example, you can pipe the discovered subdomains to [`httpx`](https://github.com/projectdiscovery/httpx) which will then find -running HTTP servers on the host. - -```console -echo hackerone.com | subfinder -silent | httpx -silent - -http://hackerone.com -http://www.hackerone.com -http://docs.hackerone.com -http://api.hackerone.com -https://docs.hackerone.com -http://mta-sts.managed.hackerone.com -``` +Learn about more ways to install subfinder here: https://docs.projectdiscovery.io/tools/subfinder/install. - - - - -
- -## Subfinder with docker - -Pull the latest tagged [subfinder](https://hub.docker.com/r/projectdiscovery/subfinder) docker image: - -```sh -docker pull projectdiscovery/subfinder:latest -``` - -Running `subfinder` using the docker image: - -```sh -docker run projectdiscovery/subfinder:latest -d hackerone.com -``` - -Running `subfinder` using the docker image, with a local config file: +## Post Installation Instructions -```sh -docker run -v $HOME/.config/subfinder:/root/.config/subfinder -t projectdiscovery/subfinder -d hackerone.com -``` +`subfinder` can be used right after the installation, however many sources required API keys to work. Learn more here: https://docs.projectdiscovery.io/tools/subfinder/install#post-install-configuration. -
+## Running Subfinder - - -
+Learn about how to run Subfinder here: https://docs.projectdiscovery.io/tools/subfinder/running. ## Subfinder Go library diff --git a/v2/go.mod b/v2/go.mod index b6196f11d..64ec10c18 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/subfinder/v2 -go 1.20 +go 1.21 require ( github.com/corpix/uarand v0.2.0 @@ -8,12 +8,12 @@ require ( github.com/json-iterator/go v1.1.12 github.com/lib/pq v1.10.9 github.com/projectdiscovery/chaos-client v0.5.1 - github.com/projectdiscovery/dnsx v1.1.4 + github.com/projectdiscovery/dnsx v1.1.6 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/gologger v1.1.11 - github.com/projectdiscovery/ratelimit v0.0.9 - github.com/projectdiscovery/retryablehttp-go v1.0.26 - github.com/projectdiscovery/utils v0.0.54 + github.com/projectdiscovery/gologger v1.1.12 + github.com/projectdiscovery/ratelimit v0.0.23 + github.com/projectdiscovery/retryablehttp-go v1.0.42 + github.com/projectdiscovery/utils v0.0.72 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.8.4 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -28,16 +28,17 @@ require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/akrylysov/pogreb v0.10.1 // indirect github.com/alecthomas/chroma v0.10.0 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect + github.com/andybalholm/brotli v1.0.6 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/charmbracelet/glamour v0.6.0 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.3.7 // indirect + github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.8.1 // indirect - github.com/dsnet/compress v0.0.1 // indirect + github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/fatih/color v1.15.0 // indirect github.com/gaukas/godicttls v0.0.4 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -46,35 +47,36 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/gorilla/css v1.0.0 // indirect github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/mholt/archiver v3.1.1+incompatible // indirect + github.com/mholt/archiver/v3 v3.5.1 // indirect github.com/microcosm-cc/bluemonday v1.0.25 // indirect github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.1 // indirect github.com/nwaples/rardecode v1.1.3 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/pierrec/lz4/v4 v4.1.2 // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.0.1 // indirect - github.com/projectdiscovery/fastdialer v0.0.37 // indirect - github.com/projectdiscovery/hmap v0.0.18 // indirect + github.com/projectdiscovery/cdncheck v1.0.9 // indirect + github.com/projectdiscovery/fastdialer v0.0.51 // indirect + github.com/projectdiscovery/hmap v0.0.33 // indirect github.com/projectdiscovery/networkpolicy v0.0.6 // indirect - github.com/quic-go/quic-go v0.37.4 // indirect - github.com/refraction-networking/utls v1.5.2 // indirect + github.com/quic-go/quic-go v0.37.7 // indirect + github.com/refraction-networking/utls v1.5.4 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/syndtr/goleveldb v1.0.0 // indirect - github.com/tidwall/btree v1.4.3 // indirect + github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.3.0 // indirect - github.com/tidwall/gjson v1.14.3 // indirect + github.com/tidwall/gjson v1.14.4 // indirect github.com/tidwall/grect v0.1.4 // indirect github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/ulikunitz/xz v0.5.11 // indirect @@ -88,11 +90,11 @@ require ( github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/tools v0.11.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.13.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect @@ -102,13 +104,13 @@ require ( github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect - github.com/miekg/dns v1.1.55 // indirect + github.com/miekg/dns v1.1.56 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/goflags v0.1.20 - github.com/projectdiscovery/retryabledns v1.0.35 // indirect + github.com/projectdiscovery/goflags v0.1.34 + github.com/projectdiscovery/retryabledns v1.0.50 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 56a6532ad..e6d312a2b 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -11,8 +11,9 @@ github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= +github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymanbagabas/go-osc52 v1.0.3/go.mod h1:zT8H+Rk4VSabYN90pWyugflM3ZhpTZNC7cASDfUCdT4= @@ -21,13 +22,15 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bits-and-blooms/bitset v1.8.0 h1:FD+XqgOZDUxxZ8hzoBFuV9+cGWY9CslN6d5MS5JVb4c= +github.com/bits-and-blooms/bitset v1.8.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bits-and-blooms/bloom/v3 v3.5.0 h1:AKDvi1V3xJCmSR6QhcBfHbCN4Vf8FfxeWkMNQfmAGhY= +github.com/bits-and-blooms/bloom/v3 v3.5.0/go.mod h1:Y8vrn7nk1tPIlmLtW2ZPV+W7StdVMor6bC1xgpjMZFs= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= github.com/charmbracelet/glamour v0.6.0/go.mod h1:taqWV4swIMMbWALc0m7AfE9JkPSU8om2538k9ITBxOc= github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= -github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= @@ -36,23 +39,27 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= +github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= -github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= +github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= -github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -61,6 +68,7 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -76,19 +84,25 @@ github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd h1:FsX+T6wA8spPe4c1K9vi7T0LvNCO1TTqiL8u7Wok2hw= github.com/hako/durafmt v0.0.0-20210316092057-3a2c319c1acd/go.mod h1:VzxiSdG6j1pi7rwGm/xYI5RbtpBgM8sARDXlvEvxlu0= github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= +github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -112,13 +126,13 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= -github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= +github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= +github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= -github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= -github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= +github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE= +github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 h1:yRZGarbxsRytL6EGgbqK2mCY+Lk5MWKQYKJT2gEglhc= github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -133,20 +147,25 @@ github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKt github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc= github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= -github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.2 h1:qvY3YFXRQE/XB8MlLzJH7mSzBs74eA2gg52YTk6jUPM= +github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -154,36 +173,36 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.0.1 h1:kv1LRsHJl8TY5zMOhhcpGaBrBAdu01nmMVYqOxGwcMU= -github.com/projectdiscovery/cdncheck v1.0.1/go.mod h1:5l2DFwQNbR3uDYiyUpUQLN0I37+JnE8rSeBpd7WQR/Y= +github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQOocUvrssFlg= +github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/chaos-client v0.5.1 h1:NFxU7cxBY7AL1OKxQQRLRvJrkXlV+SP8zv7VEkVVJ7U= github.com/projectdiscovery/chaos-client v0.5.1/go.mod h1:uU7X/hVEKpOP8OZ8meDAAetMZ4Up0lz9oCcPGe86Yx4= -github.com/projectdiscovery/dnsx v1.1.4 h1:hdvIPU8W1rwNAi6sjjN1rCtCVOhS1/cM3OgmOLFvAm8= -github.com/projectdiscovery/dnsx v1.1.4/go.mod h1:wYz+2E97h0j25FLiqpUJyVY6NW7cEDODyUAsIoXsDsA= -github.com/projectdiscovery/fastdialer v0.0.37 h1:GEn0VYD/Q+KWiUlQDPP5stvIauN8+gE/eooPzrwidic= -github.com/projectdiscovery/fastdialer v0.0.37/go.mod h1:e4Rg9mQ5mNCDFV37njgGCognM0PdLq5f8CcljBqTkRw= +github.com/projectdiscovery/dnsx v1.1.6 h1:QdKVKC0n/fpgaB4q3s6A2wn+qqg75CY0XxNkU9sVjws= +github.com/projectdiscovery/dnsx v1.1.6/go.mod h1:9rkLQzJHxQ26qiD1PhfoJDrhqCVN8lKLsxiAON1uDxM= +github.com/projectdiscovery/fastdialer v0.0.51 h1:LsRry/aSzUfgSCakJve05d6Ut83w1n1NcGS5tyUqsEY= +github.com/projectdiscovery/fastdialer v0.0.51/go.mod h1:OqJbaFL/a6kX7107K6OjZ3usi2MStZ7dQop73DUOUJU= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/goflags v0.1.20 h1:38JgrMOJWg+7G3GUvZJ5cpblBsOFgftNihFusv57A+4= -github.com/projectdiscovery/goflags v0.1.20/go.mod h1:HlJeOmWiMRmjg7PnOMiQkwgLMs88DKgymV1motcxFZw= -github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= -github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= -github.com/projectdiscovery/hmap v0.0.18 h1:L9+55rpXYXdPvTWBlXPsXM2xtivZa+CzRz6z3nfZyX8= -github.com/projectdiscovery/hmap v0.0.18/go.mod h1:kTyoFd6dyhIkBRtaLOqpVZeVLBf94FFhiLFIu+Z0g/8= +github.com/projectdiscovery/goflags v0.1.34 h1:s64GffNF7lsdMz4V4ZS2vfo+x+FxaMB0/TpqZC2q6FI= +github.com/projectdiscovery/goflags v0.1.34/go.mod h1:Br+lowrSpQuQOwDon5w1ZLq9BPJvxZhBUpUTc98UXZs= +github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= +github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= +github.com/projectdiscovery/hmap v0.0.33 h1:kDkw4xVE8uyko6Cv3Cd9MZsHByn9BtXK3y7PeLKVBs4= +github.com/projectdiscovery/hmap v0.0.33/go.mod h1:IlKSbnFKtn68STLiNwc5Kbu4GaR6aIsGaHbpFOYNFGY= github.com/projectdiscovery/networkpolicy v0.0.6 h1:yDvm0XCrS9HeemRrBS+J+22surzVczM94W5nHiOy/1o= github.com/projectdiscovery/networkpolicy v0.0.6/go.mod h1:8HJQ/33Pi7v3a3MRWIQGXzpj+zHw2d60TysEL4qdoQk= -github.com/projectdiscovery/ratelimit v0.0.9 h1:28t2xDHUnyss1irzqPG3Oxz5hkRjl+3Q2I/aes7nau8= -github.com/projectdiscovery/ratelimit v0.0.9/go.mod h1:f98UxLsHt0dWrHTbRDxos4+RvOLE0UFpyECfrfKBz1I= -github.com/projectdiscovery/retryabledns v1.0.35 h1:lPX8f7exDaiNJc/4Rc44xQfFK9BpA8ZLtpQ+te2ymLU= -github.com/projectdiscovery/retryabledns v1.0.35/go.mod h1:V4nRoHJzK2UmlGgKMRduLBkgNNMXJXmJchB5Wui8s4c= -github.com/projectdiscovery/retryablehttp-go v1.0.26 h1:eZYNRRvj7lv05+XrsQU61o1sYTcPwKbmSfiOJfUOArg= -github.com/projectdiscovery/retryablehttp-go v1.0.26/go.mod h1:lCvCUZs1MK5gLi2yUT6Lw/ciVj8Wr2SnNeIJGXxWKHo= -github.com/projectdiscovery/utils v0.0.54 h1:qwTIalrK8pKYaxFObdeSfCtwDmVCN9qswc8+7jIpnBM= -github.com/projectdiscovery/utils v0.0.54/go.mod h1:WhzbWSyGkTDn4Jvw+7jM2yP675/RARegNjoA6S7zYcc= -github.com/quic-go/quic-go v0.37.4 h1:ke8B73yMCWGq9MfrCCAw0Uzdm7GaViC3i39dsIdDlH4= -github.com/quic-go/quic-go v0.37.4/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= -github.com/refraction-networking/utls v1.5.2 h1:l6diiLbEoRqdQ+/osPDO0z0lTc8O8VZV+p82N+Hi+ws= -github.com/refraction-networking/utls v1.5.2/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw= +github.com/projectdiscovery/ratelimit v0.0.23 h1:Fz2A57UW6GK0L0huOGVXd97EhASrJV41SC1NrGImShU= +github.com/projectdiscovery/ratelimit v0.0.23/go.mod h1:042iuvdggjUnsgAIzyxM3iLFveMaXnGTRwlCpfd03I0= +github.com/projectdiscovery/retryabledns v1.0.50 h1:0nM3x29G5LAZ7urfl0jSs501RQ5q57SkPwkdY19ECn8= +github.com/projectdiscovery/retryabledns v1.0.50/go.mod h1:CbQhVC9JjtqU/89gz25gs6UgpQKYwFN2RoWoW5a/M9Q= +github.com/projectdiscovery/retryablehttp-go v1.0.42 h1:NW76U/r0pWNi6iudBqggG69sN8aguuXLLbGRkLvniyo= +github.com/projectdiscovery/retryablehttp-go v1.0.42/go.mod h1:NWR4amTNHwM+ALk1QL1HiyzhFejRTMCHapM+oSoNSv8= +github.com/projectdiscovery/utils v0.0.72 h1:sJ1lBcaWO6dJ65F+fVhSJbguhgWjixgy9mjj7jKBUW8= +github.com/projectdiscovery/utils v0.0.72/go.mod h1:VPnijH51D8wB1VJiEujUp7UZ+TUTKN8PpoW82nivUVY= +github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU= +github.com/quic-go/quic-go v0.37.7/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= +github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= +github.com/refraction-networking/utls v1.5.4/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= @@ -208,27 +227,31 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= -github.com/tidwall/btree v1.4.3 h1:Lf5U/66bk0ftNppOBjVoy/AIPBrLMkheBp4NnSNiYOo= -github.com/tidwall/btree v1.4.3/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= +github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8= +github.com/tidwall/btree v1.6.0 h1:LDZfKfQIBHGHWSwckhXI0RPSXzlo+KYdjK7FWSqOzzg= +github.com/tidwall/btree v1.6.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= github.com/tidwall/buntdb v1.3.0 h1:gdhWO+/YwoB2qZMeAU9JcWWsHSYU3OvcieYgFRS0zwA= github.com/tidwall/buntdb v1.3.0/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= -github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg= github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q= github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8= +github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= -github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= @@ -272,8 +295,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk= golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -304,6 +327,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -322,8 +346,8 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -337,14 +361,14 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= -golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -359,6 +383,7 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 210f892d8..df597e030 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -38,7 +38,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/riddler" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" @@ -80,7 +79,7 @@ var AllSources = [...]subscraping.Source{ &quake.Source{}, &rapiddns.Source{}, &redhuntlabs.Source{}, - &riddler.Source{}, + // &riddler.Source{}, // failing due to cloudfront protection &robtex.Source{}, &securitytrails.Source{}, &shodan.Source{}, diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 506fee732..e7e69214a 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -37,7 +37,7 @@ var ( "quake", "rapiddns", "redhuntlabs", - "riddler", + // "riddler", // failing due to cloudfront protection "robtex", "securitytrails", "shodan", @@ -77,7 +77,7 @@ var ( "quake", "redhuntlabs", "robtex", - "riddler", + // "riddler", // failing due to cloudfront protection "securitytrails", "shodan", "virustotal", @@ -97,6 +97,7 @@ var ( "certspotter", "crtsh", "dnsdumpster", + "dnsdb", "digitorus", "hackertarget", "passivetotal", diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go index de268e7cc..c409fb85c 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/v2/pkg/passive/sources_wo_auth_test.go @@ -25,7 +25,7 @@ func TestSourcesWithoutKeys(t *testing.T) { ignoredSources := []string{ "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ - "riddler", // Fails with 403: There might be too much traffic or a configuration error + "riddler", // failing due to cloudfront protection "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally "waybackarchive", // Fails randomly diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index 7bcb11a10..874d143b1 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.6.3` +const version = `v2.6.4` // showBanner is used to show the banner to the user func showBanner() { diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 1f4a10da1..6c8ed2c6f 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -11,25 +11,6 @@ import ( fileutil "github.com/projectdiscovery/utils/file" ) -// GetConfigDirectory gets the subfinder config directory for a user -func GetConfigDirectory() (string, error) { - var config string - - directory, err := os.UserHomeDir() - if err != nil { - return config, err - } - config = directory + "/.config/subfinder" - - // Create All directory for subfinder even if they exist - err = os.MkdirAll(config, os.ModePerm) - if err != nil { - return config, err - } - - return config, nil -} - // createProviderConfigYAML marshals the input map to the given location on the disk func createProviderConfigYAML(configFilePath string) error { configFile, err := os.Create(configFilePath) diff --git a/v2/pkg/runner/config_test.go b/v2/pkg/runner/config_test.go deleted file mode 100644 index b0c36549d..000000000 --- a/v2/pkg/runner/config_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package runner - -import ( - "github.com/stretchr/testify/require" - "os" - "testing" -) - -func TestConfigGetDirectory(t *testing.T) { - directory, err := GetConfigDirectory() - if err != nil { - t.Fatalf("Expected nil got %v while getting home\n", err) - } - home, err := os.UserHomeDir() - if err != nil { - t.Fatalf("Expected nil got %v while getting dir\n", err) - } - config := home + "/.config/subfinder" - - require.Equal(t, directory, config, "Directory and config should be equal") -} diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 89f1f00e0..63066f9da 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -6,7 +6,6 @@ import ( "io" "math" "os" - "os/user" "path/filepath" "regexp" "strings" @@ -16,13 +15,15 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" fileutil "github.com/projectdiscovery/utils/file" + folderutil "github.com/projectdiscovery/utils/folder" logutil "github.com/projectdiscovery/utils/log" updateutils "github.com/projectdiscovery/utils/update" ) var ( - defaultConfigLocation = filepath.Join(userHomeDir(), ".config/subfinder/config.yaml") - defaultProviderConfigLocation = filepath.Join(userHomeDir(), ".config/subfinder/provider-config.yaml") + configDir = folderutil.AppConfigDirOrDefault(".", "subfinder") + defaultConfigLocation = filepath.Join(configDir, "config.yaml") + defaultProviderConfigLocation = filepath.Join(configDir, "provider-config.yaml") ) // Options contains the configuration options for tuning @@ -169,6 +170,7 @@ func ParseOptions() *Options { if options.Version { gologger.Info().Msgf("Current Version: %s\n", version) + gologger.Info().Msgf("Subfinder Config Directory: %s", configDir) os.Exit(0) } @@ -239,14 +241,6 @@ func (options *Options) preProcessOptions() { } } -func userHomeDir() string { - usr, err := user.Current() - if err != nil { - gologger.Fatal().Msgf("Could not get user home directory: %s\n", err) - } - return usr.HomeDir -} - var defaultRateLimits = []string{ "github=30/m", // "gitlab=2000/m", @@ -259,4 +253,5 @@ var defaultRateLimits = []string{ // "threatminer=10/m", "waybackarchive=15/m", "whoisxmlapi=50/s", + "securitytrails=2/s", } diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index d9a6fe28e..83887ea94 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -8,11 +8,13 @@ import ( "os" "path" "regexp" + "strconv" "strings" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" + contextutil "github.com/projectdiscovery/utils/context" fileutil "github.com/projectdiscovery/utils/file" mapsutil "github.com/projectdiscovery/utils/maps" @@ -73,7 +75,8 @@ func NewRunner(options *Options) (*Runner, error) { // RunEnumeration wraps RunEnumerationWithCtx with an empty context func (r *Runner) RunEnumeration() error { - return r.RunEnumerationWithCtx(context.Background()) + ctx, _ := contextutil.WithValues(context.Background(), contextutil.ContextArg("All"), contextutil.ContextArg(strconv.FormatBool(r.options.All))) + return r.RunEnumerationWithCtx(ctx) } // RunEnumerationWithCtx runs the subdomain enumeration flow on the targets specified @@ -105,7 +108,8 @@ func (r *Runner) RunEnumerationWithCtx(ctx context.Context) error { // EnumerateMultipleDomains wraps EnumerateMultipleDomainsWithCtx with an empty context func (r *Runner) EnumerateMultipleDomains(reader io.Reader, writers []io.Writer) error { - return r.EnumerateMultipleDomainsWithCtx(context.Background(), reader, writers) + ctx, _ := contextutil.WithValues(context.Background(), contextutil.ContextArg("All"), contextutil.ContextArg(strconv.FormatBool(r.options.All))) + return r.EnumerateMultipleDomainsWithCtx(ctx, reader, writers) } // EnumerateMultipleDomainsWithCtx enumerates subdomains for multiple domains @@ -114,7 +118,7 @@ func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io. scanner := bufio.NewScanner(reader) ip, _ := regexp.Compile(`^([0-9\.]+$)`) for scanner.Scan() { - domain, err := sanitize(scanner.Text()) + domain, err := normalizeLowercase(scanner.Text()) isIp := ip.MatchString(domain) if errors.Is(err, ErrEmptyInput) || (r.options.ExcludeIps && isIp) { continue diff --git a/v2/pkg/runner/util.go b/v2/pkg/runner/util.go index 2507e8972..cc883cc82 100644 --- a/v2/pkg/runner/util.go +++ b/v2/pkg/runner/util.go @@ -5,7 +5,7 @@ import ( "github.com/pkg/errors" - "github.com/projectdiscovery/utils/file" + fileutil "github.com/projectdiscovery/utils/file" ) var ( @@ -30,9 +30,14 @@ func loadFromFile(file string) ([]string, error) { } func sanitize(data string) (string, error) { - data = strings.Trim(data, "\n\t\"' ") + data = strings.Trim(data, "\n\t\"'` ") if data == "" { return "", ErrEmptyInput } return data, nil } + +func normalizeLowercase(s string) (string, error) { + data, err := sanitize(s) + return strings.ToLower(data), err +} diff --git a/v2/pkg/subscraping/extractor.go b/v2/pkg/subscraping/extractor.go new file mode 100644 index 000000000..1a2921f9b --- /dev/null +++ b/v2/pkg/subscraping/extractor.go @@ -0,0 +1,30 @@ +package subscraping + +import ( + "regexp" + "strings" +) + +// RegexSubdomainExtractor is a concrete implementation of the SubdomainExtractor interface, using regex for extraction. +type RegexSubdomainExtractor struct { + extractor *regexp.Regexp +} + +// NewSubdomainExtractor creates a new regular expression to extract +// subdomains from text based on the given domain. +func NewSubdomainExtractor(domain string) (*RegexSubdomainExtractor, error) { + extractor, err := regexp.Compile(`(?i)[a-zA-Z0-9\*_.-]+\.` + domain) + if err != nil { + return nil, err + } + return &RegexSubdomainExtractor{extractor: extractor}, nil +} + +// Extract implements the SubdomainExtractor interface, using the regex to find subdomains in the given text. +func (re *RegexSubdomainExtractor) Extract(text string) []string { + matches := re.extractor.FindAllString(text, -1) + for i, match := range matches { + matches[i] = strings.ToLower(match) + } + return matches +} diff --git a/v2/pkg/subscraping/sources/binaryedge/binaryedge.go b/v2/pkg/subscraping/sources/binaryedge/binaryedge.go index 068d8f374..cc6b569a8 100644 --- a/v2/pkg/subscraping/sources/binaryedge/binaryedge.go +++ b/v2/pkg/subscraping/sources/binaryedge/binaryedge.go @@ -83,7 +83,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("can't get API URL"), } - s.results++ + s.errors++ return } @@ -101,7 +101,7 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba } resp, err := session.Get(ctx, pageURL.String(), "", authHeader) - if err != nil && resp == nil { + if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) @@ -120,19 +120,21 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba // Check error messages if response.Message != "" && response.Status != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf(response.Message)} - s.results++ + s.errors++ + return } resp.Body.Close() for _, subdomain := range response.Subdomains { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ } totalPages := int(math.Ceil(float64(response.Total) / float64(response.PageSize))) nextPage := response.Page + 1 - for currentPage := nextPage; currentPage <= totalPages; currentPage++ { - s.enumerate(ctx, session, baseURL, currentPage, authHeader, results) + if nextPage <= totalPages { + s.enumerate(ctx, session, baseURL, nextPage, authHeader, results) } } diff --git a/v2/pkg/subscraping/sources/bufferover/bufferover.go b/v2/pkg/subscraping/sources/bufferover/bufferover.go index f7be4834e..cf2771436 100644 --- a/v2/pkg/subscraping/sources/bufferover/bufferover.go +++ b/v2/pkg/subscraping/sources/bufferover/bufferover.go @@ -95,7 +95,7 @@ func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, s } for _, subdomain := range subdomains { - for _, value := range session.Extractor.FindAllString(subdomain, -1) { + for _, value := range session.Extractor.Extract(subdomain) { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} } s.results++ diff --git a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go index 9557e3c31..de46bbb7f 100644 --- a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ b/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -142,14 +142,15 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se continue } line, _ = url.QueryUnescape(line) - subdomain := session.Extractor.FindString(line) - if subdomain != "" { - // fix for triple encoded URL - subdomain = strings.ToLower(subdomain) - subdomain = strings.TrimPrefix(subdomain, "25") - subdomain = strings.TrimPrefix(subdomain, "2f") - - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + for _, subdomain := range session.Extractor.Extract(line) { + if subdomain != "" { + // fix for triple encoded URL + subdomain = strings.ToLower(subdomain) + subdomain = strings.TrimPrefix(subdomain, "25") + subdomain = strings.TrimPrefix(subdomain, "2f") + + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } } } resp.Body.Close() diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/v2/pkg/subscraping/sources/crtsh/crtsh.go index 22ba3a276..d01a6e99f 100644 --- a/v2/pkg/subscraping/sources/crtsh/crtsh.go +++ b/v2/pkg/subscraping/sources/crtsh/crtsh.go @@ -5,6 +5,7 @@ import ( "context" "database/sql" "fmt" + "strconv" "strings" "time" @@ -14,6 +15,7 @@ import ( _ "github.com/lib/pq" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + contextutil "github.com/projectdiscovery/utils/context" ) type subdomain struct { @@ -40,7 +42,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) - count := s.getSubdomainsFromSQL(domain, session, results) + count := s.getSubdomainsFromSQL(ctx, domain, session, results) if count > 0 { return } @@ -50,7 +52,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return results } -func (s *Source) getSubdomainsFromSQL(domain string, session *subscraping.Session, results chan subscraping.Result) int { +func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) int { db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes") if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -60,7 +62,14 @@ func (s *Source) getSubdomainsFromSQL(domain string, session *subscraping.Sessio defer db.Close() - query := `WITH ci AS ( + limitClause := "" + if all, ok := ctx.Value(contextutil.ContextArg("All")).(contextutil.ContextArg); ok { + if allBool, err := strconv.ParseBool(string(all)); err == nil && !allBool { + limitClause = "LIMIT 10000" + } + } + + query := fmt.Sprintf(`WITH ci AS ( SELECT min(sub.CERTIFICATE_ID) ID, min(sub.ISSUER_CA_ID) ISSUER_CA_ID, array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES, @@ -71,8 +80,8 @@ func (s *Source) getSubdomainsFromSQL(domain string, session *subscraping.Sessio FROM (SELECT * FROM certificate_and_identities cai WHERE plainto_tsquery('certwatch', $1) @@ identities(cai.CERTIFICATE) - AND cai.NAME_VALUE ILIKE ('%' || $1 || '%') - LIMIT 10000 + AND cai.NAME_VALUE ILIKE ('%%' || $1 || '%%') + %s ) sub GROUP BY sub.CERTIFICATE ) @@ -85,7 +94,7 @@ func (s *Source) getSubdomainsFromSQL(domain string, session *subscraping.Sessio ) le ON TRUE, ca WHERE ci.ISSUER_CA_ID = ca.ID - ORDER BY le.ENTRY_TIMESTAMP DESC NULLS LAST;` + ORDER BY le.ENTRY_TIMESTAMP DESC NULLS LAST;`, limitClause) rows, err := db.Query(query, domain) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -111,10 +120,11 @@ func (s *Source) getSubdomainsFromSQL(domain string, session *subscraping.Sessio count++ for _, subdomain := range strings.Split(data, "\n") { - value := session.Extractor.FindString(subdomain) - if value != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} - s.results++ + for _, value := range session.Extractor.Extract(subdomain) { + if value != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} + s.results++ + } } } } @@ -143,11 +153,12 @@ func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, sessi for _, subdomain := range subdomains { for _, sub := range strings.Split(subdomain.NameValue, "\n") { - value := session.Extractor.FindString(sub) - if value != "" { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} + for _, value := range session.Extractor.Extract(sub) { + if value != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} + } + s.results++ } - s.results++ } } diff --git a/v2/pkg/subscraping/sources/digitorus/digitorus.go b/v2/pkg/subscraping/sources/digitorus/digitorus.go index e8a6b141e..6cf61dbcf 100644 --- a/v2/pkg/subscraping/sources/digitorus/digitorus.go +++ b/v2/pkg/subscraping/sources/digitorus/digitorus.go @@ -49,7 +49,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if line == "" { continue } - subdomains := session.Extractor.FindAllString(line, -1) + subdomains := session.Extractor.Extract(line) for _, subdomain := range subdomains { results <- subscraping.Result{ Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimPrefix(subdomain, "."), diff --git a/v2/pkg/subscraping/sources/dnsdb/dnsdb.go b/v2/pkg/subscraping/sources/dnsdb/dnsdb.go index 0138263a3..2b18b8568 100644 --- a/v2/pkg/subscraping/sources/dnsdb/dnsdb.go +++ b/v2/pkg/subscraping/sources/dnsdb/dnsdb.go @@ -3,9 +3,12 @@ package dnsdb import ( "bufio" - "bytes" "context" + "encoding/json" "fmt" + "io" + "net/url" + "strconv" "strings" "time" @@ -14,7 +17,23 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) -type dnsdbResponse struct { +const urlBase string = "https://api.dnsdb.info/dnsdb/v2" + +type rateResponse struct { + Rate rate +} + +type rate struct { + OffsetMax json.Number `json:"offset_max"` +} + +type safResponse struct { + Condition string `json:"cond"` + Obj dnsdbObj `json:"obj"` + Msg string `json:"msg"` +} + +type dnsdbObj struct { Name string `json:"rrname"` } @@ -23,7 +42,7 @@ type Source struct { apiKeys []string timeTaken time.Duration errors int - results int + results uint64 skipped bool } @@ -39,44 +58,105 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) - randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + sourceName := s.Name() + + randomApiKey := subscraping.PickRandom(s.apiKeys, sourceName) if randomApiKey == "" { return } headers := map[string]string{ - "X-API-KEY": randomApiKey, - "Accept": "application/json", - "Content-Type": "application/json", + "X-API-KEY": randomApiKey, + "Accept": "application/x-ndjson", } - resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdb.info/lookup/rrset/name/*.%s?limit=1000000000000", domain), "", headers) + offsetMax, err := getMaxOffset(ctx, session, headers) if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ - session.DiscardHTTPResponse(resp) return } - scanner := bufio.NewScanner(resp.Body) - for scanner.Scan() { - line := scanner.Text() - if line == "" { - continue - } - var response dnsdbResponse - err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response) + path := fmt.Sprintf("lookup/rrset/name/*.%s", domain) + urlTemplate := fmt.Sprintf("%s/%s?", urlBase, path) + queryParams := url.Values{} + // ?limit=0 means DNSDB will return the maximum number of results allowed. + queryParams.Add("limit", "0") + queryParams.Add("swclient", "subfinder") + + for { + url := urlTemplate + queryParams.Encode() + + resp, err := session.Get(ctx, url, "", headers) if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ + session.DiscardHTTPResponse(resp) return } - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Name, "."), + + var respCond string + reader := bufio.NewReader(resp.Body) + for { + n, err := reader.ReadBytes('\n') + if err == io.EOF { + break + } else if err != nil { + results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} + s.errors++ + resp.Body.Close() + return + } + + var response safResponse + err = jsoniter.Unmarshal(n, &response) + if err != nil { + results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} + s.errors++ + resp.Body.Close() + return + } + + // Condition is a scalar enum of string values: {“begin”, “ongoing”, “succeeded”, “limited”, “failed”}. + // "begin" will be the initiating Condition, this can be safely ignored. The data of interest will be in + // objects with Condition "" or "ongoing". Conditions "succeeded", "limited", and "failed" are terminating conditions. + // See https://www.domaintools.com/resources/user-guides/farsight-streaming-api-framing-protocol-documentation/ + // for more details + respCond = response.Condition + if respCond == "" || respCond == "ongoing" { + if response.Obj.Name != "" { + results <- subscraping.Result{ + Source: sourceName, Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Obj.Name, "."), + } + s.results++ + } + } else if respCond != "begin" { + // if the respCond is not "", "ongoing", or "begin", then it is a terminating condition, so break out of the loop + break + } } - s.results++ + + // Check the terminating jsonl object's condition. There are 3 possible scenarios: + // 1. "limited" - There are more results available, make another query with an offset + // 2. "succeeded" - The query completed successfully and all results were sent. + // 3. anything else - This is an error and should be reported to the user. The user can then decide to use the results up to this + // point or discard and retry. + if respCond == "limited" { + if offsetMax != 0 && s.results <= offsetMax { + // Reset done to false to get more results with an offset query parameter set to s.results + queryParams.Set("offset", strconv.FormatUint(s.results, 10)) + continue + } + } else if respCond != "succeeded" { + // DNSDB's terminating jsonl object's cond is not "limited" or succeeded" (#3), this is an error, notify the user. + err = fmt.Errorf("%s terminated with condition: %s", sourceName, respCond) + results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} + s.errors++ + } + + resp.Body.Close() + break } - resp.Body.Close() }() return results @@ -92,7 +172,7 @@ func (s *Source) IsDefault() bool { } func (s *Source) HasRecursiveSupport() bool { - return false + return true } func (s *Source) NeedsKey() bool { @@ -106,8 +186,36 @@ func (s *Source) AddApiKeys(keys []string) { func (s *Source) Statistics() subscraping.Statistics { return subscraping.Statistics{ Errors: s.errors, - Results: s.results, + Results: int(s.results), TimeTaken: s.timeTaken, Skipped: s.skipped, } } + +func getMaxOffset(ctx context.Context, session *subscraping.Session, headers map[string]string) (uint64, error) { + var offsetMax uint64 + url := fmt.Sprintf("%s/rate_limit", urlBase) + resp, err := session.Get(ctx, url, "", headers) + defer session.DiscardHTTPResponse(resp) + if err != nil { + return offsetMax, err + } + data, err := io.ReadAll(resp.Body) + if err != nil { + return offsetMax, err + } + var rateResp rateResponse + err = jsoniter.Unmarshal(data, &rateResp) + if err != nil { + return offsetMax, err + } + // if the OffsetMax is "n/a" then the ?offset= query parameter is not allowed + if rateResp.Rate.OffsetMax.String() != "n/a" { + offsetMax, err = strconv.ParseUint(rateResp.Rate.OffsetMax.String(), 10, 64) + if err != nil { + return offsetMax, err + } + } + + return offsetMax, nil +} diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 8fa64a33a..8e6afbd4a 100644 --- a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -103,7 +103,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - for _, subdomain := range session.Extractor.FindAllString(data, -1) { + for _, subdomain := range session.Extractor.Extract(data) { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } diff --git a/v2/pkg/subscraping/sources/fofa/fofa.go b/v2/pkg/subscraping/sources/fofa/fofa.go index 64caafdd9..9a5294c75 100644 --- a/v2/pkg/subscraping/sources/fofa/fofa.go +++ b/v2/pkg/subscraping/sources/fofa/fofa.go @@ -5,6 +5,7 @@ import ( "context" "encoding/base64" "fmt" + "regexp" "strings" "time" @@ -85,6 +86,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") { subdomain = subdomain[strings.Index(subdomain, "//")+2:] } + re := regexp.MustCompile(`:\d+$`) + if re.MatchString(subdomain) { + subdomain = re.ReplaceAllString(subdomain, "") + } results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } diff --git a/v2/pkg/subscraping/sources/hackertarget/hackertarget.go b/v2/pkg/subscraping/sources/hackertarget/hackertarget.go index d569aca34..de20cbbf0 100644 --- a/v2/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/v2/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -45,7 +45,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if line == "" { continue } - match := session.Extractor.FindAllString(line, -1) + match := session.Extractor.Extract(line) for _, subdomain := range match { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/v2/pkg/subscraping/sources/quake/quake.go index 7aaa74895..075d35e13 100644 --- a/v2/pkg/subscraping/sources/quake/quake.go +++ b/v2/pkg/subscraping/sources/quake/quake.go @@ -58,7 +58,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } // quake api doc https://quake.360.cn/quake/#/help - var requestBody = []byte(fmt.Sprintf(`{"query":"domain: *.%s", "include":["service.http.host"], "latest": true, "start":0, "size":500}`, domain)) + var requestBody = []byte(fmt.Sprintf(`{"query":"domain: %s", "include":["service.http.host"], "latest": true, "start":0, "size":500}`, domain)) resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ "Content-Type": "application/json", "X-QuakeToken": randomApiKey, }, bytes.NewReader(requestBody)) diff --git a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go b/v2/pkg/subscraping/sources/rapiddns/rapiddns.go index 58c97b324..383ebd998 100644 --- a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go +++ b/v2/pkg/subscraping/sources/rapiddns/rapiddns.go @@ -47,7 +47,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se resp.Body.Close() src := string(body) - for _, subdomain := range session.Extractor.FindAllString(src, -1) { + for _, subdomain := range session.Extractor.Extract(src) { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } diff --git a/v2/pkg/subscraping/sources/riddler/riddler.go b/v2/pkg/subscraping/sources/riddler/riddler.go index 278baf868..0afe725eb 100644 --- a/v2/pkg/subscraping/sources/riddler/riddler.go +++ b/v2/pkg/subscraping/sources/riddler/riddler.go @@ -43,8 +43,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if line == "" { continue } - subdomain := session.Extractor.FindString(line) - if subdomain != "" { + for _, subdomain := range session.Extractor.Extract(line) { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } diff --git a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go index c8c6ca88c..3ed732989 100644 --- a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -2,8 +2,10 @@ package securitytrails import ( + "bytes" "context" "fmt" + "net/http" "strings" "time" @@ -13,6 +15,12 @@ import ( ) type response struct { + Meta struct { + ScrollID string `json:"scroll_id"` + } `json:"meta"` + Records []struct { + Hostname string `json:"hostname"` + } `json:"records"` Subdomains []string `json:"subdomains"` } @@ -43,34 +51,63 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - resp, err := session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", map[string]string{"APIKEY": randomApiKey}) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } + var scrollId string + headers := map[string]string{"Content-Type": "application/json", "APIKEY": randomApiKey} + + for { + var resp *http.Response + var err error + + if scrollId == "" { + var requestBody = []byte(fmt.Sprintf(`{"query":"apex_domain='%s'"}`, domain)) + resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "", + headers, bytes.NewReader(requestBody)) + } else { + resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/scroll/%s", scrollId), "", headers) + } + + if err != nil && resp.StatusCode == 403 { + resp, err = session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", headers) + } + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var securityTrailsResponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + resp.Body.Close() + return + } - var securityTrailsResponse response - err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ resp.Body.Close() - return - } - resp.Body.Close() + for _, record := range securityTrailsResponse.Records { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} + s.results++ + } - for _, subdomain := range securityTrailsResponse.Subdomains { - if strings.HasSuffix(subdomain, ".") { - subdomain += domain - } else { - subdomain = subdomain + "." + domain + for _, subdomain := range securityTrailsResponse.Subdomains { + if strings.HasSuffix(subdomain, ".") { + subdomain += domain + } else { + subdomain = subdomain + "." + domain + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ } - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + scrollId = securityTrailsResponse.Meta.ScrollID + + if scrollId == "" { + break + } } }() diff --git a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go index 6db725f53..c44ad1c80 100644 --- a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go +++ b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -51,7 +51,7 @@ func (a *agent) enumerate(ctx context.Context, baseURL string) { resp.Body.Close() src := string(body) - for _, match := range a.session.Extractor.FindAllString(src, -1) { + for _, match := range a.session.Extractor.Extract(src) { a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: match} } diff --git a/v2/pkg/subscraping/sources/virustotal/virustotal.go b/v2/pkg/subscraping/sources/virustotal/virustotal.go index 82b9c81bf..f996a1628 100644 --- a/v2/pkg/subscraping/sources/virustotal/virustotal.go +++ b/v2/pkg/subscraping/sources/virustotal/virustotal.go @@ -49,10 +49,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if randomApiKey == "" { return } - var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=1000", domain) - var hasMore bool = true var cursor string = "" - for hasMore { + for { + var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=1000", domain) if cursor != "" { url = fmt.Sprintf("%s&cursor=%s", url, cursor) } @@ -63,25 +62,23 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } + defer resp.Body.Close() var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() return } - resp.Body.Close() - for _, subdomain := range data.Data { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Id} s.results++ } cursor = data.Meta.Cursor if cursor == "" { - hasMore = false + break } } }() diff --git a/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go index 077b89a0d..5edc78382 100644 --- a/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go +++ b/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go @@ -48,8 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se continue } line, _ = url.QueryUnescape(line) - subdomain := session.Extractor.FindString(line) - if subdomain != "" { + for _, subdomain := range session.Extractor.Extract(line) { // fix for triple encoded URL subdomain = strings.ToLower(subdomain) subdomain = strings.TrimPrefix(subdomain, "25") diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 3d54473b4..454ad776f 100644 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -3,7 +3,6 @@ package subscraping import ( "context" "net/http" - "regexp" "time" "github.com/projectdiscovery/ratelimit" @@ -62,11 +61,16 @@ type Source interface { Statistics() Statistics } +// SubdomainExtractor is an interface that defines the contract for subdomain extraction. +type SubdomainExtractor interface { + Extract(text string) []string +} + // Session is the option passed to the source, an option is created // uniquely for each source. type Session struct { - // Extractor is the regex for subdomains created for each domain - Extractor *regexp.Regexp + //SubdomainExtractor + Extractor SubdomainExtractor // Client is the current http client Client *http.Client // Rate limit instance diff --git a/v2/pkg/subscraping/utils.go b/v2/pkg/subscraping/utils.go index cd119a8c6..218f62fc8 100644 --- a/v2/pkg/subscraping/utils.go +++ b/v2/pkg/subscraping/utils.go @@ -2,7 +2,6 @@ package subscraping import ( "math/rand" - "regexp" "strings" "github.com/projectdiscovery/gologger" @@ -10,16 +9,6 @@ import ( const MultipleKeyPartsLength = 2 -// NewSubdomainExtractor creates a new regular expression to extract -// subdomains from text based on the given domain. -func NewSubdomainExtractor(domain string) (*regexp.Regexp, error) { - extractor, err := regexp.Compile(`(?i)[a-zA-Z0-9\*_.-]+\.` + domain) - if err != nil { - return nil, err - } - return extractor, nil -} - func PickRandom[T any](v []T, sourceName string) T { var result T length := len(v)