diff --git a/Dockerfile b/Dockerfile index 3d5f71f54..eadf4f7e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.20.6-alpine AS build-env +FROM golang:1.21-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app @@ -8,9 +8,9 @@ RUN go mod download RUN go build ./cmd/subfinder # Release -FROM alpine:3.18.2 +FROM alpine:3.18.6 RUN apk upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=build-env /app/v2/subfinder /usr/local/bin/ -ENTRYPOINT ["subfinder"] \ No newline at end of file +ENTRYPOINT ["subfinder"] diff --git a/README.md b/README.md index 749098226..dc225e588 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ OPTIMIZATION: # Installation -`subfinder` requires **go1.20** to install successfully. Run the following command to install the latest version: +`subfinder` requires **go1.21** to install successfully. Run the following command to install the latest version: ```sh go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest diff --git a/v2/go.mod b/v2/go.mod index 4b1a8fd9c..3bc9734ca 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -8,14 +8,14 @@ require ( github.com/json-iterator/go v1.1.12 github.com/lib/pq v1.10.9 github.com/projectdiscovery/chaos-client v0.5.1 - github.com/projectdiscovery/dnsx v1.1.6 + github.com/projectdiscovery/dnsx v1.2.1 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.12 - github.com/projectdiscovery/ratelimit v0.0.26 - github.com/projectdiscovery/retryablehttp-go v1.0.44 - github.com/projectdiscovery/utils v0.0.76 + github.com/projectdiscovery/ratelimit v0.0.34 + github.com/projectdiscovery/retryablehttp-go v1.0.53 + github.com/projectdiscovery/utils v0.0.84 github.com/rs/xid v1.5.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 golang.org/x/exp v0.0.0-20230420155640-133eef4313cb gopkg.in/yaml.v3 v3.0.1 @@ -36,13 +36,13 @@ require ( github.com/charmbracelet/glamour v0.6.0 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/denisbrodbeck/machineid v1.0.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.8.1 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect github.com/fatih/color v1.15.0 // indirect github.com/gaukas/godicttls v0.0.4 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect @@ -52,6 +52,7 @@ require ( github.com/klauspost/pgzip v1.2.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect @@ -63,15 +64,19 @@ require ( github.com/nwaples/rardecode v1.1.3 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pierrec/lz4/v4 v4.1.2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/cdncheck v1.0.9 // indirect - github.com/projectdiscovery/fastdialer v0.0.56 // indirect - github.com/projectdiscovery/hmap v0.0.36 // indirect - github.com/projectdiscovery/networkpolicy v0.0.7 // indirect + github.com/projectdiscovery/fastdialer v0.0.64 // indirect + github.com/projectdiscovery/hmap v0.0.41 // indirect + github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect + github.com/projectdiscovery/networkpolicy v0.0.8 // indirect github.com/quic-go/quic-go v0.37.7 // indirect github.com/refraction-networking/utls v1.5.4 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect + github.com/shirou/gopsutil/v3 v3.23.7 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/tidwall/btree v1.6.0 // indirect github.com/tidwall/buntdb v1.3.0 // indirect @@ -81,13 +86,15 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/rtred v0.1.2 // indirect github.com/tidwall/tinyqueue v0.1.1 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/ulikunitz/xz v0.5.11 // indirect - github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yl2chen/cidranger v1.0.2 // indirect github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect github.com/zmap/zcrypto v0.0.0-20230422215203-9a665e1e9968 // indirect go.etcd.io/bbolt v1.3.7 // indirect @@ -95,10 +102,11 @@ require ( golang.org/x/crypto v0.17.0 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/sync v0.6.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.13.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect ) @@ -111,8 +119,8 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/goflags v0.1.34 - github.com/projectdiscovery/retryabledns v1.0.53 // indirect + github.com/projectdiscovery/goflags v0.1.42 + github.com/projectdiscovery/retryabledns v1.0.58 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.16.0 // indirect + golang.org/x/sys v0.17.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 67833e95c..a846e2020 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -41,8 +41,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ= -github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= @@ -62,6 +60,8 @@ github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXb github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -77,6 +77,7 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -121,6 +122,8 @@ github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczG github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -175,34 +178,38 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.0.9 h1:BS15gzj9gb5AVSKqTDzPamfSgStu7nJQOocUvrssFlg= github.com/projectdiscovery/cdncheck v1.0.9/go.mod h1:18SSl1w7rMj53CGeRIZTbDoa286a6xZIxGbaiEo4Fxs= github.com/projectdiscovery/chaos-client v0.5.1 h1:NFxU7cxBY7AL1OKxQQRLRvJrkXlV+SP8zv7VEkVVJ7U= github.com/projectdiscovery/chaos-client v0.5.1/go.mod h1:uU7X/hVEKpOP8OZ8meDAAetMZ4Up0lz9oCcPGe86Yx4= -github.com/projectdiscovery/dnsx v1.1.6 h1:QdKVKC0n/fpgaB4q3s6A2wn+qqg75CY0XxNkU9sVjws= -github.com/projectdiscovery/dnsx v1.1.6/go.mod h1:9rkLQzJHxQ26qiD1PhfoJDrhqCVN8lKLsxiAON1uDxM= -github.com/projectdiscovery/fastdialer v0.0.56 h1:SUw193mzYaDFtBpQOs4+EHa6r1NnBYj3xwSqK1j9Chk= -github.com/projectdiscovery/fastdialer v0.0.56/go.mod h1:o0Oeb78Bn6mpr+dTLV4Jb7hnEFQjjA7lUeDe8Ga6zq4= +github.com/projectdiscovery/dnsx v1.2.1 h1:TxslYvp1Z/YZ4CP/J0gx5RYpvXREnVmyoacmTcGu5yg= +github.com/projectdiscovery/dnsx v1.2.1/go.mod h1:6dAsMCEDu7FArZy2qjyTeUQrqpZ4ITLU11fcmUvFqt0= +github.com/projectdiscovery/fastdialer v0.0.64 h1:xivkA4g14nwQElOVsxPkGMWsdcYPcp7DPhVjvI6yQkw= +github.com/projectdiscovery/fastdialer v0.0.64/go.mod h1:S/7PAQRmVDYRaU7u4xXD0qA5a48NAZq2JcpcVoEVrlo= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= -github.com/projectdiscovery/goflags v0.1.34 h1:s64GffNF7lsdMz4V4ZS2vfo+x+FxaMB0/TpqZC2q6FI= -github.com/projectdiscovery/goflags v0.1.34/go.mod h1:Br+lowrSpQuQOwDon5w1ZLq9BPJvxZhBUpUTc98UXZs= +github.com/projectdiscovery/goflags v0.1.42 h1:C3CleYUODv5Jdn4+FTCUpzm3eOXHl+GiLHbXSw5iNsQ= +github.com/projectdiscovery/goflags v0.1.42/go.mod h1:UuS8nOpeYbVebPVWeTvcpILESC8daIwRVJ14UEt7L7c= github.com/projectdiscovery/gologger v1.1.12 h1:uX/QkQdip4PubJjjG0+uk5DtyAi1ANPJUvpmimXqv4A= github.com/projectdiscovery/gologger v1.1.12/go.mod h1:DI8nywPLERS5mo8QEA9E7gd5HZ3Je14SjJBH3F5/kLw= -github.com/projectdiscovery/hmap v0.0.36 h1:hnaiw+NZKQzoP0TXGjf5QmAtMgmrR0LHtR4xINJXWr0= -github.com/projectdiscovery/hmap v0.0.36/go.mod h1:q6UNS9PoJqZHq5s3BjFTKdYRTZvmFr1JvhMKjNGdGgo= -github.com/projectdiscovery/networkpolicy v0.0.7 h1:AwHqBRXBqDQgnWzBMuoJtHBNEYBw+NFp/4qIK688x7o= -github.com/projectdiscovery/networkpolicy v0.0.7/go.mod h1:CK0CnFoLF1Nou6mY7P4WODSAxhPN8g8g7XpapgEP8tI= -github.com/projectdiscovery/ratelimit v0.0.26 h1:sxZCh72lMpQ1YNnJOWrJ+uZE9GFWdVE58LOArOc6c+4= -github.com/projectdiscovery/ratelimit v0.0.26/go.mod h1:2NHqfqqb9xAnqW+Ztd8AzzNi+JP38Kcdhb8cnbfX9sI= -github.com/projectdiscovery/retryabledns v1.0.53 h1:eX4f7Afi2INmEaDj8F5x5T/VfEj62Q5qS+z3kcRuMBk= -github.com/projectdiscovery/retryabledns v1.0.53/go.mod h1:FbjRnVnTkyGujjnQGDVzMomIYA4apN+AR6VrlJ8toHk= -github.com/projectdiscovery/retryablehttp-go v1.0.44 h1:hicCe2h6daHt4muPovmffZE3YKBqGioreO6EpIGZ87g= -github.com/projectdiscovery/retryablehttp-go v1.0.44/go.mod h1:7ECXK2cH2/G4sstf8hacyrMdPPJ/3wCAO5tFPZ4iO4s= -github.com/projectdiscovery/utils v0.0.76 h1:6azn0Zju0taw5Y9qAjpGPxyqwJf2AI4VJjtIzPBcRzQ= -github.com/projectdiscovery/utils v0.0.76/go.mod h1:ERIYcW+h5jKIYyYkfdOpNPIUtH8Ogz4q5Wq3gx/71Zw= +github.com/projectdiscovery/hmap v0.0.41 h1:8IgTyDce3/2JzcfPVA4H+XpBRFfETULx8td3BMdSYVE= +github.com/projectdiscovery/hmap v0.0.41/go.mod h1:bCrai6x5Eijqm2U+jtcH0wZX5ZcaZhcvzoMGTZgLAf0= +github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= +github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= +github.com/projectdiscovery/networkpolicy v0.0.8 h1:XvfBaBwSDNTesSfNQP9VLk3HX9I7x7gHm028TJ5XwI8= +github.com/projectdiscovery/networkpolicy v0.0.8/go.mod h1:xnjNqhemxUPxU+UD5Jgsc3+K8IVmcqT1SJeo6UzMtkI= +github.com/projectdiscovery/ratelimit v0.0.34 h1:Hq+S4fyY23YH5RaKWZj/xlT8X5mAvrrbmbpydsKjV4M= +github.com/projectdiscovery/ratelimit v0.0.34/go.mod h1:9DpFhtNHMz7j9rhA8s9Jbq+J8KCuoDxN1SzGBY7oIbk= +github.com/projectdiscovery/retryabledns v1.0.58 h1:ut1FSB9+GZ6zQIlKJFLqIz2RZs81EmkbsHTuIrWfYLE= +github.com/projectdiscovery/retryabledns v1.0.58/go.mod h1:RobmKoNBgngAVE4H9REQtaLP1pa4TCyypHy1MWHT1mY= +github.com/projectdiscovery/retryablehttp-go v1.0.53 h1:Kp4590eOwFhUH9oIaCFMtudLSqL1BX+VNurS42+SC8Q= +github.com/projectdiscovery/retryablehttp-go v1.0.53/go.mod h1:b5zNLih0PcfvrDzTRY+QCB1dhFpODQQ2NjI4TtrmRyY= +github.com/projectdiscovery/utils v0.0.84 h1:1IcaRVOrxow7Qev57OF1CSCj5QTmx3MZL5C25A/qa8k= +github.com/projectdiscovery/utils v0.0.84/go.mod h1:8toAN5TBNqMJFS6uVVQTcmONrOBIMZpHAElrlzT+Y4I= github.com/quic-go/quic-go v0.37.7 h1:AgKsQLZ1+YCwZd2GYhBUsJDYZwEkA5gENtAjb+MxONU= github.com/quic-go/quic-go v0.37.7/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= github.com/refraction-networking/utls v1.5.4 h1:9k6EO2b8TaOGsQ7Pl7p9w6PUhx18/ZCeT0WNTZ7Uw4o= @@ -217,17 +224,28 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI= @@ -252,14 +270,18 @@ github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8= github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ= github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE= github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= -github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6/go.mod h1:h8272+G2omSmi30fBXiZDMkmHuOgonplfKIKjQWzlfs= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db h1:/WcxBne+5CbtbgWd/sV2wbravmr4sT7y52ifQaCgoLs= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= @@ -274,6 +296,9 @@ github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU= github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark-emoji v1.0.1 h1:ctuWEyzGBwiucEqxzwe0SOYDXPAucOrE9NQC18Wa1os= github.com/yuin/goldmark-emoji v1.0.1/go.mod h1:2w1E6FEWLcDQkoTE+7HU6QF1F6SLlNGjRIBbIZQFqkQ= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 h1:Nzukz5fNOBIHOsnP+6I79kPx3QhLv8nBy2mfFhBRq30= github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= @@ -330,16 +355,18 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -348,10 +375,14 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -382,8 +413,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index df597e030..948a5c8a9 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -125,7 +125,7 @@ func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSup if len(sourceNames) > 0 { for _, source := range sourceNames { if NameSourceMap[source] == nil { - gologger.Warning().Msgf("There is no source with the name: %s", source) + gologger.Fatal().Msgf("There is no source with the name: %s", source) } else { sources[source] = NameSourceMap[source] } diff --git a/v2/pkg/passive/sources_w_auth_test.go b/v2/pkg/passive/sources_w_auth_test.go index 98758a078..0a1085275 100644 --- a/v2/pkg/passive/sources_w_auth_test.go +++ b/v2/pkg/passive/sources_w_auth_test.go @@ -47,7 +47,12 @@ func TestSourcesWithKeys(t *testing.T) { continue } - apiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))) + var apiKey string + if source.Name() == "chaos" { + apiKey = os.Getenv("PDCP_API_KEY") + } else { + apiKey = os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))) + } if apiKey == "" { fmt.Printf("Skipping %s as no API key is provided\n", source.Name()) continue diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go index c409fb85c..db30aba8a 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/v2/pkg/passive/sources_wo_auth_test.go @@ -30,6 +30,7 @@ func TestSourcesWithoutKeys(t *testing.T) { "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally "waybackarchive", // Fails randomly "alienvault", // 503 Service Temporarily Unavailable + "digitorus", // failing with "Failed to retrieve certificate" } domain := "hackerone.com" diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index 52323a0fe..745b1b2cc 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.6.5` +const version = `v2.6.6` // showBanner is used to show the banner to the user func showBanner() { diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index ac3c70dd0..59c5d5041 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -18,6 +18,12 @@ import ( const maxNumCount = 2 +var replacer = strings.NewReplacer( + "*.", "", + "http://", "", + "https://", "", +) + // EnumerateSingleDomain wraps EnumerateSingleDomainWithCtx with an empty context func (r *Runner) EnumerateSingleDomain(domain string, writers []io.Writer) error { return r.EnumerateSingleDomainWithCtx(context.Background(), domain, writers) @@ -62,7 +68,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string skippedCounts[result.Source]++ continue } - subdomain := strings.ReplaceAll(strings.ToLower(result.Value), "*.", "") + subdomain := replacer.Replace(result.Value) if matchSubdomain := r.filterAndMatchSubdomain(subdomain); matchSubdomain { if _, ok := uniqueMap[subdomain]; !ok { diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 63066f9da..195165856 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -165,16 +165,13 @@ func ParseOptions() *Options { // Check if stdin pipe was given options.Stdin = fileutil.HasStdin() - // Read the inputs and configure the logging - options.configureOutput() - if options.Version { gologger.Info().Msgf("Current Version: %s\n", version) gologger.Info().Msgf("Subfinder Config Directory: %s", configDir) os.Exit(0) } - options.preProcessOptions() + options.preProcessDomains() if !options.Silent { showBanner() @@ -235,7 +232,7 @@ func listSources(options *Options) { } } -func (options *Options) preProcessOptions() { +func (options *Options) preProcessDomains() { for i, domain := range options.Domain { options.Domain[i], _ = sanitize(domain) } @@ -243,7 +240,6 @@ func (options *Options) preProcessOptions() { var defaultRateLimits = []string{ "github=30/m", - // "gitlab=2000/m", "fullhunt=60/m", fmt.Sprintf("robtex=%d/ms", uint(math.MaxUint)), "securitytrails=1/s", @@ -254,4 +250,8 @@ var defaultRateLimits = []string{ "waybackarchive=15/m", "whoisxmlapi=50/s", "securitytrails=2/s", + "sitedossier=8/m", + "netlas=1/s", + // "gitlab=2/s", + "github=83/m", } diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 83887ea94..d14dcb077 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -36,6 +36,7 @@ type Runner struct { // the configuration options, configuring sources, reading lists // and setting up loggers, etc. func NewRunner(options *Options) (*Runner, error) { + options.configureOutput() runner := &Runner{options: options} // Check if the application loading with any provider configuration, then take it diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/v2/pkg/subscraping/sources/crtsh/crtsh.go index d01a6e99f..e921bb017 100644 --- a/v2/pkg/subscraping/sources/crtsh/crtsh.go +++ b/v2/pkg/subscraping/sources/crtsh/crtsh.go @@ -95,7 +95,7 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio ca WHERE ci.ISSUER_CA_ID = ca.ID ORDER BY le.ENTRY_TIMESTAMP DESC NULLS LAST;`, limitClause) - rows, err := db.Query(query, domain) + rows, err := db.QueryContext(ctx, query, domain) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ diff --git a/v2/pkg/subscraping/sources/github/github.go b/v2/pkg/subscraping/sources/github/github.go index 85fb528a5..fc4e67df6 100644 --- a/v2/pkg/subscraping/sources/github/github.go +++ b/v2/pkg/subscraping/sources/github/github.go @@ -80,16 +80,6 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * } token := tokens.Get() - - if token.RetryAfter > 0 { - if len(tokens.pool) == 1 { - gologger.Verbose().Label(s.Name()).Msgf("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", token.RetryAfter) - time.Sleep(time.Duration(token.RetryAfter) * time.Second) - } else { - token = tokens.Get() - } - } - headers := map[string]string{ "Accept": "application/vnd.github.v3.text-match+json", "Authorization": "token " + token.Hash, } diff --git a/v2/pkg/subscraping/sources/gitlab/gitlab.go b/v2/pkg/subscraping/sources/gitlab/gitlab.go index 2dc76634a..82540adf9 100644 --- a/v2/pkg/subscraping/sources/gitlab/gitlab.go +++ b/v2/pkg/subscraping/sources/gitlab/gitlab.go @@ -9,7 +9,6 @@ import ( "regexp" "strings" "sync" - "time" jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" @@ -118,9 +117,6 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * return } - // TODO: hardcoded for testing, should be a source internal rate limit #718 - time.Sleep(2 * time.Second) - s.enumerate(ctx, nextURL, domainRegexp, headers, session, results) } } diff --git a/v2/pkg/subscraping/sources/netlas/netlas.go b/v2/pkg/subscraping/sources/netlas/netlas.go index f2132e7c5..e48e9ecd3 100644 --- a/v2/pkg/subscraping/sources/netlas/netlas.go +++ b/v2/pkg/subscraping/sources/netlas/netlas.go @@ -103,8 +103,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se //Define the API endpoint URL and query parameters for i := 0; i < domainsCount.Count; i += 20 { - - time.Sleep(1000 * time.Millisecond) offset := strconv.Itoa(i) endpoint := "https://app.netlas.io/api/domains/" diff --git a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go b/v2/pkg/subscraping/sources/rapiddns/rapiddns.go index 383ebd998..97628529d 100644 --- a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go +++ b/v2/pkg/subscraping/sources/rapiddns/rapiddns.go @@ -3,12 +3,17 @@ package rapiddns import ( "context" + "fmt" "io" + "regexp" + "strconv" "time" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) +var pagePattern = regexp.MustCompile(`class="page-link ">(\d+)`) + // Source is the passive scraping agent type Source struct { timeTaken time.Duration @@ -28,28 +33,47 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se close(results) }(time.Now()) - resp, err := session.SimpleGet(ctx, "https://rapiddns.io/subdomain/"+domain+"?full=1") - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } + page := 1 + maxPages := 1 + for { + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://rapiddns.io/subdomain/%s?page=%d", domain, page)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + resp.Body.Close() + return + } - body, err := io.ReadAll(resp.Body) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ resp.Body.Close() - return - } - - resp.Body.Close() - src := string(body) - for _, subdomain := range session.Extractor.Extract(src) { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ + src := string(body) + for _, subdomain := range session.Extractor.Extract(src) { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + + if maxPages == 1 { + matches := pagePattern.FindAllStringSubmatch(src, -1) + if len(matches) > 0 { + lastMatch := matches[len(matches)-1] + if len(lastMatch) > 1 { + maxPages, _ = strconv.Atoi(lastMatch[1]) + } + } + } + + if page >= maxPages { + break + } + page++ } }() diff --git a/v2/pkg/subscraping/sources/shodan/shodan.go b/v2/pkg/subscraping/sources/shodan/shodan.go index ebd27d1b5..688f07bee 100644 --- a/v2/pkg/subscraping/sources/shodan/shodan.go +++ b/v2/pkg/subscraping/sources/shodan/shodan.go @@ -25,6 +25,7 @@ type dnsdbLookupResponse struct { Subdomains []string `json:"subdomains"` Result int `json:"result"` Error string `json:"error"` + More bool `json:"more"` } // Run function returns all subdomains found with the service @@ -45,36 +46,45 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s", domain, randomApiKey) - resp, err := session.SimpleGet(ctx, searchURL) - if err != nil { - session.DiscardHTTPResponse(resp) - return - } + page := 1 + for { - defer resp.Body.Close() + searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s&page=%d", domain, randomApiKey, page) + resp, err := session.SimpleGet(ctx, searchURL) + if err != nil { + session.DiscardHTTPResponse(resp) + return + } - var response dnsdbLookupResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - return - } + defer resp.Body.Close() - if response.Error != "" { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error), + var response dnsdbLookupResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + if response.Error != "" { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error), + } + s.errors++ + return + } + + for _, data := range response.Subdomains { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", data, domain), + } + s.results++ } - s.errors++ - return - } - for _, data := range response.Subdomains { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", data, domain), + if !response.More { + break } - s.results++ + page++ } }() diff --git a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go index c44ad1c80..91c86298e 100644 --- a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go +++ b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io" - "math/rand" "net/http" "regexp" "time" @@ -25,44 +24,6 @@ type agent struct { session *subscraping.Session } -func (a *agent) enumerate(ctx context.Context, baseURL string) { - select { - case <-ctx.Done(): - return - default: - } - - resp, err := a.session.SimpleGet(ctx, baseURL) - isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound - if err != nil && !isnotfound { - a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} - a.errors++ - a.session.DiscardHTTPResponse(resp) - return - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} - a.errors++ - resp.Body.Close() - return - } - resp.Body.Close() - - src := string(body) - for _, match := range a.session.Extractor.Extract(src) { - a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: match} - } - - match1 := reNext.FindStringSubmatch(src) - time.Sleep(time.Duration((3 + rand.Intn(SleepRandIntn))) * time.Second) - - if len(match1) > 0 { - a.enumerate(ctx, "http://www.sitedossier.com"+match1[1]) - } -} - // Source is the passive scraping agent type Source struct { timeTaken time.Duration @@ -95,6 +56,42 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return a.results } +func (a *agent) enumerate(ctx context.Context, baseURL string) { + select { + case <-ctx.Done(): + return + default: + } + + resp, err := a.session.SimpleGet(ctx, baseURL) + isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound + if err != nil && !isnotfound { + a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} + a.errors++ + a.session.DiscardHTTPResponse(resp) + return + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} + a.errors++ + resp.Body.Close() + return + } + resp.Body.Close() + + src := string(body) + for _, subdomain := range a.session.Extractor.Extract(src) { + a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: subdomain} + } + + match := reNext.FindStringSubmatch(src) + if len(match) > 0 { + a.enumerate(ctx, fmt.Sprintf("http://www.sitedossier.com%s", match[1])) + } +} + // Name returns the name of the source func (s *Source) Name() string { return "sitedossier"