diff --git a/README.md b/README.md index 54843900..5f65f3e1 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,8 @@
- - + + @@ -173,7 +173,6 @@ export PATH=$GOPATH/bin:$GOROOT/bin:$HOME/.local/bin:$PATH AMASS_CONFIG=~/.config/amass/config.ini GITHUB_TOKENS=${tools}/.github_tokens GITLAB_TOKENS=${tools}/.gitlab_tokens -SUBGPT_COOKIE=${tools}/subgpt_cookies.json #CUSTOM_CONFIG=custom_config_path.txt # In case you use a custom config file, uncomment this line and set your files path # APIs/TOKENS - Uncomment the lines you want removing the '#' at the beginning of the line @@ -212,7 +211,6 @@ SUBBRUTE=true # DNS bruteforcing SUBSCRAPING=true # Subdomains extraction from web crawling SUBPERMUTE=true # DNS permutations SUBREGEXPERMUTE=true # Permutations by regex analysis -SUBGPT=true # Permutations by BingGPT prediction PERMUTATIONS_OPTION=gotator # The alternative is "ripgen" (faster, not deeper) GOTATOR_FLAGS=" -depth 1 -numbers 3 -mindup -adv -md" # Flags for gotator SUBTAKEOVER=false # Check subdomain takeovers, false by default cuz nuclei already check this diff --git a/Terraform/files/reconftw.cfg b/Terraform/files/reconftw.cfg index 6b5ad227..31152a36 100644 --- a/Terraform/files/reconftw.cfg +++ b/Terraform/files/reconftw.cfg @@ -15,6 +15,7 @@ fuzzing_remote_list="https://raw.githubusercontent.com/six2dez/OneListForAll/mai proxy_url="http://127.0.0.1:8080/" # Proxy url install_golang=true # Set it to false if you already have Golang configured and ready upgrade_tools=true +upgrade_before_running=false # Upgrade tools before running #dir_output=/custom/output/path # Golang Vars (Comment or change on your own) @@ -27,7 +28,6 @@ export PATH=$GOPATH/bin:$GOROOT/bin:$HOME/.local/bin:$PATH AMASS_CONFIG=~/.config/amass/config.ini GITHUB_TOKENS=${tools}/.github_tokens GITLAB_TOKENS=${tools}/.gitlab_tokens -SUBGPT_COOKIE=${tools}/subgpt_cookies.json #CUSTOM_CONFIG=custom_config_path.txt # In case you use a custom config file, uncomment this line and set your files path # APIs/TOKENS - Uncomment the lines you want removing the '#' at the beginning of the line @@ -52,6 +52,7 @@ EMAILS=true # Fetch emails from differents sites DOMAIN_INFO=true # whois info REVERSE_WHOIS=true # amass intel reverse whois info, takes some time IP_INFO=true # Reverse IP search, geolocation and whois +POSTMAN_LEAKS=true # Check for postman leaks METAFINDER_LIMIT=20 # Max 250 # Subdomains @@ -60,16 +61,16 @@ RUNSUBFINDER=true SUBDOMAINS_GENERAL=true # Enable or disable the whole Subdomains module SUBPASSIVE=true # Passive subdomains search SUBCRT=true # crtsh search -SUBNOERROR=true # Check DNS NOERROR response and BF on them +CTR_LIMIT=999999 # Limit the number of results +SUBNOERROR=false # Check DNS NOERROR response and BF on them SUBANALYTICS=true # Google Analytics search SUBBRUTE=true # DNS bruteforcing SUBSCRAPING=true # Subdomains extraction from web crawling SUBPERMUTE=true # DNS permutations SUBREGEXPERMUTE=true # Permutations by regex analysis -SUBGPT=true # Permutations by BingGPT prediction PERMUTATIONS_OPTION=gotator # The alternative is "ripgen" (faster, not deeper) GOTATOR_FLAGS=" -depth 1 -numbers 3 -mindup -adv -md" # Flags for gotator -SUBTAKEOVER=false # Check subdomain takeovers, false by default cuz nuclei already check this +SUBTAKEOVER=true # Check subdomain takeovers, false by default cuz nuclei already check this SUB_RECURSIVE_PASSIVE=false # Uses a lot of API keys queries DEEP_RECURSIVE_PASSIVE=10 # Number of top subdomains for recursion SUB_RECURSIVE_BRUTE=false # Needs big disk space and time to resolve @@ -96,8 +97,9 @@ CDN_IP=true # Check which IPs belongs to CDN # Web analysis WAF_DETECTION=true # Detect WAFs NUCLEICHECK=true # Enable or disable nuclei +NUCLEI_TEMPLATES_PATH="$HOME/nuclei-templates" # Set nuclei templates path NUCLEI_SEVERITY="info,low,medium,high,critical" # Set templates criticity -NUCLEI_FLAGS=" -silent -t $HOME/nuclei-templates/ -retries 2" # Additional nuclei extra flags, don't set the severity here but the exclusions like " -etags openssh" +NUCLEI_FLAGS=" -silent -t ${NUCLEI_TEMPLATES_PATH}/ -retries 2" # Additional nuclei extra flags, don't set the severity here but the exclusions like " -etags openssh" NUCLEI_FLAGS_JS=" -silent -tags exposure,token -severity info,low,medium,high,critical" # Additional nuclei extra flags for js secrets URL_CHECK=true # Enable or disable URL collection URL_CHECK_PASSIVE=true # Search for urls, passive methods from Archive, OTX, CommonCrawl, etc @@ -110,8 +112,8 @@ CMS_SCANNER=true # CMS scanner WORDLIST=true # Wordlist generation ROBOTSWORDLIST=true # Check historic disallow entries on waybackMachine PASSWORD_DICT=true # Generate password dictionary -PASSWORD_MIN_LENGTH=5 # Min password lenght -PASSWORD_MAX_LENGTH=14 # Max password lenght +PASSWORD_MIN_LENGTH=5 # Min password length +PASSWORD_MAX_LENGTH=14 # Max password length # Vulns VULNS_GENERAL=false # Enable or disable the vulnerability module (very intrusive and slow) @@ -133,6 +135,7 @@ PROTO_POLLUTION=true # Check for prototype pollution flaws SMUGGLING=true # Check for HTTP request smuggling flaws WEBCACHE=true # Check for Web Cache issues BYPASSER4XX=true # Check for 4XX bypasses +FUZZPARAMS=true # Fuzz parameters values # Extra features NOTIFICATION=false # Notification for every function @@ -146,7 +149,7 @@ REMOVELOG=false # Delete logs after execution PROXY=false # Send to proxy the websites found SENDZIPNOTIFY=false # Send to zip the results (over notify) PRESERVE=true # set to true to avoid deleting the .called_fn files on really large scans -FFUF_FLAGS=" -mc all -fc 404 -ac -sf" # Ffuf flags +FFUF_FLAGS=" -mc all -fc 404 -ach -sf -of json" # Ffuf flags HTTPX_FLAGS=" -follow-redirects -random-agent -status-code -silent -title -web-server -tech-detect -location -content-length" # Httpx flags for simple web probing GOWITNESS_FLAGS=" --disable-logging --timeout 5" @@ -189,6 +192,7 @@ FFUF_MAXTIME=900 # Seconds HTTPX_TIMEOUT=10 # Seconds HTTPX_UNCOMMONPORTS_TIMEOUT=10 # Seconds PERMUTATIONS_LIMIT=21474836480 # Bytes, default is 20 GB +GOWITNESS_TIMEOUT_PER_SITE=20 # Seconds # lists fuzz_wordlist=${tools}/fuzz_wordlist.txt diff --git a/install.sh b/install.sh index 6a7a42f7..9a7090f1 100755 --- a/install.sh +++ b/install.sh @@ -6,45 +6,37 @@ dir=${tools} double_check=false # ARM Detection -if [[ $(uname -m) == "amd64" ]] || [[ $(uname -m) == "x86_64" ]]; then - IS_ARM="False" -fi -if [[ $(uname -m) == "arm64" ]] || [[ $(uname -m) == "armv6l" ]]; then - IS_ARM="True" - if [[ $(uname -m) == "arm64" ]]; then - RPI_4="False" - else - RPI_3="True" - fi -fi +ARCH=$(uname -m) +case $ARCH in + amd64|x86_64) IS_ARM="False" ;; + arm64|armv6l) + IS_ARM="True" + RPI_4=$([[ $ARCH == "arm64" ]] && echo "True" || echo "False") + RPI_3=$([[ $ARCH == "arm64" ]] && echo "False" || echo "True") + ;; +esac #Mac Osx Detecting -if [[ "$OSTYPE" == "darwin"* ]]; then - IS_MAC="True" -else - IS_MAC="False" -fi - -# Check Bash version -#(bash --version | awk 'NR==1{print $4}' | cut -d'.' -f1) 2&>/dev/null || echo "Unable to get bash version, for MacOS run 'brew install bash' and rerun installer in a new terminal" && exit 1 +IS_MAC=$([[ "$OSTYPE" == "darwin"* ]] && echo "True" || echo "False") BASH_VERSION=$(bash --version | awk 'NR==1{print $4}' | cut -d'.' -f1) if [ "${BASH_VERSION}" -lt 4 ]; then printf "${bred} Your Bash version is lower than 4, please update${reset}\n" - printf "%s Your Bash version is lower than 4, please update%s\n" "${bred}" "${reset}" + printf "%s Your Bash version is lower than 4, please update%s\n" "${bred}" "${reset}" >&2 if [ "True" = "$IS_MAC" ]; then printf "${yellow} For MacOS run 'brew install bash' and rerun installer in a new terminal${reset}\n\n" exit 1; fi fi +# Declaring Go tools and their installation commands declare -A gotools gotools["gf"]="go install -v github.com/tomnomnom/gf@latest" gotools["qsreplace"]="go install -v github.com/tomnomnom/qsreplace@latest" gotools["amass"]="go install -v github.com/owasp-amass/amass/v3/...@master" gotools["ffuf"]="go install -v github.com/ffuf/ffuf/v2@latest" gotools["github-subdomains"]="go install -v github.com/gwen001/github-subdomains@latest" -gotools["gitlab-subdomains"]="go install github.com/gwen001/gitlab-subdomains@latest" +gotools["gitlab-subdomains"]="go install -v github.com/gwen001/gitlab-subdomains@latest" gotools["nuclei"]="go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest" gotools["anew"]="go install -v github.com/tomnomnom/anew@latest" gotools["notify"]="go install -v github.com/projectdiscovery/notify/cmd/notify@latest" @@ -54,7 +46,7 @@ gotools["github-endpoints"]="go install -v github.com/gwen001/github-endpoints@l gotools["dnsx"]="go install -v github.com/projectdiscovery/dnsx/cmd/dnsx@latest" gotools["subjs"]="go install -v github.com/lc/subjs@latest" gotools["Gxss"]="go install -v github.com/KathanP19/Gxss@latest" -gotools["katana"]="go install github.com/projectdiscovery/katana/cmd/katana@latest" +gotools["katana"]="go install -v github.com/projectdiscovery/katana/cmd/katana@latest" gotools["crlfuzz"]="go install -v github.com/dwisiswant0/crlfuzz/cmd/crlfuzz@latest" gotools["dalfox"]="go install -v github.com/hahwul/dalfox/v2@latest" gotools["puredns"]="go install -v github.com/d3mondev/puredns/v2@latest" @@ -66,21 +58,22 @@ gotools["mapcidr"]="go install -v github.com/projectdiscovery/mapcidr/cmd/mapcid gotools["cdncheck"]="go install -v github.com/projectdiscovery/cdncheck/cmd/cdncheck@latest" gotools["dnstake"]="go install -v github.com/pwnesia/dnstake/cmd/dnstake@latest" gotools["gowitness"]="go install -v github.com/sensepost/gowitness@latest" -gotools["tlsx"]="go install github.com/projectdiscovery/tlsx/cmd/tlsx@latest" +gotools["tlsx"]="go install -v github.com/projectdiscovery/tlsx/cmd/tlsx@latest" gotools["gitdorks_go"]="go install -v github.com/damit5/gitdorks_go@latest" gotools["smap"]="go install -v github.com/s0md3v/smap/cmd/smap@latest" gotools["dsieve"]="go install -v github.com/trickest/dsieve@master" -gotools["inscope"]="go install github.com/tomnomnom/hacks/inscope@latest" -gotools["enumerepo"]="go install github.com/trickest/enumerepo@latest" +gotools["inscope"]="go install -v github.com/tomnomnom/hacks/inscope@latest" +gotools["enumerepo"]="go install -v github.com/trickest/enumerepo@latest" gotools["Web-Cache-Vulnerability-Scanner"]="go install -v github.com/Hackmanit/Web-Cache-Vulnerability-Scanner@latest" gotools["subfinder"]="go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest" gotools["byp4xx"]="go install -v github.com/lobuhi/byp4xx@latest" -gotools["hakip2host"]="go install github.com/hakluke/hakip2host@latest" +gotools["hakip2host"]="go install -v github.com/hakluke/hakip2host@latest" gotools["gau"]="go install -v github.com/lc/gau/v2/cmd/gau@latest" -gotools["Mantra"]="go install github.com/MrEmpy/Mantra@latest" -gotools["crt"]="go install github.com/cemulus/crt@latest" +gotools["Mantra"]="go install -v github.com/MrEmpy/Mantra@latest" +gotools["crt"]="go install -v github.com/cemulus/crt@latest" gotools["s3scanner"]="go install -v github.com/sa7mon/s3scanner@latest" +# Declaring repositories and their paths declare -A repos repos["dorks_hunter"]="six2dez/dorks_hunter" repos["pwndb"]="davidtavarez/pwndb" @@ -115,7 +108,7 @@ repos["trufflehog"]="trufflesecurity/trufflehog" function banner_web(){ - echo -en "\033c" + tput clear printf "\n${bgreen}" printf " ██▀███ ▓█████ ▄████▄ ▒█████ ███▄ █ █████▒▄▄▄█████▓ █ █░\n" printf " ▓██ ▒ ██▒▓█ ▀ ▒██▀ ▀█ ▒██▒ ██▒ ██ ▀█ █ ▓██ ▒ ▓ ██▒ ▓▒▓█░ █ ░█░\n" @@ -130,8 +123,18 @@ function banner_web(){ printf " ${reconftw_version} by @six2dez\n" } +function install_ppfuzz() { + local url=$1 + local tar_file=$2 + + wget -N -c "$url" $DEBUG_STD + eval $SUDO tar -C /usr/local/bin/ -xzf "$tar_file" $DEBUG_STD + eval $SUDO rm -rf "$tar_file" $DEBUG_STD +} + +# This function installs various tools and repositories as per the configuration. function install_tools(){ - #eval ln -s /usr/local/bin/pip3 /usr/local/bin/pip3 $DEBUG_STD + eval pip3 install -I -r requirements.txt $DEBUG_STD printf "${bblue} Running: Installing Golang tools (${#gotools[@]})${reset}\n\n" @@ -219,28 +222,18 @@ function install_tools(){ if [ "True" = "$IS_ARM" ]; then if [ "True" = "$RPI_3" ]; then - eval wget -N -c https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz $DEBUG_STD - eval $SUDO tar -C /usr/local/bin/ -xzf ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz $DEBUG_STD - eval $SUDO rm -rf ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz $DEBUG_STD + install_ppfuzz "https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz" "ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz" elif [ "True" = "$RPI_4" ]; then - eval wget -N -c https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-aarch64-unknown-linux-gnueabihf.tar.gz $DEBUG_STD - eval $SUDO tar -C /usr/local/bin/ -xzf ppfuzz-v1.0.1-aarch64-unknown-linux-gnueabihf.tar.gz $DEBUG_STD - eval $SUDO rm -rf ppfuzz-v1.0.1-aarch64-unknown-linux-gnueabihf.tar.gz $DEBUG_STD + install_ppfuzz "https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-aarch64-unknown-linux-gnueabihf.tar.gz" "ppfuzz-v1.0.1-aarch64-unknown-linux-gnueabihf.tar.gz" fi elif [ "True" = "$IS_MAC" ]; then if [ "True" = "$IS_ARM" ]; then - eval wget -N -c https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz $DEBUG_STD - eval $SUDO tar -C /usr/local/bin/ -xzf ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz $DEBUG_STD - eval $SUDO rm -rf ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz $DEBUG_STD + install_ppfuzz "https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz" "ppfuzz-v1.0.1-armv7-unknown-linux-gnueabihf.tar.gz" else - eval wget -N -c https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-x86_64-apple-darwin.tar.gz $DEBUG_STD - eval $SUDO tar -C /usr/local/bin/ -xzf ppfuzz-v1.0.1-x86_64-apple-darwin.tar.gz $DEBUG_STD - eval $SUDO rm -rf ppfuzz-v1.0.1-x86_64-apple-darwin.tar.gz $DEBUG_STD + install_ppfuzz "https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-x86_64-apple-darwin.tar.gz" "ppfuzz-v1.0.1-x86_64-apple-darwin.tar.gz" fi else - eval wget -N -c https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-x86_64-unknown-linux-musl.tar.gz $DEBUG_STD - eval $SUDO tar -C /usr/local/bin/ -xzf ppfuzz-v1.0.1-x86_64-unknown-linux-musl.tar.gz $DEBUG_STD - eval $SUDO rm -rf ppfuzz-v1.0.1-x86_64-unknown-linux-musl.tar.gz $DEBUG_STD + install_ppfuzz "https://github.com/dwisiswant0/ppfuzz/releases/download/v1.0.1/ppfuzz-v1.0.1-x86_64-unknown-linux-musl.tar.gz" "ppfuzz-v1.0.1-x86_64-unknown-linux-musl.tar.gz" fi eval $SUDO chmod 755 /usr/local/bin/ppfuzz eval $SUDO strip -s /usr/local/bin/ppfuzz $DEBUG_STD @@ -272,7 +265,14 @@ install_webserver(){ $SUDO pip3 install -r $SCRIPTPATH/web/requirements.txt &>/dev/null printf "${yellow} Installing tools...${reset}\n\n" + if command -v apt > /dev/null; then $SUDO apt install redis-server -y &>/dev/null + elif command -v yum > /dev/null; then + $SUDO yum install redis -y &>/dev/null + else + printf '[ERROR] Unable to find a supported package manager. Please install redis manually.\n' + exit 1 + fi printf "${yellow} Creating WEB User...${reset}\n\n" $SUDO rm $SCRIPTPATH/web/db.sqlite3 &>/dev/null @@ -305,9 +305,15 @@ display_menu(){ printf "${bblue} 3. Setup Web Interface${reset} ${yellow}(User Interaction needed!)${reset}\n\n" printf "${bblue} 4. Exit${reset}\n\n" printf "${bgreen}#######################################################################${reset}\n\n" - read -p "$(echo -e ${bblue} "Insert option: "${reset})" option + read -p "${bblue}Insert option: ${reset}" option printf "\n\n${bgreen}#######################################################################${reset}\n\n" + option=$(echo "$option" | tr -d '[:space:]') + if ! [[ "$option" =~ ^[1-4]$ ]]; then + printf "${bred} Invalid option. Please try again.${reset}\n\n" + continue + fi + case $option in 1) web=false @@ -354,7 +360,7 @@ display_menu(){ exit 1 ;; *) - printf "${bblue} Invalid option. Exiting...${reset}\n\n" + printf "${bred} Invalid option. Please try again.${reset}\n\n" exit 1 ;; esac @@ -362,14 +368,18 @@ display_menu(){ done } -if [ "$1" = '--tools' ]; then - install_tools -fi - -if [ "$1" != '--auto' ]; then - echo "$1" - display_menu -fi +case "$1" in + --tools) + install_tools + ;; + --auto) + # possibly some other actions + ;; + *) + echo "$1" + display_menu + ;; +esac printf "${yellow} This may take time. So, go grab a coffee! ${reset}\n\n" @@ -432,7 +442,11 @@ eval git config --global --unset https.proxy $DEBUG_STD printf "${bblue} Running: Looking for new reconFTW version${reset}\n\n" -eval git fetch $DEBUG_STD +if ! eval git fetch $DEBUG_STD; then + echo "Failed to fetch updates." + exit 1 +fi + BRANCH=$(git rev-parse --abbrev-ref HEAD) HEADHASH=$(git rev-parse HEAD) UPSTREAMHASH=$(git rev-parse "${BRANCH}@{upstream}") @@ -608,6 +622,6 @@ if [ "$web" = true ]; then printf "\n${bgreen} Web server is installed, to set it up run ./install.sh and select option 3 ${reset}\n\n" fi -printf "${yellow} Remember set your api keys:\n - amass (~/.config/amass/config.ini)\n - subfinder (~/.config/subfinder/provider-config.yaml)\n - GitLab (~/Tools/.gitlab_tokens)\n - SSRF Server (COLLAB_SERVER in reconftw.cfg or env var) \n - Blind XSS Server (XSS_SERVER in reconftw.cfg or env var) \n - notify (~/.config/notify/provider-config.yaml) \n - WHOISXML API (WHOISXML_API in reconftw.cfg or env var)\n - subgpt_cookies.json (subgpt_cookies.json file, follow instructions at https://github.com/s0md3v/SubGPT#getting-bing-cookie)\n\n\n${reset}" +printf "${yellow} Remember set your api keys:\n - amass (~/.config/amass/config.ini)\n - subfinder (~/.config/subfinder/provider-config.yaml)\n - GitLab (~/Tools/.gitlab_tokens)\n - SSRF Server (COLLAB_SERVER in reconftw.cfg or env var) \n - Blind XSS Server (XSS_SERVER in reconftw.cfg or env var) \n - notify (~/.config/notify/provider-config.yaml) \n - WHOISXML API (WHOISXML_API in reconftw.cfg or env var)\n\n${reset}" printf "${bgreen} Finished!${reset}\n\n" printf "\n\n${bgreen}#######################################################################${reset}\n" diff --git a/reconftw.cfg b/reconftw.cfg index 294b38ae..40be4fdc 100644 --- a/reconftw.cfg +++ b/reconftw.cfg @@ -28,7 +28,6 @@ export PATH=$GOPATH/bin:$GOROOT/bin:$HOME/.local/bin:$PATH AMASS_CONFIG=~/.config/amass/config.ini GITHUB_TOKENS=${tools}/.github_tokens GITLAB_TOKENS=${tools}/.gitlab_tokens -SUBGPT_COOKIE=${tools}/subgpt_cookies.json #CUSTOM_CONFIG=custom_config_path.txt # In case you use a custom config file, uncomment this line and set your files path # APIs/TOKENS - Uncomment the lines you want removing the '#' at the beginning of the line @@ -53,6 +52,7 @@ EMAILS=true # Fetch emails from differents sites DOMAIN_INFO=true # whois info REVERSE_WHOIS=true # amass intel reverse whois info, takes some time IP_INFO=true # Reverse IP search, geolocation and whois +POSTMAN_LEAKS=true # Check for postman leaks METAFINDER_LIMIT=20 # Max 250 # Subdomains @@ -68,7 +68,6 @@ SUBBRUTE=true # DNS bruteforcing SUBSCRAPING=true # Subdomains extraction from web crawling SUBPERMUTE=true # DNS permutations SUBREGEXPERMUTE=true # Permutations by regex analysis -SUBGPT=true # Permutations by BingGPT prediction PERMUTATIONS_OPTION=gotator # The alternative is "ripgen" (faster, not deeper) GOTATOR_FLAGS=" -depth 1 -numbers 3 -mindup -adv -md" # Flags for gotator SUBTAKEOVER=true # Check subdomain takeovers, false by default cuz nuclei already check this diff --git a/reconftw.sh b/reconftw.sh index 217df74b..8740a4ad 100755 --- a/reconftw.sh +++ b/reconftw.sh @@ -114,7 +114,6 @@ function tools_installed(){ which hakip2host &>/dev/null || { printf "${bred} [*] hakip2host [NO]${reset}\n${reset}"; allinstalled=false;} which gau &>/dev/null || { printf "${bred} [*] gau [NO]${reset}\n${reset}"; allinstalled=false;} which crt &>/dev/null || { printf "${bred} [*] crt [NO]${reset}\n${reset}"; allinstalled=false;} - which subgpt &>/dev/null || { printf "${bred} [*] subgpt [NO]${reset}\n${reset}"; allinstalled=false;} which gitleaks &>/dev/null || { printf "${bred} [*] gitleaks [NO]${reset}\n${reset}"; allinstalled=false;} which trufflehog &>/dev/null || { printf "${bred} [*] trufflehog [NO]${reset}\n${reset}"; allinstalled=false;} which s3scanner &>/dev/null || { printf "${bred} [*] s3scanner [NO]${reset}\n${reset}"; allinstalled=false;} @@ -178,7 +177,7 @@ function github_repos(){ if [ -s "${GITHUB_TOKENS}" ]; then GH_TOKEN=$(cat ${GITHUB_TOKENS} | head -1) echo $domain | unfurl format %r > .tmp/company_name.txt - enumerepo -token-string "${GH_TOKEN}" -usernames .tmp/company_name.txt -o .tmp/company_repos.txt 2>>"$LOGFILE" >/dev/null & + enumerepo -token-string "${GH_TOKEN}" -usernames .tmp/company_name.txt -o .tmp/company_repos.txt 2>>"$LOGFILE" >/dev/null [ -s ".tmp/company_repos.txt" ] && jq -r '.[].repos[]|.url' < .tmp/company_repos.txt > .tmp/company_repos_url.txt 2>>"$LOGFILE" mkdir -p .tmp/github_repos 2>>"$LOGFILE" >>"$LOGFILE" mkdir -p .tmp/github 2>>"$LOGFILE" >>"$LOGFILE" @@ -224,6 +223,28 @@ function metadata(){ fi } +function postleaks(){ + if { [ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ] || [ "$DIFF" = true ]; } && [ "$POSTMAN_LEAKS" = true ] && [ "$OSINT" = true ] && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9] ]]; then + start_func ${FUNCNAME[0]} "Scanning for leaks in postman public directory" + + postleaksNg -k "$domain" > .tmp/postleaks.txt || { echo "postleaksNg command failed"; exit 1; } + + end_func "Results are saved in $domain/osint/[software/authors/metadata_results].txt" ${FUNCNAME[0]} + else + if [ "$POSTMAN_LEAKS" = false ] || [ "$OSINT" = false ]; then + printf "\n${yellow} ${FUNCNAME[0]} skipped in this mode or defined in reconftw.cfg ${reset}\n" + elif [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9] ]]; then + return + else + if [ "$POSTMAN_LEAKS" = false ] || [ "$OSINT" = false ]; then + printf "\n${yellow} ${FUNCNAME[0]} skipped in this mode or defined in reconftw.cfg ${reset}\n" + else + printf "${yellow} ${FUNCNAME[0]} is already processed, to force executing ${FUNCNAME[0]} delete\n $called_fn_dir/.${FUNCNAME[0]} ${reset}\n\n" + fi + fi + fi +} + function emails(){ if { [ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ] || [ "$DIFF" = true ]; } && [ "$EMAILS" = true ] && [ "$OSINT" = true ] && ! [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9] ]]; then start_func ${FUNCNAME[0]} "Searching emails/users/passwords leaks" @@ -425,7 +446,7 @@ function sub_active(){ cat .tmp/subdomains_tmp.txt | tlsx -san -cn -silent -ro -c $TLSX_THREADS | anew -q .tmp/subdomains_tmp.txt fi [[ "$INSCOPE" = true ]] && check_inscope .tmp/subdomains_tmp.txt 2>>"$LOGFILE" >/dev/null - NUMOFLINES=$(cat .tmp/subdomains_tmp.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) + NUMOFLINES=$(cat .tmp/subdomains_tmp.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) end_subfunc "${NUMOFLINES} subs DNS resolved from passive" ${FUNCNAME[0]} else printf "${yellow} ${FUNCNAME[0]} is already processed, to force executing ${FUNCNAME[0]} delete\n $called_fn_dir/.${FUNCNAME[0]} ${reset}\n\n" @@ -443,7 +464,7 @@ function sub_noerror(){ dnsx -d $domain -r $resolvers -silent -rcode noerror -w $subs_wordlist | cut -d' ' -f1 | anew -q .tmp/subs_noerror.txt 2>>"$LOGFILE" >/dev/null fi [[ "$INSCOPE" = true ]] && check_inscope .tmp/subs_noerror.txt 2>>"$LOGFILE" >/dev/null - NUMOFLINES=$(cat .tmp/subs_noerror.txt 2>>"$LOGFILE" | sed "s/*.//" | grep ".$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) + NUMOFLINES=$(cat .tmp/subs_noerror.txt 2>>"$LOGFILE" | sed "s/*.//" | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) end_subfunc "${NUMOFLINES} new subs (DNS noerror)" ${FUNCNAME[0]} else printf "\n${yellow} Detected DNSSEC black lies, skipping this technique ${reset}\n" @@ -462,22 +483,22 @@ function sub_dns(){ start_subfunc ${FUNCNAME[0]} "Running : DNS Subdomain Enumeration and PTR search" if [ ! "$AXIOM" = true ]; then [ -s "subdomains/subdomains.txt" ] && cat subdomains/subdomains.txt | dnsx -r $resolvers_trusted -a -aaaa -cname -ns -ptr -mx -soa -silent -retry 3 -json -o subdomains/subdomains_dnsregs.json 2>>"$LOGFILE" >/dev/null - [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[], try .aaaa[], try .cname[], try .ns[], try .ptr[], try .mx[], try .soa[]' 2>/dev/null | grep ".$domain$" | anew -q .tmp/subdomains_dns.txt - [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[]' | sort -u | hakip2host | cut -d' ' -f 3 | unfurl -u domains | sed -e 's/*\.//' -e 's/\.$//' -e '/\./!d' | grep ".$domain$" | anew -q .tmp/subdomains_dns.txt + [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[], try .aaaa[], try .cname[], try .ns[], try .ptr[], try .mx[], try .soa[]' 2>/dev/null | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew -q .tmp/subdomains_dns.txt + [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[]' | sort -u | hakip2host | cut -d' ' -f 3 | unfurl -u domains | sed -e 's/*\.//' -e 's/\.$//' -e '/\./!d' | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew -q .tmp/subdomains_dns.txt [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try "\(.host) - \(.a[])"' 2>/dev/null | sort -u -k2 | anew -q subdomains/subdomains_ips.txt resolvers_update_quick_local [ -s ".tmp/subdomains_dns.txt" ] && puredns resolve .tmp/subdomains_dns.txt -w .tmp/subdomains_dns_resolved.txt -r $resolvers --resolvers-trusted $resolvers_trusted -l $PUREDNS_PUBLIC_LIMIT --rate-limit-trusted $PUREDNS_TRUSTED_LIMIT --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT 2>>"$LOGFILE" >/dev/null else [ -s "subdomains/subdomains.txt" ] && axiom-scan subdomains/subdomains.txt -m dnsx -retry 3 -a -aaaa -cname -ns -ptr -mx -soa -json -o subdomains/subdomains_dnsregs.json $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[]' | sort -u | anew -q .tmp/subdomains_dns_a_records.txt - [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[]' | sort -u | hakip2host | cut -d' ' -f 3 | unfurl -u domains | sed -e 's/*\.//' -e 's/\.$//' -e '/\./!d' | grep ".$domain$" | anew -q .tmp/subdomains_dns.txt - [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[], try .aaaa[], try .cname[], try .ns[], try .ptr[], try .mx[], try .soa[]' 2>/dev/null | grep ".$domain$" | anew -q .tmp/subdomains_dns.txt + [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[]' | sort -u | hakip2host | cut -d' ' -f 3 | unfurl -u domains | sed -e 's/*\.//' -e 's/\.$//' -e '/\./!d' | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew -q .tmp/subdomains_dns.txt + [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try .a[], try .aaaa[], try .cname[], try .ns[], try .ptr[], try .mx[], try .soa[]' 2>/dev/null | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew -q .tmp/subdomains_dns.txt [ -s "subdomains/subdomains_dnsregs.json" ] && cat subdomains/subdomains_dnsregs.json | jq -r 'try "\(.host) - \(.a[])"' 2>/dev/null | sort -u -k2 | anew -q subdomains/subdomains_ips.txt resolvers_update_quick_axiom [ -s ".tmp/subdomains_dns.txt" ] && axiom-scan .tmp/subdomains_dns.txt -m puredns-resolve -r /home/op/lists/resolvers.txt --resolvers-trusted /home/op/lists/resolvers_trusted.txt --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT -o .tmp/subdomains_dns_resolved.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null fi [[ "$INSCOPE" = true ]] && check_inscope .tmp/subdomains_dns_resolved.txt 2>>"$LOGFILE" >/dev/null - NUMOFLINES=$(cat .tmp/subdomains_dns_resolved.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) + NUMOFLINES=$(cat .tmp/subdomains_dns_resolved.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) end_subfunc "${NUMOFLINES} new subs (dns resolution)" ${FUNCNAME[0]} else printf "${yellow} ${FUNCNAME[0]} is already processed, to force executing ${FUNCNAME[0]} delete\n $called_fn_dir/.${FUNCNAME[0]} ${reset}\n\n" @@ -505,7 +526,7 @@ function sub_brute(){ [ -s ".tmp/subs_brute.txt" ] && axiom-scan .tmp/subs_brute.txt -m puredns-resolve -r /home/op/lists/resolvers.txt --resolvers-trusted /home/op/lists/resolvers_trusted.txt --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT -o .tmp/subs_brute_valid.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null fi [[ "$INSCOPE" = true ]] && check_inscope .tmp/subs_brute_valid.txt 2>>"$LOGFILE" >/dev/null - NUMOFLINES=$(cat .tmp/subs_brute_valid.txt 2>>"$LOGFILE" | sed "s/*.//" | grep ".$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) + NUMOFLINES=$(cat .tmp/subs_brute_valid.txt 2>>"$LOGFILE" | sed "s/*.//" | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) end_subfunc "${NUMOFLINES} new subs (bruteforce)" ${FUNCNAME[0]} else if [ "$SUBBRUTE" = false ]; then @@ -525,9 +546,9 @@ function sub_scraping(){ if [ ! "$AXIOM" = true ]; then resolvers_update_quick_local cat subdomains/subdomains.txt | httpx -follow-host-redirects -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info1.txt 2>>"$LOGFILE" >/dev/null - [ -s ".tmp/web_full_info1.txt" ] && cat .tmp/web_full_info1.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt + [ -s ".tmp/web_full_info1.txt" ] && cat .tmp/web_full_info1.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt [ -s ".tmp/probed_tmp_scrap.txt" ] && cat .tmp/probed_tmp_scrap.txt | httpx -tls-grab -tls-probe -csp-probe -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info2.txt 2>>"$LOGFILE" >/dev/null - [ -s ".tmp/web_full_info2.txt" ] && cat .tmp/web_full_info2.txt | jq -r 'try ."tls-grab"."dns_names"[],try .csp.domains[],try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | sort -u | httpx -silent | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt + [ -s ".tmp/web_full_info2.txt" ] && cat .tmp/web_full_info2.txt | jq -r 'try ."tls-grab"."dns_names"[],try .csp.domains[],try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | sort -u | httpx -silent | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt if [ "$DEEP" = true ]; then [ -s ".tmp/probed_tmp_scrap.txt" ] && katana -silent -list .tmp/probed_tmp_scrap.txt -jc -kf all -c $KATANA_THREADS -d 3 -fs rdn -o .tmp/katana.txt 2>>"$LOGFILE" >/dev/null @@ -537,9 +558,9 @@ function sub_scraping(){ else resolvers_update_quick_axiom axiom-scan subdomains/subdomains.txt -m httpx -follow-host-redirects -random-agent -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info1.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null - [ -s ".tmp/web_full_info1.txt" ] && cat .tmp/web_full_info1.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt + [ -s ".tmp/web_full_info1.txt" ] && cat .tmp/web_full_info1.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt [ -s ".tmp/probed_tmp_scrap.txt" ] && axiom-scan .tmp/probed_tmp_scrap.txt -m httpx -tls-grab -tls-probe -csp-probe -random-agent -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info2.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null - [ -s ".tmp/web_full_info2.txt" ] && cat .tmp/web_full_info2.txt | jq -r 'try ."tls-grab"."dns_names"[],try .csp.domains[],try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | sort -u | httpx -silent | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt + [ -s ".tmp/web_full_info2.txt" ] && cat .tmp/web_full_info2.txt | jq -r 'try ."tls-grab"."dns_names"[],try .csp.domains[],try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | sort -u | httpx -silent | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt if [ "$DEEP" = true ]; then [ -s ".tmp/probed_tmp_scrap.txt" ] && axiom-scan .tmp/probed_tmp_scrap.txt -m katana -jc -kf all -d 3 -fs rdn -o .tmp/katana.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null else @@ -547,14 +568,14 @@ function sub_scraping(){ fi fi sed -i '/^.\{2048\}./d' .tmp/katana.txt - [ -s ".tmp/katana.txt" ] && cat .tmp/katana.txt | unfurl -u domains 2>>"$LOGFILE" | grep ".$domain$" | anew -q .tmp/scrap_subs.txt + [ -s ".tmp/katana.txt" ] && cat .tmp/katana.txt | unfurl -u domains 2>>"$LOGFILE" | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew -q .tmp/scrap_subs.txt [ -s ".tmp/scrap_subs.txt" ] && puredns resolve .tmp/scrap_subs.txt -w .tmp/scrap_subs_resolved.txt -r $resolvers --resolvers-trusted $resolvers_trusted -l $PUREDNS_PUBLIC_LIMIT --rate-limit-trusted $PUREDNS_TRUSTED_LIMIT --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT 2>>"$LOGFILE" >/dev/null if [ "$INSCOPE" = true ]; then check_inscope .tmp/scrap_subs_resolved.txt 2>>"$LOGFILE" >/dev/null fi - NUMOFLINES=$(cat .tmp/scrap_subs_resolved.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | anew subdomains/subdomains.txt | tee .tmp/diff_scrap.txt | sed '/^$/d' | wc -l) + NUMOFLINES=$(cat .tmp/scrap_subs_resolved.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | tee .tmp/diff_scrap.txt | sed '/^$/d' | wc -l) [ -s ".tmp/diff_scrap.txt" ] && cat .tmp/diff_scrap.txt | httpx -follow-host-redirects -random-agent -status-code -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -timeout $HTTPX_TIMEOUT -silent -retries 2 -title -web-server -tech-detect -location -no-color -json -o .tmp/web_full_info3.txt 2>>"$LOGFILE" >/dev/null - [ -s ".tmp/web_full_info3.txt" ] && cat .tmp/web_full_info3.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt + [ -s ".tmp/web_full_info3.txt" ] && cat .tmp/web_full_info3.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | anew .tmp/probed_tmp_scrap.txt | unfurl -u domains 2>>"$LOGFILE" | anew -q .tmp/scrap_subs.txt cat .tmp/web_full_info1.txt .tmp/web_full_info2.txt .tmp/web_full_info3.txt 2>>"$LOGFILE" | jq -s 'try .' | jq 'try unique_by(.input)' | jq 'try .[]' 2>>"$LOGFILE" > .tmp/web_full_info.txt end_subfunc "${NUMOFLINES} new subs (code scraping)" ${FUNCNAME[0]} else @@ -579,7 +600,7 @@ function sub_analytics(){ mkdir -p .tmp/output_analytics/ analyticsrelationships -ch < .tmp/probed_tmp_scrap.txt >> .tmp/analytics_subs_tmp.txt 2>>"$LOGFILE" - [ -s ".tmp/analytics_subs_tmp.txt" ] && cat .tmp/analytics_subs_tmp.txt | grep "\.$domain$\|^$domain$" | sed "s/|__ //" | anew -q .tmp/analytics_subs_clean.txt + [ -s ".tmp/analytics_subs_tmp.txt" ] && cat .tmp/analytics_subs_tmp.txt | grep "\.$domain$\|^$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/|__ //" | anew -q .tmp/analytics_subs_clean.txt if [ ! "$AXIOM" = true ]; then resolvers_update_quick_local [ -s ".tmp/analytics_subs_clean.txt" ] && puredns resolve .tmp/analytics_subs_clean.txt -w .tmp/analytics_subs_resolved.txt -r $resolvers --resolvers-trusted $resolvers_trusted -l $PUREDNS_PUBLIC_LIMIT --rate-limit-trusted $PUREDNS_TRUSTED_LIMIT --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT 2>>"$LOGFILE" >/dev/null @@ -643,7 +664,7 @@ function sub_permut(){ if [ -s ".tmp/permute_subs.txt" ]; then [ -s "$outOfScope_file" ] && deleteOutScoped $outOfScope_file .tmp/permute_subs.txt [[ "$INSCOPE" = true ]] && check_inscope .tmp/permute_subs.txt 2>>"$LOGFILE" >/dev/null - NUMOFLINES=$(cat .tmp/permute_subs.txt 2>>"$LOGFILE" | grep ".$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) + NUMOFLINES=$(cat .tmp/permute_subs.txt 2>>"$LOGFILE" | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) else NUMOFLINES=0 fi @@ -675,11 +696,11 @@ function sub_regex_permut(){ if [ -s ".tmp/regulator.txt" ]; then [ -s "$outOfScope_file" ] && deleteOutScoped $outOfScope_file .tmp/regulator.txt [[ "$INSCOPE" = true ]] && check_inscope .tmp/regulator.txt 2>>"$LOGFILE" >/dev/null - NUMOFLINES=$(cat .tmp/regulator.txt 2>>"$LOGFILE" | grep ".$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) + NUMOFLINES=$(cat .tmp/regulator.txt 2>>"$LOGFILE" | grep ".$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) else NUMOFLINES=0 fi - end_subfunc "${NUMOFLINES} new subs (permutations)" ${FUNCNAME[0]} + end_subfunc "${NUMOFLINES} new subs (permutations by regex)" ${FUNCNAME[0]} else if [ "$SUBREGEXPERMUTE" = false ]; then printf "\n${yellow} ${FUNCNAME[0]} skipped in this mode or defined in reconftw.cfg ${reset}\n" @@ -689,37 +710,6 @@ function sub_regex_permut(){ fi } -function sub_gpt(){ - if { [ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ] || [ "$DIFF" = true ]; } && [ "$SUBGPT" = true ] && [ -s "$SUBGPT_COOKIE" ]; then - start_subfunc ${FUNCNAME[0]} "Running : Permutations by BingGPT prediction" - subgpt -i ${dir}/subdomains/subdomains.txt -c $SUBGPT_COOKIE --dont-resolve -o ${dir}/.tmp/gpt_subs.txt 2>>"$LOGFILE" - if [ ! "$AXIOM" = true ]; then - resolvers_update_quick_local - [ -s "${dir}/.tmp/gpt_subs.txt" ] && puredns resolve ${dir}/.tmp/gpt_subs.txt -w .tmp/gpt_resolved.txt -r $resolvers --resolvers-trusted $resolvers_trusted -l $PUREDNS_PUBLIC_LIMIT --rate-limit-trusted $PUREDNS_TRUSTED_LIMIT --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT 2>>"$LOGFILE" >/dev/null - else - resolvers_update_quick_axiom - [ -s "${dir}/.tmp/gpt_subs.txt" ] && axiom-scan ${dir}/.tmp/gpt_subs.txt -m puredns-resolve -r /home/op/lists/resolvers.txt --resolvers-trusted /home/op/lists/resolvers_trusted.txt --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT -o .tmp/gpt_resolved.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null - fi - - if [ -s ".tmp/gpt_resolved.txt" ]; then - [ -s "$outOfScope_file" ] && deleteOutScoped $outOfScope_file .tmp/gpt_resolved.txt - [[ "$INSCOPE" = true ]] && check_inscope .tmp/gpt_resolved.txt 2>>"$LOGFILE" >/dev/null - NUMOFLINES=$(cat .tmp/gpt_resolved.txt 2>>"$LOGFILE" | grep ".$domain$" | anew subdomains/subdomains.txt | sed '/^$/d' | wc -l) - else - NUMOFLINES=0 - fi - end_subfunc "${NUMOFLINES} new subs (permutations)" ${FUNCNAME[0]} - else - if [ "$SUBGPT" = false ]; then - printf "\n${yellow} ${FUNCNAME[0]} skipped in this mode or defined in reconftw.cfg ${reset}\n" - elif [ ! -s "$SUBGPT_COOKIE" ]; then - printf "\n${yellow} ${FUNCNAME[0]} SUBGPT_COOKIE not defined on config file (reconftw.cfg by default) ${reset}\n" - else - printf "${yellow} ${FUNCNAME[0]} is already processed, to force executing ${FUNCNAME[0]} delete\n $called_fn_dir/.${FUNCNAME[0]} ${reset}\n\n" - fi - fi -} - function sub_recursive_passive(){ if { [ ! -f "$called_fn_dir/.${FUNCNAME[0]}" ] || [ "$DIFF" = true ]; } && [ "$SUB_RECURSIVE_PASSIVE" = true ] && [ -s "subdomains/subdomains.txt" ]; then start_subfunc ${FUNCNAME[0]} "Running : Subdomains recursive search passive" @@ -736,7 +726,7 @@ function sub_recursive_passive(){ [ -s ".tmp/passive_recursive.txt" ] && axiom-scan .tmp/passive_recursive.txt -m puredns-resolve -r /home/op/lists/resolvers.txt --resolvers-trusted /home/op/lists/resolvers_trusted.txt --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT -o .tmp/passive_recurs_tmp.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null fi [[ "$INSCOPE" = true ]] && check_inscope .tmp/passive_recurs_tmp.txt 2>>"$LOGFILE" >/dev/null - NUMOFLINES=$(cat .tmp/passive_recurs_tmp.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | sed '/^$/d' | anew subdomains/subdomains.txt | wc -l) + NUMOFLINES=$(cat .tmp/passive_recurs_tmp.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed '/^$/d' | anew subdomains/subdomains.txt | wc -l) end_subfunc "${NUMOFLINES} new subs (recursive)" ${FUNCNAME[0]} else if [ "$SUB_RECURSIVE_PASSIVE" = false ]; then @@ -802,8 +792,8 @@ function sub_recursive_brute(){ [ -s ".tmp/brute_recursive.txt" ] && axiom-scan .tmp/brute_perm_recursive.txt -m puredns-resolve -r /home/op/lists/resolvers.txt --resolvers-trusted /home/op/lists/resolvers_trusted.txt --wildcard-tests $PUREDNS_WILDCARDTEST_LIMIT --wildcard-batch $PUREDNS_WILDCARDBATCH_LIMIT -o .tmp/brute_perm_recursive_final.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null fi - NUMOFLINES=$(cat .tmp/brute_perm_recursive_final.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | sed '/^$/d' | anew subdomains/subdomains.txt | wc -l) - end_subfunc "${NUMOFLINES} new subs (recursive)" ${FUNCNAME[0]} + NUMOFLINES=$(cat .tmp/brute_perm_recursive_final.txt 2>>"$LOGFILE" | grep "\.$domain$\|^$domain$" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed '/^$/d' | anew subdomains/subdomains.txt | wc -l) + end_subfunc "${NUMOFLINES} new subs (recursive active)" ${FUNCNAME[0]} else if [ "$SUB_RECURSIVE_BRUTE" = false ]; then printf "\n${yellow} ${FUNCNAME[0]} skipped in this mode or defined in reconftw.cfg ${reset}\n" @@ -921,7 +911,7 @@ function webprobe_simple(){ axiom-scan subdomains/subdomains.txt -m httpx ${HTTPX_FLAGS} -no-color -json -random-agent -threads $HTTPX_THREADS -rl $HTTPX_RATELIMIT -retries 2 -timeout $HTTPX_TIMEOUT -o .tmp/web_full_info_probe.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null fi cat .tmp/web_full_info.txt .tmp/web_full_info_probe.txt webs/web_full_info.txt 2>>"$LOGFILE" | jq -s 'try .' | jq 'try unique_by(.input)' | jq 'try .[]' 2>>"$LOGFILE" > webs/web_full_info.txt - [ -s "webs/web_full_info.txt" ] && cat webs/web_full_info.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | anew -q .tmp/probed_tmp.txt + [ -s "webs/web_full_info.txt" ] && cat webs/web_full_info.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | anew -q .tmp/probed_tmp.txt [ -s "webs/web_full_info.txt" ] && cat webs/web_full_info.txt | jq -r 'try . |"\(.url) [\(.status_code)] [\(.title)] [\(.webserver)] \(.tech)"' | grep "$domain" | anew -q webs/web_full_info_plain.txt [ -s "$outOfScope_file" ] && deleteOutScoped $outOfScope_file .tmp/probed_tmp.txt NUMOFLINES=$(cat .tmp/probed_tmp.txt 2>>"$LOGFILE" | anew webs/webs.txt | sed '/^$/d' | wc -l) @@ -954,7 +944,7 @@ function webprobe_full(){ fi fi fi - [ -s ".tmp/web_full_info_uncommon.txt" ] && cat .tmp/web_full_info_uncommon.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | sed "s/*.//" | anew -q .tmp/probed_uncommon_ports_tmp.txt + [ -s ".tmp/web_full_info_uncommon.txt" ] && cat .tmp/web_full_info_uncommon.txt | jq -r 'try .url' 2>/dev/null | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | sed "s/*.//" | anew -q .tmp/probed_uncommon_ports_tmp.txt [ -s ".tmp/web_full_info_uncommon.txt" ] && cat .tmp/web_full_info_uncommon.txt | jq -r 'try . |"\(.url) [\(.status_code)] [\(.title)] [\(.webserver)] \(.tech)"' | anew -q webs/web_full_info_uncommon_plain.txt if [ -s ".tmp/web_full_info_uncommon.txt" ]; then if [[ $domain =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9] ]]; then @@ -1167,6 +1157,7 @@ function nuclei_check(){ do printf "${yellow}\n Running : Nuclei $crit, check results on nuclei_output folder${reset}\n\n" axiom-scan .tmp/webs_subs.txt -m nuclei --nuclei-templates ${NUCLEI_TEMPLATES_PATH} -severity ${crit} -nh -rl $NUCLEI_RATELIMIT -o nuclei_output/${crit}.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null + [ -s "nuclei_output/${crit}.txt" ] && cat nuclei_output/${crit}.txt done printf "\n\n" fi @@ -1313,11 +1304,11 @@ function urlchecks(){ fi [ -s ".tmp/katana.txt" ] && sed -i '/^.\{2048\}./d' .tmp/katana.txt [ -s ".tmp/katana.txt" ] && cat .tmp/katana.txt | anew -q .tmp/url_extract_tmp.txt - [ -s ".tmp/url_extract_tmp.txt" ] && cat .tmp/url_extract_tmp.txt | grep "${domain}" | grep -aEi "\.(js)" | anew -q .tmp/url_extract_js.txt + [ -s ".tmp/url_extract_tmp.txt" ] && cat .tmp/url_extract_tmp.txt | grep "${domain}" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | grep -aEi "\.(js)" | anew -q .tmp/url_extract_js.txt if [ "$DEEP" = true ]; then [ -s ".tmp/url_extract_js.txt" ] && interlace -tL .tmp/url_extract_js.txt -threads 10 -c "python3 $tools/JSA/jsa.py -f target | anew -q .tmp/url_extract_tmp.txt" &>/dev/null fi - [ -s ".tmp/url_extract_tmp.txt" ] && cat .tmp/url_extract_tmp.txt | grep "${domain}" | grep "=" | qsreplace -a 2>>"$LOGFILE" | grep -aEiv "\.(eot|jpg|jpeg|gif|css|tif|tiff|png|ttf|otf|woff|woff2|ico|pdf|svg|txt|js)$" | anew -q .tmp/url_extract_tmp2.txt + [ -s ".tmp/url_extract_tmp.txt" ] && cat .tmp/url_extract_tmp.txt | grep "${domain}" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | grep "=" | qsreplace -a 2>>"$LOGFILE" | grep -aEiv "\.(eot|jpg|jpeg|gif|css|tif|tiff|png|ttf|otf|woff|woff2|ico|pdf|svg|txt|js)$" | anew -q .tmp/url_extract_tmp2.txt [ -s ".tmp/url_extract_tmp2.txt" ] && cat .tmp/url_extract_tmp2.txt | python3 $tools/urless/urless/urless.py | anew -q .tmp/url_extract_uddup.txt 2>>"$LOGFILE" >/dev/null NUMOFLINES=$(cat .tmp/url_extract_uddup.txt 2>>"$LOGFILE" | anew webs/url_extract.txt | sed '/^$/d' | wc -l) notification "${NUMOFLINES} new urls with params" info @@ -1392,7 +1383,7 @@ function jschecks(){ if [ -s ".tmp/url_extract_js.txt" ]; then printf "${yellow} Running : Fetching Urls 1/5${reset}\n" if [ ! "$AXIOM" = true ]; then - cat .tmp/url_extract_js.txt | subjs -ua "Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0" -c 40 | grep "$domain" | anew -q .tmp/subjslinks.txt + cat .tmp/url_extract_js.txt | subjs -ua "Mozilla/5.0 (X11; Linux x86_64; rv:72.0) Gecko/20100101 Firefox/72.0" -c 40 | grep "$domain" | grep -E -i '^(((?!-))(xn--|_)?[a-z0-9-]{0,61}[a-z0-9]{1,1}\.)*(xn--)?([a-z0-9][a-z0-9\-]{0,60}|[a-z0-9-]{1,30}\.[a-z]{2,})' | anew -q .tmp/subjslinks.txt else axiom-scan .tmp/url_extract_js.txt -m subjs -o .tmp/subjslinks.txt $AXIOM_EXTRA_ARGS 2>>"$LOGFILE" >/dev/null fi @@ -1417,7 +1408,7 @@ function jschecks(){ if [ ! "$AXIOM" = true ]; then [ -s "js/js_livelinks.txt" ] && cat js/js_livelinks.txt | Mantra -ua ${HEADER} -s | anew -q js/js_secrets.txt else - [ -s "js/js_livelinks.txt" ] && axiom-scan js/js_livelinks.txt -m mantra -ua ${HEADER} -s -o js/js_secrets.txt $AXIOM_EXTRA_ARGS &>/dev/null + [ -s "js/js_livelinks.txt" ] && axiom-scan js/js_livelinks.txt -m mantra -ua \"${HEADER}\" -s -o js/js_secrets.txt $AXIOM_EXTRA_ARGS &>/dev/null fi [ -s "js/js_secrets.txt" ] && sed -r "s/\x1B\[([0-9]{1,3}(;[0-9]{1,2};?)?)?[mGK]//g" -i js/js_secrets.txt printf "${yellow} Running : Building wordlist 5/5${reset}\n" @@ -1952,7 +1943,7 @@ function getElapsedTime { function zipSnedOutputFolder { zip_name1=$(date +"%Y_%m_%d-%H.%M.%S") - zip_name="${zip_name1}_${domain}.zip" + zip_name="${zip_name1}_${domain}.zip" 2>>"$LOGFILE" >/dev/null (cd "$dir" && zip -r "$zip_name" .) echo "Sending zip file "${dir}/${zip_name}"" @@ -2352,7 +2343,6 @@ function passive(){ SUBSCRAPING=false SUBPERMUTE=false SUBREGEXPERMUTE=false - SUBGPT=false SUB_RECURSIVE_BRUTE=false WEBPROBESIMPLE=false if [ "$AXIOM" = true ]; then @@ -2410,6 +2400,7 @@ function vulns(){ spraying brokenLinks fuzzparams + 4xxbypass test_ssl fi } diff --git a/requirements.txt b/requirements.txt index 30305df9..e36da7fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,4 +34,4 @@ tldextract # dorks_hunter tqdm # multiple ujson # multiple urllib3 # multiple -subgpt # Tool +postleaksNeg # Tool