-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
3 changed files
with
191 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,2 +1,20 @@ | ||
# roben | ||
Robots entries parser and brute forcer. | ||
# Roben | ||
|
||
### Robots entries parser and brute forcer | ||
# | ||
#### Version | ||
##### 1.0.2 | ||
# | ||
#### Requirments | ||
- requests | ||
- bs4 | ||
- validators | ||
|
||
# | ||
|
||
#### Install requirments | ||
##### pip3 install -r requirments.txt | ||
|
||
# | ||
##### Usage: | ||
###### python3 roben.py http://example.com |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
requests | ||
bs4 | ||
validators |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,168 @@ | ||
#!/usr/bin/env python3 | ||
|
||
import requests | ||
from sys import argv, exit | ||
from bs4 import BeautifulSoup | ||
import argparse | ||
import validators | ||
|
||
script, url = argv | ||
|
||
#variables area | ||
__author__ = "@nickvourd" | ||
__version__ = "1.0.2" | ||
__license__ = "GPLv3" | ||
__team__ = "@twelvesec" | ||
__thanks__ = [ '@maldevel', 'servo' ] | ||
bannerL = "========[" | ||
bannerR = "]========" | ||
cutter_list = [] | ||
|
||
ascii_art = ''' | ||
___________ .__ _________ | ||
\__ ___/_ _ __ ____ | |___ __ ____ / _____/ ____ ____ | ||
| | \ \/ \/ // __ \| |\ \/ // __ \ \_____ \_/ __ \_/ ___\ | ||
| | \ /\ ___/| |_\ /\ ___/ / \ ___/\ \___ | ||
|____| \/\_/ \___ >____/\_/ \___ >_______ /\___ >\___ > | ||
\/ \/ \/ \/ \/ | ||
Robots Entries v.{} - Robots entries parser & brute forcer tool. | ||
Robots Entries is an open source tool licensed under {}. | ||
Written by: {} of {}. | ||
Special thanks to {} & {}. | ||
https://www.twelvesec.com/ | ||
Please visit https://github.com/twelvesec/roben for more.. | ||
'''.format(__version__, __license__, __author__, __team__, __thanks__[0], __thanks__[1]) | ||
|
||
|
||
#classes area | ||
class Bcolors: | ||
OKBLUE = '\033[94m' | ||
OKGREEN = '\033[92m' | ||
WARNING = '\033[93m' | ||
FAIL = '\033[91m' | ||
ENDC = '\033[0m' | ||
BOLD = '\033[1m' | ||
UNDERLINE = '\033[4m' | ||
OKCYAN = '\033[00m' | ||
|
||
|
||
#check_input function | ||
def check_input(url, bold, fail, endc): | ||
if not validators.url(url): | ||
print(bold + fail + "[!] Url validation [FAILED]\n" + endc) | ||
exit() | ||
else: | ||
flag = "TRUE" | ||
|
||
return flag | ||
|
||
|
||
#check_file function | ||
def check_file(url): | ||
flag = "FALSE" | ||
|
||
if "robots.txt" in url: | ||
flag = "TRUE" | ||
|
||
return flag | ||
|
||
|
||
#create_url function | ||
def create_url(flag, url): | ||
final_url = url | ||
if flag == "FALSE": | ||
url2 = url + " " | ||
for i in range(len(url2)): | ||
if url2[i] == ' ': | ||
if url2[i-1] == "/": | ||
final_url = url + "robots.txt" | ||
else: | ||
final_url = url + "/robots.txt" | ||
|
||
return final_url | ||
|
||
|
||
#main function | ||
def main(): | ||
|
||
#call objects of the class | ||
bcolor = Bcolors() | ||
|
||
print(bcolor.BOLD + bcolor.WARNING + ascii_art + bcolor.ENDC) | ||
|
||
#call check_input function | ||
flag = check_input(argv[1], bcolor.BOLD, bcolor.FAIL, bcolor.ENDC) | ||
|
||
print(bcolor.BOLD + bcolor.OKBLUE + bannerL + "Url Checks" + bannerR + "\n" + bcolor.ENDC) | ||
|
||
if flag == "TRUE": | ||
print(bcolor.BOLD + bcolor.OKGREEN + "[+] Url validation [OK]" + bcolor.ENDC) | ||
|
||
#call check_file function | ||
flag2 = check_file(argv[1]) | ||
|
||
#call create_url function | ||
url = create_url(flag2, argv[1]) | ||
|
||
print(bcolor.BOLD + bcolor.OKGREEN + "[+] String format [OK]" + bcolor.ENDC) | ||
|
||
r = requests.get(url) | ||
|
||
if r.status_code == 200: | ||
print(bcolor.BOLD + bcolor.OKGREEN + "[+] Robots file exists [OK]\n" + bcolor.ENDC) | ||
print(bcolor.BOLD + bcolor.OKBLUE + bannerL + "Robots File Content" + bannerR + "\n" + bcolor.ENDC) | ||
|
||
list = r.text | ||
print(list + "\n") | ||
|
||
#save content of robots.txt to a file | ||
with open('robots.dump.txt', 'w') as f: | ||
f.write(list) | ||
f.close | ||
|
||
print(bcolor.BOLD + bcolor.OKGREEN + "[*] Robots.txt file saved to robots.dump.txt" + bcolor.ENDC) | ||
|
||
#copy file output to a list | ||
with open('robots.dump.txt', 'r') as f: | ||
for line in f: | ||
if 'Disallow:' in line: | ||
cutter = line.split(":")[1] | ||
cutter_list.append(cutter) | ||
|
||
#delete empty lines | ||
trim_list = " ".join(cutter_list).split() | ||
|
||
print(bcolor.BOLD + bcolor.OKBLUE + "\n" + bannerL + "Responses of Robots file entries" + bannerR + "\n" + bcolor.ENDC) | ||
|
||
#create a file to save responses outputs | ||
with open('responses.dump.txt', 'w') as f: | ||
f.write("\n[+] Robots.txt responses:\n\n") | ||
f.close | ||
|
||
for z in trim_list: | ||
url = url.replace("/robots.txt","") | ||
newurl = url + z | ||
r2 = requests.get(newurl) | ||
if r2.status_code == 200: | ||
color = bcolor.OKGREEN | ||
elif r2.status_code == 301 or r2.status_code == 302: | ||
color = bcolor.OKCYAN | ||
else: | ||
color = bcolor.FAIL | ||
|
||
print(bcolor.BOLD + bcolor.OKBLUE + "[~] " + bcolor.BOLD + bcolor.WARNING + newurl + bcolor.BOLD + bcolor.OKBLUE + " -> " + bcolor.BOLD + color + str(r2.status_code) + bcolor.ENDC) | ||
|
||
#append responses to the output file | ||
with open('responses.dump.txt', 'a') as f: | ||
f.write("[*] " + newurl + " -> " + str(r2.status_code) + "\n") | ||
f.close | ||
|
||
print(bcolor.BOLD + bcolor.OKGREEN + "\n\n[*] All responses saved to responses.dump.txt\n" + bcolor.ENDC) | ||
print(bcolor.BOLD + bcolor.OKBLUE + "===================================================" + bcolor.ENDC) | ||
else: | ||
print(bcolor.BOLD + bcolor.FAIL + "[!] Robots.txt file doesn't exists [FAILED]\n" + bcolor.ENDC) | ||
exit() | ||
|
||
|
||
if __name__ == "__main__": | ||
main() |