From ee90e3b0ce0d9ff590591d46bef6f484d7c81a76 Mon Sep 17 00:00:00 2001 From: nickvourd <32869078+nickvourd@users.noreply.github.com> Date: Mon, 16 Nov 2020 12:14:36 +0200 Subject: [PATCH] Add Roben v. 1.0.2 files --- README.md | 22 ++++++- requirments.txt | 3 + roben.py | 168 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 191 insertions(+), 2 deletions(-) create mode 100644 requirments.txt create mode 100644 roben.py diff --git a/README.md b/README.md index 63fb650..f8871e2 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,20 @@ -# roben -Robots entries parser and brute forcer. +# Roben + +### Robots entries parser and brute forcer +# +#### Version +##### 1.0.2 +# +#### Requirments +- requests +- bs4 +- validators + +# + +#### Install requirments +##### pip3 install -r requirments.txt + +# +##### Usage: +###### python3 roben.py http://example.com diff --git a/requirments.txt b/requirments.txt new file mode 100644 index 0000000..6a982bb --- /dev/null +++ b/requirments.txt @@ -0,0 +1,3 @@ +requests +bs4 +validators \ No newline at end of file diff --git a/roben.py b/roben.py new file mode 100644 index 0000000..5162a1d --- /dev/null +++ b/roben.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 + +import requests +from sys import argv, exit +from bs4 import BeautifulSoup +import argparse +import validators + +script, url = argv + +#variables area +__author__ = "@nickvourd" +__version__ = "1.0.2" +__license__ = "GPLv3" +__team__ = "@twelvesec" +__thanks__ = [ '@maldevel', 'servo' ] +bannerL = "========[" +bannerR = "]========" +cutter_list = [] + +ascii_art = ''' +___________ .__ _________ +\__ ___/_ _ __ ____ | |___ __ ____ / _____/ ____ ____ + | | \ \/ \/ // __ \| |\ \/ // __ \ \_____ \_/ __ \_/ ___\ + | | \ /\ ___/| |_\ /\ ___/ / \ ___/\ \___ + |____| \/\_/ \___ >____/\_/ \___ >_______ /\___ >\___ > + \/ \/ \/ \/ \/ +Robots Entries v.{} - Robots entries parser & brute forcer tool. +Robots Entries is an open source tool licensed under {}. +Written by: {} of {}. +Special thanks to {} & {}. +https://www.twelvesec.com/ +Please visit https://github.com/twelvesec/roben for more.. +'''.format(__version__, __license__, __author__, __team__, __thanks__[0], __thanks__[1]) + + +#classes area +class Bcolors: + OKBLUE = '\033[94m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + OKCYAN = '\033[00m' + + +#check_input function +def check_input(url, bold, fail, endc): + if not validators.url(url): + print(bold + fail + "[!] Url validation [FAILED]\n" + endc) + exit() + else: + flag = "TRUE" + + return flag + + +#check_file function +def check_file(url): + flag = "FALSE" + + if "robots.txt" in url: + flag = "TRUE" + + return flag + + +#create_url function +def create_url(flag, url): + final_url = url + if flag == "FALSE": + url2 = url + " " + for i in range(len(url2)): + if url2[i] == ' ': + if url2[i-1] == "/": + final_url = url + "robots.txt" + else: + final_url = url + "/robots.txt" + + return final_url + + +#main function +def main(): + + #call objects of the class + bcolor = Bcolors() + + print(bcolor.BOLD + bcolor.WARNING + ascii_art + bcolor.ENDC) + + #call check_input function + flag = check_input(argv[1], bcolor.BOLD, bcolor.FAIL, bcolor.ENDC) + + print(bcolor.BOLD + bcolor.OKBLUE + bannerL + "Url Checks" + bannerR + "\n" + bcolor.ENDC) + + if flag == "TRUE": + print(bcolor.BOLD + bcolor.OKGREEN + "[+] Url validation [OK]" + bcolor.ENDC) + + #call check_file function + flag2 = check_file(argv[1]) + + #call create_url function + url = create_url(flag2, argv[1]) + + print(bcolor.BOLD + bcolor.OKGREEN + "[+] String format [OK]" + bcolor.ENDC) + + r = requests.get(url) + + if r.status_code == 200: + print(bcolor.BOLD + bcolor.OKGREEN + "[+] Robots file exists [OK]\n" + bcolor.ENDC) + print(bcolor.BOLD + bcolor.OKBLUE + bannerL + "Robots File Content" + bannerR + "\n" + bcolor.ENDC) + + list = r.text + print(list + "\n") + + #save content of robots.txt to a file + with open('robots.dump.txt', 'w') as f: + f.write(list) + f.close + + print(bcolor.BOLD + bcolor.OKGREEN + "[*] Robots.txt file saved to robots.dump.txt" + bcolor.ENDC) + + #copy file output to a list + with open('robots.dump.txt', 'r') as f: + for line in f: + if 'Disallow:' in line: + cutter = line.split(":")[1] + cutter_list.append(cutter) + + #delete empty lines + trim_list = " ".join(cutter_list).split() + + print(bcolor.BOLD + bcolor.OKBLUE + "\n" + bannerL + "Responses of Robots file entries" + bannerR + "\n" + bcolor.ENDC) + + #create a file to save responses outputs + with open('responses.dump.txt', 'w') as f: + f.write("\n[+] Robots.txt responses:\n\n") + f.close + + for z in trim_list: + url = url.replace("/robots.txt","") + newurl = url + z + r2 = requests.get(newurl) + if r2.status_code == 200: + color = bcolor.OKGREEN + elif r2.status_code == 301 or r2.status_code == 302: + color = bcolor.OKCYAN + else: + color = bcolor.FAIL + + print(bcolor.BOLD + bcolor.OKBLUE + "[~] " + bcolor.BOLD + bcolor.WARNING + newurl + bcolor.BOLD + bcolor.OKBLUE + " -> " + bcolor.BOLD + color + str(r2.status_code) + bcolor.ENDC) + + #append responses to the output file + with open('responses.dump.txt', 'a') as f: + f.write("[*] " + newurl + " -> " + str(r2.status_code) + "\n") + f.close + + print(bcolor.BOLD + bcolor.OKGREEN + "\n\n[*] All responses saved to responses.dump.txt\n" + bcolor.ENDC) + print(bcolor.BOLD + bcolor.OKBLUE + "===================================================" + bcolor.ENDC) + else: + print(bcolor.BOLD + bcolor.FAIL + "[!] Robots.txt file doesn't exists [FAILED]\n" + bcolor.ENDC) + exit() + + +if __name__ == "__main__": + main() \ No newline at end of file