From acef82a81d07c5b0e801d4d9cf6245a976bcdf46 Mon Sep 17 00:00:00 2001 From: epi Date: Mon, 26 Aug 2024 17:39:04 -0400 Subject: [PATCH] added --request-file, --protocol, --scan-dir-listings --- src/banner/container.rs | 27 + src/config/container.rs | 80 ++- src/config/mod.rs | 1 + src/config/tests.rs | 27 + src/config/utils.rs | 1056 +++++++++++++++++++++++++++++++++- src/main.rs | 4 +- src/parser.rs | 28 +- src/scan_manager/tests.rs | 2 + src/scanner/ferox_scanner.rs | 12 +- 9 files changed, 1202 insertions(+), 35 deletions(-) diff --git a/src/banner/container.rs b/src/banner/container.rs index a7fe5643..6eb52320 100644 --- a/src/banner/container.rs +++ b/src/banner/container.rs @@ -176,6 +176,12 @@ pub struct Banner { /// represents Configuration.collect_words force_recursion: BannerEntry, + + /// represents Configuration.protocol + protocol: BannerEntry, + + /// represents Configuration.scan_dir_listings + scan_dir_listings: BannerEntry, } /// implementation of Banner @@ -320,6 +326,12 @@ impl Banner { BannerEntry::new("🚫", "Do Not Recurse", &config.no_recursion.to_string()) }; + let protocol = if config.protocol.to_lowercase() == "http" { + BannerEntry::new("🔓", "Default Protocol", &config.protocol) + } else { + BannerEntry::new("🔒", "Default Protocol", &config.protocol) + }; + let scan_limit = BannerEntry::new( "🦥", "Concurrent Scan Limit", @@ -331,6 +343,11 @@ impl Banner { let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy); let auto_tune = BannerEntry::new("🎶", "Auto Tune", &config.auto_tune.to_string()); let auto_bail = BannerEntry::new("🙅", "Auto Bail", &config.auto_bail.to_string()); + let scan_dir_listings = BannerEntry::new( + "📂", + "Scan Dir Listings", + &config.scan_dir_listings.to_string(), + ); let cfg = BannerEntry::new("💉", "Config File", &config.config); let proxy = BannerEntry::new("💎", "Proxy", &config.proxy); let server_certs = BannerEntry::new( @@ -455,6 +472,8 @@ impl Banner { collect_words, dont_collect, config: cfg, + scan_dir_listings, + protocol, version: VERSION.to_string(), update_status: UpdateStatus::Unknown, } @@ -595,6 +614,10 @@ by Ben "epi" Risher {} ver: {}"#, } // followed by the maybe printed or variably displayed values + if !config.request_file.is_empty() || !config.target_url.starts_with("http") { + writeln!(&mut writer, "{}", self.protocol)?; + } + if !config.config.is_empty() { writeln!(&mut writer, "{}", self.config)?; } @@ -662,6 +685,10 @@ by Ben "epi" Risher {} ver: {}"#, writeln!(&mut writer, "{}", self.output)?; } + if config.scan_dir_listings { + writeln!(&mut writer, "{}", self.scan_dir_listings)?; + } + if !config.debug_log.is_empty() { writeln!(&mut writer, "{}", self.debug_log)?; } diff --git a/src/config/container.rs b/src/config/container.rs index 7bf8eb75..3e720840 100644 --- a/src/config/container.rs +++ b/src/config/container.rs @@ -1,7 +1,7 @@ use super::utils::{ - backup_extensions, depth, extract_links, ignored_extensions, methods, report_and_exit, - save_state, serialized_type, status_codes, threads, timeout, user_agent, wordlist, OutputLevel, - RequesterPolicy, + backup_extensions, depth, extract_links, ignored_extensions, methods, parse_request_file, + report_and_exit, request_protocol, save_state, serialized_type, split_header, split_query, + status_codes, threads, timeout, user_agent, wordlist, OutputLevel, RequesterPolicy, }; use crate::config::determine_output_level; use crate::config::utils::determine_requester_policy; @@ -332,6 +332,18 @@ pub struct Configuration { /// Auto update app feature #[serde(skip)] pub update_app: bool, + + /// whether to recurse into directory listings or not + #[serde(default)] + pub scan_dir_listings: bool, + + /// path to a raw request file generated by burp or similar + #[serde(skip)] + pub request_file: String, + + /// default request protocol + #[serde(default = "request_protocol")] + pub protocol: String, } impl Default for Configuration { @@ -378,6 +390,7 @@ impl Default for Configuration { resumed: false, stdin: false, json: false, + scan_dir_listings: false, verbosity: 0, scan_limit: 0, parallel: 0, @@ -403,6 +416,8 @@ impl Default for Configuration { time_limit: String::new(), resume_from: String::new(), replay_proxy: String::new(), + request_file: String::new(), + protocol: request_protocol(), server_certs: Vec::new(), queries: Vec::new(), extensions: Vec::new(), @@ -482,6 +497,9 @@ impl Configuration { /// - **replay_proxy**: `None` (no limit on concurrent scans imposed) /// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html) /// - **update_app**: `false` + /// - **scan_dir_listings**: `false` + /// - **request_file**: `None` + /// - **protocol**: `https` /// /// After which, any values defined in a /// [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file will override the @@ -555,6 +573,18 @@ impl Configuration { // merge the cli options into the config file options and return the result Self::merge_config(&mut config, cli_config); + // if the user provided a raw request file as the target, we'll need to parse out + // the provided info and update the config with those values. This call needs to + // come after the cli/config merge so we can allow the cli options to override + // the raw request values (i.e. --headers "stuff: things" should override a "stuff" + // header from the raw request). + // + // Additionally, this call needs to come before client rebuild so that the things + // like user-agent can be set at the client level instead of the header level. + if !config.request_file.is_empty() { + parse_request_file(&mut config)?; + } + // rebuild clients is the last step in either code branch Self::try_rebuild_clients(&mut config); @@ -618,6 +648,8 @@ impl Configuration { update_config_if_present!(&mut config.output, args, "output", String); update_config_if_present!(&mut config.debug_log, args, "debug_log", String); update_config_if_present!(&mut config.resume_from, args, "resume_from", String); + update_config_if_present!(&mut config.request_file, args, "request_file", String); + update_config_if_present!(&mut config.protocol, args, "protocol", String); if let Ok(Some(inner)) = args.try_get_one::("time_limit") { inner.clone_into(&mut config.time_limit); @@ -831,6 +863,10 @@ impl Configuration { config.save_state = false; } + if came_from_cli!(args, "scan_dir_listings") || came_from_cli!(args, "thorough") { + config.scan_dir_listings = true; + } + if came_from_cli!(args, "dont_filter") { config.dont_filter = true; } @@ -932,23 +968,11 @@ impl Configuration { if let Some(headers) = args.get_many::("headers") { for val in headers { - let mut split_val = val.split(':'); - - // explicitly take first split value as header's name - let name = split_val.next().unwrap().trim(); - - // all other items in the iterator returned by split, when combined with the - // original split deliminator (:), make up the header's final value - let value = split_val.collect::>().join(":"); - - if value.starts_with(' ') && !value.starts_with(" ") { - // first character is a space and the second character isn't - // we can trim the leading space - let trimmed = value.trim_start(); - config.headers.insert(name.to_string(), trimmed.to_string()); - } else { - config.headers.insert(name.to_string(), value.to_string()); - } + let Ok((name, value)) = split_header(val) else { + log::warn!("Invalid header: {}", val); + continue; + }; + config.headers.insert(name, value); } } @@ -982,14 +1006,11 @@ impl Configuration { if let Some(queries) = args.get_many::("queries") { for val in queries { - // same basic logic used as reading in the headers HashMap above - let mut split_val = val.split('='); - - let name = split_val.next().unwrap().trim(); - - let value = split_val.collect::>().join("="); - - config.queries.push((name.to_string(), value.to_string())); + let Ok((name, value)) = split_query(val) else { + log::warn!("Invalid query string: {}", val); + continue; + }; + config.queries.push((name, value)); } } @@ -1171,12 +1192,15 @@ impl Configuration { Vec::::new() ); update_if_not_default!(&mut conf.dont_filter, new.dont_filter, false); + update_if_not_default!(&mut conf.scan_dir_listings, new.scan_dir_listings, false); update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0); update_if_not_default!(&mut conf.parallel, new.parallel, 0); update_if_not_default!(&mut conf.rate_limit, new.rate_limit, 0); update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, ""); update_if_not_default!(&mut conf.debug_log, new.debug_log, ""); update_if_not_default!(&mut conf.resume_from, new.resume_from, ""); + update_if_not_default!(&mut conf.request_file, new.request_file, ""); + update_if_not_default!(&mut conf.protocol, new.protocol, request_protocol()); update_if_not_default!(&mut conf.timeout, new.timeout, timeout()); update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent()); diff --git a/src/config/mod.rs b/src/config/mod.rs index 275473c7..a3f8e0de 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -2,6 +2,7 @@ mod container; mod utils; +mod raw_request; #[cfg(test)] mod tests; diff --git a/src/config/tests.rs b/src/config/tests.rs index cbde9312..d95d3102 100644 --- a/src/config/tests.rs +++ b/src/config/tests.rs @@ -49,6 +49,9 @@ fn setup_config_test() -> Configuration { json = true save_state = false depth = 1 + protocol = "http" + request_file = "/some/request/file" + scan_dir_listings = true force_recursion = true filter_size = [4120] filter_regex = ["^ignore me$"] @@ -107,6 +110,7 @@ fn default_configuration() { assert!(!config.collect_extensions); assert!(!config.collect_backups); assert!(!config.collect_words); + assert!(!config.scan_dir_listings); assert!(config.regex_denylist.is_empty()); assert_eq!(config.queries, Vec::new()); assert_eq!(config.filter_size, Vec::::new()); @@ -125,6 +129,8 @@ fn default_configuration() { assert_eq!(config.client_cert, String::new()); assert_eq!(config.client_key, String::new()); assert_eq!(config.backup_extensions, backup_extensions()); + assert_eq!(config.protocol, request_protocol()); + assert_eq!(config.request_file, String::new()); } #[test] @@ -444,6 +450,27 @@ fn config_reads_time_limit() { assert_eq!(config.time_limit, "10m"); } +#[test] +/// parse the test config and see that the value parsed is correct +fn config_reads_scan_dir_listings() { + let config = setup_config_test(); + assert!(config.scan_dir_listings); +} + +#[test] +/// parse the test config and see that the value parsed is correct +fn config_reads_protocol() { + let config = setup_config_test(); + assert_eq!(config.protocol, "http"); +} + +#[test] +/// parse the test config and see that the value parsed is correct +fn config_reads_request_file() { + let config = setup_config_test(); + assert_eq!(config.request_file, String::new()); +} + #[test] /// parse the test config and see that the value parsed is correct fn config_reads_resume_from() { diff --git a/src/config/utils.rs b/src/config/utils.rs index f3bd3b44..128249d9 100644 --- a/src/config/utils.rs +++ b/src/config/utils.rs @@ -1,8 +1,12 @@ +use super::Configuration; use crate::{ - utils::{module_colorizer, status_colorizer}, + utils::{module_colorizer, parse_url_with_raw_path, status_colorizer}, DEFAULT_BACKUP_EXTENSIONS, DEFAULT_IGNORED_EXTENSIONS, DEFAULT_METHOD, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION, }; +use anyhow::{bail, Result}; +use std::collections::HashMap; + #[cfg(not(test))] use std::process::exit; @@ -45,6 +49,11 @@ pub(super) fn threads() -> usize { 50 } +/// default protocol value +pub(super) fn request_protocol() -> String { + String::from("https") +} + /// default status codes pub(super) fn status_codes() -> Vec { DEFAULT_STATUS_CODES @@ -179,9 +188,449 @@ pub fn determine_requester_policy(auto_tune: bool, auto_bail: bool) -> Requester } } +/// Splits a query string into a key-value pair. +/// +/// This function takes a query string in the format of `"key=value"` and splits it into +/// a tuple containing the key and value as separate strings. If the query string is +/// malformed (e.g., empty or without a key), it returns an error. +/// +/// # Arguments +/// +/// * `query` - A string slice that holds the query string to be split. +/// +/// # Returns +/// +/// * `Result<(String, String)>` - A tuple containing the key and value as `String`s, +/// or an error if the input is invalid. +/// +/// # Errors +/// +/// This function will return an error if: +/// * The input string is empty or equal to `"="`. +/// * The key part of the query string is empty (i.e., if the string starts with `"="`). +/// +/// # Examples +/// +/// ``` +/// let result = split_query("name=John"); +/// assert_eq!(result.unwrap(), ("name".to_string(), "John".to_string())); +/// +/// let result = split_query("name="); +/// assert_eq!(result.unwrap(), ("name".to_string(), "".to_string())); +/// +/// let result = split_query("name=John=Doe"); +/// assert_eq!(result.unwrap(), ("name".to_string(), "John=Doe".to_string())); +/// +/// let result = split_query("=John"); +/// assert!(result.is_err()); +/// +/// let result = split_query(""); +/// assert!(result.is_err()); +/// ``` +pub fn split_query(query: &str) -> Result<(String, String)> { + if query.is_empty() || query == "=" { + bail!("Empty query string provided"); + } + + let mut split_val = query.split('='); + + let name = split_val.next().unwrap().trim(); + + if name.is_empty() { + bail!("Empty key in query string"); + } + + let value = split_val.collect::>().join("="); + + Ok((name.to_string(), value.to_string())) +} + +/// Splits an HTTP header string into a key-value pair. +/// +/// This function takes a header string in the format of `"Key: Value"` and splits it into +/// a tuple containing the key and value as separate strings. If the header string is +/// malformed (e.g., empty or missing a key), it returns an error. +/// +/// # Arguments +/// +/// * `header` - A string slice that holds the header string to be split. +/// +/// # Returns +/// +/// * `Result<(String, String)>` - A tuple containing the key and value as `String`s, +/// or an error if the input is invalid. +/// +/// # Errors +/// +/// This function will return an error if: +/// * The input string is empty. +/// * The key part of the header string is empty (i.e., if the string starts with `":"`). +/// +/// # Examples +/// +/// ``` +/// let result = split_header("Content-Type: application/json"); +/// assert_eq!(result.unwrap(), ("Content-Type".to_string(), "application/json".to_string())); +/// +/// let result = split_header("Content-Length: 1234"); +/// assert_eq!(result.unwrap(), ("Content-Length".to_string(), "1234".to_string())); +/// +/// let result = split_header("Authorization: Bearer token"); +/// assert_eq!(result.unwrap(), ("Authorization".to_string(), "Bearer token".to_string())); +/// +/// let result = split_header("InvalidHeader"); +/// assert!(result.is_err()); +/// +/// let result = split_header(""); +/// assert!(result.is_err()); +/// ``` +pub fn split_header(header: &str) -> Result<(String, String)> { + if header.is_empty() { + bail!("Empty header provided"); + } + + let mut split_val = header.split(':'); + + // explicitly take first split value as header's name + let name = split_val.next().unwrap().trim().to_string(); + + if name.is_empty() { + bail!("Empty header name provided"); + } + + // all other items in the iterator returned by split, when combined with the + // original split deliminator (:), make up the header's final value + let value = split_val.collect::>().join(":"); + + if value.starts_with(' ') && !value.starts_with(" ") { + // first character is a space and the second character isn't + // we can trim the leading space + let trimmed = value.trim_start(); + Ok((name, trimmed.to_string())) + } else { + Ok((name, value)) + } +} + +/// Combines two `Cookie` header strings into a single, unified `Cookie` header string. +/// +/// The function parses both input strings into individual key-value pairs, ensuring that each +/// key is unique. If a key appears in both input strings, the value from the second string +/// will override the value from the first string. The resulting combined `Cookie` header string +/// is returned with all key-value pairs separated by `;`. +/// +/// # Arguments +/// +/// * `cookie1` - A string slice representing the first `Cookie` header. +/// * `cookie2` - A string slice representing the second `Cookie` header. +/// +/// # Returns +/// +/// * A `String` containing the combined `Cookie` header with unique keys. +/// +/// # Example +/// +/// ``` +/// let cookie1 = "super=duper; stuff=things"; +/// let cookie2 = "stuff=mothings; derp=tronic"; +/// let combined_cookie = combine_cookies(cookie1, cookie2); +/// assert_eq!(combined_cookie, "super=duper; stuff=mothings; derp=tronic"); +/// ``` +/// +/// The output string will contain all unique keys from both input strings, with the value +/// from the second string taking precedence in the case of key collisions. +fn combine_cookies(cookie1: &str, cookie2: &str) -> String { + let mut cookie_map = HashMap::new(); + + // Helper function to parse a cookie string and insert it into the map + let parse_cookie = |cookie_str: &str, map: &mut HashMap| { + for pair in cookie_str.split(';') { + let mut key_value = pair.trim().splitn(2, '='); + if let (Some(key), Some(value)) = (key_value.next(), key_value.next()) { + map.insert(key.to_string(), value.to_string()); + } + } + }; + + // Parse both cookie strings into the map + parse_cookie(cookie1, &mut cookie_map); + parse_cookie(cookie2, &mut cookie_map); + + // Build the final cookie header string + cookie_map + .into_iter() + .map(|(key, value)| format!("{}={}", key, value)) + .collect::>() + .join("; ") +} + +/// Parses a raw HTTP request from a file and updates the provided configuration. +/// +/// This function reads an HTTP request from the file specified by `config.request_file`, +/// parses the request line, headers, and body, and updates the `config` object +/// with the parsed values. If certain elements (e.g., headers or body) are +/// already provided via the CLI, they take precedence over the parsed values. +/// +/// # Arguments +/// +/// * `config` - A mutable reference to a `Configuration` object that will be +/// updated with the parsed request data. +/// +/// # Returns +/// +/// * `Result<()>` - Returns `Ok(())` if parsing and configuration updates +/// were successful, or an error if the raw file or request is invalid. +/// +/// # Errors +/// +/// This function will return an error if: +/// * The file specified in `config.request_file` is empty. +/// * The request is malformed (e.g., missing the request line, method, or URI). +/// * Required headers are missing (e.g., `Host` when the request line URI is not a full URL). +/// +/// # Details +/// +/// * The request body is only set if it hasn't been overridden by the CLI options. +/// * The request line method is added to `config.methods` if it's not already present. +/// * Headers from the raw request are added to `config.headers`, unless overridden +/// by CLI options. Special handling is applied to `User-Agent`, `Content-Length`, +/// and `Cookie` headers. +/// * The request URI is validated and parsed. If it's not a full URL, it will be +/// combined with the `Host` header to form a full target URL. +/// * Query parameters are extracted from the URI and added to `config.queries`, +/// unless overridden by CLI options. +/// +/// # Examples +/// +/// ```rust +/// let mut config = Configuration::default(); +/// config.request_file = "path/to/raw/request.txt".to_string(); +/// +/// let result = parse_request_file(&mut config); +/// assert!(result.is_ok()); +/// assert_eq!(config.methods, vec!["GET".to_string()]); +/// assert_eq!(config.target_url, "http://example.com/path".to_string()); +/// assert_eq!(config.headers.get("User-Agent").unwrap(), "MyCustomAgent"); +/// assert_eq!(config.data, b"key=value".to_vec()); +/// ``` +pub fn parse_request_file(config: &mut Configuration) -> Result<()> { + // read in the file located at config.request_file + // parse the file into a Request struct + let contents = std::fs::read_to_string(&config.request_file)?; + + if contents.is_empty() { + bail!("Empty --request-file file provided"); + } + + // this should split the body from the request line and headers + let lines = contents.split("\r\n\r\n").collect::>(); + + if lines.len() < 2 { + bail!("Invalid request: Missing head/body CRLF separator"); + } + + let head = lines[0]; + let body = lines[1].as_bytes().to_vec(); + + // we only want to use the request's body if the user hasn't + // overridden it on the cli + if config.data.is_empty() { + config.data = body; + } + + // begin parsing the request line and headers + let mut head_parts = head.split("\r\n"); + + let Some(request_line) = head_parts.next() else { + bail!("Invalid request: Missing request line"); + }; + + if request_line.is_empty() { + bail!("Invalid request: Empty request line"); + } + + let mut request_parts = request_line.split_whitespace(); + + let Some(method) = request_parts.next() else { + bail!("Invalid request: Missing method"); + }; + + if method.is_empty() { + bail!("Invalid request: Empty method"); + } + + let method = method.to_string(); + + if !config.methods.contains(&method) { + config.methods.push(method); + } + + let Some(uri) = request_parts.next() else { + bail!("Invalid request: Missing request line URI"); + }; + + if uri.is_empty() { + bail!("Invalid request: Empty request line URI"); + } + + for mut line in head_parts { + line = line.trim(); + + if line.is_empty() { + break; // Empty line signals the end of headers + } + + let Ok((name, value)) = split_header(line) else { + log::warn!("Invalid header: {}", line); + continue; + }; + + if name.is_empty() { + log::warn!("Invalid header name: {}", line); + continue; + } + + if name.to_lowercase() == "user-agent" { + if config.user_agent == user_agent() { + config.user_agent = value; + } + continue; + } + + if name.to_lowercase() == "content-length" { + log::debug!("Skipping content-length header, a new one will be created"); + continue; + } + + if config.headers.contains_key(&name) { + if name.to_lowercase() == "cookie" { + // the cookie header already exists, so we need to extend it with + // our values and ensure cli-provided cookie values override those + // from the request + let existing = config.headers.get_mut(&name).unwrap(); + // second param takes precedence over first + let combined = combine_cookies(&value, existing); + *existing = combined; + continue; + } + log::debug!("Found header from cli, overriding raw request with cli entry: {name}"); + continue; + } + + config.headers.insert(name, value); + } + + let url = parse_url_with_raw_path(uri); + + if url.is_err() { + // uri in request line is not a valid URL, so it's most likely a path/relative url + // we need to combine it with the host header + for (key, value) in &config.headers { + if key.to_lowercase() == "host" { + config.target_url = format!("{}{}", value, uri); + break; + } + } + + if config.target_url.is_empty() { + bail!("Invalid request: Missing Host header and request line URI isn't a full URL"); + } + + // need to parse queries from the uri, if any are present + let mut uri_parts = uri.splitn(2, '?'); + + // skip the path + uri_parts.next(); + + if let Some(queries) = uri_parts.next() { + let query_parts = queries.split("&"); + + query_parts.into_iter().for_each(|query| { + let Ok((name, value)) = split_query(query) else { + return; + }; + for (k, _) in &config.queries { + if k.to_lowercase() == name.to_lowercase() { + // allow cli options to take precedent when query names match + return; + } + } + + config.queries.push((name, value)); + }); + } + } else { + let mut url = url.unwrap(); + + if let Some(host) = config.headers.get("Host") { + url.set_host(Some(host)).unwrap(); + } + + url.query_pairs().for_each(|(key, value)| { + for (k, _) in &config.queries { + if k.to_lowercase() == key.to_lowercase() { + // allow cli options to take precedent when query names match + return; + } + } + + config.queries.push((key.to_string(), value.to_string())); + }); + + url.set_query(None); + url.set_fragment(None); + + config.target_url = url.to_string(); + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; + use std::env; + use std::fs::{self, File}; + use std::io::{self, Write}; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + struct TempSetup { + pub path: PathBuf, + pub config: Configuration, + pub file: File, + } + + impl TempSetup { + pub fn new() -> Self { + let mut temp_dir: PathBuf = env::temp_dir(); + + temp_dir.push(format!( + "temp_request_file_{}.txt", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + + let config: Configuration = Configuration { + request_file: temp_dir.to_str().unwrap().to_string(), + ..Default::default() + }; + + let file = File::create(&temp_dir).unwrap(); + + Self { + path: temp_dir, + config, + file, + } + } + + pub fn cleanup(self) { + fs::remove_file(self.path).unwrap(); + } + } #[test] /// test determine_output_level returns higher of the two levels if both given values are true @@ -233,4 +682,609 @@ mod tests { fn report_and_exit_panics_under_test() { report_and_exit("test"); } + + #[test] + fn test_split_query_simple() { + let query = "name=value"; + let result = split_query(query).unwrap(); + assert_eq!(result, ("name".to_string(), "value".to_string())); + } + + #[test] + fn test_split_query_with_spaces() { + let query = " name = value "; + let result = split_query(query).unwrap(); + assert_eq!(result, ("name".to_string(), " value ".to_string())); + } + + #[test] + fn test_split_query_empty_value() { + let query = "name="; + let result = split_query(query).unwrap(); + assert_eq!(result, ("name".to_string(), "".to_string())); + } + + #[test] + fn test_split_query_no_value() { + let query = "name"; + let result = split_query(query).unwrap(); + assert_eq!(result, ("name".to_string(), "".to_string())); + } + + #[test] + fn test_split_query_multiple_equals() { + let query = "name=value=another"; + let result = split_query(query).unwrap(); + assert_eq!(result, ("name".to_string(), "value=another".to_string())); + } + + #[test] + fn test_split_query_empty_key_and_value() { + let query = "="; + let result = split_query(query); + assert!(result.is_err()); + } + + #[test] + fn test_split_query_empty_key() { + let query = "=value"; + let result = split_query(query); + assert!(result.is_err()); + } + + #[test] + fn test_split_query_trailing_equals_in_value() { + let query = "name=value="; + let result = split_query(query).unwrap(); + assert_eq!(result, ("name".to_string(), "value=".to_string())); + } + + #[test] + fn test_split_query_no_equals() { + let query = "just_a_key"; + let result = split_query(query).unwrap(); + assert_eq!(result, ("just_a_key".to_string(), "".to_string())); + } + + #[test] + fn test_split_query_empty_input() { + let query = ""; + assert!(split_query(query).is_err()); + } + + #[test] + fn test_split_header_simple() -> Result<()> { + let header = "Content-Type: text/html"; + let result = split_header(header)?; + assert_eq!( + result, + ("Content-Type".to_string(), "text/html".to_string()) + ); + Ok(()) + } + + #[test] + fn test_split_header_with_leading_space_in_value() -> Result<()> { + let header = "Content-Type: text/html"; + let result = split_header(header)?; + assert_eq!( + result, + ("Content-Type".to_string(), " text/html".to_string()) + ); + Ok(()) + } + + #[test] + fn test_split_header_with_trimmed_leading_space() -> Result<()> { + let header = "Content-Type: text/html"; + let result = split_header(header)?; + assert_eq!( + result, + ("Content-Type".to_string(), "text/html".to_string()) + ); + Ok(()) + } + + #[test] + fn test_split_header_with_multiple_colons() -> Result<()> { + let header = "Date: Mon, 27 Jul 2009 12:28:53 GMT"; + let result = split_header(header)?; + assert_eq!( + result, + ( + "Date".to_string(), + "Mon, 27 Jul 2009 12:28:53 GMT".to_string() + ) + ); + Ok(()) + } + + #[test] + fn test_split_header_empty_value() -> Result<()> { + let header = "X-Custom-Header: "; + let result = split_header(header)?; + assert_eq!(result, ("X-Custom-Header".to_string(), "".to_string())); + Ok(()) + } + + #[test] + fn test_split_header_no_value() -> Result<()> { + let header = "X-Custom-Header:"; + let result = split_header(header)?; + assert_eq!(result, ("X-Custom-Header".to_string(), "".to_string())); + Ok(()) + } + + #[test] + fn test_split_header_no_colon() -> Result<()> { + let header = "InvalidHeader"; + let result = split_header(header)?; + assert_eq!(result, ("InvalidHeader".to_string(), "".to_string())); + Ok(()) + } + + #[test] + fn test_split_header_empty_key() { + let header = ": value"; + let result = split_header(header); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Empty header name provided" + ); + } + + #[test] + fn test_split_header_empty_key_and_value() { + let header = ": "; + let result = split_header(header); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Empty header name provided" + ); + } + + #[test] + fn test_split_header_empty_input() { + let header = ""; + let result = split_header(header); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().to_string(), "Empty header provided"); + } + + #[test] + fn test_split_header_value_with_leading_single_space() -> Result<()> { + let header = "Authorization: Bearer token"; + let result = split_header(header)?; + assert_eq!( + result, + ("Authorization".to_string(), "Bearer token".to_string()) + ); + Ok(()) + } + + #[test] + fn test_split_header_value_with_leading_multiple_spaces() -> Result<()> { + let header = "Authorization: Bearer token"; + let result = split_header(header)?; + assert_eq!( + result, + ("Authorization".to_string(), " Bearer token".to_string()) + ); + Ok(()) + } + + #[test] + fn test_parse_raw_with_empty_request() { + let mut config = Configuration::new().unwrap(); + let result = parse_request_file(&mut config); + assert!(result.is_err()); + } + #[test] + fn test_parse_raw_with_empty_file() -> io::Result<()> { + let mut tmp = TempSetup::new(); + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Empty --request-file file provided" + ); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_without_head_body_crlf() -> io::Result<()> { + let mut tmp = TempSetup::new(); + + write!(tmp.file, "GET / HTTP/1.1\r\n")?; + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Invalid request: Missing head/body CRLF separator" + ); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_with_only_head_body_crlf() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + writeln!(tmp.file, "\r\n\r\n")?; + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Invalid request: Empty request line" + ); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_body_is_overridden_by_cli() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!( + tmp.file, + "GET http://localhost/srv HTTP/1.0\r\n\r\nrequest-body" + )?; + + parse_request_file(&mut tmp.config).unwrap(); + assert_eq!(tmp.config.data, b"request-body".to_vec()); + + tmp.config.data = b"cli-data".to_vec(); + + parse_request_file(&mut tmp.config).unwrap(); + assert_eq!(tmp.config.data, b"cli-data".to_vec()); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_with_empty_request_line() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!(tmp.file, "\r\nHost: example.com\r\n\r\n")?; + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Invalid request: Empty request line" + ); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_with_missing_uri() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!(tmp.file, "GET\r\nHost: example.com\r\n\r\n")?; + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Invalid request: Missing request line URI" + ); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_with_missing_method() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!(tmp.file, " \r\nHost: example.com\r\n\r\n")?; + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Invalid request: Missing method" + ); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_methods_are_appended_if_unique() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!( + tmp.file, + "POST / HTTP/1.1\r\nHost: example.com\r\nUser-Agent: test-agent\r\n\r\n" + )?; + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_ok()); + assert_eq!(tmp.config.methods, vec!["GET", "POST"]); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_methods_are_ignored_if_already_present_from_cli() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!( + tmp.file, + "GET / HTTP/1.1\r\nHost: example.com\r\nUser-Agent: test-agent\r\n\r\n" + )?; + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_ok()); + assert_eq!(tmp.config.methods, vec!["GET"]); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_headers_added_to_config_if_missing_else_overridden_from_cli() -> io::Result<()> + { + let mut tmp: TempSetup = TempSetup::new(); + + // header from cli + tmp.config + .headers + .insert(String::from("stuff"), String::from("things")); + + // stuff header will be overridden by the one in the cli config (i.e. the raw request's + // stuff header will be ignored because of the cli config) + write!( + tmp.file, + "GET / HTTP/1.1\r\nHost: example.com\r\nstuff: mothings\r\n\r\n" + )?; + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_ok()); + assert!(tmp.config.headers.contains_key("Host")); + assert_eq!(tmp.config.headers.get("stuff").unwrap(), "things"); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_with_user_agent_in_request() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!( + tmp.file, + "GET / HTTP/1.1\r\nHost: example.com\r\nUser-Agent: test-agent\r\n\r\n" + )?; + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_ok()); + assert_eq!(tmp.config.user_agent, "test-agent"); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_with_user_agent_in_request_and_cli() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!( + tmp.file, + "GET / HTTP/1.1\r\nHost: example.com\r\nUser-Agent: test-agent\r\n\r\n" + )?; + + tmp.config.user_agent = "cli-agent".to_string(); + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_ok()); + assert_eq!(tmp.config.user_agent, "cli-agent"); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_content_length_is_always_skipped() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!( + tmp.file, + "GET / HTTP/1.1\r\nHost: example.com\r\nContent-length: 21\r\n\r\n" + )?; + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_ok()); + assert!(!tmp.config.headers.contains_key("Content-length")); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_cookie_header_appended_or_overridden() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!( + tmp.file, + "GET / HTTP/1.1\r\nHost: example.com\r\nCookie: derp=tronic2; super=duper2\r\n\r\n" + )?; + + tmp.config.headers.insert( + "Cookie".to_string(), + "derp=tronic; stuff=things".to_string(), + ); + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_ok()); + + let cookies = tmp.config.headers.get("Cookie").unwrap(); + + assert!(cookies.contains("derp=tronic")); + assert!(cookies.contains("stuff=things")); + assert!(cookies.contains("super=duper2")); + + // got overridden + assert!(!cookies.contains("derp=tronic2")); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_with_relative_path_and_partial_host_header() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!(tmp.file, "GET /srv HTTP/1.1\r\nHost: example.com\r\n\r\n")?; + + let result = parse_request_file(&mut tmp.config); + + assert!(result.is_ok()); + assert_eq!(tmp.config.target_url, "example.com/srv"); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_with_relative_path_and_no_host_header() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!(tmp.file, "GET /srv HTTP/1.1\r\n\r\n")?; + + let result: std::result::Result<(), anyhow::Error> = parse_request_file(&mut tmp.config); + + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Invalid request: Missing Host header and request line URI isn't a full URL" + ); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_with_full_url_and_no_host_header() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!(tmp.file, "GET http://localhost/srv HTTP/1.1\r\n\r\n")?; + + let result: std::result::Result<(), anyhow::Error> = parse_request_file(&mut tmp.config); + + assert!(result.is_ok()); + assert_eq!(tmp.config.target_url, "http://localhost/srv"); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_with_full_url_and_host_header() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!( + tmp.file, + "GET http://localhost/srv HTTP/1.1\r\nHost: example.com\r\n\r\n" + )?; + + let result: std::result::Result<(), anyhow::Error> = parse_request_file(&mut tmp.config); + + assert!(result.is_ok()); + assert_eq!(tmp.config.target_url, "http://example.com/srv"); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_with_partial_url_and_queries() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!( + tmp.file, + "GET /srv?mostuff=mothings&derp=tronic2 HTTP/1.1\r\nHost: example.com\r\n\r\n" + )?; + + tmp.config + .queries + .push(("derp".to_string(), "tronic".to_string())); + tmp.config + .queries + .push(("stuff".to_string(), "things".to_string())); + + let result: std::result::Result<(), anyhow::Error> = parse_request_file(&mut tmp.config); + + assert!(result.is_ok()); + assert_eq!( + tmp.config.queries, + vec![ + (String::from("derp"), String::from("tronic")), + (String::from("stuff"), String::from("things")), + (String::from("mostuff"), String::from("mothings")) + ] + ); + + tmp.cleanup(); + Ok(()) + } + + #[test] + fn test_parse_raw_with_full_url_and_queries() -> io::Result<()> { + let mut tmp: TempSetup = TempSetup::new(); + + write!( + tmp.file, + "GET http://localhost/srv?mostuff=mothings&derp=tronic2 HTTP/1.1\r\nHost: example.com\r\n\r\n" + )?; + + tmp.config + .queries + .push(("derp".to_string(), "tronic".to_string())); + tmp.config + .queries + .push(("stuff".to_string(), "things".to_string())); + + let result: std::result::Result<(), anyhow::Error> = parse_request_file(&mut tmp.config); + + assert!(result.is_ok()); + assert_eq!( + tmp.config.queries, + vec![ + (String::from("derp"), String::from("tronic")), + (String::from("stuff"), String::from("things")), + (String::from("mostuff"), String::from("mothings")) + ] + ); + + tmp.cleanup(); + Ok(()) + } } diff --git a/src/main.rs b/src/main.rs index 0e779e16..ec2cddcf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -197,9 +197,9 @@ async fn get_targets(handles: Arc) -> Result> { } } - if !target.starts_with("http") && !target.starts_with("https") { + if !target.starts_with("http") { // --url hackerone.com - *target = format!("https://{target}"); + *target = format!("{}://{target}", handles.config.protocol); } } diff --git a/src/parser.rs b/src/parser.rs index 8b4a2525..55669e60 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -40,12 +40,12 @@ pub fn initialize() -> Command { Arg::new("url") .short('u') .long("url") - .required_unless_present_any(["stdin", "resume_from", "update_app"]) + .required_unless_present_any(["stdin", "resume_from", "update_app", "request_file"]) .help_heading("Target selection") .value_name("URL") .use_value_delimiter(true) .value_hint(ValueHint::Url) - .help("The target URL (required, unless [--stdin || --resume-from] used)"), + .help("The target URL (required, unless [--stdin || --resume-from || --request-file] used)"), ) .arg( Arg::new("stdin") @@ -64,6 +64,15 @@ pub fn initialize() -> Command { .help("State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)") .conflicts_with("url") .num_args(1), + ).arg( + Arg::new("request_file") + .long("request-file") + .help_heading("Target selection") + .value_hint(ValueHint::FilePath) + .conflicts_with("url") + .num_args(1) + .value_name("REQUEST_FILE") + .help("Raw HTTP request file to use as a template for all requests"), ); ///////////////////////////////////////////////////////////////////// @@ -100,7 +109,7 @@ pub fn initialize() -> Command { .num_args(0) .help_heading("Composite settings") .conflicts_with_all(["rate_limit", "auto_bail"]) - .help("Use the same settings as --smart and set --collect-extensions to true"), + .help("Use the same settings as --smart and set --collect-extensions and --scan-dir-listings to true"), ); ///////////////////////////////////////////////////////////////////// @@ -248,6 +257,13 @@ pub fn initialize() -> Command { .help_heading("Request settings") .num_args(0) .help("Append / to each request's URL") + ).arg( + Arg::new("protocol") + .long("protocol") + .value_name("PROTOCOL") + .num_args(1) + .help_heading("Request settings") + .help("Specify the protocol to use when targeting via --request-file or --url with domain only (default: https)"), ); ///////////////////////////////////////////////////////////////////// @@ -574,6 +590,12 @@ pub fn initialize() -> Command { .help( "File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)", ), + ).arg( + Arg::new("scan_dir_listings") + .long("scan-dir-listings") + .num_args(0) + .help_heading("Scan settings") + .help("Force scans to recurse into directory listings") ); ///////////////////////////////////////////////////////////////////// diff --git a/src/scan_manager/tests.rs b/src/scan_manager/tests.rs index a3bb460f..13cdae7f 100644 --- a/src/scan_manager/tests.rs +++ b/src/scan_manager/tests.rs @@ -507,6 +507,8 @@ fn feroxstates_feroxserialize_implementation() { r#""collect_extensions":true"#, r#""collect_backups":false"#, r#""collect_words":false"#, + r#""scan_dir_listings":false"#, + r#""protocol":"https""#, r#""filters":[{"filter_code":100},{"word_count":200},{"content_length":300},{"line_count":400},{"compiled":".*","raw_string":".*"},{"hash":1,"original_url":"http://localhost:12345/"}]"#, r#""collected_extensions":["php"]"#, r#""dont_collect":["tif","tiff","ico","cur","bmp","webp","svg","png","jpg","jpeg","jfif","gif","avif","apng","pjpeg","pjp","mov","wav","mpg","mpeg","mp3","mp4","m4a","m4p","m4v","ogg","webm","ogv","oga","flac","aac","3gp","css","zip","xls","xml","gz","tgz"]"#, diff --git a/src/scanner/ferox_scanner.rs b/src/scanner/ferox_scanner.rs index 03b3a863..708fd708 100644 --- a/src/scanner/ferox_scanner.rs +++ b/src/scanner/ferox_scanner.rs @@ -283,6 +283,16 @@ impl FeroxScanner { let mut message = format!("=> {}", style("Directory listing").blue().bright()); + if !self.handles.config.scan_dir_listings { + write!( + message, + " (add {} to scan)", + style("--scan-dir-listings").bright().yellow() + )?; + } else { + // todo: need to not skip them + } + if !self.handles.config.extract_links { write!( message, @@ -291,7 +301,7 @@ impl FeroxScanner { )?; } - if !self.handles.config.force_recursion { + if !self.handles.config.force_recursion && !self.handles.config.scan_dir_listings { for handle in extraction_tasks.into_iter().flatten() { _ = handle.await; }