added --request-file, --protocol, --scan-dir-listings

src/banner/container.rs

@@ -176,6 +176,12 @@ pub struct Banner {
 
     /// represents Configuration.collect_words
     force_recursion: BannerEntry,
+
+    /// represents Configuration.protocol
+    protocol: BannerEntry,
+
+    /// represents Configuration.scan_dir_listings
+    scan_dir_listings: BannerEntry,
 }
 
 /// implementation of Banner
@@ -320,6 +326,12 @@ impl Banner {
             BannerEntry::new("🚫", "Do Not Recurse", &config.no_recursion.to_string())
         };
 
+        let protocol = if config.protocol.to_lowercase() == "http" {
+            BannerEntry::new("🔓", "Default Protocol", &config.protocol)
+        } else {
+            BannerEntry::new("🔒", "Default Protocol", &config.protocol)
+        };
+
         let scan_limit = BannerEntry::new(
             "🦥",
             "Concurrent Scan Limit",
@@ -331,6 +343,11 @@ impl Banner {
         let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy);
         let auto_tune = BannerEntry::new("🎶", "Auto Tune", &config.auto_tune.to_string());
         let auto_bail = BannerEntry::new("🙅", "Auto Bail", &config.auto_bail.to_string());
+        let scan_dir_listings = BannerEntry::new(
+            "📂",
+            "Scan Dir Listings",
+            &config.scan_dir_listings.to_string(),
+        );
         let cfg = BannerEntry::new("💉", "Config File", &config.config);
         let proxy = BannerEntry::new("💎", "Proxy", &config.proxy);
         let server_certs = BannerEntry::new(
@@ -455,6 +472,8 @@ impl Banner {
             collect_words,
             dont_collect,
             config: cfg,
+            scan_dir_listings,
+            protocol,
             version: VERSION.to_string(),
             update_status: UpdateStatus::Unknown,
         }
@@ -595,6 +614,10 @@ by Ben "epi" Risher {}                 ver: {}"#,
         }
 
         // followed by the maybe printed or variably displayed values
+        if !config.request_file.is_empty() || !config.target_url.starts_with("http") {
+            writeln!(&mut writer, "{}", self.protocol)?;
+        }
+
         if !config.config.is_empty() {
             writeln!(&mut writer, "{}", self.config)?;
         }
@@ -662,6 +685,10 @@ by Ben "epi" Risher {}                 ver: {}"#,
             writeln!(&mut writer, "{}", self.output)?;
         }
 
+        if config.scan_dir_listings {
+            writeln!(&mut writer, "{}", self.scan_dir_listings)?;
+        }
+
         if !config.debug_log.is_empty() {
             writeln!(&mut writer, "{}", self.debug_log)?;
         }

src/config/container.rs

@@ -1,7 +1,7 @@
 use super::utils::{
-    backup_extensions, depth, extract_links, ignored_extensions, methods, report_and_exit,
-    save_state, serialized_type, status_codes, threads, timeout, user_agent, wordlist, OutputLevel,
-    RequesterPolicy,
+    backup_extensions, depth, extract_links, ignored_extensions, methods, parse_request_file,
+    report_and_exit, request_protocol, save_state, serialized_type, split_header, split_query,
+    status_codes, threads, timeout, user_agent, wordlist, OutputLevel, RequesterPolicy,
 };
 use crate::config::determine_output_level;
 use crate::config::utils::determine_requester_policy;
@@ -332,6 +332,18 @@ pub struct Configuration {
     /// Auto update app feature
     #[serde(skip)]
     pub update_app: bool,
+
+    /// whether to recurse into directory listings or not
+    #[serde(default)]
+    pub scan_dir_listings: bool,
+
+    /// path to a raw request file generated by burp or similar
+    #[serde(skip)]
+    pub request_file: String,
+
+    /// default request protocol
+    #[serde(default = "request_protocol")]
+    pub protocol: String,
 }
 
 impl Default for Configuration {
@@ -378,6 +390,7 @@ impl Default for Configuration {
             resumed: false,
             stdin: false,
             json: false,
+            scan_dir_listings: false,
             verbosity: 0,
             scan_limit: 0,
             parallel: 0,
@@ -403,6 +416,8 @@ impl Default for Configuration {
             time_limit: String::new(),
             resume_from: String::new(),
             replay_proxy: String::new(),
+            request_file: String::new(),
+            protocol: request_protocol(),
             server_certs: Vec::new(),
             queries: Vec::new(),
             extensions: Vec::new(),
@@ -482,6 +497,9 @@ impl Configuration {
     /// - **replay_proxy**: `None` (no limit on concurrent scans imposed)
     /// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
     /// - **update_app**: `false`
+    /// - **scan_dir_listings**: `false`
+    /// - **request_file**: `None`
+    /// - **protocol**: `https`
     ///
     /// After which, any values defined in a
     /// [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file will override the
@@ -555,6 +573,18 @@ impl Configuration {
         // merge the cli options into the config file options and return the result
         Self::merge_config(&mut config, cli_config);
 
+        // if the user provided a raw request file as the target, we'll need to parse out
+        // the provided info and update the config with those values. This call needs to
+        // come after the cli/config merge so we can allow the cli options to override
+        // the raw request values (i.e. --headers "stuff: things" should override a "stuff"
+        // header from the raw request).
+        //
+        // Additionally, this call needs to come before client rebuild so that the things
+        // like user-agent can be set at the client level instead of the header level.
+        if !config.request_file.is_empty() {
+            parse_request_file(&mut config)?;
+        }
+
         // rebuild clients is the last step in either code branch
         Self::try_rebuild_clients(&mut config);
 
@@ -618,6 +648,8 @@ impl Configuration {
         update_config_if_present!(&mut config.output, args, "output", String);
         update_config_if_present!(&mut config.debug_log, args, "debug_log", String);
         update_config_if_present!(&mut config.resume_from, args, "resume_from", String);
+        update_config_if_present!(&mut config.request_file, args, "request_file", String);
+        update_config_if_present!(&mut config.protocol, args, "protocol", String);
 
         if let Ok(Some(inner)) = args.try_get_one::<String>("time_limit") {
             inner.clone_into(&mut config.time_limit);
@@ -831,6 +863,10 @@ impl Configuration {
             config.save_state = false;
         }
 
+        if came_from_cli!(args, "scan_dir_listings") || came_from_cli!(args, "thorough") {
+            config.scan_dir_listings = true;
+        }
+
         if came_from_cli!(args, "dont_filter") {
             config.dont_filter = true;
         }
@@ -932,23 +968,11 @@ impl Configuration {
 
         if let Some(headers) = args.get_many::<String>("headers") {
             for val in headers {
-                let mut split_val = val.split(':');
-
-                // explicitly take first split value as header's name
-                let name = split_val.next().unwrap().trim();
-
-                // all other items in the iterator returned by split, when combined with the
-                // original split deliminator (:), make up the header's final value
-                let value = split_val.collect::<Vec<&str>>().join(":");
-
-                if value.starts_with(' ') && !value.starts_with("  ") {
-                    // first character is a space and the second character isn't
-                    // we can trim the leading space
-                    let trimmed = value.trim_start();
-                    config.headers.insert(name.to_string(), trimmed.to_string());
-                } else {
-                    config.headers.insert(name.to_string(), value.to_string());
-                }
+                let Ok((name, value)) = split_header(val) else {
+                    log::warn!("Invalid header: {}", val);
+                    continue;
+                };
+                config.headers.insert(name, value);
             }
         }
 
@@ -982,14 +1006,11 @@ impl Configuration {
 
         if let Some(queries) = args.get_many::<String>("queries") {
             for val in queries {
-                // same basic logic used as reading in the headers HashMap above
-                let mut split_val = val.split('=');
-
-                let name = split_val.next().unwrap().trim();
-
-                let value = split_val.collect::<Vec<&str>>().join("=");
-
-                config.queries.push((name.to_string(), value.to_string()));
+                let Ok((name, value)) = split_query(val) else {
+                    log::warn!("Invalid query string: {}", val);
+                    continue;
+                };
+                config.queries.push((name, value));
             }
         }
 
@@ -1171,12 +1192,15 @@ impl Configuration {
             Vec::<u16>::new()
         );
         update_if_not_default!(&mut conf.dont_filter, new.dont_filter, false);
+        update_if_not_default!(&mut conf.scan_dir_listings, new.scan_dir_listings, false);
         update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0);
         update_if_not_default!(&mut conf.parallel, new.parallel, 0);
         update_if_not_default!(&mut conf.rate_limit, new.rate_limit, 0);
         update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
         update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
         update_if_not_default!(&mut conf.resume_from, new.resume_from, "");
+        update_if_not_default!(&mut conf.request_file, new.request_file, "");
+        update_if_not_default!(&mut conf.protocol, new.protocol, request_protocol());
 
         update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
         update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent());

src/config/mod.rs

@@ -2,6 +2,7 @@
 
 mod container;
 mod utils;
+mod raw_request;
 #[cfg(test)]
 mod tests;
 

src/config/tests.rs

@@ -49,6 +49,9 @@ fn setup_config_test() -> Configuration {
             json = true
             save_state = false
             depth = 1
+            protocol = "http"
+            request_file = "/some/request/file"
+            scan_dir_listings = true
             force_recursion = true
             filter_size = [4120]
             filter_regex = ["^ignore me$"]
@@ -107,6 +110,7 @@ fn default_configuration() {
     assert!(!config.collect_extensions);
     assert!(!config.collect_backups);
     assert!(!config.collect_words);
+    assert!(!config.scan_dir_listings);
     assert!(config.regex_denylist.is_empty());
     assert_eq!(config.queries, Vec::new());
     assert_eq!(config.filter_size, Vec::<u64>::new());
@@ -125,6 +129,8 @@ fn default_configuration() {
     assert_eq!(config.client_cert, String::new());
     assert_eq!(config.client_key, String::new());
     assert_eq!(config.backup_extensions, backup_extensions());
+    assert_eq!(config.protocol, request_protocol());
+    assert_eq!(config.request_file, String::new());
 }
 
 #[test]
@@ -444,6 +450,27 @@ fn config_reads_time_limit() {
     assert_eq!(config.time_limit, "10m");
 }
 
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_scan_dir_listings() {
+    let config = setup_config_test();
+    assert!(config.scan_dir_listings);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_protocol() {
+    let config = setup_config_test();
+    assert_eq!(config.protocol, "http");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_request_file() {
+    let config = setup_config_test();
+    assert_eq!(config.request_file, String::new());
+}
+
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_resume_from() {