1105 - improve json logs for post processing (#1114)

* added timestamp field to responses
* added targets field to stats object
* clippy
* write config to output file
* --data implies post request if no other method provided
* fixed issue with multiple leading spaces
* ignoring flaky test
* upgraded deps
* bumped version to 2.10.3

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.10.2"
+version = "2.10.3"
 authors = ["Ben 'epi' Risher (@epi052)"]
 license = "MIT"
 edition = "2021"
@@ -22,36 +22,36 @@ build = "build.rs"
 maintenance = { status = "actively-developed" }
 
 [build-dependencies]
-clap = { version = "4.3", features = ["wrap_help", "cargo"] }
-clap_complete = "4.3"
-regex = "1.9"
+clap = { version = "4.5", features = ["wrap_help", "cargo"] }
+clap_complete = "4.5"
+regex = "1.10"
 lazy_static = "1.4"
 dirs = "5.0"
 
 [dependencies]
 scraper = "0.18"
 futures = "0.3"
-tokio = { version = "1.29", features = ["full"] }
+tokio = { version = "1.37", features = ["full"] }
 tokio-util = { version = "0.7", features = ["codec"] }
 log = "0.4"
 env_logger = "0.10"
 reqwest = { version = "0.11", features = ["socks", "native-tls-alpn"] }
 # uses feature unification to add 'serde' to reqwest::Url
-url = { version = "2.4", features = ["serde"] }
+url = { version = "2.5", features = ["serde"] }
 serde_regex = "1.1"
-clap = { version = "4.3", features = ["wrap_help", "cargo"] }
+clap = { version = "4.5", features = ["wrap_help", "cargo"] }
 lazy_static = "1.4"
 toml = "0.8"
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
-uuid = { version = "1.4", features = ["v4"] }
+uuid = { version = "1.8", features = ["v4"] }
 # last known working version of indicatif; 0.17.5 has a bug that causes the 
 # scan menu to fail spectacularly 
-indicatif = { version = "0.17.3" }
+indicatif = { version = "0.17.8" }
 console = "0.15"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "5.0"
-regex = "1.9"
+regex = "1.10"
 crossterm = "0.27"
 rlimit = "0.10"
 ctrlc = "3.4"
@@ -69,10 +69,10 @@ self_update = { version = "0.36", features = [
 ] }
 
 [dev-dependencies]
-tempfile = "3.6"
+tempfile = "3.10"
 httpmock = "0.6"
 assert_cmd = "2.0"
-predicates = "3.0"
+predicates = "3.1"
 
 [profile.release]
 lto = true

Makefile.toml

@@ -2,6 +2,9 @@
 [tasks.upgrade]
 dependencies = ["upgrade-deps", "update"]
 
+[tasks.check]
+dependencies = ["fmt", "clippy", "test"]
+
 # cleaning
 [tasks.clean-state]
 script = """
@@ -24,6 +27,12 @@ script = """
 cargo clippy --all-targets --all-features -- -D warnings
 """
 
+[tasks.fmt]
+clear = true
+script = """
+cargo fmt --all
+"""
+
 # tests
 [tasks.test]
 clear = true

shell_completions/_feroxbuster

@@ -24,8 +24,8 @@ _feroxbuster() {
 '--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
 '*-R+[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
 '*--replay-codes=[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
-'-a+[Sets the User-Agent (default\: feroxbuster/2.10.2)]:USER_AGENT: ' \
-'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.2)]:USER_AGENT: ' \
+'-a+[Sets the User-Agent (default\: feroxbuster/2.10.3)]:USER_AGENT: ' \
+'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.3)]:USER_AGENT: ' \
 '*-x+[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
 '*--extensions=[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
 '*-m+[Which HTTP request method(s) should be sent (default\: GET)]:HTTP_METHODS: ' \

shell_completions/_feroxbuster.ps1

@@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
             [CompletionResult]::new('-R', 'R ', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
             [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
-            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.2)')
-            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.2)')
+            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.3)')
+            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.3)')
             [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
             [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
             [CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')

shell_completions/feroxbuster.bash

@@ -35,12 +35,12 @@ _feroxbuster() {
                     ;;
                 --resume-from)
                     local oldifs
-                    if [[ -v IFS ]]; then
+                    if [ -n "${IFS+x}" ]; then
                         oldifs="$IFS"
                     fi
                     IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
-                    if [[ -v oldifs ]]; then
+                    if [ -n "${oldifs+x}" ]; then
                         IFS="$oldifs"
                     fi
                     if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
@@ -190,12 +190,12 @@ _feroxbuster() {
                     ;;
                 --server-certs)
                     local oldifs
-                    if [[ -v IFS ]]; then
+                    if [ -n "${IFS+x}" ]; then
                         oldifs="$IFS"
                     fi
                     IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
-                    if [[ -v oldifs ]]; then
+                    if [ -n "${oldifs+x}" ]; then
                         IFS="$oldifs"
                     fi
                     if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
@@ -205,12 +205,12 @@ _feroxbuster() {
                     ;;
                 --client-cert)
                     local oldifs
-                    if [[ -v IFS ]]; then
+                    if [ -n "${IFS+x}" ]; then
                         oldifs="$IFS"
                     fi
                     IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
-                    if [[ -v oldifs ]]; then
+                    if [ -n "${oldifs+x}" ]; then
                         IFS="$oldifs"
                     fi
                     if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
@@ -220,12 +220,12 @@ _feroxbuster() {
                     ;;
                 --client-key)
                     local oldifs
-                    if [[ -v IFS ]]; then
+                    if [ -n "${IFS+x}" ]; then
                         oldifs="$IFS"
                     fi
                     IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
-                    if [[ -v oldifs ]]; then
+                    if [ -n "${oldifs+x}" ]; then
                         IFS="$oldifs"
                     fi
                     if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
@@ -271,12 +271,12 @@ _feroxbuster() {
                     ;;
                 --wordlist)
                     local oldifs
-                    if [[ -v IFS ]]; then
+                    if [ -n "${IFS+x}" ]; then
                         oldifs="$IFS"
                     fi
                     IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
-                    if [[ -v oldifs ]]; then
+                    if [ -n "${oldifs+x}" ]; then
                         IFS="$oldifs"
                     fi
                     if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
@@ -286,12 +286,12 @@ _feroxbuster() {
                     ;;
                 -w)
                     local oldifs
-                    if [[ -v IFS ]]; then
+                    if [ -n "${IFS+x}" ]; then
                         oldifs="$IFS"
                     fi
                     IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
-                    if [[ -v oldifs ]]; then
+                    if [ -n "${oldifs+x}" ]; then
                         IFS="$oldifs"
                     fi
                     if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
@@ -317,12 +317,12 @@ _feroxbuster() {
                     ;;
                 --output)
                     local oldifs
-                    if [[ -v IFS ]]; then
+                    if [ -n "${IFS+x}" ]; then
                         oldifs="$IFS"
                     fi
                     IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
-                    if [[ -v oldifs ]]; then
+                    if [ -n "${oldifs+x}" ]; then
                         IFS="$oldifs"
                     fi
                     if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
@@ -332,12 +332,12 @@ _feroxbuster() {
                     ;;
                 -o)
                     local oldifs
-                    if [[ -v IFS ]]; then
+                    if [ -n "${IFS+x}" ]; then
                         oldifs="$IFS"
                     fi
                     IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
-                    if [[ -v oldifs ]]; then
+                    if [ -n "${oldifs+x}" ]; then
                         IFS="$oldifs"
                     fi
                     if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
@@ -347,12 +347,12 @@ _feroxbuster() {
                     ;;
                 --debug-log)
                     local oldifs
-                    if [[ -v IFS ]]; then
+                    if [ -n "${IFS+x}" ]; then
                         oldifs="$IFS"
                     fi
                     IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
-                    if [[ -v oldifs ]]; then
+                    if [ -n "${oldifs+x}" ]; then
                         IFS="$oldifs"
                     fi
                     if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then

shell_completions/feroxbuster.elv

@@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
             cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
             cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
-            cand -a 'Sets the User-Agent (default: feroxbuster/2.10.2)'
-            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.2)'
+            cand -a 'Sets the User-Agent (default: feroxbuster/2.10.3)'
+            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.3)'
             cand -x 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
             cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
             cand -m 'Which HTTP request method(s) should be sent (default: GET)'

src/config/container.rs

@@ -704,6 +704,11 @@ impl Configuration {
             } else {
                 config.data = arg.as_bytes().to_vec();
             }
+
+            if config.methods == methods() {
+                // if the user didn't specify a method, we're going to assume they meant to use POST
+                config.methods = vec![Method::POST.as_str().to_string()];
+            }
         }
 
         if came_from_cli!(args, "stdin") {
@@ -935,7 +940,15 @@ impl Configuration {
                 // all other items in the iterator returned by split, when combined with the
                 // original split deliminator (:), make up the header's final value
                 let value = split_val.collect::<Vec<&str>>().join(":");
-                config.headers.insert(name.to_string(), value.to_string());
+
+                if value.starts_with(' ') && !value.starts_with("  ") {
+                    // first character is a space and the second character isn't
+                    // we can trim the leading space
+                    let trimmed = value.trim_start();
+                    config.headers.insert(name.to_string(), trimmed.to_string());
+                } else {
+                    config.headers.insert(name.to_string(), value.to_string());
+                }
             }
         }
 

src/event_handlers/command.rs

@@ -85,4 +85,7 @@ pub enum Command {
     /// Give a handler access to an Arc<Handles> instance after the handler has
     /// already been initialized
     AddHandles(Arc<Handles>),
+
+    /// inform the Stats object about which targets are being scanned
+    UpdateTargets(Vec<String>),
 }

src/event_handlers/outputs.rs

@@ -98,6 +98,8 @@ impl FileOutHandler {
 
         log::info!("Writing scan results to {}", self.config.output);
 
+        write_to(&*self.config, &mut file, self.config.json)?;
+
         while let Some(command) = self.receiver.recv().await {
             match command {
                 Command::Report(response) => {

src/event_handlers/statistics.rs

@@ -125,6 +125,9 @@ impl StatsHandler {
                 Command::Sync(sender) => {
                     sender.send(true).unwrap_or_default();
                 }
+                Command::UpdateTargets(targets) => {
+                    self.stats.update_targets(targets);
+                }
                 Command::Exit => break,
                 _ => {} // no more commands needed
             }

src/filters/tests.rs

@@ -271,7 +271,7 @@ fn remove_function_works_as_expected() {
 
     assert_eq!(data.filters.read().unwrap().len(), 5);
 
-    let expected = vec![
+    let expected = [
         WordsFilter { word_count: 1 },
         WordsFilter { word_count: 3 },
         WordsFilter { word_count: 5 },

src/main.rs

@@ -25,7 +25,8 @@ use feroxbuster::{
     config::{Configuration, OutputLevel},
     event_handlers::{
         Command::{
-            AddHandles, CreateBar, Exit, JoinTasks, LoadStats, ScanInitialUrls, UpdateWordlist,
+            AddHandles, CreateBar, Exit, JoinTasks, LoadStats, ScanInitialUrls, UpdateTargets,
+            UpdateWordlist,
         },
         FiltersHandler, Handles, ScanHandler, StatsHandler, Tasks, TermInputHandler,
         TermOutHandler, SCAN_COMPLETE,
@@ -507,6 +508,14 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
         return Ok(());
     }
 
+    // in order for the Stats object to know about which targets are being scanned, we need to
+    // wait until the parallel branch has been handled before sending the UpdateTargets command
+    // this ensures that only the targets being scanned are sent to the Stats object
+    //
+    // if sent before the parallel branch is handled, the Stats object will have duplicate
+    // targets
+    handles.stats.send(UpdateTargets(targets.clone()))?;
+
     if matches!(config.output_level, OutputLevel::Default) {
         // only print banner if output level is default (no banner on --quiet|--silent)
         let std_stderr = stderr(); // std::io::stderr

src/nlp/document.rs

@@ -45,9 +45,7 @@ impl Document {
 
         let html = Html::parse_document(raw_html);
 
-        let Some(element) = html.select(&selector).next() else {
-            return None;
-        };
+        let element = html.select(&selector).next()?;
 
         let text = element
             .descendants()