fixed collect backups filtering and clippy

src/event_handlers/outputs.rs

@@ -328,6 +328,21 @@ impl TermOutHandler {
                     )
                     .await;
 
+                    let Some(handles) = self.handles.as_ref() else {
+                        // shouldn't ever happen, but we'll log and return early if it does
+                        log::error!("handles were unexpectedly None, this shouldn't happen");
+                        return Ok(());
+                    };
+
+                    if handles
+                        .filters
+                        .data
+                        .should_filter_response(&ferox_response, tx_stats.clone())
+                    {
+                        // response was filtered for one reason or another, don't process it
+                        continue;
+                    }
+
                     self.process_response(
                         tx_stats.clone(),
                         Box::new(ferox_response),

src/nlp/document.rs

@@ -35,7 +35,7 @@ impl Document {
     fn add_term(&mut self, word: &str) {
         let term = Term::new(word);
 
-        let metadata = self.terms.entry(term).or_insert_with(TermMetaData::new);
+        let metadata = self.terms.entry(term).or_default();
         *metadata.count_mut() += 1;
     }
 

src/nlp/term.rs

@@ -35,11 +35,6 @@ pub(super) struct TermMetaData {
 }
 
 impl TermMetaData {
-    /// create a new metadata container
-    pub(super) fn new() -> Self {
-        Self::default()
-    }
-
     /// number of times a `Term` has appeared in any `Document` within the corpus
     pub(super) fn document_frequency(&self) -> usize {
         self.term_frequencies().len()
@@ -90,7 +85,7 @@ mod tests {
     #[test]
     /// test accessors for correctness
     fn nlp_term_metadata_accessor_test() {
-        let mut metadata = TermMetaData::new();
+        let mut metadata = TermMetaData::default();
 
         *metadata.count_mut() += 1;
         assert_eq!(metadata.count(), 1);

src/scanner/ferox_scanner.rs

@@ -251,60 +251,57 @@ impl FeroxScanner {
             // heuristics test block:
             let test = heuristics::HeuristicTests::new(self.handles.clone());
 
-            if let Ok(dirlist_result) = test.directory_listing(&self.target_url).await {
-                if dirlist_result.is_some() {
-                    let dirlist_result = dirlist_result.unwrap();
-                    // at this point, we have a DirListingType, and it's not the None variant
-                    // which means we found directory listing based on the heuristic; now we need
-                    // to process the links that are available if --extract-links was used
-
-                    if self.handles.config.extract_links {
-                        let mut extractor = ExtractorBuilder::default()
-                            .response(&dirlist_result.response)
-                            .target(ExtractionTarget::DirectoryListing)
-                            .url(&self.target_url)
-                            .handles(self.handles.clone())
-                            .build()?;
-
-                        let result = extractor.extract_from_dir_listing().await?;
-
-                        extraction_tasks.push(extractor.request_links(result).await?);
-
-                        log::trace!("exit: scan_url -> Directory listing heuristic");
-
-                        self.handles.stats.send(AddToF64Field(
-                            DirScanTimes,
-                            scan_timer.elapsed().as_secs_f64(),
-                        ))?;
-
-                        self.handles.stats.send(SubtractFromUsizeField(
-                            TotalExpected,
-                            progress_bar.length().unwrap_or(0) as usize,
-                        ))?;
-                    }
+            if let Ok(Some(dirlist_result)) = test.directory_listing(&self.target_url).await {
+                // at this point, we have a DirListingType, and it's not the None variant
+                // which means we found directory listing based on the heuristic; now we need
+                // to process the links that are available if --extract-links was used
+
+                if self.handles.config.extract_links {
+                    let mut extractor = ExtractorBuilder::default()
+                        .response(&dirlist_result.response)
+                        .target(ExtractionTarget::DirectoryListing)
+                        .url(&self.target_url)
+                        .handles(self.handles.clone())
+                        .build()?;
+
+                    let result = extractor.extract_from_dir_listing().await?;
+
+                    extraction_tasks.push(extractor.request_links(result).await?);
+
+                    log::trace!("exit: scan_url -> Directory listing heuristic");
+
+                    self.handles.stats.send(AddToF64Field(
+                        DirScanTimes,
+                        scan_timer.elapsed().as_secs_f64(),
+                    ))?;
+
+                    self.handles.stats.send(SubtractFromUsizeField(
+                        TotalExpected,
+                        progress_bar.length().unwrap_or(0) as usize,
+                    ))?;
+                }
 
-                    let mut message = format!("=> {}", style("Directory listing").blue().bright());
+                let mut message = format!("=> {}", style("Directory listing").blue().bright());
 
-                    if !self.handles.config.extract_links {
-                        write!(
-                            message,
-                            " (remove {} to scan)",
-                            style("--dont-extract-links").bright().yellow()
-                        )?;
-                    }
+                if !self.handles.config.extract_links {
+                    write!(
+                        message,
+                        " (remove {} to scan)",
+                        style("--dont-extract-links").bright().yellow()
+                    )?;
+                }
 
-                    if !self.handles.config.force_recursion {
-                        for handle in extraction_tasks.into_iter().flatten() {
-                            _ = handle.await;
-                        }
+                if !self.handles.config.force_recursion {
+                    for handle in extraction_tasks.into_iter().flatten() {
+                        _ = handle.await;
+                    }
 
-                        progress_bar.reset_eta();
-                        progress_bar.finish_with_message(message);
+                    progress_bar.reset_eta();
+                    progress_bar.finish_with_message(message);
 
-                        ferox_scan.finish()?;
+                    ferox_scan.finish()?;
 
-                        return Ok(()); // nothing left to do if we found a dir listing
-                    }
+                    return Ok(()); // nothing left to do if we found a dir listing
                 }
             }