diff --git a/src/event_handlers/outputs.rs b/src/event_handlers/outputs.rs index a93f5108..36a5e4f7 100644 --- a/src/event_handlers/outputs.rs +++ b/src/event_handlers/outputs.rs @@ -328,6 +328,21 @@ impl TermOutHandler { ) .await; + let Some(handles) = self.handles.as_ref() else { + // shouldn't ever happen, but we'll log and return early if it does + log::error!("handles were unexpectedly None, this shouldn't happen"); + return Ok(()); + }; + + if handles + .filters + .data + .should_filter_response(&ferox_response, tx_stats.clone()) + { + // response was filtered for one reason or another, don't process it + continue; + } + self.process_response( tx_stats.clone(), Box::new(ferox_response), diff --git a/src/nlp/document.rs b/src/nlp/document.rs index daeb7859..4072e89b 100644 --- a/src/nlp/document.rs +++ b/src/nlp/document.rs @@ -35,7 +35,7 @@ impl Document { fn add_term(&mut self, word: &str) { let term = Term::new(word); - let metadata = self.terms.entry(term).or_insert_with(TermMetaData::new); + let metadata = self.terms.entry(term).or_default(); *metadata.count_mut() += 1; } diff --git a/src/nlp/term.rs b/src/nlp/term.rs index 86b95919..007b8cee 100644 --- a/src/nlp/term.rs +++ b/src/nlp/term.rs @@ -35,11 +35,6 @@ pub(super) struct TermMetaData { } impl TermMetaData { - /// create a new metadata container - pub(super) fn new() -> Self { - Self::default() - } - /// number of times a `Term` has appeared in any `Document` within the corpus pub(super) fn document_frequency(&self) -> usize { self.term_frequencies().len() @@ -90,7 +85,7 @@ mod tests { #[test] /// test accessors for correctness fn nlp_term_metadata_accessor_test() { - let mut metadata = TermMetaData::new(); + let mut metadata = TermMetaData::default(); *metadata.count_mut() += 1; assert_eq!(metadata.count(), 1); diff --git a/src/scanner/ferox_scanner.rs b/src/scanner/ferox_scanner.rs index 6404708f..df00d471 100644 --- a/src/scanner/ferox_scanner.rs +++ b/src/scanner/ferox_scanner.rs @@ -251,60 +251,57 @@ impl FeroxScanner { // heuristics test block: let test = heuristics::HeuristicTests::new(self.handles.clone()); - if let Ok(dirlist_result) = test.directory_listing(&self.target_url).await { - if dirlist_result.is_some() { - let dirlist_result = dirlist_result.unwrap(); - // at this point, we have a DirListingType, and it's not the None variant - // which means we found directory listing based on the heuristic; now we need - // to process the links that are available if --extract-links was used - - if self.handles.config.extract_links { - let mut extractor = ExtractorBuilder::default() - .response(&dirlist_result.response) - .target(ExtractionTarget::DirectoryListing) - .url(&self.target_url) - .handles(self.handles.clone()) - .build()?; - - let result = extractor.extract_from_dir_listing().await?; - - extraction_tasks.push(extractor.request_links(result).await?); - - log::trace!("exit: scan_url -> Directory listing heuristic"); - - self.handles.stats.send(AddToF64Field( - DirScanTimes, - scan_timer.elapsed().as_secs_f64(), - ))?; - - self.handles.stats.send(SubtractFromUsizeField( - TotalExpected, - progress_bar.length().unwrap_or(0) as usize, - ))?; - } + if let Ok(Some(dirlist_result)) = test.directory_listing(&self.target_url).await { + // at this point, we have a DirListingType, and it's not the None variant + // which means we found directory listing based on the heuristic; now we need + // to process the links that are available if --extract-links was used + + if self.handles.config.extract_links { + let mut extractor = ExtractorBuilder::default() + .response(&dirlist_result.response) + .target(ExtractionTarget::DirectoryListing) + .url(&self.target_url) + .handles(self.handles.clone()) + .build()?; + + let result = extractor.extract_from_dir_listing().await?; + + extraction_tasks.push(extractor.request_links(result).await?); + + log::trace!("exit: scan_url -> Directory listing heuristic"); + + self.handles.stats.send(AddToF64Field( + DirScanTimes, + scan_timer.elapsed().as_secs_f64(), + ))?; + + self.handles.stats.send(SubtractFromUsizeField( + TotalExpected, + progress_bar.length().unwrap_or(0) as usize, + ))?; + } - let mut message = format!("=> {}", style("Directory listing").blue().bright()); + let mut message = format!("=> {}", style("Directory listing").blue().bright()); - if !self.handles.config.extract_links { - write!( - message, - " (remove {} to scan)", - style("--dont-extract-links").bright().yellow() - )?; - } + if !self.handles.config.extract_links { + write!( + message, + " (remove {} to scan)", + style("--dont-extract-links").bright().yellow() + )?; + } - if !self.handles.config.force_recursion { - for handle in extraction_tasks.into_iter().flatten() { - _ = handle.await; - } + if !self.handles.config.force_recursion { + for handle in extraction_tasks.into_iter().flatten() { + _ = handle.await; + } - progress_bar.reset_eta(); - progress_bar.finish_with_message(message); + progress_bar.reset_eta(); + progress_bar.finish_with_message(message); - ferox_scan.finish()?; + ferox_scan.finish()?; - return Ok(()); // nothing left to do if we found a dir listing - } + return Ok(()); // nothing left to do if we found a dir listing } }