Skip to content

Commit

Permalink
A1
Browse files Browse the repository at this point in the history
  • Loading branch information
qarmin committed Dec 29, 2024
1 parent 31e80f2 commit f0a2d3d
Show file tree
Hide file tree
Showing 16 changed files with 198 additions and 185 deletions.
124 changes: 62 additions & 62 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion czkawka_cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ repository = "https://github.com/qarmin/czkawka"
clap = { version = "4.5", features = ["derive"] }

# For enum types
image_hasher = "2.0"
image_hasher = "3.0"

log = "0.4.22"
handsome_logger = "0.8"
Expand Down
2 changes: 1 addition & 1 deletion czkawka_core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ crossbeam-channel = "0.5"
directories-next = "2.0"

# Needed by similar images
image_hasher = "2.0"
image_hasher = "3.0"
bk-tree = "0.5"
image = { version = "0.25", default-features = false, features = ["bmp", "dds", "exr", "ff", "gif", "hdr", "ico", "jpeg", "png", "pnm", "qoi", "tga", "tiff", "webp"] }
hamming-bitwise-fast = "1.0"
Expand Down
13 changes: 7 additions & 6 deletions czkawka_core/benches/hash_calculation_benchmark.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
use criterion::{black_box, criterion_group, criterion_main, Criterion};
use czkawka_core::duplicate::{hash_calculation, DuplicateEntry, HashType};
use std::env::temp_dir;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;

use criterion::{black_box, criterion_group, criterion_main, Criterion};
use czkawka_core::duplicate::{hash_calculation, DuplicateEntry, HashType};

fn setup_test_file(size: u64) -> PathBuf {
let mut path = temp_dir();
path.push("test_file");
Expand All @@ -25,25 +26,25 @@ fn get_file_entry(size: u64) -> DuplicateEntry {

fn benchmark_hash_calculation_vec<const FILE_SIZE: u64, const BUFFER_SIZE: usize>(c: &mut Criterion) {
let file_entry = get_file_entry(FILE_SIZE);
let function_name = format!("hash_calculation_vec_file_{}_buffer_{}", FILE_SIZE, BUFFER_SIZE);
let function_name = format!("hash_calculation_vec_file_{FILE_SIZE}_buffer_{BUFFER_SIZE}");

c.bench_function(&function_name, |b| {
b.iter(|| {
let mut buffer = vec![0u8; BUFFER_SIZE];
hash_calculation(black_box(&mut buffer), black_box(&file_entry), black_box(HashType::Blake3), black_box(u64::MAX)).expect("Failed to calculate hash");
})
});
});
}

fn benchmark_hash_calculation_arr<const FILE_SIZE: u64, const BUFFER_SIZE: usize>(c: &mut Criterion) {
let file_entry = get_file_entry(FILE_SIZE);
let function_name = format!("hash_calculation_arr_file_{}_buffer_{}", FILE_SIZE, BUFFER_SIZE);
let function_name = format!("hash_calculation_arr_file_{FILE_SIZE}_buffer_{BUFFER_SIZE}");

c.bench_function(&function_name, |b| {
b.iter(|| {
let mut buffer = [0u8; BUFFER_SIZE];
hash_calculation(black_box(&mut buffer), black_box(&file_entry), black_box(HashType::Blake3), black_box(u64::MAX)).expect("Failed to calculate hash");
})
});
});
}

Expand Down
20 changes: 10 additions & 10 deletions czkawka_core/src/bad_extensions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use mime_guess::get_mime_extensions;
use rayon::prelude::*;
use serde::Serialize;

use crate::common::{check_if_stop_received, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
use crate::common::{check_if_stop_received, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, WorkContinueStatus};
use crate::common_dir_traversal::{DirTraversalBuilder, DirTraversalResult, FileEntry, ToolType};
use crate::common_tool::{CommonData, CommonToolData};
use crate::common_traits::*;
Expand Down Expand Up @@ -233,19 +233,19 @@ impl BadExtensions {
#[fun_time(message = "find_bad_extensions_files", level = "info")]
pub fn find_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) {
self.prepare_items();
if !self.check_files(stop_receiver, progress_sender) {
if self.check_files(stop_receiver, progress_sender) == WorkContinueStatus::Stop {
self.common_data.stopped_search = true;
return;
}
if !self.look_for_bad_extensions_files(stop_receiver, progress_sender) {
if self.look_for_bad_extensions_files(stop_receiver, progress_sender) == WorkContinueStatus::Stop {
self.common_data.stopped_search = true;
return;
}
self.debug_print();
}

#[fun_time(message = "check_files", level = "debug")]
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> WorkContinueStatus {
let result = DirTraversalBuilder::new()
.common_data(&self.common_data)
.group_by(|_fe| ())
Expand All @@ -259,17 +259,17 @@ impl BadExtensions {
self.files_to_check = grouped_file_entries.into_values().flatten().collect();
self.common_data.text_messages.warnings.extend(warnings);

true
WorkContinueStatus::Continue
}

DirTraversalResult::Stopped => false,
DirTraversalResult::Stopped => WorkContinueStatus::Stop,
}
}

#[fun_time(message = "look_for_bad_extensions_files", level = "debug")]
fn look_for_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
fn look_for_bad_extensions_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> WorkContinueStatus {
if self.files_to_check.is_empty() {
return true;
return WorkContinueStatus::Continue;
}

let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) =
Expand All @@ -288,14 +288,14 @@ impl BadExtensions {

// Break if stop was clicked
if check_was_stopped.load(Ordering::Relaxed) {
return false;
return WorkContinueStatus::Stop;
}

self.information.number_of_files_with_bad_extension = self.bad_extensions_files.len();

debug!("Found {} files with invalid extension.", self.information.number_of_files_with_bad_extension);

true
WorkContinueStatus::Continue
}

#[fun_time(message = "verify_extensions", level = "debug")]
Expand Down
4 changes: 2 additions & 2 deletions czkawka_core/src/common_dir_traversal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ pub struct DirTraversal<'a, 'b, F> {
collect: Collect,
}

impl<'a, 'b> Default for DirTraversalBuilder<'a, 'b, ()> {
impl Default for DirTraversalBuilder<'_, '_, ()> {
fn default() -> Self {
Self::new()
}
Expand Down Expand Up @@ -302,7 +302,7 @@ fn entry_type(file_type: FileType) -> EntryType {
}
}

impl<'a, 'b, F, T> DirTraversal<'a, 'b, F>
impl<F, T> DirTraversal<'_, '_, F>
where
F: Fn(&FileEntry) -> T,
T: Ord + PartialOrd,
Expand Down
40 changes: 23 additions & 17 deletions czkawka_core/src/duplicate.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,3 @@
use crossbeam_channel::{Receiver, Sender};
use fun_time::fun_time;
use humansize::{format_size, BINARY};
use log::debug;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use std::cell::RefCell;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fmt::Debug;
Expand All @@ -16,6 +10,13 @@ use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};
use std::sync::atomic::Ordering;
use std::{fs, mem};

use crossbeam_channel::{Receiver, Sender};
use fun_time::fun_time;
use humansize::{format_size, BINARY};
use log::debug;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use xxhash_rust::xxh3::Xxh3;

use crate::common::{check_if_stop_received, delete_files_custom, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, WorkContinueStatus};
Expand Down Expand Up @@ -549,9 +550,9 @@ impl DuplicateFinder {
stop_receiver: Option<&Receiver<()>>,
progress_sender: Option<&Sender<ProgressData>>,
pre_checked_map: &mut BTreeMap<u64, Vec<DuplicateEntry>>,
) -> bool {
) -> WorkContinueStatus {
if self.files_with_identical_size.is_empty() {
return true;
return crate::common::WorkContinueStatus::Continue;
}

let check_type = self.get_params().hash_type;
Expand All @@ -562,7 +563,7 @@ impl DuplicateFinder {

send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
if check_if_stop_received(stop_receiver) {
return false;
return crate::common::WorkContinueStatus::Stop;
}
let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = prepare_thread_handler_common(
progress_sender,
Expand Down Expand Up @@ -629,10 +630,10 @@ impl DuplicateFinder {

send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
if check_was_stopped.load(Ordering::Relaxed) || check_if_stop_received(stop_receiver) {
return false;
return crate::common::WorkContinueStatus::Stop;
}

true
crate::common::WorkContinueStatus::Continue
}

fn diff_loaded_and_prechecked_files(
Expand Down Expand Up @@ -758,9 +759,14 @@ impl DuplicateFinder {
}

#[fun_time(message = "full_hashing", level = "debug")]
fn full_hashing(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>, pre_checked_map: BTreeMap<u64, Vec<DuplicateEntry>>) -> bool {
fn full_hashing(
&mut self,
stop_receiver: Option<&Receiver<()>>,
progress_sender: Option<&Sender<ProgressData>>,
pre_checked_map: BTreeMap<u64, Vec<DuplicateEntry>>,
) -> WorkContinueStatus {
if pre_checked_map.is_empty() {
return true;
return crate::common::WorkContinueStatus::Continue;
}

let (progress_thread_handle, progress_thread_run, _atomic_counter, _check_was_stopped) =
Expand All @@ -770,7 +776,7 @@ impl DuplicateFinder {

send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
if check_if_stop_received(stop_receiver) {
return false;
return crate::common::WorkContinueStatus::Stop;
}

let (progress_thread_handle, progress_thread_run, atomic_counter, check_was_stopped) = prepare_thread_handler_common(
Expand Down Expand Up @@ -835,7 +841,7 @@ impl DuplicateFinder {
}
}

true
crate::common::WorkContinueStatus::Continue
}

#[fun_time(message = "hash_reference_folders", level = "debug")]
Expand Down Expand Up @@ -894,11 +900,11 @@ impl DuplicateFinder {
assert_eq!(self.get_params().check_method, CheckingMethod::Hash);

let mut pre_checked_map: BTreeMap<u64, Vec<DuplicateEntry>> = Default::default();
if !self.prehashing(stop_receiver, progress_sender, &mut pre_checked_map) {
if self.prehashing(stop_receiver, progress_sender, &mut pre_checked_map) == crate::common::WorkContinueStatus::Stop {
return crate::common::WorkContinueStatus::Stop;
}

if !self.full_hashing(stop_receiver, progress_sender, pre_checked_map) {
if self.full_hashing(stop_receiver, progress_sender, pre_checked_map) == crate::common::WorkContinueStatus::Stop {
return crate::common::WorkContinueStatus::Stop;
}

Expand Down
9 changes: 5 additions & 4 deletions czkawka_core/src/empty_files.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use crossbeam_channel::{Receiver, Sender};
use fun_time::fun_time;
use log::debug;

use crate::common::WorkContinueStatus;
use crate::common_dir_traversal::{DirTraversalBuilder, DirTraversalResult, FileEntry, ToolType};
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
use crate::common_traits::*;
Expand Down Expand Up @@ -42,7 +43,7 @@ impl EmptyFiles {
#[fun_time(message = "find_empty_files", level = "info")]
pub fn find_empty_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) {
self.prepare_items();
if !self.check_files(stop_receiver, progress_sender) {
if self.check_files(stop_receiver, progress_sender) == WorkContinueStatus::Stop {
self.common_data.stopped_search = true;
return;
}
Expand All @@ -51,7 +52,7 @@ impl EmptyFiles {
}

#[fun_time(message = "check_files", level = "debug")]
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> WorkContinueStatus {
let result = DirTraversalBuilder::new()
.common_data(&self.common_data)
.group_by(|_fe| ())
Expand All @@ -70,10 +71,10 @@ impl EmptyFiles {

debug!("Found {} empty files.", self.information.number_of_empty_files);

true
WorkContinueStatus::Continue
}

DirTraversalResult::Stopped => false,
DirTraversalResult::Stopped => WorkContinueStatus::Stop,
}
}

Expand Down
10 changes: 5 additions & 5 deletions czkawka_core/src/empty_folder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ use fun_time::fun_time;
use log::debug;
use rayon::prelude::*;

use crate::common::{check_if_stop_received, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
use crate::common::{check_if_stop_received, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, WorkContinueStatus};
use crate::common_dir_traversal::{common_get_entry_data, common_get_metadata_dir, common_read_dir, get_modified_time, ToolType};
use crate::common_directory::Directories;
use crate::common_items::ExcludedItems;
Expand Down Expand Up @@ -72,7 +72,7 @@ impl EmptyFolder {
#[fun_time(message = "find_empty_folders", level = "info")]
pub fn find_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) {
self.prepare_items();
if !self.check_for_empty_folders(stop_receiver, progress_sender) {
if self.check_for_empty_folders(stop_receiver, progress_sender) == WorkContinueStatus::Stop {
self.common_data.stopped_search = true;
return;
}
Expand Down Expand Up @@ -102,7 +102,7 @@ impl EmptyFolder {
}

#[fun_time(message = "check_for_empty_folders", level = "debug")]
fn check_for_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
fn check_for_empty_folders(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> WorkContinueStatus {
let mut folders_to_check: Vec<PathBuf> = self.common_data.directories.included_directories.clone();

let (progress_thread_handle, progress_thread_run, atomic_counter, _check_was_stopped) =
Expand All @@ -127,7 +127,7 @@ impl EmptyFolder {
while !folders_to_check.is_empty() {
if check_if_stop_received(stop_receiver) {
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
return false;
return crate::common::WorkContinueStatus::Stop;
}

let segments: Vec<_> = folders_to_check
Expand Down Expand Up @@ -219,7 +219,7 @@ impl EmptyFolder {

debug!("Found {} empty folders.", self.empty_folder_list.len());
send_info_and_wait_for_ending_all_threads(&progress_thread_run, progress_thread_handle);
true
WorkContinueStatus::Continue
}

pub(crate) fn set_as_not_empty_folder(folder_entries: &mut HashMap<String, FolderEntry>, current_folder: &str) {
Expand Down
9 changes: 5 additions & 4 deletions czkawka_core/src/invalid_symlinks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use fun_time::fun_time;
use log::debug;
use serde::{Deserialize, Serialize};

use crate::common::WorkContinueStatus;
use crate::common_dir_traversal::{Collect, DirTraversalBuilder, DirTraversalResult, ErrorType, FileEntry, ToolType};
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
use crate::common_traits::*;
Expand Down Expand Up @@ -74,7 +75,7 @@ impl InvalidSymlinks {
#[fun_time(message = "find_invalid_links", level = "info")]
pub fn find_invalid_links(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) {
self.prepare_items();
if !self.check_files(stop_receiver, progress_sender) {
if self.check_files(stop_receiver, progress_sender) == WorkContinueStatus::Stop {
self.common_data.stopped_search = true;
return;
}
Expand All @@ -83,7 +84,7 @@ impl InvalidSymlinks {
}

#[fun_time(message = "check_files", level = "debug")]
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> WorkContinueStatus {
let result = DirTraversalBuilder::new()
.common_data(&self.common_data)
.group_by(|_fe| ())
Expand All @@ -106,9 +107,9 @@ impl InvalidSymlinks {
self.information.number_of_invalid_symlinks = self.invalid_symlinks.len();
self.common_data.text_messages.warnings.extend(warnings);
debug!("Found {} invalid symlinks.", self.information.number_of_invalid_symlinks);
true
crate::common::WorkContinueStatus::Continue
}
DirTraversalResult::Stopped => false,
DirTraversalResult::Stopped => crate::common::WorkContinueStatus::Stop,
}
}

Expand Down
Loading

0 comments on commit f0a2d3d

Please sign in to comment.