From 97f874bffd0847cc4e5e7e6de8c6d09fa59c0ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafa=C5=82=20Mikrut?= Date: Fri, 6 Oct 2023 20:02:50 +0200 Subject: [PATCH] Loading cache/save almost --- czkawka_core/src/broken_files.rs | 4 +- czkawka_core/src/common_cache.rs | 29 +++++++++--- czkawka_core/src/duplicate.rs | 46 ++++++------------- czkawka_core/src/same_music.rs | 4 +- czkawka_core/src/similar_images.rs | 4 +- czkawka_core/src/similar_videos.rs | 5 +- .../src/connect_things/connect_settings.rs | 19 ++++---- 7 files changed, 55 insertions(+), 56 deletions(-) diff --git a/czkawka_core/src/broken_files.rs b/czkawka_core/src/broken_files.rs index 201c0d9bf..92c152171 100644 --- a/czkawka_core/src/broken_files.rs +++ b/czkawka_core/src/broken_files.rs @@ -21,7 +21,7 @@ use crate::common::{ check_folder_children, create_crash_message, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS, IMAGE_RS_BROKEN_FILES_EXTENSIONS, PDF_FILES_EXTENSIONS, ZIP_FILES_EXTENSIONS, }; -use crate::common_cache::{get_broken_files_cache_file, load_cache_from_file_generalized, save_cache_to_file_generalized}; +use crate::common_cache::{get_broken_files_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized}; use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType}; use crate::common_tool::{CommonData, CommonToolData}; use crate::common_traits::*; @@ -357,7 +357,7 @@ impl BrokenFiles { let files_to_check = mem::take(&mut self.files_to_check); if self.common_data.use_cache { - let (messages, loaded_items) = load_cache_from_file_generalized::(&get_broken_files_cache_file(), self.get_delete_outdated_cache(), &files_to_check); + let (messages, loaded_items) = load_cache_from_file_generalized_by_path::(&get_broken_files_cache_file(), self.get_delete_outdated_cache(), &files_to_check); self.get_text_messages_mut().extend_with_another_messages(messages); loaded_hash_map = loaded_items.unwrap_or_default(); diff --git a/czkawka_core/src/common_cache.rs b/czkawka_core/src/common_cache.rs index 3b1e3c246..76af08dfc 100644 --- a/czkawka_core/src/common_cache.rs +++ b/czkawka_core/src/common_cache.rs @@ -83,7 +83,26 @@ where text_messages } -pub fn load_cache_from_file_generalized(cache_file_name: &str, delete_outdated_cache: bool, used_files: &BTreeMap) -> (Messages, Option>) +pub fn load_cache_from_file_generalized_by_path(cache_file_name: &str, delete_outdated_cache: bool, used_files: &BTreeMap) -> (Messages, Option>) +where + for<'a> T: Deserialize<'a> + ResultEntry + Sized + Send + Sync, +{ + let (text_messages, vec_loaded_cache) = load_cache_from_file_generalized(cache_file_name, delete_outdated_cache, used_files); + let Some(vec_loaded_entries) = vec_loaded_cache else { + return (text_messages, None); + }; + + debug!("Converting cache vec into map"); + let map_loaded_entries: BTreeMap = vec_loaded_entries + .into_iter() + .map(|file_entry| (file_entry.get_path().to_string_lossy().into_owned(), file_entry)) + .collect(); + debug!("Converted cache vec into map"); + + (text_messages, Some(map_loaded_entries)) +} + +fn load_cache_from_file_generalized(cache_file_name: &str, delete_outdated_cache: bool, used_files: &BTreeMap) -> (Messages, Option>) where for<'a> T: Deserialize<'a> + ResultEntry + Sized + Send + Sync, { @@ -150,12 +169,8 @@ where text_messages.messages.push(format!("Properly loaded {} cache entries.", vec_loaded_entries.len())); - let map_loaded_entries: BTreeMap<_, _> = vec_loaded_entries - .into_iter() - .map(|file_entry| (file_entry.get_path().to_string_lossy().into_owned(), file_entry)) - .collect(); - debug!("Loaded cache from file {cache_file_name} (or json alternative) - {} results", map_loaded_entries.len()); - return (text_messages, Some(map_loaded_entries)); + debug!("Loaded cache from file {cache_file_name} (or json alternative) - {} results", vec_loaded_entries.len()); + return (text_messages, Some(vec_loaded_entries)); } debug!("Failed to load cache from file {cache_file_name} because not exists"); (text_messages, None) diff --git a/czkawka_core/src/duplicate.rs b/czkawka_core/src/duplicate.rs index 4d6ad02d1..375d0fbbb 100644 --- a/czkawka_core/src/duplicate.rs +++ b/czkawka_core/src/duplicate.rs @@ -19,6 +19,7 @@ use rayon::prelude::*; use xxhash_rust::xxh3::Xxh3; use crate::common::{open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads}; +use crate::common_cache::{get_duplicate_cache_file, save_cache_to_file_generalized}; use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType}; use crate::common_messages::Messages; use crate::common_tool::{CommonData, CommonToolData}; @@ -596,13 +597,14 @@ impl DuplicateFinder { } } - save_hashes_to_file( + let messages = save_cache_to_file_generalized( + &get_duplicate_cache_file(&self.hash_type, true), &save_cache_to_hashmap, - &mut self.common_data.text_messages, - &self.hash_type, - true, + self.common_data.save_also_as_json, self.minimal_prehash_cache_file_size, ); + self.get_text_messages_mut().extend_with_another_messages(messages); + debug!("prehash_save_cache_at_exit - saving prehash cache end"); } } @@ -771,7 +773,15 @@ impl DuplicateFinder { } } } - save_hashes_to_file(&all_results, &mut self.common_data.text_messages, &self.hash_type, false, self.minimal_cache_file_size); + + let messages = save_cache_to_file_generalized( + &get_duplicate_cache_file(&self.hash_type, false), + &all_results, + self.common_data.save_also_as_json, + self.minimal_cache_file_size, + ); + self.get_text_messages_mut().extend_with_another_messages(messages); + debug!("full_hashing_save_cache_at_exit - end"); } @@ -1318,32 +1328,6 @@ pub fn make_hard_link(src: &Path, dst: &Path) -> io::Result<()> { result } -pub fn save_hashes_to_file(hashmap: &BTreeMap, text_messages: &mut Messages, type_of_hash: &HashType, is_prehash: bool, minimal_cache_file_size: u64) { - if let Some(((file_handler, cache_file), (_json_file, _json_name))) = open_cache_folder(&get_file_hash_name(type_of_hash, is_prehash), true, false, &mut text_messages.warnings) - { - let mut writer = BufWriter::new(file_handler.unwrap()); // Unwrap cannot fail - - let mut how_much = 0; - for file_entry in hashmap.values() { - if file_entry.size >= minimal_cache_file_size { - let string: String = format!("{}//{}//{}//{}", file_entry.path.display(), file_entry.size, file_entry.modified_date, file_entry.hash); - - if let Err(e) = writeln!(writer, "{string}") { - text_messages - .warnings - .push(format!("Failed to save some data to cache file {}, reason {}", cache_file.display(), e)); - return; - } - how_much += 1; - } - } - - text_messages - .messages - .push(flc!("core_saving_to_cache", generate_translation_hashmap(vec![("number", how_much.to_string())]))); - } -} - pub fn load_hashes_from_file(text_messages: &mut Messages, delete_outdated_cache: bool, type_of_hash: &HashType, is_prehash: bool) -> Option>> { if let Some(((file_handler, cache_file), (_json_file, _json_name))) = open_cache_folder(&get_file_hash_name(type_of_hash, is_prehash), false, false, &mut text_messages.warnings) diff --git a/czkawka_core/src/same_music.rs b/czkawka_core/src/same_music.rs index e8bb3359d..ac2a7b8bf 100644 --- a/czkawka_core/src/same_music.rs +++ b/czkawka_core/src/same_music.rs @@ -24,7 +24,7 @@ use symphonia::core::meta::MetadataOptions; use symphonia::core::probe::Hint; use crate::common::{create_crash_message, filter_reference_folders_generic, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS}; -use crate::common_cache::{get_similar_music_cache_file, load_cache_from_file_generalized, save_cache_to_file_generalized}; +use crate::common_cache::{get_similar_music_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized}; use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType}; use crate::common_tool::{CommonData, CommonToolData}; use crate::common_traits::*; @@ -234,7 +234,7 @@ impl SameMusic { if self.common_data.use_cache { let (messages, loaded_items) = - load_cache_from_file_generalized::(get_similar_music_cache_file(checking_tags), self.get_delete_outdated_cache(), &self.music_to_check); + load_cache_from_file_generalized_by_path::(get_similar_music_cache_file(checking_tags), self.get_delete_outdated_cache(), &self.music_to_check); self.get_text_messages_mut().extend_with_another_messages(messages); loaded_hash_map = loaded_items.unwrap_or_default(); diff --git a/czkawka_core/src/similar_images.rs b/czkawka_core/src/similar_images.rs index af06bc6ed..1394d323a 100644 --- a/czkawka_core/src/similar_images.rs +++ b/czkawka_core/src/similar_images.rs @@ -22,7 +22,7 @@ use crate::common::{ check_folder_children, create_crash_message, get_dynamic_image_from_raw_image, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, HEIC_EXTENSIONS, IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS, }; -use crate::common_cache::{get_similar_images_cache_file, load_cache_from_file_generalized, save_cache_to_file_generalized}; +use crate::common_cache::{get_similar_images_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized}; use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType}; use crate::common_tool::{CommonData, CommonToolData}; use crate::common_traits::{DebugPrint, PrintResults, ResultEntry, SaveResults}; @@ -287,7 +287,7 @@ impl SimilarImages { let mut non_cached_files_to_check: BTreeMap = Default::default(); if self.common_data.use_cache { - let (messages, loaded_items) = load_cache_from_file_generalized::( + let (messages, loaded_items) = load_cache_from_file_generalized_by_path::( &get_similar_images_cache_file(&self.hash_size, &self.hash_alg, &self.image_filter), self.get_delete_outdated_cache(), &self.images_to_check, diff --git a/czkawka_core/src/similar_videos.rs b/czkawka_core/src/similar_videos.rs index 9776d6794..589e035c5 100644 --- a/czkawka_core/src/similar_videos.rs +++ b/czkawka_core/src/similar_videos.rs @@ -16,7 +16,7 @@ use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo; use vid_dup_finder_lib::{NormalizedTolerance, VideoHash}; use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS}; -use crate::common_cache::{get_similar_videos_cache_file, load_cache_from_file_generalized, save_cache_to_file_generalized}; +use crate::common_cache::{get_similar_videos_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized}; use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType}; use crate::common_tool::{CommonData, CommonToolData}; use crate::common_traits::{DebugPrint, PrintResults, ResultEntry, SaveResults}; @@ -265,7 +265,8 @@ impl SimilarVideos { let mut non_cached_files_to_check: BTreeMap = Default::default(); if self.common_data.use_cache { - let (messages, loaded_items) = load_cache_from_file_generalized::(&get_similar_videos_cache_file(), self.get_delete_outdated_cache(), &self.videos_to_check); + let (messages, loaded_items) = + load_cache_from_file_generalized_by_path::(&get_similar_videos_cache_file(), self.get_delete_outdated_cache(), &self.videos_to_check); self.get_text_messages_mut().extend_with_another_messages(messages); loaded_hash_map = loaded_items.unwrap_or_default(); diff --git a/czkawka_gui/src/connect_things/connect_settings.rs b/czkawka_gui/src/connect_things/connect_settings.rs index 6aae6bdda..f9f3249a8 100644 --- a/czkawka_gui/src/connect_things/connect_settings.rs +++ b/czkawka_gui/src/connect_things/connect_settings.rs @@ -1,7 +1,9 @@ use std::collections::BTreeMap; use std::default::Default; -use czkawka_core::common_cache::{get_similar_images_cache_file, get_similar_videos_cache_file, load_cache_from_file_generalized, save_cache_to_file_generalized}; +use czkawka_core::common_cache::{ + get_duplicate_cache_file, get_similar_images_cache_file, get_similar_videos_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized, +}; use directories_next::ProjectDirs; use gtk4::prelude::*; use gtk4::{Label, ResponseType, Window}; @@ -129,13 +131,10 @@ pub fn connect_settings(gui_data: &GuiData) { hashmap_to_save.insert(file_entry.path.to_string_lossy().to_string(), file_entry); } } - czkawka_core::duplicate::save_hashes_to_file( - &hashmap_to_save, - &mut messages, - type_of_hash, - use_prehash, - entry_settings_cache_file_minimal_size.text().as_str().parse::().unwrap_or(2 * 1024 * 1024), - ); + + let minimal_cache_size = entry_settings_cache_file_minimal_size.text().as_str().parse::().unwrap_or(2 * 1024 * 1024); + + save_cache_to_file_generalized(&get_duplicate_cache_file(type_of_hash, use_prehash), &hashmap_to_save, false, minimal_cache_size); } } @@ -170,7 +169,7 @@ pub fn connect_settings(gui_data: &GuiData) { FilterType::Triangle, ] { for hash_alg in &[HashAlg::Blockhash, HashAlg::Gradient, HashAlg::DoubleGradient, HashAlg::VertGradient, HashAlg::Mean] { - let (mut messages, loaded_items) = load_cache_from_file_generalized::( + let (mut messages, loaded_items) = load_cache_from_file_generalized_by_path::( &get_similar_images_cache_file(hash_size, hash_alg, image_filter), true, &Default::default(), @@ -206,7 +205,7 @@ pub fn connect_settings(gui_data: &GuiData) { dialog.connect_response(move |dialog, response_type| { if response_type == ResponseType::Ok { let (mut messages, loaded_items) = - load_cache_from_file_generalized::(&get_similar_videos_cache_file(), true, &Default::default()); + load_cache_from_file_generalized_by_path::(&get_similar_videos_cache_file(), true, &Default::default()); if let Some(cache_entries) = loaded_items { let save_messages = save_cache_to_file_generalized(&get_similar_videos_cache_file(), &cache_entries, false, 0);