diff --git a/README.md b/README.md index 0faae8fc00..1dfbff5639 100644 --- a/README.md +++ b/README.md @@ -108,3 +108,6 @@ Timeline: Time chart: ![image](https://github.com/sepinf-inc/IPED/assets/7276994/81df1c18-361d-49f1-b755-36520437803a) + +Events correlation of 2 suspects actions and illegal activities: +![image](https://github.com/sepinf-inc/IPED/assets/7276994/e1f47b15-ba89-4436-9291-7ec354ef2b57) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index b0eb26473f..559b691e09 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -1,3 +1,97 @@ +XX/XX/2024: IPED-4.2.0: +News: +#39: Filter Manager (@patrickdalla, @lfcnassif) +#1823: New transcription implementation using Whisper (@lfcnassif, @gfd2020, @marcus6n) +#2118: New table column header filter like Excel (@patrickdalla, @lfcnassif, @wladimirleite) +#1182: Translation to French (Didier Meziat-Burdin, Christopher Etienne, @jelallt, @lfcnassif) +#1969: Utility tool to scan a folder for cases and export files matching a search criteria (@lfcnassif) +#2028: Support for UFDR generated by Physical Analyzer version 8+ (@marcosammoura) +#693: Show and jump to WhatsApp quoted messages in reply messages in WhatsApp preview (@gfd2020, @wladimirleite) +#2030: Support WhatsApp "view once" messages (@wladimirleite) +#1725: Add a notice about WhatsApp edited messages (@wladimirleite) +#1923: Handle other WhatsApp Message Types (@wladimirleite) +#2031: Extract more thumbnails from WhatsApp databases (@wladimirleite) +#2036: Preserve line breaks, text formatting and contact display name in WhatsApp messages (@wladimirleite) +#2044: Link modified WhatsApp images by long path (@wladimirleite) +#1990: Update Telegram decoder plugin (@hauck-jvsh, @lfcnassif) +#2184: Support new Telegram version for android (@hauck-jvsh, @lfcnassif) +#1998: Support some recent Telegram iOS chats not being parsed (@wladimirleite, @hauck-jvsh) +#2005: Enable Telegram iOS parser for UFDR and expose "extractMessages" parameter for Telegram Parser (@wladimirleite) +#2000: Better detection of Telegram iOS databases (@wladimirleite) +#2134: Improve Telegram Channels support on Android (@wladimirleite, @lfcnassif) +#1568: Google Chrome Cache Parser (@felipecampanini, @patrickdalla, @lfcnassif) +#1985: Carver for .torrent files and resume.dat (@wladimirleite, @patrickdalla) +#2185: Link case files to torrents and torrents to resume.dat (@wladimirleite, @patrickdalla, @lfcnassif) +#1986: Parse more information from torrents (@wladimirleite, @patrickdalla) +#2193: Better detection of Torrent client "settings.dat" +#1905: Open Financial Exchange (OFX) parser (@gfd2020, @patrickdalla, @lfcnassif) +#1953: APK Parser (@wladimirleite, @patrickdalla, @lfcnassif) +#2034: Parser for vlc-qt-interface.ini files (@lfcnassif, @marcus6n) +#1857: Shareaza Download file parser (@felipecampanini, @lfcnassif) +#1816: Carver and file signature for Shareaza .sd download control files (@wladimirleite) +#1866: Colored Bookmarks (@wladimirleite, @patrickdalla) +#1768: Import Keyword list as Bookmarks (@gfd2020, @lfcnassif) +#2124: Remove keystroke to bookmark association (@hugohmk, @lfcnassif) +#1748: Improve PDF file format carving (@wladimirleite) +#2054: Support Dahua DAV videos (@wladimirleite) +#1053: Implement "shortcut" for filter creation from results table (@patrickdalla) +#1503: Extract timestamp events from UsnJrnlParser to fill the timeline in forensic/pedo profile (@patrickdalla, @lfcnassif) +#2222: Format dates in metadata viewer (@wladimirleite, @lfcnassif) +#2189: Don't include "externally" parsed items of configured apps if phoneParsersToUse = "internal" (@patrickdalla, @lfcnassif) +#2178: Place most recent searches in the top of search history drop down list (@wladimirleite) +#2177: Add a mention to using [TAB] for properties autocomplete in the search field (@wladimirleite, @lfcnassif) +#2182: Use a shorter name for "Windows 10 User Timeline Records" and "Windows 10 User Timeline" (@wladimirleite) +#1632: Use RoaringBitmap internally to implement bookmarks (@patrickdalla, @lfcnassif, @wladimirleite) +#1650: Timeline break by bookmark counting performance improvement using RoaringBitmap bookmarks (@patrickdalla) +#1710: Load and process map data structures just when it is shown (@patrickdalla, @lfcnassif) +#1743: Delay map update when visible (@patrickdalla, @felipecostadesousa, @lfcnassif) +#1796: Do not allow more transcription requests while transcription node is restarting (@lfcnassif) +#2116: Decrease false positives in KnownMetCarver (@wladimirleite) +#1594: Allow disabling SearchHardwareWallets.py (@wladimirleite) +#1877: Improve emojis rendering in the metadata viewer (@wladimirleite) +#1996: Handle UFDR "Transfer of Funds" (@wladimirleite) +#2170: Calls subcategories (@wladimirleite, @lfcnassif) +#2021: Make Wav2Vec2 transcription compatible with new versions of huggingsound library (@lfcnassif) +#1979: Load localization configuration early (@patrickdalla) +#1865: Menu item on Timeline legend to check/uncheck corresponding items (@patrickdalla, @lfcnassif) +#1863: Better caption in selection progress dialog from timeline chart (@patrickdalla) +#1830: Select and unselect all events button on timeline page (@patrickdalla, @felipecostadesousa) +#1904: New column with item's task processing time on processing GUI (@wladimirleite) +#1754: Add TOR Parser Tests (@felipecostadesousa, @lfcnassif) +#1759: Don't count kubernetes connections checking if the service is running as requests (@hauck-jvsh) +#1696: Fix typo in "HashDBLooupConfig.txt" file name (@lfcnassif) +#2205: Increase default UI row height a bit (@wladimirleite) +#2100: Make TempFileTask maximum file length configurable (@wladimirleite) +#242: Upgrade JS interpreter to support some ECMAScript 6 features (@lfcnassif) +#1828: Move enableOCR to IPEDConfig.txt again (@lfcnassif) +Optimizations: +#1224: Optimize processing UFDR evidences (@lfcnassif) +#1958: Optimize processing small files (@lfcnassif) +#1912: Optimize performance of carved items temporary file access (@wladimirleite) +#1889: Optimize speed of WhatsApp parser (@wladimirleite) +#1763: Optimize scrolling smoothness and responsiveness for large HTML WhatsApp chats (@gfd2020, @wladimirleite) +#2160: Optimize importing NSRL "full" versions of RDS hash sets (@wladimirleite) +#1849: Optimize HashTask using parallelism (@aberenguel, @lfcnassif) +#1855: Optimize hash computation using optional AmazonCorrettoCryptoProvider plugin on Linux (@aberenguel) +#1598: Optimize memory usage by Timeline chart with very large cases (@patrickdalla, @lfcnassif) +#1265: Optimize Graph CSV (un)compression again (@lfcnassif) +#1965: Optimize Tika's SAXParser default pool size (@lfcnassif) +#1960: Minor memory optimization in WhatsApp and other parsers (@wladimirleite) +#1652: Minor memory optimization in Timeline chart when creating time index (@patrickdalla) +#2014: Minor optimization in StandardParser when processing multi-valued metadata (@wladimirleite) +Fixes: +#2035: Sometimes WhatsApp messages are out of order (@wladimirleite) +#2203: Broken links in HTML report when "thumbs only" option is used (@wladimirleite) +#2196: Exception when using "--nogui" in a never used Windows machine (@wladimirleite) +#2190: Fail to parse some iOS WhatsApp DBs (@wladimirleite) +#2121: NullPointerExceptions in AppMapPanel class (@patrickdalla) +#2175: Range queries that use localized properties for date fields are not working properly (@wladimirleite) +#2158: EmbeddedDiskProcessTask doesn't work on paths with alternate data streams (@lfcnassif) +#2033: Aborting OutOfMemoryError caused by TwelveMonkeys TIF reader plugin (@wladimirleite) +#1926: Map items checkboxes inconsistent selection (@patrickdalla) +#2003: In rare cases, Telegram iOS DBs decoding may fail and cause exceptions (@wladimirleite) + + 05/04/2024: IPED-4.1.6: Optimizations: #455: Optimization of UFDR reader memory usage (@lfcnassif) diff --git a/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java b/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java index a45496211c..a1792b07c3 100644 --- a/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java +++ b/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java @@ -57,7 +57,7 @@ public class CaseSearcherFilter extends CancelableWorker listeners = new ArrayList(); RoaringBitmap[] unionsArray; RoaringBitmap[] excludeUnionsArray; - + public static SoftReference allItemsCache; private static IPEDSource ipedCase; @@ -193,27 +193,7 @@ public MultiSearchResult doInBackground() { } IResultSetFilterer iRSFilterer = (IResultSetFilterer) iterator.next(); - if (filterManager.isFiltererEnabled(iRSFilterer)) { - IFilter rsFilter = iRSFilterer.getFilter(); - - if (rsFilter != null) { - if (rsFilter instanceof IBitmapFilter) {// if the filter exposes a internal bitmap - addBitmapFilter((IBitmapFilter) rsFilter); - } else { - RoaringBitmap[] cachedBitmaps = filterManager.getCachedBitmaps((IResultSetFilter) rsFilter); - if (cachedBitmaps != null) { // if filtermanager returned a cached bitmap - addBitmapFilter(cachedBitmaps); - } else { - MultiSearchResult newresult = filterManager.applyFilter((IResultSetFilter) rsFilter, result); - if (newresult != result) { - numFilters++; - result = newresult; - result.setIPEDSource(ipedCase); - } - } - } - } - } + applyFilterer(iRSFilterer, result); } } @@ -242,7 +222,6 @@ public MultiSearchResult doInBackground() { e.printStackTrace(); } return new MultiSearchResult(new ItemId[0], new float[0]); - } result.setIpedSearcher(searcher); @@ -253,7 +232,62 @@ public MultiSearchResult doInBackground() { return result; } + } + + private void applyFilterer(IResultSetFilterer iRSFilterer, MultiSearchResult result2) { + if (filterManager.isFiltererEnabled(iRSFilterer)) { + IFilter rsFilter = iRSFilterer.getFilter(); + + if (rsFilter != null) { + if (rsFilter instanceof IBitmapFilter) {// if the filter exposes a internal bitmap + applyBitmapFilter((IBitmapFilter) rsFilter); + } else { + RoaringBitmap[] cachedBitmaps = filterManager.getCachedBitmaps((IResultSetFilter) rsFilter); + if (cachedBitmaps != null) { // if filtermanager returned a cached bitmap + applyBitmapFilter(cachedBitmaps); + } else { + MultiSearchResult newresult = filterManager.applyFilter((IResultSetFilter) rsFilter, result); + if (newresult != result) { + numFilters++; + result = newresult; + result.setIPEDSource(ipedCase); + } + } + } + } + } + } + + private void applyBitmapFilter(RoaringBitmap[] cachedBitmaps) { + RoaringBitmap[] lunionsArray = result.getCasesBitSets((IPEDMultiSource) ipedCase); + for (int i = 0; i < unionsArray.length; i++) { + lunionsArray[i].and(cachedBitmaps[i]); + } + + MultiSearchResult newresult = filterManager.applyFilter(lunionsArray, result); + if (newresult != result) { + numFilters++; + result = newresult; + result.setIPEDSource(ipedCase); + } + } + + private void applyBitmapFilter(IBitmapFilter rsFilter) { + RoaringBitmap[] lunionsArray = result.getCasesBitSets((IPEDMultiSource) ipedCase); + for (int i = 0; i < lunionsArray.length; i++) { + if (rsFilter.isToFilterOut()) { + lunionsArray[i].andNot(((IBitmapFilter) rsFilter).getBitmap()[i]); + } else { + lunionsArray[i].and(((IBitmapFilter) rsFilter).getBitmap()[i]); + } + } + MultiSearchResult newresult = filterManager.applyFilter(lunionsArray, result); + if (newresult != result) { + numFilters++; + result = newresult; + result.setIPEDSource(ipedCase); + } } @Override @@ -317,24 +351,6 @@ public void setAppyUIFilters(boolean applyUIFilters) { this.applyUIFilters = applyUIFilters; } - public void addBitmapFilter(IBitmapFilter filter) { - RoaringBitmap[] lunionsArray = filter.getBitmap(); - - if (unionsArray == null) { - // creates the unionsArray and puts the cases bitmaps as the initial bitmap - // array - addBitmapFilter(result.getCasesBitSets((IPEDMultiSource) ipedCase)); - } - - for (int i = 0; i < unionsArray.length; i++) { - if (filter.isToFilterOut()) { - unionsArray[i].andNot(lunionsArray[i]); - } else { - unionsArray[i].and(lunionsArray[i]); - } - } - } - public void addBitmapExcludeFilter(RoaringBitmap[] lunionsArray) { if (excludeUnionsArray == null) { excludeUnionsArray = new RoaringBitmap[lunionsArray.length]; diff --git a/iped-engine/src/main/java/iped/engine/data/IPEDSource.java b/iped-engine/src/main/java/iped/engine/data/IPEDSource.java index 7819e955d6..ed8ac42264 100644 --- a/iped-engine/src/main/java/iped/engine/data/IPEDSource.java +++ b/iped-engine/src/main/java/iped/engine/data/IPEDSource.java @@ -421,29 +421,27 @@ private int countNumItems(Category category) { if (category.getNumItems() != -1) return category.getNumItems(); - if (!category.getChildren().isEmpty()) { - int num = 0; - for (Category child : category.getChildren()) { - num += countNumItems(child); - } - category.setNumItems(num); - - } else { - String query = IndexItem.CATEGORY + ":\"" + category.getName() + "\""; - IPEDSearcher searcher = new IPEDSearcher(this, query); - searcher.setNoScoring(true); - try { - if (this instanceof IPEDMultiSource) { - category.setNumItems(searcher.multiSearch().getLength()); - } else { - category.setNumItems(searcher.search().getLength()); - } + int num = 0; + for (Category child : category.getChildren()) { + num += countNumItems(child); + } - } catch (Exception e) { - e.printStackTrace(); + String query = IndexItem.CATEGORY + ":\"" + category.getName() + "\""; + IPEDSearcher searcher = new IPEDSearcher(this, query); + searcher.setNoScoring(true); + try { + if (this instanceof IPEDMultiSource) { + num += searcher.multiSearch().getLength(); + } else { + num += searcher.search().getLength(); } + + } catch (Exception e) { + e.printStackTrace(); } - return category.getNumItems(); + + category.setNumItems(num); + return num; } private void loadKeywords() { diff --git a/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java b/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java index 4b466b1c2c..3dd1843e21 100644 --- a/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java +++ b/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java @@ -232,7 +232,18 @@ public RoaringBitmap[] getCasesBitSets(IPEDMultiSource multiSource) { } bitset.add(ids[i].getId()); } + } - return casesBitSet; + return clone(casesBitSet); } + + private RoaringBitmap[] clone(RoaringBitmap[] casesBitSet2) { + RoaringBitmap[] lcasesBitSet = new RoaringBitmap[casesBitSet.length]; + for (int i = 0; i < casesBitSet.length; i++) { + lcasesBitSet[i] = new RoaringBitmap(); + lcasesBitSet[i].or(casesBitSet[i]); + } + return lcasesBitSet; + } + }