From 3928caa1dd93ac273df9f93dd991883d3e980a60 Mon Sep 17 00:00:00 2001 From: tc-wleite Date: Wed, 29 May 2024 17:06:44 -0300 Subject: [PATCH 1/6] '#2232: Count items in parent categories and add to children's totals. --- .../java/iped/engine/data/IPEDSource.java | 38 +++++++++---------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/iped-engine/src/main/java/iped/engine/data/IPEDSource.java b/iped-engine/src/main/java/iped/engine/data/IPEDSource.java index 7819e955d6..ed8ac42264 100644 --- a/iped-engine/src/main/java/iped/engine/data/IPEDSource.java +++ b/iped-engine/src/main/java/iped/engine/data/IPEDSource.java @@ -421,29 +421,27 @@ private int countNumItems(Category category) { if (category.getNumItems() != -1) return category.getNumItems(); - if (!category.getChildren().isEmpty()) { - int num = 0; - for (Category child : category.getChildren()) { - num += countNumItems(child); - } - category.setNumItems(num); - - } else { - String query = IndexItem.CATEGORY + ":\"" + category.getName() + "\""; - IPEDSearcher searcher = new IPEDSearcher(this, query); - searcher.setNoScoring(true); - try { - if (this instanceof IPEDMultiSource) { - category.setNumItems(searcher.multiSearch().getLength()); - } else { - category.setNumItems(searcher.search().getLength()); - } + int num = 0; + for (Category child : category.getChildren()) { + num += countNumItems(child); + } - } catch (Exception e) { - e.printStackTrace(); + String query = IndexItem.CATEGORY + ":\"" + category.getName() + "\""; + IPEDSearcher searcher = new IPEDSearcher(this, query); + searcher.setNoScoring(true); + try { + if (this instanceof IPEDMultiSource) { + num += searcher.multiSearch().getLength(); + } else { + num += searcher.search().getLength(); } + + } catch (Exception e) { + e.printStackTrace(); } - return category.getNumItems(); + + category.setNumItems(num); + return num; } private void loadKeywords() { From 1adab43950e9e65c945846146a7198f47fb15ac0 Mon Sep 17 00:00:00 2001 From: Luis Nassif Date: Mon, 17 Jun 2024 23:46:18 -0300 Subject: [PATCH 2/6] update release notes --- ReleaseNotes.txt | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index b0eb26473f..4125f2e0b2 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -1,3 +1,96 @@ +XX/XX/2024: IPED-4.2.0: +News: +#1823: New transcription implementation using Whisper (@lfcnassif, @gfd2020, @marcus6n) +#2118: New table column header filter like Excel (@patrickdalla, @lfcnassif, @wladimirleite) +#1182: Translation to French (Didier Meziat-Burdin, Christopher Etienne, @jelallt, @lfcnassif) +#1969: Utility tool to scan a folder for cases and export files matching a search criteria (@lfcnassif) +#2028: Support for UFDR generated by Physical Analyzer version 8+ (@marcosammoura) +#693: Show and jump to WhatsApp quoted messages in reply messages in WhatsApp preview (@gfd2020, @wladimirleite) +#2030: Support WhatsApp "view once" messages (@wladimirleite) +#1725: Add a notice about WhatsApp edited messages (@wladimirleite) +#1923: Handle other WhatsApp Message Types (@wladimirleite) +#2031: Extract more thumbnails from WhatsApp databases (@wladimirleite) +#2036: Preserve line breaks, text formatting and contact display name in WhatsApp messages (@wladimirleite) +#2044: Link modified WhatsApp images by long path (@wladimirleite) +#1990: Update Telegram decoder plugin (@hauck-jvsh, @lfcnassif) +#2184: Support new Telegram version for android (@hauck-jvsh, @lfcnassif) +#1998: Support some recent Telegram iOS chats not being parsed (@wladimirleite, @hauck-jvsh) +#2005: Enable Telegram iOS parser for UFDR and expose "extractMessages" parameter for Telegram Parser (@wladimirleite) +#2000: Better detection of Telegram iOS databases (@wladimirleite) +#2134: Improve Telegram Channels support on Android (@wladimirleite, @lfcnassif) +#1568: Google Chrome Cache Parser (@felipecampanini, @patrickdalla, @lfcnassif) +#1985: Carver for .torrent files and resume.dat (@wladimirleite, @patrickdalla) +#2185: Link case files to torrents and torrents to resume.dat (@wladimirleite, @patrickdalla, @lfcnassif) +#1986: Parse more information from torrents (@wladimirleite, @patrickdalla) +#2193: Better detection of Torrent client "settings.dat" +#1905: Open Financial Exchange (OFX) parser (@gfd2020, @patrickdalla, @lfcnassif) +#1953: APK Parser (@wladimirleite, @patrickdalla, @lfcnassif) +#2034: Parser for vlc-qt-interface.ini files (@lfcnassif, @marcus6n) +#1857: Shareaza Download file parser (@felipecampanini, @lfcnassif) +#1816: Carver and file signature for Shareaza .sd download control files (@wladimirleite) +#1866: Colored Bookmarks (@wladimirleite, @patrickdalla) +#1768: Import Keyword list as Bookmarks (@gfd2020, @lfcnassif) +#2124: Remove keystroke to bookmark association (@hugohmk, @lfcnassif) +#1748: Improve PDF file format carving (@wladimirleite) +#2054: Support Dahua DAV videos (@wladimirleite) +#1053: Implement "shortcut" for filter creation from results table (@patrickdalla) +#1503: Extract timestamp events from UsnJrnlParser to fill the timeline in forensic/pedo profile (@patrickdalla, @lfcnassif) +#2222: Format dates in metadata viewer (@wladimirleite, @lfcnassif) +#2189: Don't include "externally" parsed items of configured apps if phoneParsersToUse = "internal" (@patrickdalla, @lfcnassif) +#2178: Place most recent searches in the top of search history drop down list (@wladimirleite) +#2177: Add a mention to using [TAB] for properties autocomplete in the search field (@wladimirleite, @lfcnassif) +#2182: Use a shorter name for "Windows 10 User Timeline Records" and "Windows 10 User Timeline" (@wladimirleite) +#1632: Use RoaringBitmap internally to implement bookmarks (@patrickdalla, @lfcnassif, @wladimirleite) +#1650: Timeline break by bookmark counting performance improvement using RoaringBitmap bookmarks (@patrickdalla) +#1710: Load and process map data structures just when it is shown (@patrickdalla, @lfcnassif) +#1743: Delay map update when visible (@patrickdalla, @felipecostadesousa, @lfcnassif) +#1796: Do not allow more transcription requests while transcription node is restarting (@lfcnassif) +#2116: Decrease false positives in KnownMetCarver (@wladimirleite) +#1594: Allow disabling SearchHardwareWallets.py (@wladimirleite) +#1877: Improve emojis rendering in the metadata viewer (@wladimirleite) +#1996: Handle UFDR "Transfer of Funds" (@wladimirleite) +#2170: Calls subcategories (@wladimirleite, @lfcnassif) +#2021: Make Wav2Vec2 transcription compatible with new versions of huggingsound library (@lfcnassif) +#1979: Load localization configuration early (@patrickdalla) +#1865: Menu item on Timeline legend to check/uncheck corresponding items (@patrickdalla, @lfcnassif) +#1863: Better caption in selection progress dialog from timeline chart (@patrickdalla) +#1830: Select and unselect all events button on timeline page (@patrickdalla, @felipecostadesousa) +#1904: New column with item's task processing time on processing GUI (@wladimirleite) +#1754: Add TOR Parser Tests (@felipecostadesousa, @lfcnassif) +#1759: Don't count kubernetes connections checking if the service is running as requests (@hauck-jvsh) +#1696: Fix typo in "HashDBLooupConfig.txt" file name (@lfcnassif) +#2205: Increase default UI row height a bit (@wladimirleite) +#2100: Make TempFileTask maximum file length configurable (@wladimirleite) +#242: Upgrade JS interpreter to support some ECMAScript 6 features (@lfcnassif) +#1828: Move enableOCR to IPEDConfig.txt again (@lfcnassif) +Optimizations: +#1224: Optimize processing UFDR evidences (@lfcnassif) +#1958: Optimize processing small files (@lfcnassif) +#1912: Optimize performance of carved items temporary file access (@wladimirleite) +#1889: Optimize speed of WhatsApp parser (@wladimirleite) +#1763: Optimize scrolling smoothness and responsiveness for large HTML WhatsApp chats (@gfd2020, @wladimirleite) +#2160: Optimize importing NSRL "full" versions of RDS hash sets (@wladimirleite) +#1849: Optimize HashTask using parallelism (@aberenguel, @lfcnassif) +#1855: Optimize hash computation using optional AmazonCorrettoCryptoProvider plugin on Linux (@aberenguel) +#1598: Optimize memory usage by Timeline chart with very large cases (@patrickdalla, @lfcnassif) +#1265: Optimize Graph CSV (un)compression again (@lfcnassif) +#1965: Optimize Tika's SAXParser default pool size (@lfcnassif) +#1960: Minor memory optimization in WhatsApp and other parsers (@wladimirleite) +#1652: Minor memory optimization in Timeline chart when creating time index (@patrickdalla) +#2014: Minor optimization in StandardParser when processing multi-valued metadata (@wladimirleite) +Fixes: +#2035: Sometimes WhatsApp messages are out of order (@wladimirleite) +#2203: Broken links in HTML report when "thumbs only" option is used (@wladimirleite) +#2196: Exception when using "--nogui" in a never used Windows machine (@wladimirleite) +#2190: Fail to parse some iOS WhatsApp DBs (@wladimirleite) +#2121: NullPointerExceptions in AppMapPanel class (@patrickdalla) +#2175: Range queries that use localized properties for date fields are not working properly (@wladimirleite) +#2158: EmbeddedDiskProcessTask doesn't work on paths with alternate data streams (@lfcnassif) +#2033: Aborting OutOfMemoryError caused by TwelveMonkeys TIF reader plugin (@wladimirleite) +#1926: Map items checkboxes inconsistent selection (@patrickdalla) +#2003: In rare cases, Telegram iOS DBs decoding may fail and cause exceptions (@wladimirleite) + + 05/04/2024: IPED-4.1.6: Optimizations: #455: Optimization of UFDR reader memory usage (@lfcnassif) From a27e82c2415280e3804c82621c99946cfbefec13 Mon Sep 17 00:00:00 2001 From: Luis Nassif Date: Tue, 18 Jun 2024 00:01:52 -0300 Subject: [PATCH 3/6] update release notes --- ReleaseNotes.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt index 4125f2e0b2..559b691e09 100644 --- a/ReleaseNotes.txt +++ b/ReleaseNotes.txt @@ -1,6 +1,7 @@ XX/XX/2024: IPED-4.2.0: News: -#1823: New transcription implementation using Whisper (@lfcnassif, @gfd2020, @marcus6n) +#39: Filter Manager (@patrickdalla, @lfcnassif) +#1823: New transcription implementation using Whisper (@lfcnassif, @gfd2020, @marcus6n) #2118: New table column header filter like Excel (@patrickdalla, @lfcnassif, @wladimirleite) #1182: Translation to French (Didier Meziat-Burdin, Christopher Etienne, @jelallt, @lfcnassif) #1969: Utility tool to scan a folder for cases and export files matching a search criteria (@lfcnassif) From d436a740da72e56d87da6b6bdc1b77c4ddbff156 Mon Sep 17 00:00:00 2001 From: Luis Filipe Nassif Date: Tue, 18 Jun 2024 14:45:57 -0300 Subject: [PATCH 4/6] Update README.md Update readme with events correlation picture --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 0faae8fc00..1dfbff5639 100644 --- a/README.md +++ b/README.md @@ -108,3 +108,6 @@ Timeline: Time chart: ![image](https://github.com/sepinf-inc/IPED/assets/7276994/81df1c18-361d-49f1-b755-36520437803a) + +Events correlation of 2 suspects actions and illegal activities: +![image](https://github.com/sepinf-inc/IPED/assets/7276994/e1f47b15-ba89-4436-9291-7ec354ef2b57) From 2300bd847a6cae9ae83d09a404aa83c2300a65f3 Mon Sep 17 00:00:00 2001 From: patrick Date: Thu, 27 Jun 2024 23:27:31 -0400 Subject: [PATCH 5/6] '#2251 Remove scheme to 'cache' bitmaps in unionsArray field, so the actual filter appliance order is done. --- .../java/iped/app/ui/CaseSearcherFilter.java | 104 ++++++++++-------- .../iped/engine/search/MultiSearchResult.java | 15 ++- 2 files changed, 73 insertions(+), 46 deletions(-) diff --git a/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java b/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java index a45496211c..ffc8bb7708 100644 --- a/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java +++ b/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java @@ -193,27 +193,7 @@ public MultiSearchResult doInBackground() { } IResultSetFilterer iRSFilterer = (IResultSetFilterer) iterator.next(); - if (filterManager.isFiltererEnabled(iRSFilterer)) { - IFilter rsFilter = iRSFilterer.getFilter(); - - if (rsFilter != null) { - if (rsFilter instanceof IBitmapFilter) {// if the filter exposes a internal bitmap - addBitmapFilter((IBitmapFilter) rsFilter); - } else { - RoaringBitmap[] cachedBitmaps = filterManager.getCachedBitmaps((IResultSetFilter) rsFilter); - if (cachedBitmaps != null) { // if filtermanager returned a cached bitmap - addBitmapFilter(cachedBitmaps); - } else { - MultiSearchResult newresult = filterManager.applyFilter((IResultSetFilter) rsFilter, result); - if (newresult != result) { - numFilters++; - result = newresult; - result.setIPEDSource(ipedCase); - } - } - } - } - } + applyFilterer(iRSFilterer, result); } } @@ -236,13 +216,12 @@ public MultiSearchResult doInBackground() { result.setIPEDSource(ipedCase); } } - + } catch (Throwable e) { if (!(e instanceof CancellationException)) { e.printStackTrace(); } return new MultiSearchResult(new ItemId[0], new float[0]); - } result.setIpedSearcher(searcher); @@ -253,10 +232,65 @@ public MultiSearchResult doInBackground() { return result; } - } - @Override + private void applyFilterer(IResultSetFilterer iRSFilterer, MultiSearchResult result2) { + if (filterManager.isFiltererEnabled(iRSFilterer)) { + IFilter rsFilter = iRSFilterer.getFilter(); + + if (rsFilter != null) { + if (rsFilter instanceof IBitmapFilter) {// if the filter exposes a internal bitmap + applyBitmapFilter((IBitmapFilter)rsFilter); + } else { + RoaringBitmap[] cachedBitmaps = filterManager.getCachedBitmaps((IResultSetFilter) rsFilter); + if (cachedBitmaps != null) { // if filtermanager returned a cached bitmap + applyBitmapFilter(cachedBitmaps); + } else { + MultiSearchResult newresult = filterManager.applyFilter((IResultSetFilter) rsFilter, result); + if (newresult != result) { + numFilters++; + result = newresult; + result.setIPEDSource(ipedCase); + } + } + } + } + } + } + + private void applyBitmapFilter(RoaringBitmap[] cachedBitmaps) { + RoaringBitmap[] lunionsArray = result.getCasesBitSets((IPEDMultiSource) ipedCase); + for (int i = 0; i < unionsArray.length; i++) { + lunionsArray[i].and(cachedBitmaps[i]); + } + + MultiSearchResult newresult = filterManager.applyFilter(lunionsArray, result); + if (newresult != result) { + numFilters++; + result = newresult; + result.setIPEDSource(ipedCase); + } + } + + private void applyBitmapFilter(IBitmapFilter rsFilter) { + RoaringBitmap[] lunionsArray = result.getCasesBitSets((IPEDMultiSource) ipedCase); + for (int i = 0; i < lunionsArray.length; i++) { + if (rsFilter.isToFilterOut()) { + lunionsArray[i].andNot(((IBitmapFilter) rsFilter).getBitmap()[i]); + } else { + lunionsArray[i].and(((IBitmapFilter) rsFilter).getBitmap()[i]); + } + } + + MultiSearchResult newresult = filterManager.applyFilter(lunionsArray, result); + if (newresult != result) { + numFilters++; + result = newresult; + result.setIPEDSource(ipedCase); + } + } + + @Override public void done() { for (CaseSearchFilterListener caseSearchFilterListener : listeners) { if (isCancelled()) { @@ -317,24 +351,6 @@ public void setAppyUIFilters(boolean applyUIFilters) { this.applyUIFilters = applyUIFilters; } - public void addBitmapFilter(IBitmapFilter filter) { - RoaringBitmap[] lunionsArray = filter.getBitmap(); - - if (unionsArray == null) { - // creates the unionsArray and puts the cases bitmaps as the initial bitmap - // array - addBitmapFilter(result.getCasesBitSets((IPEDMultiSource) ipedCase)); - } - - for (int i = 0; i < unionsArray.length; i++) { - if (filter.isToFilterOut()) { - unionsArray[i].andNot(lunionsArray[i]); - } else { - unionsArray[i].and(lunionsArray[i]); - } - } - } - public void addBitmapExcludeFilter(RoaringBitmap[] lunionsArray) { if (excludeUnionsArray == null) { excludeUnionsArray = new RoaringBitmap[lunionsArray.length]; @@ -360,6 +376,6 @@ public void addBitmapFilter(RoaringBitmap[] lunionsArray) { } else { unionsArray[i].and(lunionsArray[i]); } - } + } } } diff --git a/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java b/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java index 4b466b1c2c..b48f65efb3 100644 --- a/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java +++ b/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java @@ -232,7 +232,18 @@ public RoaringBitmap[] getCasesBitSets(IPEDMultiSource multiSource) { } bitset.add(ids[i].getId()); } - } - return casesBitSet; + + } + return clone(casesBitSet); } + + private RoaringBitmap[] clone(RoaringBitmap[] casesBitSet2) { + RoaringBitmap[] lcasesBitSet = new RoaringBitmap[casesBitSet.length]; + for(int i=0; i Date: Fri, 28 Jun 2024 18:11:30 -0300 Subject: [PATCH 6/6] '#2251: fix indentation mixing tabs and remove trailing spaces --- .../java/iped/app/ui/CaseSearcherFilter.java | 26 +++++++++---------- .../iped/engine/search/MultiSearchResult.java | 18 ++++++------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java b/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java index ffc8bb7708..a1792b07c3 100644 --- a/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java +++ b/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java @@ -57,7 +57,7 @@ public class CaseSearcherFilter extends CancelableWorker listeners = new ArrayList(); RoaringBitmap[] unionsArray; RoaringBitmap[] excludeUnionsArray; - + public static SoftReference allItemsCache; private static IPEDSource ipedCase; @@ -216,7 +216,7 @@ public MultiSearchResult doInBackground() { result.setIPEDSource(ipedCase); } } - + } catch (Throwable e) { if (!(e instanceof CancellationException)) { e.printStackTrace(); @@ -240,11 +240,11 @@ private void applyFilterer(IResultSetFilterer iRSFilterer, MultiSearchResult res if (rsFilter != null) { if (rsFilter instanceof IBitmapFilter) {// if the filter exposes a internal bitmap - applyBitmapFilter((IBitmapFilter)rsFilter); + applyBitmapFilter((IBitmapFilter) rsFilter); } else { RoaringBitmap[] cachedBitmaps = filterManager.getCachedBitmaps((IResultSetFilter) rsFilter); if (cachedBitmaps != null) { // if filtermanager returned a cached bitmap - applyBitmapFilter(cachedBitmaps); + applyBitmapFilter(cachedBitmaps); } else { MultiSearchResult newresult = filterManager.applyFilter((IResultSetFilter) rsFilter, result); if (newresult != result) { @@ -256,10 +256,10 @@ private void applyFilterer(IResultSetFilterer iRSFilterer, MultiSearchResult res } } } - } + } - private void applyBitmapFilter(RoaringBitmap[] cachedBitmaps) { - RoaringBitmap[] lunionsArray = result.getCasesBitSets((IPEDMultiSource) ipedCase); + private void applyBitmapFilter(RoaringBitmap[] cachedBitmaps) { + RoaringBitmap[] lunionsArray = result.getCasesBitSets((IPEDMultiSource) ipedCase); for (int i = 0; i < unionsArray.length; i++) { lunionsArray[i].and(cachedBitmaps[i]); } @@ -270,10 +270,10 @@ private void applyBitmapFilter(RoaringBitmap[] cachedBitmaps) { result = newresult; result.setIPEDSource(ipedCase); } - } + } - private void applyBitmapFilter(IBitmapFilter rsFilter) { - RoaringBitmap[] lunionsArray = result.getCasesBitSets((IPEDMultiSource) ipedCase); + private void applyBitmapFilter(IBitmapFilter rsFilter) { + RoaringBitmap[] lunionsArray = result.getCasesBitSets((IPEDMultiSource) ipedCase); for (int i = 0; i < lunionsArray.length; i++) { if (rsFilter.isToFilterOut()) { lunionsArray[i].andNot(((IBitmapFilter) rsFilter).getBitmap()[i]); @@ -288,9 +288,9 @@ private void applyBitmapFilter(IBitmapFilter rsFilter) { result = newresult; result.setIPEDSource(ipedCase); } - } + } - @Override + @Override public void done() { for (CaseSearchFilterListener caseSearchFilterListener : listeners) { if (isCancelled()) { @@ -376,6 +376,6 @@ public void addBitmapFilter(RoaringBitmap[] lunionsArray) { } else { unionsArray[i].and(lunionsArray[i]); } - } + } } } diff --git a/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java b/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java index b48f65efb3..3dd1843e21 100644 --- a/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java +++ b/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java @@ -233,17 +233,17 @@ public RoaringBitmap[] getCasesBitSets(IPEDMultiSource multiSource) { bitset.add(ids[i].getId()); } - } + } return clone(casesBitSet); } - private RoaringBitmap[] clone(RoaringBitmap[] casesBitSet2) { - RoaringBitmap[] lcasesBitSet = new RoaringBitmap[casesBitSet.length]; - for(int i=0; i