Merge branch 'master' into #2234_ResumeDatEntries

sepinf-inc · Jul 5, 2024 · a5e60df · a5e60df
2 parents 2d608bd + bb92776
commit a5e60df
Show file tree

Hide file tree

Showing 5 changed files with 184 additions and 62 deletions.
diff --git a/README.md b/README.md
@@ -108,3 +108,6 @@ Timeline:
 
 Time chart:
 ![image](https://github.com/sepinf-inc/IPED/assets/7276994/81df1c18-361d-49f1-b755-36520437803a)
+
+Events correlation of 2 suspects actions and illegal activities:
+![image](https://github.com/sepinf-inc/IPED/assets/7276994/e1f47b15-ba89-4436-9291-7ec354ef2b57)
diff --git a/ReleaseNotes.txt b/ReleaseNotes.txt
@@ -1,3 +1,97 @@
+XX/XX/2024: IPED-4.2.0:
+News:
+#39:   Filter Manager (@patrickdalla, @lfcnassif)
+#1823: New transcription implementation using Whisper (@lfcnassif, @gfd2020, @marcus6n) 
+#2118: New table column header filter like Excel (@patrickdalla, @lfcnassif, @wladimirleite)
+#1182: Translation to French (Didier Meziat-Burdin, Christopher Etienne, @jelallt, @lfcnassif)
+#1969: Utility tool to scan a folder for cases and export files matching a search criteria (@lfcnassif)
+#2028: Support for UFDR generated by Physical Analyzer version 8+ (@marcosammoura)
+#693:  Show and jump to WhatsApp quoted messages in reply messages in WhatsApp preview (@gfd2020, @wladimirleite)
+#2030: Support WhatsApp "view once" messages (@wladimirleite)
+#1725: Add a notice about WhatsApp edited messages (@wladimirleite)
+#1923: Handle other WhatsApp Message Types (@wladimirleite)
+#2031: Extract more thumbnails from WhatsApp databases (@wladimirleite)
+#2036: Preserve line breaks, text formatting and contact display name in WhatsApp messages (@wladimirleite)
+#2044: Link modified WhatsApp images by long path (@wladimirleite)
+#1990: Update Telegram decoder plugin (@hauck-jvsh, @lfcnassif)
+#2184: Support new Telegram version for android (@hauck-jvsh, @lfcnassif)
+#1998: Support some recent Telegram iOS chats not being parsed (@wladimirleite, @hauck-jvsh)
+#2005: Enable Telegram iOS parser for UFDR and expose "extractMessages" parameter for Telegram Parser (@wladimirleite)
+#2000: Better detection of Telegram iOS databases (@wladimirleite)
+#2134: Improve Telegram Channels support on Android (@wladimirleite, @lfcnassif)
+#1568: Google Chrome Cache Parser (@felipecampanini, @patrickdalla, @lfcnassif)
+#1985: Carver for .torrent files and resume.dat (@wladimirleite, @patrickdalla)
+#2185: Link case files to torrents and torrents to resume.dat (@wladimirleite, @patrickdalla, @lfcnassif)
+#1986: Parse more information from torrents (@wladimirleite, @patrickdalla)
+#2193: Better detection of Torrent client "settings.dat"
+#1905: Open Financial Exchange (OFX) parser (@gfd2020, @patrickdalla, @lfcnassif)
+#1953: APK Parser (@wladimirleite, @patrickdalla, @lfcnassif)
+#2034: Parser for vlc-qt-interface.ini files (@lfcnassif, @marcus6n)
+#1857: Shareaza Download file parser (@felipecampanini, @lfcnassif)
+#1816: Carver and file signature for Shareaza .sd download control files (@wladimirleite)
+#1866: Colored Bookmarks (@wladimirleite, @patrickdalla)
+#1768: Import Keyword list as Bookmarks (@gfd2020, @lfcnassif)
+#2124: Remove keystroke to bookmark association (@hugohmk, @lfcnassif)
+#1748: Improve PDF file format carving (@wladimirleite)
+#2054: Support Dahua DAV videos (@wladimirleite)
+#1053: Implement "shortcut" for filter creation from results table (@patrickdalla)
+#1503: Extract timestamp events from UsnJrnlParser to fill the timeline in forensic/pedo profile (@patrickdalla, @lfcnassif)
+#2222: Format dates in metadata viewer (@wladimirleite, @lfcnassif)
+#2189: Don't include "externally" parsed items of configured apps if phoneParsersToUse = "internal" (@patrickdalla, @lfcnassif)
+#2178: Place most recent searches in the top of search history drop down list (@wladimirleite)
+#2177: Add a mention to using [TAB] for properties autocomplete in the search field (@wladimirleite, @lfcnassif)
+#2182: Use a shorter name for "Windows 10 User Timeline Records" and "Windows 10 User Timeline" (@wladimirleite)
+#1632: Use RoaringBitmap internally to implement bookmarks (@patrickdalla, @lfcnassif, @wladimirleite)
+#1650: Timeline break by bookmark counting performance improvement using RoaringBitmap bookmarks (@patrickdalla)
+#1710: Load and process map data structures just when it is shown (@patrickdalla, @lfcnassif)
+#1743: Delay map update when visible (@patrickdalla, @felipecostadesousa, @lfcnassif)
+#1796: Do not allow more transcription requests while transcription node is restarting (@lfcnassif)
+#2116: Decrease false positives in KnownMetCarver (@wladimirleite)
+#1594: Allow disabling SearchHardwareWallets.py (@wladimirleite)
+#1877: Improve emojis rendering in the metadata viewer (@wladimirleite)
+#1996: Handle UFDR "Transfer of Funds" (@wladimirleite)
+#2170: Calls subcategories (@wladimirleite, @lfcnassif)
+#2021: Make Wav2Vec2 transcription compatible with new versions of huggingsound library (@lfcnassif)
+#1979: Load localization configuration early (@patrickdalla)
+#1865: Menu item on Timeline legend to check/uncheck corresponding items (@patrickdalla, @lfcnassif)
+#1863: Better caption in selection progress dialog from timeline chart (@patrickdalla)
+#1830: Select and unselect all events button on timeline page (@patrickdalla, @felipecostadesousa)
+#1904: New column with item's task processing time on processing GUI (@wladimirleite)
+#1754: Add TOR Parser Tests (@felipecostadesousa, @lfcnassif)
+#1759: Don't count kubernetes connections checking if the service is running as requests (@hauck-jvsh)
+#1696: Fix typo in "HashDBLooupConfig.txt" file name (@lfcnassif)
+#2205: Increase default UI row height a bit (@wladimirleite)
+#2100: Make TempFileTask maximum file length configurable (@wladimirleite)
+#242:  Upgrade JS interpreter to support some ECMAScript 6 features (@lfcnassif)
+#1828: Move enableOCR to IPEDConfig.txt again (@lfcnassif)
+Optimizations:
+#1224: Optimize processing UFDR evidences (@lfcnassif)
+#1958: Optimize processing small files (@lfcnassif)
+#1912: Optimize performance of carved items temporary file access (@wladimirleite)
+#1889: Optimize speed of WhatsApp parser (@wladimirleite)
+#1763: Optimize scrolling smoothness and responsiveness for large HTML WhatsApp chats (@gfd2020, @wladimirleite) 
+#2160: Optimize importing NSRL "full" versions of RDS hash sets (@wladimirleite)
+#1849: Optimize HashTask using parallelism (@aberenguel, @lfcnassif)
+#1855: Optimize hash computation using optional AmazonCorrettoCryptoProvider plugin on Linux (@aberenguel)
+#1598: Optimize memory usage by Timeline chart with very large cases (@patrickdalla, @lfcnassif)
+#1265: Optimize Graph CSV (un)compression again (@lfcnassif)
+#1965: Optimize Tika's SAXParser default pool size (@lfcnassif)
+#1960: Minor memory optimization in WhatsApp and other parsers (@wladimirleite)
+#1652: Minor memory optimization in Timeline chart when creating time index (@patrickdalla)
+#2014: Minor optimization in StandardParser when processing multi-valued metadata (@wladimirleite)
+Fixes:
+#2035: Sometimes WhatsApp messages are out of order (@wladimirleite)
+#2203: Broken links in HTML report when "thumbs only" option is used (@wladimirleite)
+#2196: Exception when using "--nogui" in a never used Windows machine (@wladimirleite)
+#2190: Fail to parse some iOS WhatsApp DBs (@wladimirleite)
+#2121: NullPointerExceptions in AppMapPanel class (@patrickdalla)
+#2175: Range queries that use localized properties for date fields are not working properly (@wladimirleite)
+#2158: EmbeddedDiskProcessTask doesn't work on paths with alternate data streams (@lfcnassif)
+#2033: Aborting OutOfMemoryError caused by TwelveMonkeys TIF reader plugin (@wladimirleite)
+#1926: Map items checkboxes inconsistent selection (@patrickdalla)
+#2003: In rare cases, Telegram iOS DBs decoding may fail and cause exceptions (@wladimirleite)
+
+
 05/04/2024: IPED-4.1.6:
 Optimizations:
 #455:  Optimization of UFDR reader memory usage (@lfcnassif)

diff --git a/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java b/iped-app/src/main/java/iped/app/ui/CaseSearcherFilter.java
@@ -57,7 +57,7 @@ public class CaseSearcherFilter extends CancelableWorker<MultiSearchResult, Obje
     ArrayList<CaseSearchFilterListener> listeners = new ArrayList<CaseSearchFilterListener>();
     RoaringBitmap[] unionsArray;
     RoaringBitmap[] excludeUnionsArray;
-    
+
     public static SoftReference<MultiSearchResult> allItemsCache;
     private static IPEDSource ipedCase;
 
@@ -193,27 +193,7 @@ public MultiSearchResult doInBackground() {
                         }
 
                         IResultSetFilterer iRSFilterer = (IResultSetFilterer) iterator.next();
-                        if (filterManager.isFiltererEnabled(iRSFilterer)) {
-                            IFilter rsFilter = iRSFilterer.getFilter();
-
-                            if (rsFilter != null) {
-                                if (rsFilter instanceof IBitmapFilter) {// if the filter exposes a internal bitmap
-                                    addBitmapFilter((IBitmapFilter) rsFilter);
-                                } else {
-                                    RoaringBitmap[] cachedBitmaps = filterManager.getCachedBitmaps((IResultSetFilter) rsFilter);
-                                    if (cachedBitmaps != null) { // if filtermanager returned a cached bitmap
-                                        addBitmapFilter(cachedBitmaps);
-                                    } else {
-                                        MultiSearchResult newresult = filterManager.applyFilter((IResultSetFilter) rsFilter, result);
-                                        if (newresult != result) {
-                                            numFilters++;
-                                            result = newresult;
-                                            result.setIPEDSource(ipedCase);
-                                        }
-                                    }
-                                }
-                            }
-                        }
+                        applyFilterer(iRSFilterer, result);
                     }
                 }
 
@@ -242,7 +222,6 @@ public MultiSearchResult doInBackground() {
                     e.printStackTrace();
                 }
                 return new MultiSearchResult(new ItemId[0], new float[0]);
-
             }
 
             result.setIpedSearcher(searcher);
@@ -253,7 +232,62 @@ public MultiSearchResult doInBackground() {
 
             return result;
         }
+    }
+
+    private void applyFilterer(IResultSetFilterer iRSFilterer, MultiSearchResult result2) {
+        if (filterManager.isFiltererEnabled(iRSFilterer)) {
+            IFilter rsFilter = iRSFilterer.getFilter();
+
+            if (rsFilter != null) {
+                if (rsFilter instanceof IBitmapFilter) {// if the filter exposes a internal bitmap
+                    applyBitmapFilter((IBitmapFilter) rsFilter);
+                } else {
+                    RoaringBitmap[] cachedBitmaps = filterManager.getCachedBitmaps((IResultSetFilter) rsFilter);
+                    if (cachedBitmaps != null) { // if filtermanager returned a cached bitmap
+                        applyBitmapFilter(cachedBitmaps);
+                    } else {
+                        MultiSearchResult newresult = filterManager.applyFilter((IResultSetFilter) rsFilter, result);
+                        if (newresult != result) {
+                            numFilters++;
+                            result = newresult;
+                            result.setIPEDSource(ipedCase);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private void applyBitmapFilter(RoaringBitmap[] cachedBitmaps) {
+        RoaringBitmap[] lunionsArray = result.getCasesBitSets((IPEDMultiSource) ipedCase);
+        for (int i = 0; i < unionsArray.length; i++) {
+            lunionsArray[i].and(cachedBitmaps[i]);
+        }
+
+        MultiSearchResult newresult = filterManager.applyFilter(lunionsArray, result);
+        if (newresult != result) {
+            numFilters++;
+            result = newresult;
+            result.setIPEDSource(ipedCase);
+        }
+    }
+
+    private void applyBitmapFilter(IBitmapFilter rsFilter) {
+        RoaringBitmap[] lunionsArray = result.getCasesBitSets((IPEDMultiSource) ipedCase);
+        for (int i = 0; i < lunionsArray.length; i++) {
+            if (rsFilter.isToFilterOut()) {
+                lunionsArray[i].andNot(((IBitmapFilter) rsFilter).getBitmap()[i]);
+            } else {
+                lunionsArray[i].and(((IBitmapFilter) rsFilter).getBitmap()[i]);
+            }
+        }
 
+        MultiSearchResult newresult = filterManager.applyFilter(lunionsArray, result);
+        if (newresult != result) {
+            numFilters++;
+            result = newresult;
+            result.setIPEDSource(ipedCase);
+        }
     }
 
     @Override
@@ -317,24 +351,6 @@ public void setAppyUIFilters(boolean applyUIFilters) {
         this.applyUIFilters = applyUIFilters;
     }
 
-    public void addBitmapFilter(IBitmapFilter filter) {
-        RoaringBitmap[] lunionsArray = filter.getBitmap();
-
-        if (unionsArray == null) {
-            // creates the unionsArray and puts the cases bitmaps as the initial bitmap
-            // array
-            addBitmapFilter(result.getCasesBitSets((IPEDMultiSource) ipedCase));
-        }
-
-        for (int i = 0; i < unionsArray.length; i++) {
-            if (filter.isToFilterOut()) {
-                unionsArray[i].andNot(lunionsArray[i]);
-            } else {
-                unionsArray[i].and(lunionsArray[i]);
-            }
-        }
-    }
-
     public void addBitmapExcludeFilter(RoaringBitmap[] lunionsArray) {
         if (excludeUnionsArray == null) {
             excludeUnionsArray = new RoaringBitmap[lunionsArray.length];

diff --git a/iped-engine/src/main/java/iped/engine/data/IPEDSource.java b/iped-engine/src/main/java/iped/engine/data/IPEDSource.java
@@ -421,29 +421,27 @@ private int countNumItems(Category category) {
         if (category.getNumItems() != -1)
             return category.getNumItems();
 
-        if (!category.getChildren().isEmpty()) {
-            int num = 0;
-            for (Category child : category.getChildren()) {
-                num += countNumItems(child);
-            }
-            category.setNumItems(num);
-
-        } else {
-            String query = IndexItem.CATEGORY + ":\"" + category.getName() + "\"";
-            IPEDSearcher searcher = new IPEDSearcher(this, query);
-            searcher.setNoScoring(true);
-            try {
-                if (this instanceof IPEDMultiSource) {
-                    category.setNumItems(searcher.multiSearch().getLength());
-                } else {
-                    category.setNumItems(searcher.search().getLength());
-                }
+        int num = 0;
+        for (Category child : category.getChildren()) {
+            num += countNumItems(child);
+        }
 
-            } catch (Exception e) {
-                e.printStackTrace();
+        String query = IndexItem.CATEGORY + ":\"" + category.getName() + "\"";
+        IPEDSearcher searcher = new IPEDSearcher(this, query);
+        searcher.setNoScoring(true);
+        try {
+            if (this instanceof IPEDMultiSource) {
+                num += searcher.multiSearch().getLength();
+            } else {
+                num += searcher.search().getLength();
             }
+
+        } catch (Exception e) {
+            e.printStackTrace();
         }
-        return category.getNumItems();
+
+        category.setNumItems(num);
+        return num;
     }
 
     private void loadKeywords() {

diff --git a/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java b/iped-engine/src/main/java/iped/engine/search/MultiSearchResult.java
@@ -232,7 +232,18 @@ public RoaringBitmap[] getCasesBitSets(IPEDMultiSource multiSource) {
                 }
                 bitset.add(ids[i].getId());
             }
+
         }
-        return casesBitSet;
+        return clone(casesBitSet);
     }
+
+    private RoaringBitmap[] clone(RoaringBitmap[] casesBitSet2) {
+        RoaringBitmap[] lcasesBitSet = new RoaringBitmap[casesBitSet.length];
+        for (int i = 0; i < casesBitSet.length; i++) {
+            lcasesBitSet[i] = new RoaringBitmap();
+            lcasesBitSet[i].or(casesBitSet[i]);
+        }
+        return lcasesBitSet;
+    }
+
 }