Remove duplicated suggestions in awesome bar

We remove duplicated suggestions based on url, and remove those entries that only differ in 'http'/'https', and not end with '/' Signed-off-by: Songlin Jiang <[email protected]>
Igalia · Oct 18, 2023 · 391cdf2 · 391cdf2
1 parent 727defe
commit 391cdf2
Show file tree

Hide file tree

Showing 2 changed files with 79 additions and 1 deletion.
diff --git a/app/src/common/shared/com/igalia/wolvic/search/suggestions/SuggestionsProvider.java b/app/src/common/shared/com/igalia/wolvic/search/suggestions/SuggestionsProvider.java
@@ -14,10 +14,14 @@
 
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executor;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 public class SuggestionsProvider {
 
@@ -180,11 +184,59 @@ private CompletableFuture<List<SuggestionItem>> getSearchEngineSuggestions(@NonN
         return future;
     }
 
+    private CompletableFuture<List<SuggestionItem>> removeDuplicatedItems(@NonNull final List<SuggestionItem> items) {
+        CompletableFuture<List<SuggestionItem>> future = new CompletableFuture<>();
+
+        Map<String, List<Integer>> category = new HashMap<>();
+        Map<String, Integer> best = new HashMap<>();
+
+        // Filter out duplicate items based on the URL
+        for (int i = 0; i < items.size(); i++) {
+            SuggestionItem thisItem = items.get(i);
+            String normalizedUrl = UrlUtils.normalize(thisItem.url);
+            if (best.containsKey(normalizedUrl)) {
+                SuggestionItem bestItem = items.get(best.get(normalizedUrl));
+                // Select the best one that uses 'https' and contains trailing slashes
+                if (Objects.equals(UrlUtils.getProtocol(thisItem.url), "https") && Objects.equals(UrlUtils.getProtocol(bestItem.url), "http") ||
+                        thisItem.url.endsWith("/") && !bestItem.url.endsWith("/") ) {
+                    best.put(normalizedUrl, i);
+                }
+            } else {
+                category.put(normalizedUrl, new ArrayList<>());
+                best.put(normalizedUrl, i);
+            }
+            category.get(normalizedUrl).add(i);
+        }
+
+        List<Integer> toRemove = new ArrayList<>();
+        // Calculate the index to remove (category - best)
+        for (String url : category.keySet()) {
+            toRemove.addAll(category.get(url).stream().filter(
+                    index -> !Objects.equals(index, best.get(url))).collect(Collectors.toList()));
+        }
+
+        List<SuggestionItem> uniqueItems = new ArrayList<>();
+        for (int i = 0; i < items.size(); i++) {
+            if (!toRemove.contains(i)) {
+                uniqueItems.add(items.get(i));
+            }
+        }
+
+        if (mComparator != null) {
+            items.sort(mComparator);
+        }
+
+        future.complete(uniqueItems);
+
+        return future;
+    }
+
     public CompletableFuture<List<SuggestionItem>> getSuggestions() {
         return CompletableFuture.supplyAsync((Supplier<ArrayList<SuggestionItem>>) ArrayList::new)
                 .thenComposeAsync(this::getSearchEngineSuggestions)
                 .thenComposeAsync(this::getBookmarkSuggestions)
-                .thenComposeAsync(this::getHistorySuggestions);
+                .thenComposeAsync(this::getHistorySuggestions)
+                .thenComposeAsync(this::removeDuplicatedItems);
     }
 
 }
diff --git a/app/src/common/shared/com/igalia/wolvic/utils/UrlUtils.java b/app/src/common/shared/com/igalia/wolvic/utils/UrlUtils.java
@@ -237,6 +237,32 @@ public static String getHost(String uri) {
         }
     }
 
+    public static String getProtocol(String uri) {
+        try {
+            URL url = new URL(uri);
+            return url.getProtocol();
+        } catch (MalformedURLException e) {
+            return uri;
+        }
+    }
+
+    public static String normalize(String uri) {
+        // Remove the protocol and trailing '/'
+        try {
+            URL url = new URL(uri);
+            String normalized = url.getHost() + url.getPath();
+            if (url.getQuery() != null) {
+                normalized += "?" + url.getQuery();
+            }
+            if (normalized.endsWith("/")) {
+                normalized = normalized.substring(0, normalized.length() - 1);
+            }
+            return normalized;
+        } catch (MalformedURLException e) {
+            return uri;
+        }
+    }
+
     public static URI parseUri(String aUri) throws URISyntaxException {
         try {
             return new URI(aUri);