Adjust similarity threshold

logancyang · Dec 30, 2024 · afad3ca · afad3ca
1 parent 8bf5a62
commit afad3ca
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 34 deletions.
diff --git a/src/components/chat-components/RelevantNotes.tsx b/src/components/chat-components/RelevantNotes.tsx
@@ -70,7 +70,7 @@ function inSameFolder(path1: string, path2: string) {
 
 function SimilarityBadge({ score }: { score: number }) {
   let text = "🔴 Low Similarity";
-  if (score > 0.5) text = "🟠 Medium Similarity";
+  if (score > 0.55) text = "🟠 Medium Similarity";
   if (score > 0.7) text = "🟢 High Similarity";
   return (
     <Tooltip>

diff --git a/src/main.ts b/src/main.ts
@@ -15,7 +15,7 @@ import { CHAT_VIEWTYPE, DEFAULT_OPEN_AREA, EVENT_NAMES } from "@/constants";
 import { CustomPromptProcessor } from "@/customPromptProcessor";
 import { encryptAllKeys } from "@/encryptionService";
 import { CustomError } from "@/error";
-import { calculateScoreDistribution, findRelevantNotes } from "@/search/findRelevantNotes";
+import { findRelevantNotes } from "@/search/findRelevantNotes";
 import { HybridRetriever } from "@/search/hybridRetriever";
 import { getAllQAMarkdownContent } from "@/search/searchUtils";
 import VectorStoreManager from "@/search/vectorStoreManager";
@@ -318,7 +318,6 @@ export default class CopilotPlugin extends Plugin {
         }
 
         const db = await this.vectorStoreManager.getDb();
-        await calculateScoreDistribution(db);
         const relevantNotes = await findRelevantNotes({
           db,
           filePath: activeFile.path,

diff --git a/src/search/findRelevantNotes.ts b/src/search/findRelevantNotes.ts
@@ -208,34 +208,3 @@ export async function findRelevantNotes({
     })
     .filter((entry) => entry !== null);
 }
-
-function computeHistogram(scores: number[], binCount = 10): number[] {
-  // Initialize all bins to zero
-  const histogram = new Array<number>(binCount).fill(0);
-
-  // Fill each bin with the count of scores that fall into that range
-  for (const score of scores) {
-    // Determine the correct bin index based on the score
-    let index = Math.floor(score * binCount);
-    if (index === binCount) {
-      // Edge case: a score of exactly 1 goes into the last bin
-      index = binCount - 1;
-    }
-    histogram[index]++;
-  }
-
-  return histogram;
-}
-
-export async function calculateScoreDistribution(db: Orama<any>) {
-  const files = app.vault.getMarkdownFiles();
-  const scores = [];
-  for (const file of files) {
-    const similarityScoreMap = await calculateSimilarityScore({ db, filePath: file.path });
-    for (const score of similarityScoreMap.values()) {
-      scores.push(score);
-    }
-  }
-  const result = computeHistogram(scores);
-  console.log(result);
-}