Enhance time query

src/LLMProviders/chainRunner.ts

@@ -1,4 +1,11 @@
-import { ABORT_REASON, AI_SENDER, EMPTY_INDEX_ERROR_MESSAGE, LOADING_MESSAGES } from "@/constants";
+import {
+  ABORT_REASON,
+  AI_SENDER,
+  EMPTY_INDEX_ERROR_MESSAGE,
+  LOADING_MESSAGES,
+  MAX_CHARS_FOR_LOCAL_SEARCH_CONTEXT,
+} from "@/constants";
+import { BrevilabsClient } from "@/LLMProviders/brevilabsClient";
 import { getSystemPrompt } from "@/settings/model";
 import { ChatMessage } from "@/sharedState";
 import { ToolManager } from "@/tools/toolManager";
@@ -11,7 +18,6 @@ import {
 import { Notice } from "obsidian";
 import ChainManager from "./chainManager";
 import { COPILOT_TOOL_NAMES, IntentAnalyzer } from "./intentAnalyzer";
-import { BrevilabsClient } from "@/LLMProviders/brevilabsClient";
 
 export interface ChainRunner {
   run(
@@ -421,7 +427,7 @@ class CopilotPlusChainRunner extends BaseChainRunner {
 
         if (debug) console.log("==== Step 4: Preparing context ====");
         const timeExpression = this.getTimeExpression(toolCalls);
-        const context = this.formatLocalSearchResult(documents, timeExpression);
+        const context = this.prepareLocalSearchResult(documents, timeExpression);
 
         const currentTimeOutputs = toolOutputs.filter((output) => output.tool === "getCurrentTime");
         const enhancedQuestion = this.prepareEnhancedUserMessage(
@@ -593,11 +599,27 @@ class CopilotPlusChainRunner extends BaseChainRunner {
     return timeRangeCall ? timeRangeCall.args.timeExpression : "";
   }
 
-  private formatLocalSearchResult(documents: any[], timeExpression: string): string {
-    const formattedDocs = documents
-      .filter((doc) => doc.includeInContext)
+  private prepareLocalSearchResult(documents: any[], timeExpression: string): string {
+    // First filter documents with includeInContext
+    const includedDocs = documents.filter((doc) => doc.includeInContext);
+
+    // Calculate total content length
+    const totalLength = includedDocs.reduce((sum, doc) => sum + doc.content.length, 0);
+
+    // If total length exceeds threshold, calculate truncation ratio
+    let truncatedDocs = includedDocs;
+    if (totalLength > MAX_CHARS_FOR_LOCAL_SEARCH_CONTEXT) {
+      const truncationRatio = MAX_CHARS_FOR_LOCAL_SEARCH_CONTEXT / totalLength;
+      truncatedDocs = includedDocs.map((doc) => ({
+        ...doc,
+        content: doc.content.slice(0, Math.floor(doc.content.length * truncationRatio)),
+      }));
+    }
+
+    const formattedDocs = truncatedDocs
       .map((doc: any) => `Note in Vault: ${doc.content}`)
       .join("\n\n");
+
     return timeExpression
       ? `Local Search Result for ${timeExpression}:\n${formattedDocs}`
       : `Local Search Result:\n${formattedDocs}`;

src/constants.ts

@@ -23,6 +23,7 @@ export const CHUNK_SIZE = 4000;
 export const CONTEXT_SCORE_THRESHOLD = 0.4;
 export const TEXT_WEIGHT = 0.4;
 export const PLUS_MODE_DEFAULT_SOURCE_CHUNKS = 15;
+export const MAX_CHARS_FOR_LOCAL_SEARCH_CONTEXT = 512000;
 export const LOADING_MESSAGES = {
   DEFAULT: "",
   READING_FILES: "Reading files",

src/search/indexOperations.ts

@@ -9,7 +9,7 @@ import { App, Notice, TFile } from "obsidian";
 import { DBOperations } from "./dbOperations";
 import { extractAppIgnoreSettings, getFilePathsForQA } from "./searchUtils";
 
-const EMBEDDING_BATCH_SIZE = 64;
+const EMBEDDING_BATCH_SIZE = 16;
 const CHECKPOINT_INTERVAL = 8 * EMBEDDING_BATCH_SIZE;
 
 export interface IndexingState {
@@ -132,7 +132,20 @@ export class IndexOperations {
             console.log("Copilot index checkpoint save completed.");
           }
         } catch (err) {
-          this.handleIndexingError(err, batch[0].fileInfo.path, errors, rateLimitNoticeShown);
+          console.error("Batch processing error:", {
+            error: err,
+            batchSize: batch?.length || 0,
+            firstChunk: batch?.[0]
+              ? {
+                  path: batch[0].fileInfo?.path,
+                  contentLength: batch[0].content?.length,
+                  hasFileInfo: !!batch[0].fileInfo,
+                }
+              : "No chunks in batch",
+            errorType: err?.constructor?.name,
+            errorMessage: err?.message,
+          });
+          this.handleIndexingError(err, batch?.[0]?.fileInfo?.path, errors, rateLimitNoticeShown);
           if (this.isRateLimitError(err)) {
             rateLimitNoticeShown = true;
             break;
@@ -389,14 +402,14 @@ export class IndexOperations {
 
   private handleIndexingError(
     err: any,
-    file: TFile,
+    filePath: string,
     errors: string[],
     rateLimitNoticeShown: boolean
   ): void {
-    console.error(`Error indexing file ${file.path}:`, err);
-    errors.push(file.path);
+    console.error(`Error indexing file ${filePath || "unknown"}:`, err);
+    errors.push(filePath || "unknown");
     if (!rateLimitNoticeShown) {
-      new Notice(`Error indexing file ${file.path}. Check console for details.`);
+      new Notice(`Error indexing file ${filePath || "unknown"}. Check console for details.`);
     }
   }
 

src/tools/SearchTools.ts

@@ -33,9 +33,13 @@ const localSearchTool = tool(
         ? PLUS_MODE_DEFAULT_SOURCE_CHUNKS
         : getSettings().maxSourceChunks;
 
+    if (getSettings().debug) {
+      console.log("returnAll:", returnAll);
+    }
+
     const hybridRetriever = new HybridRetriever({
       minSimilarityScore: returnAll ? 0.0 : 0.1,
-      maxK: returnAll ? 100 : maxSourceChunks,
+      maxK: returnAll ? 1000 : maxSourceChunks,
       salientTerms,
       timeRange: timeRange
         ? {