tweaking

badass-courses · Mar 27, 2024 · 35fc420 · 35fc420
1 parent 871b8b0
commit 35fc420
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/apps/course-builder-web/src/utils/vector-utils/rag.ts b/apps/course-builder-web/src/utils/vector-utils/rag.ts
@@ -36,7 +36,10 @@ const vectorStore = new PineconeVectorStore({
 	indexName: 'rag',
 })
 
-const nodeParser = new MarkdownNodeParser()
+const nodeParser = new MarkdownNodeParser({
+	includeMetadata: true,
+	includePrevNextRel: true,
+})
 
 const storageContext = await storageContextFromDefaults({
 	vectorStore,
@@ -48,6 +51,8 @@ const serviceContext = serviceContextFromDefaults({
 		model: 'gpt-4-0125-preview',
 	}),
 	embedModel: new OpenAIEmbedding(),
+	chunkSize: 10000,
+	chunkOverlap: 100,
 })
 
 export async function ingest(source_filename: string) {