Skip to content

Commit

Permalink
Milvus bug fix (#2183)
Browse files Browse the repository at this point in the history
* patch no text results for milvus chunks

* wrap addDocumentToNamespace in try catch for handling milvus errors

* lint

* revert milvus db changes

* add try catch to handle grpc error from milvus
  • Loading branch information
shatfield4 authored Sep 9, 2024
1 parent cf6928f commit a58f271
Showing 1 changed file with 29 additions and 21 deletions.
50 changes: 29 additions & 21 deletions server/utils/vectorDbProviders/milvus/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -156,30 +156,38 @@ const Milvus = {
vectorDimension = chunks[0][0].values.length || null;

await this.getOrCreateCollection(client, namespace, vectorDimension);
for (const chunk of chunks) {
// Before sending to Pinecone and saving the records to our db
// we need to assign the id of each chunk that is stored in the cached file.
const newChunks = chunk.map((chunk) => {
const id = uuidv4();
documentVectors.push({ docId, vectorId: id });
return { id, vector: chunk.values, metadata: chunk.metadata };
});
const insertResult = await client.insert({
collection_name: this.normalize(namespace),
data: newChunks,
});
try {
for (const chunk of chunks) {
// Before sending to Milvus and saving the records to our db
// we need to assign the id of each chunk that is stored in the cached file.
const newChunks = chunk.map((chunk) => {
const id = uuidv4();
documentVectors.push({ docId, vectorId: id });
return { id, vector: chunk.values, metadata: chunk.metadata };
});
const insertResult = await client.insert({
collection_name: this.normalize(namespace),
data: newChunks,
});

if (insertResult?.status.error_code !== "Success") {
throw new Error(
`Error embedding into Milvus! Reason:${insertResult?.status.reason}`
);
if (insertResult?.status.error_code !== "Success") {
throw new Error(
`Error embedding into Milvus! Reason:${insertResult?.status.reason}`
);
}
}
await DocumentVectors.bulkInsert(documentVectors);
await client.flushSync({
collection_names: [this.normalize(namespace)],
});
return { vectorized: true, error: null };
} catch (insertError) {
console.error(
"Error inserting cached chunks:",
insertError.message
);
return { vectorized: false, error: insertError.message };
}
await DocumentVectors.bulkInsert(documentVectors);
await client.flushSync({
collection_names: [this.normalize(namespace)],
});
return { vectorized: true, error: null };
}
}

Expand Down

0 comments on commit a58f271

Please sign in to comment.