From 695779bb9812aa73421cc454cb57e16fdfbb6d34 Mon Sep 17 00:00:00 2001
From: Alex Komoroske <jkomoros@gmail.com>
Date: Sun, 29 Oct 2023 07:33:34 -0700
Subject: [PATCH] Hook up a document handler for cards when they change.

This is the first time exercising the current pipeline, and revels a few problems, like "invalid-card-id" is somehow in the document?

Part of #646.
---
 functions/src/embeddings.ts | 27 +++++++++++++++++++++++++--
 functions/src/index.ts      |  9 +++++++++
 gulpfile.js                 |  2 +-
 3 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/functions/src/embeddings.ts b/functions/src/embeddings.ts
index 97364626..e6b515df 100644
--- a/functions/src/embeddings.ts
+++ b/functions/src/embeddings.ts
@@ -16,9 +16,14 @@ import {
 
 import {
     FieldValue,
-    Timestamp
+    Timestamp,
 } from 'firebase-admin/firestore';
 
+import {
+    Change,
+    firestore
+} from 'firebase-functions'
+
 const DOM = new JSDOM();
 
 const EMBEDDINGS_COLLECTION = 'embeddings';
@@ -79,6 +84,7 @@ const innerTextForHTML = (html : string) : string => {
 
 const textContentForEmbeddingForCard = (card : Card) : string => {
     //TODO: ideally this would literally be the cardPlainContent implementation from src/util.ts
+    //TODO: this shouldn't use the title for working-notes cards, since it's computed.
     if (card.card_type != CARD_TYPE_CONTENT && card.card_type != CARD_TYPE_WORKING_NOTES) return '';
     const body = innerTextForHTML(card[TEXT_FIELD_BODY]);
     const title = card[TEXT_FIELD_TITLE] || '';
@@ -116,7 +122,7 @@ const embeddingInfoIDForCard = (card : Card, embeddingType : EmbeddingType = DEF
     return card.id + '+' + embeddingType + '+' + version;
 };
 
-export const processCard = async (card : Card) : Promise<void> => {
+const processCard = async (card : Card) : Promise<void> => {
     const id = embeddingInfoIDForCard(card);
     const record = await db.collection(EMBEDDINGS_COLLECTION).doc(id).get();
     const text = textContentForEmbeddingForCard(card);
@@ -141,4 +147,21 @@ export const processCard = async (card : Card) : Promise<void> => {
 
     await db.collection(EMBEDDINGS_COLLECTION).doc(id).update(info);
 
+};
+
+const deleteCard = async (card : Card) : Promise<void> => {
+    const id = embeddingInfoIDForCard(card);
+    await db.collection(EMBEDDINGS_COLLECTION).doc(id).delete();
+    //TODO: also delete item from hnsw index.
+};
+
+export const processCardEmbedding = async (change : Change<firestore.DocumentSnapshot>) : Promise<void> => {
+    //TODO: if openai key not set, then silently exit.
+    if (!change.after.exists) {
+        const card = {id : change.before.id, ...change.before.data()} as Card;
+        await deleteCard(card);
+        return;
+    }
+    const card = {id : change.after.id, ...change.after.data()} as Card;
+    await processCard(card);
 };
\ No newline at end of file
diff --git a/functions/src/index.ts b/functions/src/index.ts
index 060c24db..1ba0470f 100644
--- a/functions/src/index.ts
+++ b/functions/src/index.ts
@@ -17,6 +17,10 @@ import {
 	slug
 } from "./legal.js";
 
+import {
+	processCardEmbedding
+} from "./embeddings.js";
+
 import * as openaiimpl from "./openai.js";
 
 //Runs every three hours
@@ -41,6 +45,11 @@ export const emailAdminOnMessage = functions.firestore.
 	document('messages/{messageId}').
 	onCreate(email.onMessage);
 
+//TODO: only allow a single instance
+export const updateCardEmbedding = functions.firestore.
+	document('cards/{cardID}').
+	onWrite(processCardEmbedding);
+
 const screenshotApp = express();
 screenshotApp.get('/:id', async (req, res) => {
 	const png = await fetchScreenshotByIDOrSlug(req.params.id);
diff --git a/gulpfile.js b/gulpfile.js
index 97e4d776..5e9e310b 100644
--- a/gulpfile.js
+++ b/gulpfile.js
@@ -202,7 +202,7 @@ gulp.task(BUILD_TASK, makeExecutor('npm run build'));
 
 gulp.task(GENERATE_SEO_PAGES, makeExecutor('npm run generate:seo:pages'));
 
-gulp.task(FIREBASE_DEPLOY_TASK, makeExecutor(ENABLE_TWITTER ? 'firebase deploy' : 'firebase deploy --only hosting,storage,firestore,functions:emailAdminOnMessage,functions:emailAdminOnStar,functions:legal' + (OPENAI_ENABLED ? ',functions:openai' : '')));
+gulp.task(FIREBASE_DEPLOY_TASK, makeExecutor(ENABLE_TWITTER ? 'firebase deploy' : 'firebase deploy --only hosting,storage,firestore,functions:emailAdminOnMessage,functions:emailAdminOnStar,functions:legal' + (OPENAI_ENABLED ? ',functions:openai,functions:updateCardEmbedding' : '')));
 
 gulp.task(FIREBASE_SET_CONFIG_LAST_DEPLOY_AFFECTING_RENDERING, makeExecutor('firebase functions:config:set site.last_deploy_affecting_rendering=' + RELEASE_TAG));