patch text.substring bug from compressor (#1918)

Mintplex-Labs · Jul 22, 2024 · 18171bc · 18171bc
1 parent a43fa8a
commit 18171bc
Showing 1 changed file with 7 additions and 2 deletions.
diff --git a/server/utils/helpers/tiktoken.js b/server/utils/helpers/tiktoken.js
@@ -19,8 +19,13 @@ class TokenManager {
   // https://github.com/openai/tiktoken/blob/9e79899bc248d5313c7dd73562b5e211d728723d/tiktoken/core.py#L91C20-L91C38
   // Returns number[]
   tokensFromString(input = "") {
-    const tokens = this.encoder.encode(input, undefined, []);
-    return tokens;
+    try {
+      const tokens = this.encoder.encode(String(input), undefined, []);
+      return tokens;
+    } catch (e) {
+      console.error(e);
+      return [];
+    }
   }
 
   bytesFromTokens(tokens = []) {