use llama3-tokenizer-js

package.json

@@ -19,7 +19,7 @@
     "ai": "^3.2.0",
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.0",
-    "mistral-tokenizer-js": "^1.0.0",
+    "llama3-tokenizer-js": "^1.1.3",
     "next": "14.1.4",
     "next-themes": "^0.3.0",
     "react": "^18",

src/components/chat/chat-bottombar.tsx

@@ -4,7 +4,7 @@ import React from "react";
 
 import { PaperPlaneIcon, StopIcon } from "@radix-ui/react-icons";
 import { ChatRequestOptions } from "ai";
-import mistralTokenizer from "mistral-tokenizer-js";
+import llama3Tokenizer from "llama3-tokenizer-js";
 import TextareaAutosize from "react-textarea-autosize";
 
 import { basePath, useHasMounted } from "@/lib/utils";
@@ -41,7 +41,7 @@ export default function ChatBottombar({
       handleSubmit(e as unknown as React.FormEvent<HTMLFormElement>);
     }
   };
-  const tokenCount = input ? mistralTokenizer.encode(input).length - 1 : 0;
+  const tokenCount = input ? llama3Tokenizer.encode(input).length - 1 : 0;
 
   const [tokenLimit, setTokenLimit] = React.useState<number>(4096);
   React.useEffect(() => {

src/lib/token-counter.ts

@@ -1,5 +1,5 @@
 import { CoreMessage, Message } from "ai";
-import mistralTokenizer from "mistral-tokenizer-js";
+import llama3Tokenizer from "llama3-tokenizer-js";
 
 export const getTokenLimit = async (basePath: string) => {
   const res = await fetch(basePath + "/api/settings");
@@ -18,9 +18,9 @@ export const encodeChat = (messages: Message[] | CoreMessage[]): number => {
   let numTokens = 0;
   for (const message of messages) {
     numTokens += tokensPerMessage;
-    numTokens += mistralTokenizer.encode(message.role).length;
+    numTokens += llama3Tokenizer.encode(message.role).length;
     if (typeof message.content === "string") {
-      numTokens += mistralTokenizer.encode(message.content).length;
+      numTokens += llama3Tokenizer.encode(message.content).length;
     }
   }
   numTokens += 3;

yarn.lock

@@ -2603,6 +2603,11 @@ lines-and-columns@^1.1.6:
   resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632"
   integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==
 
+llama3-tokenizer-js@^1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/llama3-tokenizer-js/-/llama3-tokenizer-js-1.1.3.tgz#db5d905879180c847917c50e7386e75a0b2530c7"
+  integrity sha512-ST7tpVSPw4oO/ibZxwPlNyEOpsr4StZwJQh1zIVvmRmtbeGVeOssSEcNhd9Ul61sZEkx1zQ4iSBo9rDUX0gWlA==
+
 locate-path@^6.0.0:
   version "6.0.0"
   resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286"
@@ -3149,11 +3154,6 @@ minimist@^1.2.0, minimist@^1.2.6:
   resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707"
   integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==
 
-mistral-tokenizer-js@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/mistral-tokenizer-js/-/mistral-tokenizer-js-1.0.0.tgz#d501a3ecbe5ab4ea3cd2061e303510b864052e32"
-  integrity sha512-9+M/2Ul5M5oRFwX+QmwUIxpi7iptLgkIs87f3DEwVqmt/hQ7j4RGkawOBjs3LsmJzEbpSQcv8GGOMG1lwoAKyw==
-
 [email protected]:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"