truncation option for OpenPlugin.fetchPlugin complete

npm-core/openplugincore/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openplugincore",
-  "version": "0.5.1",
+  "version": "0.6.1",
   "license": "MIT",
   "author": "Sebastian Sosa",
   "main": "dist/index.js",

npm-core/openplugincore/src/openPlugin.ts

@@ -10,6 +10,8 @@ import {
   HumanMessagePromptTemplate,
 } from 'oplangchain/prompts';
 import { JsonOutputFunctionsParser } from 'oplangchain/output_parsers';
+import { estimateTokens, truncateJsonRoot } from './util/prompting';
+import { openaiModelsInfo } from './util/constants';
 
 type OpenPluginManifest = {
   description_for_model: string;
@@ -36,6 +38,12 @@ type ChatgptFunctionMessage = {
   name: string;
   content: string;
 };
+type fetchPluginArgs = {
+  prompt: string;
+  truncate?: boolean | number;
+  truncateOffset?: number;
+  [key: string]: any;
+};
 
 export class OpenPlugin {
   pluginName: string | undefined;
@@ -179,8 +187,9 @@ export class OpenPlugin {
     }
     return [openai_fns, callApiFn as Callable];
   }
-  async fetchPlugin(args: any = {}): Promise<ChatgptFunctionMessage> {
-    const { prompt, ...chatgpt_args } = args;
+
+  async fetchPlugin(args: fetchPluginArgs): Promise<ChatgptFunctionMessage> {
+    const { prompt, truncate, truncateOffset, ...chatgpt_args } = args;
     const model = chatgpt_args['model'];
     if (model !== 'gpt-3.5-turbo-0613' && model !== 'gpt-4-0613') {
       throw new Error('Model must be either gpt-3.5-turbo-0613 or gpt-4-0613');
@@ -252,6 +261,25 @@ export class OpenPlugin {
     );
     let json_response = await JSON.parse(request_out);
 
+    if (truncate) {
+      let truncateTo;
+      if (typeof truncate === 'number') {
+        truncateTo = truncate;
+      } else {
+        const tokenSlack = 56;
+        const dummyChatgptMessage = {
+          role: 'user',
+          content: prompt,
+        };
+        truncateTo =
+          openaiModelsInfo[model].maxTokens -
+          estimateTokens(JSON.stringify(dummyChatgptMessage)) -
+          tokenSlack -
+          (truncateOffset || 0);
+      }
+      json_response = truncateJsonRoot(json_response, truncateTo);
+    }
+
     if (this.verbose) {
       console.log(
         `${this.pluginName} json_response: `,

npm-core/openplugincore/src/util/constants.ts

@@ -0,0 +1,21 @@
+type ModelInfo = {
+  maxTokens: number;
+};
+type ModelsInfo = {
+  [key: string]: ModelInfo;
+};
+
+export const openaiModelsInfo: ModelsInfo = {
+  'gpt-3.5-turbo-0613': {
+    maxTokens: 4096,
+  },
+  'gpt-3.5-turbo-16k-0613': {
+    maxTokens: 16384,
+  },
+  'gpt-4-0613': {
+    maxTokens: 8192,
+  },
+  'gpt-4-32k-0613': {
+    maxTokens: 32768,
+  },
+};

npm-core/openplugincore/src/util/prompting.ts

@@ -0,0 +1,101 @@
+export const estimateTokens = (str: string): number => {
+  return Math.ceil(str.length / 2);
+};
+
+const truncateString = (str: string, truncateBy: number): string => {
+  return str.slice(0, -truncateBy);
+};
+
+const truncateArray = (arr: any[], truncateBy: number): any[] => {
+  // slice a random index in the array
+  const randomIndex = Math.floor(Math.random() * arr.length);
+  arr.splice(randomIndex, truncateBy);
+  return arr;
+};
+
+const handleArray = (arr: any[], truncateBy: number): any[] => {
+  const reducedArray = arr.map((item) => {
+    if (typeof item === 'object' && estimateTokens(JSON.stringify(item)) > 20) {
+      return truncateJson(item, truncateBy);
+    } else if (typeof item === 'string') {
+      return truncateString(item, truncateBy);
+    } else {
+      return item;
+    }
+  });
+
+  // if the nothing changed in the reducedArray delete a random item from the array
+  if (JSON.stringify(reducedArray) === JSON.stringify(arr)) {
+    return truncateArray(arr, truncateBy);
+  } else {
+    return reducedArray;
+  }
+};
+
+const truncateJson = (json: any, truncateBy: number): any => {
+  if (typeof json === 'string') {
+    return truncateString(json, truncateBy);
+  } else if (Array.isArray(json)) {
+    return handleArray(json, truncateBy);
+  } else if (typeof json === 'object') {
+    // ensure keys can get deleted
+    let hasDeletedMisc = false;
+    for (let key in json) {
+      // if null or empty object, delete
+      if (
+        !json[key] ||
+        (typeof json[key] === 'object' && Object.keys(json[key]).length === 0)
+      ) {
+        delete json[key];
+        continue;
+      }
+
+      // if not string, array, or object, delete. Also delete if empty string or empty array
+      if (
+        !hasDeletedMisc &&
+        (typeof json[key] !== 'string' ||
+          (typeof json[key] === 'string' && !json[key].length)) &&
+        (!Array.isArray(json[key]) ||
+          (Array.isArray(json[key]) && !json[key].length)) &&
+        typeof json[key] !== 'object'
+      ) {
+        hasDeletedMisc = true;
+        delete json[key];
+      } else {
+        json[key] = truncateJson(json[key], truncateBy);
+      }
+    }
+    if (Object.keys(json).length === 0) {
+      return null; // remove empty objects
+    }
+    return json;
+  } else {
+    return json;
+  }
+};
+
+export const truncateJsonRoot = (
+  json: any,
+  truncateTo: number,
+  verbose = false
+): any => {
+  if (verbose)
+    console.log(
+      'original json token count: ' + estimateTokens(JSON.stringify(json))
+    );
+  let truncateBy = 1; // adjust this value as needed
+  let prevJson;
+  while (estimateTokens(JSON.stringify(json)) > truncateTo) {
+    prevJson = JSON.stringify(json);
+    json = truncateJson(json, truncateBy);
+    if (JSON.stringify(json) === prevJson) {
+      console.log('Failed to complete truncation.');
+      break; // break out of the loop if truncateJson is not able to make any changes
+    }
+  }
+  if (verbose)
+    console.log(
+      'final json token count: ' + estimateTokens(JSON.stringify(json))
+    );
+  return json;
+};

npm-core/openplugincore/test/openPlugin.test.ts

@@ -3,6 +3,7 @@ import { beforeEach, describe, expect, test } from '@jest/globals';
 import { OpenPlugin } from '../src/index'; // Adjust the import path as necessary
 import { todo_plugin as mock_todo_plugin } from './mockData';
 import dotenv from 'dotenv';
+import { estimateTokens } from '../src/util/prompting';
 dotenv.config();
 
 const openaiApiKey = process.env.OPENAI_API_KEY;
@@ -70,3 +71,14 @@ describe('OpenPlugin todo', () => {
     }, 30000);
   });
 });
+
+test('truncated response test', async () => {
+  const ytPlugin = new OpenPlugin('yt_caption_retriever');
+  await ytPlugin.init();
+  const response = await ytPlugin.fetchPlugin({
+    prompt: 'summarize this video https://www.youtube.com/watch?v=oAknbBFo-U0',
+    model: 'gpt-3.5-turbo-0613',
+    truncate: true,
+  });
+  expect(estimateTokens(response.content)).toBe(3951);
+}, 30000);

npm-core/openplugincore/yarn.lock

@@ -2755,16 +2755,11 @@ domexception@^4.0.0:
   dependencies:
     webidl-conversions "^7.0.0"
 
-dotenv@*:
+dotenv@*, dotenv@^16.3.1:
   version "16.3.1"
   resolved "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz"
   integrity sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==
 
-dotenv@^16.3.1:
-  version "16.3.1"
-  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.3.1.tgz#369034de7d7e5b120972693352a3bf112172cc3e"
-  integrity sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==
-
 dts-cli@^2.0.3:
   version "2.0.3"
   resolved "https://registry.npmjs.org/dts-cli/-/dts-cli-2.0.3.tgz"