From 4299facb0d1bc5bc35bbd4d8b4f7250ba8631a13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=20Keskim=C3=A4ki?= Date: Tue, 25 Apr 2023 10:59:45 +0300 Subject: [PATCH] Check for model max context --- src/server/router.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/server/router.ts b/src/server/router.ts index b6fc13eb..b4337882 100644 --- a/src/server/router.ts +++ b/src/server/router.ts @@ -73,6 +73,13 @@ router.post('/stream', async (req, res) => { // const usageAllowed = await checkUsage(user, service) // if (!usageAllowed) return res.status(403).send('Usage limit reached') + const encoding = getEncoding(options.model) + let tokenCount = calculateUsage(options, encoding) + + // gpt-3.5-turbo has maximum context of 4096 tokens + if (tokenCount > 4000) + return res.status(403).send('Model maximum context reached') + options.user = hashData(user.id) const stream = await completionStream(options) @@ -80,9 +87,6 @@ router.post('/stream', async (req, res) => { res.setHeader('content-type', 'text/plain') - const encoding = getEncoding(options.model) - - let tokenCount = calculateUsage(options, encoding) // https://github.com/openai/openai-node/issues/18#issuecomment-1493132878 stream.on('data', (chunk: Buffer) => { // Messages in the event stream are separated by a pair of newline characters.