From fb5698801426f045e46dfc228f1adca70ed79bbc Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Thu, 1 Feb 2024 09:50:48 -0800 Subject: [PATCH] account for image projection in token count --- server/routes.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/routes.go b/server/routes.go index f29d9b2b..d2c7323e 100644 --- a/server/routes.go +++ b/server/routes.go @@ -1273,6 +1273,10 @@ func trimmedPrompt(ctx context.Context, chat *ChatHistory, model *Model) (string promptsToAdd = append(promptsToAdd, promptInfo{vars: chat.Prompts[i], tokenLen: len(encodedTokens)}) images = append(images, chat.Prompts[i].Images...) + + // clip has a projection dimension of 768 + // TODO: use kv['clip.vision.projection_dim'] from projection instead + totalTokenLength += 768 * len(chat.Prompts[i].Images) } // ensure the system prompt is included, if not already