From 3fe7a25759bb2960997de81f9fd433ddf72987b6 Mon Sep 17 00:00:00 2001 From: Sean Hatfield <seanhatfield5@gmail.com> Date: Wed, 17 Jan 2024 16:25:30 -0800 Subject: [PATCH] add token context limit for native llm settings (#614) Co-authored-by: timothycarambat <rambat1010@gmail.com> --- .../LLMSelection/NativeLLMOptions/index.jsx | 70 ++++++++++++------- server/models/systemSettings.js | 1 + server/utils/AiProviders/native/index.js | 2 - server/utils/helpers/updateENV.js | 5 ++ 4 files changed, 50 insertions(+), 28 deletions(-) diff --git a/frontend/src/components/LLMSelection/NativeLLMOptions/index.jsx b/frontend/src/components/LLMSelection/NativeLLMOptions/index.jsx index a41a81fe8..457c09322 100644 --- a/frontend/src/components/LLMSelection/NativeLLMOptions/index.jsx +++ b/frontend/src/components/LLMSelection/NativeLLMOptions/index.jsx @@ -54,31 +54,49 @@ function NativeModelSelection({ settings }) { } return ( - <div className="flex flex-col w-60"> - <label className="text-white text-sm font-semibold block mb-4"> - Model Selection - </label> - <select - name="NativeLLMModelPref" - required={true} - className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5" - > - {customModels.length > 0 && ( - <optgroup label="Your loaded models"> - {customModels.map((model) => { - return ( - <option - key={model.id} - value={model.id} - selected={settings.NativeLLMModelPref === model.id} - > - {model.id} - </option> - ); - })} - </optgroup> - )} - </select> - </div> + <> + <div className="flex flex-col w-60"> + <label className="text-white text-sm font-semibold block mb-4"> + Model Selection + </label> + <select + name="NativeLLMModelPref" + required={true} + className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5" + > + {customModels.length > 0 && ( + <optgroup label="Your loaded models"> + {customModels.map((model) => { + return ( + <option + key={model.id} + value={model.id} + selected={settings.NativeLLMModelPref === model.id} + > + {model.id} + </option> + ); + })} + </optgroup> + )} + </select> + </div> + <div className="flex flex-col w-60"> + <label className="text-white text-sm font-semibold block mb-4"> + Token context window + </label> + <input + type="number" + name="NativeLLMTokenLimit" + className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5" + placeholder="4096" + min={1} + onScroll={(e) => e.target.blur()} + defaultValue={settings?.NativeLLMTokenLimit} + required={true} + autoComplete="off" + /> + </div> + </> ); } diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 53d42f2e2..1c4069ac9 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -174,6 +174,7 @@ const SystemSettings = { ...(llmProvider === "native" ? { NativeLLMModelPref: process.env.NATIVE_LLM_MODEL_PREF, + NativeLLMTokenLimit: process.env.NATIVE_LLM_MODEL_TOKEN_LIMIT, // For embedding credentials when ollama is selected. OpenAiKey: !!process.env.OPEN_AI_KEY, diff --git a/server/utils/AiProviders/native/index.js b/server/utils/AiProviders/native/index.js index fff904c46..de1a97f3d 100644 --- a/server/utils/AiProviders/native/index.js +++ b/server/utils/AiProviders/native/index.js @@ -94,8 +94,6 @@ class NativeLLM { } // Ensure the user set a value for the token limit - // and if undefined - assume 4096 window. - // DEV: Currently this ENV is not configurable. promptWindowLimit() { const limit = process.env.NATIVE_LLM_MODEL_TOKEN_LIMIT || 4096; if (!limit || isNaN(Number(limit))) diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 54e684029..f44b040b7 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -110,6 +110,11 @@ const KEY_MAPPING = { checks: [isDownloadedModel], }, + NativeLLMTokenLimit: { + envKey: "NATIVE_LLM_MODEL_TOKEN_LIMIT", + checks: [nonZero], + }, + EmbeddingEngine: { envKey: "EMBEDDING_ENGINE", checks: [supportedEmbeddingModel],