add token context limit for native llm settings (#614)

Co-authored-by: timothycarambat <rambat1010@gmail.com>

frontend/src/components/LLMSelection/NativeLLMOptions/index.jsx

@@ -54,6 +54,7 @@ function NativeModelSelection({ settings }) {
   }
 
   return (
+    <>
       <div className="flex flex-col w-60">
         <label className="text-white text-sm font-semibold block mb-4">
           Model Selection
@@ -80,5 +81,22 @@ function NativeModelSelection({ settings }) {
           )}
         </select>
       </div>
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          Token context window
+        </label>
+        <input
+          type="number"
+          name="NativeLLMTokenLimit"
+          className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
+          placeholder="4096"
+          min={1}
+          onScroll={(e) => e.target.blur()}
+          defaultValue={settings?.NativeLLMTokenLimit}
+          required={true}
+          autoComplete="off"
+        />
+      </div>
+    </>
   );
 }

server/models/systemSettings.js

@@ -174,6 +174,7 @@ const SystemSettings = {
       ...(llmProvider === "native"
         ? {
             NativeLLMModelPref: process.env.NATIVE_LLM_MODEL_PREF,
+            NativeLLMTokenLimit: process.env.NATIVE_LLM_MODEL_TOKEN_LIMIT,
 
             // For embedding credentials when ollama is selected.
             OpenAiKey: !!process.env.OPEN_AI_KEY,

server/utils/AiProviders/native/index.js

@@ -94,8 +94,6 @@ class NativeLLM {
   }
 
   // Ensure the user set a value for the token limit
-  // and if undefined - assume 4096 window.
-  // DEV: Currently this ENV is not configurable.
   promptWindowLimit() {
     const limit = process.env.NATIVE_LLM_MODEL_TOKEN_LIMIT || 4096;
     if (!limit || isNaN(Number(limit)))

server/utils/helpers/updateENV.js

@@ -110,6 +110,11 @@ const KEY_MAPPING = {
     checks: [isDownloadedModel],
   },
 
+  NativeLLMTokenLimit: {
+    envKey: "NATIVE_LLM_MODEL_TOKEN_LIMIT",
+    checks: [nonZero],
+  },
+
   EmbeddingEngine: {
     envKey: "EMBEDDING_ENGINE",
     checks: [supportedEmbeddingModel],