sij/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2025-04-23 21:18:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions1

persist token window for NIM and not only on model change

Browse source
This commit is contained in:
timothycarambat 2024-12-05 11:57:07 -08:00
parent b2dd35fe15
commit 4b09a06590
2 changed files with 5 additions and 2 deletions
server/utils
AiProviders/nvidiaNim
index.js
helpers
updateENV.js

4
server/utils/AiProviders/nvidiaNim/index.js
View file

@ -45,7 +45,7 @@ class NvidiaNimLLM {
}
/**
* Set the model token limit `NVIDIA_NIM_LLM_TOKEN_LIMIT` for the given model ID
* Set the model token limit `NVIDIA_NIM_LLM_MODEL_TOKEN_LIMIT` for the given model ID
* @param {string} modelId
* @param {string} basePath
* @returns {Promise<void>}
@ -69,7 +69,7 @@ class NvidiaNimLLM {
if (!model.length) return;
const modelInfo = model.find((model) => model.id === modelId);
if (!modelInfo) return;
process.env.NVIDIA_NIM_LLM_TOKEN_LIMIT = Number(
process.env.NVIDIA_NIM_LLM_MODEL_TOKEN_LIMIT = Number(
modelInfo.max_model_len || 4096
);
}

3
server/utils/helpers/updateENV.js
View file

@ -965,6 +965,9 @@ function dumpENV() {
"SIMPLE_SSO_ENABLED",
// Community Hub
"COMMUNITY_HUB_BUNDLE_DOWNLOADS_ENABLED",
// Nvidia NIM Keys that are automatically managed
"NVIDIA_NIM_LLM_MODEL_TOKEN_LIMIT",
];
// Simple sanitization of each value to prevent ENV injection via newline or quote escaping.