Enable editing of OpenRouter stream timeout for slower connections (#1994)

frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx

@@ -1,27 +1,72 @@
 import System from "@/models/system";
+import { CaretDown, CaretUp } from "@phosphor-icons/react";
 import { useState, useEffect } from "react";
 
 export default function OpenRouterOptions({ settings }) {
+  const [showAdvancedControls, setShowAdvancedControls] = useState(false);
+
   return (
-    <div className="flex gap-[36px] mt-1.5">
-      <div className="flex flex-col w-60">
-        <label className="text-white text-sm font-semibold block mb-3">
-          OpenRouter API Key
-        </label>
-        <input
-          type="password"
-          name="OpenRouterApiKey"
-          className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
-          placeholder="OpenRouter API Key"
-          defaultValue={settings?.OpenRouterApiKey ? "*".repeat(20) : ""}
-          required={true}
-          autoComplete="off"
-          spellCheck={false}
-        />
+    <div className="flex flex-col gap-y-4 mt-1.5">
+      <div className="flex gap-[36px]">
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            OpenRouter API Key
+          </label>
+          <input
+            type="password"
+            name="OpenRouterApiKey"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="OpenRouter API Key"
+            defaultValue={settings?.OpenRouterApiKey ? "*".repeat(20) : ""}
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        {!settings?.credentialsOnly && (
+          <OpenRouterModelSelection settings={settings} />
+        )}
+      </div>
+      <AdvancedControls settings={settings} />
+    </div>
+  );
+}
+
+function AdvancedControls({ settings }) {
+  const [showAdvancedControls, setShowAdvancedControls] = useState(false);
+
+  return (
+    <div className="flex flex-col gap-y-4">
+      <button
+        type="button"
+        onClick={() => setShowAdvancedControls(!showAdvancedControls)}
+        className="text-white hover:text-white/70 flex items-center text-sm"
+      >
+        {showAdvancedControls ? "Hide" : "Show"} advanced controls
+        {showAdvancedControls ? (
+          <CaretUp size={14} className="ml-1" />
+        ) : (
+          <CaretDown size={14} className="ml-1" />
+        )}
+      </button>
+      <div hidden={!showAdvancedControls}>
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-3">
+            Stream Timeout (ms)
+          </label>
+          <input
+            type="number"
+            name="OpenRouterTimeout"
+            className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+            placeholder="Timeout value between token responses to auto-timeout the stream"
+            defaultValue={settings?.OpenRouterTimeout ?? 500}
+            autoComplete="off"
+            onScroll={(e) => e.target.blur()}
+            min={500}
+            step={1}
+          />
+        </div>
       </div>
-      {!settings?.credentialsOnly && (
-        <OpenRouterModelSelection settings={settings} />
-      )}
     </div>
   );
 }

server/models/systemSettings.js

@@ -424,6 +424,7 @@ const SystemSettings = {
       // OpenRouter Keys
       OpenRouterApiKey: !!process.env.OPENROUTER_API_KEY,
       OpenRouterModelPref: process.env.OPENROUTER_MODEL_PREF,
+      OpenRouterTimeout: process.env.OPENROUTER_TIMEOUT_MS,
 
       // Mistral AI (API) Keys
       MistralApiKey: !!process.env.MISTRAL_API_KEY,

server/utils/AiProviders/openRouter/index.js

@@ -38,6 +38,7 @@ class OpenRouterLLM {
 
     this.embedder = embedder ?? new NativeEmbedder();
     this.defaultTemp = 0.7;
+    this.timeout = this.#parseTimeout();
 
     if (!fs.existsSync(cacheFolder))
       fs.mkdirSync(cacheFolder, { recursive: true });
@@ -49,6 +50,22 @@ class OpenRouterLLM {
     console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args);
   }
 
+  /**
+   * OpenRouter has various models that never return `finish_reasons` and thus leave the stream open
+   * which causes issues in subsequent messages. This timeout value forces us to close the stream after
+   * x milliseconds. This is a configurable value via the OPENROUTER_TIMEOUT_MS value
+   * @returns {number} The timeout value in milliseconds (default: 500)
+   */
+  #parseTimeout() {
+    this.log(
+      `OpenRouter timeout is set to ${process.env.OPENROUTER_TIMEOUT_MS ?? 500}ms`
+    );
+    if (isNaN(Number(process.env.OPENROUTER_TIMEOUT_MS))) return 500;
+    const setValue = Number(process.env.OPENROUTER_TIMEOUT_MS);
+    if (setValue < 500) return 500;
+    return setValue;
+  }
+
   // This checks if the .cached_at file has a timestamp that is more than 1Week (in millis)
   // from the current date. If it is, then we will refetch the API so that all the models are up
   // to date.
@@ -161,7 +178,7 @@ class OpenRouterLLM {
   }
 
   handleStream(response, stream, responseProps) {
-    const timeoutThresholdMs = 500;
+    const timeoutThresholdMs = this.timeout;
     const { uuid = uuidv4(), sources = [] } = responseProps;
 
     return new Promise(async (resolve) => {

server/utils/helpers/updateENV.js

@@ -365,6 +365,10 @@ const KEY_MAPPING = {
     envKey: "OPENROUTER_MODEL_PREF",
     checks: [isNotEmpty],
   },
+  OpenRouterTimeout: {
+    envKey: "OPENROUTER_TIMEOUT_MS",
+    checks: [],
+  },
 
   // Groq Options
   GroqApiKey: {