From 633f4252067a1b38a63f375c3fe8d4a6e78e9bc3 Mon Sep 17 00:00:00 2001
From: Sean Hatfield <seanhatfield5@gmail.com>
Date: Fri, 23 Feb 2024 17:18:58 -0800
Subject: [PATCH] [FEAT] OpenRouter integration (#784)
* WIP openrouter integration
* add OpenRouter options to onboarding flow and data handling
* add todo to fix headers for rankings
* OpenRouter LLM support complete
* Fix hanging response stream with OpenRouter
update tagline
update comment
* update timeout comment
* wait for first chunk to start timer
* sort OpenRouter models by organization
* uppercase first letter of organization
* sort grouped models by org
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
---
README.md | 1 +
docker/.env.example | 4 +
.../LLMSelection/OpenRouterOptions/index.jsx | 97 +++
.../LLMSelection/TogetherAiOptions/index.jsx | 28 +-
.../src/media/llmprovider/openrouter.jpeg | Bin 0 -> 6366 bytes
.../GeneralSettings/LLMPreference/index.jsx | 9 +
.../Steps/DataHandling/index.jsx | 9 +
.../Steps/LLMPreference/index.jsx | 9 +
server/.env.example | 4 +
server/models/systemSettings.js | 12 +
server/utils/AiProviders/openRouter/index.js | 334 ++++++++++
server/utils/AiProviders/openRouter/models.js | 622 ++++++++++++++++++
.../AiProviders/openRouter/scripts/.gitignore | 1 +
.../AiProviders/openRouter/scripts/parse.mjs | 37 ++
server/utils/helpers/customModels.js | 19 +
server/utils/helpers/index.js | 3 +
server/utils/helpers/updateENV.js | 11 +
17 files changed, 1187 insertions(+), 13 deletions(-)
create mode 100644 frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx
create mode 100644 frontend/src/media/llmprovider/openrouter.jpeg
create mode 100644 server/utils/AiProviders/openRouter/index.js
create mode 100644 server/utils/AiProviders/openRouter/models.js
create mode 100644 server/utils/AiProviders/openRouter/scripts/.gitignore
create mode 100644 server/utils/AiProviders/openRouter/scripts/parse.mjs
diff --git a/README.md b/README.md
index 200355707..f77cdf2b8 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,7 @@ Some cool features of AnythingLLM
- [LocalAi (all models)](https://localai.io/)
- [Together AI (chat models)](https://www.together.ai/)
- [Perplexity (chat models)](https://www.perplexity.ai/)
+- [OpenRouter (chat models)](https://openrouter.ai/)
- [Mistral](https://mistral.ai/)
**Supported Embedding models:**
diff --git a/docker/.env.example b/docker/.env.example
index eed505782..16413ad3c 100644
--- a/docker/.env.example
+++ b/docker/.env.example
@@ -52,6 +52,10 @@ GID='1000'
# PERPLEXITY_API_KEY='my-perplexity-key'
# PERPLEXITY_MODEL_PREF='codellama-34b-instruct'
+# LLM_PROVIDER='openrouter'
+# OPENROUTER_API_KEY='my-openrouter-key'
+# OPENROUTER_MODEL_PREF='openrouter/auto'
+
# LLM_PROVIDER='huggingface'
# HUGGING_FACE_LLM_ENDPOINT=https://uuid-here.us-east-1.aws.endpoints.huggingface.cloud
# HUGGING_FACE_LLM_API_KEY=hf_xxxxxx
diff --git a/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx b/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx
new file mode 100644
index 000000000..aa4ccdb2e
--- /dev/null
+++ b/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx
@@ -0,0 +1,97 @@
+import System from "@/models/system";
+import { useState, useEffect } from "react";
+
+export default function OpenRouterOptions({ settings }) {
+ return (
+ <div className="flex gap-x-4">
+ <div className="flex flex-col w-60">
+ <label className="text-white text-sm font-semibold block mb-4">
+ OpenRouter API Key
+ </label>
+ <input
+ type="password"
+ name="OpenRouterApiKey"
+ className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
+ placeholder="OpenRouter API Key"
+ defaultValue={settings?.OpenRouterApiKey ? "*".repeat(20) : ""}
+ required={true}
+ autoComplete="off"
+ spellCheck={false}
+ />
+ </div>
+ <OpenRouterModelSelection settings={settings} />
+ </div>
+ );
+}
+
+function OpenRouterModelSelection({ settings }) {
+ const [groupedModels, setGroupedModels] = useState({});
+ const [loading, setLoading] = useState(true);
+
+ useEffect(() => {
+ async function findCustomModels() {
+ setLoading(true);
+ const { models } = await System.customModels("openrouter");
+ if (models?.length > 0) {
+ const modelsByOrganization = models.reduce((acc, model) => {
+ acc[model.organization] = acc[model.organization] || [];
+ acc[model.organization].push(model);
+ return acc;
+ }, {});
+
+ setGroupedModels(modelsByOrganization);
+ }
+
+ setLoading(false);
+ }
+ findCustomModels();
+ }, []);
+
+ if (loading || Object.keys(groupedModels).length === 0) {
+ return (
+ <div className="flex flex-col w-60">
+ <label className="text-white text-sm font-semibold block mb-4">
+ Chat Model Selection
+ </label>
+ <select
+ name="OpenRouterModelPref"
+ disabled={true}
+ className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+ >
+ <option disabled={true} selected={true}>
+ -- loading available models --
+ </option>
+ </select>
+ </div>
+ );
+ }
+
+ return (
+ <div className="flex flex-col w-60">
+ <label className="text-white text-sm font-semibold block mb-4">
+ Chat Model Selection
+ </label>
+ <select
+ name="OpenRouterModelPref"
+ required={true}
+ className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+ >
+ {Object.keys(groupedModels)
+ .sort()
+ .map((organization) => (
+ <optgroup key={organization} label={organization}>
+ {groupedModels[organization].map((model) => (
+ <option
+ key={model.id}
+ value={model.id}
+ selected={settings.OpenRouterModelPref === model.id}
+ >
+ {model.name}
+ </option>
+ ))}
+ </optgroup>
+ ))}
+ </select>
+ </div>
+ );
+}
diff --git a/frontend/src/components/LLMSelection/TogetherAiOptions/index.jsx b/frontend/src/components/LLMSelection/TogetherAiOptions/index.jsx
index e526b3afe..66ba1715e 100644
--- a/frontend/src/components/LLMSelection/TogetherAiOptions/index.jsx
+++ b/frontend/src/components/LLMSelection/TogetherAiOptions/index.jsx
@@ -76,19 +76,21 @@ function TogetherAiModelSelection({ settings }) {
required={true}
className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
>
- {Object.entries(groupedModels).map(([organization, models]) => (
- <optgroup key={organization} label={organization}>
- {models.map((model) => (
- <option
- key={model.id}
- value={model.id}
- selected={settings.TogetherAiModelPref === model.id}
- >
- {model.name}
- </option>
- ))}
- </optgroup>
- ))}
+ {Object.keys(groupedModels)
+ .sort()
+ .map((organization) => (
+ <optgroup key={organization} label={organization}>
+ {groupedModels[organization].map((model) => (
+ <option
+ key={model.id}
+ value={model.id}
+ selected={settings.OpenRouterModelPref === model.id}
+ >
+ {model.name}
+ </option>
+ ))}
+ </optgroup>
+ ))}
</select>
</div>
);
diff --git a/frontend/src/media/llmprovider/openrouter.jpeg b/frontend/src/media/llmprovider/openrouter.jpeg
new file mode 100644
index 0000000000000000000000000000000000000000..ce272ac81e15a41ebc3b2724085275a103bb0837
GIT binary patch
literal 6366
zcmcIo2UJsElYdF5F=9gRgsOB95$O??CM7`V%@C;q(m_BeQj{vvRS-m^N=J&FDjlSW
zbU{R=BTd@AVEz8P=j{2;emhCt`@Oj{b7yX5?%ZVm-TnlCR99A01|SdsfPgPx{{u8b
z?aUcVtgg1Qn#Q?<0q8A9f|EOm7yvjqd${SUD4~svP0+w!geSPWoW^1`{$l+Dt`88c
z13<4Z(bhi(|F_3O7p>h1po0{k;3<NOn+J$7AlCBsa3SJbAf_eQ;V**tA&7<CKnFqG
zP2^ksz|%x*`vY$hu^UeJ3;;k8AV%B%z(Pc9^#c=QK@;rUoIn{z5c4=WdV)S25HS*j
zwX?n+cxOHMxC6R?3ZMj_0V}{0um>Cg4?q~aJA-TXKkFSo(9;4ggE|E8>H=H=+&~Rm
zzz)<E0=b@mJ75joZ9u*?_;Lg#K>D-xf7aC9MuKP$3>l#g0Hjm<`>O%~Kz<hhcEk7g
zw;$~9?`8o2>>U8SO8BFHIRRYG1(1L15057e0FGP-fG02h@T{@{pbA`L#xWPX8~$J(
z7<h%+*Z{yv4gefB1^^mxt>LEs*YO|g4caE!2eJkM0Otn)8XW+TdItdb!1<1t?GFQI
z08$tU2?>l8ypWQTl93}Q$U!=C=ny3Wb%dH4bp(Z?p=V{Fp=G8+p%^(Cnc3JlI5=n+
zxOlkOd05#w*oj6Upei{TITZy36+118mi^zR{W<_e0l5Xa1%q$_P!t4)g6uZ{EMTHT
zVZ?&^O{8Q{5^@-EkP&h~zehj-%98BA2M{n&4GBYn-{D-w-{b!=3@IY!@ju9*urfcT
zO_OG!=3AnAM@;$@m=k|UfdjkLFLhW(F|nh|A~&zx|M8&{j;7DxQ-g0zY-^gvyaUd*
z5i(?cE5O)f5;fM`^`D$JJK#~5YKact8}kzu))qDU_EQW}IMe7WB5JnMFz}P3(6L&w
zF8w^pqnV3Xf*`@P9_vZrR-A4bf(xYU?2wtZ4~z+>@6KvTkKb1<926v|vhY<z?Q|Mu
zqOE^Az|1T^z1W&yuGs`X{LuQQ8vt-Pn$77K2M9@0W&p%M?+myNTy?)*I@65#_+uUj
zrf<0K+g5!B0FbpO7RdI*i+8gD0Q<GTALHQcL3veqpI5^%09i+VrDN+fnaS@lq6{fa
z@3LO+C#pbzd_1`2;>`Be=zLC1OyDd4$i-rRSwVfLSy)M9>JhJ*`C3TYpjra}5*2;D
zV+q+NrGt)|05sJ-d&J3Ddc3mhET+@?(HXD|e^PY*xOm>oSNTi+rEC*`!VZnZi9W+K
z>oY?g9^SI9-Jf0G;dYqM`<*qP-YV>rF*A8;{O<P`azd&T>+xOQ(s6<etY0FLc`*;C
z&5qq$?}~}PeBbKT0TaA%Ii!&vig777T<>b|4kU6R>walrW$oL9oUVsPM8*LjB#<r}
z=d`6)KQnvNO19P2;EWsbcGnkAG`cu3m&LDa*=N^UUjT#uo3cmwy-^DB^iEv#ujASN
zitUFGWD?wj{?h~&8!r-of&_t){whXdtpTKB7%UC?1QJC}#~BD#9tBvd5K`zq;8L1F
zI_-VMWJ)h_o_~32BS3Y}+9@cEKci=<SJ6>cl~p*@r)8*E(Vl5rWA~9oCH>V%d}02a
zm5U8r4$-bG^Zg7<sVlxTyv?FsPqLW*`PTgkqjvxyeNbDuav(?WonBUzLu(JcllW@i
z&6GRC)KSKf<fZO-ROoTNej`+(xrZ_W(`>w8qC8sfZSmw;7j<^7@A@LNwD2SYE2WmN
zaM!t-&k}rm^8NXeLnirg?@IA%k%mSKCv@D!P5NS>IQG%OQhvgtO48zz;Tsl;R&_0@
z{F9H3WAF6SrXL>fCW*@bT0(;HjjA%^Syzv8ZfT72Gj8S>$tm8RXh66)msOHUAVVL?
z3@R9MQ8&}6N2;&}ws?v77s+@~rex5?hSaQ7W5Z+|bWHr^yAtaSdQp;2!h7tWT0el%
z_#v2X#!|X}bknbB(DXaxvwV%Pwa%o2h#wGfz4IwXT*H!)38MFsys<nrvfqoB@d=E)
zT%3mLL6~&}l_i^Lp%zzN8vAZ>d*g{deJ|h&%|=ZBod_@E{61}jMnl$x?#4N!k+89!
z*Gq}7CcQe6Mf{^RH&UvK>uxel?X<}o*ygtHbRFGZD7YHgAnzR0pWo(hT-2bAQw?*N
zb3thM@o^E{LSoHDtkj>pcAOu6SVef-kgRk5WTLrG%-O416;?;-C94}t-Meq3-l_c{
zr2CpM#z_-?31{@49!7z<a!L16*>h_yIo1X(HuaD3Ma9xGjkaH!6%Cobm$pTOD1M2U
z^B8zg=`#uj5X7WEc=kdR!e=yRect>^Ixi-3dGD6*{dsLLB{@(BDGBcNzX<|GQH!B5
zNO2vPgJc9-hazO>xJ9S{&5OIYB_wP#TDJ5={Sv7RA2rGs1xRLFg;fc#ybut2^~$wa
zfWy;7;!OO9#0beci%=Ho$Cq!%YA#26a_}moaYGu5_`9?HkIw4RJnuXyX@*NpP5F2Y
zx3}mzkOnus=jc$8Z54jHk&#h?{#511Gm9qILQw)c44|8wD6km;WPf!rVoyWSpv82M
z;uz{5jzWo!?lEb*W;Eq3Cm83PXxZe%u1v1ci1_y4v6C88;e*2V<IEpfwW?bS1z1c&
z`tIDW>R9>U*|>!ds|%`hORAmAsUG3Bo|1R2T#3Bj;5+QUcV^(LUUBTVdQ@ohC9^5q
zQ24@$PCm2#TBV_RI|(iO)?QH&4zsnF{&wOI+zqyr>g#Hhw{}S1I%q0gdt>BZo#y}U
z+LTMg%hv&56vvSO1O_F6QjkFp&J*CC2DWKxE*K|9M+}QTE^!*~78HjRcfFnYKq*ky
zs`Te(K&A-YTdFMb&|jG`a**kGSD+zFXcyaK5^!~9*~la0#LtvBI&8(ymJyN(+ak^D
z=CN~k5szf(#2e_ePO5kcAD5<gV7annP!D}Rz3k{^dhDdE<((Y(>Zak--g9K@n+0s?
z@7q^ph1#X?*|+d|(J7<2dTu>sD%+ie>Xu&JrWSpI?L_7Svh<u3tgy{{-%=+Q?_2^w
zM~kgfU;eF6>WQL`v9@yWvM;hjHr(0b=e-3vMMl1edie>Qi=J$uJwkUy)L^p0*TdyI
zvZ=LzdgGcT(+ln3C~=IgMmpUCmfBmq!|7xkbxAh`3Ii$xOwjhKUVM0yL1`nH{^HT_
z+r=F7SQ08K=Dc;OmCWd91<{mce2}*B_EP-~lGvmihIXGsE0gi>c^86OT>YCv5e|8S
zit!(|V&NmhWpC=AOmDs*)W_Y<;#kl{#GH?ukuxiIZZ&cr4l~B)dON7OO9%{B3*uYX
zQT2I`;91Orm+N>Kr3GBeS@KgpGv^BOb^FskS>JT|?AbnKG`Bt>JF7q^-ksN2u{iS3
z;Zy#6UG;fyR(>*Cg{AVOUH$54h35=c$(4G!t+MjxV~XISueNb0@|y#KMT?$;-nC!p
z?yDz{^$U4FDw?DkH9J18Da=Esr8q*TiH_D6p{DbZj;>f~R*E{$w%Ok&>>FNn{yYn>
zezfZlroTQU^wiPrTtC(BSJe4U6^?lX4D+T72|r%Q7#<#6^U(F;`_a}4+m}4;A}uc>
zw~pv1aSu9NO?LWT)VIwYhWI9QIX~HA!DKOGQ@821zR}P=kRNt~wYoXDfWD{7H#<7!
zf-8UZ#y;RlOjSW7*n^-Xq$J?Z@FQ0tC_o%@TE~@B42xevqJ!cxOM3!ePcHvzSc;rr
z!-};{b<XzgEEbrbO|A_cS3H}in#d?R5a7Tc`!y>tf!ueO)uwP*fPK-2zt!SovFHxg
zeQC3Z^Hfb&NlH>8nf&8id+Ime(7ceboSq}$qf#s?^l51wGdsD0k~B|>$v?4EWFX#O
zp_)G`a{mk|<L=#v#0{z$iJRZ~)+)IM=r6{l>Ttgw3rj2)hm#c2an+_C&bvhYHH<av
zO)hED{jJQ0hsfBWP0CIR9T{EzVuBE1m6nY}c>T(Uqg;b;9y&W+NhT~S4fk9*)c5pm
ztou?HrA+_a+1L^W2UBZyk)4brc;o`dO!aNk*R$<FiJ$1!_R*($aXKZ}NLeYw+8OdD
zPE>#KH%YdyyZvRQYDpHSh}IuhJbmaAsj)U<7SqNJKg<&n_A;98HTkx~z<X^fBOIwc
z8fm?YjVHYx-_-p4T-!)(MuFbl@MV)#pM@{eI%Ojtt3=ktq^<l7p1qO&wwm!aDUG4=
zQfqvUAy<YJU(+ZD`0&9dtCF0#{-w&Y$x4erB<~rHBWswDB(2B~l{3qC%?DP<-IEk5
zB&>Ya>~)G;w3$;Tw&m<8Mmmip6K`j9zj?EghDB=TllYnTsd|;_pf#D3Mg+O*%AbBs
zW~vc!(nI$Zxyd!9$Dai=gYw827D_QBZd3B;J>*h?nreHlo!>?g`*>MwT14<kZDYw|
zR8F~zaDH@>C=(Ff6kT$8A5b@$s9uGX?0heauv~nYr@pcNK4VY$<LS4bQX1zbq;egH
z4&M_$Uf7g`iLbLO8z4krd8+%K)oiu;;B#1X;&o78Y%*_a;5z4w$Ff~|TI!<p?6G-6
z?;fCm605$nGIc(}mPL-!Vi3>6X&lr`>Tv?SeA|GUuEU8Mb+NB9HY^M7gd?kW^(Q}h
z&xX7@>Q=;ACeZDu$0HS1A@_99bIMo3A(xt*VTgXhrVw}S`@7FC2->IL7}G(^dK}=G
z5|0m2=Q*;K)HV{SR94mit9zfe4Z)N@_;qwp8OYTxx&M!M?|(C!C$jVRM3dohH^c%}
z6#vZLDi%zE4yF+5|0Txe3_c#bKmW9LSOXo*7*&i!Gi?0eXvTvAP*J&+$;&^rUMW$a
z!_@Wxxz)DcrXirv%YSu<1RYGtc@@-|`pvJWI))BzkCj`EuK4AF-0}q|1y9*2xjOR4
z{=b;=H!^;>6`1o|w^Q|pqo=}uPbybt)p1NLRA>AThiIXVP2#uKPayQm^1n*{Q|#Xv
z1@}NM;-LfbX9py<YjG43%_#=%gZK+BLBEej$i1jCxx;eeD$`MlMw#ITB9Nk$kSf&}
zRn2t~K4+O0)pKhJ@=x+P9GmBwIz14d0l{ohS9rwx$Q!2gDizG0LnDhqfS8Yg*Zs8*
z#-ov86oVmExv{2k_J{1OCiJQ&ohq<9?bkFM38%Hxz65o6Wh^yI%M9EaUN~jyk<H04
zAt2hDx%uk&>>(2jd^y~iRL(y@<Yxd782D@B_vsD-pg6_RVi+VCfa`@H0T7Q($lVmf
z_}GUwJ|XOL++QPbGBz|m=TrhF+MVCP#UmFWj_Ojw6a`FArd(xiF7saFy`4rYA&brL
zAEF^qwyk(;Q@AN3-iDWu^{G4@W+m`U$i#BBp)uFAj$0*!j8W)IuvCBE$T^(%3k0{*
z+0C+-#~)SB2J58Fo-MCE))zeNl$Kn#-9or|a|mT1A6hNMKFz%tft}6t^2*dNXmSdj
z@s70|{6|*;|F$5WSi!EuDGqieF$_NFH4+<_iSAh{4g9MsA@_l5n`zxjKD|VWV6Ia~
zt9Oqu`+U6_)``1-XoNZW`Gt2ce`9WlV^oi-x0}H3xm~~7WYOx&d#n*lryzX%?wTI?
zMrz++z_r%)-P!az5~2<{H!be=vK^^E#eFP2CYwS8kaD(G-k8B^ez_W&Iwla9Q^-c&
z<{jZE&oJkY3%Tm?Xb0n9E_F3ORcWnRopk|c495j#Dbbr>->Zlhyw0M+WeUH?SuqY%
z4pesZGhY8h`zU*Ucvg2(vo>bl0=N5-l5#A^7B1^8Q7GiO$6D?iaS_9706eIBt>hp5
zdF)xf{>Kur3!0R7x!;Z6caVAs3zW7>+n4~e%&(<#Lt;3I=V1xSV~ulTZ7kMfl?@tN
zQ)zN}>Le8sF0<C(Yo5b9CeKp^B~!5p(0|RkL~H&nU6HS(bX137j_2qKT`rWIAfv#V
zD1Qg%_BF`na^-dwb+Lx0&m<aJ1W^ip&MRY;3qh#t7;a}mxy8CVPn>`e1`Zz!frQ6S
z*vv(-HKkuzq;z-O2LFr`qcW7i<ihB3OW6WC>9`i^YXd?h{Zo0pCn>EMt5_U*uqQRL
zy4!-j$z4T~zWQj-a?&cGm=r&ic|8#s#ug_ombdI&;&3|P)b`!0xMoYmdMd^NGJw~<
zS=`<7a4ru53QQ#64Z7zI-!EFngqeR)X$XC>I{U16xzZIQYl!c+Q=`1$Po|#?56FQV
z+qdrnEQScWz@$nkiuX&{sR9^#!QHu+P5pHd(|D(6edO>Z%h}breE{SDTII1$S!eJU
zi$CWbaO!olkA>L7cwU`fzjDl%<iXi>%PEHsQFJWZ&UQUU8LeJBeM_;(GeQua@zm#Z
z!gQ{#5z^*wAj{_;`7^#ave!2#@8ad-B=Zqv>|@5Z%aM=bxoAHgI1D|@4cEDMY<{FO
zq;{rcT+7xMQ+1KYjwdoSehN;T-9XP4p!QH1#ZKd>{?_=*vfdMtA+KtVk)Bx&DSVZZ
ztr9tmqg-8x^jMs9RgeaZAumle``+7em21>}{M6JBx`j`sZ*63RKF4hxTS`{GcFf+O
ztN8S)2JY;VBG7yOj%86lUwdb>EqTujh6Tf8cgrT;);lbL9m9FbDk?&9&1V)tUYpz~
zTR_XU)*s3rfWk)bQDRzzcrRt$_mx1ZoB9+=7ak{J?^xp>X{@~8_w8d^scPmiCR*rB
zSN{c={W($5<P6G9l4u1D3z4y$QBp20E4j+q4Vr0*+?c24>(*m|$g^=Qay>|Ku1yR3
znN(-ySb<3bZ%jd)Hr$YAW*}G=gQjv2q|a6tSc|fK(mbyfSz$c%dfFfpLtSB7;7;xN
zHkjg~$igSJE?U|UHC4WSAZ2_Z?hxLUMXKm-W~U&9{6zI^XC_~m_1;sQALVT2Ftt*4
zGV2cWZOeejgvC4`xJ{W%Sjb>lRL?th`*UivLf1DnSB%QW?6SyxE_=vTvJdTcu;1({
zq}c~rjr{SBqg|CW8i5!;_jV;9vFy6Zf>LwLHypN>rq67*PSy^Y0Hx@ys@^Et2i7?D
GhyDw4i6SZh
literal 0
HcmV?d00001
diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx
index 8c51e559e..579c3903b 100644
--- a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx
+++ b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx
@@ -15,6 +15,7 @@ import TogetherAILogo from "@/media/llmprovider/togetherai.png";
import MistralLogo from "@/media/llmprovider/mistral.jpeg";
import HuggingFaceLogo from "@/media/llmprovider/huggingface.png";
import PerplexityLogo from "@/media/llmprovider/perplexity.png";
+import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
import PreLoader from "@/components/Preloader";
import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions";
@@ -31,6 +32,7 @@ import HuggingFaceOptions from "@/components/LLMSelection/HuggingFaceOptions";
import LLMItem from "@/components/LLMSelection/LLMItem";
import { MagnifyingGlass } from "@phosphor-icons/react";
import PerplexityOptions from "@/components/LLMSelection/PerplexityOptions";
+import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions";
export default function GeneralLLMPreference() {
const [saving, setSaving] = useState(false);
@@ -164,6 +166,13 @@ export default function GeneralLLMPreference() {
description:
"Run powerful and internet-connected models hosted by Perplexity AI.",
},
+ {
+ name: "OpenRouter",
+ value: "openrouter",
+ logo: OpenRouterLogo,
+ options: <OpenRouterOptions settings={settings} />,
+ description: "A unified interface for LLMs.",
+ },
{
name: "Native",
value: "native",
diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx
index f9c4c4169..51dc73004 100644
--- a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx
+++ b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx
@@ -12,6 +12,7 @@ import LocalAiLogo from "@/media/llmprovider/localai.png";
import MistralLogo from "@/media/llmprovider/mistral.jpeg";
import HuggingFaceLogo from "@/media/llmprovider/huggingface.png";
import PerplexityLogo from "@/media/llmprovider/perplexity.png";
+import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
import ZillizLogo from "@/media/vectordbs/zilliz.png";
import AstraDBLogo from "@/media/vectordbs/astraDB.png";
import ChromaLogo from "@/media/vectordbs/chroma.png";
@@ -118,6 +119,14 @@ const LLM_SELECTION_PRIVACY = {
],
logo: PerplexityLogo,
},
+ openrouter: {
+ name: "OpenRouter",
+ description: [
+ "Your chats will not be used for training",
+ "Your prompts and document text used in response creation are visible to OpenRouter",
+ ],
+ logo: OpenRouterLogo,
+ },
};
const VECTOR_DB_PRIVACY = {
diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
index 296a28d9e..df94652a8 100644
--- a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
+++ b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
@@ -12,6 +12,7 @@ import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
import MistralLogo from "@/media/llmprovider/mistral.jpeg";
import HuggingFaceLogo from "@/media/llmprovider/huggingface.png";
import PerplexityLogo from "@/media/llmprovider/perplexity.png";
+import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg";
import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions";
import AnthropicAiOptions from "@/components/LLMSelection/AnthropicAiOptions";
@@ -29,6 +30,7 @@ import System from "@/models/system";
import paths from "@/utils/paths";
import showToast from "@/utils/toast";
import { useNavigate } from "react-router-dom";
+import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions";
const TITLE = "LLM Preference";
const DESCRIPTION =
@@ -138,6 +140,13 @@ export default function LLMPreference({
description:
"Run powerful and internet-connected models hosted by Perplexity AI.",
},
+ {
+ name: "OpenRouter",
+ value: "openrouter",
+ logo: OpenRouterLogo,
+ options: <OpenRouterOptions settings={settings} />,
+ description: "A unified interface for LLMs.",
+ },
{
name: "Native",
value: "native",
diff --git a/server/.env.example b/server/.env.example
index 863486ad4..bed943925 100644
--- a/server/.env.example
+++ b/server/.env.example
@@ -45,6 +45,10 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
# PERPLEXITY_API_KEY='my-perplexity-key'
# PERPLEXITY_MODEL_PREF='codellama-34b-instruct'
+# LLM_PROVIDER='openrouter'
+# OPENROUTER_API_KEY='my-openrouter-key'
+# OPENROUTER_MODEL_PREF='openrouter/auto'
+
# LLM_PROVIDER='mistral'
# MISTRAL_API_KEY='example-mistral-ai-api-key'
# MISTRAL_MODEL_PREF='mistral-tiny'
diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
index 415448282..31d5c59a8 100644
--- a/server/models/systemSettings.js
+++ b/server/models/systemSettings.js
@@ -188,6 +188,18 @@ const SystemSettings = {
PerplexityApiKey: !!process.env.PERPLEXITY_API_KEY,
PerplexityModelPref: process.env.PERPLEXITY_MODEL_PREF,
+ // For embedding credentials when ollama is selected.
+ OpenAiKey: !!process.env.OPEN_AI_KEY,
+ AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT,
+ AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY,
+ AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF,
+ }
+ : {}),
+ ...(llmProvider === "openrouter"
+ ? {
+ OpenRouterApiKey: !!process.env.OPENROUTER_API_KEY,
+ OpenRouterModelPref: process.env.OPENROUTER_MODEL_PREF,
+
// For embedding credentials when ollama is selected.
OpenAiKey: !!process.env.OPEN_AI_KEY,
AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT,
diff --git a/server/utils/AiProviders/openRouter/index.js b/server/utils/AiProviders/openRouter/index.js
new file mode 100644
index 000000000..38a6f9f09
--- /dev/null
+++ b/server/utils/AiProviders/openRouter/index.js
@@ -0,0 +1,334 @@
+const { NativeEmbedder } = require("../../EmbeddingEngines/native");
+const { chatPrompt } = require("../../chats");
+const { v4: uuidv4 } = require("uuid");
+const { writeResponseChunk } = require("../../helpers/chat/responses");
+
+function openRouterModels() {
+ const { MODELS } = require("./models.js");
+ return MODELS || {};
+}
+
+class OpenRouterLLM {
+ constructor(embedder = null, modelPreference = null) {
+ const { Configuration, OpenAIApi } = require("openai");
+ if (!process.env.OPENROUTER_API_KEY)
+ throw new Error("No OpenRouter API key was set.");
+
+ const config = new Configuration({
+ basePath: "https://openrouter.ai/api/v1",
+ apiKey: process.env.OPENROUTER_API_KEY,
+ baseOptions: {
+ headers: {
+ "HTTP-Referer": "https://useanything.com",
+ "X-Title": "AnythingLLM",
+ },
+ },
+ });
+ this.openai = new OpenAIApi(config);
+ this.model =
+ modelPreference || process.env.OPENROUTER_MODEL_PREF || "openrouter/auto";
+ this.limits = {
+ history: this.promptWindowLimit() * 0.15,
+ system: this.promptWindowLimit() * 0.15,
+ user: this.promptWindowLimit() * 0.7,
+ };
+
+ this.embedder = !embedder ? new NativeEmbedder() : embedder;
+ this.defaultTemp = 0.7;
+ }
+
+ #appendContext(contextTexts = []) {
+ if (!contextTexts || !contextTexts.length) return "";
+ return (
+ "\nContext:\n" +
+ contextTexts
+ .map((text, i) => {
+ return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
+ })
+ .join("")
+ );
+ }
+
+ allModelInformation() {
+ return openRouterModels();
+ }
+
+ streamingEnabled() {
+ return "streamChat" in this && "streamGetChatCompletion" in this;
+ }
+
+ promptWindowLimit() {
+ const availableModels = this.allModelInformation();
+ return availableModels[this.model]?.maxLength || 4096;
+ }
+
+ async isValidChatCompletionModel(model = "") {
+ const availableModels = this.allModelInformation();
+ return availableModels.hasOwnProperty(model);
+ }
+
+ constructPrompt({
+ systemPrompt = "",
+ contextTexts = [],
+ chatHistory = [],
+ userPrompt = "",
+ }) {
+ const prompt = {
+ role: "system",
+ content: `${systemPrompt}${this.#appendContext(contextTexts)}`,
+ };
+ return [prompt, ...chatHistory, { role: "user", content: userPrompt }];
+ }
+
+ async isSafe(_input = "") {
+ // Not implemented so must be stubbed
+ return { safe: true, reasons: [] };
+ }
+
+ async sendChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) {
+ if (!(await this.isValidChatCompletionModel(this.model)))
+ throw new Error(
+ `OpenRouter chat: ${this.model} is not valid for chat completion!`
+ );
+
+ const textResponse = await this.openai
+ .createChatCompletion({
+ model: this.model,
+ temperature: Number(workspace?.openAiTemp ?? this.defaultTemp),
+ n: 1,
+ messages: await this.compressMessages(
+ {
+ systemPrompt: chatPrompt(workspace),
+ userPrompt: prompt,
+ chatHistory,
+ },
+ rawHistory
+ ),
+ })
+ .then((json) => {
+ const res = json.data;
+ if (!res.hasOwnProperty("choices"))
+ throw new Error("OpenRouter chat: No results!");
+ if (res.choices.length === 0)
+ throw new Error("OpenRouter chat: No results length!");
+ return res.choices[0].message.content;
+ })
+ .catch((error) => {
+ throw new Error(
+ `OpenRouter::createChatCompletion failed with: ${error.message}`
+ );
+ });
+
+ return textResponse;
+ }
+
+ async streamChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) {
+ if (!(await this.isValidChatCompletionModel(this.model)))
+ throw new Error(
+ `OpenRouter chat: ${this.model} is not valid for chat completion!`
+ );
+
+ const streamRequest = await this.openai.createChatCompletion(
+ {
+ model: this.model,
+ stream: true,
+ temperature: Number(workspace?.openAiTemp ?? this.defaultTemp),
+ n: 1,
+ messages: await this.compressMessages(
+ {
+ systemPrompt: chatPrompt(workspace),
+ userPrompt: prompt,
+ chatHistory,
+ },
+ rawHistory
+ ),
+ },
+ { responseType: "stream" }
+ );
+ return streamRequest;
+ }
+
+ async getChatCompletion(messages = null, { temperature = 0.7 }) {
+ if (!(await this.isValidChatCompletionModel(this.model)))
+ throw new Error(
+ `OpenRouter chat: ${this.model} is not valid for chat completion!`
+ );
+
+ const { data } = await this.openai
+ .createChatCompletion({
+ model: this.model,
+ messages,
+ temperature,
+ })
+ .catch((e) => {
+ throw new Error(e.response.data.error.message);
+ });
+
+ if (!data.hasOwnProperty("choices")) return null;
+ return data.choices[0].message.content;
+ }
+
+ async streamGetChatCompletion(messages = null, { temperature = 0.7 }) {
+ if (!(await this.isValidChatCompletionModel(this.model)))
+ throw new Error(
+ `OpenRouter chat: ${this.model} is not valid for chat completion!`
+ );
+
+ const streamRequest = await this.openai.createChatCompletion(
+ {
+ model: this.model,
+ stream: true,
+ messages,
+ temperature,
+ },
+ { responseType: "stream" }
+ );
+ return streamRequest;
+ }
+
+ handleStream(response, stream, responseProps) {
+ const timeoutThresholdMs = 500;
+ const { uuid = uuidv4(), sources = [] } = responseProps;
+
+ return new Promise((resolve) => {
+ let fullText = "";
+ let chunk = "";
+ let lastChunkTime = null; // null when first token is still not received.
+
+ // NOTICE: Not all OpenRouter models will return a stop reason
+ // which keeps the connection open and so the model never finalizes the stream
+ // like the traditional OpenAI response schema does. So in the case the response stream
+ // never reaches a formal close state we maintain an interval timer that if we go >=timeoutThresholdMs with
+ // no new chunks then we kill the stream and assume it to be complete. OpenRouter is quite fast
+ // so this threshold should permit most responses, but we can adjust `timeoutThresholdMs` if
+ // we find it is too aggressive.
+ const timeoutCheck = setInterval(() => {
+ if (lastChunkTime === null) return;
+
+ const now = Number(new Date());
+ const diffMs = now - lastChunkTime;
+ if (diffMs >= timeoutThresholdMs) {
+ console.log(
+ `OpenRouter stream did not self-close and has been stale for >${timeoutThresholdMs}ms. Closing response stream.`
+ );
+ writeResponseChunk(response, {
+ uuid,
+ sources,
+ type: "textResponseChunk",
+ textResponse: "",
+ close: true,
+ error: false,
+ });
+ clearInterval(timeoutCheck);
+ resolve(fullText);
+ }
+ }, 500);
+
+ stream.data.on("data", (data) => {
+ const lines = data
+ ?.toString()
+ ?.split("\n")
+ .filter((line) => line.trim() !== "");
+
+ for (const line of lines) {
+ let validJSON = false;
+ const message = chunk + line.replace(/^data: /, "");
+
+ // JSON chunk is incomplete and has not ended yet
+ // so we need to stitch it together. You would think JSON
+ // chunks would only come complete - but they don't!
+ try {
+ JSON.parse(message);
+ validJSON = true;
+ } catch {}
+
+ if (!validJSON) {
+ // It can be possible that the chunk decoding is running away
+ // and the message chunk fails to append due to string length.
+ // In this case abort the chunk and reset so we can continue.
+ // ref: https://github.com/Mintplex-Labs/anything-llm/issues/416
+ try {
+ chunk += message;
+ } catch (e) {
+ console.error(`Chunk appending error`, e);
+ chunk = "";
+ }
+ continue;
+ } else {
+ chunk = "";
+ }
+
+ if (message == "[DONE]") {
+ lastChunkTime = Number(new Date());
+ writeResponseChunk(response, {
+ uuid,
+ sources,
+ type: "textResponseChunk",
+ textResponse: "",
+ close: true,
+ error: false,
+ });
+ clearInterval(timeoutCheck);
+ resolve(fullText);
+ } else {
+ let finishReason = null;
+ let token = "";
+ try {
+ const json = JSON.parse(message);
+ token = json?.choices?.[0]?.delta?.content;
+ finishReason = json?.choices?.[0]?.finish_reason || null;
+ } catch {
+ continue;
+ }
+
+ if (token) {
+ fullText += token;
+ lastChunkTime = Number(new Date());
+ writeResponseChunk(response, {
+ uuid,
+ sources: [],
+ type: "textResponseChunk",
+ textResponse: token,
+ close: false,
+ error: false,
+ });
+ }
+
+ if (finishReason !== null) {
+ lastChunkTime = Number(new Date());
+ writeResponseChunk(response, {
+ uuid,
+ sources,
+ type: "textResponseChunk",
+ textResponse: "",
+ close: true,
+ error: false,
+ });
+ clearInterval(timeoutCheck);
+ resolve(fullText);
+ }
+ }
+ }
+ });
+ });
+ }
+
+ // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
+ async embedTextInput(textInput) {
+ return await this.embedder.embedTextInput(textInput);
+ }
+ async embedChunks(textChunks = []) {
+ return await this.embedder.embedChunks(textChunks);
+ }
+
+ async compressMessages(promptArgs = {}, rawHistory = []) {
+ const { messageArrayCompressor } = require("../../helpers/chat");
+ const messageArray = this.constructPrompt(promptArgs);
+ return await messageArrayCompressor(this, messageArray, rawHistory);
+ }
+}
+
+module.exports = {
+ OpenRouterLLM,
+ openRouterModels,
+};
diff --git a/server/utils/AiProviders/openRouter/models.js b/server/utils/AiProviders/openRouter/models.js
new file mode 100644
index 000000000..c920b88a4
--- /dev/null
+++ b/server/utils/AiProviders/openRouter/models.js
@@ -0,0 +1,622 @@
+const MODELS = {
+ "nousresearch/nous-capybara-34b": {
+ id: "nousresearch/nous-capybara-34b",
+ name: "Nous: Capybara 34B",
+ organization: "Nousresearch",
+ maxLength: 32768,
+ },
+ "openrouter/auto": {
+ id: "openrouter/auto",
+ name: "Auto (best for prompt)",
+ organization: "Openrouter",
+ maxLength: 128000,
+ },
+ "nousresearch/nous-capybara-7b:free": {
+ id: "nousresearch/nous-capybara-7b:free",
+ name: "Nous: Capybara 7B (free)",
+ organization: "Nousresearch",
+ maxLength: 4096,
+ },
+ "mistralai/mistral-7b-instruct:free": {
+ id: "mistralai/mistral-7b-instruct:free",
+ name: "Mistral 7B Instruct (free)",
+ organization: "Mistralai",
+ maxLength: 8192,
+ },
+ "gryphe/mythomist-7b:free": {
+ id: "gryphe/mythomist-7b:free",
+ name: "MythoMist 7B (free)",
+ organization: "Gryphe",
+ maxLength: 32768,
+ },
+ "undi95/toppy-m-7b:free": {
+ id: "undi95/toppy-m-7b:free",
+ name: "Toppy M 7B (free)",
+ organization: "Undi95",
+ maxLength: 4096,
+ },
+ "openrouter/cinematika-7b:free": {
+ id: "openrouter/cinematika-7b:free",
+ name: "Cinematika 7B (alpha) (free)",
+ organization: "Openrouter",
+ maxLength: 8000,
+ },
+ "google/gemma-7b-it:free": {
+ id: "google/gemma-7b-it:free",
+ name: "Google: Gemma 7B (free)",
+ organization: "Google",
+ maxLength: 8000,
+ },
+ "jondurbin/bagel-34b": {
+ id: "jondurbin/bagel-34b",
+ name: "Bagel 34B v0.2",
+ organization: "Jondurbin",
+ maxLength: 8000,
+ },
+ "jebcarter/psyfighter-13b": {
+ id: "jebcarter/psyfighter-13b",
+ name: "Psyfighter 13B",
+ organization: "Jebcarter",
+ maxLength: 4096,
+ },
+ "koboldai/psyfighter-13b-2": {
+ id: "koboldai/psyfighter-13b-2",
+ name: "Psyfighter v2 13B",
+ organization: "Koboldai",
+ maxLength: 4096,
+ },
+ "neversleep/noromaid-mixtral-8x7b-instruct": {
+ id: "neversleep/noromaid-mixtral-8x7b-instruct",
+ name: "Noromaid Mixtral 8x7B Instruct",
+ organization: "Neversleep",
+ maxLength: 8000,
+ },
+ "nousresearch/nous-hermes-llama2-13b": {
+ id: "nousresearch/nous-hermes-llama2-13b",
+ name: "Nous: Hermes 13B",
+ organization: "Nousresearch",
+ maxLength: 4096,
+ },
+ "meta-llama/codellama-34b-instruct": {
+ id: "meta-llama/codellama-34b-instruct",
+ name: "Meta: CodeLlama 34B Instruct",
+ organization: "Meta-llama",
+ maxLength: 8192,
+ },
+ "phind/phind-codellama-34b": {
+ id: "phind/phind-codellama-34b",
+ name: "Phind: CodeLlama 34B v2",
+ organization: "Phind",
+ maxLength: 4096,
+ },
+ "intel/neural-chat-7b": {
+ id: "intel/neural-chat-7b",
+ name: "Neural Chat 7B v3.1",
+ organization: "Intel",
+ maxLength: 4096,
+ },
+ "mistralai/mixtral-8x7b-instruct": {
+ id: "mistralai/mixtral-8x7b-instruct",
+ name: "Mistral: Mixtral 8x7B Instruct",
+ organization: "Mistralai",
+ maxLength: 32768,
+ },
+ "nousresearch/nous-hermes-2-mixtral-8x7b-dpo": {
+ id: "nousresearch/nous-hermes-2-mixtral-8x7b-dpo",
+ name: "Nous: Hermes 2 Mixtral 8x7B DPO",
+ organization: "Nousresearch",
+ maxLength: 32000,
+ },
+ "nousresearch/nous-hermes-2-mixtral-8x7b-sft": {
+ id: "nousresearch/nous-hermes-2-mixtral-8x7b-sft",
+ name: "Nous: Hermes 2 Mixtral 8x7B SFT",
+ organization: "Nousresearch",
+ maxLength: 32000,
+ },
+ "haotian-liu/llava-13b": {
+ id: "haotian-liu/llava-13b",
+ name: "Llava 13B",
+ organization: "Haotian-liu",
+ maxLength: 2048,
+ },
+ "nousresearch/nous-hermes-2-vision-7b": {
+ id: "nousresearch/nous-hermes-2-vision-7b",
+ name: "Nous: Hermes 2 Vision 7B (alpha)",
+ organization: "Nousresearch",
+ maxLength: 4096,
+ },
+ "meta-llama/llama-2-13b-chat": {
+ id: "meta-llama/llama-2-13b-chat",
+ name: "Meta: Llama v2 13B Chat",
+ organization: "Meta-llama",
+ maxLength: 4096,
+ },
+ "migtissera/synthia-70b": {
+ id: "migtissera/synthia-70b",
+ name: "Synthia 70B",
+ organization: "Migtissera",
+ maxLength: 8192,
+ },
+ "pygmalionai/mythalion-13b": {
+ id: "pygmalionai/mythalion-13b",
+ name: "Pygmalion: Mythalion 13B",
+ organization: "Pygmalionai",
+ maxLength: 8192,
+ },
+ "undi95/remm-slerp-l2-13b-6k": {
+ id: "undi95/remm-slerp-l2-13b-6k",
+ name: "ReMM SLERP 13B 6k",
+ organization: "Undi95",
+ maxLength: 6144,
+ },
+ "gryphe/mythomax-l2-13b": {
+ id: "gryphe/mythomax-l2-13b",
+ name: "MythoMax 13B",
+ organization: "Gryphe",
+ maxLength: 4096,
+ },
+ "xwin-lm/xwin-lm-70b": {
+ id: "xwin-lm/xwin-lm-70b",
+ name: "Xwin 70B",
+ organization: "Xwin-lm",
+ maxLength: 8192,
+ },
+ "gryphe/mythomax-l2-13b-8k": {
+ id: "gryphe/mythomax-l2-13b-8k",
+ name: "MythoMax 13B 8k",
+ organization: "Gryphe",
+ maxLength: 8192,
+ },
+ "alpindale/goliath-120b": {
+ id: "alpindale/goliath-120b",
+ name: "Goliath 120B",
+ organization: "Alpindale",
+ maxLength: 6144,
+ },
+ "neversleep/noromaid-20b": {
+ id: "neversleep/noromaid-20b",
+ name: "Noromaid 20B",
+ organization: "Neversleep",
+ maxLength: 8192,
+ },
+ "gryphe/mythomist-7b": {
+ id: "gryphe/mythomist-7b",
+ name: "MythoMist 7B",
+ organization: "Gryphe",
+ maxLength: 32768,
+ },
+ "mancer/weaver": {
+ id: "mancer/weaver",
+ name: "Mancer: Weaver (alpha)",
+ organization: "Mancer",
+ maxLength: 8000,
+ },
+ "nousresearch/nous-hermes-llama2-70b": {
+ id: "nousresearch/nous-hermes-llama2-70b",
+ name: "Nous: Hermes 70B",
+ organization: "Nousresearch",
+ maxLength: 4096,
+ },
+ "nousresearch/nous-capybara-7b": {
+ id: "nousresearch/nous-capybara-7b",
+ name: "Nous: Capybara 7B",
+ organization: "Nousresearch",
+ maxLength: 4096,
+ },
+ "codellama/codellama-70b-instruct": {
+ id: "codellama/codellama-70b-instruct",
+ name: "Meta: CodeLlama 70B Instruct",
+ organization: "Codellama",
+ maxLength: 2048,
+ },
+ "teknium/openhermes-2-mistral-7b": {
+ id: "teknium/openhermes-2-mistral-7b",
+ name: "OpenHermes 2 Mistral 7B",
+ organization: "Teknium",
+ maxLength: 4096,
+ },
+ "teknium/openhermes-2.5-mistral-7b": {
+ id: "teknium/openhermes-2.5-mistral-7b",
+ name: "OpenHermes 2.5 Mistral 7B",
+ organization: "Teknium",
+ maxLength: 4096,
+ },
+ "undi95/remm-slerp-l2-13b": {
+ id: "undi95/remm-slerp-l2-13b",
+ name: "ReMM SLERP 13B",
+ organization: "Undi95",
+ maxLength: 4096,
+ },
+ "undi95/toppy-m-7b": {
+ id: "undi95/toppy-m-7b",
+ name: "Toppy M 7B",
+ organization: "Undi95",
+ maxLength: 4096,
+ },
+ "openrouter/cinematika-7b": {
+ id: "openrouter/cinematika-7b",
+ name: "Cinematika 7B (alpha)",
+ organization: "Openrouter",
+ maxLength: 8000,
+ },
+ "01-ai/yi-34b-chat": {
+ id: "01-ai/yi-34b-chat",
+ name: "Yi 34B Chat",
+ organization: "01-ai",
+ maxLength: 4096,
+ },
+ "01-ai/yi-34b": {
+ id: "01-ai/yi-34b",
+ name: "Yi 34B (base)",
+ organization: "01-ai",
+ maxLength: 4096,
+ },
+ "01-ai/yi-6b": {
+ id: "01-ai/yi-6b",
+ name: "Yi 6B (base)",
+ organization: "01-ai",
+ maxLength: 4096,
+ },
+ "togethercomputer/stripedhyena-nous-7b": {
+ id: "togethercomputer/stripedhyena-nous-7b",
+ name: "StripedHyena Nous 7B",
+ organization: "Togethercomputer",
+ maxLength: 32768,
+ },
+ "togethercomputer/stripedhyena-hessian-7b": {
+ id: "togethercomputer/stripedhyena-hessian-7b",
+ name: "StripedHyena Hessian 7B (base)",
+ organization: "Togethercomputer",
+ maxLength: 32768,
+ },
+ "mistralai/mixtral-8x7b": {
+ id: "mistralai/mixtral-8x7b",
+ name: "Mistral: Mixtral 8x7B (base)",
+ organization: "Mistralai",
+ maxLength: 32768,
+ },
+ "nousresearch/nous-hermes-yi-34b": {
+ id: "nousresearch/nous-hermes-yi-34b",
+ name: "Nous: Hermes 2 Yi 34B",
+ organization: "Nousresearch",
+ maxLength: 4096,
+ },
+ "nousresearch/nous-hermes-2-mistral-7b-dpo": {
+ id: "nousresearch/nous-hermes-2-mistral-7b-dpo",
+ name: "Nous: Hermes 2 Mistral 7B DPO",
+ organization: "Nousresearch",
+ maxLength: 8192,
+ },
+ "open-orca/mistral-7b-openorca": {
+ id: "open-orca/mistral-7b-openorca",
+ name: "Mistral OpenOrca 7B",
+ organization: "Open-orca",
+ maxLength: 8192,
+ },
+ "huggingfaceh4/zephyr-7b-beta": {
+ id: "huggingfaceh4/zephyr-7b-beta",
+ name: "Hugging Face: Zephyr 7B",
+ organization: "Huggingfaceh4",
+ maxLength: 4096,
+ },
+ "openai/gpt-3.5-turbo": {
+ id: "openai/gpt-3.5-turbo",
+ name: "OpenAI: GPT-3.5 Turbo",
+ organization: "Openai",
+ maxLength: 4095,
+ },
+ "openai/gpt-3.5-turbo-0125": {
+ id: "openai/gpt-3.5-turbo-0125",
+ name: "OpenAI: GPT-3.5 Turbo 16k",
+ organization: "Openai",
+ maxLength: 16385,
+ },
+ "openai/gpt-3.5-turbo-1106": {
+ id: "openai/gpt-3.5-turbo-1106",
+ name: "OpenAI: GPT-3.5 Turbo 16k (older v1106)",
+ organization: "Openai",
+ maxLength: 16385,
+ },
+ "openai/gpt-3.5-turbo-0613": {
+ id: "openai/gpt-3.5-turbo-0613",
+ name: "OpenAI: GPT-3.5 Turbo (older v0613)",
+ organization: "Openai",
+ maxLength: 4095,
+ },
+ "openai/gpt-3.5-turbo-0301": {
+ id: "openai/gpt-3.5-turbo-0301",
+ name: "OpenAI: GPT-3.5 Turbo (older v0301)",
+ organization: "Openai",
+ maxLength: 4095,
+ },
+ "openai/gpt-3.5-turbo-16k": {
+ id: "openai/gpt-3.5-turbo-16k",
+ name: "OpenAI: GPT-3.5 Turbo 16k",
+ organization: "Openai",
+ maxLength: 16385,
+ },
+ "openai/gpt-4-turbo-preview": {
+ id: "openai/gpt-4-turbo-preview",
+ name: "OpenAI: GPT-4 Turbo (preview)",
+ organization: "Openai",
+ maxLength: 128000,
+ },
+ "openai/gpt-4-1106-preview": {
+ id: "openai/gpt-4-1106-preview",
+ name: "OpenAI: GPT-4 Turbo (older v1106)",
+ organization: "Openai",
+ maxLength: 128000,
+ },
+ "openai/gpt-4": {
+ id: "openai/gpt-4",
+ name: "OpenAI: GPT-4",
+ organization: "Openai",
+ maxLength: 8191,
+ },
+ "openai/gpt-4-0314": {
+ id: "openai/gpt-4-0314",
+ name: "OpenAI: GPT-4 (older v0314)",
+ organization: "Openai",
+ maxLength: 8191,
+ },
+ "openai/gpt-4-32k": {
+ id: "openai/gpt-4-32k",
+ name: "OpenAI: GPT-4 32k",
+ organization: "Openai",
+ maxLength: 32767,
+ },
+ "openai/gpt-4-32k-0314": {
+ id: "openai/gpt-4-32k-0314",
+ name: "OpenAI: GPT-4 32k (older v0314)",
+ organization: "Openai",
+ maxLength: 32767,
+ },
+ "openai/gpt-4-vision-preview": {
+ id: "openai/gpt-4-vision-preview",
+ name: "OpenAI: GPT-4 Vision (preview)",
+ organization: "Openai",
+ maxLength: 128000,
+ },
+ "openai/gpt-3.5-turbo-instruct": {
+ id: "openai/gpt-3.5-turbo-instruct",
+ name: "OpenAI: GPT-3.5 Turbo Instruct",
+ organization: "Openai",
+ maxLength: 4095,
+ },
+ "google/palm-2-chat-bison": {
+ id: "google/palm-2-chat-bison",
+ name: "Google: PaLM 2 Chat",
+ organization: "Google",
+ maxLength: 36864,
+ },
+ "google/palm-2-codechat-bison": {
+ id: "google/palm-2-codechat-bison",
+ name: "Google: PaLM 2 Code Chat",
+ organization: "Google",
+ maxLength: 28672,
+ },
+ "google/palm-2-chat-bison-32k": {
+ id: "google/palm-2-chat-bison-32k",
+ name: "Google: PaLM 2 Chat 32k",
+ organization: "Google",
+ maxLength: 131072,
+ },
+ "google/palm-2-codechat-bison-32k": {
+ id: "google/palm-2-codechat-bison-32k",
+ name: "Google: PaLM 2 Code Chat 32k",
+ organization: "Google",
+ maxLength: 131072,
+ },
+ "google/gemini-pro": {
+ id: "google/gemini-pro",
+ name: "Google: Gemini Pro (preview)",
+ organization: "Google",
+ maxLength: 131040,
+ },
+ "google/gemini-pro-vision": {
+ id: "google/gemini-pro-vision",
+ name: "Google: Gemini Pro Vision (preview)",
+ organization: "Google",
+ maxLength: 65536,
+ },
+ "perplexity/pplx-70b-online": {
+ id: "perplexity/pplx-70b-online",
+ name: "Perplexity: PPLX 70B Online",
+ organization: "Perplexity",
+ maxLength: 4096,
+ },
+ "perplexity/pplx-7b-online": {
+ id: "perplexity/pplx-7b-online",
+ name: "Perplexity: PPLX 7B Online",
+ organization: "Perplexity",
+ maxLength: 4096,
+ },
+ "perplexity/pplx-7b-chat": {
+ id: "perplexity/pplx-7b-chat",
+ name: "Perplexity: PPLX 7B Chat",
+ organization: "Perplexity",
+ maxLength: 8192,
+ },
+ "perplexity/pplx-70b-chat": {
+ id: "perplexity/pplx-70b-chat",
+ name: "Perplexity: PPLX 70B Chat",
+ organization: "Perplexity",
+ maxLength: 4096,
+ },
+ "meta-llama/llama-2-70b-chat": {
+ id: "meta-llama/llama-2-70b-chat",
+ name: "Meta: Llama v2 70B Chat",
+ organization: "Meta-llama",
+ maxLength: 4096,
+ },
+ "jondurbin/airoboros-l2-70b": {
+ id: "jondurbin/airoboros-l2-70b",
+ name: "Airoboros 70B",
+ organization: "Jondurbin",
+ maxLength: 4096,
+ },
+ "austism/chronos-hermes-13b": {
+ id: "austism/chronos-hermes-13b",
+ name: "Chronos Hermes 13B v2",
+ organization: "Austism",
+ maxLength: 4096,
+ },
+ "mistralai/mistral-7b-instruct": {
+ id: "mistralai/mistral-7b-instruct",
+ name: "Mistral 7B Instruct",
+ organization: "Mistralai",
+ maxLength: 8192,
+ },
+ "openchat/openchat-7b": {
+ id: "openchat/openchat-7b",
+ name: "OpenChat 3.5",
+ organization: "Openchat",
+ maxLength: 8192,
+ },
+ "lizpreciatior/lzlv-70b-fp16-hf": {
+ id: "lizpreciatior/lzlv-70b-fp16-hf",
+ name: "lzlv 70B",
+ organization: "Lizpreciatior",
+ maxLength: 4096,
+ },
+ "cognitivecomputations/dolphin-mixtral-8x7b": {
+ id: "cognitivecomputations/dolphin-mixtral-8x7b",
+ name: "Dolphin 2.6 Mixtral 8x7B 🐬",
+ organization: "Cognitivecomputations",
+ maxLength: 32000,
+ },
+ "rwkv/rwkv-5-world-3b": {
+ id: "rwkv/rwkv-5-world-3b",
+ name: "RWKV v5 World 3B",
+ organization: "Rwkv",
+ maxLength: 10000,
+ },
+ "recursal/rwkv-5-3b-ai-town": {
+ id: "recursal/rwkv-5-3b-ai-town",
+ name: "RWKV v5 3B AI Town",
+ organization: "Recursal",
+ maxLength: 10000,
+ },
+ "recursal/eagle-7b": {
+ id: "recursal/eagle-7b",
+ name: "RWKV v5: Eagle 7B",
+ organization: "Recursal",
+ maxLength: 10000,
+ },
+ "google/gemma-7b-it": {
+ id: "google/gemma-7b-it",
+ name: "Google: Gemma 7B",
+ organization: "Google",
+ maxLength: 8000,
+ },
+ "anthropic/claude-2": {
+ id: "anthropic/claude-2",
+ name: "Anthropic: Claude v2",
+ organization: "Anthropic",
+ maxLength: 200000,
+ },
+ "anthropic/claude-2.1": {
+ id: "anthropic/claude-2.1",
+ name: "Anthropic: Claude v2.1",
+ organization: "Anthropic",
+ maxLength: 200000,
+ },
+ "anthropic/claude-2.0": {
+ id: "anthropic/claude-2.0",
+ name: "Anthropic: Claude v2.0",
+ organization: "Anthropic",
+ maxLength: 100000,
+ },
+ "anthropic/claude-instant-1": {
+ id: "anthropic/claude-instant-1",
+ name: "Anthropic: Claude Instant v1",
+ organization: "Anthropic",
+ maxLength: 100000,
+ },
+ "anthropic/claude-instant-1.2": {
+ id: "anthropic/claude-instant-1.2",
+ name: "Anthropic: Claude Instant v1.2",
+ organization: "Anthropic",
+ maxLength: 100000,
+ },
+ "anthropic/claude-1": {
+ id: "anthropic/claude-1",
+ name: "Anthropic: Claude v1",
+ organization: "Anthropic",
+ maxLength: 100000,
+ },
+ "anthropic/claude-1.2": {
+ id: "anthropic/claude-1.2",
+ name: "Anthropic: Claude (older v1)",
+ organization: "Anthropic",
+ maxLength: 100000,
+ },
+ "anthropic/claude-instant-1.0": {
+ id: "anthropic/claude-instant-1.0",
+ name: "Anthropic: Claude Instant (older v1)",
+ organization: "Anthropic",
+ maxLength: 100000,
+ },
+ "anthropic/claude-instant-1.1": {
+ id: "anthropic/claude-instant-1.1",
+ name: "Anthropic: Claude Instant (older v1.1)",
+ organization: "Anthropic",
+ maxLength: 100000,
+ },
+ "anthropic/claude-2:beta": {
+ id: "anthropic/claude-2:beta",
+ name: "Anthropic: Claude v2 (experimental)",
+ organization: "Anthropic",
+ maxLength: 200000,
+ },
+ "anthropic/claude-2.1:beta": {
+ id: "anthropic/claude-2.1:beta",
+ name: "Anthropic: Claude v2.1 (experimental)",
+ organization: "Anthropic",
+ maxLength: 200000,
+ },
+ "anthropic/claude-2.0:beta": {
+ id: "anthropic/claude-2.0:beta",
+ name: "Anthropic: Claude v2.0 (experimental)",
+ organization: "Anthropic",
+ maxLength: 100000,
+ },
+ "anthropic/claude-instant-1:beta": {
+ id: "anthropic/claude-instant-1:beta",
+ name: "Anthropic: Claude Instant v1 (experimental)",
+ organization: "Anthropic",
+ maxLength: 100000,
+ },
+ "huggingfaceh4/zephyr-7b-beta:free": {
+ id: "huggingfaceh4/zephyr-7b-beta:free",
+ name: "Hugging Face: Zephyr 7B (free)",
+ organization: "Huggingfaceh4",
+ maxLength: 4096,
+ },
+ "openchat/openchat-7b:free": {
+ id: "openchat/openchat-7b:free",
+ name: "OpenChat 3.5 (free)",
+ organization: "Openchat",
+ maxLength: 8192,
+ },
+ "mistralai/mistral-tiny": {
+ id: "mistralai/mistral-tiny",
+ name: "Mistral: Tiny",
+ organization: "Mistralai",
+ maxLength: 32000,
+ },
+ "mistralai/mistral-small": {
+ id: "mistralai/mistral-small",
+ name: "Mistral: Small",
+ organization: "Mistralai",
+ maxLength: 32000,
+ },
+ "mistralai/mistral-medium": {
+ id: "mistralai/mistral-medium",
+ name: "Mistral: Medium",
+ organization: "Mistralai",
+ maxLength: 32000,
+ },
+};
+
+module.exports.MODELS = MODELS;
diff --git a/server/utils/AiProviders/openRouter/scripts/.gitignore b/server/utils/AiProviders/openRouter/scripts/.gitignore
new file mode 100644
index 000000000..94a2dd146
--- /dev/null
+++ b/server/utils/AiProviders/openRouter/scripts/.gitignore
@@ -0,0 +1 @@
+*.json
\ No newline at end of file
diff --git a/server/utils/AiProviders/openRouter/scripts/parse.mjs b/server/utils/AiProviders/openRouter/scripts/parse.mjs
new file mode 100644
index 000000000..fb3b562b5
--- /dev/null
+++ b/server/utils/AiProviders/openRouter/scripts/parse.mjs
@@ -0,0 +1,37 @@
+// OpenRouter has lots of models we can use so we use this script
+// to cache all the models. We can see the list of all the models
+// here: https://openrouter.ai/docs#models
+
+// To run, cd into this directory and run `node parse.mjs`
+// copy outputs into the export in ../models.js
+
+// Update the date below if you run this again because OpenRouter added new models.
+// Last Collected: Feb 23, 2024
+
+import fs from "fs";
+
+async function parseChatModels() {
+ const models = {};
+ const response = await fetch("https://openrouter.ai/api/v1/models");
+ const data = await response.json();
+ data.data.forEach((model) => {
+ models[model.id] = {
+ id: model.id,
+ name: model.name,
+ // capitalize first letter
+ organization:
+ model.id.split("/")[0].charAt(0).toUpperCase() +
+ model.id.split("/")[0].slice(1),
+ maxLength: model.context_length,
+ };
+ });
+
+ fs.writeFileSync(
+ "chat_models.json",
+ JSON.stringify(models, null, 2),
+ "utf-8"
+ );
+ return models;
+}
+
+parseChatModels();
diff --git a/server/utils/helpers/customModels.js b/server/utils/helpers/customModels.js
index 8f8ca0657..f434ac078 100644
--- a/server/utils/helpers/customModels.js
+++ b/server/utils/helpers/customModels.js
@@ -1,3 +1,4 @@
+const { openRouterModels } = require("../AiProviders/openRouter");
const { perplexityModels } = require("../AiProviders/perplexity");
const { togetherAiModels } = require("../AiProviders/togetherAi");
const SUPPORT_CUSTOM_MODELS = [
@@ -8,6 +9,7 @@ const SUPPORT_CUSTOM_MODELS = [
"togetherai",
"mistral",
"perplexity",
+ "openrouter",
];
async function getCustomModels(provider = "", apiKey = null, basePath = null) {
@@ -29,6 +31,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) {
return nativeLLMModels();
case "perplexity":
return await getPerplexityModels();
+ case "openrouter":
+ return await getOpenRouterModels();
default:
return { models: [], error: "Invalid provider for custom models" };
}
@@ -138,6 +142,21 @@ async function getPerplexityModels() {
return { models, error: null };
}
+async function getOpenRouterModels() {
+ const knownModels = await openRouterModels();
+ if (!Object.keys(knownModels).length === 0)
+ return { models: [], error: null };
+
+ const models = Object.values(knownModels).map((model) => {
+ return {
+ id: model.id,
+ organization: model.organization,
+ name: model.name,
+ };
+ });
+ return { models, error: null };
+}
+
async function getMistralModels(apiKey = null) {
const { Configuration, OpenAIApi } = require("openai");
const config = new Configuration({
diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js
index 818d92dbc..8bda716aa 100644
--- a/server/utils/helpers/index.js
+++ b/server/utils/helpers/index.js
@@ -61,6 +61,9 @@ function getLLMProvider(modelPreference = null) {
case "perplexity":
const { PerplexityLLM } = require("../AiProviders/perplexity");
return new PerplexityLLM(embedder, modelPreference);
+ case "openrouter":
+ const { OpenRouterLLM } = require("../AiProviders/openRouter");
+ return new OpenRouterLLM(embedder, modelPreference);
case "mistral":
const { MistralLLM } = require("../AiProviders/mistral");
return new MistralLLM(embedder, modelPreference);
diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js
index 5a384740b..247e3ba48 100644
--- a/server/utils/helpers/updateENV.js
+++ b/server/utils/helpers/updateENV.js
@@ -249,6 +249,16 @@ const KEY_MAPPING = {
checks: [isNotEmpty],
},
+ // OpenRouter Options
+ OpenRouterApiKey: {
+ envKey: "OPENROUTER_API_KEY",
+ checks: [isNotEmpty],
+ },
+ OpenRouterModelPref: {
+ envKey: "OPENROUTER_MODEL_PREF",
+ checks: [isNotEmpty],
+ },
+
// System Settings
AuthToken: {
envKey: "AUTH_TOKEN",
@@ -325,6 +335,7 @@ function supportedLLM(input = "") {
"mistral",
"huggingface",
"perplexity",
+ "openrouter",
].includes(input);
return validSelection ? null : `${input} is not a valid LLM provider.`;
}