sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-11-23 15:38:55 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Merge branch 'features/multi-user-support-khoj' of github.com:khoj-ai/khoj into features/multi-user-support-khoj

Browse source
This commit is contained in:
sabaimran 2023-11-04 14:29:34 -07:00
commit d1d210605e
44 changed files with 1648 additions and 161 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
docs/chat.md
View file

@ -10,7 +10,8 @@
Offline chat stays completely private and works without internet. But it is slower, lower quality and more compute intensive.
> **System Requirements**:
> - Machine with at least **6 GB of RAM** and **4 GB of Disk** available
> - Minimum 8 GB RAM. Recommend **16Gb VRAM**
> - Minimum **5 GB of Disk** available
> - A CPU supporting [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) is required
> - A Mac M1+ or [Vulcan supported GPU](https://vulkan.gpuinfo.org/) should significantly speed up chat response times

5
pyproject.toml
View file

@ -62,8 +62,8 @@ dependencies = [
"pymupdf >= 1.23.5",
"django == 4.2.5",
"authlib == 1.2.1",
"gpt4all == 1.0.12; platform_system == 'Linux' and platform_machine == 'x86_64'",
"gpt4all == 1.0.12; platform_system == 'Windows' or platform_system == 'Darwin'",
"gpt4all >= 2.0.0; platform_system == 'Linux' and platform_machine == 'x86_64'",
"gpt4all >= 2.0.0; platform_system == 'Windows' or platform_system == 'Darwin'",
"itsdangerous == 2.1.2",
"httpx == 0.25.0",
"pgvector == 0.2.3",
@ -72,6 +72,7 @@ dependencies = [
"python-multipart == 0.0.6",
"gunicorn == 21.2.0",
"lxml == 4.9.3",
"tzdata == 2023.3",
]
dynamic = ["version"]

23
src/app/settings.py
View file

@ -24,10 +24,29 @@ BASE_DIR = Path(__file__).resolve().parent.parent.parent
SECRET_KEY = os.getenv("DJANGO_SECRET_KEY")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
DEBUG = os.getenv("DJANGO_DEBUG", "False") == "True"
ALLOWED_HOSTS = []
ALLOWED_HOSTS = [".khoj.dev", "localhost", "127.0.0.1", "[::1]", "beta.khoj.dev"]
CSRF_TRUSTED_ORIGINS = [
"https://app.khoj.dev",
"https://beta.khoj.dev",
"https://khoj.dev",
"https://*.khoj.dev",
]
COOKIE_SAMESITE = "None"
if DEBUG:
SESSION_COOKIE_DOMAIN = "localhost"
CSRF_COOKIE_DOMAIN = "localhost"
else:
SESSION_COOKIE_DOMAIN = "khoj.dev"
CSRF_COOKIE_DOMAIN = "khoj.dev"
SESSION_COOKIE_SECURE = True
CSRF_COOKIE_SECURE = True
COOKIE_SAMESITE = "None"
SESSION_COOKIE_SAMESITE = "None"
# Application definition

8
src/database/adapters/__init__.py
View file

@ -265,6 +265,10 @@ class ConversationAdapters:
@staticmethod
async def get_openai_chat():
return await ChatModelOptions.objects.filter(model_type="openai").afirst()
@staticmethod
async def get_openai_chat_config():
return await OpenAIProcessorConversationConfig.objects.filter().afirst()
@staticmethod
@ -340,11 +344,11 @@ class EntryAdapters:
if min_date is not None:
# Convert the min_date timestamp to yyyy-mm-dd format
formatted_min_date = date.fromtimestamp(min_date).strftime("%Y-%m-%d")
q_filter_terms &= Q(entry_dates__date__gte=formatted_min_date)
q_filter_terms &= Q(embeddings_dates__date__gte=formatted_min_date)
if max_date is not None:
# Convert the max_date timestamp to yyyy-mm-dd format
formatted_max_date = date.fromtimestamp(max_date).strftime("%Y-%m-%d")
q_filter_terms &= Q(entry_dates__date__lte=formatted_max_date)
q_filter_terms &= Q(embeddings_dates__date__lte=formatted_max_date)
relevant_entries = Entry.objects.filter(user=user).filter(
q_filter_terms,

88
src/interface/desktop/about.html Normal file
View file

@ -0,0 +1,88 @@
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
<title>Khoj - About</title>
<link rel="icon" type="image/png" sizes="128x128" href="./assets/icons/favicon-128x128.png">
<link rel="manifest" href="/static/khoj_chat.webmanifest">
<link rel="stylesheet" href="./assets/khoj.css">
</head>
<script type="text/javascript" src="./utils.js"></script>
<style>
html, body {
height: 100%;
width: 100%;
padding: 0px;
margin: 0px;
}
body {
display: grid;
grid-template-rows: auto;
background: var(--background-color);
color: var(--main-text-color);
text-align: center;
font-family: roboto, karma, segoe ui, sans-serif;
font-size: small;
font-weight: 300;
line-height: 1.5em;
}
header > *,
body > * {
padding: 0px;
margin: 0px;
}
header > * {
margin-top: 20px;
}
img {
width: 100px;
height: 100px;
margin-top: 32px;
}
p {
font-size: 14px;
}
#about-page-version {
margin: 0;
}
.button {
display: block;
width: 60%;
padding: 10px 16px;
margin: 10px auto;
background-color: var(--primary);
border: none;
border-radius: 8px;
cursor: pointer;
transition: background-color 0.3s;
}
.button:hover {
background-color: var(--primary-hover);
}
footer {
font-size: 10px;
color: slategray;
margin-top: 10px;
}
</style>
<body>
<header>
<img id="logo" src="./assets/icons/favicon-128x128.png" alt="Khoj Logo">
<p id="about-page-title"><b>Khoj for Desktop</b>
<p id="about-page-version"></p>
</header>
<div class="action">
<button class="button" onclick="window.open('https://khoj.dev/terms-of-service', '_blank')">Terms of Service</button>
<button class="button" onclick="window.open('https://khoj.dev/privacy-policy', '_blank')">Privacy Policy</button>
</div>
<footer>
© 2023 Khoj Inc. All rights reserved.
</footer>
</body>
</html>

BIN
src/interface/desktop/assets/icons/khoj-logo-sideways-500.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 29 KiB

36
src/interface/desktop/assets/khoj.css
View file

@ -2,29 +2,44 @@
/* Can be forced with data-theme="light" */
[data-theme="light"],
:root:not([data-theme="dark"]) {
--primary: #ffb300;
--primary-hover: #ffa000;
--primary: #fee285;
--primary-hover: #fcc50b;
--primary-focus: rgba(255, 179, 0, 0.125);
--primary-inverse: rgba(0, 0, 0, 0.75);
--background-color: #f5f4f3;
--main-text-color: #475569;
--water: #44b9da;
--leaf: #7b990a;
--flower: #d1684e;
}
/* Amber Dark scheme (Auto) */
/* Automatically enabled if user has Dark mode enabled */
@media only screen and (prefers-color-scheme: dark) {
:root:not([data-theme]) {
--primary: #ffb300;
--primary-hover: #ffc107;
--primary: #fee285;
--primary-hover: #fcc50b;
--primary-focus: rgba(255, 179, 0, 0.25);
--primary-inverse: rgba(0, 0, 0, 0.75);
--background-color: #f5f4f3;
--main-text-color: #475569;
--water: #44b9da;
--leaf: #7b990a;
--flower: #d1684e;
}
}
/* Amber Dark scheme (Forced) */
/* Enabled if forced with data-theme="dark" */
[data-theme="dark"] {
--primary: #ffb300;
--primary-hover: #ffc107;
--primary: #fee285;
--primary-hover: #fcc50b;
--primary-focus: rgba(255, 179, 0, 0.25);
--primary-inverse: rgba(0, 0, 0, 0.75);
--background-color: #f5f4f3;
--main-text-color: #475569;
--water: #44b9da;
--leaf: #7b990a;
--flower: #d1684e;
}
/* Amber (Common styles) */
:root {
@ -37,8 +52,10 @@
.khoj-configure {
display: grid;
grid-template-columns: 1fr;
padding: 0 24px;
font-family: roboto, karma, segoe ui, sans-serif;
font-weight: 300;
}
.khoj-header {
display: grid;
grid-auto-flow: column;
@ -64,7 +81,7 @@ a.khoj-logo {
}
.khoj-nav a {
color: #333;
color: var(--main-text-color);
text-decoration: none;
font-size: small;
font-weight: normal;
@ -75,8 +92,9 @@ a.khoj-logo {
}
.khoj-nav a:hover {
background-color: var(--primary-hover);
color: var(--main-text-color);
}
.khoj-nav-selected {
a.khoj-nav-selected {
background-color: var(--primary);
}
img.khoj-logo {

991
src/interface/desktop/assets/three.min.js vendored Normal file
View file

File diff suppressed because one or more lines are too long

14
src/interface/desktop/chat.html
View file

@ -8,6 +8,8 @@
<link rel="manifest" href="/static/khoj_chat.webmanifest">
<link rel="stylesheet" href="./assets/khoj.css">
</head>
<script src="./utils.js"></script>
<script>
let chatOptions = [];
function copyProgrammaticOutput(event) {
@ -66,6 +68,8 @@
// Replace any ** with <b> and __ with <u>
newHTML = newHTML.replace(/\*\*([\s\S]*?)\*\*/g, '<b>$1</b>');
newHTML = newHTML.replace(/__([\s\S]*?)__/g, '<u>$1</u>');
// Remove any text between <s>[INST] and </s> tags. These are spurious instructions for the AI chat model.
newHTML = newHTML.replace(/<s>\[INST\].+(<\/s>)?/g, '');
return newHTML;
}
@ -166,6 +170,7 @@
function incrementalChat(event) {
if (!event.shiftKey && event.key === 'Enter') {
event.preventDefault();
chat();
}
}
@ -278,7 +283,7 @@
</a>
<nav class="khoj-nav">
<a class="khoj-nav khoj-nav-selected" href="./chat.html">💬 Chat</a>
<a class="khoj-nav" href="./index.html">🔎 Search</a>
<a class="khoj-nav" href="./search.html">🔎 Search</a>
<a class="khoj-nav" href="./config.html">⚙️ Settings</a>
</nav>
</div>
@ -289,8 +294,7 @@
<!-- Chat Footer -->
<div id="chat-footer">
<div id="chat-tooltip" style="display: none;"></div>
<textarea id="chat-input" class="option" oninput="onChatInput()" onkeyup=incrementalChat(event) autofocus="autofocus" placeholder="Type / to see a list of commands, or just type your questions and hit enter.">
</textarea>
<textarea id="chat-input" class="option" oninput="onChatInput()" onkeydown=incrementalChat(event) autofocus="autofocus" placeholder="Type / to see a list of commands, or just type your questions and hit enter."></textarea>
</div>
</body>
@ -303,8 +307,8 @@
}
body {
display: grid;
background: #fff;
color: #475569;
background: var(--background-color);
color: var(--main-text-color);
text-align: center;
font-family: roboto, karma, segoe ui, sans-serif;
font-size: small;

45
src/interface/desktop/config.html
View file

@ -8,18 +8,17 @@
<link rel="manifest" href="./khoj.webmanifest">
<link rel="stylesheet" href="./assets/khoj.css">
</head>
<script type="text/javascript" src="./assets/org.min.js"></script>
<script type="text/javascript" src="./assets/markdown-it.min.js"></script>
<script src="./utils.js"></script>
<body>
<!--Add Header Logo and Nav Pane-->
<div class="khoj-header">
<a class="khoj-logo" href="./index.html">
<a class="khoj-logo" href="./chat.html">
<img class="khoj-logo" src="./assets/icons/khoj-logo-sideways-500.png" alt="Khoj"></img>
</a>
<nav class="khoj-nav">
<a class="khoj-nav" href="./chat.html">💬 Chat</a>
<a class="khoj-nav" href="./index.html">🔎 Search</a>
<a class="khoj-nav" href="./search.html">🔎 Search</a>
<a class="khoj-nav khoj-nav-selected" href="./config.html">⚙️ Settings</a>
</nav>
</div>
@ -38,11 +37,11 @@
<div class="card-title-row">
<img class="card-icon" src="./assets/icons/key.svg" alt="Khoj Access Key">
<h3 class="card-title">
Access Key
API Key
</h3>
</div>
<div class="card-description-row">
<input id="khoj-access-key" class="card-input" type="text" placeholder="Enter key to access your Khoj">
<input id="khoj-access-key" class="card-input" type="text" placeholder="Enter API key to access your Khoj">
</div>
</div>
</div>
@ -131,7 +130,7 @@
body, input {
padding: 0px;
margin: 0px;
background: #fff;
background: var(--background-color);
color: #475569;
font-family: roboto, karma, segoe ui, sans-serif;
font-size: small;
@ -181,7 +180,7 @@
.card-input {
padding: 4px;
box-shadow: 0 0 2px 1px rgba(0, 0, 0, 0.2);
box-shadow: 0 0 2px 1px rgba(0, 0, 0, 0.3);
border: none;
width: 450px;
}
@ -191,7 +190,7 @@
gap: 8px;
padding: 24px 16px;
width: 450px;
background: white;
background: var(--background-color);
border: 1px solid rgb(229, 229, 229);
border-radius: 4px;
box-shadow: 0px 1px 3px 0px rgba(0,0,0,0.1),0px 1px 2px -1px rgba(0,0,0,0.1);
@ -259,7 +258,7 @@
}
.primary-button {
border: none;
color: white;
color: var(--background-color);
padding: 15px 32px;
text-align: center;
text-decoration: none;
@ -268,7 +267,7 @@
}
button.card-button.disabled {
color: rgb(255, 136, 136);
color: var(--flower);
background: transparent;
font-size: small;
cursor: pointer;
@ -280,11 +279,7 @@
}
button.card-button.happy {
color: rgb(0, 146, 0);
}
button.card-button.happy {
color: rgb(0, 146, 0);
color: var(--leaf);
}
img.configured-icon {
@ -308,7 +303,9 @@
div.folder-element {
display: grid;
grid-template-columns: auto 1fr;
box-shadow: 0 0 2px 1px rgba(0, 0, 0, 0.2);
border: 1px solid rgb(229, 229, 229);
border-radius: 4px;
box-shadow: 0px 1px 3px 0px rgba(0,0,0,0.1),0px 1px 2px -1px rgba(0,0,0,0.8);
padding: 4px;
margin-bottom: 8px;
}
@ -326,7 +323,7 @@
background-color: rgb(253 214 214);
border-radius: 3px;
border: none;
color: rgb(207, 67, 59);
color: var(--flower);
padding: 4px;
}
@ -335,14 +332,14 @@
background-color: rgb(255 235 235);
border-radius: 3px;
border: none;
color: rgb(207, 67, 59);
color: var(--flower);
padding: 4px;
cursor: pointer;
}
#sync-data {
background-color: #ffb300;
background-color: var(--primary);
border: none;
color: white;
color: var(--main-text-color);
padding: 12px;
text-align: center;
text-decoration: none;
@ -351,12 +348,12 @@
border-radius: 4px;
cursor: pointer;
transition: background-color 0.3s ease;
box-shadow: 0px 5px 0px #f9f5de;
box-shadow: 0px 5px 0px var(--background-color);
}
#sync-data:hover {
background-color: #ffcc00;
box-shadow: 0px 3px 0px #f9f5de;
background-color: var(--primary-hover);
box-shadow: 0px 3px 0px var(--background-color);
}
.sync-force-toggle {
align-content: center;

129
src/interface/desktop/loading-animation.js Normal file
View file

@ -0,0 +1,129 @@
var $wrap = document.getElementById('loading-animation'),
canvassize = 380,
length = 40,
radius = 6.8,
rotatevalue = 0.02,
acceleration = 0,
animatestep = 0,
toend = false,
pi2 = Math.PI*2,
group = new THREE.Group(),
mesh, ringcover, ring,
camera, scene, renderer;
camera = new THREE.PerspectiveCamera(65, 1, 1, 10000);
camera.position.z = 120;
scene = new THREE.Scene();
// scene.add(new THREE.AxisHelper(30));
scene.add(group);
mesh = new THREE.Mesh(
new THREE.TubeGeometry(new (THREE.Curve.create(function() {},
function(percent) {
var x = length*Math.sin(pi2*percent),
y = radius*Math.cos(pi2*3*percent),
z, t;
t = percent%0.25/0.25;
t = percent%0.25-(2*(1-t)*t* -0.0185 +t*t*0.25);
if (Math.floor(percent/0.25) == 0 || Math.floor(percent/0.25) == 2) {
t *= -1;
}
z = radius*Math.sin(pi2*2* (percent-t));
return new THREE.Vector3(x, y, z);
}
))(), 200, 1.1, 2, true),
new THREE.MeshBasicMaterial({
color: 0xfcc50b
// , wireframe: true
})
);
group.add(mesh);
ringcover = new THREE.Mesh(new THREE.PlaneGeometry(50, 15, 1), new THREE.MeshBasicMaterial({color: 0xd1684e, opacity: 0, transparent: true}));
ringcover.position.x = length+1;
ringcover.rotation.y = Math.PI/2;
group.add(ringcover);
ring = new THREE.Mesh(new THREE.RingGeometry(4.3, 5.55, 32), new THREE.MeshBasicMaterial({color: 0xfcc50b, opacity: 0, transparent: true}));
ring.position.x = length+1.1;
ring.rotation.y = Math.PI/2;
group.add(ring);
// fake shadow
(function() {
var plain, i;
for (i = 0; i < 10; i++) {
plain = new THREE.Mesh(new THREE.PlaneGeometry(length*2+1, radius*3, 1), new THREE.MeshBasicMaterial({color: 0xd1684e, transparent: true, opacity: 0.15}));
plain.position.z = -2.5+i*0.5;
group.add(plain);
}
})();
renderer = new THREE.WebGLRenderer({
antialias: true
});
renderer.setPixelRatio(window.devicePixelRatio);
renderer.setSize(canvassize, canvassize);
renderer.setClearColor('#d1684e');
$wrap.appendChild(renderer.domElement);
function start() {
toend = true;
}
function back() {
toend = false;
}
function tilt(percent) {
group.rotation.y = percent*0.5;
}
function render() {
var progress;
animatestep = Math.max(0, Math.min(240, toend ? animatestep+1 : animatestep-4));
acceleration = easing(animatestep, 0, 1, 240);
if (acceleration > 0.35) {
progress = (acceleration-0.35)/0.65;
group.rotation.y = -Math.PI/2 *progress;
group.position.z = 20*progress;
progress = Math.max(0, (acceleration-0.99)/0.01);
mesh.material.opacity = 1-progress;
ringcover.material.opacity = ring.material.opacity = progress;
ring.scale.x = ring.scale.y = 0.9 + 0.1*progress;
}
renderer.render(scene, camera);
}
function animate() {
mesh.rotation.x += rotatevalue + acceleration*Math.sin(Math.PI*acceleration);
render();
requestAnimationFrame(animate);
}
function easing(t, b, c, d) {
if ((t /= d/2) < 1)
return c/2*t*t+b;
return c/2*((t-=2)*t*t+2)+b;
}
animate();
setTimeout(start, 30);

76
src/interface/desktop/main.js
View file

@ -1,5 +1,6 @@
const { app, BrowserWindow, ipcMain, Tray, Menu, nativeImage } = require('electron');
const { app, BrowserWindow, ipcMain, Tray, Menu, nativeImage, shell } = require('electron');
const todesktop = require("@todesktop/runtime");
const khojPackage = require('./package.json');
todesktop.init();
@ -305,11 +306,13 @@ async function syncData (regenerate = false) {
}
}
let firstRun = true;
let win = null;
const createWindow = (tab = 'index.html') => {
const createWindow = (tab = 'chat.html') => {
win = new BrowserWindow({
width: 800,
height: 800,
show: false,
// titleBarStyle: 'hidden',
webPreferences: {
preload: path.join(__dirname, 'preload.js'),
@ -330,12 +333,30 @@ const createWindow = (tab = 'index.html') => {
win.setResizable(true);
win.setOpacity(0.95);
win.setBackgroundColor('#FFFFFF');
win.setBackgroundColor('#f5f4f3');
win.setHasShadow(true);
job.start();
win.loadFile(tab)
if (firstRun === true) {
firstRun = false;
// Create splash screen
var splash = new BrowserWindow({width: 400, height: 400, transparent: true, frame: false, alwaysOnTop: true});
splash.setOpacity(1.0);
splash.setBackgroundColor('#d16b4e');
splash.loadFile('splash.html');
// Show splash screen on app load
win.once('ready-to-show', () => {
setTimeout(function(){ splash.close(); win.show(); }, 4500);
});
} else {
// Show main window directly if not first run
win.once('ready-to-show', () => { win.show(); });
}
}
app.whenReady().then(() => {
@ -370,11 +391,12 @@ app.whenReady().then(() => {
app.setAboutPanelOptions({
applicationName: "Khoj",
applicationVersion: "0.0.1",
version: "0.0.1",
authors: "Khoj Team",
applicationVersion: khojPackage.version,
version: khojPackage.version,
authors: "Saba Imran, Debanjum Singh Solanky and contributors",
website: "https://khoj.dev",
iconPath: path.join(__dirname, 'assets', 'khoj.png')
copyright: "GPL v3",
iconPath: path.join(__dirname, 'assets', 'icons', 'favicon-128x128.png')
});
app.on('ready', async() => {
@ -398,6 +420,43 @@ app.on('window-all-closed', () => {
if (process.platform !== 'darwin') app.quit()
})
/*
** About Page
*/
let aboutWindow;
function openAboutWindow() {
if (aboutWindow) { aboutWindow.focus(); return; }
aboutWindow = new BrowserWindow({
width: 400,
height: 400,
titleBarStyle: 'hidden',
show: false,
webPreferences: {
preload: path.join(__dirname, 'preload.js'),
nodeIntegration: true,
},
});
aboutWindow.loadFile('about.html');
// Pass OS, Khoj version to About page
aboutWindow.webContents.on('did-finish-load', () => {
aboutWindow.webContents.send('appInfo', { version: khojPackage.version, platform: process.platform });
});
// Open links in external browser
aboutWindow.webContents.setWindowOpenHandler(({ url }) => {
shell.openExternal(url);
return { action: 'deny' };
});
aboutWindow.once('ready-to-show', () => { aboutWindow.show(); });
aboutWindow.on('closed', () => { aboutWindow = null; });
}
/*
** System Tray Icon
*/
@ -418,9 +477,10 @@ app.whenReady().then(() => {
const contextMenu = Menu.buildFromTemplate([
{ label: 'Chat', type: 'normal', click: () => { openWindow('chat.html'); }},
{ label: 'Search', type: 'normal', click: () => { openWindow('index.html') }},
{ label: 'Search', type: 'normal', click: () => { openWindow('search.html') }},
{ label: 'Configure', type: 'normal', click: () => { openWindow('config.html') }},
{ type: 'separator' },
{ label: 'About Khoj', type: 'normal', click: () => { openAboutWindow(); } },
{ label: 'Quit', type: 'normal', click: () => { app.quit() } }
])

4
src/interface/desktop/preload.js
View file

@ -52,3 +52,7 @@ contextBridge.exposeInMainWorld('tokenAPI', {
setToken: (token) => ipcRenderer.invoke('setToken', token),
getToken: () => ipcRenderer.invoke('getToken')
})
contextBridge.exposeInMainWorld('appInfoAPI', {
getInfo: (callback) => ipcRenderer.on('appInfo', callback)
})

2
src/interface/desktop/renderer.js
View file

@ -61,6 +61,7 @@ toggleFoldersButton.addEventListener('click', () => {
function makeFileElement(file) {
let fileElement = document.createElement("div");
fileElement.classList.add("file-element");
let fileNameElement = document.createElement("div");
fileNameElement.classList.add("content-name");
fileNameElement.innerHTML = file.path;
@ -82,6 +83,7 @@ function makeFileElement(file) {
function makeFolderElement(folder) {
let folderElement = document.createElement("div");
folderElement.classList.add("folder-element");
let folderNameElement = document.createElement("div");
folderNameElement.classList.add("content-name");
folderNameElement.innerHTML = folder.path;

7
src/interface/desktop/index.html → src/interface/desktop/search.html
View file

@ -10,6 +10,7 @@
</head>
<script type="text/javascript" src="./assets/org.min.js"></script>
<script type="text/javascript" src="./assets/markdown-it.min.js"></script>
<script src="./utils.js"></script>
<script>
function render_image(item) {
@ -264,7 +265,7 @@
</a>
<nav class="khoj-nav">
<a class="khoj-nav" href="./chat.html">💬 Chat</a>
<a class="khoj-nav khoj-nav-selected" href="./index.html">🔎 Search</a>
<a class="khoj-nav khoj-nav-selected" href="./search.html">🔎 Search</a>
<a class="khoj-nav" href="./config.html">⚙️ Settings</a>
</nav>
</div>
@ -302,8 +303,8 @@
body {
padding: 0px;
margin: 0px;
background: #fff;
color: #475569;
background: var(--background-color);
color: var(--main-text-color);
font-family: roboto, karma, segoe ui, sans-serif;
font-size: small;
font-weight: 300;

15
src/interface/desktop/splash.html Normal file
View file

@ -0,0 +1,15 @@
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
<title>Khoj</title>
<link rel="icon" type="image/png" sizes="128x128" href="./assets/icons/favicon-128x128.png">
<link rel="manifest" href="./khoj.webmanifest">
</head>
<script type="text/javascript" src="./assets/three.min.js"></script>
<body>
<div id="loading-animation"></div>
</body>
<script src="./loading-animation.js"></script>
</html>

26
src/interface/desktop/utils.js Normal file
View file

@ -0,0 +1,26 @@
console.log(`%c %s`, "font-family:monospace", `
__ __ __ __ ______ __ _____ __
/\\ \\/ / /\\ \\_\\ \\ /\\ __ \\ /\\ \\ /\\ __ \\ /\\ \\
\\ \\ _"-. \\ \\ __ \\ \\ \\ \\/\\ \\ _\\_\\ \\ \\ \\ __ \\ \\ \\ \\
\\ \\_\\ \\_\\ \\ \\_\\ \\_\\ \\ \\_____\\ /\\_____\\ \\ \\_\\ \\_\\ \\ \\_\\
\\/_/\\/_/ \\/_/\\/_/ \\/_____/ \\/_____/ \\/_/\\/_/ \\/_/
Greetings traveller,
I am ✨Khoj✨, your open-source, personal AI copilot.
See my source code at https://github.com/khoj-ai/khoj
Read my operating manual at https://docs.khoj.dev
`);
window.appInfoAPI.getInfo((_, info) => {
let khojVersionElement = document.getElementById("about-page-version");
if (khojVersionElement) {
khojVersionElement.innerHTML = `<code>${info.version}</code>`;
}
let khojTitleElement = document.getElementById("about-page-title");
if (khojTitleElement) {
khojTitleElement.innerHTML = '<b>Khoj for ' + (info.platform === 'win32' ? 'Windows' : info.platform === 'darwin' ? 'macOS' : 'Linux') + '</b>';
}
});

5
src/interface/obsidian/src/chat_modal.ts
View file

@ -38,7 +38,7 @@ export class KhojChatModal extends Modal {
await this.getChatHistory();
// Add chat input field
contentEl.createEl("input",
const chatInput = contentEl.createEl("input",
{
attr: {
type: "text",
@ -48,10 +48,11 @@ export class KhojChatModal extends Modal {
class: "khoj-chat-input option"
}
})
.addEventListener('change', (event) => { this.result = (<HTMLInputElement>event.target).value });
chatInput.addEventListener('change', (event) => { this.result = (<HTMLInputElement>event.target).value });
// Scroll to bottom of modal, till the send message input box
this.modalEl.scrollTop = this.modalEl.scrollHeight;
chatInput.focus();
}
generateReference(messageEl: any, reference: string, index: number) {

2
src/interface/obsidian/styles.css
View file

@ -8,7 +8,7 @@ If your plugin does not need CSS, delete this file.
*/
:root {
--khoj-chat-primary: #ffb300;
--khoj-chat-primary: #fee285;
--khoj-chat-dark-grey: #475569;
}

BIN
src/khoj/interface/web/assets/icons/khoj-logo-sideways-500.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 29 KiB

39
src/khoj/interface/web/assets/khoj.css
View file

@ -2,35 +2,44 @@
/* Can be forced with data-theme="light" */
[data-theme="light"],
:root:not([data-theme="dark"]) {
--primary: #ffb300;
--primary-hover: #ffa000;
--primary: #fee285;
--primary-hover: #fcc50b;
--primary-focus: rgba(255, 179, 0, 0.125);
--primary-inverse: rgba(0, 0, 0, 0.75);
--background-color: #fff;
--background-color: #f5f4f3;
--main-text-color: #475569;
--water: #44b9da;
--leaf: #7b990a;
--flower: #ffaeae;
}
/* Amber Dark scheme (Auto) */
/* Automatically enabled if user has Dark mode enabled */
@media only screen and (prefers-color-scheme: dark) {
:root:not([data-theme]) {
--primary: #ffb300;
--primary-hover: #ffc107;
--primary: #fee285;
--primary-hover: #fcc50b;
--primary-focus: rgba(255, 179, 0, 0.25);
--primary-inverse: rgba(0, 0, 0, 0.75);
--background-color: #fff;
--background-color: #f5f4f3;
--main-text-color: #475569;
--water: #44b9da;
--leaf: #7b990a;
--flower: #ffaeae;
}
}
/* Amber Dark scheme (Forced) */
/* Enabled if forced with data-theme="dark" */
[data-theme="dark"] {
--primary: #ffb300;
--primary-hover: #ffc107;
--primary: #fee285;
--primary-hover: #fcc50b;
--primary-focus: rgba(255, 179, 0, 0.25);
--primary-inverse: rgba(0, 0, 0, 0.75);
--background-color: #fff;
--background-color: #f5f4f3;
--main-text-color: #475569;
--water: #44b9da;
--leaf: #7b990a;
--flower: #ffaeae;
}
/* Amber (Common styles) */
:root {
@ -46,6 +55,8 @@
font-family: roboto, karma, segoe ui, sans-serif;
font-weight: 300;
}
.khoj-footer,
.khoj-header {
display: grid;
grid-auto-flow: column;
@ -53,6 +64,9 @@
padding: 16px 0;
margin: 0 0 16px 0;
}
.khoj-footer {
margin: 16px 0 0 0;
}
nav.khoj-nav {
display: grid;
@ -71,7 +85,7 @@ a.khoj-logo {
}
.khoj-nav a {
color: #333;
color: var(--main-text-color);
text-decoration: none;
font-size: 20px;
font-weight: normal;
@ -148,10 +162,9 @@ p#khoj-banner {
}
.circle {
border-radius: 50%;
border: 2px solid var(--primary-inverse);
border: 3px solid var(--primary-hover);
width: 40px;
height: 40px;
vertical-align: text-top;
padding: 3px;
cursor: pointer;
}
@ -159,7 +172,7 @@ p#khoj-banner {
background-color: var(--primary-hover);
}
.user-initial {
background-color: white;
background-color: var(--background-color);
color: black;
display: grid;
justify-content: center;

15
src/khoj/interface/web/assets/khoj.js
View file

@ -1,15 +0,0 @@
// Toggle the navigation menu
function toggleMenu() {
var menu = document.getElementById("khoj-nav-menu");
menu.classList.toggle("show");
}
// Close the dropdown menu if the user clicks outside of it
document.addEventListener('click', function(event) {
let menu = document.getElementById("khoj-nav-menu");
let menuContainer = document.getElementById("khoj-nav-menu-container");
let isClickOnMenu = menuContainer.contains(event.target) || menuContainer === event.target;
if (isClickOnMenu === false && menu.classList.contains("show")) {
menu.classList.remove("show");
}
});

31
src/khoj/interface/web/assets/utils.js Normal file
View file

@ -0,0 +1,31 @@
// Toggle the navigation menu
function toggleMenu() {
var menu = document.getElementById("khoj-nav-menu");
menu.classList.toggle("show");
}
// Close the dropdown menu if the user clicks outside of it
document.addEventListener('click', function(event) {
let menu = document.getElementById("khoj-nav-menu");
let menuContainer = document.getElementById("khoj-nav-menu-container");
let isClickOnMenu = menuContainer.contains(event.target) || menuContainer === event.target;
if (isClickOnMenu === false && menu.classList.contains("show")) {
menu.classList.remove("show");
}
});
console.log(`%c %s`, "font-family:monospace", `
__ __ __ __ ______ __ _____ __
/\\ \\/ / /\\ \\_\\ \\ /\\ __ \\ /\\ \\ /\\ __ \\ /\\ \\
\\ \\ _"-. \\ \\ __ \\ \\ \\ \\/\\ \\ _\\_\\ \\ \\ \\ __ \\ \\ \\ \\
\\ \\_\\ \\_\\ \\ \\_\\ \\_\\ \\ \\_____\\ /\\_____\\ \\ \\_\\ \\_\\ \\ \\_\\
\\/_/\\/_/ \\/_/\\/_/ \\/_____/ \\/_____/ \\/_/\\/_/ \\/_/
Greetings traveller,
I am ✨Khoj✨, your open-source, personal AI copilot.
See my source code at https://github.com/khoj-ai/khoj
Read my operating manual at https://docs.khoj.dev
`);

21
src/khoj/interface/web/base_config.html
View file

@ -8,7 +8,7 @@
<link rel="stylesheet" href="/static/assets/pico.min.css">
<link rel="stylesheet" href="/static/assets/khoj.css">
</head>
<script type="text/javascript" src="/static/assets/khoj.js"></script>
<script type="text/javascript" src="/static/assets/utils.js"></script>
<body class="khoj-configure">
<div class="khoj-header-wrapper">
<div class="filler"></div>
@ -38,9 +38,9 @@
display: grid;
grid-template-columns: 1fr min(70vw, 100%) 1fr;
}
img.circle {
width: 49px;
height: 49px;
.circle {
width: 51px;
height: 51px;
}
.page {
@ -60,10 +60,10 @@
justify-items: start;
gap: 8px;
padding: 24px 24px;
background: white;
background: var(--background-color);
border: 1px solid rgb(229, 229, 229);
border-radius: 4px;
box-shadow: 0px 1px 3px 0px rgba(0,0,0,0.1),0px 1px 2px -1px rgba(0,0,0,0.1);
box-shadow: 0px 1px 3px 0px rgba(0,0,0,0.1),0px 1px 2px -1px rgba(0,0,0,0.8);
}
#api-settings-card-description {
margin: 8px 0 0 0;
@ -95,10 +95,10 @@
padding: 24px 16px;
width: 320px;
height: 180px;
background: white;
background: var(--background-color);
border: 1px solid rgb(229, 229, 229);
border-radius: 4px;
box-shadow: 0px 1px 3px 0px rgba(0,0,0,0.1),0px 1px 2px -1px rgba(0,0,0,0.1);
box-shadow: 0px 1px 3px 0px rgba(0,0,0,0.1),0px 1px 2px -1px rgba(0,0,0,0.8);
overflow: hidden;
}
div.finalize-buttons {
@ -106,7 +106,6 @@
gap: 8px;
padding: 24px 16px;
width: 320px;
background: white;
border-radius: 4px;
overflow: hidden;
}
@ -174,7 +173,7 @@
}
button.card-button {
color: rgb(255, 136, 136);
color: var(--flower);
background: transparent;
font-size: 16px;
cursor: pointer;
@ -186,7 +185,7 @@
}
button.card-button.happy {
color: rgb(0, 146, 0);
color: var(--leaf);
}
img.configured-icon {

8
src/khoj/interface/web/chat.html
View file

@ -8,7 +8,7 @@
<link rel="manifest" href="/static/khoj_chat.webmanifest">
<link rel="stylesheet" href="/static/assets/khoj.css">
</head>
<script type="text/javascript" src="/static/assets/khoj.js"></script>
<script type="text/javascript" src="/static/assets/utils.js"></script>
<script>
let chatOptions = [];
function copyProgrammaticOutput(event) {
@ -67,6 +67,8 @@
// Replace any ** with <b> and __ with <u>
newHTML = newHTML.replace(/\*\*([\s\S]*?)\*\*/g, '<b>$1</b>');
newHTML = newHTML.replace(/__([\s\S]*?)__/g, '<u>$1</u>');
// Remove any text between <s>[INST] and </s> tags. These are spurious instructions for the AI chat model.
newHTML = newHTML.replace(/<s>\[INST\].+(<\/s>)?/g, '');
return newHTML;
}
@ -163,6 +165,7 @@
function incrementalChat(event) {
if (!event.shiftKey && event.key === 'Enter') {
e.preventDefault();
chat();
}
}
@ -281,8 +284,7 @@
<!-- Chat Footer -->
<div id="chat-footer">
<div id="chat-tooltip" style="display: none;"></div>
<textarea id="chat-input" class="option" oninput="onChatInput()" onkeyup=incrementalChat(event) autofocus="autofocus" placeholder="Type / to see a list of commands, or just type your questions and hit enter.">
</textarea>
<textarea id="chat-input" class="option" oninput="onChatInput()" onkeydown=incrementalChat(event) autofocus="autofocus" placeholder="Type / to see a list of commands, or just type your questions and hit enter."></textarea>
</div>
</body>

60
src/khoj/interface/web/login.html
View file

@ -22,36 +22,30 @@
<button id="khoj-banner-submit" class="khoj-banner-button">Submit</button>
</div>
{% endif %}
<!--Add Header Logo and Nav Pane-->
<div class="khoj-header">
{% if demo %}
<a class="khoj-logo" href="https://khoj.dev" target="_blank">
<img class="khoj-logo" src="/static/assets/icons/khoj-logo-sideways-500.png" alt="Khoj"></img>
</a>
{% else %}
<a class="khoj-logo" href="/">
<img class="khoj-logo" src="/static/assets/icons/khoj-logo-sideways-500.png" alt="Khoj"></img>
</a>
{% endif %}
</div>
<div class="khoj-header"></div>
<!-- Sign Up button for Google OAuth -->
<!-- Login Modal -->
<div id="login-modal">
<h1 class="login-modal-title">Become superhuman with your personal knowledge base copilot</h1>
<img class="khoj-logo" src="/static/assets/icons/favicon-128x128.png" alt="Khoj"></img>
<div class="login-modal-title">Log in to Khoj</div>
<!-- Sign Up/Login with Google OAuth -->
<div
class="g_id_signin"
data-shape="circle"
data-text="continue_with"
data-logo_alignment="center"
data-size="large"
data-width="300"
data-type="standard">
</div>
<div id="g_id_onload"
data-client_id="{{ google_client_id }}"
data-ux_mode="redirect"
data-login_uri="{{ redirect_uri }}"
data-auto-select="true">
</div>
</div>
<div id="g_id_onload"
data-client_id="{{ google_client_id }}"
data-ux_mode="redirect"
data-login_uri="{{ redirect_uri }}">
<div class="khoj-footer"></div>
</div>
@ -62,7 +56,7 @@
body {
display: grid;
grid-template-columns: 1fr;
grid-template-rows: 1fr auto auto auto minmax(80px, 100%);
grid-template-rows: 1fr auto 1fr;
font-size: small!important;
}
body > * {
@ -73,8 +67,7 @@
body {
display: grid;
grid-template-columns: 1fr min(70vw, 100%) 1fr;
grid-template-rows: 1fr auto auto auto minmax(80px, 100%);
padding-top: 60vw;
grid-template-rows: 1fr auto 1fr;
}
body > * {
grid-column: 2;
@ -83,8 +76,9 @@
body {
padding: 0px;
margin: 0px;
background: #fff;
color: #475569;
height: 100%;
background: var(--background-color);
color: var(--main-text-color);
font-family: roboto, karma, segoe ui, sans-serif;
font-size: 20px;
font-weight: 300;
@ -109,6 +103,7 @@
a.khoj-logo {
text-align: center;
justify-self: center;
}
button#khoj-banner-submit,
@ -127,14 +122,10 @@
div#login-modal {
display: grid;
grid-template-columns: 1fr;
grid-template-rows: 1fr auto auto auto;
gap: 10px;
padding: 10px;
margin: 10px;
background: #fff;
border-radius: 5px;
border: 1px solid #475569;
box-shadow: 0 0 11px #aaa;
grid-template-rows: 1fr auto auto 1fr;
gap: 32px;
min-height: 300px;
background: var(--background-color);
margin-left: 25%;
margin-right: 25%;
}
@ -144,10 +135,11 @@
display: block;
}
h1.login-modal-title {
div.login-modal-title {
text-align: center;
line-height: 28px;
font-size: x-large;
font-size: 24px;
font-weight: 500;
}
@media only screen and (max-width: 700px) {

2
src/khoj/interface/web/index.html → src/khoj/interface/web/search.html
View file

@ -10,7 +10,7 @@
</head>
<script type="text/javascript" src="/static/assets/org.min.js"></script>
<script type="text/javascript" src="/static/assets/markdown-it.min.js"></script>
<script type="text/javascript" src="/static/assets/khoj.js"></script>
<script type="text/javascript" src="/static/assets/utils.js"></script>
<script>
function render_image(item) {

2
src/khoj/interface/web/utils.html
View file

@ -5,7 +5,7 @@
</a>
<nav class="khoj-nav">
<a class="khoj-nav" href="/chat">💬 Chat</a>
<a class="khoj-nav" href="/">🔎 Search</a>
<a class="khoj-nav" href="/search">🔎 Search</a>
<!-- Dropdown Menu -->
<div id="khoj-nav-menu-container" class="khoj-nav dropdown">
{% if user_photo and user_photo != "None" %}

1
src/khoj/main.py
View file

@ -122,6 +122,7 @@ def set_state(args):
state.demo = args.demo
state.anonymous_mode = args.anonymous_mode
state.khoj_version = version("khoj-assistant")
state.chat_on_gpu = args.chat_on_gpu
def start_server(app, host=None, port=None, socket=None):

69
src/khoj/migrations/migrate_offline_chat_default_model.py Normal file
View file

@ -0,0 +1,69 @@
"""
Current format of khoj.yml
---
app:
...
content-type:
...
processor:
conversation:
offline-chat:
enable-offline-chat: false
chat-model: llama-2-7b-chat.ggmlv3.q4_0.bin
...
search-type:
...
New format of khoj.yml
---
app:
...
content-type:
...
processor:
conversation:
offline-chat:
enable-offline-chat: false
chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
...
search-type:
...
"""
import logging
from packaging import version
from khoj.utils.yaml import load_config_from_file, save_config_to_file
logger = logging.getLogger(__name__)
def migrate_offline_chat_default_model(args):
schema_version = "0.12.4"
raw_config = load_config_from_file(args.config_file)
previous_version = raw_config.get("version")
if "processor" not in raw_config:
return args
if raw_config["processor"] is None:
return args
if "conversation" not in raw_config["processor"]:
return args
if "offline-chat" not in raw_config["processor"]["conversation"]:
return args
if "chat-model" not in raw_config["processor"]["conversation"]["offline-chat"]:
return args
if previous_version is None or version.parse(previous_version) < version.parse("0.12.4"):
logger.info(
f"Upgrading config schema to {schema_version} from {previous_version} to change default (offline) chat model to mistral GGUF"
)
raw_config["version"] = schema_version
# Update offline chat model to mistral in GGUF format to use latest GPT4All
offline_chat_model = raw_config["processor"]["conversation"]["offline-chat"]["chat-model"]
if offline_chat_model.endswith(".bin"):
raw_config["processor"]["conversation"]["offline-chat"]["chat-model"] = "mistral-7b-instruct-v0.1.Q4_0.gguf"
save_config_to_file(raw_config, args.config_file)
return args

4
src/khoj/migrations/migrate_server_pg.py
View file

@ -9,7 +9,7 @@ processor:
conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
max-prompt-size: null
offline-chat:
chat-model: llama-2-7b-chat.ggmlv3.q4_0.bin
chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
enable-offline-chat: false
openai:
api-key: sk-blah
@ -46,7 +46,7 @@ processor:
- chat-model: gpt-3.5-turbo
tokenizer: null
type: openai
- chat-model: llama-2-7b-chat.ggmlv3.q4_0.bin
- chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
tokenizer: null
type: offline
search-type:

4
src/khoj/processor/conversation/gpt4all/chat_model.py
View file

@ -16,7 +16,7 @@ logger = logging.getLogger(__name__)
def extract_questions_offline(
text: str,
model: str = "llama-2-7b-chat.ggmlv3.q4_0.bin",
model: str = "mistral-7b-instruct-v0.1.Q4_0.gguf",
loaded_model: Union[Any, None] = None,
conversation_log={},
use_history: bool = True,
@ -123,7 +123,7 @@ def converse_offline(
references,
user_query,
conversation_log={},
model: str = "llama-2-7b-chat.ggmlv3.q4_0.bin",
model: str = "mistral-7b-instruct-v0.1.Q4_0.gguf",
loaded_model: Union[Any, None] = None,
completion_func=None,
conversation_command=ConversationCommand.Default,

10
src/khoj/processor/conversation/gpt4all/utils.py
View file

@ -1,5 +1,6 @@
import logging
from khoj.utils import state
logger = logging.getLogger(__name__)
@ -16,8 +17,13 @@ def download_model(model_name: str):
# Decide whether to load model to GPU or CPU
try:
# Check if machine has GPU and GPU has enough free memory to load the chat model
device = "gpu" if gpt4all.pyllmodel.LLModel().list_gpu(chat_model_config["path"]) else "cpu"
# Try load chat model to GPU if:
# 1. Loading chat model to GPU isn't disabled via CLI and
# 2. Machine has GPU
# 3. GPU has enough free memory to load the chat model
device = (
"gpu" if state.chat_on_gpu and gpt4all.pyllmodel.LLModel().list_gpu(chat_model_config["path"]) else "cpu"
)
except ValueError:
device = "cpu"

2
src/khoj/processor/conversation/utils.py
View file

@ -20,9 +20,11 @@ model_to_prompt_size = {
"gpt-4": 8192,
"llama-2-7b-chat.ggmlv3.q4_0.bin": 1548,
"gpt-3.5-turbo-16k": 15000,
"mistral-7b-instruct-v0.1.Q4_0.gguf": 1548,
}
model_to_tokenizer = {
"llama-2-7b-chat.ggmlv3.q4_0.bin": "hf-internal-testing/llama-tokenizer",
"mistral-7b-instruct-v0.1.Q4_0.gguf": "mistralai/Mistral-7B-Instruct-v0.1",
}

16
src/khoj/processor/text_to_entries.py
View file

@ -121,12 +121,12 @@ class TextToEntries(ABC):
batcher(entry_batches, batch_size), desc="Processing embeddings in batches"
):
batch_embeddings_to_create = []
for entry_hash, embedding in entry_batch:
for entry_hash, new_entry in entry_batch:
entry = hash_to_current_entries[entry_hash]
batch_embeddings_to_create.append(
DbEntry(
user=user,
embeddings=embedding,
embeddings=new_entry,
raw=entry.raw,
compiled=entry.compiled,
heading=entry.heading[:1000], # Truncate to max chars of field allowed
@ -136,19 +136,19 @@ class TextToEntries(ABC):
corpus_id=entry.corpus_id,
)
)
new_embeddings = DbEntry.objects.bulk_create(batch_embeddings_to_create)
logger.debug(f"Created {len(new_embeddings)} new embeddings")
num_new_embeddings += len(new_embeddings)
new_entries = DbEntry.objects.bulk_create(batch_embeddings_to_create)
logger.debug(f"Created {len(new_entries)} new embeddings")
num_new_embeddings += len(new_entries)
dates_to_create = []
with timer("Create new date associations for new embeddings", logger):
for embedding in new_embeddings:
dates = self.date_filter.extract_dates(embedding.raw)
for new_entry in new_entries:
dates = self.date_filter.extract_dates(new_entry.raw)
for date in dates:
dates_to_create.append(
EntryDates(
date=date,
embeddings=embedding,
entry=new_entry,
)
)
new_dates = EntryDates.objects.bulk_create(dates_to_create)

3
src/khoj/routers/api.py
View file

@ -670,8 +670,9 @@ async def extract_references_and_questions(
defiltered_query, loaded_model=loaded_model, conversation_log=meta_log, should_extract_questions=False
)
elif await ConversationAdapters.has_openai_chat():
openai_chat_config = await ConversationAdapters.get_openai_chat_config()
openai_chat = await ConversationAdapters.get_openai_chat()
api_key = openai_chat.api_key
api_key = openai_chat_config.api_key
chat_model = openai_chat.chat_model
inferred_queries = extract_questions(
defiltered_query, model=chat_model, api_key=api_key, conversation_log=meta_log

22
src/khoj/routers/web_client.py
View file

@ -10,7 +10,6 @@ from fastapi.templating import Jinja2Templates
from starlette.authentication import requires
from khoj.utils.rawconfig import (
TextContentConfig,
FullConfig,
GithubContentConfig,
GithubRepoConfig,
NotionContentConfig,
@ -36,7 +35,7 @@ def index(request: Request):
user_picture = request.session.get("user", {}).get("picture")
return templates.TemplateResponse(
"index.html",
"chat.html",
context={
"request": request,
"demo": state.demo,
@ -53,7 +52,24 @@ def index_post(request: Request):
user_picture = request.session.get("user", {}).get("picture")
return templates.TemplateResponse(
"index.html",
"chat.html",
context={
"request": request,
"demo": state.demo,
"username": user.username,
"user_photo": user_picture,
},
)
@web_client.get("/search", response_class=FileResponse)
@requires(["authenticated"], redirect="login_page")
def search_page(request: Request):
user = request.user.object
user_picture = request.session.get("user", {}).get("picture")
return templates.TemplateResponse(
"search.html",
context={
"request": request,
"demo": state.demo,

8
src/khoj/utils/cli.py
View file

@ -14,6 +14,7 @@ from khoj.migrations.migrate_version import migrate_config_to_version
from khoj.migrations.migrate_processor_config_openai import migrate_processor_conversation_schema
from khoj.migrations.migrate_offline_model import migrate_offline_model
from khoj.migrations.migrate_offline_chat_schema import migrate_offline_chat_schema
from khoj.migrations.migrate_offline_chat_default_model import migrate_offline_chat_default_model
from khoj.migrations.migrate_server_pg import migrate_server_pg
@ -38,6 +39,9 @@ def cli(args=None):
help="Path to UNIX socket for server. Use to run server behind reverse proxy. Default: /tmp/uvicorn.sock",
)
parser.add_argument("--version", "-V", action="store_true", help="Print the installed Khoj version and exit")
parser.add_argument(
"--disable-chat-on-gpu", action="store_true", default=False, help="Disable using GPU for the offline chat model"
)
parser.add_argument("--demo", action="store_true", default=False, help="Run Khoj in demo mode")
parser.add_argument(
"--anonymous-mode",
@ -50,6 +54,9 @@ def cli(args=None):
logger.debug(f"Ignoring unknown commandline args: {remaining_args}")
# Set default values for arguments
args.chat_on_gpu = not args.disable_chat_on_gpu
args.version_no = version("khoj-assistant")
if args.version:
# Show version of khoj installed and exit
@ -76,6 +83,7 @@ def run_migrations(args):
migrate_processor_conversation_schema,
migrate_offline_model,
migrate_offline_chat_schema,
migrate_offline_chat_default_model,
migrate_server_pg,
]
for migration in migrations:

2
src/khoj/utils/rawconfig.py
View file

@ -84,7 +84,7 @@ class OpenAIProcessorConfig(ConfigBase):
class OfflineChatProcessorConfig(ConfigBase):
enable_offline_chat: Optional[bool] = False
chat_model: Optional[str] = "llama-2-7b-chat.ggmlv3.q4_0.bin"
chat_model: Optional[str] = "mistral-7b-instruct-v0.1.Q4_0.gguf"
class ConversationProcessorConfig(ConfigBase):

3
src/khoj/utils/state.py
View file

@ -33,5 +33,6 @@ SearchType = utils_config.SearchType
telemetry: List[Dict[str, str]] = []
demo: bool = False
khoj_version: str = None
anonymous_mode: bool = False
device = get_device()
chat_on_gpu: bool = True
anonymous_mode: bool = False

2
tests/conftest.py
View file

@ -224,7 +224,7 @@ def chat_client_no_background(search_config: SearchConfig, default_user2: KhojUs
# Initialize Processor from Config
if os.getenv("OPENAI_API_KEY"):
OpenAIProcessorConversationConfigFactory(user=default_user2)
OpenAIProcessorConversationConfigFactory()
state.anonymous_mode = True

2
tests/data/config.yml vendored
View file

@ -14,4 +14,4 @@ search-type:
asymmetric:
cross-encoder: cross-encoder/ms-marco-MiniLM-L-6-v2
encoder: sentence-transformers/msmarco-MiniLM-L-6-v3
version: 0.10.1
version: 0.14.0

2
tests/helpers.py
View file

@ -37,7 +37,7 @@ class ChatModelOptionsFactory(factory.django.DjangoModelFactory):
max_prompt_size = 2000
tokenizer = None
chat_model = "llama-2-7b-chat.ggmlv3.q4_0.bin"
chat_model = "mistral-7b-instruct-v0.1.Q4_0.gguf"
model_type = "offline"

2
tests/test_gpt4all_chat_actors.py
View file

@ -24,7 +24,7 @@ from khoj.processor.conversation.gpt4all.utils import download_model
from khoj.processor.conversation.utils import message_to_log
MODEL_NAME = "llama-2-7b-chat.ggmlv3.q4_0.bin"
MODEL_NAME = "mistral-7b-instruct-v0.1.Q4_0.gguf"
@pytest.fixture(scope="session")