sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-11-23 23:48:56 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 16

Fix, Improve Configuring Khoj from Obsidian Plugin

Browse source 
### Details
- 1c813a6 Convert *Results Count* setting to `Slider` from `Text` in plugin settings pane
- 4e1abd1 Disable `Update` button in plugin settings while indexing vault
- 513c86c Set index file paths relative to current or default path on Khoj backend
- 4407e23 Only index current vault on Khoj. Remove `ObsidianVaultPath` setting from plugin
- 86a1e43 Return HTTP Exception on */api/update* API call failure
- 5af2b68 Update plugin notifications for errors. Remove notification for success
This commit is contained in:
Debanjum 2023-01-11 17:01:33 -03:00 committed by GitHub
commit e28af68cbd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 82 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/test.yml vendored
View file

@ -34,7 +34,7 @@ jobs:
- name: Install Dependencies
run: |
sudo apt install libegl1 -y
sudo apt update && sudo apt install -y libegl1
python -m pip install --upgrade pip
pip install pytest

2
src/interface/obsidian/src/main.ts
View file

@ -45,7 +45,7 @@ export default class Khoj extends Plugin {
}
async saveSettings() {
await configureKhojBackend(this.settings)
await configureKhojBackend(this.settings, false)
.then(() => this.saveData(this.settings));
}
}

60
src/interface/obsidian/src/settings.ts
View file

@ -1,18 +1,15 @@
import { App, PluginSettingTab, request, Setting } from 'obsidian';
import { App, Notice, PluginSettingTab, request, Setting } from 'obsidian';
import Khoj from 'src/main';
import { getVaultAbsolutePath } from 'src/utils';
export interface KhojSetting {
resultsCount: number;
khojUrl: string;
obsidianVaultPath: string;
connectedToBackend: boolean;
}
export const DEFAULT_SETTINGS: KhojSetting = {
resultsCount: 6,
khojUrl: 'http://localhost:8000',
obsidianVaultPath: getVaultAbsolutePath(),
connectedToBackend: false,
}
@ -28,49 +25,58 @@ export class KhojSettingTab extends PluginSettingTab {
const { containerEl } = this;
containerEl.empty();
// Add notice if unable to connect to khoj backend
if (!this.plugin.settings.connectedToBackend) {
containerEl.createEl('small', { text: '❗Ensure Khoj backend is running and Khoj URL is correctly set below' });
}
// Add notice whether able to connect to khoj backend or not
containerEl.createEl('small', { text: this.getBackendStatusMessage() });
// Add khoj settings configurable from the plugin settings tab
new Setting(containerEl)
.setName('Vault Path')
.setDesc('The Obsidian Vault to search with Khoj')
.addText(text => text
.setValue(`${this.plugin.settings.obsidianVaultPath}`)
.onChange(async (value) => {
this.plugin.settings.obsidianVaultPath = value;
await this.plugin.saveSettings();
}));
new Setting(containerEl)
.setName('Khoj URL')
.setDesc('The URL of the Khoj backend')
.addText(text => text
.setValue(`${this.plugin.settings.khojUrl}`)
.onChange(async (value) => {
this.plugin.settings.khojUrl = value;
await this.plugin.saveSettings();
this.plugin.settings.khojUrl = value.trim();
await this.plugin.saveSettings()
.finally(() => containerEl.firstElementChild?.setText(this.getBackendStatusMessage()));
}));
new Setting(containerEl)
.setName('Results Count')
.setDesc('The number of search results to show')
.addText(text => text
.setPlaceholder('6')
.setValue(`${this.plugin.settings.resultsCount}`)
.addSlider(slider => slider
.setLimits(1, 10, 1)
.setValue(this.plugin.settings.resultsCount)
.setDynamicTooltip()
.onChange(async (value) => {
this.plugin.settings.resultsCount = parseInt(value);
this.plugin.settings.resultsCount = value;
await this.plugin.saveSettings();
}));
new Setting(containerEl)
let indexVaultSetting = new Setting(containerEl);
indexVaultSetting
.setName('Index Vault')
.setDesc('Manually force Khoj to re-index your Obsidian Vault')
.addButton(button => button
.setButtonText('Update')
.setCta()
.onClick(async () => {
await request(`${this.plugin.settings.khojUrl}/api/update?t=markdown&force=true`);
}
));
// Disable button while updating index
button.setButtonText('Updating...');
button.removeCta()
indexVaultSetting = indexVaultSetting.setDisabled(true);
await request(`${this.plugin.settings.khojUrl}/api/update?t=markdown&force=true`)
.then(() => new Notice('✅ Updated Khoj index.'));
// Re-enable button once index is updated
button.setButtonText('Update');
button.setCta()
indexVaultSetting = indexVaultSetting.setDisabled(false);
})
);
}
getBackendStatusMessage() {
return !this.plugin.settings.connectedToBackend
? '❗Disconnected from Khoj backend. Ensure Khoj backend is running and Khoj URL is correctly set below.'
: '✅ Connected to Khoj backend.';
}
}

46
src/interface/obsidian/src/utils.ts
View file

@ -9,11 +9,11 @@ export function getVaultAbsolutePath(): string {
return '';
}
export async function configureKhojBackend(setting: KhojSetting) {
let mdInVault = `${setting.obsidianVaultPath}/**/*.md`;
export async function configureKhojBackend(setting: KhojSetting, notify: boolean = true) {
let mdInVault = `${getVaultAbsolutePath()}/**/*.md`;
let khojConfigUrl = `${setting.khojUrl}/api/config/data`;
// Check if khoj backend is configured, show error if backend is not running
// Check if khoj backend is configured, note if cannot connect to backend
let khoj_already_configured = await request(khojConfigUrl)
.then(response => {
setting.connectedToBackend = true;
@ -21,11 +21,19 @@ export async function configureKhojBackend(setting: KhojSetting) {
})
.catch(error => {
setting.connectedToBackend = false;
new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`);
if (notify)
new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`);
})
// Short-circuit configuring khoj if unable to connect to khoj backend
if (!setting.connectedToBackend) return;
// Set index name from the path of the current vault
let indexName = getVaultAbsolutePath().replace(/\//g, '_').replace(/ /g, '_');
// Get default index directory from khoj backend
let khojDefaultIndexDirectory = await request(`${khojConfigUrl}/default`)
.then(response => JSON.parse(response))
.then(data => { return getIndexDirectoryFromBackendConfig(data); });
// Get current config if khoj backend configured, else get default config from khoj backend
await request(khoj_already_configured ? khojConfigUrl : `${khojConfigUrl}/default`)
.then(response => JSON.parse(response))
@ -33,13 +41,12 @@ export async function configureKhojBackend(setting: KhojSetting) {
// If khoj backend not configured yet
if (!khoj_already_configured) {
// Create khoj content-type config with only markdown configured
let khojObsidianPluginPath = `${setting.obsidianVaultPath}/${this.app.vault.configDir}/plugins/khoj/`;
data["content-type"] = {
"markdown": {
"input-filter": [mdInVault],
"input-files": null,
"embeddings-file": `${khojObsidianPluginPath}/markdown_embeddings.pt`,
"compressed-jsonl": `${khojObsidianPluginPath}/markdown.jsonl.gz`,
"embeddings-file": `${khojDefaultIndexDirectory}/${indexName}.pt`,
"compressed-jsonl": `${khojDefaultIndexDirectory}/${indexName}.jsonl.gz`,
}
}
// Disable khoj processors, as not required
@ -54,12 +61,11 @@ export async function configureKhojBackend(setting: KhojSetting) {
else if (!data["content-type"]["markdown"]) {
// Add markdown config to khoj content-type config
// Set markdown config to index markdown files in configured obsidian vault
let khojObsidianPluginPath = `${setting.obsidianVaultPath}/${this.app.vault.configDir}/plugins/khoj/`;
data["content-type"]["markdown"] = {
"input-filter": [mdInVault],
"input-files": null,
"embeddings-file": `${khojObsidianPluginPath}/markdown_embeddings.pt`,
"compressed-jsonl": `${khojObsidianPluginPath}/markdown.jsonl.gz`,
"embeddings-file": `${khojDefaultIndexDirectory}/${indexName}.pt`,
"compressed-jsonl": `${khojDefaultIndexDirectory}/${indexName}.jsonl.gz`,
}
// Save updated config and refresh index on khoj backend
@ -72,17 +78,21 @@ export async function configureKhojBackend(setting: KhojSetting) {
data["content-type"]["markdown"]["input-filter"][0] !== mdInVault) {
// Update markdown config in khoj content-type config
// Set markdown config to only index markdown files in configured obsidian vault
data["content-type"]["markdown"]["input-filter"] = [mdInVault]
data["content-type"]["markdown"]["input-files"] = null
let khojIndexDirectory = getIndexDirectoryFromBackendConfig(data);
data["content-type"]["markdown"] = {
"input-filter": [mdInVault],
"input-files": null,
"embeddings-file": `${khojIndexDirectory}/${indexName}.pt`,
"compressed-jsonl": `${khojIndexDirectory}/${indexName}.jsonl.gz`,
}
// Save updated config and refresh index on khoj backend
updateKhojBackend(setting.khojUrl, data);
console.log(`Khoj: Updated markdown config in khoj backend config:\n${JSON.stringify(data["content-type"]["markdown"])}`)
}
new Notice(`✅ Successfully Setup Khoj`);
})
.catch(error => {
new Notice(`❗️Failed to configure Khoj backend. Contact developer on Github.\n\nError: ${error}`);
if (notify)
new Notice(`❗️Failed to configure Khoj backend. Contact developer on Github.\n\nError: ${error}`);
})
}
@ -98,5 +108,9 @@ export async function updateKhojBackend(khojUrl: string, khojConfig: Object) {
// Save khojConfig on khoj backend at khojConfigUrl
await request(requestContent)
// Refresh khoj search index after updating config
.then(_ => request(`${khojUrl}/api/update?t=markdown`));
.then(_ => request(`${khojUrl}/api/update?t=markdown&force=true`));
}
function getIndexDirectoryFromBackendConfig(khojConfig: any) {
return khojConfig["content-type"]["markdown"]["embeddings-file"].split("/").slice(0, -1).join("/");
}

24
src/routers/api.py
View file

@ -1,11 +1,11 @@
# Standard Packages
import yaml
import time
import logging
from typing import Optional
# External Packages
from fastapi import APIRouter
from fastapi import HTTPException
# Internal Packages
from src.configure import configure_processor, configure_search
@ -114,12 +114,22 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti
@api.get('/update')
def update(t: Optional[SearchType] = None, force: Optional[bool] = False):
state.search_index_lock.acquire()
state.model = configure_search(state.model, state.config, regenerate=force, t=t)
state.search_index_lock.release()
logger.info("Search Index updated via API call")
try:
state.search_index_lock.acquire()
state.model = configure_search(state.model, state.config, regenerate=force, t=t)
state.search_index_lock.release()
except ValueError as e:
logger.error(e)
raise HTTPException(status_code=500, detail=str(e))
else:
logger.info("Search Index updated via API call")
state.processor_config = configure_processor(state.config.processor)
logger.info("Processor reconfigured via API call")
try:
state.processor_config = configure_processor(state.config.processor)
except ValueError as e:
logger.error(e)
raise HTTPException(status_code=500, detail=str(e))
else:
logger.info("Processor reconfigured via API call")
return {'status': 'ok', 'message': 'khoj reloaded'}