diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8c712dc0..2902a56f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ jobs: - name: Install Dependencies run: | - sudo apt install libegl1 -y + sudo apt update && sudo apt install -y libegl1 python -m pip install --upgrade pip pip install pytest diff --git a/src/interface/obsidian/src/main.ts b/src/interface/obsidian/src/main.ts index 23082895..1725b5ae 100644 --- a/src/interface/obsidian/src/main.ts +++ b/src/interface/obsidian/src/main.ts @@ -45,7 +45,7 @@ export default class Khoj extends Plugin { } async saveSettings() { - await configureKhojBackend(this.settings) + await configureKhojBackend(this.settings, false) .then(() => this.saveData(this.settings)); } } diff --git a/src/interface/obsidian/src/settings.ts b/src/interface/obsidian/src/settings.ts index 74ae9c76..edfbb123 100644 --- a/src/interface/obsidian/src/settings.ts +++ b/src/interface/obsidian/src/settings.ts @@ -1,18 +1,15 @@ -import { App, PluginSettingTab, request, Setting } from 'obsidian'; +import { App, Notice, PluginSettingTab, request, Setting } from 'obsidian'; import Khoj from 'src/main'; -import { getVaultAbsolutePath } from 'src/utils'; export interface KhojSetting { resultsCount: number; khojUrl: string; - obsidianVaultPath: string; connectedToBackend: boolean; } export const DEFAULT_SETTINGS: KhojSetting = { resultsCount: 6, khojUrl: 'http://localhost:8000', - obsidianVaultPath: getVaultAbsolutePath(), connectedToBackend: false, } @@ -28,49 +25,58 @@ export class KhojSettingTab extends PluginSettingTab { const { containerEl } = this; containerEl.empty(); - // Add notice if unable to connect to khoj backend - if (!this.plugin.settings.connectedToBackend) { - containerEl.createEl('small', { text: '❗Ensure Khoj backend is running and Khoj URL is correctly set below' }); - } + // Add notice whether able to connect to khoj backend or not + containerEl.createEl('small', { text: this.getBackendStatusMessage() }); // Add khoj settings configurable from the plugin settings tab - new Setting(containerEl) - .setName('Vault Path') - .setDesc('The Obsidian Vault to search with Khoj') - .addText(text => text - .setValue(`${this.plugin.settings.obsidianVaultPath}`) - .onChange(async (value) => { - this.plugin.settings.obsidianVaultPath = value; - await this.plugin.saveSettings(); - })); new Setting(containerEl) .setName('Khoj URL') .setDesc('The URL of the Khoj backend') .addText(text => text .setValue(`${this.plugin.settings.khojUrl}`) .onChange(async (value) => { - this.plugin.settings.khojUrl = value; - await this.plugin.saveSettings(); + this.plugin.settings.khojUrl = value.trim(); + await this.plugin.saveSettings() + .finally(() => containerEl.firstElementChild?.setText(this.getBackendStatusMessage())); })); new Setting(containerEl) .setName('Results Count') .setDesc('The number of search results to show') - .addText(text => text - .setPlaceholder('6') - .setValue(`${this.plugin.settings.resultsCount}`) + .addSlider(slider => slider + .setLimits(1, 10, 1) + .setValue(this.plugin.settings.resultsCount) + .setDynamicTooltip() .onChange(async (value) => { - this.plugin.settings.resultsCount = parseInt(value); + this.plugin.settings.resultsCount = value; await this.plugin.saveSettings(); })); - new Setting(containerEl) + let indexVaultSetting = new Setting(containerEl); + indexVaultSetting .setName('Index Vault') .setDesc('Manually force Khoj to re-index your Obsidian Vault') .addButton(button => button .setButtonText('Update') .setCta() .onClick(async () => { - await request(`${this.plugin.settings.khojUrl}/api/update?t=markdown&force=true`); - } - )); + // Disable button while updating index + button.setButtonText('Updating...'); + button.removeCta() + indexVaultSetting = indexVaultSetting.setDisabled(true); + + await request(`${this.plugin.settings.khojUrl}/api/update?t=markdown&force=true`) + .then(() => new Notice('✅ Updated Khoj index.')); + + // Re-enable button once index is updated + button.setButtonText('Update'); + button.setCta() + indexVaultSetting = indexVaultSetting.setDisabled(false); + }) + ); + } + + getBackendStatusMessage() { + return !this.plugin.settings.connectedToBackend + ? '❗Disconnected from Khoj backend. Ensure Khoj backend is running and Khoj URL is correctly set below.' + : '✅ Connected to Khoj backend.'; } } diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index 3d4220b2..377297f1 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -9,11 +9,11 @@ export function getVaultAbsolutePath(): string { return ''; } -export async function configureKhojBackend(setting: KhojSetting) { - let mdInVault = `${setting.obsidianVaultPath}/**/*.md`; +export async function configureKhojBackend(setting: KhojSetting, notify: boolean = true) { + let mdInVault = `${getVaultAbsolutePath()}/**/*.md`; let khojConfigUrl = `${setting.khojUrl}/api/config/data`; - // Check if khoj backend is configured, show error if backend is not running + // Check if khoj backend is configured, note if cannot connect to backend let khoj_already_configured = await request(khojConfigUrl) .then(response => { setting.connectedToBackend = true; @@ -21,11 +21,19 @@ export async function configureKhojBackend(setting: KhojSetting) { }) .catch(error => { setting.connectedToBackend = false; - new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`); + if (notify) + new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`); }) // Short-circuit configuring khoj if unable to connect to khoj backend if (!setting.connectedToBackend) return; + // Set index name from the path of the current vault + let indexName = getVaultAbsolutePath().replace(/\//g, '_').replace(/ /g, '_'); + // Get default index directory from khoj backend + let khojDefaultIndexDirectory = await request(`${khojConfigUrl}/default`) + .then(response => JSON.parse(response)) + .then(data => { return getIndexDirectoryFromBackendConfig(data); }); + // Get current config if khoj backend configured, else get default config from khoj backend await request(khoj_already_configured ? khojConfigUrl : `${khojConfigUrl}/default`) .then(response => JSON.parse(response)) @@ -33,13 +41,12 @@ export async function configureKhojBackend(setting: KhojSetting) { // If khoj backend not configured yet if (!khoj_already_configured) { // Create khoj content-type config with only markdown configured - let khojObsidianPluginPath = `${setting.obsidianVaultPath}/${this.app.vault.configDir}/plugins/khoj/`; data["content-type"] = { "markdown": { "input-filter": [mdInVault], "input-files": null, - "embeddings-file": `${khojObsidianPluginPath}/markdown_embeddings.pt`, - "compressed-jsonl": `${khojObsidianPluginPath}/markdown.jsonl.gz`, + "embeddings-file": `${khojDefaultIndexDirectory}/${indexName}.pt`, + "compressed-jsonl": `${khojDefaultIndexDirectory}/${indexName}.jsonl.gz`, } } // Disable khoj processors, as not required @@ -54,12 +61,11 @@ export async function configureKhojBackend(setting: KhojSetting) { else if (!data["content-type"]["markdown"]) { // Add markdown config to khoj content-type config // Set markdown config to index markdown files in configured obsidian vault - let khojObsidianPluginPath = `${setting.obsidianVaultPath}/${this.app.vault.configDir}/plugins/khoj/`; data["content-type"]["markdown"] = { "input-filter": [mdInVault], "input-files": null, - "embeddings-file": `${khojObsidianPluginPath}/markdown_embeddings.pt`, - "compressed-jsonl": `${khojObsidianPluginPath}/markdown.jsonl.gz`, + "embeddings-file": `${khojDefaultIndexDirectory}/${indexName}.pt`, + "compressed-jsonl": `${khojDefaultIndexDirectory}/${indexName}.jsonl.gz`, } // Save updated config and refresh index on khoj backend @@ -72,17 +78,21 @@ export async function configureKhojBackend(setting: KhojSetting) { data["content-type"]["markdown"]["input-filter"][0] !== mdInVault) { // Update markdown config in khoj content-type config // Set markdown config to only index markdown files in configured obsidian vault - data["content-type"]["markdown"]["input-filter"] = [mdInVault] - data["content-type"]["markdown"]["input-files"] = null - + let khojIndexDirectory = getIndexDirectoryFromBackendConfig(data); + data["content-type"]["markdown"] = { + "input-filter": [mdInVault], + "input-files": null, + "embeddings-file": `${khojIndexDirectory}/${indexName}.pt`, + "compressed-jsonl": `${khojIndexDirectory}/${indexName}.jsonl.gz`, + } // Save updated config and refresh index on khoj backend updateKhojBackend(setting.khojUrl, data); console.log(`Khoj: Updated markdown config in khoj backend config:\n${JSON.stringify(data["content-type"]["markdown"])}`) } - new Notice(`✅ Successfully Setup Khoj`); }) .catch(error => { - new Notice(`❗️Failed to configure Khoj backend. Contact developer on Github.\n\nError: ${error}`); + if (notify) + new Notice(`❗️Failed to configure Khoj backend. Contact developer on Github.\n\nError: ${error}`); }) } @@ -98,5 +108,9 @@ export async function updateKhojBackend(khojUrl: string, khojConfig: Object) { // Save khojConfig on khoj backend at khojConfigUrl await request(requestContent) // Refresh khoj search index after updating config - .then(_ => request(`${khojUrl}/api/update?t=markdown`)); + .then(_ => request(`${khojUrl}/api/update?t=markdown&force=true`)); } + +function getIndexDirectoryFromBackendConfig(khojConfig: any) { + return khojConfig["content-type"]["markdown"]["embeddings-file"].split("/").slice(0, -1).join("/"); +} \ No newline at end of file diff --git a/src/routers/api.py b/src/routers/api.py index 0da30eb3..9480c914 100644 --- a/src/routers/api.py +++ b/src/routers/api.py @@ -1,11 +1,11 @@ # Standard Packages import yaml -import time import logging from typing import Optional # External Packages from fastapi import APIRouter +from fastapi import HTTPException # Internal Packages from src.configure import configure_processor, configure_search @@ -114,12 +114,22 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti @api.get('/update') def update(t: Optional[SearchType] = None, force: Optional[bool] = False): - state.search_index_lock.acquire() - state.model = configure_search(state.model, state.config, regenerate=force, t=t) - state.search_index_lock.release() - logger.info("Search Index updated via API call") + try: + state.search_index_lock.acquire() + state.model = configure_search(state.model, state.config, regenerate=force, t=t) + state.search_index_lock.release() + except ValueError as e: + logger.error(e) + raise HTTPException(status_code=500, detail=str(e)) + else: + logger.info("Search Index updated via API call") - state.processor_config = configure_processor(state.config.processor) - logger.info("Processor reconfigured via API call") + try: + state.processor_config = configure_processor(state.config.processor) + except ValueError as e: + logger.error(e) + raise HTTPException(status_code=500, detail=str(e)) + else: + logger.info("Processor reconfigured via API call") return {'status': 'ok', 'message': 'khoj reloaded'}