Fix, Improve Configuring Khoj from Obsidian Plugin

### Details
- 1c813a6 Convert *Results Count* setting to `Slider` from `Text` in plugin settings pane
- 4e1abd1 Disable `Update` button in plugin settings while indexing vault
- 513c86c Set index file paths relative to current or default path on Khoj backend
- 4407e23 Only index current vault on Khoj. Remove `ObsidianVaultPath` setting from plugin
- 86a1e43 Return HTTP Exception on */api/update* API call failure
- 5af2b68 Update plugin notifications for errors. Remove notification for success

.github/workflows/test.yml

@@ -34,7 +34,7 @@ jobs:
 
       - name: Install Dependencies
         run: |
-          sudo apt install libegl1 -y
+          sudo apt update && sudo apt install -y libegl1
           python -m pip install --upgrade pip
           pip install pytest
 

src/interface/obsidian/src/main.ts

@@ -45,7 +45,7 @@ export default class Khoj extends Plugin {
     }
 
     async saveSettings() {
-        await configureKhojBackend(this.settings)
+        await configureKhojBackend(this.settings, false)
             .then(() => this.saveData(this.settings));
     }
 }

src/interface/obsidian/src/settings.ts

@@ -1,18 +1,15 @@
-import { App, PluginSettingTab, request, Setting } from 'obsidian';
+import { App, Notice, PluginSettingTab, request, Setting } from 'obsidian';
 import Khoj from 'src/main';
-import { getVaultAbsolutePath } from 'src/utils';
 
 export interface KhojSetting {
     resultsCount: number;
     khojUrl: string;
-    obsidianVaultPath: string;
     connectedToBackend: boolean;
 }
 
 export const DEFAULT_SETTINGS: KhojSetting = {
     resultsCount: 6,
     khojUrl: 'http://localhost:8000',
-    obsidianVaultPath: getVaultAbsolutePath(),
     connectedToBackend: false,
 }
 
@@ -28,49 +25,58 @@ export class KhojSettingTab extends PluginSettingTab {
         const { containerEl } = this;
         containerEl.empty();
 
-        // Add notice if unable to connect to khoj backend
+        // Add notice whether able to connect to khoj backend or not
-        if (!this.plugin.settings.connectedToBackend) {
+        containerEl.createEl('small', { text: this.getBackendStatusMessage() });
-            containerEl.createEl('small', { text: '❗Ensure Khoj backend is running and Khoj URL is correctly set below' });
-        }
 
         // Add khoj settings configurable from the plugin settings tab
-        new Setting(containerEl)
-            .setName('Vault Path')
-            .setDesc('The Obsidian Vault to search with Khoj')
-            .addText(text => text
-                .setValue(`${this.plugin.settings.obsidianVaultPath}`)
-                .onChange(async (value) => {
-                    this.plugin.settings.obsidianVaultPath = value;
-                    await this.plugin.saveSettings();
-                }));
         new Setting(containerEl)
             .setName('Khoj URL')
             .setDesc('The URL of the Khoj backend')
             .addText(text => text
                 .setValue(`${this.plugin.settings.khojUrl}`)
                 .onChange(async (value) => {
-                    this.plugin.settings.khojUrl = value;
+                    this.plugin.settings.khojUrl = value.trim();
-                    await this.plugin.saveSettings();
+                    await this.plugin.saveSettings()
+                    .finally(() => containerEl.firstElementChild?.setText(this.getBackendStatusMessage()));
                 }));
          new Setting(containerEl)
             .setName('Results Count')
             .setDesc('The number of search results to show')
-            .addText(text => text
+            .addSlider(slider => slider
-                .setPlaceholder('6')
+                .setLimits(1, 10, 1)
-                .setValue(`${this.plugin.settings.resultsCount}`)
+                .setValue(this.plugin.settings.resultsCount)
+                .setDynamicTooltip()
                 .onChange(async (value) => {
-                    this.plugin.settings.resultsCount = parseInt(value);
+                    this.plugin.settings.resultsCount = value;
                     await this.plugin.saveSettings();
                 }));
-        new Setting(containerEl)
+        let indexVaultSetting = new Setting(containerEl);
+        indexVaultSetting
             .setName('Index Vault')
             .setDesc('Manually force Khoj to re-index your Obsidian Vault')
             .addButton(button => button
                 .setButtonText('Update')
                 .setCta()
                 .onClick(async () => {
-                    await request(`${this.plugin.settings.khojUrl}/api/update?t=markdown&force=true`);
+                    // Disable button while updating index
-                }
+                    button.setButtonText('Updating...');
-            ));
+                    button.removeCta()
+                    indexVaultSetting = indexVaultSetting.setDisabled(true);
+
+                    await request(`${this.plugin.settings.khojUrl}/api/update?t=markdown&force=true`)
+                    .then(() => new Notice('✅ Updated Khoj index.'));
+
+                    // Re-enable button once index is updated
+                    button.setButtonText('Update');
+                    button.setCta()
+                    indexVaultSetting = indexVaultSetting.setDisabled(false);
+                })
+            );
+    }
+
+    getBackendStatusMessage() {
+        return !this.plugin.settings.connectedToBackend
+        ? '❗Disconnected from Khoj backend. Ensure Khoj backend is running and Khoj URL is correctly set below.'
+        : '✅ Connected to Khoj backend.';
     }
 }

src/interface/obsidian/src/utils.ts

@@ -9,11 +9,11 @@ export function getVaultAbsolutePath(): string {
     return '';
 }
 
-export async function configureKhojBackend(setting: KhojSetting) {
+export async function configureKhojBackend(setting: KhojSetting, notify: boolean = true) {
-    let mdInVault = `${setting.obsidianVaultPath}/**/*.md`;
+    let mdInVault = `${getVaultAbsolutePath()}/**/*.md`;
     let khojConfigUrl = `${setting.khojUrl}/api/config/data`;
 
-    // Check if khoj backend is configured, show error if backend is not running
+    // Check if khoj backend is configured, note if cannot connect to backend
     let khoj_already_configured = await request(khojConfigUrl)
         .then(response => {
             setting.connectedToBackend = true;
@@ -21,11 +21,19 @@ export async function configureKhojBackend(setting: KhojSetting) {
         })
         .catch(error => {
             setting.connectedToBackend = false;
-            new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`);
+            if (notify)
+                new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`);
         })
     // Short-circuit configuring khoj if unable to connect to khoj backend
     if (!setting.connectedToBackend) return;
 
+    // Set index name from the path of the current vault
+    let indexName = getVaultAbsolutePath().replace(/\//g, '_').replace(/ /g, '_');
+    // Get default index directory from khoj backend
+    let khojDefaultIndexDirectory = await request(`${khojConfigUrl}/default`)
+        .then(response => JSON.parse(response))
+        .then(data => { return getIndexDirectoryFromBackendConfig(data); });
+
     // Get current config if khoj backend configured, else get default config from khoj backend
     await request(khoj_already_configured ? khojConfigUrl : `${khojConfigUrl}/default`)
         .then(response => JSON.parse(response))
@@ -33,13 +41,12 @@ export async function configureKhojBackend(setting: KhojSetting) {
             // If khoj backend not configured yet
             if (!khoj_already_configured) {
                 // Create khoj content-type config with only markdown configured
-                let khojObsidianPluginPath = `${setting.obsidianVaultPath}/${this.app.vault.configDir}/plugins/khoj/`;
                 data["content-type"] = {
                     "markdown": {
                         "input-filter": [mdInVault],
                         "input-files": null,
-                        "embeddings-file": `${khojObsidianPluginPath}/markdown_embeddings.pt`,
+                        "embeddings-file": `${khojDefaultIndexDirectory}/${indexName}.pt`,
-                        "compressed-jsonl": `${khojObsidianPluginPath}/markdown.jsonl.gz`,
+                        "compressed-jsonl": `${khojDefaultIndexDirectory}/${indexName}.jsonl.gz`,
                     }
                 }
                 // Disable khoj processors, as not required
@@ -54,12 +61,11 @@ export async function configureKhojBackend(setting: KhojSetting) {
             else if (!data["content-type"]["markdown"]) {
                 // Add markdown config to khoj content-type config
                 // Set markdown config to index markdown files in configured obsidian vault
-                let khojObsidianPluginPath = `${setting.obsidianVaultPath}/${this.app.vault.configDir}/plugins/khoj/`;
                 data["content-type"]["markdown"] = {
                     "input-filter": [mdInVault],
                     "input-files": null,
-                    "embeddings-file": `${khojObsidianPluginPath}/markdown_embeddings.pt`,
+                    "embeddings-file": `${khojDefaultIndexDirectory}/${indexName}.pt`,
-                    "compressed-jsonl": `${khojObsidianPluginPath}/markdown.jsonl.gz`,
+                    "compressed-jsonl": `${khojDefaultIndexDirectory}/${indexName}.jsonl.gz`,
                 }
 
                 // Save updated config and refresh index on khoj backend
@@ -72,17 +78,21 @@ export async function configureKhojBackend(setting: KhojSetting) {
                 data["content-type"]["markdown"]["input-filter"][0] !== mdInVault) {
                 // Update markdown config in khoj content-type config
                 // Set markdown config to only index markdown files in configured obsidian vault
-                data["content-type"]["markdown"]["input-filter"] = [mdInVault]
+                let khojIndexDirectory = getIndexDirectoryFromBackendConfig(data);
-                data["content-type"]["markdown"]["input-files"] = null
+                data["content-type"]["markdown"] = {
-
+                    "input-filter": [mdInVault],
+                    "input-files": null,
+                    "embeddings-file": `${khojIndexDirectory}/${indexName}.pt`,
+                    "compressed-jsonl": `${khojIndexDirectory}/${indexName}.jsonl.gz`,
+                }
                 // Save updated config and refresh index on khoj backend
                 updateKhojBackend(setting.khojUrl, data);
                 console.log(`Khoj: Updated markdown config in khoj backend config:\n${JSON.stringify(data["content-type"]["markdown"])}`)
             }
-            new Notice(`✅ Successfully Setup Khoj`);
         })
         .catch(error => {
-            new Notice(`❗️Failed to configure Khoj backend. Contact developer on Github.\n\nError: ${error}`);
+            if (notify)
+                new Notice(`❗️Failed to configure Khoj backend. Contact developer on Github.\n\nError: ${error}`);
         })
 }
 
@@ -98,5 +108,9 @@ export async function updateKhojBackend(khojUrl: string, khojConfig: Object) {
     // Save khojConfig on khoj backend at khojConfigUrl
     await request(requestContent)
         // Refresh khoj search index after updating config
-        .then(_ => request(`${khojUrl}/api/update?t=markdown`));
+        .then(_ => request(`${khojUrl}/api/update?t=markdown&force=true`));
+}
+
+function getIndexDirectoryFromBackendConfig(khojConfig: any) {
+    return khojConfig["content-type"]["markdown"]["embeddings-file"].split("/").slice(0, -1).join("/");
 }

src/routers/api.py

@@ -1,11 +1,11 @@
 # Standard Packages
 import yaml
-import time
 import logging
 from typing import Optional
 
 # External Packages
 from fastapi import APIRouter
+from fastapi import HTTPException
 
 # Internal Packages
 from src.configure import configure_processor, configure_search
@@ -114,12 +114,22 @@ def search(q: str, n: Optional[int] = 5, t: Optional[SearchType] = None, r: Opti
 
 @api.get('/update')
 def update(t: Optional[SearchType] = None, force: Optional[bool] = False):
-    state.search_index_lock.acquire()
+    try:
-    state.model = configure_search(state.model, state.config, regenerate=force, t=t)
+        state.search_index_lock.acquire()
-    state.search_index_lock.release()
+        state.model = configure_search(state.model, state.config, regenerate=force, t=t)
-    logger.info("Search Index updated via API call")
+        state.search_index_lock.release()
+    except ValueError as e:
+        logger.error(e)
+        raise HTTPException(status_code=500, detail=str(e))
+    else:
+        logger.info("Search Index updated via API call")
 
-    state.processor_config = configure_processor(state.config.processor)
+    try:
-    logger.info("Processor reconfigured via API call")
+        state.processor_config = configure_processor(state.config.processor)
+    except ValueError as e:
+        logger.error(e)
+        raise HTTPException(status_code=500, detail=str(e))
+    else:
+        logger.info("Processor reconfigured via API call")
 
     return {'status': 'ok', 'message': 'khoj reloaded'}