Enable Voice, Keyboard Shortcuts in Khoj Obsidian Plugin (#837)

- Simplify quick jump between Khoj side pane and main editor view using keyboard shortcuts
- Enable voice chat in Obsidian to make interactions with Khoj more seamless

src/interface/obsidian/src/chat_view.ts

@@ -1,4 +1,4 @@
-import { ItemView, MarkdownRenderer, WorkspaceLeaf, request, requestUrl, setIcon } from 'obsidian';
+import { ItemView, MarkdownRenderer, Scope, WorkspaceLeaf, request, requestUrl, setIcon } from 'obsidian';
 import * as DOMPurify from 'dompurify';
 import { KhojSetting } from 'src/settings';
 import { KhojPaneView } from 'src/pane_view';
@@ -28,6 +28,10 @@ export class KhojChatView extends KhojPaneView {
     constructor(leaf: WorkspaceLeaf, setting: KhojSetting) {
         super(leaf, setting);
 
+        // Register Modal Keybindings to send voice message
+        this.scope = new Scope(this.app.scope);
+        this.scope.register(["Mod"], 's', async (event) => { await this.speechToText(event); });
+
         this.waitingForLocation = true;
 
         fetch("https://ipapi.co/json")
@@ -61,7 +65,7 @@ export class KhojChatView extends KhojPaneView {
         return "message-circle";
     }
 
-    async chat() {
+    async chat(isVoice: boolean = false) {
 
         // Get text in chat input element
         let input_el = <HTMLTextAreaElement>this.contentEl.getElementsByClassName("khoj-chat-input")[0];
@@ -72,7 +76,7 @@ export class KhojChatView extends KhojPaneView {
         this.autoResize();
 
         // Get and render chat response to user message
-        await this.getChatResponse(user_message);
+        await this.getChatResponse(user_message, isVoice);
     }
 
     async onOpen() {
@@ -294,6 +298,60 @@ export class KhojChatView extends KhojPaneView {
         return referenceButton;
     }
 
+    textToSpeech(message: string, event: MouseEvent | null = null): void {
+        // Replace the speaker with a loading icon.
+        let loader = document.createElement("span");
+        loader.classList.add("loader");
+
+        let speechButton: HTMLButtonElement;
+        let speechIcon: Element;
+
+        if (event === null) {
+            // Pick the last speech button if none is provided
+            let speechButtons = document.getElementsByClassName("speech-button");
+            speechButton = speechButtons[speechButtons.length - 1] as HTMLButtonElement;
+
+            let speechIcons = document.getElementsByClassName("speech-icon");
+            speechIcon = speechIcons[speechIcons.length - 1];
+        } else {
+            speechButton = event.currentTarget as HTMLButtonElement;
+            speechIcon = event.target as Element;
+        }
+
+        speechButton.innerHTML = "";
+        speechButton.appendChild(loader);
+        speechButton.disabled = true;
+
+        const context = new AudioContext();
+        let textToSpeechApi = `${this.setting.khojUrl}/api/chat/speech?text=${encodeURIComponent(message)}`;
+        fetch(textToSpeechApi, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                "Authorization": `Bearer ${this.setting.khojApiKey}`,
+            },
+        })
+        .then(response => response.arrayBuffer())
+        .then(arrayBuffer => context.decodeAudioData(arrayBuffer))
+        .then(audioBuffer => {
+            const source = context.createBufferSource();
+            source.buffer = audioBuffer;
+            source.connect(context.destination);
+            source.start(0);
+            source.onended = function() {
+                speechButton.innerHTML = "";
+                speechButton.appendChild(speechIcon);
+                speechButton.disabled = false;
+            };
+        })
+        .catch(err => {
+            console.error("Error playing speech:", err);
+            speechButton.innerHTML = "";
+            speechButton.appendChild(speechIcon);
+            speechButton.disabled = false; // Consider enabling the button again to allow retrying
+        });
+    }
+
     formatHTMLMessage(message: string, raw = false, willReplace = true) {
         // Remove any text between <s>[INST] and </s> tags. These are spurious instructions for some AI chat model.
         message = message.replace(/<s>\[INST\].+(<\/s>)?/g, '');
@@ -461,19 +519,36 @@ export class KhojChatView extends KhojPaneView {
 
     renderActionButtons(message: string, chat_message_body_text_el: HTMLElement) {
         let copyButton = this.contentEl.createEl('button');
-        copyButton.classList.add("copy-button");
+        copyButton.classList.add("chat-action-button");
         copyButton.title = "Copy Message to Clipboard";
         setIcon(copyButton, "copy-plus");
         copyButton.addEventListener('click', createCopyParentText(message));
-        chat_message_body_text_el.append(copyButton);
 
         // Add button to paste into current buffer
         let pasteToFile = this.contentEl.createEl('button');
-        pasteToFile.classList.add("copy-button");
+        pasteToFile.classList.add("chat-action-button");
         pasteToFile.title = "Paste Message to File";
         setIcon(pasteToFile, "clipboard-paste");
         pasteToFile.addEventListener('click', (event) => { pasteTextAtCursor(createCopyParentText(message, 'clipboard-paste')(event)); });
-        chat_message_body_text_el.append(pasteToFile);
+
+        // Only enable the speech feature if the user is subscribed
+        let speechButton = null;
+
+        if (this.setting.userInfo?.is_active) {
+            // Create a speech button icon to play the message out loud
+            speechButton = this.contentEl.createEl('button');
+            speechButton.classList.add("chat-action-button", "speech-button");
+            speechButton.title = "Listen to Message";
+            setIcon(speechButton, "speech")
+            speechButton.addEventListener('click', (event) => this.textToSpeech(message, event));
+        }
+
+        // Append buttons to parent element
+        chat_message_body_text_el.append(copyButton, pasteToFile);
+
+        if (speechButton) {
+            chat_message_body_text_el.append(speechButton);
+        }
     }
 
     formatDate(date: Date): string {
@@ -727,7 +802,7 @@ export class KhojChatView extends KhojPaneView {
         return true;
     }
 
-    async readChatStream(response: Response, responseElement: HTMLDivElement): Promise<void> {
+    async readChatStream(response: Response, responseElement: HTMLDivElement, isVoice: boolean = false): Promise<void> {
         // Exit if response body is empty
         if (response.body == null) return;
 
@@ -737,8 +812,12 @@ export class KhojChatView extends KhojPaneView {
         while (true) {
             const { value, done } = await reader.read();
 
-            // Break if the stream is done
+            if (done) {
-            if (done) break;
+                // Automatically respond with voice if the subscribed user has sent voice message
+                if (isVoice && this.setting.userInfo?.is_active) this.textToSpeech(this.result);
+                // Break if the stream is done
+                break;
+            }
 
             let responseText = decoder.decode(value);
             if (responseText.includes("### compiled references:")) {
@@ -756,7 +835,7 @@ export class KhojChatView extends KhojPaneView {
         }
     }
 
-    async getChatResponse(query: string | undefined | null): Promise<void> {
+    async getChatResponse(query: string | undefined | null, isVoice: boolean = false): Promise<void> {
         // Exit if query is empty
         if (!query || query === "") return;
 
@@ -835,7 +914,7 @@ export class KhojChatView extends KhojPaneView {
                 }
             } else {
                 // Stream and render chat response
-                await this.readChatStream(response, responseElement);
+                await this.readChatStream(response, responseElement, isVoice);
             }
         } catch (err) {
             console.log(`Khoj chat response failed with\n${err}`);
@@ -883,7 +962,7 @@ export class KhojChatView extends KhojPaneView {
 
     sendMessageTimeout: NodeJS.Timeout | undefined;
     mediaRecorder: MediaRecorder | undefined;
-    async speechToText(event: MouseEvent | TouchEvent) {
+    async speechToText(event: MouseEvent | TouchEvent | KeyboardEvent) {
         event.preventDefault();
         const transcribeButton = <HTMLButtonElement>this.contentEl.getElementsByClassName("khoj-transcribe")[0];
         const chatInput = <HTMLTextAreaElement>this.contentEl.getElementsByClassName("khoj-chat-input")[0];
@@ -947,7 +1026,7 @@ export class KhojChatView extends KhojPaneView {
                 sendImg.addEventListener('click', async (_) => { await this.chat() });
 
                 // Send message
-                this.chat();
+                this.chat(true);
             }, 3000);
         };
 

src/interface/obsidian/src/main.ts

@@ -2,7 +2,8 @@ import { Plugin, WorkspaceLeaf } from 'obsidian';
 import { KhojSetting, KhojSettingTab, DEFAULT_SETTINGS } from 'src/settings'
 import { KhojSearchModal } from 'src/search_modal'
 import { KhojChatView } from 'src/chat_view'
-import { updateContentIndex, canConnectToBackend, KhojView } from './utils';
+import { updateContentIndex, canConnectToBackend, KhojView, jumpToPreviousView } from './utils';
+import { KhojPaneView } from './pane_view';
 
 
 export default class Khoj extends Plugin {
@@ -79,16 +80,30 @@ export default class Khoj extends Plugin {
         const leaves = workspace.getLeavesOfType(viewType);
 
         if (leaves.length > 0) {
-          // A leaf with our view already exists, use that
+            // A leaf with our view already exists, use that
-          leaf = leaves[0];
+            leaf = leaves[0];
         } else {
-          // Our view could not be found in the workspace, create a new leaf
+            // Our view could not be found in the workspace, create a new leaf
-          // in the right sidebar for it
+            // in the right sidebar for it
-          leaf = workspace.getRightLeaf(false);
+            leaf = workspace.getRightLeaf(false);
-          await leaf?.setViewState({ type: viewType, active: true });
+            await leaf?.setViewState({ type: viewType, active: true });
         }
 
-        // "Reveal" the leaf in case it is in a collapsed sidebar
+        if (leaf) {
-        if (leaf) workspace.revealLeaf(leaf);
+            const activeKhojLeaf = workspace.getActiveViewOfType(KhojPaneView)?.leaf;
-      }
+            // Jump to the previous view if the current view is Khoj Side Pane
+            if (activeKhojLeaf === leaf) jumpToPreviousView();
+            // Else Reveal the leaf in case it is in a collapsed sidebar
+            else {
+                workspace.revealLeaf(leaf);
+
+                if (viewType === KhojView.CHAT) {
+                    // focus on the chat input when the chat view is opened
+                    let chatView = leaf.view as KhojChatView;
+                    let chatInput = <HTMLTextAreaElement>chatView.contentEl.getElementsByClassName("khoj-chat-input")[0];
+                    if (chatInput) chatInput.focus();
+                }
+            }
+        }
+    }
 }

src/interface/obsidian/src/pane_view.ts

@@ -38,16 +38,24 @@ export abstract class KhojPaneView extends ItemView {
         const leaves = workspace.getLeavesOfType(viewType);
 
         if (leaves.length > 0) {
-          // A leaf with our view already exists, use that
+            // A leaf with our view already exists, use that
-          leaf = leaves[0];
+            leaf = leaves[0];
         } else {
-          // Our view could not be found in the workspace, create a new leaf
+            // Our view could not be found in the workspace, create a new leaf
-          // in the right sidebar for it
+            // in the right sidebar for it
-          leaf = workspace.getRightLeaf(false);
+            leaf = workspace.getRightLeaf(false);
-          await leaf?.setViewState({ type: viewType, active: true });
+            await leaf?.setViewState({ type: viewType, active: true });
         }
 
-        // "Reveal" the leaf in case it is in a collapsed sidebar
+        if (leaf) {
-        if (leaf) workspace.revealLeaf(leaf);
+            if (viewType === KhojView.CHAT) {
-      }
+                // focus on the chat input when the chat view is opened
+                let chatInput = <HTMLTextAreaElement>this.contentEl.getElementsByClassName("khoj-chat-input")[0];
+                if (chatInput) chatInput.focus();
+            }
+
+            // "Reveal" the leaf in case it is in a collapsed sidebar
+            workspace.revealLeaf(leaf);
+        }
+    }
 }

src/interface/obsidian/src/utils.ts

@@ -333,6 +333,12 @@ export function createCopyParentText(message: string, originalButton: string = '
     }
 }
 
+export function jumpToPreviousView() {
+    const editor: Editor = this.app.workspace.getActiveFileView()?.editor
+    if (!editor) return;
+    editor.focus();
+}
+
 export function pasteTextAtCursor(text: string | undefined) {
     // Get the current active file's editor
     const editor: Editor = this.app.workspace.getActiveFileView()?.editor

src/interface/obsidian/styles.css

@@ -477,7 +477,7 @@ span.khoj-nav-item-text {
 }
 
 /* Copy button */
-button.copy-button {
+button.chat-action-button {
     display: block;
     border-radius: 4px;
     color: var(--text-muted);
@@ -491,20 +491,54 @@ button.copy-button {
     margin-top: 8px;
     float: right;
 }
-button.copy-button span {
+button.chat-action-button span {
     cursor: pointer;
     display: inline-block;
     position: relative;
     transition: 0.5s;
 }
+button.chat-action-button:hover {
+    background-color: var(--background-modifier-active-hover);
+    color: var(--text-normal);
+}
+
 img.copy-icon {
     width: 16px;
     height: 16px;
 }
 
-button.copy-button:hover {
+/* Circular Loading Spinner */
-    background-color: var(--background-modifier-active-hover);
+.loader {
-    color: var(--text-normal);
+    width: 18px;
+    height: 18px;
+    border: 3px solid #FFF;
+    border-radius: 50%;
+    display: inline-block;
+    position: relative;
+    box-sizing: border-box;
+    animation: rotation 1s linear infinite;
+}
+.loader::after {
+    content: '';
+    box-sizing: border-box;
+    position: absolute;
+    left: 50%;
+    top: 50%;
+    transform: translate(-50%, -50%);
+    width: 18px;
+    height: 18px;
+    border-radius: 50%;
+    border: 3px solid transparent;
+    border-bottom-color: var(--flower);
+}
+
+@keyframes rotation {
+    0% {
+        transform: rotate(0deg);
+    }
+    100% {
+        transform: rotate(360deg);
+    }
 }
 
 /* Loading Spinner */