diff --git a/src/interface/obsidian/src/chat_view.ts b/src/interface/obsidian/src/chat_view.ts index 4a4bf0ae..ca0c95e8 100644 --- a/src/interface/obsidian/src/chat_view.ts +++ b/src/interface/obsidian/src/chat_view.ts @@ -1,4 +1,4 @@ -import { ItemView, MarkdownRenderer, WorkspaceLeaf, request, requestUrl, setIcon } from 'obsidian'; +import { ItemView, MarkdownRenderer, Scope, WorkspaceLeaf, request, requestUrl, setIcon } from 'obsidian'; import * as DOMPurify from 'dompurify'; import { KhojSetting } from 'src/settings'; import { KhojPaneView } from 'src/pane_view'; @@ -28,6 +28,10 @@ export class KhojChatView extends KhojPaneView { constructor(leaf: WorkspaceLeaf, setting: KhojSetting) { super(leaf, setting); + // Register Modal Keybindings to send voice message + this.scope = new Scope(this.app.scope); + this.scope.register(["Mod"], 's', async (event) => { await this.speechToText(event); }); + this.waitingForLocation = true; fetch("https://ipapi.co/json") @@ -61,7 +65,7 @@ export class KhojChatView extends KhojPaneView { return "message-circle"; } - async chat() { + async chat(isVoice: boolean = false) { // Get text in chat input element let input_el = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; @@ -72,7 +76,7 @@ export class KhojChatView extends KhojPaneView { this.autoResize(); // Get and render chat response to user message - await this.getChatResponse(user_message); + await this.getChatResponse(user_message, isVoice); } async onOpen() { @@ -294,6 +298,60 @@ export class KhojChatView extends KhojPaneView { return referenceButton; } + textToSpeech(message: string, event: MouseEvent | null = null): void { + // Replace the speaker with a loading icon. + let loader = document.createElement("span"); + loader.classList.add("loader"); + + let speechButton: HTMLButtonElement; + let speechIcon: Element; + + if (event === null) { + // Pick the last speech button if none is provided + let speechButtons = document.getElementsByClassName("speech-button"); + speechButton = speechButtons[speechButtons.length - 1] as HTMLButtonElement; + + let speechIcons = document.getElementsByClassName("speech-icon"); + speechIcon = speechIcons[speechIcons.length - 1]; + } else { + speechButton = event.currentTarget as HTMLButtonElement; + speechIcon = event.target as Element; + } + + speechButton.innerHTML = ""; + speechButton.appendChild(loader); + speechButton.disabled = true; + + const context = new AudioContext(); + let textToSpeechApi = `${this.setting.khojUrl}/api/chat/speech?text=${encodeURIComponent(message)}`; + fetch(textToSpeechApi, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + "Authorization": `Bearer ${this.setting.khojApiKey}`, + }, + }) + .then(response => response.arrayBuffer()) + .then(arrayBuffer => context.decodeAudioData(arrayBuffer)) + .then(audioBuffer => { + const source = context.createBufferSource(); + source.buffer = audioBuffer; + source.connect(context.destination); + source.start(0); + source.onended = function() { + speechButton.innerHTML = ""; + speechButton.appendChild(speechIcon); + speechButton.disabled = false; + }; + }) + .catch(err => { + console.error("Error playing speech:", err); + speechButton.innerHTML = ""; + speechButton.appendChild(speechIcon); + speechButton.disabled = false; // Consider enabling the button again to allow retrying + }); + } + formatHTMLMessage(message: string, raw = false, willReplace = true) { // Remove any text between [INST] and tags. These are spurious instructions for some AI chat model. message = message.replace(/\[INST\].+(<\/s>)?/g, ''); @@ -461,19 +519,36 @@ export class KhojChatView extends KhojPaneView { renderActionButtons(message: string, chat_message_body_text_el: HTMLElement) { let copyButton = this.contentEl.createEl('button'); - copyButton.classList.add("copy-button"); + copyButton.classList.add("chat-action-button"); copyButton.title = "Copy Message to Clipboard"; setIcon(copyButton, "copy-plus"); copyButton.addEventListener('click', createCopyParentText(message)); - chat_message_body_text_el.append(copyButton); // Add button to paste into current buffer let pasteToFile = this.contentEl.createEl('button'); - pasteToFile.classList.add("copy-button"); + pasteToFile.classList.add("chat-action-button"); pasteToFile.title = "Paste Message to File"; setIcon(pasteToFile, "clipboard-paste"); pasteToFile.addEventListener('click', (event) => { pasteTextAtCursor(createCopyParentText(message, 'clipboard-paste')(event)); }); - chat_message_body_text_el.append(pasteToFile); + + // Only enable the speech feature if the user is subscribed + let speechButton = null; + + if (this.setting.userInfo?.is_active) { + // Create a speech button icon to play the message out loud + speechButton = this.contentEl.createEl('button'); + speechButton.classList.add("chat-action-button", "speech-button"); + speechButton.title = "Listen to Message"; + setIcon(speechButton, "speech") + speechButton.addEventListener('click', (event) => this.textToSpeech(message, event)); + } + + // Append buttons to parent element + chat_message_body_text_el.append(copyButton, pasteToFile); + + if (speechButton) { + chat_message_body_text_el.append(speechButton); + } } formatDate(date: Date): string { @@ -727,7 +802,7 @@ export class KhojChatView extends KhojPaneView { return true; } - async readChatStream(response: Response, responseElement: HTMLDivElement): Promise { + async readChatStream(response: Response, responseElement: HTMLDivElement, isVoice: boolean = false): Promise { // Exit if response body is empty if (response.body == null) return; @@ -737,8 +812,12 @@ export class KhojChatView extends KhojPaneView { while (true) { const { value, done } = await reader.read(); - // Break if the stream is done - if (done) break; + if (done) { + // Automatically respond with voice if the subscribed user has sent voice message + if (isVoice && this.setting.userInfo?.is_active) this.textToSpeech(this.result); + // Break if the stream is done + break; + } let responseText = decoder.decode(value); if (responseText.includes("### compiled references:")) { @@ -756,7 +835,7 @@ export class KhojChatView extends KhojPaneView { } } - async getChatResponse(query: string | undefined | null): Promise { + async getChatResponse(query: string | undefined | null, isVoice: boolean = false): Promise { // Exit if query is empty if (!query || query === "") return; @@ -835,7 +914,7 @@ export class KhojChatView extends KhojPaneView { } } else { // Stream and render chat response - await this.readChatStream(response, responseElement); + await this.readChatStream(response, responseElement, isVoice); } } catch (err) { console.log(`Khoj chat response failed with\n${err}`); @@ -883,7 +962,7 @@ export class KhojChatView extends KhojPaneView { sendMessageTimeout: NodeJS.Timeout | undefined; mediaRecorder: MediaRecorder | undefined; - async speechToText(event: MouseEvent | TouchEvent) { + async speechToText(event: MouseEvent | TouchEvent | KeyboardEvent) { event.preventDefault(); const transcribeButton = this.contentEl.getElementsByClassName("khoj-transcribe")[0]; const chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; @@ -947,7 +1026,7 @@ export class KhojChatView extends KhojPaneView { sendImg.addEventListener('click', async (_) => { await this.chat() }); // Send message - this.chat(); + this.chat(true); }, 3000); }; diff --git a/src/interface/obsidian/src/main.ts b/src/interface/obsidian/src/main.ts index a97c1466..bf7cad54 100644 --- a/src/interface/obsidian/src/main.ts +++ b/src/interface/obsidian/src/main.ts @@ -2,7 +2,8 @@ import { Plugin, WorkspaceLeaf } from 'obsidian'; import { KhojSetting, KhojSettingTab, DEFAULT_SETTINGS } from 'src/settings' import { KhojSearchModal } from 'src/search_modal' import { KhojChatView } from 'src/chat_view' -import { updateContentIndex, canConnectToBackend, KhojView } from './utils'; +import { updateContentIndex, canConnectToBackend, KhojView, jumpToPreviousView } from './utils'; +import { KhojPaneView } from './pane_view'; export default class Khoj extends Plugin { @@ -79,16 +80,30 @@ export default class Khoj extends Plugin { const leaves = workspace.getLeavesOfType(viewType); if (leaves.length > 0) { - // A leaf with our view already exists, use that - leaf = leaves[0]; + // A leaf with our view already exists, use that + leaf = leaves[0]; } else { - // Our view could not be found in the workspace, create a new leaf - // in the right sidebar for it - leaf = workspace.getRightLeaf(false); - await leaf?.setViewState({ type: viewType, active: true }); + // Our view could not be found in the workspace, create a new leaf + // in the right sidebar for it + leaf = workspace.getRightLeaf(false); + await leaf?.setViewState({ type: viewType, active: true }); } - // "Reveal" the leaf in case it is in a collapsed sidebar - if (leaf) workspace.revealLeaf(leaf); - } + if (leaf) { + const activeKhojLeaf = workspace.getActiveViewOfType(KhojPaneView)?.leaf; + // Jump to the previous view if the current view is Khoj Side Pane + if (activeKhojLeaf === leaf) jumpToPreviousView(); + // Else Reveal the leaf in case it is in a collapsed sidebar + else { + workspace.revealLeaf(leaf); + + if (viewType === KhojView.CHAT) { + // focus on the chat input when the chat view is opened + let chatView = leaf.view as KhojChatView; + let chatInput = chatView.contentEl.getElementsByClassName("khoj-chat-input")[0]; + if (chatInput) chatInput.focus(); + } + } + } + } } diff --git a/src/interface/obsidian/src/pane_view.ts b/src/interface/obsidian/src/pane_view.ts index 3912ce4d..64a167dd 100644 --- a/src/interface/obsidian/src/pane_view.ts +++ b/src/interface/obsidian/src/pane_view.ts @@ -38,16 +38,24 @@ export abstract class KhojPaneView extends ItemView { const leaves = workspace.getLeavesOfType(viewType); if (leaves.length > 0) { - // A leaf with our view already exists, use that - leaf = leaves[0]; + // A leaf with our view already exists, use that + leaf = leaves[0]; } else { - // Our view could not be found in the workspace, create a new leaf - // in the right sidebar for it - leaf = workspace.getRightLeaf(false); - await leaf?.setViewState({ type: viewType, active: true }); + // Our view could not be found in the workspace, create a new leaf + // in the right sidebar for it + leaf = workspace.getRightLeaf(false); + await leaf?.setViewState({ type: viewType, active: true }); } - // "Reveal" the leaf in case it is in a collapsed sidebar - if (leaf) workspace.revealLeaf(leaf); - } + if (leaf) { + if (viewType === KhojView.CHAT) { + // focus on the chat input when the chat view is opened + let chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; + if (chatInput) chatInput.focus(); + } + + // "Reveal" the leaf in case it is in a collapsed sidebar + workspace.revealLeaf(leaf); + } + } } diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index 5f3e5acb..4a969793 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -333,6 +333,12 @@ export function createCopyParentText(message: string, originalButton: string = ' } } +export function jumpToPreviousView() { + const editor: Editor = this.app.workspace.getActiveFileView()?.editor + if (!editor) return; + editor.focus(); +} + export function pasteTextAtCursor(text: string | undefined) { // Get the current active file's editor const editor: Editor = this.app.workspace.getActiveFileView()?.editor diff --git a/src/interface/obsidian/styles.css b/src/interface/obsidian/styles.css index a79e0116..8e3d2c6b 100644 --- a/src/interface/obsidian/styles.css +++ b/src/interface/obsidian/styles.css @@ -477,7 +477,7 @@ span.khoj-nav-item-text { } /* Copy button */ -button.copy-button { +button.chat-action-button { display: block; border-radius: 4px; color: var(--text-muted); @@ -491,20 +491,54 @@ button.copy-button { margin-top: 8px; float: right; } -button.copy-button span { +button.chat-action-button span { cursor: pointer; display: inline-block; position: relative; transition: 0.5s; } +button.chat-action-button:hover { + background-color: var(--background-modifier-active-hover); + color: var(--text-normal); +} + img.copy-icon { width: 16px; height: 16px; } -button.copy-button:hover { - background-color: var(--background-modifier-active-hover); - color: var(--text-normal); +/* Circular Loading Spinner */ +.loader { + width: 18px; + height: 18px; + border: 3px solid #FFF; + border-radius: 50%; + display: inline-block; + position: relative; + box-sizing: border-box; + animation: rotation 1s linear infinite; +} +.loader::after { + content: ''; + box-sizing: border-box; + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + width: 18px; + height: 18px; + border-radius: 50%; + border: 3px solid transparent; + border-bottom-color: var(--flower); +} + +@keyframes rotation { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } } /* Loading Spinner */