diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html index eee29a99..5078904b 100644 --- a/src/interface/desktop/chat.html +++ b/src/interface/desktop/chat.html @@ -62,7 +62,10 @@ return `${time_string}, ${date_string}`; } - function generateReference(reference, index) { + function generateReference(referenceJson, index) { + let reference = referenceJson.hasOwnProperty("compiled") ? referenceJson.compiled : referenceJson; + let referenceFile = referenceJson.hasOwnProperty("file") ? referenceJson.file : null; + // Escape reference for HTML rendering let escaped_ref = reference.replaceAll('"', '"'); @@ -219,98 +222,44 @@ } function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) { + let chatEl; + if (intentType?.includes("text-to-image")) { + let imageMarkdown = generateImageMarkdown(message, intentType, inferredQueries); + chatEl = renderMessage(imageMarkdown, by, dt, null, false, "return"); + } else { + chatEl = renderMessage(message, by, dt, null, false, "return"); + } + // If no document or online context is provided, render the message as is - if ((context == null || context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) { - if (intentType?.includes("text-to-image")) { - let imageMarkdown; - if (intentType === "text-to-image") { - imageMarkdown = `![](data:image/png;base64,${message})`; - } else if (intentType === "text-to-image2") { - imageMarkdown = `![](${message})`; - } else if (intentType === "text-to-image-v3") { - imageMarkdown = `![](data:image/webp;base64,${message})`; - } - - const inferredQuery = inferredQueries?.[0]; - if (inferredQuery) { - imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`; - } - return renderMessage(imageMarkdown, by, dt, null, false, "return"); - } - - return renderMessage(message, by, dt, null, false, "return"); - } - - if (context == null && onlineContext == null) { - return renderMessage(message, by, dt, null, false, "return"); - } - - if ((context && context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) { - return renderMessage(message, by, dt, null, false, "return"); + if ((context == null || context?.length == 0) + && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) { + return chatEl; } // If document or online context is provided, render the message with its references - let references = document.createElement('div'); + let references = {}; + if (!!context) references["notes"] = context; + if (!!onlineContext) references["online"] = onlineContext; + let chatMessageEl = chatEl.getElementsByClassName("chat-message-text")[0]; + chatMessageEl.appendChild(createReferenceSection(references)); - let referenceExpandButton = document.createElement('button'); - referenceExpandButton.classList.add("reference-expand-button"); - let numReferences = 0; + return chatEl; + } - if (context) { - numReferences += context.length; + function generateImageMarkdown(message, intentType, inferredQueries=null) { + let imageMarkdown; + if (intentType === "text-to-image") { + imageMarkdown = `![](data:image/png;base64,${message})`; + } else if (intentType === "text-to-image2") { + imageMarkdown = `![](${message})`; + } else if (intentType === "text-to-image-v3") { + imageMarkdown = `![](data:image/webp;base64,${message})`; } - - references.appendChild(referenceExpandButton); - - let referenceSection = document.createElement('div'); - referenceSection.classList.add("reference-section"); - referenceSection.classList.add("collapsed"); - - referenceExpandButton.addEventListener('click', function() { - if (referenceSection.classList.contains("collapsed")) { - referenceSection.classList.remove("collapsed"); - referenceSection.classList.add("expanded"); - } else { - referenceSection.classList.add("collapsed"); - referenceSection.classList.remove("expanded"); - } - }); - - references.classList.add("references"); - if (context) { - for (let index in context) { - let reference = context[index]; - let polishedReference = generateReference(reference, index); - referenceSection.appendChild(polishedReference); - } + const inferredQuery = inferredQueries?.[0]; + if (inferredQuery) { + imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`; } - - if (onlineContext) { - numReferences += processOnlineReferences(referenceSection, onlineContext); - } - - let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`; - referenceExpandButton.innerHTML = expandButtonText; - - references.appendChild(referenceSection); - - if (intentType?.includes("text-to-image")) { - let imageMarkdown; - if (intentType === "text-to-image") { - imageMarkdown = `![](data:image/png;base64,${message})`; - } else if (intentType === "text-to-image2") { - imageMarkdown = `![](${message})`; - } else if (intentType === "text-to-image-v3") { - imageMarkdown = `![](data:image/webp;base64,${message})`; - } - const inferredQuery = inferredQueries?.[0]; - if (inferredQuery) { - imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`; - } - return renderMessage(imageMarkdown, by, dt, references, false, "return"); - } - - return renderMessage(message, by, dt, references, false, "return"); + return imageMarkdown; } function formatHTMLMessage(message, raw=false, willReplace=true) { diff --git a/src/interface/obsidian/src/chat_modal.ts b/src/interface/obsidian/src/chat_modal.ts deleted file mode 100644 index 31b938a1..00000000 --- a/src/interface/obsidian/src/chat_modal.ts +++ /dev/null @@ -1,643 +0,0 @@ -import { App, MarkdownRenderer, Modal, request, requestUrl, setIcon } from 'obsidian'; -import { KhojSetting } from 'src/settings'; - -export interface ChatJsonResult { - image?: string; - detail?: string; - intentType?: string; - inferredQueries?: string[]; -} - - -export class KhojChatModal extends Modal { - result: string; - setting: KhojSetting; - region: string; - city: string; - countryName: string; - timezone: string; - - constructor(app: App, setting: KhojSetting) { - super(app); - this.setting = setting; - - // Register Modal Keybindings to send user message - this.scope.register([], 'Enter', async () => { await this.chat() }); - - - fetch("https://ipapi.co/json") - .then(response => response.json()) - .then(data => { - this.region = data.region; - this.city = data.city; - this.countryName = data.country_name; - this.timezone = data.timezone; - }) - .catch(err => { - console.log(err); - return; - }); - } - - async chat() { - // Get text in chat input element - let input_el = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; - - // Clear text after extracting message to send - let user_message = input_el.value.trim(); - input_el.value = ""; - this.autoResize(); - - // Get and render chat response to user message - await this.getChatResponse(user_message); - } - - async onOpen() { - let { contentEl } = this; - contentEl.addClass("khoj-chat"); - - // Add title to the Khoj Chat modal - contentEl.createEl("h1", ({ attr: { id: "khoj-chat-title" }, text: "Khoj Chat" })); - - // Create area for chat logs - let chatBodyEl = contentEl.createDiv({ attr: { id: "khoj-chat-body", class: "khoj-chat-body" } }); - - // Add chat input field - let inputRow = contentEl.createDiv("khoj-input-row"); - let clearChat = inputRow.createEl("button", { - text: "Clear History", - attr: { - class: "khoj-input-row-button clickable-icon", - }, - }) - clearChat.addEventListener('click', async (_) => { await this.clearConversationHistory() }); - setIcon(clearChat, "trash"); - - let chatInput = inputRow.createEl("textarea", { - attr: { - id: "khoj-chat-input", - autofocus: "autofocus", - class: "khoj-chat-input option", - }, - }) - chatInput.addEventListener('input', (_) => { this.onChatInput() }); - chatInput.addEventListener('keydown', (event) => { this.incrementalChat(event) }); - - let transcribe = inputRow.createEl("button", { - text: "Transcribe", - attr: { - id: "khoj-transcribe", - class: "khoj-transcribe khoj-input-row-button clickable-icon ", - }, - }) - transcribe.addEventListener('mousedown', async (event) => { await this.speechToText(event) }); - transcribe.addEventListener('touchstart', async (event) => { await this.speechToText(event) }); - transcribe.addEventListener('touchend', async (event) => { await this.speechToText(event) }); - transcribe.addEventListener('touchcancel', async (event) => { await this.speechToText(event) }); - setIcon(transcribe, "mic"); - - let send = inputRow.createEl("button", { - text: "Send", - attr: { - id: "khoj-chat-send", - class: "khoj-chat-send khoj-input-row-button clickable-icon", - }, - }) - setIcon(send, "arrow-up-circle"); - let sendImg = send.getElementsByClassName("lucide-arrow-up-circle")[0] - sendImg.addEventListener('click', async (_) => { await this.chat() }); - - // Get chat history from Khoj backend and set chat input state - let getChatHistorySucessfully = await this.getChatHistory(chatBodyEl); - let placeholderText = getChatHistorySucessfully ? "Message" : "Configure Khoj to enable chat"; - chatInput.placeholder = placeholderText; - chatInput.disabled = !getChatHistorySucessfully; - - // Scroll to bottom of modal, till the send message input box - this.scrollChatToBottom(); - chatInput.focus(); - } - - generateReference(messageEl: Element, reference: string, index: number) { - // Escape reference for HTML rendering - let escaped_ref = reference.replace(/"/g, """) - - // Generate HTML for Chat Reference - let short_ref = escaped_ref.slice(0, 100); - short_ref = short_ref.length < escaped_ref.length ? short_ref + "..." : short_ref; - let referenceButton = messageEl.createEl('button'); - referenceButton.textContent = short_ref; - referenceButton.id = `ref-${index}`; - referenceButton.classList.add("reference-button"); - referenceButton.classList.add("collapsed"); - referenceButton.tabIndex = 0; - - // Add event listener to toggle full reference on click - referenceButton.addEventListener('click', function() { - console.log(`Toggling ref-${index}`) - if (this.classList.contains("collapsed")) { - this.classList.remove("collapsed"); - this.classList.add("expanded"); - this.textContent = escaped_ref; - } else { - this.classList.add("collapsed"); - this.classList.remove("expanded"); - this.textContent = short_ref; - } - }); - - return referenceButton; - } - - renderMessageWithReferences(chatEl: Element, message: string, sender: string, context?: string[], dt?: Date, intentType?: string, inferredQueries?: string) { - if (!message) { - return; - } else if (intentType?.includes("text-to-image")) { - let imageMarkdown = ""; - if (intentType === "text-to-image") { - imageMarkdown = `![](data:image/png;base64,${message})`; - } else if (intentType === "text-to-image2") { - imageMarkdown = `![](${message})`; - } else if (intentType === "text-to-image-v3") { - imageMarkdown = `![](data:image/webp;base64,${message})`; - } - if (inferredQueries) { - imageMarkdown += "\n\n**Inferred Query**:"; - for (let inferredQuery of inferredQueries) { - imageMarkdown += `\n\n${inferredQuery}`; - } - } - this.renderMessage(chatEl, imageMarkdown, sender, dt); - return; - } else if (!context) { - this.renderMessage(chatEl, message, sender, dt); - return; - } else if (!!context && context?.length === 0) { - this.renderMessage(chatEl, message, sender, dt); - return; - } - let chatMessageEl = this.renderMessage(chatEl, message, sender, dt); - let chatMessageBodyEl = chatMessageEl.getElementsByClassName("khoj-chat-message-text")[0] - let references = chatMessageBodyEl.createDiv(); - - let referenceExpandButton = references.createEl('button'); - referenceExpandButton.classList.add("reference-expand-button"); - let numReferences = 0; - - if (context) { - numReferences += context.length; - } - - let referenceSection = references.createEl('div'); - referenceSection.classList.add("reference-section"); - referenceSection.classList.add("collapsed"); - - referenceExpandButton.addEventListener('click', function() { - if (referenceSection.classList.contains("collapsed")) { - referenceSection.classList.remove("collapsed"); - referenceSection.classList.add("expanded"); - } else { - referenceSection.classList.add("collapsed"); - referenceSection.classList.remove("expanded"); - } - }); - - references.classList.add("references"); - if (context) { - context.map((reference, index) => { - this.generateReference(referenceSection, reference, index + 1); - }); - } - - let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`; - referenceExpandButton.innerHTML = expandButtonText; - } - - renderMessage(chatEl: Element, message: string, sender: string, dt?: Date, raw: boolean=false): Element { - let message_time = this.formatDate(dt ?? new Date()); - let emojified_sender = sender == "khoj" ? "🏮 Khoj" : "🤔 You"; - - // Append message to conversation history HTML element. - // The chat logs should display above the message input box to follow standard UI semantics - let chatMessageEl = chatEl.createDiv({ - attr: { - "data-meta": `${emojified_sender} at ${message_time}`, - class: `khoj-chat-message ${sender}` - }, - }) - let chat_message_body_el = chatMessageEl.createDiv(); - chat_message_body_el.addClasses(["khoj-chat-message-text", sender]); - let chat_message_body_text_el = chat_message_body_el.createDiv(); - if (raw) { - chat_message_body_text_el.innerHTML = message; - } else { - // @ts-ignore - MarkdownRenderer.renderMarkdown(message, chat_message_body_text_el, '', null); - } - - // Remove user-select: none property to make text selectable - chatMessageEl.style.userSelect = "text"; - - // Scroll to bottom after inserting chat messages - this.scrollChatToBottom(); - - return chatMessageEl - } - - createKhojResponseDiv(dt?: Date): HTMLDivElement { - let message_time = this.formatDate(dt ?? new Date()); - - // Append message to conversation history HTML element. - // The chat logs should display above the message input box to follow standard UI semantics - let chat_body_el = this.contentEl.getElementsByClassName("khoj-chat-body")[0]; - let chat_message_el = chat_body_el.createDiv({ - attr: { - "data-meta": `🏮 Khoj at ${message_time}`, - class: `khoj-chat-message khoj` - }, - }).createDiv({ - attr: { - class: `khoj-chat-message-text khoj` - }, - }) - - // Scroll to bottom after inserting chat messages - this.scrollChatToBottom(); - - return chat_message_el - } - - async renderIncrementalMessage(htmlElement: HTMLDivElement, additionalMessage: string) { - this.result += additionalMessage; - htmlElement.innerHTML = ""; - // @ts-ignore - await MarkdownRenderer.renderMarkdown(this.result, htmlElement, '', null); - // Scroll to bottom of modal, till the send message input box - this.scrollChatToBottom(); - } - - formatDate(date: Date): string { - // Format date in HH:MM, DD MMM YYYY format - let time_string = date.toLocaleTimeString('en-IN', { hour: '2-digit', minute: '2-digit', hour12: false }); - let date_string = date.toLocaleString('en-IN', { year: 'numeric', month: 'short', day: '2-digit' }).replace(/-/g, ' '); - return `${time_string}, ${date_string}`; - } - - async getChatHistory(chatBodyEl: Element): Promise { - // Get chat history from Khoj backend - let chatUrl = `${this.setting.khojUrl}/api/chat/history?client=obsidian`; - - try { - let response = await fetch(chatUrl, { - method: "GET", - headers: { "Authorization": `Bearer ${this.setting.khojApiKey}` }, - }); - - let responseJson: any = await response.json(); - - if (responseJson.detail) { - // If the server returns error details in response, render a setup hint. - let setupMsg = "Hi 👋🏾, to start chatting add available chat models options via [the Django Admin panel](/server/admin) on the Server"; - this.renderMessage(chatBodyEl, setupMsg, "khoj", undefined); - - return false; - } else if (responseJson.response) { - let chatLogs = responseJson.response?.conversation_id ? responseJson.response.chat ?? [] : responseJson.response; - chatLogs.forEach((chatLog: any) => { - this.renderMessageWithReferences( - chatBodyEl, - chatLog.message, - chatLog.by, - chatLog.context, - new Date(chatLog.created), - chatLog.intent?.type, - chatLog.intent?.["inferred-queries"], - ); - }); - } - } catch (err) { - let errorMsg = "Unable to get response from Khoj server ❤️‍🩹. Ensure server is running or contact developers for help at [team@khoj.dev](mailto:team@khoj.dev) or in [Discord](https://discord.gg/BDgyabRM6e)"; - this.renderMessage(chatBodyEl, errorMsg, "khoj", undefined); - return false; - } - return true; - } - - async readChatStream(response: Response, responseElement: HTMLDivElement): Promise { - // Exit if response body is empty - if (response.body == null) return; - - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - - while (true) { - const { value, done } = await reader.read(); - - // Break if the stream is done - if (done) break; - - let responseText = decoder.decode(value); - if (responseText.includes("### compiled references:")) { - // Render any references used to generate the response - const [additionalResponse, rawReference] = responseText.split("### compiled references:", 2); - await this.renderIncrementalMessage(responseElement, additionalResponse); - - const rawReferenceAsJson = JSON.parse(rawReference); - let references = responseElement.createDiv(); - references.classList.add("references"); - - let referenceExpandButton = references.createEl('button'); - referenceExpandButton.classList.add("reference-expand-button"); - - let referenceSection = references.createDiv(); - referenceSection.classList.add("reference-section"); - referenceSection.classList.add("collapsed"); - - let numReferences = 0; - - // If rawReferenceAsJson is a list, then count the length - if (Array.isArray(rawReferenceAsJson)) { - numReferences = rawReferenceAsJson.length; - - rawReferenceAsJson.forEach((reference, index) => { - this.generateReference(referenceSection, reference, index); - }); - } - references.appendChild(referenceExpandButton); - - referenceExpandButton.addEventListener('click', function() { - if (referenceSection.classList.contains("collapsed")) { - referenceSection.classList.remove("collapsed"); - referenceSection.classList.add("expanded"); - } else { - referenceSection.classList.add("collapsed"); - referenceSection.classList.remove("expanded"); - } - }); - - let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`; - referenceExpandButton.innerHTML = expandButtonText; - references.appendChild(referenceSection); - } else { - // Render incremental chat response - await this.renderIncrementalMessage(responseElement, responseText); - } - } - } - - async getChatResponse(query: string | undefined | null): Promise { - // Exit if query is empty - if (!query || query === "") return; - - // Render user query as chat message - let chatBodyEl = this.contentEl.getElementsByClassName("khoj-chat-body")[0]; - this.renderMessage(chatBodyEl, query, "you"); - - // Get chat response from Khoj backend - let encodedQuery = encodeURIComponent(query); - let chatUrl = `${this.setting.khojUrl}/api/chat?q=${encodedQuery}&n=${this.setting.resultsCount}&client=obsidian&stream=true®ion=${this.region}&city=${this.city}&country=${this.countryName}&timezone=${this.timezone}`; - let responseElement = this.createKhojResponseDiv(); - - // Temporary status message to indicate that Khoj is thinking - this.result = ""; - await this.renderIncrementalMessage(responseElement, "🤔"); - - let response = await fetch(chatUrl, { - method: "GET", - headers: { - "Content-Type": "text/event-stream", - "Authorization": `Bearer ${this.setting.khojApiKey}`, - }, - }) - - try { - if (response.body === null) { - throw new Error("Response body is null"); - } - - // Clear thinking status message - if (responseElement.innerHTML === "🤔") { - responseElement.innerHTML = ""; - } - - // Reset collated chat result to empty string - this.result = ""; - responseElement.innerHTML = ""; - if (response.headers.get("content-type") === "application/json") { - let responseText = "" - try { - const responseAsJson = await response.json() as ChatJsonResult; - if (responseAsJson.image) { - // If response has image field, response is a generated image. - if (responseAsJson.intentType === "text-to-image") { - responseText += `![${query}](data:image/png;base64,${responseAsJson.image})`; - } else if (responseAsJson.intentType === "text-to-image2") { - responseText += `![${query}](${responseAsJson.image})`; - } else if (responseAsJson.intentType === "text-to-image-v3") { - responseText += `![${query}](data:image/webp;base64,${responseAsJson.image})`; - } - const inferredQuery = responseAsJson.inferredQueries?.[0]; - if (inferredQuery) { - responseText += `\n\n**Inferred Query**:\n\n${inferredQuery}`; - } - } else if (responseAsJson.detail) { - responseText = responseAsJson.detail; - } - } catch (error) { - // If the chunk is not a JSON object, just display it as is - responseText = await response.text(); - } finally { - await this.renderIncrementalMessage(responseElement, responseText); - } - } else { - // Stream and render chat response - await this.readChatStream(response, responseElement); - } - } catch (err) { - console.log(`Khoj chat response failed with\n${err}`); - let errorMsg = "Sorry, unable to get response from Khoj backend ❤️‍🩹. Retry or contact developers for help at team@khoj.dev or on Discord"; - responseElement.innerHTML = errorMsg - } - } - - flashStatusInChatInput(message: string) { - // Get chat input element and original placeholder - let chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; - let originalPlaceholder = chatInput.placeholder; - // Set placeholder to message - chatInput.placeholder = message; - // Reset placeholder after 2 seconds - setTimeout(() => { - chatInput.placeholder = originalPlaceholder; - }, 2000); - } - - async clearConversationHistory() { - let chatBody = this.contentEl.getElementsByClassName("khoj-chat-body")[0]; - - let response = await request({ - url: `${this.setting.khojUrl}/api/chat/history?client=obsidian`, - method: "DELETE", - headers: { "Authorization": `Bearer ${this.setting.khojApiKey}` }, - }) - try { - let result = JSON.parse(response); - if (result.status !== "ok") { - // Throw error if conversation history isn't cleared - throw new Error("Failed to clear conversation history"); - } else { - let getChatHistoryStatus = await this.getChatHistory(chatBody); - // If conversation history is cleared successfully, clear chat logs from modal - if (getChatHistoryStatus) chatBody.innerHTML = ""; - let statusMsg = getChatHistoryStatus ? result.message : "Failed to clear conversation history"; - this.flashStatusInChatInput(statusMsg); - } - } catch (err) { - this.flashStatusInChatInput("Failed to clear conversation history"); - } - } - - sendMessageTimeout: NodeJS.Timeout | undefined; - mediaRecorder: MediaRecorder | undefined; - async speechToText(event: MouseEvent | TouchEvent) { - event.preventDefault(); - const transcribeButton = this.contentEl.getElementsByClassName("khoj-transcribe")[0]; - const chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; - const sendButton = this.modalEl.getElementsByClassName("khoj-chat-send")[0] - - const generateRequestBody = async (audioBlob: Blob, boundary_string: string) => { - const boundary = `------${boundary_string}`; - const chunks: ArrayBuffer[] = []; - - chunks.push(new TextEncoder().encode(`${boundary}\r\n`)); - chunks.push(new TextEncoder().encode(`Content-Disposition: form-data; name="file"; filename="blob"\r\nContent-Type: "application/octet-stream"\r\n\r\n`)); - chunks.push(await audioBlob.arrayBuffer()); - chunks.push(new TextEncoder().encode('\r\n')); - - await Promise.all(chunks); - chunks.push(new TextEncoder().encode(`${boundary}--\r\n`)); - return await new Blob(chunks).arrayBuffer(); - }; - - const sendToServer = async (audioBlob: Blob) => { - const boundary_string = `Boundary${Math.random().toString(36).slice(2)}`; - const requestBody = await generateRequestBody(audioBlob, boundary_string); - - const response = await requestUrl({ - url: `${this.setting.khojUrl}/api/transcribe?client=obsidian`, - method: 'POST', - headers: { "Authorization": `Bearer ${this.setting.khojApiKey}` }, - contentType: `multipart/form-data; boundary=----${boundary_string}`, - body: requestBody, - }); - - // Parse response from Khoj backend - if (response.status === 200) { - console.log(response); - chatInput.value += response.json.text.trimStart(); - this.autoResize(); - } else if (response.status === 501) { - throw new Error("⛔️ Configure speech-to-text model on server."); - } else if (response.status === 422) { - throw new Error("⛔️ Audio file to large to process."); - } else { - throw new Error("⛔️ Failed to transcribe audio."); - } - - // Don't auto-send empty messages - if (chatInput.value.length === 0) return; - - // Show stop auto-send button. It stops auto-send when clicked - setIcon(sendButton, "stop-circle"); - let stopSendButtonImg = sendButton.getElementsByClassName("lucide-stop-circle")[0] - stopSendButtonImg.addEventListener('click', (_) => { this.cancelSendMessage() }); - - // Start the countdown timer UI - stopSendButtonImg.getElementsByTagName("circle")[0].style.animation = "countdown 3s linear 1 forwards"; - - // Auto send message after 3 seconds - this.sendMessageTimeout = setTimeout(() => { - // Stop the countdown timer UI - setIcon(sendButton, "arrow-up-circle") - let sendImg = sendButton.getElementsByClassName("lucide-arrow-up-circle")[0] - sendImg.addEventListener('click', async (_) => { await this.chat() }); - - // Send message - this.chat(); - }, 3000); - }; - - const handleRecording = (stream: MediaStream) => { - const audioChunks: Blob[] = []; - const recordingConfig = { mimeType: 'audio/webm' }; - this.mediaRecorder = new MediaRecorder(stream, recordingConfig); - - this.mediaRecorder.addEventListener("dataavailable", function(event) { - if (event.data.size > 0) audioChunks.push(event.data); - }); - - this.mediaRecorder.addEventListener("stop", async function() { - const audioBlob = new Blob(audioChunks, { type: 'audio/webm' }); - await sendToServer(audioBlob); - }); - - this.mediaRecorder.start(); - setIcon(transcribeButton, "mic-off"); - }; - - // Toggle recording - if (!this.mediaRecorder || this.mediaRecorder.state === 'inactive' || event.type === 'touchstart') { - navigator.mediaDevices - .getUserMedia({ audio: true }) - ?.then(handleRecording) - .catch((e) => { - this.flashStatusInChatInput("⛔️ Failed to access microphone"); - }); - } else if (this.mediaRecorder.state === 'recording' || event.type === 'touchend' || event.type === 'touchcancel') { - this.mediaRecorder.stop(); - this.mediaRecorder.stream.getTracks().forEach(track => track.stop()); - this.mediaRecorder = undefined; - setIcon(transcribeButton, "mic"); - } - } - - cancelSendMessage() { - // Cancel the auto-send chat message timer if the stop-send-button is clicked - clearTimeout(this.sendMessageTimeout); - - // Revert to showing send-button and hide the stop-send-button - let sendButton = this.modalEl.getElementsByClassName("khoj-chat-send")[0]; - setIcon(sendButton, "arrow-up-circle"); - let sendImg = sendButton.getElementsByClassName("lucide-arrow-up-circle")[0] - sendImg.addEventListener('click', async (_) => { await this.chat() }); - }; - - incrementalChat(event: KeyboardEvent) { - if (!event.shiftKey && event.key === 'Enter') { - event.preventDefault(); - this.chat(); - } - } - - onChatInput() { - const chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; - chatInput.value = chatInput.value.trimStart(); - - this.autoResize(); - } - - autoResize() { - const chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; - const scrollTop = chatInput.scrollTop; - chatInput.style.height = '0'; - const scrollHeight = chatInput.scrollHeight + 8; // +8 accounts for padding - chatInput.style.height = Math.min(scrollHeight, 200) + 'px'; - chatInput.scrollTop = scrollTop; - this.scrollChatToBottom(); - } - - scrollChatToBottom() { - let sendButton = this.modalEl.getElementsByClassName("khoj-chat-send")[0]; - sendButton.scrollIntoView({ behavior: "auto", block: "center" }); - } -} diff --git a/src/interface/obsidian/src/chat_view.ts b/src/interface/obsidian/src/chat_view.ts new file mode 100644 index 00000000..f56fd7fb --- /dev/null +++ b/src/interface/obsidian/src/chat_view.ts @@ -0,0 +1,1146 @@ +import { MarkdownRenderer, WorkspaceLeaf, request, requestUrl, setIcon } from 'obsidian'; +import { KhojSetting } from 'src/settings'; +import { KhojPaneView } from 'src/pane_view'; +import { KhojView, createCopyParentText, getLinkToEntry, pasteTextAtCursor } from 'src/utils'; + +export interface ChatJsonResult { + image?: string; + detail?: string; + intentType?: string; + inferredQueries?: string[]; +} + + +interface Location { + region: string; + city: string; + countryName: string; + timezone: string; +} + +export class KhojChatView extends KhojPaneView { + result: string; + setting: KhojSetting; + waitingForLocation: boolean; + location: Location; + + constructor(leaf: WorkspaceLeaf, setting: KhojSetting) { + super(leaf, setting); + + this.waitingForLocation = true; + + fetch("https://ipapi.co/json") + .then(response => response.json()) + .then(data => { + this.location = { + region: data.region, + city: data.city, + countryName: data.country_name, + timezone: data.timezone, + }; + }) + .catch(err => { + console.log(err); + }) + .finally(() => { + this.waitingForLocation = false; + }); + + } + + getViewType(): string { + return KhojView.CHAT; + } + + getDisplayText(): string { + return "Khoj Chat"; + } + + getIcon(): string { + return "message-circle"; + } + + async chat() { + + // Get text in chat input element + let input_el = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; + + // Clear text after extracting message to send + let user_message = input_el.value.trim(); + input_el.value = ""; + this.autoResize(); + + // Get and render chat response to user message + await this.getChatResponse(user_message); + } + + async onOpen() { + let { contentEl } = this; + contentEl.addClass("khoj-chat"); + + super.onOpen(); + + // Create area for chat logs + let chatBodyEl = contentEl.createDiv({ attr: { id: "khoj-chat-body", class: "khoj-chat-body" } }); + + // Add chat input field + let inputRow = contentEl.createDiv("khoj-input-row"); + let chatSessions = inputRow.createEl("button", { + text: "Chat Sessions", + attr: { + class: "khoj-input-row-button clickable-icon", + }, + }) + chatSessions.addEventListener('click', async (_) => { await this.toggleChatSessions(chatBodyEl) }); + setIcon(chatSessions, "history"); + + let chatInput = inputRow.createEl("textarea", { + attr: { + id: "khoj-chat-input", + autofocus: "autofocus", + class: "khoj-chat-input option", + }, + }) + chatInput.addEventListener('input', (_) => { this.onChatInput() }); + chatInput.addEventListener('keydown', (event) => { this.incrementalChat(event) }); + + let transcribe = inputRow.createEl("button", { + text: "Transcribe", + attr: { + id: "khoj-transcribe", + class: "khoj-transcribe khoj-input-row-button clickable-icon ", + }, + }) + transcribe.addEventListener('mousedown', async (event) => { await this.speechToText(event) }); + transcribe.addEventListener('touchstart', async (event) => { await this.speechToText(event) }); + transcribe.addEventListener('touchend', async (event) => { await this.speechToText(event) }); + transcribe.addEventListener('touchcancel', async (event) => { await this.speechToText(event) }); + setIcon(transcribe, "mic"); + + let send = inputRow.createEl("button", { + text: "Send", + attr: { + id: "khoj-chat-send", + class: "khoj-chat-send khoj-input-row-button clickable-icon", + }, + }) + setIcon(send, "arrow-up-circle"); + let sendImg = send.getElementsByClassName("lucide-arrow-up-circle")[0] + sendImg.addEventListener('click', async (_) => { await this.chat() }); + + // Get chat history from Khoj backend and set chat input state + let getChatHistorySucessfully = await this.getChatHistory(chatBodyEl); + let placeholderText = getChatHistorySucessfully ? "Message" : "Configure Khoj to enable chat"; + chatInput.placeholder = placeholderText; + chatInput.disabled = !getChatHistorySucessfully; + + // Scroll to bottom of chat messages and focus on chat input field, once messages rendered + requestAnimationFrame(() => { + // Ensure layout and paint have occurred + requestAnimationFrame(() => { + this.scrollChatToBottom(); + const chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; + chatInput?.focus(); + }); + }); + } + + processOnlineReferences(referenceSection: HTMLElement, onlineContext: any) { + let numOnlineReferences = 0; + for (let subquery in onlineContext) { + let onlineReference = onlineContext[subquery]; + if (onlineReference.organic && onlineReference.organic.length > 0) { + numOnlineReferences += onlineReference.organic.length; + for (let key in onlineReference.organic) { + let reference = onlineReference.organic[key]; + let polishedReference = this.generateOnlineReference(referenceSection, reference, key); + referenceSection.appendChild(polishedReference); + } + } + + if (onlineReference.knowledgeGraph && onlineReference.knowledgeGraph.length > 0) { + numOnlineReferences += onlineReference.knowledgeGraph.length; + for (let key in onlineReference.knowledgeGraph) { + let reference = onlineReference.knowledgeGraph[key]; + let polishedReference = this.generateOnlineReference(referenceSection, reference, key); + referenceSection.appendChild(polishedReference); + } + } + + if (onlineReference.peopleAlsoAsk && onlineReference.peopleAlsoAsk.length > 0) { + numOnlineReferences += onlineReference.peopleAlsoAsk.length; + for (let key in onlineReference.peopleAlsoAsk) { + let reference = onlineReference.peopleAlsoAsk[key]; + let polishedReference = this.generateOnlineReference(referenceSection, reference, key); + referenceSection.appendChild(polishedReference); + } + } + + if (onlineReference.webpages && onlineReference.webpages.length > 0) { + numOnlineReferences += onlineReference.webpages.length; + for (let key in onlineReference.webpages) { + let reference = onlineReference.webpages[key]; + let polishedReference = this.generateOnlineReference(referenceSection, reference, key); + referenceSection.appendChild(polishedReference); + } + } + } + + return numOnlineReferences; + } + + generateOnlineReference(messageEl: Element, reference: any, index: string) { + // Generate HTML for Chat Reference + let title = reference.title || reference.link; + let link = reference.link; + let snippet = reference.snippet; + let question = reference.question ? `Question: ${reference.question}

` : ""; + + let referenceButton = messageEl.createEl('button'); + let linkElement = referenceButton.createEl('a'); + linkElement.setAttribute('href', link); + linkElement.setAttribute('target', '_blank'); + linkElement.setAttribute('rel', 'noopener noreferrer'); + linkElement.classList.add("reference-link"); + linkElement.setAttribute('title', title); + linkElement.textContent = title; + + referenceButton.id = `ref-${index}`; + referenceButton.classList.add("reference-button"); + referenceButton.classList.add("collapsed"); + referenceButton.tabIndex = 0; + + // Add event listener to toggle full reference on click + referenceButton.addEventListener('click', function() { + if (this.classList.contains("collapsed")) { + this.classList.remove("collapsed"); + this.classList.add("expanded"); + this.innerHTML = linkElement.outerHTML + `

${question + snippet}`; + } else { + this.classList.add("collapsed"); + this.classList.remove("expanded"); + this.innerHTML = linkElement.outerHTML; + } + }); + + return referenceButton; + } + + generateReference(messageEl: Element, referenceJson: any, index: number) { + let reference: string = referenceJson.hasOwnProperty("compiled") ? referenceJson.compiled : referenceJson; + let referenceFile = referenceJson.hasOwnProperty("file") ? referenceJson.file : null; + + // Get all markdown and PDF files in vault + const mdFiles = this.app.vault.getMarkdownFiles(); + const pdfFiles = this.app.vault.getFiles().filter(file => file.extension === 'pdf'); + + // Escape reference for HTML rendering + reference = reference.split('\n').slice(1).join('\n'); + let escaped_ref = reference.replace(/"/g, """) + + // Generate HTML for Chat Reference + let referenceButton = messageEl.createEl('button'); + + if (referenceFile) { + // Find vault file associated with current reference + let linkToEntry = getLinkToEntry(mdFiles.concat(pdfFiles), referenceFile, reference); + + let linkElement: Element; + linkElement = referenceButton.createEl('span'); + linkElement.setAttribute('title', escaped_ref); + linkElement.textContent = referenceFile; + if (linkToEntry && linkToEntry) { + linkElement.classList.add("reference-link"); + linkElement.addEventListener('click', (event) => { + event.stopPropagation(); + this.app.workspace.openLinkText(linkToEntry, ''); + }); + } + } + + let referenceText = referenceButton.createDiv(); + referenceText.textContent = escaped_ref; + + referenceButton.id = `ref-${index}`; + referenceButton.classList.add("reference-button"); + referenceButton.classList.add("collapsed"); + referenceButton.tabIndex = 0; + + // Add event listener to toggle full reference on click + referenceButton.addEventListener('click', function() { + if (this.classList.contains("collapsed")) { + this.classList.remove("collapsed"); + this.classList.add("expanded"); + } else { + this.classList.add("collapsed"); + this.classList.remove("expanded"); + } + }); + + return referenceButton; + } + + formatHTMLMessage(message: string, raw = false, willReplace = true) { + let rendered_msg = message; + + // Replace LaTeX delimiters with placeholders + rendered_msg = rendered_msg.replace(/\\\(/g, 'LEFTPAREN').replace(/\\\)/g, 'RIGHTPAREN') + .replace(/\\\[/g, 'LEFTBRACKET').replace(/\\\]/g, 'RIGHTBRACKET'); + + // Remove any text between [INST] and tags. These are spurious instructions for the AI chat model. + rendered_msg = rendered_msg.replace(/\[INST\].+(<\/s>)?/g, ''); + + // Render markdow to HTML DOM element + let chat_message_body_text_el = this.contentEl.createDiv(); + chat_message_body_text_el.className = "chat-message-text-response"; + MarkdownRenderer.renderMarkdown(message, chat_message_body_text_el, '', null); + + // Replace placeholders with LaTeX delimiters + rendered_msg = chat_message_body_text_el.innerHTML; + chat_message_body_text_el.innerHTML = rendered_msg.replace(/LEFTPAREN/g, '\\(').replace(/RIGHTPAREN/g, '\\)') + .replace(/LEFTBRACKET/g, '\\[').replace(/RIGHTBRACKET/g, '\\]'); + + // Add a copy button to each chat message, if it doesn't already exist + if (willReplace === true) { + this.renderActionButtons(message, chat_message_body_text_el); + } + + return chat_message_body_text_el; + } + + renderMessageWithReferences( + chatEl: Element, + message: string, + sender: string, + context?: string[], + onlineContext?: object, + dt?: Date, + intentType?: string, + inferredQueries?: string[], + ) { + if (!message) return; + + let chatMessageEl; + if (intentType?.includes("text-to-image")) { + let imageMarkdown = this.generateImageMarkdown(message, intentType, inferredQueries); + chatMessageEl = this.renderMessage(chatEl, imageMarkdown, sender, dt); + } else { + chatMessageEl = this.renderMessage(chatEl, message, sender, dt); + } + + // If no document or online context is provided, skip rendering the reference section + if ((context == null || context.length == 0) + && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) { + return; + } + + // If document or online context is provided, render the message with its references + let references: any = {}; + if (!!context) references["notes"] = context; + if (!!onlineContext) references["online"] = onlineContext; + let chatMessageBodyEl = chatMessageEl.getElementsByClassName("khoj-chat-message-text")[0]; + chatMessageBodyEl.appendChild(this.createReferenceSection(references)); + } + + generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[]) { + let imageMarkdown = ""; + if (intentType === "text-to-image") { + imageMarkdown = `![](data:image/png;base64,${message})`; + } else if (intentType === "text-to-image2") { + imageMarkdown = `![](${message})`; + } else if (intentType === "text-to-image-v3") { + imageMarkdown = `![](data:image/webp;base64,${message})`; + } + if (inferredQueries) { + imageMarkdown += "\n\n**Inferred Query**:"; + for (let inferredQuery of inferredQueries) { + imageMarkdown += `\n\n${inferredQuery}`; + } + } + return imageMarkdown; + } + + renderMessage(chatBodyEl: Element, message: string, sender: string, dt?: Date, raw: boolean = false, willReplace: boolean = true): Element { + let message_time = this.formatDate(dt ?? new Date()); + let emojified_sender = sender == "khoj" ? "🏮 Khoj" : "🤔 You"; + + // Append message to conversation history HTML element. + // The chat logs should display above the message input box to follow standard UI semantics + let chatMessageEl = chatBodyEl.createDiv({ + attr: { + "data-meta": `${emojified_sender} at ${message_time}`, + class: `khoj-chat-message ${sender}` + }, + }) + let chat_message_body_el = chatMessageEl.createDiv(); + chat_message_body_el.addClasses(["khoj-chat-message-text", sender]); + let chat_message_body_text_el = chat_message_body_el.createDiv(); + if (raw) { + chat_message_body_text_el.innerHTML = message; + } else { + // @ts-ignore + MarkdownRenderer.renderMarkdown(message, chat_message_body_text_el, '', null); + } + + // Add action buttons to each chat message element + if (willReplace === true) { + this.renderActionButtons(message, chat_message_body_text_el); + } + + // Remove user-select: none property to make text selectable + chatMessageEl.style.userSelect = "text"; + + // Scroll to bottom after inserting chat messages + this.scrollChatToBottom(); + + return chatMessageEl; + } + + createKhojResponseDiv(dt?: Date): HTMLDivElement { + let message_time = this.formatDate(dt ?? new Date()); + + // Append message to conversation history HTML element. + // The chat logs should display above the message input box to follow standard UI semantics + let chat_body_el = this.contentEl.getElementsByClassName("khoj-chat-body")[0]; + let chat_message_el = chat_body_el.createDiv({ + attr: { + "data-meta": `🏮 Khoj at ${message_time}`, + class: `khoj-chat-message khoj` + }, + }).createDiv({ + attr: { + class: `khoj-chat-message-text khoj` + }, + }).createDiv(); + + // Scroll to bottom after inserting chat messages + this.scrollChatToBottom(); + + return chat_message_el; + } + + async renderIncrementalMessage(htmlElement: HTMLDivElement, additionalMessage: string) { + this.result += additionalMessage; + htmlElement.innerHTML = ""; + // @ts-ignore + await MarkdownRenderer.renderMarkdown(this.result, htmlElement, '', null); + // Render action buttons for the message + this.renderActionButtons(this.result, htmlElement); + // Scroll to bottom of modal, till the send message input box + this.scrollChatToBottom(); + } + + renderActionButtons(message: string, chat_message_body_text_el: HTMLElement) { + let copyButton = this.contentEl.createEl('button'); + copyButton.classList.add("copy-button"); + copyButton.title = "Copy Message to Clipboard"; + setIcon(copyButton, "copy-plus"); + copyButton.addEventListener('click', createCopyParentText(message)); + chat_message_body_text_el.append(copyButton); + + // Add button to paste into current buffer + let pasteToFile = this.contentEl.createEl('button'); + pasteToFile.classList.add("copy-button"); + pasteToFile.title = "Paste Message to File"; + setIcon(pasteToFile, "clipboard-paste"); + pasteToFile.addEventListener('click', (event) => { pasteTextAtCursor(createCopyParentText(message, 'clipboard-paste')(event)); }); + chat_message_body_text_el.append(pasteToFile); + } + + formatDate(date: Date): string { + // Format date in HH:MM, DD MMM YYYY format + let time_string = date.toLocaleTimeString('en-IN', { hour: '2-digit', minute: '2-digit', hour12: false }); + let date_string = date.toLocaleString('en-IN', { year: 'numeric', month: 'short', day: '2-digit' }).replace(/-/g, ' '); + return `${time_string}, ${date_string}`; + } + + createNewConversation(chatBodyEl: HTMLElement) { + chatBodyEl.innerHTML = ""; + chatBodyEl.dataset.conversationId = ""; + chatBodyEl.dataset.conversationTitle = ""; + this.renderMessage(chatBodyEl, "Hey 👋🏾, what's up?", "khoj"); + } + + async toggleChatSessions(chatBodyEl: HTMLElement, forceShow: boolean = false): Promise { + if (!forceShow && this.contentEl.getElementsByClassName("side-panel")?.length > 0) { + chatBodyEl.innerHTML = ""; + return this.getChatHistory(chatBodyEl); + } + chatBodyEl.innerHTML = ""; + const sidePanelEl = this.contentEl.createDiv("side-panel"); + const newConversationEl = sidePanelEl.createDiv("new-conversation"); + const conversationHeaderTitleEl = newConversationEl.createDiv("conversation-header-title"); + conversationHeaderTitleEl.textContent = "Conversations"; + + const newConversationButtonEl = newConversationEl.createEl("button"); + newConversationButtonEl.classList.add("new-conversation-button"); + newConversationButtonEl.classList.add("side-panel-button"); + newConversationButtonEl.addEventListener('click', (_) => this.createNewConversation(chatBodyEl)); + setIcon(newConversationButtonEl, "plus"); + newConversationButtonEl.innerHTML += "New"; + + const existingConversationsEl = sidePanelEl.createDiv("existing-conversations"); + const conversationListEl = existingConversationsEl.createDiv("conversation-list"); + const conversationListBodyHeaderEl = conversationListEl.createDiv("conversation-list-header"); + const conversationListBodyEl = conversationListEl.createDiv("conversation-list-body"); + + const chatSessionsUrl = `${this.setting.khojUrl}/api/chat/sessions?client=obsidian`; + const headers = { 'Authorization': `Bearer ${this.setting.khojApiKey}` }; + try { + let response = await fetch(chatSessionsUrl, { method: "GET", headers: headers }); + let responseJson: any = await response.json(); + let conversationId = chatBodyEl.dataset.conversationId; + + if (responseJson.length > 0) { + conversationListBodyHeaderEl.style.display = "block"; + for (let key in responseJson) { + let conversation = responseJson[key]; + let conversationSessionEl = this.contentEl.createEl('div'); + let incomingConversationId = conversation["conversation_id"]; + conversationSessionEl.classList.add("conversation-session"); + if (incomingConversationId == conversationId) { + conversationSessionEl.classList.add("selected-conversation"); + } + const conversationTitle = conversation["slug"] || `New conversation 🌱`; + const conversationSessionTitleEl = conversationSessionEl.createDiv("conversation-session-title"); + conversationSessionTitleEl.textContent = conversationTitle; + conversationSessionTitleEl.addEventListener('click', () => { + chatBodyEl.innerHTML = ""; + chatBodyEl.dataset.conversationId = incomingConversationId; + chatBodyEl.dataset.conversationTitle = conversationTitle; + this.getChatHistory(chatBodyEl); + }); + + let conversationMenuEl = this.contentEl.createEl('div'); + conversationMenuEl = this.addConversationMenu( + conversationMenuEl, + conversationSessionEl, + conversationTitle, + conversationSessionTitleEl, + chatBodyEl, + incomingConversationId, + incomingConversationId == conversationId, + ); + + conversationSessionEl.appendChild(conversationMenuEl); + conversationListBodyEl.appendChild(conversationSessionEl); + chatBodyEl.appendChild(sidePanelEl); + } + } + } catch (err) { + return false; + } + return true; + } + + addConversationMenu( + conversationMenuEl: HTMLDivElement, + conversationSessionEl: HTMLElement, + conversationTitle: string, + conversationSessionTitleEl: HTMLElement, + chatBodyEl: HTMLElement, + incomingConversationId: string, + selectedConversation: boolean, + ) { + conversationMenuEl.classList.add("conversation-menu"); + + const headers = { 'Authorization': `Bearer ${this.setting.khojApiKey}` }; + + let editConversationTitleButtonEl = this.contentEl.createEl('button'); + setIcon(editConversationTitleButtonEl, "edit"); + editConversationTitleButtonEl.title = "Rename"; + editConversationTitleButtonEl.classList.add("edit-title-button"); + editConversationTitleButtonEl.classList.add("three-dot-menu-button-item"); + if (selectedConversation) editConversationTitleButtonEl.classList.add("selected-conversation"); + editConversationTitleButtonEl.addEventListener('click', (event) => { + event.stopPropagation(); + + let conversationMenuChildren = conversationMenuEl.children; + let totalItems = conversationMenuChildren.length; + + for (let i = totalItems - 1; i >= 0; i--) { + conversationMenuChildren[i].remove(); + } + + // Create a dialog box to get new title for conversation + let editConversationTitleInputEl = this.contentEl.createEl('input'); + editConversationTitleInputEl.classList.add("conversation-title-input"); + editConversationTitleInputEl.value = conversationTitle; + editConversationTitleInputEl.addEventListener('click', function(event) { + event.stopPropagation(); + }); + editConversationTitleInputEl.addEventListener('keydown', function(event) { + if (event.key === "Enter") { + event.preventDefault(); + editConversationTitleSaveButtonEl.click(); + } + }); + let editConversationTitleSaveButtonEl = this.contentEl.createEl('button'); + conversationSessionTitleEl.replaceWith(editConversationTitleInputEl); + editConversationTitleSaveButtonEl.innerHTML = "Save"; + editConversationTitleSaveButtonEl.classList.add("three-dot-menu-button-item"); + if (selectedConversation) editConversationTitleSaveButtonEl.classList.add("selected-conversation"); + editConversationTitleSaveButtonEl.addEventListener('click', (event) => { + event.stopPropagation(); + let newTitle = editConversationTitleInputEl.value; + if (newTitle != null) { + let editURL = `/api/chat/title?client=web&conversation_id=${incomingConversationId}&title=${newTitle}`; + fetch(`${this.setting.khojUrl}${editURL}`, { method: "PATCH", headers }) + .then(response => response.ok ? response.json() : Promise.reject(response)) + .then(data => { + conversationSessionTitleEl.textContent = newTitle; + }) + .catch(err => { + return; + }); + const conversationSessionTitleEl = conversationSessionEl.createDiv("conversation-session-title"); + conversationSessionTitleEl.textContent = newTitle; + conversationSessionTitleEl.addEventListener('click', () => { + chatBodyEl.innerHTML = ""; + chatBodyEl.dataset.conversationId = incomingConversationId; + chatBodyEl.dataset.conversationTitle = conversationTitle; + this.getChatHistory(chatBodyEl); + }); + + let newConversationMenuEl = this.contentEl.createEl('div'); + newConversationMenuEl = this.addConversationMenu( + newConversationMenuEl, + conversationSessionEl, + newTitle, + conversationSessionTitleEl, + chatBodyEl, + incomingConversationId, + selectedConversation, + ); + + conversationMenuEl.replaceWith(newConversationMenuEl); + editConversationTitleInputEl.replaceWith(conversationSessionTitleEl); + } + }); + conversationMenuEl.appendChild(editConversationTitleSaveButtonEl); + }); + + conversationMenuEl.appendChild(editConversationTitleButtonEl); + + let deleteConversationButtonEl = this.contentEl.createEl('button'); + setIcon(deleteConversationButtonEl, "trash"); + deleteConversationButtonEl.title = "Delete"; + deleteConversationButtonEl.classList.add("delete-conversation-button"); + deleteConversationButtonEl.classList.add("three-dot-menu-button-item"); + if (selectedConversation) deleteConversationButtonEl.classList.add("selected-conversation"); + deleteConversationButtonEl.addEventListener('click', () => { + // Ask for confirmation before deleting chat session + let confirmation = confirm('Are you sure you want to delete this chat session?'); + if (!confirmation) return; + let deleteURL = `/api/chat/history?client=obsidian&conversation_id=${incomingConversationId}`; + fetch(`${this.setting.khojUrl}${deleteURL}`, { method: "DELETE", headers }) + .then(response => response.ok ? response.json() : Promise.reject(response)) + .then(data => { + chatBodyEl.innerHTML = ""; + chatBodyEl.dataset.conversationId = ""; + chatBodyEl.dataset.conversationTitle = ""; + this.toggleChatSessions(chatBodyEl, true); + }) + .catch(err => { + return; + }); + }); + + conversationMenuEl.appendChild(deleteConversationButtonEl); + return conversationMenuEl; + } + + async getChatHistory(chatBodyEl: HTMLElement): Promise { + // Get chat history from Khoj backend + let chatUrl = `${this.setting.khojUrl}/api/chat/history?client=obsidian`; + if (chatBodyEl.dataset.conversationId) { + chatUrl += `&conversation_id=${chatBodyEl.dataset.conversationId}`; + } + + try { + let response = await fetch(chatUrl, { + method: "GET", + headers: { "Authorization": `Bearer ${this.setting.khojApiKey}` }, + }); + + let responseJson: any = await response.json(); + chatBodyEl.dataset.conversationId = responseJson.conversation_id; + + if (responseJson.detail) { + // If the server returns error details in response, render a setup hint. + let setupMsg = "Hi 👋🏾, to start chatting add available chat models options via [the Django Admin panel](/server/admin) on the Server"; + this.renderMessage(chatBodyEl, setupMsg, "khoj", undefined); + + return false; + } else if (responseJson.response) { + // Render conversation history, if any + chatBodyEl.dataset.conversationId = responseJson.response.conversation_id; + chatBodyEl.dataset.conversationTitle = responseJson.response.slug || `New conversation 🌱`; + + + let chatLogs = responseJson.response?.conversation_id ? responseJson.response.chat ?? [] : responseJson.response; + chatLogs.forEach((chatLog: any) => { + this.renderMessageWithReferences( + chatBodyEl, + chatLog.message, + chatLog.by, + chatLog.context, + chatLog.onlineContext, + new Date(chatLog.created), + chatLog.intent?.type, + chatLog.intent?.["inferred-queries"], + ); + }); + } + } catch (err) { + let errorMsg = "Unable to get response from Khoj server ❤️‍🩹. Ensure server is running or contact developers for help at [team@khoj.dev](mailto:team@khoj.dev) or in [Discord](https://discord.gg/BDgyabRM6e)"; + this.renderMessage(chatBodyEl, errorMsg, "khoj", undefined); + return false; + } + return true; + } + + async readChatStream(response: Response, responseElement: HTMLDivElement): Promise { + // Exit if response body is empty + if (response.body == null) return; + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + + while (true) { + const { value, done } = await reader.read(); + + // Break if the stream is done + if (done) break; + + let responseText = decoder.decode(value); + if (responseText.includes("### compiled references:")) { + // Render any references used to generate the response + const [additionalResponse, rawReference] = responseText.split("### compiled references:", 2); + await this.renderIncrementalMessage(responseElement, additionalResponse); + + const rawReferenceAsJson = JSON.parse(rawReference); + let references = this.extractReferences(rawReferenceAsJson); + responseElement.appendChild(this.createReferenceSection(references)); + } else { + // Render incremental chat response + await this.renderIncrementalMessage(responseElement, responseText); + } + } + } + + async getChatResponse(query: string | undefined | null): Promise { + // Exit if query is empty + if (!query || query === "") return; + + // Render user query as chat message + let chatBodyEl = this.contentEl.getElementsByClassName("khoj-chat-body")[0] as HTMLElement; + this.renderMessage(chatBodyEl, query, "you"); + + let conversationID = chatBodyEl.dataset.conversationId; + if (!conversationID) { + let chatUrl = `${this.setting.khojUrl}/api/chat/sessions?client=obsidian`; + let response = await fetch(chatUrl, { + method: "POST", + headers: { "Authorization": `Bearer ${this.setting.khojApiKey}` }, + }); + let data = await response.json(); + conversationID = data.conversation_id; + chatBodyEl.dataset.conversationId = conversationID; + } + + // Get chat response from Khoj backend + let encodedQuery = encodeURIComponent(query); + let chatUrl = `${this.setting.khojUrl}/api/chat?q=${encodedQuery}&n=${this.setting.resultsCount}&client=obsidian&stream=true®ion=${this.location.region}&city=${this.location.city}&country=${this.location.countryName}&timezone=${this.location.timezone}`; + let responseElement = this.createKhojResponseDiv(); + + // Temporary status message to indicate that Khoj is thinking + this.result = ""; + let loadingEllipsis = this.createLoadingEllipse(); + responseElement.appendChild(loadingEllipsis); + + let response = await fetch(chatUrl, { + method: "GET", + headers: { + "Content-Type": "text/event-stream", + "Authorization": `Bearer ${this.setting.khojApiKey}`, + }, + }) + + try { + if (response.body === null) { + throw new Error("Response body is null"); + } + + // Clear loading status message + if (responseElement.getElementsByClassName("lds-ellipsis").length > 0 && loadingEllipsis) { + responseElement.removeChild(loadingEllipsis); + } + + // Reset collated chat result to empty string + this.result = ""; + responseElement.innerHTML = ""; + if (response.headers.get("content-type") === "application/json") { + let responseText = "" + try { + const responseAsJson = await response.json() as ChatJsonResult; + if (responseAsJson.image) { + // If response has image field, response is a generated image. + if (responseAsJson.intentType === "text-to-image") { + responseText += `![${query}](data:image/png;base64,${responseAsJson.image})`; + } else if (responseAsJson.intentType === "text-to-image2") { + responseText += `![${query}](${responseAsJson.image})`; + } else if (responseAsJson.intentType === "text-to-image-v3") { + responseText += `![${query}](data:image/webp;base64,${responseAsJson.image})`; + } + const inferredQuery = responseAsJson.inferredQueries?.[0]; + if (inferredQuery) { + responseText += `\n\n**Inferred Query**:\n\n${inferredQuery}`; + } + } else if (responseAsJson.detail) { + responseText = responseAsJson.detail; + } + } catch (error) { + // If the chunk is not a JSON object, just display it as is + responseText = await response.text(); + } finally { + await this.renderIncrementalMessage(responseElement, responseText); + } + } else { + // Stream and render chat response + await this.readChatStream(response, responseElement); + } + } catch (err) { + console.log(`Khoj chat response failed with\n${err}`); + let errorMsg = "Sorry, unable to get response from Khoj backend ❤️‍🩹. Retry or contact developers for help at team@khoj.dev or on Discord"; + responseElement.innerHTML = errorMsg + } + } + + flashStatusInChatInput(message: string) { + // Get chat input element and original placeholder + let chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; + let originalPlaceholder = chatInput.placeholder; + // Set placeholder to message + chatInput.placeholder = message; + // Reset placeholder after 2 seconds + setTimeout(() => { + chatInput.placeholder = originalPlaceholder; + }, 2000); + } + + async clearConversationHistory() { + let chatBody = this.contentEl.getElementsByClassName("khoj-chat-body")[0] as HTMLElement; + + let response = await request({ + url: `${this.setting.khojUrl}/api/chat/history?client=obsidian`, + method: "DELETE", + headers: { "Authorization": `Bearer ${this.setting.khojApiKey}` }, + }) + try { + let result = JSON.parse(response); + if (result.status !== "ok") { + // Throw error if conversation history isn't cleared + throw new Error("Failed to clear conversation history"); + } else { + let getChatHistoryStatus = await this.getChatHistory(chatBody); + // If conversation history is cleared successfully, clear chat logs from modal + if (getChatHistoryStatus) chatBody.innerHTML = ""; + let statusMsg = getChatHistoryStatus ? result.message : "Failed to clear conversation history"; + this.flashStatusInChatInput(statusMsg); + } + } catch (err) { + this.flashStatusInChatInput("Failed to clear conversation history"); + } + } + + sendMessageTimeout: NodeJS.Timeout | undefined; + mediaRecorder: MediaRecorder | undefined; + async speechToText(event: MouseEvent | TouchEvent) { + event.preventDefault(); + const transcribeButton = this.contentEl.getElementsByClassName("khoj-transcribe")[0]; + const chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; + const sendButton = this.contentEl.getElementsByClassName("khoj-chat-send")[0] + + const generateRequestBody = async (audioBlob: Blob, boundary_string: string) => { + const boundary = `------${boundary_string}`; + const chunks: ArrayBuffer[] = []; + + chunks.push(new TextEncoder().encode(`${boundary}\r\n`)); + chunks.push(new TextEncoder().encode(`Content-Disposition: form-data; name="file"; filename="blob"\r\nContent-Type: "application/octet-stream"\r\n\r\n`)); + chunks.push(await audioBlob.arrayBuffer()); + chunks.push(new TextEncoder().encode('\r\n')); + + await Promise.all(chunks); + chunks.push(new TextEncoder().encode(`${boundary}--\r\n`)); + return await new Blob(chunks).arrayBuffer(); + }; + + const sendToServer = async (audioBlob: Blob) => { + const boundary_string = `Boundary${Math.random().toString(36).slice(2)}`; + const requestBody = await generateRequestBody(audioBlob, boundary_string); + + const response = await requestUrl({ + url: `${this.setting.khojUrl}/api/transcribe?client=obsidian`, + method: 'POST', + headers: { "Authorization": `Bearer ${this.setting.khojApiKey}` }, + contentType: `multipart/form-data; boundary=----${boundary_string}`, + body: requestBody, + }); + + // Parse response from Khoj backend + if (response.status === 200) { + console.log(response); + chatInput.value += response.json.text.trimStart(); + this.autoResize(); + } else if (response.status === 501) { + throw new Error("⛔️ Configure speech-to-text model on server."); + } else if (response.status === 422) { + throw new Error("⛔️ Audio file to large to process."); + } else { + throw new Error("⛔️ Failed to transcribe audio."); + } + + // Don't auto-send empty messages + if (chatInput.value.length === 0) return; + + // Show stop auto-send button. It stops auto-send when clicked + setIcon(sendButton, "stop-circle"); + let stopSendButtonImg = sendButton.getElementsByClassName("lucide-stop-circle")[0] + stopSendButtonImg.addEventListener('click', (_) => { this.cancelSendMessage() }); + + // Start the countdown timer UI + stopSendButtonImg.getElementsByTagName("circle")[0].style.animation = "countdown 3s linear 1 forwards"; + + // Auto send message after 3 seconds + this.sendMessageTimeout = setTimeout(() => { + // Stop the countdown timer UI + setIcon(sendButton, "arrow-up-circle") + let sendImg = sendButton.getElementsByClassName("lucide-arrow-up-circle")[0] + sendImg.addEventListener('click', async (_) => { await this.chat() }); + + // Send message + this.chat(); + }, 3000); + }; + + const handleRecording = (stream: MediaStream) => { + const audioChunks: Blob[] = []; + const recordingConfig = { mimeType: 'audio/webm' }; + this.mediaRecorder = new MediaRecorder(stream, recordingConfig); + + this.mediaRecorder.addEventListener("dataavailable", function(event) { + if (event.data.size > 0) audioChunks.push(event.data); + }); + + this.mediaRecorder.addEventListener("stop", async function() { + const audioBlob = new Blob(audioChunks, { type: 'audio/webm' }); + await sendToServer(audioBlob); + }); + + this.mediaRecorder.start(); + setIcon(transcribeButton, "mic-off"); + }; + + // Toggle recording + if (!this.mediaRecorder || this.mediaRecorder.state === 'inactive' || event.type === 'touchstart') { + navigator.mediaDevices + .getUserMedia({ audio: true }) + ?.then(handleRecording) + .catch((e) => { + this.flashStatusInChatInput("⛔️ Failed to access microphone"); + }); + } else if (this.mediaRecorder.state === 'recording' || event.type === 'touchend' || event.type === 'touchcancel') { + this.mediaRecorder.stop(); + this.mediaRecorder.stream.getTracks().forEach(track => track.stop()); + this.mediaRecorder = undefined; + setIcon(transcribeButton, "mic"); + } + } + + cancelSendMessage() { + // Cancel the auto-send chat message timer if the stop-send-button is clicked + clearTimeout(this.sendMessageTimeout); + + // Revert to showing send-button and hide the stop-send-button + let sendButton = this.contentEl.getElementsByClassName("khoj-chat-send")[0]; + setIcon(sendButton, "arrow-up-circle"); + let sendImg = sendButton.getElementsByClassName("lucide-arrow-up-circle")[0] + sendImg.addEventListener('click', async (_) => { await this.chat() }); + }; + + incrementalChat(event: KeyboardEvent) { + if (!event.shiftKey && event.key === 'Enter') { + event.preventDefault(); + this.chat(); + } + } + + onChatInput() { + const chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; + chatInput.value = chatInput.value.trimStart(); + + this.autoResize(); + } + + autoResize() { + const chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; + const scrollTop = chatInput.scrollTop; + chatInput.style.height = '0'; + const scrollHeight = chatInput.scrollHeight + 8; // +8 accounts for padding + chatInput.style.height = Math.min(scrollHeight, 200) + 'px'; + chatInput.scrollTop = scrollTop; + this.scrollChatToBottom(); + } + + scrollChatToBottom() { + const chat_body_el = this.contentEl.getElementsByClassName("khoj-chat-body")[0]; + if (!!chat_body_el) chat_body_el.scrollTop = chat_body_el.scrollHeight; + } + + createLoadingEllipse() { + // Temporary status message to indicate that Khoj is thinking + let loadingEllipsis = this.contentEl.createEl("div"); + loadingEllipsis.classList.add("lds-ellipsis"); + + let firstEllipsis = this.contentEl.createEl("div"); + firstEllipsis.classList.add("lds-ellipsis-item"); + + let secondEllipsis = this.contentEl.createEl("div"); + secondEllipsis.classList.add("lds-ellipsis-item"); + + let thirdEllipsis = this.contentEl.createEl("div"); + thirdEllipsis.classList.add("lds-ellipsis-item"); + + let fourthEllipsis = this.contentEl.createEl("div"); + fourthEllipsis.classList.add("lds-ellipsis-item"); + + loadingEllipsis.appendChild(firstEllipsis); + loadingEllipsis.appendChild(secondEllipsis); + loadingEllipsis.appendChild(thirdEllipsis); + loadingEllipsis.appendChild(fourthEllipsis); + + return loadingEllipsis; + } + + handleStreamResponse(newResponseElement: HTMLElement | null, rawResponse: string, loadingEllipsis: HTMLElement | null, replace = true) { + if (!newResponseElement) return; + if (newResponseElement.getElementsByClassName("lds-ellipsis").length > 0 && loadingEllipsis) { + newResponseElement.removeChild(loadingEllipsis); + } + if (replace) { + newResponseElement.innerHTML = ""; + } + newResponseElement.appendChild(this.formatHTMLMessage(rawResponse, false, replace)); + this.scrollChatToBottom(); + } + + handleCompiledReferences(rawResponseElement: HTMLElement | null, chunk: string, references: any, rawResponse: string) { + if (!rawResponseElement || !chunk) return { rawResponse, references }; + + const [additionalResponse, rawReference] = chunk.split("### compiled references:", 2); + rawResponse += additionalResponse; + rawResponseElement.innerHTML = ""; + rawResponseElement.appendChild(this.formatHTMLMessage(rawResponse)); + + const rawReferenceAsJson = JSON.parse(rawReference); + references = this.extractReferences(rawReferenceAsJson); + + return { rawResponse, references }; + } + + handleImageResponse(imageJson: any, rawResponse: string) { + if (imageJson.image) { + const inferredQuery = imageJson.inferredQueries?.[0] ?? "generated image"; + + // If response has image field, response is a generated image. + if (imageJson.intentType === "text-to-image") { + rawResponse += `![generated_image](data:image/png;base64,${imageJson.image})`; + } else if (imageJson.intentType === "text-to-image2") { + rawResponse += `![generated_image](${imageJson.image})`; + } else if (imageJson.intentType === "text-to-image-v3") { + rawResponse = `![](data:image/webp;base64,${imageJson.image})`; + } + if (inferredQuery) { + rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`; + } + } + let references = {}; + if (imageJson.context && imageJson.context.length > 0) { + references = this.extractReferences(imageJson.context); + } + if (imageJson.detail) { + // If response has detail field, response is an error message. + rawResponse += imageJson.detail; + } + return { rawResponse, references }; + } + + extractReferences(rawReferenceAsJson: any): object { + let references: any = {}; + if (rawReferenceAsJson instanceof Array) { + references["notes"] = rawReferenceAsJson; + } else if (typeof rawReferenceAsJson === "object" && rawReferenceAsJson !== null) { + references["online"] = rawReferenceAsJson; + } + return references; + } + + addMessageToChatBody(rawResponse: string, newResponseElement: HTMLElement | null, references: any) { + if (!newResponseElement) return; + newResponseElement.innerHTML = ""; + newResponseElement.appendChild(this.formatHTMLMessage(rawResponse)); + + this.finalizeChatBodyResponse(references, newResponseElement); + } + + finalizeChatBodyResponse(references: object, newResponseElement: HTMLElement | null) { + if (!!newResponseElement && references != null && Object.keys(references).length > 0) { + newResponseElement.appendChild(this.createReferenceSection(references)); + } + this.scrollChatToBottom(); + let chatInput = this.contentEl.getElementsByClassName("khoj-chat-input")[0]; + if (chatInput) chatInput.removeAttribute("disabled"); + } + + createReferenceSection(references: any) { + let referenceSection = this.contentEl.createEl('div'); + referenceSection.classList.add("reference-section"); + referenceSection.classList.add("collapsed"); + + let numReferences = 0; + + if (references.hasOwnProperty("notes")) { + numReferences += references["notes"].length; + + references["notes"].forEach((reference: any, index: number) => { + let polishedReference = this.generateReference(referenceSection, reference, index); + referenceSection.appendChild(polishedReference); + }); + } + if (references.hasOwnProperty("online")) { + numReferences += this.processOnlineReferences(referenceSection, references["online"]); + } + + let referenceExpandButton = this.contentEl.createEl('button'); + referenceExpandButton.classList.add("reference-expand-button"); + referenceExpandButton.innerHTML = numReferences == 1 ? "1 reference" : `${numReferences} references`; + + referenceExpandButton.addEventListener('click', function() { + if (referenceSection.classList.contains("collapsed")) { + referenceSection.classList.remove("collapsed"); + referenceSection.classList.add("expanded"); + } else { + referenceSection.classList.add("collapsed"); + referenceSection.classList.remove("expanded"); + } + }); + + let referencesDiv = this.contentEl.createEl('div'); + referencesDiv.classList.add("references"); + referencesDiv.appendChild(referenceExpandButton); + referencesDiv.appendChild(referenceSection); + + return referencesDiv; + } +} diff --git a/src/interface/obsidian/src/main.ts b/src/interface/obsidian/src/main.ts index 7e152c49..83ac17b8 100644 --- a/src/interface/obsidian/src/main.ts +++ b/src/interface/obsidian/src/main.ts @@ -1,8 +1,8 @@ -import { Plugin } from 'obsidian'; +import { Plugin, WorkspaceLeaf } from 'obsidian'; import { KhojSetting, KhojSettingTab, DEFAULT_SETTINGS } from 'src/settings' import { KhojSearchModal } from 'src/search_modal' -import { KhojChatModal } from 'src/chat_modal' -import { updateContentIndex, canConnectToBackend } from './utils'; +import { KhojChatView } from 'src/chat_view' +import { updateContentIndex, canConnectToBackend, KhojView } from './utils'; export default class Khoj extends Plugin { @@ -30,12 +30,14 @@ export default class Khoj extends Plugin { this.addCommand({ id: 'chat', name: 'Chat', - callback: () => { new KhojChatModal(this.app, this.settings).open(); } + callback: () => { this.activateView(KhojView.CHAT); } }); + this.registerView(KhojView.CHAT, (leaf) => new KhojChatView(leaf, this.settings)); + // Create an icon in the left ribbon. this.addRibbonIcon('message-circle', 'Khoj', (_: MouseEvent) => { - new KhojChatModal(this.app, this.settings).open() + this.activateView(KhojView.CHAT); }); // Add a settings tab so the user can configure khoj @@ -69,4 +71,24 @@ export default class Khoj extends Plugin { this.unload(); } + + async activateView(viewType: KhojView) { + const { workspace } = this.app; + + let leaf: WorkspaceLeaf | null = null; + const leaves = workspace.getLeavesOfType(viewType); + + if (leaves.length > 0) { + // A leaf with our view already exists, use that + leaf = leaves[0]; + } else { + // Our view could not be found in the workspace, create a new leaf + // in the right sidebar for it + leaf = workspace.getRightLeaf(false); + await leaf.setViewState({ type: viewType, active: true }); + } + + // "Reveal" the leaf in case it is in a collapsed sidebar + workspace.revealLeaf(leaf); + } } diff --git a/src/interface/obsidian/src/pane_view.ts b/src/interface/obsidian/src/pane_view.ts new file mode 100644 index 00000000..40659572 --- /dev/null +++ b/src/interface/obsidian/src/pane_view.ts @@ -0,0 +1,53 @@ +import { ItemView, WorkspaceLeaf } from 'obsidian'; +import { KhojSetting } from 'src/settings'; +import { KhojSearchModal } from 'src/search_modal'; +import { KhojView, populateHeaderPane } from './utils'; + +export abstract class KhojPaneView extends ItemView { + setting: KhojSetting; + + constructor(leaf: WorkspaceLeaf, setting: KhojSetting) { + super(leaf); + + this.setting = setting; + + // Register Modal Keybindings to send user message + // this.scope.register([], 'Enter', async () => { await this.chat() }); + } + + async onOpen() { + let { contentEl } = this; + + // Add title to the Khoj Chat modal + let headerEl = contentEl.createDiv(({ attr: { id: "khoj-header", class: "khoj-header" } })); + // Setup the header pane + await populateHeaderPane(headerEl, this.setting); + // Set the active nav pane + headerEl.getElementsByClassName("chat-nav")[0]?.classList.add("khoj-nav-selected"); + headerEl.getElementsByClassName("chat-nav")[0]?.addEventListener("click", (_) => { this.activateView(KhojView.CHAT); }); + headerEl.getElementsByClassName("search-nav")[0]?.addEventListener("click", (_) => { new KhojSearchModal(this.app, this.setting).open(); }); + headerEl.getElementsByClassName("similar-nav")[0]?.addEventListener("click", (_) => { new KhojSearchModal(this.app, this.setting, true).open(); }); + let similarNavSvgEl = headerEl.getElementsByClassName("khoj-nav-icon-similar")[0]?.firstElementChild; + if (!!similarNavSvgEl) similarNavSvgEl.id = "similar-nav-icon-svg"; + } + + async activateView(viewType: string) { + const { workspace } = this.app; + + let leaf: WorkspaceLeaf | null = null; + const leaves = workspace.getLeavesOfType(viewType); + + if (leaves.length > 0) { + // A leaf with our view already exists, use that + leaf = leaves[0]; + } else { + // Our view could not be found in the workspace, create a new leaf + // in the right sidebar for it + leaf = workspace.getRightLeaf(false); + await leaf.setViewState({ type: viewType, active: true }); + } + + // "Reveal" the leaf in case it is in a collapsed sidebar + workspace.revealLeaf(leaf); + } +} diff --git a/src/interface/obsidian/src/search_modal.ts b/src/interface/obsidian/src/search_modal.ts index c1b09db9..7d791204 100644 --- a/src/interface/obsidian/src/search_modal.ts +++ b/src/interface/obsidian/src/search_modal.ts @@ -1,6 +1,6 @@ import { App, SuggestModal, request, MarkdownRenderer, Instruction, Platform } from 'obsidian'; import { KhojSetting } from 'src/settings'; -import { createNoteAndCloseModal } from 'src/utils'; +import { createNoteAndCloseModal, getLinkToEntry } from 'src/utils'; export interface SearchResult { entry: string; @@ -132,21 +132,8 @@ export class KhojSearchModal extends SuggestModal { const mdFiles = this.app.vault.getMarkdownFiles(); const pdfFiles = this.app.vault.getFiles().filter(file => file.extension === 'pdf'); - // Find the vault file matching file of chosen search result - let file_match = mdFiles.concat(pdfFiles) - // Sort by descending length of path - // This finds longest path match when multiple files have same name - .sort((a, b) => b.path.length - a.path.length) - // The first match is the best file match across OS - // e.g Khoj server on Linux, Obsidian vault on Android - .find(file => result.file.replace(/\\/g, "/").endsWith(file.path)) - - // Open vault file at heading of chosen search result - if (file_match) { - let resultHeading = file_match.extension !== 'pdf' ? result.entry.split('\n', 1)[0] : ''; - let linkToEntry = resultHeading.startsWith('#') ? `${file_match.path}${resultHeading}` : file_match.path; - this.app.workspace.openLinkText(linkToEntry, ''); - console.log(`Link: ${linkToEntry}, File: ${file_match.path}, Heading: ${resultHeading}`); - } + // Find, Open vault file at heading of chosen search result + let linkToEntry = getLinkToEntry(mdFiles.concat(pdfFiles), result.file, result.entry); + if (linkToEntry) this.app.workspace.openLinkText(linkToEntry, ''); } } diff --git a/src/interface/obsidian/src/settings.ts b/src/interface/obsidian/src/settings.ts index 875bd40a..5e0e3494 100644 --- a/src/interface/obsidian/src/settings.ts +++ b/src/interface/obsidian/src/settings.ts @@ -2,6 +2,15 @@ import { App, Notice, PluginSettingTab, Setting, TFile } from 'obsidian'; import Khoj from 'src/main'; import { canConnectToBackend, getBackendStatusMessage, updateContentIndex } from './utils'; +export interface UserInfo { + username?: string; + photo?: string; + is_active?: boolean; + has_documents?: boolean; + email?: string; +} + + export interface KhojSetting { resultsCount: number; khojUrl: string; @@ -9,7 +18,7 @@ export interface KhojSetting { connectedToBackend: boolean; autoConfigure: boolean; lastSync: Map; - userEmail: string; + userInfo: UserInfo | null; } export const DEFAULT_SETTINGS: KhojSetting = { @@ -19,7 +28,7 @@ export const DEFAULT_SETTINGS: KhojSetting = { connectedToBackend: false, autoConfigure: true, lastSync: new Map(), - userEmail: '', + userInfo: null, } export class KhojSettingTab extends PluginSettingTab { @@ -38,7 +47,7 @@ export class KhojSettingTab extends PluginSettingTab { let backendStatusEl = containerEl.createEl('small', { text: getBackendStatusMessage( this.plugin.settings.connectedToBackend, - this.plugin.settings.userEmail, + this.plugin.settings.userInfo?.email, this.plugin.settings.khojUrl, this.plugin.settings.khojApiKey )} @@ -55,7 +64,7 @@ export class KhojSettingTab extends PluginSettingTab { this.plugin.settings.khojUrl = value.trim().replace(/\/$/, ''); ({ connectedToBackend: this.plugin.settings.connectedToBackend, - userEmail: this.plugin.settings.userEmail, + userInfo: this.plugin.settings.userInfo, statusMessage: backendStatusMessage, } = await canConnectToBackend(this.plugin.settings.khojUrl, this.plugin.settings.khojApiKey)); @@ -71,7 +80,7 @@ export class KhojSettingTab extends PluginSettingTab { this.plugin.settings.khojApiKey = value.trim(); ({ connectedToBackend: this.plugin.settings.connectedToBackend, - userEmail: this.plugin.settings.userEmail, + userInfo: this.plugin.settings.userInfo, statusMessage: backendStatusMessage, } = await canConnectToBackend(this.plugin.settings.khojUrl, this.plugin.settings.khojApiKey)); await this.plugin.saveSettings(); diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index 11b6ac07..cfdbc431 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -1,5 +1,5 @@ -import { FileSystemAdapter, Notice, Vault, Modal, TFile, request } from 'obsidian'; -import { KhojSetting } from 'src/settings' +import { FileSystemAdapter, Notice, Vault, Modal, TFile, request, setIcon, Editor } from 'obsidian'; +import { KhojSetting, UserInfo } from 'src/settings' export function getVaultAbsolutePath(vault: Vault): string { let adaptor = vault.adapter; @@ -173,31 +173,30 @@ export async function canConnectToBackend( khojUrl: string, khojApiKey: string, showNotice: boolean = false -): Promise<{ connectedToBackend: boolean; statusMessage: string, userEmail: string }> { +): Promise<{ connectedToBackend: boolean; statusMessage: string, userInfo: UserInfo | null }> { let connectedToBackend = false; - let userEmail: string = ''; + let userInfo: UserInfo | null = null; if (!!khojUrl) { let headers = !!khojApiKey ? { "Authorization": `Bearer ${khojApiKey}` } : undefined; - await request({ url: `${khojUrl}/api/health`, method: "GET", headers: headers }) - .then(response => { + try { + let response = await request({ url: `${khojUrl}/api/v1/user`, method: "GET", headers: headers }) connectedToBackend = true; - userEmail = JSON.parse(response)?.email; - }) - .catch(error => { + userInfo = JSON.parse(response); + } catch (error) { connectedToBackend = false; console.log(`Khoj connection error:\n\n${error}`); - }); + }; } - let statusMessage: string = getBackendStatusMessage(connectedToBackend, userEmail, khojUrl, khojApiKey); + let statusMessage: string = getBackendStatusMessage(connectedToBackend, userInfo?.email, khojUrl, khojApiKey); if (showNotice) new Notice(statusMessage); - return { connectedToBackend, statusMessage, userEmail }; + return { connectedToBackend, statusMessage, userInfo }; } export function getBackendStatusMessage( connectedToServer: boolean, - userEmail: string, + userEmail: string | undefined, khojUrl: string, khojApiKey: string ): string { @@ -215,3 +214,155 @@ export function getBackendStatusMessage( else return `✅ Signed in to Khoj as ${userEmail}`; } + +export async function populateHeaderPane(headerEl: Element, setting: KhojSetting): Promise { + let userInfo: UserInfo | null = null; + try { + const { userInfo: extractedUserInfo } = await canConnectToBackend(setting.khojUrl, setting.khojApiKey, false); + userInfo = extractedUserInfo; + } catch (error) { + console.error("❗️Could not connect to Khoj"); + } + + // Add Khoj title to header element + const titleEl = headerEl.createDiv(); + titleEl.className = 'khoj-logo'; + titleEl.textContent = "KHOJ" + + // Populate the header element with the navigation pane + // Create the nav element + const nav = headerEl.createEl('nav'); + nav.className = 'khoj-nav'; + + // Create the chat link + const chatLink = nav.createEl('a'); + chatLink.id = 'chat-nav'; + chatLink.className = 'khoj-nav chat-nav'; + + // Create the chat icon + const chatIcon = chatLink.createEl('span'); + chatIcon.className = 'khoj-nav-icon khoj-nav-icon-chat'; + setIcon(chatIcon, 'khoj-chat'); + + // Create the chat text + const chatText = chatLink.createEl('span'); + chatText.className = 'khoj-nav-item-text'; + chatText.textContent = 'Chat'; + + // Append the chat icon and text to the chat link + chatLink.appendChild(chatIcon); + chatLink.appendChild(chatText); + + // Create the search link + const searchLink = nav.createEl('a'); + searchLink.id = 'search-nav'; + searchLink.className = 'khoj-nav search-nav'; + + // Create the search icon + const searchIcon = searchLink.createEl('span'); + searchIcon.className = 'khoj-nav-icon khoj-nav-icon-search'; + + // Create the search text + const searchText = searchLink.createEl('span'); + searchText.className = 'khoj-nav-item-text'; + searchText.textContent = 'Search'; + + // Append the search icon and text to the search link + searchLink.appendChild(searchIcon); + searchLink.appendChild(searchText); + + // Create the search link + const similarLink = nav.createEl('a'); + similarLink.id = 'similar-nav'; + similarLink.className = 'khoj-nav similar-nav'; + + // Create the search icon + const similarIcon = searchLink.createEl('span'); + similarIcon.id = 'similar-nav-icon'; + similarIcon.className = 'khoj-nav-icon khoj-nav-icon-similar'; + setIcon(similarIcon, 'webhook'); + + // Create the search text + const similarText = searchLink.createEl('span'); + similarText.className = 'khoj-nav-item-text'; + similarText.textContent = 'Similar'; + + // Append the search icon and text to the search link + similarLink.appendChild(similarIcon); + similarLink.appendChild(similarText); + + // Append the nav items to the nav element + nav.appendChild(chatLink); + nav.appendChild(searchLink); + nav.appendChild(similarLink); + + // Append the title, nav items to the header element + headerEl.appendChild(titleEl); + headerEl.appendChild(nav); +} + +export enum KhojView { + CHAT = "khoj-chat-view", +} + +function copyParentText(event: MouseEvent, message: string, originalButton: string) { + const button = event.currentTarget as HTMLElement; + if (!button || !button?.parentNode?.textContent) return; + if (!!button.firstChild) button.removeChild(button.firstChild as HTMLImageElement); + const textContent = message ?? button.parentNode.textContent.trim(); + navigator.clipboard.writeText(textContent).then(() => { + setIcon((button as HTMLElement), 'copy-check'); + setTimeout(() => { + setIcon((button as HTMLElement), originalButton); + }, 1000); + }).catch((error) => { + console.error("Error copying text to clipboard:", error); + const originalButtonText = button.innerHTML; + button.innerHTML = "⛔️"; + setTimeout(() => { + button.innerHTML = originalButtonText; + setIcon((button as HTMLElement), originalButton); + }, 2000); + }); + + return textContent; +} + +export function createCopyParentText(message: string, originalButton: string = 'copy-plus') { + return function(event: MouseEvent) { + return copyParentText(event, message, originalButton); + } +} + +export function pasteTextAtCursor(text: string | undefined) { + // Get the current active file's editor + const editor: Editor = this.app.workspace.getActiveFileView()?.editor + if (!editor || !text) return; + const cursor = editor.getCursor(); + // If there is a selection, replace it with the text + if (editor?.getSelection()) { + editor.replaceSelection(text); + // If there is no selection, insert the text at the cursor position + } else if (cursor) { + editor.replaceRange(text, cursor); + } +} + +export function getLinkToEntry(sourceFiles: TFile[], chosenFile: string, chosenEntry: string): string | undefined { + // Find the vault file matching file of chosen file, entry + let fileMatch = sourceFiles + // Sort by descending length of path + // This finds longest path match when multiple files have same name + .sort((a, b) => b.path.length - a.path.length) + // The first match is the best file match across OS + // e.g Khoj server on Linux, Obsidian vault on Android + .find(file => chosenFile.replace(/\\/g, "/").endsWith(file.path)) + + // Return link to vault file at heading of chosen search result + if (fileMatch) { + let resultHeading = fileMatch.extension !== 'pdf' ? chosenEntry.split('\n', 1)[0] : ''; + let linkToEntry = resultHeading.startsWith('#') ? `${fileMatch.path}${resultHeading}` : fileMatch.path; + console.log(`Link: ${linkToEntry}, File: ${fileMatch.path}, Heading: ${resultHeading}`); + return linkToEntry; + } +} diff --git a/src/interface/obsidian/styles.css b/src/interface/obsidian/styles.css index ec28eebd..31314271 100644 --- a/src/interface/obsidian/styles.css +++ b/src/interface/obsidian/styles.css @@ -11,6 +11,8 @@ If your plugin does not need CSS, delete this file. --khoj-winter-sun: #f9f5de; --khoj-sun: #fee285; --khoj-storm-grey: #475569; + --chat-icon: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='24px' height='24px' viewBox='0 0 24 24' fill='currentColor' stroke-linecap='round' stroke-linejoin='round' class='svg-icon' version='1.1'%3E%3Cpath d='m 14.024348,9.8497703 0.04627,1.9750167' stroke='%231c274c' stroke-width='2' stroke-linecap='round' /%3E%3Cpath d='m 9.6453624,9.7953624 0.046275,1.9750166' stroke='%231c274c' stroke-width='2' stroke-linecap='round' /%3E%3Cpath d='m 11.90538,2.3619994 c -5.4939109,0 -9.6890976,4.0608185 -9.6890976,9.8578926 0,1.477202 0.2658016,2.542848 0.6989332,3.331408 0.433559,0.789293 1.0740097,1.372483 1.9230615,1.798517 1.7362861,0.87132 4.1946007,1.018626 7.0671029,1.018626 0.317997,0 0.593711,0.167879 0.784844,0.458501 0.166463,0.253124 0.238617,0.552748 0.275566,0.787233 0.07263,0.460801 0.05871,1.030165 0.04785,1.474824 v 4.8e-5 l -2.26e-4,0.0091 c -0.0085,0.348246 -0.01538,0.634247 -0.0085,0.861186 0.105589,-0.07971 0.227925,-0.185287 0.36735,-0.31735 0.348613,-0.330307 0.743513,-0.767362 1.176607,-1.246635 l 0.07837,-0.08673 c 0.452675,-0.500762 0.941688,-1.037938 1.41216,-1.473209 0.453774,-0.419787 0.969948,-0.822472 1.476003,-0.953853 1.323661,-0.343655 2.330132,-0.904027 3.005749,-1.76381 0.658957,-0.838568 1.073167,-2.051868 1.073167,-3.898667 0,-5.7970748 -4.195186,-9.8578946 -9.689097,-9.8578946 z M 0.92440678,12.219892 c 0,-7.0067939 5.05909412,-11.47090892 10.98097322,-11.47090892 5.921878,0 10.980972,4.46411502 10.980972,11.47090892 0,2.172259 -0.497596,3.825405 -1.442862,5.028357 -0.928601,1.181693 -2.218843,1.837914 -3.664937,2.213334 -0.211641,0.05502 -0.53529,0.268579 -0.969874,0.670658 -0.417861,0.386604 -0.865628,0.876836 -1.324566,1.384504 l -0.09131,0.101202 c -0.419252,0.464136 -0.849637,0.94059 -1.239338,1.309807 -0.210187,0.199169 -0.425281,0.383422 -0.635348,0.523424 -0.200911,0.133819 -0.449635,0.263369 -0.716376,0.281474 -0.327812,0.02226 -0.61539,-0.149209 -0.804998,-0.457293 -0.157614,-0.255993 -0.217622,-0.557143 -0.246564,-0.778198 -0.0542,-0.414027 -0.04101,-0.933065 -0.03027,-1.355183 l 0.0024,-0.0922 c 0.01099,-0.463865 0.01489,-0.820507 -0.01611,-1.06842 C 8.9434608,19.975238 6.3139711,19.828758 4.356743,18.84659 3.3355029,18.334136 2.4624526,17.578678 1.8500164,16.463713 1.2372016,15.348029 0.92459928,13.943803 0.92459928,12.219967 Z' clip-rule='evenodd' stroke-width='2' fill='currentColor' fill-rule='evenodd' fill-opacity='1' /%3E%3C/svg%3E%0A"); + --search-icon: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='24px' height='24px' viewBox='0 0 24 24' fill='currentColor' stroke-linecap='round' stroke-linejoin='round' class='svg-icon' version='1.1'%3E%3Cpath d='m 18.562765,17.147843 c 1.380497,-1.679442 2.307667,-4.013099 2.307667,-6.330999 C 20.870432,5.3951476 16.353958,1 10.782674,1 5.2113555,1 0.69491525,5.3951476 0.69491525,10.816844 c 0,5.421663 4.51644025,9.816844 10.08775875,9.816844 2.381867,0 4.570922,-0.803307 6.296712,-2.14673 0.508475,-0.508475 4.514633,4.192839 4.514633,4.192839 1.036377,1.008544 2.113087,-0.02559 1.07671,-1.034139 z m -7.780091,1.925408 c -4.3394583,0 -8.6708434,-4.033489 -8.6708434,-8.256407 0,-4.2229187 4.3313851,-8.2564401 8.6708434,-8.2564401 4.339458,0 8.670809,4.2369112 8.670809,8.4598301 0,4.222918 -4.331351,8.053017 -8.670809,8.053017 z' fill='currentColor' fill-rule='evenodd' clip-rule='evenodd' fill-opacity='1' stroke-width='1.10519' stroke-dasharray='none' /%3E%3Cpath d='m 13.337351,9.3402647 0.05184,2.1532893' stroke='%231c274c' stroke-width='2' stroke-linecap='round' /%3E%3Cpath d='M 8.431347,9.2809457 8.483191,11.434235' stroke='%231c274c' stroke-width='2' stroke-linecap='round' /%3E%3C/svg%3E%0A"); } .khoj-chat p { @@ -22,6 +24,7 @@ If your plugin does not need CSS, delete this file. .khoj-chat { display: grid; + grid-template-rows: auto 1fr auto; background: var(--background-primary); color: var(--text-normal); text-align: center; @@ -73,6 +76,7 @@ If your plugin does not need CSS, delete this file. display: inline-block; max-width: 80%; text-align: left; + user-select: text; } /* color chat bubble by khoj blue */ .khoj-chat-message-text.khoj { @@ -113,7 +117,7 @@ If your plugin does not need CSS, delete this file. .khoj-chat-message-text ul, .khoj-chat-message-text ol { - margin: -20px 0 0; + margin: 0px 0 0; } .khoj-chat-message-text ol li { white-space: normal; @@ -153,14 +157,14 @@ div.expanded.reference-section { grid-auto-flow: row; grid-column-gap: 10px; grid-row-gap: 10px; - margin: 10px; + margin: 10px 0; } button.reference-button { - background: var(--color-base-00); - color: var(--color-base-100); + background: var(--khoj-winter-sun); + color: var(--khoj-storm-grey); border: 1px solid var(--khoj-storm-grey); border-radius: 5px; - padding: 5px; + padding: 4px; font-size: 14px; font-weight: 300; line-height: 1.5em; @@ -179,6 +183,13 @@ button.reference-button.expanded { max-height: none; white-space: pre-wrap; } +button.reference-button.expanded > :nth-child(2) { + display: block; +} +button.reference-button.collapsed > :nth-child(2) { + display: none; +} + button.reference-button::before { content: "▶"; margin-right: 5px; @@ -191,11 +202,12 @@ button.reference-button[aria-expanded="true"]::before { transform: rotate(90deg); } button.reference-expand-button { - background: var(--color-base-00); - color: var(--color-base-100); + background: var(--khoj-winter-sun); + color: var(--khoj-storm-grey); border: 1px solid var(--khoj-storm-grey); border-radius: 5px; - padding: 5px; + padding: 8px; + margin-top: 8px; font-size: 14px; font-weight: 300; line-height: 1.5em; @@ -205,14 +217,14 @@ button.reference-expand-button { } button.reference-expand-button:hover { background: var(--khoj-sun); - color: var(--color-base-00); + color: var(--khoj-storm-grey); } a.inline-chat-link { color: #475569; text-decoration: none; border-bottom: 1px dotted #475569; } -a.reference-link { +.reference-link { color: var(--khoj-storm-grey); border-bottom: 1px dotted var(--khoj-storm-grey); } @@ -230,6 +242,98 @@ img { max-width: 60%; } +div.new-conversation { + display: grid; + grid-auto-flow: column; + grid-template-columns: 1fr auto; + margin-bottom: 16px; +} +div.conversation-header-title { + text-align: left; + font-size: larger; + line-height: 1.5em; +} +div.conversation-session { + color: var(--color-base-90); + border: 1px solid var(--khoj-storm-grey); + border-radius: 5px; + margin-top: 8px; + padding: 5px; + font-size: 14px; + font-weight: 300; + line-height: 1.5em; + cursor: pointer; + transition: background 0.2s ease-in-out; + text-align: left; + display: grid; + grid-auto-flow: column; + grid-template-columns: 1fr auto; +} + +.three-dot-menu { + display: block; + /* background: var(--background-color); */ + /* border: 1px solid var(--main-text-color); */ + border-radius: 5px; + /* position: relative; */ +} + +button.selected-conversation { + background: var(--khoj-winter-sun); +} +button.three-dot-menu-button-item { + color: var(--color-base-90); + border: none; + box-shadow: none; + font-size: 14px; + font-weight: 300; + line-height: 1.5em; + cursor: pointer; + transition: background 0.3s ease-in-out; + font-family: var(--font-family); + border-radius: 4px; + right: 0; +} + +button.three-dot-menu-button-item:hover { + background: var(--khoj-storm-grey); + color: var(--khoj-winter-sun); +} + +.three-dot-menu-button { + background: var(--khoj-winter-sun); + border: none; + box-shadow: none; + font-size: 14px; + font-weight: 300; + line-height: 1.5em; + cursor: pointer; + transition: background 0.3s ease-in-out; + font-family: var(--font-family); + border-radius: 4px; + right: 0; +} + +.conversation-button:hover .three-dot-menu { + display: block; +} + +div.conversation-menu { + z-index: 1; + top: 100%; + right: 0; + text-align: right; + border-radius: 5px; + padding: 5px; +} +div.conversation-session:hover { + transform: scale(1.03); +} +div.selected-conversation { + background: var(--khoj-winter-sun) !important; + color: var(--khoj-storm-grey) !important; +} + #khoj-chat-footer { padding: 0; display: grid; @@ -345,3 +449,187 @@ img { .khoj-result-entry p br { display: none; } + +/* Khoj Header, Navigation Pane */ +div.khoj-header { + display: grid; + grid-auto-flow: column; + gap: 20px; + padding: 0 0 10px 0; + margin: 0; + align-items: center; + user-select: none; + -webkit-user-select: none; + -webkit-app-region: drag; +} + +/* Keeps the navigation menu clickable */ +a.khoj-nav { + -webkit-app-region: no-drag; +} +div.khoj-nav { + -webkit-app-region: no-drag; +} +nav.khoj-nav { + display: grid; + grid-auto-flow: column; + grid-gap: 32px; + justify-self: right; + align-items: center; +} + +a.khoj-nav { + display: flex; + align-items: center; +} + +div.khoj-logo { + justify-self: left; +} + +.khoj-nav a { + color: var(--main-text-color); + text-decoration: none; + font-size: small; + font-weight: normal; + padding: 0 4px; + border-radius: 4px; + justify-self: center; + margin: 0; +} +.khoj-nav a:hover { + background-color: var(--khoj-sun); + color: var(--main-text-color); +} +a.khoj-nav-selected { + background-color: var(--khoj-winter-sun); +} +#similar-nav-icon-svg, +.khoj-nav-icon { + width: 24px; + height: 24px; +} +.khoj-nav-icon-chat { + background-image: var(--chat-icon); +} +.khoj-nav-icon-search { + background-image: var(--search-icon); +} +span.khoj-nav-item-text { + padding-left: 8px; +} + +/* Copy button */ +button.copy-button { + border-radius: 4px; + background-color: var(--background-color); + border: 1px solid var(--main-text-color); + text-align: center; + font-size: 16px; + transition: all 0.5s; + cursor: pointer; + padding: 4px; + margin-top: 8px; + float: right; +} +button.copy-button span { + cursor: pointer; + display: inline-block; + position: relative; + transition: 0.5s; +} +img.copy-icon { + width: 16px; + height: 16px; +} +.you button.copy-button { + color: var(--text-on-accent); +} +.khoj button.copy-button { + color: var(--khoj-storm-grey); +} +.you button.copy-button:hover { + color: var(--khoj-storm-grey); + background: var(--text-on-accent); +} +.khoj button.copy-button:hover { + background: var(--text-on-accent); +} + +/* Loading Spinner */ +.lds-ellipsis { + display: inline-block; + position: relative; + width: 60px; + height: 32px; +} +.lds-ellipsis div { + position: absolute; + top: 12px; + width: 8px; + height: 8px; + border-radius: 50%; + background: var(--color-base-70); + animation-timing-function: cubic-bezier(0, 1, 1, 0); +} +.lds-ellipsis div:nth-child(1) { + left: 8px; + animation: lds-ellipsis1 0.6s infinite; +} +.lds-ellipsis div:nth-child(2) { + left: 8px; + animation: lds-ellipsis2 0.6s infinite; +} +.lds-ellipsis div:nth-child(3) { + left: 32px; + animation: lds-ellipsis2 0.6s infinite; +} +.lds-ellipsis div:nth-child(4) { + left: 56px; + animation: lds-ellipsis3 0.6s infinite; +} +@keyframes lds-ellipsis1 { + 0% { + transform: scale(0); + } + 100% { + transform: scale(1); + } +} +@keyframes lds-ellipsis3 { + 0% { + transform: scale(1); + } + 100% { + transform: scale(0); + } +} +@keyframes lds-ellipsis2 { + 0% { + transform: translate(0, 0); + } + 100% { + transform: translate(24px, 0); + } +} + +@media only screen and (max-width: 600px) { + div.khoj-header { + display: grid; + grid-auto-flow: column; + gap: 20px; + padding: 24px 10px 10px 10px; + margin: 0 0 16px 0; + } + + nav.khoj-nav { + grid-gap: 0px; + justify-content: space-between; + } + a.khoj-nav { + padding: 0 16px; + } + span.khoj-nav-item-text { + display: none; + } +} diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index e4a172ff..06a8aa98 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -102,7 +102,10 @@ To get started, just start typing below. You can also type / to see a list of co return `${time_string}, ${date_string}`; } - function generateReference(reference, index) { + function generateReference(referenceJson, index) { + let reference = referenceJson.hasOwnProperty("compiled") ? referenceJson.compiled : referenceJson; + let referenceFile = referenceJson.hasOwnProperty("file") ? referenceJson.file : null; + // Escape reference for HTML rendering let escaped_ref = reference.replaceAll('"', '"'); @@ -277,95 +280,47 @@ To get started, just start typing below. You can also type / to see a list of co return numOnlineReferences; } - function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null, userQuery) { - // If no document or online context is provided, render the message as is - if ((context == null || context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) { - if (intentType?.includes("text-to-image")) { - let imageMarkdown; - if (intentType === "text-to-image") { - imageMarkdown = `![](data:image/png;base64,${message})`; - } else if (intentType === "text-to-image2") { - imageMarkdown = `![](${message})`; - } else if (intentType === "text-to-image-v3") { - imageMarkdown = `![](data:image/webp;base64,${message})`; - } - const inferredQuery = inferredQueries?.[0]; - if (inferredQuery) { - imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`; - } - return renderMessage(imageMarkdown, by, dt, null, false, "return", userQuery); - } - - return renderMessage(message, by, dt, null, false, "return", userQuery); + function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) { + let chatEl; + if (intentType?.includes("text-to-image")) { + let imageMarkdown = generateImageMarkdown(message, intentType, inferredQueries); + chatEl = renderMessage(imageMarkdown, by, dt, null, false, "return"); + } else { + chatEl = renderMessage(message, by, dt, null, false, "return"); } - if ((context && context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) { - return renderMessage(message, by, dt, null, false, "return", userQuery); + // If no document or online context is provided, render the message as is + if ((context == null || context?.length == 0) + && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) { + return chatEl; } // If document or online context is provided, render the message with its references - let references = document.createElement('div'); + let references = {}; + if (!!context) references["notes"] = context; + if (!!onlineContext) references["online"] = onlineContext; + let chatMessageEl = chatEl.getElementsByClassName("chat-message-text")[0]; + chatMessageEl.appendChild(createReferenceSection(references)); - let referenceExpandButton = document.createElement('button'); - referenceExpandButton.classList.add("reference-expand-button"); - let numReferences = 0; - - if (context) { - numReferences += context.length; - } - - references.appendChild(referenceExpandButton); - - let referenceSection = document.createElement('div'); - referenceSection.classList.add("reference-section"); - referenceSection.classList.add("collapsed"); - - referenceExpandButton.addEventListener('click', function() { - if (referenceSection.classList.contains("collapsed")) { - referenceSection.classList.remove("collapsed"); - referenceSection.classList.add("expanded"); - } else { - referenceSection.classList.add("collapsed"); - referenceSection.classList.remove("expanded"); - } - }); - - references.classList.add("references"); - if (context) { - for (let index in context) { - let reference = context[index]; - let polishedReference = generateReference(reference, index); - referenceSection.appendChild(polishedReference); - } - } - - if (onlineContext) { - numReferences += processOnlineReferences(referenceSection, onlineContext); - } - - let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`; - referenceExpandButton.innerHTML = expandButtonText; - - references.appendChild(referenceSection); - - if (intentType?.includes("text-to-image")) { - let imageMarkdown; - if (intentType === "text-to-image") { - imageMarkdown = `![](data:image/png;base64,${message})`; - } else if (intentType === "text-to-image2") { - imageMarkdown = `![](${message})`; - } else if (intentType === "text-to-image-v3") { - imageMarkdown = `![](data:image/webp;base64,${message})`; - } - const inferredQuery = inferredQueries?.[0]; - if (inferredQuery) { - imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`; - } - return renderMessage(imageMarkdown, by, dt, references, false, "return", userQuery); - } - - return renderMessage(message, by, dt, references, false, "return", userQuery); + return chatEl; } + + function generateImageMarkdown(message, intentType, inferredQueries=null) { + let imageMarkdown; + if (intentType === "text-to-image") { + imageMarkdown = `![](data:image/png;base64,${message})`; + } else if (intentType === "text-to-image2") { + imageMarkdown = `![](${message})`; + } else if (intentType === "text-to-image-v3") { + imageMarkdown = `![](data:image/webp;base64,${message})`; + } + const inferredQuery = inferredQueries?.[0]; + if (inferredQuery) { + imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`; + } + return imageMarkdown; + } + //handler function for posting feedback data to endpoint function sendFeedback(_uquery="", _kquery="", _sentiment="") { const uquery = _uquery; @@ -387,7 +342,7 @@ To get started, just start typing below. You can also type / to see a list of co // Replace LaTeX delimiters with placeholders newHTML = newHTML.replace(/\\\(/g, 'LEFTPAREN').replace(/\\\)/g, 'RIGHTPAREN') - .replace(/\\\[/g, 'LEFTBRACKET').replace(/\\\]/g, 'RIGHTBRACKET'); + .replace(/\\\[/g, 'LEFTBRACKET').replace(/\\\]/g, 'RIGHTBRACKET'); // Remove any text between [INST] and tags. These are spurious instructions for the AI chat model. newHTML = newHTML.replace(/\[INST\].+(<\/s>)?/g, ''); @@ -408,7 +363,7 @@ To get started, just start typing below. You can also type / to see a list of co // Replace placeholders with LaTeX delimiters newHTML = newHTML.replace(/LEFTPAREN/g, '\\(').replace(/RIGHTPAREN/g, '\\)') - .replace(/LEFTBRACKET/g, '\\[').replace(/RIGHTBRACKET/g, '\\]'); + .replace(/LEFTBRACKET/g, '\\[').replace(/RIGHTBRACKET/g, '\\]'); // Set rendered markdown to HTML DOM element let element = document.createElement('div'); @@ -1101,11 +1056,7 @@ To get started, just start typing below. You can also type / to see a list of co } } - function sendMessageViaWebSocket(event) { - if (event) { - event.preventDefault(); - } - + function sendMessageViaWebSocket() { let chatBody = document.getElementById("chat-body"); var query = document.getElementById("chat-input").value.trim(); @@ -2645,7 +2596,7 @@ To get started, just start typing below. You can also type / to see a list of co text-align: left; display: flex; position: relative; - margin: 0 8px; + margin-right: 8px; } .three-dot-menu { @@ -3030,7 +2981,5 @@ To get started, just start typing below. You can also type / to see a list of co transform: translate(24px, 0); } } - - diff --git a/src/khoj/processor/conversation/offline/chat_model.py b/src/khoj/processor/conversation/offline/chat_model.py index a559df22..edc2d9f0 100644 --- a/src/khoj/processor/conversation/offline/chat_model.py +++ b/src/khoj/processor/conversation/offline/chat_model.py @@ -142,7 +142,7 @@ def converse_offline( # Initialize Variables assert loaded_model is None or isinstance(loaded_model, Llama), "loaded_model must be of type Llama, if configured" offline_chat_model = loaded_model or download_model(model, max_tokens=max_prompt_size) - compiled_references_message = "\n\n".join({f"{item}" for item in references}) + compiled_references_message = "\n\n".join({f"{item['compiled']}" for item in references}) current_date = datetime.now().strftime("%Y-%m-%d") diff --git a/src/khoj/processor/conversation/openai/gpt.py b/src/khoj/processor/conversation/openai/gpt.py index f5ff5384..e424a5cb 100644 --- a/src/khoj/processor/conversation/openai/gpt.py +++ b/src/khoj/processor/conversation/openai/gpt.py @@ -126,7 +126,7 @@ def converse( """ # Initialize Variables current_date = datetime.now().strftime("%Y-%m-%d") - compiled_references = "\n\n".join({f"# {item}" for item in references}) + compiled_references = "\n\n".join({f"# {item['compiled']}" for item in references}) conversation_primer = prompts.query_prompt.format(query=user_query) diff --git a/src/khoj/processor/conversation/utils.py b/src/khoj/processor/conversation/utils.py index 775848c8..5d68d17d 100644 --- a/src/khoj/processor/conversation/utils.py +++ b/src/khoj/processor/conversation/utils.py @@ -96,7 +96,7 @@ def save_to_conversation_log( user: KhojUser, meta_log: Dict, user_message_time: str = None, - compiled_references: List[str] = [], + compiled_references: List[Dict[str, Any]] = [], online_results: Dict[str, Any] = {}, inferred_queries: List[str] = [], intent_type: str = "remember", diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index 0344c81d..bce956ef 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -358,14 +358,14 @@ async def extract_references_and_questions( # Collate search results as context for GPT with timer("Searching knowledge base took", logger): - result_list = [] + search_results = [] logger.info(f"🔍 Searching knowledge base with queries: {inferred_queries}") if send_status_func: inferred_queries_str = "\n- " + "\n- ".join(inferred_queries) await send_status_func(f"**🔍 Searching Documents for:** {inferred_queries_str}") for query in inferred_queries: n_items = min(n, 3) if using_offline_chat else n - result_list.extend( + search_results.extend( await execute_search( user, f"{query} {filters_in_query}", @@ -376,8 +376,10 @@ async def extract_references_and_questions( dedupe=False, ) ) - result_list = text_search.deduplicated_search_responses(result_list) - compiled_references = [item.additional["compiled"] for item in result_list] + search_results = text_search.deduplicated_search_responses(search_results) + compiled_references = [ + {"compiled": item.additional["compiled"], "file": item.additional["file"]} for item in search_results + ] return compiled_references, inferred_queries, defiltered_query diff --git a/src/khoj/routers/api_chat.py b/src/khoj/routers/api_chat.py index ce167713..c428502f 100644 --- a/src/khoj/routers/api_chat.py +++ b/src/khoj/routers/api_chat.py @@ -586,7 +586,7 @@ async def websocket_endpoint( if compiled_references: headings = "\n- " + "\n- ".join( - set([" ".join(c.split("Path: ")[1:]).split("\n ")[0] for c in compiled_references]) + set([" ".join(c.get("compiled", c).split("Path: ")[1:]).split("\n ")[0] for c in compiled_references]) ) await send_status_update(f"**📜 Found Relevant Notes**: {headings}") @@ -944,6 +944,12 @@ async def chat( actual_response = aggregated_gpt_response.split("### compiled references:")[0] - response_obj = {"response": actual_response, "context": compiled_references} + response_obj = { + "response": actual_response, + "intentType": intent_type, + "inferredQueries": inferred_queries, + "context": compiled_references, + "online_results": online_results, + } return Response(content=json.dumps(response_obj), media_type="application/json", status_code=200) diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py index 6581d253..d8102517 100644 --- a/src/khoj/routers/helpers.py +++ b/src/khoj/routers/helpers.py @@ -414,7 +414,7 @@ async def generate_better_image_prompt( q: str, conversation_history: str, location_data: LocationData, - note_references: List[str], + note_references: List[Dict[str, Any]], online_results: Optional[dict] = None, ) -> str: """ @@ -429,7 +429,7 @@ async def generate_better_image_prompt( else: location_prompt = "Unknown" - user_references = "\n\n".join([f"# {item}" for item in note_references]) + user_references = "\n\n".join([f"# {item['compiled']}" for item in note_references]) simplified_online_results = {} @@ -596,7 +596,7 @@ def generate_chat_response( q: str, meta_log: dict, conversation: Conversation, - compiled_references: List[str] = [], + compiled_references: List[Dict] = [], online_results: Dict[str, Dict] = {}, inferred_queries: List[str] = [], conversation_commands: List[ConversationCommand] = [ConversationCommand.Default], @@ -698,7 +698,7 @@ async def text_to_image( user: KhojUser, conversation_log: dict, location_data: LocationData, - references: List[str], + references: List[Dict[str, Any]], online_results: Dict[str, Any], send_status_func: Optional[Callable] = None, ) -> Tuple[Optional[str], int, Optional[str], str]: