sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-12-18 18:47:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 23

Update response handling in Obsidian to work with new format

Browse source
This commit is contained in:
sabaimran 2024-11-29 18:10:47 -08:00
parent 512cf535e0
commit df855adc98
2 changed files with 97 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

154
src/interface/obsidian/src/chat_view.ts
View file

@ -1,4 +1,4 @@
import {ItemView, MarkdownRenderer, Scope, WorkspaceLeaf, request, requestUrl, setIcon, Platform} from 'obsidian'; import { ItemView, MarkdownRenderer, Scope, WorkspaceLeaf, request, requestUrl, setIcon, Platform } from 'obsidian';
import * as DOMPurify from 'dompurify'; import * as DOMPurify from 'dompurify';
import { KhojSetting } from 'src/settings'; import { KhojSetting } from 'src/settings';
import { KhojPaneView } from 'src/pane_view'; import { KhojPaneView } from 'src/pane_view';
@ -46,10 +46,10 @@ export class KhojChatView extends KhojPaneView {
waitingForLocation: boolean; waitingForLocation: boolean;
location: Location = { timezone: Intl.DateTimeFormat().resolvedOptions().timeZone }; location: Location = { timezone: Intl.DateTimeFormat().resolvedOptions().timeZone };
keyPressTimeout: NodeJS.Timeout | null = null; keyPressTimeout: NodeJS.Timeout | null = null;
userMessages: string[] = []; // Store user sent messages for input history cycling userMessages: string[] = []; // Store user sent messages for input history cycling
currentMessageIndex: number = -1; // Track current message index in userMessages array currentMessageIndex: number = -1; // Track current message index in userMessages array
private currentUserInput: string = ""; // Stores the current user input that is being typed in chat private currentUserInput: string = ""; // Stores the current user input that is being typed in chat
private startingMessage: string = "Message"; private startingMessage: string = "Message";
chatMessageState: ChatMessageState; chatMessageState: ChatMessageState;
constructor(leaf: WorkspaceLeaf, setting: KhojSetting) { constructor(leaf: WorkspaceLeaf, setting: KhojSetting) {
@ -102,14 +102,14 @@ export class KhojChatView extends KhojPaneView {
// Clear text after extracting message to send // Clear text after extracting message to send
let user_message = input_el.value.trim(); let user_message = input_el.value.trim();
// Store the message in the array if it's not empty // Store the message in the array if it's not empty
if (user_message) { if (user_message) {
this.userMessages.push(user_message); this.userMessages.push(user_message);
// Update starting message after sending a new message // Update starting message after sending a new message
const modifierKey = Platform.isMacOS ? '⌘' : '^'; const modifierKey = Platform.isMacOS ? '⌘' : '^';
this.startingMessage = `(${modifierKey}+↑/↓) for prev messages`; this.startingMessage = `(${modifierKey}+↑/↓) for prev messages`;
input_el.placeholder = this.startingMessage; input_el.placeholder = this.startingMessage;
} }
input_el.value = ""; input_el.value = "";
this.autoResize(); this.autoResize();
@ -162,9 +162,9 @@ export class KhojChatView extends KhojPaneView {
}) })
chatInput.addEventListener('input', (_) => { this.onChatInput() }); chatInput.addEventListener('input', (_) => { this.onChatInput() });
chatInput.addEventListener('keydown', (event) => { chatInput.addEventListener('keydown', (event) => {
this.incrementalChat(event); this.incrementalChat(event);
this.handleArrowKeys(event); this.handleArrowKeys(event);
}); });
// Add event listeners for long press keybinding // Add event listeners for long press keybinding
this.contentEl.addEventListener('keydown', this.handleKeyDown.bind(this)); this.contentEl.addEventListener('keydown', this.handleKeyDown.bind(this));
@ -199,7 +199,7 @@ export class KhojChatView extends KhojPaneView {
// Get chat history from Khoj backend and set chat input state // Get chat history from Khoj backend and set chat input state
let getChatHistorySucessfully = await this.getChatHistory(chatBodyEl); let getChatHistorySucessfully = await this.getChatHistory(chatBodyEl);
let placeholderText : string = getChatHistorySucessfully ? this.startingMessage : "Configure Khoj to enable chat"; let placeholderText: string = getChatHistorySucessfully ? this.startingMessage : "Configure Khoj to enable chat";
chatInput.placeholder = placeholderText; chatInput.placeholder = placeholderText;
chatInput.disabled = !getChatHistorySucessfully; chatInput.disabled = !getChatHistorySucessfully;
@ -214,7 +214,7 @@ export class KhojChatView extends KhojPaneView {
}); });
} }
startSpeechToText(event: KeyboardEvent | MouseEvent | TouchEvent, timeout=200) { startSpeechToText(event: KeyboardEvent | MouseEvent | TouchEvent, timeout = 200) {
if (!this.keyPressTimeout) { if (!this.keyPressTimeout) {
this.keyPressTimeout = setTimeout(async () => { this.keyPressTimeout = setTimeout(async () => {
// Reset auto send voice message timer, UI if running // Reset auto send voice message timer, UI if running
@ -320,7 +320,7 @@ export class KhojChatView extends KhojPaneView {
referenceButton.tabIndex = 0; referenceButton.tabIndex = 0;
// Add event listener to toggle full reference on click // Add event listener to toggle full reference on click
referenceButton.addEventListener('click', function() { referenceButton.addEventListener('click', function () {
if (this.classList.contains("collapsed")) { if (this.classList.contains("collapsed")) {
this.classList.remove("collapsed"); this.classList.remove("collapsed");
this.classList.add("expanded"); this.classList.add("expanded");
@ -375,7 +375,7 @@ export class KhojChatView extends KhojPaneView {
referenceButton.tabIndex = 0; referenceButton.tabIndex = 0;
// Add event listener to toggle full reference on click // Add event listener to toggle full reference on click
referenceButton.addEventListener('click', function() { referenceButton.addEventListener('click', function () {
if (this.classList.contains("collapsed")) { if (this.classList.contains("collapsed")) {
this.classList.remove("collapsed"); this.classList.remove("collapsed");
this.classList.add("expanded"); this.classList.add("expanded");
@ -420,23 +420,23 @@ export class KhojChatView extends KhojPaneView {
"Authorization": `Bearer ${this.setting.khojApiKey}`, "Authorization": `Bearer ${this.setting.khojApiKey}`,
}, },
}) })
.then(response => response.arrayBuffer()) .then(response => response.arrayBuffer())
.then(arrayBuffer => context.decodeAudioData(arrayBuffer)) .then(arrayBuffer => context.decodeAudioData(arrayBuffer))
.then(audioBuffer => { .then(audioBuffer => {
const source = context.createBufferSource(); const source = context.createBufferSource();
source.buffer = audioBuffer; source.buffer = audioBuffer;
source.connect(context.destination); source.connect(context.destination);
source.start(0); source.start(0);
source.onended = function() { source.onended = function () {
speechButton.removeChild(loader);
speechButton.disabled = false;
};
})
.catch(err => {
console.error("Error playing speech:", err);
speechButton.removeChild(loader); speechButton.removeChild(loader);
speechButton.disabled = false; speechButton.disabled = false; // Consider enabling the button again to allow retrying
}; });
})
.catch(err => {
console.error("Error playing speech:", err);
speechButton.removeChild(loader);
speechButton.disabled = false; // Consider enabling the button again to allow retrying
});
} }
formatHTMLMessage(message: string, raw = false, willReplace = true) { formatHTMLMessage(message: string, raw = false, willReplace = true) {
@ -485,12 +485,18 @@ export class KhojChatView extends KhojPaneView {
intentType?: string, intentType?: string,
inferredQueries?: string[], inferredQueries?: string[],
conversationId?: string, conversationId?: string,
images?: string[],
excalidrawDiagram?: string
) { ) {
if (!message) return; if (!message) return;
let chatMessageEl; let chatMessageEl;
if (intentType?.includes("text-to-image") || intentType === "excalidraw") { if (
let imageMarkdown = this.generateImageMarkdown(message, intentType, inferredQueries, conversationId); intentType?.includes("text-to-image") ||
intentType === "excalidraw" ||
(images && images.length > 0) ||
excalidrawDiagram) {
let imageMarkdown = this.generateImageMarkdown(message, intentType ?? "", inferredQueries, conversationId, images, excalidrawDiagram);
chatMessageEl = this.renderMessage(chatEl, imageMarkdown, sender, dt); chatMessageEl = this.renderMessage(chatEl, imageMarkdown, sender, dt);
} else { } else {
chatMessageEl = this.renderMessage(chatEl, message, sender, dt); chatMessageEl = this.renderMessage(chatEl, message, sender, dt);
@ -510,7 +516,7 @@ export class KhojChatView extends KhojPaneView {
chatMessageBodyEl.appendChild(this.createReferenceSection(references)); chatMessageBodyEl.appendChild(this.createReferenceSection(references));
} }
generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[], conversationId?: string): string { generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[], conversationId?: string, images?: string[], excalidrawDiagram?: string): string {
let imageMarkdown = ""; let imageMarkdown = "";
if (intentType === "text-to-image") { if (intentType === "text-to-image") {
imageMarkdown = `![](data:image/png;base64,${message})`; imageMarkdown = `![](data:image/png;base64,${message})`;
@ -518,11 +524,20 @@ export class KhojChatView extends KhojPaneView {
imageMarkdown = `![](${message})`; imageMarkdown = `![](${message})`;
} else if (intentType === "text-to-image-v3") { } else if (intentType === "text-to-image-v3") {
imageMarkdown = `![](data:image/webp;base64,${message})`; imageMarkdown = `![](data:image/webp;base64,${message})`;
} else if (intentType === "excalidraw") { } else if (intentType === "excalidraw" || excalidrawDiagram) {
const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`; const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`;
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}chat?conversationId=${conversationId}`; const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}chat?conversationId=${conversationId}`;
imageMarkdown = redirectMessage; imageMarkdown = redirectMessage;
} else if (images && images.length > 0) {
for (let image of images) {
if (image.startsWith("https://")) {
imageMarkdown += `![](${image})\n\n`;
} else {
imageMarkdown += `![](data:image/png;base64,${image})\n\n`;
}
}
} }
if (inferredQueries) { if (inferredQueries) {
imageMarkdown += "\n\n**Inferred Query**:"; imageMarkdown += "\n\n**Inferred Query**:";
for (let inferredQuery of inferredQueries) { for (let inferredQuery of inferredQueries) {
@ -650,19 +665,19 @@ export class KhojChatView extends KhojPaneView {
chatBodyEl.innerHTML = ""; chatBodyEl.innerHTML = "";
chatBodyEl.dataset.conversationId = ""; chatBodyEl.dataset.conversationId = "";
chatBodyEl.dataset.conversationTitle = ""; chatBodyEl.dataset.conversationTitle = "";
this.userMessages = []; this.userMessages = [];
this.startingMessage = "Message"; this.startingMessage = "Message";
// Update the placeholder of the chat input // Update the placeholder of the chat input
const chatInput = this.contentEl.querySelector('.khoj-chat-input') as HTMLTextAreaElement; const chatInput = this.contentEl.querySelector('.khoj-chat-input') as HTMLTextAreaElement;
if (chatInput) { if (chatInput) {
chatInput.placeholder = this.startingMessage; chatInput.placeholder = this.startingMessage;
} }
this.renderMessage(chatBodyEl, "Hey 👋🏾, what's up?", "khoj"); this.renderMessage(chatBodyEl, "Hey 👋🏾, what's up?", "khoj");
} }
async toggleChatSessions(forceShow: boolean = false): Promise<boolean> { async toggleChatSessions(forceShow: boolean = false): Promise<boolean> {
this.userMessages = []; // clear user previous message history this.userMessages = []; // clear user previous message history
let chatBodyEl = this.contentEl.getElementsByClassName("khoj-chat-body")[0] as HTMLElement; let chatBodyEl = this.contentEl.getElementsByClassName("khoj-chat-body")[0] as HTMLElement;
if (!forceShow && this.contentEl.getElementsByClassName("side-panel")?.length > 0) { if (!forceShow && this.contentEl.getElementsByClassName("side-panel")?.length > 0) {
chatBodyEl.innerHTML = ""; chatBodyEl.innerHTML = "";
@ -768,10 +783,10 @@ export class KhojChatView extends KhojPaneView {
let editConversationTitleInputEl = this.contentEl.createEl('input'); let editConversationTitleInputEl = this.contentEl.createEl('input');
editConversationTitleInputEl.classList.add("conversation-title-input"); editConversationTitleInputEl.classList.add("conversation-title-input");
editConversationTitleInputEl.value = conversationTitle; editConversationTitleInputEl.value = conversationTitle;
editConversationTitleInputEl.addEventListener('click', function(event) { editConversationTitleInputEl.addEventListener('click', function (event) {
event.stopPropagation(); event.stopPropagation();
}); });
editConversationTitleInputEl.addEventListener('keydown', function(event) { editConversationTitleInputEl.addEventListener('keydown', function (event) {
if (event.key === "Enter") { if (event.key === "Enter") {
event.preventDefault(); event.preventDefault();
editConversationTitleSaveButtonEl.click(); editConversationTitleSaveButtonEl.click();
@ -890,15 +905,17 @@ export class KhojChatView extends KhojPaneView {
chatLog.intent?.type, chatLog.intent?.type,
chatLog.intent?.["inferred-queries"], chatLog.intent?.["inferred-queries"],
chatBodyEl.dataset.conversationId ?? "", chatBodyEl.dataset.conversationId ?? "",
chatLog.images,
chatLog.excalidrawDiagram,
); );
// push the user messages to the chat history // push the user messages to the chat history
if(chatLog.by === "you"){ if (chatLog.by === "you") {
this.userMessages.push(chatLog.message); this.userMessages.push(chatLog.message);
} }
}); });
// Update starting message after loading history // Update starting message after loading history
const modifierKey: string = Platform.isMacOS ? '⌘' : '^'; const modifierKey: string = Platform.isMacOS ? '⌘' : '^';
this.startingMessage = this.userMessages.length > 0 this.startingMessage = this.userMessages.length > 0
? `(${modifierKey}+↑/↓) for prev messages` ? `(${modifierKey}+↑/↓) for prev messages`
: "Message"; : "Message";
@ -922,15 +939,15 @@ export class KhojChatView extends KhojPaneView {
try { try {
let jsonChunk = JSON.parse(rawChunk); let jsonChunk = JSON.parse(rawChunk);
if (!jsonChunk.type) if (!jsonChunk.type)
jsonChunk = {type: 'message', data: jsonChunk}; jsonChunk = { type: 'message', data: jsonChunk };
return jsonChunk; return jsonChunk;
} catch (e) { } catch (e) {
return {type: 'message', data: rawChunk}; return { type: 'message', data: rawChunk };
} }
} else if (rawChunk.length > 0) { } else if (rawChunk.length > 0) {
return {type: 'message', data: rawChunk}; return { type: 'message', data: rawChunk };
} }
return {type: '', data: ''}; return { type: '', data: '' };
} }
processMessageChunk(rawChunk: string): void { processMessageChunk(rawChunk: string): void {
@ -965,7 +982,7 @@ export class KhojChatView extends KhojPaneView {
isVoice: false, isVoice: false,
}; };
} else if (chunk.type === "references") { } else if (chunk.type === "references") {
this.chatMessageState.references = {"notes": chunk.data.context, "online": chunk.data.onlineContext}; this.chatMessageState.references = { "notes": chunk.data.context, "online": chunk.data.onlineContext };
} else if (chunk.type === 'message') { } else if (chunk.type === 'message') {
const chunkData = chunk.data; const chunkData = chunk.data;
if (typeof chunkData === 'object' && chunkData !== null) { if (typeof chunkData === 'object' && chunkData !== null) {
@ -988,7 +1005,7 @@ export class KhojChatView extends KhojPaneView {
} }
handleJsonResponse(jsonData: any): void { handleJsonResponse(jsonData: any): void {
if (jsonData.image || jsonData.detail) { if (jsonData.image || jsonData.detail || jsonData.images || jsonData.excalidrawDiagram) {
this.chatMessageState.rawResponse = this.handleImageResponse(jsonData, this.chatMessageState.rawResponse); this.chatMessageState.rawResponse = this.handleImageResponse(jsonData, this.chatMessageState.rawResponse);
} else if (jsonData.response) { } else if (jsonData.response) {
this.chatMessageState.rawResponse = jsonData.response; this.chatMessageState.rawResponse = jsonData.response;
@ -1234,11 +1251,11 @@ export class KhojChatView extends KhojPaneView {
const recordingConfig = { mimeType: 'audio/webm' }; const recordingConfig = { mimeType: 'audio/webm' };
this.mediaRecorder = new MediaRecorder(stream, recordingConfig); this.mediaRecorder = new MediaRecorder(stream, recordingConfig);
this.mediaRecorder.addEventListener("dataavailable", function(event) { this.mediaRecorder.addEventListener("dataavailable", function (event) {
if (event.data.size > 0) audioChunks.push(event.data); if (event.data.size > 0) audioChunks.push(event.data);
}); });
this.mediaRecorder.addEventListener("stop", async function() { this.mediaRecorder.addEventListener("stop", async function () {
const audioBlob = new Blob(audioChunks, { type: 'audio/webm' }); const audioBlob = new Blob(audioChunks, { type: 'audio/webm' });
await sendToServer(audioBlob); await sendToServer(audioBlob);
}); });
@ -1368,7 +1385,22 @@ export class KhojChatView extends KhojPaneView {
if (inferredQuery) { if (inferredQuery) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`; rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
} }
} else if (imageJson.images) {
// If response has images field, response is a list of generated images.
imageJson.images.forEach((image: any) => {
if (image.startsWith("http")) {
rawResponse += `![generated_image](${image})\n\n`;
} else {
rawResponse += `![generated_image](data:image/png;base64,${image})\n\n`;
}
});
} else if (imageJson.excalidrawDiagram) {
const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`;
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}`;
rawResponse += redirectMessage;
} }
// If response has detail field, response is an error message. // If response has detail field, response is an error message.
if (imageJson.detail) rawResponse += imageJson.detail; if (imageJson.detail) rawResponse += imageJson.detail;
@ -1407,7 +1439,7 @@ export class KhojChatView extends KhojPaneView {
referenceExpandButton.classList.add("reference-expand-button"); referenceExpandButton.classList.add("reference-expand-button");
referenceExpandButton.innerHTML = numReferences == 1 ? "1 reference" : `${numReferences} references`; referenceExpandButton.innerHTML = numReferences == 1 ? "1 reference" : `${numReferences} references`;
referenceExpandButton.addEventListener('click', function() { referenceExpandButton.addEventListener('click', function () {
if (referenceSection.classList.contains("collapsed")) { if (referenceSection.classList.contains("collapsed")) {
referenceSection.classList.remove("collapsed"); referenceSection.classList.remove("collapsed");
referenceSection.classList.add("expanded"); referenceSection.classList.add("expanded");

6
src/interface/obsidian/styles.css
View file

@ -82,7 +82,8 @@ If your plugin does not need CSS, delete this file.
} }
/* color chat bubble by khoj blue */ /* color chat bubble by khoj blue */
.khoj-chat-message-text.khoj { .khoj-chat-message-text.khoj {
border: 1px solid var(--khoj-sun); border-top: 1px solid var(--khoj-sun);
border-radius: 0px;
margin-left: auto; margin-left: auto;
white-space: pre-line; white-space: pre-line;
} }
@ -104,8 +105,9 @@ If your plugin does not need CSS, delete this file.
} }
/* color chat bubble by you dark grey */ /* color chat bubble by you dark grey */
.khoj-chat-message-text.you { .khoj-chat-message-text.you {
border: 1px solid var(--color-accent); color: var(--text-normal);
margin-right: auto; margin-right: auto;
background-color: var(--background-modifier-cover);
} }
/* add right protrusion to you chat bubble */ /* add right protrusion to you chat bubble */
.khoj-chat-message-text.you:after { .khoj-chat-message-text.you:after {