sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-12-18 18:47:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 23

Update response handling in Obsidian to work with new format

Browse source
This commit is contained in:
sabaimran 2024-11-29 18:10:47 -08:00
parent 512cf535e0
commit df855adc98
2 changed files with 97 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
src/interface/obsidian/src/chat_view.ts
View file

@ -1,4 +1,4 @@
import {ItemView, MarkdownRenderer, Scope, WorkspaceLeaf, request, requestUrl, setIcon, Platform} from 'obsidian'; import { ItemView, MarkdownRenderer, Scope, WorkspaceLeaf, request, requestUrl, setIcon, Platform } from 'obsidian';
import * as DOMPurify from 'dompurify'; import * as DOMPurify from 'dompurify';
import { KhojSetting } from 'src/settings'; import { KhojSetting } from 'src/settings';
import { KhojPaneView } from 'src/pane_view'; import { KhojPaneView } from 'src/pane_view';
@ -199,7 +199,7 @@ export class KhojChatView extends KhojPaneView {
// Get chat history from Khoj backend and set chat input state // Get chat history from Khoj backend and set chat input state
let getChatHistorySucessfully = await this.getChatHistory(chatBodyEl); let getChatHistorySucessfully = await this.getChatHistory(chatBodyEl);
let placeholderText : string = getChatHistorySucessfully ? this.startingMessage : "Configure Khoj to enable chat"; let placeholderText: string = getChatHistorySucessfully ? this.startingMessage : "Configure Khoj to enable chat";
chatInput.placeholder = placeholderText; chatInput.placeholder = placeholderText;
chatInput.disabled = !getChatHistorySucessfully; chatInput.disabled = !getChatHistorySucessfully;
@ -214,7 +214,7 @@ export class KhojChatView extends KhojPaneView {
}); });
} }
startSpeechToText(event: KeyboardEvent | MouseEvent | TouchEvent, timeout=200) { startSpeechToText(event: KeyboardEvent | MouseEvent | TouchEvent, timeout = 200) {
if (!this.keyPressTimeout) { if (!this.keyPressTimeout) {
this.keyPressTimeout = setTimeout(async () => { this.keyPressTimeout = setTimeout(async () => {
// Reset auto send voice message timer, UI if running // Reset auto send voice message timer, UI if running
@ -320,7 +320,7 @@ export class KhojChatView extends KhojPaneView {
referenceButton.tabIndex = 0; referenceButton.tabIndex = 0;
// Add event listener to toggle full reference on click // Add event listener to toggle full reference on click
referenceButton.addEventListener('click', function() { referenceButton.addEventListener('click', function () {
if (this.classList.contains("collapsed")) { if (this.classList.contains("collapsed")) {
this.classList.remove("collapsed"); this.classList.remove("collapsed");
this.classList.add("expanded"); this.classList.add("expanded");
@ -375,7 +375,7 @@ export class KhojChatView extends KhojPaneView {
referenceButton.tabIndex = 0; referenceButton.tabIndex = 0;
// Add event listener to toggle full reference on click // Add event listener to toggle full reference on click
referenceButton.addEventListener('click', function() { referenceButton.addEventListener('click', function () {
if (this.classList.contains("collapsed")) { if (this.classList.contains("collapsed")) {
this.classList.remove("collapsed"); this.classList.remove("collapsed");
this.classList.add("expanded"); this.classList.add("expanded");
@ -427,7 +427,7 @@ export class KhojChatView extends KhojPaneView {
source.buffer = audioBuffer; source.buffer = audioBuffer;
source.connect(context.destination); source.connect(context.destination);
source.start(0); source.start(0);
source.onended = function() { source.onended = function () {
speechButton.removeChild(loader); speechButton.removeChild(loader);
speechButton.disabled = false; speechButton.disabled = false;
}; };
@ -485,12 +485,18 @@ export class KhojChatView extends KhojPaneView {
intentType?: string, intentType?: string,
inferredQueries?: string[], inferredQueries?: string[],
conversationId?: string, conversationId?: string,
images?: string[],
excalidrawDiagram?: string
) { ) {
if (!message) return; if (!message) return;
let chatMessageEl; let chatMessageEl;
if (intentType?.includes("text-to-image") || intentType === "excalidraw") { if (
let imageMarkdown = this.generateImageMarkdown(message, intentType, inferredQueries, conversationId); intentType?.includes("text-to-image") ||
intentType === "excalidraw" ||
(images && images.length > 0) ||
excalidrawDiagram) {
let imageMarkdown = this.generateImageMarkdown(message, intentType ?? "", inferredQueries, conversationId, images, excalidrawDiagram);
chatMessageEl = this.renderMessage(chatEl, imageMarkdown, sender, dt); chatMessageEl = this.renderMessage(chatEl, imageMarkdown, sender, dt);
} else { } else {
chatMessageEl = this.renderMessage(chatEl, message, sender, dt); chatMessageEl = this.renderMessage(chatEl, message, sender, dt);
@ -510,7 +516,7 @@ export class KhojChatView extends KhojPaneView {
chatMessageBodyEl.appendChild(this.createReferenceSection(references)); chatMessageBodyEl.appendChild(this.createReferenceSection(references));
} }
generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[], conversationId?: string): string { generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[], conversationId?: string, images?: string[], excalidrawDiagram?: string): string {
let imageMarkdown = ""; let imageMarkdown = "";
if (intentType === "text-to-image") { if (intentType === "text-to-image") {
imageMarkdown = `![](data:image/png;base64,${message})`; imageMarkdown = `![](data:image/png;base64,${message})`;
@ -518,11 +524,20 @@ export class KhojChatView extends KhojPaneView {
imageMarkdown = `![](${message})`; imageMarkdown = `![](${message})`;
} else if (intentType === "text-to-image-v3") { } else if (intentType === "text-to-image-v3") {
imageMarkdown = `![](data:image/webp;base64,${message})`; imageMarkdown = `![](data:image/webp;base64,${message})`;
} else if (intentType === "excalidraw") { } else if (intentType === "excalidraw" || excalidrawDiagram) {
const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`; const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`;
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}chat?conversationId=${conversationId}`; const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}chat?conversationId=${conversationId}`;
imageMarkdown = redirectMessage; imageMarkdown = redirectMessage;
} else if (images && images.length > 0) {
for (let image of images) {
if (image.startsWith("https://")) {
imageMarkdown += `![](${image})\n\n`;
} else {
imageMarkdown += `![](data:image/png;base64,${image})\n\n`;
} }
}
}
if (inferredQueries) { if (inferredQueries) {
imageMarkdown += "\n\n**Inferred Query**:"; imageMarkdown += "\n\n**Inferred Query**:";
for (let inferredQuery of inferredQueries) { for (let inferredQuery of inferredQueries) {
@ -768,10 +783,10 @@ export class KhojChatView extends KhojPaneView {
let editConversationTitleInputEl = this.contentEl.createEl('input'); let editConversationTitleInputEl = this.contentEl.createEl('input');
editConversationTitleInputEl.classList.add("conversation-title-input"); editConversationTitleInputEl.classList.add("conversation-title-input");
editConversationTitleInputEl.value = conversationTitle; editConversationTitleInputEl.value = conversationTitle;
editConversationTitleInputEl.addEventListener('click', function(event) { editConversationTitleInputEl.addEventListener('click', function (event) {
event.stopPropagation(); event.stopPropagation();
}); });
editConversationTitleInputEl.addEventListener('keydown', function(event) { editConversationTitleInputEl.addEventListener('keydown', function (event) {
if (event.key === "Enter") { if (event.key === "Enter") {
event.preventDefault(); event.preventDefault();
editConversationTitleSaveButtonEl.click(); editConversationTitleSaveButtonEl.click();
@ -890,9 +905,11 @@ export class KhojChatView extends KhojPaneView {
chatLog.intent?.type, chatLog.intent?.type,
chatLog.intent?.["inferred-queries"], chatLog.intent?.["inferred-queries"],
chatBodyEl.dataset.conversationId ?? "", chatBodyEl.dataset.conversationId ?? "",
chatLog.images,
chatLog.excalidrawDiagram,
); );
// push the user messages to the chat history // push the user messages to the chat history
if(chatLog.by === "you"){ if (chatLog.by === "you") {
this.userMessages.push(chatLog.message); this.userMessages.push(chatLog.message);
} }
}); });
@ -922,15 +939,15 @@ export class KhojChatView extends KhojPaneView {
try { try {
let jsonChunk = JSON.parse(rawChunk); let jsonChunk = JSON.parse(rawChunk);
if (!jsonChunk.type) if (!jsonChunk.type)
jsonChunk = {type: 'message', data: jsonChunk}; jsonChunk = { type: 'message', data: jsonChunk };
return jsonChunk; return jsonChunk;
} catch (e) { } catch (e) {
return {type: 'message', data: rawChunk}; return { type: 'message', data: rawChunk };
} }
} else if (rawChunk.length > 0) { } else if (rawChunk.length > 0) {
return {type: 'message', data: rawChunk}; return { type: 'message', data: rawChunk };
} }
return {type: '', data: ''}; return { type: '', data: '' };
} }
processMessageChunk(rawChunk: string): void { processMessageChunk(rawChunk: string): void {
@ -965,7 +982,7 @@ export class KhojChatView extends KhojPaneView {
isVoice: false, isVoice: false,
}; };
} else if (chunk.type === "references") { } else if (chunk.type === "references") {
this.chatMessageState.references = {"notes": chunk.data.context, "online": chunk.data.onlineContext}; this.chatMessageState.references = { "notes": chunk.data.context, "online": chunk.data.onlineContext };
} else if (chunk.type === 'message') { } else if (chunk.type === 'message') {
const chunkData = chunk.data; const chunkData = chunk.data;
if (typeof chunkData === 'object' && chunkData !== null) { if (typeof chunkData === 'object' && chunkData !== null) {
@ -988,7 +1005,7 @@ export class KhojChatView extends KhojPaneView {
} }
handleJsonResponse(jsonData: any): void { handleJsonResponse(jsonData: any): void {
if (jsonData.image || jsonData.detail) { if (jsonData.image || jsonData.detail || jsonData.images || jsonData.excalidrawDiagram) {
this.chatMessageState.rawResponse = this.handleImageResponse(jsonData, this.chatMessageState.rawResponse); this.chatMessageState.rawResponse = this.handleImageResponse(jsonData, this.chatMessageState.rawResponse);
} else if (jsonData.response) { } else if (jsonData.response) {
this.chatMessageState.rawResponse = jsonData.response; this.chatMessageState.rawResponse = jsonData.response;
@ -1234,11 +1251,11 @@ export class KhojChatView extends KhojPaneView {
const recordingConfig = { mimeType: 'audio/webm' }; const recordingConfig = { mimeType: 'audio/webm' };
this.mediaRecorder = new MediaRecorder(stream, recordingConfig); this.mediaRecorder = new MediaRecorder(stream, recordingConfig);
this.mediaRecorder.addEventListener("dataavailable", function(event) { this.mediaRecorder.addEventListener("dataavailable", function (event) {
if (event.data.size > 0) audioChunks.push(event.data); if (event.data.size > 0) audioChunks.push(event.data);
}); });
this.mediaRecorder.addEventListener("stop", async function() { this.mediaRecorder.addEventListener("stop", async function () {
const audioBlob = new Blob(audioChunks, { type: 'audio/webm' }); const audioBlob = new Blob(audioChunks, { type: 'audio/webm' });
await sendToServer(audioBlob); await sendToServer(audioBlob);
}); });
@ -1368,7 +1385,22 @@ export class KhojChatView extends KhojPaneView {
if (inferredQuery) { if (inferredQuery) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`; rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
} }
} else if (imageJson.images) {
// If response has images field, response is a list of generated images.
imageJson.images.forEach((image: any) => {
if (image.startsWith("http")) {
rawResponse += `![generated_image](${image})\n\n`;
} else {
rawResponse += `![generated_image](data:image/png;base64,${image})\n\n`;
} }
});
} else if (imageJson.excalidrawDiagram) {
const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`;
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}`;
rawResponse += redirectMessage;
}
// If response has detail field, response is an error message. // If response has detail field, response is an error message.
if (imageJson.detail) rawResponse += imageJson.detail; if (imageJson.detail) rawResponse += imageJson.detail;
@ -1407,7 +1439,7 @@ export class KhojChatView extends KhojPaneView {
referenceExpandButton.classList.add("reference-expand-button"); referenceExpandButton.classList.add("reference-expand-button");
referenceExpandButton.innerHTML = numReferences == 1 ? "1 reference" : `${numReferences} references`; referenceExpandButton.innerHTML = numReferences == 1 ? "1 reference" : `${numReferences} references`;
referenceExpandButton.addEventListener('click', function() { referenceExpandButton.addEventListener('click', function () {
if (referenceSection.classList.contains("collapsed")) { if (referenceSection.classList.contains("collapsed")) {
referenceSection.classList.remove("collapsed"); referenceSection.classList.remove("collapsed");
referenceSection.classList.add("expanded"); referenceSection.classList.add("expanded");

6
src/interface/obsidian/styles.css
View file

@ -82,7 +82,8 @@ If your plugin does not need CSS, delete this file.
} }
/* color chat bubble by khoj blue */ /* color chat bubble by khoj blue */
.khoj-chat-message-text.khoj { .khoj-chat-message-text.khoj {
border: 1px solid var(--khoj-sun); border-top: 1px solid var(--khoj-sun);
border-radius: 0px;
margin-left: auto; margin-left: auto;
white-space: pre-line; white-space: pre-line;
} }
@ -104,8 +105,9 @@ If your plugin does not need CSS, delete this file.
} }
/* color chat bubble by you dark grey */ /* color chat bubble by you dark grey */
.khoj-chat-message-text.you { .khoj-chat-message-text.you {
border: 1px solid var(--color-accent); color: var(--text-normal);
margin-right: auto; margin-right: auto;
background-color: var(--background-modifier-cover);
} }
/* add right protrusion to you chat bubble */ /* add right protrusion to you chat bubble */
.khoj-chat-message-text.you:after { .khoj-chat-message-text.you:after {