sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-12-18 18:47:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 23

Update response handling in Obsidian to work with new format

Browse source
This commit is contained in:
sabaimran 2024-11-29 18:10:47 -08:00
parent 512cf535e0
commit df855adc98
2 changed files with 97 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
src/interface/obsidian/src/chat_view.ts
View file

@ -1,4 +1,4 @@
import {ItemView, MarkdownRenderer, Scope, WorkspaceLeaf, request, requestUrl, setIcon, Platform} from 'obsidian';
import { ItemView, MarkdownRenderer, Scope, WorkspaceLeaf, request, requestUrl, setIcon, Platform } from 'obsidian';
import * as DOMPurify from 'dompurify';
import { KhojSetting } from 'src/settings';
import { KhojPaneView } from 'src/pane_view';
@ -199,7 +199,7 @@ export class KhojChatView extends KhojPaneView {
// Get chat history from Khoj backend and set chat input state
let getChatHistorySucessfully = await this.getChatHistory(chatBodyEl);
let placeholderText : string = getChatHistorySucessfully ? this.startingMessage : "Configure Khoj to enable chat";
let placeholderText: string = getChatHistorySucessfully ? this.startingMessage : "Configure Khoj to enable chat";
chatInput.placeholder = placeholderText;
chatInput.disabled = !getChatHistorySucessfully;
@ -214,7 +214,7 @@ export class KhojChatView extends KhojPaneView {
});
}
startSpeechToText(event: KeyboardEvent | MouseEvent | TouchEvent, timeout=200) {
startSpeechToText(event: KeyboardEvent | MouseEvent | TouchEvent, timeout = 200) {
if (!this.keyPressTimeout) {
this.keyPressTimeout = setTimeout(async () => {
// Reset auto send voice message timer, UI if running
@ -320,7 +320,7 @@ export class KhojChatView extends KhojPaneView {
referenceButton.tabIndex = 0;
// Add event listener to toggle full reference on click
referenceButton.addEventListener('click', function() {
referenceButton.addEventListener('click', function () {
if (this.classList.contains("collapsed")) {
this.classList.remove("collapsed");
this.classList.add("expanded");
@ -375,7 +375,7 @@ export class KhojChatView extends KhojPaneView {
referenceButton.tabIndex = 0;
// Add event listener to toggle full reference on click
referenceButton.addEventListener('click', function() {
referenceButton.addEventListener('click', function () {
if (this.classList.contains("collapsed")) {
this.classList.remove("collapsed");
this.classList.add("expanded");
@ -427,7 +427,7 @@ export class KhojChatView extends KhojPaneView {
source.buffer = audioBuffer;
source.connect(context.destination);
source.start(0);
source.onended = function() {
source.onended = function () {
speechButton.removeChild(loader);
speechButton.disabled = false;
};
@ -485,12 +485,18 @@ export class KhojChatView extends KhojPaneView {
intentType?: string,
inferredQueries?: string[],
conversationId?: string,
images?: string[],
excalidrawDiagram?: string
) {
if (!message) return;
let chatMessageEl;
if (intentType?.includes("text-to-image") || intentType === "excalidraw") {
let imageMarkdown = this.generateImageMarkdown(message, intentType, inferredQueries, conversationId);
if (
intentType?.includes("text-to-image") ||
intentType === "excalidraw" ||
(images && images.length > 0) ||
excalidrawDiagram) {
let imageMarkdown = this.generateImageMarkdown(message, intentType ?? "", inferredQueries, conversationId, images, excalidrawDiagram);
chatMessageEl = this.renderMessage(chatEl, imageMarkdown, sender, dt);
} else {
chatMessageEl = this.renderMessage(chatEl, message, sender, dt);
@ -510,7 +516,7 @@ export class KhojChatView extends KhojPaneView {
chatMessageBodyEl.appendChild(this.createReferenceSection(references));
}
generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[], conversationId?: string): string {
generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[], conversationId?: string, images?: string[], excalidrawDiagram?: string): string {
let imageMarkdown = "";
if (intentType === "text-to-image") {
imageMarkdown = `![](data:image/png;base64,${message})`;
@ -518,11 +524,20 @@ export class KhojChatView extends KhojPaneView {
imageMarkdown = `![](${message})`;
} else if (intentType === "text-to-image-v3") {
imageMarkdown = `![](data:image/webp;base64,${message})`;
} else if (intentType === "excalidraw") {
} else if (intentType === "excalidraw" || excalidrawDiagram) {
const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`;
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}chat?conversationId=${conversationId}`;
imageMarkdown = redirectMessage;
} else if (images && images.length > 0) {
for (let image of images) {
if (image.startsWith("https://")) {
imageMarkdown += `![](${image})\n\n`;
} else {
imageMarkdown += `![](data:image/png;base64,${image})\n\n`;
}
}
}
if (inferredQueries) {
imageMarkdown += "\n\n**Inferred Query**:";
for (let inferredQuery of inferredQueries) {
@ -768,10 +783,10 @@ export class KhojChatView extends KhojPaneView {
let editConversationTitleInputEl = this.contentEl.createEl('input');
editConversationTitleInputEl.classList.add("conversation-title-input");
editConversationTitleInputEl.value = conversationTitle;
editConversationTitleInputEl.addEventListener('click', function(event) {
editConversationTitleInputEl.addEventListener('click', function (event) {
event.stopPropagation();
});
editConversationTitleInputEl.addEventListener('keydown', function(event) {
editConversationTitleInputEl.addEventListener('keydown', function (event) {
if (event.key === "Enter") {
event.preventDefault();
editConversationTitleSaveButtonEl.click();
@ -890,9 +905,11 @@ export class KhojChatView extends KhojPaneView {
chatLog.intent?.type,
chatLog.intent?.["inferred-queries"],
chatBodyEl.dataset.conversationId ?? "",
chatLog.images,
chatLog.excalidrawDiagram,
);
// push the user messages to the chat history
if(chatLog.by === "you"){
if (chatLog.by === "you") {
this.userMessages.push(chatLog.message);
}
});
@ -922,15 +939,15 @@ export class KhojChatView extends KhojPaneView {
try {
let jsonChunk = JSON.parse(rawChunk);
if (!jsonChunk.type)
jsonChunk = {type: 'message', data: jsonChunk};
jsonChunk = { type: 'message', data: jsonChunk };
return jsonChunk;
} catch (e) {
return {type: 'message', data: rawChunk};
return { type: 'message', data: rawChunk };
}
} else if (rawChunk.length > 0) {
return {type: 'message', data: rawChunk};
return { type: 'message', data: rawChunk };
}
return {type: '', data: ''};
return { type: '', data: '' };
}
processMessageChunk(rawChunk: string): void {
@ -965,7 +982,7 @@ export class KhojChatView extends KhojPaneView {
isVoice: false,
};
} else if (chunk.type === "references") {
this.chatMessageState.references = {"notes": chunk.data.context, "online": chunk.data.onlineContext};
this.chatMessageState.references = { "notes": chunk.data.context, "online": chunk.data.onlineContext };
} else if (chunk.type === 'message') {
const chunkData = chunk.data;
if (typeof chunkData === 'object' && chunkData !== null) {
@ -988,7 +1005,7 @@ export class KhojChatView extends KhojPaneView {
}
handleJsonResponse(jsonData: any): void {
if (jsonData.image || jsonData.detail) {
if (jsonData.image || jsonData.detail || jsonData.images || jsonData.excalidrawDiagram) {
this.chatMessageState.rawResponse = this.handleImageResponse(jsonData, this.chatMessageState.rawResponse);
} else if (jsonData.response) {
this.chatMessageState.rawResponse = jsonData.response;
@ -1234,11 +1251,11 @@ export class KhojChatView extends KhojPaneView {
const recordingConfig = { mimeType: 'audio/webm' };
this.mediaRecorder = new MediaRecorder(stream, recordingConfig);
this.mediaRecorder.addEventListener("dataavailable", function(event) {
this.mediaRecorder.addEventListener("dataavailable", function (event) {
if (event.data.size > 0) audioChunks.push(event.data);
});
this.mediaRecorder.addEventListener("stop", async function() {
this.mediaRecorder.addEventListener("stop", async function () {
const audioBlob = new Blob(audioChunks, { type: 'audio/webm' });
await sendToServer(audioBlob);
});
@ -1368,7 +1385,22 @@ export class KhojChatView extends KhojPaneView {
if (inferredQuery) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
} else if (imageJson.images) {
// If response has images field, response is a list of generated images.
imageJson.images.forEach((image: any) => {
if (image.startsWith("http")) {
rawResponse += `![generated_image](${image})\n\n`;
} else {
rawResponse += `![generated_image](data:image/png;base64,${image})\n\n`;
}
});
} else if (imageJson.excalidrawDiagram) {
const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`;
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}`;
rawResponse += redirectMessage;
}
// If response has detail field, response is an error message.
if (imageJson.detail) rawResponse += imageJson.detail;
@ -1407,7 +1439,7 @@ export class KhojChatView extends KhojPaneView {
referenceExpandButton.classList.add("reference-expand-button");
referenceExpandButton.innerHTML = numReferences == 1 ? "1 reference" : `${numReferences} references`;
referenceExpandButton.addEventListener('click', function() {
referenceExpandButton.addEventListener('click', function () {
if (referenceSection.classList.contains("collapsed")) {
referenceSection.classList.remove("collapsed");
referenceSection.classList.add("expanded");

6
src/interface/obsidian/styles.css
View file

@ -82,7 +82,8 @@ If your plugin does not need CSS, delete this file.
}
/* color chat bubble by khoj blue */
.khoj-chat-message-text.khoj {
border: 1px solid var(--khoj-sun);
border-top: 1px solid var(--khoj-sun);
border-radius: 0px;
margin-left: auto;
white-space: pre-line;
}
@ -104,8 +105,9 @@ If your plugin does not need CSS, delete this file.
}
/* color chat bubble by you dark grey */
.khoj-chat-message-text.you {
border: 1px solid var(--color-accent);
color: var(--text-normal);
margin-right: auto;
background-color: var(--background-modifier-cover);
}
/* add right protrusion to you chat bubble */
.khoj-chat-message-text.you:after {