Fix, Improve rendering images in Obsidian, Desktop, Web clients (#659)

- Improve render of inferred query in image chat messages in Web, Desktop apps
- Add inferred queries to image chat responses in Obsidian client
- Fix rendering images from Khoj response in Obsidian client

src/interface/desktop/chat.html

@@ -199,10 +199,9 @@
         function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) {
             if (intentType === "text-to-image") {
                 let imageMarkdown = `![](data:image/png;base64,${message})`;
-                imageMarkdown += "\n\n";
-                if (inferredQueries) {
-                    const inferredQuery = inferredQueries?.[0];
-                    imageMarkdown += `**Inferred Query**: ${inferredQuery}`;
+                const inferredQuery = inferredQueries?.[0];
+                if (inferredQuery) {
+                    imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
                 }
                 renderMessage(imageMarkdown, by, dt);
                 return;
@@ -392,10 +391,9 @@
                     if (responseAsJson.image) {
                         // If response has image field, response is a generated image.
                         rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`;
-                        rawResponse += "\n\n";
                         const inferredQueries = responseAsJson.inferredQueries?.[0];
                         if (inferredQueries) {
-                            rawResponse += `**Inferred Query**: ${inferredQueries}`;
+                            rawResponse += `\n\n**Inferred Query**:\n\n${inferredQueries}`;
                         }
                     }
                     if (responseAsJson.detail) {
@@ -496,10 +494,9 @@
                                     if (responseAsJson.image) {
                                         // If response has image field, response is a generated image.
                                         rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`;
-                                        rawResponse += "\n\n";
-                                        const inferredQueries = responseAsJson.inferredQueries?.[0];
-                                        if (inferredQueries) {
-                                            rawResponse += `**Inferred Query**: ${inferredQueries}`;
+                                        const inferredQuery = responseAsJson.inferredQueries?.[0];
+                                        if (inferredQuery) {
+                                            rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
                                         }
                                     }
                                     if (responseAsJson.detail) {

src/interface/obsidian/src/chat_modal.ts

@@ -4,6 +4,8 @@ import { KhojSetting } from 'src/settings';
 export interface ChatJsonResult {
     image?: string;
     detail?: string;
+    intentType?: string;
+    inferredQueries?: string[];
 }
 
 
@@ -145,11 +147,17 @@ export class KhojChatModal extends Modal {
         return referenceButton;
     }
 
-    renderMessageWithReferences(chatEl: Element, message: string, sender: string, context?: string[], dt?: Date, intentType?: string) {
+    renderMessageWithReferences(chatEl: Element, message: string, sender: string, context?: string[], dt?: Date, intentType?: string, inferredQueries?: string) {
         if (!message) {
             return;
         } else if (intentType === "text-to-image") {
             let imageMarkdown = `![](data:image/png;base64,${message})`;
+            if (inferredQueries) {
+                imageMarkdown += "\n\n**Inferred Query**:";
+                for (let inferredQuery of inferredQueries) {
+                    imageMarkdown += `\n\n${inferredQuery}`;
+                }
+            }
             this.renderMessage(chatEl, imageMarkdown, sender, dt);
             return;
         } else if (!context) {
@@ -287,7 +295,15 @@ export class KhojChatModal extends Modal {
             } else if (responseJson.response) {
                 let chatLogs = responseJson.response?.conversation_id ? responseJson.response.chat ?? [] : responseJson.response;
                 chatLogs.forEach((chatLog: any) => {
-                    this.renderMessageWithReferences(chatBodyEl, chatLog.message, chatLog.by, chatLog.context, new Date(chatLog.created), chatLog.intent?.type);
+                    this.renderMessageWithReferences(
+                        chatBodyEl,
+                        chatLog.message,
+                        chatLog.by,
+                        chatLog.context,
+                        new Date(chatLog.created),
+                        chatLog.intent?.type,
+                        chatLog.intent?.["inferred-queries"],
+                    );
                 });
             }
         } catch (err) {
@@ -398,12 +414,16 @@ export class KhojChatModal extends Modal {
             // Reset collated chat result to empty string
             this.result = "";
             responseElement.innerHTML = "";
-            if (response.headers.get("content-type") == "application/json") {
+            if (response.headers.get("content-type") === "application/json") {
                 let responseText = ""
                 try {
                     const responseAsJson = await response.json() as ChatJsonResult;
                     if (responseAsJson.image) {
                         responseText = `![${query}](data:image/png;base64,${responseAsJson.image})`;
+                        const inferredQuery = responseAsJson.inferredQueries?.[0];
+                        if (inferredQuery) {
+                            responseText += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
+                        }
                     } else if (responseAsJson.detail) {
                         responseText = responseAsJson.detail;
                     }
@@ -413,10 +433,10 @@ export class KhojChatModal extends Modal {
                 } finally {
                     await this.renderIncrementalMessage(responseElement, responseText);
                 }
+            } else {
+                // Stream and render chat response
+                await this.readChatStream(response, responseElement);
             }
-
-            // Stream and render chat response
-            await this.readChatStream(response, responseElement);
         } catch (err) {
             console.log(`Khoj chat response failed with\n${err}`);
             let errorMsg = "Sorry, unable to get response from Khoj backend ❤️‍🩹. Retry or contact developers for help at <a href=mailto:'team@khoj.dev'>team@khoj.dev</a> or <a href='https://discord.gg/BDgyabRM6e'>on Discord</a>";

src/khoj/interface/web/chat.html

@@ -211,10 +211,9 @@ To get started, just start typing below. You can also type / to see a list of co
         function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) {
             if (intentType === "text-to-image") {
                 let imageMarkdown = `![](data:image/png;base64,${message})`;
-                imageMarkdown += "\n\n";
-                if (inferredQueries) {
-                    const inferredQuery = inferredQueries?.[0];
-                    imageMarkdown += `**Inferred Query**: ${inferredQuery}`;
+                const inferredQuery = inferredQueries?.[0];
+                if (inferredQuery) {
+                    imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
                 }
                 renderMessage(imageMarkdown, by, dt);
                 return;
@@ -400,10 +399,9 @@ To get started, just start typing below. You can also type / to see a list of co
                     if (responseAsJson.image) {
                         // If response has image field, response is a generated image.
                         rawResponse += `![${query}](data:image/png;base64,${responseAsJson.image})`;
-                        rawResponse += "\n\n";
-                        const inferredQueries = responseAsJson.inferredQueries?.[0];
-                        if (inferredQueries) {
-                            rawResponse += `**Inferred Query**: ${inferredQueries}`;
+                        const inferredQuery = responseAsJson.inferredQueries?.[0];
+                        if (inferredQuery) {
+                            rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
                         }
                     }
                     if (responseAsJson.detail) {