mirror of
https://github.com/khoj-ai/khoj.git
synced 2025-02-18 23:54:20 +00:00
Generate dynamic diagrams (via Excalidraw) (#940)
Add support for generating dynamic diagrams in flow with Excalidraw (https://github.com/excalidraw/excalidraw). This happens in three steps: 1. Default information collection & intent determination step. 2. Improving the overall guidance of the prompt for generating a JSON, Excalidraw-compatible declaration. 3. Generation of the diagram to output to the final UI. Add support in the web UI.
This commit is contained in:
parent
1e993d561b
commit
0dad4212fa
24 changed files with 689 additions and 100 deletions
src
interface
desktop
obsidian/src
web
khoj
|
@ -326,7 +326,7 @@
|
|||
entries.forEach(entry => {
|
||||
// If the element is in the viewport, fetch the remaining message and unobserve the element
|
||||
if (entry.isIntersecting) {
|
||||
fetchRemainingChatMessages(chatHistoryUrl, headers);
|
||||
fetchRemainingChatMessages(chatHistoryUrl, headers, chatBody.dataset.conversation_id, hostURL);
|
||||
observer.unobserve(entry.target);
|
||||
}
|
||||
});
|
||||
|
@ -342,7 +342,11 @@
|
|||
new Date(chat_log.created),
|
||||
chat_log.onlineContext,
|
||||
chat_log.intent?.type,
|
||||
chat_log.intent?.["inferred-queries"]);
|
||||
chat_log.intent?.["inferred-queries"],
|
||||
chatBody.dataset.conversationId ?? "",
|
||||
hostURL,
|
||||
);
|
||||
|
||||
chatBody.appendChild(messageElement);
|
||||
|
||||
// When the 4th oldest message is within viewing distance (~60% scrolled up)
|
||||
|
@ -421,7 +425,7 @@
|
|||
}
|
||||
}
|
||||
|
||||
function fetchRemainingChatMessages(chatHistoryUrl, headers) {
|
||||
function fetchRemainingChatMessages(chatHistoryUrl, headers, conversationId, hostURL) {
|
||||
// Create a new IntersectionObserver
|
||||
let observer = new IntersectionObserver((entries, observer) => {
|
||||
entries.forEach(entry => {
|
||||
|
@ -435,7 +439,9 @@
|
|||
new Date(chat_log.created),
|
||||
chat_log.onlineContext,
|
||||
chat_log.intent?.type,
|
||||
chat_log.intent?.["inferred-queries"]
|
||||
chat_log.intent?.["inferred-queries"],
|
||||
chatBody.dataset.conversationId ?? "",
|
||||
hostURL,
|
||||
);
|
||||
entry.target.replaceWith(messageElement);
|
||||
|
||||
|
|
|
@ -189,11 +189,19 @@ function processOnlineReferences(referenceSection, onlineContext) { //same
|
|||
return numOnlineReferences;
|
||||
}
|
||||
|
||||
function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) { //same
|
||||
function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null, conversationId=null, hostURL=null) {
|
||||
let chatEl;
|
||||
if (intentType?.includes("text-to-image")) {
|
||||
let imageMarkdown = generateImageMarkdown(message, intentType, inferredQueries);
|
||||
chatEl = renderMessage(imageMarkdown, by, dt, null, false, "return");
|
||||
} else if (intentType === "excalidraw") {
|
||||
let domain = hostURL ?? "https://app.khoj.dev/";
|
||||
|
||||
if (!domain.endsWith("/")) domain += "/";
|
||||
|
||||
let excalidrawMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in the web app at ${domain}chat?conversationId=${conversationId}`;
|
||||
|
||||
chatEl = renderMessage(excalidrawMessage, by, dt, null, false, "return");
|
||||
} else {
|
||||
chatEl = renderMessage(message, by, dt, null, false, "return");
|
||||
}
|
||||
|
@ -312,7 +320,6 @@ function formatHTMLMessage(message, raw=false, willReplace=true) { //same
|
|||
}
|
||||
|
||||
function createReferenceSection(references, createLinkerSection=false) {
|
||||
console.log("linker data: ", createLinkerSection);
|
||||
let referenceSection = document.createElement('div');
|
||||
referenceSection.classList.add("reference-section");
|
||||
referenceSection.classList.add("collapsed");
|
||||
|
@ -417,7 +424,11 @@ function handleImageResponse(imageJson, rawResponse) {
|
|||
rawResponse += ``;
|
||||
} else if (imageJson.intentType === "text-to-image-v3") {
|
||||
rawResponse = ``;
|
||||
} else if (imageJson.intentType === "excalidraw") {
|
||||
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in the web app`;
|
||||
rawResponse += redirectMessage;
|
||||
}
|
||||
|
||||
if (inferredQuery) {
|
||||
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
|
||||
}
|
||||
|
|
|
@ -484,12 +484,13 @@ export class KhojChatView extends KhojPaneView {
|
|||
dt?: Date,
|
||||
intentType?: string,
|
||||
inferredQueries?: string[],
|
||||
conversationId?: string,
|
||||
) {
|
||||
if (!message) return;
|
||||
|
||||
let chatMessageEl;
|
||||
if (intentType?.includes("text-to-image")) {
|
||||
let imageMarkdown = this.generateImageMarkdown(message, intentType, inferredQueries);
|
||||
if (intentType?.includes("text-to-image") || intentType === "excalidraw") {
|
||||
let imageMarkdown = this.generateImageMarkdown(message, intentType, inferredQueries, conversationId);
|
||||
chatMessageEl = this.renderMessage(chatEl, imageMarkdown, sender, dt);
|
||||
} else {
|
||||
chatMessageEl = this.renderMessage(chatEl, message, sender, dt);
|
||||
|
@ -509,7 +510,7 @@ export class KhojChatView extends KhojPaneView {
|
|||
chatMessageBodyEl.appendChild(this.createReferenceSection(references));
|
||||
}
|
||||
|
||||
generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[]) {
|
||||
generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[], conversationId?: string): string {
|
||||
let imageMarkdown = "";
|
||||
if (intentType === "text-to-image") {
|
||||
imageMarkdown = ``;
|
||||
|
@ -517,6 +518,10 @@ export class KhojChatView extends KhojPaneView {
|
|||
imageMarkdown = ``;
|
||||
} else if (intentType === "text-to-image-v3") {
|
||||
imageMarkdown = ``;
|
||||
} else if (intentType === "excalidraw") {
|
||||
const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`;
|
||||
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}chat?conversationId=${conversationId}`;
|
||||
imageMarkdown = redirectMessage;
|
||||
}
|
||||
if (inferredQueries) {
|
||||
imageMarkdown += "\n\n**Inferred Query**:";
|
||||
|
@ -884,6 +889,7 @@ export class KhojChatView extends KhojPaneView {
|
|||
new Date(chatLog.created),
|
||||
chatLog.intent?.type,
|
||||
chatLog.intent?.["inferred-queries"],
|
||||
chatBodyEl.dataset.conversationId ?? "",
|
||||
);
|
||||
// push the user messages to the chat history
|
||||
if(chatLog.by === "you"){
|
||||
|
@ -1354,6 +1360,10 @@ export class KhojChatView extends KhojPaneView {
|
|||
rawResponse += ``;
|
||||
} else if (imageJson.intentType === "text-to-image-v3") {
|
||||
rawResponse = ``;
|
||||
} else if (imageJson.intentType === "excalidraw") {
|
||||
const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`;
|
||||
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}`;
|
||||
rawResponse += redirectMessage;
|
||||
}
|
||||
if (inferredQuery) {
|
||||
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
|
||||
|
|
|
@ -79,7 +79,7 @@ div.titleBar {
|
|||
div.chatBoxBody {
|
||||
display: grid;
|
||||
height: 100%;
|
||||
width: 70%;
|
||||
width: 95%;
|
||||
margin: auto;
|
||||
}
|
||||
|
||||
|
|
|
@ -47,7 +47,14 @@ export default function RootLayout({
|
|||
child-src 'none';
|
||||
object-src 'none';"
|
||||
></meta>
|
||||
<body className={inter.className}>{children}</body>
|
||||
<body className={inter.className}>
|
||||
{children}
|
||||
<script
|
||||
dangerouslySetInnerHTML={{
|
||||
__html: `window.EXCALIDRAW_ASSET_PATH = 'https://assets.khoj.dev/@excalidraw/excalidraw/dist/';`,
|
||||
}}
|
||||
/>
|
||||
</body>
|
||||
</html>
|
||||
);
|
||||
}
|
||||
|
|
|
@ -17,8 +17,6 @@ import { useIPLocationData, useIsMobileWidth, welcomeConsole } from "../common/u
|
|||
import ChatInputArea, { ChatOptions } from "../components/chatInputArea/chatInputArea";
|
||||
import { useAuthenticatedData } from "../common/auth";
|
||||
import { AgentData } from "../agents/page";
|
||||
import { DotsThreeVertical } from "@phosphor-icons/react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
||||
interface ChatBodyDataProps {
|
||||
chatOptionsData: ChatOptions | null;
|
||||
|
@ -43,6 +41,8 @@ function ChatBodyData(props: ChatBodyDataProps) {
|
|||
const setQueryToProcess = props.setQueryToProcess;
|
||||
const onConversationIdChange = props.onConversationIdChange;
|
||||
|
||||
const chatHistoryCustomClassName = props.isMobileWidth ? "w-full" : "w-4/6";
|
||||
|
||||
useEffect(() => {
|
||||
if (image) {
|
||||
props.setImage64(encodeURIComponent(image));
|
||||
|
@ -103,10 +103,11 @@ function ChatBodyData(props: ChatBodyDataProps) {
|
|||
setAgent={setAgentMetadata}
|
||||
pendingMessage={processingMessage ? message : ""}
|
||||
incomingMessages={props.streamedMessages}
|
||||
customClassName={chatHistoryCustomClassName}
|
||||
/>
|
||||
</div>
|
||||
<div
|
||||
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl h-fit`}
|
||||
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl h-fit ${chatHistoryCustomClassName} mr-auto ml-auto`}
|
||||
>
|
||||
<ChatInputArea
|
||||
agentColor={agentMetadata?.color}
|
||||
|
|
|
@ -5,10 +5,10 @@ export interface RawReferenceData {
|
|||
onlineContext?: OnlineContext;
|
||||
}
|
||||
|
||||
export interface ResponseWithReferences {
|
||||
context?: Context[];
|
||||
online?: OnlineContext;
|
||||
response?: string;
|
||||
export interface ResponseWithIntent {
|
||||
intentType: string;
|
||||
response: string;
|
||||
inferredQueries?: string[];
|
||||
}
|
||||
|
||||
interface MessageChunk {
|
||||
|
@ -49,10 +49,14 @@ export function convertMessageChunkToJson(chunk: string): MessageChunk {
|
|||
function handleJsonResponse(chunkData: any) {
|
||||
const jsonData = chunkData as any;
|
||||
if (jsonData.image || jsonData.detail) {
|
||||
let responseWithReference = handleImageResponse(chunkData, true);
|
||||
if (responseWithReference.response) return responseWithReference.response;
|
||||
let responseWithIntent = handleImageResponse(chunkData, true);
|
||||
return responseWithIntent;
|
||||
} else if (jsonData.response) {
|
||||
return jsonData.response;
|
||||
return {
|
||||
response: jsonData.response,
|
||||
intentType: "",
|
||||
inferredQueries: [],
|
||||
};
|
||||
} else {
|
||||
throw new Error("Invalid JSON response");
|
||||
}
|
||||
|
@ -80,8 +84,18 @@ export function processMessageChunk(
|
|||
return { context, onlineContext };
|
||||
} else if (chunk.type === "message") {
|
||||
const chunkData = chunk.data;
|
||||
// Here, handle if the response is a JSON response with an image, but the intentType is excalidraw
|
||||
if (chunkData !== null && typeof chunkData === "object") {
|
||||
currentMessage.rawResponse += handleJsonResponse(chunkData);
|
||||
let responseWithIntent = handleJsonResponse(chunkData);
|
||||
|
||||
if (responseWithIntent.intentType && responseWithIntent.intentType === "excalidraw") {
|
||||
currentMessage.rawResponse = responseWithIntent.response;
|
||||
} else {
|
||||
currentMessage.rawResponse += responseWithIntent.response;
|
||||
}
|
||||
|
||||
currentMessage.intentType = responseWithIntent.intentType;
|
||||
currentMessage.inferredQueries = responseWithIntent.inferredQueries;
|
||||
} else if (
|
||||
typeof chunkData === "string" &&
|
||||
chunkData.trim()?.startsWith("{") &&
|
||||
|
@ -89,7 +103,10 @@ export function processMessageChunk(
|
|||
) {
|
||||
try {
|
||||
const jsonData = JSON.parse(chunkData.trim());
|
||||
currentMessage.rawResponse += handleJsonResponse(jsonData);
|
||||
let responseWithIntent = handleJsonResponse(jsonData);
|
||||
currentMessage.rawResponse += responseWithIntent.response;
|
||||
currentMessage.intentType = responseWithIntent.intentType;
|
||||
currentMessage.inferredQueries = responseWithIntent.inferredQueries;
|
||||
} catch (e) {
|
||||
currentMessage.rawResponse += JSON.stringify(chunkData);
|
||||
}
|
||||
|
@ -111,42 +128,26 @@ export function processMessageChunk(
|
|||
return { context, onlineContext };
|
||||
}
|
||||
|
||||
export function handleImageResponse(imageJson: any, liveStream: boolean): ResponseWithReferences {
|
||||
export function handleImageResponse(imageJson: any, liveStream: boolean): ResponseWithIntent {
|
||||
let rawResponse = "";
|
||||
|
||||
if (imageJson.image) {
|
||||
const inferredQuery = imageJson.inferredQueries?.[0] ?? "generated image";
|
||||
|
||||
// If response has image field, response is a generated image.
|
||||
if (imageJson.intentType === "text-to-image") {
|
||||
rawResponse += ``;
|
||||
} else if (imageJson.intentType === "text-to-image2") {
|
||||
rawResponse += ``;
|
||||
} else if (imageJson.intentType === "text-to-image-v3") {
|
||||
rawResponse = ``;
|
||||
}
|
||||
if (inferredQuery && !liveStream) {
|
||||
rawResponse += `\n\n${inferredQuery}`;
|
||||
}
|
||||
// If response has image field, response may be a generated image
|
||||
rawResponse = imageJson.image;
|
||||
}
|
||||
|
||||
let reference: ResponseWithReferences = {};
|
||||
let responseWithIntent: ResponseWithIntent = {
|
||||
intentType: imageJson.intentType,
|
||||
response: rawResponse,
|
||||
inferredQueries: imageJson.inferredQueries,
|
||||
};
|
||||
|
||||
if (imageJson.context && imageJson.context.length > 0) {
|
||||
const rawReferenceAsJson = imageJson.context;
|
||||
if (rawReferenceAsJson instanceof Array) {
|
||||
reference.context = rawReferenceAsJson;
|
||||
} else if (typeof rawReferenceAsJson === "object" && rawReferenceAsJson !== null) {
|
||||
reference.online = rawReferenceAsJson;
|
||||
}
|
||||
}
|
||||
if (imageJson.detail) {
|
||||
// The detail field contains the improved image prompt
|
||||
rawResponse += imageJson.detail;
|
||||
}
|
||||
|
||||
reference.response = rawResponse;
|
||||
return reference;
|
||||
return responseWithIntent;
|
||||
}
|
||||
|
||||
export function modifyFileFilterForConversation(
|
||||
|
|
|
@ -48,6 +48,7 @@ import {
|
|||
Oven,
|
||||
Gavel,
|
||||
Broadcast,
|
||||
KeyReturn,
|
||||
} from "@phosphor-icons/react";
|
||||
import { Markdown, OrgMode, Pdf, Word } from "@/app/components/logo/fileLogo";
|
||||
|
||||
|
@ -193,6 +194,10 @@ export function getIconForSlashCommand(command: string, customClassName: string
|
|||
}
|
||||
|
||||
if (command.includes("default")) {
|
||||
return <KeyReturn className={className} />;
|
||||
}
|
||||
|
||||
if (command.includes("diagram")) {
|
||||
return <Shapes className={className} />;
|
||||
}
|
||||
|
||||
|
|
|
@ -2,12 +2,7 @@ div.chatHistory {
|
|||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
div.chatLayout {
|
||||
height: 80vh;
|
||||
overflow-y: auto;
|
||||
margin: 0 auto;
|
||||
margin: auto;
|
||||
}
|
||||
|
||||
div.agentIndicator a {
|
||||
|
|
|
@ -37,6 +37,7 @@ interface ChatHistoryProps {
|
|||
pendingMessage?: string;
|
||||
publicConversationSlug?: string;
|
||||
setAgent: (agent: AgentData) => void;
|
||||
customClassName?: string;
|
||||
}
|
||||
|
||||
function constructTrainOfThought(
|
||||
|
@ -255,7 +256,7 @@ export default function ChatHistory(props: ChatHistoryProps) {
|
|||
return (
|
||||
<ScrollArea className={`h-[80vh] relative`} ref={scrollAreaRef}>
|
||||
<div>
|
||||
<div className={styles.chatHistory}>
|
||||
<div className={`${styles.chatHistory} ${props.customClassName}`}>
|
||||
<div ref={sentinelRef} style={{ height: "1px" }}>
|
||||
{fetchingData && (
|
||||
<InlineLoading message="Loading Conversation" className="opacity-50" />
|
||||
|
@ -322,6 +323,12 @@ export default function ChatHistory(props: ChatHistoryProps) {
|
|||
by: "khoj",
|
||||
automationId: "",
|
||||
rawQuery: message.rawQuery,
|
||||
intent: {
|
||||
type: message.intentType || "",
|
||||
query: message.rawQuery,
|
||||
"memory-type": "",
|
||||
"inferred-queries": message.inferredQueries || [],
|
||||
},
|
||||
}}
|
||||
customClassName="fullHistory"
|
||||
borderLeftColor={`${data?.agent?.color}-500`}
|
||||
|
@ -366,18 +373,20 @@ export default function ChatHistory(props: ChatHistoryProps) {
|
|||
</div>
|
||||
)}
|
||||
</div>
|
||||
{!isNearBottom && (
|
||||
<button
|
||||
title="Scroll to bottom"
|
||||
className="absolute bottom-4 right-5 bg-white dark:bg-[hsl(var(--background))] text-neutral-500 dark:text-white p-2 rounded-full shadow-xl"
|
||||
onClick={() => {
|
||||
scrollToBottom();
|
||||
setIsNearBottom(true);
|
||||
}}
|
||||
>
|
||||
<ArrowDown size={24} />
|
||||
</button>
|
||||
)}
|
||||
<div className={`${props.customClassName} fixed bottom-[15%] z-10`}>
|
||||
{!isNearBottom && (
|
||||
<button
|
||||
title="Scroll to bottom"
|
||||
className="absolute bottom-0 right-0 bg-white dark:bg-[hsl(var(--background))] text-neutral-500 dark:text-white p-2 rounded-full shadow-xl"
|
||||
onClick={() => {
|
||||
scrollToBottom();
|
||||
setIsNearBottom(true);
|
||||
}}
|
||||
>
|
||||
<ArrowDown size={24} />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</ScrollArea>
|
||||
);
|
||||
|
|
|
@ -26,6 +26,7 @@ import {
|
|||
Palette,
|
||||
ClipboardText,
|
||||
Check,
|
||||
Shapes,
|
||||
} from "@phosphor-icons/react";
|
||||
|
||||
import DOMPurify from "dompurify";
|
||||
|
@ -35,6 +36,7 @@ import { AgentData } from "@/app/agents/page";
|
|||
|
||||
import renderMathInElement from "katex/contrib/auto-render";
|
||||
import "katex/dist/katex.min.css";
|
||||
import ExcalidrawComponent from "../excalidraw/excalidraw";
|
||||
|
||||
const md = new markdownIt({
|
||||
html: true,
|
||||
|
@ -127,6 +129,9 @@ export interface StreamMessage {
|
|||
timestamp: string;
|
||||
agent?: AgentData;
|
||||
uploadedImageData?: string;
|
||||
intentType?: string;
|
||||
inferredQueries?: string[];
|
||||
image?: string;
|
||||
}
|
||||
|
||||
export interface ChatHistoryData {
|
||||
|
@ -252,6 +257,10 @@ function chooseIconFromHeader(header: string, iconColor: string) {
|
|||
return <Aperture className={`${classNames}`} />;
|
||||
}
|
||||
|
||||
if (compareHeader.includes("diagram")) {
|
||||
return <Shapes className={`${classNames}`} />;
|
||||
}
|
||||
|
||||
if (compareHeader.includes("paint")) {
|
||||
return <Palette className={`${classNames}`} />;
|
||||
}
|
||||
|
@ -283,6 +292,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
|
|||
const [markdownRendered, setMarkdownRendered] = useState<string>("");
|
||||
const [isPlaying, setIsPlaying] = useState<boolean>(false);
|
||||
const [interrupted, setInterrupted] = useState<boolean>(false);
|
||||
const [excalidrawData, setExcalidrawData] = useState<string>("");
|
||||
|
||||
const interruptedRef = useRef<boolean>(false);
|
||||
const messageRef = useRef<HTMLDivElement>(null);
|
||||
|
@ -321,6 +331,11 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
|
|||
useEffect(() => {
|
||||
let message = props.chatMessage.message;
|
||||
|
||||
if (props.chatMessage.intent && props.chatMessage.intent.type == "excalidraw") {
|
||||
message = props.chatMessage.intent["inferred-queries"][0];
|
||||
setExcalidrawData(props.chatMessage.message);
|
||||
}
|
||||
|
||||
// Replace LaTeX delimiters with placeholders
|
||||
message = message
|
||||
.replace(/\\\(/g, "LEFTPAREN")
|
||||
|
@ -332,22 +347,27 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
|
|||
message = `\n\n${message}`;
|
||||
}
|
||||
|
||||
if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image") {
|
||||
message = ``;
|
||||
} else if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image2") {
|
||||
message = ``;
|
||||
} else if (
|
||||
props.chatMessage.intent &&
|
||||
props.chatMessage.intent.type == "text-to-image-v3"
|
||||
) {
|
||||
message = ``;
|
||||
}
|
||||
if (
|
||||
props.chatMessage.intent &&
|
||||
props.chatMessage.intent.type.includes("text-to-image") &&
|
||||
props.chatMessage.intent["inferred-queries"]?.length > 0
|
||||
) {
|
||||
message += `\n\n${props.chatMessage.intent["inferred-queries"][0]}`;
|
||||
const intentTypeHandlers = {
|
||||
"text-to-image": (msg: string) => ``,
|
||||
"text-to-image2": (msg: string) => ``,
|
||||
"text-to-image-v3": (msg: string) =>
|
||||
``,
|
||||
excalidraw: (msg: string) => {
|
||||
return msg;
|
||||
},
|
||||
};
|
||||
|
||||
if (props.chatMessage.intent) {
|
||||
const { type, "inferred-queries": inferredQueries } = props.chatMessage.intent;
|
||||
|
||||
console.log("intent type", type);
|
||||
if (type in intentTypeHandlers) {
|
||||
message = intentTypeHandlers[type as keyof typeof intentTypeHandlers](message);
|
||||
}
|
||||
|
||||
if (type.includes("text-to-image") && inferredQueries?.length > 0) {
|
||||
message += `\n\n${inferredQueries[0]}`;
|
||||
}
|
||||
}
|
||||
|
||||
setTextRendered(message);
|
||||
|
@ -554,6 +574,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
|
|||
className={styles.chatMessage}
|
||||
dangerouslySetInnerHTML={{ __html: markdownRendered }}
|
||||
/>
|
||||
{excalidrawData && <ExcalidrawComponent data={excalidrawData} />}
|
||||
</div>
|
||||
<div className={styles.teaserReferencesContainer}>
|
||||
<TeaserReferencesSection
|
||||
|
|
24
src/interface/web/app/components/excalidraw/excalidraw.tsx
Normal file
24
src/interface/web/app/components/excalidraw/excalidraw.tsx
Normal file
|
@ -0,0 +1,24 @@
|
|||
"use client";
|
||||
|
||||
import dynamic from "next/dynamic";
|
||||
import { Suspense } from "react";
|
||||
import Loading from "../../components/loading/loading";
|
||||
|
||||
// Since client components get prerenderd on server as well hence importing
|
||||
// the excalidraw stuff dynamically with ssr false
|
||||
|
||||
const ExcalidrawWrapper = dynamic(() => import("./excalidrawWrapper").then((mod) => mod.default), {
|
||||
ssr: false,
|
||||
});
|
||||
|
||||
interface ExcalidrawComponentProps {
|
||||
data: any;
|
||||
}
|
||||
|
||||
export default function ExcalidrawComponent(props: ExcalidrawComponentProps) {
|
||||
return (
|
||||
<Suspense fallback={<Loading />}>
|
||||
<ExcalidrawWrapper data={props.data} />
|
||||
</Suspense>
|
||||
);
|
||||
}
|
|
@ -0,0 +1,149 @@
|
|||
"use client";
|
||||
|
||||
import { useState, useEffect } from "react";
|
||||
|
||||
import dynamic from "next/dynamic";
|
||||
|
||||
import { ExcalidrawProps } from "@excalidraw/excalidraw/types/types";
|
||||
import { ExcalidrawElement } from "@excalidraw/excalidraw/types/element/types";
|
||||
import { ExcalidrawElementSkeleton } from "@excalidraw/excalidraw/types/data/transform";
|
||||
|
||||
const Excalidraw = dynamic<ExcalidrawProps>(
|
||||
async () => (await import("@excalidraw/excalidraw")).Excalidraw,
|
||||
{
|
||||
ssr: false,
|
||||
},
|
||||
);
|
||||
|
||||
import { convertToExcalidrawElements } from "@excalidraw/excalidraw";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
|
||||
import { ArrowsInSimple, ArrowsOutSimple } from "@phosphor-icons/react";
|
||||
|
||||
interface ExcalidrawWrapperProps {
|
||||
data: ExcalidrawElementSkeleton[];
|
||||
}
|
||||
|
||||
export default function ExcalidrawWrapper(props: ExcalidrawWrapperProps) {
|
||||
const [excalidrawElements, setExcalidrawElements] = useState<ExcalidrawElement[]>([]);
|
||||
const [expanded, setExpanded] = useState<boolean>(false);
|
||||
|
||||
const isValidExcalidrawElement = (element: ExcalidrawElementSkeleton): boolean => {
|
||||
return (
|
||||
element.x !== undefined &&
|
||||
element.y !== undefined &&
|
||||
element.id !== undefined &&
|
||||
element.type !== undefined
|
||||
);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (expanded) {
|
||||
onkeydown = (e) => {
|
||||
if (e.key === "Escape") {
|
||||
setExpanded(false);
|
||||
// Trigger a resize event to make Excalidraw adjust its size
|
||||
window.dispatchEvent(new Event("resize"));
|
||||
}
|
||||
};
|
||||
} else {
|
||||
onkeydown = null;
|
||||
}
|
||||
}, [expanded]);
|
||||
|
||||
useEffect(() => {
|
||||
// Do some basic validation
|
||||
const basicValidSkeletons: ExcalidrawElementSkeleton[] = [];
|
||||
|
||||
for (const element of props.data) {
|
||||
if (isValidExcalidrawElement(element as ExcalidrawElementSkeleton)) {
|
||||
basicValidSkeletons.push(element as ExcalidrawElementSkeleton);
|
||||
}
|
||||
}
|
||||
|
||||
const validSkeletons: ExcalidrawElementSkeleton[] = [];
|
||||
for (const element of basicValidSkeletons) {
|
||||
if (element.type === "frame") {
|
||||
continue;
|
||||
}
|
||||
if (element.type === "arrow") {
|
||||
const start = basicValidSkeletons.find((child) => child.id === element.start?.id);
|
||||
const end = basicValidSkeletons.find((child) => child.id === element.end?.id);
|
||||
if (start && end) {
|
||||
validSkeletons.push(element);
|
||||
}
|
||||
} else {
|
||||
validSkeletons.push(element);
|
||||
}
|
||||
}
|
||||
|
||||
for (const element of basicValidSkeletons) {
|
||||
if (element.type === "frame") {
|
||||
const children = element.children?.map((childId) => {
|
||||
return validSkeletons.find((child) => child.id === childId);
|
||||
});
|
||||
// Get the valid children, filter out any undefined values
|
||||
const validChildrenIds: readonly string[] = children
|
||||
?.map((child) => child?.id)
|
||||
.filter((id) => id !== undefined) as string[];
|
||||
|
||||
if (validChildrenIds === undefined || validChildrenIds.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
validSkeletons.push({
|
||||
...element,
|
||||
children: validChildrenIds,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const elements = convertToExcalidrawElements(validSkeletons);
|
||||
setExcalidrawElements(elements);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="relative">
|
||||
<div
|
||||
className={`${expanded ? "fixed inset-0 bg-black bg-opacity-50 backdrop-blur-sm z-50 flex items-center justify-center" : ""}`}
|
||||
>
|
||||
<Button
|
||||
onClick={() => {
|
||||
setExpanded(!expanded);
|
||||
// Trigger a resize event to make Excalidraw adjust its size
|
||||
window.dispatchEvent(new Event("resize"));
|
||||
}}
|
||||
variant={"outline"}
|
||||
className={`${expanded ? "absolute top-2 left-2 z-[60]" : ""}`}
|
||||
>
|
||||
{expanded ? (
|
||||
<ArrowsInSimple className="h-4 w-4" />
|
||||
) : (
|
||||
<ArrowsOutSimple className="h-4 w-4" />
|
||||
)}
|
||||
</Button>
|
||||
<div
|
||||
className={`
|
||||
${expanded ? "w-[80vw] h-[80vh]" : "w-full h-[500px]"}
|
||||
bg-white overflow-hidden rounded-lg relative
|
||||
`}
|
||||
>
|
||||
<Excalidraw
|
||||
initialData={{
|
||||
elements: excalidrawElements,
|
||||
appState: { zenModeEnabled: true },
|
||||
scrollToContent: true,
|
||||
}}
|
||||
// TODO - Create a common function to detect if the theme is dark?
|
||||
theme={localStorage.getItem("theme") === "dark" ? "dark" : "light"}
|
||||
validateEmbeddable={true}
|
||||
renderTopRightUI={(isMobile, appState) => {
|
||||
return <></>;
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
|
@ -27,7 +27,14 @@ export default function RootLayout({
|
|||
child-src 'none';
|
||||
object-src 'none';"
|
||||
></meta>
|
||||
<body className={inter.className}>{children}</body>
|
||||
<body className={inter.className}>
|
||||
{children}
|
||||
<script
|
||||
dangerouslySetInnerHTML={{
|
||||
__html: `window.EXCALIDRAW_ASSET_PATH = 'https://assets.khoj.dev/@excalidraw/excalidraw/dist/';`,
|
||||
}}
|
||||
/>
|
||||
</body>
|
||||
</html>
|
||||
);
|
||||
}
|
||||
|
|
|
@ -39,6 +39,8 @@ function ChatBodyData(props: ChatBodyDataProps) {
|
|||
const setQueryToProcess = props.setQueryToProcess;
|
||||
const streamedMessages = props.streamedMessages;
|
||||
|
||||
const chatHistoryCustomClassName = props.isMobileWidth ? "w-full" : "w-4/6";
|
||||
|
||||
useEffect(() => {
|
||||
if (image) {
|
||||
props.setImage64(encodeURIComponent(image));
|
||||
|
@ -78,10 +80,11 @@ function ChatBodyData(props: ChatBodyDataProps) {
|
|||
setTitle={props.setTitle}
|
||||
pendingMessage={processingMessage ? message : ""}
|
||||
incomingMessages={props.streamedMessages}
|
||||
customClassName={chatHistoryCustomClassName}
|
||||
/>
|
||||
</div>
|
||||
<div
|
||||
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl`}
|
||||
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl h-fit ${chatHistoryCustomClassName} mr-auto ml-auto`}
|
||||
>
|
||||
<ChatInputArea
|
||||
isLoggedIn={props.isLoggedIn}
|
||||
|
@ -275,6 +278,19 @@ export default function SharedChat() {
|
|||
|
||||
<div className={styles.chatBox}>
|
||||
<div className={styles.chatBoxBody}>
|
||||
{!isMobileWidth && title && (
|
||||
<div
|
||||
className={`${styles.chatTitleWrapper} text-nowrap text-ellipsis overflow-hidden max-w-screen-md grid items-top font-bold mr-8 pt-6 col-auto h-fit`}
|
||||
>
|
||||
{title && (
|
||||
<h2
|
||||
className={`text-lg text-ellipsis whitespace-nowrap overflow-x-hidden`}
|
||||
>
|
||||
{title}
|
||||
</h2>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
<Suspense fallback={<Loading />}>
|
||||
<ChatBodyData
|
||||
conversationId={conversationId}
|
||||
|
|
|
@ -75,7 +75,7 @@ div.titleBar {
|
|||
div.chatBoxBody {
|
||||
display: grid;
|
||||
height: 100%;
|
||||
width: 70%;
|
||||
width: 95%;
|
||||
margin: auto;
|
||||
}
|
||||
|
||||
|
|
|
@ -63,7 +63,8 @@
|
|||
"swr": "^2.2.5",
|
||||
"typescript": "^5",
|
||||
"vaul": "^0.9.1",
|
||||
"zod": "^3.23.8"
|
||||
"zod": "^3.23.8",
|
||||
"@excalidraw/excalidraw": "^0.17.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/dompurify": "^3.0.5",
|
||||
|
|
|
@ -286,6 +286,11 @@
|
|||
resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.1.tgz#de633db3ec2ef6a3c89e2f19038063e8a122e2c2"
|
||||
integrity sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==
|
||||
|
||||
"@excalidraw/excalidraw@^0.17.6":
|
||||
version "0.17.6"
|
||||
resolved "https://registry.yarnpkg.com/@excalidraw/excalidraw/-/excalidraw-0.17.6.tgz#5fd208ce69d33ca712d1804b50d7d06d5c46ac4d"
|
||||
integrity sha512-fyCl+zG/Z5yhHDh5Fq2ZGmphcrALmuOdtITm8gN4d8w4ntnaopTXcTfnAAaU3VleDC6LhTkoLOTG6P5kgREiIg==
|
||||
|
||||
"@floating-ui/core@^1.6.0":
|
||||
version "1.6.8"
|
||||
resolved "https://registry.yarnpkg.com/@floating-ui/core/-/core-1.6.8.tgz#aa43561be075815879305965020f492cdb43da12"
|
||||
|
|
|
@ -176,6 +176,150 @@ Improved Prompt:
|
|||
""".strip()
|
||||
)
|
||||
|
||||
## Diagram Generation
|
||||
## --
|
||||
|
||||
improve_diagram_description_prompt = PromptTemplate.from_template(
|
||||
"""
|
||||
you are an architect working with a novice artist using a diagramming tool.
|
||||
{personality_context}
|
||||
|
||||
you need to convert the user's query to a description format that the novice artist can use very well. you are allowed to use primitives like
|
||||
- text
|
||||
- rectangle
|
||||
- diamond
|
||||
- ellipse
|
||||
- line
|
||||
- arrow
|
||||
- frame
|
||||
|
||||
use these primitives to describe what sort of diagram the drawer should create. the artist must recreate the diagram every time, so include all relevant prior information in your description.
|
||||
|
||||
use simple, concise language.
|
||||
|
||||
Today's Date: {current_date}
|
||||
User's Location: {location}
|
||||
|
||||
User's Notes:
|
||||
{references}
|
||||
|
||||
Online References:
|
||||
{online_results}
|
||||
|
||||
Conversation Log:
|
||||
{chat_history}
|
||||
|
||||
Query: {query}
|
||||
|
||||
|
||||
""".strip()
|
||||
)
|
||||
|
||||
excalidraw_diagram_generation_prompt = PromptTemplate.from_template(
|
||||
"""
|
||||
You are a program manager with the ability to describe diagrams to compose in professional, fine detail.
|
||||
{personality_context}
|
||||
|
||||
You need to create a declarative description of the diagram and relevant components, using this base schema. Use the `label` property to specify the text to be rendered in the respective elements. Always use light colors for the `backgroundColor` property, like white, or light blue, green, red. "type", "x", "y", "id", are required properties for all elements.
|
||||
|
||||
{{
|
||||
type: string,
|
||||
x: number,
|
||||
y: number,
|
||||
strokeColor: string,
|
||||
backgroundColor: string,
|
||||
width: number,
|
||||
height: number,
|
||||
id: string,
|
||||
label: {{
|
||||
text: string,
|
||||
}}
|
||||
}}
|
||||
|
||||
Valid types:
|
||||
- text
|
||||
- rectangle
|
||||
- diamond
|
||||
- ellipse
|
||||
- line
|
||||
- arrow
|
||||
|
||||
For arrows and lines, you can use the `points` property to specify the start and end points of the arrow. You may also use the `label` property to specify the text to be rendered. You may use the `start` and `end` properties to connect the linear elements to other elements. The start and end point can either be the ID to map to an existing object, or the `type` to create a new object. Mapping to an existing object is useful if you want to connect it to multiple objects. Lines and arrows can only start and end at rectangle, text, diamond, or ellipse elements.
|
||||
|
||||
{{
|
||||
type: "arrow",
|
||||
id: string,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
strokeColor: string,
|
||||
start: {{
|
||||
id: string,
|
||||
type: string,
|
||||
}},
|
||||
end: {{
|
||||
id: string,
|
||||
type: string,
|
||||
}},
|
||||
label: {{
|
||||
text: string,
|
||||
}}
|
||||
points: [
|
||||
[number, number],
|
||||
[number, number],
|
||||
]
|
||||
}}
|
||||
|
||||
For text, you must use the `text` property to specify the text to be rendered. You may also use `fontSize` property to specify the font size of the text. Only use the `text` element for titles, subtitles, and overviews. For labels, use the `label` property in the respective elements.
|
||||
|
||||
{{
|
||||
type: "text",
|
||||
id: string,
|
||||
x: number,
|
||||
y: number,
|
||||
fontSize: number,
|
||||
text: string,
|
||||
}}
|
||||
|
||||
For frames, use the `children` property to specify the elements that are inside the frame by their ids.
|
||||
|
||||
{{
|
||||
type: "frame",
|
||||
id: string,
|
||||
x: number,
|
||||
y: number,
|
||||
width: number,
|
||||
height: number,
|
||||
name: string,
|
||||
children: [
|
||||
string
|
||||
]
|
||||
}}
|
||||
|
||||
Here's an example of a valid diagram:
|
||||
|
||||
Design Description: Create a diagram describing a circular development process with 3 stages: design, implementation and feedback. The design stage is connected to the implementation stage and the implementation stage is connected to the feedback stage and the feedback stage is connected to the design stage. Each stage should be labeled with the stage name.
|
||||
|
||||
Response:
|
||||
|
||||
[
|
||||
{{"type":"text","x":-150,"y":50,"width":300,"height":40,"id":"title_text","text":"Circular Development Process","fontSize":24}},
|
||||
{{"type":"ellipse","x":-169,"y":113,"width":188,"height":202,"id":"design_ellipse", "label": {{"text": "Design"}}}},
|
||||
{{"type":"ellipse","x":62,"y":394,"width":186,"height":188,"id":"implement_ellipse", "label": {{"text": "Implement"}}}},
|
||||
{{"type":"ellipse","x":-348,"y":430,"width":184,"height":170,"id":"feedback_ellipse", "label": {{"text": "Feedback"}}}},
|
||||
{{"type":"arrow","x":21,"y":273,"id":"design_to_implement_arrow","points":[[0,0],[86,105]],"start":{{"id":"design_ellipse"}}, "end":{{"id":"implement_ellipse"}}}},
|
||||
{{"type":"arrow","x":50,"y":519,"id":"implement_to_feedback_arrow","points":[[0,0],[-198,-6]],"start":{{"id":"implement_ellipse"}}, "end":{{"id":"feedback_ellipse"}}}},
|
||||
{{"type":"arrow","x":-228,"y":417,"id":"feedback_to_design_arrow","points":[[0,0],[85,-123]],"start":{{"id":"feedback_ellipse"}}, "end":{{"id":"design_ellipse"}}}},
|
||||
]
|
||||
|
||||
Create a detailed diagram from the provided context and user prompt below. Return a valid JSON object:
|
||||
|
||||
Diagram Description: {query}
|
||||
|
||||
""".strip()
|
||||
)
|
||||
|
||||
## Online Search Conversation
|
||||
## --
|
||||
online_search_conversation = PromptTemplate.from_template(
|
||||
|
|
|
@ -181,7 +181,10 @@ def generate_chatml_messages_with_context(
|
|||
message_notes = f'\n\n Notes:\n{chat.get("context")}' if chat.get("context") else "\n"
|
||||
role = "user" if chat["by"] == "you" else "assistant"
|
||||
|
||||
message_content = chat["message"] + message_notes
|
||||
if chat["by"] == "khoj" and "excalidraw" in chat["intent"].get("type"):
|
||||
message_content = chat.get("intent").get("inferred-queries")[0] + message_notes
|
||||
else:
|
||||
message_content = chat["message"] + message_notes
|
||||
|
||||
message_content = construct_structured_message(
|
||||
message_content, chat.get("uploadedImageData"), model_type, vision_enabled
|
||||
|
|
|
@ -40,6 +40,7 @@ from khoj.routers.helpers import (
|
|||
construct_automation_created_message,
|
||||
create_automation,
|
||||
extract_relevant_summary,
|
||||
generate_excalidraw_diagram,
|
||||
get_conversation_command,
|
||||
is_query_empty,
|
||||
is_ready_to_chat,
|
||||
|
@ -1008,6 +1009,57 @@ async def chat(
|
|||
yield result
|
||||
return
|
||||
|
||||
if ConversationCommand.Diagram in conversation_commands:
|
||||
async for result in send_event(ChatEvent.STATUS, f"Creating diagram"):
|
||||
yield result
|
||||
|
||||
intent_type = "excalidraw"
|
||||
inferred_queries = []
|
||||
diagram_description = ""
|
||||
|
||||
async for result in generate_excalidraw_diagram(
|
||||
q=defiltered_query,
|
||||
conversation_history=meta_log,
|
||||
location_data=location,
|
||||
note_references=compiled_references,
|
||||
online_results=online_results,
|
||||
uploaded_image_url=uploaded_image_url,
|
||||
user=user,
|
||||
agent=agent,
|
||||
send_status_func=partial(send_event, ChatEvent.STATUS),
|
||||
):
|
||||
if isinstance(result, dict) and ChatEvent.STATUS in result:
|
||||
yield result[ChatEvent.STATUS]
|
||||
else:
|
||||
better_diagram_description_prompt, excalidraw_diagram_description = result
|
||||
inferred_queries.append(better_diagram_description_prompt)
|
||||
diagram_description = excalidraw_diagram_description
|
||||
|
||||
content_obj = {
|
||||
"intentType": intent_type,
|
||||
"inferredQueries": inferred_queries,
|
||||
"image": diagram_description,
|
||||
}
|
||||
|
||||
await sync_to_async(save_to_conversation_log)(
|
||||
q,
|
||||
excalidraw_diagram_description,
|
||||
user,
|
||||
meta_log,
|
||||
user_message_time,
|
||||
intent_type="excalidraw",
|
||||
inferred_queries=[better_diagram_description_prompt],
|
||||
client_application=request.user.client_app,
|
||||
conversation_id=conversation_id,
|
||||
compiled_references=compiled_references,
|
||||
online_results=online_results,
|
||||
uploaded_image_url=uploaded_image_url,
|
||||
)
|
||||
|
||||
async for result in send_llm_response(json.dumps(content_obj)):
|
||||
yield result
|
||||
return
|
||||
|
||||
## Generate Text Output
|
||||
async for result in send_event(ChatEvent.STATUS, f"**Generating a well-informed response**"):
|
||||
yield result
|
||||
|
|
|
@ -14,6 +14,7 @@ from typing import (
|
|||
Annotated,
|
||||
Any,
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
Dict,
|
||||
Iterator,
|
||||
List,
|
||||
|
@ -215,6 +216,9 @@ def construct_chat_history(conversation_history: dict, n: int = 4, agent_name="A
|
|||
elif chat["by"] == "khoj" and ("text-to-image" in chat["intent"].get("type")):
|
||||
chat_history += f"User: {chat['intent']['query']}\n"
|
||||
chat_history += f"{agent_name}: [generated image redacted for space]\n"
|
||||
elif chat["by"] == "khoj" and ("excalidraw" in chat["intent"].get("type")):
|
||||
chat_history += f"User: {chat['intent']['query']}\n"
|
||||
chat_history += f"{agent_name}: {chat['intent']['inferred-queries'][0]}\n"
|
||||
return chat_history
|
||||
|
||||
|
||||
|
@ -235,6 +239,8 @@ def get_conversation_command(query: str, any_references: bool = False) -> Conver
|
|||
return ConversationCommand.AutomatedTask
|
||||
elif query.startswith("/summarize"):
|
||||
return ConversationCommand.Summarize
|
||||
elif query.startswith("/diagram"):
|
||||
return ConversationCommand.Diagram
|
||||
# If no relevant notes found for the given query
|
||||
elif not any_references:
|
||||
return ConversationCommand.General
|
||||
|
@ -617,6 +623,129 @@ async def extract_relevant_summary(
|
|||
return response.strip()
|
||||
|
||||
|
||||
async def generate_excalidraw_diagram(
|
||||
q: str,
|
||||
conversation_history: Dict[str, Any],
|
||||
location_data: LocationData,
|
||||
note_references: List[Dict[str, Any]],
|
||||
online_results: Optional[dict] = None,
|
||||
uploaded_image_url: Optional[str] = None,
|
||||
user: KhojUser = None,
|
||||
agent: Agent = None,
|
||||
send_status_func: Optional[Callable] = None,
|
||||
):
|
||||
if send_status_func:
|
||||
async for event in send_status_func("**Enhancing the Diagramming Prompt**"):
|
||||
yield {ChatEvent.STATUS: event}
|
||||
|
||||
better_diagram_description_prompt = await generate_better_diagram_description(
|
||||
q=q,
|
||||
conversation_history=conversation_history,
|
||||
location_data=location_data,
|
||||
note_references=note_references,
|
||||
online_results=online_results,
|
||||
uploaded_image_url=uploaded_image_url,
|
||||
user=user,
|
||||
agent=agent,
|
||||
)
|
||||
|
||||
if send_status_func:
|
||||
async for event in send_status_func(f"**Diagram to Create:**:\n{better_diagram_description_prompt}"):
|
||||
yield {ChatEvent.STATUS: event}
|
||||
|
||||
excalidraw_diagram_description = await generate_excalidraw_diagram_from_description(
|
||||
q=better_diagram_description_prompt,
|
||||
user=user,
|
||||
agent=agent,
|
||||
)
|
||||
|
||||
yield better_diagram_description_prompt, excalidraw_diagram_description
|
||||
|
||||
|
||||
async def generate_better_diagram_description(
|
||||
q: str,
|
||||
conversation_history: Dict[str, Any],
|
||||
location_data: LocationData,
|
||||
note_references: List[Dict[str, Any]],
|
||||
online_results: Optional[dict] = None,
|
||||
uploaded_image_url: Optional[str] = None,
|
||||
user: KhojUser = None,
|
||||
agent: Agent = None,
|
||||
) -> str:
|
||||
"""
|
||||
Generate a diagram description from the given query and context
|
||||
"""
|
||||
|
||||
today_date = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d, %A")
|
||||
personality_context = (
|
||||
prompts.personality_context.format(personality=agent.personality) if agent and agent.personality else ""
|
||||
)
|
||||
|
||||
if location_data:
|
||||
location_prompt = prompts.user_location.format(location=f"{location_data}")
|
||||
else:
|
||||
location_prompt = "Unknown"
|
||||
|
||||
user_references = "\n\n".join([f"# {item['compiled']}" for item in note_references])
|
||||
|
||||
chat_history = construct_chat_history(conversation_history)
|
||||
|
||||
simplified_online_results = {}
|
||||
|
||||
if online_results:
|
||||
for result in online_results:
|
||||
if online_results[result].get("answerBox"):
|
||||
simplified_online_results[result] = online_results[result]["answerBox"]
|
||||
elif online_results[result].get("webpages"):
|
||||
simplified_online_results[result] = online_results[result]["webpages"]
|
||||
|
||||
improve_diagram_description_prompt = prompts.improve_diagram_description_prompt.format(
|
||||
query=q,
|
||||
chat_history=chat_history,
|
||||
location=location_prompt,
|
||||
current_date=today_date,
|
||||
references=user_references,
|
||||
online_results=simplified_online_results,
|
||||
personality_context=personality_context,
|
||||
)
|
||||
|
||||
with timer("Chat actor: Generate better diagram description", logger):
|
||||
response = await send_message_to_model_wrapper(
|
||||
improve_diagram_description_prompt, uploaded_image_url=uploaded_image_url, user=user
|
||||
)
|
||||
response = response.strip()
|
||||
if response.startswith(('"', "'")) and response.endswith(('"', "'")):
|
||||
response = response[1:-1]
|
||||
|
||||
return response
|
||||
|
||||
|
||||
async def generate_excalidraw_diagram_from_description(
|
||||
q: str,
|
||||
user: KhojUser = None,
|
||||
agent: Agent = None,
|
||||
) -> str:
|
||||
personality_context = (
|
||||
prompts.personality_context.format(personality=agent.personality) if agent and agent.personality else ""
|
||||
)
|
||||
|
||||
excalidraw_diagram_generation = prompts.excalidraw_diagram_generation_prompt.format(
|
||||
personality_context=personality_context,
|
||||
query=q,
|
||||
)
|
||||
|
||||
with timer("Chat actor: Generate excalidraw diagram", logger):
|
||||
raw_response = await send_message_to_model_wrapper(message=excalidraw_diagram_generation, user=user)
|
||||
raw_response = raw_response.strip()
|
||||
raw_response = remove_json_codeblock(raw_response)
|
||||
response: Dict[str, str] = json.loads(raw_response)
|
||||
if not response or not isinstance(response, List) or not isinstance(response[0], Dict):
|
||||
# TODO Some additional validation here that it's a valid Excalidraw diagram
|
||||
raise AssertionError(f"Invalid response for improving diagram description: {response}")
|
||||
|
||||
return response
|
||||
|
||||
|
||||
async def generate_better_image_prompt(
|
||||
q: str,
|
||||
conversation_history: str,
|
||||
|
|
|
@ -51,17 +51,6 @@ def chat_page(request: Request):
|
|||
)
|
||||
|
||||
|
||||
@web_client.get("/experimental", response_class=FileResponse)
|
||||
@requires(["authenticated"], redirect="login_page")
|
||||
def experimental_page(request: Request):
|
||||
return templates.TemplateResponse(
|
||||
"index.html",
|
||||
context={
|
||||
"request": request,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@web_client.get("/factchecker", response_class=FileResponse)
|
||||
def fact_checker_page(request: Request):
|
||||
return templates.TemplateResponse(
|
||||
|
|
|
@ -318,6 +318,7 @@ class ConversationCommand(str, Enum):
|
|||
Automation = "automation"
|
||||
AutomatedTask = "automated_task"
|
||||
Summarize = "summarize"
|
||||
Diagram = "diagram"
|
||||
|
||||
|
||||
command_descriptions = {
|
||||
|
@ -326,10 +327,11 @@ command_descriptions = {
|
|||
ConversationCommand.Default: "The default command when no command specified. It intelligently auto-switches between general and notes mode.",
|
||||
ConversationCommand.Online: "Search for information on the internet.",
|
||||
ConversationCommand.Webpage: "Get information from webpage suggested by you.",
|
||||
ConversationCommand.Image: "Generate images by describing your imagination in words.",
|
||||
ConversationCommand.Image: "Generate illustrative, creative images by describing your imagination in words.",
|
||||
ConversationCommand.Automation: "Automatically run your query at a specified time or interval.",
|
||||
ConversationCommand.Help: "Get help with how to use or setup Khoj from the documentation",
|
||||
ConversationCommand.Summarize: "Get help with a question pertaining to an entire document.",
|
||||
ConversationCommand.Diagram: "Draw a flowchart, diagram, or any other visual representation best expressed with primitives like lines, rectangles, and text.",
|
||||
}
|
||||
|
||||
command_descriptions_for_agent = {
|
||||
|
@ -353,12 +355,14 @@ mode_descriptions_for_llm = {
|
|||
ConversationCommand.Image: "Use this if the user is requesting you to generate a picture based on their description.",
|
||||
ConversationCommand.Automation: "Use this if you are confident the user is requesting a response at a scheduled date, time and frequency",
|
||||
ConversationCommand.Text: "Use this if the other response modes don't seem to fit the query.",
|
||||
ConversationCommand.Diagram: "Use this if the user is requesting a visual representation that requires primitives like lines, rectangles, and text.",
|
||||
}
|
||||
|
||||
mode_descriptions_for_agent = {
|
||||
ConversationCommand.Image: "Agent can generate image in response.",
|
||||
ConversationCommand.Automation: "Agent can schedule a task to run at a scheduled date, time and frequency in response.",
|
||||
ConversationCommand.Text: "Agent can generate text in response.",
|
||||
ConversationCommand.Diagram: "Agent can generate a visual representation that requires primitives like lines, rectangles, and text.",
|
||||
}
|
||||
|
||||
|
||||
|
|
Loading…
Add table
Reference in a new issue