diff --git a/src/interface/web/app/chat/page.tsx b/src/interface/web/app/chat/page.tsx index 34d2ccc7..deb26105 100644 --- a/src/interface/web/app/chat/page.tsx +++ b/src/interface/web/app/chat/page.tsx @@ -12,7 +12,12 @@ import { processMessageChunk } from "../common/chatFunctions"; import "katex/dist/katex.min.css"; -import { Context, OnlineContext, StreamMessage } from "../components/chatMessage/chatMessage"; +import { + CodeContext, + Context, + OnlineContext, + StreamMessage, +} from "../components/chatMessage/chatMessage"; import { useIPLocationData, useIsMobileWidth, welcomeConsole } from "../common/utils"; import { ChatInputArea, ChatOptions } from "../components/chatInputArea/chatInputArea"; import { useAuthenticatedData } from "../common/auth"; @@ -37,6 +42,7 @@ function ChatBodyData(props: ChatBodyDataProps) { const [images, setImages] = useState([]); const [processingMessage, setProcessingMessage] = useState(false); const [agentMetadata, setAgentMetadata] = useState(null); + const [isInResearchMode, setIsInResearchMode] = useState(false); const chatInputRef = useRef(null); const setQueryToProcess = props.setQueryToProcess; @@ -65,6 +71,10 @@ function ChatBodyData(props: ChatBodyDataProps) { if (storedMessage) { setProcessingMessage(true); setQueryToProcess(storedMessage); + + if (storedMessage.trim().startsWith("/research")) { + setIsInResearchMode(true); + } } }, [setQueryToProcess, props.setImages]); @@ -125,6 +135,7 @@ function ChatBodyData(props: ChatBodyDataProps) { isMobileWidth={props.isMobileWidth} setUploadedFiles={props.setUploadedFiles} ref={chatInputRef} + isResearchModeEnabled={isInResearchMode} /> @@ -174,6 +185,7 @@ export default function Chat() { trainOfThought: [], context: [], onlineContext: {}, + codeContext: {}, completed: false, timestamp: new Date().toISOString(), rawQuery: queryToProcess || "", @@ -202,6 +214,7 @@ export default function Chat() { // Track context used for chat response let context: Context[] = []; let onlineContext: OnlineContext = {}; + let codeContext: CodeContext = {}; while (true) { const { done, value } = await reader.read(); @@ -228,11 +241,12 @@ export default function Chat() { } // Track context used for chat response. References are rendered at the end of the chat - ({ context, onlineContext } = processMessageChunk( + ({ context, onlineContext, codeContext } = processMessageChunk( event, currentMessage, context, onlineContext, + codeContext, )); setMessages([...messages]); diff --git a/src/interface/web/app/common/chatFunctions.ts b/src/interface/web/app/common/chatFunctions.ts index c05e5859..8c016c1a 100644 --- a/src/interface/web/app/common/chatFunctions.ts +++ b/src/interface/web/app/common/chatFunctions.ts @@ -1,8 +1,14 @@ -import { Context, OnlineContext, StreamMessage } from "../components/chatMessage/chatMessage"; +import { + CodeContext, + Context, + OnlineContext, + StreamMessage, +} from "../components/chatMessage/chatMessage"; export interface RawReferenceData { context?: Context[]; onlineContext?: OnlineContext; + codeContext?: CodeContext; } export interface ResponseWithIntent { @@ -67,10 +73,11 @@ export function processMessageChunk( currentMessage: StreamMessage, context: Context[] = [], onlineContext: OnlineContext = {}, -): { context: Context[]; onlineContext: OnlineContext } { + codeContext: CodeContext = {}, +): { context: Context[]; onlineContext: OnlineContext; codeContext: CodeContext } { const chunk = convertMessageChunkToJson(rawChunk); - if (!currentMessage || !chunk || !chunk.type) return { context, onlineContext }; + if (!currentMessage || !chunk || !chunk.type) return { context, onlineContext, codeContext }; if (chunk.type === "status") { console.log(`status: ${chunk.data}`); @@ -81,7 +88,8 @@ export function processMessageChunk( if (references.context) context = references.context; if (references.onlineContext) onlineContext = references.onlineContext; - return { context, onlineContext }; + if (references.codeContext) codeContext = references.codeContext; + return { context, onlineContext, codeContext }; } else if (chunk.type === "message") { const chunkData = chunk.data; // Here, handle if the response is a JSON response with an image, but the intentType is excalidraw @@ -119,13 +127,41 @@ export function processMessageChunk( console.log(`Completed streaming: ${new Date()}`); // Append any references after all the data has been streamed + if (codeContext) currentMessage.codeContext = codeContext; if (onlineContext) currentMessage.onlineContext = onlineContext; if (context) currentMessage.context = context; + // Replace file links with base64 data + currentMessage.rawResponse = renderCodeGenImageInline( + currentMessage.rawResponse, + codeContext, + ); + + // Add code context files to the message + if (codeContext) { + Object.entries(codeContext).forEach(([key, value]) => { + value.results.output_files?.forEach((file) => { + if (file.filename.endsWith(".png") || file.filename.endsWith(".jpg")) { + // Don't add the image again if it's already in the message! + if (!currentMessage.rawResponse.includes(`![${file.filename}](`)) { + currentMessage.rawResponse += `\n\n![${file.filename}](data:image/png;base64,${file.b64_data})`; + } + } else if ( + file.filename.endsWith(".txt") || + file.filename.endsWith(".org") || + file.filename.endsWith(".md") + ) { + const decodedText = atob(file.b64_data); + currentMessage.rawResponse += `\n\n\`\`\`\n${decodedText}\n\`\`\``; + } + }); + }); + } + // Mark current message streaming as completed currentMessage.completed = true; } - return { context, onlineContext }; + return { context, onlineContext, codeContext }; } export function handleImageResponse(imageJson: any, liveStream: boolean): ResponseWithIntent { @@ -150,6 +186,22 @@ export function handleImageResponse(imageJson: any, liveStream: boolean): Respon return responseWithIntent; } +export function renderCodeGenImageInline(message: string, codeContext: CodeContext) { + if (!codeContext) return message; + + Object.values(codeContext).forEach((contextData) => { + contextData.results.output_files?.forEach((file) => { + const regex = new RegExp(`!?\\[.*?\\]\\(.*${file.filename}\\)`, "g"); + if (file.filename.match(/\.(png|jpg|jpeg|gif|webp)$/i)) { + const replacement = `![${file.filename}](data:image/${file.filename.split(".").pop()};base64,${file.b64_data})`; + message = message.replace(regex, replacement); + } + }); + }); + + return message; +} + export function modifyFileFilterForConversation( conversationId: string | null, filenames: string[], diff --git a/src/interface/web/app/common/colorUtils.ts b/src/interface/web/app/common/colorUtils.ts index c534097f..6cd57a64 100644 --- a/src/interface/web/app/common/colorUtils.ts +++ b/src/interface/web/app/common/colorUtils.ts @@ -42,6 +42,13 @@ export function converColorToBgGradient(color: string) { return `${convertToBGGradientClass(color)} dark:border dark:border-neutral-700`; } +export function convertColorToRingClass(color: string | undefined) { + if (color && tailwindColors.includes(color)) { + return `focus-visible:ring-${color}-500`; + } + return `focus-visible:ring-orange-500`; +} + export function convertColorToBorderClass(color: string) { if (tailwindColors.includes(color)) { return `border-${color}-500`; diff --git a/src/interface/web/app/components/chatHistory/chatHistory.tsx b/src/interface/web/app/components/chatHistory/chatHistory.tsx index 944aed9f..9e156142 100644 --- a/src/interface/web/app/components/chatHistory/chatHistory.tsx +++ b/src/interface/web/app/components/chatHistory/chatHistory.tsx @@ -13,13 +13,14 @@ import { ScrollArea } from "@/components/ui/scroll-area"; import { InlineLoading } from "../loading/loading"; -import { Lightbulb, ArrowDown } from "@phosphor-icons/react"; +import { Lightbulb, ArrowDown, XCircle } from "@phosphor-icons/react"; import AgentProfileCard from "../profileCard/profileCard"; import { getIconFromIconName } from "@/app/common/iconUtils"; import { AgentData } from "@/app/agents/page"; import React from "react"; import { useIsMobileWidth } from "@/app/common/utils"; +import { Button } from "@/components/ui/button"; interface ChatResponse { status: string; @@ -40,26 +41,54 @@ interface ChatHistoryProps { customClassName?: string; } -function constructTrainOfThought( - trainOfThought: string[], - lastMessage: boolean, - agentColor: string, - key: string, - completed: boolean = false, -) { - const lastIndex = trainOfThought.length - 1; - return ( -
- {!completed && } +interface TrainOfThoughtComponentProps { + trainOfThought: string[]; + lastMessage: boolean; + agentColor: string; + key: string; + completed?: boolean; +} - {trainOfThought.map((train, index) => ( - - ))} +function TrainOfThoughtComponent(props: TrainOfThoughtComponentProps) { + const lastIndex = props.trainOfThought.length - 1; + const [collapsed, setCollapsed] = useState(props.completed); + + return ( +
+ {!props.completed && } + {props.completed && + (collapsed ? ( + + ) : ( + + ))} + {!collapsed && + props.trainOfThought.map((train, index) => ( + + ))}
); } @@ -254,36 +283,48 @@ export default function ChatHistory(props: ChatHistoryProps) { } return ( - +
- {fetchingData && ( - - )} + {fetchingData && }
{data && data.chat && data.chat.map((chatMessage, index) => ( - + <> + + {chatMessage.trainOfThought && chatMessage.by === "khoj" && ( + train.data, + )} + lastMessage={false} + agentColor={data?.agent?.color || "orange"} + key={`${index}trainOfThought`} + completed={true} + /> + )} + ))} {props.incomingMessages && props.incomingMessages.map((message, index) => { @@ -296,6 +337,7 @@ export default function ChatHistory(props: ChatHistoryProps) { message: message.rawQuery, context: [], onlineContext: {}, + codeContext: {}, created: message.timestamp, by: "you", automationId: "", @@ -304,14 +346,15 @@ export default function ChatHistory(props: ChatHistoryProps) { customClassName="fullHistory" borderLeftColor={`${data?.agent?.color}-500`} /> - {message.trainOfThought && - constructTrainOfThought( - message.trainOfThought, - index === incompleteIncomingMessageIndex, - data?.agent?.color || "orange", - `${index}trainOfThought`, - message.completed, - )} + {message.trainOfThought && ( + + )} )}
-
+
{!isNearBottom && ( -
-
-
- {imageUploaded && - imagePaths.map((path, index) => ( -
- {`img-${index}`} - -
- ))} -
-