Add support for converting an attached doc and chatting with it

- Document is first converted in the chatinputarea, then sent to the chat component. From there, it's sent in the chat API body and then processed by the backend
- We couldn't directly use a UploadFile type in the backend API because we'd have to convert the api type to a multipart form. This would require other client side migrations without uniform benefit, which is why we do it in this two-phase process. This also gives us capacity to repurpose the moe generic interface down the road.

src/interface/web/app/chat/page.tsx

@@ -19,7 +19,11 @@ import {
     StreamMessage,
 } from "../components/chatMessage/chatMessage";
 import { useIPLocationData, useIsMobileWidth, welcomeConsole } from "../common/utils";
-import { ChatInputArea, ChatOptions } from "../components/chatInputArea/chatInputArea";
+import {
+    AttachedFileText,
+    ChatInputArea,
+    ChatOptions,
+} from "../components/chatInputArea/chatInputArea";
 import { useAuthenticatedData } from "../common/auth";
 import { AgentData } from "../agents/page";
 
@@ -30,7 +34,7 @@ interface ChatBodyDataProps {
     setQueryToProcess: (query: string) => void;
     streamedMessages: StreamMessage[];
     setStreamedMessages: (messages: StreamMessage[]) => void;
-    setUploadedFiles: (files: string[]) => void;
+    setUploadedFiles: (files: AttachedFileText[] | undefined) => void;
     isMobileWidth?: boolean;
     isLoggedIn: boolean;
     setImages: (images: string[]) => void;
@@ -77,6 +81,20 @@ function ChatBodyData(props: ChatBodyDataProps) {
                 setIsInResearchMode(true);
             }
         }
+
+        const storedUploadedFiles = localStorage.getItem("uploadedFiles");
+        const parsedFiles = storedUploadedFiles ? JSON.parse(storedUploadedFiles) : [];
+
+        const uploadedFiles: AttachedFileText[] = [];
+        for (const file of parsedFiles) {
+            uploadedFiles.push({
+                name: file.name,
+                file_type: file.file_type,
+                content: file.content,
+                size: file.size,
+            });
+        }
+        props.setUploadedFiles(uploadedFiles);
     }, [setQueryToProcess, props.setImages]);
 
     useEffect(() => {
@@ -100,6 +118,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
         ) {
             setProcessingMessage(false);
             setImages([]); // Reset images after processing
+            props.setUploadedFiles(undefined); // Reset uploaded files after processing
         } else {
             setMessage("");
         }
@@ -153,7 +172,7 @@ export default function Chat() {
     const [messages, setMessages] = useState<StreamMessage[]>([]);
     const [queryToProcess, setQueryToProcess] = useState<string>("");
     const [processQuerySignal, setProcessQuerySignal] = useState(false);
-    const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
+    const [uploadedFiles, setUploadedFiles] = useState<AttachedFileText[] | undefined>(undefined);
     const [images, setImages] = useState<string[]>([]);
 
     const locationData = useIPLocationData() || {
@@ -192,6 +211,7 @@ export default function Chat() {
                 timestamp: new Date().toISOString(),
                 rawQuery: queryToProcess || "",
                 images: images,
+                attachedFiles: uploadedFiles,
             };
             setMessages((prevMessages) => [...prevMessages, newStreamMessage]);
             setProcessQuerySignal(true);
@@ -273,6 +293,7 @@ export default function Chat() {
                 timezone: locationData.timezone,
             }),
             ...(images.length > 0 && { images: images }),
+            ...(uploadedFiles && { files: uploadedFiles }),
         };
 
         const response = await fetch(chatAPI, {
@@ -325,7 +346,7 @@ export default function Chat() {
             <div>
                 <SidePanel
                     conversationId={conversationId}
-                    uploadedFiles={uploadedFiles}
+                    uploadedFiles={[]}
                     isMobileWidth={isMobileWidth}
                 />
             </div>

src/interface/web/app/common/chatFunctions.ts

@@ -267,6 +267,58 @@ export async function createNewConversation(slug: string) {
     }
 }
 
+export async function packageFilesForUpload(files: FileList): Promise<FormData> {
+    const formData = new FormData();
+
+    const fileReadPromises = Array.from(files).map((file) => {
+        return new Promise<void>((resolve, reject) => {
+            let reader = new FileReader();
+            reader.onload = function (event) {
+                if (event.target === null) {
+                    reject();
+                    return;
+                }
+
+                let fileContents = event.target.result;
+                let fileType = file.type;
+                let fileName = file.name;
+                if (fileType === "") {
+                    let fileExtension = fileName.split(".").pop();
+                    if (fileExtension === "org") {
+                        fileType = "text/org";
+                    } else if (fileExtension === "md") {
+                        fileType = "text/markdown";
+                    } else if (fileExtension === "txt") {
+                        fileType = "text/plain";
+                    } else if (fileExtension === "html") {
+                        fileType = "text/html";
+                    } else if (fileExtension === "pdf") {
+                        fileType = "application/pdf";
+                    } else {
+                        // Skip this file if its type is not supported
+                        resolve();
+                        return;
+                    }
+                }
+
+                if (fileContents === null) {
+                    reject();
+                    return;
+                }
+
+                let fileObj = new Blob([fileContents], { type: fileType });
+                formData.append("files", fileObj, file.name);
+                resolve();
+            };
+            reader.onerror = reject;
+            reader.readAsArrayBuffer(file);
+        });
+    });
+
+    await Promise.all(fileReadPromises);
+    return formData;
+}
+
 export function uploadDataForIndexing(
     files: FileList,
     setWarning: (warning: string) => void,

src/interface/web/app/common/utils.ts

@@ -71,6 +71,16 @@ export function useIsMobileWidth() {
     return isMobileWidth;
 }
 
+export const convertBytesToText = (fileSize: number) => {
+    if (fileSize < 1024) {
+        return `${fileSize} B`;
+    } else if (fileSize < 1024 * 1024) {
+        return `${(fileSize / 1024).toFixed(2)} KB`;
+    } else {
+        return `${(fileSize / (1024 * 1024)).toFixed(2)} MB`;
+    }
+};
+
 export function useDebounce<T>(value: T, delay: number): T {
     const [debouncedValue, setDebouncedValue] = useState<T>(value);
 

src/interface/web/app/components/chatInputArea/chatInputArea.tsx

@@ -40,19 +40,27 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp
 import { convertColorToTextClass, convertToBGClass } from "@/app/common/colorUtils";
 
 import LoginPrompt from "../loginPrompt/loginPrompt";
-import { uploadDataForIndexing } from "../../common/chatFunctions";
 import { InlineLoading } from "../loading/loading";
-import { getIconForSlashCommand } from "@/app/common/iconUtils";
+import { getIconForSlashCommand, getIconFromFileType } from "@/app/common/iconUtils";
+import { packageFilesForUpload } from "@/app/common/chatFunctions";
+import { convertBytesToText } from "@/app/common/utils";
 
 export interface ChatOptions {
     [key: string]: string;
 }
 
+export interface AttachedFileText {
+    name: string;
+    content: string;
+    file_type: string;
+    size: number;
+}
+
 interface ChatInputProps {
     sendMessage: (message: string) => void;
     sendImage: (image: string) => void;
     sendDisabled: boolean;
-    setUploadedFiles?: (files: string[]) => void;
+    setUploadedFiles: (files: AttachedFileText[]) => void;
     conversationId?: string | null;
     chatOptionsData?: ChatOptions | null;
     isMobileWidth?: boolean;
@@ -75,6 +83,9 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
     const [imagePaths, setImagePaths] = useState<string[]>([]);
     const [imageData, setImageData] = useState<string[]>([]);
 
+    const [attachedFiles, setAttachedFiles] = useState<FileList | null>(null);
+    const [convertedAttachedFiles, setConvertedAttachedFiles] = useState<AttachedFileText[]>([]);
+
     const [recording, setRecording] = useState(false);
     const [mediaRecorder, setMediaRecorder] = useState<MediaRecorder | null>(null);
 
@@ -154,6 +165,8 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
         }
 
         props.sendMessage(messageToSend);
+        setAttachedFiles(null);
+        setConvertedAttachedFiles([]);
         setMessage("");
     }
 
@@ -203,22 +216,57 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
             setImagePaths((prevPaths) => [...prevPaths, ...newImagePaths]);
             // Set focus to the input for user message after uploading files
             chatInputRef?.current?.focus();
-            return;
         }
 
-        uploadDataForIndexing(
-            files,
-            setWarning,
-            setUploading,
-            setError,
-            props.setUploadedFiles,
-            props.conversationId,
+        // Process all non-image files
+        const nonImageFiles = Array.from(files).filter(
+            (file) => !image_endings.includes(file.name.split(".").pop() || ""),
         );
 
+        // Concatenate attachedFiles and files
+        const newFiles = nonImageFiles
+            ? Array.from(nonImageFiles).concat(Array.from(attachedFiles || []))
+            : Array.from(attachedFiles || []);
+
+        const dataTransfer = new DataTransfer();
+        newFiles.forEach((file) => dataTransfer.items.add(file));
+        setAttachedFiles(dataTransfer.files);
+
+        // Extract text from files
+        extractTextFromFiles(dataTransfer.files).then((data) => {
+            props.setUploadedFiles(data);
+            setConvertedAttachedFiles(data);
+        });
+
+        const totalSize = Array.from(files).reduce((acc, file) => acc + file.size, 0);
+        const totalSizeInMB = totalSize / (1024 * 1024);
+
         // Set focus to the input for user message after uploading files
         chatInputRef?.current?.focus();
     }
 
+    async function extractTextFromFiles(files: FileList): Promise<AttachedFileText[]> {
+        const formData = await packageFilesForUpload(files);
+        setUploading(true);
+
+        try {
+            const response = await fetch("/api/content/convert", {
+                method: "POST",
+                body: formData,
+            });
+            setUploading(false);
+
+            if (!response.ok) {
+                throw new Error(`HTTP error! status: ${response.status}`);
+            }
+
+            return await response.json();
+        } catch (error) {
+            console.error("Error converting files:", error);
+            return [];
+        }
+    }
+
     // Assuming this function is added within the same context as the provided excerpt
     async function startRecordingAndTranscribe() {
         try {
@@ -445,6 +493,73 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
                 </div>
             )}
             <div>
+                <div className="flex items-center gap-2 overflow-x-auto">
+                    {imageUploaded &&
+                        imagePaths.map((path, index) => (
+                            <div key={index} className="relative flex-shrink-0 pb-3 pt-2 group">
+                                <img
+                                    src={path}
+                                    alt={`img-${index}`}
+                                    className="w-auto h-16 object-cover rounded-xl"
+                                />
+                                <Button
+                                    variant="ghost"
+                                    size="icon"
+                                    className="absolute -top-0 -right-2 h-5 w-5 rounded-full bg-neutral-200 dark:bg-neutral-600 hover:bg-neutral-300 dark:hover:bg-neutral-500 opacity-0 group-hover:opacity-100 transition-opacity"
+                                    onClick={() => removeImageUpload(index)}
+                                >
+                                    <X className="h-3 w-3" />
+                                </Button>
+                            </div>
+                        ))}
+                    {convertedAttachedFiles &&
+                        Array.from(convertedAttachedFiles).map((file, index) => (
+                            <div key={index} className="relative flex-shrink-0 p-2 group">
+                                <div
+                                    className={`w-auto h-16 object-cover rounded-xl ${props.agentColor ? convertToBGClass(props.agentColor) : "bg-orange-300 hover:bg-orange-500"} bg-opacity-15`}
+                                >
+                                    <div className="flex p-2 flex-col justify-start items-start h-full">
+                                        <span className="text-sm font-bold text-neutral-500 dark:text-neutral-400 text-ellipsis truncate max-w-[200px] break-words">
+                                            {file.name}
+                                        </span>
+                                        <span className="flex items-center gap-1">
+                                            {getIconFromFileType(file.file_type)}
+                                            <span className="text-xs text-neutral-500 dark:text-neutral-400">
+                                                {convertBytesToText(file.size)}
+                                            </span>
+                                        </span>
+                                    </div>
+                                </div>
+                                <Button
+                                    variant="ghost"
+                                    size="icon"
+                                    className="absolute -top-0 -right-2 h-5 w-5 rounded-full bg-neutral-200 dark:bg-neutral-600 hover:bg-neutral-300 dark:hover:bg-neutral-500 opacity-0 group-hover:opacity-100 transition-opacity"
+                                    onClick={() => {
+                                        setAttachedFiles((prevFiles) => {
+                                            const removeFile = file.name;
+                                            if (!prevFiles) return null;
+                                            const updatedFiles = Array.from(prevFiles).filter(
+                                                (file) => file.name !== removeFile,
+                                            );
+                                            const dataTransfer = new DataTransfer();
+                                            updatedFiles.forEach((file) =>
+                                                dataTransfer.items.add(file),
+                                            );
+                                            extractTextFromFiles(dataTransfer.files).then(
+                                                (data) => {
+                                                    props.setUploadedFiles(data);
+                                                    setConvertedAttachedFiles(data);
+                                                },
+                                            );
+                                            return dataTransfer.files;
+                                        });
+                                    }}
+                                >
+                                    <X className="h-3 w-3" />
+                                </Button>
+                            </div>
+                        ))}
+                </div>
                 <div
                     className={`${styles.actualInputArea} justify-between dark:bg-neutral-700 relative ${isDragAndDropping && "animate-pulse"}`}
                     onDragOver={handleDragOver}
@@ -458,6 +573,7 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
                         onChange={handleFileChange}
                         style={{ display: "none" }}
                     />
+
                     <div className="flex items-center">
                         <Button
                             variant={"ghost"}
@@ -469,29 +585,6 @@ export const ChatInputArea = forwardRef<HTMLTextAreaElement, ChatInputProps>((pr
                         </Button>
                     </div>
                     <div className="flex-grow flex flex-col w-full gap-1.5 relative">
-                        <div className="flex items-center gap-2 overflow-x-auto">
-                            {imageUploaded &&
-                                imagePaths.map((path, index) => (
-                                    <div
-                                        key={index}
-                                        className="relative flex-shrink-0 pb-3 pt-2 group"
-                                    >
-                                        <img
-                                            src={path}
-                                            alt={`img-${index}`}
-                                            className="w-auto h-16 object-cover rounded-xl"
-                                        />
-                                        <Button
-                                            variant="ghost"
-                                            size="icon"
-                                            className="absolute -top-0 -right-2 h-5 w-5 rounded-full bg-neutral-200 dark:bg-neutral-600 hover:bg-neutral-300 dark:hover:bg-neutral-500 opacity-0 group-hover:opacity-100 transition-opacity"
-                                            onClick={() => removeImageUpload(index)}
-                                        >
-                                            <X className="h-3 w-3" />
-                                        </Button>
-                                    </div>
-                                ))}
-                        </div>
                         <Textarea
                             ref={chatInputRef}
                             className={`border-none focus:border-none

src/interface/web/app/page.tsx

@@ -11,7 +11,11 @@ import { Card, CardTitle } from "@/components/ui/card";
 import SuggestionCard from "@/app/components/suggestions/suggestionCard";
 import SidePanel from "@/app/components/sidePanel/chatHistorySidePanel";
 import Loading from "@/app/components/loading/loading";
-import { ChatInputArea, ChatOptions } from "@/app/components/chatInputArea/chatInputArea";
+import {
+    AttachedFileText,
+    ChatInputArea,
+    ChatOptions,
+} from "@/app/components/chatInputArea/chatInputArea";
 import { Suggestion, suggestionsData } from "@/app/components/suggestions/suggestionsData";
 import LoginPrompt from "@/app/components/loginPrompt/loginPrompt";
 
@@ -30,12 +34,11 @@ import { useRouter, useSearchParams } from "next/navigation";
 import { ScrollArea, ScrollBar } from "@/components/ui/scroll-area";
 import { AgentCard } from "@/app/components/agentCard/agentCard";
 import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
-import { modifyFileFilterForConversation } from "./common/chatFunctions";
 
 interface ChatBodyDataProps {
     chatOptionsData: ChatOptions | null;
     onConversationIdChange?: (conversationId: string) => void;
-    setUploadedFiles: (files: string[]) => void;
+    setUploadedFiles: (files: AttachedFileText[]) => void;
     isMobileWidth?: boolean;
     isLoggedIn: boolean;
     userConfig: UserConfig | null;
@@ -151,26 +154,13 @@ function ChatBodyData(props: ChatBodyDataProps) {
                 setProcessingMessage(true);
                 try {
                     const newConversationId = await createNewConversation(selectedAgent || "khoj");
-                    const uploadedFiles = localStorage.getItem("uploadedFiles");
                     onConversationIdChange?.(newConversationId);
                     localStorage.setItem("message", message);
                     if (images.length > 0) {
                         localStorage.setItem("images", JSON.stringify(images));
                     }
 
-                    if (uploadedFiles) {
-                        modifyFileFilterForConversation(
-                            newConversationId,
-                            JSON.parse(uploadedFiles),
-                            () => {
-                                window.location.href = `/chat?conversationId=${newConversationId}`;
-                            },
-                            "add",
-                        );
-                        localStorage.removeItem("uploadedFiles");
-                    } else {
-                        window.location.href = `/chat?conversationId=${newConversationId}`;
-                    }
+                    window.location.href = `/chat?conversationId=${newConversationId}`;
                 } catch (error) {
                     console.error("Error creating new conversation:", error);
                     setProcessingMessage(false);
@@ -416,7 +406,7 @@ export default function Home() {
     const [chatOptionsData, setChatOptionsData] = useState<ChatOptions | null>(null);
     const [isLoading, setLoading] = useState(true);
     const [conversationId, setConversationID] = useState<string | null>(null);
-    const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
+    const [uploadedFiles, setUploadedFiles] = useState<AttachedFileText[] | null>(null);
     const isMobileWidth = useIsMobileWidth();
 
     const { userConfig: initialUserConfig, isLoadingUserConfig } = useUserConfig(true);
@@ -433,7 +423,9 @@ export default function Home() {
     }, [initialUserConfig]);
 
     useEffect(() => {
-        localStorage.setItem("uploadedFiles", JSON.stringify(uploadedFiles));
+        if (uploadedFiles) {
+            localStorage.setItem("uploadedFiles", JSON.stringify(uploadedFiles));
+        }
     }, [uploadedFiles]);
 
     useEffect(() => {
@@ -461,7 +453,7 @@ export default function Home() {
             <div className={`${styles.sidePanel}`}>
                 <SidePanel
                     conversationId={conversationId}
-                    uploadedFiles={uploadedFiles}
+                    uploadedFiles={[]}
                     isMobileWidth={isMobileWidth}
                 />
             </div>