1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2025-02-18 23:54:20 +00:00

Allow sharing multiple images as part of user query from the web app

Previously the web app only expected a single image to be shared by
the user as part of their query.

This change allows sharing multiple images from the web app.

Closes 
This commit is contained in:
Debanjum Singh Solanky 2024-10-17 23:08:20 -07:00
parent e2abc1a257
commit 0d6a54c10f
6 changed files with 122 additions and 77 deletions
src/interface/web/app
chat
components
chatHistory
chatInputArea
chatMessage
page.tsx
share/chat

View file

@ -17,8 +17,6 @@ import { useIPLocationData, useIsMobileWidth, welcomeConsole } from "../common/u
import ChatInputArea, { ChatOptions } from "../components/chatInputArea/chatInputArea";
import { useAuthenticatedData } from "../common/auth";
import { AgentData } from "../agents/page";
import { DotsThreeVertical } from "@phosphor-icons/react";
import { Button } from "@/components/ui/button";
interface ChatBodyDataProps {
chatOptionsData: ChatOptions | null;
@ -29,14 +27,14 @@ interface ChatBodyDataProps {
setUploadedFiles: (files: string[]) => void;
isMobileWidth?: boolean;
isLoggedIn: boolean;
setImage64: (image64: string) => void;
setImages: (images: string[]) => void;
}
function ChatBodyData(props: ChatBodyDataProps) {
const searchParams = useSearchParams();
const conversationId = searchParams.get("conversationId");
const [message, setMessage] = useState("");
const [image, setImage] = useState<string | null>(null);
const [images, setImages] = useState<string[]>([]);
const [processingMessage, setProcessingMessage] = useState(false);
const [agentMetadata, setAgentMetadata] = useState<AgentData | null>(null);
@ -44,17 +42,20 @@ function ChatBodyData(props: ChatBodyDataProps) {
const onConversationIdChange = props.onConversationIdChange;
useEffect(() => {
if (image) {
props.setImage64(encodeURIComponent(image));
if (images.length > 0) {
const encodedImages = images.map((image) => encodeURIComponent(image));
props.setImages(encodedImages);
}
}, [image, props.setImage64]);
}, [images, props.setImages]);
useEffect(() => {
const storedImage = localStorage.getItem("image");
if (storedImage) {
setImage(storedImage);
props.setImage64(encodeURIComponent(storedImage));
localStorage.removeItem("image");
const storedImages = localStorage.getItem("images");
if (storedImages) {
const parsedImages: string[] = JSON.parse(storedImages);
setImages(parsedImages);
const encodedImages = parsedImages.map((img: string) => encodeURIComponent(img));
props.setImages(encodedImages);
localStorage.removeItem("images");
}
const storedMessage = localStorage.getItem("message");
@ -62,7 +63,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
setProcessingMessage(true);
setQueryToProcess(storedMessage);
}
}, [setQueryToProcess]);
}, [setQueryToProcess, props.setImages]);
useEffect(() => {
if (message) {
@ -112,7 +113,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
agentColor={agentMetadata?.color}
isLoggedIn={props.isLoggedIn}
sendMessage={(message) => setMessage(message)}
sendImage={(image) => setImage(image)}
sendImage={(image) => setImages((prevImages) => [...prevImages, image])}
sendDisabled={processingMessage}
chatOptionsData={props.chatOptionsData}
conversationId={conversationId}
@ -134,7 +135,7 @@ export default function Chat() {
const [queryToProcess, setQueryToProcess] = useState<string>("");
const [processQuerySignal, setProcessQuerySignal] = useState(false);
const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
const [image64, setImage64] = useState<string>("");
const [images, setImages] = useState<string[]>([]);
const locationData = useIPLocationData() || {
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
@ -170,7 +171,7 @@ export default function Chat() {
completed: false,
timestamp: new Date().toISOString(),
rawQuery: queryToProcess || "",
uploadedImageData: decodeURIComponent(image64),
images: images,
};
setMessages((prevMessages) => [...prevMessages, newStreamMessage]);
setProcessQuerySignal(true);
@ -201,7 +202,7 @@ export default function Chat() {
if (done) {
setQueryToProcess("");
setProcessQuerySignal(false);
setImage64("");
setImages([]);
break;
}
@ -249,7 +250,7 @@ export default function Chat() {
country_code: locationData.countryCode,
timezone: locationData.timezone,
}),
...(image64 && { image: image64 }),
...(images.length > 0 && { images: images }),
};
const response = await fetch(chatAPI, {
@ -331,7 +332,7 @@ export default function Chat() {
setUploadedFiles={setUploadedFiles}
isMobileWidth={isMobileWidth}
onConversationIdChange={handleConversationIdChange}
setImage64={setImage64}
setImages={setImages}
/>
</Suspense>
</div>

View file

@ -298,7 +298,7 @@ export default function ChatHistory(props: ChatHistoryProps) {
created: message.timestamp,
by: "you",
automationId: "",
uploadedImageData: message.uploadedImageData,
images: message.images,
}}
customClassName="fullHistory"
borderLeftColor={`${data?.agent?.color}-500`}
@ -341,7 +341,6 @@ export default function ChatHistory(props: ChatHistoryProps) {
created: new Date().getTime().toString(),
by: "you",
automationId: "",
uploadedImageData: props.pendingMessage,
}}
customClassName="fullHistory"
borderLeftColor={`${data?.agent?.color}-500`}

View file

@ -78,10 +78,11 @@ export default function ChatInputArea(props: ChatInputProps) {
const [loginRedirectMessage, setLoginRedirectMessage] = useState<string | null>(null);
const [showLoginPrompt, setShowLoginPrompt] = useState(false);
const [recording, setRecording] = useState(false);
const [imageUploaded, setImageUploaded] = useState(false);
const [imagePath, setImagePath] = useState<string>("");
const [imageData, setImageData] = useState<string | null>(null);
const [imagePaths, setImagePaths] = useState<string[]>([]);
const [imageData, setImageData] = useState<string[]>([]);
const [recording, setRecording] = useState(false);
const [mediaRecorder, setMediaRecorder] = useState<MediaRecorder | null>(null);
const [progressValue, setProgressValue] = useState(0);
@ -106,27 +107,31 @@ export default function ChatInputArea(props: ChatInputProps) {
useEffect(() => {
async function fetchImageData() {
if (imagePath) {
const response = await fetch(imagePath);
const blob = await response.blob();
const reader = new FileReader();
reader.onload = function () {
const base64data = reader.result;
setImageData(base64data as string);
};
reader.readAsDataURL(blob);
if (imagePaths.length > 0) {
const newImageData = await Promise.all(
imagePaths.map(async (path) => {
const response = await fetch(path);
const blob = await response.blob();
return new Promise<string>((resolve) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result as string);
reader.readAsDataURL(blob);
});
}),
);
setImageData(newImageData);
}
setUploading(false);
}
setUploading(true);
fetchImageData();
}, [imagePath]);
}, [imagePaths]);
function onSendMessage() {
if (imageUploaded) {
setImageUploaded(false);
setImagePath("");
props.sendImage(imageData || "");
setImagePaths([]);
imageData.forEach((data) => props.sendImage(data));
}
if (!message.trim()) return;
@ -172,18 +177,23 @@ export default function ChatInputArea(props: ChatInputProps) {
setShowLoginPrompt(true);
return;
}
// check for image file
// check for image files
const image_endings = ["jpg", "jpeg", "png", "webp"];
const newImagePaths: string[] = [];
for (let i = 0; i < files.length; i++) {
const file = files[i];
const file_extension = file.name.split(".").pop();
if (image_endings.includes(file_extension || "")) {
setImageUploaded(true);
setImagePath(DOMPurify.sanitize(URL.createObjectURL(file)));
return;
newImagePaths.push(DOMPurify.sanitize(URL.createObjectURL(file)));
}
}
if (newImagePaths.length > 0) {
setImageUploaded(true);
setImagePaths((prevPaths) => [...prevPaths, ...newImagePaths]);
return;
}
uploadDataForIndexing(
files,
setWarning,
@ -288,9 +298,12 @@ export default function ChatInputArea(props: ChatInputProps) {
setIsDragAndDropping(false);
}
function removeImageUpload() {
setImageUploaded(false);
setImagePath("");
function removeImageUpload(index: number) {
setImagePaths((prevPaths) => prevPaths.filter((_, i) => i !== index));
setImageData((prevData) => prevData.filter((_, i) => i !== index));
if (imagePaths.length === 1) {
setImageUploaded(false);
}
}
return (
@ -413,16 +426,24 @@ export default function ChatInputArea(props: ChatInputProps) {
onDrop={handleDragAndDropFiles}
>
{imageUploaded && (
<div className="absolute bottom-[80px] left-0 right-0 dark:bg-neutral-700 bg-white pt-5 pb-5 w-full rounded-lg border dark:border-none grid grid-cols-2">
<div className="pl-4 pr-4">
<img src={imagePath} alt="img" className="w-auto max-h-[100px]" />
</div>
<div className="pl-4 pr-4">
<X
className="w-6 h-6 float-right dark:hover:bg-[hsl(var(--background))] hover:bg-neutral-100 rounded-sm"
onClick={removeImageUpload}
/>
</div>
<div className="absolute bottom-full left-0 right-0 px-12 py-2 dark:bg-neutral-700 bg-white w-full rounded-t-lg border dark:border-none flex items-center space-x-2 overflow-x-auto">
{imagePaths.map((path, index) => (
<div key={index} className="relative flex-shrink-0 group">
<img
src={path}
alt={`img-${index}`}
className="w-auto h-16 object-cover rounded-xl"
/>
<Button
variant="ghost"
size="icon"
className="absolute -top-2 -right-2 h-5 w-5 rounded-full bg-neutral-200 dark:bg-neutral-600 hover:bg-neutral-300 dark:hover:bg-neutral-500 opacity-0 group-hover:opacity-100 transition-opacity"
onClick={() => removeImageUpload(index)}
>
<X className="h-3 w-3" />
</Button>
</div>
))}
</div>
)}
<input
@ -451,7 +472,7 @@ export default function ChatInputArea(props: ChatInputProps) {
onKeyDown={(e) => {
if (e.key === "Enter" && !e.shiftKey) {
setImageUploaded(false);
setImagePath("");
setImagePaths([]);
e.preventDefault();
onSendMessage();
}

View file

@ -114,7 +114,7 @@ export interface SingleChatMessage {
rawQuery?: string;
intent?: Intent;
agent?: AgentData;
uploadedImageData?: string;
images?: string[];
}
export interface StreamMessage {
@ -126,7 +126,7 @@ export interface StreamMessage {
rawQuery: string;
timestamp: string;
agent?: AgentData;
uploadedImageData?: string;
images?: string[];
}
export interface ChatHistoryData {
@ -208,7 +208,6 @@ interface ChatMessageProps {
borderLeftColor?: string;
isLastMessage?: boolean;
agent?: AgentData;
uploadedImageData?: string;
}
interface TrainOfThoughtProps {
@ -328,8 +327,14 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
.replace(/\\\[/g, "LEFTBRACKET")
.replace(/\\\]/g, "RIGHTBRACKET");
if (props.chatMessage.uploadedImageData) {
message = `![uploaded image](${props.chatMessage.uploadedImageData})\n\n${message}`;
if (props.chatMessage.images && props.chatMessage.images.length > 0) {
const imagesInMd = props.chatMessage.images
.map(
(image) =>
`![uploaded image](${image.startsWith("data%3Aimage") ? decodeURIComponent(image) : image})`,
)
.join("\n\n");
message = `${imagesInMd}\n\n${message}`;
}
if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image") {
@ -364,7 +369,7 @@ const ChatMessage = forwardRef<HTMLDivElement, ChatMessageProps>((props, ref) =>
// Sanitize and set the rendered markdown
setMarkdownRendered(DOMPurify.sanitize(markdownRendered));
}, [props.chatMessage.message, props.chatMessage.intent]);
}, [props.chatMessage.message, props.chatMessage.images, props.chatMessage.intent]);
useEffect(() => {
if (copySuccess) {

View file

@ -44,7 +44,7 @@ function FisherYatesShuffle(array: any[]) {
function ChatBodyData(props: ChatBodyDataProps) {
const [message, setMessage] = useState("");
const [image, setImage] = useState<string | null>(null);
const [images, setImages] = useState<string[]>([]);
const [processingMessage, setProcessingMessage] = useState(false);
const [greeting, setGreeting] = useState("");
const [shuffledOptions, setShuffledOptions] = useState<Suggestion[]>([]);
@ -140,18 +140,19 @@ function ChatBodyData(props: ChatBodyDataProps) {
onConversationIdChange?.(newConversationId);
window.location.href = `/chat?conversationId=${newConversationId}`;
localStorage.setItem("message", message);
if (image) {
localStorage.setItem("image", image);
if (images.length > 0) {
localStorage.setItem("images", JSON.stringify(images));
}
} catch (error) {
console.error("Error creating new conversation:", error);
setProcessingMessage(false);
}
setMessage("");
setImages([]);
}
};
processMessage();
if (message) {
if (message || images.length > 0) {
setProcessingMessage(true);
}
}, [selectedAgent, message, processingMessage, onConversationIdChange]);
@ -232,7 +233,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
<ChatInputArea
isLoggedIn={props.isLoggedIn}
sendMessage={(message) => setMessage(message)}
sendImage={(image) => setImage(image)}
sendImage={(image) => setImages((prevImages) => [...prevImages, image])}
sendDisabled={processingMessage}
chatOptionsData={props.chatOptionsData}
conversationId={null}
@ -313,7 +314,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
<ChatInputArea
isLoggedIn={props.isLoggedIn}
sendMessage={(message) => setMessage(message)}
sendImage={(image) => setImage(image)}
sendImage={(image) => setImages((prevImages) => [...prevImages, image])}
sendDisabled={processingMessage}
chatOptionsData={props.chatOptionsData}
conversationId={null}

View file

@ -28,22 +28,40 @@ interface ChatBodyDataProps {
isLoggedIn: boolean;
conversationId?: string;
setQueryToProcess: (query: string) => void;
setImage64: (image64: string) => void;
setImages: (images: string[]) => void;
}
function ChatBodyData(props: ChatBodyDataProps) {
const [message, setMessage] = useState("");
const [image, setImage] = useState<string | null>(null);
const [images, setImages] = useState<string[]>([]);
const [processingMessage, setProcessingMessage] = useState(false);
const [agentMetadata, setAgentMetadata] = useState<AgentData | null>(null);
const setQueryToProcess = props.setQueryToProcess;
const streamedMessages = props.streamedMessages;
useEffect(() => {
if (image) {
props.setImage64(encodeURIComponent(image));
if (images.length > 0) {
const encodedImages = images.map((image) => encodeURIComponent(image));
props.setImages(encodedImages);
}
}, [image, props.setImage64]);
}, [images, props.setImages]);
useEffect(() => {
const storedImages = localStorage.getItem("images");
if (storedImages) {
const parsedImages: string[] = JSON.parse(storedImages);
setImages(parsedImages);
const encodedImages = parsedImages.map((img: string) => encodeURIComponent(img));
props.setImages(encodedImages);
localStorage.removeItem("images");
}
const storedMessage = localStorage.getItem("message");
if (storedMessage) {
setProcessingMessage(true);
setQueryToProcess(storedMessage);
}
}, [setQueryToProcess, props.setImages]);
useEffect(() => {
if (message) {
@ -86,7 +104,7 @@ function ChatBodyData(props: ChatBodyDataProps) {
<ChatInputArea
isLoggedIn={props.isLoggedIn}
sendMessage={(message) => setMessage(message)}
sendImage={(image) => setImage(image)}
sendImage={(image) => setImages((prevImages) => [...prevImages, image])}
sendDisabled={processingMessage}
chatOptionsData={props.chatOptionsData}
conversationId={props.conversationId}
@ -109,7 +127,7 @@ export default function SharedChat() {
const [processQuerySignal, setProcessQuerySignal] = useState(false);
const [uploadedFiles, setUploadedFiles] = useState<string[]>([]);
const [paramSlug, setParamSlug] = useState<string | undefined>(undefined);
const [image64, setImage64] = useState<string>("");
const [images, setImages] = useState<string[]>([]);
const locationData = useIPLocationData() || {
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
@ -167,7 +185,7 @@ export default function SharedChat() {
completed: false,
timestamp: new Date().toISOString(),
rawQuery: queryToProcess || "",
uploadedImageData: decodeURIComponent(image64),
images: images,
};
setMessages((prevMessages) => [...prevMessages, newStreamMessage]);
setProcessQuerySignal(true);
@ -194,7 +212,7 @@ export default function SharedChat() {
if (done) {
setQueryToProcess("");
setProcessQuerySignal(false);
setImage64("");
setImages([]);
break;
}
@ -236,7 +254,7 @@ export default function SharedChat() {
country_code: locationData.countryCode,
timezone: locationData.timezone,
}),
...(image64 && { image: image64 }),
...(images.length > 0 && { image: images }),
};
const response = await fetch(chatAPI, {
@ -286,7 +304,7 @@ export default function SharedChat() {
setTitle={setTitle}
setUploadedFiles={setUploadedFiles}
isMobileWidth={isMobileWidth}
setImage64={setImage64}
setImages={setImages}
/>
</Suspense>
</div>