From 7b39f2014afb6e28b9430acc2b05d58530233e9c Mon Sep 17 00:00:00 2001 From: Debanjum Date: Mon, 4 Nov 2024 16:59:19 -0800 Subject: [PATCH 1/7] Enable analysing user documents in code sandbox and other improvements - Run one program at a time, instead of allowing model to pass multiple programs to run in parallel to simplify logic for model - Update prompt to give more example of complex, multi-line code - Allow passing user files as input into code sandbox for analysis - Log code execution timer at info level to evaluate execution latencies in production - Type the generated code for easier processing by caller functions --- src/khoj/processor/conversation/prompts.py | 33 ++++++++---- src/khoj/processor/tools/run_code.py | 61 +++++++++++++++------- 2 files changed, 65 insertions(+), 29 deletions(-) diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py index e69da533..3c46a6f8 100644 --- a/src/khoj/processor/conversation/prompts.py +++ b/src/khoj/processor/conversation/prompts.py @@ -870,25 +870,40 @@ Khoj: # -- python_code_generation_prompt = PromptTemplate.from_template( """ -You are Khoj, an advanced python programmer. You are tasked with constructing **up to three** python programs to best answer the user query. +You are Khoj, an advanced python programmer. You are tasked with constructing a python program to best answer the user query. - The python program will run in a pyodide python sandbox with no network access. -- You can write programs to run complex calculations, analyze data, create charts, generate documents to meticulously answer the query -- The sandbox has access to the standard library, matplotlib, panda, numpy, scipy, bs4, sympy, brotli, cryptography, fast-parquet +- You can write programs to run complex calculations, analyze data, create charts, generate documents to meticulously answer the query. +- The sandbox has access to the standard library, matplotlib, panda, numpy, scipy, bs4, sympy, brotli, cryptography, fast-parquet. +- List known file paths to required user documents in "input_files" and known links to required documents from the web in the "input_links" field. +- The python program should be self-contained. It can only read data generated by the program itself and from provided input_files, input_links by their basename (i.e filename excluding file path). - Do not try display images or plots in the code directly. The code should save the image or plot to a file instead. - Write any document, charts etc. to be shared with the user to file. These files can be seen by the user. - Use as much context from the previous questions and answers as required to generate your code. {personality_context} -What code will you need to write, if any, to answer the user's question? -Provide code programs as a list of strings in a JSON object with key "codes". +What code will you need to write to answer the user's question? + Current Date: {current_date} User's Location: {location} {username} -The JSON schema is of the form {{"codes": ["code1", "code2", "code3"]}} -For example: -{{"codes": ["print('Hello, World!')", "print('Goodbye, World!')"]}} +The response JSON schema is of the form {{"code": "", "input_files": ["file_path_1", "file_path_2"], "input_links": ["link_1", "link_2"]}} +Examples: +--- +{{ +"code": "# Input values\\nprincipal = 43235\\nrate = 5.24\\nyears = 5\\n\\n# Convert rate to decimal\\nrate_decimal = rate / 100\\n\\n# Calculate final amount\\nfinal_amount = principal * (1 + rate_decimal) ** years\\n\\n# Calculate interest earned\\ninterest_earned = final_amount - principal\\n\\n# Print results with formatting\\nprint(f"Interest Earned: ${{interest_earned:,.2f}}")\\nprint(f"Final Amount: ${{final_amount:,.2f}}")" +}} -Now it's your turn to construct python programs to answer the user's question. Provide them as a list of strings in a JSON object. Do not say anything else. +{{ +"code": "import re\\n\\n# Read org file\\nfile_path = 'tasks.org'\\nwith open(file_path, 'r') as f:\\n content = f.read()\\n\\n# Get today's date in YYYY-MM-DD format\\ntoday = datetime.now().strftime('%Y-%m-%d')\\npattern = r'\*+\s+.*\\n.*SCHEDULED:\s+<' + today + r'.*>'\\n\\n# Find all matches using multiline mode\\nmatches = re.findall(pattern, content, re.MULTILINE)\\ncount = len(matches)\\n\\n# Display count\\nprint(f'Count of scheduled tasks for today: {{count}}')", +"input_files": ["/home/linux/tasks.org"] +}} + +{{ +"code": "import pandas as pd\\nimport matplotlib.pyplot as plt\\n\\n# Load the CSV file\\ndf = pd.read_csv('world_population_by_year.csv')\\n\\n# Plot the data\\nplt.figure(figsize=(10, 6))\\nplt.plot(df['Year'], df['Population'], marker='o')\\n\\n# Add titles and labels\\nplt.title('Population by Year')\\nplt.xlabel('Year')\\nplt.ylabel('Population')\\n\\n# Save the plot to a file\\nplt.savefig('population_by_year_plot.png')", +"input_links": ["https://population.un.org/world_population_by_year.csv"] +}} + +Now it's your turn to construct a python program to answer the user's question. Provide the code, required input files and input links in a JSON object. Do not say anything else. Context: --- {context} diff --git a/src/khoj/processor/tools/run_code.py b/src/khoj/processor/tools/run_code.py index e24b3b53..fe89da1c 100644 --- a/src/khoj/processor/tools/run_code.py +++ b/src/khoj/processor/tools/run_code.py @@ -1,14 +1,14 @@ -import asyncio +import base64 import datetime import json import logging import os -from typing import Any, Callable, List, Optional +from typing import Any, Callable, List, NamedTuple, Optional import aiohttp -from khoj.database.adapters import ais_user_subscribed -from khoj.database.models import Agent, KhojUser +from khoj.database.adapters import FileObjectAdapters +from khoj.database.models import Agent, FileObject, KhojUser from khoj.processor.conversation import prompts from khoj.processor.conversation.utils import ( ChatEvent, @@ -17,7 +17,7 @@ from khoj.processor.conversation.utils import ( construct_chat_history, ) from khoj.routers.helpers import send_message_to_model_wrapper -from khoj.utils.helpers import timer +from khoj.utils.helpers import is_none_or_empty, timer from khoj.utils.rawconfig import LocationData logger = logging.getLogger(__name__) @@ -26,6 +26,12 @@ logger = logging.getLogger(__name__) SANDBOX_URL = os.getenv("KHOJ_TERRARIUM_URL", "http://localhost:8080") +class GeneratedCode(NamedTuple): + code: str + input_files: List[str] + input_links: List[str] + + async def run_code( query: str, conversation_history: dict, @@ -41,11 +47,11 @@ async def run_code( ): # Generate Code if send_status_func: - async for event in send_status_func(f"**Generate code snippets** for {query}"): + async for event in send_status_func(f"**Generate code snippet** for {query}"): yield {ChatEvent.STATUS: event} try: with timer("Chat actor: Generate programs to execute", logger): - codes = await generate_python_code( + generated_code = await generate_python_code( query, conversation_history, context, @@ -59,15 +65,26 @@ async def run_code( except Exception as e: raise ValueError(f"Failed to generate code for {query} with error: {e}") + # Prepare Input Data + input_data = [] + user_input_files: List[FileObject] = [] + for input_file in generated_code.input_files: + user_input_files += await FileObjectAdapters.aget_file_objects_by_name(user, input_file) + for f in user_input_files: + input_data.append( + { + "filename": os.path.basename(f.file_name), + "b64_data": base64.b64encode(f.raw_text.encode("utf-8")).decode("utf-8"), + } + ) + # Run Code if send_status_func: - async for event in send_status_func(f"**Running {len(codes)} code snippets**"): + async for event in send_status_func(f"**Running code snippet**"): yield {ChatEvent.STATUS: event} try: - tasks = [execute_sandboxed_python(code, sandbox_url) for code in codes] - with timer("Chat actor: Execute generated programs", logger): - results = await asyncio.gather(*tasks) - for result in results: + with timer("Chat actor: Execute generated program", logger, log_level=logging.INFO): + result = await execute_sandboxed_python(generated_code.code, input_data, sandbox_url) code = result.pop("code") logger.info(f"Executed Code:\n--@@--\n{code}\n--@@--Result:\n--@@--\n{result}\n--@@--") yield {query: {"code": code, "results": result}} @@ -81,14 +98,13 @@ async def generate_python_code( context: str, location_data: LocationData, user: KhojUser, - query_images: List[str] = None, + query_images: list[str] = None, agent: Agent = None, tracer: dict = {}, query_files: str = None, -) -> List[str]: +) -> GeneratedCode: location = f"{location_data}" if location_data else "Unknown" username = prompts.user_name.format(name=user.get_full_name()) if user.get_full_name() else "" - subscribed = await ais_user_subscribed(user) chat_history = construct_chat_history(conversation_history) utc_date = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d") @@ -118,21 +134,26 @@ async def generate_python_code( # Validate that the response is a non-empty, JSON-serializable list response = clean_json(response) response = json.loads(response) - codes = [code.strip() for code in response["codes"] if code.strip()] + code = response.get("code", "").strip() + input_files = response.get("input_files", []) + input_links = response.get("input_links", []) - if not isinstance(codes, list) or not codes or len(codes) == 0: + if not isinstance(code, str) or is_none_or_empty(code): raise ValueError - return codes + return GeneratedCode(code, input_files, input_links) -async def execute_sandboxed_python(code: str, sandbox_url: str = SANDBOX_URL) -> dict[str, Any]: +async def execute_sandboxed_python(code: str, input_data: list[dict], sandbox_url: str = SANDBOX_URL) -> dict[str, Any]: """ Takes code to run as a string and calls the terrarium API to execute it. Returns the result of the code execution as a dictionary. + + Reference data i/o format based on Terrarium example client code at: + https://github.com/cohere-ai/cohere-terrarium/blob/main/example-clients/python/terrarium_client.py """ headers = {"Content-Type": "application/json"} cleaned_code = clean_code_python(code) - data = {"code": cleaned_code} + data = {"code": cleaned_code, "files": input_data} async with aiohttp.ClientSession() as session: async with session.post(sandbox_url, json=data, headers=headers) as response: From af0215765ca59373f08c491e6f927d9f64dbb3c6 Mon Sep 17 00:00:00 2001 From: Debanjum Date: Fri, 8 Nov 2024 17:19:42 -0800 Subject: [PATCH 2/7] Decode code text output files from b64 to str to ease client processing --- src/interface/web/app/common/chatFunctions.ts | 4 ++++ .../web/app/components/chatMessage/chatMessage.tsx | 4 ++-- src/khoj/processor/tools/run_code.py | 9 +++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/interface/web/app/common/chatFunctions.ts b/src/interface/web/app/common/chatFunctions.ts index b4777ba8..5e9ac321 100644 --- a/src/interface/web/app/common/chatFunctions.ts +++ b/src/interface/web/app/common/chatFunctions.ts @@ -203,6 +203,10 @@ export function renderCodeGenImageInline(message: string, codeContext: CodeConte if (file.filename.match(/\.(png|jpg|jpeg|gif|webp)$/i)) { const replacement = `![${file.filename}](data:image/${file.filename.split(".").pop()};base64,${file.b64_data})`; message = message.replace(regex, replacement); + } else if (file.filename.match(/\.(txt|org|md)$/i)) { + // render output files generated by codegen as downloadable links + const replacement = `![${file.filename}](data:text/plain;base64,${file.b64_data})`; + message = message.replace(regex, replacement); } }); }); diff --git a/src/interface/web/app/components/chatMessage/chatMessage.tsx b/src/interface/web/app/components/chatMessage/chatMessage.tsx index aae271b1..4869b5fa 100644 --- a/src/interface/web/app/components/chatMessage/chatMessage.tsx +++ b/src/interface/web/app/components/chatMessage/chatMessage.tsx @@ -420,6 +420,7 @@ const ChatMessage = forwardRef((props, ref) => message += `\n\n${inferredQueries[0]}`; } } + // Handle user attached images rendering let messageForClipboard = message; let messageToRender = message; @@ -480,8 +481,7 @@ const ChatMessage = forwardRef((props, ref) => file.filename.endsWith(".org") || file.filename.endsWith(".md") ) { - const decodedText = atob(file.b64_data); - message += `\n\n\`\`\`\n${decodedText}\n\`\`\``; + message += `\n\n## ${file.filename}\n\`\`\`\n${file.b64_data}\n\`\`\`\n`; } }); }); diff --git a/src/khoj/processor/tools/run_code.py b/src/khoj/processor/tools/run_code.py index fe89da1c..5366cd02 100644 --- a/src/khoj/processor/tools/run_code.py +++ b/src/khoj/processor/tools/run_code.py @@ -2,7 +2,9 @@ import base64 import datetime import json import logging +import mimetypes import os +from pathlib import Path from typing import Any, Callable, List, NamedTuple, Optional import aiohttp @@ -160,6 +162,13 @@ async def execute_sandboxed_python(code: str, input_data: list[dict], sandbox_ur if response.status == 200: result: dict[str, Any] = await response.json() result["code"] = cleaned_code + # Store decoded output files + for output_file in result.get("output_files", []): + # Decode text files as UTF-8 + if mimetypes.guess_type(output_file["filename"])[0].startswith("text/") or Path( + output_file["filename"] + ).suffix in [".org", ".md", ".json"]: + output_file["b64_data"] = base64.b64decode(output_file["b64_data"]).decode("utf-8") return result else: return { From 92c1efe6eeb4fe776e90d231ee81e13054586003 Mon Sep 17 00:00:00 2001 From: Debanjum Date: Sun, 10 Nov 2024 21:03:03 -0800 Subject: [PATCH 3/7] Fixes to render & save code context with non text based output modes - Fix to render code generated chart with images, excalidraw diagrams - Fix to save code context to chat history in image, diagram output modes - Fix bug in image markdown being wrapped twice in markdown syntax - Render newline in code references shown on chat page of web app Previously newlines weren't getting rendered. This made the code executed by Khoj hard to read in references. This changes fixes that. `dangerouslySetInnerHTML' usage is justified as rendered code snippet is being sanitized by DOMPurify before rendering. --- src/interface/web/app/common/chatFunctions.ts | 31 +-------- .../components/chatMessage/chatMessage.tsx | 66 +++++++------------ .../referencePanel/referencePanel.tsx | 12 ++-- src/khoj/routers/api_chat.py | 2 + 4 files changed, 36 insertions(+), 75 deletions(-) diff --git a/src/interface/web/app/common/chatFunctions.ts b/src/interface/web/app/common/chatFunctions.ts index 5e9ac321..aeb74ee8 100644 --- a/src/interface/web/app/common/chatFunctions.ts +++ b/src/interface/web/app/common/chatFunctions.ts @@ -139,33 +139,6 @@ export function processMessageChunk( if (onlineContext) currentMessage.onlineContext = onlineContext; if (context) currentMessage.context = context; - // Replace file links with base64 data - currentMessage.rawResponse = renderCodeGenImageInline( - currentMessage.rawResponse, - codeContext, - ); - - // Add code context files to the message - if (codeContext) { - Object.entries(codeContext).forEach(([key, value]) => { - value.results.output_files?.forEach((file) => { - if (file.filename.endsWith(".png") || file.filename.endsWith(".jpg")) { - // Don't add the image again if it's already in the message! - if (!currentMessage.rawResponse.includes(`![${file.filename}](`)) { - currentMessage.rawResponse += `\n\n![${file.filename}](data:image/png;base64,${file.b64_data})`; - } - } else if ( - file.filename.endsWith(".txt") || - file.filename.endsWith(".org") || - file.filename.endsWith(".md") - ) { - const decodedText = atob(file.b64_data); - currentMessage.rawResponse += `\n\n\`\`\`\n${decodedText}\n\`\`\``; - } - }); - }); - } - // Mark current message streaming as completed currentMessage.completed = true; } @@ -200,10 +173,10 @@ export function renderCodeGenImageInline(message: string, codeContext: CodeConte Object.values(codeContext).forEach((contextData) => { contextData.results.output_files?.forEach((file) => { const regex = new RegExp(`!?\\[.*?\\]\\(.*${file.filename}\\)`, "g"); - if (file.filename.match(/\.(png|jpg|jpeg|gif|webp)$/i)) { + if (file.filename.match(/\.(png|jpg|jpeg)$/i)) { const replacement = `![${file.filename}](data:image/${file.filename.split(".").pop()};base64,${file.b64_data})`; message = message.replace(regex, replacement); - } else if (file.filename.match(/\.(txt|org|md)$/i)) { + } else if (file.filename.match(/\.(txt|org|md|csv|json)$/i)) { // render output files generated by codegen as downloadable links const replacement = `![${file.filename}](data:text/plain;base64,${file.b64_data})`; message = message.replace(regex, replacement); diff --git a/src/interface/web/app/components/chatMessage/chatMessage.tsx b/src/interface/web/app/components/chatMessage/chatMessage.tsx index 4869b5fa..a8fb2f6f 100644 --- a/src/interface/web/app/components/chatMessage/chatMessage.tsx +++ b/src/interface/web/app/components/chatMessage/chatMessage.tsx @@ -421,6 +421,31 @@ const ChatMessage = forwardRef((props, ref) => } } + // Replace file links with base64 data + message = renderCodeGenImageInline(message, props.chatMessage.codeContext); + + // Add code context files to the message + if (props.chatMessage.codeContext) { + Object.entries(props.chatMessage.codeContext).forEach(([key, value]) => { + value.results.output_files?.forEach((file) => { + if (file.filename.endsWith(".png") || file.filename.endsWith(".jpg")) { + // Don't add the image again if it's already in the message! + if (!message.includes(`![${file.filename}](`)) { + message += `\n\n![${file.filename}](data:image/png;base64,${file.b64_data})`; + } + } else if ( + file.filename.endsWith(".txt") || + file.filename.endsWith(".org") || + file.filename.endsWith(".md") || + file.filename.endsWith(".csv") || + file.filename.endsWith(".json") + ) { + message += `\n\n## ${file.filename}\n\`\`\`\n${file.b64_data}\n\`\`\`\n`; + } + }); + }); + } + // Handle user attached images rendering let messageForClipboard = message; let messageToRender = message; @@ -446,47 +471,6 @@ const ChatMessage = forwardRef((props, ref) => messageToRender = `${userImagesInHtml}${messageToRender}`; } - if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image") { - message = `![generated image](data:image/png;base64,${message})`; - } else if (props.chatMessage.intent && props.chatMessage.intent.type == "text-to-image2") { - message = `![generated image](${message})`; - } else if ( - props.chatMessage.intent && - props.chatMessage.intent.type == "text-to-image-v3" - ) { - message = `![generated image](data:image/webp;base64,${message})`; - } - if ( - props.chatMessage.intent && - props.chatMessage.intent.type.includes("text-to-image") && - props.chatMessage.intent["inferred-queries"]?.length > 0 - ) { - message += `\n\n${props.chatMessage.intent["inferred-queries"][0]}`; - } - - // Replace file links with base64 data - message = renderCodeGenImageInline(message, props.chatMessage.codeContext); - - // Add code context files to the message - if (props.chatMessage.codeContext) { - Object.entries(props.chatMessage.codeContext).forEach(([key, value]) => { - value.results.output_files?.forEach((file) => { - if (file.filename.endsWith(".png") || file.filename.endsWith(".jpg")) { - // Don't add the image again if it's already in the message! - if (!message.includes(`![${file.filename}](`)) { - message += `\n\n![${file.filename}](data:image/png;base64,${file.b64_data})`; - } - } else if ( - file.filename.endsWith(".txt") || - file.filename.endsWith(".org") || - file.filename.endsWith(".md") - ) { - message += `\n\n## ${file.filename}\n\`\`\`\n${file.b64_data}\n\`\`\`\n`; - } - }); - }); - } - // Set the message text setTextRendered(messageForClipboard); diff --git a/src/interface/web/app/components/referencePanel/referencePanel.tsx b/src/interface/web/app/components/referencePanel/referencePanel.tsx index 785d05d8..53016cfc 100644 --- a/src/interface/web/app/components/referencePanel/referencePanel.tsx +++ b/src/interface/web/app/components/referencePanel/referencePanel.tsx @@ -103,7 +103,7 @@ interface CodeContextReferenceCardProps { function CodeContextReferenceCard(props: CodeContextReferenceCardProps) { const fileIcon = getIconFromFilename(".py", "w-6 h-6 text-muted-foreground inline-flex mr-2"); - const snippet = DOMPurify.sanitize(props.code); + const sanitizedCodeSnippet = DOMPurify.sanitize(props.code.replace(/\n/g, "
")); const [isHovering, setIsHovering] = useState(false); return ( @@ -123,9 +123,8 @@ function CodeContextReferenceCard(props: CodeContextReferenceCardProps) {

- {snippet} -

+ dangerouslySetInnerHTML={{ __html: sanitizedCodeSnippet }} + >

@@ -136,7 +135,10 @@ function CodeContextReferenceCard(props: CodeContextReferenceCardProps) { {fileIcon} Code -

{snippet}

+

diff --git a/src/khoj/routers/api_chat.py b/src/khoj/routers/api_chat.py index 33476a97..7c79809f 100644 --- a/src/khoj/routers/api_chat.py +++ b/src/khoj/routers/api_chat.py @@ -1131,6 +1131,7 @@ async def chat( conversation_id=conversation_id, compiled_references=compiled_references, online_results=online_results, + code_results=code_results, query_images=uploaded_images, train_of_thought=train_of_thought, attached_file_context=attached_file_context, @@ -1192,6 +1193,7 @@ async def chat( conversation_id=conversation_id, compiled_references=compiled_references, online_results=online_results, + code_results=code_results, query_images=uploaded_images, train_of_thought=train_of_thought, attached_file_context=attached_file_context, From 8e9f4262a985eeeb29b0be7dd9ba6072dff78596 Mon Sep 17 00:00:00 2001 From: Debanjum Date: Mon, 11 Nov 2024 00:22:23 -0800 Subject: [PATCH 4/7] Render code output files with code references in reference section - Improve rendering code reference with better icons, smaller text and different line clamps for better visibility - Show code output files as sub card of code card in reference section - Allow downloading files generated by code instead of rendering it in chat message directly - Show executed code before online references in reference panel --- src/interface/web/app/common/iconUtils.tsx | 8 +- .../components/chatMessage/chatMessage.tsx | 8 -- .../referencePanel/referencePanel.tsx | 129 +++++++++++++++--- 3 files changed, 118 insertions(+), 27 deletions(-) diff --git a/src/interface/web/app/common/iconUtils.tsx b/src/interface/web/app/common/iconUtils.tsx index f266cac5..4fd7d443 100644 --- a/src/interface/web/app/common/iconUtils.tsx +++ b/src/interface/web/app/common/iconUtils.tsx @@ -40,7 +40,6 @@ import { Leaf, NewspaperClipping, OrangeSlice, - Rainbow, SmileyMelting, YinYang, SneakerMove, @@ -247,6 +246,13 @@ function getIconFromFilename( case "doc": case "docx": return ; + case "csv": + case "json": + return ; + case "txt": + return ; + case "py": + return ; case "jpg": case "jpeg": case "png": diff --git a/src/interface/web/app/components/chatMessage/chatMessage.tsx b/src/interface/web/app/components/chatMessage/chatMessage.tsx index a8fb2f6f..2d63d58c 100644 --- a/src/interface/web/app/components/chatMessage/chatMessage.tsx +++ b/src/interface/web/app/components/chatMessage/chatMessage.tsx @@ -433,14 +433,6 @@ const ChatMessage = forwardRef((props, ref) => if (!message.includes(`![${file.filename}](`)) { message += `\n\n![${file.filename}](data:image/png;base64,${file.b64_data})`; } - } else if ( - file.filename.endsWith(".txt") || - file.filename.endsWith(".org") || - file.filename.endsWith(".md") || - file.filename.endsWith(".csv") || - file.filename.endsWith(".json") - ) { - message += `\n\n## ${file.filename}\n\`\`\`\n${file.b64_data}\n\`\`\`\n`; } }); }); diff --git a/src/interface/web/app/components/referencePanel/referencePanel.tsx b/src/interface/web/app/components/referencePanel/referencePanel.tsx index 53016cfc..0237f18f 100644 --- a/src/interface/web/app/components/referencePanel/referencePanel.tsx +++ b/src/interface/web/app/components/referencePanel/referencePanel.tsx @@ -2,7 +2,7 @@ import { useEffect, useState } from "react"; -import { ArrowRight } from "@phosphor-icons/react"; +import { ArrowCircleDown, ArrowRight } from "@phosphor-icons/react"; import markdownIt from "markdown-it"; const md = new markdownIt({ @@ -11,7 +11,13 @@ const md = new markdownIt({ typographer: true, }); -import { Context, WebPage, OnlineContext, CodeContext } from "../chatMessage/chatMessage"; +import { + Context, + WebPage, + OnlineContext, + CodeContext, + CodeContextFile, +} from "../chatMessage/chatMessage"; import { Card } from "@/components/ui/card"; import { @@ -97,6 +103,7 @@ function NotesContextReferenceCard(props: NotesContextReferenceCardProps) { interface CodeContextReferenceCardProps { code: string; output: string; + output_files: CodeContextFile[]; error: string; showFullContent: boolean; } @@ -105,6 +112,38 @@ function CodeContextReferenceCard(props: CodeContextReferenceCardProps) { const fileIcon = getIconFromFilename(".py", "w-6 h-6 text-muted-foreground inline-flex mr-2"); const sanitizedCodeSnippet = DOMPurify.sanitize(props.code.replace(/\n/g, "
")); const [isHovering, setIsHovering] = useState(false); + const [isDownloadHover, setIsDownloadHover] = useState(false); + + const handleDownload = (file: CodeContextFile) => { + // Determine MIME type + let mimeType = "text/plain"; + let byteString = file.b64_data; + if (file.filename.match(/\.(png|jpg|jpeg|webp)$/)) { + mimeType = `image/${file.filename.split(".").pop()}`; + byteString = atob(file.b64_data); + } else if (file.filename.endsWith(".json")) { + mimeType = "application/json"; + } else if (file.filename.endsWith(".csv")) { + mimeType = "text/csv"; + } + + const arrayBuffer = new ArrayBuffer(byteString.length); + const bytes = new Uint8Array(arrayBuffer); + + for (let i = 0; i < byteString.length; i++) { + bytes[i] = byteString.charCodeAt(i); + } + + const blob = new Blob([arrayBuffer], { type: mimeType }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = file.filename; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + }; return ( <> @@ -122,9 +161,66 @@ function CodeContextReferenceCard(props: CodeContextReferenceCardProps) { Code

+ {props.output_files && props.output_files.length > 0 && ( +
+ {props.output_files + .slice(0, props.showFullContent ? undefined : 1) + .map((file, index) => { + const fileIcon = getIconFromFilename( + file.filename, + "w-4 h-4 text-muted-foreground inline-flex", + ); + return ( +
+

+ {fileIcon} + + {file.filename} + + +

+ {file.filename.match(/\.(txt|org|md|csv|json)$/) ? ( + 
+                                                        {file.b64_data}
+
+ ) : file.filename.match( + /\.(png|jpg|jpeg|webp)$/, + ) ? ( + {file.filename} + ) : null} +
+ ); + })} +
+ )} @@ -136,7 +232,7 @@ function CodeContextReferenceCard(props: CodeContextReferenceCardProps) { Code

@@ -146,14 +242,10 @@ function CodeContextReferenceCard(props: CodeContextReferenceCardProps) { ); } -export interface ReferencePanelData { - notesReferenceCardData: NotesContextReferenceData[]; - onlineReferenceCardData: OnlineReferenceData[]; -} - export interface CodeReferenceData { code: string; output: string; + output_files: CodeContextFile[]; error: string; } @@ -289,6 +381,7 @@ export function constructAllReferences( codeReferences.push({ code: value.code, output: value.results.std_out, + output_files: value.results.output_files, error: value.results.std_err, }); } @@ -497,15 +590,6 @@ export default function ReferencePanel(props: ReferencePanelDataProps) { /> ); })} - {props.onlineReferenceCardData.map((online, index) => { - return ( - - ); - })} {props.codeReferenceCardData.map((code, index) => { return ( ); })} + {props.onlineReferenceCardData.map((online, index) => { + return ( + + ); + })} From b970cfd4b3e07b98a0c3b255e53040b7b77c5af9 Mon Sep 17 00:00:00 2001 From: Debanjum Date: Mon, 11 Nov 2024 15:35:08 -0800 Subject: [PATCH 5/7] Align styling of reference panel card across code, docs, web results - Add a border below heading - Show code snippet in pre block - Overflow-x when reference side panel open to allow seeing whole text via x-scroll - Align header, body position of reference cards with each other - Only show filename in doc reference cards at message bottom. Show full file path in hover and reference side panel --- .../referencePanel/referencePanel.tsx | 200 +++++++++--------- 1 file changed, 103 insertions(+), 97 deletions(-) diff --git a/src/interface/web/app/components/referencePanel/referencePanel.tsx b/src/interface/web/app/components/referencePanel/referencePanel.tsx index 0237f18f..dc6d8360 100644 --- a/src/interface/web/app/components/referencePanel/referencePanel.tsx +++ b/src/interface/web/app/components/referencePanel/referencePanel.tsx @@ -57,6 +57,7 @@ function NotesContextReferenceCard(props: NotesContextReferenceCardProps) { props.title || ".txt", "w-6 h-6 text-muted-foreground inline-flex mr-2", ); + const fileName = props.title.split("/").pop() || props.title; const snippet = extractSnippet(props); const [isHovering, setIsHovering] = useState(false); @@ -67,30 +68,30 @@ function NotesContextReferenceCard(props: NotesContextReferenceCardProps) { setIsHovering(true)} onMouseLeave={() => setIsHovering(false)} - className={`${props.showFullContent ? "w-auto" : "w-[200px]"} overflow-hidden break-words text-balance rounded-lg p-2 bg-muted border-none`} + className={`${props.showFullContent ? "w-auto" : "w-[200px]"} overflow-hidden break-words text-balance rounded-lg border-none p-2 bg-muted`} >

{fileIcon} - {props.title} + {props.showFullContent ? props.title : fileName}

{fileIcon} {props.title}

@@ -109,8 +110,8 @@ interface CodeContextReferenceCardProps { } function CodeContextReferenceCard(props: CodeContextReferenceCardProps) { - const fileIcon = getIconFromFilename(".py", "w-6 h-6 text-muted-foreground inline-flex mr-2"); - const sanitizedCodeSnippet = DOMPurify.sanitize(props.code.replace(/\n/g, "
")); + const fileIcon = getIconFromFilename(".py", "!w-4 h-4 text-muted-foreground flex-shrink-0"); + const sanitizedCodeSnippet = DOMPurify.sanitize(props.code); const [isHovering, setIsHovering] = useState(false); const [isDownloadHover, setIsDownloadHover] = useState(false); @@ -152,89 +153,98 @@ function CodeContextReferenceCard(props: CodeContextReferenceCardProps) { setIsHovering(true)} onMouseLeave={() => setIsHovering(false)} - className={`${props.showFullContent ? "w-auto" : "w-[200px]"} overflow-hidden break-words text-balance rounded-lg p-2 bg-muted border-none`} + className={`${props.showFullContent ? "w-auto" : "w-[200px]"} overflow-hidden break-words text-balance rounded-lg border-none p-2 bg-muted`} > -

- {fileIcon} - Code -

-

- {props.output_files && props.output_files.length > 0 && ( -
- {props.output_files - .slice(0, props.showFullContent ? undefined : 1) - .map((file, index) => { - const fileIcon = getIconFromFilename( - file.filename, - "w-4 h-4 text-muted-foreground inline-flex", - ); - return ( -
-

- {fileIcon} - - {file.filename} - - -

- {file.filename.match(/\.(txt|org|md|csv|json)$/) ? ( - 
-                                                        {file.b64_data}
-
- ) : file.filename.match( - /\.(png|jpg|jpeg|webp)$/, - ) ? ( - {file.filename} - ) : null} -
- ); - })} +
+
+ {fileIcon} +

+ code artifacts +

- )} + 
 0 ? "hidden" : "overflow-hidden line-clamp-3"}`}
+                            >
+                                {sanitizedCodeSnippet}
+
+ {props.output_files?.length > 0 && ( +
+ {props.output_files + .slice(0, props.showFullContent ? undefined : 1) + .map((file, index) => { + return ( +
+

+ + {file.filename} + + +

+ {file.filename.match( + /\.(txt|org|md|csv|json)$/, + ) ? ( + 
+                                                            {file.b64_data}
+
+ ) : file.filename.match( + /\.(png|jpg|jpeg|webp)$/, + ) ? ( + {file.filename} + ) : null} +
+ ); + })} +
+ )} +
-

+
{fileIcon} - Code -

-

+

+ code +

+
+ 
+                            {sanitizedCodeSnippet}
+
@@ -291,21 +301,17 @@ function GenericOnlineReferenceCard(props: OnlineReferenceCardProps) {
- +

@@ -313,12 +319,12 @@ function GenericOnlineReferenceCard(props: OnlineReferenceCardProps) {

{props.title}

{props.description}

@@ -335,23 +341,23 @@ function GenericOnlineReferenceCard(props: OnlineReferenceCardProps) { href={props.link} target="_blank" rel="noreferrer" - className="!no-underline p-2" + className="!no-underline px-1" > -
- +
+

{domain}

{props.title}

{props.description}

From 218eed83cd9e8b22fad995aadb3d26bad87f96e5 Mon Sep 17 00:00:00 2001 From: Debanjum Date: Mon, 11 Nov 2024 16:59:53 -0800 Subject: [PATCH 6/7] Show output file not code on hover. Remove reference card title border --- .../referencePanel/referencePanel.tsx | 129 +++++++++--------- 1 file changed, 63 insertions(+), 66 deletions(-) diff --git a/src/interface/web/app/components/referencePanel/referencePanel.tsx b/src/interface/web/app/components/referencePanel/referencePanel.tsx index dc6d8360..2f7650b3 100644 --- a/src/interface/web/app/components/referencePanel/referencePanel.tsx +++ b/src/interface/web/app/components/referencePanel/referencePanel.tsx @@ -77,7 +77,7 @@ function NotesContextReferenceCard(props: NotesContextReferenceCardProps) { {props.showFullContent ? props.title : fileName}

@@ -146,6 +146,57 @@ function CodeContextReferenceCard(props: CodeContextReferenceCardProps) { URL.revokeObjectURL(url); }; + const renderOutputFiles = (files: CodeContextFile[], hoverCard: boolean) => { + if (files?.length == 0) return null; + return ( +
+ {files.slice(0, props.showFullContent ? undefined : 1).map((file, index) => { + return ( +
+

+ + {file.filename} + + +

+ {file.filename.match(/\.(txt|org|md|csv|json)$/) ? ( + 
+                                    {file.b64_data}
+
+ ) : file.filename.match(/\.(png|jpg|jpeg|webp)$/) ? ( + {file.filename} + ) : null} +
+ ); + })} +
+ ); + }; + return ( <> @@ -161,72 +212,15 @@ function CodeContextReferenceCard(props: CodeContextReferenceCardProps) {

- code artifacts + code {props.output_files?.length > 0 ? "artifacts" : ""} 

 0 ? "hidden" : "overflow-hidden line-clamp-3"}`}
+                                className={`text-xs pb-2 ${props.showFullContent ? "block overflow-x-auto" : props.output_files?.length > 0 ? "hidden" : "overflow-hidden line-clamp-3"}`}
                             >
                                 {sanitizedCodeSnippet}
- {props.output_files?.length > 0 && ( -
- {props.output_files - .slice(0, props.showFullContent ? undefined : 1) - .map((file, index) => { - return ( -
-

- - {file.filename} - - -

- {file.filename.match( - /\.(txt|org|md|csv|json)$/, - ) ? ( - 
-                                                            {file.b64_data}
-
- ) : file.filename.match( - /\.(png|jpg|jpeg|webp)$/, - ) ? ( - {file.filename} - ) : null} -
- ); - })} -
- )} + {renderOutputFiles(props.output_files, false)}
@@ -239,12 +233,15 @@ function CodeContextReferenceCard(props: CodeContextReferenceCardProps) {

- code + code {props.output_files?.length > 0 ? "artifact" : ""}

- 
-                            {sanitizedCodeSnippet}
-
+ {(props.output_files.length > 0 && + renderOutputFiles(props.output_files?.slice(0, 1), true)) || ( + 
+                                {sanitizedCodeSnippet}
+
+ )} @@ -319,7 +316,7 @@ function GenericOnlineReferenceCard(props: OnlineReferenceCardProps) {

{props.title}

From a52500d2893ce62a2489a7c8d843516a2de697ee Mon Sep 17 00:00:00 2001 From: Debanjum Date: Mon, 11 Nov 2024 17:30:06 -0800 Subject: [PATCH 7/7] Show generated code artifacts before notes and online references --- .../referencePanel/referencePanel.tsx | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/interface/web/app/components/referencePanel/referencePanel.tsx b/src/interface/web/app/components/referencePanel/referencePanel.tsx index 2f7650b3..e8942bb4 100644 --- a/src/interface/web/app/components/referencePanel/referencePanel.tsx +++ b/src/interface/web/app/components/referencePanel/referencePanel.tsx @@ -488,10 +488,10 @@ export function TeaserReferencesSection(props: TeaserReferenceSectionProps) { setNumTeaserSlots(props.isMobileWidth ? 1 : 3); }, [props.isMobileWidth]); - const notesDataToShow = props.notesReferenceCardData.slice(0, numTeaserSlots); - const codeDataToShow = props.codeReferenceCardData.slice( + const codeDataToShow = props.codeReferenceCardData.slice(0, numTeaserSlots); + const notesDataToShow = props.notesReferenceCardData.slice( 0, - numTeaserSlots - notesDataToShow.length, + numTeaserSlots - codeDataToShow.length, ); const onlineDataToShow = notesDataToShow.length + codeDataToShow.length < numTeaserSlots @@ -522,15 +522,6 @@ export function TeaserReferencesSection(props: TeaserReferenceSectionProps) {

{numReferences} sources

- {notesDataToShow.map((note, index) => { - return ( - - ); - })} {codeDataToShow.map((code, index) => { return ( ); })} + {notesDataToShow.map((note, index) => { + return ( + + ); + })} {onlineDataToShow.map((online, index) => { return ( View all references for this response
- {props.notesReferenceCardData.map((note, index) => { - return ( - - ); - })} {props.codeReferenceCardData.map((code, index) => { return ( ); })} + {props.notesReferenceCardData.map((note, index) => { + return ( + + ); + })} {props.onlineReferenceCardData.map((online, index) => { return (