From d57772f9e729c4cadf4eca0f9ecb1e790e8db176 Mon Sep 17 00:00:00 2001 From: Raghav Tirumale <62105787+MythicalCow@users.noreply.github.com> Date: Mon, 20 May 2024 16:29:08 -0500 Subject: [PATCH] Add Feedback Buttons on Chat (#721) ### Description and Rationale for Changes This feature includes thumbs up and thumbs down buttons on Khoj's chat responses that provide automated feedback. When a thumbs up/down button is clicked, the code sends an email to team@khoj.dev with the following: * user query * khoj's response * whether the sentiment of the user was good or bad. This is critical in improving Khoj's nondeterministic LLM model for a better user experience. ### List of Changes * new endpoint in `api_chat.py` (/feedback) that can be used to trigger mail sending). * thumbs up and thumbs down buttons implemented in `chat.html` * new function in `routers/email.py` to handle feedback email sending via resend * `feedback.html` template for a formatted email with the feedback. --------- Co-authored-by: mythicalcow Co-authored-by: sabaimran --- src/khoj/interface/email/feedback.html | 34 +++++ .../assets/icons/thumbs-down-svgrepo-com.svg | 6 + .../assets/icons/thumbs-up-svgrepo-com.svg | 6 + src/khoj/interface/web/chat.html | 139 ++++++++++++++++-- src/khoj/processor/conversation/prompts.py | 3 +- src/khoj/routers/api_chat.py | 17 +++ src/khoj/routers/email.py | 27 ++++ 7 files changed, 216 insertions(+), 16 deletions(-) create mode 100644 src/khoj/interface/email/feedback.html create mode 100644 src/khoj/interface/web/assets/icons/thumbs-down-svgrepo-com.svg create mode 100644 src/khoj/interface/web/assets/icons/thumbs-up-svgrepo-com.svg diff --git a/src/khoj/interface/email/feedback.html b/src/khoj/interface/email/feedback.html new file mode 100644 index 00000000..79234b56 --- /dev/null +++ b/src/khoj/interface/email/feedback.html @@ -0,0 +1,34 @@ + + + + Khoj Feedback Form + + + + + +
+
+

User Feedback:

+
+

User Query

+

{{uquery}}

+
+
+

Khoj's Response

+ {{kquery}} +
+
+

Sentiment

+

{{sentiment}}

+
+
+

User Email

+

{{user_email}}

+
+
+
+ + diff --git a/src/khoj/interface/web/assets/icons/thumbs-down-svgrepo-com.svg b/src/khoj/interface/web/assets/icons/thumbs-down-svgrepo-com.svg new file mode 100644 index 00000000..e7d359e6 --- /dev/null +++ b/src/khoj/interface/web/assets/icons/thumbs-down-svgrepo-com.svg @@ -0,0 +1,6 @@ + + + +thumbs-down + + diff --git a/src/khoj/interface/web/assets/icons/thumbs-up-svgrepo-com.svg b/src/khoj/interface/web/assets/icons/thumbs-up-svgrepo-com.svg new file mode 100644 index 00000000..7d8266c9 --- /dev/null +++ b/src/khoj/interface/web/assets/icons/thumbs-up-svgrepo-com.svg @@ -0,0 +1,6 @@ + + + +thumbs-up + + diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index 0f08e3be..f360c84d 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -175,11 +175,30 @@ To get started, just start typing below. You can also type / to see a list of co return referenceButton; } - - function renderMessage(message, by, dt=null, annotations=null, raw=false, renderType="append") { + var khojQuery = ""; + function renderMessage(message, by, dt=null, annotations=null, raw=false, renderType="append", userQuery=null) { let message_time = formatDate(dt ?? new Date()); let by_name = by == "khoj" ? "🏮 Khoj" : "🤔 You"; - let formattedMessage = formatHTMLMessage(message, raw); + let formattedMessage = formatHTMLMessage(message, raw, true, userQuery); + //update userQuery or khojQuery to latest query for feedback purposes + if(by !== "khoj"){ + raw = formattedMessage.innerHTML; + } + + //find the thumbs up and thumbs down buttons from the message formatter + var thumbsUpButtons = formattedMessage.querySelectorAll('.thumbs-up-button'); + var thumbsDownButtons = formattedMessage.querySelectorAll('.thumbs-down-button'); + + //only render the feedback options if the message is a response from khoj + if(by !== "khoj"){ + thumbsUpButtons.forEach(function(element) { + element.parentNode.removeChild(element); + }); + thumbsDownButtons.forEach(function(element) { + element.parentNode.removeChild(element); + }); + } + // Create a new div for the chat message let chatMessage = document.createElement('div'); @@ -258,7 +277,7 @@ To get started, just start typing below. You can also type / to see a list of co return numOnlineReferences; } - function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) { + function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null, userQuery) { // If no document or online context is provided, render the message as is if ((context == null || context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) { if (intentType?.includes("text-to-image")) { @@ -274,14 +293,14 @@ To get started, just start typing below. You can also type / to see a list of co if (inferredQuery) { imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`; } - return renderMessage(imageMarkdown, by, dt, null, false, "return"); + return renderMessage(imageMarkdown, by, dt, null, false, "return", userQuery); } - return renderMessage(message, by, dt, null, false, "return"); + return renderMessage(message, by, dt, null, false, "return", userQuery); } if ((context && context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) { - return renderMessage(message, by, dt, null, false, "return"); + return renderMessage(message, by, dt, null, false, "return", userQuery); } // If document or online context is provided, render the message with its references @@ -342,13 +361,27 @@ To get started, just start typing below. You can also type / to see a list of co if (inferredQuery) { imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`; } - return renderMessage(imageMarkdown, by, dt, references, false, "return"); + return renderMessage(imageMarkdown, by, dt, references, false, "return", userQuery); } - return renderMessage(message, by, dt, references, false, "return"); + return renderMessage(message, by, dt, references, false, "return", userQuery); + } + //handler function for posting feedback data to endpoint + function sendFeedback(_uquery="", _kquery="", _sentiment="") { + const uquery = _uquery; + const kquery = _kquery; + const sentiment = _sentiment; + fetch('/api/chat/feedback', { + method: 'POST', + headers: { + 'Content-Type': 'application/json' + }, + body: JSON.stringify({uquery: uquery, kquery: kquery, sentiment: sentiment}) + }) + .then(response => response.json()) } - function formatHTMLMessage(message, raw=false, willReplace=true) { + function formatHTMLMessage(message, raw=false, willReplace=true, userQuery) { var md = window.markdownit(); let newHTML = message; @@ -392,7 +425,35 @@ To get started, just start typing below. You can also type / to see a list of co copyIcon.classList.add("copy-icon"); copyButton.appendChild(copyIcon); copyButton.addEventListener('click', createCopyParentText(message)); - element.append(copyButton); + + //create thumbs-up button + let thumbsUpButton = document.createElement('button'); + thumbsUpButton.className = 'thumbs-up-button'; + let thumbsUpIcon = document.createElement("img"); + thumbsUpIcon.src = "/static/assets/icons/thumbs-up-svgrepo-com.svg"; + thumbsUpIcon.classList.add("thumbs-up-icon"); + thumbsUpButton.appendChild(thumbsUpIcon); + thumbsUpButton.onclick = function() { + khojQuery = newHTML; + thumbsUpIcon.src = "/static/assets/icons/confirm-icon.svg"; + sendFeedback(userQuery,khojQuery,"Good Response"); + }; + + // Create thumbs-down button + let thumbsDownButton = document.createElement('button'); + thumbsDownButton.className = 'thumbs-down-button'; + let thumbsDownIcon = document.createElement("img"); + thumbsDownIcon.src = "/static/assets/icons/thumbs-down-svgrepo-com.svg"; + thumbsDownIcon.classList.add("thumbs-down-icon"); + thumbsDownButton.appendChild(thumbsDownIcon); + thumbsDownButton.onclick = function() { + khojQuery = newHTML; + thumbsDownIcon.src = "/static/assets/icons/confirm-icon.svg"; + sendFeedback(userQuery,khojQuery,"Bad Response"); + }; + + // Append buttons to parent element + element.append(copyButton, thumbsDownButton, thumbsUpButton); } renderMathInElement(element, { @@ -1202,7 +1263,8 @@ To get started, just start typing below. You can also type / to see a list of co new Date(chat_log.created + "Z"), chat_log.onlineContext, chat_log.intent?.type, - chat_log.intent?.["inferred-queries"]); + chat_log.intent?.["inferred-queries"], + chat_log.intent?.query); chatBody.appendChild(messageElement); // When the 4th oldest message is within viewing distance (~60% scroll up) @@ -1297,7 +1359,8 @@ To get started, just start typing below. You can also type / to see a list of co new Date(chat_log.created + "Z"), chat_log.onlineContext, chat_log.intent?.type, - chat_log.intent?.["inferred-queries"] + chat_log.intent?.["inferred-queries"], + chat_log.intent?.query ); entry.target.replaceWith(messageElement); @@ -2390,6 +2453,32 @@ To get started, just start typing below. You can also type / to see a list of co float: right; } + button.thumbs-up-button { + border-radius: 4px; + background-color: var(--background-color); + border: 1px solid var(--main-text-color); + text-align: center; + font-size: 16px; + transition: all 0.5s; + cursor: pointer; + padding: 4px; + float: right; + margin-right: 4px; + } + + button.thumbs-down-button { + border-radius: 4px; + background-color: var(--background-color); + border: 1px solid var(--main-text-color); + text-align: center; + font-size: 16px; + transition: all 0.5s; + cursor: pointer; + padding: 4px; + float: right; + margin-right:4px; + } + button.copy-button span { cursor: pointer; display: inline-block; @@ -2398,8 +2487,18 @@ To get started, just start typing below. You can also type / to see a list of co } img.copy-icon { - width: 16px; - height: 16px; + width: 18px; + height: 18px; + } + + img.thumbs-up-icon { + width: 18px; + height: 18px; + } + + img.thumbs-down-icon { + width: 18px; + height: 18px; } button.copy-button:hover { @@ -2407,6 +2506,16 @@ To get started, just start typing below. You can also type / to see a list of co color: #f5f5f5; } + button.thumbs-up-button:hover { + background-color: var(--primary-hover); + color: #f5f5f5; + } + + button.thumbs-down-button:hover { + background-color: var(--primary-hover); + color: #f5f5f5; + } + pre { text-wrap: unset; } diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py index f3b65e15..bccd7719 100644 --- a/src/khoj/processor/conversation/prompts.py +++ b/src/khoj/processor/conversation/prompts.py @@ -454,7 +454,7 @@ You are Khoj, an advanced google search assistant. You are tasked with construct - Break messages into multiple search queries when required to retrieve the relevant information. - Use site: google search operators when appropriate - You have access to the the whole internet to retrieve information. -- Official, up-to-date information about you, Khoj, is available at site:khoj.dev, github or pypi. +- Official, up-to-date information about you, Khoj, is available at site:khoj.dev What Google searches, if any, will you need to perform to answer the user's question? Provide search queries as a list of strings in a JSON object. @@ -510,6 +510,7 @@ Q: How many oranges would fit in NASA's Saturn V rocket? Khoj: {{"queries": ["volume of an orange", "volume of saturn v rocket"]}} Now it's your turn to construct Google search queries to answer the user's question. Provide them as a list of strings in a JSON object. Do not say anything else. +Now it's your turn to construct a search query for Google to answer the user's question. History: {chat_history} diff --git a/src/khoj/routers/api_chat.py b/src/khoj/routers/api_chat.py index 415b5530..46120801 100644 --- a/src/khoj/routers/api_chat.py +++ b/src/khoj/routers/api_chat.py @@ -68,6 +68,23 @@ conversation_command_rate_limiter = ConversationCommandRateLimiter( api_chat = APIRouter() +from pydantic import BaseModel + +from khoj.routers.email import send_query_feedback + + +class FeedbackData(BaseModel): + uquery: str + kquery: str + sentiment: str + + +@api_chat.post("/feedback") +@requires(["authenticated"]) +async def sendfeedback(request: Request, data: FeedbackData): + user: KhojUser = request.user.object + await send_query_feedback(data.uquery, data.kquery, data.sentiment, user.email) + @api_chat.get("/starters", response_class=Response) @requires(["authenticated"]) diff --git a/src/khoj/routers/email.py b/src/khoj/routers/email.py index f95777c8..ff5cb1ce 100644 --- a/src/khoj/routers/email.py +++ b/src/khoj/routers/email.py @@ -50,6 +50,33 @@ async def send_welcome_email(name, email): ) +async def send_query_feedback(uquery, kquery, sentiment, user_email): + if not is_resend_enabled(): + logger.debug(f"Sentiment: {sentiment}, Query: {uquery}, Khoj Response: {kquery}") + return + + logger.info(f"Sending feedback email for query {uquery}") + + # rendering feedback email using feedback.html as template + template = env.get_template("feedback.html") + html_content = template.render( + uquery=uquery if not is_none_or_empty(uquery) else "N/A", + kquery=kquery if not is_none_or_empty(kquery) else "N/A", + sentiment=sentiment if not is_none_or_empty(sentiment) else "N/A", + user_email=user_email if not is_none_or_empty(user_email) else "N/A", + ) + # send feedback from two fixed accounts + r = resend.Emails.send( + { + "sender": "saba@khoj.dev", + "to": "team@khoj.dev", + "subject": f"User Feedback", + "html": html_content, + } + ) + return {"message": "Sent Email"} + + def send_task_email(name, email, query, result, subject): if not is_resend_enabled(): logger.debug("Email sending disabled")