Add Feedback Buttons on Chat (#721)

### Description and Rationale for Changes
This feature includes thumbs up and thumbs down buttons on Khoj's chat responses that provide automated feedback. When a thumbs up/down button is clicked, the code sends an email to team@khoj.dev with the following:
* user query
* khoj's response
* whether the sentiment of the user was good or bad. 

This is critical in improving Khoj's nondeterministic LLM model for a better user experience.
### List of Changes
* new endpoint in `api_chat.py` (/feedback) that can be used to trigger mail sending).
* thumbs up and thumbs down buttons implemented in `chat.html`
* new function in `routers/email.py` to handle feedback email sending via resend
* `feedback.html` template for a formatted email with the feedback.

---------

Co-authored-by: mythicalcow <mythicalcow@linux.myguest.virtualbox.org>
Co-authored-by: sabaimran <narmiabas@gmail.com>
This commit is contained in:
Raghav Tirumale 2024-05-20 16:29:08 -05:00 committed by GitHub
parent f941948d11
commit d57772f9e7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 216 additions and 16 deletions

View file

@ -0,0 +1,34 @@
<!DOCTYPE html>
<html>
<head>
<title>Khoj Feedback Form</title>
</head>
<body>
<body style="font-family: 'Verdana', sans-serif; font-weight: 400; font-style: normal; padding: 0; text-align: left; width: 600px; margin: 20px auto;">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
<a class="logo" href="https://khoj.dev" target="_blank" style="text-decoration: none; text-decoration: underline dotted;">
<img src="https://khoj.dev/khoj-logo-sideways-500.png" alt="Khoj Logo" style="width: 100px;">
</a>
<div class="calls-to-action" style="margin-top: 20px;">
<div>
<h1 style="color: #333; font-size: large; font-weight: bold; margin: 10px; line-height: 1.5; background-color: #fee285; padding: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.5);">User Feedback:</h1>
<div>
<h3>User Query</h3>
<p>{{uquery}}</p>
</div>
<div>
<h3>Khoj's Response</h3>
{{kquery}}
</div>
<div>
<h3>Sentiment</h3>
<p>{{sentiment}}</p>
</div>
<div>
<h3>User Email</h3>
<p>{{user_email}}</p>
</div>
</div>
</div>
</body>
</html>

View file

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
<svg fill="#000000" width="800px" height="800px" viewBox="-7.5 0 32 32" version="1.1" xmlns="http://www.w3.org/2000/svg">
<title>thumbs-down</title>
<path d="M5.92 25.24c-0.64 0-1.36-0.2-2.2-0.64-0.28-0.16-0.44-0.44-0.44-0.76v-14.96c0-0.36 0.24-0.72 0.6-0.8 0.2-0.040 4.8-1.32 8.16-1.32 1.36 0 2.36 0.2 3 0.6 0.88 0.56 1.44 1.52 1.6 2.92 0.36 2.44-0.52 5.92-1.44 6.96-0.8 0.88-2.36 1.040-3.84 1.2-0.72 0.080-1.72 0.2-2 0.36s-0.44 1.4-0.52 2.12c-0.24 1.84-0.6 4.32-2.92 4.32zM4.92 23.32c0.48 0.2 0.8 0.24 1 0.24 0.72 0 1-0.88 1.24-2.84 0.2-1.36 0.36-2.68 1.24-3.28 0.6-0.4 1.6-0.52 2.76-0.64 0.96-0.12 2.44-0.28 2.8-0.68 0.48-0.52 1.36-3.36 1.040-5.6-0.080-0.6-0.32-1.4-0.84-1.72-0.24-0.12-0.8-0.36-2.12-0.36-2.44 0-5.76 0.76-7.12 1.080 0 0 0 13.8 0 13.8zM0.84 18.64c-0.48 0-0.84-0.36-0.84-0.84v-8.92c0-0.48 0.36-0.84 0.84-0.84s0.84 0.36 0.84 0.84v8.96c0 0.44-0.36 0.8-0.84 0.8z"></path>
</svg>

After

Width:  |  Height:  |  Size: 1,019 B

View file

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
<svg fill="#000000" width="800px" height="800px" viewBox="-7.5 0 32 32" version="1.1" xmlns="http://www.w3.org/2000/svg">
<title>thumbs-up</title>
<path d="M12.040 25.24v0c-3.36 0-7.96-1.24-8.16-1.32-0.36-0.080-0.6-0.44-0.6-0.8v-14.96c0-0.32 0.16-0.6 0.44-0.76 0.84-0.44 1.56-0.64 2.2-0.64 2.32 0 2.68 2.48 2.92 4.28 0.080 0.72 0.24 1.92 0.52 2.12s1.28 0.28 2 0.36c1.52 0.16 3.080 0.32 3.84 1.2 0.92 1.040 1.8 4.52 1.44 6.96-0.2 1.4-0.76 2.36-1.6 2.92-0.68 0.44-1.68 0.64-3 0.64zM4.92 22.48c1.36 0.32 4.64 1.080 7.12 1.080 1.32 0 1.92-0.24 2.12-0.36 0.52-0.32 0.76-1.12 0.84-1.72 0.32-2.24-0.56-5.080-1.040-5.6-0.36-0.4-1.84-0.56-2.8-0.68-1.16-0.12-2.12-0.24-2.76-0.64-0.88-0.6-1.080-1.92-1.24-3.28-0.28-1.96-0.52-2.84-1.24-2.84-0.2 0-0.52 0.040-1 0.24 0 0 0 13.8 0 13.8zM0.84 23.96c-0.48 0-0.84-0.36-0.84-0.84v-8.92c0-0.48 0.36-0.84 0.84-0.84s0.84 0.36 0.84 0.84v8.96c0 0.44-0.36 0.8-0.84 0.8z"></path>
</svg>

After

Width:  |  Height:  |  Size: 1 KiB

View file

@ -175,11 +175,30 @@ To get started, just start typing below. You can also type / to see a list of co
return referenceButton; return referenceButton;
} }
var khojQuery = "";
function renderMessage(message, by, dt=null, annotations=null, raw=false, renderType="append") { function renderMessage(message, by, dt=null, annotations=null, raw=false, renderType="append", userQuery=null) {
let message_time = formatDate(dt ?? new Date()); let message_time = formatDate(dt ?? new Date());
let by_name = by == "khoj" ? "🏮 Khoj" : "🤔 You"; let by_name = by == "khoj" ? "🏮 Khoj" : "🤔 You";
let formattedMessage = formatHTMLMessage(message, raw); let formattedMessage = formatHTMLMessage(message, raw, true, userQuery);
//update userQuery or khojQuery to latest query for feedback purposes
if(by !== "khoj"){
raw = formattedMessage.innerHTML;
}
//find the thumbs up and thumbs down buttons from the message formatter
var thumbsUpButtons = formattedMessage.querySelectorAll('.thumbs-up-button');
var thumbsDownButtons = formattedMessage.querySelectorAll('.thumbs-down-button');
//only render the feedback options if the message is a response from khoj
if(by !== "khoj"){
thumbsUpButtons.forEach(function(element) {
element.parentNode.removeChild(element);
});
thumbsDownButtons.forEach(function(element) {
element.parentNode.removeChild(element);
});
}
// Create a new div for the chat message // Create a new div for the chat message
let chatMessage = document.createElement('div'); let chatMessage = document.createElement('div');
@ -258,7 +277,7 @@ To get started, just start typing below. You can also type / to see a list of co
return numOnlineReferences; return numOnlineReferences;
} }
function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) { function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null, userQuery) {
// If no document or online context is provided, render the message as is // If no document or online context is provided, render the message as is
if ((context == null || context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) { if ((context == null || context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) {
if (intentType?.includes("text-to-image")) { if (intentType?.includes("text-to-image")) {
@ -274,14 +293,14 @@ To get started, just start typing below. You can also type / to see a list of co
if (inferredQuery) { if (inferredQuery) {
imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`; imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
} }
return renderMessage(imageMarkdown, by, dt, null, false, "return"); return renderMessage(imageMarkdown, by, dt, null, false, "return", userQuery);
} }
return renderMessage(message, by, dt, null, false, "return"); return renderMessage(message, by, dt, null, false, "return", userQuery);
} }
if ((context && context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) { if ((context && context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) {
return renderMessage(message, by, dt, null, false, "return"); return renderMessage(message, by, dt, null, false, "return", userQuery);
} }
// If document or online context is provided, render the message with its references // If document or online context is provided, render the message with its references
@ -342,13 +361,27 @@ To get started, just start typing below. You can also type / to see a list of co
if (inferredQuery) { if (inferredQuery) {
imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`; imageMarkdown += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
} }
return renderMessage(imageMarkdown, by, dt, references, false, "return"); return renderMessage(imageMarkdown, by, dt, references, false, "return", userQuery);
} }
return renderMessage(message, by, dt, references, false, "return"); return renderMessage(message, by, dt, references, false, "return", userQuery);
}
//handler function for posting feedback data to endpoint
function sendFeedback(_uquery="", _kquery="", _sentiment="") {
const uquery = _uquery;
const kquery = _kquery;
const sentiment = _sentiment;
fetch('/api/chat/feedback', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({uquery: uquery, kquery: kquery, sentiment: sentiment})
})
.then(response => response.json())
} }
function formatHTMLMessage(message, raw=false, willReplace=true) { function formatHTMLMessage(message, raw=false, willReplace=true, userQuery) {
var md = window.markdownit(); var md = window.markdownit();
let newHTML = message; let newHTML = message;
@ -392,7 +425,35 @@ To get started, just start typing below. You can also type / to see a list of co
copyIcon.classList.add("copy-icon"); copyIcon.classList.add("copy-icon");
copyButton.appendChild(copyIcon); copyButton.appendChild(copyIcon);
copyButton.addEventListener('click', createCopyParentText(message)); copyButton.addEventListener('click', createCopyParentText(message));
element.append(copyButton);
//create thumbs-up button
let thumbsUpButton = document.createElement('button');
thumbsUpButton.className = 'thumbs-up-button';
let thumbsUpIcon = document.createElement("img");
thumbsUpIcon.src = "/static/assets/icons/thumbs-up-svgrepo-com.svg";
thumbsUpIcon.classList.add("thumbs-up-icon");
thumbsUpButton.appendChild(thumbsUpIcon);
thumbsUpButton.onclick = function() {
khojQuery = newHTML;
thumbsUpIcon.src = "/static/assets/icons/confirm-icon.svg";
sendFeedback(userQuery,khojQuery,"Good Response");
};
// Create thumbs-down button
let thumbsDownButton = document.createElement('button');
thumbsDownButton.className = 'thumbs-down-button';
let thumbsDownIcon = document.createElement("img");
thumbsDownIcon.src = "/static/assets/icons/thumbs-down-svgrepo-com.svg";
thumbsDownIcon.classList.add("thumbs-down-icon");
thumbsDownButton.appendChild(thumbsDownIcon);
thumbsDownButton.onclick = function() {
khojQuery = newHTML;
thumbsDownIcon.src = "/static/assets/icons/confirm-icon.svg";
sendFeedback(userQuery,khojQuery,"Bad Response");
};
// Append buttons to parent element
element.append(copyButton, thumbsDownButton, thumbsUpButton);
} }
renderMathInElement(element, { renderMathInElement(element, {
@ -1202,7 +1263,8 @@ To get started, just start typing below. You can also type / to see a list of co
new Date(chat_log.created + "Z"), new Date(chat_log.created + "Z"),
chat_log.onlineContext, chat_log.onlineContext,
chat_log.intent?.type, chat_log.intent?.type,
chat_log.intent?.["inferred-queries"]); chat_log.intent?.["inferred-queries"],
chat_log.intent?.query);
chatBody.appendChild(messageElement); chatBody.appendChild(messageElement);
// When the 4th oldest message is within viewing distance (~60% scroll up) // When the 4th oldest message is within viewing distance (~60% scroll up)
@ -1297,7 +1359,8 @@ To get started, just start typing below. You can also type / to see a list of co
new Date(chat_log.created + "Z"), new Date(chat_log.created + "Z"),
chat_log.onlineContext, chat_log.onlineContext,
chat_log.intent?.type, chat_log.intent?.type,
chat_log.intent?.["inferred-queries"] chat_log.intent?.["inferred-queries"],
chat_log.intent?.query
); );
entry.target.replaceWith(messageElement); entry.target.replaceWith(messageElement);
@ -2390,6 +2453,32 @@ To get started, just start typing below. You can also type / to see a list of co
float: right; float: right;
} }
button.thumbs-up-button {
border-radius: 4px;
background-color: var(--background-color);
border: 1px solid var(--main-text-color);
text-align: center;
font-size: 16px;
transition: all 0.5s;
cursor: pointer;
padding: 4px;
float: right;
margin-right: 4px;
}
button.thumbs-down-button {
border-radius: 4px;
background-color: var(--background-color);
border: 1px solid var(--main-text-color);
text-align: center;
font-size: 16px;
transition: all 0.5s;
cursor: pointer;
padding: 4px;
float: right;
margin-right:4px;
}
button.copy-button span { button.copy-button span {
cursor: pointer; cursor: pointer;
display: inline-block; display: inline-block;
@ -2398,8 +2487,18 @@ To get started, just start typing below. You can also type / to see a list of co
} }
img.copy-icon { img.copy-icon {
width: 16px; width: 18px;
height: 16px; height: 18px;
}
img.thumbs-up-icon {
width: 18px;
height: 18px;
}
img.thumbs-down-icon {
width: 18px;
height: 18px;
} }
button.copy-button:hover { button.copy-button:hover {
@ -2407,6 +2506,16 @@ To get started, just start typing below. You can also type / to see a list of co
color: #f5f5f5; color: #f5f5f5;
} }
button.thumbs-up-button:hover {
background-color: var(--primary-hover);
color: #f5f5f5;
}
button.thumbs-down-button:hover {
background-color: var(--primary-hover);
color: #f5f5f5;
}
pre { pre {
text-wrap: unset; text-wrap: unset;
} }

View file

@ -454,7 +454,7 @@ You are Khoj, an advanced google search assistant. You are tasked with construct
- Break messages into multiple search queries when required to retrieve the relevant information. - Break messages into multiple search queries when required to retrieve the relevant information.
- Use site: google search operators when appropriate - Use site: google search operators when appropriate
- You have access to the the whole internet to retrieve information. - You have access to the the whole internet to retrieve information.
- Official, up-to-date information about you, Khoj, is available at site:khoj.dev, github or pypi. - Official, up-to-date information about you, Khoj, is available at site:khoj.dev
What Google searches, if any, will you need to perform to answer the user's question? What Google searches, if any, will you need to perform to answer the user's question?
Provide search queries as a list of strings in a JSON object. Provide search queries as a list of strings in a JSON object.
@ -510,6 +510,7 @@ Q: How many oranges would fit in NASA's Saturn V rocket?
Khoj: {{"queries": ["volume of an orange", "volume of saturn v rocket"]}} Khoj: {{"queries": ["volume of an orange", "volume of saturn v rocket"]}}
Now it's your turn to construct Google search queries to answer the user's question. Provide them as a list of strings in a JSON object. Do not say anything else. Now it's your turn to construct Google search queries to answer the user's question. Provide them as a list of strings in a JSON object. Do not say anything else.
Now it's your turn to construct a search query for Google to answer the user's question.
History: History:
{chat_history} {chat_history}

View file

@ -68,6 +68,23 @@ conversation_command_rate_limiter = ConversationCommandRateLimiter(
api_chat = APIRouter() api_chat = APIRouter()
from pydantic import BaseModel
from khoj.routers.email import send_query_feedback
class FeedbackData(BaseModel):
uquery: str
kquery: str
sentiment: str
@api_chat.post("/feedback")
@requires(["authenticated"])
async def sendfeedback(request: Request, data: FeedbackData):
user: KhojUser = request.user.object
await send_query_feedback(data.uquery, data.kquery, data.sentiment, user.email)
@api_chat.get("/starters", response_class=Response) @api_chat.get("/starters", response_class=Response)
@requires(["authenticated"]) @requires(["authenticated"])

View file

@ -50,6 +50,33 @@ async def send_welcome_email(name, email):
) )
async def send_query_feedback(uquery, kquery, sentiment, user_email):
if not is_resend_enabled():
logger.debug(f"Sentiment: {sentiment}, Query: {uquery}, Khoj Response: {kquery}")
return
logger.info(f"Sending feedback email for query {uquery}")
# rendering feedback email using feedback.html as template
template = env.get_template("feedback.html")
html_content = template.render(
uquery=uquery if not is_none_or_empty(uquery) else "N/A",
kquery=kquery if not is_none_or_empty(kquery) else "N/A",
sentiment=sentiment if not is_none_or_empty(sentiment) else "N/A",
user_email=user_email if not is_none_or_empty(user_email) else "N/A",
)
# send feedback from two fixed accounts
r = resend.Emails.send(
{
"sender": "saba@khoj.dev",
"to": "team@khoj.dev",
"subject": f"User Feedback",
"html": html_content,
}
)
return {"message": "Sent Email"}
def send_task_email(name, email, query, result, subject): def send_task_email(name, email, query, result, subject):
if not is_resend_enabled(): if not is_resend_enabled():
logger.debug("Email sending disabled") logger.debug("Email sending disabled")