Allow Khoj Chat to respond to general queries not in reference notes

- Khoj chat will now respond to general queries if:
  1. no relevant reference notes available or
  2. when explicitly induced by prefixing the chat message with "@general"

- Previously Khoj Chat would a lot of times refuse to respond to
  general queries not answerable from reference notes or chat history

- Make chat quality tests more robust
  - Add more equivalent chat response options refusing to answer
  - Force haiku writing to not give any preable, just the haiku
This commit is contained in:
Debanjum Singh Solanky 2023-05-12 18:35:14 +08:00
parent cc75f986b2
commit 131b8407b5
3 changed files with 28 additions and 6 deletions

View file

@ -244,14 +244,28 @@ def converse(references, user_query, conversation_log={}, model="gpt-3.5-turbo",
compiled_references = "\n\n".join({f"# {item}" for item in references})
personality_primer = "You are Khoj, a friendly, smart and helpful personal assistant."
conversation_primer = f"""
conversation_primers = {
"general": f"""
Using your general knowledge and our past conversations as context, answer the following question.
Current Date: {datetime.now().strftime("%Y-%m-%d")}
Question: {user_query}
""".strip(),
"notes": f"""
Using the notes and our past conversations as context, answer the following question.
Current Date: {datetime.now().strftime("%Y-%m-%d")}
Notes:
{compiled_references}
Question: {user_query}"""
Question: {user_query}
""".strip(),
}
# Get Conversation Primer appropriate to Conversation Type
conversation_type = "general" if user_query.startswith("@general") or compiled_references.strip() == "" else "notes"
logger.debug(f"Conversation Type: {conversation_type}")
conversation_primer = conversation_primers.get(conversation_type)
# Setup Prompt with Primer or Conversation History
messages = generate_chatml_messages_with_context(

View file

@ -277,8 +277,8 @@ def test_answer_from_chat_history_and_currently_retrieved_content():
# ----------------------------------------------------------------------------------------------------
@pytest.mark.chatquality
def test_no_answer_in_chat_history_or_retrieved_content():
"Chat actor should say don't know as not enough contexts in chat history or retrieved to answer question"
def test_refuse_answering_unanswerable_question():
"Chat actor should not try make up answers to unanswerable questions."
# Arrange
message_list = [
("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []),
@ -294,7 +294,15 @@ def test_no_answer_in_chat_history_or_retrieved_content():
)
# Assert
expected_responses = ["don't know", "do not know", "no information", "do not have", "don't have"]
expected_responses = [
"don't know",
"do not know",
"no information",
"do not have",
"don't have",
"cannot answer",
"I'm sorry",
]
assert len(response) > 0
assert any([expected_response in response for expected_response in expected_responses]), (
"Expected chat actor to say they don't know in response, but got: " + response

View file

@ -212,7 +212,7 @@ def test_answer_general_question_not_in_chat_history_or_retrieved_content(chat_c
populate_chat_history(message_list)
# Act
response = chat_client.get(f'/api/chat?q=""Write a haiku about unit testing"')
response = chat_client.get(f'/api/chat?q=""Write a haiku about unit testing. Do not say anything else."')
response_message = response.json()["response"]
# Assert