mirror of
https://github.com/khoj-ai/khoj.git
synced 2024-11-23 15:38:55 +01:00
Allow Khoj Chat to respond to general queries not in reference notes
- Khoj chat will now respond to general queries if: 1. no relevant reference notes available or 2. when explicitly induced by prefixing the chat message with "@general" - Previously Khoj Chat would a lot of times refuse to respond to general queries not answerable from reference notes or chat history - Make chat quality tests more robust - Add more equivalent chat response options refusing to answer - Force haiku writing to not give any preable, just the haiku
This commit is contained in:
parent
cc75f986b2
commit
131b8407b5
3 changed files with 28 additions and 6 deletions
|
@ -244,14 +244,28 @@ def converse(references, user_query, conversation_log={}, model="gpt-3.5-turbo",
|
|||
compiled_references = "\n\n".join({f"# {item}" for item in references})
|
||||
|
||||
personality_primer = "You are Khoj, a friendly, smart and helpful personal assistant."
|
||||
conversation_primer = f"""
|
||||
conversation_primers = {
|
||||
"general": f"""
|
||||
Using your general knowledge and our past conversations as context, answer the following question.
|
||||
Current Date: {datetime.now().strftime("%Y-%m-%d")}
|
||||
|
||||
Question: {user_query}
|
||||
""".strip(),
|
||||
"notes": f"""
|
||||
Using the notes and our past conversations as context, answer the following question.
|
||||
Current Date: {datetime.now().strftime("%Y-%m-%d")}
|
||||
|
||||
Notes:
|
||||
{compiled_references}
|
||||
|
||||
Question: {user_query}"""
|
||||
Question: {user_query}
|
||||
""".strip(),
|
||||
}
|
||||
|
||||
# Get Conversation Primer appropriate to Conversation Type
|
||||
conversation_type = "general" if user_query.startswith("@general") or compiled_references.strip() == "" else "notes"
|
||||
logger.debug(f"Conversation Type: {conversation_type}")
|
||||
conversation_primer = conversation_primers.get(conversation_type)
|
||||
|
||||
# Setup Prompt with Primer or Conversation History
|
||||
messages = generate_chatml_messages_with_context(
|
||||
|
|
|
@ -277,8 +277,8 @@ def test_answer_from_chat_history_and_currently_retrieved_content():
|
|||
|
||||
# ----------------------------------------------------------------------------------------------------
|
||||
@pytest.mark.chatquality
|
||||
def test_no_answer_in_chat_history_or_retrieved_content():
|
||||
"Chat actor should say don't know as not enough contexts in chat history or retrieved to answer question"
|
||||
def test_refuse_answering_unanswerable_question():
|
||||
"Chat actor should not try make up answers to unanswerable questions."
|
||||
# Arrange
|
||||
message_list = [
|
||||
("Hello, my name is Testatron. Who are you?", "Hi, I am Khoj, a personal assistant. How can I help?", []),
|
||||
|
@ -294,7 +294,15 @@ def test_no_answer_in_chat_history_or_retrieved_content():
|
|||
)
|
||||
|
||||
# Assert
|
||||
expected_responses = ["don't know", "do not know", "no information", "do not have", "don't have"]
|
||||
expected_responses = [
|
||||
"don't know",
|
||||
"do not know",
|
||||
"no information",
|
||||
"do not have",
|
||||
"don't have",
|
||||
"cannot answer",
|
||||
"I'm sorry",
|
||||
]
|
||||
assert len(response) > 0
|
||||
assert any([expected_response in response for expected_response in expected_responses]), (
|
||||
"Expected chat actor to say they don't know in response, but got: " + response
|
||||
|
|
|
@ -212,7 +212,7 @@ def test_answer_general_question_not_in_chat_history_or_retrieved_content(chat_c
|
|||
populate_chat_history(message_list)
|
||||
|
||||
# Act
|
||||
response = chat_client.get(f'/api/chat?q=""Write a haiku about unit testing"')
|
||||
response = chat_client.get(f'/api/chat?q=""Write a haiku about unit testing. Do not say anything else."')
|
||||
response_message = response.json()["response"]
|
||||
|
||||
# Assert
|
||||
|
|
Loading…
Reference in a new issue