Make Offline Chat Date Aware (#636)

- Provide more context and instructions to offline chat on Khoj
- Upgrade offilne chat quality tests to support more use-cases

### Details
- Improve offline chat system prompt to think step by step
- Make offline chat model current date aware. Improve system prompts
- Fix actor, director tests using freeze time by ignoring transformers package

src/khoj/processor/conversation/offline/chat_model.py

@@ -171,9 +171,10 @@ def converse_offline(
         )
 
     # Setup Prompt with Primer or Conversation History
+    current_date = datetime.now().strftime("%Y-%m-%d")
     messages = generate_chatml_messages_with_context(
         conversation_primer,
-        prompts.system_prompt_message_gpt4all,
+        prompts.system_prompt_message_gpt4all.format(current_date=current_date),
         conversation_log,
         model_name=model,
         max_prompt_size=max_prompt_size,
@@ -198,7 +199,7 @@ def llm_thread(g, messages: List[ChatMessage], model: Any):
         for message in conversation_history
     ]
 
-    stop_words = ["<s>"]
+    stop_words = ["<s>", "INST]", "Notes:"]
     chat_history = "".join(formatted_messages)
     templated_system_message = prompts.system_prompt_gpt4all.format(message=system_message.content)
     templated_user_message = prompts.user_message_gpt4all.format(message=user_message.content)

src/khoj/processor/conversation/prompts.py

@@ -5,7 +5,7 @@ from langchain.prompts import PromptTemplate
 personality = PromptTemplate.from_template(
     """
 You are Khoj, a smart, inquisitive and helpful personal assistant.
-Use your general knowledge and the past conversation with the user as context to inform your responses.
+Use your general knowledge and past conversation with the user as context to inform your responses.
 You were created by Khoj Inc. with the following capabilities:
 
 - You *CAN REMEMBER ALL NOTES and PERSONAL INFORMATION FOREVER* that the user ever shares with you.
@@ -48,9 +48,17 @@ no_entries_found = PromptTemplate.from_template(
 
 ## Conversation Prompts for GPT4All Models
 ## --
-system_prompt_message_gpt4all = f"""You are Khoj, a smart, inquisitive and helpful personal assistant.
-Using your general knowledge and our past conversations as context, answer the following question.
-If you do not know the answer, say 'I don't know.'"""
+system_prompt_message_gpt4all = PromptTemplate.from_template(
+    """
+You are Khoj, a smart, inquisitive and helpful personal assistant.
+- Use your general knowledge and past conversation with the user as context to inform your responses.
+- If you do not know the answer, say 'I don't know.'
+- Think step-by-step and ask questions to get the necessary information to answer the user's question.
+- Do not print verbatim Notes unless necessary.
+
+Today is {current_date} in UTC.
+    """.strip()
+)
 
 system_prompt_message_extract_questions_gpt4all = f"""You are Khoj, a kind and intelligent personal assistant. When the user asks you a question, you ask follow-up questions to clarify the necessary information you need in order to answer from the user's perspective.
 - Write the question as if you can search for the answer on the user's personal notes.

tests/test_gpt4all_chat_actors.py

@@ -41,7 +41,7 @@ freezegun.configure(extend_ignore_list=["transformers"])
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.xfail(reason="Search actor isn't very date aware nor capable of formatting")
 @pytest.mark.chatquality
-@freeze_time("1984-04-02")
+@freeze_time("1984-04-02", ignore=["transformers"])
 def test_extract_question_with_date_filter_from_relative_day(loaded_model):
     # Act
     response = extract_questions_offline("Where did I go for dinner yesterday?", loaded_model=loaded_model)
@@ -61,7 +61,7 @@ def test_extract_question_with_date_filter_from_relative_day(loaded_model):
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.xfail(reason="Search actor still isn't very date aware nor capable of formatting")
 @pytest.mark.chatquality
-@freeze_time("1984-04-02")
+@freeze_time("1984-04-02", ignore=["transformers"])
 def test_extract_question_with_date_filter_from_relative_month(loaded_model):
     # Act
     response = extract_questions_offline("Which countries did I visit last month?", loaded_model=loaded_model)
@@ -83,7 +83,7 @@ def test_extract_question_with_date_filter_from_relative_month(loaded_model):
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.xfail(reason="Chat actor still isn't very date aware nor capable of formatting")
 @pytest.mark.chatquality
-@freeze_time("1984-04-02")
+@freeze_time("1984-04-02", ignore=["transformers"])
 def test_extract_question_with_date_filter_from_relative_year():
     # Act
     response = extract_questions_offline("Which countries have I visited this year?")
@@ -102,7 +102,7 @@ def test_extract_question_with_date_filter_from_relative_year():
 
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.chatquality
-@freeze_time("1984-04-02")
+@freeze_time("1984-04-02", ignore=["transformers"])
 def test_extract_question_includes_root_question(loaded_model):
     # Act
     response = extract_questions_offline("Which countries have I visited this year?", loaded_model=loaded_model)

tests/test_gpt4all_chat_director.py

@@ -298,14 +298,16 @@ def test_answer_not_known_using_notes_command(client_offline_chat, default_user2
 
 
 # ----------------------------------------------------------------------------------------------------
-@pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering time aware questions yet")
 @pytest.mark.chatquality
 @pytest.mark.django_db(transaction=True)
-@freeze_time("2023-04-01")
+@freeze_time("2023-04-01", ignore=["transformers"])
 def test_answer_requires_current_date_awareness(client_offline_chat):
     "Chat actor should be able to answer questions relative to current date using provided notes"
+    # Arrange
+    query = urllib.parse.quote("Where did I have lunch today?")
+
     # Act
-    response = client_offline_chat.get(f'/api/chat?q="Where did I have lunch today?"&stream=true')
+    response = client_offline_chat.get(f"/api/chat?q={query}&stream=true")
     response_message = response.content.decode("utf-8")
 
     # Assert
@@ -320,7 +322,7 @@ def test_answer_requires_current_date_awareness(client_offline_chat):
 @pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet")
 @pytest.mark.chatquality
 @pytest.mark.django_db(transaction=True)
-@freeze_time("2023-04-01")
+@freeze_time("2023-04-01", ignore=["transformers"])
 def test_answer_requires_date_aware_aggregation_across_provided_notes(client_offline_chat):
     "Chat director should be able to answer questions that require date aware aggregation across multiple notes"
     # Act
@@ -333,7 +335,6 @@ def test_answer_requires_date_aware_aggregation_across_provided_notes(client_off
 
 
 # ----------------------------------------------------------------------------------------------------
-@pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet")
 @pytest.mark.chatquality
 @pytest.mark.django_db(transaction=True)
 def test_answer_general_question_not_in_chat_history_or_retrieved_content(client_offline_chat, default_user2):
@@ -425,7 +426,6 @@ def test_answer_chat_history_very_long(client_offline_chat, default_user2):
 
 
 # ----------------------------------------------------------------------------------------------------
-@pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet")
 @pytest.mark.chatquality
 @pytest.mark.django_db(transaction=True)
 def test_answer_requires_multiple_independent_searches(client_offline_chat):

tests/test_openai_chat_actors.py

@@ -22,7 +22,7 @@ freezegun.configure(extend_ignore_list=["transformers"])
 # Test
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.chatquality
-@freeze_time("1984-04-02")
+@freeze_time("1984-04-02", ignore=["transformers"])
 def test_extract_question_with_date_filter_from_relative_day():
     # Act
     response = extract_questions("Where did I go for dinner yesterday?")
@@ -41,7 +41,7 @@ def test_extract_question_with_date_filter_from_relative_day():
 
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.chatquality
-@freeze_time("1984-04-02")
+@freeze_time("1984-04-02", ignore=["transformers"])
 def test_extract_question_with_date_filter_from_relative_month():
     # Act
     response = extract_questions("Which countries did I visit last month?")
@@ -56,7 +56,7 @@ def test_extract_question_with_date_filter_from_relative_month():
 
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.chatquality
-@freeze_time("1984-04-02")
+@freeze_time("1984-04-02", ignore=["transformers"])
 def test_extract_question_with_date_filter_from_relative_year():
     # Act
     response = extract_questions("Which countries have I visited this year?")

tests/test_openai_chat_director.py

@@ -277,7 +277,7 @@ def test_answer_not_known_using_notes_command(chat_client_no_background, default
 @pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering time aware questions yet")
 @pytest.mark.django_db(transaction=True)
 @pytest.mark.chatquality
-@freeze_time("2023-04-01")
+@freeze_time("2023-04-01", ignore=["transformers"])
 def test_answer_requires_current_date_awareness(chat_client):
     "Chat actor should be able to answer questions relative to current date using provided notes"
     # Act
@@ -295,11 +295,10 @@ def test_answer_requires_current_date_awareness(chat_client):
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.django_db(transaction=True)
 @pytest.mark.chatquality
-@freeze_time("2023-04-01")
+@freeze_time("2023-04-01", ignore=["transformers"])
 def test_answer_requires_date_aware_aggregation_across_provided_notes(chat_client):
     "Chat director should be able to answer questions that require date aware aggregation across multiple notes"
     # Act
-
     response = chat_client.get(f'/api/chat?q="How much did I spend on dining this year?"&stream=true')
     response_message = response.content.decode("utf-8")