Configure tests to mark chat quality tests & filter unhelpful warnings

- Mark chat quality tests, register custom mark for chat quality
- Filter unhelpful deprecation warnings from within dateparser library
- Error if tests use unregistered marks

pyproject.toml

@@ -95,3 +95,9 @@ warn_unused_ignores = false
 
 [tool.black]
 line-length = 120
+
+[tool.pytest.ini_options]
+addopts = "--strict-markers"
+markers = [
+    "chatquality: Evaluate chatbot capabilities and quality",
+]

tests/test_chatbot.py

@@ -30,6 +30,7 @@ def test_message_to_understand_prompt():
 
 
 # ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
 @pytest.mark.skipif(
     api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys"
 )
@@ -50,6 +51,7 @@ def test_chat_with_no_chat_history_or_retrieved_content():
 
 
 # ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
 @pytest.mark.skipif(
     api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys"
 )
@@ -81,6 +83,7 @@ def test_answer_from_chat_history_and_no_content():
 
 
 # ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
 @pytest.mark.skipif(
     api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys"
 )
@@ -111,6 +114,7 @@ def test_answer_from_chat_history_and_previously_retrieved_content():
 
 
 # ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
 @pytest.mark.skipif(
     api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys"
 )
@@ -140,6 +144,7 @@ def test_answer_from_chat_history_and_currently_retrieved_content():
 
 
 # ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
 @pytest.mark.skipif(
     api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys"
 )
@@ -170,6 +175,7 @@ def test_no_answer_in_chat_history_or_retrieved_content():
 
 
 # ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
 @pytest.mark.skipif(
     api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys"
 )
@@ -206,6 +212,7 @@ def test_answer_requires_current_date_awareness():
 
 
 # ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
 @pytest.mark.skipif(
     api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys"
 )
@@ -239,6 +246,7 @@ def test_answer_requires_date_aware_aggregation_across_provided_notes():
 
 
 # ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
 @pytest.mark.skipif(
     api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys"
 )
@@ -273,6 +281,7 @@ def test_answer_general_question_not_in_chat_history_or_retrieved_content():
 
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.xfail(reason="Chatbot not consistently capable of asking for clarification yet.")
+@pytest.mark.chatquality
 @pytest.mark.skipif(
     api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys"
 )

tests/test_date_filter.py

@@ -3,11 +3,15 @@ import re
 from datetime import datetime
 from math import inf
 
-# Application Packages
+# External Packages
+import pytest
+
+# Internal Packages
 from khoj.search_filter.date_filter import DateFilter
 from khoj.utils.rawconfig import Entry
 
 
+@pytest.mark.filterwarnings("ignore:The localize method is no longer necessary.")
 def test_date_filter():
     entries = [
         Entry(compiled="", raw="Entry with no date"),
@@ -46,6 +50,7 @@ def test_date_filter():
     assert entry_indices == {1, 2}
 
 
+@pytest.mark.filterwarnings("ignore:The localize method is no longer necessary.")
 def test_extract_date_range():
     assert DateFilter().extract_date_range('head dt>"1984-01-04" dt<"1984-01-07" tail') == [
         datetime(1984, 1, 5, 0, 0, 0).timestamp(),
@@ -68,6 +73,7 @@ def test_extract_date_range():
     assert DateFilter().extract_date_range('head dt>"1984-01-01" dt<"1984-01-01" tail') == None
 
 
+@pytest.mark.filterwarnings("ignore:The localize method is no longer necessary.")
 def test_parse():
     test_now = datetime(1984, 4, 1, 21, 21, 21)