From dfb277ee37f9381ed48fea55ec219e43b6213b9d Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Thu, 16 Mar 2023 12:20:33 -0600 Subject: [PATCH] Set skipif at module level if OpenAI API key not set for chat tests - Remove stale message_to_prompt test It is too broad, reduces maintainability. Remove as it doesn't really need its own test right now - Setting skipif at module level for chat actor, director tests reduces code duplication as earlier was using decorator on each chat test --- tests/test_chat_actors.py | 51 +++++----------------------------- tests/test_chat_director.py | 55 ++++--------------------------------- 2 files changed, 13 insertions(+), 93 deletions(-) diff --git a/tests/test_chat_actors.py b/tests/test_chat_actors.py index 7639252c..365643cc 100644 --- a/tests/test_chat_actors.py +++ b/tests/test_chat_actors.py @@ -7,34 +7,21 @@ import pytest # Internal Packages from khoj.processor.conversation.gpt import converse -from khoj.processor.conversation.utils import message_to_log, message_to_prompt +from khoj.processor.conversation.utils import message_to_log # Initialize variables for tests -api_key = os.getenv("OPENAI_API_KEY") # Set your OPENAI_API_KEY as environment variable to run the tests below +api_key = os.getenv("OPENAI_API_KEY") +if api_key is None: + pytest.skip( + reason="Set OPENAI_API_KEY environment variable to run tests below. Get OpenAI API key from https://platform.openai.com/account/api-keys", + allow_module_level=True, + ) # Test -# ---------------------------------------------------------------------------------------------------- -def test_message_to_understand_prompt(): - # Arrange - understand_primer = 'Extract information from each chat message\n\nremember(memory-type, data);\nmemory-type=["companion", "notes", "ledger", "image", "music"]\nsearch(search-type, data);\nsearch-type=["google", "youtube"]\ngenerate(activity);\nactivity=["paint","write", "chat"]\ntrigger-emotion(emotion);\nemotion=["happy","confidence","fear","surprise","sadness","disgust","anger", "curiosity", "calm"]\n\nQ: How are you doing?\nA: activity("chat"); trigger-emotion("surprise")\nQ: Do you remember what I told you about my brother Antoine when we were at the beach?\nA: remember("notes", "Brother Antoine when we were at the beach"); trigger-emotion("curiosity");\nQ: what did we talk about last time?\nA: remember("notes", "talk last time"); trigger-emotion("curiosity");\nQ: Let\'s make some drawings!\nA: generate("paint"); trigger-emotion("happy");\nQ: Do you know anything about Lebanon?\nA: search("google", "lebanon"); trigger-emotion("confidence");\nQ: Find a video about a panda rolling in the grass\nA: search("youtube","panda rolling in the grass"); trigger-emotion("happy"); \nQ: Tell me a scary story\nA: generate("write" "A story about some adventure"); trigger-emotion("fear");\nQ: What fiction book was I reading last week about AI starship?\nA: remember("notes", "read fiction book about AI starship last week"); trigger-emotion("curiosity");\nQ: How much did I spend at Subway for dinner last time?\nA: remember("ledger", "last Subway dinner"); trigger-emotion("curiosity");\nQ: I\'m feeling sleepy\nA: activity("chat"); trigger-emotion("calm")\nQ: What was that popular Sri lankan song that Alex showed me recently?\nA: remember("music", "popular Sri lankan song that Alex showed recently"); trigger-emotion("curiosity"); \nQ: You\'re pretty funny!\nA: activity("chat"); trigger-emotion("pride")' - expected_response = 'Extract information from each chat message\n\nremember(memory-type, data);\nmemory-type=["companion", "notes", "ledger", "image", "music"]\nsearch(search-type, data);\nsearch-type=["google", "youtube"]\ngenerate(activity);\nactivity=["paint","write", "chat"]\ntrigger-emotion(emotion);\nemotion=["happy","confidence","fear","surprise","sadness","disgust","anger", "curiosity", "calm"]\n\nQ: How are you doing?\nA: activity("chat"); trigger-emotion("surprise")\nQ: Do you remember what I told you about my brother Antoine when we were at the beach?\nA: remember("notes", "Brother Antoine when we were at the beach"); trigger-emotion("curiosity");\nQ: what did we talk about last time?\nA: remember("notes", "talk last time"); trigger-emotion("curiosity");\nQ: Let\'s make some drawings!\nA: generate("paint"); trigger-emotion("happy");\nQ: Do you know anything about Lebanon?\nA: search("google", "lebanon"); trigger-emotion("confidence");\nQ: Find a video about a panda rolling in the grass\nA: search("youtube","panda rolling in the grass"); trigger-emotion("happy"); \nQ: Tell me a scary story\nA: generate("write" "A story about some adventure"); trigger-emotion("fear");\nQ: What fiction book was I reading last week about AI starship?\nA: remember("notes", "read fiction book about AI starship last week"); trigger-emotion("curiosity");\nQ: How much did I spend at Subway for dinner last time?\nA: remember("ledger", "last Subway dinner"); trigger-emotion("curiosity");\nQ: I\'m feeling sleepy\nA: activity("chat"); trigger-emotion("calm")\nQ: What was that popular Sri lankan song that Alex showed me recently?\nA: remember("music", "popular Sri lankan song that Alex showed recently"); trigger-emotion("curiosity"); \nQ: You\'re pretty funny!\nA: activity("chat"); trigger-emotion("pride")\nQ: When did I last dine at Burger King?\nA:' - - # Act - actual_response = message_to_prompt( - "When did I last dine at Burger King?", understand_primer, start_sequence="\nA:", restart_sequence="\nQ:" - ) - - # Assert - assert actual_response == expected_response - - # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys" -) def test_chat_with_no_chat_history_or_retrieved_content(): # Act response = converse( @@ -53,9 +40,6 @@ def test_chat_with_no_chat_history_or_retrieved_content(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys" -) def test_answer_from_chat_history_and_no_content(): # Arrange conversation_log = {"chat": []} @@ -85,9 +69,6 @@ def test_answer_from_chat_history_and_no_content(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys" -) def test_answer_from_chat_history_and_previously_retrieved_content(): "Chat actor needs to use context in previous notes and chat history to answer question" # Arrange @@ -116,9 +97,6 @@ def test_answer_from_chat_history_and_previously_retrieved_content(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys" -) def test_answer_from_chat_history_and_currently_retrieved_content(): "Chat actor needs to use context across currently retrieved notes and chat history to answer question" # Arrange @@ -146,9 +124,6 @@ def test_answer_from_chat_history_and_currently_retrieved_content(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys" -) def test_no_answer_in_chat_history_or_retrieved_content(): "Chat actor should say don't know as not enough contexts in chat history or retrieved to answer question" # Arrange @@ -179,9 +154,6 @@ def test_no_answer_in_chat_history_or_retrieved_content(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys" -) def test_answer_requires_current_date_awareness(): "Chat actor should be able to answer questions relative to current date using provided notes" # Arrange @@ -216,9 +188,6 @@ def test_answer_requires_current_date_awareness(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys" -) def test_answer_requires_date_aware_aggregation_across_provided_notes(): "Chat actor should be able to answer questions that require date aware aggregation across multiple notes" # Arrange @@ -250,9 +219,6 @@ def test_answer_requires_date_aware_aggregation_across_provided_notes(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys" -) def test_answer_general_question_not_in_chat_history_or_retrieved_content(): "Chat actor should be able to answer general questions not requiring looking at chat history or notes" # Arrange @@ -285,9 +251,6 @@ def test_answer_general_question_not_in_chat_history_or_retrieved_content(): # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(reason="Chat actor not consistently capable of asking for clarification yet.") @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, reason="Set api_key variable to your OpenAI API key from https://beta.openai.com/account/api-keys" -) def test_ask_for_clarification_if_not_enough_context_in_question(): "Chat actor should ask for clarification if question cannot be answered unambiguously with the provided context" # Arrange diff --git a/tests/test_chat_director.py b/tests/test_chat_director.py index ec56f8e4..35e08c88 100644 --- a/tests/test_chat_director.py +++ b/tests/test_chat_director.py @@ -11,7 +11,12 @@ from khoj.utils import state # Initialize variables for tests -api_key = os.getenv("OPENAI_API_KEY") # Set your OPENAI_API_KEY as environment variable to run the tests below +api_key = os.getenv("OPENAI_API_KEY") +if api_key is None: + pytest.skip( + reason="Set OPENAI_API_KEY environment variable to run tests below. Get OpenAI API key from https://platform.openai.com/account/api-keys", + allow_module_level=True, + ) # Helpers @@ -29,10 +34,6 @@ def populate_chat_history(message_list): # Tests # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, - reason="Set api_key variable to your OpenAI API key from https://platform.openai.com/account/api-keys", -) def test_chat_with_no_chat_history_or_retrieved_content(chat_client): # Act response = chat_client.get(f'/api/chat?q="Hello, my name is Testatron. Who are you?"') @@ -48,10 +49,6 @@ def test_chat_with_no_chat_history_or_retrieved_content(chat_client): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, - reason="Set api_key variable to your OpenAI API key from https://platform.openai.com/account/api-keys", -) def test_answer_from_chat_history(chat_client): # Arrange message_list = [ @@ -74,10 +71,6 @@ def test_answer_from_chat_history(chat_client): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, - reason="Set api_key variable to your OpenAI API key from https://platform.openai.com/account/api-keys", -) def test_answer_from_currently_retrieved_content(chat_client): # Arrange message_list = [ @@ -97,10 +90,6 @@ def test_answer_from_currently_retrieved_content(chat_client): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, - reason="Set api_key variable to your OpenAI API key from https://platform.openai.com/account/api-keys", -) def test_answer_from_chat_history_and_previously_retrieved_content(chat_client): # Arrange message_list = [ @@ -123,10 +112,6 @@ def test_answer_from_chat_history_and_previously_retrieved_content(chat_client): # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering this question yet") @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, - reason="Set api_key variable to your OpenAI API key from https://platform.openai.com/account/api-keys", -) def test_answer_from_chat_history_and_currently_retrieved_content(chat_client): # Arrange message_list = [ @@ -150,10 +135,6 @@ def test_answer_from_chat_history_and_currently_retrieved_content(chat_client): # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, - reason="Set api_key variable to your OpenAI API key from https://platform.openai.com/account/api-keys", -) def test_no_answer_in_chat_history_or_retrieved_content(chat_client): "Chat director should say don't know as not enough contexts in chat history or retrieved to answer question" # Arrange @@ -178,10 +159,6 @@ def test_no_answer_in_chat_history_or_retrieved_content(chat_client): # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering time aware questions yet") @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, - reason="Set api_key variable to your OpenAI API key from https://platform.openai.com/account/api-keys", -) @freeze_time("2023-04-01") def test_answer_requires_current_date_awareness(chat_client): "Chat actor should be able to answer questions relative to current date using provided notes" @@ -200,10 +177,6 @@ def test_answer_requires_current_date_awareness(chat_client): # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(AssertionError, reason="Chat director not capable of answering time aware questions yet") @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, - reason="Set api_key variable to your OpenAI API key from https://platform.openai.com/account/api-keys", -) @freeze_time("2023-04-01") def test_answer_requires_date_aware_aggregation_across_provided_notes(chat_client): "Chat director should be able to answer questions that require date aware aggregation across multiple notes" @@ -218,10 +191,6 @@ def test_answer_requires_date_aware_aggregation_across_provided_notes(chat_clien # ---------------------------------------------------------------------------------------------------- @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, - reason="Set api_key variable to your OpenAI API key from https://platform.openai.com/account/api-keys", -) def test_answer_general_question_not_in_chat_history_or_retrieved_content(chat_client): # Arrange message_list = [ @@ -247,10 +216,6 @@ def test_answer_general_question_not_in_chat_history_or_retrieved_content(chat_c # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(reason="Chat director not consistently capable of asking for clarification yet.") @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, - reason="Set api_key variable to your OpenAI API key from https://platform.openai.com/account/api-keys", -) def test_ask_for_clarification_if_not_enough_context_in_question(chat_client): # Act response = chat_client.get(f'/api/chat?q="What is the name of Namitas older son"') @@ -272,10 +237,6 @@ def test_ask_for_clarification_if_not_enough_context_in_question(chat_client): # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(reason="Chat director not capable of answering this question yet") @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, - reason="Set api_key variable to your OpenAI API key from https://platform.openai.com/account/api-keys", -) def test_answer_in_chat_history_beyond_lookback_window(chat_client): # Arrange message_list = [ @@ -300,10 +261,6 @@ def test_answer_in_chat_history_beyond_lookback_window(chat_client): # ---------------------------------------------------------------------------------------------------- @pytest.mark.xfail(reason="Chat director not capable of answering this question yet") @pytest.mark.chatquality -@pytest.mark.skipif( - api_key is None, - reason="Set api_key variable to your OpenAI API key from https://platform.openai.com/account/api-keys", -) def test_answer_requires_multiple_independent_searches(chat_client): "Chat director should be able to answer by doing multiple independent searches for required information" # Act