Improve and simplify Khoj Chat using ChatGPT

- Set context by either including last 2 chat messages from active
  session or past 2 conversation summaries from conversation logs

- Set personality in system message
- Place personality system message before last completed back & forth
  This may stop ChatGPT forgetting its personality as conversation progresses given:
  - The conditioning based on system role messages is light
  - If system message is too far back in conversation history, the
    model may forget its personality conditioning
  - If system message at end of conversation, the model can think its
    the start of a new conversation
  - Inserting the system message before last completed back & forth should
    prevent ChatGPT from assuming its the start of a new conversation
    while not losing personality conditioning from the system message

- Simplfy the Khoj Chat API to for now just answer from users notes
  instead of trying to infer other potential interaction types.
  - This is the default expected behavior from the feature anyway
  - Use the compiled text of the top 2 search results for context

- Benefits of using ChatGPT
  - Better model
  - 1/10th the price
  - No hand rolled prompt required to make GPT provide more chatty,
    assistant type responses

pyproject.toml

@@ -40,7 +40,7 @@ dependencies = [
     "defusedxml == 0.7.1",
     "fastapi == 0.77.1",
     "jinja2 == 3.1.2",
-    "openai == 0.20.0",
+    "openai >= 0.27.0",
     "pillow == 9.3.0",
     "pydantic == 1.9.1",
     "pyqt6 == 6.3.1",

src/khoj/processor/conversation/gpt.py

@@ -114,104 +114,75 @@ A:{ "search-type": "notes" }"""
     return json.loads(story.strip(empty_escape_sequences))
 
 
-def understand(text, model, api_key=None, temperature=0.5, max_tokens=100, verbose=0):
+def converse(text, user_query, active_session_length=0, conversation_log=None, api_key=None, temperature=0):
     """
-    Understand user input using OpenAI's GPT
+    Converse with user using OpenAI's ChatGPT
     """
     # Initialize Variables
-    openai.api_key = api_key or os.getenv("OPENAI_API_KEY")
-    understand_primer = """
-Objective: Extract intent and trigger emotion information as JSON from each chat message
-
-Potential intent types and valid argument values are listed below:
-- intent
-  - remember(memory-type, query);
-     - memory-type=["companion","notes","ledger","image","music"]
-  - search(search-type, query);
-     - search-type=["google"]
-  - generate(activity, query);
-     - activity=["paint","write","chat"]
-- trigger-emotion(emotion)
-   - emotion=["happy","confidence","fear","surprise","sadness","disgust","anger","shy","curiosity","calm"]
-
-Some examples are given below for reference:
-Q: How are you doing?
-A: { "intent": {"type": "generate", "activity": "chat", "query": "How are you doing?"}, "trigger-emotion": "happy" }
-Q: Do you remember what I told you about my brother Antoine when we were at the beach?
-A: { "intent": {"type": "remember", "memory-type": "companion", "query": "Brother Antoine when we were at the beach"}, "trigger-emotion": "curiosity" }
-Q: what was that fantasy story you told me last time?
-A: { "intent": {"type": "remember", "memory-type": "companion", "query": "fantasy story told last time"}, "trigger-emotion": "curiosity" }
-Q: Let's make some drawings about the stars on a clear full moon night!
-A: { "intent": {"type": "generate", "activity": "paint", "query": "stars on a clear full moon night"}, "trigger-emotion: "happy" }
-Q: Do you know anything about Lebanon cuisine in the 18th century?
-A: { "intent": {"type": "search", "search-type": "google", "query": "lebanon cusine in the 18th century"}, "trigger-emotion; "confidence" }
-Q: Tell me a scary story
-A: { "intent": {"type": "generate", "activity": "write", "query": "A scary story"}, "trigger-emotion": "fear" }
-Q: What fiction book was I reading last week about AI starship?
-A: { "intent": {"type": "remember", "memory-type": "notes", "query": "fiction book about AI starship last week"}, "trigger-emotion": "curiosity" }
-Q: How much did I spend at Subway for dinner last time?
-A: { "intent": {"type": "remember", "memory-type": "ledger", "query": "last Subway dinner"}, "trigger-emotion": "calm" }
-Q: I'm feeling sleepy
-A: { "intent": {"type": "generate", "activity": "chat", "query": "I'm feeling sleepy"}, "trigger-emotion": "calm" }
-Q: What was that popular Sri lankan song that Alex had mentioned?
-A: { "intent": {"type": "remember", "memory-type": "music", "query": "popular Sri lankan song mentioned by Alex"}, "trigger-emotion": "curiosity" }
-Q: You're pretty funny!
-A: { "intent": {"type": "generate", "activity": "chat", "query": "You're pretty funny!"}, "trigger-emotion": "shy" }
-Q: Can you recommend a movie to watch from my notes?
-A: { "intent": {"type": "remember", "memory-type": "notes", "query": "recommend movie to watch"}, "trigger-emotion": "curiosity" }
-Q: When did I go surfing last?
-A: { "intent": {"type": "remember", "memory-type": "notes", "query": "When did I go surfing last"}, "trigger-emotion": "calm" }
-Q: Can you dance for me?
-A: { "intent": {"type": "generate", "activity": "chat", "query": "Can you dance for me?"}, "trigger-emotion": "sad" }"""
-
-    # Setup Prompt with Understand Primer
-    prompt = message_to_prompt(text, understand_primer, start_sequence="\nA:", restart_sequence="\nQ:")
-    if verbose > 1:
-        print(f"Message -> Prompt: {text} -> {prompt}")
-
-    # Get Response from GPT
-    response = openai.Completion.create(
-        prompt=prompt, model=model, temperature=temperature, max_tokens=max_tokens, frequency_penalty=0.2, stop=["\n"]
-    )
-
-    # Extract, Clean Message from GPT's Response
-    story = str(response["choices"][0]["text"])
-    return json.loads(story.strip(empty_escape_sequences))
-
-
-def converse(text, model, conversation_history=None, api_key=None, temperature=0.9, max_tokens=150):
-    """
-    Converse with user using OpenAI's GPT
-    """
-    # Initialize Variables
-    max_words = 500
+    model = "gpt-3.5-turbo"
     openai.api_key = api_key or os.getenv("OPENAI_API_KEY")
 
+    personality_primer = "You are a friendly, helpful personal assistant."
     conversation_primer = f"""
-The following is a conversation with an AI assistant. The assistant is helpful, creative, clever, and a very friendly companion.
+Using my notes below, answer the following question. If the answer is not contained within the notes, say "I don't know."
 
-Human: Hello, who are you?
-AI: Hi, I am an AI conversational companion created by OpenAI. How can I help you today?"""
+Notes:
+{text}
+
+Question: {user_query}"""
 
     # Setup Prompt with Primer or Conversation History
-    prompt = message_to_prompt(text, conversation_history or conversation_primer)
-    prompt = " ".join(prompt.split()[:max_words])
+    messages = generate_chatml_messages_with_context(
+        conversation_primer,
+        personality_primer,
+        active_session_length,
+        conversation_log,
+    )
 
     # Get Response from GPT
-    response = openai.Completion.create(
-        prompt=prompt,
+    response = openai.ChatCompletion.create(
+        messages=messages,
         model=model,
         temperature=temperature,
-        max_tokens=max_tokens,
-        presence_penalty=0.6,
-        stop=["\n", "Human:", "AI:"],
     )
 
     # Extract, Clean Message from GPT's Response
-    story = str(response["choices"][0]["text"])
+    story = str(response["choices"][0]["message"]["content"])
     return story.strip(empty_escape_sequences)
 
 
+def generate_chatml_messages_with_context(user_message, system_message, active_session_length=0, conversation_log=None):
+    """Generate messages for ChatGPT with context from previous conversation"""
+    # Extract Chat History for Context
+    chat_logs = [chat["message"] for chat in conversation_log.get("chat", [])]
+    session_summaries = [session["summary"] for session in conversation_log.get("session", {})]
+    if active_session_length == 0:
+        last_backnforth = list(map(message_to_chatml, session_summaries[-1:]))
+        rest_backnforth = list(map(message_to_chatml, session_summaries[-2:-1]))
+    elif active_session_length == 1:
+        last_backnforth = reciprocal_conversation_to_chatml(chat_logs[-2:])
+        rest_backnforth = list(map(message_to_chatml, session_summaries[-1:]))
+    else:
+        last_backnforth = reciprocal_conversation_to_chatml(chat_logs[-2:])
+        rest_backnforth = reciprocal_conversation_to_chatml(chat_logs[-4:-2])
+
+    # Format user and system messages to chatml format
+    system_chatml_message = [message_to_chatml(system_message, "system")]
+    user_chatml_message = [message_to_chatml(user_message, "user")]
+
+    return rest_backnforth + system_chatml_message + last_backnforth + user_chatml_message
+
+
+def reciprocal_conversation_to_chatml(message_pair):
+    """Convert a single back and forth between user and assistant to chatml format"""
+    return [message_to_chatml(message, role) for message, role in zip(message_pair, ["user", "assistant"])]
+
+
+def message_to_chatml(message, role="assistant"):
+    """Create chatml message from message and role"""
+    return {"role": role, "content": message}
+
+
 def message_to_prompt(
     user_message, conversation_history="", gpt_message=None, start_sequence="\nAI:", restart_sequence="\nHuman:"
 ):

src/khoj/routers/api_beta.py

@@ -15,7 +15,6 @@ from khoj.processor.conversation.gpt import (
     extract_search_type,
     message_to_log,
     message_to_prompt,
-    understand,
     summarize,
 )
 from khoj.utils.state import SearchType
@@ -84,12 +83,12 @@ def answer_beta(q: str):
 @api_beta.get("/chat")
 def chat(q: Optional[str] = None):
     # Initialize Variables
-    model = state.processor_config.conversation.model
     api_key = state.processor_config.conversation.openai_api_key
 
     # Load Conversation History
     chat_session = state.processor_config.conversation.chat_session
     meta_log = state.processor_config.conversation.meta_log
+    active_session_length = len(chat_session.split("\nAI:")) - 1 if chat_session else 0
 
     # If user query is empty, return chat history
     if not q:
@@ -98,33 +97,22 @@ def chat(q: Optional[str] = None):
         else:
             return {"status": "ok", "response": []}
 
-    # Converse with OpenAI GPT
-    metadata = understand(q, model=model, api_key=api_key, verbose=state.verbose)
-    logger.debug(f'Understood: {get_from_dict(metadata, "intent")}')
+    # Collate context for GPT
+    result_list = search(q, n=2, r=True)
+    collated_result = "\n\n".join([f"# {item.additional['compiled']}" for item in result_list])
+    logger.debug(f"Reference Context:\n{collated_result}")
 
-    if get_from_dict(metadata, "intent", "memory-type") == "notes":
-        query = get_from_dict(metadata, "intent", "query")
-        result_list = search(query, n=1, t=SearchType.Org, r=True)
-        collated_result = "\n".join([item.entry for item in result_list])
-        logger.debug(f"Semantically Similar Notes:\n{collated_result}")
-        try:
-            gpt_response = summarize(collated_result, summary_type="notes", user_query=q, model=model, api_key=api_key)
-            status = "ok"
-        except Exception as e:
-            gpt_response = str(e)
-            status = "error"
-    else:
-        try:
-            gpt_response = converse(q, model, chat_session, api_key=api_key)
-            status = "ok"
-        except Exception as e:
-            gpt_response = str(e)
-            status = "error"
+    try:
+        gpt_response = converse(collated_result, q, active_session_length, meta_log, api_key=api_key)
+        status = "ok"
+    except Exception as e:
+        gpt_response = str(e)
+        status = "error"
 
     # Update Conversation History
     state.processor_config.conversation.chat_session = message_to_prompt(q, chat_session, gpt_message=gpt_response)
     state.processor_config.conversation.meta_log["chat"] = message_to_log(
-        q, gpt_response, metadata, meta_log.get("chat", [])
+        q, gpt_response, conversation_log=meta_log.get("chat", [])
     )
 
     return {"status": status, "response": gpt_response}