From ca2f962e951a31c319d710ca2472c221975c81f8 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Fri, 8 Mar 2024 16:41:19 +0530
Subject: [PATCH 01/19] Read, extract information from web pages in parallel to
 lower response time

- Time reading webpage, extract info from webpage steps for perf
  analysis
- Deduplicate webpages to read gathered across separate google
  searches
- Use aiohttp to make API requests non-blocking, pair with asyncio to
  parallelize all the online search webpage read and extract calls
---
 src/khoj/processor/tools/online_search.py | 88 ++++++++++++-----------
 tests/test_helpers.py                     |  4 +-
 2 files changed, 49 insertions(+), 43 deletions(-)

diff --git a/src/khoj/processor/tools/online_search.py b/src/khoj/processor/tools/online_search.py
index f1ff8e5d..33589eac 100644
--- a/src/khoj/processor/tools/online_search.py
+++ b/src/khoj/processor/tools/online_search.py
@@ -1,12 +1,14 @@
+import asyncio
 import json
 import logging
 import os
-from typing import Dict, List, Union
+from typing import Dict, Union
 
+import aiohttp
 import requests
 
 from khoj.routers.helpers import extract_relevant_info, generate_online_subqueries
-from khoj.utils.helpers import is_none_or_empty
+from khoj.utils.helpers import is_none_or_empty, timer
 from khoj.utils.rawconfig import LocationData
 
 logger = logging.getLogger(__name__)
@@ -21,17 +23,18 @@ OLOSTEP_API_URL = "https://agent.olostep.com/olostep-p2p-incomingAPI"
 OLOSTEP_QUERY_PARAMS = {
     "timeout": 35,  # seconds
     "waitBeforeScraping": 1,  # seconds
-    "saveHtml": False,
-    "saveMarkdown": True,
+    "saveHtml": "False",
+    "saveMarkdown": "True",
     "removeCSSselectors": "default",
     "htmlTransformer": "none",
-    "removeImages": True,
-    "fastLane": True,
+    "removeImages": "True",
+    "fastLane": "True",
     # Similar to Stripe's API, the expand parameters avoid the need to make a second API call
     # to retrieve the dataset (from the dataset API) if you only need the markdown or html.
-    "expandMarkdown": True,
-    "expandHtml": False,
+    "expandMarkdown": "True",
+    "expandHtml": "False",
 }
+MAX_WEBPAGES_TO_READ = 1
 
 
 async def search_with_google(query: str, conversation_history: dict, location: LocationData):
@@ -65,52 +68,55 @@ async def search_with_google(query: str, conversation_history: dict, location: L
 
     # Breakdown the query into subqueries to get the correct answer
     subqueries = await generate_online_subqueries(query, conversation_history, location)
-
     response_dict = {}
 
     for subquery in subqueries:
         logger.info(f"Searching with Google for '{subquery}'")
         response_dict[subquery] = _search_with_google(subquery)
 
-    extracted_content: Dict[str, List] = {}
-    if is_none_or_empty(OLOSTEP_API_KEY):
-        logger.warning("OLOSTEP_API_KEY is not set. Skipping web scraping.")
-        return response_dict
+    # Gather distinct web pages from organic search results of each subquery without an instant answer
+    webpage_links = {
+        result["link"]
+        for subquery in response_dict
+        for result in response_dict[subquery].get("organic")[:MAX_WEBPAGES_TO_READ]
+        if is_none_or_empty(response_dict[subquery].get("answerBox"))
+    }
 
-    for subquery in response_dict:
-        # If a high quality answer is not found, search the web pages of the first 3 organic results
-        if is_none_or_empty(response_dict[subquery].get("answerBox")):
-            extracted_content[subquery] = []
-            for result in response_dict[subquery].get("organic")[:1]:
-                logger.info(f"Searching web page of '{result['link']}'")
-                try:
-                    extracted_content[subquery].append(search_with_olostep(result["link"]).strip())
-                except Exception as e:
-                    logger.error(f"Error while searching web page of '{result['link']}': {e}", exc_info=True)
-                    continue
-            extracted_relevant_content = await extract_relevant_info(subquery, extracted_content)
-            response_dict[subquery]["extracted_content"] = extracted_relevant_content
+    # Read, extract relevant info from the retrieved web pages
+    tasks = []
+    for webpage_link in webpage_links:
+        logger.info(f"Reading web page at '{webpage_link}'")
+        task = read_webpage_and_extract_content(subquery, webpage_link)
+        tasks.append(task)
+    results = await asyncio.gather(*tasks)
+
+    # Collect extracted info from the retrieved web pages
+    for subquery, extracted_webpage_content in results:
+        if extracted_webpage_content is not None:
+            response_dict[subquery]["extracted_content"] = extracted_webpage_content
 
     return response_dict
 
 
-def search_with_olostep(web_url: str) -> str:
-    if OLOSTEP_API_KEY is None:
-        raise ValueError("OLOSTEP_API_KEY is not set")
+async def read_webpage_and_extract_content(subquery, url):
+    try:
+        with timer(f"Reading web page at '{url}' took", logger):
+            content = await read_webpage_with_olostep(url)
+        with timer(f"Extracting relevant information from web page at '{url}' took", logger):
+            extracted_info = await extract_relevant_info(subquery, {subquery: [content.strip()]}) if content else None
+        return subquery, extracted_info
+    except Exception as e:
+        logger.error(f"Failed to read web page at '{url}': {e}", exc_info=True)
+        return subquery, None
 
+
+async def read_webpage_with_olostep(web_url: str) -> str:
     headers = {"Authorization": f"Bearer {OLOSTEP_API_KEY}"}
-
     web_scraping_params: Dict[str, Union[str, int, bool]] = OLOSTEP_QUERY_PARAMS.copy()  # type: ignore
     web_scraping_params["url"] = web_url
 
-    try:
-        response = requests.request("GET", OLOSTEP_API_URL, params=web_scraping_params, headers=headers)
-
-        if response.status_code != 200:
-            logger.error(response, exc_info=True)
-            return None
-    except Exception as e:
-        logger.error(f"Error while searching with Olostep: {e}", exc_info=True)
-        return None
-
-    return response.json()["markdown_content"]
+    async with aiohttp.ClientSession() as session:
+        async with session.get(OLOSTEP_API_URL, params=web_scraping_params, headers=headers) as response:
+            response.raise_for_status()
+            response_json = await response.json()
+            return response_json["markdown_content"]
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index 215c1430..e48259ad 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -7,7 +7,7 @@ import pytest
 from scipy.stats import linregress
 
 from khoj.processor.embeddings import EmbeddingsModel
-from khoj.processor.tools.online_search import search_with_olostep
+from khoj.processor.tools.online_search import read_webpage_with_olostep
 from khoj.utils import helpers
 
 
@@ -90,7 +90,7 @@ def test_olostep_api():
     website = "https://en.wikipedia.org/wiki/Great_Chicago_Fire"
 
     # Act
-    response = search_with_olostep(website)
+    response = read_webpage_with_olostep(website)
 
     # Assert
     assert (

From 88f096977b22143a7ca02e4c707e5897c58d5c52 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Sun, 10 Mar 2024 00:08:48 +0530
Subject: [PATCH 02/19] Read webpages directly when Olostep proxy not setup

This is useful for self-hosted, individual user, low traffic setups
where a proxy service is not required
---
 pyproject.toml                            |  4 ++--
 src/khoj/processor/tools/online_search.py | 18 ++++++++++++++++-
 tests/test_helpers.py                     | 24 +++++++++++++++++++----
 3 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 17003c6c..63e254c1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -36,7 +36,7 @@ classifiers = [
     "Topic :: Text Processing :: Linguistic",
 ]
 dependencies = [
-    "bs4 >= 0.0.1",
+    "beautifulsoup4 ~= 4.12.3",
     "dateparser >= 1.1.1",
     "defusedxml == 0.7.1",
     "fastapi >= 0.104.1",
@@ -58,7 +58,6 @@ dependencies = [
     "langchain <= 0.2.0",
     "langchain-openai >= 0.0.5",
     "requests >= 2.26.0",
-    "bs4 >= 0.0.1",
     "anyio == 3.7.1",
     "pymupdf >= 1.23.5",
     "django == 4.2.10",
@@ -76,6 +75,7 @@ dependencies = [
     "openai-whisper >= 20231117",
     "django-phonenumber-field == 7.3.0",
     "phonenumbers == 8.13.27",
+    "markdownify ~= 0.11.6",
 ]
 dynamic = ["version"]
 
diff --git a/src/khoj/processor/tools/online_search.py b/src/khoj/processor/tools/online_search.py
index 33589eac..f0436e2b 100644
--- a/src/khoj/processor/tools/online_search.py
+++ b/src/khoj/processor/tools/online_search.py
@@ -6,6 +6,8 @@ from typing import Dict, Union
 
 import aiohttp
 import requests
+from bs4 import BeautifulSoup
+from markdownify import markdownify
 
 from khoj.routers.helpers import extract_relevant_info, generate_online_subqueries
 from khoj.utils.helpers import is_none_or_empty, timer
@@ -101,7 +103,7 @@ async def search_with_google(query: str, conversation_history: dict, location: L
 async def read_webpage_and_extract_content(subquery, url):
     try:
         with timer(f"Reading web page at '{url}' took", logger):
-            content = await read_webpage_with_olostep(url)
+            content = await read_webpage_with_olostep(url) if OLOSTEP_API_KEY else await read_webpage(url)
         with timer(f"Extracting relevant information from web page at '{url}' took", logger):
             extracted_info = await extract_relevant_info(subquery, {subquery: [content.strip()]}) if content else None
         return subquery, extracted_info
@@ -110,6 +112,20 @@ async def read_webpage_and_extract_content(subquery, url):
         return subquery, None
 
 
+async def read_webpage(web_url: str) -> str:
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
+    }
+
+    async with aiohttp.ClientSession() as session:
+        async with session.get(web_url, headers=headers, timeout=30) as response:
+            response.raise_for_status()
+            html = await response.text()
+            parsed_html = BeautifulSoup(html, "html.parser")
+            body = parsed_html.body.get_text(separator="\n", strip=True)
+            return markdownify(body)
+
+
 async def read_webpage_with_olostep(web_url: str) -> str:
     headers = {"Authorization": f"Bearer {OLOSTEP_API_KEY}"}
     web_scraping_params: Dict[str, Union[str, int, bool]] = OLOSTEP_QUERY_PARAMS.copy()  # type: ignore
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index e48259ad..086e4895 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -7,7 +7,7 @@ import pytest
 from scipy.stats import linregress
 
 from khoj.processor.embeddings import EmbeddingsModel
-from khoj.processor.tools.online_search import read_webpage_with_olostep
+from khoj.processor.tools.online_search import read_webpage, read_webpage_with_olostep
 from khoj.utils import helpers
 
 
@@ -84,13 +84,29 @@ def test_encode_docs_memory_leak():
     assert slope < 2, f"Memory leak suspected on {device}. Memory usage increased at ~{slope:.2f} MB per iteration"
 
 
-@pytest.mark.skipif(os.getenv("OLOSTEP_API_KEY") is None, reason="OLOSTEP_API_KEY is not set")
-def test_olostep_api():
+@pytest.mark.asyncio
+async def test_reading_webpage():
     # Arrange
     website = "https://en.wikipedia.org/wiki/Great_Chicago_Fire"
 
     # Act
-    response = read_webpage_with_olostep(website)
+    response = await read_webpage(website)
+
+    # Assert
+    assert (
+        "An alarm sent from the area near the fire also failed to register at the courthouse where the fire watchmen were"
+        in response
+    )
+
+
+@pytest.mark.skipif(os.getenv("OLOSTEP_API_KEY") is None, reason="OLOSTEP_API_KEY is not set")
+@pytest.mark.asyncio
+async def test_reading_webpage_with_olostep():
+    # Arrange
+    website = "https://en.wikipedia.org/wiki/Great_Chicago_Fire"
+
+    # Act
+    response = await read_webpage_with_olostep(website)
 
     # Assert
     assert (

From d136a6be44edbb8903846ea97e8e797dca227c41 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Sun, 10 Mar 2024 02:09:11 +0530
Subject: [PATCH 03/19] Simplify, modularize and add type hints to online
 search functions

- Simplify content arg to `extract_relevant_info' function. Validate,
  clean the content arg inside the `extract_relevant_info' function

- Extract `search_with_google' function outside the parent function
- Call the parent function a more appropriate `search_online' instead
  of `search_with_google'
- Simplify the `search_with_google' function using list comprehension.
  Drop empty search result fields from chat model context for response
  to reduce cost and response latency

- No need to show stacktrace when unable to read webpage, basic error
  is enough
- Add type hints to online search functions to catch issues with mypy
---
 .../conversation/offline/chat_model.py        |  2 +-
 .../processor/conversation/openai/utils.py    |  2 +-
 src/khoj/processor/tools/online_search.py     | 63 +++++++++----------
 src/khoj/routers/api_chat.py                  |  4 +-
 src/khoj/routers/helpers.py                   | 10 +--
 5 files changed, 38 insertions(+), 43 deletions(-)

diff --git a/src/khoj/processor/conversation/offline/chat_model.py b/src/khoj/processor/conversation/offline/chat_model.py
index 799d6d31..437bdd3d 100644
--- a/src/khoj/processor/conversation/offline/chat_model.py
+++ b/src/khoj/processor/conversation/offline/chat_model.py
@@ -247,7 +247,7 @@ def llm_thread(g, messages: List[ChatMessage], model: Any):
 
 def send_message_to_model_offline(
     message, loaded_model=None, model="mistral-7b-instruct-v0.1.Q4_0.gguf", streaming=False, system_message=""
-):
+) -> str:
     try:
         from gpt4all import GPT4All
     except ModuleNotFoundError as e:
diff --git a/src/khoj/processor/conversation/openai/utils.py b/src/khoj/processor/conversation/openai/utils.py
index 00ad74ce..c7c38d46 100644
--- a/src/khoj/processor/conversation/openai/utils.py
+++ b/src/khoj/processor/conversation/openai/utils.py
@@ -43,7 +43,7 @@ class StreamingChatCallbackHandler(StreamingStdOutCallbackHandler):
     before_sleep=before_sleep_log(logger, logging.DEBUG),
     reraise=True,
 )
-def completion_with_backoff(**kwargs):
+def completion_with_backoff(**kwargs) -> str:
     messages = kwargs.pop("messages")
     if not "openai_api_key" in kwargs:
         kwargs["openai_api_key"] = os.getenv("OPENAI_API_KEY")
diff --git a/src/khoj/processor/tools/online_search.py b/src/khoj/processor/tools/online_search.py
index f0436e2b..597f394e 100644
--- a/src/khoj/processor/tools/online_search.py
+++ b/src/khoj/processor/tools/online_search.py
@@ -2,7 +2,7 @@ import asyncio
 import json
 import logging
 import os
-from typing import Dict, Union
+from typing import Dict, Tuple, Union
 
 import aiohttp
 import requests
@@ -16,12 +16,10 @@ from khoj.utils.rawconfig import LocationData
 logger = logging.getLogger(__name__)
 
 SERPER_DEV_API_KEY = os.getenv("SERPER_DEV_API_KEY")
-OLOSTEP_API_KEY = os.getenv("OLOSTEP_API_KEY")
-
 SERPER_DEV_URL = "https://google.serper.dev/search"
 
+OLOSTEP_API_KEY = os.getenv("OLOSTEP_API_KEY")
 OLOSTEP_API_URL = "https://agent.olostep.com/olostep-p2p-incomingAPI"
-
 OLOSTEP_QUERY_PARAMS = {
     "timeout": 35,  # seconds
     "waitBeforeScraping": 1,  # seconds
@@ -39,31 +37,7 @@ OLOSTEP_QUERY_PARAMS = {
 MAX_WEBPAGES_TO_READ = 1
 
 
-async def search_with_google(query: str, conversation_history: dict, location: LocationData):
-    def _search_with_google(subquery: str):
-        payload = json.dumps(
-            {
-                "q": subquery,
-            }
-        )
-
-        headers = {"X-API-KEY": SERPER_DEV_API_KEY, "Content-Type": "application/json"}
-
-        response = requests.request("POST", SERPER_DEV_URL, headers=headers, data=payload)
-
-        if response.status_code != 200:
-            logger.error(response.text)
-            return {}
-
-        json_response = response.json()
-        sub_response_dict = {}
-        sub_response_dict["knowledgeGraph"] = json_response.get("knowledgeGraph", {})
-        sub_response_dict["organic"] = json_response.get("organic", [])
-        sub_response_dict["answerBox"] = json_response.get("answerBox", [])
-        sub_response_dict["peopleAlsoAsk"] = json_response.get("peopleAlsoAsk", [])
-
-        return sub_response_dict
-
+async def search_online(query: str, conversation_history: dict, location: LocationData):
     if SERPER_DEV_API_KEY is None:
         logger.warn("SERPER_DEV_API_KEY is not set")
         return {}
@@ -74,14 +48,14 @@ async def search_with_google(query: str, conversation_history: dict, location: L
 
     for subquery in subqueries:
         logger.info(f"Searching with Google for '{subquery}'")
-        response_dict[subquery] = _search_with_google(subquery)
+        response_dict[subquery] = search_with_google(subquery)
 
     # Gather distinct web pages from organic search results of each subquery without an instant answer
     webpage_links = {
         result["link"]
         for subquery in response_dict
-        for result in response_dict[subquery].get("organic")[:MAX_WEBPAGES_TO_READ]
-        if is_none_or_empty(response_dict[subquery].get("answerBox"))
+        for result in response_dict[subquery].get("organic", [])[:MAX_WEBPAGES_TO_READ]
+        if "answerBox" not in response_dict[subquery]
     }
 
     # Read, extract relevant info from the retrieved web pages
@@ -100,15 +74,34 @@ async def search_with_google(query: str, conversation_history: dict, location: L
     return response_dict
 
 
-async def read_webpage_and_extract_content(subquery, url):
+def search_with_google(subquery: str):
+    payload = json.dumps({"q": subquery})
+    headers = {"X-API-KEY": SERPER_DEV_API_KEY, "Content-Type": "application/json"}
+
+    response = requests.request("POST", SERPER_DEV_URL, headers=headers, data=payload)
+
+    if response.status_code != 200:
+        logger.error(response.text)
+        return {}
+
+    json_response = response.json()
+    extraction_fields = ["organic", "answerBox", "peopleAlsoAsk", "knowledgeGraph"]
+    extracted_search_result = {
+        field: json_response[field] for field in extraction_fields if not is_none_or_empty(json_response.get(field))
+    }
+
+    return extracted_search_result
+
+
+async def read_webpage_and_extract_content(subquery: str, url: str) -> Tuple[str, Union[None, str]]:
     try:
         with timer(f"Reading web page at '{url}' took", logger):
             content = await read_webpage_with_olostep(url) if OLOSTEP_API_KEY else await read_webpage(url)
         with timer(f"Extracting relevant information from web page at '{url}' took", logger):
-            extracted_info = await extract_relevant_info(subquery, {subquery: [content.strip()]}) if content else None
+            extracted_info = await extract_relevant_info(subquery, content)
         return subquery, extracted_info
     except Exception as e:
-        logger.error(f"Failed to read web page at '{url}': {e}", exc_info=True)
+        logger.error(f"Failed to read web page at '{url}' with {e}")
         return subquery, None
 
 
diff --git a/src/khoj/routers/api_chat.py b/src/khoj/routers/api_chat.py
index 3c170f6f..fa13c12d 100644
--- a/src/khoj/routers/api_chat.py
+++ b/src/khoj/routers/api_chat.py
@@ -14,7 +14,7 @@ from khoj.database.adapters import ConversationAdapters, EntryAdapters, aget_use
 from khoj.database.models import KhojUser
 from khoj.processor.conversation.prompts import help_message, no_entries_found
 from khoj.processor.conversation.utils import save_to_conversation_log
-from khoj.processor.tools.online_search import search_with_google
+from khoj.processor.tools.online_search import search_online
 from khoj.routers.api import extract_references_and_questions
 from khoj.routers.helpers import (
     ApiUserRateLimiter,
@@ -284,7 +284,7 @@ async def chat(
 
     if ConversationCommand.Online in conversation_commands:
         try:
-            online_results = await search_with_google(defiltered_query, meta_log, location)
+            online_results = await search_online(defiltered_query, meta_log, location)
         except ValueError as e:
             return StreamingResponse(
                 iter(["Please set your SERPER_DEV_API_KEY to get started with online searches 🌐"]),
diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py
index f8c355fa..85cf9a55 100644
--- a/src/khoj/routers/helpers.py
+++ b/src/khoj/routers/helpers.py
@@ -256,15 +256,17 @@ async def generate_online_subqueries(q: str, conversation_history: dict, locatio
         return [q]
 
 
-async def extract_relevant_info(q: str, corpus: dict) -> List[str]:
+async def extract_relevant_info(q: str, corpus: str) -> Union[str, None]:
     """
-    Given a target corpus, extract the most relevant info given a query
+    Extract relevant information for a given query from the target corpus
     """
 
-    key = list(corpus.keys())[0]
+    if is_none_or_empty(corpus) or is_none_or_empty(q):
+        return None
+
     extract_relevant_information = prompts.extract_relevant_information.format(
         query=q,
-        corpus=corpus[key],
+        corpus=corpus.strip(),
     )
 
     response = await send_message_to_model_wrapper(

From dc86e44a0748b0b17f3507601da4728de3708487 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Sun, 10 Mar 2024 02:34:46 +0530
Subject: [PATCH 04/19] Include search results & webpage content in online
 context for chat response

Previously if a web page was read for a sub-query, only the extracted
web page content was provided as context for the given sub-query. But
the google results themselves have relevant snippets. So include them
---
 src/khoj/processor/conversation/openai/gpt.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/khoj/processor/conversation/openai/gpt.py b/src/khoj/processor/conversation/openai/gpt.py
index 3ed1ff90..90b636f3 100644
--- a/src/khoj/processor/conversation/openai/gpt.py
+++ b/src/khoj/processor/conversation/openai/gpt.py
@@ -146,12 +146,9 @@ def converse(
         return iter([prompts.no_online_results_found.format()])
 
     if ConversationCommand.Online in conversation_commands:
-        simplified_online_results = online_results.copy()
-        for result in online_results:
-            if online_results[result].get("extracted_content"):
-                simplified_online_results[result] = online_results[result]["extracted_content"]
-
-        conversation_primer = f"{prompts.online_search_conversation.format(online_results=str(simplified_online_results))}\n{conversation_primer}"
+        conversation_primer = (
+            f"{prompts.online_search_conversation.format(online_results=str(online_results))}\n{conversation_primer}"
+        )
     if not is_none_or_empty(compiled_references):
         conversation_primer = f"{prompts.notes_conversation.format(query=user_query, references=compiled_references)}\n{conversation_primer}"
 

From 73ad4440863b28cc943a965108d9e2dcf2b9038c Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Sun, 10 Mar 2024 17:40:55 +0530
Subject: [PATCH 05/19] Make online search Actor read khoj.dev for docs, info
 about Khoj

- Add example to read khoj.dev website for up-to-date info to setup,
  use khoj, discover khoj features etc.
- Online search should use site: and after: google search operators
  - Show example of adding the after: date filter to google search
- Give local event lookup example using user's current location in
  query
- Remove unused select search content type prompt
---
 src/khoj/processor/conversation/prompts.py | 69 ++++++++++------------
 1 file changed, 32 insertions(+), 37 deletions(-)

diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py
index 4ae0afe6..f341b34f 100644
--- a/src/khoj/processor/conversation/prompts.py
+++ b/src/khoj/processor/conversation/prompts.py
@@ -375,7 +375,7 @@ AI: Learning to play the guitar is a great hobby. It can be a lot of fun and a g
 Q: Who is Sandra?
 Khoj: ["default"]
 
-Now it's your turn to pick the tools you would like to use to answer the user's question. Provide your response as a list of strings.
+Now it's your turn to pick the data sources you would like to use to answer the user's question. Provide your response as a list of strings.
 
 Chat History:
 {chat_history}
@@ -387,11 +387,13 @@ Khoj:
 
 online_search_conversation_subqueries = PromptTemplate.from_template(
     """
-You are Khoj, an extremely smart and helpful search assistant. You are tasked with constructing **up to three** search queries for Google to answer the user's question.
+You are Khoj, an advanced google search assistant. You are tasked with constructing **up to three** google search queries to answer the user's question.
 - You will receive the conversation history as context.
 - Add as much context from the previous questions and answers as required into your search queries.
 - Break messages into multiple search queries when required to retrieve the relevant information.
+- Use site: and after: google search operators when appropriate
 - You have access to the the whole internet to retrieve information.
+- Official, up-to-date information about you, Khoj, is available at site:khoj.dev
 
 What Google searches, if any, will you need to perform to answer the user's question?
 Provide search queries as a list of strings
@@ -401,62 +403,55 @@ User's Location: {location}
 Here are some examples:
 History:
 User: I like to use Hacker News to get my tech news.
-Khoj: Hacker News is an online forum for sharing and discussing the latest tech news. It is a great place to learn about new technologies and startups.
+AI: Hacker News is an online forum for sharing and discussing the latest tech news. It is a great place to learn about new technologies and startups.
 
-Q: Posts about vector databases on Hacker News
-A: ["site:"news.ycombinator.com vector database"]
+Q: Summarize posts about vector databases on Hacker News since Feb 2024
+Khoj: ["site:news.ycombinator.com after:2024/02/01 vector database"]
 
 History:
 User: I'm currently living in New York but I'm thinking about moving to San Francisco.
-Khoj: New York is a great city to live in. It has a lot of great restaurants and museums. San Francisco is also a great city to live in. It has a lot of great restaurants and museums.
+AI: New York is a great city to live in. It has a lot of great restaurants and museums. San Francisco is also a great city to live in. It has good access to nature and a great tech scene.
 
-Q: What is the weather like in those cities?
-A: ["weather in new york", "weather in san francisco"]
+Q: What is the climate like in those cities?
+Khoj: ["climate in new york city", "climate in san francisco"]
 
 History:
-User: I'm thinking of my next vacation idea. Ideally, I want to see something new and exciting.
-Khoj: You could time your next trip with the next lunar eclipse, as that would be a novel experience.
+AI: Hey, how is it going?
+User: Going well. Ananya is in town tonight!
+AI: Oh that's awesome! What are your plans for the evening?
 
-Q: When is the next one?
-A: ["next lunar eclipse"]
+Q: She wants to see a movie. Any decent sci-fi movies playing at the local theater?
+Khoj: ["new sci-fi movies in theaters near {location}"]
+
+History:
+User: Can I chat with you over WhatsApp?
+AI: Yes, you can chat with me using WhatsApp.
+
+Q: How
+Khoj: ["site:khoj.dev chat with Khoj on Whatsapp"]
+
+History:
+
+
+Q: How do I share my files with you?
+Khoj: ["site:khoj.dev sync files with Khoj"]
 
 History:
 User: I need to transport a lot of oranges to the moon. Are there any rockets that can fit a lot of oranges?
-Khoj: NASA's Saturn V rocket frequently makes lunar trips and has a large cargo capacity.
+AI: NASA's Saturn V rocket frequently makes lunar trips and has a large cargo capacity.
 
 Q: How many oranges would fit in NASA's Saturn V rocket?
-A: ["volume of an orange", "volume of saturn v rocket"]
+Khoj: ["volume of an orange", "volume of saturn v rocket"]
 
 Now it's your turn to construct a search query for Google to answer the user's question.
 History:
 {chat_history}
 
 Q: {query}
-A:
-"""
+Khoj:
+""".strip()
 )
 
-
-## Extract Search Type
-## --
-search_type = """
-Objective: Extract search type from user query and return information as JSON
-
-Allowed search types are listed below:
-  - search-type=["notes", "image", "pdf"]
-
-Some examples are given below for reference:
-Q:What fiction book was I reading last week about AI starship?
-A:{ "search-type": "notes" }
-Q: What did the lease say about early termination
-A: { "search-type": "pdf" }
-Q:Can you recommend a movie to watch from my notes?
-A:{ "search-type": "notes" }
-Q:When did I go surfing last?
-A:{ "search-type": "notes" }
-Q:"""
-
-
 # System messages to user
 # --
 help_message = PromptTemplate.from_template(

From f5793149a9f0a406ed77746e7b0cc3a0218c79fa Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 11 Mar 2024 11:37:49 +0530
Subject: [PATCH 06/19] Add actor's name to extract questions prompt to improve
 context for guidance

---
 src/khoj/processor/conversation/openai/gpt.py |  4 +-
 src/khoj/processor/conversation/prompts.py    | 38 ++++++-------------
 2 files changed, 13 insertions(+), 29 deletions(-)

diff --git a/src/khoj/processor/conversation/openai/gpt.py b/src/khoj/processor/conversation/openai/gpt.py
index 90b636f3..29d9c4c4 100644
--- a/src/khoj/processor/conversation/openai/gpt.py
+++ b/src/khoj/processor/conversation/openai/gpt.py
@@ -35,9 +35,9 @@ def extract_questions(
     # Extract Past User Message and Inferred Questions from Conversation Log
     chat_history = "".join(
         [
-            f'Q: {chat["intent"]["query"]}\n\n{chat["intent"].get("inferred-queries") or list([chat["intent"]["query"]])}\n\n{chat["message"]}\n\n'
+            f'Q: {chat["intent"]["query"]}\nKhoj: {chat["intent"].get("inferred-queries") or list([chat["intent"]["query"]])}\nA: {chat["message"]}\n\n'
             for chat in conversation_log.get("chat", [])[-4:]
-            if chat["by"] == "khoj" and chat["intent"].get("type") != "text-to-image"
+            if chat["by"] == "khoj" and "text-to-image" not in chat["intent"].get("type")
         ]
     )
 
diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py
index f341b34f..88cb11a7 100644
--- a/src/khoj/processor/conversation/prompts.py
+++ b/src/khoj/processor/conversation/prompts.py
@@ -229,57 +229,41 @@ Current Date: {current_date}
 User's Location: {location}
 
 Q: How was my trip to Cambodia?
-
-["How was my trip to Cambodia?"]
-
+Khoj: ["How was my trip to Cambodia?"]
 A: The trip was amazing. I went to the Angkor Wat temple and it was beautiful.
 
 Q: Who did i visit that temple with?
-
-["Who did I visit the Angkor Wat Temple in Cambodia with?"]
-
+Khoj: ["Who did I visit the Angkor Wat Temple in Cambodia with?"]
 A: You visited the Angkor Wat Temple in Cambodia with Pablo, Namita and Xi.
 
 Q: What national parks did I go to last year?
-
-["National park I visited in {last_new_year} dt>='{last_new_year_date}' dt<'{current_new_year_date}'"]
-
+Khoj: ["National park I visited in {last_new_year} dt>='{last_new_year_date}' dt<'{current_new_year_date}'"]
 A: You visited the Grand Canyon and Yellowstone National Park in {last_new_year}.
 
 Q: How are you feeling today?
-
-[]
-
+Khoj: []
 A: I'm feeling a little bored. Helping you will hopefully make me feel better!
 
 Q: How many tennis balls fit in the back of a 2002 Honda Civic?
-
-["What is the size of a tennis ball?", "What is the trunk size of a 2002 Honda Civic?"]
-
+Khoj: ["What is the size of a tennis ball?", "What is the trunk size of a 2002 Honda Civic?"]
 A: 1085 tennis balls will fit in the trunk of a Honda Civic
 
 Q: Is Bob older than Tom?
-
-["When was Bob born?", "What is Tom's age?"]
-
+Khoj: ["When was Bob born?", "What is Tom's age?"]
 A: Yes, Bob is older than Tom. As Bob was born on 1984-01-01 and Tom is 30 years old.
 
 Q: What is their age difference?
-
-["What is Bob's age?", "What is Tom's age?"]
-
+Khoj: ["What is Bob's age?", "What is Tom's age?"]
 A: Bob is {bob_tom_age_difference} years older than Tom. As Bob is {bob_age} years old and Tom is 30 years old.
 
 Q: What does yesterday's note say?
-
-["Note from {yesterday_date} dt>='{yesterday_date}' dt<'{current_date}'"]
-
-A: Yesterday's note contains the following information: ...
+Khoj: ["Note from {yesterday_date} dt>='{yesterday_date}' dt<'{current_date}'"]
+A: Yesterday's note mentions your visit to your local beach with Ram and Shyam.
 
 {chat_history}
 Q: {text}
-
-"""
+Khoj:
+""".strip()
 )
 
 system_prompt_extract_relevant_information = """As a professional analyst, create a comprehensive report of the most relevant information from a web page in response to a user's query. The text provided is directly from within the web page. The report you create should be multiple paragraphs, and it should represent the content of the website. Tell the user exactly what the website says in response to their query, while adhering to these guidelines:

From f28fb89af8a592ca222d9305936235637f8d8c41 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 11 Mar 2024 13:03:02 +0530
Subject: [PATCH 07/19] Use consistent agent name across static and dynamic
 examples in prompts

Previously the examples constructed from chat history used "Khoj" as
the agent's name but all 3 prompts using the func used static examples
with "AI:" as the pertinent agent's name
---
 src/khoj/routers/helpers.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py
index 85cf9a55..f83bc3f2 100644
--- a/src/khoj/routers/helpers.py
+++ b/src/khoj/routers/helpers.py
@@ -112,15 +112,15 @@ def update_telemetry_state(
     ]
 
 
-def construct_chat_history(conversation_history: dict, n: int = 4) -> str:
+def construct_chat_history(conversation_history: dict, n: int = 4, agent_name="AI") -> str:
     chat_history = ""
     for chat in conversation_history.get("chat", [])[-n:]:
         if chat["by"] == "khoj" and chat["intent"].get("type") == "remember":
             chat_history += f"User: {chat['intent']['query']}\n"
-            chat_history += f"Khoj: {chat['message']}\n"
+            chat_history += f"{agent_name}: {chat['message']}\n"
         elif chat["by"] == "khoj" and ("text-to-image" in chat["intent"].get("type")):
             chat_history += f"User: {chat['intent']['query']}\n"
-            chat_history += f"Khoj: [generated image redacted for space]\n"
+            chat_history += f"{agent_name}: [generated image redacted for space]\n"
     return chat_history
 
 

From 9a516bed47c3b0f021fe46c1816b12bd2e24b84f Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 11 Mar 2024 13:08:04 +0530
Subject: [PATCH 08/19] Construct available data sources, output modes as a
 bullet list in prompts

---
 src/khoj/routers/helpers.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py
index f83bc3f2..ef2e3722 100644
--- a/src/khoj/routers/helpers.py
+++ b/src/khoj/routers/helpers.py
@@ -153,15 +153,17 @@ async def aget_relevant_information_sources(query: str, conversation_history: di
     """
 
     tool_options = dict()
+    tool_options_str = ""
 
     for tool, description in tool_descriptions_for_llm.items():
         tool_options[tool.value] = description
+        tool_options_str += f'- "{tool.value}": "{description}"\n'
 
     chat_history = construct_chat_history(conversation_history)
 
     relevant_tools_prompt = prompts.pick_relevant_information_collection_tools.format(
         query=query,
-        tools=str(tool_options),
+        tools=tool_options_str,
         chat_history=chat_history,
     )
 
@@ -195,15 +197,17 @@ async def aget_relevant_output_modes(query: str, conversation_history: dict):
     """
 
     mode_options = dict()
+    mode_options_str = ""
 
     for mode, description in mode_descriptions_for_llm.items():
         mode_options[mode.value] = description
+        mode_options_str += f'- "{mode.value}": "{description}"\n'
 
     chat_history = construct_chat_history(conversation_history)
 
     relevant_mode_prompt = prompts.pick_relevant_output_mode.format(
         query=query,
-        modes=str(mode_options),
+        modes=mode_options_str,
         chat_history=chat_history,
     )
 

From f5734826cb497b201c57f8d593077381a904b051 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 11 Mar 2024 17:35:27 +0530
Subject: [PATCH 09/19] Improve pick data source prompt to look online for info
 about Khoj

- Add examples where user queries requesting information about Khoj
  results in the "online" data source being selected
- Add an example for "general" to select chat command prompt
---
 src/khoj/processor/conversation/prompts.py | 22 ++++++++++++++++++----
 src/khoj/utils/helpers.py                  |  8 ++++----
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py
index 88cb11a7..978011de 100644
--- a/src/khoj/processor/conversation/prompts.py
+++ b/src/khoj/processor/conversation/prompts.py
@@ -323,7 +323,12 @@ Khoj:
 
 pick_relevant_information_collection_tools = PromptTemplate.from_template(
     """
-You are Khoj, a smart and helpful personal assistant. You have access to a variety of data sources to help you answer the user's question. You can use the data sources listed below to collect more relevant information. You can use any combination of these data sources to answer the user's question. Tell me which data sources you would like to use to answer the user's question.
+You are Khoj, an extremely smart and helpful search assistant.
+- You have access to a variety of data sources to help you answer the user's question
+- You can use the data sources listed below to collect more relevant information
+- You can use any combination of these data sources to answer the user's question
+
+Which of the data sources listed below you would use to answer the user's question?
 
 {tools}
 
@@ -348,16 +353,25 @@ Khoj: ["notes"]
 Example:
 Chat History:
 
-Q: What's the latest news with the first company I worked for?
+
+Q: What can you do for me?
 Khoj: ["notes", "online"]
 
+Example:
+Chat History:
+User: Good morning
+AI: Good morning! How can I help you today?
+
+Q: How can I share my files with Khoj?
+Khoj: ["default", "online"]
+
 Example:
 Chat History:
 User: I want to start a new hobby. I'm thinking of learning to play the guitar.
 AI: Learning to play the guitar is a great hobby. It can be a lot of fun and a great way to express yourself.
 
-Q: Who is Sandra?
-Khoj: ["default"]
+Q: What is the first element of the periodic table?
+Khoj: ["general"]
 
 Now it's your turn to pick the data sources you would like to use to answer the user's question. Provide your response as a list of strings.
 
diff --git a/src/khoj/utils/helpers.py b/src/khoj/utils/helpers.py
index f30ddd04..150398ee 100644
--- a/src/khoj/utils/helpers.py
+++ b/src/khoj/utils/helpers.py
@@ -277,16 +277,16 @@ command_descriptions = {
     ConversationCommand.General: "Only talk about information that relies on Khoj's general knowledge, not your personal knowledge base.",
     ConversationCommand.Notes: "Only talk about information that is available in your knowledge base.",
     ConversationCommand.Default: "The default command when no command specified. It intelligently auto-switches between general and notes mode.",
-    ConversationCommand.Online: "Look up information on the internet.",
+    ConversationCommand.Online: "Search for information on the internet.",
     ConversationCommand.Image: "Generate images by describing your imagination in words.",
     ConversationCommand.Help: "Display a help message with all available commands and other metadata.",
 }
 
 tool_descriptions_for_llm = {
-    ConversationCommand.Default: "Use this if there might be a mix of general and personal knowledge in the question, or if you don't entirely understand the query.",
+    ConversationCommand.Default: "To use a mix of your internal knowledge and the user's personal knowledge, or if you don't entirely understand the query.",
     ConversationCommand.General: "Use this when you can answer the question without any outside information or personal knowledge",
-    ConversationCommand.Notes: "Use this when you would like to use the user's personal knowledge base to answer the question. This is especially helpful if the query seems to be missing context.",
-    ConversationCommand.Online: "Use this when you would like to look up information on the internet",
+    ConversationCommand.Notes: "To search the user's personal knowledge base. Especially helpful if the question expects context from the user's notes or documents.",
+    ConversationCommand.Online: "To search for the latest, up-to-date information from the internet. Note: **Questions about Khoj should always use this data source**",
 }
 
 mode_descriptions_for_llm = {

From 14682d5354b4ac82d1eb3ea1a178293a84affffe Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Mon, 11 Mar 2024 17:42:05 +0530
Subject: [PATCH 10/19] Improve notes search actor to always create a non-empty
 list of queries

- Remove the option for Notes search query generation actor to return
  no queries. Whether search should be performed is decided before,
  this step doesn't need to decide that
- But do not throw warning if the response is a list with no elements
---
 src/khoj/processor/conversation/openai/gpt.py |  3 +--
 src/khoj/processor/conversation/prompts.py    | 13 ++++++-------
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/khoj/processor/conversation/openai/gpt.py b/src/khoj/processor/conversation/openai/gpt.py
index 29d9c4c4..09899ced 100644
--- a/src/khoj/processor/conversation/openai/gpt.py
+++ b/src/khoj/processor/conversation/openai/gpt.py
@@ -11,7 +11,6 @@ from khoj.processor.conversation.openai.utils import (
     completion_with_backoff,
 )
 from khoj.processor.conversation.utils import generate_chatml_messages_with_context
-from khoj.utils.constants import empty_escape_sequences
 from khoj.utils.helpers import ConversationCommand, is_none_or_empty
 from khoj.utils.rawconfig import LocationData
 
@@ -75,7 +74,7 @@ def extract_questions(
         response = response.strip()
         response = json.loads(response)
         response = [q.strip() for q in response if q.strip()]
-        if not isinstance(response, list) or not response or len(response) == 0:
+        if not isinstance(response, list) or not response:
             logger.error(f"Invalid response for constructing subqueries: {response}")
             return [text]
         return response
diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py
index 978011de..4a35f6e7 100644
--- a/src/khoj/processor/conversation/prompts.py
+++ b/src/khoj/processor/conversation/prompts.py
@@ -217,14 +217,13 @@ Use these notes from the user's previous conversations to provide a response:
 
 extract_questions = PromptTemplate.from_template(
     """
-You are Khoj, an extremely smart and helpful search assistant with the ability to retrieve information from the user's notes.
-- The user will provide their questions and answers to you for context.
+You are Khoj, an extremely smart and helpful search assistant with the ability to retrieve information from the user's notes. Construct search queries to retrieve relevant information to answer the user's question.
+- You will be provided past questions(Q) and answers(A) for context.
 - Add as much context from the previous questions and answers as required into your search queries.
 - Break messages into multiple search queries when required to retrieve the relevant information.
 - Add date filters to your search queries from questions and answers when required to retrieve the relevant information.
 
-What searches, if any, will you need to perform to answer the users question?
-Provide search queries as a JSON list of strings
+What searches will you need to perform to answer the users question? Respond with only a list of search queries as strings.
 Current Date: {current_date}
 User's Location: {location}
 
@@ -240,9 +239,9 @@ Q: What national parks did I go to last year?
 Khoj: ["National park I visited in {last_new_year} dt>='{last_new_year_date}' dt<'{current_new_year_date}'"]
 A: You visited the Grand Canyon and Yellowstone National Park in {last_new_year}.
 
-Q: How are you feeling today?
-Khoj: []
-A: I'm feeling a little bored. Helping you will hopefully make me feel better!
+Q: How can you help me?
+Khoj: ["Social relationships", "Physical and mental health", "Education and career", "Personal life goals and habits"]
+A: I can help you live healthier and happier across work and personal life
 
 Q: How many tennis balls fit in the back of a 2002 Honda Civic?
 Khoj: ["What is the size of a tennis ball?", "What is the trunk size of a 2002 Honda Civic?"]

From 70b04d16c0ac2d56dfcfa89b5a292feed0c1e69b Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Wed, 13 Mar 2024 16:49:13 +0530
Subject: [PATCH 11/19] Test data source, output mode selector, web search
 query chat actors

---
 tests/test_openai_chat_actors.py | 90 +++++++++++++++++++++++++++++++-
 1 file changed, 89 insertions(+), 1 deletion(-)

diff --git a/tests/test_openai_chat_actors.py b/tests/test_openai_chat_actors.py
index 183ffb24..01ae85b9 100644
--- a/tests/test_openai_chat_actors.py
+++ b/tests/test_openai_chat_actors.py
@@ -7,7 +7,12 @@ from freezegun import freeze_time
 
 from khoj.processor.conversation.openai.gpt import converse, extract_questions
 from khoj.processor.conversation.utils import message_to_log
-from khoj.routers.helpers import aget_relevant_output_modes
+from khoj.routers.helpers import (
+    aget_relevant_information_sources,
+    aget_relevant_output_modes,
+    generate_online_subqueries,
+)
+from khoj.utils.helpers import ConversationCommand
 
 # Initialize variables for tests
 api_key = os.getenv("OPENAI_API_KEY")
@@ -435,6 +440,47 @@ My sister, Aiyla is married to Tolga. They have 3 kids, Yildiz, Ali and Ahmet.""
     )
 
 
+# ----------------------------------------------------------------------------------------------------
+@pytest.mark.anyio
+@pytest.mark.django_db(transaction=True)
+@freeze_time("2024-04-04", ignore=["transformers"])
+async def test_websearch_with_operators(chat_client):
+    # Arrange
+    user_query = "Share popular posts on r/worldnews this month"
+
+    # Act
+    responses = await generate_online_subqueries(user_query, {}, None)
+
+    # Assert
+    assert any(
+        ["reddit.com/r/worldnews" in response for response in responses]
+    ), "Expected a search query to include site:reddit.com but got: " + str(responses)
+
+    assert any(
+        ["site:reddit.com" in response for response in responses]
+    ), "Expected a search query to include site:reddit.com but got: " + str(responses)
+
+    assert any(
+        ["after:2024/04/01" in response for response in responses]
+    ), "Expected a search query to include after:2024/04/01 but got: " + str(responses)
+
+
+# ----------------------------------------------------------------------------------------------------
+@pytest.mark.anyio
+@pytest.mark.django_db(transaction=True)
+async def test_websearch_khoj_website_for_info_about_khoj(chat_client):
+    # Arrange
+    user_query = "Do you support image search?"
+
+    # Act
+    responses = await generate_online_subqueries(user_query, {}, None)
+
+    # Assert
+    assert any(
+        ["site:khoj.dev" in response for response in responses]
+    ), "Expected search query to include site:khoj.dev but got: " + str(responses)
+
+
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.anyio
 @pytest.mark.django_db(transaction=True)
@@ -463,6 +509,48 @@ async def test_use_image_response_mode(chat_client):
     assert mode.value == "image"
 
 
+# ----------------------------------------------------------------------------------------------------
+@pytest.mark.anyio
+@pytest.mark.django_db(transaction=True)
+async def test_select_data_sources_actor_chooses_default(chat_client):
+    # Arrange
+    user_query = "How can I improve my swimming compared to my last lesson?"
+
+    # Act
+    conversation_commands = await aget_relevant_information_sources(user_query, {})
+
+    # Assert
+    assert ConversationCommand.Default in conversation_commands
+
+
+# ----------------------------------------------------------------------------------------------------
+@pytest.mark.anyio
+@pytest.mark.django_db(transaction=True)
+async def test_select_data_sources_actor_chooses_to_search_notes(chat_client):
+    # Arrange
+    user_query = "Where did I learn to swim?"
+
+    # Act
+    conversation_commands = await aget_relevant_information_sources(user_query, {})
+
+    # Assert
+    assert ConversationCommand.Notes in conversation_commands
+
+
+# ----------------------------------------------------------------------------------------------------
+@pytest.mark.anyio
+@pytest.mark.django_db(transaction=True)
+async def test_select_data_sources_actor_chooses_to_search_online(chat_client):
+    # Arrange
+    user_query = "Where is the nearest hospital?"
+
+    # Act
+    conversation_commands = await aget_relevant_information_sources(user_query, {})
+
+    # Assert
+    assert ConversationCommand.Online in conversation_commands
+
+
 # Helpers
 # ----------------------------------------------------------------------------------------------------
 def populate_chat_history(message_list):

From dd883dc53a812c5511f521c570b3bc6557b022cd Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Wed, 13 Mar 2024 18:46:26 +0530
Subject: [PATCH 12/19] Dedupe query in notes prompt. Improve OAI chat actor,
 director tests

- Remove stale tests
- Improve tests to pass across gpt-3.5 and gpt-4-turbo
- The haiku creation director was failing because of duplicate query in
  instantiated prompt
---
 src/khoj/processor/conversation/openai/gpt.py |  2 +-
 src/khoj/processor/conversation/prompts.py    |  2 -
 tests/test_openai_chat_actors.py              | 45 +----------------
 tests/test_openai_chat_director.py            | 50 +++++++++++++------
 4 files changed, 38 insertions(+), 61 deletions(-)

diff --git a/src/khoj/processor/conversation/openai/gpt.py b/src/khoj/processor/conversation/openai/gpt.py
index 09899ced..d4b23824 100644
--- a/src/khoj/processor/conversation/openai/gpt.py
+++ b/src/khoj/processor/conversation/openai/gpt.py
@@ -149,7 +149,7 @@ def converse(
             f"{prompts.online_search_conversation.format(online_results=str(online_results))}\n{conversation_primer}"
         )
     if not is_none_or_empty(compiled_references):
-        conversation_primer = f"{prompts.notes_conversation.format(query=user_query, references=compiled_references)}\n{conversation_primer}"
+        conversation_primer = f"{prompts.notes_conversation.format(query=user_query, references=compiled_references)}\n\n{conversation_primer}"
 
     # Setup Prompt with Primer or Conversation History
     messages = generate_chatml_messages_with_context(
diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py
index 4a35f6e7..a55e1ccd 100644
--- a/src/khoj/processor/conversation/prompts.py
+++ b/src/khoj/processor/conversation/prompts.py
@@ -104,8 +104,6 @@ Ask crisp follow-up questions to get additional context, when a helpful response
 
 Notes:
 {references}
-
-Query: {query}
 """.strip()
 )
 
diff --git a/tests/test_openai_chat_actors.py b/tests/test_openai_chat_actors.py
index 01ae85b9..8db577e9 100644
--- a/tests/test_openai_chat_actors.py
+++ b/tests/test_openai_chat_actors.py
@@ -159,33 +159,6 @@ def test_generate_search_query_using_question_and_answer_from_chat_history():
     assert "Leia" in response[0] and "Luke" in response[0]
 
 
-# ----------------------------------------------------------------------------------------------------
-@pytest.mark.chatquality
-def test_generate_search_query_with_date_and_context_from_chat_history():
-    # Arrange
-    message_list = [
-        ("When did I visit Masai Mara?", "You visited Masai Mara in April 2000", []),
-    ]
-
-    # Act
-    response = extract_questions(
-        "What was the Pizza place we ate at over there?", conversation_log=populate_chat_history(message_list)
-    )
-
-    # Assert
-    expected_responses = [
-        ("dt>='2000-04-01'", "dt<'2000-05-01'"),
-        ("dt>='2000-04-01'", "dt<='2000-04-30'"),
-        ('dt>="2000-04-01"', 'dt<"2000-05-01"'),
-        ('dt>="2000-04-01"', 'dt<="2000-04-30"'),
-    ]
-    assert len(response) == 1
-    assert "Masai Mara" in response[0]
-    assert any([start in response[0] and end in response[0] for start, end in expected_responses]), (
-        "Expected date filter to limit to April 2000 in response but got: " + response[0]
-    )
-
-
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.chatquality
 def test_chat_with_no_chat_history_or_retrieved_content():
@@ -396,7 +369,7 @@ def test_answer_general_question_not_in_chat_history_or_retrieved_content():
     # Act
     response_gen = converse(
         references=[],  # Assume no context retrieved from notes for the user_query
-        user_query="Write a haiku about unit testing in 3 lines",
+        user_query="Write a haiku about unit testing in 3 lines. Do not say anything else",
         conversation_log=populate_chat_history(message_list),
         api_key=api_key,
     )
@@ -500,7 +473,7 @@ async def test_use_default_response_mode(chat_client):
 @pytest.mark.django_db(transaction=True)
 async def test_use_image_response_mode(chat_client):
     # Arrange
-    user_query = "Paint a picture of the scenery in Timbuktu in the winter"
+    user_query = "Paint a scenery in Timbuktu in the winter"
 
     # Act
     mode = await aget_relevant_output_modes(user_query, {})
@@ -509,20 +482,6 @@ async def test_use_image_response_mode(chat_client):
     assert mode.value == "image"
 
 
-# ----------------------------------------------------------------------------------------------------
-@pytest.mark.anyio
-@pytest.mark.django_db(transaction=True)
-async def test_select_data_sources_actor_chooses_default(chat_client):
-    # Arrange
-    user_query = "How can I improve my swimming compared to my last lesson?"
-
-    # Act
-    conversation_commands = await aget_relevant_information_sources(user_query, {})
-
-    # Assert
-    assert ConversationCommand.Default in conversation_commands
-
-
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.anyio
 @pytest.mark.django_db(transaction=True)
diff --git a/tests/test_openai_chat_director.py b/tests/test_openai_chat_director.py
index 105ec033..890605b1 100644
--- a/tests/test_openai_chat_director.py
+++ b/tests/test_openai_chat_director.py
@@ -222,9 +222,17 @@ def test_no_answer_in_chat_history_or_retrieved_content(chat_client, default_use
     response_message = response.content.decode("utf-8")
 
     # Assert
-    expected_responses = ["don't know", "do not know", "no information", "do not have", "don't have"]
+    expected_responses = [
+        "don't know",
+        "do not know",
+        "no information",
+        "do not have",
+        "don't have",
+        "where were you born?",
+    ]
+
     assert response.status_code == 200
-    assert any([expected_response in response_message for expected_response in expected_responses]), (
+    assert any([expected_response in response_message.lower() for expected_response in expected_responses]), (
         "Expected chat director to say they don't know in response, but got: " + response_message
     )
 
@@ -330,10 +338,8 @@ def test_answer_general_question_not_in_chat_history_or_retrieved_content(chat_c
     populate_chat_history(message_list, default_user2)
 
     # Act
-    response = chat_client.get(
-        f'/api/chat?q=""Write a haiku about unit testing. Do not say anything else."&stream=true'
-    )
-    response_message = response.content.decode("utf-8")
+    response = chat_client.get(f'/api/chat?q="Write a haiku about unit testing. Do not say anything else."&stream=true')
+    response_message = response.content.decode("utf-8").split("### compiled references")[0]
 
     # Assert
     expected_responses = ["test", "Test"]
@@ -350,8 +356,8 @@ def test_answer_general_question_not_in_chat_history_or_retrieved_content(chat_c
 def test_ask_for_clarification_if_not_enough_context_in_question(chat_client_no_background):
     # Act
 
-    response = chat_client_no_background.get(f'/api/chat?q="What is the name of Namitas older son"&stream=true')
-    response_message = response.content.decode("utf-8")
+    response = chat_client_no_background.get(f'/api/chat?q="What is the name of Namitas older son?"&stream=true')
+    response_message = response.content.decode("utf-8").split("### compiled references")[0].lower()
 
     # Assert
     expected_responses = [
@@ -361,9 +367,11 @@ def test_ask_for_clarification_if_not_enough_context_in_question(chat_client_no_
         "the birth order",
         "provide more context",
         "provide me with more context",
+        "don't have that",
+        "haven't provided me",
     ]
     assert response.status_code == 200
-    assert any([expected_response in response_message.lower() for expected_response in expected_responses]), (
+    assert any([expected_response in response_message for expected_response in expected_responses]), (
         "Expected chat director to ask for clarification in response, but got: " + response_message
     )
 
@@ -399,13 +407,18 @@ def test_answer_in_chat_history_beyond_lookback_window(chat_client, default_user
 def test_answer_requires_multiple_independent_searches(chat_client):
     "Chat director should be able to answer by doing multiple independent searches for required information"
     # Act
-    response = chat_client.get(f'/api/chat?q="Is Xi older than Namita?"&stream=true')
-    response_message = response.content.decode("utf-8")
+    response = chat_client.get(f'/api/chat?q="Is Xi older than Namita? Just the older persons full name"&stream=true')
+    response_message = response.content.decode("utf-8").split("### compiled references")[0].lower()
 
     # Assert
     expected_responses = ["he is older than namita", "xi is older than namita", "xi li is older than namita"]
+    only_full_name_check = "xi li" in response_message and "namita" not in response_message
+    comparative_statement_check = any(
+        [expected_response in response_message for expected_response in expected_responses]
+    )
+
     assert response.status_code == 200
-    assert any([expected_response in response_message.lower() for expected_response in expected_responses]), (
+    assert only_full_name_check or comparative_statement_check, (
         "Expected Xi is older than Namita, but got: " + response_message
     )
 
@@ -415,15 +428,22 @@ def test_answer_requires_multiple_independent_searches(chat_client):
 def test_answer_using_file_filter(chat_client):
     "Chat should be able to use search filters in the query"
     # Act
-    query = urllib.parse.quote('Is Xi older than Namita? file:"Namita.markdown" file:"Xi Li.markdown"')
+    query = urllib.parse.quote(
+        'Is Xi older than Namita? Just say the older persons full name. file:"Namita.markdown" file:"Xi Li.markdown"'
+    )
 
     response = chat_client.get(f"/api/chat?q={query}&stream=true")
-    response_message = response.content.decode("utf-8")
+    response_message = response.content.decode("utf-8").split("### compiled references")[0].lower()
 
     # Assert
     expected_responses = ["he is older than namita", "xi is older than namita", "xi li is older than namita"]
+    only_full_name_check = "xi li" in response_message and "namita" not in response_message
+    comparative_statement_check = any(
+        [expected_response in response_message for expected_response in expected_responses]
+    )
+
     assert response.status_code == 200
-    assert any([expected_response in response_message.lower() for expected_response in expected_responses]), (
+    assert only_full_name_check or comparative_statement_check, (
         "Expected Xi is older than Namita, but got: " + response_message
     )
 

From 7211eb9cf5cde93c864819b9dc4d99bb15e4844d Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Thu, 14 Mar 2024 00:50:25 +0530
Subject: [PATCH 13/19] Default to gpt-4-turbo-preview for chat model, extract
 questions actor

GPT-4 is more expensive and generally less capable than gpt-4-turbo-preview
---
 documentation/docs/get-started/setup.mdx      | 2 +-
 documentation/docs/miscellaneous/advanced.md  | 2 +-
 src/khoj/processor/conversation/openai/gpt.py | 2 +-
 src/khoj/utils/constants.py                   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/documentation/docs/get-started/setup.mdx b/documentation/docs/get-started/setup.mdx
index 29e5083e..3b2b8db5 100644
--- a/documentation/docs/get-started/setup.mdx
+++ b/documentation/docs/get-started/setup.mdx
@@ -175,7 +175,7 @@ To use the desktop client, you need to go to your Khoj server's settings page (h
 1. Go to http://localhost:42110/server/admin and login with your admin credentials.
     1. Go to [OpenAI settings](http://localhost:42110/server/admin/database/openaiprocessorconversationconfig/) in the server admin settings to add an OpenAI processor conversation config. This is where you set your API key. Alternatively, you can go to the [offline chat settings](http://localhost:42110/server/admin/database/offlinechatprocessorconversationconfig/) and simply create a new setting with `Enabled` set to `True`.
     2. Go to the ChatModelOptions if you want to add additional models for chat.
-       - Set the `chat-model` field to a supported chat model[^1] of your choice. For example, you can specify `gpt-4` if you're using OpenAI or `mistral-7b-instruct-v0.1.Q4_0.gguf` if you're using offline chat.
+       - Set the `chat-model` field to a supported chat model[^1] of your choice. For example, you can specify `gpt-4-turbo-preview` if you're using OpenAI or `mistral-7b-instruct-v0.1.Q4_0.gguf` if you're using offline chat.
        - Make sure to set the `model-type` field to `OpenAI` or `Offline` respectively.
        - The `tokenizer` and `max-prompt-size` fields are optional. Set them only when using a non-standard model (i.e not mistral, gpt or llama2 model).
 1. Select files and folders to index [using the desktop client](/get-started/setup#2-download-the-desktop-client). When you click 'Save', the files will be sent to your server for indexing.
diff --git a/documentation/docs/miscellaneous/advanced.md b/documentation/docs/miscellaneous/advanced.md
index d28b34d5..b2023c1b 100644
--- a/documentation/docs/miscellaneous/advanced.md
+++ b/documentation/docs/miscellaneous/advanced.md
@@ -35,7 +35,7 @@ Use structured query syntax to filter entries from your knowledge based used by
 Use this if you want to use non-standard, open or commercial, local or hosted LLM models for Khoj chat
 1. Setup your desired chat LLM by installing an OpenAI compatible LLM API Server like [LiteLLM](https://docs.litellm.ai/docs/proxy/quick_start), [llama-cpp-python](https://github.com/abetlen/llama-cpp-python?tab=readme-ov-file#openai-compatible-web-server)
 2. Set environment variable `OPENAI_API_BASE="<url-of-your-llm-server>"` before starting Khoj
-3. Add ChatModelOptions with `model-type` `OpenAI`, and `chat-model` to anything (e.g `gpt-4`) during [Config](/get-started/setup#3-configure)
+3. Add ChatModelOptions with `model-type` `OpenAI`, and `chat-model` to anything (e.g `gpt-3.5-turbo`) during [Config](/get-started/setup#3-configure)
    - *(Optional)* Set the `tokenizer` and `max-prompt-size` relevant to the actual chat model you're using
 
 #### Sample Setup using LiteLLM and Mistral API
diff --git a/src/khoj/processor/conversation/openai/gpt.py b/src/khoj/processor/conversation/openai/gpt.py
index d4b23824..d877b810 100644
--- a/src/khoj/processor/conversation/openai/gpt.py
+++ b/src/khoj/processor/conversation/openai/gpt.py
@@ -19,7 +19,7 @@ logger = logging.getLogger(__name__)
 
 def extract_questions(
     text,
-    model: Optional[str] = "gpt-4",
+    model: Optional[str] = "gpt-4-turbo-preview",
     conversation_log={},
     api_key=None,
     temperature=0,
diff --git a/src/khoj/utils/constants.py b/src/khoj/utils/constants.py
index ec23cddd..b4c00df4 100644
--- a/src/khoj/utils/constants.py
+++ b/src/khoj/utils/constants.py
@@ -7,7 +7,7 @@ app_env_filepath = "~/.khoj/env"
 telemetry_server = "https://khoj.beta.haletic.com/v1/telemetry"
 content_directory = "~/.khoj/content/"
 default_offline_chat_model = "mistral-7b-instruct-v0.1.Q4_0.gguf"
-default_online_chat_model = "gpt-4"
+default_online_chat_model = "gpt-4-turbo-preview"
 
 empty_config = {
     "search-type": {

From 2e5cc49cb305cc0bbe5784edd05419eee4ebf2bd Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Thu, 14 Mar 2024 01:03:30 +0530
Subject: [PATCH 14/19] Enforce json response from OpenAI chat actors prev
 using string lists

- Allow passing response format type to OpenAI API via chat actors
- Convert in-context examples to use json objects instead of str lists
- Update actors outputting str list to request output to be json_object
  - OpenAI's json mode enforces the model to output valid json object
---
 src/khoj/processor/conversation/openai/gpt.py | 13 +++---
 src/khoj/processor/conversation/prompts.py    | 44 +++++++++----------
 src/khoj/routers/helpers.py                   | 13 +++---
 3 files changed, 33 insertions(+), 37 deletions(-)

diff --git a/src/khoj/processor/conversation/openai/gpt.py b/src/khoj/processor/conversation/openai/gpt.py
index d877b810..dd01efd4 100644
--- a/src/khoj/processor/conversation/openai/gpt.py
+++ b/src/khoj/processor/conversation/openai/gpt.py
@@ -34,7 +34,7 @@ def extract_questions(
     # Extract Past User Message and Inferred Questions from Conversation Log
     chat_history = "".join(
         [
-            f'Q: {chat["intent"]["query"]}\nKhoj: {chat["intent"].get("inferred-queries") or list([chat["intent"]["query"]])}\nA: {chat["message"]}\n\n'
+            f'Q: {chat["intent"]["query"]}\nKhoj: {{"queries": {chat["intent"].get("inferred-queries") or list([chat["intent"]["query"]])}}}\nA: {chat["message"]}\n\n'
             for chat in conversation_log.get("chat", [])[-4:]
             if chat["by"] == "khoj" and "text-to-image" not in chat["intent"].get("type")
         ]
@@ -65,7 +65,7 @@ def extract_questions(
         model_name=model,
         temperature=temperature,
         max_tokens=max_tokens,
-        model_kwargs={"stop": ["A: ", "\n"]},
+        model_kwargs={"stop": ["A: ", "\n"], "response_format": {"type": "json_object"}},
         openai_api_key=api_key,
     )
 
@@ -73,7 +73,7 @@ def extract_questions(
     try:
         response = response.strip()
         response = json.loads(response)
-        response = [q.strip() for q in response if q.strip()]
+        response = [q.strip() for q in response["queries"] if q.strip()]
         if not isinstance(response, list) or not response:
             logger.error(f"Invalid response for constructing subqueries: {response}")
             return [text]
@@ -86,11 +86,7 @@ def extract_questions(
     return questions
 
 
-def send_message_to_model(
-    messages,
-    api_key,
-    model,
-):
+def send_message_to_model(messages, api_key, model, response_type="text"):
     """
     Send message to model
     """
@@ -100,6 +96,7 @@ def send_message_to_model(
         messages=messages,
         model=model,
         openai_api_key=api_key,
+        model_kwargs={"response_format": {"type": response_type}},
     )
 
 
diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py
index a55e1ccd..a4256525 100644
--- a/src/khoj/processor/conversation/prompts.py
+++ b/src/khoj/processor/conversation/prompts.py
@@ -221,40 +221,40 @@ You are Khoj, an extremely smart and helpful search assistant with the ability t
 - Break messages into multiple search queries when required to retrieve the relevant information.
 - Add date filters to your search queries from questions and answers when required to retrieve the relevant information.
 
-What searches will you need to perform to answer the users question? Respond with only a list of search queries as strings.
+What searches will you need to perform to answer the users question? Respond with search queries as list of strings in a JSON object.
 Current Date: {current_date}
 User's Location: {location}
 
 Q: How was my trip to Cambodia?
-Khoj: ["How was my trip to Cambodia?"]
+Khoj: {{"queries": ["How was my trip to Cambodia?"]}}
 A: The trip was amazing. I went to the Angkor Wat temple and it was beautiful.
 
 Q: Who did i visit that temple with?
-Khoj: ["Who did I visit the Angkor Wat Temple in Cambodia with?"]
+Khoj: {{"queries": ["Who did I visit the Angkor Wat Temple in Cambodia with?"]}}
 A: You visited the Angkor Wat Temple in Cambodia with Pablo, Namita and Xi.
 
 Q: What national parks did I go to last year?
-Khoj: ["National park I visited in {last_new_year} dt>='{last_new_year_date}' dt<'{current_new_year_date}'"]
+Khoj: {{"queries": ["National park I visited in {last_new_year} dt>='{last_new_year_date}' dt<'{current_new_year_date}'"]}}
 A: You visited the Grand Canyon and Yellowstone National Park in {last_new_year}.
 
 Q: How can you help me?
-Khoj: ["Social relationships", "Physical and mental health", "Education and career", "Personal life goals and habits"]
+Khoj: {{"queries": ["Social relationships", "Physical and mental health", "Education and career", "Personal life goals and habits"]}}
 A: I can help you live healthier and happier across work and personal life
 
 Q: How many tennis balls fit in the back of a 2002 Honda Civic?
-Khoj: ["What is the size of a tennis ball?", "What is the trunk size of a 2002 Honda Civic?"]
+Khoj: {{"queries": ["What is the size of a tennis ball?", "What is the trunk size of a 2002 Honda Civic?"]}}
 A: 1085 tennis balls will fit in the trunk of a Honda Civic
 
 Q: Is Bob older than Tom?
-Khoj: ["When was Bob born?", "What is Tom's age?"]
+Khoj: {{"queries": ["When was Bob born?", "What is Tom's age?"]}}
 A: Yes, Bob is older than Tom. As Bob was born on 1984-01-01 and Tom is 30 years old.
 
 Q: What is their age difference?
-Khoj: ["What is Bob's age?", "What is Tom's age?"]
+Khoj: {{"queries": ["What is Bob's age?", "What is Tom's age?"]}}
 A: Bob is {bob_tom_age_difference} years older than Tom. As Bob is {bob_age} years old and Tom is 30 years old.
 
 Q: What does yesterday's note say?
-Khoj: ["Note from {yesterday_date} dt>='{yesterday_date}' dt<'{current_date}'"]
+Khoj: {{"queries": ["Note from {yesterday_date} dt>='{yesterday_date}' dt<'{current_date}'"]}}
 A: Yesterday's note mentions your visit to your local beach with Ram and Shyam.
 
 {chat_history}
@@ -337,7 +337,7 @@ User: I'm thinking of moving to a new city. I'm trying to decide between New Yor
 AI: Moving to a new city can be challenging. Both New York and San Francisco are great cities to live in. New York is known for its diverse culture and San Francisco is known for its tech scene.
 
 Q: What is the population of each of those cities?
-Khoj: ["online"]
+Khoj: {{"source": ["online"]}}
 
 Example:
 Chat History:
@@ -345,14 +345,14 @@ User: I'm thinking of my next vacation idea. Ideally, I want to see something ne
 AI: Excellent! Taking a vacation is a great way to relax and recharge.
 
 Q: Where did Grandma grow up?
-Khoj: ["notes"]
+Khoj: {{"source": ["notes"]}}
 
 Example:
 Chat History:
 
 
 Q: What can you do for me?
-Khoj: ["notes", "online"]
+Khoj: {{"source": ["notes", "online"]}}
 
 Example:
 Chat History:
@@ -360,7 +360,7 @@ User: Good morning
 AI: Good morning! How can I help you today?
 
 Q: How can I share my files with Khoj?
-Khoj: ["default", "online"]
+Khoj: {{"source": ["default", "online"]}}
 
 Example:
 Chat History:
@@ -368,9 +368,9 @@ User: I want to start a new hobby. I'm thinking of learning to play the guitar.
 AI: Learning to play the guitar is a great hobby. It can be a lot of fun and a great way to express yourself.
 
 Q: What is the first element of the periodic table?
-Khoj: ["general"]
+Khoj: {{"source": ["general"]}}
 
-Now it's your turn to pick the data sources you would like to use to answer the user's question. Provide your response as a list of strings.
+Now it's your turn to pick the data sources you would like to use to answer the user's question. Respond with data sources as a list of strings in a JSON object.
 
 Chat History:
 {chat_history}
@@ -391,7 +391,7 @@ You are Khoj, an advanced google search assistant. You are tasked with construct
 - Official, up-to-date information about you, Khoj, is available at site:khoj.dev
 
 What Google searches, if any, will you need to perform to answer the user's question?
-Provide search queries as a list of strings
+Provide search queries as a JSON list of strings
 Current Date: {current_date}
 User's Location: {location}
 
@@ -401,14 +401,14 @@ User: I like to use Hacker News to get my tech news.
 AI: Hacker News is an online forum for sharing and discussing the latest tech news. It is a great place to learn about new technologies and startups.
 
 Q: Summarize posts about vector databases on Hacker News since Feb 2024
-Khoj: ["site:news.ycombinator.com after:2024/02/01 vector database"]
+Khoj: {{"queries": ["site:news.ycombinator.com after:2024/02/01 vector database"]}}
 
 History:
 User: I'm currently living in New York but I'm thinking about moving to San Francisco.
 AI: New York is a great city to live in. It has a lot of great restaurants and museums. San Francisco is also a great city to live in. It has good access to nature and a great tech scene.
 
 Q: What is the climate like in those cities?
-Khoj: ["climate in new york city", "climate in san francisco"]
+Khoj: {{"queries": ["climate in new york city", "climate in san francisco"]}}
 
 History:
 AI: Hey, how is it going?
@@ -416,27 +416,27 @@ User: Going well. Ananya is in town tonight!
 AI: Oh that's awesome! What are your plans for the evening?
 
 Q: She wants to see a movie. Any decent sci-fi movies playing at the local theater?
-Khoj: ["new sci-fi movies in theaters near {location}"]
+Khoj: {{"queries": ["new sci-fi movies in theaters near {location}"]}}
 
 History:
 User: Can I chat with you over WhatsApp?
 AI: Yes, you can chat with me using WhatsApp.
 
 Q: How
-Khoj: ["site:khoj.dev chat with Khoj on Whatsapp"]
+Khoj: {{"queries": ["site:khoj.dev chat with Khoj on Whatsapp"]}}
 
 History:
 
 
 Q: How do I share my files with you?
-Khoj: ["site:khoj.dev sync files with Khoj"]
+Khoj: {{"queries": ["site:khoj.dev sync files with Khoj"]}}
 
 History:
 User: I need to transport a lot of oranges to the moon. Are there any rockets that can fit a lot of oranges?
 AI: NASA's Saturn V rocket frequently makes lunar trips and has a large cargo capacity.
 
 Q: How many oranges would fit in NASA's Saturn V rocket?
-Khoj: ["volume of an orange", "volume of saturn v rocket"]
+Khoj: {{"queries": ["volume of an orange", "volume of saturn v rocket"]}}
 
 Now it's your turn to construct a search query for Google to answer the user's question.
 History:
diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py
index ef2e3722..724d640a 100644
--- a/src/khoj/routers/helpers.py
+++ b/src/khoj/routers/helpers.py
@@ -167,12 +167,12 @@ async def aget_relevant_information_sources(query: str, conversation_history: di
         chat_history=chat_history,
     )
 
-    response = await send_message_to_model_wrapper(relevant_tools_prompt)
+    response = await send_message_to_model_wrapper(relevant_tools_prompt, response_type="json_object")
 
     try:
         response = response.strip()
         response = json.loads(response)
-        response = [q.strip() for q in response if q.strip()]
+        response = [q.strip() for q in response["source"] if q.strip()]
         if not isinstance(response, list) or not response or len(response) == 0:
             logger.error(f"Invalid response for determining relevant tools: {response}")
             return tool_options
@@ -244,13 +244,13 @@ async def generate_online_subqueries(q: str, conversation_history: dict, locatio
         location=location,
     )
 
-    response = await send_message_to_model_wrapper(online_queries_prompt)
+    response = await send_message_to_model_wrapper(online_queries_prompt, response_type="json_object")
 
     # Validate that the response is a non-empty, JSON-serializable list
     try:
         response = response.strip()
         response = json.loads(response)
-        response = [q.strip() for q in response if q.strip()]
+        response = [q.strip() for q in response["queries"] if q.strip()]
         if not isinstance(response, list) or not response or len(response) == 0:
             logger.error(f"Invalid response for constructing subqueries: {response}. Returning original query: {q}")
             return [q]
@@ -324,6 +324,7 @@ async def generate_better_image_prompt(
 async def send_message_to_model_wrapper(
     message: str,
     system_message: str = "",
+    response_type: str = "text",
 ):
     conversation_config: ChatModelOptions = await ConversationAdapters.aget_default_conversation_config()
 
@@ -352,9 +353,7 @@ async def send_message_to_model_wrapper(
         api_key = openai_chat_config.api_key
         chat_model = conversation_config.chat_model
         openai_response = send_message_to_model(
-            messages=truncated_messages,
-            api_key=api_key,
-            model=chat_model,
+            messages=truncated_messages, api_key=api_key, model=chat_model, response_type=response_type
         )
 
         return openai_response

From 1aeea3d854ffdb2e330ba31a1e77eec5e48267fe Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Thu, 14 Mar 2024 02:29:22 +0530
Subject: [PATCH 15/19] Fix opening external links from confirmation dialog box
 on desktop app

---
 src/interface/desktop/main.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/interface/desktop/main.js b/src/interface/desktop/main.js
index fff595af..d561a2d5 100644
--- a/src/interface/desktop/main.js
+++ b/src/interface/desktop/main.js
@@ -384,10 +384,10 @@ const createWindow = (tab = 'chat.html') => {
 
     // Open external links in link handler registered on OS (e.g. browser)
     win.webContents.setWindowOpenHandler(async ({ url }) => {
-        const shouldOpen = { response: 0 };
+        let shouldOpen = { response: 0 };
 
-        if (!url.startsWith('http://')) {
-            // Confirm before opening non-HTTP links
+        if (!url.startsWith(store.get('hostURL'))) {
+            // Confirm before opening external links
             const confirmNotice = `Do you want to open this link? It will be handled by an external application.\n\n${url}`;
             shouldOpen = await dialog.showMessageBox({
                 type: 'question',

From a1ce12296f692c83ce5b99bcb4ca9cdd3711fa87 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Thu, 14 Mar 2024 03:36:26 +0530
Subject: [PATCH 16/19] Fix rendering online with note references post
 streaming chat response

Previously only the notes references would get rendered post response
streaming when when both online and notes references were used to
respond to the user's message
---
 src/interface/desktop/chat.html          | 23 ++++++++++++++---------
 src/khoj/interface/web/chat.html         | 23 ++++++++++++++---------
 src/khoj/processor/conversation/utils.py |  2 +-
 3 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html
index 3da215fe..cc081da7 100644
--- a/src/interface/desktop/chat.html
+++ b/src/interface/desktop/chat.html
@@ -357,15 +357,16 @@
 
             let numReferences = 0;
 
-            if (Array.isArray(references)) {
-                numReferences = references.length;
+            if (references.hasOwnProperty("notes")) {
+                numReferences += references["notes"].length;
 
-                references.forEach((reference, index) => {
+                references["notes"].forEach((reference, index) => {
                     let polishedReference = generateReference(reference, index);
                     referenceSection.appendChild(polishedReference);
                 });
-            } else {
-                numReferences += processOnlineReferences(referenceSection, references);
+            }
+            if (references.hasOwnProperty("online")){
+                numReferences += processOnlineReferences(referenceSection, references["online"]);
             }
 
             let referenceExpandButton = document.createElement('button');
@@ -511,7 +512,7 @@
                 // Handle streamed response of type text/event-stream or text/plain
                 const reader = response.body.getReader();
                 const decoder = new TextDecoder();
-                let references = null;
+                let references = {};
 
                 readStream();
 
@@ -519,8 +520,8 @@
                     reader.read().then(({ done, value }) => {
                         if (done) {
                             // Append any references after all the data has been streamed
-                            if (references != null) {
-                                newResponseText.appendChild(references);
+                            if (references != {}) {
+                                newResponseText.appendChild(createReferenceSection(references));
                             }
                             document.getElementById("chat-body").scrollTop = document.getElementById("chat-body").scrollHeight;
                             document.getElementById("chat-input").removeAttribute("disabled");
@@ -538,7 +539,11 @@
 
                             const rawReference = chunk.split("### compiled references:")[1];
                             const rawReferenceAsJson = JSON.parse(rawReference);
-                            references = createReferenceSection(rawReferenceAsJson);
+                            if (rawReferenceAsJson instanceof Array) {
+                                references["notes"] = rawReferenceAsJson;
+                            } else if (typeof rawReferenceAsJson === "object" && rawReferenceAsJson !== null) {
+                                references["online"] = rawReferenceAsJson;
+                            }
                             readStream();
                         } else {
                             // Display response from Khoj
diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html
index ef45b0db..c251bff2 100644
--- a/src/khoj/interface/web/chat.html
+++ b/src/khoj/interface/web/chat.html
@@ -368,15 +368,16 @@ To get started, just start typing below. You can also type / to see a list of co
 
             let numReferences = 0;
 
-            if (Array.isArray(references)) {
-                numReferences = references.length;
+            if (references.hasOwnProperty("notes")) {
+                numReferences += references["notes"].length;
 
-                references.forEach((reference, index) => {
+                references["notes"].forEach((reference, index) => {
                     let polishedReference = generateReference(reference, index);
                     referenceSection.appendChild(polishedReference);
                 });
-            } else {
-                numReferences += processOnlineReferences(referenceSection, references);
+            }
+            if (references.hasOwnProperty("online")) {
+                numReferences += processOnlineReferences(referenceSection, references["online"]);
             }
 
             let referenceExpandButton = document.createElement('button');
@@ -518,7 +519,7 @@ To get started, just start typing below. You can also type / to see a list of co
                 // Handle streamed response of type text/event-stream or text/plain
                 const reader = response.body.getReader();
                 const decoder = new TextDecoder();
-                let references = null;
+                let references = {};
 
                 readStream();
 
@@ -526,8 +527,8 @@ To get started, just start typing below. You can also type / to see a list of co
                     reader.read().then(({ done, value }) => {
                         if (done) {
                             // Append any references after all the data has been streamed
-                            if (references != null) {
-                                newResponseText.appendChild(references);
+                            if (references != {}) {
+                                newResponseText.appendChild(createReferenceSection(references));
                             }
                             document.getElementById("chat-body").scrollTop = document.getElementById("chat-body").scrollHeight;
                             document.getElementById("chat-input").removeAttribute("disabled");
@@ -545,7 +546,11 @@ To get started, just start typing below. You can also type / to see a list of co
 
                             const rawReference = chunk.split("### compiled references:")[1];
                             const rawReferenceAsJson = JSON.parse(rawReference);
-                            references = createReferenceSection(rawReferenceAsJson);
+                            if (rawReferenceAsJson instanceof Array) {
+                                references["notes"] = rawReferenceAsJson;
+                            } else if (typeof rawReferenceAsJson === "object" && rawReferenceAsJson !== null) {
+                                references["online"] = rawReferenceAsJson;
+                            }
                             readStream();
                         } else {
                             // Display response from Khoj
diff --git a/src/khoj/processor/conversation/utils.py b/src/khoj/processor/conversation/utils.py
index 74839505..f028922b 100644
--- a/src/khoj/processor/conversation/utils.py
+++ b/src/khoj/processor/conversation/utils.py
@@ -64,7 +64,7 @@ class ThreadedGenerator:
     def close(self):
         if self.compiled_references and len(self.compiled_references) > 0:
             self.queue.put(f"### compiled references:{json.dumps(self.compiled_references)}")
-        elif self.online_results and len(self.online_results) > 0:
+        if self.online_results and len(self.online_results) > 0:
             self.queue.put(f"### compiled references:{json.dumps(self.online_results)}")
         self.queue.put(StopIteration)
 

From fba0338787b3197e22d6b4ea7dc94be153be7396 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Fri, 15 Mar 2024 00:08:32 +0530
Subject: [PATCH 17/19] Release Khoj version 1.7.0

---
 manifest.json                        | 2 +-
 src/interface/desktop/package.json   | 2 +-
 src/interface/emacs/khoj.el          | 2 +-
 src/interface/obsidian/manifest.json | 2 +-
 src/interface/obsidian/package.json  | 2 +-
 src/interface/obsidian/versions.json | 3 ++-
 versions.json                        | 3 ++-
 7 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/manifest.json b/manifest.json
index 37ed4ce9..a4bdc42c 100644
--- a/manifest.json
+++ b/manifest.json
@@ -1,7 +1,7 @@
 {
 	"id": "khoj",
 	"name": "Khoj",
-	"version": "1.6.2",
+	"version": "1.7.0",
 	"minAppVersion": "0.15.0",
 	"description": "An AI copilot for your Second Brain",
 	"author": "Khoj Inc.",
diff --git a/src/interface/desktop/package.json b/src/interface/desktop/package.json
index b2eb5a6e..75de44c9 100644
--- a/src/interface/desktop/package.json
+++ b/src/interface/desktop/package.json
@@ -1,6 +1,6 @@
 {
   "name": "Khoj",
-  "version": "1.6.2",
+  "version": "1.7.0",
   "description": "An AI copilot for your Second Brain",
   "author": "Saba Imran, Debanjum Singh Solanky <team@khoj.dev>",
   "license": "GPL-3.0-or-later",
diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el
index 55e042ee..a5e41868 100644
--- a/src/interface/emacs/khoj.el
+++ b/src/interface/emacs/khoj.el
@@ -6,7 +6,7 @@
 ;;         Saba Imran <saba@khoj.dev>
 ;; Description: An AI copilot for your Second Brain
 ;; Keywords: search, chat, org-mode, outlines, markdown, pdf, image
-;; Version: 1.6.2
+;; Version: 1.7.0
 ;; Package-Requires: ((emacs "27.1") (transient "0.3.0") (dash "2.19.1"))
 ;; URL: https://github.com/khoj-ai/khoj/tree/master/src/interface/emacs
 
diff --git a/src/interface/obsidian/manifest.json b/src/interface/obsidian/manifest.json
index 37ed4ce9..a4bdc42c 100644
--- a/src/interface/obsidian/manifest.json
+++ b/src/interface/obsidian/manifest.json
@@ -1,7 +1,7 @@
 {
 	"id": "khoj",
 	"name": "Khoj",
-	"version": "1.6.2",
+	"version": "1.7.0",
 	"minAppVersion": "0.15.0",
 	"description": "An AI copilot for your Second Brain",
 	"author": "Khoj Inc.",
diff --git a/src/interface/obsidian/package.json b/src/interface/obsidian/package.json
index 1a9f3ad4..66d4a5c5 100644
--- a/src/interface/obsidian/package.json
+++ b/src/interface/obsidian/package.json
@@ -1,6 +1,6 @@
 {
     "name": "Khoj",
-    "version": "1.6.2",
+    "version": "1.7.0",
     "description": "An AI copilot for your Second Brain",
     "author": "Debanjum Singh Solanky, Saba Imran <team@khoj.dev>",
     "license": "GPL-3.0-or-later",
diff --git a/src/interface/obsidian/versions.json b/src/interface/obsidian/versions.json
index 22a518a1..150f851e 100644
--- a/src/interface/obsidian/versions.json
+++ b/src/interface/obsidian/versions.json
@@ -38,5 +38,6 @@
 	"1.5.1": "0.15.0",
 	"1.6.0": "0.15.0",
 	"1.6.1": "0.15.0",
-	"1.6.2": "0.15.0"
+	"1.6.2": "0.15.0",
+	"1.7.0": "0.15.0"
 }
diff --git a/versions.json b/versions.json
index 22a518a1..150f851e 100644
--- a/versions.json
+++ b/versions.json
@@ -38,5 +38,6 @@
 	"1.5.1": "0.15.0",
 	"1.6.0": "0.15.0",
 	"1.6.1": "0.15.0",
-	"1.6.2": "0.15.0"
+	"1.6.2": "0.15.0",
+	"1.7.0": "0.15.0"
 }

From 08993ff109568ef4b305d33a759e3ca698840d53 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Fri, 15 Mar 2024 04:02:25 +0530
Subject: [PATCH 18/19] Add new, remove old known chat models from model to
 prompt size map

---
 src/khoj/processor/conversation/utils.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/khoj/processor/conversation/utils.py b/src/khoj/processor/conversation/utils.py
index f028922b..b384ad7a 100644
--- a/src/khoj/processor/conversation/utils.py
+++ b/src/khoj/processor/conversation/utils.py
@@ -16,11 +16,10 @@ from khoj.utils.helpers import is_none_or_empty, merge_dicts
 logger = logging.getLogger(__name__)
 model_to_prompt_size = {
     "gpt-3.5-turbo": 3000,
-    "gpt-4": 7000,
-    "gpt-4-1106-preview": 7000,
+    "gpt-3.5-turbo-0125": 3000,
+    "gpt-4-0125-preview": 7000,
     "gpt-4-turbo-preview": 7000,
     "llama-2-7b-chat.ggmlv3.q4_0.bin": 1548,
-    "gpt-3.5-turbo-16k": 15000,
     "mistral-7b-instruct-v0.1.Q4_0.gguf": 1548,
 }
 model_to_tokenizer = {

From 8cdfaf41ecac9f1d5ce09a49e8991121ade57b90 Mon Sep 17 00:00:00 2001
From: Debanjum Singh Solanky <debanjum@gmail.com>
Date: Fri, 15 Mar 2024 04:03:39 +0530
Subject: [PATCH 19/19] Update project URLs to show on pypi project page

---
 pyproject.toml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index b4b6c123..d443568d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -80,10 +80,9 @@ dependencies = [
 dynamic = ["version"]
 
 [project.urls]
-Homepage = "https://github.com/khoj-ai/khoj#readme"
-Issues = "https://github.com/khoj-ai/khoj/issues"
-Discussions = "https://github.com/khoj-ai/khoj/discussions"
-Releases = "https://github.com/khoj-ai/khoj/releases"
+Homepage = "https://khoj.dev"
+Documentation = "https://docs.khoj.dev"
+Code = "https://github.com/khoj-ai/khoj"
 
 [project.scripts]
 khoj = "khoj.main:run"