From ec06d2c446e466b9f45af570990344faa4b96028 Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Thu, 16 Nov 2023 17:19:55 -0800
Subject: [PATCH 01/16] Move data indexer files into a separate folder under
 processor. Update assoc UTs

---
 .../processor/{github => data_sources}/__init__.py   |  0
 .../{markdown => data_sources/github}/__init__.py    |  0
 .../{ => data_sources}/github/github_to_entries.py   |  4 ++--
 .../{org_mode => data_sources/markdown}/__init__.py  |  0
 .../markdown/markdown_to_entries.py                  |  0
 .../{ => data_sources}/notion/notion_to_entries.py   |  0
 .../{pdf => data_sources/org_mode}/__init__.py       |  0
 .../{ => data_sources}/org_mode/org_to_entries.py    |  2 +-
 .../processor/{ => data_sources}/org_mode/orgnode.py |  0
 .../{plaintext => data_sources/pdf}/__init__.py      |  0
 .../{ => data_sources}/pdf/pdf_to_entries.py         |  0
 .../processor/data_sources/plaintext/__init__.py     |  0
 .../plaintext/plaintext_to_entries.py                |  0
 src/khoj/routers/indexer.py                          | 12 ++++++------
 src/khoj/utils/helpers.py                            |  1 +
 tests/conftest.py                                    |  4 ++--
 tests/data/config.yml                                |  2 +-
 tests/test_client.py                                 |  2 +-
 tests/test_markdown_to_entries.py                    |  2 +-
 tests/test_multiple_users.py                         |  2 +-
 tests/test_org_to_entries.py                         |  2 +-
 tests/test_orgnode.py                                |  2 +-
 tests/test_pdf_to_entries.py                         |  2 +-
 tests/test_plaintext_to_entries.py                   |  2 +-
 tests/test_text_search.py                            |  4 ++--
 25 files changed, 22 insertions(+), 21 deletions(-)
 rename src/khoj/processor/{github => data_sources}/__init__.py (100%)
 rename src/khoj/processor/{markdown => data_sources/github}/__init__.py (100%)
 rename src/khoj/processor/{ => data_sources}/github/github_to_entries.py (98%)
 rename src/khoj/processor/{org_mode => data_sources/markdown}/__init__.py (100%)
 rename src/khoj/processor/{ => data_sources}/markdown/markdown_to_entries.py (100%)
 rename src/khoj/processor/{ => data_sources}/notion/notion_to_entries.py (100%)
 rename src/khoj/processor/{pdf => data_sources/org_mode}/__init__.py (100%)
 rename src/khoj/processor/{ => data_sources}/org_mode/org_to_entries.py (99%)
 rename src/khoj/processor/{ => data_sources}/org_mode/orgnode.py (100%)
 rename src/khoj/processor/{plaintext => data_sources/pdf}/__init__.py (100%)
 rename src/khoj/processor/{ => data_sources}/pdf/pdf_to_entries.py (100%)
 create mode 100644 src/khoj/processor/data_sources/plaintext/__init__.py
 rename src/khoj/processor/{ => data_sources}/plaintext/plaintext_to_entries.py (100%)

diff --git a/src/khoj/processor/github/__init__.py b/src/khoj/processor/data_sources/__init__.py
similarity index 100%
rename from src/khoj/processor/github/__init__.py
rename to src/khoj/processor/data_sources/__init__.py
diff --git a/src/khoj/processor/markdown/__init__.py b/src/khoj/processor/data_sources/github/__init__.py
similarity index 100%
rename from src/khoj/processor/markdown/__init__.py
rename to src/khoj/processor/data_sources/github/__init__.py
diff --git a/src/khoj/processor/github/github_to_entries.py b/src/khoj/processor/data_sources/github/github_to_entries.py
similarity index 98%
rename from src/khoj/processor/github/github_to_entries.py
rename to src/khoj/processor/data_sources/github/github_to_entries.py
index 56279453..592cfcf0 100644
--- a/src/khoj/processor/github/github_to_entries.py
+++ b/src/khoj/processor/data_sources/github/github_to_entries.py
@@ -10,8 +10,8 @@ import requests
 # Internal Packages
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry, GithubContentConfig, GithubRepoConfig
-from khoj.processor.markdown.markdown_to_entries import MarkdownToEntries
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.text_to_entries import TextToEntries
 from database.models import Entry as DbEntry, GithubConfig, KhojUser
 
diff --git a/src/khoj/processor/org_mode/__init__.py b/src/khoj/processor/data_sources/markdown/__init__.py
similarity index 100%
rename from src/khoj/processor/org_mode/__init__.py
rename to src/khoj/processor/data_sources/markdown/__init__.py
diff --git a/src/khoj/processor/markdown/markdown_to_entries.py b/src/khoj/processor/data_sources/markdown/markdown_to_entries.py
similarity index 100%
rename from src/khoj/processor/markdown/markdown_to_entries.py
rename to src/khoj/processor/data_sources/markdown/markdown_to_entries.py
diff --git a/src/khoj/processor/notion/notion_to_entries.py b/src/khoj/processor/data_sources/notion/notion_to_entries.py
similarity index 100%
rename from src/khoj/processor/notion/notion_to_entries.py
rename to src/khoj/processor/data_sources/notion/notion_to_entries.py
diff --git a/src/khoj/processor/pdf/__init__.py b/src/khoj/processor/data_sources/org_mode/__init__.py
similarity index 100%
rename from src/khoj/processor/pdf/__init__.py
rename to src/khoj/processor/data_sources/org_mode/__init__.py
diff --git a/src/khoj/processor/org_mode/org_to_entries.py b/src/khoj/processor/data_sources/org_mode/org_to_entries.py
similarity index 99%
rename from src/khoj/processor/org_mode/org_to_entries.py
rename to src/khoj/processor/data_sources/org_mode/org_to_entries.py
index 04ce97e4..0aef9b67 100644
--- a/src/khoj/processor/org_mode/org_to_entries.py
+++ b/src/khoj/processor/data_sources/org_mode/org_to_entries.py
@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Iterable, List, Tuple
 
 # Internal Packages
-from khoj.processor.org_mode import orgnode
+from khoj.processor.data_sources.org_mode import orgnode
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry
diff --git a/src/khoj/processor/org_mode/orgnode.py b/src/khoj/processor/data_sources/org_mode/orgnode.py
similarity index 100%
rename from src/khoj/processor/org_mode/orgnode.py
rename to src/khoj/processor/data_sources/org_mode/orgnode.py
diff --git a/src/khoj/processor/plaintext/__init__.py b/src/khoj/processor/data_sources/pdf/__init__.py
similarity index 100%
rename from src/khoj/processor/plaintext/__init__.py
rename to src/khoj/processor/data_sources/pdf/__init__.py
diff --git a/src/khoj/processor/pdf/pdf_to_entries.py b/src/khoj/processor/data_sources/pdf/pdf_to_entries.py
similarity index 100%
rename from src/khoj/processor/pdf/pdf_to_entries.py
rename to src/khoj/processor/data_sources/pdf/pdf_to_entries.py
diff --git a/src/khoj/processor/data_sources/plaintext/__init__.py b/src/khoj/processor/data_sources/plaintext/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/khoj/processor/plaintext/plaintext_to_entries.py b/src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py
similarity index 100%
rename from src/khoj/processor/plaintext/plaintext_to_entries.py
rename to src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py
diff --git a/src/khoj/routers/indexer.py b/src/khoj/routers/indexer.py
index a7a1249d..a2cb0381 100644
--- a/src/khoj/routers/indexer.py
+++ b/src/khoj/routers/indexer.py
@@ -10,12 +10,12 @@ from starlette.authentication import requires
 
 # Internal Packages
 from khoj.utils import state, constants
-from khoj.processor.markdown.markdown_to_entries import MarkdownToEntries
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
-from khoj.processor.pdf.pdf_to_entries import PdfToEntries
-from khoj.processor.github.github_to_entries import GithubToEntries
-from khoj.processor.notion.notion_to_entries import NotionToEntries
-from khoj.processor.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries
+from khoj.processor.data_sources.github.github_to_entries import GithubToEntries
+from khoj.processor.data_sources.notion.notion_to_entries import NotionToEntries
+from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
 from khoj.search_type import text_search, image_search
 from khoj.routers.helpers import update_telemetry_state
 from khoj.utils.yaml import save_config_to_file_updated_state
diff --git a/src/khoj/utils/helpers.py b/src/khoj/utils/helpers.py
index 3bce67a0..a41de361 100644
--- a/src/khoj/utils/helpers.py
+++ b/src/khoj/utils/helpers.py
@@ -272,6 +272,7 @@ class ConversationCommand(str, Enum):
     General = "general"
     Notes = "notes"
     Help = "help"
+    Online = "online"
 
 
 command_descriptions = {
diff --git a/tests/conftest.py b/tests/conftest.py
index d90bae95..16f0ef1b 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -15,7 +15,7 @@ app = FastAPI()
 # Internal Packages
 from khoj.configure import configure_routes, configure_search_types, configure_middleware
 from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
-from khoj.processor.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
 from khoj.search_type import image_search, text_search
 from khoj.utils.config import SearchModels
 from khoj.utils.constants import web_directory
@@ -28,7 +28,7 @@ from khoj.utils.rawconfig import (
 )
 from khoj.utils import state, fs_syncer
 from khoj.routers.indexer import configure_content
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from database.models import (
     KhojApiUser,
     LocalOrgConfig,
diff --git a/tests/data/config.yml b/tests/data/config.yml
index 2d642a09..bb6736ab 100644
--- a/tests/data/config.yml
+++ b/tests/data/config.yml
@@ -14,4 +14,4 @@ search-type:
   asymmetric:
     cross-encoder: cross-encoder/ms-marco-MiniLM-L-6-v2
     encoder: sentence-transformers/msmarco-MiniLM-L-6-v3
-version: 0.14.0
+version: 0.15.0
diff --git a/tests/test_client.py b/tests/test_client.py
index f642a727..9c02a05a 100644
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -15,7 +15,7 @@ from khoj.utils import state
 from khoj.utils.state import search_models, content_index, config
 from khoj.search_type import text_search, image_search
 from khoj.utils.rawconfig import ContentConfig, SearchConfig
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from database.models import KhojUser, KhojApiUser
 from database.adapters import EntryAdapters
 
diff --git a/tests/test_markdown_to_entries.py b/tests/test_markdown_to_entries.py
index 4593b23a..9ec88382 100644
--- a/tests/test_markdown_to_entries.py
+++ b/tests/test_markdown_to_entries.py
@@ -4,7 +4,7 @@ from pathlib import Path
 import os
 
 # Internal Packages
-from khoj.processor.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
 from khoj.utils.fs_syncer import get_markdown_files
 from khoj.utils.rawconfig import TextContentConfig
 
diff --git a/tests/test_multiple_users.py b/tests/test_multiple_users.py
index 95a2535f..2b1eb2f4 100644
--- a/tests/test_multiple_users.py
+++ b/tests/test_multiple_users.py
@@ -16,7 +16,7 @@ from khoj.utils import state
 from khoj.utils.state import search_models, content_index, config
 from khoj.search_type import text_search, image_search
 from khoj.utils.rawconfig import ContentConfig, SearchConfig
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from database.models import KhojUser, KhojApiUser
 from database.adapters import EntryAdapters
 
diff --git a/tests/test_org_to_entries.py b/tests/test_org_to_entries.py
index 1eddcf95..3b80873a 100644
--- a/tests/test_org_to_entries.py
+++ b/tests/test_org_to_entries.py
@@ -3,7 +3,7 @@ import json
 import os
 
 # Internal Packages
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.helpers import is_none_or_empty
 from khoj.utils.rawconfig import Entry
diff --git a/tests/test_orgnode.py b/tests/test_orgnode.py
index c6ed3447..7ee948e2 100644
--- a/tests/test_orgnode.py
+++ b/tests/test_orgnode.py
@@ -2,7 +2,7 @@
 import datetime
 
 # Internal Packages
-from khoj.processor.org_mode import orgnode
+from khoj.processor.data_sources.org_mode import orgnode
 
 
 # Test
diff --git a/tests/test_pdf_to_entries.py b/tests/test_pdf_to_entries.py
index 3ab44639..ebf53025 100644
--- a/tests/test_pdf_to_entries.py
+++ b/tests/test_pdf_to_entries.py
@@ -3,7 +3,7 @@ import json
 import os
 
 # Internal Packages
-from khoj.processor.pdf.pdf_to_entries import PdfToEntries
+from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries
 
 from khoj.utils.fs_syncer import get_pdf_files
 from khoj.utils.rawconfig import TextContentConfig
diff --git a/tests/test_plaintext_to_entries.py b/tests/test_plaintext_to_entries.py
index 23b0d652..91a6569c 100644
--- a/tests/test_plaintext_to_entries.py
+++ b/tests/test_plaintext_to_entries.py
@@ -6,7 +6,7 @@ from pathlib import Path
 # Internal Packages
 from khoj.utils.fs_syncer import get_plaintext_files
 from khoj.utils.rawconfig import TextContentConfig
-from khoj.processor.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
 from database.models import LocalPlaintextConfig, KhojUser
 
 
diff --git a/tests/test_text_search.py b/tests/test_text_search.py
index 3d729ab5..ac24c9a0 100644
--- a/tests/test_text_search.py
+++ b/tests/test_text_search.py
@@ -10,8 +10,8 @@ import pytest
 # Internal Packages
 from khoj.search_type import text_search
 from khoj.utils.rawconfig import ContentConfig, SearchConfig
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
-from khoj.processor.github.github_to_entries import GithubToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.github.github_to_entries import GithubToEntries
 from khoj.utils.fs_syncer import collect_files, get_org_files
 from database.models import LocalOrgConfig, KhojUser, Entry, GithubConfig
 

From a0b12b001a34afda3fedfaef3f324a84e74bed75 Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Fri, 17 Nov 2023 11:04:36 -0800
Subject: [PATCH 02/16] Provide in-line rendering when output matches certain
 views

---
 src/interface/desktop/chat.html  | 12 ++++++++++++
 src/khoj/interface/web/chat.html | 12 ++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html
index ebf93195..09bc6ff7 100644
--- a/src/interface/desktop/chat.html
+++ b/src/interface/desktop/chat.html
@@ -139,6 +139,8 @@
             newHTML = newHTML.replace(/__([\s\S]*?)__/g, '<u>$1</u>');
             // Remove any text between <s>[INST] and </s> tags. These are spurious instructions for the AI chat model.
             newHTML = newHTML.replace(/<s>\[INST\].+(<\/s>)?/g, '');
+            // For any text that has single backticks, replace them with <code> tags
+            newHTML = newHTML.replace(/`([^`]+)`/g, '<code class="chat-response">$1</code>');
             return newHTML;
         }
 
@@ -573,6 +575,16 @@
             margin: 10px;
         }
 
+        code.chat-response {
+            background: var(--primary-hover);
+            color: var(--primary-inverse);
+            border-radius: 5px;
+            padding: 5px;
+            font-size: 14px;
+            font-weight: 300;
+            line-height: 1.5em;
+        }
+
         button.reference-button {
             background: var(--background-color);
             color: var(--main-text-color);
diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html
index 82e3233d..fb7c0b05 100644
--- a/src/khoj/interface/web/chat.html
+++ b/src/khoj/interface/web/chat.html
@@ -148,6 +148,8 @@ To get started, just start typing below. You can also type / to see a list of co
             newHTML = newHTML.replace(/__([\s\S]*?)__/g, '<u>$1</u>');
             // Remove any text between <s>[INST] and </s> tags. These are spurious instructions for the AI chat model.
             newHTML = newHTML.replace(/<s>\[INST\].+(<\/s>)?/g, '');
+            // For any text that has single backticks, replace them with <code> tags
+            newHTML = newHTML.replace(/`([^`]+)`/g, '<code class="chat-response">$1</code>');
             return newHTML;
         }
 
@@ -490,6 +492,16 @@ To get started, just start typing below. You can also type / to see a list of co
             background: var(--primary-hover);
         }
 
+        code.chat-response {
+            background: var(--primary-hover);
+            color: var(--primary-inverse);
+            border-radius: 5px;
+            padding: 5px;
+            font-size: 14px;
+            font-weight: 300;
+            line-height: 1.5em;
+        }
+
         #chat-body {
             font-size: medium;
             margin: 0px;

From 9ddf3b58c3ac119dd7db914a7192250cffc747a0 Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Fri, 17 Nov 2023 12:14:02 -0800
Subject: [PATCH 03/16] Use the markdown parser for rendering the chat messages
 in the web interface

---
 src/khoj/interface/web/chat.html | 48 +++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 10 deletions(-)

diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html
index fb7c0b05..95118895 100644
--- a/src/khoj/interface/web/chat.html
+++ b/src/khoj/interface/web/chat.html
@@ -9,6 +9,7 @@
         <link rel="stylesheet" href="/static/assets/khoj.css">
     </head>
     <script type="text/javascript" src="/static/assets/utils.js"></script>
+    <script type="text/javascript" src="/static/assets/markdown-it.min.js"></script>
     <script>
         let welcome_message = `
 Hi, I am Khoj, your open, personal AI 👋🏽. I can help:
@@ -83,7 +84,7 @@ To get started, just start typing below. You can also type / to see a list of co
             // Create a new div for the chat message text and append it to the chat message
             let chatMessageText = document.createElement('div');
             chatMessageText.className = `chat-message-text ${by}`;
-            chatMessageText.innerHTML = formattedMessage;
+            chatMessageText.appendChild(formattedMessage);
             chatMessage.appendChild(chatMessageText);
 
             // Append annotations div to the chat message
@@ -141,16 +142,42 @@ To get started, just start typing below. You can also type / to see a list of co
         }
 
         function formatHTMLMessage(htmlMessage) {
-            // Replace any ``` with <div class="programmatic-output">
-            let newHTML = htmlMessage.replace(/```([\s\S]*?)```/g, '<div class="programmatic-output"><button class="copy-button" onclick="copyProgrammaticOutput(event)">Copy</button>$1</div>');
-            // Replace any ** with <b> and __ with <u>
-            newHTML = newHTML.replace(/\*\*([\s\S]*?)\*\*/g, '<b>$1</b>');
-            newHTML = newHTML.replace(/__([\s\S]*?)__/g, '<u>$1</u>');
+            var md = window.markdownit();
+            let newHTML = htmlMessage;
+
             // Remove any text between <s>[INST] and </s> tags. These are spurious instructions for the AI chat model.
             newHTML = newHTML.replace(/<s>\[INST\].+(<\/s>)?/g, '');
-            // For any text that has single backticks, replace them with <code> tags
-            newHTML = newHTML.replace(/`([^`]+)`/g, '<code class="chat-response">$1</code>');
-            return newHTML;
+
+            // Render markdown
+            newHTML = md.render(newHTML);
+            // Get any elements with a class that starts with "language"
+            let element = document.createElement('div');
+            element.innerHTML = newHTML;
+            let codeBlockElements = element.querySelectorAll('[class^="language-"]');
+            // For each element, add a parent div with the class "programmatic-output"
+            codeBlockElements.forEach((codeElement) => {
+                // Create the parent div
+                let parentDiv = document.createElement('div');
+                parentDiv.classList.add("programmatic-output");
+                // Add the parent div before the code element
+                codeElement.parentNode.insertBefore(parentDiv, codeElement);
+                // Move the code element into the parent div
+                parentDiv.appendChild(codeElement);
+                // Add a copy button to each element
+                let copyButton = document.createElement('button');
+                copyButton.classList.add("copy-button");
+                copyButton.innerHTML = "Copy";
+                copyButton.addEventListener('click', copyProgrammaticOutput);
+                codeElement.prepend(copyButton);
+            });
+
+            // Get all code elements that have no class.
+            let codeElements = element.querySelectorAll('code:not([class])');
+            codeElements.forEach((codeElement) => {
+                // Add the class "chat-response" to each element
+                codeElement.classList.add("chat-response");
+            });
+            return element
         }
 
         function chat() {
@@ -206,7 +233,8 @@ To get started, just start typing below. You can also type / to see a list of co
                             if (done) {
                                 // Evaluate the contents of new_response_text.innerHTML after all the data has been streamed
                                 const currentHTML = newResponseText.innerHTML;
-                                newResponseText.innerHTML = formatHTMLMessage(currentHTML);
+                                newResponseText.innerHTML = "";
+                                newResponseText.appendChild(formatHTMLMessage(currentHTML));
                                 if (references != null) {
                                     newResponseText.appendChild(references);
                                 }

From bfbe273ffd58a3020c5a344d1563d61396486587 Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Fri, 17 Nov 2023 12:18:35 -0800
Subject: [PATCH 04/16] Add some styling to the copy button for programmatic
 output

---
 src/khoj/interface/web/chat.html | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html
index 95118895..18a03403 100644
--- a/src/khoj/interface/web/chat.html
+++ b/src/khoj/interface/web/chat.html
@@ -665,6 +665,17 @@ To get started, just start typing below. You can also type / to see a list of co
             border-bottom: 1px dotted var(--main-text-color);
         }
 
+        button.copy-button {
+            display: block;
+            border-radius: 4px;
+            background-color: var(--background-color);
+        }
+
+        button.copy-button:hover {
+            background: #f5f5f5;
+            cursor: pointer;
+        }
+
         @media (pointer: coarse), (hover: none) {
             abbr[title] {
                 position: relative;

From 0fcf234f076f237cf59ce03c3893759c79b5bd11 Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Fri, 17 Nov 2023 16:19:11 -0800
Subject: [PATCH 05/16] Add support for using serper.dev for online queries

- Use the knowledgeGraph, answerBox, peopleAlsoAsk and organic responses of serper.dev to provide online context for queries made with the /online command
- Add it as an additional tool for doing Google searches
- Render the results appropriately in the chat web window
- Pass appropriate reference data down to the LLM
---
 src/khoj/interface/web/chat.html              | 145 ++++++++++++++++--
 .../conversation/gpt4all/chat_model.py        |  10 +-
 src/khoj/processor/conversation/openai/gpt.py |   9 ++
 .../processor/conversation/openai/utils.py    |  11 +-
 src/khoj/processor/conversation/prompts.py    |  20 ++-
 src/khoj/processor/conversation/utils.py      |   5 +-
 src/khoj/processor/tools/__init__.py          |   0
 src/khoj/processor/tools/online_search.py     |  38 +++++
 src/khoj/routers/api.py                       |  17 +-
 src/khoj/routers/helpers.py                   |  15 +-
 src/khoj/utils/helpers.py                     |   1 +
 11 files changed, 250 insertions(+), 21 deletions(-)
 create mode 100644 src/khoj/processor/tools/__init__.py
 create mode 100644 src/khoj/processor/tools/online_search.py

diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html
index 18a03403..4f2b89c8 100644
--- a/src/khoj/interface/web/chat.html
+++ b/src/khoj/interface/web/chat.html
@@ -70,6 +70,52 @@ To get started, just start typing below. You can also type / to see a list of co
             return referenceButton;
         }
 
+        function generateOnlineReference(reference, index) {
+
+            // Generate HTML for Chat Reference
+            let title = reference.title;
+            let link = reference.link;
+            let snippet = reference.snippet;
+            let question = reference.question;
+            if (question) {
+                question = `<b>Question:</b> ${question}<br><br>`;
+            } else {
+                question = "";
+            }
+
+            let linkElement = document.createElement('a');
+            linkElement.setAttribute('href', link);
+            linkElement.setAttribute('target', '_blank');
+            linkElement.setAttribute('rel', 'noopener noreferrer');
+            linkElement.classList.add("inline-chat-link");
+            linkElement.classList.add("reference-link");
+            linkElement.setAttribute('title', title);
+            linkElement.innerHTML = title;
+
+            let referenceButton = document.createElement('button');
+            referenceButton.innerHTML = linkElement.outerHTML;
+            referenceButton.id = `ref-${index}`;
+            referenceButton.classList.add("reference-button");
+            referenceButton.classList.add("collapsed");
+            referenceButton.tabIndex = 0;
+
+            // Add event listener to toggle full reference on click
+            referenceButton.addEventListener('click', function() {
+                console.log(`Toggling ref-${index}`)
+                if (this.classList.contains("collapsed")) {
+                    this.classList.remove("collapsed");
+                    this.classList.add("expanded");
+                    this.innerHTML = linkElement.outerHTML + `<br><br>${question + snippet}`;
+                } else {
+                    this.classList.add("collapsed");
+                    this.classList.remove("expanded");
+                    this.innerHTML = linkElement.outerHTML;
+                }
+            });
+
+            return referenceButton;
+        }
+
         function renderMessage(message, by, dt=null, annotations=null) {
             let message_time = formatDate(dt ?? new Date());
             let by_name =  by == "khoj" ? "🏮 Khoj" : "🤔 You";
@@ -99,8 +145,45 @@ To get started, just start typing below. You can also type / to see a list of co
             chatBody.scrollTop = chatBody.scrollHeight;
         }
 
-        function renderMessageWithReference(message, by, context=null, dt=null) {
-            if (context == null || context.length == 0) {
+        function processOnlineReferences(referenceSection, onlineContext) {
+            let numOnlineReferences = 0;
+            if (onlineContext.organic && onlineContext.organic.length > 0) {
+                numOnlineReferences += onlineContext.organic.length;
+                for (let index in onlineContext.organic) {
+                    let reference = onlineContext.organic[index];
+                    let polishedReference = generateOnlineReference(reference, index);
+                    referenceSection.appendChild(polishedReference);
+                }
+            }
+
+            if (onlineContext.knowledgeGraph && onlineContext.knowledgeGraph.length > 0) {
+                numOnlineReferences += onlineContext.knowledgeGraph.length;
+                for (let index in onlineContext.knowledgeGraph) {
+                    let reference = onlineContext.knowledgeGraph[index];
+                    let polishedReference = generateOnlineReference(reference, index);
+                    referenceSection.appendChild(polishedReference);
+                }
+            }
+
+            if (onlineContext.peopleAlsoAsk && onlineContext.peopleAlsoAsk.length > 0) {
+                numOnlineReferences += onlineContext.peopleAlsoAsk.length;
+                for (let index in onlineContext.peopleAlsoAsk) {
+                    let reference = onlineContext.peopleAlsoAsk[index];
+                    let polishedReference = generateOnlineReference(reference, index);
+                    referenceSection.appendChild(polishedReference);
+                }
+            }
+
+            return numOnlineReferences;
+        }
+
+        function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null) {
+            if (context == null && onlineContext == null) {
+                renderMessage(message, by, dt);
+                return;
+            }
+
+            if ((context && context.length == 0) && (onlineContext == null || (onlineContext && onlineContext.organic.length == 0 && onlineContext.knowledgeGraph.length == 0 && onlineContext.peopleAlsoAsk.length == 0))) {
                 renderMessage(message, by, dt);
                 return;
             }
@@ -109,8 +192,11 @@ To get started, just start typing below. You can also type / to see a list of co
 
             let referenceExpandButton = document.createElement('button');
             referenceExpandButton.classList.add("reference-expand-button");
-            let expandButtonText = context.length == 1 ? "1 reference" : `${context.length} references`;
-            referenceExpandButton.innerHTML = expandButtonText;
+            let numReferences = 0;
+
+            if (context) {
+                numReferences += context.length;
+            }
 
             references.appendChild(referenceExpandButton);
 
@@ -136,6 +222,14 @@ To get started, just start typing below. You can also type / to see a list of co
                     referenceSection.appendChild(polishedReference);
                 }
             }
+
+            if (onlineContext) {
+                numReferences += processOnlineReferences(referenceSection, onlineContext);
+            }
+
+            let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`;
+            referenceExpandButton.innerHTML = expandButtonText;
+
             references.appendChild(referenceSection);
 
             renderMessage(message, by, dt, references);
@@ -177,6 +271,13 @@ To get started, just start typing below. You can also type / to see a list of co
                 // Add the class "chat-response" to each element
                 codeElement.classList.add("chat-response");
             });
+
+            let anchorElements = element.querySelectorAll('a');
+            anchorElements.forEach((anchorElement) => {
+                // Add the class "inline-chat-link" to each element
+                anchorElement.classList.add("inline-chat-link");
+            });
+
             return element
         }
 
@@ -258,15 +359,28 @@ To get started, just start typing below. You can also type / to see a list of co
 
                                 let referenceExpandButton = document.createElement('button');
                                 referenceExpandButton.classList.add("reference-expand-button");
-                                let expandButtonText = rawReferenceAsJson.length == 1 ? "1 reference" : `${rawReferenceAsJson.length} references`;
-                                referenceExpandButton.innerHTML = expandButtonText;
 
-                                references.appendChild(referenceExpandButton);
 
                                 let referenceSection = document.createElement('div');
                                 referenceSection.classList.add("reference-section");
                                 referenceSection.classList.add("collapsed");
 
+                                let numReferences = 0;
+
+                                // If rawReferenceAsJson is a list, then count the length
+                                if (Array.isArray(rawReferenceAsJson)) {
+                                    numReferences = rawReferenceAsJson.length;
+
+                                    rawReferenceAsJson.forEach((reference, index) => {
+                                    let polishedReference = generateReference(reference, index);
+                                    referenceSection.appendChild(polishedReference);
+                                });
+                                } else {
+                                    numReferences += processOnlineReferences(referenceSection, rawReferenceAsJson);
+                                }
+
+                                references.appendChild(referenceExpandButton);
+
                                 referenceExpandButton.addEventListener('click', function() {
                                     if (referenceSection.classList.contains("collapsed")) {
                                         referenceSection.classList.remove("collapsed");
@@ -277,10 +391,8 @@ To get started, just start typing below. You can also type / to see a list of co
                                     }
                                 });
 
-                                rawReferenceAsJson.forEach((reference, index) => {
-                                    let polishedReference = generateReference(reference, index);
-                                    referenceSection.appendChild(polishedReference);
-                                });
+                                let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`;
+                                referenceExpandButton.innerHTML = expandButtonText;
                                 references.appendChild(referenceSection);
                                 readStream();
                             } else {
@@ -371,7 +483,7 @@ To get started, just start typing below. You can also type / to see a list of co
                 .then(response => {
                     // Render conversation history, if any
                     response.forEach(chat_log => {
-                        renderMessageWithReference(chat_log.message, chat_log.by, chat_log.context, new Date(chat_log.created));
+                        renderMessageWithReference(chat_log.message, chat_log.by, chat_log.context, new Date(chat_log.created), chat_log.onlineContext);
                     });
                 })
                 .catch(err => {
@@ -665,6 +777,11 @@ To get started, just start typing below. You can also type / to see a list of co
             border-bottom: 1px dotted var(--main-text-color);
         }
 
+        a.reference-link {
+            color: var(--main-text-color);
+            border-bottom: 1px dotted var(--main-text-color);
+        }
+
         button.copy-button {
             display: block;
             border-radius: 4px;
@@ -749,6 +866,10 @@ To get started, just start typing below. You can also type / to see a list of co
             padding: 0px;
         }
 
+        p {
+            margin: 0;
+        }
+
         div.programmatic-output {
             background-color: #f5f5f5;
             border: 1px solid #ddd;
diff --git a/src/khoj/processor/conversation/gpt4all/chat_model.py b/src/khoj/processor/conversation/gpt4all/chat_model.py
index d3eaa01a..6ab02318 100644
--- a/src/khoj/processor/conversation/gpt4all/chat_model.py
+++ b/src/khoj/processor/conversation/gpt4all/chat_model.py
@@ -121,6 +121,7 @@ def filter_questions(questions: List[str]):
 
 def converse_offline(
     references,
+    online_results,
     user_query,
     conversation_log={},
     model: str = "mistral-7b-instruct-v0.1.Q4_0.gguf",
@@ -147,6 +148,13 @@ def converse_offline(
     # Get Conversation Primer appropriate to Conversation Type
     if conversation_command == ConversationCommand.Notes and is_none_or_empty(compiled_references_message):
         return iter([prompts.no_notes_found.format()])
+    elif conversation_command == ConversationCommand.Online and is_none_or_empty(online_results):
+        completion_func(chat_response=prompts.no_online_results_found.format())
+        return iter([prompts.no_online_results_found.format()])
+    elif conversation_command == ConversationCommand.Online:
+        conversation_primer = prompts.online_search_conversation.format(
+            query=user_query, online_results=str(online_results)
+        )
     elif conversation_command == ConversationCommand.General or is_none_or_empty(compiled_references_message):
         conversation_primer = user_query
     else:
@@ -164,7 +172,7 @@ def converse_offline(
         tokenizer_name=tokenizer_name,
     )
 
-    g = ThreadedGenerator(references, completion_func=completion_func)
+    g = ThreadedGenerator(references, online_results, completion_func=completion_func)
     t = Thread(target=llm_thread, args=(g, messages, gpt4all_model))
     t.start()
     return g
diff --git a/src/khoj/processor/conversation/openai/gpt.py b/src/khoj/processor/conversation/openai/gpt.py
index b86ebc6b..128488ac 100644
--- a/src/khoj/processor/conversation/openai/gpt.py
+++ b/src/khoj/processor/conversation/openai/gpt.py
@@ -88,6 +88,7 @@ def extract_questions(
 
 def converse(
     references,
+    online_results,
     user_query,
     conversation_log={},
     model: str = "gpt-3.5-turbo",
@@ -109,6 +110,13 @@ def converse(
     if conversation_command == ConversationCommand.Notes and is_none_or_empty(compiled_references):
         completion_func(chat_response=prompts.no_notes_found.format())
         return iter([prompts.no_notes_found.format()])
+    elif conversation_command == ConversationCommand.Online and is_none_or_empty(online_results):
+        completion_func(chat_response=prompts.no_online_results_found.format())
+        return iter([prompts.no_online_results_found.format()])
+    elif conversation_command == ConversationCommand.Online:
+        conversation_primer = prompts.online_search_conversation.format(
+            query=user_query, online_results=str(online_results)
+        )
     elif conversation_command == ConversationCommand.General or is_none_or_empty(compiled_references):
         conversation_primer = prompts.general_conversation.format(query=user_query)
     else:
@@ -130,6 +138,7 @@ def converse(
     return chat_completion_with_backoff(
         messages=messages,
         compiled_references=references,
+        online_results=online_results,
         model_name=model,
         temperature=temperature,
         openai_api_key=api_key,
diff --git a/src/khoj/processor/conversation/openai/utils.py b/src/khoj/processor/conversation/openai/utils.py
index dce72e1f..69fab7e5 100644
--- a/src/khoj/processor/conversation/openai/utils.py
+++ b/src/khoj/processor/conversation/openai/utils.py
@@ -69,9 +69,16 @@ def completion_with_backoff(**kwargs):
     reraise=True,
 )
 def chat_completion_with_backoff(
-    messages, compiled_references, model_name, temperature, openai_api_key=None, completion_func=None, model_kwargs=None
+    messages,
+    compiled_references,
+    online_results,
+    model_name,
+    temperature,
+    openai_api_key=None,
+    completion_func=None,
+    model_kwargs=None,
 ):
-    g = ThreadedGenerator(compiled_references, completion_func=completion_func)
+    g = ThreadedGenerator(compiled_references, online_results, completion_func=completion_func)
     t = Thread(target=llm_thread, args=(g, messages, model_name, temperature, openai_api_key, model_kwargs))
     t.start()
     return g
diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py
index f6b84804..7fe1ce48 100644
--- a/src/khoj/processor/conversation/prompts.py
+++ b/src/khoj/processor/conversation/prompts.py
@@ -10,7 +10,7 @@ You are Khoj, a smart, inquisitive and helpful personal assistant.
 Use your general knowledge and the past conversation with the user as context to inform your responses.
 You were created by Khoj Inc. with the following capabilities:
 
-- You *CAN REMEMBER ALL NOTES and PERSONAL INFORMATION FOREVER* that the user ever shares with you.
+- You *CAN REMEMBER ALL NOTES and PERSONAL INFORMATION FOREVER* that the user ever shares with you. They can share files with you using the Khoj desktop application.
 - You cannot set reminders.
 - Say "I don't know" or "I don't understand" if you don't know what to say or if you don't know the answer to a question.
 - Ask crisp follow-up questions to get additional context, when the answer cannot be inferred from the provided notes or past conversations.
@@ -35,6 +35,12 @@ no_notes_found = PromptTemplate.from_template(
     """.strip()
 )
 
+no_online_results_found = PromptTemplate.from_template(
+    """
+    I'm sorry, I couldn't find any relevant information from the internet to respond to your message.
+    """.strip()
+)
+
 no_entries_found = PromptTemplate.from_template(
     """
     It looks like you haven't added any notes yet. No worries, you can fix that by downloading the Khoj app from <a href=https://khoj.dev/downloads>here</a>.
@@ -103,6 +109,18 @@ Question: {query}
 """.strip()
 )
 
+## Online Search Conversation
+## --
+online_search_conversation = PromptTemplate.from_template(
+    """
+Use this up-to-date information from the internet to inform your response.
+Ask crisp follow-up questions to get additional context, when a helpful response cannot be provided from the online data or past conversations.
+
+Information from the internet: {online_results}
+
+Query: {query}""".strip()
+)
+
 
 ## Summarize Notes
 ## --
diff --git a/src/khoj/processor/conversation/utils.py b/src/khoj/processor/conversation/utils.py
index b0d401fa..b44195fd 100644
--- a/src/khoj/processor/conversation/utils.py
+++ b/src/khoj/processor/conversation/utils.py
@@ -29,9 +29,10 @@ model_to_tokenizer = {
 
 
 class ThreadedGenerator:
-    def __init__(self, compiled_references, completion_func=None):
+    def __init__(self, compiled_references, online_results, completion_func=None):
         self.queue = queue.Queue()
         self.compiled_references = compiled_references
+        self.online_results = online_results
         self.completion_func = completion_func
         self.response = ""
         self.start_time = perf_counter()
@@ -62,6 +63,8 @@ class ThreadedGenerator:
     def close(self):
         if self.compiled_references and len(self.compiled_references) > 0:
             self.queue.put(f"### compiled references:{json.dumps(self.compiled_references)}")
+        elif self.online_results and len(self.online_results) > 0:
+            self.queue.put(f"### compiled references:{json.dumps(self.online_results)}")
         self.queue.put(StopIteration)
 
 
diff --git a/src/khoj/processor/tools/__init__.py b/src/khoj/processor/tools/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/khoj/processor/tools/online_search.py b/src/khoj/processor/tools/online_search.py
new file mode 100644
index 00000000..32a2650a
--- /dev/null
+++ b/src/khoj/processor/tools/online_search.py
@@ -0,0 +1,38 @@
+import requests
+import json
+import os
+import logging
+
+logger = logging.getLogger(__name__)
+
+SERPER_DEV_API_KEY = os.getenv("SERPER_DEV_API_KEY")
+
+url = "https://google.serper.dev/search"
+
+
+def search_with_google(query: str):
+    if SERPER_DEV_API_KEY is None:
+        raise ValueError("SERPER_DEV_API_KEY is not set")
+
+    payload = json.dumps(
+        {
+            "q": query,
+        }
+    )
+
+    headers = {"X-API-KEY": SERPER_DEV_API_KEY, "Content-Type": "application/json"}
+
+    response = requests.request("POST", url, headers=headers, data=payload)
+
+    if response.status_code != 200:
+        logger.error(response.text)
+        return {}
+
+    response = response.json()
+
+    response_dict = {}
+    response_dict["knowledgeGraph"] = response.get("knowledgeGraph", {})
+    response_dict["organic"] = response.get("organic", [])
+    response_dict["answerBox"] = response.get("answerBox", [])
+    response_dict["peopleAlsoAsk"] = response.get("peopleAlsoAsk", [])
+    return response_dict
diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py
index 85fba38c..c8962938 100644
--- a/src/khoj/routers/api.py
+++ b/src/khoj/routers/api.py
@@ -4,7 +4,7 @@ import math
 import time
 import logging
 import json
-from typing import List, Optional, Union, Any
+from typing import List, Optional, Union, Any, Dict
 
 # External Packages
 from fastapi import APIRouter, Depends, HTTPException, Header, Request
@@ -41,6 +41,7 @@ from khoj.routers.helpers import (
 from khoj.processor.conversation.prompts import help_message, no_entries_found
 from khoj.processor.conversation.openai.gpt import extract_questions
 from khoj.processor.conversation.gpt4all.chat_model import extract_questions_offline
+from khoj.processor.tools.online_search import search_with_google
 from fastapi.requests import Request
 
 from database import adapters
@@ -602,6 +603,7 @@ async def chat(
     compiled_references, inferred_queries, defiltered_query = await extract_references_and_questions(
         request, meta_log, q, (n or 5), (d or math.inf), conversation_command
     )
+    online_results: Dict = dict()
 
     if conversation_command == ConversationCommand.Default and is_none_or_empty(compiled_references):
         conversation_command = ConversationCommand.General
@@ -618,11 +620,22 @@ async def chat(
         no_entries_found_format = no_entries_found.format()
         return StreamingResponse(iter([no_entries_found_format]), media_type="text/event-stream", status_code=200)
 
+    elif conversation_command == ConversationCommand.Online:
+        try:
+            online_results = search_with_google(defiltered_query)
+        except ValueError as e:
+            return StreamingResponse(
+                iter(["Please set your SERPER_DEV_API_KEY to get started with online searches 🌐"]),
+                media_type="text/event-stream",
+                status_code=200,
+            )
+
     # Get the (streamed) chat response from the LLM of choice.
     llm_response, chat_metadata = await agenerate_chat_response(
         defiltered_query,
         meta_log,
         compiled_references,
+        online_results,
         inferred_queries,
         conversation_command,
         user,
@@ -677,7 +690,7 @@ async def extract_references_and_questions(
     compiled_references: List[Any] = []
     inferred_queries: List[str] = []
 
-    if conversation_type == ConversationCommand.General:
+    if conversation_type == ConversationCommand.General or conversation_type == ConversationCommand.Online:
         return compiled_references, inferred_queries, q
 
     if not await sync_to_async(EntryAdapters.user_has_entries)(user=user):
diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py
index b52098e7..207c87a1 100644
--- a/src/khoj/routers/helpers.py
+++ b/src/khoj/routers/helpers.py
@@ -6,7 +6,7 @@ from datetime import datetime
 from functools import partial
 import logging
 from time import time
-from typing import Iterator, List, Optional, Union, Tuple, Dict
+from typing import Iterator, List, Optional, Union, Tuple, Dict, Any
 
 # External Packages
 from fastapi import HTTPException, Request
@@ -96,6 +96,8 @@ def get_conversation_command(query: str, any_references: bool = False) -> Conver
         return ConversationCommand.Help
     elif query.startswith("/general"):
         return ConversationCommand.General
+    elif query.startswith("/online"):
+        return ConversationCommand.Online
     # If no relevant notes found for the given query
     elif not any_references:
         return ConversationCommand.General
@@ -116,6 +118,7 @@ def generate_chat_response(
     q: str,
     meta_log: dict,
     compiled_references: List[str] = [],
+    online_results: Dict[str, Any] = {},
     inferred_queries: List[str] = [],
     conversation_command: ConversationCommand = ConversationCommand.Default,
     user: KhojUser = None,
@@ -125,6 +128,7 @@ def generate_chat_response(
         chat_response: str,
         user_message_time: str,
         compiled_references: List[str],
+        online_results: Dict[str, Any],
         inferred_queries: List[str],
         meta_log,
     ):
@@ -132,7 +136,11 @@ def generate_chat_response(
             user_message=q,
             chat_response=chat_response,
             user_message_metadata={"created": user_message_time},
-            khoj_message_metadata={"context": compiled_references, "intent": {"inferred-queries": inferred_queries}},
+            khoj_message_metadata={
+                "context": compiled_references,
+                "intent": {"inferred-queries": inferred_queries},
+                "onlineContext": online_results,
+            },
             conversation_log=meta_log.get("chat", []),
         )
         ConversationAdapters.save_conversation(user, {"chat": updated_conversation})
@@ -150,6 +158,7 @@ def generate_chat_response(
             q,
             user_message_time=user_message_time,
             compiled_references=compiled_references,
+            online_results=online_results,
             inferred_queries=inferred_queries,
             meta_log=meta_log,
         )
@@ -166,6 +175,7 @@ def generate_chat_response(
             loaded_model = state.gpt4all_processor_config.loaded_model
             chat_response = converse_offline(
                 references=compiled_references,
+                online_results=online_results,
                 user_query=q,
                 loaded_model=loaded_model,
                 conversation_log=meta_log,
@@ -181,6 +191,7 @@ def generate_chat_response(
             chat_model = conversation_config.chat_model
             chat_response = converse(
                 compiled_references,
+                online_results,
                 q,
                 meta_log,
                 model=chat_model,
diff --git a/src/khoj/utils/helpers.py b/src/khoj/utils/helpers.py
index a41de361..799184ae 100644
--- a/src/khoj/utils/helpers.py
+++ b/src/khoj/utils/helpers.py
@@ -279,6 +279,7 @@ command_descriptions = {
     ConversationCommand.General: "Only talk about information that relies on Khoj's general knowledge, not your personal knowledge base.",
     ConversationCommand.Notes: "Only talk about information that is available in your knowledge base.",
     ConversationCommand.Default: "The default command when no command specified. It intelligently auto-switches between general and notes mode.",
+    ConversationCommand.Online: "Look up information on the internet.",
     ConversationCommand.Help: "Display a help message with all available commands and other metadata.",
 }
 

From a7e00898cbd539522073fd3a1d27c19f30355a99 Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Fri, 17 Nov 2023 16:41:28 -0800
Subject: [PATCH 06/16] Fix rendering even when no online context references
 are returned

---
 src/interface/desktop/chat.html  | 104 +++++++++++++++++++++++++++++--
 src/khoj/interface/web/chat.html |   2 +-
 2 files changed, 100 insertions(+), 6 deletions(-)

diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html
index 09bc6ff7..ad4a2c49 100644
--- a/src/interface/desktop/chat.html
+++ b/src/interface/desktop/chat.html
@@ -60,6 +60,52 @@
             return referenceButton;
         }
 
+        function generateOnlineReference(reference, index) {
+
+            // Generate HTML for Chat Reference
+            let title = reference.title;
+            let link = reference.link;
+            let snippet = reference.snippet;
+            let question = reference.question;
+            if (question) {
+                question = `<b>Question:</b> ${question}<br><br>`;
+            } else {
+                question = "";
+            }
+
+            let linkElement = document.createElement('a');
+            linkElement.setAttribute('href', link);
+            linkElement.setAttribute('target', '_blank');
+            linkElement.setAttribute('rel', 'noopener noreferrer');
+            linkElement.classList.add("inline-chat-link");
+            linkElement.classList.add("reference-link");
+            linkElement.setAttribute('title', title);
+            linkElement.innerHTML = title;
+
+            let referenceButton = document.createElement('button');
+            referenceButton.innerHTML = linkElement.outerHTML;
+            referenceButton.id = `ref-${index}`;
+            referenceButton.classList.add("reference-button");
+            referenceButton.classList.add("collapsed");
+            referenceButton.tabIndex = 0;
+
+            // Add event listener to toggle full reference on click
+            referenceButton.addEventListener('click', function() {
+                console.log(`Toggling ref-${index}`)
+                if (this.classList.contains("collapsed")) {
+                    this.classList.remove("collapsed");
+                    this.classList.add("expanded");
+                    this.innerHTML = linkElement.outerHTML + `<br><br>${question + snippet}`;
+                } else {
+                    this.classList.add("collapsed");
+                    this.classList.remove("expanded");
+                    this.innerHTML = linkElement.outerHTML;
+                }
+            });
+
+            return referenceButton;
+        }
+
         function renderMessage(message, by, dt=null, annotations=null) {
             let message_time = formatDate(dt ?? new Date());
             let by_name =  by == "khoj" ? "🏮 Khoj" : "🤔 You";
@@ -89,8 +135,45 @@
             chatBody.scrollTop = chatBody.scrollHeight;
         }
 
-        function renderMessageWithReference(message, by, context=null, dt=null) {
-            if (context == null || context.length == 0) {
+        function processOnlineReferences(referenceSection, onlineContext) {
+            let numOnlineReferences = 0;
+            if (onlineContext.organic && onlineContext.organic.length > 0) {
+                numOnlineReferences += onlineContext.organic.length;
+                for (let index in onlineContext.organic) {
+                    let reference = onlineContext.organic[index];
+                    let polishedReference = generateOnlineReference(reference, index);
+                    referenceSection.appendChild(polishedReference);
+                }
+            }
+
+            if (onlineContext.knowledgeGraph && onlineContext.knowledgeGraph.length > 0) {
+                numOnlineReferences += onlineContext.knowledgeGraph.length;
+                for (let index in onlineContext.knowledgeGraph) {
+                    let reference = onlineContext.knowledgeGraph[index];
+                    let polishedReference = generateOnlineReference(reference, index);
+                    referenceSection.appendChild(polishedReference);
+                }
+            }
+
+            if (onlineContext.peopleAlsoAsk && onlineContext.peopleAlsoAsk.length > 0) {
+                numOnlineReferences += onlineContext.peopleAlsoAsk.length;
+                for (let index in onlineContext.peopleAlsoAsk) {
+                    let reference = onlineContext.peopleAlsoAsk[index];
+                    let polishedReference = generateOnlineReference(reference, index);
+                    referenceSection.appendChild(polishedReference);
+                }
+            }
+
+            return numOnlineReferences;
+        }
+
+        function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null) {
+            if (context == null && onlineContext == null) {
+                renderMessage(message, by, dt);
+                return;
+            }
+
+            if ((context && context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) {
                 renderMessage(message, by, dt);
                 return;
             }
@@ -99,8 +182,11 @@
 
             let referenceExpandButton = document.createElement('button');
             referenceExpandButton.classList.add("reference-expand-button");
-            let expandButtonText = context.length == 1 ? "1 reference" : `${context.length} references`;
-            referenceExpandButton.innerHTML = expandButtonText;
+            let numReferences = 0;
+
+            if (context) {
+                numReferences += context.length;
+            }
 
             references.appendChild(referenceExpandButton);
 
@@ -126,6 +212,14 @@
                     referenceSection.appendChild(polishedReference);
                 }
             }
+
+            if (onlineContext) {
+                numReferences += processOnlineReferences(referenceSection, onlineContext);
+            }
+
+            let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`;
+            referenceExpandButton.innerHTML = expandButtonText;
+
             references.appendChild(referenceSection);
 
             renderMessage(message, by, dt, references);
@@ -352,7 +446,7 @@
                 .then(response => {
                     // Render conversation history, if any
                     response.forEach(chat_log => {
-                        renderMessageWithReference(chat_log.message, chat_log.by, chat_log.context, new Date(chat_log.created));
+                        renderMessageWithReference(chat_log.message, chat_log.by, chat_log.context, new Date(chat_log.created), chat_log.onlineContext);
                     });
                 })
                 .catch(err => {
diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html
index 4f2b89c8..bc75d6c2 100644
--- a/src/khoj/interface/web/chat.html
+++ b/src/khoj/interface/web/chat.html
@@ -183,7 +183,7 @@ To get started, just start typing below. You can also type / to see a list of co
                 return;
             }
 
-            if ((context && context.length == 0) && (onlineContext == null || (onlineContext && onlineContext.organic.length == 0 && onlineContext.knowledgeGraph.length == 0 && onlineContext.peopleAlsoAsk.length == 0))) {
+            if ((context && context.length == 0) && (onlineContext == null || (onlineContext && Object.keys(onlineContext).length == 0))) {
                 renderMessage(message, by, dt);
                 return;
             }

From 262f3ccb596825247dc49e048d6c7914ca13b5ad Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Fri, 17 Nov 2023 17:11:00 -0800
Subject: [PATCH 07/16] Resolve mypy issues with formatting

---
 src/khoj/processor/tools/online_search.py | 10 +++++-----
 src/khoj/routers/api.py                   |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/khoj/processor/tools/online_search.py b/src/khoj/processor/tools/online_search.py
index 32a2650a..7ac8273e 100644
--- a/src/khoj/processor/tools/online_search.py
+++ b/src/khoj/processor/tools/online_search.py
@@ -28,11 +28,11 @@ def search_with_google(query: str):
         logger.error(response.text)
         return {}
 
-    response = response.json()
+    json_response = response.json()
 
     response_dict = {}
-    response_dict["knowledgeGraph"] = response.get("knowledgeGraph", {})
-    response_dict["organic"] = response.get("organic", [])
-    response_dict["answerBox"] = response.get("answerBox", [])
-    response_dict["peopleAlsoAsk"] = response.get("peopleAlsoAsk", [])
+    response_dict["knowledgeGraph"] = json_response.get("knowledgeGraph", {})
+    response_dict["organic"] = json_response.get("organic", [])
+    response_dict["answerBox"] = json_response.get("answerBox", [])
+    response_dict["peopleAlsoAsk"] = json_response.get("peopleAlsoAsk", [])
     return response_dict
diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py
index c8962938..ae13ed70 100644
--- a/src/khoj/routers/api.py
+++ b/src/khoj/routers/api.py
@@ -296,7 +296,7 @@ async def get_all_filenames(
         client=client,
     )
 
-    return await sync_to_async(list)(EntryAdapters.aget_all_filenames_by_source(user, content_source))
+    return await sync_to_async(list)(EntryAdapters.aget_all_filenames_by_source(user, content_source))  # type: ignore[call-arg]
 
 
 @api.post("/config/data/conversation/model", status_code=200)

From 11ccb927557a4a34601fdf94fff3ef8b5f67ffb5 Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Fri, 17 Nov 2023 18:55:59 -0800
Subject: [PATCH 08/16] Fix formatting of welcome message to use markdown

---
 src/khoj/interface/web/chat.html | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html
index bc75d6c2..b27312aa 100644
--- a/src/khoj/interface/web/chat.html
+++ b/src/khoj/interface/web/chat.html
@@ -13,11 +13,11 @@
     <script>
         let welcome_message = `
 Hi, I am Khoj, your open, personal AI 👋🏽. I can help:
-• 🧠 Answer general knowledge questions
-• 💡 Be a sounding board for your ideas
-• 📜 Chat with your notes & documents
+- 🧠 Answer general knowledge questions
+- 💡 Be a sounding board for your ideas
+- 📜 Chat with your notes & documents
 
-Download the <a class='inline-chat-link' href='https://khoj.dev/downloads'>🖥️ Desktop app</a> to chat with your computer docs.
+Download the [🖥️ Desktop app](https://khoj.dev/downloads) to chat with your computer docs.
 
 To get started, just start typing below. You can also type / to see a list of commands.
 `.trim()

From fee99779bf06457d15c0c227310a5703089e0046 Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Mon, 20 Nov 2023 15:19:15 -0800
Subject: [PATCH 09/16] Add subqueries for internet-connected search results
 and update client-side code accordingly

- Add a wrapper method to help make direct queries to the LLM and determine any intermediate responses needed for handling the request
---
 src/database/adapters/__init__.py             | 21 +++++
 src/interface/desktop/chat.html               | 70 +++++++++-------
 src/khoj/interface/web/chat.html              | 40 ++++-----
 .../conversation/gpt4all/chat_model.py        | 24 ++++--
 src/khoj/processor/conversation/openai/gpt.py | 21 +++++
 src/khoj/processor/conversation/prompts.py    | 26 ++++++
 src/khoj/processor/tools/online_search.py     | 54 ++++++++-----
 src/khoj/routers/api.py                       |  2 +-
 src/khoj/routers/helpers.py                   | 81 ++++++++++++++++---
 9 files changed, 253 insertions(+), 86 deletions(-)

diff --git a/src/database/adapters/__init__.py b/src/database/adapters/__init__.py
index 4141d3bb..7fae4ac7 100644
--- a/src/database/adapters/__init__.py
+++ b/src/database/adapters/__init__.py
@@ -4,6 +4,7 @@ from datetime import date, datetime
 import secrets
 from typing import Type, List
 from datetime import date, timezone
+import random
 
 from django.db import models
 from django.contrib.sessions.backends.db import SessionStore
@@ -339,6 +340,26 @@ class ConversationAdapters:
     async def get_openai_chat_config():
         return await OpenAIProcessorConversationConfig.objects.filter().afirst()
 
+    @staticmethod
+    def get_valid_conversation_config(user: KhojUser):
+        offline_chat_config = ConversationAdapters.get_offline_chat_conversation_config()
+        conversation_config = ConversationAdapters.get_conversation_config(user)
+        if conversation_config is None:
+            conversation_config = ConversationAdapters.get_default_conversation_config()
+
+        if offline_chat_config and offline_chat_config.enabled and conversation_config.model_type == "offline":
+            if state.gpt4all_processor_config is None or state.gpt4all_processor_config.loaded_model is None:
+                state.gpt4all_processor_config = GPT4AllProcessorModel(conversation_config.chat_model)
+
+            return conversation_config
+
+        openai_chat_config = ConversationAdapters.get_openai_conversation_config()
+        if openai_chat_config and conversation_config.model_type == "openai":
+            return conversation_config
+
+        else:
+            raise ValueError("Invalid conversation config - either configure offline chat or openai chat")
+
 
 class EntryAdapters:
     word_filer = WordFilter()
diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html
index ad4a2c49..21acc4a4 100644
--- a/src/interface/desktop/chat.html
+++ b/src/interface/desktop/chat.html
@@ -137,30 +137,33 @@
 
         function processOnlineReferences(referenceSection, onlineContext) {
             let numOnlineReferences = 0;
-            if (onlineContext.organic && onlineContext.organic.length > 0) {
-                numOnlineReferences += onlineContext.organic.length;
-                for (let index in onlineContext.organic) {
-                    let reference = onlineContext.organic[index];
-                    let polishedReference = generateOnlineReference(reference, index);
-                    referenceSection.appendChild(polishedReference);
+            for (let subquery in onlineContext) {
+                let onlineReference = onlineContext[subquery];
+                if (onlineReference.organic && onlineReference.organic.length > 0) {
+                    numOnlineReferences += onlineReference.organic.length;
+                    for (let index in onlineReference.organic) {
+                        let reference = onlineReference.organic[index];
+                        let polishedReference = generateOnlineReference(reference, index);
+                        referenceSection.appendChild(polishedReference);
+                    }
                 }
-            }
 
-            if (onlineContext.knowledgeGraph && onlineContext.knowledgeGraph.length > 0) {
-                numOnlineReferences += onlineContext.knowledgeGraph.length;
-                for (let index in onlineContext.knowledgeGraph) {
-                    let reference = onlineContext.knowledgeGraph[index];
-                    let polishedReference = generateOnlineReference(reference, index);
-                    referenceSection.appendChild(polishedReference);
+                if (onlineReference.knowledgeGraph && onlineReference.knowledgeGraph.length > 0) {
+                    numOnlineReferences += onlineReference.knowledgeGraph.length;
+                    for (let index in onlineReference.knowledgeGraph) {
+                        let reference = onlineReference.knowledgeGraph[index];
+                        let polishedReference = generateOnlineReference(reference, index);
+                        referenceSection.appendChild(polishedReference);
+                    }
                 }
-            }
 
-            if (onlineContext.peopleAlsoAsk && onlineContext.peopleAlsoAsk.length > 0) {
-                numOnlineReferences += onlineContext.peopleAlsoAsk.length;
-                for (let index in onlineContext.peopleAlsoAsk) {
-                    let reference = onlineContext.peopleAlsoAsk[index];
-                    let polishedReference = generateOnlineReference(reference, index);
-                    referenceSection.appendChild(polishedReference);
+                if (onlineReference.peopleAlsoAsk && onlineReference.peopleAlsoAsk.length > 0) {
+                    numOnlineReferences += onlineReference.peopleAlsoAsk.length;
+                    for (let index in onlineReference.peopleAlsoAsk) {
+                        let reference = onlineReference.peopleAlsoAsk[index];
+                        let polishedReference = generateOnlineReference(reference, index);
+                        referenceSection.appendChild(polishedReference);
+                    }
                 }
             }
 
@@ -316,15 +319,28 @@
 
                                 let referenceExpandButton = document.createElement('button');
                                 referenceExpandButton.classList.add("reference-expand-button");
-                                let expandButtonText = rawReferenceAsJson.length == 1 ? "1 reference" : `${rawReferenceAsJson.length} references`;
-                                referenceExpandButton.innerHTML = expandButtonText;
 
-                                references.appendChild(referenceExpandButton);
 
                                 let referenceSection = document.createElement('div');
                                 referenceSection.classList.add("reference-section");
                                 referenceSection.classList.add("collapsed");
 
+                                let numReferences = 0;
+
+                                // If rawReferenceAsJson is a list, then count the length
+                                if (Array.isArray(rawReferenceAsJson)) {
+                                    numReferences = rawReferenceAsJson.length;
+
+                                    rawReferenceAsJson.forEach((reference, index) => {
+                                    let polishedReference = generateReference(reference, index);
+                                    referenceSection.appendChild(polishedReference);
+                                });
+                                } else {
+                                    numReferences += processOnlineReferences(referenceSection, rawReferenceAsJson);
+                                }
+
+                                references.appendChild(referenceExpandButton);
+
                                 referenceExpandButton.addEventListener('click', function() {
                                     if (referenceSection.classList.contains("collapsed")) {
                                         referenceSection.classList.remove("collapsed");
@@ -335,10 +351,8 @@
                                     }
                                 });
 
-                                rawReferenceAsJson.forEach((reference, index) => {
-                                    let polishedReference = generateReference(reference, index);
-                                    referenceSection.appendChild(polishedReference);
-                                });
+                                let expandButtonText = numReferences == 1 ? "1 reference" : `${numReferences} references`;
+                                referenceExpandButton.innerHTML = expandButtonText;
                                 references.appendChild(referenceSection);
                                 readStream();
                             } else {
@@ -419,7 +433,7 @@
             const khojToken = await window.tokenAPI.getToken();
             const headers = { 'Authorization': `Bearer ${khojToken}` };
 
-            fetch(`${hostURL}/api/chat/history?client=web`, { headers })
+            fetch(`${hostURL}/api/chat/history?client=desktop`, { headers })
                 .then(response => response.json())
                 .then(data => {
                     if (data.detail) {
diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html
index b27312aa..7afa7bee 100644
--- a/src/khoj/interface/web/chat.html
+++ b/src/khoj/interface/web/chat.html
@@ -147,35 +147,38 @@ To get started, just start typing below. You can also type / to see a list of co
 
         function processOnlineReferences(referenceSection, onlineContext) {
             let numOnlineReferences = 0;
-            if (onlineContext.organic && onlineContext.organic.length > 0) {
-                numOnlineReferences += onlineContext.organic.length;
-                for (let index in onlineContext.organic) {
-                    let reference = onlineContext.organic[index];
+            for (let subquery in onlineContext) {
+                let onlineReference = onlineContext[subquery];
+                if (onlineReference.organic && onlineReference.organic.length > 0) {
+                    numOnlineReferences += onlineReference.organic.length;
+                    for (let index in onlineReference.organic) {
+                        let reference = onlineReference.organic[index];
                     let polishedReference = generateOnlineReference(reference, index);
                     referenceSection.appendChild(polishedReference);
+                    }
                 }
-            }
 
-            if (onlineContext.knowledgeGraph && onlineContext.knowledgeGraph.length > 0) {
-                numOnlineReferences += onlineContext.knowledgeGraph.length;
-                for (let index in onlineContext.knowledgeGraph) {
-                    let reference = onlineContext.knowledgeGraph[index];
+                if (onlineReference.knowledgeGraph && onlineReference.knowledgeGraph.length > 0) {
+                    numOnlineReferences += onlineReference.knowledgeGraph.length;
+                    for (let index in onlineReference.knowledgeGraph) {
+                        let reference = onlineReference.knowledgeGraph[index];
                     let polishedReference = generateOnlineReference(reference, index);
                     referenceSection.appendChild(polishedReference);
+                    }
                 }
-            }
 
-            if (onlineContext.peopleAlsoAsk && onlineContext.peopleAlsoAsk.length > 0) {
-                numOnlineReferences += onlineContext.peopleAlsoAsk.length;
-                for (let index in onlineContext.peopleAlsoAsk) {
-                    let reference = onlineContext.peopleAlsoAsk[index];
-                    let polishedReference = generateOnlineReference(reference, index);
-                    referenceSection.appendChild(polishedReference);
+                if (onlineReference.peopleAlsoAsk && onlineReference.peopleAlsoAsk.length > 0) {
+                    numOnlineReferences += onlineReference.peopleAlsoAsk.length;
+                    for (let index in onlineReference.peopleAlsoAsk) {
+                        let reference = onlineReference.peopleAlsoAsk[index];
+                        let polishedReference = generateOnlineReference(reference, index);
+                        referenceSection.appendChild(polishedReference);
+                    }
                 }
-            }
+        }
 
             return numOnlineReferences;
-        }
+                    }
 
         function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null) {
             if (context == null && onlineContext == null) {
@@ -356,7 +359,6 @@ To get started, just start typing below. You can also type / to see a list of co
                                 references = document.createElement('div');
                                 references.classList.add("references");
 
-
                                 let referenceExpandButton = document.createElement('button');
                                 referenceExpandButton.classList.add("reference-expand-button");
 
diff --git a/src/khoj/processor/conversation/gpt4all/chat_model.py b/src/khoj/processor/conversation/gpt4all/chat_model.py
index 6ab02318..41b1844b 100644
--- a/src/khoj/processor/conversation/gpt4all/chat_model.py
+++ b/src/khoj/processor/conversation/gpt4all/chat_model.py
@@ -179,13 +179,6 @@ def converse_offline(
 
 
 def llm_thread(g, messages: List[ChatMessage], model: Any):
-    try:
-        from gpt4all import GPT4All
-    except ModuleNotFoundError as e:
-        logger.info("There was an error importing GPT4All. Please run pip install gpt4all in order to install it.")
-        raise e
-
-    assert isinstance(model, GPT4All), "model should be of type GPT4All"
     user_message = messages[-1]
     system_message = messages[0]
     conversation_history = messages[1:-1]
@@ -204,7 +197,7 @@ def llm_thread(g, messages: List[ChatMessage], model: Any):
     prompted_message = templated_system_message + chat_history + templated_user_message
 
     state.chat_lock.acquire()
-    response_iterator = model.generate(prompted_message, streaming=True, max_tokens=500, n_batch=512)
+    response_iterator = send_message_to_model_offline(prompted_message, loaded_model=model, streaming=True)
     try:
         for response in response_iterator:
             if any(stop_word in response.strip() for stop_word in stop_words):
@@ -214,3 +207,18 @@ def llm_thread(g, messages: List[ChatMessage], model: Any):
     finally:
         state.chat_lock.release()
     g.close()
+
+
+def send_message_to_model_offline(
+    message, loaded_model=None, model="mistral-7b-instruct-v0.1.Q4_0.gguf", streaming=False
+):
+    try:
+        from gpt4all import GPT4All
+    except ModuleNotFoundError as e:
+        logger.info("There was an error importing GPT4All. Please run pip install gpt4all in order to install it.")
+        raise e
+
+    assert loaded_model is None or isinstance(loaded_model, GPT4All), "loaded_model must be of type GPT4All or None"
+    gpt4all_model = loaded_model or GPT4All(model)
+
+    return gpt4all_model.generate(message, max_tokens=200, top_k=2, temp=0, n_batch=512, streaming=streaming)
diff --git a/src/khoj/processor/conversation/openai/gpt.py b/src/khoj/processor/conversation/openai/gpt.py
index 909c304f..fed110f7 100644
--- a/src/khoj/processor/conversation/openai/gpt.py
+++ b/src/khoj/processor/conversation/openai/gpt.py
@@ -100,6 +100,27 @@ def extract_questions(
     return questions
 
 
+def send_message_to_model(
+    message,
+    api_key,
+    model,
+):
+    """
+    Send message to model
+    """
+    messages = [ChatMessage(content=message, role="assistant")]
+
+    # Get Response from GPT
+    return completion_with_backoff(
+        messages=messages,
+        model_name=model,
+        temperature=0,
+        max_tokens=100,
+        model_kwargs={"stop": ["A: ", "\n"]},
+        openai_api_key=api_key,
+    )
+
+
 def converse(
     references,
     online_results,
diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py
index 7fe1ce48..cba7cb59 100644
--- a/src/khoj/processor/conversation/prompts.py
+++ b/src/khoj/processor/conversation/prompts.py
@@ -121,6 +121,32 @@ Information from the internet: {online_results}
 Query: {query}""".strip()
 )
 
+online_search_conversation_subqueries = PromptTemplate.from_template(
+    """
+The user has a question which you can use the internet to respond to. Can you break down the question into subqueries to get the correct answer? Provide search queries as a JSON list of strings
+
+Today's date in UTC: {current_date}
+
+Here are some examples of questions and subqueries:
+Q: What is the weather like in New York?
+A: ["weather in new york"]
+
+Q: What is the weather like in New York and San Francisco?
+A: ["weather in new york", "weather in san francisco"]
+
+Q: What is the latest news about Google stock?
+A: ["google stock news"]
+
+Q: When is the next lunar eclipse?
+A: ["next lunar eclipse"]
+
+Q: How many oranges would fit in NASA's Saturn V rocket?
+A: ["volume of an orange", "volume of saturn v rocket"]
+
+This is the user's query:
+Q: {query}
+A: """.strip()
+)
 
 ## Summarize Notes
 ## --
diff --git a/src/khoj/processor/tools/online_search.py b/src/khoj/processor/tools/online_search.py
index 7ac8273e..f014fb4a 100644
--- a/src/khoj/processor/tools/online_search.py
+++ b/src/khoj/processor/tools/online_search.py
@@ -3,6 +3,8 @@ import json
 import os
 import logging
 
+from khoj.routers.helpers import generate_online_subqueries
+
 logger = logging.getLogger(__name__)
 
 SERPER_DEV_API_KEY = os.getenv("SERPER_DEV_API_KEY")
@@ -10,29 +12,41 @@ SERPER_DEV_API_KEY = os.getenv("SERPER_DEV_API_KEY")
 url = "https://google.serper.dev/search"
 
 
-def search_with_google(query: str):
+async def search_with_google(query: str):
+    def _search_with_google(subquery: str):
+        payload = json.dumps(
+            {
+                "q": subquery,
+            }
+        )
+
+        headers = {"X-API-KEY": SERPER_DEV_API_KEY, "Content-Type": "application/json"}
+
+        response = requests.request("POST", url, headers=headers, data=payload)
+
+        if response.status_code != 200:
+            logger.error(response.text)
+            return {}
+
+        json_response = response.json()
+        sub_response_dict = {}
+        sub_response_dict["knowledgeGraph"] = json_response.get("knowledgeGraph", {})
+        sub_response_dict["organic"] = json_response.get("organic", [])
+        sub_response_dict["answerBox"] = json_response.get("answerBox", [])
+        sub_response_dict["peopleAlsoAsk"] = json_response.get("peopleAlsoAsk", [])
+
+        return sub_response_dict
+
     if SERPER_DEV_API_KEY is None:
         raise ValueError("SERPER_DEV_API_KEY is not set")
 
-    payload = json.dumps(
-        {
-            "q": query,
-        }
-    )
-
-    headers = {"X-API-KEY": SERPER_DEV_API_KEY, "Content-Type": "application/json"}
-
-    response = requests.request("POST", url, headers=headers, data=payload)
-
-    if response.status_code != 200:
-        logger.error(response.text)
-        return {}
-
-    json_response = response.json()
+    # Breakdown the query into subqueries to get the correct answer
+    subqueries = await generate_online_subqueries(query)
 
     response_dict = {}
-    response_dict["knowledgeGraph"] = json_response.get("knowledgeGraph", {})
-    response_dict["organic"] = json_response.get("organic", [])
-    response_dict["answerBox"] = json_response.get("answerBox", [])
-    response_dict["peopleAlsoAsk"] = json_response.get("peopleAlsoAsk", [])
+
+    for subquery in subqueries:
+        logger.info(f"Searching with Google for '{subquery}'")
+        response_dict[subquery] = _search_with_google(subquery)
+
     return response_dict
diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py
index d9b80756..10db3a83 100644
--- a/src/khoj/routers/api.py
+++ b/src/khoj/routers/api.py
@@ -597,7 +597,7 @@ async def chat(
 
     elif conversation_command == ConversationCommand.Online:
         try:
-            online_results = search_with_google(defiltered_query)
+            online_results = await search_with_google(defiltered_query)
         except ValueError as e:
             return StreamingResponse(
                 iter(["Please set your SERPER_DEV_API_KEY to get started with online searches 🌐"]),
diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py
index b609e977..cdb91b6f 100644
--- a/src/khoj/routers/helpers.py
+++ b/src/khoj/routers/helpers.py
@@ -6,8 +6,13 @@ from datetime import datetime
 from functools import partial
 import logging
 from time import time
+import json
 from typing import Annotated, Iterator, List, Optional, Union, Tuple, Dict, Any
 
+from datetime import datetime
+
+from khoj.processor.conversation import prompts
+
 # External Packages
 from fastapi import HTTPException, Header, Request, Depends
 
@@ -15,10 +20,10 @@ from fastapi import HTTPException, Header, Request, Depends
 from khoj.utils import state
 from khoj.utils.config import GPT4AllProcessorModel
 from khoj.utils.helpers import ConversationCommand, log_telemetry
-from khoj.processor.conversation.openai.gpt import converse
-from khoj.processor.conversation.gpt4all.chat_model import converse_offline
+from khoj.processor.conversation.openai.gpt import converse, send_message_to_model
+from khoj.processor.conversation.gpt4all.chat_model import converse_offline, send_message_to_model_offline
 from khoj.processor.conversation.utils import message_to_log, ThreadedGenerator
-from database.models import KhojUser, Subscription
+from database.models import KhojUser, Subscription, ChatModelOptions
 from database.adapters import ConversationAdapters
 
 
@@ -114,6 +119,65 @@ async def agenerate_chat_response(*args):
     return await loop.run_in_executor(executor, generate_chat_response, *args)
 
 
+async def generate_online_subqueries(q: str) -> List[str]:
+    """
+    Generate subqueries from the given query
+    """
+    utc_date = datetime.utcnow().strftime("%Y-%m-%d")
+    online_queries_prompt = prompts.online_search_conversation_subqueries.format(
+        current_date=utc_date,
+        query=q,
+    )
+
+    response = await send_message_to_model_wrapper(online_queries_prompt)
+
+    # Validate that the response is a non-empty, JSON-serializable list
+    try:
+        response = response.strip()
+        response = json.loads(response)
+        response = [q.strip() for q in response if q.strip()]
+        if not isinstance(response, list) or not response or len(response) == 0:
+            logger.error(f"Invalid response for constructing subqueries: {response}")
+            return [q]
+        return response
+    except Exception as e:
+        logger.error(f"Invalid response for constructing subqueries: {response}")
+        return [q]
+
+
+async def send_message_to_model_wrapper(
+    message: str,
+):
+    conversation_config = await ConversationAdapters.aget_default_conversation_config()
+
+    if conversation_config is None:
+        raise HTTPException(status_code=500, detail="Contact the server administrator to set a default chat model.")
+
+    if conversation_config.model_type == "offline":
+        if state.gpt4all_processor_config is None or state.gpt4all_processor_config.loaded_model is None:
+            state.gpt4all_processor_config = GPT4AllProcessorModel(conversation_config.chat_model)
+
+        loaded_model = state.gpt4all_processor_config.loaded_model
+        return send_message_to_model_offline(
+            message=message,
+            loaded_model=loaded_model,
+            model=conversation_config.chat_model,
+            streaming=False,
+        )
+
+    elif conversation_config.model_type == "openai":
+        openai_chat_config = await ConversationAdapters.aget_openai_conversation_config()
+        api_key = openai_chat_config.api_key
+        chat_model = conversation_config.chat_model
+        return send_message_to_model(
+            message=message,
+            api_key=api_key,
+            model=chat_model,
+        )
+    else:
+        raise HTTPException(status_code=500, detail="Invalid conversation config")
+
+
 def generate_chat_response(
     q: str,
     meta_log: dict,
@@ -163,12 +227,8 @@ def generate_chat_response(
             meta_log=meta_log,
         )
 
-        offline_chat_config = ConversationAdapters.get_offline_chat_conversation_config()
-        conversation_config = ConversationAdapters.get_conversation_config(user)
-        if conversation_config is None:
-            conversation_config = ConversationAdapters.get_default_conversation_config()
-        openai_chat_config = ConversationAdapters.get_openai_conversation_config()
-        if offline_chat_config and offline_chat_config.enabled and conversation_config.model_type == "offline":
+        conversation_config = ConversationAdapters.get_valid_conversation_config(user)
+        if conversation_config.model_type == "offline":
             if state.gpt4all_processor_config is None or state.gpt4all_processor_config.loaded_model is None:
                 state.gpt4all_processor_config = GPT4AllProcessorModel(conversation_config.chat_model)
 
@@ -186,7 +246,8 @@ def generate_chat_response(
                 tokenizer_name=conversation_config.tokenizer,
             )
 
-        elif openai_chat_config and conversation_config.model_type == "openai":
+        elif conversation_config.model_type == "openai":
+            openai_chat_config = ConversationAdapters.get_openai_conversation_config()
             api_key = openai_chat_config.api_key
             chat_model = conversation_config.chat_model
             chat_response = converse(

From 8fa0b69c673c128f98b5e83a865d51993215608e Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Mon, 20 Nov 2023 15:21:06 -0800
Subject: [PATCH 10/16] Resolve merge issue with adapters methods

---
 src/database/adapters/__init__.py             | 22 +++++
 .../migrations/0020_reflectivequestion.py     | 36 ++++++++
 src/database/models/__init__.py               |  5 ++
 src/interface/desktop/chat.html               | 53 +++++++++++
 src/khoj/interface/web/chat.html              | 89 +++++++++++++++++++
 src/khoj/routers/api.py                       | 11 +++
 6 files changed, 216 insertions(+)
 create mode 100644 src/database/migrations/0020_reflectivequestion.py

diff --git a/src/database/adapters/__init__.py b/src/database/adapters/__init__.py
index 7fae4ac7..53f69bc8 100644
--- a/src/database/adapters/__init__.py
+++ b/src/database/adapters/__init__.py
@@ -33,8 +33,11 @@ from database.models import (
     UserConversationConfig,
     OpenAIProcessorConversationConfig,
     OfflineChatProcessorConversationConfig,
+    ReflectiveQuestion,
 )
 from khoj.utils.helpers import generate_random_name
+from khoj.utils import state
+from khoj.utils.config import GPT4AllProcessorModel
 from khoj.search_filter.word_filter import WordFilter
 from khoj.search_filter.file_filter import FileFilter
 from khoj.search_filter.date_filter import DateFilter
@@ -340,6 +343,25 @@ class ConversationAdapters:
     async def get_openai_chat_config():
         return await OpenAIProcessorConversationConfig.objects.filter().afirst()
 
+    @staticmethod
+    async def aget_conversation_starters(user: KhojUser):
+        all_questions = []
+        if await ReflectiveQuestion.objects.filter(user=user).aexists():
+            all_questions = await sync_to_async(ReflectiveQuestion.objects.filter(user=user).values_list)(
+                "question", flat=True
+            )
+
+        all_questions = await sync_to_async(ReflectiveQuestion.objects.filter(user=None).values_list)(
+            "question", flat=True
+        )
+
+        max_results = 3
+        all_questions = await sync_to_async(list)(all_questions)
+        if len(all_questions) < max_results:
+            return all_questions
+
+        return random.sample(all_questions, max_results)
+
     @staticmethod
     def get_valid_conversation_config(user: KhojUser):
         offline_chat_config = ConversationAdapters.get_offline_chat_conversation_config()
diff --git a/src/database/migrations/0020_reflectivequestion.py b/src/database/migrations/0020_reflectivequestion.py
new file mode 100644
index 00000000..aefb73bf
--- /dev/null
+++ b/src/database/migrations/0020_reflectivequestion.py
@@ -0,0 +1,36 @@
+# Generated by Django 4.2.7 on 2023-11-20 01:13
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0019_alter_googleuser_family_name_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ReflectiveQuestion",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("question", models.CharField(max_length=500)),
+                (
+                    "user",
+                    models.ForeignKey(
+                        blank=True,
+                        default=None,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]
diff --git a/src/database/models/__init__.py b/src/database/models/__init__.py
index 92da3e6e..b0463df8 100644
--- a/src/database/models/__init__.py
+++ b/src/database/models/__init__.py
@@ -141,6 +141,11 @@ class Conversation(BaseModel):
     conversation_log = models.JSONField(default=dict)
 
 
+class ReflectiveQuestion(BaseModel):
+    question = models.CharField(max_length=500)
+    user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True)
+
+
 class Entry(BaseModel):
     class EntryType(models.TextChoices):
         IMAGE = "image"
diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html
index 21acc4a4..0b50555b 100644
--- a/src/interface/desktop/chat.html
+++ b/src/interface/desktop/chat.html
@@ -385,6 +385,9 @@
             let chatInput = document.getElementById("chat-input");
             chatInput.value = chatInput.value.trimStart();
 
+            let questionStarterSuggestions = document.getElementById("question-starters");
+            questionStarterSuggestions.style.display = "none";
+
             if (chatInput.value.startsWith("/") && chatInput.value.split(" ").length === 1) {
                 let chatTooltip = document.getElementById("chat-tooltip");
                 chatTooltip.style.display = "block";
@@ -467,6 +470,31 @@
                     return;
                 });
 
+            fetch(`${hostURL}/api/chat/starters?client=desktop`, { headers })
+                .then(response => response.json())
+                .then(data => {
+                    // Render chat options, if any
+                    if (data) {
+                        let questionStarterSuggestions = document.getElementById("question-starters");
+                        for (let index in data) {
+                            let questionStarter = data[index];
+                            let questionStarterButton = document.createElement('button');
+                            questionStarterButton.innerHTML = questionStarter;
+                            questionStarterButton.classList.add("question-starter");
+                            questionStarterButton.addEventListener('click', function() {
+                                questionStarterSuggestions.style.display = "none";
+                                document.getElementById("chat-input").value = questionStarter;
+                                chat();
+                            });
+                            questionStarterSuggestions.appendChild(questionStarterButton);
+                        }
+                        questionStarterSuggestions.style.display = "grid";
+                    }
+                })
+                .catch(err => {
+                    return;
+                });
+
             fetch(`${hostURL}/api/chat/options`, { headers })
                 .then(response => response.json())
                 .then(data => {
@@ -506,6 +534,9 @@
         <!-- Chat Body -->
         <div id="chat-body"></div>
 
+        <!-- Chat Suggestions -->
+        <div id="question-starters" style="display: none;"></div>
+
         <!-- Chat Footer -->
         <div id="chat-footer">
             <div id="chat-tooltip" style="display: none;"></div>
@@ -683,6 +714,28 @@
             margin: 10px;
         }
 
+        div#question-starters {
+            grid-template-columns: repeat(auto-fit, minmax(100px, 1fr));
+            grid-column-gap: 8px;
+        }
+
+        button.question-starter {
+            background: var(--background-color);
+            color: var(--main-text-color);
+            border: 1px solid var(--main-text-color);
+            border-radius: 5px;
+            padding: 5px;
+            font-size: 14px;
+            font-weight: 300;
+            line-height: 1.5em;
+            cursor: pointer;
+            transition: background 0.2s ease-in-out;
+            text-align: left;
+            max-height: 75px;
+            transition: max-height 0.3s ease-in-out;
+            overflow: hidden;
+        }
+
         code.chat-response {
             background: var(--primary-hover);
             color: var(--primary-inverse);
diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html
index 7afa7bee..8224db65 100644
--- a/src/khoj/interface/web/chat.html
+++ b/src/khoj/interface/web/chat.html
@@ -147,6 +147,7 @@ To get started, just start typing below. You can also type / to see a list of co
 
         function processOnlineReferences(referenceSection, onlineContext) {
             let numOnlineReferences = 0;
+<<<<<<< Updated upstream
             for (let subquery in onlineContext) {
                 let onlineReference = onlineContext[subquery];
                 if (onlineReference.organic && onlineReference.organic.length > 0) {
@@ -156,8 +157,27 @@ To get started, just start typing below. You can also type / to see a list of co
                     let polishedReference = generateOnlineReference(reference, index);
                     referenceSection.appendChild(polishedReference);
                     }
+||||||| Stash base
+            if (onlineContext.organic && onlineContext.organic.length > 0) {
+                numOnlineReferences += onlineContext.organic.length;
+                for (let index in onlineContext.organic) {
+                    let reference = onlineContext.organic[index];
+                    let polishedReference = generateOnlineReference(reference, index);
+                    referenceSection.appendChild(polishedReference);
+=======
+            for (let subquery in onlineContext) {
+                let onlineReference = onlineContext[subquery];
+                if (onlineReference.organic && onlineReference.organic.length > 0) {
+                    numOnlineReferences += onlineReference.organic.length;
+                    for (let index in onlineReference.organic) {
+                        let reference = onlineReference.organic[index];
+                        let polishedReference = generateOnlineReference(reference, index);
+                        referenceSection.appendChild(polishedReference);
+                    }
+>>>>>>> Stashed changes
                 }
 
+<<<<<<< Updated upstream
                 if (onlineReference.knowledgeGraph && onlineReference.knowledgeGraph.length > 0) {
                     numOnlineReferences += onlineReference.knowledgeGraph.length;
                     for (let index in onlineReference.knowledgeGraph) {
@@ -165,6 +185,22 @@ To get started, just start typing below. You can also type / to see a list of co
                     let polishedReference = generateOnlineReference(reference, index);
                     referenceSection.appendChild(polishedReference);
                     }
+||||||| Stash base
+            if (onlineContext.knowledgeGraph && onlineContext.knowledgeGraph.length > 0) {
+                numOnlineReferences += onlineContext.knowledgeGraph.length;
+                for (let index in onlineContext.knowledgeGraph) {
+                    let reference = onlineContext.knowledgeGraph[index];
+                    let polishedReference = generateOnlineReference(reference, index);
+                    referenceSection.appendChild(polishedReference);
+=======
+                if (onlineReference.knowledgeGraph && onlineReference.knowledgeGraph.length > 0) {
+                    numOnlineReferences += onlineReference.knowledgeGraph.length;
+                    for (let index in onlineReference.knowledgeGraph) {
+                        let reference = onlineReference.knowledgeGraph[index];
+                        let polishedReference = generateOnlineReference(reference, index);
+                        referenceSection.appendChild(polishedReference);
+                    }
+>>>>>>> Stashed changes
                 }
 
                 if (onlineReference.peopleAlsoAsk && onlineReference.peopleAlsoAsk.length > 0) {
@@ -426,6 +462,9 @@ To get started, just start typing below. You can also type / to see a list of co
             let chatInput = document.getElementById("chat-input");
             chatInput.value = chatInput.value.trimStart();
 
+            let questionStarterSuggestions = document.getElementById("question-starters");
+            questionStarterSuggestions.style.display = "none";
+
             if (chatInput.value.startsWith("/") && chatInput.value.split(" ").length === 1) {
                 let chatTooltip = document.getElementById("chat-tooltip");
                 chatTooltip.style.display = "block";
@@ -505,6 +544,31 @@ To get started, just start typing below. You can also type / to see a list of co
                     return;
                 });
 
+            fetch('/api/chat/starters')
+                .then(response => response.json())
+                .then(data => {
+                    // Render chat options, if any
+                    if (data) {
+                        let questionStarterSuggestions = document.getElementById("question-starters");
+                        for (let index in data) {
+                            let questionStarter = data[index];
+                            let questionStarterButton = document.createElement('button');
+                            questionStarterButton.innerHTML = questionStarter;
+                            questionStarterButton.classList.add("question-starter");
+                            questionStarterButton.addEventListener('click', function() {
+                                questionStarterSuggestions.style.display = "none";
+                                document.getElementById("chat-input").value = questionStarter;
+                                chat();
+                            });
+                            questionStarterSuggestions.appendChild(questionStarterButton);
+                        }
+                        questionStarterSuggestions.style.display = "grid";
+                    }
+                })
+                .catch(err => {
+                    return;
+                });
+
             // Fill query field with value passed in URL query parameters, if any.
             var query_via_url = new URLSearchParams(window.location.search).get("q");
             if (query_via_url) {
@@ -524,6 +588,9 @@ To get started, just start typing below. You can also type / to see a list of co
         <!-- Chat Body -->
         <div id="chat-body"></div>
 
+        <!-- Chat Suggestions -->
+        <div id="question-starters" style="display: none;"></div>
+
         <!-- Chat Footer -->
         <div id="chat-footer">
             <div id="chat-tooltip" style="display: none;"></div>
@@ -584,6 +651,28 @@ To get started, just start typing below. You can also type / to see a list of co
             margin: 10px;
         }
 
+        div#question-starters {
+            grid-template-columns: repeat(auto-fit, minmax(100px, 1fr));
+            grid-column-gap: 8px;
+        }
+
+        button.question-starter {
+            background: var(--background-color);
+            color: var(--main-text-color);
+            border: 1px solid var(--main-text-color);
+            border-radius: 5px;
+            padding: 5px;
+            font-size: 14px;
+            font-weight: 300;
+            line-height: 1.5em;
+            cursor: pointer;
+            transition: background 0.2s ease-in-out;
+            text-align: left;
+            max-height: 75px;
+            transition: max-height 0.3s ease-in-out;
+            overflow: hidden;
+        }
+
         button.reference-button {
             background: var(--background-color);
             color: var(--main-text-color);
diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py
index 10db3a83..5b05c675 100644
--- a/src/khoj/routers/api.py
+++ b/src/khoj/routers/api.py
@@ -513,6 +513,17 @@ def update(
     return {"status": "ok", "message": "khoj reloaded"}
 
 
+@api.get("/chat/starters", response_class=Response)
+@requires(["authenticated"])
+async def chat_starters(
+    request: Request,
+    common: CommonQueryParams,
+) -> Response:
+    user: KhojUser = request.user.object
+    starter_questions = await ConversationAdapters.aget_conversation_starters(user)
+    return Response(content=json.dumps(starter_questions), media_type="application/json", status_code=200)
+
+
 @api.get("/chat/history")
 @requires(["authenticated"])
 def chat_history(

From a8f13f334f882f02ba62d780a276e8bd57c8a46b Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Mon, 20 Nov 2023 15:22:50 -0800
Subject: [PATCH 11/16] Fix merging issues with base after popping the stash

---
 src/khoj/interface/web/chat.html | 36 --------------------------------
 1 file changed, 36 deletions(-)

diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html
index 8224db65..abab83ab 100644
--- a/src/khoj/interface/web/chat.html
+++ b/src/khoj/interface/web/chat.html
@@ -147,24 +147,6 @@ To get started, just start typing below. You can also type / to see a list of co
 
         function processOnlineReferences(referenceSection, onlineContext) {
             let numOnlineReferences = 0;
-<<<<<<< Updated upstream
-            for (let subquery in onlineContext) {
-                let onlineReference = onlineContext[subquery];
-                if (onlineReference.organic && onlineReference.organic.length > 0) {
-                    numOnlineReferences += onlineReference.organic.length;
-                    for (let index in onlineReference.organic) {
-                        let reference = onlineReference.organic[index];
-                    let polishedReference = generateOnlineReference(reference, index);
-                    referenceSection.appendChild(polishedReference);
-                    }
-||||||| Stash base
-            if (onlineContext.organic && onlineContext.organic.length > 0) {
-                numOnlineReferences += onlineContext.organic.length;
-                for (let index in onlineContext.organic) {
-                    let reference = onlineContext.organic[index];
-                    let polishedReference = generateOnlineReference(reference, index);
-                    referenceSection.appendChild(polishedReference);
-=======
             for (let subquery in onlineContext) {
                 let onlineReference = onlineContext[subquery];
                 if (onlineReference.organic && onlineReference.organic.length > 0) {
@@ -174,25 +156,8 @@ To get started, just start typing below. You can also type / to see a list of co
                         let polishedReference = generateOnlineReference(reference, index);
                         referenceSection.appendChild(polishedReference);
                     }
->>>>>>> Stashed changes
                 }
 
-<<<<<<< Updated upstream
-                if (onlineReference.knowledgeGraph && onlineReference.knowledgeGraph.length > 0) {
-                    numOnlineReferences += onlineReference.knowledgeGraph.length;
-                    for (let index in onlineReference.knowledgeGraph) {
-                        let reference = onlineReference.knowledgeGraph[index];
-                    let polishedReference = generateOnlineReference(reference, index);
-                    referenceSection.appendChild(polishedReference);
-                    }
-||||||| Stash base
-            if (onlineContext.knowledgeGraph && onlineContext.knowledgeGraph.length > 0) {
-                numOnlineReferences += onlineContext.knowledgeGraph.length;
-                for (let index in onlineContext.knowledgeGraph) {
-                    let reference = onlineContext.knowledgeGraph[index];
-                    let polishedReference = generateOnlineReference(reference, index);
-                    referenceSection.appendChild(polishedReference);
-=======
                 if (onlineReference.knowledgeGraph && onlineReference.knowledgeGraph.length > 0) {
                     numOnlineReferences += onlineReference.knowledgeGraph.length;
                     for (let index in onlineReference.knowledgeGraph) {
@@ -200,7 +165,6 @@ To get started, just start typing below. You can also type / to see a list of co
                         let polishedReference = generateOnlineReference(reference, index);
                         referenceSection.appendChild(polishedReference);
                     }
->>>>>>> Stashed changes
                 }
 
                 if (onlineReference.peopleAlsoAsk && onlineReference.peopleAlsoAsk.length > 0) {

From a9623ef85a60f5ac365f3a988211d41e06bc7b0f Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Mon, 20 Nov 2023 15:27:42 -0800
Subject: [PATCH 12/16] Add requisite imports in order to instantiate offline
 model in adapters file

---
 src/database/adapters/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/database/adapters/__init__.py b/src/database/adapters/__init__.py
index 7fae4ac7..4cbf2d8b 100644
--- a/src/database/adapters/__init__.py
+++ b/src/database/adapters/__init__.py
@@ -4,7 +4,6 @@ from datetime import date, datetime
 import secrets
 from typing import Type, List
 from datetime import date, timezone
-import random
 
 from django.db import models
 from django.contrib.sessions.backends.db import SessionStore
@@ -35,6 +34,8 @@ from database.models import (
     OfflineChatProcessorConversationConfig,
 )
 from khoj.utils.helpers import generate_random_name
+from khoj.utils import state
+from khoj.utils.config import GPT4AllProcessorModel
 from khoj.search_filter.word_filter import WordFilter
 from khoj.search_filter.file_filter import FileFilter
 from khoj.search_filter.date_filter import DateFilter

From 1e2af083f0efe88aba29801464b6b0d727ae922f Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Tue, 21 Nov 2023 22:11:32 -0800
Subject: [PATCH 13/16] Rename the data_sources module to content

---
 .../processor/{data_sources => content}/__init__.py  |  0
 .../{data_sources => content}/github/__init__.py     |  0
 .../github/github_to_entries.py                      |  4 ++--
 .../{data_sources => content}/markdown/__init__.py   |  0
 .../markdown/markdown_to_entries.py                  |  0
 .../notion/notion_to_entries.py                      |  0
 .../{data_sources => content}/org_mode/__init__.py   |  0
 .../org_mode/org_to_entries.py                       |  2 +-
 .../{data_sources => content}/org_mode/orgnode.py    |  0
 .../{data_sources => content}/pdf/__init__.py        |  0
 .../{data_sources => content}/pdf/pdf_to_entries.py  |  0
 .../{data_sources => content}/plaintext/__init__.py  |  0
 .../plaintext/plaintext_to_entries.py                |  0
 src/khoj/routers/indexer.py                          | 12 ++++++------
 tests/conftest.py                                    |  4 ++--
 tests/test_client.py                                 |  2 +-
 tests/test_markdown_to_entries.py                    |  2 +-
 tests/test_multiple_users.py                         |  2 +-
 tests/test_org_to_entries.py                         |  2 +-
 tests/test_orgnode.py                                |  2 +-
 tests/test_pdf_to_entries.py                         |  2 +-
 tests/test_plaintext_to_entries.py                   |  2 +-
 tests/test_text_search.py                            |  4 ++--
 23 files changed, 20 insertions(+), 20 deletions(-)
 rename src/khoj/processor/{data_sources => content}/__init__.py (100%)
 rename src/khoj/processor/{data_sources => content}/github/__init__.py (100%)
 rename src/khoj/processor/{data_sources => content}/github/github_to_entries.py (98%)
 rename src/khoj/processor/{data_sources => content}/markdown/__init__.py (100%)
 rename src/khoj/processor/{data_sources => content}/markdown/markdown_to_entries.py (100%)
 rename src/khoj/processor/{data_sources => content}/notion/notion_to_entries.py (100%)
 rename src/khoj/processor/{data_sources => content}/org_mode/__init__.py (100%)
 rename src/khoj/processor/{data_sources => content}/org_mode/org_to_entries.py (99%)
 rename src/khoj/processor/{data_sources => content}/org_mode/orgnode.py (100%)
 rename src/khoj/processor/{data_sources => content}/pdf/__init__.py (100%)
 rename src/khoj/processor/{data_sources => content}/pdf/pdf_to_entries.py (100%)
 rename src/khoj/processor/{data_sources => content}/plaintext/__init__.py (100%)
 rename src/khoj/processor/{data_sources => content}/plaintext/plaintext_to_entries.py (100%)

diff --git a/src/khoj/processor/data_sources/__init__.py b/src/khoj/processor/content/__init__.py
similarity index 100%
rename from src/khoj/processor/data_sources/__init__.py
rename to src/khoj/processor/content/__init__.py
diff --git a/src/khoj/processor/data_sources/github/__init__.py b/src/khoj/processor/content/github/__init__.py
similarity index 100%
rename from src/khoj/processor/data_sources/github/__init__.py
rename to src/khoj/processor/content/github/__init__.py
diff --git a/src/khoj/processor/data_sources/github/github_to_entries.py b/src/khoj/processor/content/github/github_to_entries.py
similarity index 98%
rename from src/khoj/processor/data_sources/github/github_to_entries.py
rename to src/khoj/processor/content/github/github_to_entries.py
index 05d18d8a..b0854850 100644
--- a/src/khoj/processor/data_sources/github/github_to_entries.py
+++ b/src/khoj/processor/content/github/github_to_entries.py
@@ -9,8 +9,8 @@ import requests
 
 from khoj.database.models import Entry as DbEntry
 from khoj.database.models import GithubConfig, KhojUser
-from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.text_to_entries import TextToEntries
 
 # Internal Packages
diff --git a/src/khoj/processor/data_sources/markdown/__init__.py b/src/khoj/processor/content/markdown/__init__.py
similarity index 100%
rename from src/khoj/processor/data_sources/markdown/__init__.py
rename to src/khoj/processor/content/markdown/__init__.py
diff --git a/src/khoj/processor/data_sources/markdown/markdown_to_entries.py b/src/khoj/processor/content/markdown/markdown_to_entries.py
similarity index 100%
rename from src/khoj/processor/data_sources/markdown/markdown_to_entries.py
rename to src/khoj/processor/content/markdown/markdown_to_entries.py
diff --git a/src/khoj/processor/data_sources/notion/notion_to_entries.py b/src/khoj/processor/content/notion/notion_to_entries.py
similarity index 100%
rename from src/khoj/processor/data_sources/notion/notion_to_entries.py
rename to src/khoj/processor/content/notion/notion_to_entries.py
diff --git a/src/khoj/processor/data_sources/org_mode/__init__.py b/src/khoj/processor/content/org_mode/__init__.py
similarity index 100%
rename from src/khoj/processor/data_sources/org_mode/__init__.py
rename to src/khoj/processor/content/org_mode/__init__.py
diff --git a/src/khoj/processor/data_sources/org_mode/org_to_entries.py b/src/khoj/processor/content/org_mode/org_to_entries.py
similarity index 99%
rename from src/khoj/processor/data_sources/org_mode/org_to_entries.py
rename to src/khoj/processor/content/org_mode/org_to_entries.py
index 9e0d8e54..37c569e9 100644
--- a/src/khoj/processor/data_sources/org_mode/org_to_entries.py
+++ b/src/khoj/processor/content/org_mode/org_to_entries.py
@@ -7,7 +7,7 @@ from khoj.database.models import Entry as DbEntry
 from khoj.database.models import KhojUser
 
 # Internal Packages
-from khoj.processor.data_sources.org_mode import orgnode
+from khoj.processor.content.org_mode import orgnode
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils import state
 from khoj.utils.helpers import timer
diff --git a/src/khoj/processor/data_sources/org_mode/orgnode.py b/src/khoj/processor/content/org_mode/orgnode.py
similarity index 100%
rename from src/khoj/processor/data_sources/org_mode/orgnode.py
rename to src/khoj/processor/content/org_mode/orgnode.py
diff --git a/src/khoj/processor/data_sources/pdf/__init__.py b/src/khoj/processor/content/pdf/__init__.py
similarity index 100%
rename from src/khoj/processor/data_sources/pdf/__init__.py
rename to src/khoj/processor/content/pdf/__init__.py
diff --git a/src/khoj/processor/data_sources/pdf/pdf_to_entries.py b/src/khoj/processor/content/pdf/pdf_to_entries.py
similarity index 100%
rename from src/khoj/processor/data_sources/pdf/pdf_to_entries.py
rename to src/khoj/processor/content/pdf/pdf_to_entries.py
diff --git a/src/khoj/processor/data_sources/plaintext/__init__.py b/src/khoj/processor/content/plaintext/__init__.py
similarity index 100%
rename from src/khoj/processor/data_sources/plaintext/__init__.py
rename to src/khoj/processor/content/plaintext/__init__.py
diff --git a/src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py b/src/khoj/processor/content/plaintext/plaintext_to_entries.py
similarity index 100%
rename from src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py
rename to src/khoj/processor/content/plaintext/plaintext_to_entries.py
diff --git a/src/khoj/routers/indexer.py b/src/khoj/routers/indexer.py
index 6fcf3348..0432eed0 100644
--- a/src/khoj/routers/indexer.py
+++ b/src/khoj/routers/indexer.py
@@ -7,12 +7,12 @@ from pydantic import BaseModel
 from starlette.authentication import requires
 
 from khoj.database.models import GithubConfig, KhojUser, NotionConfig
-from khoj.processor.data_sources.github.github_to_entries import GithubToEntries
-from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
-from khoj.processor.data_sources.notion.notion_to_entries import NotionToEntries
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
-from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries
-from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.content.github.github_to_entries import GithubToEntries
+from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.content.notion.notion_to_entries import NotionToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries
+from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries
 from khoj.routers.helpers import update_telemetry_state
 from khoj.search_type import image_search, text_search
 from khoj.utils import constants, state
diff --git a/tests/conftest.py b/tests/conftest.py
index c18c5f80..54c664d5 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -18,8 +18,8 @@ from khoj.database.models import (
     LocalOrgConfig,
     LocalPlaintextConfig,
 )
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
-from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries
 from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
 from khoj.routers.indexer import configure_content
 from khoj.search_type import image_search, text_search
diff --git a/tests/test_client.py b/tests/test_client.py
index aecd0498..19aba03b 100644
--- a/tests/test_client.py
+++ b/tests/test_client.py
@@ -13,7 +13,7 @@ from PIL import Image
 from khoj.configure import configure_routes, configure_search_types
 from khoj.database.adapters import EntryAdapters
 from khoj.database.models import KhojApiUser, KhojUser
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 from khoj.search_type import image_search, text_search
 from khoj.utils import state
 from khoj.utils.rawconfig import ContentConfig, SearchConfig
diff --git a/tests/test_markdown_to_entries.py b/tests/test_markdown_to_entries.py
index 9ec88382..b20040f9 100644
--- a/tests/test_markdown_to_entries.py
+++ b/tests/test_markdown_to_entries.py
@@ -4,7 +4,7 @@ from pathlib import Path
 import os
 
 # Internal Packages
-from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries
 from khoj.utils.fs_syncer import get_markdown_files
 from khoj.utils.rawconfig import TextContentConfig
 
diff --git a/tests/test_multiple_users.py b/tests/test_multiple_users.py
index d2d7737b..dae881bf 100644
--- a/tests/test_multiple_users.py
+++ b/tests/test_multiple_users.py
@@ -5,7 +5,7 @@ from urllib.parse import quote
 import pytest
 
 from khoj.database.models import KhojApiUser, KhojUser
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 
 # Internal Packages
 from khoj.search_type import text_search
diff --git a/tests/test_org_to_entries.py b/tests/test_org_to_entries.py
index fb37426a..742c9f8e 100644
--- a/tests/test_org_to_entries.py
+++ b/tests/test_org_to_entries.py
@@ -3,7 +3,7 @@ import json
 import os
 
 # Internal Packages
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.helpers import is_none_or_empty
 from khoj.utils.rawconfig import Entry
diff --git a/tests/test_orgnode.py b/tests/test_orgnode.py
index aa6a3cb9..a543ceb5 100644
--- a/tests/test_orgnode.py
+++ b/tests/test_orgnode.py
@@ -2,7 +2,7 @@
 import datetime
 
 # Internal Packages
-from khoj.processor.data_sources.org_mode import orgnode
+from khoj.processor.content.org_mode import orgnode
 
 
 # Test
diff --git a/tests/test_pdf_to_entries.py b/tests/test_pdf_to_entries.py
index ebf53025..9aeb438f 100644
--- a/tests/test_pdf_to_entries.py
+++ b/tests/test_pdf_to_entries.py
@@ -3,7 +3,7 @@ import json
 import os
 
 # Internal Packages
-from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries
+from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries
 
 from khoj.utils.fs_syncer import get_pdf_files
 from khoj.utils.rawconfig import TextContentConfig
diff --git a/tests/test_plaintext_to_entries.py b/tests/test_plaintext_to_entries.py
index 393fac17..d3d50c94 100644
--- a/tests/test_plaintext_to_entries.py
+++ b/tests/test_plaintext_to_entries.py
@@ -4,7 +4,7 @@ import os
 from pathlib import Path
 
 from khoj.database.models import KhojUser, LocalPlaintextConfig
-from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries
 
 # Internal Packages
 from khoj.utils.fs_syncer import get_plaintext_files
diff --git a/tests/test_text_search.py b/tests/test_text_search.py
index f19903ae..186ef57c 100644
--- a/tests/test_text_search.py
+++ b/tests/test_text_search.py
@@ -8,8 +8,8 @@ from pathlib import Path
 import pytest
 
 from khoj.database.models import Entry, GithubConfig, KhojUser, LocalOrgConfig
-from khoj.processor.data_sources.github.github_to_entries import GithubToEntries
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.github.github_to_entries import GithubToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 
 # Internal Packages
 from khoj.search_type import text_search

From c652a7fd2d72120ee319f8dce01a6e3803cd687e Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Tue, 21 Nov 2023 22:25:17 -0800
Subject: [PATCH 14/16] Move text_to_entries under the new content folder

---
 src/khoj/processor/content/github/github_to_entries.py     | 2 +-
 src/khoj/processor/content/markdown/markdown_to_entries.py | 2 +-
 src/khoj/processor/content/notion/notion_to_entries.py     | 2 +-
 src/khoj/processor/content/org_mode/org_to_entries.py      | 2 +-
 src/khoj/processor/content/pdf/pdf_to_entries.py           | 2 +-
 .../processor/content/plaintext/plaintext_to_entries.py    | 2 +-
 src/khoj/processor/{ => content}/text_to_entries.py        | 0
 src/khoj/processor/conversation/prompts.py                 | 7 ++++---
 src/khoj/search_type/text_search.py                        | 4 ++--
 tests/test_org_to_entries.py                               | 2 +-
 10 files changed, 13 insertions(+), 12 deletions(-)
 rename src/khoj/processor/{ => content}/text_to_entries.py (100%)

diff --git a/src/khoj/processor/content/github/github_to_entries.py b/src/khoj/processor/content/github/github_to_entries.py
index b0854850..1149227e 100644
--- a/src/khoj/processor/content/github/github_to_entries.py
+++ b/src/khoj/processor/content/github/github_to_entries.py
@@ -11,7 +11,7 @@ from khoj.database.models import Entry as DbEntry
 from khoj.database.models import GithubConfig, KhojUser
 from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries
 from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
-from khoj.processor.text_to_entries import TextToEntries
+from khoj.processor.content.text_to_entries import TextToEntries
 
 # Internal Packages
 from khoj.utils.helpers import timer
diff --git a/src/khoj/processor/content/markdown/markdown_to_entries.py b/src/khoj/processor/content/markdown/markdown_to_entries.py
index 05531d70..2a4b6348 100644
--- a/src/khoj/processor/content/markdown/markdown_to_entries.py
+++ b/src/khoj/processor/content/markdown/markdown_to_entries.py
@@ -10,7 +10,7 @@ from khoj.database.models import Entry as DbEntry
 from khoj.database.models import KhojUser
 
 # Internal Packages
-from khoj.processor.text_to_entries import TextToEntries
+from khoj.processor.content.text_to_entries import TextToEntries
 from khoj.utils.constants import empty_escape_sequences
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry
diff --git a/src/khoj/processor/content/notion/notion_to_entries.py b/src/khoj/processor/content/notion/notion_to_entries.py
index 4a2fd817..21c7232d 100644
--- a/src/khoj/processor/content/notion/notion_to_entries.py
+++ b/src/khoj/processor/content/notion/notion_to_entries.py
@@ -8,7 +8,7 @@ import requests
 
 from khoj.database.models import Entry as DbEntry
 from khoj.database.models import KhojUser, NotionConfig
-from khoj.processor.text_to_entries import TextToEntries
+from khoj.processor.content.text_to_entries import TextToEntries
 
 # Internal Packages
 from khoj.utils.helpers import timer
diff --git a/src/khoj/processor/content/org_mode/org_to_entries.py b/src/khoj/processor/content/org_mode/org_to_entries.py
index 37c569e9..c3f345f0 100644
--- a/src/khoj/processor/content/org_mode/org_to_entries.py
+++ b/src/khoj/processor/content/org_mode/org_to_entries.py
@@ -8,7 +8,7 @@ from khoj.database.models import KhojUser
 
 # Internal Packages
 from khoj.processor.content.org_mode import orgnode
-from khoj.processor.text_to_entries import TextToEntries
+from khoj.processor.content.text_to_entries import TextToEntries
 from khoj.utils import state
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry
diff --git a/src/khoj/processor/content/pdf/pdf_to_entries.py b/src/khoj/processor/content/pdf/pdf_to_entries.py
index f1769a49..caa93636 100644
--- a/src/khoj/processor/content/pdf/pdf_to_entries.py
+++ b/src/khoj/processor/content/pdf/pdf_to_entries.py
@@ -11,7 +11,7 @@ from khoj.database.models import Entry as DbEntry
 from khoj.database.models import KhojUser
 
 # Internal Packages
-from khoj.processor.text_to_entries import TextToEntries
+from khoj.processor.content.text_to_entries import TextToEntries
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry
 
diff --git a/src/khoj/processor/content/plaintext/plaintext_to_entries.py b/src/khoj/processor/content/plaintext/plaintext_to_entries.py
index 39966bef..cae88837 100644
--- a/src/khoj/processor/content/plaintext/plaintext_to_entries.py
+++ b/src/khoj/processor/content/plaintext/plaintext_to_entries.py
@@ -9,7 +9,7 @@ from khoj.database.models import Entry as DbEntry
 from khoj.database.models import KhojUser
 
 # Internal Packages
-from khoj.processor.text_to_entries import TextToEntries
+from khoj.processor.content.text_to_entries import TextToEntries
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry
 
diff --git a/src/khoj/processor/text_to_entries.py b/src/khoj/processor/content/text_to_entries.py
similarity index 100%
rename from src/khoj/processor/text_to_entries.py
rename to src/khoj/processor/content/text_to_entries.py
diff --git a/src/khoj/processor/conversation/prompts.py b/src/khoj/processor/conversation/prompts.py
index cba7cb59..b0e316da 100644
--- a/src/khoj/processor/conversation/prompts.py
+++ b/src/khoj/processor/conversation/prompts.py
@@ -10,7 +10,7 @@ You are Khoj, a smart, inquisitive and helpful personal assistant.
 Use your general knowledge and the past conversation with the user as context to inform your responses.
 You were created by Khoj Inc. with the following capabilities:
 
-- You *CAN REMEMBER ALL NOTES and PERSONAL INFORMATION FOREVER* that the user ever shares with you. They can share files with you using the Khoj desktop application.
+- You *CAN REMEMBER ALL NOTES and PERSONAL INFORMATION FOREVER* that the user ever shares with you. They can share files with you using any Khoj client, including the native Desktop app, the Obsidian or Emacs plugins, or the web app.
 - You cannot set reminders.
 - Say "I don't know" or "I don't understand" if you don't know what to say or if you don't know the answer to a question.
 - Ask crisp follow-up questions to get additional context, when the answer cannot be inferred from the provided notes or past conversations.
@@ -128,8 +128,9 @@ The user has a question which you can use the internet to respond to. Can you br
 Today's date in UTC: {current_date}
 
 Here are some examples of questions and subqueries:
-Q: What is the weather like in New York?
-A: ["weather in new york"]
+
+Q: Posts about vector databases on Hacker News
+A: ["site:"news.ycombinator.com vector database"]
 
 Q: What is the weather like in New York and San Francisco?
 A: ["weather in new york", "weather in san francisco"]
diff --git a/src/khoj/search_type/text_search.py b/src/khoj/search_type/text_search.py
index ca0ac6d3..d04d4c6a 100644
--- a/src/khoj/search_type/text_search.py
+++ b/src/khoj/search_type/text_search.py
@@ -18,7 +18,7 @@ from khoj.utils.models import BaseEncoder
 from khoj.utils.state import SearchType
 from khoj.utils.rawconfig import SearchResponse, Entry
 from khoj.utils.jsonl import load_jsonl
-from khoj.processor.text_to_entries import TextToEntries
+from khoj.processor.content.text_to_entries import TextToEntries
 from khoj.database.adapters import EntryAdapters
 from khoj.database.models import KhojUser, Entry as DbEntry
 
@@ -141,7 +141,7 @@ def collate_results(hits, dedupe=True):
 
         else:
             hit_ids.add(hit.corpus_id)
-            yield SearchResponse.parse_obj(
+            yield SearchResponse.model_validate(
                 {
                     "entry": hit.raw,
                     "score": hit.distance,
diff --git a/tests/test_org_to_entries.py b/tests/test_org_to_entries.py
index 742c9f8e..6cd5e3ce 100644
--- a/tests/test_org_to_entries.py
+++ b/tests/test_org_to_entries.py
@@ -4,7 +4,7 @@ import os
 
 # Internal Packages
 from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
-from khoj.processor.text_to_entries import TextToEntries
+from khoj.processor.content.text_to_entries import TextToEntries
 from khoj.utils.helpers import is_none_or_empty
 from khoj.utils.rawconfig import Entry
 from khoj.utils.fs_syncer import get_org_files

From 60c23d9e3a73e1381f11b88e0b52045c352a3aea Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Tue, 21 Nov 2023 23:08:36 -0800
Subject: [PATCH 15/16] Add online search chat director tests

---
 tests/test_gpt4all_chat_director.py | 21 +++++++++++++++++++++
 tests/test_openai_chat_director.py  | 21 +++++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/tests/test_gpt4all_chat_director.py b/tests/test_gpt4all_chat_director.py
index d978fc99..d44a3c07 100644
--- a/tests/test_gpt4all_chat_director.py
+++ b/tests/test_gpt4all_chat_director.py
@@ -1,5 +1,6 @@
 # Standard Packages
 import urllib.parse
+from urllib.parse import quote
 
 # External Packages
 import pytest
@@ -54,6 +55,26 @@ def test_chat_with_no_chat_history_or_retrieved_content_gpt4all(client_offline_c
     )
 
 
+# ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
+@pytest.mark.django_db(transaction=True)
+def test_chat_with_online_content(chat_client):
+    # Act
+    q = "/online give me the link to paul graham's essay how to do great work"
+    encoded_q = quote(q, safe="")
+    response = chat_client.get(f"/api/chat?q={encoded_q}&stream=true")
+    response_message = response.content.decode("utf-8")
+
+    response_message = response_message.split("### compiled references")[0]
+
+    # Assert
+    expected_responses = ["http://www.paulgraham.com/greatwork.html"]
+    assert response.status_code == 200
+    assert any([expected_response in response_message for expected_response in expected_responses]), (
+        "Expected assistants name, [K|k]hoj, in response but got: " + response_message
+    )
+
+
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.chatquality
 @pytest.mark.django_db(transaction=True)
diff --git a/tests/test_openai_chat_director.py b/tests/test_openai_chat_director.py
index b4e63364..7b98d4da 100644
--- a/tests/test_openai_chat_director.py
+++ b/tests/test_openai_chat_director.py
@@ -1,6 +1,7 @@
 # Standard Packages
 import os
 import urllib.parse
+from urllib.parse import quote
 
 # External Packages
 import pytest
@@ -54,6 +55,26 @@ def test_chat_with_no_chat_history_or_retrieved_content(chat_client):
     )
 
 
+# ----------------------------------------------------------------------------------------------------
+@pytest.mark.chatquality
+@pytest.mark.django_db(transaction=True)
+def test_chat_with_online_content(chat_client):
+    # Act
+    q = "/online give me the link to paul graham's essay how to do great work"
+    encoded_q = quote(q, safe="")
+    response = chat_client.get(f"/api/chat?q={encoded_q}&stream=true")
+    response_message = response.content.decode("utf-8")
+
+    response_message = response_message.split("### compiled references")[0]
+
+    # Assert
+    expected_responses = ["http://www.paulgraham.com/greatwork.html"]
+    assert response.status_code == 200
+    assert any([expected_response in response_message for expected_response in expected_responses]), (
+        "Expected assistants name, [K|k]hoj, in response but got: " + response_message
+    )
+
+
 # ----------------------------------------------------------------------------------------------------
 @pytest.mark.django_db(transaction=True)
 @pytest.mark.chatquality

From e0949e232b84974577ec82671fcc13bd90aec66c Mon Sep 17 00:00:00 2001
From: sabaimran <narmiabas@gmail.com>
Date: Wed, 22 Nov 2023 07:52:51 -0800
Subject: [PATCH 16/16] Import random in adapters file for selecting reflective
 question

---
 src/khoj/database/adapters/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/khoj/database/adapters/__init__.py b/src/khoj/database/adapters/__init__.py
index ef8ebf45..7fd04006 100644
--- a/src/khoj/database/adapters/__init__.py
+++ b/src/khoj/database/adapters/__init__.py
@@ -1,4 +1,5 @@
 import math
+import random
 import secrets
 from datetime import date, datetime, timezone
 from typing import List, Optional, Type