Clean Logs, Improve Message Rendering and Make Khoj Trusted Host Configurable (#561)

- Append chat message to chat logs as TextNodes in web, desktop clients

- Simplify Code to Identify Files from Github, Notion on Web, Desktop Client
  - Use file source to find entries from github, notion on web, desktop client
  - Pass file source to clients via text search API response

- Make Django Logs Follow Khoj Log Format, Verbosity
  - Handle image search setup related warning
  - Format Django initializing outputs using Khoj logger format

- Use `KHOJ_HOST` env var to set allowed/trusted domains to host Khoj

src/interface/desktop/chat.html

@@ -74,7 +74,8 @@
             // Create a new div for the chat message text and append it to the chat message
             let chatMessageText = document.createElement('div');
             chatMessageText.className = `chat-message-text ${by}`;
-            chatMessageText.innerHTML = formattedMessage;
+            let textNode = document.createTextNode(formattedMessage);
+            chatMessageText.appendChild(textNode);
             chatMessage.appendChild(chatMessageText);
 
             // Append annotations div to the chat message

src/interface/desktop/search.html

@@ -112,14 +112,14 @@
                 } else if (
                     item.additional.file.endsWith(".md") ||
                     item.additional.file.endsWith(".markdown") ||
-                        (item.additional.file.includes("issues") && item.additional.file.includes("github.com")) ||
-                        (item.additional.file.includes("commit") && item.additional.file.includes("github.com"))
+                        (item.additional.file.includes("issues") && item.additional.source === "github") ||
+                        (item.additional.file.includes("commit") && item.additional.source === "github")
                     )
                 {
                     html += render_markdown(query, [item]);
                 } else if (item.additional.file.endsWith(".pdf")) {
                     html += render_pdf(query, [item]);
-                } else if (item.additional.file.includes("notion.so")) {
+                } else if (item.additional.source == "notion") {
                     html += `<div class="results-notion">` + `<b><a href="${item.additional.file}">${item.additional.heading}</a></b>` + `<p>${item.entry}</p>` + `</div>`;
                 } else if (item.additional.file.endsWith(".html")) {
                     html += render_html(query, [item]);

src/khoj/app/settings.py

@@ -24,15 +24,15 @@ BASE_DIR = Path(__file__).resolve().parent.parent
 SECRET_KEY = os.getenv("KHOJ_DJANGO_SECRET_KEY")
 
 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = os.getenv("KHOJ_DEBUG", "False") == "True"
+DEBUG = os.getenv("KHOJ_DEBUG") == "True"
 
-ALLOWED_HOSTS = [".khoj.dev", "localhost", "127.0.0.1", "[::1]", "beta.khoj.dev"]
+# All Subdomains of KHOJ_DOMAIN are trusted
+KHOJ_DOMAIN = os.getenv("KHOJ_DOMAIN", "khoj.dev")
+ALLOWED_HOSTS = [f".{KHOJ_DOMAIN}", "localhost", "127.0.0.1", "[::1]"]
 
 CSRF_TRUSTED_ORIGINS = [
-    "https://app.khoj.dev",
-    "https://beta.khoj.dev",
-    "https://khoj.dev",
-    "https://*.khoj.dev",
+    f"https://*.{KHOJ_DOMAIN}",
+    f"https://{KHOJ_DOMAIN}",
 ]
 
 COOKIE_SAMESITE = "None"
@@ -40,8 +40,8 @@ if DEBUG:
     SESSION_COOKIE_DOMAIN = "localhost"
     CSRF_COOKIE_DOMAIN = "localhost"
 else:
-    SESSION_COOKIE_DOMAIN = "khoj.dev"
-    CSRF_COOKIE_DOMAIN = "khoj.dev"
+    SESSION_COOKIE_DOMAIN = KHOJ_DOMAIN
+    CSRF_COOKIE_DOMAIN = KHOJ_DOMAIN
 
 SESSION_COOKIE_SECURE = True
 CSRF_COOKIE_SECURE = True

src/khoj/interface/web/chat.html

@@ -83,7 +83,8 @@ To get started, just start typing below. You can also type / to see a list of co
             // Create a new div for the chat message text and append it to the chat message
             let chatMessageText = document.createElement('div');
             chatMessageText.className = `chat-message-text ${by}`;
-            chatMessageText.innerHTML = formattedMessage;
+            let textNode = document.createTextNode(formattedMessage);
+            chatMessageText.appendChild(textNode);
             chatMessage.appendChild(chatMessageText);
 
             // Append annotations div to the chat message

src/khoj/interface/web/search.html

@@ -112,14 +112,14 @@
                 } else if (
                     item.additional.file.endsWith(".md") ||
                     item.additional.file.endsWith(".markdown") ||
-                        (item.additional.file.includes("issues") && item.additional.file.includes("github.com")) ||
-                        (item.additional.file.includes("commit") && item.additional.file.includes("github.com"))
+                        (item.additional.file.includes("issues") && item.additional.source === "github") ||
+                        (item.additional.file.includes("commit") && item.additional.source === "github")
                     )
                 {
                     html += render_markdown(query, [item]);
                 } else if (item.additional.file.endsWith(".pdf")) {
                     html += render_pdf(query, [item]);
-                } else if (item.additional.file.includes("notion.so")) {
+                } else if (item.additional.source === "notion") {
                     html += `<div class="results-notion">` + `<b><a href="${item.additional.file}">${item.additional.heading}</a></b>` + `<p>${item.entry}</p>` + `</div>`;
                 } else if (item.additional.file.endsWith(".html")) {
                     html += render_html(query, [item]);

src/khoj/main.py

@@ -3,6 +3,8 @@
 """
 
 # Standard Packages
+from contextlib import redirect_stdout
+import io
 import os
 import sys
 import locale
@@ -33,10 +35,14 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings")
 django.setup()
 
 # Initialize Django Database
-call_command("migrate", "--noinput")
+db_migrate_output = io.StringIO()
+with redirect_stdout(db_migrate_output):
+    call_command("migrate", "--noinput")
 
 # Initialize Django Static Files
-call_command("collectstatic", "--noinput")
+collectstatic_output = io.StringIO()
+with redirect_stdout(collectstatic_output):
+    call_command("collectstatic", "--noinput")
 
 # Initialize the Application Server
 app = FastAPI()
@@ -45,9 +51,16 @@ app = FastAPI()
 django_app = get_asgi_application()
 
 # Add CORS middleware
+KHOJ_DOMAIN = os.getenv("KHOJ_DOMAIN", "app.khoj.dev")
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["app://obsidian.md", "http://localhost:*", "https://app.khoj.dev/*", "app://khoj.dev"],
+    allow_origins=[
+        "app://obsidian.md",
+        "http://localhost:*",
+        "http://127.0.0.1:*",
+        f"https://{KHOJ_DOMAIN}",
+        "app://khoj.dev",
+    ],
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
@@ -79,14 +92,16 @@ def run(should_start_server=True):
     args = cli(state.cli_args)
     set_state(args)
 
-    logger.info(f"🚒 Initializing Khoj v{state.khoj_version}")
-
     # Set Logging Level
     if args.verbose == 0:
         logger.setLevel(logging.INFO)
     elif args.verbose >= 1:
         logger.setLevel(logging.DEBUG)
 
+    logger.info(f"🚒 Initializing Khoj v{state.khoj_version}")
+    logger.info(f"📦 Initializing DB:\n{db_migrate_output.getvalue().strip()}")
+    logger.debug(f"🌍 Initializing Web Client:\n{collectstatic_output.getvalue().strip()}")
+
     initialization()
 
     # Create app directory, if it doesn't exist

src/khoj/search_type/image_search.py

@@ -12,7 +12,6 @@ from sentence_transformers import SentenceTransformer, util
 from PIL import Image
 from tqdm import trange
 import torch
-from khoj.utils import state
 
 # Internal Packages
 from khoj.utils.helpers import get_absolute_path, get_from_dict, resolve_absolute_path, load_model, timer
@@ -26,9 +25,6 @@ logger = logging.getLogger(__name__)
 
 
 def initialize_model(search_config: ImageSearchConfig):
-    # Initialize Model
-    torch.set_num_threads(4)
-
     # Convert model directory to absolute path
     search_config.model_directory = resolve_absolute_path(search_config.model_directory)
 

src/khoj/search_type/text_search.py

@@ -147,6 +147,7 @@ def collate_results(hits, dedupe=True):
                     "score": hit.distance,
                     "corpus_id": str(hit.corpus_id),
                     "additional": {
+                        "source": hit.file_source,
                         "file": hit.file_path,
                         "compiled": hit.compiled,
                         "heading": hit.heading,
@@ -169,6 +170,7 @@ def deduplicated_search_responses(hits: List[SearchResponse]):
                     "score": hit.score,
                     "corpus_id": hit.corpus_id,
                     "additional": {
+                        "source": hit.additional["source"],
                         "file": hit.additional["file"],
                         "compiled": hit.additional["compiled"],
                         "heading": hit.additional["heading"],

src/khoj/utils/rawconfig.py

@@ -72,6 +72,9 @@ class ImageSearchConfig(ConfigBase):
     encoder_type: Optional[str] = None
     model_directory: Optional[Path] = None
 
+    class Config:
+        protected_namespaces = ()
+
 
 class SearchConfig(ConfigBase):
     image: Optional[ImageSearchConfig] = None