Resolve merge conflicts and fix some import ordering

.github/workflows/pypi.yml

@@ -48,7 +48,7 @@ jobs:
       - name: 🌡️ Validate Python Package
         run: |
           # Validate PyPi Package
-          pipx run check-wheel-contents dist/*.whl
+          pipx run check-wheel-contents dist/*.whl --ignore W004
           pipx run twine check dist/*
 
       - name: ⏫ Upload Python Package Artifacts

.github/workflows/test.yml

@@ -3,7 +3,7 @@ name: test
 on:
   pull_request:
     paths:
-      - src/**
+      - src/khoj/**
       - tests/**
       - config/**
       - pyproject.toml

.pre-commit-config.yaml

@@ -4,6 +4,12 @@ repos:
   hooks:
   - id: black
 
+# - repo: https://github.com/pycqa/isort
+#   rev: 5.12.0
+#   hooks:
+#   - id: isort
+#     name: isort (python)
+
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v4.4.0
   hooks:

Dockerfile

@@ -17,7 +17,7 @@ RUN sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && \
 COPY . .
 
 # Set the PYTHONPATH environment variable in order for it to find the Django app.
-ENV PYTHONPATH=/app/src:$PYTHONPATH
+ENV PYTHONPATH=/app/src/khoj:$PYTHONPATH
 
 # Run the Application
 # There are more arguments required for the application to run,

prod.Dockerfile

@@ -20,7 +20,7 @@ COPY . .
 RUN apt install vim -y
 
 # Set the PYTHONPATH environment variable in order for it to find the Django app.
-ENV PYTHONPATH=/app/src:$PYTHONPATH
+ENV PYTHONPATH=/app/src/khoj:$PYTHONPATH
 
 # Run the Application
 # There are more arguments required for the application to run,

pyproject.toml

@@ -46,7 +46,7 @@ dependencies = [
     "tiktoken >= 0.3.2",
     "tenacity >= 8.2.2",
     "pillow == 9.3.0",
-    "pydantic >= 1.10.10",
+    "pydantic >= 2.0.0",
     "pyyaml == 6.0",
     "rich >= 13.3.1",
     "schedule == 1.1.0",

pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-DJANGO_SETTINGS_MODULE = app.settings
+DJANGO_SETTINGS_MODULE = khoj.app.settings
 pythonpath = . src
 testpaths = tests
 markers =

src/interface/obsidian/src/main.ts

@@ -73,21 +73,19 @@ export default class Khoj extends Plugin {
         // Check if khoj backend is configured, note if cannot connect to backend
         let headers = { "Authorization": `Bearer ${this.settings.khojApiKey}` };
 
-        if (this.settings.khojUrl === "https://app.khoj.dev") {
+        if (this.settings.khojApiKey === "" && this.settings.khojUrl === "https://app.khoj.dev") {
-            if (this.settings.khojApiKey === "") {
+            new Notice(`❗️Khoj API key is not configured. Please visit https://app.khoj.dev/config#clients to get an API key.`);
-                new Notice(`❗️Khoj API key is not configured. Please visit https://app.khoj.dev/config#clients to get an API key.`);
+            return;
-                return;
-            }
-
-            await request({ url: this.settings.khojUrl ,method: "GET", headers: headers })
-                .then(response => {
-                    this.settings.connectedToBackend = true;
-                })
-                .catch(error => {
-                    this.settings.connectedToBackend = false;
-                    new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`);
-                });
         }
+
+        await request({ url: this.settings.khojUrl ,method: "GET", headers: headers })
+            .then(response => {
+                this.settings.connectedToBackend = true;
+            })
+            .catch(error => {
+                this.settings.connectedToBackend = false;
+                new Notice(`❗️Ensure Khoj backend is running and Khoj URL is pointing to it in the plugin settings.\n\n${error}`);
+            });
     }
 
     async saveSettings() {

src/interface/obsidian/src/search_modal.ts

@@ -87,27 +87,18 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
     }
 
     async getSuggestions(query: string): Promise<SearchResult[]> {
-        // Query Khoj backend for search results
+        // Setup Query Khoj backend for search results
         let encodedQuery = encodeURIComponent(query);
         let searchUrl = `${this.setting.khojUrl}/api/search?q=${encodedQuery}&n=${this.setting.resultsCount}&r=${this.rerank}&client=obsidian`;
         let headers = { 'Authorization': `Bearer ${this.setting.khojApiKey}` }
 
-        // Get search results for markdown and pdf files
+        // Get search results from Khoj backend
-        let mdResponse = await request({ url: `${searchUrl}&t=markdown`, headers: headers });
+        let response = await request({ url: `${searchUrl}`, headers: headers });
-        let pdfResponse = await request({ url: `${searchUrl}&t=pdf`, headers: headers });
 
         // Parse search results
-        let mdData = JSON.parse(mdResponse)
+        let results = JSON.parse(response)
             .filter((result: any) => !this.find_similar_notes || !result.additional.file.endsWith(this.app.workspace.getActiveFile()?.path))
-            .map((result: any) => { return { entry: result.entry, score: result.score, file: result.additional.file }; });
+            .map((result: any) => { return { entry: result.entry, file: result.additional.file } as SearchResult; });
-        let pdfData = JSON.parse(pdfResponse)
-            .filter((result: any) => !this.find_similar_notes || !result.additional.file.endsWith(this.app.workspace.getActiveFile()?.path))
-            .map((result: any) => { return { entry: `## ${result.additional.compiled}`, score: result.score, file: result.additional.file } as SearchResult; })
-
-        // Combine markdown and PDF results and sort them by score
-        let results = mdData.concat(pdfData)
-            .sort((a: any, b: any) => b.score - a.score)
-            .map((result: any) => { return { entry: result.entry, file: result.file } as SearchResult; })
 
         this.query = query;
         return results;

src/khoj/app/settings.py

@@ -14,7 +14,7 @@ from pathlib import Path
 import os
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
-BASE_DIR = Path(__file__).resolve().parent.parent.parent
+BASE_DIR = Path(__file__).resolve().parent.parent.parent.parent
 
 
 # Quick-start development settings - unsuitable for production
@@ -53,7 +53,7 @@ SESSION_COOKIE_SAMESITE = "None"
 INSTALLED_APPS = [
     "django.contrib.auth",
     "django.contrib.contenttypes",
-    "database.apps.DatabaseConfig",
+    "khoj.database.apps.DatabaseConfig",
     "django.contrib.admin",
     "django.contrib.sessions",
     "django.contrib.messages",

src/khoj/configure.py

@@ -20,8 +20,8 @@ from starlette.authentication import (
 )
 
 # Internal Packages
-from database.models import KhojUser, Subscription
+from khoj.database.models import KhojUser, Subscription
-from database.adapters import get_all_users, get_or_create_search_model
+from khoj.database.adapters import get_all_users, get_or_create_search_model
 from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
 from khoj.routers.indexer import configure_content, load_content, configure_search
 from khoj.utils import constants, state
@@ -45,7 +45,7 @@ class UserAuthenticationBackend(AuthenticationBackend):
     def __init__(
         self,
     ):
-        from database.models import KhojUser, KhojApiUser
+        from khoj.database.models import KhojUser, KhojApiUser
 
         self.khojuser_manager = KhojUser.objects
         self.khojapiuser_manager = KhojApiUser.objects

src/khoj/database/adapters/__init__.py

@@ -1,44 +1,40 @@
 import math
-from typing import Optional, Type, List
-from datetime import date, datetime
 import secrets
-from typing import Type, List
+from datetime import date, datetime, timezone
-from datetime import date, timezone
+from typing import List, Optional, Type
-
-from django.db import models
-from django.contrib.sessions.backends.db import SessionStore
-from pgvector.django import CosineDistance
-from django.db.models.manager import BaseManager
-from django.db.models import Q
-from torch import Tensor
 
 # Import sync_to_async from Django Channels
 from asgiref.sync import sync_to_async
-
+from django.contrib.sessions.backends.db import SessionStore
+from django.db import models
+from django.db.models import Q
+from django.db.models.manager import BaseManager
 from fastapi import HTTPException
+from pgvector.django import CosineDistance
+from torch import Tensor
 
-from database.models import (
+from khoj.database.models import (
-    KhojUser,
+    ChatModelOptions,
+    Conversation,
+    Entry,
+    GithubConfig,
+    GithubRepoConfig,
     GoogleUser,
     KhojApiUser,
+    KhojUser,
     NotionConfig,
-    GithubConfig,
+    OfflineChatProcessorConversationConfig,
-    Entry,
+    OpenAIProcessorConversationConfig,
-    GithubRepoConfig,
-    Conversation,
-    ChatModelOptions,
     SearchModelConfig,
     Subscription,
     UserConversationConfig,
-    OpenAIProcessorConversationConfig,
-    OfflineChatProcessorConversationConfig,
 )
-from khoj.utils.helpers import generate_random_name
+from khoj.search_filter.date_filter import DateFilter
+from khoj.search_filter.file_filter import FileFilter
+from khoj.search_filter.word_filter import WordFilter
 from khoj.utils import state
 from khoj.utils.config import GPT4AllProcessorModel
-from khoj.search_filter.word_filter import WordFilter
+from khoj.utils.helpers import generate_random_name
-from khoj.search_filter.file_filter import FileFilter
-from khoj.search_filter.date_filter import DateFilter
 
 
 async def set_notion_config(token: str, user: KhojUser):

src/khoj/database/admin.py

@@ -3,7 +3,7 @@ from django.contrib.auth.admin import UserAdmin
 
 # Register your models here.
 
-from database.models import (
+from khoj.database.models import (
     KhojUser,
     ChatModelOptions,
     OpenAIProcessorConversationConfig,

src/khoj/database/apps.py

@@ -3,4 +3,4 @@ from django.apps import AppConfig
 
 class DatabaseConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
-    name = "database"
+    name = "khoj.database"

src/khoj/database/migrations/0011_merge_20231102_0138.py

@@ -2,6 +2,8 @@
 
 from django.db import migrations
 
+from typing import List, Any
+
 
 class Migration(migrations.Migration):
     dependencies = [
@@ -9,4 +11,4 @@ class Migration(migrations.Migration):
         ("database", "0010_rename_embeddings_entry_and_more"),
     ]
 
-    operations = []
+    operations: List[Any] = []

src/khoj/main.py

@@ -1,3 +1,7 @@
+""" Main module for Khoj Assistant
+   isort:skip_file
+"""
+
 # Standard Packages
 import os
 import sys
@@ -25,7 +29,7 @@ from django.core.asgi import get_asgi_application
 from django.core.management import call_command
 
 # Initialize Django
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings")
 django.setup()
 
 # Initialize Django Database

src/khoj/manage.py

@@ -6,7 +6,7 @@ import sys
 
 def main():
     """Run administrative tasks."""
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings")
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:

src/khoj/migrations/migrate_server_pg.py

@@ -60,7 +60,7 @@ import logging
 from packaging import version
 
 from khoj.utils.yaml import load_config_from_file, save_config_to_file
-from database.models import (
+from khoj.database.models import (
     OpenAIProcessorConversationConfig,
     OfflineChatProcessorConversationConfig,
     ChatModelOptions,

src/khoj/processor/data_sources/github/github_to_entries.py

@@ -2,19 +2,20 @@
 import logging
 import time
 from datetime import datetime
-from typing import Dict, List, Union, Tuple
+from typing import Dict, List, Tuple, Union
 
 # External Packages
 import requests
 
-# Internal Packages
+from khoj.database.models import Entry as DbEntry
-from khoj.utils.helpers import timer
+from khoj.database.models import GithubConfig, KhojUser
-from khoj.utils.rawconfig import Entry, GithubContentConfig, GithubRepoConfig
 from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
 from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.text_to_entries import TextToEntries
-from database.models import Entry as DbEntry, GithubConfig, KhojUser
 
+# Internal Packages
+from khoj.utils.helpers import timer
+from khoj.utils.rawconfig import Entry, GithubContentConfig, GithubRepoConfig
 
 logger = logging.getLogger(__name__)
 

src/khoj/processor/data_sources/markdown/markdown_to_entries.py

@@ -1,17 +1,19 @@
 # Standard Packages
 import logging
 import re
-import urllib3
 from pathlib import Path
-from typing import Tuple, List
+from typing import List, Tuple
+
+import urllib3
+
+from khoj.database.models import Entry as DbEntry
+from khoj.database.models import KhojUser
 
 # Internal Packages
 from khoj.processor.text_to_entries import TextToEntries
-from khoj.utils.helpers import timer
 from khoj.utils.constants import empty_escape_sequences
+from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry
-from database.models import Entry as DbEntry, KhojUser
-
 
 logger = logging.getLogger(__name__)
 

src/khoj/processor/data_sources/notion/notion_to_entries.py

@@ -1,19 +1,18 @@
 # Standard Packages
 import logging
+from enum import Enum
 from typing import Tuple
 
 # External Packages
 import requests
 
+from khoj.database.models import Entry as DbEntry
+from khoj.database.models import KhojUser, NotionConfig
+from khoj.processor.text_to_entries import TextToEntries
+
 # Internal Packages
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry, NotionContentConfig
-from khoj.processor.text_to_entries import TextToEntries
-from khoj.utils.rawconfig import Entry
-from database.models import Entry as DbEntry, KhojUser, NotionConfig
-
-from enum import Enum
-
 
 logger = logging.getLogger(__name__)
 

src/khoj/processor/data_sources/org_mode/org_to_entries.py

@@ -3,14 +3,15 @@ import logging
 from pathlib import Path
 from typing import Iterable, List, Tuple
 
+from khoj.database.models import Entry as DbEntry
+from khoj.database.models import KhojUser
+
 # Internal Packages
 from khoj.processor.data_sources.org_mode import orgnode
 from khoj.processor.text_to_entries import TextToEntries
+from khoj.utils import state
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry
-from khoj.utils import state
-from database.models import Entry as DbEntry, KhojUser
-
 
 logger = logging.getLogger(__name__)
 

src/khoj/processor/data_sources/pdf/pdf_to_entries.py

@@ -1,18 +1,19 @@
 # Standard Packages
-import os
-import logging
-from typing import List, Tuple
 import base64
+import logging
+import os
+from typing import List, Tuple
 
 # External Packages
 from langchain.document_loaders import PyMuPDFLoader
 
+from khoj.database.models import Entry as DbEntry
+from khoj.database.models import KhojUser
+
 # Internal Packages
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry
-from database.models import Entry as DbEntry, KhojUser
-
 
 logger = logging.getLogger(__name__)
 

src/khoj/processor/data_sources/plaintext/plaintext_to_entries.py

@@ -2,15 +2,16 @@
 import logging
 from pathlib import Path
 from typing import List, Tuple
+
 from bs4 import BeautifulSoup
 
+from khoj.database.models import Entry as DbEntry
+from khoj.database.models import KhojUser
 
 # Internal Packages
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry
-from database.models import Entry as DbEntry, KhojUser
-
 
 logger = logging.getLogger(__name__)
 

src/khoj/processor/text_to_entries.py

@@ -13,8 +13,8 @@ from khoj.utils.helpers import is_none_or_empty, timer, batcher
 # Internal Packages
 from khoj.utils.rawconfig import Entry
 from khoj.search_filter.date_filter import DateFilter
-from database.models import KhojUser, Entry as DbEntry, EntryDates
+from khoj.database.models import KhojUser, Entry as DbEntry, EntryDates
-from database.adapters import EntryAdapters
+from khoj.database.adapters import EntryAdapters
 
 
 logger = logging.getLogger(__name__)

src/khoj/routers/api.py

@@ -1,64 +1,63 @@
 # Standard Packages
 import concurrent.futures
+import json
+import logging
 import math
 import time
-import logging
+from typing import Any, Dict, List, Optional, Union
-import json
+
-from typing import List, Optional, Union, Any, Dict
+from asgiref.sync import sync_to_async
 
 # External Packages
-from fastapi import APIRouter, Depends, HTTPException, Header, Request
+from fastapi import APIRouter, Depends, Header, HTTPException, Request
+from fastapi.requests import Request
+from fastapi.responses import Response, StreamingResponse
 from starlette.authentication import requires
-from asgiref.sync import sync_to_async
 
 # Internal Packages
 from khoj.configure import configure_server
-from khoj.search_type import image_search, text_search
+from khoj.database import adapters
-from khoj.search_filter.date_filter import DateFilter
+from khoj.database.adapters import ConversationAdapters, EntryAdapters
-from khoj.search_filter.file_filter import FileFilter
+from khoj.database.models import ChatModelOptions
-from khoj.search_filter.word_filter import WordFilter
+from khoj.database.models import Entry as DbEntry
-from khoj.utils.config import TextSearchModel, GPT4AllProcessorModel
+from khoj.database.models import (
-from khoj.utils.helpers import ConversationCommand, is_none_or_empty, timer, command_descriptions
+    GithubConfig,
-from khoj.utils.rawconfig import (
+    KhojUser,
-    FullConfig,
-    SearchConfig,
-    SearchResponse,
-    GithubContentConfig,
-    NotionContentConfig,
-)
-from khoj.utils.state import SearchType
-from khoj.utils import state, constants
-from khoj.utils.helpers import AsyncIteratorWrapper, get_device
-from fastapi.responses import StreamingResponse, Response
-from khoj.routers.helpers import (
-    CommonQueryParams,
-    get_conversation_command,
-    validate_conversation_config,
-    agenerate_chat_response,
-    update_telemetry_state,
-    is_ready_to_chat,
-    ApiUserRateLimiter,
-)
-from khoj.processor.conversation.prompts import help_message, no_entries_found
-from khoj.processor.conversation.openai.gpt import extract_questions
-from khoj.processor.conversation.gpt4all.chat_model import extract_questions_offline
-from khoj.processor.tools.online_search import search_with_google
-from fastapi.requests import Request
-
-from database import adapters
-from database.adapters import EntryAdapters, ConversationAdapters
-from database.models import (
     LocalMarkdownConfig,
     LocalOrgConfig,
     LocalPdfConfig,
     LocalPlaintextConfig,
-    KhojUser,
-    Entry as DbEntry,
-    GithubConfig,
     NotionConfig,
-    ChatModelOptions,
 )
-
+from khoj.processor.conversation.gpt4all.chat_model import extract_questions_offline
+from khoj.processor.conversation.openai.gpt import extract_questions
+from khoj.processor.conversation.prompts import help_message, no_entries_found
+from khoj.processor.tools.online_search import search_with_google
+from khoj.routers.helpers import (
+    ApiUserRateLimiter,
+    CommonQueryParams,
+    agenerate_chat_response,
+    get_conversation_command,
+    is_ready_to_chat,
+    update_telemetry_state,
+    validate_conversation_config,
+)
+from khoj.search_filter.date_filter import DateFilter
+from khoj.search_filter.file_filter import FileFilter
+from khoj.search_filter.word_filter import WordFilter
+from khoj.search_type import image_search, text_search
+from khoj.utils import constants, state
+from khoj.utils.config import GPT4AllProcessorModel, TextSearchModel
+from khoj.utils.helpers import (
+    AsyncIteratorWrapper,
+    ConversationCommand,
+    command_descriptions,
+    get_device,
+    is_none_or_empty,
+    timer,
+)
+from khoj.utils.rawconfig import FullConfig, GithubContentConfig, NotionContentConfig, SearchConfig, SearchResponse
+from khoj.utils.state import SearchType
 
 # Initialize Router
 api = APIRouter()

src/khoj/routers/auth.py

@@ -15,8 +15,8 @@ from google.oauth2 import id_token
 from google.auth.transport import requests as google_requests
 
 # Internal Packages
-from database.adapters import get_khoj_tokens, get_or_create_user, create_khoj_token, delete_khoj_token
+from khoj.database.adapters import get_khoj_tokens, get_or_create_user, create_khoj_token, delete_khoj_token
-from database.models import KhojApiUser
+from khoj.database.models import KhojApiUser
 from khoj.routers.helpers import update_telemetry_state
 from khoj.utils import state
 

src/khoj/routers/helpers.py

@@ -1,31 +1,28 @@
 # Standard Packages
 import asyncio
+import json
+import logging
 from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 from functools import partial
-import logging
 from time import time
-import json
+from typing import Annotated, Any, Dict, Iterator, List, Optional, Tuple, Union
-from typing import Annotated, Iterator, List, Optional, Union, Tuple, Dict, Any
-
-from datetime import datetime
-
-from khoj.processor.conversation import prompts
 
 # External Packages
-from fastapi import HTTPException, Header, Request, Depends
+from fastapi import Depends, Header, HTTPException, Request
+
+from khoj.database.adapters import ConversationAdapters
+from khoj.database.models import KhojUser, Subscription
+from khoj.processor.conversation import prompts
+from khoj.processor.conversation.gpt4all.chat_model import converse_offline, send_message_to_model_offline
+from khoj.processor.conversation.openai.gpt import converse, send_message_to_model
+from khoj.processor.conversation.utils import ThreadedGenerator, message_to_log
 
 # Internal Packages
 from khoj.utils import state
 from khoj.utils.config import GPT4AllProcessorModel
 from khoj.utils.helpers import ConversationCommand, log_telemetry
-from khoj.processor.conversation.openai.gpt import converse, send_message_to_model
-from khoj.processor.conversation.gpt4all.chat_model import converse_offline, send_message_to_model_offline
-from khoj.processor.conversation.utils import message_to_log, ThreadedGenerator
-from database.models import KhojUser, Subscription, ChatModelOptions
-from database.adapters import ConversationAdapters
-
 
 logger = logging.getLogger(__name__)
 

src/khoj/routers/indexer.py

@@ -1,40 +1,26 @@
 # Standard Packages
-import logging
-from typing import Optional, Union, Dict
 import asyncio
+import logging
+from typing import Dict, Optional, Union
 
-# External Packages
 from fastapi import APIRouter, Header, Request, Response, UploadFile
 from pydantic import BaseModel
 from starlette.authentication import requires
 
-# Internal Packages
+from khoj.database.models import GithubConfig, KhojUser, NotionConfig
-from khoj.utils import state, constants
+from khoj.processor.data_sources.github.github_to_entries import GithubToEntries
 from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.data_sources.notion.notion_to_entries import NotionToEntries
 from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries
-from khoj.processor.data_sources.github.github_to_entries import GithubToEntries
-from khoj.processor.data_sources.notion.notion_to_entries import NotionToEntries
 from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
-from khoj.search_type import text_search, image_search
 from khoj.routers.helpers import update_telemetry_state
-from khoj.utils.yaml import save_config_to_file_updated_state
+from khoj.search_type import image_search, text_search
-from khoj.utils.config import SearchModels
+from khoj.utils import constants, state
+from khoj.utils.config import ContentIndex, SearchModels
 from khoj.utils.helpers import LRU, get_file_type
-from khoj.utils.rawconfig import (
+from khoj.utils.rawconfig import ContentConfig, FullConfig, SearchConfig
-    ContentConfig,
+from khoj.utils.yaml import save_config_to_file_updated_state
-    FullConfig,
-    SearchConfig,
-)
-from khoj.utils.config import (
-    ContentIndex,
-    SearchModels,
-)
-from database.models import (
-    KhojUser,
-    GithubConfig,
-    NotionConfig,
-)
 
 logger = logging.getLogger(__name__)
 

src/khoj/routers/subscription.py

@@ -10,7 +10,7 @@ from starlette.authentication import requires
 import stripe
 
 # Internal Packages
-from database import adapters
+from khoj.database import adapters
 
 
 # Stripe integration for Khoj Cloud Subscription

src/khoj/routers/web_client.py

@@ -8,8 +8,8 @@ from fastapi import Request
 from fastapi.responses import HTMLResponse, FileResponse, RedirectResponse
 from fastapi.templating import Jinja2Templates
 from starlette.authentication import requires
-from database import adapters
+from khoj.database import adapters
-from database.models import KhojUser
+from khoj.database.models import KhojUser
 from khoj.utils.rawconfig import (
     GithubContentConfig,
     GithubRepoConfig,
@@ -18,7 +18,7 @@ from khoj.utils.rawconfig import (
 
 # Internal Packages
 from khoj.utils import constants, state
-from database.adapters import (
+from khoj.database.adapters import (
     EntryAdapters,
     get_user_github_config,
     get_user_notion_config,

src/khoj/search_type/text_search.py

@@ -19,8 +19,8 @@ from khoj.utils.state import SearchType
 from khoj.utils.rawconfig import SearchResponse, Entry
 from khoj.utils.jsonl import load_jsonl
 from khoj.processor.text_to_entries import TextToEntries
-from database.adapters import EntryAdapters
+from khoj.database.adapters import EntryAdapters
-from database.models import KhojUser, Entry as DbEntry
+from khoj.database.models import KhojUser, Entry as DbEntry
 
 logger = logging.getLogger(__name__)
 

src/khoj/utils/constants.py

@@ -7,6 +7,7 @@ app_env_filepath = "~/.khoj/env"
 telemetry_server = "https://khoj.beta.haletic.com/v1/telemetry"
 content_directory = "~/.khoj/content/"
 default_offline_chat_model = "mistral-7b-instruct-v0.1.Q4_0.gguf"
+default_online_chat_model = "gpt-4"
 
 empty_config = {
     "search-type": {

src/khoj/utils/fs_syncer.py

@@ -7,7 +7,7 @@ from bs4 import BeautifulSoup
 from khoj.utils.helpers import get_absolute_path, is_none_or_empty
 from khoj.utils.rawconfig import TextContentConfig
 from khoj.utils.config import SearchType
-from database.models import LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, LocalPlaintextConfig
+from khoj.database.models import LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, LocalPlaintextConfig
 
 logger = logging.getLogger(__name__)
 

src/khoj/utils/initialization.py

@@ -1,16 +1,17 @@
 import logging
 import os
 
-from database.models import (
+from khoj.database.models import (
     KhojUser,
     OfflineChatProcessorConversationConfig,
     OpenAIProcessorConversationConfig,
     ChatModelOptions,
 )
 
-from khoj.utils.constants import default_offline_chat_model
+from khoj.utils.constants import default_offline_chat_model, default_online_chat_model
+from khoj.processor.conversation.utils import model_to_prompt_size, model_to_tokenizer
 
-from database.adapters import ConversationAdapters
+from khoj.database.adapters import ConversationAdapters
 
 
 logger = logging.getLogger(__name__)
@@ -30,11 +31,6 @@ def initialization():
         logger.info(
             "🗣️  Configure chat models available to your server. You can always update these at /server/admin using the credentials of your admin account"
         )
-        try:
-            # Some environments don't support interactive input. We catch the exception and return if that's the case. The admin can still configure their settings from the admin page.
-            input()
-        except EOFError:
-            return
 
         try:
             # Note: gpt4all package is not available on all devices.
@@ -47,15 +43,27 @@ def initialization():
                 OfflineChatProcessorConversationConfig.objects.create(enabled=True)
 
                 offline_chat_model = input(
-                    f"Enter the name of the offline chat model you want to use, based on the models in HuggingFace (press enter to use the default: {default_offline_chat_model}): "
+                    f"Enter the offline chat model you want to use, See GPT4All for supported models (default: {default_offline_chat_model}): "
                 )
                 if offline_chat_model == "":
                     ChatModelOptions.objects.create(
                         chat_model=default_offline_chat_model, model_type=ChatModelOptions.ModelType.OFFLINE
                     )
                 else:
-                    max_tokens = input("Enter the maximum number of tokens to use for the offline chat model:")
+                    default_max_tokens = model_to_prompt_size.get(offline_chat_model, 2000)
-                    tokenizer = input("Enter the tokenizer to use for the offline chat model:")
+                    max_tokens = input(
+                        f"Enter the maximum number of tokens to use for the offline chat model (default {default_max_tokens}):"
+                    )
+                    max_tokens = max_tokens or default_max_tokens
+
+                    default_tokenizer = model_to_tokenizer.get(
+                        offline_chat_model, "hf-internal-testing/llama-tokenizer"
+                    )
+                    tokenizer = input(
+                        f"Enter the tokenizer to use for the offline chat model (default: {default_tokenizer}):"
+                    )
+                    tokenizer = tokenizer or default_tokenizer
+
                     ChatModelOptions.objects.create(
                         chat_model=offline_chat_model,
                         model_type=ChatModelOptions.ModelType.OFFLINE,
@@ -71,10 +79,19 @@ def initialization():
             logger.info("🗣️ Setting up OpenAI chat model")
             api_key = input("Enter your OpenAI API key: ")
             OpenAIProcessorConversationConfig.objects.create(api_key=api_key)
-            openai_chat_model = input("Enter the name of the OpenAI chat model you want to use: ")
+
-            max_tokens = input("Enter the maximum number of tokens to use for the OpenAI chat model:")
+            openai_chat_model = input(
+                f"Enter the OpenAI chat model you want to use (default: {default_online_chat_model}): "
+            )
+            openai_chat_model = openai_chat_model or default_online_chat_model
+
+            default_max_tokens = model_to_prompt_size.get(openai_chat_model, 2000)
+            max_tokens = input(
+                f"Enter the maximum number of tokens to use for the OpenAI chat model (default: {default_max_tokens}): "
+            )
+            max_tokens = max_tokens or default_max_tokens
             ChatModelOptions.objects.create(
-                chat_model=openai_chat_model, model_type=ChatModelOptions.ModelType.OPENAI, max_tokens=max_tokens
+                chat_model=openai_chat_model, model_type=ChatModelOptions.ModelType.OPENAI, max_prompt_size=max_tokens
             )
 
         logger.info("🗣️  Chat model configuration complete")
@@ -94,5 +111,8 @@ def initialization():
             try:
                 _create_chat_configuration()
                 break
+            # Some environments don't support interactive input. We catch the exception and return if that's the case. The admin can still configure their settings from the admin page.
+            except EOFError:
+                return
             except Exception as e:
                 logger.error(f"🚨 Failed to create chat configuration: {e}", exc_info=True)

tests/conftest.py

@@ -1,48 +1,40 @@
 # External Packages
 import os
-from fastapi.testclient import TestClient
 from pathlib import Path
-import pytest
-from fastapi.staticfiles import StaticFiles
-from fastapi import FastAPI
-import os
-from fastapi import FastAPI
 
+import pytest
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from fastapi.testclient import TestClient
 
 # Internal Packages
-from khoj.configure import configure_routes, configure_search_types, configure_middleware
+from khoj.configure import configure_middleware, configure_routes, configure_search_types
-from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
+from khoj.database.models import (
+    GithubConfig,
+    GithubRepoConfig,
+    KhojApiUser,
+    KhojUser,
+    LocalMarkdownConfig,
+    LocalOrgConfig,
+    LocalPlaintextConfig,
+)
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
+from khoj.routers.indexer import configure_content
 from khoj.search_type import image_search, text_search
+from khoj.utils import fs_syncer, state
 from khoj.utils.config import SearchModels
 from khoj.utils.constants import web_directory
 from khoj.utils.helpers import resolve_absolute_path
-from khoj.utils.rawconfig import (
+from khoj.utils.rawconfig import ContentConfig, ImageContentConfig, ImageSearchConfig, SearchConfig
-    ContentConfig,
-    ImageContentConfig,
-    SearchConfig,
-    ImageSearchConfig,
-)
-from khoj.utils import state, fs_syncer
-from khoj.routers.indexer import configure_content
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
-from database.models import (
-    KhojApiUser,
-    LocalOrgConfig,
-    LocalMarkdownConfig,
-    LocalPlaintextConfig,
-    GithubConfig,
-    KhojUser,
-    GithubRepoConfig,
-)
-
 from tests.helpers import (
-    UserFactory,
     ChatModelOptionsFactory,
-    OpenAIProcessorConversationConfigFactory,
     OfflineChatProcessorConversationConfigFactory,
-    UserConversationProcessorConfigFactory,
+    OpenAIProcessorConversationConfigFactory,
     SubscriptionFactory,
+    UserConversationProcessorConfigFactory,
+    UserFactory,
 )
 
 

tests/helpers.py

@@ -1,7 +1,7 @@
 import factory
 import os
 
-from database.models import (
+from khoj.database.models import (
     KhojUser,
     KhojApiUser,
     ChatModelOptions,

tests/test_client.py

@@ -1,23 +1,23 @@
 # Standard Modules
 from io import BytesIO
-from PIL import Image
 from urllib.parse import quote
+
 import pytest
+from fastapi import FastAPI
 
 # External Packages
 from fastapi.testclient import TestClient
-from fastapi import FastAPI
+from PIL import Image
-import pytest
 
 # Internal Packages
 from khoj.configure import configure_routes, configure_search_types
-from khoj.utils import state
+from khoj.database.adapters import EntryAdapters
-from khoj.utils.state import search_models, content_index, config
+from khoj.database.models import KhojApiUser, KhojUser
-from khoj.search_type import text_search, image_search
-from khoj.utils.rawconfig import ContentConfig, SearchConfig
 from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
-from database.models import KhojUser, KhojApiUser
+from khoj.search_type import image_search, text_search
-from database.adapters import EntryAdapters
+from khoj.utils import state
+from khoj.utils.rawconfig import ContentConfig, SearchConfig
+from khoj.utils.state import config, content_index, search_models
 
 
 # Test

tests/test_multiple_users.py

@@ -1,24 +1,14 @@
 # Standard Modules
-from io import BytesIO
-from PIL import Image
 from urllib.parse import quote
-import pytest
 
 # External Packages
-from fastapi.testclient import TestClient
-from fastapi import FastAPI, UploadFile
-from io import BytesIO
 import pytest
 
-# Internal Packages
+from khoj.database.models import KhojApiUser, KhojUser
-from khoj.configure import configure_routes, configure_search_types
-from khoj.utils import state
-from khoj.utils.state import search_models, content_index, config
-from khoj.search_type import text_search, image_search
-from khoj.utils.rawconfig import ContentConfig, SearchConfig
 from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
-from database.models import KhojUser, KhojApiUser
+
-from database.adapters import EntryAdapters
+# Internal Packages
+from khoj.search_type import text_search
 
 
 # ----------------------------------------------------------------------------------------------------

tests/test_openai_chat_director.py

@@ -10,7 +10,7 @@ from khoj.processor.conversation import prompts
 # Internal Packages
 from khoj.processor.conversation.utils import message_to_log
 from tests.helpers import ConversationFactory
-from database.models import KhojUser
+from khoj.database.models import KhojUser
 
 # Initialize variables for tests
 api_key = os.getenv("OPENAI_API_KEY")

tests/test_plaintext_to_entries.py

@@ -3,11 +3,12 @@ import json
 import os
 from pathlib import Path
 
+from khoj.database.models import KhojUser, LocalPlaintextConfig
+from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
+
 # Internal Packages
 from khoj.utils.fs_syncer import get_plaintext_files
 from khoj.utils.rawconfig import TextContentConfig
-from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
-from database.models import LocalPlaintextConfig, KhojUser
 
 
 def test_plaintext_file(tmp_path):

tests/test_text_search.py

@@ -1,19 +1,20 @@
 # System Packages
-import logging
-from pathlib import Path
-import os
 import asyncio
+import logging
+import os
+from pathlib import Path
 
 # External Packages
 import pytest
 
+from khoj.database.models import Entry, GithubConfig, KhojUser, LocalOrgConfig
+from khoj.processor.data_sources.github.github_to_entries import GithubToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+
 # Internal Packages
 from khoj.search_type import text_search
-from khoj.utils.rawconfig import ContentConfig, SearchConfig
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
-from khoj.processor.data_sources.github.github_to_entries import GithubToEntries
 from khoj.utils.fs_syncer import collect_files, get_org_files
-from database.models import LocalOrgConfig, KhojUser, Entry, GithubConfig
+from khoj.utils.rawconfig import ContentConfig, SearchConfig
 
 logger = logging.getLogger(__name__)