Move the django app into the src/khoj folder for better organization and functionality

- Our pypi package currently does not work because the django app and associated database is not included. To remedy this issue, move the app into the src/khoj folder. This has the added benefit of improved organization of the codebase, as all server related code is now in a single folder
- Update associated file paths and system references

Dockerfile

@@ -17,7 +17,7 @@ RUN sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && \
 COPY . .
 
 # Set the PYTHONPATH environment variable in order for it to find the Django app.
-ENV PYTHONPATH=/app/src:$PYTHONPATH
+ENV PYTHONPATH=/app/src/khoj:$PYTHONPATH
 
 # Run the Application
 # There are more arguments required for the application to run,

prod.Dockerfile

@@ -20,7 +20,7 @@ COPY . .
 RUN apt install vim -y
 
 # Set the PYTHONPATH environment variable in order for it to find the Django app.
-ENV PYTHONPATH=/app/src:$PYTHONPATH
+ENV PYTHONPATH=/app/src/khoj:$PYTHONPATH
 
 # Run the Application
 # There are more arguments required for the application to run,

pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
-DJANGO_SETTINGS_MODULE = app.settings
+DJANGO_SETTINGS_MODULE = khoj.app.settings
 pythonpath = . src
 testpaths = tests
 markers =

src/khoj/app/settings.py

@@ -14,7 +14,7 @@ from pathlib import Path
 import os
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
-BASE_DIR = Path(__file__).resolve().parent.parent.parent
+BASE_DIR = Path(__file__).resolve().parent.parent.parent.parent
 
 
 # Quick-start development settings - unsuitable for production
@@ -53,7 +53,7 @@ SESSION_COOKIE_SAMESITE = "None"
 INSTALLED_APPS = [
     "django.contrib.auth",
     "django.contrib.contenttypes",
-    "database.apps.DatabaseConfig",
+    "khoj.database.apps.DatabaseConfig",
     "django.contrib.admin",
     "django.contrib.sessions",
     "django.contrib.messages",

src/khoj/configure.py

@@ -20,8 +20,8 @@ from starlette.authentication import (
 )
 
 # Internal Packages
-from database.models import KhojUser, Subscription
+from khoj.database.models import KhojUser, Subscription
-from database.adapters import get_all_users, get_or_create_search_model
+from khoj.database.adapters import get_all_users, get_or_create_search_model
 from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
 from khoj.routers.indexer import configure_content, load_content, configure_search
 from khoj.utils import constants, state
@@ -45,7 +45,7 @@ class UserAuthenticationBackend(AuthenticationBackend):
     def __init__(
         self,
     ):
-        from database.models import KhojUser, KhojApiUser
+        from khoj.database.models import KhojUser, KhojApiUser
 
         self.khojuser_manager = KhojUser.objects
         self.khojapiuser_manager = KhojApiUser.objects

src/khoj/database/adapters/__init__.py

@@ -17,7 +17,7 @@ from asgiref.sync import sync_to_async
 
 from fastapi import HTTPException
 
-from database.models import (
+from khoj.database.models import (
     KhojUser,
     GoogleUser,
     KhojApiUser,

src/khoj/database/admin.py

@@ -3,7 +3,7 @@ from django.contrib.auth.admin import UserAdmin
 
 # Register your models here.
 
-from database.models import (
+from khoj.database.models import (
     KhojUser,
     ChatModelOptions,
     OpenAIProcessorConversationConfig,

src/khoj/database/apps.py

@@ -3,4 +3,4 @@ from django.apps import AppConfig
 
 class DatabaseConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
-    name = "database"
+    name = "khoj.database"

src/khoj/main.py

@@ -26,7 +26,7 @@ from django.core.management import call_command
 
 # Initialize Django
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings")
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings")
 django.setup()
 
 # Initialize Django Database

src/khoj/manage.py

@@ -6,7 +6,7 @@ import sys
 
 def main():
     """Run administrative tasks."""
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "app.settings")
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings")
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:

src/khoj/migrations/migrate_server_pg.py

@@ -60,7 +60,7 @@ import logging
 from packaging import version
 
 from khoj.utils.yaml import load_config_from_file, save_config_to_file
-from database.models import (
+from khoj.database.models import (
     OpenAIProcessorConversationConfig,
     OfflineChatProcessorConversationConfig,
     ChatModelOptions,

src/khoj/processor/github/github_to_entries.py

@@ -13,7 +13,7 @@ from khoj.utils.rawconfig import Entry, GithubContentConfig, GithubRepoConfig
 from khoj.processor.markdown.markdown_to_entries import MarkdownToEntries
 from khoj.processor.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.text_to_entries import TextToEntries
-from database.models import Entry as DbEntry, GithubConfig, KhojUser
+from khoj.database.models import Entry as DbEntry, GithubConfig, KhojUser
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/markdown/markdown_to_entries.py

@@ -10,7 +10,7 @@ from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.helpers import timer
 from khoj.utils.constants import empty_escape_sequences
 from khoj.utils.rawconfig import Entry
-from database.models import Entry as DbEntry, KhojUser
+from khoj.database.models import Entry as DbEntry, KhojUser
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/notion/notion_to_entries.py

@@ -10,7 +10,7 @@ from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry, NotionContentConfig
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.rawconfig import Entry
-from database.models import Entry as DbEntry, KhojUser, NotionConfig
+from khoj.database.models import Entry as DbEntry, KhojUser, NotionConfig
 
 from enum import Enum
 

src/khoj/processor/org_mode/org_to_entries.py

@@ -9,7 +9,7 @@ from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry
 from khoj.utils import state
-from database.models import Entry as DbEntry, KhojUser
+from khoj.database.models import Entry as DbEntry, KhojUser
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/pdf/pdf_to_entries.py

@@ -11,7 +11,7 @@ from langchain.document_loaders import PyMuPDFLoader
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry
-from database.models import Entry as DbEntry, KhojUser
+from khoj.database.models import Entry as DbEntry, KhojUser
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/plaintext/plaintext_to_entries.py

@@ -9,7 +9,7 @@ from bs4 import BeautifulSoup
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry
-from database.models import Entry as DbEntry, KhojUser
+from khoj.database.models import Entry as DbEntry, KhojUser
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/text_to_entries.py

@@ -13,8 +13,8 @@ from khoj.utils.helpers import is_none_or_empty, timer, batcher
 # Internal Packages
 from khoj.utils.rawconfig import Entry
 from khoj.search_filter.date_filter import DateFilter
-from database.models import KhojUser, Entry as DbEntry, EntryDates
+from khoj.database.models import KhojUser, Entry as DbEntry, EntryDates
-from database.adapters import EntryAdapters
+from khoj.database.adapters import EntryAdapters
 
 
 logger = logging.getLogger(__name__)

src/khoj/routers/api.py

@@ -44,9 +44,9 @@ from khoj.processor.conversation.openai.gpt import extract_questions
 from khoj.processor.conversation.gpt4all.chat_model import extract_questions_offline
 from fastapi.requests import Request
 
-from database import adapters
+from khoj.database import adapters
-from database.adapters import EntryAdapters, ConversationAdapters
+from khoj.database.adapters import EntryAdapters, ConversationAdapters
-from database.models import (
+from khoj.database.models import (
     LocalMarkdownConfig,
     LocalOrgConfig,
     LocalPdfConfig,

src/khoj/routers/auth.py

@@ -15,8 +15,8 @@ from google.oauth2 import id_token
 from google.auth.transport import requests as google_requests
 
 # Internal Packages
-from database.adapters import get_khoj_tokens, get_or_create_user, create_khoj_token, delete_khoj_token
+from khoj.database.adapters import get_khoj_tokens, get_or_create_user, create_khoj_token, delete_khoj_token
-from database.models import KhojApiUser
+from khoj.database.models import KhojApiUser
 from khoj.routers.helpers import update_telemetry_state
 from khoj.utils import state
 

src/khoj/routers/helpers.py

@@ -18,8 +18,8 @@ from khoj.utils.helpers import ConversationCommand, log_telemetry
 from khoj.processor.conversation.openai.gpt import converse
 from khoj.processor.conversation.gpt4all.chat_model import converse_offline
 from khoj.processor.conversation.utils import message_to_log, ThreadedGenerator
-from database.models import KhojUser, Subscription
+from khoj.database.models import KhojUser, Subscription
-from database.adapters import ConversationAdapters
+from khoj.database.adapters import ConversationAdapters
 
 
 logger = logging.getLogger(__name__)

src/khoj/routers/indexer.py

@@ -30,7 +30,7 @@ from khoj.utils.config import (
     ContentIndex,
     SearchModels,
 )
-from database.models import (
+from khoj.database.models import (
     KhojUser,
     GithubConfig,
     NotionConfig,

src/khoj/routers/subscription.py

@@ -10,7 +10,7 @@ from starlette.authentication import requires
 import stripe
 
 # Internal Packages
-from database import adapters
+from khoj.database import adapters
 
 
 # Stripe integration for Khoj Cloud Subscription

src/khoj/routers/web_client.py

@@ -8,8 +8,8 @@ from fastapi import Request
 from fastapi.responses import HTMLResponse, FileResponse, RedirectResponse
 from fastapi.templating import Jinja2Templates
 from starlette.authentication import requires
-from database import adapters
+from khoj.database import adapters
-from database.models import KhojUser
+from khoj.database.models import KhojUser
 from khoj.utils.rawconfig import (
     GithubContentConfig,
     GithubRepoConfig,
@@ -18,7 +18,7 @@ from khoj.utils.rawconfig import (
 
 # Internal Packages
 from khoj.utils import constants, state
-from database.adapters import (
+from khoj.database.adapters import (
     EntryAdapters,
     get_user_github_config,
     get_user_notion_config,

src/khoj/search_type/text_search.py

@@ -19,8 +19,8 @@ from khoj.utils.state import SearchType
 from khoj.utils.rawconfig import SearchResponse, Entry
 from khoj.utils.jsonl import load_jsonl
 from khoj.processor.text_to_entries import TextToEntries
-from database.adapters import EntryAdapters
+from khoj.database.adapters import EntryAdapters
-from database.models import KhojUser, Entry as DbEntry
+from khoj.database.models import KhojUser, Entry as DbEntry
 
 logger = logging.getLogger(__name__)
 

src/khoj/utils/fs_syncer.py

@@ -7,7 +7,7 @@ from bs4 import BeautifulSoup
 from khoj.utils.helpers import get_absolute_path, is_none_or_empty
 from khoj.utils.rawconfig import TextContentConfig
 from khoj.utils.config import SearchType
-from database.models import LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, LocalPlaintextConfig
+from khoj.database.models import LocalMarkdownConfig, LocalOrgConfig, LocalPdfConfig, LocalPlaintextConfig
 
 logger = logging.getLogger(__name__)
 

src/khoj/utils/initialization.py

@@ -1,7 +1,7 @@
 import logging
 import os
 
-from database.models import (
+from khoj.database.models import (
     KhojUser,
     OfflineChatProcessorConversationConfig,
     OpenAIProcessorConversationConfig,
@@ -11,7 +11,7 @@ from database.models import (
 from khoj.utils.constants import default_offline_chat_model, default_online_chat_model
 from khoj.processor.conversation.utils import model_to_prompt_size, model_to_tokenizer
 
-from database.adapters import ConversationAdapters
+from khoj.database.adapters import ConversationAdapters
 
 
 logger = logging.getLogger(__name__)

tests/conftest.py

@@ -26,7 +26,7 @@ from khoj.utils.rawconfig import (
 from khoj.utils import state, fs_syncer
 from khoj.routers.indexer import configure_content
 from khoj.processor.org_mode.org_to_entries import OrgToEntries
-from database.models import (
+from khoj.database.models import (
     KhojApiUser,
     LocalOrgConfig,
     LocalMarkdownConfig,

tests/helpers.py

@@ -1,7 +1,7 @@
 import factory
 import os
 
-from database.models import (
+from khoj.database.models import (
     KhojUser,
     KhojApiUser,
     ChatModelOptions,

tests/test_client.py

@@ -16,8 +16,8 @@ from khoj.utils.state import search_models, content_index, config
 from khoj.search_type import text_search, image_search
 from khoj.utils.rawconfig import ContentConfig, SearchConfig
 from khoj.processor.org_mode.org_to_entries import OrgToEntries
-from database.models import KhojUser, KhojApiUser
+from khoj.database.models import KhojUser, KhojApiUser
-from database.adapters import EntryAdapters
+from khoj.database.adapters import EntryAdapters
 
 
 # Test

tests/test_multiple_users.py

@@ -17,8 +17,8 @@ from khoj.utils.state import search_models, content_index, config
 from khoj.search_type import text_search, image_search
 from khoj.utils.rawconfig import ContentConfig, SearchConfig
 from khoj.processor.org_mode.org_to_entries import OrgToEntries
-from database.models import KhojUser, KhojApiUser
+from khoj.database.models import KhojUser, KhojApiUser
-from database.adapters import EntryAdapters
+from khoj.database.adapters import EntryAdapters
 
 
 # ----------------------------------------------------------------------------------------------------

tests/test_openai_chat_director.py

@@ -10,7 +10,7 @@ from khoj.processor.conversation import prompts
 # Internal Packages
 from khoj.processor.conversation.utils import message_to_log
 from tests.helpers import ConversationFactory
-from database.models import KhojUser
+from khoj.database.models import KhojUser
 
 # Initialize variables for tests
 api_key = os.getenv("OPENAI_API_KEY")

tests/test_plaintext_to_entries.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from khoj.utils.fs_syncer import get_plaintext_files
 from khoj.utils.rawconfig import TextContentConfig
 from khoj.processor.plaintext.plaintext_to_entries import PlaintextToEntries
-from database.models import LocalPlaintextConfig, KhojUser
+from khoj.database.models import LocalPlaintextConfig, KhojUser
 
 
 def test_plaintext_file(tmp_path):

tests/test_text_search.py

@@ -13,7 +13,7 @@ from khoj.utils.rawconfig import ContentConfig, SearchConfig
 from khoj.processor.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.github.github_to_entries import GithubToEntries
 from khoj.utils.fs_syncer import collect_files, get_org_files
-from database.models import LocalOrgConfig, KhojUser, Entry, GithubConfig
+from khoj.database.models import LocalOrgConfig, KhojUser, Entry, GithubConfig
 
 logger = logging.getLogger(__name__)