Move data indexer files into a separate folder under processor. Update assoc UTs

src/khoj/processor/data_sources/github/github_to_entries.py

@@ -10,8 +10,8 @@ import requests
 # Internal Packages
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry, GithubContentConfig, GithubRepoConfig
-from khoj.processor.markdown.markdown_to_entries import MarkdownToEntries
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.text_to_entries import TextToEntries
 from database.models import Entry as DbEntry, GithubConfig, KhojUser
 

src/khoj/processor/data_sources/org_mode/org_to_entries.py

@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Iterable, List, Tuple
 
 # Internal Packages
-from khoj.processor.org_mode import orgnode
+from khoj.processor.data_sources.org_mode import orgnode
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.helpers import timer
 from khoj.utils.rawconfig import Entry

src/khoj/routers/indexer.py

@@ -10,12 +10,12 @@ from starlette.authentication import requires
 
 # Internal Packages
 from khoj.utils import state, constants
-from khoj.processor.markdown.markdown_to_entries import MarkdownToEntries
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
-from khoj.processor.pdf.pdf_to_entries import PdfToEntries
-from khoj.processor.github.github_to_entries import GithubToEntries
-from khoj.processor.notion.notion_to_entries import NotionToEntries
-from khoj.processor.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries
+from khoj.processor.data_sources.github.github_to_entries import GithubToEntries
+from khoj.processor.data_sources.notion.notion_to_entries import NotionToEntries
+from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
 from khoj.search_type import text_search, image_search
 from khoj.routers.helpers import update_telemetry_state
 from khoj.utils.yaml import save_config_to_file_updated_state

src/khoj/utils/helpers.py

@@ -272,6 +272,7 @@ class ConversationCommand(str, Enum):
     General = "general"
     Notes = "notes"
     Help = "help"
+    Online = "online"
 
 
 command_descriptions = {

tests/conftest.py

@@ -15,7 +15,7 @@ app = FastAPI()
 # Internal Packages
 from khoj.configure import configure_routes, configure_search_types, configure_middleware
 from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
-from khoj.processor.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
 from khoj.search_type import image_search, text_search
 from khoj.utils.config import SearchModels
 from khoj.utils.constants import web_directory
@@ -28,7 +28,7 @@ from khoj.utils.rawconfig import (
 )
 from khoj.utils import state, fs_syncer
 from khoj.routers.indexer import configure_content
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from database.models import (
     KhojApiUser,
     LocalOrgConfig,

tests/data/config.yml

@@ -14,4 +14,4 @@ search-type:
   asymmetric:
     cross-encoder: cross-encoder/ms-marco-MiniLM-L-6-v2
     encoder: sentence-transformers/msmarco-MiniLM-L-6-v3
-version: 0.14.0
+version: 0.15.0

tests/test_client.py

@@ -15,7 +15,7 @@ from khoj.utils import state
 from khoj.utils.state import search_models, content_index, config
 from khoj.search_type import text_search, image_search
 from khoj.utils.rawconfig import ContentConfig, SearchConfig
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from database.models import KhojUser, KhojApiUser
 from database.adapters import EntryAdapters
 

tests/test_markdown_to_entries.py

@@ -4,7 +4,7 @@ from pathlib import Path
 import os
 
 # Internal Packages
-from khoj.processor.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
 from khoj.utils.fs_syncer import get_markdown_files
 from khoj.utils.rawconfig import TextContentConfig
 

tests/test_multiple_users.py

@@ -16,7 +16,7 @@ from khoj.utils import state
 from khoj.utils.state import search_models, content_index, config
 from khoj.search_type import text_search, image_search
 from khoj.utils.rawconfig import ContentConfig, SearchConfig
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from database.models import KhojUser, KhojApiUser
 from database.adapters import EntryAdapters
 

tests/test_org_to_entries.py

@@ -3,7 +3,7 @@ import json
 import os
 
 # Internal Packages
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.helpers import is_none_or_empty
 from khoj.utils.rawconfig import Entry

tests/test_orgnode.py

@@ -2,7 +2,7 @@
 import datetime
 
 # Internal Packages
-from khoj.processor.org_mode import orgnode
+from khoj.processor.data_sources.org_mode import orgnode
 
 
 # Test

tests/test_pdf_to_entries.py

@@ -3,7 +3,7 @@ import json
 import os
 
 # Internal Packages
-from khoj.processor.pdf.pdf_to_entries import PdfToEntries
+from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries
 
 from khoj.utils.fs_syncer import get_pdf_files
 from khoj.utils.rawconfig import TextContentConfig

tests/test_plaintext_to_entries.py

@@ -6,7 +6,7 @@ from pathlib import Path
 # Internal Packages
 from khoj.utils.fs_syncer import get_plaintext_files
 from khoj.utils.rawconfig import TextContentConfig
-from khoj.processor.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
 from database.models import LocalPlaintextConfig, KhojUser
 
 

tests/test_text_search.py

@@ -10,8 +10,8 @@ import pytest
 # Internal Packages
 from khoj.search_type import text_search
 from khoj.utils.rawconfig import ContentConfig, SearchConfig
-from khoj.processor.org_mode.org_to_entries import OrgToEntries
-from khoj.processor.github.github_to_entries import GithubToEntries
+from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.data_sources.github.github_to_entries import GithubToEntries
 from khoj.utils.fs_syncer import collect_files, get_org_files
 from database.models import LocalOrgConfig, KhojUser, Entry, GithubConfig