Rename the data_sources module to content

src/khoj/processor/content/github/github_to_entries.py

@@ -9,8 +9,8 @@ import requests
 
 from khoj.database.models import Entry as DbEntry
 from khoj.database.models import GithubConfig, KhojUser
-from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.text_to_entries import TextToEntries
 
 # Internal Packages

src/khoj/processor/content/org_mode/org_to_entries.py

@@ -7,7 +7,7 @@ from khoj.database.models import Entry as DbEntry
 from khoj.database.models import KhojUser
 
 # Internal Packages
-from khoj.processor.data_sources.org_mode import orgnode
+from khoj.processor.content.org_mode import orgnode
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils import state
 from khoj.utils.helpers import timer

src/khoj/routers/indexer.py

@@ -7,12 +7,12 @@ from pydantic import BaseModel
 from starlette.authentication import requires
 
 from khoj.database.models import GithubConfig, KhojUser, NotionConfig
-from khoj.processor.data_sources.github.github_to_entries import GithubToEntries
-from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
-from khoj.processor.data_sources.notion.notion_to_entries import NotionToEntries
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
-from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries
-from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.content.github.github_to_entries import GithubToEntries
+from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.content.notion.notion_to_entries import NotionToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries
+from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries
 from khoj.routers.helpers import update_telemetry_state
 from khoj.search_type import image_search, text_search
 from khoj.utils import constants, state

tests/conftest.py

@@ -18,8 +18,8 @@ from khoj.database.models import (
     LocalOrgConfig,
     LocalPlaintextConfig,
 )
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
-from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries
 from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
 from khoj.routers.indexer import configure_content
 from khoj.search_type import image_search, text_search

tests/test_client.py

@@ -13,7 +13,7 @@ from PIL import Image
 from khoj.configure import configure_routes, configure_search_types
 from khoj.database.adapters import EntryAdapters
 from khoj.database.models import KhojApiUser, KhojUser
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 from khoj.search_type import image_search, text_search
 from khoj.utils import state
 from khoj.utils.rawconfig import ContentConfig, SearchConfig

tests/test_markdown_to_entries.py

@@ -4,7 +4,7 @@ from pathlib import Path
 import os
 
 # Internal Packages
-from khoj.processor.data_sources.markdown.markdown_to_entries import MarkdownToEntries
+from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries
 from khoj.utils.fs_syncer import get_markdown_files
 from khoj.utils.rawconfig import TextContentConfig
 

tests/test_multiple_users.py

@@ -5,7 +5,7 @@ from urllib.parse import quote
 import pytest
 
 from khoj.database.models import KhojApiUser, KhojUser
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 
 # Internal Packages
 from khoj.search_type import text_search

tests/test_org_to_entries.py

@@ -3,7 +3,7 @@ import json
 import os
 
 # Internal Packages
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 from khoj.processor.text_to_entries import TextToEntries
 from khoj.utils.helpers import is_none_or_empty
 from khoj.utils.rawconfig import Entry

tests/test_orgnode.py

@@ -2,7 +2,7 @@
 import datetime
 
 # Internal Packages
-from khoj.processor.data_sources.org_mode import orgnode
+from khoj.processor.content.org_mode import orgnode
 
 
 # Test

tests/test_pdf_to_entries.py

@@ -3,7 +3,7 @@ import json
 import os
 
 # Internal Packages
-from khoj.processor.data_sources.pdf.pdf_to_entries import PdfToEntries
+from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries
 
 from khoj.utils.fs_syncer import get_pdf_files
 from khoj.utils.rawconfig import TextContentConfig

tests/test_plaintext_to_entries.py

@@ -4,7 +4,7 @@ import os
 from pathlib import Path
 
 from khoj.database.models import KhojUser, LocalPlaintextConfig
-from khoj.processor.data_sources.plaintext.plaintext_to_entries import PlaintextToEntries
+from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries
 
 # Internal Packages
 from khoj.utils.fs_syncer import get_plaintext_files

tests/test_text_search.py

@@ -8,8 +8,8 @@ from pathlib import Path
 import pytest
 
 from khoj.database.models import Entry, GithubConfig, KhojUser, LocalOrgConfig
-from khoj.processor.data_sources.github.github_to_entries import GithubToEntries
-from khoj.processor.data_sources.org_mode.org_to_entries import OrgToEntries
+from khoj.processor.content.github.github_to_entries import GithubToEntries
+from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
 
 # Internal Packages
 from khoj.search_type import text_search