Use the src/ layout to fix packaging Khoj for PyPi

- Why
  The khoj pypi packages should be installed in `khoj' directory.
  Previously it was being installed into `src' directory, which is a
  generic top level directory name that is discouraged from being used

- Changes
 - move src/* to src/khoj/*
 - update `setup.py' to `find_packages' in `src' instead of project root
 - rename imports to form `from khoj.*' in complete project
 - update `constants.web_directory' path to use `khoj' directory
 - rename root logger to `khoj' in `main.py'
 - fix image_search tests to use the newly rename `khoj' logger
 - update config, docs, workflows to reference new path `src/khoj'

.github/workflows/build.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - master
     paths:
-      - src/**
+      - src/khoj/**
       - config/**
       - setup.py
       - Dockerfile

.github/workflows/build_khoj_el.yml

@@ -7,13 +7,13 @@ on:
     branches:
       - 'master'
     paths:
-      - src/interface/emacs/*.el
+      - src/khoj/interface/emacs/*.el
       - .github/workflows/build_khoj_el.yml
   pull_request:
     branches:
       - 'master'
     paths:
-      - src/interface/emacs/*.el
+      - src/khoj/interface/emacs/*.el
       - .github/workflows/build_khoj_el.yml
 
 jobs:
@@ -33,7 +33,7 @@ jobs:
     - name: Run
       env:
         # Khoj recipe from https://github.com/melpa/melpa/pull/8321/files
-        RECIPE: (khoj :fetcher github :repo "debanjum/khoj" :files ("src/interface/emacs/*.el"))
+        RECIPE: (khoj :fetcher github :repo "debanjum/khoj" :files ("src/khoj/interface/emacs/*.el"))
         EXIST_OK: true
         LOCAL_REPO: ${{ github.workspace }}
       run: echo $GITHUB_REF && make -C ~/melpazoid

.github/workflows/publish.yml

@@ -7,14 +7,14 @@ on:
     branches:
       - 'master'
     paths:
-      - src/**
+      - src/khoj/**
       - setup.py
       - .github/workflows/publish.yml
   pull_request:
     branches:
       - 'master'
     paths:
-      - src/**
+      - src/khoj/**
       - setup.py
       - .github/workflows/publish.yml
 

.github/workflows/release.yml

@@ -66,7 +66,7 @@ jobs:
           # Create disk image with the app
           create-dmg \
             --volname "Khoj" \
-            --volicon "src/interface/web/assets/icons/favicon.icns" \
+            --volicon "src/khoj/interface/web/assets/icons/favicon.icns" \
             --window-pos 200 120 \
             --window-size 600 300 \
             --icon-size 100 \
@@ -92,7 +92,7 @@ jobs:
           # Copy app files into expected output directory structure
           mkdir -p package/opt package/usr/share/applications package/usr/share/icons/hicolor/128x128/apps
           cp -r dist/Khoj package/opt/Khoj
-          cp src/interface/web/assets/icons/favicon-128x128.png package/usr/share/icons/hicolor/128x128/apps/Khoj.png
+          cp src/khoj/interface/web/assets/icons/favicon-128x128.png package/usr/share/icons/hicolor/128x128/apps/Khoj.png
           cp Khoj.desktop package/usr/share/applications
 
           # Fix permissions to be usable by non-root users

.github/workflows/test.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - 'master'
     paths:
-      - src/**
+      - src/khoj/**
       - tests/**
       - config/**
       - setup.py
@@ -14,7 +14,7 @@ on:
     branches:
       - 'master'
     paths:
-      - src/**
+      - src/khoj/**
       - tests/**
       - config/**
       - setup.py

.github/workflows/test_khoj_el.yml

@@ -5,15 +5,15 @@ on:
     branches:
       - 'master'
     paths:
-      - src/interface/emacs/*.el
+      - src/khoj/interface/emacs/*.el
-      - src/interface/emacs/tests/*.el
+      - src/khoj/interface/emacs/tests/*.el
       - .github/workflows/test_khoj_el.yml
   pull_request:
     branches:
       - 'master'
     paths:
-      - src/interface/emacs/*.el
+      - src/khoj/interface/emacs/*.el
-      - src/interface/emacs/tests/*.el
+      - src/khoj/interface/emacs/tests/*.el
       - .github/workflows/test_khoj_el.yml
 
 jobs:
@@ -44,6 +44,6 @@ jobs:
                     (unless package-archive-contents (package-refresh-contents)) \
                     (unless (package-installed-p 'transient) (package-install 'transient)))" \
            -l ert \
-           -l ./src/interface/emacs/khoj.el \
+           -l ./src/khoj/interface/emacs/khoj.el \
-           -l ./src/interface/emacs/tests/khoj-tests.el \
+           -l ./src/khoj/interface/emacs/tests/khoj-tests.el \
            -f ert-run-tests-batch-and-exit

.gitignore

@@ -1,7 +1,6 @@
 # Khoj artifacts
 *.gz
 *.pt
-src/.data
 tests/data/models
 tests/data/embeddings
 
@@ -13,10 +12,10 @@ __pycache__
 .vscode
 
 # Build artifacts
-/src/interface/web/images
+/src/khoj/interface/web/images
 /build/
 /dist/
-/khoj_assistant.egg-info/
+khoj_assistant.egg-info
 /config/khoj*.yml
 .pytest_cache
 khoj.log

.mypy.ini

@@ -5,9 +5,9 @@ install_types = True
 non_interactive = True
 show_error_codes = True
 exclude = (?x)(
-    src/interface/desktop/main_window.py
+    src/khoj/interface/desktop/main_window.py
-    | src/interface/desktop/file_browser.py
+    | src/khoj/interface/desktop/file_browser.py
-    | src/interface/desktop/system_tray.py
+    | src/khoj/interface/desktop/system_tray.py
     | build/*
     | tests/*
   )

Khoj.spec

@@ -5,7 +5,7 @@ from PyInstaller.utils.hooks import copy_metadata
 import sysconfig
 
 datas = [
-    ('src/interface/web', 'src/interface/web'),
+    ('src/khoj/interface/web', 'src/khoj/interface/web'),
     (f'{sysconfig.get_paths()["purelib"]}/transformers', 'transformers')
 ]
 datas += copy_metadata('tqdm')
@@ -19,7 +19,7 @@ datas += copy_metadata('tokenizers')
 block_cipher = None
 
 a = Analysis(
-    ['src/main.py'],
+    ['src/khoj/main.py'],
     pathex=[],
     binaries=[],
     datas=datas,
@@ -50,7 +50,7 @@ pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)
 if system() != 'Darwin':
     # Add Splash screen to show on app launch
     splash = Splash(
-        'src/interface/web/assets/icons/favicon-144x144.png',
+        'src/khoj/interface/web/assets/icons/favicon-144x144.png',
         binaries=a.binaries,
         datas=a.datas,
         text_pos=(10, 160),
@@ -82,7 +82,7 @@ if system() != 'Darwin':
         target_arch='x86_64',
         codesign_identity=None,
         entitlements_file=None,
-        icon='src/interface/web/assets/icons/favicon-144x144.ico',
+        icon='src/khoj/interface/web/assets/icons/favicon-144x144.ico',
     )
 else:
     exe = EXE(
@@ -105,11 +105,11 @@ else:
         target_arch='x86_64',
         codesign_identity=None,
         entitlements_file=None,
-        icon='src/interface/web/assets/icons/favicon.icns',
+        icon='src/khoj/interface/web/assets/icons/favicon.icns',
     )
     app = BUNDLE(
         exe,
         name='Khoj.app',
-        icon='src/interface/web/assets/icons/favicon.icns',
+        icon='src/khoj/interface/web/assets/icons/favicon.icns',
         bundle_identifier=None,
     )

MANIFEST.in

@@ -1,5 +1,7 @@
 include README.md
-graft src/interface/*
+graft src/khoj/interface/*
-prune src/interface/web/images*
+prune src/khoj/interface/web/images*
 prune docs*
+prune src/khoj/interface/obsidian*
+prune src/khoj/interface/emacs*
 global-exclude .DS_Store *.py[cod]

README.md

@@ -7,8 +7,8 @@
 
 **Supported Plugins**
 
-[![Khoj on Obsidian](https://img.shields.io/badge/Obsidian-%23483699.svg?style=for-the-badge&logo=obsidian&logoColor=white)](https://github.com/debanjum/khoj/tree/master/src/interface/obsidian#readme)
+[![Khoj on Obsidian](https://img.shields.io/badge/Obsidian-%23483699.svg?style=for-the-badge&logo=obsidian&logoColor=white)](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/obsidian#readme)
-[![Khoj on Emacs](https://img.shields.io/badge/Emacs-%237F5AB6.svg?&style=for-the-badge&logo=gnu-emacs&logoColor=white)](https://github.com/debanjum/khoj/tree/master/src/interface/emacs#readme)
+[![Khoj on Emacs](https://img.shields.io/badge/Emacs-%237F5AB6.svg?&style=for-the-badge&logo=gnu-emacs&logoColor=white)](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#readme)
 
 ## Table of Contents
 
@@ -59,7 +59,7 @@
 - **Incremental**: Incremental search for a fast, search-as-you-type experience
 - **Pluggable**: Modular architecture makes it easy to plug in new data sources, frontends and ML models
 - **Multiple Sources**: Search your Org-mode and Markdown notes, Beancount transactions and Photos
-- **Multiple Interfaces**: Search using a [Web Browser](./src/interface/web/index.html), [Emacs](./src/interface/emacs/khoj.el) or the [API](http://localhost:8000/docs)
+- **Multiple Interfaces**: Search using a [Web Browser](./src/khoj/interface/web/index.html), [Emacs](./src/khoj/interface/emacs/khoj.el) or the [API](http://localhost:8000/docs)
 
 ## Demos
 ### Khoj in Obsidian
@@ -83,9 +83,9 @@ https://user-images.githubusercontent.com/6413477/184735169-92c78bf1-d827-4663-9
 
 - Install Khoj via pip
 - Start Khoj app
-- Add this readme and [khoj.el readme](https://github.com/debanjum/khoj/tree/master/src/interface/emacs) as org-mode for Khoj to index
+- Add this readme and [khoj.el readme](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs) as org-mode for Khoj to index
 - Search \"*Setup editor*\" on the Web and Emacs. Re-rank the results for better accuracy
-- Top result is what we are looking for, the [section to Install Khoj.el on Emacs](https://github.com/debanjum/khoj/tree/master/src/interface/emacs#2-Install-Khojel)
+- Top result is what we are looking for, the [section to Install Khoj.el on Emacs](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#2-Install-Khojel)
 </details>
 
 <details><summary>Analysis</summary>
@@ -107,8 +107,8 @@ https://user-images.githubusercontent.com/6413477/184735169-92c78bf1-d827-4663-9
 ## Setup
 These are the general setup instructions for Khoj.
 
-- Check the [Khoj.el Readme](https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Setup) to setup Khoj with Emacs
+- Check the [Khoj.el Readme](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Setup) to setup Khoj with Emacs
-- Check the [Khoj Obsidian Readme](https://github.com/debanjum/khoj/tree/master/src/interface/obsidian#Setup) to setup Khoj with Obsidian<br />
+- Check the [Khoj Obsidian Readme](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/obsidian#Setup) to setup Khoj with Obsidian<br />
   Its simpler as it can skip the configure step below.
 
 ### 1. Install
@@ -132,10 +132,10 @@ khoj
 ### Interfaces
 
 - **Khoj via Obsidian**
-  - [Install](https://github.com/debanjum/khoj/tree/master/src/interface/obsidian#2-Setup-Plugin) the Khoj Obsidian plugin
+  - [Install](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/obsidian#2-Setup-Plugin) the Khoj Obsidian plugin
   - Click the *Khoj search* icon 🔎 on the [Ribbon](https://help.obsidian.md/User+interface/Workspace/Ribbon) or Search for *Khoj: Search* in the [Command Palette](https://help.obsidian.md/Plugins/Command+palette)
 - **Khoj via Emacs**
-  - [Install](https://github.com/debanjum/khoj/tree/master/src/interface/emacs#installation) [khoj.el](./src/interface/emacs/khoj.el)
+  - [Install](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#installation) [khoj.el](./src/khoj/interface/emacs/khoj.el)
   - Run `M-x khoj <user-query>`
 - **Khoj via Web**
   - Open <http://localhost:8000/> via desktop interface or directly
@@ -169,11 +169,11 @@ pip install --upgrade khoj-assistant
 
 ### Upgrade Khoj on Emacs
 - Use your Emacs Package Manager to Upgrade
-- See [khoj.el readme](https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Upgrade) for details
+- See [khoj.el readme](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Upgrade) for details
 
 ### Upgrade Khoj on Obsidian
 - Upgrade via the Community plugins tab on the settings pane in the Obsidian app
-- See the [khoj plugin readme](https://github.com/debanjum/khoj/tree/master/src/interface/obsidian#2-Setup-Plugin) for details
+- See the [khoj plugin readme](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/obsidian#2-Setup-Plugin) for details
 
 ## Uninstall Khoj
 1. (Optional) Hit `Ctrl-C` in the terminal running the khoj server to stop it
@@ -238,7 +238,7 @@ pip install --upgrade khoj-assistant
       asymmetric:
    -    encoder: "sentence-transformers/multi-qa-MiniLM-L6-cos-v1"
    +    encoder: text-embedding-ada-002
-   +    encoder-type: src.utils.models.OpenAI
+   +    encoder-type: src.khoj.utils.models.OpenAI
         cross-encoder: "cross-encoder/ms-marco-MiniLM-L-6-v2"
    -    encoder-type: sentence_transformers.SentenceTransformer
    -    model_directory: "~/.khoj/search/asymmetric/"
@@ -408,7 +408,7 @@ python3 -m pip install pyqt6  # As conda does not support pyqt6 yet
 
 ##### 4. Run
 ```shell
-python3 -m src.main -vv
+python3 -m src.khoj.main -vv
 ```
   Load ML model, generate embeddings and expose API to query notes, images, transactions etc specified in config YAML
 

setup.py

@@ -17,10 +17,10 @@ setup(
     license="GPLv3",
     keywords="search semantic-search productivity NLP org-mode markdown beancount images",
     python_requires=">=3.8, <3.11",
+    package_dir={"": "src"},
     packages=find_packages(
-        where=".",
+        where="src",
-        exclude=["tests*"],
+        include=["khoj*"]
-        include=["src*"]
     ),
     install_requires=[
         "torch == 1.13.1",
@@ -39,7 +39,7 @@ setup(
         'schedule == 1.1.0',
     ],
     include_package_data=True,
-    entry_points={"console_scripts": ["khoj = src.main:run"]},
+    entry_points={"console_scripts": ["khoj = khoj.main:run"]},
     classifiers=[
         "Development Status :: 4 - Beta",
         "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",

src/khoj/configure.py

@@ -7,17 +7,17 @@ import json
 import schedule
 
 # Internal Packages
-from src.processor.ledger.beancount_to_jsonl import BeancountToJsonl
+from khoj.processor.ledger.beancount_to_jsonl import BeancountToJsonl
-from src.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
+from khoj.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
-from src.processor.org_mode.org_to_jsonl import OrgToJsonl
+from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl
-from src.search_type import image_search, text_search
+from khoj.search_type import image_search, text_search
-from src.utils.config import SearchType, SearchModels, ProcessorConfigModel, ConversationProcessorConfigModel
+from khoj.utils.config import SearchType, SearchModels, ProcessorConfigModel, ConversationProcessorConfigModel
-from src.utils import state
+from khoj.utils import state
-from src.utils.helpers import LRU, resolve_absolute_path
+from khoj.utils.helpers import LRU, resolve_absolute_path
-from src.utils.rawconfig import FullConfig, ProcessorConfig
+from khoj.utils.rawconfig import FullConfig, ProcessorConfig
-from src.search_filter.date_filter import DateFilter
+from khoj.search_filter.date_filter import DateFilter
-from src.search_filter.word_filter import WordFilter
+from khoj.search_filter.word_filter import WordFilter
-from src.search_filter.file_filter import FileFilter
+from khoj.search_filter.file_filter import FileFilter
 
 
 logger = logging.getLogger(__name__)

src/khoj/interface/desktop/file_browser.py

@@ -3,8 +3,8 @@ from PyQt6 import QtWidgets
 from PyQt6.QtCore import QDir
 
 # Internal Packages
-from src.utils.config import SearchType
+from khoj.utils.config import SearchType
-from src.utils.helpers import is_none_or_empty
+from khoj.utils.helpers import is_none_or_empty
 
 
 class FileBrowser(QtWidgets.QWidget):

src/khoj/interface/desktop/labelled_text_field.py

@@ -2,7 +2,7 @@
 from PyQt6 import QtWidgets
 
 # Internal Packages
-from src.utils.config import ProcessorType
+from khoj.utils.config import ProcessorType
 
 
 class LabelledTextField(QtWidgets.QWidget):

src/khoj/interface/desktop/main_window.py

@@ -9,13 +9,13 @@ from PyQt6 import QtGui, QtWidgets
 from PyQt6.QtCore import Qt, QThread, QObject, pyqtSignal
 
 # Internal Packages
-from src.configure import configure_server
+from khoj.configure import configure_server
-from src.interface.desktop.file_browser import FileBrowser
+from khoj.interface.desktop.file_browser import FileBrowser
-from src.interface.desktop.labelled_text_field import LabelledTextField
+from khoj.interface.desktop.labelled_text_field import LabelledTextField
-from src.utils import constants, state, yaml as yaml_utils
+from khoj.utils import constants, state, yaml as yaml_utils
-from src.utils.cli import cli
+from khoj.utils.cli import cli
-from src.utils.config import SearchType, ProcessorType
+from khoj.utils.config import SearchType, ProcessorType
-from src.utils.helpers import merge_dicts, resolve_absolute_path
+from khoj.utils.helpers import merge_dicts, resolve_absolute_path
 
 
 class MainWindow(QtWidgets.QMainWindow):

src/khoj/interface/desktop/system_tray.py

@@ -5,8 +5,8 @@ import webbrowser
 from PyQt6 import QtGui, QtWidgets
 
 # Internal Packages
-from src.utils import constants, state
+from khoj.utils import constants, state
-from src.interface.desktop.main_window import MainWindow
+from khoj.interface.desktop.main_window import MainWindow
 
 
 def create_system_tray(gui: QtWidgets.QApplication, main_window: MainWindow):

src/khoj/interface/emacs/README.org

@@ -4,19 +4,19 @@
   /Natural, Incremental Search for your Second Brain/
 
 ** Table of Contents
-  - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#features][Features]]
+  - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#features][Features]]
-  - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Interface][Interface]]
+  - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Interface][Interface]]
-  - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Setup][Setup]]
+  - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Setup][Setup]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#1-Setup-Backend][Setup Backend]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#1-Setup-Backend][Setup Backend]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#2-Install-Khojel][Install Khoj.el]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#2-Install-Khojel][Install Khoj.el]]
-  - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Use][Use]]
+  - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Use][Use]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Search][Search]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Search][Search]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Find-similar-entries][Find Similar Entries]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Find-similar-entries][Find Similar Entries]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Advanced-usage][Advanced Usage]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Advanced-usage][Advanced Usage]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Khoj-menu][Khoj Menu]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Khoj-menu][Khoj Menu]]
-  - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Upgrade][Upgrade]]
+  - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Upgrade][Upgrade]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Upgrade-Khoj-Backend][Upgrade Backend]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Upgrade-Khoj-Backend][Upgrade Backend]]
-    - [[https://github.com/debanjum/khoj/tree/master/src/interface/emacs#Upgrade-Khojel][Upgrade Khoj.el]]
+    - [[https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#Upgrade-Khojel][Upgrade Khoj.el]]
 
 ** Features
   - *Natural*: Advanced natural language understanding using Transformer based ML Models
@@ -58,7 +58,7 @@
     ;; Install Khoj Package using Straight.el
     (use-package khoj
       :after org
-      :straight (khoj :type git :host github :repo "debanjum/khoj" :files (:defaults "src/interface/emacs/khoj.el"))
+      :straight (khoj :type git :host github :repo "debanjum/khoj" :files (:defaults "src/khoj/interface/emacs/khoj.el"))
       :bind ("C-c s" . 'khoj))
   #+end_src
 

src/khoj/interface/emacs/khoj.el

@@ -7,7 +7,7 @@
 ;; Keywords: search, org-mode, outlines, markdown, beancount, ledger, image
 ;; Version: 0.3.0
 ;; Package-Requires: ((emacs "27.1") (transient "0.3.0"))
-;; URL: https://github.com/debanjum/khoj/tree/master/src/interface/emacs
+;; URL: https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs
 
 ;; This file is NOT part of GNU Emacs.
 

src/khoj/interface/emacs/tests/khoj-tests.el

@@ -5,7 +5,7 @@
 ;; Author: Debanjum Singh Solanky <debanjum@gmail.com>
 ;; Version: 0.0.0
 ;; Package-Requires: ((emacs "27.1") (transient "0.3.0"))
-;; URL: https://github.com/debanjum/khoj/tree/master/src/interface/emacs
+;; URL: https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs
 
 ;;; License:
 

src/khoj/main.py

@@ -20,15 +20,15 @@ from PyQt6.QtCore import QThread, QTimer
 import schedule
 
 # Internal Packages
-from src.configure import configure_server
+from khoj.configure import configure_server
-from src.routers.api import api
+from khoj.routers.api import api
-from src.routers.api_beta import api_beta
+from khoj.routers.api_beta import api_beta
-from src.routers.web_client import web_client
+from khoj.routers.web_client import web_client
-from src.utils import constants, state
+from khoj.utils import constants, state
-from src.utils.cli import cli
+from khoj.utils.cli import cli
-from src.utils.helpers import CustomFormatter
+from khoj.utils.helpers import CustomFormatter
-from src.interface.desktop.main_window import MainWindow
+from khoj.interface.desktop.main_window import MainWindow
-from src.interface.desktop.system_tray import create_system_tray
+from khoj.interface.desktop.system_tray import create_system_tray
 
 
 # Initialize the Application Server
@@ -38,7 +38,7 @@ app.include_router(api, prefix="/api")
 app.include_router(api_beta, prefix="/api/beta")
 app.include_router(web_client)
 
-logger = logging.getLogger('src')
+logger = logging.getLogger('khoj')
 
 
 def run():

src/khoj/processor/conversation/gpt.py

@@ -7,7 +7,7 @@ from datetime import datetime
 import openai
 
 # Internal Packages
-from src.utils.constants import empty_escape_sequences
+from khoj.utils.constants import empty_escape_sequences
 
 
 def summarize(text, summary_type, model, user_query=None, api_key=None, temperature=0.5, max_tokens=200):

src/khoj/processor/ledger/beancount_to_jsonl.py

@@ -5,11 +5,11 @@ import logging
 from typing import List
 
 # Internal Packages
-from src.processor.text_to_jsonl import TextToJsonl
+from khoj.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import get_absolute_path, is_none_or_empty, timer
+from khoj.utils.helpers import get_absolute_path, is_none_or_empty, timer
-from src.utils.constants import empty_escape_sequences
+from khoj.utils.constants import empty_escape_sequences
-from src.utils.jsonl import dump_jsonl, compress_jsonl_data
+from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/markdown/markdown_to_jsonl.py

@@ -6,11 +6,11 @@ import time
 from typing import List
 
 # Internal Packages
-from src.processor.text_to_jsonl import TextToJsonl
+from khoj.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import get_absolute_path, is_none_or_empty, timer
+from khoj.utils.helpers import get_absolute_path, is_none_or_empty, timer
-from src.utils.constants import empty_escape_sequences
+from khoj.utils.constants import empty_escape_sequences
-from src.utils.jsonl import dump_jsonl, compress_jsonl_data
+from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/org_mode/org_to_jsonl.py

@@ -5,12 +5,12 @@ import time
 from typing import Iterable, List
 
 # Internal Packages
-from src.processor.org_mode import orgnode
+from khoj.processor.org_mode import orgnode
-from src.processor.text_to_jsonl import TextToJsonl
+from khoj.processor.text_to_jsonl import TextToJsonl
-from src.utils.helpers import get_absolute_path, is_none_or_empty, timer
+from khoj.utils.helpers import get_absolute_path, is_none_or_empty, timer
-from src.utils.jsonl import dump_jsonl, compress_jsonl_data
+from khoj.utils.jsonl import dump_jsonl, compress_jsonl_data
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
-from src.utils import state
+from khoj.utils import state
 
 
 logger = logging.getLogger(__name__)

src/khoj/processor/text_to_jsonl.py

@@ -3,10 +3,10 @@ from abc import ABC, abstractmethod
 import hashlib
 import logging
 from typing import Callable, List, Tuple
-from src.utils.helpers import timer
+from khoj.utils.helpers import timer
 
 # Internal Packages
-from src.utils.rawconfig import Entry, TextContentConfig
+from khoj.utils.rawconfig import Entry, TextContentConfig
 
 
 logger = logging.getLogger(__name__)

src/khoj/routers/api.py

@@ -8,12 +8,12 @@ from fastapi import APIRouter
 from fastapi import HTTPException
 
 # Internal Packages
-from src.configure import configure_processor, configure_search
+from khoj.configure import configure_processor, configure_search
-from src.search_type import image_search, text_search
+from khoj.search_type import image_search, text_search
-from src.utils.helpers import timer
+from khoj.utils.helpers import timer
-from src.utils.rawconfig import FullConfig, SearchResponse
+from khoj.utils.rawconfig import FullConfig, SearchResponse
-from src.utils.config import SearchType
+from khoj.utils.config import SearchType
-from src.utils import state, constants
+from khoj.utils import state, constants
 
 
 # Initialize Router

src/khoj/routers/api_beta.py

@@ -8,11 +8,11 @@ import schedule
 from fastapi import APIRouter
 
 # Internal Packages
-from src.routers.api import search
+from khoj.routers.api import search
-from src.processor.conversation.gpt import converse, extract_search_type, message_to_log, message_to_prompt, understand, summarize
+from khoj.processor.conversation.gpt import converse, extract_search_type, message_to_log, message_to_prompt, understand, summarize
-from src.utils.config import SearchType
+from khoj.utils.config import SearchType
-from src.utils.helpers import get_from_dict, resolve_absolute_path
+from khoj.utils.helpers import get_from_dict, resolve_absolute_path
-from src.utils import state
+from khoj.utils import state
 
 
 # Initialize Router

src/khoj/routers/web_client.py

@@ -5,7 +5,7 @@ from fastapi.responses import HTMLResponse, FileResponse
 from fastapi.templating import Jinja2Templates
 
 # Internal Packages
-from src.utils import constants
+from khoj.utils import constants
 
 
 # Initialize Router

src/khoj/search_filter/base_filter.py

@@ -3,7 +3,7 @@ from abc import ABC, abstractmethod
 from typing import List, Set, Tuple
 
 # Internal Packages
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 class BaseFilter(ABC):

src/khoj/search_filter/date_filter.py

@@ -11,8 +11,8 @@ from math import inf
 import dateparser as dtparse
 
 # Internal Packages
-from src.search_filter.base_filter import BaseFilter
+from khoj.search_filter.base_filter import BaseFilter
-from src.utils.helpers import LRU, timer
+from khoj.utils.helpers import LRU, timer
 
 
 logger = logging.getLogger(__name__)

src/khoj/search_filter/file_filter.py

@@ -6,8 +6,8 @@ import logging
 from collections import defaultdict
 
 # Internal Packages
-from src.search_filter.base_filter import BaseFilter
+from khoj.search_filter.base_filter import BaseFilter
-from src.utils.helpers import LRU, timer
+from khoj.utils.helpers import LRU, timer
 
 
 logger = logging.getLogger(__name__)

src/khoj/search_filter/word_filter.py

@@ -5,8 +5,8 @@ import logging
 from collections import defaultdict
 
 # Internal Packages
-from src.search_filter.base_filter import BaseFilter
+from khoj.search_filter.base_filter import BaseFilter
-from src.utils.helpers import LRU, timer
+from khoj.utils.helpers import LRU, timer
 
 
 logger = logging.getLogger(__name__)

src/khoj/search_type/image_search.py

@@ -14,9 +14,9 @@ from tqdm import trange
 import torch
 
 # Internal Packages
-from src.utils.helpers import get_absolute_path, get_from_dict, resolve_absolute_path, load_model, timer
+from khoj.utils.helpers import get_absolute_path, get_from_dict, resolve_absolute_path, load_model, timer
-from src.utils.config import ImageSearchModel
+from khoj.utils.config import ImageSearchModel
-from src.utils.rawconfig import ImageContentConfig, ImageSearchConfig, SearchResponse
+from khoj.utils.rawconfig import ImageContentConfig, ImageSearchConfig, SearchResponse
 
 
 # Create Logger

src/khoj/search_type/text_search.py

@@ -7,16 +7,16 @@ from typing import List, Tuple, Type
 # External Packages
 import torch
 from sentence_transformers import SentenceTransformer, CrossEncoder, util
-from src.processor.text_to_jsonl import TextToJsonl
+from khoj.processor.text_to_jsonl import TextToJsonl
-from src.search_filter.base_filter import BaseFilter
+from khoj.search_filter.base_filter import BaseFilter
 
 # Internal Packages
-from src.utils import state
+from khoj.utils import state
-from src.utils.helpers import get_absolute_path, is_none_or_empty, resolve_absolute_path, load_model, timer
+from khoj.utils.helpers import get_absolute_path, is_none_or_empty, resolve_absolute_path, load_model, timer
-from src.utils.config import TextSearchModel
+from khoj.utils.config import TextSearchModel
-from src.utils.models import BaseEncoder
+from khoj.utils.models import BaseEncoder
-from src.utils.rawconfig import SearchResponse, TextSearchConfig, TextContentConfig, Entry
+from khoj.utils.rawconfig import SearchResponse, TextSearchConfig, TextContentConfig, Entry
-from src.utils.jsonl import load_jsonl
+from khoj.utils.jsonl import load_jsonl
 
 
 logger = logging.getLogger(__name__)

src/khoj/utils/cli.py

@@ -4,8 +4,8 @@ import pathlib
 from importlib.metadata import version
 
 # Internal Packages
-from src.utils.helpers import resolve_absolute_path
+from khoj.utils.helpers import resolve_absolute_path
-from src.utils.yaml import parse_config_from_file
+from khoj.utils.yaml import parse_config_from_file
 
 
 def cli(args=None):

src/khoj/utils/config.py

@@ -11,9 +11,9 @@ import torch
 # Internal Packages
 if TYPE_CHECKING:
     from sentence_transformers import CrossEncoder
-    from src.search_filter.base_filter import BaseFilter
+    from khoj.search_filter.base_filter import BaseFilter
-    from src.utils.models import BaseEncoder
+    from khoj.utils.models import BaseEncoder
-    from src.utils.rawconfig import ConversationProcessorConfig, Entry
+    from khoj.utils.rawconfig import ConversationProcessorConfig, Entry
 
 
 class SearchType(str, Enum):

src/khoj/utils/constants.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
 app_root_directory = Path(__file__).parent.parent.parent
-web_directory = app_root_directory / 'src/interface/web/'
+web_directory = app_root_directory / 'khoj/interface/web/'
 empty_escape_sequences = '\n|\r|\t| '
 
 # default app config to use

src/khoj/utils/helpers.py

@@ -14,7 +14,7 @@ if TYPE_CHECKING:
     # External Packages
     from sentence_transformers import CrossEncoder
     # Internal Packages
-    from src.utils.models import BaseEncoder
+    from khoj.utils.models import BaseEncoder
 
 
 def is_none_or_empty(item):

src/khoj/utils/jsonl.py

@@ -4,8 +4,8 @@ import gzip
 import logging
 
 # Internal Packages
-from src.utils.constants import empty_escape_sequences
+from khoj.utils.constants import empty_escape_sequences
-from src.utils.helpers import get_absolute_path
+from khoj.utils.helpers import get_absolute_path
 
 
 logger = logging.getLogger(__name__)

src/khoj/utils/models.py

@@ -8,7 +8,7 @@ import torch
 from tqdm import trange
 
 # Internal Packages
-from src.utils.state import processor_config, config_file
+from khoj.utils.state import processor_config, config_file
 
 
 class BaseEncoder(ABC):

src/khoj/utils/rawconfig.py

@@ -7,7 +7,7 @@ from typing import List, Optional
 from pydantic import BaseModel, validator
 
 # Internal Packages
-from src.utils.helpers import to_snake_case_from_dash, is_none_or_empty
+from khoj.utils.helpers import to_snake_case_from_dash, is_none_or_empty
 
 class ConfigBase(BaseModel):
     class Config:

src/khoj/utils/state.py

@@ -8,9 +8,9 @@ import torch
 from pathlib import Path
 
 # Internal Packages
-from src.utils.config import SearchModels, ProcessorConfigModel
+from khoj.utils.config import SearchModels, ProcessorConfigModel
-from src.utils.helpers import LRU
+from khoj.utils.helpers import LRU
-from src.utils.rawconfig import FullConfig
+from khoj.utils.rawconfig import FullConfig
 
 # Application Global State
 config = FullConfig()

src/khoj/utils/yaml.py

@@ -5,7 +5,7 @@ from pathlib import Path
 import yaml
 
 # Internal Packages
-from src.utils.rawconfig import FullConfig
+from khoj.utils.rawconfig import FullConfig
 
 
 # Do not emit tags when dumping to YAML

tests/conftest.py

@@ -4,13 +4,13 @@ from pathlib import Path
 import pytest
 
 # Internal Packages
-from src.search_type import image_search, text_search
+from khoj.search_type import image_search, text_search
-from src.utils.helpers import resolve_absolute_path
+from khoj.utils.helpers import resolve_absolute_path
-from src.utils.rawconfig import ContentConfig, TextContentConfig, ImageContentConfig, SearchConfig, TextSearchConfig, ImageSearchConfig
+from khoj.utils.rawconfig import ContentConfig, TextContentConfig, ImageContentConfig, SearchConfig, TextSearchConfig, ImageSearchConfig
-from src.processor.org_mode.org_to_jsonl import OrgToJsonl
+from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl
-from src.search_filter.date_filter import DateFilter
+from khoj.search_filter.date_filter import DateFilter
-from src.search_filter.word_filter import WordFilter
+from khoj.search_filter.word_filter import WordFilter
-from src.search_filter.file_filter import FileFilter
+from khoj.search_filter.file_filter import FileFilter
 
 
 @pytest.fixture(scope='session')

tests/data/markdown/interface_emacs_readme.md

@@ -28,7 +28,7 @@
         ;; Khoj Package for Semantic Search
         (use-package khoj
           :after org
-          :straight (khoj :type git :host github :repo "debanjum/khoj" :files (:defaults "src/interface/emacs/khoj.el"))
+          :straight (khoj :type git :host github :repo "debanjum/khoj" :files (:defaults "src/khoj/interface/emacs/khoj.el"))
           :bind ("C-c s" . 'khoj))
         ```
 -   With [Quelpa](https://github.com/quelpa/quelpa#installation)

tests/data/markdown/main_readme.md

@@ -6,8 +6,8 @@
 *Allow natural language search on user content like notes, images,
 transactions using transformer ML models*
 
-User can interface with Khoj via [Web](./src/interface/web/index.html),
+User can interface with Khoj via [Web](./src/khoj/interface/web/index.html),
-[Emacs](./src/interface/emacs/khoj.el) or the API. All search is done
+[Emacs](./src/khoj/interface/emacs/khoj.el) or the API. All search is done
 locally[\*](https://github.com/debanjum/khoj#miscellaneous)
 
 ## Demo
@@ -47,8 +47,8 @@ just generating embeddings*
     -   [Update Index](http://localhost:8000/api/update?t=ledger)
     -   [Configure Application](https://localhost:8000/ui)
 -   **Khoj via Emacs**
-    -   [Install](https://github.com/debanjum/khoj/tree/master/src/interface/emacs#installation)
+    -   [Install](https://github.com/debanjum/khoj/tree/master/src/khoj/interface/emacs#installation)
-        [khoj.el](./src/interface/emacs/khoj.el)
+        [khoj.el](./src/khoj/interface/emacs/khoj.el)
     -   Run `M-x khoj <user-query>`
 
 ## Run Unit tests
@@ -118,7 +118,7 @@ docker-compose build --pull
     images, transactions etc specified in config YAML
 
     ``` shell
-    python3 -m src.main -c=config/khoj_sample.yml -vv
+    python3 -m src.khoj.main -c=config/khoj_sample.yml -vv
     ```
 
 ### Upgrade On Local Machine

tests/test_beancount_to_jsonl.py

@@ -2,7 +2,7 @@
 import json
 
 # Internal Packages
-from src.processor.ledger.beancount_to_jsonl import BeancountToJsonl
+from khoj.processor.ledger.beancount_to_jsonl import BeancountToJsonl
 
 
 def test_no_transactions_in_file(tmp_path):

tests/test_chatbot.py

@@ -2,7 +2,7 @@
 import pytest
 
 # Internal Packages
-from src.processor.conversation.gpt import converse, understand, message_to_prompt
+from khoj.processor.conversation.gpt import converse, understand, message_to_prompt
 
 
 # Initialize variables for tests

tests/test_cli.py

@@ -3,8 +3,8 @@ from pathlib import Path
 from random import random
 
 # Internal Packages
-from src.utils.cli import cli
+from khoj.utils.cli import cli
-from src.utils.helpers import resolve_absolute_path
+from khoj.utils.helpers import resolve_absolute_path
 
 
 # Test

tests/test_client.py

@@ -8,13 +8,13 @@ from urllib.parse import quote
 from fastapi.testclient import TestClient
 
 # Internal Packages
-from src.main import app
+from khoj.main import app
-from src.utils.state import model, config
+from khoj.utils.state import model, config
-from src.search_type import text_search, image_search
+from khoj.search_type import text_search, image_search
-from src.utils.rawconfig import ContentConfig, SearchConfig
+from khoj.utils.rawconfig import ContentConfig, SearchConfig
-from src.processor.org_mode.org_to_jsonl import OrgToJsonl
+from khoj.processor.org_mode.org_to_jsonl import OrgToJsonl
-from src.search_filter.word_filter import WordFilter
+from khoj.search_filter.word_filter import WordFilter
-from src.search_filter.file_filter import FileFilter
+from khoj.search_filter.file_filter import FileFilter
 
 
 # Arrange

tests/test_date_filter.py

@@ -4,8 +4,8 @@ from datetime import datetime
 from math import inf
 
 # Application Packages
-from src.search_filter.date_filter import DateFilter
+from khoj.search_filter.date_filter import DateFilter
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 def test_date_filter():

tests/test_file_filter.py

@@ -1,6 +1,6 @@
 # Application Packages
-from src.search_filter.file_filter import FileFilter
+from khoj.search_filter.file_filter import FileFilter
-from src.utils.rawconfig import Entry
+from khoj.utils.rawconfig import Entry
 
 
 def test_no_file_filter():

tests/test_helpers.py

@@ -1,4 +1,4 @@
-from src.utils import helpers
+from khoj.utils import helpers
 
 def test_get_from_null_dict():
     # null handling

tests/test_image_search.py

@@ -4,11 +4,11 @@ from pathlib import Path
 from PIL import Image
 
 # Internal Packages
-from src.utils.state import model
+from khoj.utils.state import model
-from src.utils.constants import web_directory
+from khoj.utils.constants import web_directory
-from src.search_type import image_search
+from khoj.search_type import image_search
-from src.utils.helpers import resolve_absolute_path
+from khoj.utils.helpers import resolve_absolute_path
-from src.utils.rawconfig import ContentConfig, SearchConfig
+from khoj.utils.rawconfig import ContentConfig, SearchConfig
 
 
 # Test
@@ -91,7 +91,7 @@ def test_image_search_query_truncated(content_config: ContentConfig, search_conf
 
     # Act
     try:
-        with caplog.at_level(logging.INFO, logger="src.search_type.image_search"):
+        with caplog.at_level(logging.INFO, logger="khoj.search_type.image_search"):
             image_search.query(
                 query,
                 count = 1,
@@ -114,7 +114,7 @@ def test_image_search_by_filepath(content_config: ContentConfig, search_config:
     expected_image_path = f"{image_directory.joinpath('kitten_park.jpg')}"
 
     # Act
-    with caplog.at_level(logging.INFO, logger="src.search_type.image_search"):
+    with caplog.at_level(logging.INFO, logger="khoj.search_type.image_search"):
         hits = image_search.query(
             query,
             count = 1,

tests/test_markdown_to_jsonl.py

@@ -2,7 +2,7 @@
 import json
 
 # Internal Packages
-from src.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
+from khoj.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
 
 
 def test_markdown_file_with_no_headings_to_jsonl(tmp_path):