Install, Start and Configure Khoj Server from Emacs

Merge pull request #193 from debanjum/simplify-khoj-server-setup-on-emacs

## Major Changes
- ae535a0 Configure Khoj chat using khoj.el by setting OpenAI API key in Emacs
- 82eb4bf Setup Khoj server on opening khoj.el
- 99d19dc Start Khoj server from Emacs using khoj.el
- c92d791 Install Khoj server from Emacs using khoj.el
  *This assumes you have python (<3.11) and pip installed in a system path*

### Sample Config
- Enable Khoj Chat by configuring you OpenAI API Key
- Specify Org Files, Directories to Index for Search (and Chat)
  By default, your org-agenda-files (include archive files)) are indexed
- Invoke khoj by calling `C-c s`

``` emacs-lisp
(use-package khoj
  :after org
  :straight (khoj
             :type git
             :host github
             :repo "debanjum/khoj"
             :files ("src/interface/emacs/khoj.el"))
  :bind ("C-c s" . 'khoj)
  :config (setq
           khoj-openai-api-key "<YOUR_OPENAI_API_KEY_FOR_KHOJ_CHAT>"
           khoj-org-directories '("~/docs/notes" "~/docs/journals")
           khoj-org-files '("~/docs/tasks.org" "~/docs/journal.org" "~/docs/archive.org")))
```

src/interface/emacs/khoj.el

@@ -65,6 +65,11 @@
   :group 'khoj
   :type 'string)
 
+(defcustom khoj-server-is-local t
+  "Is Khoj server on local machine?."
+  :group 'khoj
+  :type 'boolean)
+
 (defcustom khoj-image-width 156
   "Width of rendered images returned by Khoj."
   :group 'khoj
@@ -171,6 +176,305 @@ Use `which-key` if available, else display simple message in echo area"
                                 nil t t))
     (message "%s" (khoj--keybindings-info-message))))
 
+
+;; ----------------
+;; Khoj Setup
+;; ----------------
+(defcustom khoj-server-command
+  (or (executable-find "khoj")
+      (executable-find "khoj.exe")
+      "khoj")
+  "Command to interact with Khoj server."
+  :type 'string
+  :group 'khoj)
+
+(defcustom khoj-server-args '("--no-gui")
+  "Arguments to pass to Khoj server on startup."
+  :type '(repeat string)
+  :group 'khoj)
+
+(defcustom khoj-server-python-command
+  (if (equal system-type 'windows-nt)
+      (or (executable-find "py")
+          (executable-find "pythonw")
+          "python")
+    (if (executable-find "python")
+        "python"
+      ;; Fallback on systems where python is not
+      ;; symlinked to python3.
+      "python3"))
+  "The Python interpreter used for the Khoj server.
+
+Khoj will try to use the system interpreter if it exists. If you wish
+to use a specific python interpreter (from a virtual environment
+for example), set this to the full interpreter path."
+  :type '(choice (const :tag "python" "python")
+                 (const :tag "python3" "python3")
+                 (const :tag "pythonw (Python on Windows)" "pythonw")
+                 (const :tag "py (other Python on Windows)" "py")
+                 (string :tag "Other"))
+  :safe (lambda (val)
+          (member val '("python" "python3" "pythonw" "py")))
+  :group 'khoj)
+
+(defcustom khoj-org-files (org-agenda-files t t)
+  "List of org-files to index on khoj server."
+  :type '(repeat string)
+  :group 'khoj)
+
+(defcustom khoj-org-directories nil
+  "List of directories with org-mode files to index on khoj server."
+  :type '(repeat string)
+  :group 'khoj)
+
+(defcustom khoj-openai-api-key nil
+  "OpenAI API key used to configure chat on khoj server."
+  :type 'string
+  :group 'khoj)
+
+(defcustom khoj-auto-setup t
+  "Automate install, configure and start of khoj server.
+Auto invokes setup steps on calling main entrypoint."
+  :type 'string
+  :group 'khoj)
+
+(defvar khoj--server-process nil "Track khoj server process.")
+(defvar khoj--server-name "*khoj-server*" "Track khoj server buffer.")
+(defvar khoj--server-ready? nil "Track if khoj server is ready to receive API calls.")
+(defvar khoj--server-configured? t "Track if khoj server is configured to receive API calls.")
+(defvar khoj--progressbar '(🌑 🌘 🌗 🌖 🌕 🌔 🌓 🌒) "Track progress via moon phase animations.")
+
+(defun khoj--server-get-version ()
+  "Return the khoj server version."
+  (with-temp-buffer
+    (call-process khoj-server-command nil t nil "--version")
+    (goto-char (point-min))
+    (re-search-forward "\\([a-z0-9.]+\\)")
+    (match-string 1)))
+
+(defun khoj--server-install-upgrade ()
+  "Install or upgrade the khoj server."
+  (with-temp-buffer
+    (message "khoj.el: Installing server...")
+    (if (/= (apply 'call-process khoj-server-python-command
+                     nil t nil
+                     "-m" "pip" "install" "--upgrade"
+                     '("khoj-assistant"))
+            0)
+        (message "khoj.el: Failed to install Khoj server. Please install it manually using pip install `khoj-assistant'.\n%s" (buffer-string))
+      (message "khoj.el: Installed and upgraded Khoj server version: %s" (khoj--server-get-version)))))
+
+(defun khoj--server-start ()
+  "Start the khoj server."
+  (interactive)
+  (let* ((url-parts (split-string (cadr (split-string khoj-server-url "://")) ":"))
+         (server-host (nth 0 url-parts))
+         (server-port (or (nth 1 url-parts) "80"))
+         (server-args (append khoj-server-args
+                              (list (format "--host=%s" server-host)
+                                    (format "--port=%s" server-port)))))
+    (message "khoj.el: Starting server at %s %s..." server-host server-port)
+    (setq khoj--server-process
+          (make-process
+           :name khoj--server-name
+           :buffer khoj--server-name
+           :command (append (list khoj-server-command) server-args)
+           :sentinel (lambda (process event)
+                       (message "khoj.el: khoj server stopped with: %s" event)
+                       (setq khoj--server-ready? nil))
+           :filter (lambda (process msg)
+                     (cond ((string-match (format "Uvicorn running on %s" khoj-server-url) msg)
+                            (progn
+                              (setq khoj--server-ready? t)
+                              (khoj--server-configure)))
+                           ((string-match "Batches:  " msg)
+                            (when (string-match "\\([0-9]+\\.[0-9]+\\|\\([0-9]+\\)\\)%?" msg)
+                              (message "khoj.el: %s updating index %s"
+                                       (nth (% (string-to-number (match-string 1 msg)) (length khoj--progressbar)) khoj--progressbar)
+                                       (match-string 0 msg)))
+                            (setq khoj--server-configured? nil))
+                           ((and (not khoj--server-configured?)
+                                 (string-match "Processor reconfigured via API" msg))
+                            (setq khoj--server-configured? t))
+                           ((and (not khoj--server-ready?)
+                                 (or (string-match "configure.py" msg)
+                                     (string-match "main.py" msg)
+                                     (string-match "api.py" msg)))
+                            (dolist (line (split-string msg "\n"))
+                              (message "khoj.el: %s" (nth 1 (split-string msg "  " t " *"))))))
+                     ;; call default process filter to write output to process buffer
+                     (internal-default-process-filter process msg))))
+    (set-process-query-on-exit-flag khoj--server-process nil)
+    (when (not khoj--server-process)
+        (message "khoj.el: Failed to start Khoj server. Please start it manually by running `khoj' on terminal.\n%s" (buffer-string)))))
+
+(defun khoj--server-started? ()
+  "Check if the khoj server has been started."
+  ;; check for when server process handled from within emacs
+  (if (and khoj--server-process
+           (not (null (process-live-p khoj--server-process))))
+      t
+    ;; else general check via ping to khoj-server-url
+    (if (ignore-errors
+          (not (null (url-retrieve-synchronously (format "%s/api/config/data/default" khoj-server-url)))))
+        ;; Successful ping to non-emacs khoj server indicates it is started and ready.
+        ;; So update ready state tracker variable (and implicitly return true for started)
+        (setq khoj--server-ready? t)
+      nil)))
+
+(defun khoj--server-restart ()
+  "Restart the khoj server."
+  (interactive)
+  (khoj--server-stop)
+  (khoj--server-start))
+
+(defun khoj--server-stop ()
+  "Stop the khoj server."
+  (interactive)
+  (when (khoj--server-started?)
+    (message "khoj.el: Stopping server...")
+    (kill-process khoj--server-process)
+    (message "khoj.el: Stopped server.")))
+
+(defun khoj--server-setup ()
+  "Install and start the khoj server, if required."
+  (interactive)
+  ;; Install khoj server, if not available but expected on local machine
+  (when (and khoj-server-is-local
+             (or (not (executable-find khoj-server-command))
+                 (not (khoj--server-get-version))))
+      (khoj--server-install-upgrade))
+  ;; Start khoj server if not already started
+  (when (not (khoj--server-started?))
+    (khoj--server-start)))
+
+(defun khoj--get-directory-from-config (config keys &optional level)
+  "Extract directory under specified KEYS in CONFIG and trim it to LEVEL.
+CONFIG is json obtained from Khoj config API."
+  (let ((item config))
+    (dolist (key keys)
+      (setq item (cdr (assoc key item))))
+      (-> item
+          (split-string "/")
+          (butlast (or level nil))
+          (string-join "/"))))
+
+(defun khoj--server-configure ()
+  "Configure the the Khoj server for search and chat."
+  (interactive)
+  (let* ((org-directory-regexes (or (mapcar (lambda (dir) (format "%s/**/*.org" dir)) khoj-org-directories) json-null))
+         (current-config
+          (with-temp-buffer
+            (url-insert-file-contents (format "%s/api/config/data" khoj-server-url))
+            (ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
+         (default-config
+           (with-temp-buffer
+             (url-insert-file-contents (format "%s/api/config/data/default" khoj-server-url))
+             (ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
+         (default-index-dir (khoj--get-directory-from-config default-config '(content-type org embeddings-file)))
+         (default-chat-dir (khoj--get-directory-from-config default-config '(processor conversation conversation-logfile)))
+         (default-model (or (alist-get 'model (alist-get 'conversation (alist-get 'processor default-config))) "text-davinci-003"))
+         (config (or current-config default-config)))
+
+    ;; Configure content types
+    (cond
+     ;; If khoj backend is not configured yet
+     ((not current-config)
+      (setq config (delq (assoc 'content-type config) config))
+      (add-to-list 'config
+                   `(content-type . ((org . ((input-files . ,khoj-org-files)
+                                             (input-filter . ,org-directory-regexes)
+                                             (compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir))
+                                             (embeddings-file . ,(format "%s/org.pt" default-index-dir))
+                                             (index-heading-entries . ,json-false)))))))
+
+     ;; Else if khoj config has no org content config
+     ((not (alist-get 'org (alist-get 'content-type config)))
+      (let ((new-content-type (alist-get 'content-type config)))
+        (setq new-content-type (delq (assoc 'org new-content-type) new-content-type))
+        (add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files)
+                                                (input-filter . ,org-directory-regexes)
+                                                (compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir))
+                                                (embeddings-file . ,(format "%s/org.pt" default-index-dir))
+                                                (index-heading-entries . ,json-false))))
+        (setq config (delq (assoc 'content-type config) config))
+        (add-to-list 'config `(content-type . ,new-content-type))))
+
+     ;; Else if khoj is not configured to index specified org files
+     ((not (and (equal (alist-get 'input-files (alist-get 'org (alist-get 'content-type config))) khoj-org-files)
+                (equal (alist-get 'input-filter (alist-get 'org (alist-get 'content-type config))) org-directory-regexes)))
+      (let* ((index-directory (khoj--get-directory-from-config config '(content-type org embeddings-file)))
+             (new-content-type (alist-get 'content-type config)))
+        (setq new-content-type (delq (assoc 'org new-content-type) new-content-type))
+        (add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files)
+                                                (input-filter . ,org-directory-regexes)
+                                                (compressed-jsonl . ,(format "%s/org.jsonl.gz" index-directory))
+                                                (embeddings-file . ,(format "%s/org.pt" index-directory))
+                                                (index-heading-entries . ,json-false))))
+        (setq config (delq (assoc 'content-type config) config))
+        (add-to-list 'config `(content-type . ,new-content-type)))))
+
+    ;; Configure processors
+    (cond
+     ((not khoj-openai-api-key)
+      (setq config (delq (assoc 'processor config) config)))
+
+     ((not current-config)
+      (setq config (delq (assoc 'processor config) config))
+      (add-to-list 'config
+                   `(processor . ((conversation . ((conversation-logfile . ,(format "%s/conversation.json" default-chat-dir))
+                                                   (model . ,default-model)
+                                                   (openai-api-key . ,khoj-openai-api-key)))))))
+
+     ((not (alist-get 'conversation (alist-get 'processor config)))
+       (let ((new-processor-type (alist-get 'processor config)))
+         (setq new-processor-type (delq (assoc 'conversation new-processor-type) new-processor-type))
+         (add-to-list 'new-processor-type `(conversation . ((conversation-logfile . ,(format "%s/conversation.json" default-chat-dir))
+                                                            (model . ,default-model)
+                                                            (openai-api-key . ,khoj-openai-api-key))))
+        (setq config (delq (assoc 'processor config) config))
+        (add-to-list 'config `(processor . ,new-processor-type))))
+
+     ;; Else if khoj is not configured with specified openai api key
+     ((not (equal (alist-get 'openai-api-key (alist-get 'conversation (alist-get 'processor config))) khoj-openai-api-key))
+      (let* ((chat-directory (khoj--get-directory-from-config config '(processor conversation conversation-logfile)))
+             (model-name (khoj--get-directory-from-config config '(processor conversation model)))
+             (new-processor-type (alist-get 'processor config)))
+        (setq new-processor-type (delq (assoc 'conversation new-processor-type) new-processor-type))
+        (add-to-list 'new-processor-type `(conversation . ((conversation-logfile . ,(format "%s/conversation.json" chat-directory))
+                                                           (model . ,model-name)
+                                                           (openai-api-key . ,khoj-openai-api-key))))
+        (setq config (delq (assoc 'processor config) config))
+        (add-to-list 'config `(processor . ,new-processor-type)))))
+
+     ;; Update server with latest configuration
+     (khoj--post-new-config config)
+     (cond ((not current-config)
+            (message "khoj.el: ⚙️ Generated new khoj server configuration."))
+           ((not (equal config current-config))
+            (message "Khoj: ⚙️ Updated khoj server configuration")))))
+
+(defun khoj-setup (&optional interact)
+  "Install, start and configure Khoj server."
+  (interactive "p")
+  ;; Setup khoj server if not running
+  (let* ((not-started (not (khoj--server-started?)))
+         (permitted (if (and not-started interact)
+                        (y-or-n-p "Could not connect to Khoj server. Should I install, start and configure it for you?")
+                      t)))
+    ;; Install, start server if user permitted and server not ready
+    (when (and permitted not-started)
+      (khoj--server-setup))
+
+    ;; Server can be started but not ready (to use/configure)
+    ;; Wait until server is ready if setup was permitted
+    (while (and permitted (not khoj--server-ready?))
+      (sit-for 0.5))
+
+    ;; Configure server once server ready if user permitted
+    (when permitted
+      (khoj--server-configure))))
+
 
 ;; -----------------------------------------------
 ;; Extract and Render Entries of each Content Type
@@ -196,22 +500,21 @@ Use `which-key` if available, else display simple message in echo area"
 
 (defun khoj--extract-entries-as-org (json-response query)
   "Convert JSON-RESPONSE, QUERY from API to `org-mode' entries."
-  (let ((org-results-buffer-format-str "* %s\n%s\n#+STARTUP: showall hidestars inlineimages"))
+  (thread-last
-    (thread-last
+    json-response
-      json-response
+    ;; Extract and render each org-mode entry from response
-      ;; Extract and render each org-mode entry from response
+    (mapcar (lambda (json-response-item)
-      (mapcar (lambda (json-response-item)
+              (thread-last
-                (thread-last
+                ;; Extract org entry from each item in json response
-                  ;; Extract org entry from each item in json response
+                (cdr (assoc 'entry json-response-item))
-                  (cdr (assoc 'entry json-response-item))
+                ;; Format org entry as a string
-                  ;; Format org entry as a string
+                (format "%s")
-                  (format "%s")
+                ;; Standardize results to 2nd level heading for consistent rendering
-                  ;; Standardize results to 2nd level heading for consistent rendering
+                (replace-regexp-in-string "^\*+" "**"))))
-                  (replace-regexp-in-string "^\*+" "**"))))
+    ;; Render entries into org formatted string with query set as as top level heading
-      ;; Render entries into org formatted string with query set as as top level heading
+    (format "* %s\n%s\n" query)
-      (format org-results-buffer-format-str query)
+    ;; remove leading (, ) or SPC from extracted entries string
-      ;; remove leading (, ) or SPC from extracted entries string
+    (replace-regexp-in-string "^[\(\) ]" "")))
-      (replace-regexp-in-string "^[\(\) ]" ""))))
 
 (defun khoj--extract-entries-as-ledger (json-response query)
   "Convert JSON-RESPONSE, QUERY from API to ledger entries."
@@ -281,12 +584,24 @@ Use `which-key` if available, else display simple message in echo area"
 ;; Query Khoj API
 ;; --------------
 
+(defun khoj--post-new-config (config)
+  "Configure khoj server with provided CONFIG."
+  ;; POST provided config to khoj server
+  (let ((url-request-method "POST")
+        (url-request-extra-headers '(("Content-Type" . "application/json")))
+        (url-request-data (json-encode-alist config))
+        (config-url (format "%s/api/config/data" khoj-server-url)))
+    (with-current-buffer (url-retrieve-synchronously config-url)
+      (buffer-string)))
+  ;; Update index on khoj server after configuration update
+  (let ((khoj--server-ready? nil))
+    (url-retrieve (format "%s/api/update?t=org" khoj-server-url) #'identity)))
+
 (defun khoj--get-enabled-content-types ()
   "Get content types enabled for search from API."
   (let ((config-url (format "%s/api/config/types" khoj-server-url))
         (url-request-method "GET"))
     (with-temp-buffer
-      (erase-buffer)
       (url-insert-file-contents config-url)
       (thread-last
         (json-parse-buffer :object-type 'alist)
@@ -319,8 +634,13 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
              ((equal content-type "ledger") (khoj--extract-entries-as-ledger json-response query))
              ((equal content-type "image") (khoj--extract-entries-as-images json-response query))
              (t (khoj--extract-entries json-response query))))
-      (cond ((equal content-type "org") (progn (org-mode)
+      (cond ((equal content-type "org") (progn (visual-line-mode)
-                                               (visual-line-mode)))
+                                               (org-mode)
+                                               (setq-local
+                                                org-startup-folded "showall"
+                                                org-hide-leading-stars t
+                                                org-startup-with-inline-images t)
+                                               (org-set-startup-visibility)))
             ((equal content-type "markdown") (progn (markdown-mode)
                                                     (visual-line-mode)))
             ((equal content-type "ledger") (beancount-mode))
@@ -428,9 +748,14 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
          (encoded-query (url-hexify-string query))
          (query-url (format "%s/api/chat?q=%s" khoj-server-url encoded-query)))
     (with-temp-buffer
-      (erase-buffer)
+      (condition-case ex
-      (url-insert-file-contents query-url)
+          (progn
-      (json-parse-buffer :object-type 'alist))))
+            (url-insert-file-contents query-url)
+            (json-parse-buffer :object-type 'alist))
+        ('file-error (cond ((string-match "Internal server error" (nth 2 ex))
+                      (message "Chat processor not configured. Configure OpenAI API key and restart it. Exception: [%s]" ex))
+                     (t (message "Chat exception: [%s]" ex))))))))
+
 
 (defun khoj--render-chat-message (message sender &optional receive-date)
   "Render chat messages as `org-mode' list item.
@@ -671,7 +996,7 @@ Paragraph only starts at first text after blank line."
   (interactive (list (transient-args transient-current-command)))
   (khoj--chat))
 
-(transient-define-prefix khoj-menu ()
+(transient-define-prefix khoj--menu ()
   "Create Khoj Menu to Configure and Execute Commands."
   [["Configure Search"
     ("n" "Results Count" "--results-count=" :init-value (lambda (obj) (oset obj value (format "%s" khoj-results-count))))
@@ -692,9 +1017,11 @@ Paragraph only starts at first text after blank line."
 
 ;;;###autoload
 (defun khoj ()
-  "Natural, Incremental Search for your personal notes, transactions and images."
+  "Provide natural, search assistance for your notes, transactions and images."
   (interactive)
-  (khoj-menu))
+  (when khoj-auto-setup
+    (khoj-setup t))
+  (khoj--menu))
 
 (provide 'khoj)
 

src/interface/emacs/tests/khoj-tests.el

@@ -109,8 +109,7 @@ Penance to Immortality\n\
 ** Act\n\
 \n\
 Rule everything\n\
-\n\
+\n"))))
-#+STARTUP: showall hidestars inlineimages"))))
 
 
 (ert-deftest khoj-tests--extract-entries-as-ledger ()

src/khoj/routers/api.py

@@ -33,6 +33,12 @@ def get_default_config_data():
 @api.get("/config/types", response_model=List[str])
 def get_config_types():
     """Get configured content types"""
+    if state.config is None or state.config.content_type is None:
+        raise HTTPException(
+            status_code=500,
+            detail="Content types not configured. Configure at least one content type on server and restart it.",
+        )
+
     configured_content_types = state.config.content_type.dict(exclude_none=True)
     return [
         search_type.value
@@ -190,6 +196,15 @@ def update(t: Optional[SearchType] = None, force: Optional[bool] = False):
 
 @api.get("/chat")
 def chat(q: Optional[str] = None):
+    if (
+        state.processor_config is None
+        or state.processor_config.conversation is None
+        or state.processor_config.conversation.openai_api_key is None
+    ):
+        raise HTTPException(
+            status_code=500, detail="Chat processor not configured. Configure OpenAI API key on server and restart it."
+        )
+
     # Initialize Variables
     api_key = state.processor_config.conversation.openai_api_key
     model = state.processor_config.conversation.model

src/khoj/utils/constants.py

@@ -56,6 +56,7 @@ default_config = {
     "processor": {
         "conversation": {
             "openai-api-key": None,
+            "model": "text-davinci-003",
             "conversation-logfile": "~/.khoj/processor/conversation/conversation_logs.json",
         }
     },