sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-11-23 23:48:56 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 16

Install, Start and Configure Khoj Server from Emacs

Browse source 
Merge pull request #193 from debanjum/simplify-khoj-server-setup-on-emacs

## Major Changes
- ae535a0 Configure Khoj chat using khoj.el by setting OpenAI API key in Emacs
- 82eb4bf Setup Khoj server on opening khoj.el
- 99d19dc Start Khoj server from Emacs using khoj.el
- c92d791 Install Khoj server from Emacs using khoj.el
  *This assumes you have python (<3.11) and pip installed in a system path*

### Sample Config
- Enable Khoj Chat by configuring you OpenAI API Key
- Specify Org Files, Directories to Index for Search (and Chat)
  By default, your org-agenda-files (include archive files)) are indexed
- Invoke khoj by calling `C-c s`

``` emacs-lisp
(use-package khoj
  :after org
  :straight (khoj
             :type git
             :host github
             :repo "debanjum/khoj"
             :files ("src/interface/emacs/khoj.el"))
  :bind ("C-c s" . 'khoj)
  :config (setq
           khoj-openai-api-key "<YOUR_OPENAI_API_KEY_FOR_KHOJ_CHAT>"
           khoj-org-directories '("~/docs/notes" "~/docs/journals")
           khoj-org-files '("~/docs/tasks.org" "~/docs/journal.org" "~/docs/archive.org")))
```
This commit is contained in:
Debanjum 2023-03-27 18:49:43 +07:00 committed by GitHub
commit 87090531da
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 369 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

377
src/interface/emacs/khoj.el
View file

@ -65,6 +65,11 @@
:group 'khoj
:type 'string)
(defcustom khoj-server-is-local t
"Is Khoj server on local machine?."
:group 'khoj
:type 'boolean)
(defcustom khoj-image-width 156
"Width of rendered images returned by Khoj."
:group 'khoj
@ -171,6 +176,305 @@ Use `which-key` if available, else display simple message in echo area"
nil t t))
(message "%s" (khoj--keybindings-info-message))))
;; ----------------
;; Khoj Setup
;; ----------------
(defcustom khoj-server-command
(or (executable-find "khoj")
(executable-find "khoj.exe")
"khoj")
"Command to interact with Khoj server."
:type 'string
:group 'khoj)
(defcustom khoj-server-args '("--no-gui")
"Arguments to pass to Khoj server on startup."
:type '(repeat string)
:group 'khoj)
(defcustom khoj-server-python-command
(if (equal system-type 'windows-nt)
(or (executable-find "py")
(executable-find "pythonw")
"python")
(if (executable-find "python")
"python"
;; Fallback on systems where python is not
;; symlinked to python3.
"python3"))
"The Python interpreter used for the Khoj server.
Khoj will try to use the system interpreter if it exists. If you wish
to use a specific python interpreter (from a virtual environment
for example), set this to the full interpreter path."
:type '(choice (const :tag "python" "python")
(const :tag "python3" "python3")
(const :tag "pythonw (Python on Windows)" "pythonw")
(const :tag "py (other Python on Windows)" "py")
(string :tag "Other"))
:safe (lambda (val)
(member val '("python" "python3" "pythonw" "py")))
:group 'khoj)
(defcustom khoj-org-files (org-agenda-files t t)
"List of org-files to index on khoj server."
:type '(repeat string)
:group 'khoj)
(defcustom khoj-org-directories nil
"List of directories with org-mode files to index on khoj server."
:type '(repeat string)
:group 'khoj)
(defcustom khoj-openai-api-key nil
"OpenAI API key used to configure chat on khoj server."
:type 'string
:group 'khoj)
(defcustom khoj-auto-setup t
"Automate install, configure and start of khoj server.
Auto invokes setup steps on calling main entrypoint."
:type 'string
:group 'khoj)
(defvar khoj--server-process nil "Track khoj server process.")
(defvar khoj--server-name "*khoj-server*" "Track khoj server buffer.")
(defvar khoj--server-ready? nil "Track if khoj server is ready to receive API calls.")
(defvar khoj--server-configured? t "Track if khoj server is configured to receive API calls.")
(defvar khoj--progressbar '(🌑 🌘 🌗 🌖 🌕 🌔 🌓 🌒) "Track progress via moon phase animations.")
(defun khoj--server-get-version ()
"Return the khoj server version."
(with-temp-buffer
(call-process khoj-server-command nil t nil "--version")
(goto-char (point-min))
(re-search-forward "\\([a-z0-9.]+\\)")
(match-string 1)))
(defun khoj--server-install-upgrade ()
"Install or upgrade the khoj server."
(with-temp-buffer
(message "khoj.el: Installing server...")
(if (/= (apply 'call-process khoj-server-python-command
nil t nil
"-m" "pip" "install" "--upgrade"
'("khoj-assistant"))
0)
(message "khoj.el: Failed to install Khoj server. Please install it manually using pip install `khoj-assistant'.\n%s" (buffer-string))
(message "khoj.el: Installed and upgraded Khoj server version: %s" (khoj--server-get-version)))))
(defun khoj--server-start ()
"Start the khoj server."
(interactive)
(let* ((url-parts (split-string (cadr (split-string khoj-server-url "://")) ":"))
(server-host (nth 0 url-parts))
(server-port (or (nth 1 url-parts) "80"))
(server-args (append khoj-server-args
(list (format "--host=%s" server-host)
(format "--port=%s" server-port)))))
(message "khoj.el: Starting server at %s %s..." server-host server-port)
(setq khoj--server-process
(make-process
:name khoj--server-name
:buffer khoj--server-name
:command (append (list khoj-server-command) server-args)
:sentinel (lambda (process event)
(message "khoj.el: khoj server stopped with: %s" event)
(setq khoj--server-ready? nil))
:filter (lambda (process msg)
(cond ((string-match (format "Uvicorn running on %s" khoj-server-url) msg)
(progn
(setq khoj--server-ready? t)
(khoj--server-configure)))
((string-match "Batches: " msg)
(when (string-match "\\([0-9]+\\.[0-9]+\\|\\([0-9]+\\)\\)%?" msg)
(message "khoj.el: %s updating index %s"
(nth (% (string-to-number (match-string 1 msg)) (length khoj--progressbar)) khoj--progressbar)
(match-string 0 msg)))
(setq khoj--server-configured? nil))
((and (not khoj--server-configured?)
(string-match "Processor reconfigured via API" msg))
(setq khoj--server-configured? t))
((and (not khoj--server-ready?)
(or (string-match "configure.py" msg)
(string-match "main.py" msg)
(string-match "api.py" msg)))
(dolist (line (split-string msg "\n"))
(message "khoj.el: %s" (nth 1 (split-string msg " " t " *"))))))
;; call default process filter to write output to process buffer
(internal-default-process-filter process msg))))
(set-process-query-on-exit-flag khoj--server-process nil)
(when (not khoj--server-process)
(message "khoj.el: Failed to start Khoj server. Please start it manually by running `khoj' on terminal.\n%s" (buffer-string)))))
(defun khoj--server-started? ()
"Check if the khoj server has been started."
;; check for when server process handled from within emacs
(if (and khoj--server-process
(not (null (process-live-p khoj--server-process))))
t
;; else general check via ping to khoj-server-url
(if (ignore-errors
(not (null (url-retrieve-synchronously (format "%s/api/config/data/default" khoj-server-url)))))
;; Successful ping to non-emacs khoj server indicates it is started and ready.
;; So update ready state tracker variable (and implicitly return true for started)
(setq khoj--server-ready? t)
nil)))
(defun khoj--server-restart ()
"Restart the khoj server."
(interactive)
(khoj--server-stop)
(khoj--server-start))
(defun khoj--server-stop ()
"Stop the khoj server."
(interactive)
(when (khoj--server-started?)
(message "khoj.el: Stopping server...")
(kill-process khoj--server-process)
(message "khoj.el: Stopped server.")))
(defun khoj--server-setup ()
"Install and start the khoj server, if required."
(interactive)
;; Install khoj server, if not available but expected on local machine
(when (and khoj-server-is-local
(or (not (executable-find khoj-server-command))
(not (khoj--server-get-version))))
(khoj--server-install-upgrade))
;; Start khoj server if not already started
(when (not (khoj--server-started?))
(khoj--server-start)))
(defun khoj--get-directory-from-config (config keys &optional level)
"Extract directory under specified KEYS in CONFIG and trim it to LEVEL.
CONFIG is json obtained from Khoj config API."
(let ((item config))
(dolist (key keys)
(setq item (cdr (assoc key item))))
(-> item
(split-string "/")
(butlast (or level nil))
(string-join "/"))))
(defun khoj--server-configure ()
"Configure the the Khoj server for search and chat."
(interactive)
(let* ((org-directory-regexes (or (mapcar (lambda (dir) (format "%s/**/*.org" dir)) khoj-org-directories) json-null))
(current-config
(with-temp-buffer
(url-insert-file-contents (format "%s/api/config/data" khoj-server-url))
(ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
(default-config
(with-temp-buffer
(url-insert-file-contents (format "%s/api/config/data/default" khoj-server-url))
(ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
(default-index-dir (khoj--get-directory-from-config default-config '(content-type org embeddings-file)))
(default-chat-dir (khoj--get-directory-from-config default-config '(processor conversation conversation-logfile)))
(default-model (or (alist-get 'model (alist-get 'conversation (alist-get 'processor default-config))) "text-davinci-003"))
(config (or current-config default-config)))
;; Configure content types
(cond
;; If khoj backend is not configured yet
((not current-config)
(setq config (delq (assoc 'content-type config) config))
(add-to-list 'config
`(content-type . ((org . ((input-files . ,khoj-org-files)
(input-filter . ,org-directory-regexes)
(compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir))
(embeddings-file . ,(format "%s/org.pt" default-index-dir))
(index-heading-entries . ,json-false)))))))
;; Else if khoj config has no org content config
((not (alist-get 'org (alist-get 'content-type config)))
(let ((new-content-type (alist-get 'content-type config)))
(setq new-content-type (delq (assoc 'org new-content-type) new-content-type))
(add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files)
(input-filter . ,org-directory-regexes)
(compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir))
(embeddings-file . ,(format "%s/org.pt" default-index-dir))
(index-heading-entries . ,json-false))))
(setq config (delq (assoc 'content-type config) config))
(add-to-list 'config `(content-type . ,new-content-type))))
;; Else if khoj is not configured to index specified org files
((not (and (equal (alist-get 'input-files (alist-get 'org (alist-get 'content-type config))) khoj-org-files)
(equal (alist-get 'input-filter (alist-get 'org (alist-get 'content-type config))) org-directory-regexes)))
(let* ((index-directory (khoj--get-directory-from-config config '(content-type org embeddings-file)))
(new-content-type (alist-get 'content-type config)))
(setq new-content-type (delq (assoc 'org new-content-type) new-content-type))
(add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files)
(input-filter . ,org-directory-regexes)
(compressed-jsonl . ,(format "%s/org.jsonl.gz" index-directory))
(embeddings-file . ,(format "%s/org.pt" index-directory))
(index-heading-entries . ,json-false))))
(setq config (delq (assoc 'content-type config) config))
(add-to-list 'config `(content-type . ,new-content-type)))))
;; Configure processors
(cond
((not khoj-openai-api-key)
(setq config (delq (assoc 'processor config) config)))
((not current-config)
(setq config (delq (assoc 'processor config) config))
(add-to-list 'config
`(processor . ((conversation . ((conversation-logfile . ,(format "%s/conversation.json" default-chat-dir))
(model . ,default-model)
(openai-api-key . ,khoj-openai-api-key)))))))
((not (alist-get 'conversation (alist-get 'processor config)))
(let ((new-processor-type (alist-get 'processor config)))
(setq new-processor-type (delq (assoc 'conversation new-processor-type) new-processor-type))
(add-to-list 'new-processor-type `(conversation . ((conversation-logfile . ,(format "%s/conversation.json" default-chat-dir))
(model . ,default-model)
(openai-api-key . ,khoj-openai-api-key))))
(setq config (delq (assoc 'processor config) config))
(add-to-list 'config `(processor . ,new-processor-type))))
;; Else if khoj is not configured with specified openai api key
((not (equal (alist-get 'openai-api-key (alist-get 'conversation (alist-get 'processor config))) khoj-openai-api-key))
(let* ((chat-directory (khoj--get-directory-from-config config '(processor conversation conversation-logfile)))
(model-name (khoj--get-directory-from-config config '(processor conversation model)))
(new-processor-type (alist-get 'processor config)))
(setq new-processor-type (delq (assoc 'conversation new-processor-type) new-processor-type))
(add-to-list 'new-processor-type `(conversation . ((conversation-logfile . ,(format "%s/conversation.json" chat-directory))
(model . ,model-name)
(openai-api-key . ,khoj-openai-api-key))))
(setq config (delq (assoc 'processor config) config))
(add-to-list 'config `(processor . ,new-processor-type)))))
;; Update server with latest configuration
(khoj--post-new-config config)
(cond ((not current-config)
(message "khoj.el: ⚙️ Generated new khoj server configuration."))
((not (equal config current-config))
(message "Khoj: ⚙️ Updated khoj server configuration")))))
(defun khoj-setup (&optional interact)
"Install, start and configure Khoj server."
(interactive "p")
;; Setup khoj server if not running
(let* ((not-started (not (khoj--server-started?)))
(permitted (if (and not-started interact)
(y-or-n-p "Could not connect to Khoj server. Should I install, start and configure it for you?")
t)))
;; Install, start server if user permitted and server not ready
(when (and permitted not-started)
(khoj--server-setup))
;; Server can be started but not ready (to use/configure)
;; Wait until server is ready if setup was permitted
(while (and permitted (not khoj--server-ready?))
(sit-for 0.5))
;; Configure server once server ready if user permitted
(when permitted
(khoj--server-configure))))
;; -----------------------------------------------
;; Extract and Render Entries of each Content Type
@ -196,22 +500,21 @@ Use `which-key` if available, else display simple message in echo area"
(defun khoj--extract-entries-as-org (json-response query)
"Convert JSON-RESPONSE, QUERY from API to `org-mode' entries."
(let ((org-results-buffer-format-str "* %s\n%s\n#+STARTUP: showall hidestars inlineimages"))
(thread-last
json-response
;; Extract and render each org-mode entry from response
(mapcar (lambda (json-response-item)
(thread-last
;; Extract org entry from each item in json response
(cdr (assoc 'entry json-response-item))
;; Format org entry as a string
(format "%s")
;; Standardize results to 2nd level heading for consistent rendering
(replace-regexp-in-string "^\*+" "**"))))
;; Render entries into org formatted string with query set as as top level heading
(format org-results-buffer-format-str query)
;; remove leading (, ) or SPC from extracted entries string
(replace-regexp-in-string "^[\(\) ]" ""))))
(thread-last
json-response
;; Extract and render each org-mode entry from response
(mapcar (lambda (json-response-item)
(thread-last
;; Extract org entry from each item in json response
(cdr (assoc 'entry json-response-item))
;; Format org entry as a string
(format "%s")
;; Standardize results to 2nd level heading for consistent rendering
(replace-regexp-in-string "^\*+" "**"))))
;; Render entries into org formatted string with query set as as top level heading
(format "* %s\n%s\n" query)
;; remove leading (, ) or SPC from extracted entries string
(replace-regexp-in-string "^[\(\) ]" "")))
(defun khoj--extract-entries-as-ledger (json-response query)
"Convert JSON-RESPONSE, QUERY from API to ledger entries."
@ -281,12 +584,24 @@ Use `which-key` if available, else display simple message in echo area"
;; Query Khoj API
;; --------------
(defun khoj--post-new-config (config)
"Configure khoj server with provided CONFIG."
;; POST provided config to khoj server
(let ((url-request-method "POST")
(url-request-extra-headers '(("Content-Type" . "application/json")))
(url-request-data (json-encode-alist config))
(config-url (format "%s/api/config/data" khoj-server-url)))
(with-current-buffer (url-retrieve-synchronously config-url)
(buffer-string)))
;; Update index on khoj server after configuration update
(let ((khoj--server-ready? nil))
(url-retrieve (format "%s/api/update?t=org" khoj-server-url) #'identity)))
(defun khoj--get-enabled-content-types ()
"Get content types enabled for search from API."
(let ((config-url (format "%s/api/config/types" khoj-server-url))
(url-request-method "GET"))
(with-temp-buffer
(erase-buffer)
(url-insert-file-contents config-url)
(thread-last
(json-parse-buffer :object-type 'alist)
@ -319,8 +634,13 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
((equal content-type "ledger") (khoj--extract-entries-as-ledger json-response query))
((equal content-type "image") (khoj--extract-entries-as-images json-response query))
(t (khoj--extract-entries json-response query))))
(cond ((equal content-type "org") (progn (org-mode)
(visual-line-mode)))
(cond ((equal content-type "org") (progn (visual-line-mode)
(org-mode)
(setq-local
org-startup-folded "showall"
org-hide-leading-stars t
org-startup-with-inline-images t)
(org-set-startup-visibility)))
((equal content-type "markdown") (progn (markdown-mode)
(visual-line-mode)))
((equal content-type "ledger") (beancount-mode))
@ -428,9 +748,14 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
(encoded-query (url-hexify-string query))
(query-url (format "%s/api/chat?q=%s" khoj-server-url encoded-query)))
(with-temp-buffer
(erase-buffer)
(url-insert-file-contents query-url)
(json-parse-buffer :object-type 'alist))))
(condition-case ex
(progn
(url-insert-file-contents query-url)
(json-parse-buffer :object-type 'alist))
('file-error (cond ((string-match "Internal server error" (nth 2 ex))
(message "Chat processor not configured. Configure OpenAI API key and restart it. Exception: [%s]" ex))
(t (message "Chat exception: [%s]" ex))))))))
(defun khoj--render-chat-message (message sender &optional receive-date)
"Render chat messages as `org-mode' list item.
@ -671,7 +996,7 @@ Paragraph only starts at first text after blank line."
(interactive (list (transient-args transient-current-command)))
(khoj--chat))
(transient-define-prefix khoj-menu ()
(transient-define-prefix khoj--menu ()
"Create Khoj Menu to Configure and Execute Commands."
[["Configure Search"
("n" "Results Count" "--results-count=" :init-value (lambda (obj) (oset obj value (format "%s" khoj-results-count))))
@ -692,9 +1017,11 @@ Paragraph only starts at first text after blank line."
;;;###autoload
(defun khoj ()
"Natural, Incremental Search for your personal notes, transactions and images."
"Provide natural, search assistance for your notes, transactions and images."
(interactive)
(khoj-menu))
(when khoj-auto-setup
(khoj-setup t))
(khoj--menu))
(provide 'khoj)

3
src/interface/emacs/tests/khoj-tests.el
View file

@ -109,8 +109,7 @@ Penance to Immortality\n\
** Act\n\
\n\
Rule everything\n\
\n\
#+STARTUP: showall hidestars inlineimages"))))
\n"))))
(ert-deftest khoj-tests--extract-entries-as-ledger ()

15
src/khoj/routers/api.py
View file

@ -33,6 +33,12 @@ def get_default_config_data():
@api.get("/config/types", response_model=List[str])
def get_config_types():
"""Get configured content types"""
if state.config is None or state.config.content_type is None:
raise HTTPException(
status_code=500,
detail="Content types not configured. Configure at least one content type on server and restart it.",
)
configured_content_types = state.config.content_type.dict(exclude_none=True)
return [
search_type.value
@ -190,6 +196,15 @@ def update(t: Optional[SearchType] = None, force: Optional[bool] = False):
@api.get("/chat")
def chat(q: Optional[str] = None):
if (
state.processor_config is None
or state.processor_config.conversation is None
or state.processor_config.conversation.openai_api_key is None
):
raise HTTPException(
status_code=500, detail="Chat processor not configured. Configure OpenAI API key on server and restart it."
)
# Initialize Variables
api_key = state.processor_config.conversation.openai_api_key
model = state.processor_config.conversation.model

1
src/khoj/utils/constants.py
View file

@ -56,6 +56,7 @@ default_config = {
"processor": {
"conversation": {
"openai-api-key": None,
"model": "text-davinci-003",
"conversation-logfile": "~/.khoj/processor/conversation/conversation_logs.json",
}
},