sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2025-02-17 08:04:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Index configured org files from khoj.el

Browse source 
- Set `khoj-org-files-index' to list of files to index
- Defaults to indexing org-agenda-files
- Uses khoj server api to configure org files to index
This commit is contained in:
Debanjum Singh Solanky 2023-03-26 18:44:10 +07:00
parent 50760acc37
commit cb40a96c85
1 changed files with 84 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

84
src/interface/emacs/khoj.el
View file

@ -217,6 +217,11 @@ for example), set this to the full interpreter path."
(member val '("python" "python3" "pythonw" "py")))
:group 'khoj)
(defcustom khoj-org-files-index (org-agenda-files t t)
"List of org-files to index on khoj server"
:group 'khoj
:type '(repeat string))
(defvar khoj--server-process nil "Track Khoj server process.")
(defvar khoj--server-name "*khoj-server*" "Track Khoj server buffer.")
(defvar khoj--server-ready? nil "Track if khoj server is ready to receive API calls.")
@ -304,6 +309,72 @@ for example), set this to the full interpreter path."
(when (not (khoj--server-running?))
(khoj--server-start)))
(defun khoj--get-content-index-dir (config type)
"Extract directory containing index files of specified content TYPE from CONFIG.
CONFIG is json obtained from Khoj config API."
(--> config
(cdr (assoc 'content-type it))
(cdr (assoc type it))
(cdr (assoc 'embeddings-file it))
(split-string it "/")
(butlast it)
(string-join it "/")))
(defun khoj--server-configure ()
"Configure the khoj server to index specified files."
(interactive)
(let* ((current-config
(with-temp-buffer
(url-insert-file-contents (format "%s/api/config/data" khoj-server-url))
(ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
(default-config
(with-temp-buffer
(url-insert-file-contents (format "%s/api/config/data/default" khoj-server-url))
(ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false))))
(default-index-dir (khoj--get-content-index-dir default-config 'org))
(config (or current-config default-config)))
(cond
;; If khoj backend is not configured yet
((not current-config)
(setq config (delq (assoc 'content-type config) config))
(setq config (delq (assoc 'processor config) config))
(add-to-list 'config `(content-type . ((org . ((input-files . ,khoj-org-files-index)
(input-filter . ,json-null)
(compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir))
(embeddings-file . ,(format "%s/org.pt" default-index-dir))
(index-heading-entries . ,json-false))))))
(khoj--post-new-config config)
(message "khoj.el: ⚙️ Generated new khoj server configuration."))
;; Else if khoj config has no org content config
((not (alist-get 'org (alist-get 'content-type config)))
(let ((new-content-type (alist-get 'content-type config)))
(setq new-content-type (delq (assoc 'org new-content-type) new-content-type))
(add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files-index)
(input-filter . ,json-null)
(compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir))
(embeddings-file . ,(format "%s/org.pt" default-index-dir))
(index-heading-entries . ,json-false))))
(setq config (delq (assoc 'content-type config) config))
(add-to-list 'config `(content-type . ,new-content-type)))
(khoj--post-new-config config)
(message "Khoj: ⚙️ Added org content to index on khoj server."))
;; Else if khoj is not configured to index org files
((not (equal (alist-get 'input-files (alist-get 'org (alist-get 'content-type config))) khoj-org-files-index))
(let* ((index-directory (khoj--get-content-index-dir config 'org))
(new-content-type (alist-get 'content-type config)))
(setq new-content-type (delq (assoc 'org new-content-type) new-content-type))
(add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files-index)
(input-filter . ,json-null)
(compressed-jsonl . ,(format "%s/org.jsonl.gz" index-directory))
(embeddings-file . ,(format "%s/org.pt" index-directory))
(index-heading-entries . ,json-false))))
(setq config (delq (assoc 'content-type config) config))
(add-to-list 'config `(content-type . ,new-content-type)))
(khoj--post-new-config config)
(message "Khoj: ⚙️ Updated org content in index on khoj server")))))
;; -----------------------------------------------
;; Extract and Render Entries of each Content Type
@ -413,6 +484,19 @@ for example), set this to the full interpreter path."
;; Query Khoj API
;; --------------
(defun khoj--post-new-config (config)
"Configure khoj server with provided CONFIG."
;; POST provided config to khoj server
(let ((url-request-method "POST")
(url-request-extra-headers '(("Content-Type" . "application/json")))
(url-request-data (json-encode-alist config))
(config-url (format "%s/api/config/data" khoj-server-url)))
(with-current-buffer (url-retrieve-synchronously config-url)
(buffer-string)))
;; Update index on khoj server after configuration update
(let ((khoj--server-ready? nil))
(url-retrieve (format "%s/api/update?t=org" khoj-server-url) #'identity)))
(defun khoj--get-enabled-content-types ()
"Get content types enabled for search from API."
(let ((config-url (format "%s/api/config/types" khoj-server-url))