sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2025-02-17 16:14:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Use `-difference' to get files to delete. Make batch size defcustom

Browse source 
Improve docstrings to align with `checkdoc' requirement for all args
being mentioned
This commit is contained in:
Debanjum Singh Solanky 2024-06-05 18:52:26 +05:30
parent bfacd65971
commit 21987f60c7
1 changed files with 13 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/interface/emacs/khoj.el
View file

@ -108,6 +108,11 @@
:group 'khoj :group 'khoj
:type 'number) :type 'number)
(defcustom khoj-index-files-batch 30
"Number of files to send for indexing in each request."
:group 'khoj
:type 'number)
(defcustom khoj-default-content-type "org" (defcustom khoj-default-content-type "org"
"The default content type to perform search on." "The default content type to perform search on."
:group 'khoj :group 'khoj
@ -416,29 +421,20 @@ Auto invokes setup steps on calling main entrypoint."
(files-to-index (or file-paths (files-to-index (or file-paths
(append (mapcan (lambda (dir) (directory-files-recursively dir "\\.\\(org\\|md\\|markdown\\|pdf\\|txt\\|rst\\|xml\\|htm\\|html\\)$")) content-directories) content-files))) (append (mapcan (lambda (dir) (directory-files-recursively dir "\\.\\(org\\|md\\|markdown\\|pdf\\|txt\\|rst\\|xml\\|htm\\|html\\)$")) content-directories) content-files)))
(type-query (if (or (equal content-type "all") (not content-type)) "" (format "t=%s" content-type))) (type-query (if (or (equal content-type "all") (not content-type)) "" (format "t=%s" content-type)))
(delete-files (khoj--get-delete-file-list khoj--indexed-files files-to-index)) (delete-files (-difference khoj--indexed-files files-to-index))
(inhibit-message t) (inhibit-message t)
(message-log-max nil) (message-log-max nil)
(batch-size 30)) (batch-size khoj-index-files-batch))
(dolist (files (-partition-all batch-size files-to-index)) (dolist (files (-partition-all batch-size files-to-index))
(khoj--send-index-update-request (khoj--render-update-files-as-request-body files boundary) boundary content-type type-query force)) (khoj--send-index-update-request (khoj--render-update-files-as-request-body files boundary) boundary content-type type-query force))
(when delete-files (when delete-files
(khoj--send-index-update-request (khoj--render-delete-files-as-request-body delete-files boundary) boundary content-type type-query force)) (khoj--send-index-update-request (khoj--render-delete-files-as-request-body delete-files boundary) boundary content-type type-query force))
(setq khoj--indexed-files files-to-index))) (setq khoj--indexed-files files-to-index)))
(defun khoj--get-delete-file-list (indexed-files upload-files)
"Get delete file list. when `INDEXED-FILES' no longer in `UPLOAD-FILES'.
delete them. return delete-file-list."
(let (delete-files '())
(dolist (indexed-file indexed-files)
(when (not (member indexed-file upload-files))
(push indexed-file delete-files)))
delete-files))
(defun khoj--send-index-update-request (body boundary &optional content-type type-query force) (defun khoj--send-index-update-request (body boundary &optional content-type type-query force)
"Send `BODY' request to khoj server. 'TYPE-QUERY' is appended to the URL. "Send multi-part form `BODY' of `CONTENT-TYPE' in request to khoj server.
Use `BOUNDARY' to add headder conte Append 'TYPE-QUERY' as query parameter in request url.
nt-type." Specify `BOUNDARY' used to separate files in request header."
(let ((url-request-method "POST") (let ((url-request-method "POST")
(url-request-data body) (url-request-data body)
(url-request-extra-headers `(("content-type" . ,(format "multipart/form-data; boundary=%s" boundary)) (url-request-extra-headers `(("content-type" . ,(format "multipart/form-data; boundary=%s" boundary))
@ -1087,8 +1083,8 @@ Paragraph only starts at first text after blank line."
(defun khoj () (defun khoj ()
"Search and chat with your knowledge base using your personal AI copilot. "Search and chat with your knowledge base using your personal AI copilot.
Collaborate with Khoj to search, understand, create, review and update your knowledge base. Collaborate with Khoj to search, create, review and update your knowledge base.
Khoj can research across your org-mode, markdown notes, plaintext documents and the internet." Research across the internet & your documents from the comfort of Emacs."
(interactive) (interactive)
(when khoj-auto-setup (when khoj-auto-setup
(khoj-setup t)) (khoj-setup t))