diff --git a/src/interface/emacs/khoj.el b/src/interface/emacs/khoj.el index b0b999f0..ddcd50c6 100644 --- a/src/interface/emacs/khoj.el +++ b/src/interface/emacs/khoj.el @@ -65,6 +65,11 @@ :group 'khoj :type 'string) +(defcustom khoj-server-is-local t + "Is Khoj server on local machine?." + :group 'khoj + :type 'boolean) + (defcustom khoj-image-width 156 "Width of rendered images returned by Khoj." :group 'khoj @@ -171,6 +176,305 @@ Use `which-key` if available, else display simple message in echo area" nil t t)) (message "%s" (khoj--keybindings-info-message)))) + +;; ---------------- +;; Khoj Setup +;; ---------------- +(defcustom khoj-server-command + (or (executable-find "khoj") + (executable-find "khoj.exe") + "khoj") + "Command to interact with Khoj server." + :type 'string + :group 'khoj) + +(defcustom khoj-server-args '("--no-gui") + "Arguments to pass to Khoj server on startup." + :type '(repeat string) + :group 'khoj) + +(defcustom khoj-server-python-command + (if (equal system-type 'windows-nt) + (or (executable-find "py") + (executable-find "pythonw") + "python") + (if (executable-find "python") + "python" + ;; Fallback on systems where python is not + ;; symlinked to python3. + "python3")) + "The Python interpreter used for the Khoj server. + +Khoj will try to use the system interpreter if it exists. If you wish +to use a specific python interpreter (from a virtual environment +for example), set this to the full interpreter path." + :type '(choice (const :tag "python" "python") + (const :tag "python3" "python3") + (const :tag "pythonw (Python on Windows)" "pythonw") + (const :tag "py (other Python on Windows)" "py") + (string :tag "Other")) + :safe (lambda (val) + (member val '("python" "python3" "pythonw" "py"))) + :group 'khoj) + +(defcustom khoj-org-files (org-agenda-files t t) + "List of org-files to index on khoj server." + :type '(repeat string) + :group 'khoj) + +(defcustom khoj-org-directories nil + "List of directories with org-mode files to index on khoj server." + :type '(repeat string) + :group 'khoj) + +(defcustom khoj-openai-api-key nil + "OpenAI API key used to configure chat on khoj server." + :type 'string + :group 'khoj) + +(defcustom khoj-auto-setup t + "Automate install, configure and start of khoj server. +Auto invokes setup steps on calling main entrypoint." + :type 'string + :group 'khoj) + +(defvar khoj--server-process nil "Track khoj server process.") +(defvar khoj--server-name "*khoj-server*" "Track khoj server buffer.") +(defvar khoj--server-ready? nil "Track if khoj server is ready to receive API calls.") +(defvar khoj--server-configured? t "Track if khoj server is configured to receive API calls.") +(defvar khoj--progressbar '(🌑 🌘 🌗 🌖 🌕 🌔 🌓 🌒) "Track progress via moon phase animations.") + +(defun khoj--server-get-version () + "Return the khoj server version." + (with-temp-buffer + (call-process khoj-server-command nil t nil "--version") + (goto-char (point-min)) + (re-search-forward "\\([a-z0-9.]+\\)") + (match-string 1))) + +(defun khoj--server-install-upgrade () + "Install or upgrade the khoj server." + (with-temp-buffer + (message "khoj.el: Installing server...") + (if (/= (apply 'call-process khoj-server-python-command + nil t nil + "-m" "pip" "install" "--upgrade" + '("khoj-assistant")) + 0) + (message "khoj.el: Failed to install Khoj server. Please install it manually using pip install `khoj-assistant'.\n%s" (buffer-string)) + (message "khoj.el: Installed and upgraded Khoj server version: %s" (khoj--server-get-version))))) + +(defun khoj--server-start () + "Start the khoj server." + (interactive) + (let* ((url-parts (split-string (cadr (split-string khoj-server-url "://")) ":")) + (server-host (nth 0 url-parts)) + (server-port (or (nth 1 url-parts) "80")) + (server-args (append khoj-server-args + (list (format "--host=%s" server-host) + (format "--port=%s" server-port))))) + (message "khoj.el: Starting server at %s %s..." server-host server-port) + (setq khoj--server-process + (make-process + :name khoj--server-name + :buffer khoj--server-name + :command (append (list khoj-server-command) server-args) + :sentinel (lambda (process event) + (message "khoj.el: khoj server stopped with: %s" event) + (setq khoj--server-ready? nil)) + :filter (lambda (process msg) + (cond ((string-match (format "Uvicorn running on %s" khoj-server-url) msg) + (progn + (setq khoj--server-ready? t) + (khoj--server-configure))) + ((string-match "Batches: " msg) + (when (string-match "\\([0-9]+\\.[0-9]+\\|\\([0-9]+\\)\\)%?" msg) + (message "khoj.el: %s updating index %s" + (nth (% (string-to-number (match-string 1 msg)) (length khoj--progressbar)) khoj--progressbar) + (match-string 0 msg))) + (setq khoj--server-configured? nil)) + ((and (not khoj--server-configured?) + (string-match "Processor reconfigured via API" msg)) + (setq khoj--server-configured? t)) + ((and (not khoj--server-ready?) + (or (string-match "configure.py" msg) + (string-match "main.py" msg) + (string-match "api.py" msg))) + (dolist (line (split-string msg "\n")) + (message "khoj.el: %s" (nth 1 (split-string msg " " t " *")))))) + ;; call default process filter to write output to process buffer + (internal-default-process-filter process msg)))) + (set-process-query-on-exit-flag khoj--server-process nil) + (when (not khoj--server-process) + (message "khoj.el: Failed to start Khoj server. Please start it manually by running `khoj' on terminal.\n%s" (buffer-string))))) + +(defun khoj--server-started? () + "Check if the khoj server has been started." + ;; check for when server process handled from within emacs + (if (and khoj--server-process + (not (null (process-live-p khoj--server-process)))) + t + ;; else general check via ping to khoj-server-url + (if (ignore-errors + (not (null (url-retrieve-synchronously (format "%s/api/config/data/default" khoj-server-url))))) + ;; Successful ping to non-emacs khoj server indicates it is started and ready. + ;; So update ready state tracker variable (and implicitly return true for started) + (setq khoj--server-ready? t) + nil))) + +(defun khoj--server-restart () + "Restart the khoj server." + (interactive) + (khoj--server-stop) + (khoj--server-start)) + +(defun khoj--server-stop () + "Stop the khoj server." + (interactive) + (when (khoj--server-started?) + (message "khoj.el: Stopping server...") + (kill-process khoj--server-process) + (message "khoj.el: Stopped server."))) + +(defun khoj--server-setup () + "Install and start the khoj server, if required." + (interactive) + ;; Install khoj server, if not available but expected on local machine + (when (and khoj-server-is-local + (or (not (executable-find khoj-server-command)) + (not (khoj--server-get-version)))) + (khoj--server-install-upgrade)) + ;; Start khoj server if not already started + (when (not (khoj--server-started?)) + (khoj--server-start))) + +(defun khoj--get-directory-from-config (config keys &optional level) + "Extract directory under specified KEYS in CONFIG and trim it to LEVEL. +CONFIG is json obtained from Khoj config API." + (let ((item config)) + (dolist (key keys) + (setq item (cdr (assoc key item)))) + (-> item + (split-string "/") + (butlast (or level nil)) + (string-join "/")))) + +(defun khoj--server-configure () + "Configure the the Khoj server for search and chat." + (interactive) + (let* ((org-directory-regexes (or (mapcar (lambda (dir) (format "%s/**/*.org" dir)) khoj-org-directories) json-null)) + (current-config + (with-temp-buffer + (url-insert-file-contents (format "%s/api/config/data" khoj-server-url)) + (ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false)))) + (default-config + (with-temp-buffer + (url-insert-file-contents (format "%s/api/config/data/default" khoj-server-url)) + (ignore-error json-end-of-file (json-parse-buffer :object-type 'alist :array-type 'list :null-object json-null :false-object json-false)))) + (default-index-dir (khoj--get-directory-from-config default-config '(content-type org embeddings-file))) + (default-chat-dir (khoj--get-directory-from-config default-config '(processor conversation conversation-logfile))) + (default-model (or (alist-get 'model (alist-get 'conversation (alist-get 'processor default-config))) "text-davinci-003")) + (config (or current-config default-config))) + + ;; Configure content types + (cond + ;; If khoj backend is not configured yet + ((not current-config) + (setq config (delq (assoc 'content-type config) config)) + (add-to-list 'config + `(content-type . ((org . ((input-files . ,khoj-org-files) + (input-filter . ,org-directory-regexes) + (compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir)) + (embeddings-file . ,(format "%s/org.pt" default-index-dir)) + (index-heading-entries . ,json-false))))))) + + ;; Else if khoj config has no org content config + ((not (alist-get 'org (alist-get 'content-type config))) + (let ((new-content-type (alist-get 'content-type config))) + (setq new-content-type (delq (assoc 'org new-content-type) new-content-type)) + (add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files) + (input-filter . ,org-directory-regexes) + (compressed-jsonl . ,(format "%s/org.jsonl.gz" default-index-dir)) + (embeddings-file . ,(format "%s/org.pt" default-index-dir)) + (index-heading-entries . ,json-false)))) + (setq config (delq (assoc 'content-type config) config)) + (add-to-list 'config `(content-type . ,new-content-type)))) + + ;; Else if khoj is not configured to index specified org files + ((not (and (equal (alist-get 'input-files (alist-get 'org (alist-get 'content-type config))) khoj-org-files) + (equal (alist-get 'input-filter (alist-get 'org (alist-get 'content-type config))) org-directory-regexes))) + (let* ((index-directory (khoj--get-directory-from-config config '(content-type org embeddings-file))) + (new-content-type (alist-get 'content-type config))) + (setq new-content-type (delq (assoc 'org new-content-type) new-content-type)) + (add-to-list 'new-content-type `(org . ((input-files . ,khoj-org-files) + (input-filter . ,org-directory-regexes) + (compressed-jsonl . ,(format "%s/org.jsonl.gz" index-directory)) + (embeddings-file . ,(format "%s/org.pt" index-directory)) + (index-heading-entries . ,json-false)))) + (setq config (delq (assoc 'content-type config) config)) + (add-to-list 'config `(content-type . ,new-content-type))))) + + ;; Configure processors + (cond + ((not khoj-openai-api-key) + (setq config (delq (assoc 'processor config) config))) + + ((not current-config) + (setq config (delq (assoc 'processor config) config)) + (add-to-list 'config + `(processor . ((conversation . ((conversation-logfile . ,(format "%s/conversation.json" default-chat-dir)) + (model . ,default-model) + (openai-api-key . ,khoj-openai-api-key))))))) + + ((not (alist-get 'conversation (alist-get 'processor config))) + (let ((new-processor-type (alist-get 'processor config))) + (setq new-processor-type (delq (assoc 'conversation new-processor-type) new-processor-type)) + (add-to-list 'new-processor-type `(conversation . ((conversation-logfile . ,(format "%s/conversation.json" default-chat-dir)) + (model . ,default-model) + (openai-api-key . ,khoj-openai-api-key)))) + (setq config (delq (assoc 'processor config) config)) + (add-to-list 'config `(processor . ,new-processor-type)))) + + ;; Else if khoj is not configured with specified openai api key + ((not (equal (alist-get 'openai-api-key (alist-get 'conversation (alist-get 'processor config))) khoj-openai-api-key)) + (let* ((chat-directory (khoj--get-directory-from-config config '(processor conversation conversation-logfile))) + (model-name (khoj--get-directory-from-config config '(processor conversation model))) + (new-processor-type (alist-get 'processor config))) + (setq new-processor-type (delq (assoc 'conversation new-processor-type) new-processor-type)) + (add-to-list 'new-processor-type `(conversation . ((conversation-logfile . ,(format "%s/conversation.json" chat-directory)) + (model . ,model-name) + (openai-api-key . ,khoj-openai-api-key)))) + (setq config (delq (assoc 'processor config) config)) + (add-to-list 'config `(processor . ,new-processor-type))))) + + ;; Update server with latest configuration + (khoj--post-new-config config) + (cond ((not current-config) + (message "khoj.el: ⚙️ Generated new khoj server configuration.")) + ((not (equal config current-config)) + (message "Khoj: ⚙️ Updated khoj server configuration"))))) + +(defun khoj-setup (&optional interact) + "Install, start and configure Khoj server." + (interactive "p") + ;; Setup khoj server if not running + (let* ((not-started (not (khoj--server-started?))) + (permitted (if (and not-started interact) + (y-or-n-p "Could not connect to Khoj server. Should I install, start and configure it for you?") + t))) + ;; Install, start server if user permitted and server not ready + (when (and permitted not-started) + (khoj--server-setup)) + + ;; Server can be started but not ready (to use/configure) + ;; Wait until server is ready if setup was permitted + (while (and permitted (not khoj--server-ready?)) + (sit-for 0.5)) + + ;; Configure server once server ready if user permitted + (when permitted + (khoj--server-configure)))) + ;; ----------------------------------------------- ;; Extract and Render Entries of each Content Type @@ -196,22 +500,21 @@ Use `which-key` if available, else display simple message in echo area" (defun khoj--extract-entries-as-org (json-response query) "Convert JSON-RESPONSE, QUERY from API to `org-mode' entries." - (let ((org-results-buffer-format-str "* %s\n%s\n#+STARTUP: showall hidestars inlineimages")) - (thread-last - json-response - ;; Extract and render each org-mode entry from response - (mapcar (lambda (json-response-item) - (thread-last - ;; Extract org entry from each item in json response - (cdr (assoc 'entry json-response-item)) - ;; Format org entry as a string - (format "%s") - ;; Standardize results to 2nd level heading for consistent rendering - (replace-regexp-in-string "^\*+" "**")))) - ;; Render entries into org formatted string with query set as as top level heading - (format org-results-buffer-format-str query) - ;; remove leading (, ) or SPC from extracted entries string - (replace-regexp-in-string "^[\(\) ]" "")))) + (thread-last + json-response + ;; Extract and render each org-mode entry from response + (mapcar (lambda (json-response-item) + (thread-last + ;; Extract org entry from each item in json response + (cdr (assoc 'entry json-response-item)) + ;; Format org entry as a string + (format "%s") + ;; Standardize results to 2nd level heading for consistent rendering + (replace-regexp-in-string "^\*+" "**")))) + ;; Render entries into org formatted string with query set as as top level heading + (format "* %s\n%s\n" query) + ;; remove leading (, ) or SPC from extracted entries string + (replace-regexp-in-string "^[\(\) ]" ""))) (defun khoj--extract-entries-as-ledger (json-response query) "Convert JSON-RESPONSE, QUERY from API to ledger entries." @@ -281,12 +584,24 @@ Use `which-key` if available, else display simple message in echo area" ;; Query Khoj API ;; -------------- +(defun khoj--post-new-config (config) + "Configure khoj server with provided CONFIG." + ;; POST provided config to khoj server + (let ((url-request-method "POST") + (url-request-extra-headers '(("Content-Type" . "application/json"))) + (url-request-data (json-encode-alist config)) + (config-url (format "%s/api/config/data" khoj-server-url))) + (with-current-buffer (url-retrieve-synchronously config-url) + (buffer-string))) + ;; Update index on khoj server after configuration update + (let ((khoj--server-ready? nil)) + (url-retrieve (format "%s/api/update?t=org" khoj-server-url) #'identity))) + (defun khoj--get-enabled-content-types () "Get content types enabled for search from API." (let ((config-url (format "%s/api/config/types" khoj-server-url)) (url-request-method "GET")) (with-temp-buffer - (erase-buffer) (url-insert-file-contents config-url) (thread-last (json-parse-buffer :object-type 'alist) @@ -319,8 +634,13 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE." ((equal content-type "ledger") (khoj--extract-entries-as-ledger json-response query)) ((equal content-type "image") (khoj--extract-entries-as-images json-response query)) (t (khoj--extract-entries json-response query)))) - (cond ((equal content-type "org") (progn (org-mode) - (visual-line-mode))) + (cond ((equal content-type "org") (progn (visual-line-mode) + (org-mode) + (setq-local + org-startup-folded "showall" + org-hide-leading-stars t + org-startup-with-inline-images t) + (org-set-startup-visibility))) ((equal content-type "markdown") (progn (markdown-mode) (visual-line-mode))) ((equal content-type "ledger") (beancount-mode)) @@ -428,9 +748,14 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE." (encoded-query (url-hexify-string query)) (query-url (format "%s/api/chat?q=%s" khoj-server-url encoded-query))) (with-temp-buffer - (erase-buffer) - (url-insert-file-contents query-url) - (json-parse-buffer :object-type 'alist)))) + (condition-case ex + (progn + (url-insert-file-contents query-url) + (json-parse-buffer :object-type 'alist)) + ('file-error (cond ((string-match "Internal server error" (nth 2 ex)) + (message "Chat processor not configured. Configure OpenAI API key and restart it. Exception: [%s]" ex)) + (t (message "Chat exception: [%s]" ex)))))))) + (defun khoj--render-chat-message (message sender &optional receive-date) "Render chat messages as `org-mode' list item. @@ -671,7 +996,7 @@ Paragraph only starts at first text after blank line." (interactive (list (transient-args transient-current-command))) (khoj--chat)) -(transient-define-prefix khoj-menu () +(transient-define-prefix khoj--menu () "Create Khoj Menu to Configure and Execute Commands." [["Configure Search" ("n" "Results Count" "--results-count=" :init-value (lambda (obj) (oset obj value (format "%s" khoj-results-count)))) @@ -692,9 +1017,11 @@ Paragraph only starts at first text after blank line." ;;;###autoload (defun khoj () - "Natural, Incremental Search for your personal notes, transactions and images." + "Provide natural, search assistance for your notes, transactions and images." (interactive) - (khoj-menu)) + (when khoj-auto-setup + (khoj-setup t)) + (khoj--menu)) (provide 'khoj) diff --git a/src/interface/emacs/tests/khoj-tests.el b/src/interface/emacs/tests/khoj-tests.el index 577f2c5c..83985b67 100644 --- a/src/interface/emacs/tests/khoj-tests.el +++ b/src/interface/emacs/tests/khoj-tests.el @@ -109,8 +109,7 @@ Penance to Immortality\n\ ** Act\n\ \n\ Rule everything\n\ -\n\ -#+STARTUP: showall hidestars inlineimages")))) +\n")))) (ert-deftest khoj-tests--extract-entries-as-ledger () diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index 312c1ffc..8ee6de13 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -33,6 +33,12 @@ def get_default_config_data(): @api.get("/config/types", response_model=List[str]) def get_config_types(): """Get configured content types""" + if state.config is None or state.config.content_type is None: + raise HTTPException( + status_code=500, + detail="Content types not configured. Configure at least one content type on server and restart it.", + ) + configured_content_types = state.config.content_type.dict(exclude_none=True) return [ search_type.value @@ -190,6 +196,15 @@ def update(t: Optional[SearchType] = None, force: Optional[bool] = False): @api.get("/chat") def chat(q: Optional[str] = None): + if ( + state.processor_config is None + or state.processor_config.conversation is None + or state.processor_config.conversation.openai_api_key is None + ): + raise HTTPException( + status_code=500, detail="Chat processor not configured. Configure OpenAI API key on server and restart it." + ) + # Initialize Variables api_key = state.processor_config.conversation.openai_api_key model = state.processor_config.conversation.model diff --git a/src/khoj/utils/constants.py b/src/khoj/utils/constants.py index 40e01c5d..71722c1a 100644 --- a/src/khoj/utils/constants.py +++ b/src/khoj/utils/constants.py @@ -56,6 +56,7 @@ default_config = { "processor": { "conversation": { "openai-api-key": None, + "model": "text-davinci-003", "conversation-logfile": "~/.khoj/processor/conversation/conversation_logs.json", } },