Create Khoj Chat Interface in Emacs

Merge pull request #191 from debanjum/create-chat-interface-on-emacs

- Render conversation history in a read-only org-mode buffer for Khoj Chat
- Add `chat` as a transient action in the Khoj transient menu
- Style chat messages as org-mode entries
  - Put received date in property drawer and keep it hidden/folded by default
  - Add Khoj chat response as child entry of the users associated question org entry
    This allows folding back-n-forth between user and Khoj for easier viewing
  - Render source notes snippets used as references for response as org-mode links
    Hovering mouse on link or opening links shows reference note snippets used

.github/workflows/test_khoj_el.yml

@@ -42,7 +42,10 @@ jobs:
                     (push '(\"melpa\" . \"https://melpa.org/packages/\") package-archives) \
                     (package-initialize) \
                     (unless package-archive-contents (package-refresh-contents)) \
-                    (unless (package-installed-p 'transient) (package-install 'transient)))" \
+                    (unless (package-installed-p 'transient) (package-install 'transient)) \
+                    (unless (package-installed-p 'dash) (package-install 'dash)) \
+                    (unless (package-installed-p 'org) (package-install 'org)) \
+                   )" \
            -l ert \
            -l ./src/interface/emacs/khoj.el \
            -l ./src/interface/emacs/tests/khoj-tests.el \

src/interface/emacs/khoj.el

@@ -6,7 +6,7 @@
 ;; Description: Natural, Incremental Search for your Second Brain
 ;; Keywords: search, org-mode, outlines, markdown, beancount, ledger, image
 ;; Version: 0.4.1
-;; Package-Requires: ((emacs "27.1") (transient "0.3.0"))
+;; Package-Requires: ((emacs "27.1") (transient "0.3.0") (dash "2.19.1"))
 ;; URL: https://github.com/debanjum/khoj/tree/master/src/interface/emacs
 
 ;; This file is NOT part of GNU Emacs.
@@ -50,6 +50,9 @@
 (require 'json)
 (require 'transient)
 (require 'outline)
+(require 'dash)
+(require 'org)
+
 (eval-when-compile (require 'subr-x)) ;; for string-trim before Emacs 28.2
 
 
@@ -97,8 +100,11 @@
 (defconst khoj--query-prompt "🦅Khoj: "
   "Query prompt shown in the minibuffer.")
 
-(defconst khoj--buffer-name "*🦅Khoj*"
-  "Name of buffer to show results from Khoj.")
+(defconst khoj--search-buffer-name "*🦅Khoj Search*"
+  "Name of buffer to show search results from Khoj.")
+
+(defconst khoj--chat-buffer-name "*🦅Khoj Chat*"
+  "Name of chat buffer for Khoj.")
 
 (defvar khoj--content-type "org"
   "The type of content to perform search on.")
@@ -281,16 +287,18 @@ Use `which-key` if available, else display simple message in echo area"
       (url-insert-file-contents config-url)
       (thread-last
         (json-parse-buffer :object-type 'alist)
-        (mapcar 'intern)))))
+        (mapcar #'intern)))))
 
-(defun khoj--construct-api-query (query content-type &optional rerank)
-  "Construct API Query from QUERY, CONTENT-TYPE and (optional) RERANK params."
+(defun khoj--construct-search-api-query (query content-type &optional rerank)
+  "Construct Search API Query.
+Use QUERY, CONTENT-TYPE and (optional) RERANK as query params"
   (let ((rerank (or rerank "false"))
         (encoded-query (url-hexify-string query)))
     (format "%s/api/search?q=%s&t=%s&r=%s&n=%s" khoj-server-url encoded-query content-type rerank khoj-results-count)))
 
-(defun khoj--query-api-and-render-results (query-url content-type query buffer-name)
-  "Query Khoj QUERY-URL. Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
+(defun khoj--query-search-api-and-render-results (query-url content-type query buffer-name)
+  "Query Khoj Search with QUERY-URL.
+Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
   ;; get json response from api
   (with-current-buffer buffer-name
     (let ((inhibit-read-only t)
@@ -320,6 +328,100 @@ Use `which-key` if available, else display simple message in echo area"
             (t (fundamental-mode))))
     (read-only-mode t)))
 
+
+;; ----------------
+;; Khoj Chat
+;; ----------------
+
+(defun khoj--chat ()
+  "Chat with Khoj."
+  (let ((query (read-string "Query: ")))
+    (khoj--query-chat-api-and-render-messages query khoj--chat-buffer-name)
+    (switch-to-buffer khoj--chat-buffer-name)))
+
+(defun khoj--load-chat-history (buffer-name)
+  (let ((json-response (cdr (assoc 'response (khoj--query-chat-api "")))))
+    (with-current-buffer (get-buffer-create buffer-name)
+      (erase-buffer)
+      (insert "#+STARTUP: showall hidestars\n")
+      (thread-last
+        json-response
+        ;; generate chat messages from Khoj Chat API response
+        (mapcar #'khoj--render-chat-response)
+        ;; insert chat messages into Khoj Chat Buffer
+        (mapc #'insert))
+      (progn (org-mode)
+             (visual-line-mode)
+             (read-only-mode t)))))
+
+(defun khoj--query-chat-api-and-render-messages (query buffer-name)
+  "Send QUERY to Khoj Chat. Render the chat messages from exchange in BUFFER-NAME."
+  ;; render json response into formatted chat messages
+  (if (not (get-buffer buffer-name))
+      (khoj--load-chat-history buffer-name)
+    (with-current-buffer (get-buffer buffer-name)
+      (let ((inhibit-read-only t)
+            (json-response (khoj--query-chat-api query)))
+        (goto-char (point-max))
+        (insert
+         (khoj--render-chat-message query "you")
+         (khoj--render-chat-response json-response)))
+        (progn (org-mode)
+               (visual-line-mode))
+    (read-only-mode t))))
+
+(defun khoj--query-chat-api (query)
+  "Send QUERY to Khoj Chat API."
+  (let* ((url-request-method "GET")
+         (encoded-query (url-hexify-string query))
+         (query-url (format "%s/api/chat?q=%s" khoj-server-url encoded-query)))
+    (with-temp-buffer
+      (erase-buffer)
+      (url-insert-file-contents query-url)
+      (json-parse-buffer :object-type 'alist))))
+
+(defun khoj--render-chat-message (message sender &optional receive-date)
+  "Render chat messages as `org-mode' list item.
+MESSAGE is the text of the chat message.
+SENDER is the message sender.
+RECEIVE-DATE is the message receive date."
+  (let ((first-message-line (car (split-string message "\n" t)))
+        (rest-message-lines (string-join (cdr (split-string message "\n" t)) "\n   "))
+        (heading-level (if (equal sender "you") "**" "***"))
+        (emojified-by (if (equal sender "you") "🤔 *You*" "🦅 *Khoj*"))
+        (received (or receive-date (format-time-string "%F %T"))))
+    (format "%s %s: %s\n   :PROPERTIES:\n   :RECEIVED: [%s]\n   :END:\n   %s\n"
+            heading-level
+            emojified-by
+            first-message-line
+            received
+            rest-message-lines)))
+
+(defun khoj--generate-reference (index reference)
+  "Create `org-mode' links with REFERENCE as link and INDEX as link description."
+  (with-temp-buffer
+    (org-insert-link
+     nil
+     (format "%s" (replace-regexp-in-string "\n" " " reference))
+     (format "%s" index))
+    (format "[%s]" (buffer-substring-no-properties (point-min) (point-max)))))
+
+(defun khoj--render-chat-response (json-response)
+  "Render chat message using JSON-RESPONSE from Khoj Chat API."
+  (let* ((message (cdr (or (assoc 'response json-response) (assoc 'message json-response))))
+         (sender (cdr (assoc 'by json-response)))
+         (receive-date (cdr (assoc 'created json-response)))
+         (context (or (cdr (assoc 'context json-response)) ""))
+         (reference-texts (split-string context "\n\n# " t))
+         (reference-links (-map-indexed #'khoj--generate-reference reference-texts)))
+    (thread-first
+      ;; extract khoj message from API response and make it bold
+      (format "%s" message)
+      ;; append references to khoj message
+      (concat " " (string-join reference-links " "))
+      ;; Render chat message using data obtained from API
+      (khoj--render-chat-message sender receive-date))))
+
 
 ;; ------------------
 ;; Incremental Search
@@ -328,9 +430,9 @@ Use `which-key` if available, else display simple message in echo area"
 (defun khoj--incremental-search (&optional rerank)
   "Perform Incremental Search on Khoj. Allow optional RERANK of results."
   (let* ((rerank-str (cond (rerank "true") (t "false")))
-         (khoj-buffer-name (get-buffer-create khoj--buffer-name))
+         (khoj-buffer-name (get-buffer-create khoj--search-buffer-name))
          (query (minibuffer-contents-no-properties))
-         (query-url (khoj--construct-api-query query khoj--content-type rerank-str)))
+         (query-url (khoj--construct-search-api-query query khoj--content-type rerank-str)))
     ;; Query khoj API only when user in khoj minibuffer and non-empty query
     ;; Prevents querying if
     ;;   1. user hasn't started typing query
@@ -349,7 +451,7 @@ Use `which-key` if available, else display simple message in echo area"
         (when rerank
           (setq khoj--rerank t)
           (message "Khoj: Rerank Results"))
-        (khoj--query-api-and-render-results
+        (khoj--query-search-api-and-render-results
          query-url
          khoj--content-type
          query
@@ -377,7 +479,7 @@ Use `which-key` if available, else display simple message in echo area"
 (defun khoj-incremental ()
   "Natural, Incremental Search for your personal notes, transactions and music."
   (interactive)
-  (let* ((khoj-buffer-name (get-buffer-create khoj--buffer-name)))
+  (let* ((khoj-buffer-name (get-buffer-create khoj--search-buffer-name)))
     ;; switch to khoj results buffer
     (switch-to-buffer khoj-buffer-name)
     ;; open and setup minibuffer for incremental search
@@ -442,14 +544,14 @@ Paragraph only starts at first text after blank line."
                  ;; get paragraph, if in text mode
                  (t
                   (khoj--get-current-paragraph-text))))
-         (query-url (khoj--construct-api-query query content-type rerank))
+         (query-url (khoj--construct-search-api-query query content-type rerank))
          ;; extract heading to show in result buffer from query
          (query-title
           (format "Similar to: %s"
                   (replace-regexp-in-string "^[#\\*]* " "" (car (split-string query "\n")))))
-         (buffer-name (get-buffer-create khoj--buffer-name)))
+         (buffer-name (get-buffer-create khoj--search-buffer-name)))
     (progn
-      (khoj--query-api-and-render-results
+      (khoj--query-search-api-and-render-results
        query-url
        content-type
        query-title
@@ -503,15 +605,20 @@ Paragraph only starts at first text after blank line."
       (setq khoj--content-type content-type)
       (url-retrieve update-url (lambda (_) (message "Khoj %s index %supdated!" content-type (if (member "--force-update" args) "force " "")))))))
 
+(transient-define-suffix khoj--chat-command (&optional _)
+  "Command to Chat with Khoj."
+  (interactive (list (transient-args transient-current-command)))
+  (khoj--chat))
+
 (transient-define-prefix khoj-menu ()
   "Create Khoj Menu to Configure and Execute Commands."
-  [["Configure General"
+  [["Configure Search"
+    ("n" "Results Count" "--results-count=" :init-value (lambda (obj) (oset obj value (format "%s" khoj-results-count))))
     ("t" "Content Type" khoj--content-type-switch)]
-   ["Configure Search"
-    ("n" "Results Count" "--results-count=" :init-value (lambda (obj) (oset obj value (format "%s" khoj-results-count))))]
    ["Configure Update"
     ("-f" "Force Update" "--force-update")]]
   [["Act"
+    ("c" "Chat" khoj--chat-command)
     ("s" "Search" khoj--search-command)
     ("f" "Find Similar" khoj--find-similar-command)
     ("u" "Update" khoj--update-command)

src/interface/emacs/tests/khoj-tests.el

@@ -4,7 +4,7 @@
 
 ;; Author: Debanjum Singh Solanky <debanjum@gmail.com>
 ;; Version: 0.0.0
-;; Package-Requires: ((emacs "27.1") (transient "0.3.0"))
+;; Package-Requires: ((emacs "27.1") (transient "0.3.0") (dash "2.19.1") (org "9.0.0"))
 ;; URL: https://github.com/debanjum/khoj/tree/master/src/interface/emacs
 
 ;;; License:
@@ -28,8 +28,10 @@
 
 ;;; Code:
 
+(require 'dash)
 (require 'ert)
 (require 'khoj)
+(require 'org)