Upgrade Khoj Emacs UX (#812)

- Open Khoj in Emacs Side pane
   Open Khoj chat, search in right pane to allow for ambient engagement
- Improve Khoj Chat
  - Show online references used for chat
  - Make chat API call async to not block user interactions
  - Fix loading chat history, references in khoj.el chat buffer
- Improve Khoj Search, Find Similar functions
   - Make calls to Khoj search API async to not block user interactions
- Support Conversation Sessions
  - Create transient menu to open, create, delete conversation sessions from the Khoj Emacs client

src/interface/emacs/khoj.el

@@ -113,7 +113,7 @@
   :group 'khoj
   :type 'number)
 
-(defcustom khoj-default-content-type "org"
+(defcustom khoj-default-content-type "all"
   "The default content type to perform search on."
   :group 'khoj
   :type '(choice (const "org")
@@ -163,6 +163,7 @@ NO-PAGING FILTER))
      "
      Set Content Type
 -------------------------\n"
+     ("C-c RET | improve sort \n")
      (when (member 'markdown enabled-content-types)
        "C-x m  | markdown\n")
      (when (member 'org enabled-content-types)
@@ -174,24 +175,12 @@ NO-PAGING FILTER))
 
 (defvar khoj--rerank nil "Track when re-rank of results triggered.")
 (defvar khoj--reference-count 0 "Track number of references currently in chat bufffer.")
-(defun khoj--search-markdown () "Set content-type to `markdown'." (interactive) (setq khoj--content-type "markdown"))
+(defun khoj--improve-sort () "Use cross-encoder to improve sorting of search results." (interactive) (khoj--incremental-search t))
-(defun khoj--search-org () "Set content-type to `org-mode'." (interactive) (setq khoj--content-type "org"))
-(defun khoj--search-images () "Set content-type to image." (interactive) (setq khoj--content-type "image"))
-(defun khoj--search-pdf () "Set content-type to pdf." (interactive) (setq khoj--content-type "pdf"))
-(defun khoj--improve-rank () "Use cross-encoder to rerank search results." (interactive) (khoj--incremental-search t))
 (defun khoj--make-search-keymap (&optional existing-keymap)
   "Setup keymap to configure Khoj search. Build of EXISTING-KEYMAP when passed."
   (let ((enabled-content-types (khoj--get-enabled-content-types))
         (kmap (or existing-keymap (make-sparse-keymap))))
-    (define-key kmap (kbd "C-c RET") #'khoj--improve-rank)
+    (define-key kmap (kbd "C-c RET") #'khoj--improve-sort)
-    (when (member 'markdown enabled-content-types)
-      (define-key kmap (kbd "C-x m") #'khoj--search-markdown))
-    (when (member 'org enabled-content-types)
-      (define-key kmap (kbd "C-x o") #'khoj--search-org))
-    (when (member 'image enabled-content-types)
-      (define-key kmap (kbd "C-x i") #'khoj--search-images))
-    (when (member 'pdf enabled-content-types)
-      (define-key kmap (kbd "C-x p") #'khoj--search-pdf))
     kmap))
 
 (defvar khoj--keymap nil "Track Khoj keymap in this variable.")
@@ -629,38 +618,67 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request.
 ;; --------------
 ;; Query Khoj API
 ;; --------------
+(defun khoj--call-api (path &optional method params callback &rest cbargs)
+  "Sync call API at PATH with METHOD and query PARAMS as kv assoc list.
+Return json parsed response as alist."
+  (let* ((url-request-method (or method "GET"))
+         (url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key))))
+         (param-string (if params (url-build-query-string params) ""))
+         (query-url (format "%s%s?%s&client=emacs" khoj-server-url path param-string))
+         (cbargs (if (and (listp cbargs) (listp (car cbargs))) (car cbargs) cbargs))) ; normalize cbargs to (a b) from ((a b)) if required
+    (with-temp-buffer
+      (condition-case ex
+          (progn
+            (url-insert-file-contents query-url)
+            (if (and callback cbargs)
+                (apply callback (json-parse-buffer :object-type 'alist) cbargs)
+              (if callback
+                  (funcall callback (json-parse-buffer :object-type 'alist))
+            (json-parse-buffer :object-type 'alist))))
+        ('file-error (message "Chat exception: [%s]" ex))))))
+
+(defun khoj--call-api-async (path &optional method params callback &rest cbargs)
+  "Async call to API at PATH with METHOD and query PARAMS as kv assoc list.
+Return json parsed response as alist."
+  (let* ((url-request-method (or method "GET"))
+         (url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key))))
+         (param-string (if params (url-build-query-string params) ""))
+         (cbargs (if (and (listp cbargs) (listp (car cbargs))) (car cbargs) cbargs)) ; normalize cbargs to (a b) from ((a b)) if required
+         (query-url (format "%s%s?%s&client=emacs" khoj-server-url path param-string)))
+    (url-retrieve query-url
+                  (lambda (status)
+                    (if (plist-get status :error)
+                        (message "Chat exception: [%s]" (plist-get status :error))
+                      (goto-char (point-min))
+                      (re-search-forward "^$")
+                      (delete-region (point) (point-min))
+                      (if (and callback cbargs)
+                          (apply callback (json-parse-buffer :object-type 'alist) cbargs)
+                        (if callback
+                            (funcall callback (json-parse-buffer :object-type 'alist))
+                          (json-parse-buffer :object-type 'alist))))))))
+
 (defun khoj--get-enabled-content-types ()
   "Get content types enabled for search from API."
-  (let ((config-url (format "%s/api/config/types" khoj-server-url))
+  (khoj--call-api "/api/config/types" "GET" nil `(lambda (item) (mapcar #'intern item))))
-        (url-request-method "GET")
-        (url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key)))))
-    (with-temp-buffer
-      (url-insert-file-contents config-url)
-      (thread-last
-        (json-parse-buffer :object-type 'alist)
-        (mapcar #'intern)))))
 
-(defun khoj--construct-search-api-query (query content-type &optional rerank)
+(defun khoj--query-search-api-and-render-results (query content-type buffer-name &optional rerank title)
-  "Construct Search API Query.
+  "Query Khoj Search API with QUERY, CONTENT-TYPE and (optional) RERANK as query params
-Use QUERY, CONTENT-TYPE and (optional) RERANK as query params"
+Render results in BUFFER-NAME using search results, CONTENT-TYPE and (optional) TITLE."
-  (let ((rerank (or rerank "false"))
+  (let ((title (or title query))
-        (encoded-query (url-hexify-string query)))
+        (rerank (or rerank "false"))
-    (format "%s/api/search?q=%s&t=%s&r=%s&n=%s&client=emacs" khoj-server-url encoded-query content-type rerank khoj-results-count)))
+        (params `((q ,query) (t ,content-type) (r ,rerank) (n ,khoj-results-count)))
+        (path "/api/search"))
+    (khoj--call-api-async path
+                    "GET"
+                    params
+                    'khoj--render-search-results
+                    content-type title buffer-name)))
 
-(defun khoj--query-search-api-and-render-results (query-url content-type query buffer-name)
+(defun khoj--render-search-results (json-response content-type query buffer-name)
-  "Query Khoj Search with QUERY-URL.
-Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
-  ;; get json response from api
-  (with-current-buffer buffer-name
-    (let ((inhibit-read-only t)
-          (url-request-method "GET")
-          (url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key)))))
-      (erase-buffer)
-      (url-insert-file-contents query-url)))
   ;; render json response into formatted entries
   (with-current-buffer buffer-name
-    (let ((inhibit-read-only t)
+    (let ((inhibit-read-only t))
-          (json-response (json-parse-buffer :object-type 'alist)))
       (erase-buffer)
       (insert
        (cond ((equal content-type "org") (khoj--extract-entries-as-org json-response query))
@@ -673,16 +691,20 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
                  (equal content-type "org"))
              (progn (visual-line-mode)
                     (org-mode)
-                   (setq-local
+                    (setq-local
-                    org-startup-folded "showall"
+                     org-startup-folded "showall"
-                    org-hide-leading-stars t
+                     org-hide-leading-stars t
-                    org-startup-with-inline-images t)
+                     org-startup-with-inline-images t)
-                   (org-set-startup-visibility)))
+                    (org-set-startup-visibility)))
             ((equal content-type "markdown") (progn (markdown-mode)
                                                     (visual-line-mode)))
             ((equal content-type "image") (progn (shr-render-region (point-min) (point-max))
-                                                (goto-char (point-min))))
+                                                 (goto-char (point-min))))
             (t (fundamental-mode))))
+    ;; keep cursor at top of khoj buffer by default
+    (goto-char (point-min))
+    ;; enable minor modes for khoj chat
+    (visual-line-mode)
     (read-only-mode t)))
 
 
@@ -694,24 +716,52 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
   "Chat with Khoj."
   (interactive)
   (when (not (get-buffer khoj--chat-buffer-name))
-      (khoj--load-chat-history khoj--chat-buffer-name))
+      (khoj--load-chat-session khoj--chat-buffer-name))
-  (switch-to-buffer khoj--chat-buffer-name)
+  (khoj--open-side-pane khoj--chat-buffer-name)
   (let ((query (read-string "Query: ")))
     (when (not (string-empty-p query))
       (khoj--query-chat-api-and-render-messages query khoj--chat-buffer-name))))
 
-(defun khoj--load-chat-history (buffer-name)
+(defun khoj--open-side-pane (buffer-name)
+  "Open Khoj BUFFER-NAME in right side pane."
+  (if (get-buffer-window-list buffer-name)
+      ;; if window is already open, switch to it
+      (progn
+        (select-window (get-buffer-window buffer-name))
+        (switch-to-buffer buffer-name))
+    ;; else if window is not open, open it as a right-side window pane
+    (let ((bottomright-window (some-window (lambda (window) (and (window-at-side-p window 'right) (window-at-side-p window 'bottom))))))
+      (progn
+        ;; Select the right-most window
+        (select-window bottomright-window)
+        ;; if bottom-right window is not a vertical pane, split it vertically, else use the existing bottom-right vertical window
+        (let ((khoj-window (if (window-at-side-p bottomright-window 'left)
+                               (split-window-right)
+                             bottomright-window)))
+          ;; Set the buffer in the khoj window
+          (set-window-buffer khoj-window buffer-name)
+          ;; Switch to the khoj window
+          (select-window khoj-window)
+          ;; Resize the window to 1/3 of the frame width
+          (window-resize khoj-window
+                         (- (truncate (* 0.33 (frame-width))) (window-width))
+                         t))))))
+
+(defun khoj--load-chat-session (buffer-name &optional session-id)
   "Load Khoj Chat conversation history into BUFFER-NAME."
-  (let ((json-response (cdr (assoc 'response (khoj--get-chat-history-api)))))
+  (setq khoj--reference-count 0)
+  (let ((inhibit-read-only t)
+        (json-response (cdr (assoc 'chat (cdr (assoc 'response (khoj--get-chat-session session-id)))))))
     (with-current-buffer (get-buffer-create buffer-name)
       (erase-buffer)
       (insert "* Khoj Chat\n")
-      (thread-last
+      (when json-response
-        json-response
+        (thread-last
-        ;; generate chat messages from Khoj Chat API response
+          json-response
-        (mapcar #'khoj--render-chat-response)
+          ;; generate chat messages from Khoj Chat API response
-        ;; insert chat messages into Khoj Chat Buffer
+          (mapcar #'khoj--format-chat-response)
-        (mapc #'insert))
+          ;; insert chat messages into Khoj Chat Buffer
+          (mapc #'insert)))
       (progn
         (org-mode)
         (khoj--add-hover-text-to-footnote-refs (point-min))
@@ -727,6 +777,7 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
 
         ;; create khoj chat shortcut keybindings
         (use-local-map (copy-keymap org-mode-map))
+        (local-set-key (kbd "q") #'khoj--close)
         (local-set-key (kbd "m") #'khoj--chat)
         (local-set-key (kbd "C-x m") #'khoj--chat)
 
@@ -734,6 +785,13 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
         (visual-line-mode)
         (read-only-mode t)))))
 
+(defun khoj--close ()
+  "Kill Khoj buffer and window"
+  (interactive)
+  (progn
+    (kill-buffer (current-buffer))
+    (delete-window)))
+
 (defun khoj--add-hover-text-to-footnote-refs (start-pos)
   "Show footnote defs on mouse hover on footnote refs from START-POS."
   (org-with-wide-buffer
@@ -763,49 +821,72 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
   ;; render json response into formatted chat messages
   (with-current-buffer (get-buffer buffer-name)
     (let ((inhibit-read-only t)
-          (new-content-start-pos (point-max))
+          (query-time (format-time-string "%F %T")))
-          (query-time (format-time-string "%F %T"))
+      (goto-char (point-max))
-          (json-response (khoj--query-chat-api query)))
-      (goto-char new-content-start-pos)
       (insert
-       (khoj--render-chat-message query "you" query-time)
+       (khoj--render-chat-message query "you" query-time))
-       (khoj--render-chat-response json-response))
+      (khoj--query-chat-api query
-      (khoj--add-hover-text-to-footnote-refs new-content-start-pos))
+                            #'khoj--format-chat-response
-    (progn
+                            #'khoj--render-chat-response buffer-name))))
-      (org-set-startup-visibility)
-      (visual-line-mode)
-      (re-search-backward "^\*+ 🏮" nil t))))
 
-(defun khoj--query-chat-api (query)
+(defun khoj--query-chat-api (query callback &rest cbargs)
-  "Send QUERY to Khoj Chat API."
+  "Send QUERY to Khoj Chat API and call CALLBACK with the response.
-  (let* ((url-request-method "GET")
+CBARGS are optional additional arguments to pass to CALLBACK."
-         (encoded-query (url-hexify-string query))
+  (khoj--call-api-async "/api/chat"
-         (url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key))))
+                        "GET"
-         (query-url (format "%s/api/chat?q=%s&n=%s&client=emacs" khoj-server-url encoded-query khoj-results-count)))
+                        `(("q" ,query) ("n" ,khoj-results-count))
-    (with-temp-buffer
+                        callback cbargs))
-      (condition-case ex
-          (progn
-            (url-insert-file-contents query-url)
-            (json-parse-buffer :object-type 'alist))
-        ('file-error (cond ((string-match "Internal server error" (nth 2 ex))
-                      (message "Chat processor not configured. Configure OpenAI API key and restart it. Exception: [%s]" ex))
-                     (t (message "Chat exception: [%s]" ex))))))))
 
+(defun khoj--get-chat-sessions ()
+  "Get all chat sessions from Khoj server."
+  (khoj--call-api "/api/chat/sessions" "GET"))
 
-(defun khoj--get-chat-history-api ()
+(defun khoj--get-chat-session (&optional session-id)
-  "Send QUERY to Khoj Chat History API."
+  "Get chat messages from default or SESSION-ID chat session."
-  (let* ((url-request-method "GET")
+  (khoj--call-api "/api/chat/history"
-         (url-request-extra-headers `(("Authorization" . ,(format "Bearer %s" khoj-api-key))))
+                  "GET"
-         (query-url (format "%s/api/chat/history?client=emacs" khoj-server-url)))
+                  (when session-id `(("conversation_id" ,session-id)))))
-    (with-temp-buffer
-      (condition-case ex
-          (progn
-            (url-insert-file-contents query-url)
-            (json-parse-buffer :object-type 'alist))
-        ('file-error (cond ((string-match "Internal server error" (nth 2 ex))
-                      (message "Chat processor not configured. Configure OpenAI API key and restart it. Exception: [%s]" ex))
-                     (t (message "Chat exception: [%s]" ex))))))))
 
+(defun khoj--select-conversation-session (&optional completion-action)
+  "Select Khoj conversation session to perform COMPLETION-ACTION on."
+  (let* ((completion-text (format "%s Conversation:" (or completion-action "Open")))
+         (sessions (khoj--get-chat-sessions))
+         (session-alist (-map (lambda (session)
+                                (cons (if (not (equal :null (cdr (assoc 'slug session))))
+                                          (cdr (assoc 'slug session))
+                                        (format "New Conversation (%s)" (cdr (assoc 'conversation_id session))))
+                                      (cdr (assoc 'conversation_id session))))
+                              sessions))
+         (selected-session-slug (completing-read completion-text session-alist nil t)))
+    (cdr (assoc selected-session-slug session-alist))))
+
+(defun khoj--open-conversation-session ()
+  "Menu to select Khoj conversation session to open."
+  (let ((selected-session-id (khoj--select-conversation-session "Open")))
+    (khoj--load-chat-session khoj--chat-buffer-name selected-session-id)
+    (khoj--open-side-pane khoj--chat-buffer-name)))
+
+(defun khoj--create-chat-session ()
+  "Create new chat session."
+  (khoj--call-api "/api/chat/sessions" "POST"))
+
+(defun khoj--new-conversation-session ()
+  "Create new Khoj conversation session."
+  (let* ((session (khoj--create-chat-session))
+         (new-session-id (cdr (assoc 'conversation_id session))))
+    (khoj--load-chat-session khoj--chat-buffer-name new-session-id)
+    (khoj--open-side-pane khoj--chat-buffer-name)))
+
+(defun khoj--delete-chat-session (session-id)
+  "Delete new chat session."
+  (khoj--call-api "/api/chat/history" "DELETE" `(("conversation_id" ,session-id))))
+
+(defun khoj--delete-conversation-session ()
+  "Delete new Khoj conversation session."
+  (let* ((selected-session-id (khoj--select-conversation-session "Delete"))
+         (session (khoj--delete-chat-session selected-session-id)))
+    (khoj--load-chat-session khoj--chat-buffer-name)
+    (khoj--open-side-pane khoj--chat-buffer-name)))
 
 (defun khoj--render-chat-message (message sender &optional receive-date)
   "Render chat messages as `org-mode' list item.
@@ -829,38 +910,99 @@ RECEIVE-DATE is the message receive date."
 (defun khoj--generate-reference (reference)
   "Create `org-mode' footnotes with REFERENCE."
   (setq khoj--reference-count (1+ khoj--reference-count))
-  (cons
+  (let ((compiled-reference (if (stringp reference) reference (cdr (assoc 'compiled reference)))))
-   (propertize (format "^{ [fn:%x]}" khoj--reference-count) 'help-echo reference)
+    (cons
-   (thread-last
+     (propertize (format "^{ [fn:%x]}" khoj--reference-count) 'help-echo compiled-reference)
-     reference
+     (thread-last
-     ;; remove filename top heading line from reference
+       compiled-reference
-     ;; prevents actual reference heading in next line jumping out of references footnote section
+       ;; remove filename top heading line from reference
-     (replace-regexp-in-string "^\* .*\n" "")
+       ;; prevents actual reference heading in next line jumping out of references footnote section
-     ;; remove multiple, consecutive empty lines from reference
+       (replace-regexp-in-string "^\* .*\n" "")
-     (replace-regexp-in-string "\n\n" "\n")
+       ;; remove multiple, consecutive empty lines from reference
-     (format "\n[fn:%x] %s" khoj--reference-count))))
+       (replace-regexp-in-string "\n\n" "\n")
+       (format "\n[fn:%x] %s" khoj--reference-count)))))
 
-(defun khoj--render-chat-response (json-response)
+(defun khoj--generate-online-reference (reference)
+  (setq khoj--reference-count (1+ khoj--reference-count))
+  (let ((link (cdr (assoc 'link reference)))
+        (title (cdr (assoc 'title reference)))
+        (description (cdr (assoc 'description reference))))
+    (cons
+     (propertize (format "^{ [fn:%x]}" khoj--reference-count) 'help-echo (format "%s\n%s" link description))
+     (thread-last
+       description
+       ;; remove multiple, consecutive empty lines from reference
+       (replace-regexp-in-string "\n\n" "\n")
+       (format "\n[fn:%x] [[%s][%s]]\n%s\n" khoj--reference-count link title)))))
+
+(defun khoj--extract-online-references (result-types searches)
+  "Extract link, title, and description of specified RESULT-TYPES from SEARCHES."
+  (let ((result '()))
+    (-map
+     (lambda (search)
+      (let ((search-q (car search))
+            (search-results (cdr search)))
+        (-map-when
+         ;; filter search results by specified result types
+         (lambda (search-result) (member (car search-result) result-types))
+         ;; extract link, title, and description from search results
+         (lambda (search-result)
+           (-map
+            (lambda (entry)
+              (let ((link (cdr (or (assoc 'link entry) (assoc 'descriptionLink entry))))
+                    (title (cdr (or (assoc 'title entry) '(title . ,link))))
+                    (description (cdr (or (assoc 'snippet entry) (assoc 'description entry)))))
+                (setq result (append result `(((title . ,title) (link . ,link) (description . ,description) (search . ,search-q)))))))
+            ;; wrap search results in a list if it is not already a list
+            (if (or (equal 'knowledgeGraph (car search-result)) (equal 'webpages (car search-result)))
+                (list (cdr search-result))
+              (cdr search-result))))
+         search-results)))
+     searches)
+    result))
+
+(defun khoj--render-chat-response (response buffer-name)
+  (with-current-buffer (get-buffer buffer-name)
+    (let ((start-pos (point))
+          (inhibit-read-only t))
+      (goto-char (point-max))
+      (insert
+       response
+       (or (khoj--add-hover-text-to-footnote-refs start-pos) ""))
+      (progn
+        (org-set-startup-visibility)
+        (visual-line-mode)
+        (re-search-backward "^\*+ 🏮" nil t)))))
+
+(defun khoj--format-chat-response (json-response &optional callback &rest cbargs)
   "Render chat message using JSON-RESPONSE from Khoj Chat API."
   (let* ((message (cdr (or (assoc 'response json-response) (assoc 'message json-response))))
          (sender (cdr (assoc 'by json-response)))
          (receive-date (cdr (assoc 'created json-response)))
-         (references (or (cdr (assoc 'context json-response)) '()))
+         (online-references  (or (cdr (assoc 'onlineContext json-response)) '()))
-         (footnotes (mapcar #'khoj--generate-reference references))
+         (online-footnotes (-map #'khoj--generate-online-reference
-         (footnote-links (mapcar #'car footnotes))
+                                 (khoj--extract-online-references '(organic knowledgeGraph peopleAlsoAsk webpages)
-         (footnote-defs (mapcar #'cdr footnotes)))
+                                                                  online-references)))
-    (thread-first
+         (doc-references (or (cdr (assoc 'context json-response)) '()))
-      ;; concatenate khoj message and references from API
+         (doc-footnotes (mapcar #'khoj--generate-reference doc-references))
-      (concat
+         (footnote-links (mapcar #'car (append doc-footnotes online-footnotes)))
-       message
+         (footnote-defs (mapcar #'cdr (append doc-footnotes online-footnotes)))
-       ;; append reference links to khoj message
+         (formatted-response
-       (string-join footnote-links "")
+          (thread-first
-       ;; append reference sub-section to khoj message and fold it
+            ;; concatenate khoj message and references from API
-       (if footnote-defs "\n**** References\n:PROPERTIES:\n:VISIBILITY: folded\n:END:" "")
+            (concat
-       ;; append reference definitions to references subsection
+             message
-       (string-join footnote-defs " "))
+             ;; append reference links to khoj message
-      ;; Render chat message using data obtained from API
+             (string-join footnote-links "")
-      (khoj--render-chat-message sender receive-date))))
+             ;; append reference sub-section to khoj message and fold it
+             (if footnote-defs "\n**** References\n:PROPERTIES:\n:VISIBILITY: folded\n:END:" "")
+             ;; append reference definitions to references subsection
+             (string-join footnote-defs " "))
+            ;; Render chat message using data obtained from API
+            (khoj--render-chat-message sender receive-date))))
+    (if callback
+        (apply callback formatted-response cbargs)
+        formatted-response)))
 
 
 ;; ------------------
@@ -871,8 +1013,7 @@ RECEIVE-DATE is the message receive date."
   "Perform Incremental Search on Khoj. Allow optional RERANK of results."
   (let* ((rerank-str (cond (rerank "true") (t "false")))
          (khoj-buffer-name (get-buffer-create khoj--search-buffer-name))
-         (query (minibuffer-contents-no-properties))
+         (query (minibuffer-contents-no-properties)))
-         (query-url (khoj--construct-search-api-query query khoj--content-type rerank-str)))
     ;; Query khoj API only when user in khoj minibuffer and non-empty query
     ;; Prevents querying if
     ;;   1. user hasn't started typing query
@@ -892,10 +1033,10 @@ RECEIVE-DATE is the message receive date."
           (setq khoj--rerank t)
           (message "khoj.el: Rerank Results"))
         (khoj--query-search-api-and-render-results
-         query-url
-         khoj--content-type
          query
-         khoj-buffer-name))))))
+         khoj--content-type
+         khoj-buffer-name
+         rerank-str))))))
 
 (defun khoj--delete-open-network-connections-to-server ()
   "Delete all network connections to khoj server."
@@ -923,8 +1064,8 @@ RECEIVE-DATE is the message receive date."
   "Natural, Incremental Search for your personal notes and documents."
   (interactive)
   (let* ((khoj-buffer-name (get-buffer-create khoj--search-buffer-name)))
-    ;; switch to khoj results buffer
+    ;; switch to khoj search buffer
-    (switch-to-buffer khoj-buffer-name)
+    (khoj--open-side-pane khoj-buffer-name)
     ;; open and setup minibuffer for incremental search
     (minibuffer-with-setup-hook
         (lambda ()
@@ -989,7 +1130,6 @@ Paragraph only starts at first text after blank line."
                  ;; get paragraph, if in text mode
                  (t
                   (khoj--get-current-paragraph-text))))
-         (query-url (khoj--construct-search-api-query query content-type rerank))
          ;; extract heading to show in result buffer from query
          (query-title
           (format "Similar to: %s"
@@ -997,11 +1137,12 @@ Paragraph only starts at first text after blank line."
          (buffer-name (get-buffer-create khoj--search-buffer-name)))
     (progn
       (khoj--query-search-api-and-render-results
-       query-url
+       query
        content-type
-       query-title
+       buffer-name
-       buffer-name)
+       rerank
-      (switch-to-buffer buffer-name)
+       query-title)
+      (khoj--open-side-pane buffer-name)
       (goto-char (point-min)))))
 
 
@@ -1010,7 +1151,7 @@ Paragraph only starts at first text after blank line."
 ;; ---------
 
 (defun khoj--setup-and-show-menu ()
-  "Create Transient menu for khoj and show it."
+  "Create main Transient menu for Khoj and show it."
   ;; Create the Khoj Transient menu
   (transient-define-argument khoj--content-type-switch ()
     :class 'transient-switches
@@ -1057,15 +1198,40 @@ Paragraph only starts at first text after blank line."
     (interactive (list (transient-args transient-current-command)))
     (khoj--chat))
 
+  (transient-define-suffix khoj--open-conversation-session-command (&optional _)
+    "Command to select Khoj conversation sessions to open."
+    (interactive (list (transient-args transient-current-command)))
+    (khoj--open-conversation-session))
+
+  (transient-define-suffix khoj--new-conversation-session-command (&optional _)
+    "Command to select Khoj conversation sessions to open."
+    (interactive (list (transient-args transient-current-command)))
+    (khoj--new-conversation-session))
+
+  (transient-define-suffix khoj--delete-conversation-session-command (&optional _)
+    "Command to select Khoj conversation sessions to delete."
+    (interactive (list (transient-args transient-current-command)))
+    (khoj--delete-conversation-session))
+
+  (transient-define-prefix khoj--chat-menu ()
+    "Open the Khoj chat menu."
+    ["Act"
+     ("c" "Chat" khoj--chat-command)
+     ("o" "Open Conversation" khoj--open-conversation-session-command)
+     ("n" "New Conversation" khoj--new-conversation-session-command)
+     ("d" "Delete Conversation" khoj--delete-conversation-session-command)
+     ("q" "Quit" transient-quit-one)
+     ])
+
   (transient-define-prefix khoj--menu ()
     "Create Khoj Menu to Configure and Execute Commands."
     [["Configure Search"
-      ("n" "Results Count" "--results-count=" :init-value (lambda (obj) (oset obj value (format "%s" khoj-results-count))))
+      ("-n" "Results Count" "--results-count=" :init-value (lambda (obj) (oset obj value (format "%s" khoj-results-count))))
       ("t" "Content Type" khoj--content-type-switch)]
      ["Configure Update"
       ("-f" "Force Update" "--force-update")]]
     [["Act"
-      ("c" "Chat" khoj--chat-command)
+      ("c" "Chat" khoj--chat-menu)
       ("s" "Search" khoj--search-command)
       ("f" "Find Similar" khoj--find-similar-command)
       ("u" "Update" khoj--update-command)

src/interface/emacs/tests/khoj-tests.el

@@ -221,7 +221,7 @@ Rule everything\n")
            (equal
             (khoj--render-update-files-as-request-body (list upgrade-file act-file) "khoj")
             (format
-            "\n--khoj\r\n\
+             "\n--khoj\r\n\
 Content-Disposition: form-data; name=\"files\"; filename=\"%s\"\r\n\
 Content-Type: text/org\r\n\r\n\
 # Become God\n\
@@ -246,7 +246,7 @@ Rule everything\n\n\r\n\
            (equal
             (khoj--render-delete-files-as-request-body (list upgrade-file act-file "/tmp/deleted-file.org") "khoj")
             (format
-            "\n--khoj\r\n\
+             "\n--khoj\r\n\
 Content-Disposition: form-data; name=\"files\"; filename=\"%s\"\r\n\
 Content-Type: text/org\r\n\r\n\
 \r
@@ -262,6 +262,79 @@ Content-Type: text/org\r\n\r\n\
       (delete-file upgrade-file)
       (delete-file act-file))))
 
+(ert-deftest khoj-tests--extract-online-references ()
+  (let* (;; Arrange
+         (onlineContext '((Albert\ Einstein\ wife
+                           (organic . [((link . "https://en.wikipedia.org/wiki/Mileva_Mari%C4%87")
+                                        (title . "Mileva Marić - Wikipedia")
+                                        (snippet . "Marić married Einstein in 1903 and they had three ...")
+                                        (position . 1)
+                                        (sitelinks . [((link . "https://en.wikipedia.org/wiki/Mileva_Mari%C4%87#Bi...")
+                                                       (title . "Biography"))
+                                                      ((link . "https://en.wikipedia.org/wiki/Mileva_Mari%C4%87#De...")
+                                                       (title . "Debate over collaboration with..."))]))
+                                       ((link . "https://en.wikipedia.org/wiki/Elsa_Einstein")
+                                        (title . "Elsa Einstein - Wikipedia")
+                                        (snippet . "Elsa Einstein (18 January 1876 – 20 December 1936)...")
+                                        (position . 2))
+                                       ((link . "https://www.amazon.com/Einsteins-Wife-Story-Mileva...")
+                                        (price . 25.86)
+                                        (title . "Einstein's Wife: The Real Story of Mileva Einstein...")
+                                        (rating . 3.8)
+                                        (snippet . "Albert Einstein's first wife, Mileva Einstein-Mari...")
+                                        (currency . "$")
+                                        (position . 3)
+                                        (ratingCount . 80))])
+                           (peopleAlsoAsk . [((link . "https://en.wikipedia.org/wiki/Mileva_Mari%C4%87")
+                                              (title . "Mileva Marić - Wikipedia")
+                                              (snippet . "Death. Mileva Marić suffered a severe stroke and d...")
+                                              (question . "What happened to Einstein's first wife?"))
+                                             ((link . "https://www.redalyc.org/journal/5117/511767145014/...")
+                                              (title . "The story of Mileva Marić: Did Einstein's first wi...")
+                                              (snippet . "The couple were married in 1903. Some claim that M...")
+                                              (question . "Did Albert Einstein's wife do all his work?"))
+                                             ((link . "https://en.wikipedia.org/wiki/Hans_Albert_Einstein")
+                                              (title . "Hans Albert Einstein - Wikipedia")
+                                              (snippet . "Klaus Martin Einstein (1932–1939), died of diphthe...")
+                                              (question . "What happened to Einstein's children?"))])
+                           (knowledgeGraph (type . "Theoretical physicist")
+                                           (title . "Albert Einstein")
+                                           (attributes (Born . "March 14, 1879, Ulm, Germany")
+                                                       (Died . "April 18, 1955 (age 76 years), Princeton, NJ")
+                                                       (Spouse . "Elsa Einstein (m. 1919–1936) and Mileva Marić (m. ...")
+                                                       (Children . "Eduard Einstein, Hans Albert Einstein, and Lieserl...")
+                                                       (Education . "University of Zurich (1905), ETH Zürich (1897–1900...")
+                                                       (Nationality . "American, German, Hungarian, and more")
+                                                       (Grandchildren . "Evelyn Einstein, Bernhard Caesar Einstein, Klaus M..."))
+                                           (description . "Albert Einstein was a German-born theoretical phys...")
+                                           (descriptionLink . "https://en.wikipedia.org/wiki/Albert_Einstein")
+                                           (descriptionSource . "Wikipedia")))
+                          (Prince\ Albert\ spouse
+                           (webpages (link . "https://en.wikipedia.org/wiki/Prince_Albert_of_Sax...") (snippet . "Prince Albert of Saxe-Coburg and Gotha was the hus..."))
+                           (organic . [((date . "Feb 4, 2024")
+                                        (link . "https://en.wikipedia.org/wiki/Prince_Albert_of_Sax...")
+                                        (title . "Prince Albert of Saxe-Coburg and Gotha - Wikipedia")
+                                        (snippet . "Prince Albert of Saxe-Coburg and Gotha was the hus...")
+                                        (position . 1)
+                                        (sitelinks . [((link . "https://en.wikipedia.org/wiki/Prince_consort")
+                                                       (title . "Prince consort"))
+                                                      ((link . "https://en.wikipedia.org/wiki/Nellie_Clifden")
+                                                       (title . "Nellie Clifden"))]))])
+                           (answerBox (title . "Prince Albert of Saxe-Coburg and Gotha / Spouse")
+                                      (answer . "Queen Victoria"))
+                           (peopleAlsoAsk . [((link . "https://en.wikipedia.org/wiki/Albert_II,_Prince_of...")
+                                              (title . "Albert II, Prince of Monaco - Wikipedia")
+                                              (snippet . "In July 2011, Prince Albert married South African ...")
+                                              (question . "How many children does Prince Albert of Monaco hav..."))]))))
+         ;; Act
+         (result (khoj--extract-online-references
+                  '(organic knowledgeGraph peopleAlsoAsk webpages)
+                  onlineContext)))
+    ;; Assert
+    (progn
+      (should (equal (length result) 10))
+      result)))
+
 (provide 'khoj-tests)
 
 ;;; khoj-tests.el ends here