Improve Styling of Khoj Search Modal on Obsidian and Indexing of Markdown

Merge pull request #198 from debanjum/improve-khoj-search-for-markdown-obsidian

### Overview
- Copied Khoj Search Modal styling from Jim Prince's PR #135 with minor improvements
- Implements improvements to the Khoj Search in Markdown/Obsidian suggested by folks. Specifically:
  - #133
  - #134
  - #142

### Changes
- 5673bd5 Keep original formatting in compiled text entry strings
- a2ab68a Include filename of markdown entries for search indexing
- 6712996 Create Note with Query as title from within Khoj Search Modal
- d3257cb Style the search result. Use Obsidian theme colors and font-size
- 4009148 For each result: snip it by lines, show filename, remove frontmatter

src/interface/obsidian/src/search_modal.ts

@@ -1,5 +1,6 @@
 import { App, SuggestModal, request, MarkdownRenderer, Instruction, Platform } from 'obsidian';
 import { KhojSetting } from 'src/settings';
+import { createNoteAndCloseModal } from 'src/utils';
 
 export interface SearchResult {
     entry: string;
@@ -10,6 +11,7 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
     setting: KhojSetting;
     rerank: boolean = false;
     find_similar_notes: boolean;
+    query: string = "";
     app: App;
 
     constructor(app: App, setting: KhojSetting, find_similar_notes: boolean = false) {
@@ -31,6 +33,14 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
             this.rerank = false
         });
 
+        // Register Modal Keybindings to Create New Note with Query as Title
+        this.scope.register(['Shift'], 'Enter', async () => {
+            if (this.query != "") createNoteAndCloseModal(this.query, this);
+        });
+        this.scope.register(['Ctrl', 'Shift'], 'Enter', async () => {
+            if (this.query != "") createNoteAndCloseModal(this.query, this, { newLeaf: true });
+        });
+
         // Add Hints to Modal for available Keybindings
         const modalInstructions: Instruction[] = [
             {
@@ -86,16 +96,31 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
             .filter((result: any) => !this.find_similar_notes || !result.additional.file.endsWith(this.app.workspace.getActiveFile()?.path))
             .map((result: any) => { return { entry: result.entry, file: result.additional.file } as SearchResult; });
 
+        this.query = query;
         return results;
     }
 
     async renderSuggestion(result: SearchResult, el: HTMLElement) {
-        let words_to_render = 30;
+        // Max number of lines to render
-        let entry_words = result.entry.split(' ')
+        let lines_to_render = 8;
-        let entry_snipped_indicator = entry_words.length > words_to_render ? ' **...**' : '';
+
-        let snipped_entry = entry_words.slice(0, words_to_render).join(' ');
+        // Extract filename of result
+        let os_path_separator = result.file.includes('\\') ? '\\' : '/';
+        let filename = result.file.split(os_path_separator).pop();
+
+        // Remove YAML frontmatter when rendering string
+        result.entry = result.entry.replace(/---[\n\r][\s\S]*---[\n\r]/, '');
+
+        // Truncate search results to lines_to_render
+        let entry_snipped_indicator = result.entry.split('\n').length > lines_to_render ? ' **...**' : '';
+        let snipped_entry = result.entry.split('\n').slice(0, lines_to_render).join('\n');
+
+        // Show filename of each search result for context
+        el.createEl("div",{ cls: 'khoj-result-file' }).setText(filename ?? "");
+        let result_el = el.createEl("div", { cls: 'khoj-result-entry' })
+
         // @ts-ignore
-        MarkdownRenderer.renderMarkdown(snipped_entry + entry_snipped_indicator, el, null, null);
+        MarkdownRenderer.renderMarkdown(snipped_entry + entry_snipped_indicator, result_el, null, null);
     }
 
     async onChooseSuggestion(result: SearchResult, _: MouseEvent | KeyboardEvent) {

src/interface/obsidian/src/utils.ts

@@ -1,4 +1,4 @@
-import { FileSystemAdapter, Notice, RequestUrlParam, request, Vault } from 'obsidian';
+import { FileSystemAdapter, Notice, RequestUrlParam, request, Vault, Modal } from 'obsidian';
 import { KhojSetting } from 'src/settings'
 
 export function getVaultAbsolutePath(vault: Vault): string {
@@ -139,3 +139,35 @@ export async function updateKhojBackend(khojUrl: string, khojConfig: Object) {
 function getIndexDirectoryFromBackendConfig(filepath: string) {
     return filepath.split("/").slice(0, -1).join("/");
 }
+
+export async function createNote(name: string, newLeaf = false): Promise<void> {
+    try {
+      let pathPrefix: string
+      switch (app.vault.getConfig('newFileLocation')) {
+        case 'current':
+          pathPrefix = (app.workspace.getActiveFile()?.parent.path ?? '') + '/'
+          break
+        case 'folder':
+          pathPrefix = this.app.vault.getConfig('newFileFolderPath') + '/'
+          break
+        default: // 'root'
+          pathPrefix = ''
+          break
+      }
+      await app.workspace.openLinkText(`${pathPrefix}${name}.md`, '', newLeaf)
+    } catch (e) {
+      console.error('Khoj: Could not create note.\n' + (e as any).message);
+      throw e
+    }
+  }
+
+export async function createNoteAndCloseModal(query: string, modal: Modal, opt?: { newLeaf: boolean }): Promise<void> {
+    try {
+        await createNote(query, opt?.newLeaf);
+    }
+    catch (e) {
+        new Notice((e as Error).message)
+        return
+    }
+    modal.close();
+}

src/interface/obsidian/styles.css

@@ -145,3 +145,32 @@ If your plugin does not need CSS, delete this file.
         padding: 2px 4px;
     }
 }
+
+.khoj-result-file {
+    font-weight: 600;
+  }
+
+  .khoj-result-entry {
+    color: var(--text-muted);
+    margin-left: 2em;
+    padding-left: 0.5em;
+    line-height: normal;
+    margin-top: 0.2em;
+    margin-bottom: 0.2em;
+    border-left-style: solid;
+    border-left-color: var(--color-accent-2);
+    white-space: normal;
+  }
+
+  .khoj-result-entry > * {
+    font-size: var(--font-ui-medium);
+  }
+
+  .khoj-result-entry > p {
+    margin-top: 0.2em;
+    margin-bottom: 0.2em;
+  }
+
+  .khoj-result-entry p br {
+    display: none;
+  }

src/khoj/processor/markdown/markdown_to_jsonl.py

@@ -1,8 +1,9 @@
 # Standard Packages
 import glob
-import re
 import logging
+import re
 import time
+from pathlib import Path
 from typing import List
 
 # Internal Packages
@@ -124,7 +125,10 @@ class MarkdownToJsonl(TextToJsonl):
         "Convert each Markdown entries into a dictionary"
         entries = []
         for parsed_entry in parsed_entries:
-            entries.append(Entry(compiled=parsed_entry, raw=parsed_entry, file=f"{entry_to_file_map[parsed_entry]}"))
+            entry_filename = Path(entry_to_file_map[parsed_entry])
+            # Append base filename to compiled entry for context to model
+            compiled_entry = f"{parsed_entry}\n{entry_filename.stem}"
+            entries.append(Entry(compiled=compiled_entry, raw=parsed_entry, file=f"{entry_filename}"))
 
         logger.debug(f"Converted {len(parsed_entries)} markdown entries to dictionaries")
 

src/khoj/processor/text_to_jsonl.py

@@ -31,7 +31,7 @@ class TextToJsonl(ABC):
         "Split entries if compiled entry length exceeds the max tokens supported by the ML model."
         chunked_entries: List[Entry] = []
         for entry in entries:
-            compiled_entry_words = entry.compiled.split()
+            compiled_entry_words = [word for word in entry.compiled.split(" ") if word != ""]
             # Drop long words instead of having entry truncated to maintain quality of entry processed by models
             compiled_entry_words = [word for word in compiled_entry_words if len(word) <= max_word_length]
             for chunk_index in range(0, len(compiled_entry_words), max_tokens):

tests/test_markdown_to_jsonl.py

@@ -1,5 +1,6 @@
 # Standard Packages
 import json
+from pathlib import Path
 
 # Internal Packages
 from khoj.processor.markdown.markdown_to_jsonl import MarkdownToJsonl
@@ -66,16 +67,17 @@ def test_multiple_markdown_entries_to_jsonl(tmp_path):
 
     # Act
     # Extract Entries from specified Markdown files
-    entries, entry_to_file_map = MarkdownToJsonl.extract_markdown_entries(markdown_files=[markdownfile])
+    entry_strings, entry_to_file_map = MarkdownToJsonl.extract_markdown_entries(markdown_files=[markdownfile])
+    entries = MarkdownToJsonl.convert_markdown_entries_to_maps(entry_strings, entry_to_file_map)
 
     # Process Each Entry from All Notes Files
-    jsonl_string = MarkdownToJsonl.convert_markdown_maps_to_jsonl(
+    jsonl_string = MarkdownToJsonl.convert_markdown_maps_to_jsonl(entries)
-        MarkdownToJsonl.convert_markdown_entries_to_maps(entries, entry_to_file_map)
-    )
     jsonl_data = [json.loads(json_string) for json_string in jsonl_string.splitlines()]
 
     # Assert
     assert len(jsonl_data) == 2
+    # Ensure entry compiled strings include the markdown files they originate from
+    assert all([markdownfile.stem in entry.compiled for entry in entries])
 
 
 def test_get_markdown_files(tmp_path):

tests/test_org_to_jsonl.py

@@ -44,7 +44,7 @@ def test_entry_split_when_exceeds_max_words(tmp_path):
     # Arrange
     entry = f"""*** Heading
     \t\r
-    Body Line 1
+    Body Line
     """
     orgfile = create_file(tmp_path, entry)