Improve Syncing Obsidian Vault, Invalidate Static Assets in Browser Cache in Web Client (#657)

- Improve
  - Only send files modified since their last sync for indexing on server from the Obsidian client
- Fix 
  - Invalidate static asset browser cache in Web client when Khoj version changes

src/interface/desktop/main.js

@@ -106,6 +106,7 @@ function filenameToMimeType (filename) {
         case 'org':
             return 'text/org';
         default:
+            console.warn(`Unknown file type: ${extension}. Defaulting to text/plain.`);
             return 'text/plain';
     }
 }

src/interface/obsidian/src/main.ts

@@ -44,9 +44,7 @@ export default class Khoj extends Plugin {
         // Add scheduled job to update index every 60 minutes
         this.indexingTimer = setInterval(async () => {
             if (this.settings.autoConfigure) {
-                this.settings.lastSyncedFiles = await updateContentIndex(
+                this.settings.lastSync = await updateContentIndex(this.app.vault, this.settings, this.settings.lastSync);
-                    this.app.vault, this.settings, this.settings.lastSyncedFiles
-                );
             }
         }, 60 * 60 * 1000);
     }

src/interface/obsidian/src/settings.ts

@@ -8,7 +8,7 @@ export interface KhojSetting {
     khojApiKey: string;
     connectedToBackend: boolean;
     autoConfigure: boolean;
-    lastSyncedFiles: TFile[];
+    lastSync: Map<TFile, number>;
     userEmail: string;
 }
 
@@ -18,7 +18,7 @@ export const DEFAULT_SETTINGS: KhojSetting = {
     khojApiKey: '',
     connectedToBackend: false,
     autoConfigure: true,
-    lastSyncedFiles: [],
+    lastSync: new Map(),
     userEmail: '',
 }
 
@@ -132,8 +132,8 @@ export class KhojSettingTab extends PluginSettingTab {
                     }, 300);
                     this.plugin.registerInterval(progress_indicator);
 
-                    this.plugin.settings.lastSyncedFiles = await updateContentIndex(
+                    this.plugin.settings.lastSync = await updateContentIndex(
-                        this.app.vault, this.plugin.settings, this.plugin.settings.lastSyncedFiles, true
+                        this.app.vault, this.plugin.settings, this.plugin.settings.lastSync, true
                     );
                     new Notice('✅ Updated Khoj index.');
 

src/interface/obsidian/src/utils.ts

@@ -28,17 +28,43 @@ function fileExtensionToMimeType (extension: string): string {
     }
 }
 
-export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSyncedFiles: TFile[], regenerate: boolean = false): Promise<TFile[]> {
+function filenameToMimeType (filename: TFile): string {
+    switch (filename.extension) {
+        case 'pdf':
+            return 'application/pdf';
+        case 'png':
+            return 'image/png';
+        case 'jpg':
+        case 'jpeg':
+            return 'image/jpeg';
+        case 'md':
+        case 'markdown':
+            return 'text/markdown';
+        case 'org':
+            return 'text/org';
+        default:
+            console.warn(`Unknown file type: ${filename.extension}. Defaulting to text/plain.`);
+            return 'text/plain';
+    }
+}
+
+export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map<TFile, number>, regenerate: boolean = false): Promise<Map<TFile, number>> {
     // Get all markdown, pdf files in the vault
     console.log(`Khoj: Updating Khoj content index...`)
     const files = vault.getFiles().filter(file => file.extension === 'md' || file.extension === 'markdown' || file.extension === 'pdf');
     const binaryFileTypes = ['pdf']
     let countOfFilesToIndex = 0;
     let countOfFilesToDelete = 0;
+    lastSync = lastSync.size > 0 ? lastSync : new Map<TFile, number>();
 
     // Add all files to index as multipart form data
     const fileData = [];
     for (const file of files) {
+        // Only push files that have been modified since last sync if not regenerating
+        if (!regenerate && file.stat.mtime < (lastSync.get(file) ?? 0)){
+            continue;
+        }
+
         countOfFilesToIndex++;
         const encoding = binaryFileTypes.includes(file.extension) ? "binary" : "utf8";
         const mimeType = fileExtensionToMimeType(file.extension) + (encoding === "utf8" ? "; charset=UTF-8" : "");
@@ -47,14 +73,18 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
     }
 
     // Add any previously synced files to be deleted to multipart form data
-    for (const lastSyncedFile of lastSyncedFiles) {
+    let filesToDelete: TFile[] = [];
+    for (const lastSyncedFile of lastSync.keys()) {
         if (!files.includes(lastSyncedFile)) {
             countOfFilesToDelete++;
-            fileData.push({blob: new Blob([]), path: lastSyncedFile.path});
+            let fileObj = new Blob([""], { type: filenameToMimeType(lastSyncedFile) });
+            fileData.push({blob: fileObj, path: lastSyncedFile.path});
+            filesToDelete.push(lastSyncedFile);
         }
     }
 
     // Iterate through all indexable files in vault, 1000 at a time
+    let responses: string[]  = [];
     let error_message = null;
     for (let i = 0; i < fileData.length; i += 1000) {
         const filesGroup = fileData.slice(i, i + 1000);
@@ -79,16 +109,31 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
             } else {
                 error_message = `❗️Failed to sync your content with Khoj server. Raise issue on Khoj Discord or Github\nError: ${response.statusText}`;
             }
+        } else {
+            responses.push(await response.text());
         }
     }
 
+    // Update last sync time for each successfully indexed file
+   files
+    .filter(file => responses.find(response => response.includes(file.path)))
+    .reduce((newSync, file) => {
+        newSync.set(file, new Date().getTime());
+        return newSync;
+    }, lastSync);
+
+    // Remove files that were deleted from last sync
+    filesToDelete
+    .filter(file => responses.find(response => response.includes(file.path)))
+    .forEach(file => lastSync.delete(file));
+
     if (error_message) {
         new Notice(error_message);
     } else {
         console.log(`✅ Refreshed Khoj content index. Updated: ${countOfFilesToIndex} files, Deleted: ${countOfFilesToDelete} files.`);
     }
 
-    return files;
+    return lastSync;
 }
 
 export async function createNote(name: string, newLeaf = false): Promise<void> {

src/khoj/interface/web/base_config.html

@@ -3,17 +3,17 @@
     <head>
         <meta charset="utf-8">
         <meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
-        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png">
+        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png?v={{ khoj_version }}">
         <title>Khoj - Settings</title>
-        <link rel="stylesheet" href="/static/assets/pico.min.css">
+        <link rel="stylesheet" href="/static/assets/pico.min.css?v={{ khoj_version }}">
-        <link rel="stylesheet" href="/static/assets/khoj.css">
+        <link rel="stylesheet" href="/static/assets/khoj.css?v={{ khoj_version }}">
         <script
             integrity="sha384-05IkdNHoAlkhrFVUCCN805WC/h4mcI98GUBssmShF2VJAXKyZTrO/TmJ+4eBo0Cy"
             crossorigin="anonymous"
             src="https://cdnjs.cloudflare.com/ajax/libs/intl-tel-input/17.0.13/js/intlTelInput.min.js"></script>
         <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/intl-tel-input/17.0.13/css/intlTelInput.css">
     </head>
-    <script type="text/javascript" src="/static/assets/utils.js"></script>
+    <script type="text/javascript" src="/static/assets/utils.js?v={{ khoj_version }}"></script>
     <body class="khoj-configure">
         <div class="khoj-header-wrapper">
             <div class="filler"></div>

src/khoj/interface/web/chat.html

@@ -4,12 +4,12 @@
         <meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
         <title>Khoj - Chat</title>
 
-        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png">
+        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png?v={{ khoj_version }}">
-        <link rel="manifest" href="/static/khoj.webmanifest">
+        <link rel="manifest" href="/static/khoj.webmanifest?v={{ khoj_version }}">
-        <link rel="stylesheet" href="/static/assets/khoj.css">
+        <link rel="stylesheet" href="/static/assets/khoj.css?v={{ khoj_version }}">
     </head>
-    <script type="text/javascript" src="/static/assets/utils.js"></script>
+    <script type="text/javascript" src="/static/assets/utils.js?v={{ khoj_version }}"></script>
-    <script type="text/javascript" src="/static/assets/markdown-it.min.js"></script>
+    <script type="text/javascript" src="/static/assets/markdown-it.min.js?v={{ khoj_version }}"></script>
     <script>
         let welcome_message = `
 Hi, I am Khoj, your open, personal AI 👋🏽. I can help:

src/khoj/interface/web/content_source_computer_input.html

@@ -3,7 +3,7 @@
 <div class="page">
     <div class="section">
         <h2 class="section-title">
-            <img class="card-icon" src="/static/assets/icons/computer.png" alt="files">
+            <img class="card-icon" src="/static/assets/icons/computer.png?v={{ khoj_version }}" alt="files">
             <span class="card-title-text">Files</span>
             <div class="instructions">
                 <p class="card-description">Manage files from your computer</p>

src/khoj/interface/web/content_source_github_input.html

@@ -3,7 +3,7 @@
 <div class="page">
     <div class="section">
         <h2 class="section-title">
-            <img class="card-icon" src="/static/assets/icons/github.svg" alt="Github">
+            <img class="card-icon" src="/static/assets/icons/github.svg?v={{ khoj_version }}" alt="Github">
             <span class="card-title-text">Github</span>
             <div class="instructions">
                 <a href="https://docs.khoj.dev/#/github_integration">ⓘ Help</a>

src/khoj/interface/web/content_source_notion_input.html

@@ -3,7 +3,7 @@
 <div class="page">
     <div class="section">
         <h2 class="section-title">
-            <img class="card-icon" src="/static/assets/icons/notion.svg" alt="Notion">
+            <img class="card-icon" src="/static/assets/icons/notion.svg?v={{ khoj_version }}" alt="Notion">
             <span class="card-title-text">Notion</span>
             <div class="instructions">
                 <a href="https://docs.khoj.dev/#/notion_integration">ⓘ Help</a>

src/khoj/interface/web/search.html

@@ -4,13 +4,13 @@
         <meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
         <title>Khoj - Search</title>
 
-        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png">
+        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png?v={{ khoj_version }}">
-        <link rel="manifest" href="/static/khoj.webmanifest">
+        <link rel="manifest" href="/static/khoj.webmanifest?v={{ khoj_version }}">
-        <link rel="stylesheet" href="/static/assets/khoj.css">
+        <link rel="stylesheet" href="/static/assets/khoj.css?v={{ khoj_version }}">
     </head>
-    <script type="text/javascript" src="/static/assets/org.min.js"></script>
+    <script type="text/javascript" src="/static/assets/org.min.js?v={{ khoj_version }}"></script>
-    <script type="text/javascript" src="/static/assets/markdown-it.min.js"></script>
+    <script type="text/javascript" src="/static/assets/markdown-it.min.js?v={{ khoj_version }}"></script>
-    <script type="text/javascript" src="/static/assets/utils.js"></script>
+    <script type="text/javascript" src="/static/assets/utils.js?v={{ khoj_version }}"></script>
 
     <script>
         function render_image(item) {

src/khoj/interface/web/utils.html

@@ -1,7 +1,7 @@
 {% macro heading_pane(user_photo, username, is_active, has_documents) -%}
     <div class="khoj-header">
         <a class="khoj-logo" href="/">
-            <img class="khoj-logo" src="/static/assets/icons/khoj-logo-sideways-500.png" alt="Khoj"></img>
+            <img class="khoj-logo" src="/static/assets/icons/khoj-logo-sideways-500.png?v={{ khoj_version }}" alt="Khoj"></img>
         </a>
         <nav class="khoj-nav">
             {% if has_documents %}

src/khoj/routers/web_client.py

@@ -47,6 +47,7 @@ def index(request: Request):
             "user_photo": user_picture,
             "is_active": has_required_scope(request, ["premium"]),
             "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
         },
     )
 
@@ -66,6 +67,7 @@ def index_post(request: Request):
             "user_photo": user_picture,
             "is_active": has_required_scope(request, ["premium"]),
             "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
         },
     )
 
@@ -85,6 +87,7 @@ def search_page(request: Request):
             "user_photo": user_picture,
             "is_active": has_required_scope(request, ["premium"]),
             "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
         },
     )
 
@@ -104,6 +107,7 @@ def chat_page(request: Request):
             "user_photo": user_picture,
             "is_active": has_required_scope(request, ["premium"]),
             "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
         },
     )
 
@@ -183,6 +187,7 @@ def config_page(request: Request):
             "is_twilio_enabled": is_twilio_enabled(),
             "phone_number": user.phone_number,
             "is_phone_number_verified": user.verified_phone_number,
+            "khoj_version": state.khoj_version,
         },
     )
 
@@ -223,6 +228,7 @@ def github_config_page(request: Request):
             "user_photo": user_picture,
             "is_active": has_required_scope(request, ["premium"]),
             "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
         },
     )
 
@@ -250,6 +256,7 @@ def notion_config_page(request: Request):
             "user_photo": user_picture,
             "is_active": has_required_scope(request, ["premium"]),
             "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
         },
     )
 
@@ -269,5 +276,6 @@ def computer_config_page(request: Request):
             "user_photo": user_picture,
             "is_active": has_required_scope(request, ["premium"]),
             "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
         },
     )

src/khoj/search_type/text_search.py

@@ -211,7 +211,7 @@ def setup(
         file_names = [file_name for file_name in files]
 
         logger.info(
-            f"Deleted {num_deleted_embeddings} entries. Created {num_new_embeddings} new entries for user {user} from files {file_names}"
+            f"Deleted {num_deleted_embeddings} entries. Created {num_new_embeddings} new entries for user {user} from files {file_names[:10]} ..."
         )