Improve Syncing Obsidian Vault, Invalidate Static Assets in Browser Cache in Web Client (#657)

- Improve - Only send files modified since their last sync for indexing on server from the Obsidian client - Fix - Invalidate static asset browser cache in Web client when Khoj version changes
2024-11-23 23:48:56 +01:00 · 2024-02-24 20:20:30 +05:30 · 2024-02-24 20:20:30 +05:30 · 8855529637
commit 8855529637
parent 42773e808c a46f70c4b0
13 changed files with 83 additions and 31 deletions
--- a/src/interface/desktop/main.js
+++ b/src/interface/desktop/main.js
@ -106,6 +106,7 @@ function filenameToMimeType (filename) {
        case 'org':
            return 'text/org';
        default:
+            console.warn(`Unknown file type: ${extension}. Defaulting to text/plain.`);
            return 'text/plain';
    }
 }
--- a/src/interface/obsidian/src/main.ts
+++ b/src/interface/obsidian/src/main.ts
@ -44,9 +44,7 @@ export default class Khoj extends Plugin {
        // Add scheduled job to update index every 60 minutes
        this.indexingTimer = setInterval(async () => {
            if (this.settings.autoConfigure) {
-                this.settings.lastSyncedFiles = await updateContentIndex(
-                    this.app.vault, this.settings, this.settings.lastSyncedFiles
-                );
+                this.settings.lastSync = await updateContentIndex(this.app.vault, this.settings, this.settings.lastSync);
            }
        }, 60 * 60 * 1000);
    }
--- a/src/interface/obsidian/src/settings.ts
+++ b/src/interface/obsidian/src/settings.ts
@ -8,7 +8,7 @@ export interface KhojSetting {
    khojApiKey: string;
    connectedToBackend: boolean;
    autoConfigure: boolean;
-    lastSyncedFiles: TFile[];
+    lastSync: Map<TFile, number>;
    userEmail: string;
 }

@ -18,7 +18,7 @@ export const DEFAULT_SETTINGS: KhojSetting = {
    khojApiKey: '',
    connectedToBackend: false,
    autoConfigure: true,
-    lastSyncedFiles: [],
+    lastSync: new Map(),
    userEmail: '',
 }

@ -132,8 +132,8 @@ export class KhojSettingTab extends PluginSettingTab {
                    }, 300);
                    this.plugin.registerInterval(progress_indicator);

-                    this.plugin.settings.lastSyncedFiles = await updateContentIndex(
-                        this.app.vault, this.plugin.settings, this.plugin.settings.lastSyncedFiles, true
+                    this.plugin.settings.lastSync = await updateContentIndex(
+                        this.app.vault, this.plugin.settings, this.plugin.settings.lastSync, true
                    );
                    new Notice('✅ Updated Khoj index.');

--- a/src/interface/obsidian/src/utils.ts
+++ b/src/interface/obsidian/src/utils.ts
@ -28,17 +28,43 @@ function fileExtensionToMimeType (extension: string): string {
    }
 }

-export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSyncedFiles: TFile[], regenerate: boolean = false): Promise<TFile[]> {
+function filenameToMimeType (filename: TFile): string {
+    switch (filename.extension) {
+        case 'pdf':
+            return 'application/pdf';
+        case 'png':
+            return 'image/png';
+        case 'jpg':
+        case 'jpeg':
+            return 'image/jpeg';
+        case 'md':
+        case 'markdown':
+            return 'text/markdown';
+        case 'org':
+            return 'text/org';
+        default:
+            console.warn(`Unknown file type: ${filename.extension}. Defaulting to text/plain.`);
+            return 'text/plain';
+    }
+}
+
+export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map<TFile, number>, regenerate: boolean = false): Promise<Map<TFile, number>> {
    // Get all markdown, pdf files in the vault
    console.log(`Khoj: Updating Khoj content index...`)
    const files = vault.getFiles().filter(file => file.extension === 'md' || file.extension === 'markdown' || file.extension === 'pdf');
    const binaryFileTypes = ['pdf']
    let countOfFilesToIndex = 0;
    let countOfFilesToDelete = 0;
+    lastSync = lastSync.size > 0 ? lastSync : new Map<TFile, number>();

    // Add all files to index as multipart form data
    const fileData = [];
    for (const file of files) {
+        // Only push files that have been modified since last sync if not regenerating
+        if (!regenerate && file.stat.mtime < (lastSync.get(file) ?? 0)){
+            continue;
+        }
+
        countOfFilesToIndex++;
        const encoding = binaryFileTypes.includes(file.extension) ? "binary" : "utf8";
        const mimeType = fileExtensionToMimeType(file.extension) + (encoding === "utf8" ? "; charset=UTF-8" : "");
@ -47,14 +73,18 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
    }

    // Add any previously synced files to be deleted to multipart form data
-    for (const lastSyncedFile of lastSyncedFiles) {
+    let filesToDelete: TFile[] = [];
+    for (const lastSyncedFile of lastSync.keys()) {
        if (!files.includes(lastSyncedFile)) {
            countOfFilesToDelete++;
-            fileData.push({blob: new Blob([]), path: lastSyncedFile.path});
+            let fileObj = new Blob([""], { type: filenameToMimeType(lastSyncedFile) });
+            fileData.push({blob: fileObj, path: lastSyncedFile.path});
+            filesToDelete.push(lastSyncedFile);
        }
    }

    // Iterate through all indexable files in vault, 1000 at a time
+    let responses: string[]  = [];
    let error_message = null;
    for (let i = 0; i < fileData.length; i += 1000) {
        const filesGroup = fileData.slice(i, i + 1000);
@ -79,16 +109,31 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las
            } else {
                error_message = `❗️Failed to sync your content with Khoj server. Raise issue on Khoj Discord or Github\nError: ${response.statusText}`;
            }
+        } else {
+            responses.push(await response.text());
        }
    }

+    // Update last sync time for each successfully indexed file
+   files
+    .filter(file => responses.find(response => response.includes(file.path)))
+    .reduce((newSync, file) => {
+        newSync.set(file, new Date().getTime());
+        return newSync;
+    }, lastSync);
+
+    // Remove files that were deleted from last sync
+    filesToDelete
+    .filter(file => responses.find(response => response.includes(file.path)))
+    .forEach(file => lastSync.delete(file));
+
    if (error_message) {
        new Notice(error_message);
    } else {
        console.log(`✅ Refreshed Khoj content index. Updated: ${countOfFilesToIndex} files, Deleted: ${countOfFilesToDelete} files.`);
    }

-    return files;
+    return lastSync;
 }

 export async function createNote(name: string, newLeaf = false): Promise<void> {
--- a/src/khoj/interface/web/base_config.html
+++ b/src/khoj/interface/web/base_config.html
@ -3,17 +3,17 @@
    <head>
        <meta charset="utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
-        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png">
+        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png?v={{ khoj_version }}">
        <title>Khoj - Settings</title>
-        <link rel="stylesheet" href="/static/assets/pico.min.css">
-        <link rel="stylesheet" href="/static/assets/khoj.css">
+        <link rel="stylesheet" href="/static/assets/pico.min.css?v={{ khoj_version }}">
+        <link rel="stylesheet" href="/static/assets/khoj.css?v={{ khoj_version }}">
        <script
            integrity="sha384-05IkdNHoAlkhrFVUCCN805WC/h4mcI98GUBssmShF2VJAXKyZTrO/TmJ+4eBo0Cy"
            crossorigin="anonymous"
            src="https://cdnjs.cloudflare.com/ajax/libs/intl-tel-input/17.0.13/js/intlTelInput.min.js"></script>
        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/intl-tel-input/17.0.13/css/intlTelInput.css">
    </head>
-    <script type="text/javascript" src="/static/assets/utils.js"></script>
+    <script type="text/javascript" src="/static/assets/utils.js?v={{ khoj_version }}"></script>
    <body class="khoj-configure">
        <div class="khoj-header-wrapper">
            <div class="filler"></div>
--- a/src/khoj/interface/web/chat.html
+++ b/src/khoj/interface/web/chat.html
@ -4,12 +4,12 @@
        <meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
        <title>Khoj - Chat</title>

-        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png">
-        <link rel="manifest" href="/static/khoj.webmanifest">
-        <link rel="stylesheet" href="/static/assets/khoj.css">
+        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png?v={{ khoj_version }}">
+        <link rel="manifest" href="/static/khoj.webmanifest?v={{ khoj_version }}">
+        <link rel="stylesheet" href="/static/assets/khoj.css?v={{ khoj_version }}">
    </head>
-    <script type="text/javascript" src="/static/assets/utils.js"></script>
-    <script type="text/javascript" src="/static/assets/markdown-it.min.js"></script>
+    <script type="text/javascript" src="/static/assets/utils.js?v={{ khoj_version }}"></script>
+    <script type="text/javascript" src="/static/assets/markdown-it.min.js?v={{ khoj_version }}"></script>
    <script>
        let welcome_message = `
 Hi, I am Khoj, your open, personal AI 👋🏽. I can help:
--- a/src/khoj/interface/web/content_source_computer_input.html
+++ b/src/khoj/interface/web/content_source_computer_input.html
@ -3,7 +3,7 @@
 <div class="page">
    <div class="section">
        <h2 class="section-title">
-            <img class="card-icon" src="/static/assets/icons/computer.png" alt="files">
+            <img class="card-icon" src="/static/assets/icons/computer.png?v={{ khoj_version }}" alt="files">
            <span class="card-title-text">Files</span>
            <div class="instructions">
                <p class="card-description">Manage files from your computer</p>
--- a/src/khoj/interface/web/content_source_github_input.html
+++ b/src/khoj/interface/web/content_source_github_input.html
@ -3,7 +3,7 @@
 <div class="page">
    <div class="section">
        <h2 class="section-title">
-            <img class="card-icon" src="/static/assets/icons/github.svg" alt="Github">
+            <img class="card-icon" src="/static/assets/icons/github.svg?v={{ khoj_version }}" alt="Github">
            <span class="card-title-text">Github</span>
            <div class="instructions">
                <a href="https://docs.khoj.dev/#/github_integration">ⓘ Help</a>
--- a/src/khoj/interface/web/content_source_notion_input.html
+++ b/src/khoj/interface/web/content_source_notion_input.html
@ -3,7 +3,7 @@
 <div class="page">
    <div class="section">
        <h2 class="section-title">
-            <img class="card-icon" src="/static/assets/icons/notion.svg" alt="Notion">
+            <img class="card-icon" src="/static/assets/icons/notion.svg?v={{ khoj_version }}" alt="Notion">
            <span class="card-title-text">Notion</span>
            <div class="instructions">
                <a href="https://docs.khoj.dev/#/notion_integration">ⓘ Help</a>
--- a/src/khoj/interface/web/search.html
+++ b/src/khoj/interface/web/search.html
@ -4,13 +4,13 @@
        <meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
        <title>Khoj - Search</title>

-        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png">
-        <link rel="manifest" href="/static/khoj.webmanifest">
-        <link rel="stylesheet" href="/static/assets/khoj.css">
+        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png?v={{ khoj_version }}">
+        <link rel="manifest" href="/static/khoj.webmanifest?v={{ khoj_version }}">
+        <link rel="stylesheet" href="/static/assets/khoj.css?v={{ khoj_version }}">
    </head>
-    <script type="text/javascript" src="/static/assets/org.min.js"></script>
-    <script type="text/javascript" src="/static/assets/markdown-it.min.js"></script>
-    <script type="text/javascript" src="/static/assets/utils.js"></script>
+    <script type="text/javascript" src="/static/assets/org.min.js?v={{ khoj_version }}"></script>
+    <script type="text/javascript" src="/static/assets/markdown-it.min.js?v={{ khoj_version }}"></script>
+    <script type="text/javascript" src="/static/assets/utils.js?v={{ khoj_version }}"></script>

    <script>
        function render_image(item) {
--- a/src/khoj/interface/web/utils.html
+++ b/src/khoj/interface/web/utils.html
@ -1,7 +1,7 @@
 {% macro heading_pane(user_photo, username, is_active, has_documents) -%}
    <div class="khoj-header">
        <a class="khoj-logo" href="/">
-            <img class="khoj-logo" src="/static/assets/icons/khoj-logo-sideways-500.png" alt="Khoj"></img>
+            <img class="khoj-logo" src="/static/assets/icons/khoj-logo-sideways-500.png?v={{ khoj_version }}" alt="Khoj"></img>
        </a>
        <nav class="khoj-nav">
            {% if has_documents %}
--- a/src/khoj/routers/web_client.py
+++ b/src/khoj/routers/web_client.py
@ -47,6 +47,7 @@ def index(request: Request):
            "user_photo": user_picture,
            "is_active": has_required_scope(request, ["premium"]),
            "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
        },
    )

@ -66,6 +67,7 @@ def index_post(request: Request):
            "user_photo": user_picture,
            "is_active": has_required_scope(request, ["premium"]),
            "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
        },
    )

@ -85,6 +87,7 @@ def search_page(request: Request):
            "user_photo": user_picture,
            "is_active": has_required_scope(request, ["premium"]),
            "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
        },
    )

@ -104,6 +107,7 @@ def chat_page(request: Request):
            "user_photo": user_picture,
            "is_active": has_required_scope(request, ["premium"]),
            "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
        },
    )

@ -183,6 +187,7 @@ def config_page(request: Request):
            "is_twilio_enabled": is_twilio_enabled(),
            "phone_number": user.phone_number,
            "is_phone_number_verified": user.verified_phone_number,
+            "khoj_version": state.khoj_version,
        },
    )

@ -223,6 +228,7 @@ def github_config_page(request: Request):
            "user_photo": user_picture,
            "is_active": has_required_scope(request, ["premium"]),
            "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
        },
    )

@ -250,6 +256,7 @@ def notion_config_page(request: Request):
            "user_photo": user_picture,
            "is_active": has_required_scope(request, ["premium"]),
            "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
        },
    )

@ -269,5 +276,6 @@ def computer_config_page(request: Request):
            "user_photo": user_picture,
            "is_active": has_required_scope(request, ["premium"]),
            "has_documents": has_documents,
+            "khoj_version": state.khoj_version,
        },
    )
--- a/src/khoj/search_type/text_search.py
+++ b/src/khoj/search_type/text_search.py
@ -211,7 +211,7 @@ def setup(
        file_names = [file_name for file_name in files]

        logger.info(
-            f"Deleted {num_deleted_embeddings} entries. Created {num_new_embeddings} new entries for user {user} from files {file_names}"
+            f"Deleted {num_deleted_embeddings} entries. Created {num_new_embeddings} new entries for user {user} from files {file_names[:10]} ..."
        )