diff --git a/src/interface/desktop/main.js b/src/interface/desktop/main.js index f455db25..32c58a23 100644 --- a/src/interface/desktop/main.js +++ b/src/interface/desktop/main.js @@ -106,6 +106,7 @@ function filenameToMimeType (filename) { case 'org': return 'text/org'; default: + console.warn(`Unknown file type: ${extension}. Defaulting to text/plain.`); return 'text/plain'; } } diff --git a/src/interface/obsidian/src/main.ts b/src/interface/obsidian/src/main.ts index 6b24ee37..7e152c49 100644 --- a/src/interface/obsidian/src/main.ts +++ b/src/interface/obsidian/src/main.ts @@ -44,9 +44,7 @@ export default class Khoj extends Plugin { // Add scheduled job to update index every 60 minutes this.indexingTimer = setInterval(async () => { if (this.settings.autoConfigure) { - this.settings.lastSyncedFiles = await updateContentIndex( - this.app.vault, this.settings, this.settings.lastSyncedFiles - ); + this.settings.lastSync = await updateContentIndex(this.app.vault, this.settings, this.settings.lastSync); } }, 60 * 60 * 1000); } diff --git a/src/interface/obsidian/src/settings.ts b/src/interface/obsidian/src/settings.ts index 4eb6553e..875bd40a 100644 --- a/src/interface/obsidian/src/settings.ts +++ b/src/interface/obsidian/src/settings.ts @@ -8,7 +8,7 @@ export interface KhojSetting { khojApiKey: string; connectedToBackend: boolean; autoConfigure: boolean; - lastSyncedFiles: TFile[]; + lastSync: Map; userEmail: string; } @@ -18,7 +18,7 @@ export const DEFAULT_SETTINGS: KhojSetting = { khojApiKey: '', connectedToBackend: false, autoConfigure: true, - lastSyncedFiles: [], + lastSync: new Map(), userEmail: '', } @@ -132,8 +132,8 @@ export class KhojSettingTab extends PluginSettingTab { }, 300); this.plugin.registerInterval(progress_indicator); - this.plugin.settings.lastSyncedFiles = await updateContentIndex( - this.app.vault, this.plugin.settings, this.plugin.settings.lastSyncedFiles, true + this.plugin.settings.lastSync = await updateContentIndex( + this.app.vault, this.plugin.settings, this.plugin.settings.lastSync, true ); new Notice('βœ… Updated Khoj index.'); diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index 5c1061bd..11b6ac07 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -28,17 +28,43 @@ function fileExtensionToMimeType (extension: string): string { } } -export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSyncedFiles: TFile[], regenerate: boolean = false): Promise { +function filenameToMimeType (filename: TFile): string { + switch (filename.extension) { + case 'pdf': + return 'application/pdf'; + case 'png': + return 'image/png'; + case 'jpg': + case 'jpeg': + return 'image/jpeg'; + case 'md': + case 'markdown': + return 'text/markdown'; + case 'org': + return 'text/org'; + default: + console.warn(`Unknown file type: ${filename.extension}. Defaulting to text/plain.`); + return 'text/plain'; + } +} + +export async function updateContentIndex(vault: Vault, setting: KhojSetting, lastSync: Map, regenerate: boolean = false): Promise> { // Get all markdown, pdf files in the vault console.log(`Khoj: Updating Khoj content index...`) const files = vault.getFiles().filter(file => file.extension === 'md' || file.extension === 'markdown' || file.extension === 'pdf'); const binaryFileTypes = ['pdf'] let countOfFilesToIndex = 0; let countOfFilesToDelete = 0; + lastSync = lastSync.size > 0 ? lastSync : new Map(); // Add all files to index as multipart form data const fileData = []; for (const file of files) { + // Only push files that have been modified since last sync if not regenerating + if (!regenerate && file.stat.mtime < (lastSync.get(file) ?? 0)){ + continue; + } + countOfFilesToIndex++; const encoding = binaryFileTypes.includes(file.extension) ? "binary" : "utf8"; const mimeType = fileExtensionToMimeType(file.extension) + (encoding === "utf8" ? "; charset=UTF-8" : ""); @@ -47,14 +73,18 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las } // Add any previously synced files to be deleted to multipart form data - for (const lastSyncedFile of lastSyncedFiles) { + let filesToDelete: TFile[] = []; + for (const lastSyncedFile of lastSync.keys()) { if (!files.includes(lastSyncedFile)) { countOfFilesToDelete++; - fileData.push({blob: new Blob([]), path: lastSyncedFile.path}); + let fileObj = new Blob([""], { type: filenameToMimeType(lastSyncedFile) }); + fileData.push({blob: fileObj, path: lastSyncedFile.path}); + filesToDelete.push(lastSyncedFile); } } // Iterate through all indexable files in vault, 1000 at a time + let responses: string[] = []; let error_message = null; for (let i = 0; i < fileData.length; i += 1000) { const filesGroup = fileData.slice(i, i + 1000); @@ -79,16 +109,31 @@ export async function updateContentIndex(vault: Vault, setting: KhojSetting, las } else { error_message = `❗️Failed to sync your content with Khoj server. Raise issue on Khoj Discord or Github\nError: ${response.statusText}`; } + } else { + responses.push(await response.text()); } } + // Update last sync time for each successfully indexed file + files + .filter(file => responses.find(response => response.includes(file.path))) + .reduce((newSync, file) => { + newSync.set(file, new Date().getTime()); + return newSync; + }, lastSync); + + // Remove files that were deleted from last sync + filesToDelete + .filter(file => responses.find(response => response.includes(file.path))) + .forEach(file => lastSync.delete(file)); + if (error_message) { new Notice(error_message); } else { console.log(`βœ… Refreshed Khoj content index. Updated: ${countOfFilesToIndex} files, Deleted: ${countOfFilesToDelete} files.`); } - return files; + return lastSync; } export async function createNote(name: string, newLeaf = false): Promise { diff --git a/src/khoj/interface/web/base_config.html b/src/khoj/interface/web/base_config.html index 4126ad71..5c26e060 100644 --- a/src/khoj/interface/web/base_config.html +++ b/src/khoj/interface/web/base_config.html @@ -3,17 +3,17 @@ - + Khoj - Settings - - + + - +
diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index 0a154603..a2229376 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -4,12 +4,12 @@ Khoj - Chat - - - + + + - - + + - - + + +