sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2025-02-17 16:14:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Push 1000 files at a time from the Desktop client for indexing

Browse source 
FastAPI API endpoints only support uploading 1000 files at a time.
So split all files to index into groups of 1000 for upload to
index/update API endpoint
This commit is contained in:
Debanjum Singh Solanky 2024-01-03 23:08:20 +05:30
parent fca7a5ff32
commit efe41aaaca
1 changed files with 36 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
src/interface/desktop/main.js
View file

@ -151,7 +151,7 @@ function pushDataToKhoj (regenerate = false) {
} }
const lastSync = store.get('lastSync') || []; const lastSync = store.get('lastSync') || [];
const formData = new FormData(); const filesDataToPush = [];
for (const file of filesToPush) { for (const file of filesToPush) {
const stats = fs.statSync(file); const stats = fs.statSync(file);
if (!regenerate) { if (!regenerate) {
@ -167,7 +167,7 @@ function pushDataToKhoj (regenerate = false) {
let mimeType = filenameToMimeType(file) + (encoding === "utf8" ? "; charset=UTF-8" : ""); let mimeType = filenameToMimeType(file) + (encoding === "utf8" ? "; charset=UTF-8" : "");
let fileContent = Buffer.from(fs.readFileSync(file, { encoding: encoding }), encoding); let fileContent = Buffer.from(fs.readFileSync(file, { encoding: encoding }), encoding);
let fileObj = new Blob([fileContent], { type: mimeType }); let fileObj = new Blob([fileContent], { type: mimeType });
formData.append('files', fileObj, file); filesDataToPush.push({blob: fileObj, path: file});
state[file] = { state[file] = {
success: true, success: true,
} }
@ -184,36 +184,41 @@ function pushDataToKhoj (regenerate = false) {
for (const syncedFile of lastSync) { for (const syncedFile of lastSync) {
if (!filesToPush.includes(syncedFile.path)) { if (!filesToPush.includes(syncedFile.path)) {
fileObj = new Blob([""], { type: filenameToMimeType(syncedFile.path) }); fileObj = new Blob([""], { type: filenameToMimeType(syncedFile.path) });
formData.append('files', fileObj, syncedFile.path); filesDataToPush.push({blob: fileObj, path: syncedFile.path});
} }
} }
// Send collected files to Khoj server for indexing // Send collected files to Khoj server for indexing
if (!!formData?.entries()?.next().value) {
const hostURL = store.get('hostURL') || KHOJ_URL; const hostURL = store.get('hostURL') || KHOJ_URL;
const headers = { const headers = { 'Authorization': `Bearer ${store.get("khojToken")}` };
'Authorization': `Bearer ${store.get("khojToken")}` let requests = [];
};
axios.post(`${hostURL}/api/v1/index/update?force=${regenerate}&client=desktop`, formData, { headers }) // Request indexing files on server. With upto 1000 files in each request
.then(response => { for (let i = 0; i < filesDataToPush.length; i += 1000) {
console.log(response.data); const filesDataGroup = filesDataToPush.slice(i, i + 1000);
let lastSync = []; const formData = new FormData();
for (const file of filesToPush) { filesDataGroup.forEach(fileData => { formData.append('files', fileData.blob, fileData.path) });
lastSync.push({ let request = axios.post(`${hostURL}/api/v1/index/update?force=${regenerate}&client=desktop`, formData, { headers });
path: file, requests.push(request);
datetime: new Date().toISOString()
});
} }
// Wait for requests batch to finish
Promise
.all(requests)
.then(responses => {
const lastSync = filesToPush
.filter(file => responses.find(response => response.data.includes(file)))
.map(file => ({ path: file, datetime: new Date().toISOString() }));
store.set('lastSync', lastSync); store.set('lastSync', lastSync);
}) })
.catch(error => { .catch(error => {
console.error(error); console.error(error);
if (error.response.status == 429) { state["completed"] = false;
const win = BrowserWindow.getAllWindows()[0]; if (error.response.status === 429) {
win = BrowserWindow.getAllWindows()[0]
if (win) win.webContents.send('needsSubscription', true); if (win) win.webContents.send('needsSubscription', true);
if (win) win.webContents.send('update-state', state); if (win) win.webContents.send('update-state', state);
} }
state['completed'] = false
}) })
.finally(() => { .finally(() => {
// Syncing complete // Syncing complete
@ -221,12 +226,6 @@ function pushDataToKhoj (regenerate = false) {
const win = BrowserWindow.getAllWindows()[0]; const win = BrowserWindow.getAllWindows()[0];
if (win) win.webContents.send('update-state', state); if (win) win.webContents.send('update-state', state);
}); });
} else {
// Syncing complete
syncing = false;
const win = BrowserWindow.getAllWindows()[0];
if (win) win.webContents.send('update-state', state);
}
} }
pushDataToKhoj(); pushDataToKhoj();