mirror of
https://github.com/khoj-ai/khoj.git
synced 2024-12-04 04:43:01 +01:00
Add new API to batch delete a list of files by filename
- Rearrange DELETE content API definitions order to go from more specific to more general - Create batched file deletion DB adapter
This commit is contained in:
parent
9d86cb57ac
commit
ccc46a09b5
2 changed files with 69 additions and 34 deletions
|
@ -1106,6 +1106,16 @@ class EntryAdapters:
|
||||||
async def adelete_entry_by_file(user: KhojUser, file_path: str):
|
async def adelete_entry_by_file(user: KhojUser, file_path: str):
|
||||||
return await Entry.objects.filter(user=user, file_path=file_path).adelete()
|
return await Entry.objects.filter(user=user, file_path=file_path).adelete()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def adelete_entries_by_filenames(user: KhojUser, filenames: List[str], batch_size=1000):
|
||||||
|
deleted_count = 0
|
||||||
|
for i in range(0, len(filenames), batch_size):
|
||||||
|
batch = filenames[i : i + batch_size]
|
||||||
|
count, _ = await Entry.objects.filter(user=user, file_path__in=batch).adelete()
|
||||||
|
deleted_count += count
|
||||||
|
|
||||||
|
return deleted_count
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_all_filenames_by_source(user: KhojUser, file_source: str):
|
def get_all_filenames_by_source(user: KhojUser, file_source: str):
|
||||||
return (
|
return (
|
||||||
|
|
|
@ -236,42 +236,9 @@ async def set_content_notion(
|
||||||
return {"status": "ok"}
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
@api_content.delete("/{content_source}", status_code=200)
|
|
||||||
@requires(["authenticated"])
|
|
||||||
async def delete_content_source(
|
|
||||||
request: Request,
|
|
||||||
content_source: str,
|
|
||||||
client: Optional[str] = None,
|
|
||||||
):
|
|
||||||
user = request.user.object
|
|
||||||
|
|
||||||
content_object = map_config_to_object(content_source)
|
|
||||||
if content_object is None:
|
|
||||||
raise ValueError(f"Invalid content source: {content_source}")
|
|
||||||
elif content_object != "Computer":
|
|
||||||
await content_object.objects.filter(user=user).adelete()
|
|
||||||
await sync_to_async(EntryAdapters.delete_all_entries)(user, file_source=content_source)
|
|
||||||
|
|
||||||
if content_source == DbEntry.EntrySource.NOTION:
|
|
||||||
await NotionConfig.objects.filter(user=user).adelete()
|
|
||||||
elif content_source == DbEntry.EntrySource.GITHUB:
|
|
||||||
await GithubConfig.objects.filter(user=user).adelete()
|
|
||||||
|
|
||||||
update_telemetry_state(
|
|
||||||
request=request,
|
|
||||||
telemetry_type="api",
|
|
||||||
api="delete_content_config",
|
|
||||||
client=client,
|
|
||||||
metadata={"content_source": content_source},
|
|
||||||
)
|
|
||||||
|
|
||||||
enabled_content = await sync_to_async(EntryAdapters.get_unique_file_types)(user)
|
|
||||||
return {"status": "ok"}
|
|
||||||
|
|
||||||
|
|
||||||
@api_content.delete("/file", status_code=201)
|
@api_content.delete("/file", status_code=201)
|
||||||
@requires(["authenticated"])
|
@requires(["authenticated"])
|
||||||
async def delete_content_file(
|
async def delete_content_files(
|
||||||
request: Request,
|
request: Request,
|
||||||
filename: str,
|
filename: str,
|
||||||
client: Optional[str] = None,
|
client: Optional[str] = None,
|
||||||
|
@ -290,6 +257,31 @@ async def delete_content_file(
|
||||||
return {"status": "ok"}
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
class DeleteFilesRequest(BaseModel):
|
||||||
|
files: List[str]
|
||||||
|
|
||||||
|
|
||||||
|
@api_content.delete("/files", status_code=201)
|
||||||
|
@requires(["authenticated"])
|
||||||
|
async def delete_content_file(
|
||||||
|
request: Request,
|
||||||
|
files: DeleteFilesRequest,
|
||||||
|
client: Optional[str] = None,
|
||||||
|
):
|
||||||
|
user = request.user.object
|
||||||
|
|
||||||
|
update_telemetry_state(
|
||||||
|
request=request,
|
||||||
|
telemetry_type="api",
|
||||||
|
api="delete_file",
|
||||||
|
client=client,
|
||||||
|
)
|
||||||
|
|
||||||
|
deleted_count = await EntryAdapters.adelete_entries_by_filenames(user, files.files)
|
||||||
|
|
||||||
|
return {"status": "ok", "deleted_count": deleted_count}
|
||||||
|
|
||||||
|
|
||||||
@api_content.get("/size", response_model=Dict[str, int])
|
@api_content.get("/size", response_model=Dict[str, int])
|
||||||
@requires(["authenticated"])
|
@requires(["authenticated"])
|
||||||
async def get_content_size(request: Request, common: CommonQueryParams, client: Optional[str] = None):
|
async def get_content_size(request: Request, common: CommonQueryParams, client: Optional[str] = None):
|
||||||
|
@ -336,6 +328,39 @@ async def get_content_source(
|
||||||
return await sync_to_async(list)(EntryAdapters.get_all_filenames_by_source(user, content_source)) # type: ignore[call-arg]
|
return await sync_to_async(list)(EntryAdapters.get_all_filenames_by_source(user, content_source)) # type: ignore[call-arg]
|
||||||
|
|
||||||
|
|
||||||
|
@api_content.delete("/{content_source}", status_code=200)
|
||||||
|
@requires(["authenticated"])
|
||||||
|
async def delete_content_source(
|
||||||
|
request: Request,
|
||||||
|
content_source: str,
|
||||||
|
client: Optional[str] = None,
|
||||||
|
):
|
||||||
|
user = request.user.object
|
||||||
|
|
||||||
|
content_object = map_config_to_object(content_source)
|
||||||
|
if content_object is None:
|
||||||
|
raise ValueError(f"Invalid content source: {content_source}")
|
||||||
|
elif content_object != "Computer":
|
||||||
|
await content_object.objects.filter(user=user).adelete()
|
||||||
|
await sync_to_async(EntryAdapters.delete_all_entries)(user, file_source=content_source)
|
||||||
|
|
||||||
|
if content_source == DbEntry.EntrySource.NOTION:
|
||||||
|
await NotionConfig.objects.filter(user=user).adelete()
|
||||||
|
elif content_source == DbEntry.EntrySource.GITHUB:
|
||||||
|
await GithubConfig.objects.filter(user=user).adelete()
|
||||||
|
|
||||||
|
update_telemetry_state(
|
||||||
|
request=request,
|
||||||
|
telemetry_type="api",
|
||||||
|
api="delete_content_config",
|
||||||
|
client=client,
|
||||||
|
metadata={"content_source": content_source},
|
||||||
|
)
|
||||||
|
|
||||||
|
enabled_content = await sync_to_async(EntryAdapters.get_unique_file_types)(user)
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
async def indexer(
|
async def indexer(
|
||||||
request: Request,
|
request: Request,
|
||||||
files: list[UploadFile],
|
files: list[UploadFile],
|
||||||
|
|
Loading…
Reference in a new issue