From d527b644f4a36f607f46b7ef56c68ffc2a3903db Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Mon, 6 Nov 2023 23:51:43 -0800 Subject: [PATCH] Update content by source via API. Make web client use this API for config --- docs/github_integration.md | 2 +- docs/notion_integration.md | 2 +- src/khoj/interface/web/config.html | 15 +++++----- ....html => content_source_github_input.html} | 2 +- ....html => content_source_notion_input.html} | 2 +- src/khoj/routers/api.py | 30 ++++++++++--------- src/khoj/routers/web_client.py | 8 ++--- 7 files changed, 32 insertions(+), 29 deletions(-) rename src/khoj/interface/web/{content_type_github_input.html => content_source_github_input.html} (99%) rename src/khoj/interface/web/{content_type_notion_input.html => content_source_notion_input.html} (97%) diff --git a/docs/github_integration.md b/docs/github_integration.md index 6b8dce48..413dd41e 100644 --- a/docs/github_integration.md +++ b/docs/github_integration.md @@ -9,6 +9,6 @@ The Github integration allows you to index as many repositories as you want. It' ## Use the Github plugin 1. Generate a [classic PAT (personal access token)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) from [Github](https://github.com/settings/tokens) with `repo` and `admin:org` scopes at least. -2. Navigate to [http://localhost:42110/config/content_type/github](http://localhost:42110/config/content_type/github) to configure your Github settings. Enter in your PAT, along with details for each repository you want to index. +2. Navigate to [http://localhost:42110/config/content-source/github](http://localhost:42110/config/content-source/github) to configure your Github settings. Enter in your PAT, along with details for each repository you want to index. 3. Click `Save`. Go back to the settings page and click `Configure`. 4. Go to [http://localhost:42110/](http://localhost:42110/) and start searching! diff --git a/docs/notion_integration.md b/docs/notion_integration.md index 5fee7ff6..d3b645ca 100644 --- a/docs/notion_integration.md +++ b/docs/notion_integration.md @@ -8,7 +8,7 @@ We haven't setup a fancy integration with OAuth yet, so this integration still r ![setup_new_integration](https://github.com/khoj-ai/khoj/assets/65192171/b056e057-d4dc-47dc-aad3-57b59a22c68b) 3. Share all the workspaces that you want to integrate with the Khoj integration you just made in the previous step ![enable_workspace](https://github.com/khoj-ai/khoj/assets/65192171/98290303-b5b8-4cb0-b32c-f68c6923a3d0) -4. In the first step, you generated an API key. Use the newly generated API Key in your Khoj settings, by default at http://localhost:42110/config/content_type/notion. Click `Save`. +4. In the first step, you generated an API key. Use the newly generated API Key in your Khoj settings, by default at http://localhost:42110/config/content-source/notion. Click `Save`. 5. Click `Configure` in http://localhost:42110/config to index your Notion workspace(s). That's it! You should be ready to start searching and chatting. Make sure you've configured your OpenAI API Key for chat. diff --git a/src/khoj/interface/web/config.html b/src/khoj/interface/web/config.html index b19bbff6..4e77a8ef 100644 --- a/src/khoj/interface/web/config.html +++ b/src/khoj/interface/web/config.html @@ -19,7 +19,7 @@

Set repositories to index

- + {% if current_model_state.github %} Update {% else %} @@ -50,7 +50,7 @@

Configure your settings from Notion

- + {% if current_model_state.content %} Update {% else %} @@ -176,8 +176,9 @@ }) }; - function clearContentType(content_type) { - fetch('/api/config/data/content_type/' + content_type, { + function clearContentType(content_source) { + + fetch('/api/config/data/content-source/' + content_source, { method: 'DELETE', headers: { 'Content-Type': 'application/json', @@ -186,15 +187,15 @@ .then(response => response.json()) .then(data => { if (data.status == "ok") { - var contentTypeClearButton = document.getElementById("clear-" + content_type); + var contentTypeClearButton = document.getElementById("clear-" + content_source); contentTypeClearButton.style.display = "none"; - var configuredIcon = document.getElementById("configured-icon-" + content_type); + var configuredIcon = document.getElementById("configured-icon-" + content_source); if (configuredIcon) { configuredIcon.style.display = "none"; } - var misconfiguredIcon = document.getElementById("misconfigured-icon-" + content_type); + var misconfiguredIcon = document.getElementById("misconfigured-icon-" + content_source); if (misconfiguredIcon) { misconfiguredIcon.style.display = "none"; } diff --git a/src/khoj/interface/web/content_type_github_input.html b/src/khoj/interface/web/content_source_github_input.html similarity index 99% rename from src/khoj/interface/web/content_type_github_input.html rename to src/khoj/interface/web/content_source_github_input.html index 0e41645a..ff82b1f2 100644 --- a/src/khoj/interface/web/content_type_github_input.html +++ b/src/khoj/interface/web/content_source_github_input.html @@ -125,7 +125,7 @@ } const csrfToken = document.cookie.split('; ').find(row => row.startsWith('csrftoken'))?.split('=')[1]; - fetch('/api/config/data/content_type/github', { + fetch('/api/config/data/content-source/github', { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/src/khoj/interface/web/content_type_notion_input.html b/src/khoj/interface/web/content_source_notion_input.html similarity index 97% rename from src/khoj/interface/web/content_type_notion_input.html rename to src/khoj/interface/web/content_source_notion_input.html index 965c1ef5..18eb5a7f 100644 --- a/src/khoj/interface/web/content_type_notion_input.html +++ b/src/khoj/interface/web/content_source_notion_input.html @@ -42,7 +42,7 @@ } const csrfToken = document.cookie.split('; ').find(row => row.startsWith('csrftoken'))?.split('=')[1]; - fetch('/api/config/data/content_type/notion', { + fetch('/api/config/data/content-source/notion', { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index 84e63b09..c2002048 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -61,11 +61,13 @@ api = APIRouter() logger = logging.getLogger(__name__) -def map_config_to_object(content_type: str): - if content_type == "github": +def map_config_to_object(content_source: str): + if content_source == "github": return GithubConfig - if content_type == "notion": + if content_source == "notion": return NotionConfig + if content_source == "computer": + return "Computer" async def map_config_to_db(config: FullConfig, user: KhojUser): @@ -164,7 +166,7 @@ async def set_config_data( return state.config -@api.post("/config/data/content_type/github", status_code=200) +@api.post("/config/data/content-source/github", status_code=200) @requires(["authenticated"]) async def set_content_config_github_data( request: Request, @@ -192,7 +194,7 @@ async def set_content_config_github_data( return {"status": "ok"} -@api.post("/config/data/content_type/notion", status_code=200) +@api.post("/config/data/content-source/notion", status_code=200) @requires(["authenticated"]) async def set_content_config_notion_data( request: Request, @@ -219,11 +221,11 @@ async def set_content_config_notion_data( return {"status": "ok"} -@api.delete("/config/data/content_type/{content_type}", status_code=200) +@api.delete("/config/data/content-source/{content_source}", status_code=200) @requires(["authenticated"]) -async def remove_content_config_data( +async def remove_content_source_data( request: Request, - content_type: str, + content_source: str, client: Optional[str] = None, ): user = request.user.object @@ -233,15 +235,15 @@ async def remove_content_config_data( telemetry_type="api", api="delete_content_config", client=client, - metadata={"content_type": content_type}, + metadata={"content_source": content_source}, ) - content_object = map_config_to_object(content_type) + content_object = map_config_to_object(content_source) if content_object is None: - raise ValueError(f"Invalid content type: {content_type}") - - await content_object.objects.filter(user=user).adelete() - await sync_to_async(EntryAdapters.delete_all_entries)(user, content_type) + raise ValueError(f"Invalid content source: {content_source}") + elif content_object != "Computer": + await content_object.objects.filter(user=user).adelete() + await sync_to_async(EntryAdapters.delete_all_entries_by_source)(user, content_source) enabled_content = await sync_to_async(EntryAdapters.get_unique_file_types)(user) return {"status": "ok"} diff --git a/src/khoj/routers/web_client.py b/src/khoj/routers/web_client.py index 65292ccf..8016cfce 100644 --- a/src/khoj/routers/web_client.py +++ b/src/khoj/routers/web_client.py @@ -150,7 +150,7 @@ def config_page(request: Request): ) -@web_client.get("/config/content_type/github", response_class=HTMLResponse) +@web_client.get("/config/content-source/github", response_class=HTMLResponse) @requires(["authenticated"], redirect="login_page") def github_config_page(request: Request): user = request.user.object @@ -177,7 +177,7 @@ def github_config_page(request: Request): current_config = {} # type: ignore return templates.TemplateResponse( - "content_type_github_input.html", + "content_source_github_input.html", context={ "request": request, "current_config": current_config, @@ -187,7 +187,7 @@ def github_config_page(request: Request): ) -@web_client.get("/config/content_type/notion", response_class=HTMLResponse) +@web_client.get("/config/content-source/notion", response_class=HTMLResponse) @requires(["authenticated"], redirect="login_page") def notion_config_page(request: Request): user = request.user.object @@ -201,7 +201,7 @@ def notion_config_page(request: Request): current_config = json.loads(current_config.json()) return templates.TemplateResponse( - "content_type_notion_input.html", + "content_source_notion_input.html", context={ "request": request, "current_config": current_config,