sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-11-23 15:38:55 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Move API endpoints under /api/configure/content/ to /api/content/

Browse source 
Pull out /api/configure/content API endpoints into /api/content to
allow for more logical organization of API path hierarchy

This should make the url more succinct and API request intent more
understandable by using existing HTTP method semantics along with the
path.

The /configure URL path segment was either
- redundant (e.g POST /configure/notion) or
- incorrect (e.g GET /configure/files)

Some example of naming improvements:
- GET /configure/types -> GET /content/types
- GET /configure/files -> GET /content/files
- DELETE /configure/files -> DELETE /content/files

This should also align, merge better the the content indexing API
triggered via PUT, PATCH /content

Refactor Flow
1. Rename /api/configure/types -> /api/content/types
2. Rename /api/configure -> /api
3. Move /api/content to api_content from under api_config
This commit is contained in:
Debanjum Singh Solanky 2024-07-19 00:00:49 +05:30
parent bba4e0b529
commit 469a1cb6a2
11 changed files with 331 additions and 340 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/interface/desktop/search.html
View file

@ -212,7 +212,7 @@
const headers = { 'Authorization': `Bearer ${khojToken}` };
// Populate type dropdown field with enabled content types only
fetch(`${hostURL}/api/configure/types`, { headers })
fetch(`${hostURL}/api/content/types`, { headers })
.then(response => response.json())
.then(enabled_types => {
// Show warning if no content types are enabled

2
src/interface/emacs/khoj.el
View file

@ -697,7 +697,7 @@ Optionally apply CALLBACK with JSON parsed response and CBARGS."
(defun khoj--get-enabled-content-types ()
"Get content types enabled for search from API."
(khoj--call-api "/api/configure/types" "GET" nil `(lambda (item) (mapcar #'intern item))))
(khoj--call-api "/api/content/types" "GET" nil `(lambda (item) (mapcar #'intern item))))
(defun khoj--query-search-api-and-render-results (query content-type buffer-name &optional rerank is-find-similar)
"Query Khoj Search API with QUERY, CONTENT-TYPE and RERANK as query params.

2
src/khoj/interface/web/chat.html
View file

@ -1954,7 +1954,7 @@ To get started, just start typing below. You can also type / to see a list of co
}
var allFiles;
function renderAllFiles() {
fetch('/api/configure/content/computer')
fetch('/api/content/computer')
.then(response => response.json())
.then(data => {
var indexedFiles = document.getElementsByClassName("indexed-files")[0];

6
src/khoj/interface/web/content_source_computer_input.html
View file

@ -32,7 +32,7 @@
</style>
<script>
function removeFile(path) {
fetch('/api/configure/content/file?filename=' + path, {
fetch('/api/content/file?filename=' + path, {
method: 'DELETE',
headers: {
'Content-Type': 'application/json',
@ -48,7 +48,7 @@
// Get all currently indexed files
function getAllComputerFilenames() {
fetch('/api/configure/content/computer')
fetch('/api/content/computer')
.then(response => response.json())
.then(data => {
var indexedFiles = document.getElementsByClassName("indexed-files")[0];
@ -122,7 +122,7 @@
deleteAllComputerFilesButton.textContent = "🗑️ Deleting...";
deleteAllComputerFilesButton.disabled = true;
fetch('/api/configure/content/computer', {
fetch('/api/content/computer', {
method: 'DELETE',
headers: {
'Content-Type': 'application/json',

2
src/khoj/interface/web/content_source_github_input.html
View file

@ -165,7 +165,7 @@
// Save Github config on server
const csrfToken = document.cookie.split('; ').find(row => row.startsWith('csrftoken'))?.split('=')[1];
fetch('/api/configure/content/github', {
fetch('/api/content/github', {
method: 'POST',
headers: {
'Content-Type': 'application/json',

2
src/khoj/interface/web/content_source_notion_input.html
View file

@ -45,7 +45,7 @@
// Save Notion config on server
const csrfToken = document.cookie.split('; ').find(row => row.startsWith('csrftoken'))?.split('=')[1];
fetch('/api/configure/content/notion', {
fetch('/api/content/notion', {
method: 'POST',
headers: {
'Content-Type': 'application/json',

2
src/khoj/interface/web/search.html
View file

@ -209,7 +209,7 @@
function populate_type_dropdown() {
// Populate type dropdown field with enabled content types only
fetch("/api/configure/types")
fetch("/api/content/types")
.then(response => response.json())
.then(enabled_types => {
// Show warning if no content types are enabled, or just one ("all")

8
src/khoj/interface/web/settings.html
View file

@ -553,7 +553,7 @@
};
function clearContentType(content_source) {
fetch('/api/configure/content/' + content_source, {
fetch('/api/content/' + content_source, {
method: 'DELETE',
headers: {
'Content-Type': 'application/json',
@ -676,7 +676,7 @@
content_sources = ["computer", "github", "notion"];
content_sources.forEach(content_source => {
fetch(`/api/configure/content/${content_source}`, {
fetch(`/api/content/${content_source}`, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
@ -807,7 +807,7 @@
function getIndexedDataSize() {
document.getElementById("indexed-data-size").textContent = "Calculating...";
fetch('/api/configure/content/size')
fetch('/api/content/size')
.then(response => response.json())
.then(data => {
document.getElementById("indexed-data-size").textContent = data.indexed_data_size_in_mb + " MB used";
@ -815,7 +815,7 @@
}
function removeFile(path) {
fetch('/api/configure/content/file?filename=' + path, {
fetch('/api/content/file?filename=' + path, {
method: 'DELETE',
headers: {
'Content-Type': 'application/json',

324
src/khoj/routers/api_config.py
View file

@ -1,306 +1,20 @@
import json
import logging
import math
from typing import Dict, List, Optional, Union
from typing import Dict, Optional, Union
from asgiref.sync import sync_to_async
from fastapi import APIRouter, HTTPException, Request
from fastapi.requests import Request
from fastapi.responses import Response
from starlette.authentication import has_required_scope, requires
from khoj.database import adapters
from khoj.database.adapters import (
ConversationAdapters,
EntryAdapters,
get_user_github_config,
get_user_notion_config,
)
from khoj.database.models import Entry as DbEntry
from khoj.database.models import (
GithubConfig,
GithubRepoConfig,
KhojUser,
LocalMarkdownConfig,
LocalOrgConfig,
LocalPdfConfig,
LocalPlaintextConfig,
NotionConfig,
)
from khoj.routers.helpers import (
CommonQueryParams,
get_user_config,
update_telemetry_state,
)
from khoj.utils import constants, state
from khoj.utils.rawconfig import (
FullConfig,
GithubContentConfig,
NotionContentConfig,
SearchConfig,
)
from khoj.utils.state import SearchType
from khoj.database.adapters import ConversationAdapters, EntryAdapters
from khoj.routers.helpers import update_telemetry_state
api_config = APIRouter()
logger = logging.getLogger(__name__)
def map_config_to_object(content_source: str):
if content_source == DbEntry.EntrySource.GITHUB:
return GithubConfig
if content_source == DbEntry.EntrySource.NOTION:
return NotionConfig
if content_source == DbEntry.EntrySource.COMPUTER:
return "Computer"
async def map_config_to_db(config: FullConfig, user: KhojUser):
if config.content_type:
if config.content_type.org:
await LocalOrgConfig.objects.filter(user=user).adelete()
await LocalOrgConfig.objects.acreate(
input_files=config.content_type.org.input_files,
input_filter=config.content_type.org.input_filter,
index_heading_entries=config.content_type.org.index_heading_entries,
user=user,
)
if config.content_type.markdown:
await LocalMarkdownConfig.objects.filter(user=user).adelete()
await LocalMarkdownConfig.objects.acreate(
input_files=config.content_type.markdown.input_files,
input_filter=config.content_type.markdown.input_filter,
index_heading_entries=config.content_type.markdown.index_heading_entries,
user=user,
)
if config.content_type.pdf:
await LocalPdfConfig.objects.filter(user=user).adelete()
await LocalPdfConfig.objects.acreate(
input_files=config.content_type.pdf.input_files,
input_filter=config.content_type.pdf.input_filter,
index_heading_entries=config.content_type.pdf.index_heading_entries,
user=user,
)
if config.content_type.plaintext:
await LocalPlaintextConfig.objects.filter(user=user).adelete()
await LocalPlaintextConfig.objects.acreate(
input_files=config.content_type.plaintext.input_files,
input_filter=config.content_type.plaintext.input_filter,
index_heading_entries=config.content_type.plaintext.index_heading_entries,
user=user,
)
if config.content_type.github:
await adapters.set_user_github_config(
user=user,
pat_token=config.content_type.github.pat_token,
repos=config.content_type.github.repos,
)
if config.content_type.notion:
await adapters.set_notion_config(
user=user,
token=config.content_type.notion.token,
)
def _initialize_config():
if state.config is None:
state.config = FullConfig()
state.config.search_type = SearchConfig.model_validate(constants.default_config["search-type"])
@api_config.get("", response_class=Response)
@requires(["authenticated"])
def get_config(request: Request, detailed: Optional[bool] = False) -> Response:
user = request.user.object
user_config = get_user_config(user, request, is_detailed=detailed)
del user_config["request"]
# Return config data as a JSON response
return Response(content=json.dumps(user_config), media_type="application/json", status_code=200)
@api_config.get("/content/github", response_class=Response)
@requires(["authenticated"])
def get_content_github(request: Request) -> Response:
user = request.user.object
user_config = get_user_config(user, request)
del user_config["request"]
current_github_config = get_user_github_config(user)
if current_github_config:
raw_repos = current_github_config.githubrepoconfig.all()
repos = []
for repo in raw_repos:
repos.append(
GithubRepoConfig(
name=repo.name,
owner=repo.owner,
branch=repo.branch,
)
)
current_config = GithubContentConfig(
pat_token=current_github_config.pat_token,
repos=repos,
)
current_config = json.loads(current_config.json())
else:
current_config = {} # type: ignore
user_config["current_config"] = current_config
# Return config data as a JSON response
return Response(content=json.dumps(user_config), media_type="application/json", status_code=200)
@api_config.get("/content/notion", response_class=Response)
@requires(["authenticated"])
def get_content_notion(request: Request) -> Response:
user = request.user.object
user_config = get_user_config(user, request)
del user_config["request"]
current_notion_config = get_user_notion_config(user)
token = current_notion_config.token if current_notion_config else ""
current_config = NotionContentConfig(token=token)
current_config = json.loads(current_config.model_dump_json())
user_config["current_config"] = current_config
# Return config data as a JSON response
return Response(content=json.dumps(user_config), media_type="application/json", status_code=200)
@api_config.post("/content/github", status_code=200)
@requires(["authenticated"])
async def set_content_github(
request: Request,
updated_config: Union[GithubContentConfig, None],
client: Optional[str] = None,
):
_initialize_config()
user = request.user.object
try:
await adapters.set_user_github_config(
user=user,
pat_token=updated_config.pat_token,
repos=updated_config.repos,
)
except Exception as e:
logger.error(e, exc_info=True)
raise HTTPException(status_code=500, detail="Failed to set Github config")
update_telemetry_state(
request=request,
telemetry_type="api",
api="set_content_config",
client=client,
metadata={"content_type": "github"},
)
return {"status": "ok"}
@api_config.post("/content/notion", status_code=200)
@requires(["authenticated"])
async def set_content_notion(
request: Request,
updated_config: Union[NotionContentConfig, None],
client: Optional[str] = None,
):
_initialize_config()
user = request.user.object
try:
await adapters.set_notion_config(
user=user,
token=updated_config.token,
)
except Exception as e:
logger.error(e, exc_info=True)
raise HTTPException(status_code=500, detail="Failed to set Github config")
update_telemetry_state(
request=request,
telemetry_type="api",
api="set_content_config",
client=client,
metadata={"content_type": "notion"},
)
return {"status": "ok"}
@api_config.delete("/content/{content_source}", status_code=200)
@requires(["authenticated"])
async def delete_content_source(
request: Request,
content_source: str,
client: Optional[str] = None,
):
user = request.user.object
update_telemetry_state(
request=request,
telemetry_type="api",
api="delete_content_config",
client=client,
metadata={"content_source": content_source},
)
content_object = map_config_to_object(content_source)
if content_object is None:
raise ValueError(f"Invalid content source: {content_source}")
elif content_object != "Computer":
await content_object.objects.filter(user=user).adelete()
await sync_to_async(EntryAdapters.delete_all_entries)(user, file_source=content_source)
enabled_content = await sync_to_async(EntryAdapters.get_unique_file_types)(user)
return {"status": "ok"}
@api_config.delete("/content/file", status_code=201)
@requires(["authenticated"])
async def delete_content_file(
request: Request,
filename: str,
client: Optional[str] = None,
):
user = request.user.object
update_telemetry_state(
request=request,
telemetry_type="api",
api="delete_file",
client=client,
)
await EntryAdapters.adelete_entry_by_file(user, filename)
return {"status": "ok"}
@api_config.get("/content/{content_source}", response_model=List[str])
@requires(["authenticated"])
async def get_content_source(
request: Request,
content_source: str,
client: Optional[str] = None,
):
user = request.user.object
update_telemetry_state(
request=request,
telemetry_type="api",
api="get_all_filenames",
client=client,
)
return await sync_to_async(list)(EntryAdapters.get_all_filenames_by_source(user, content_source)) # type: ignore[call-arg]
@api_config.get("/chat/model/options", response_model=Dict[str, Union[str, int]])
def get_chat_model_options(
request: Request,
@ -442,18 +156,6 @@ async def update_paint_model(
return {"status": "ok"}
@api_config.get("/content/size", response_model=Dict[str, int])
@requires(["authenticated"])
async def get_content_size(request: Request, common: CommonQueryParams):
user = request.user.object
indexed_data_size_in_mb = await sync_to_async(EntryAdapters.get_size_of_indexed_data_in_mb)(user)
return Response(
content=json.dumps({"indexed_data_size_in_mb": math.ceil(indexed_data_size_in_mb)}),
media_type="application/json",
status_code=200,
)
@api_config.post("/user/name", status_code=200)
@requires(["authenticated"])
def set_user_name(
@ -484,23 +186,3 @@ def set_user_name(
)
return {"status": "ok"}
@api_config.get("/types", response_model=List[str])
@requires(["authenticated"])
def get_config_types(
request: Request,
):
user = request.user.object
enabled_file_types = EntryAdapters.get_unique_file_types(user)
configured_content_types = list(enabled_file_types)
if state.config and state.config.content_type:
for ctype in state.config.content_type.model_dump(exclude_none=True):
configured_content_types.append(ctype)
return [
search_type.value
for search_type in SearchType
if (search_type.value in configured_content_types) or search_type == SearchType.All
]

315
src/khoj/routers/api_content.py
View file

@ -1,20 +1,57 @@
import asyncio
import json
import logging
from typing import Dict, Optional, Union
import math
from typing import Dict, List, Optional, Union
from fastapi import APIRouter, Depends, Header, Request, Response, UploadFile
from asgiref.sync import sync_to_async
from fastapi import (
APIRouter,
Depends,
Header,
HTTPException,
Request,
Response,
UploadFile,
)
from pydantic import BaseModel
from starlette.authentication import requires
from khoj.database import adapters
from khoj.database.adapters import (
EntryAdapters,
get_user_github_config,
get_user_notion_config,
)
from khoj.database.models import Entry as DbEntry
from khoj.database.models import (
GithubConfig,
GithubRepoConfig,
KhojUser,
LocalMarkdownConfig,
LocalOrgConfig,
LocalPdfConfig,
LocalPlaintextConfig,
NotionConfig,
)
from khoj.routers.helpers import (
ApiIndexedDataLimiter,
CommonQueryParams,
configure_content,
get_user_config,
update_telemetry_state,
)
from khoj.utils import constants, state
from khoj.utils.config import SearchModels
from khoj.utils.helpers import get_file_type
from khoj.utils.rawconfig import ContentConfig, FullConfig, SearchConfig
from khoj.utils.rawconfig import (
ContentConfig,
FullConfig,
GithubContentConfig,
NotionContentConfig,
SearchConfig,
)
from khoj.utils.state import SearchType
from khoj.utils.yaml import save_config_to_file_updated_state
logger = logging.getLogger(__name__)
@ -84,6 +121,216 @@ async def patch_content(
return await indexer(request, files, t, False, client, user_agent, referer, host)
@api_content.get("/github", response_class=Response)
@requires(["authenticated"])
def get_content_github(request: Request) -> Response:
user = request.user.object
user_config = get_user_config(user, request)
del user_config["request"]
current_github_config = get_user_github_config(user)
if current_github_config:
raw_repos = current_github_config.githubrepoconfig.all()
repos = []
for repo in raw_repos:
repos.append(
GithubRepoConfig(
name=repo.name,
owner=repo.owner,
branch=repo.branch,
)
)
current_config = GithubContentConfig(
pat_token=current_github_config.pat_token,
repos=repos,
)
current_config = json.loads(current_config.json())
else:
current_config = {} # type: ignore
user_config["current_config"] = current_config
# Return config data as a JSON response
return Response(content=json.dumps(user_config), media_type="application/json", status_code=200)
@api_content.get("/notion", response_class=Response)
@requires(["authenticated"])
def get_content_notion(request: Request) -> Response:
user = request.user.object
user_config = get_user_config(user, request)
del user_config["request"]
current_notion_config = get_user_notion_config(user)
token = current_notion_config.token if current_notion_config else ""
current_config = NotionContentConfig(token=token)
current_config = json.loads(current_config.model_dump_json())
user_config["current_config"] = current_config
# Return config data as a JSON response
return Response(content=json.dumps(user_config), media_type="application/json", status_code=200)
@api_content.post("/github", status_code=200)
@requires(["authenticated"])
async def set_content_github(
request: Request,
updated_config: Union[GithubContentConfig, None],
client: Optional[str] = None,
):
_initialize_config()
user = request.user.object
try:
await adapters.set_user_github_config(
user=user,
pat_token=updated_config.pat_token,
repos=updated_config.repos,
)
except Exception as e:
logger.error(e, exc_info=True)
raise HTTPException(status_code=500, detail="Failed to set Github config")
update_telemetry_state(
request=request,
telemetry_type="api",
api="set_content_config",
client=client,
metadata={"content_type": "github"},
)
return {"status": "ok"}
@api_content.post("/notion", status_code=200)
@requires(["authenticated"])
async def set_content_notion(
request: Request,
updated_config: Union[NotionContentConfig, None],
client: Optional[str] = None,
):
_initialize_config()
user = request.user.object
try:
await adapters.set_notion_config(
user=user,
token=updated_config.token,
)
except Exception as e:
logger.error(e, exc_info=True)
raise HTTPException(status_code=500, detail="Failed to set Notion config")
update_telemetry_state(
request=request,
telemetry_type="api",
api="set_content_config",
client=client,
metadata={"content_type": "notion"},
)
return {"status": "ok"}
@api_content.delete("/{content_source}", status_code=200)
@requires(["authenticated"])
async def delete_content_source(
request: Request,
content_source: str,
client: Optional[str] = None,
):
user = request.user.object
update_telemetry_state(
request=request,
telemetry_type="api",
api="delete_content_config",
client=client,
metadata={"content_source": content_source},
)
content_object = map_config_to_object(content_source)
if content_object is None:
raise ValueError(f"Invalid content source: {content_source}")
elif content_object != "Computer":
await content_object.objects.filter(user=user).adelete()
await sync_to_async(EntryAdapters.delete_all_entries)(user, file_source=content_source)
enabled_content = await sync_to_async(EntryAdapters.get_unique_file_types)(user)
return {"status": "ok"}
@api_content.delete("/file", status_code=201)
@requires(["authenticated"])
async def delete_content_file(
request: Request,
filename: str,
client: Optional[str] = None,
):
user = request.user.object
update_telemetry_state(
request=request,
telemetry_type="api",
api="delete_file",
client=client,
)
await EntryAdapters.adelete_entry_by_file(user, filename)
return {"status": "ok"}
@api_content.get("/size", response_model=Dict[str, int])
@requires(["authenticated"])
async def get_content_size(request: Request, common: CommonQueryParams, client: Optional[str] = None):
user = request.user.object
indexed_data_size_in_mb = await sync_to_async(EntryAdapters.get_size_of_indexed_data_in_mb)(user)
return Response(
content=json.dumps({"indexed_data_size_in_mb": math.ceil(indexed_data_size_in_mb)}),
media_type="application/json",
status_code=200,
)
@api_content.get("/types", response_model=List[str])
@requires(["authenticated"])
def get_content_types(request: Request, client: Optional[str] = None):
user = request.user.object
all_content_types = {s.value for s in SearchType}
configured_content_types = set(EntryAdapters.get_unique_file_types(user))
configured_content_types |= {"all"}
if state.config and state.config.content_type:
for ctype in state.config.content_type.model_dump(exclude_none=True):
configured_content_types.add(ctype)
return list(configured_content_types & all_content_types)
@api_content.get("/{content_source}", response_model=List[str])
@requires(["authenticated"])
async def get_content_source(
request: Request,
content_source: str,
client: Optional[str] = None,
):
user = request.user.object
update_telemetry_state(
request=request,
telemetry_type="api",
api="get_all_filenames",
client=client,
)
return await sync_to_async(list)(EntryAdapters.get_all_filenames_by_source(user, content_source)) # type: ignore[call-arg]
async def indexer(
request: Request,
files: list[UploadFile],
@ -198,3 +445,65 @@ def configure_search(search_models: SearchModels, search_config: Optional[Search
search_models = SearchModels()
return search_models
def map_config_to_object(content_source: str):
if content_source == DbEntry.EntrySource.GITHUB:
return GithubConfig
if content_source == DbEntry.EntrySource.NOTION:
return NotionConfig
if content_source == DbEntry.EntrySource.COMPUTER:
return "Computer"
async def map_config_to_db(config: FullConfig, user: KhojUser):
if config.content_type:
if config.content_type.org:
await LocalOrgConfig.objects.filter(user=user).adelete()
await LocalOrgConfig.objects.acreate(
input_files=config.content_type.org.input_files,
input_filter=config.content_type.org.input_filter,
index_heading_entries=config.content_type.org.index_heading_entries,
user=user,
)
if config.content_type.markdown:
await LocalMarkdownConfig.objects.filter(user=user).adelete()
await LocalMarkdownConfig.objects.acreate(
input_files=config.content_type.markdown.input_files,
input_filter=config.content_type.markdown.input_filter,
index_heading_entries=config.content_type.markdown.index_heading_entries,
user=user,
)
if config.content_type.pdf:
await LocalPdfConfig.objects.filter(user=user).adelete()
await LocalPdfConfig.objects.acreate(
input_files=config.content_type.pdf.input_files,
input_filter=config.content_type.pdf.input_filter,
index_heading_entries=config.content_type.pdf.index_heading_entries,
user=user,
)
if config.content_type.plaintext:
await LocalPlaintextConfig.objects.filter(user=user).adelete()
await LocalPlaintextConfig.objects.acreate(
input_files=config.content_type.plaintext.input_files,
input_filter=config.content_type.plaintext.input_filter,
index_heading_entries=config.content_type.plaintext.index_heading_entries,
user=user,
)
if config.content_type.github:
await adapters.set_user_github_config(
user=user,
pat_token=config.content_type.github.pat_token,
repos=config.content_type.github.repos,
)
if config.content_type.notion:
await adapters.set_notion_config(
user=user,
token=config.content_type.notion.token,
)
def _initialize_config():
if state.config is None:
state.config = FullConfig()
state.config.search_type = SearchConfig.model_validate(constants.default_config["search-type"])

6
tests/test_client.py
View file

@ -269,11 +269,11 @@ def test_get_api_config_types(client, sample_org_data, default_user: KhojUser):
text_search.setup(OrgToEntries, sample_org_data, regenerate=False, user=default_user)
# Act
response = client.get(f"/api/configure/types", headers=headers)
response = client.get(f"/api/content/types", headers=headers)
# Assert
assert response.status_code == 200
assert response.json() == ["all", "org", "plaintext"]
assert set(response.json()) == {"all", "org", "plaintext"}
# ----------------------------------------------------------------------------------------------------
@ -289,7 +289,7 @@ def test_get_configured_types_with_no_content_config(fastapi_app: FastAPI):
client = TestClient(fastapi_app)
# Act
response = client.get(f"/api/configure/types")
response = client.get(f"/api/content/types")
# Assert
assert response.status_code == 200