mirror of
https://github.com/khoj-ai/khoj.git
synced 2024-11-27 17:35:07 +01:00
Move API endpoints under /api/configure/content/ to /api/content/
Pull out /api/configure/content API endpoints into /api/content to allow for more logical organization of API path hierarchy This should make the url more succinct and API request intent more understandable by using existing HTTP method semantics along with the path. The /configure URL path segment was either - redundant (e.g POST /configure/notion) or - incorrect (e.g GET /configure/files) Some example of naming improvements: - GET /configure/types -> GET /content/types - GET /configure/files -> GET /content/files - DELETE /configure/files -> DELETE /content/files This should also align, merge better the the content indexing API triggered via PUT, PATCH /content Refactor Flow 1. Rename /api/configure/types -> /api/content/types 2. Rename /api/configure -> /api 3. Move /api/content to api_content from under api_config
This commit is contained in:
parent
bba4e0b529
commit
469a1cb6a2
11 changed files with 331 additions and 340 deletions
|
@ -212,7 +212,7 @@
|
||||||
const headers = { 'Authorization': `Bearer ${khojToken}` };
|
const headers = { 'Authorization': `Bearer ${khojToken}` };
|
||||||
|
|
||||||
// Populate type dropdown field with enabled content types only
|
// Populate type dropdown field with enabled content types only
|
||||||
fetch(`${hostURL}/api/configure/types`, { headers })
|
fetch(`${hostURL}/api/content/types`, { headers })
|
||||||
.then(response => response.json())
|
.then(response => response.json())
|
||||||
.then(enabled_types => {
|
.then(enabled_types => {
|
||||||
// Show warning if no content types are enabled
|
// Show warning if no content types are enabled
|
||||||
|
|
|
@ -697,7 +697,7 @@ Optionally apply CALLBACK with JSON parsed response and CBARGS."
|
||||||
|
|
||||||
(defun khoj--get-enabled-content-types ()
|
(defun khoj--get-enabled-content-types ()
|
||||||
"Get content types enabled for search from API."
|
"Get content types enabled for search from API."
|
||||||
(khoj--call-api "/api/configure/types" "GET" nil `(lambda (item) (mapcar #'intern item))))
|
(khoj--call-api "/api/content/types" "GET" nil `(lambda (item) (mapcar #'intern item))))
|
||||||
|
|
||||||
(defun khoj--query-search-api-and-render-results (query content-type buffer-name &optional rerank is-find-similar)
|
(defun khoj--query-search-api-and-render-results (query content-type buffer-name &optional rerank is-find-similar)
|
||||||
"Query Khoj Search API with QUERY, CONTENT-TYPE and RERANK as query params.
|
"Query Khoj Search API with QUERY, CONTENT-TYPE and RERANK as query params.
|
||||||
|
|
|
@ -1954,7 +1954,7 @@ To get started, just start typing below. You can also type / to see a list of co
|
||||||
}
|
}
|
||||||
var allFiles;
|
var allFiles;
|
||||||
function renderAllFiles() {
|
function renderAllFiles() {
|
||||||
fetch('/api/configure/content/computer')
|
fetch('/api/content/computer')
|
||||||
.then(response => response.json())
|
.then(response => response.json())
|
||||||
.then(data => {
|
.then(data => {
|
||||||
var indexedFiles = document.getElementsByClassName("indexed-files")[0];
|
var indexedFiles = document.getElementsByClassName("indexed-files")[0];
|
||||||
|
|
|
@ -32,7 +32,7 @@
|
||||||
</style>
|
</style>
|
||||||
<script>
|
<script>
|
||||||
function removeFile(path) {
|
function removeFile(path) {
|
||||||
fetch('/api/configure/content/file?filename=' + path, {
|
fetch('/api/content/file?filename=' + path, {
|
||||||
method: 'DELETE',
|
method: 'DELETE',
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
|
@ -48,7 +48,7 @@
|
||||||
|
|
||||||
// Get all currently indexed files
|
// Get all currently indexed files
|
||||||
function getAllComputerFilenames() {
|
function getAllComputerFilenames() {
|
||||||
fetch('/api/configure/content/computer')
|
fetch('/api/content/computer')
|
||||||
.then(response => response.json())
|
.then(response => response.json())
|
||||||
.then(data => {
|
.then(data => {
|
||||||
var indexedFiles = document.getElementsByClassName("indexed-files")[0];
|
var indexedFiles = document.getElementsByClassName("indexed-files")[0];
|
||||||
|
@ -122,7 +122,7 @@
|
||||||
deleteAllComputerFilesButton.textContent = "🗑️ Deleting...";
|
deleteAllComputerFilesButton.textContent = "🗑️ Deleting...";
|
||||||
deleteAllComputerFilesButton.disabled = true;
|
deleteAllComputerFilesButton.disabled = true;
|
||||||
|
|
||||||
fetch('/api/configure/content/computer', {
|
fetch('/api/content/computer', {
|
||||||
method: 'DELETE',
|
method: 'DELETE',
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
|
|
|
@ -165,7 +165,7 @@
|
||||||
|
|
||||||
// Save Github config on server
|
// Save Github config on server
|
||||||
const csrfToken = document.cookie.split('; ').find(row => row.startsWith('csrftoken'))?.split('=')[1];
|
const csrfToken = document.cookie.split('; ').find(row => row.startsWith('csrftoken'))?.split('=')[1];
|
||||||
fetch('/api/configure/content/github', {
|
fetch('/api/content/github', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
|
|
|
@ -45,7 +45,7 @@
|
||||||
|
|
||||||
// Save Notion config on server
|
// Save Notion config on server
|
||||||
const csrfToken = document.cookie.split('; ').find(row => row.startsWith('csrftoken'))?.split('=')[1];
|
const csrfToken = document.cookie.split('; ').find(row => row.startsWith('csrftoken'))?.split('=')[1];
|
||||||
fetch('/api/configure/content/notion', {
|
fetch('/api/content/notion', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
|
|
|
@ -209,7 +209,7 @@
|
||||||
|
|
||||||
function populate_type_dropdown() {
|
function populate_type_dropdown() {
|
||||||
// Populate type dropdown field with enabled content types only
|
// Populate type dropdown field with enabled content types only
|
||||||
fetch("/api/configure/types")
|
fetch("/api/content/types")
|
||||||
.then(response => response.json())
|
.then(response => response.json())
|
||||||
.then(enabled_types => {
|
.then(enabled_types => {
|
||||||
// Show warning if no content types are enabled, or just one ("all")
|
// Show warning if no content types are enabled, or just one ("all")
|
||||||
|
|
|
@ -553,7 +553,7 @@
|
||||||
};
|
};
|
||||||
|
|
||||||
function clearContentType(content_source) {
|
function clearContentType(content_source) {
|
||||||
fetch('/api/configure/content/' + content_source, {
|
fetch('/api/content/' + content_source, {
|
||||||
method: 'DELETE',
|
method: 'DELETE',
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
|
@ -676,7 +676,7 @@
|
||||||
|
|
||||||
content_sources = ["computer", "github", "notion"];
|
content_sources = ["computer", "github", "notion"];
|
||||||
content_sources.forEach(content_source => {
|
content_sources.forEach(content_source => {
|
||||||
fetch(`/api/configure/content/${content_source}`, {
|
fetch(`/api/content/${content_source}`, {
|
||||||
method: 'GET',
|
method: 'GET',
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
|
@ -807,7 +807,7 @@
|
||||||
|
|
||||||
function getIndexedDataSize() {
|
function getIndexedDataSize() {
|
||||||
document.getElementById("indexed-data-size").textContent = "Calculating...";
|
document.getElementById("indexed-data-size").textContent = "Calculating...";
|
||||||
fetch('/api/configure/content/size')
|
fetch('/api/content/size')
|
||||||
.then(response => response.json())
|
.then(response => response.json())
|
||||||
.then(data => {
|
.then(data => {
|
||||||
document.getElementById("indexed-data-size").textContent = data.indexed_data_size_in_mb + " MB used";
|
document.getElementById("indexed-data-size").textContent = data.indexed_data_size_in_mb + " MB used";
|
||||||
|
@ -815,7 +815,7 @@
|
||||||
}
|
}
|
||||||
|
|
||||||
function removeFile(path) {
|
function removeFile(path) {
|
||||||
fetch('/api/configure/content/file?filename=' + path, {
|
fetch('/api/content/file?filename=' + path, {
|
||||||
method: 'DELETE',
|
method: 'DELETE',
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/json',
|
'Content-Type': 'application/json',
|
||||||
|
|
|
@ -1,306 +1,20 @@
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import math
|
from typing import Dict, Optional, Union
|
||||||
from typing import Dict, List, Optional, Union
|
|
||||||
|
|
||||||
from asgiref.sync import sync_to_async
|
|
||||||
from fastapi import APIRouter, HTTPException, Request
|
from fastapi import APIRouter, HTTPException, Request
|
||||||
from fastapi.requests import Request
|
from fastapi.requests import Request
|
||||||
from fastapi.responses import Response
|
from fastapi.responses import Response
|
||||||
from starlette.authentication import has_required_scope, requires
|
from starlette.authentication import has_required_scope, requires
|
||||||
|
|
||||||
from khoj.database import adapters
|
from khoj.database import adapters
|
||||||
from khoj.database.adapters import (
|
from khoj.database.adapters import ConversationAdapters, EntryAdapters
|
||||||
ConversationAdapters,
|
from khoj.routers.helpers import update_telemetry_state
|
||||||
EntryAdapters,
|
|
||||||
get_user_github_config,
|
|
||||||
get_user_notion_config,
|
|
||||||
)
|
|
||||||
from khoj.database.models import Entry as DbEntry
|
|
||||||
from khoj.database.models import (
|
|
||||||
GithubConfig,
|
|
||||||
GithubRepoConfig,
|
|
||||||
KhojUser,
|
|
||||||
LocalMarkdownConfig,
|
|
||||||
LocalOrgConfig,
|
|
||||||
LocalPdfConfig,
|
|
||||||
LocalPlaintextConfig,
|
|
||||||
NotionConfig,
|
|
||||||
)
|
|
||||||
from khoj.routers.helpers import (
|
|
||||||
CommonQueryParams,
|
|
||||||
get_user_config,
|
|
||||||
update_telemetry_state,
|
|
||||||
)
|
|
||||||
from khoj.utils import constants, state
|
|
||||||
from khoj.utils.rawconfig import (
|
|
||||||
FullConfig,
|
|
||||||
GithubContentConfig,
|
|
||||||
NotionContentConfig,
|
|
||||||
SearchConfig,
|
|
||||||
)
|
|
||||||
from khoj.utils.state import SearchType
|
|
||||||
|
|
||||||
api_config = APIRouter()
|
api_config = APIRouter()
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def map_config_to_object(content_source: str):
|
|
||||||
if content_source == DbEntry.EntrySource.GITHUB:
|
|
||||||
return GithubConfig
|
|
||||||
if content_source == DbEntry.EntrySource.NOTION:
|
|
||||||
return NotionConfig
|
|
||||||
if content_source == DbEntry.EntrySource.COMPUTER:
|
|
||||||
return "Computer"
|
|
||||||
|
|
||||||
|
|
||||||
async def map_config_to_db(config: FullConfig, user: KhojUser):
|
|
||||||
if config.content_type:
|
|
||||||
if config.content_type.org:
|
|
||||||
await LocalOrgConfig.objects.filter(user=user).adelete()
|
|
||||||
await LocalOrgConfig.objects.acreate(
|
|
||||||
input_files=config.content_type.org.input_files,
|
|
||||||
input_filter=config.content_type.org.input_filter,
|
|
||||||
index_heading_entries=config.content_type.org.index_heading_entries,
|
|
||||||
user=user,
|
|
||||||
)
|
|
||||||
if config.content_type.markdown:
|
|
||||||
await LocalMarkdownConfig.objects.filter(user=user).adelete()
|
|
||||||
await LocalMarkdownConfig.objects.acreate(
|
|
||||||
input_files=config.content_type.markdown.input_files,
|
|
||||||
input_filter=config.content_type.markdown.input_filter,
|
|
||||||
index_heading_entries=config.content_type.markdown.index_heading_entries,
|
|
||||||
user=user,
|
|
||||||
)
|
|
||||||
if config.content_type.pdf:
|
|
||||||
await LocalPdfConfig.objects.filter(user=user).adelete()
|
|
||||||
await LocalPdfConfig.objects.acreate(
|
|
||||||
input_files=config.content_type.pdf.input_files,
|
|
||||||
input_filter=config.content_type.pdf.input_filter,
|
|
||||||
index_heading_entries=config.content_type.pdf.index_heading_entries,
|
|
||||||
user=user,
|
|
||||||
)
|
|
||||||
if config.content_type.plaintext:
|
|
||||||
await LocalPlaintextConfig.objects.filter(user=user).adelete()
|
|
||||||
await LocalPlaintextConfig.objects.acreate(
|
|
||||||
input_files=config.content_type.plaintext.input_files,
|
|
||||||
input_filter=config.content_type.plaintext.input_filter,
|
|
||||||
index_heading_entries=config.content_type.plaintext.index_heading_entries,
|
|
||||||
user=user,
|
|
||||||
)
|
|
||||||
if config.content_type.github:
|
|
||||||
await adapters.set_user_github_config(
|
|
||||||
user=user,
|
|
||||||
pat_token=config.content_type.github.pat_token,
|
|
||||||
repos=config.content_type.github.repos,
|
|
||||||
)
|
|
||||||
if config.content_type.notion:
|
|
||||||
await adapters.set_notion_config(
|
|
||||||
user=user,
|
|
||||||
token=config.content_type.notion.token,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _initialize_config():
|
|
||||||
if state.config is None:
|
|
||||||
state.config = FullConfig()
|
|
||||||
state.config.search_type = SearchConfig.model_validate(constants.default_config["search-type"])
|
|
||||||
|
|
||||||
|
|
||||||
@api_config.get("", response_class=Response)
|
|
||||||
@requires(["authenticated"])
|
|
||||||
def get_config(request: Request, detailed: Optional[bool] = False) -> Response:
|
|
||||||
user = request.user.object
|
|
||||||
user_config = get_user_config(user, request, is_detailed=detailed)
|
|
||||||
del user_config["request"]
|
|
||||||
|
|
||||||
# Return config data as a JSON response
|
|
||||||
return Response(content=json.dumps(user_config), media_type="application/json", status_code=200)
|
|
||||||
|
|
||||||
|
|
||||||
@api_config.get("/content/github", response_class=Response)
|
|
||||||
@requires(["authenticated"])
|
|
||||||
def get_content_github(request: Request) -> Response:
|
|
||||||
user = request.user.object
|
|
||||||
user_config = get_user_config(user, request)
|
|
||||||
del user_config["request"]
|
|
||||||
|
|
||||||
current_github_config = get_user_github_config(user)
|
|
||||||
|
|
||||||
if current_github_config:
|
|
||||||
raw_repos = current_github_config.githubrepoconfig.all()
|
|
||||||
repos = []
|
|
||||||
for repo in raw_repos:
|
|
||||||
repos.append(
|
|
||||||
GithubRepoConfig(
|
|
||||||
name=repo.name,
|
|
||||||
owner=repo.owner,
|
|
||||||
branch=repo.branch,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
current_config = GithubContentConfig(
|
|
||||||
pat_token=current_github_config.pat_token,
|
|
||||||
repos=repos,
|
|
||||||
)
|
|
||||||
current_config = json.loads(current_config.json())
|
|
||||||
else:
|
|
||||||
current_config = {} # type: ignore
|
|
||||||
|
|
||||||
user_config["current_config"] = current_config
|
|
||||||
|
|
||||||
# Return config data as a JSON response
|
|
||||||
return Response(content=json.dumps(user_config), media_type="application/json", status_code=200)
|
|
||||||
|
|
||||||
|
|
||||||
@api_config.get("/content/notion", response_class=Response)
|
|
||||||
@requires(["authenticated"])
|
|
||||||
def get_content_notion(request: Request) -> Response:
|
|
||||||
user = request.user.object
|
|
||||||
user_config = get_user_config(user, request)
|
|
||||||
del user_config["request"]
|
|
||||||
|
|
||||||
current_notion_config = get_user_notion_config(user)
|
|
||||||
token = current_notion_config.token if current_notion_config else ""
|
|
||||||
current_config = NotionContentConfig(token=token)
|
|
||||||
current_config = json.loads(current_config.model_dump_json())
|
|
||||||
|
|
||||||
user_config["current_config"] = current_config
|
|
||||||
|
|
||||||
# Return config data as a JSON response
|
|
||||||
return Response(content=json.dumps(user_config), media_type="application/json", status_code=200)
|
|
||||||
|
|
||||||
|
|
||||||
@api_config.post("/content/github", status_code=200)
|
|
||||||
@requires(["authenticated"])
|
|
||||||
async def set_content_github(
|
|
||||||
request: Request,
|
|
||||||
updated_config: Union[GithubContentConfig, None],
|
|
||||||
client: Optional[str] = None,
|
|
||||||
):
|
|
||||||
_initialize_config()
|
|
||||||
|
|
||||||
user = request.user.object
|
|
||||||
|
|
||||||
try:
|
|
||||||
await adapters.set_user_github_config(
|
|
||||||
user=user,
|
|
||||||
pat_token=updated_config.pat_token,
|
|
||||||
repos=updated_config.repos,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(e, exc_info=True)
|
|
||||||
raise HTTPException(status_code=500, detail="Failed to set Github config")
|
|
||||||
|
|
||||||
update_telemetry_state(
|
|
||||||
request=request,
|
|
||||||
telemetry_type="api",
|
|
||||||
api="set_content_config",
|
|
||||||
client=client,
|
|
||||||
metadata={"content_type": "github"},
|
|
||||||
)
|
|
||||||
|
|
||||||
return {"status": "ok"}
|
|
||||||
|
|
||||||
|
|
||||||
@api_config.post("/content/notion", status_code=200)
|
|
||||||
@requires(["authenticated"])
|
|
||||||
async def set_content_notion(
|
|
||||||
request: Request,
|
|
||||||
updated_config: Union[NotionContentConfig, None],
|
|
||||||
client: Optional[str] = None,
|
|
||||||
):
|
|
||||||
_initialize_config()
|
|
||||||
|
|
||||||
user = request.user.object
|
|
||||||
|
|
||||||
try:
|
|
||||||
await adapters.set_notion_config(
|
|
||||||
user=user,
|
|
||||||
token=updated_config.token,
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(e, exc_info=True)
|
|
||||||
raise HTTPException(status_code=500, detail="Failed to set Github config")
|
|
||||||
|
|
||||||
update_telemetry_state(
|
|
||||||
request=request,
|
|
||||||
telemetry_type="api",
|
|
||||||
api="set_content_config",
|
|
||||||
client=client,
|
|
||||||
metadata={"content_type": "notion"},
|
|
||||||
)
|
|
||||||
|
|
||||||
return {"status": "ok"}
|
|
||||||
|
|
||||||
|
|
||||||
@api_config.delete("/content/{content_source}", status_code=200)
|
|
||||||
@requires(["authenticated"])
|
|
||||||
async def delete_content_source(
|
|
||||||
request: Request,
|
|
||||||
content_source: str,
|
|
||||||
client: Optional[str] = None,
|
|
||||||
):
|
|
||||||
user = request.user.object
|
|
||||||
|
|
||||||
update_telemetry_state(
|
|
||||||
request=request,
|
|
||||||
telemetry_type="api",
|
|
||||||
api="delete_content_config",
|
|
||||||
client=client,
|
|
||||||
metadata={"content_source": content_source},
|
|
||||||
)
|
|
||||||
|
|
||||||
content_object = map_config_to_object(content_source)
|
|
||||||
if content_object is None:
|
|
||||||
raise ValueError(f"Invalid content source: {content_source}")
|
|
||||||
elif content_object != "Computer":
|
|
||||||
await content_object.objects.filter(user=user).adelete()
|
|
||||||
await sync_to_async(EntryAdapters.delete_all_entries)(user, file_source=content_source)
|
|
||||||
|
|
||||||
enabled_content = await sync_to_async(EntryAdapters.get_unique_file_types)(user)
|
|
||||||
return {"status": "ok"}
|
|
||||||
|
|
||||||
|
|
||||||
@api_config.delete("/content/file", status_code=201)
|
|
||||||
@requires(["authenticated"])
|
|
||||||
async def delete_content_file(
|
|
||||||
request: Request,
|
|
||||||
filename: str,
|
|
||||||
client: Optional[str] = None,
|
|
||||||
):
|
|
||||||
user = request.user.object
|
|
||||||
|
|
||||||
update_telemetry_state(
|
|
||||||
request=request,
|
|
||||||
telemetry_type="api",
|
|
||||||
api="delete_file",
|
|
||||||
client=client,
|
|
||||||
)
|
|
||||||
|
|
||||||
await EntryAdapters.adelete_entry_by_file(user, filename)
|
|
||||||
|
|
||||||
return {"status": "ok"}
|
|
||||||
|
|
||||||
|
|
||||||
@api_config.get("/content/{content_source}", response_model=List[str])
|
|
||||||
@requires(["authenticated"])
|
|
||||||
async def get_content_source(
|
|
||||||
request: Request,
|
|
||||||
content_source: str,
|
|
||||||
client: Optional[str] = None,
|
|
||||||
):
|
|
||||||
user = request.user.object
|
|
||||||
|
|
||||||
update_telemetry_state(
|
|
||||||
request=request,
|
|
||||||
telemetry_type="api",
|
|
||||||
api="get_all_filenames",
|
|
||||||
client=client,
|
|
||||||
)
|
|
||||||
|
|
||||||
return await sync_to_async(list)(EntryAdapters.get_all_filenames_by_source(user, content_source)) # type: ignore[call-arg]
|
|
||||||
|
|
||||||
|
|
||||||
@api_config.get("/chat/model/options", response_model=Dict[str, Union[str, int]])
|
@api_config.get("/chat/model/options", response_model=Dict[str, Union[str, int]])
|
||||||
def get_chat_model_options(
|
def get_chat_model_options(
|
||||||
request: Request,
|
request: Request,
|
||||||
|
@ -442,18 +156,6 @@ async def update_paint_model(
|
||||||
return {"status": "ok"}
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
@api_config.get("/content/size", response_model=Dict[str, int])
|
|
||||||
@requires(["authenticated"])
|
|
||||||
async def get_content_size(request: Request, common: CommonQueryParams):
|
|
||||||
user = request.user.object
|
|
||||||
indexed_data_size_in_mb = await sync_to_async(EntryAdapters.get_size_of_indexed_data_in_mb)(user)
|
|
||||||
return Response(
|
|
||||||
content=json.dumps({"indexed_data_size_in_mb": math.ceil(indexed_data_size_in_mb)}),
|
|
||||||
media_type="application/json",
|
|
||||||
status_code=200,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@api_config.post("/user/name", status_code=200)
|
@api_config.post("/user/name", status_code=200)
|
||||||
@requires(["authenticated"])
|
@requires(["authenticated"])
|
||||||
def set_user_name(
|
def set_user_name(
|
||||||
|
@ -484,23 +186,3 @@ def set_user_name(
|
||||||
)
|
)
|
||||||
|
|
||||||
return {"status": "ok"}
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
@api_config.get("/types", response_model=List[str])
|
|
||||||
@requires(["authenticated"])
|
|
||||||
def get_config_types(
|
|
||||||
request: Request,
|
|
||||||
):
|
|
||||||
user = request.user.object
|
|
||||||
enabled_file_types = EntryAdapters.get_unique_file_types(user)
|
|
||||||
configured_content_types = list(enabled_file_types)
|
|
||||||
|
|
||||||
if state.config and state.config.content_type:
|
|
||||||
for ctype in state.config.content_type.model_dump(exclude_none=True):
|
|
||||||
configured_content_types.append(ctype)
|
|
||||||
|
|
||||||
return [
|
|
||||||
search_type.value
|
|
||||||
for search_type in SearchType
|
|
||||||
if (search_type.value in configured_content_types) or search_type == SearchType.All
|
|
||||||
]
|
|
||||||
|
|
|
@ -1,20 +1,57 @@
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
from typing import Dict, Optional, Union
|
import math
|
||||||
|
from typing import Dict, List, Optional, Union
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Header, Request, Response, UploadFile
|
from asgiref.sync import sync_to_async
|
||||||
|
from fastapi import (
|
||||||
|
APIRouter,
|
||||||
|
Depends,
|
||||||
|
Header,
|
||||||
|
HTTPException,
|
||||||
|
Request,
|
||||||
|
Response,
|
||||||
|
UploadFile,
|
||||||
|
)
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from starlette.authentication import requires
|
from starlette.authentication import requires
|
||||||
|
|
||||||
|
from khoj.database import adapters
|
||||||
|
from khoj.database.adapters import (
|
||||||
|
EntryAdapters,
|
||||||
|
get_user_github_config,
|
||||||
|
get_user_notion_config,
|
||||||
|
)
|
||||||
|
from khoj.database.models import Entry as DbEntry
|
||||||
|
from khoj.database.models import (
|
||||||
|
GithubConfig,
|
||||||
|
GithubRepoConfig,
|
||||||
|
KhojUser,
|
||||||
|
LocalMarkdownConfig,
|
||||||
|
LocalOrgConfig,
|
||||||
|
LocalPdfConfig,
|
||||||
|
LocalPlaintextConfig,
|
||||||
|
NotionConfig,
|
||||||
|
)
|
||||||
from khoj.routers.helpers import (
|
from khoj.routers.helpers import (
|
||||||
ApiIndexedDataLimiter,
|
ApiIndexedDataLimiter,
|
||||||
|
CommonQueryParams,
|
||||||
configure_content,
|
configure_content,
|
||||||
|
get_user_config,
|
||||||
update_telemetry_state,
|
update_telemetry_state,
|
||||||
)
|
)
|
||||||
from khoj.utils import constants, state
|
from khoj.utils import constants, state
|
||||||
from khoj.utils.config import SearchModels
|
from khoj.utils.config import SearchModels
|
||||||
from khoj.utils.helpers import get_file_type
|
from khoj.utils.helpers import get_file_type
|
||||||
from khoj.utils.rawconfig import ContentConfig, FullConfig, SearchConfig
|
from khoj.utils.rawconfig import (
|
||||||
|
ContentConfig,
|
||||||
|
FullConfig,
|
||||||
|
GithubContentConfig,
|
||||||
|
NotionContentConfig,
|
||||||
|
SearchConfig,
|
||||||
|
)
|
||||||
|
from khoj.utils.state import SearchType
|
||||||
from khoj.utils.yaml import save_config_to_file_updated_state
|
from khoj.utils.yaml import save_config_to_file_updated_state
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
@ -84,6 +121,216 @@ async def patch_content(
|
||||||
return await indexer(request, files, t, False, client, user_agent, referer, host)
|
return await indexer(request, files, t, False, client, user_agent, referer, host)
|
||||||
|
|
||||||
|
|
||||||
|
@api_content.get("/github", response_class=Response)
|
||||||
|
@requires(["authenticated"])
|
||||||
|
def get_content_github(request: Request) -> Response:
|
||||||
|
user = request.user.object
|
||||||
|
user_config = get_user_config(user, request)
|
||||||
|
del user_config["request"]
|
||||||
|
|
||||||
|
current_github_config = get_user_github_config(user)
|
||||||
|
|
||||||
|
if current_github_config:
|
||||||
|
raw_repos = current_github_config.githubrepoconfig.all()
|
||||||
|
repos = []
|
||||||
|
for repo in raw_repos:
|
||||||
|
repos.append(
|
||||||
|
GithubRepoConfig(
|
||||||
|
name=repo.name,
|
||||||
|
owner=repo.owner,
|
||||||
|
branch=repo.branch,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
current_config = GithubContentConfig(
|
||||||
|
pat_token=current_github_config.pat_token,
|
||||||
|
repos=repos,
|
||||||
|
)
|
||||||
|
current_config = json.loads(current_config.json())
|
||||||
|
else:
|
||||||
|
current_config = {} # type: ignore
|
||||||
|
|
||||||
|
user_config["current_config"] = current_config
|
||||||
|
|
||||||
|
# Return config data as a JSON response
|
||||||
|
return Response(content=json.dumps(user_config), media_type="application/json", status_code=200)
|
||||||
|
|
||||||
|
|
||||||
|
@api_content.get("/notion", response_class=Response)
|
||||||
|
@requires(["authenticated"])
|
||||||
|
def get_content_notion(request: Request) -> Response:
|
||||||
|
user = request.user.object
|
||||||
|
user_config = get_user_config(user, request)
|
||||||
|
del user_config["request"]
|
||||||
|
|
||||||
|
current_notion_config = get_user_notion_config(user)
|
||||||
|
token = current_notion_config.token if current_notion_config else ""
|
||||||
|
current_config = NotionContentConfig(token=token)
|
||||||
|
current_config = json.loads(current_config.model_dump_json())
|
||||||
|
|
||||||
|
user_config["current_config"] = current_config
|
||||||
|
|
||||||
|
# Return config data as a JSON response
|
||||||
|
return Response(content=json.dumps(user_config), media_type="application/json", status_code=200)
|
||||||
|
|
||||||
|
|
||||||
|
@api_content.post("/github", status_code=200)
|
||||||
|
@requires(["authenticated"])
|
||||||
|
async def set_content_github(
|
||||||
|
request: Request,
|
||||||
|
updated_config: Union[GithubContentConfig, None],
|
||||||
|
client: Optional[str] = None,
|
||||||
|
):
|
||||||
|
_initialize_config()
|
||||||
|
|
||||||
|
user = request.user.object
|
||||||
|
|
||||||
|
try:
|
||||||
|
await adapters.set_user_github_config(
|
||||||
|
user=user,
|
||||||
|
pat_token=updated_config.pat_token,
|
||||||
|
repos=updated_config.repos,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(e, exc_info=True)
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to set Github config")
|
||||||
|
|
||||||
|
update_telemetry_state(
|
||||||
|
request=request,
|
||||||
|
telemetry_type="api",
|
||||||
|
api="set_content_config",
|
||||||
|
client=client,
|
||||||
|
metadata={"content_type": "github"},
|
||||||
|
)
|
||||||
|
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@api_content.post("/notion", status_code=200)
|
||||||
|
@requires(["authenticated"])
|
||||||
|
async def set_content_notion(
|
||||||
|
request: Request,
|
||||||
|
updated_config: Union[NotionContentConfig, None],
|
||||||
|
client: Optional[str] = None,
|
||||||
|
):
|
||||||
|
_initialize_config()
|
||||||
|
|
||||||
|
user = request.user.object
|
||||||
|
|
||||||
|
try:
|
||||||
|
await adapters.set_notion_config(
|
||||||
|
user=user,
|
||||||
|
token=updated_config.token,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(e, exc_info=True)
|
||||||
|
raise HTTPException(status_code=500, detail="Failed to set Notion config")
|
||||||
|
|
||||||
|
update_telemetry_state(
|
||||||
|
request=request,
|
||||||
|
telemetry_type="api",
|
||||||
|
api="set_content_config",
|
||||||
|
client=client,
|
||||||
|
metadata={"content_type": "notion"},
|
||||||
|
)
|
||||||
|
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@api_content.delete("/{content_source}", status_code=200)
|
||||||
|
@requires(["authenticated"])
|
||||||
|
async def delete_content_source(
|
||||||
|
request: Request,
|
||||||
|
content_source: str,
|
||||||
|
client: Optional[str] = None,
|
||||||
|
):
|
||||||
|
user = request.user.object
|
||||||
|
|
||||||
|
update_telemetry_state(
|
||||||
|
request=request,
|
||||||
|
telemetry_type="api",
|
||||||
|
api="delete_content_config",
|
||||||
|
client=client,
|
||||||
|
metadata={"content_source": content_source},
|
||||||
|
)
|
||||||
|
|
||||||
|
content_object = map_config_to_object(content_source)
|
||||||
|
if content_object is None:
|
||||||
|
raise ValueError(f"Invalid content source: {content_source}")
|
||||||
|
elif content_object != "Computer":
|
||||||
|
await content_object.objects.filter(user=user).adelete()
|
||||||
|
await sync_to_async(EntryAdapters.delete_all_entries)(user, file_source=content_source)
|
||||||
|
|
||||||
|
enabled_content = await sync_to_async(EntryAdapters.get_unique_file_types)(user)
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@api_content.delete("/file", status_code=201)
|
||||||
|
@requires(["authenticated"])
|
||||||
|
async def delete_content_file(
|
||||||
|
request: Request,
|
||||||
|
filename: str,
|
||||||
|
client: Optional[str] = None,
|
||||||
|
):
|
||||||
|
user = request.user.object
|
||||||
|
|
||||||
|
update_telemetry_state(
|
||||||
|
request=request,
|
||||||
|
telemetry_type="api",
|
||||||
|
api="delete_file",
|
||||||
|
client=client,
|
||||||
|
)
|
||||||
|
|
||||||
|
await EntryAdapters.adelete_entry_by_file(user, filename)
|
||||||
|
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@api_content.get("/size", response_model=Dict[str, int])
|
||||||
|
@requires(["authenticated"])
|
||||||
|
async def get_content_size(request: Request, common: CommonQueryParams, client: Optional[str] = None):
|
||||||
|
user = request.user.object
|
||||||
|
indexed_data_size_in_mb = await sync_to_async(EntryAdapters.get_size_of_indexed_data_in_mb)(user)
|
||||||
|
return Response(
|
||||||
|
content=json.dumps({"indexed_data_size_in_mb": math.ceil(indexed_data_size_in_mb)}),
|
||||||
|
media_type="application/json",
|
||||||
|
status_code=200,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@api_content.get("/types", response_model=List[str])
|
||||||
|
@requires(["authenticated"])
|
||||||
|
def get_content_types(request: Request, client: Optional[str] = None):
|
||||||
|
user = request.user.object
|
||||||
|
all_content_types = {s.value for s in SearchType}
|
||||||
|
configured_content_types = set(EntryAdapters.get_unique_file_types(user))
|
||||||
|
configured_content_types |= {"all"}
|
||||||
|
|
||||||
|
if state.config and state.config.content_type:
|
||||||
|
for ctype in state.config.content_type.model_dump(exclude_none=True):
|
||||||
|
configured_content_types.add(ctype)
|
||||||
|
|
||||||
|
return list(configured_content_types & all_content_types)
|
||||||
|
|
||||||
|
|
||||||
|
@api_content.get("/{content_source}", response_model=List[str])
|
||||||
|
@requires(["authenticated"])
|
||||||
|
async def get_content_source(
|
||||||
|
request: Request,
|
||||||
|
content_source: str,
|
||||||
|
client: Optional[str] = None,
|
||||||
|
):
|
||||||
|
user = request.user.object
|
||||||
|
|
||||||
|
update_telemetry_state(
|
||||||
|
request=request,
|
||||||
|
telemetry_type="api",
|
||||||
|
api="get_all_filenames",
|
||||||
|
client=client,
|
||||||
|
)
|
||||||
|
|
||||||
|
return await sync_to_async(list)(EntryAdapters.get_all_filenames_by_source(user, content_source)) # type: ignore[call-arg]
|
||||||
|
|
||||||
|
|
||||||
async def indexer(
|
async def indexer(
|
||||||
request: Request,
|
request: Request,
|
||||||
files: list[UploadFile],
|
files: list[UploadFile],
|
||||||
|
@ -198,3 +445,65 @@ def configure_search(search_models: SearchModels, search_config: Optional[Search
|
||||||
search_models = SearchModels()
|
search_models = SearchModels()
|
||||||
|
|
||||||
return search_models
|
return search_models
|
||||||
|
|
||||||
|
|
||||||
|
def map_config_to_object(content_source: str):
|
||||||
|
if content_source == DbEntry.EntrySource.GITHUB:
|
||||||
|
return GithubConfig
|
||||||
|
if content_source == DbEntry.EntrySource.NOTION:
|
||||||
|
return NotionConfig
|
||||||
|
if content_source == DbEntry.EntrySource.COMPUTER:
|
||||||
|
return "Computer"
|
||||||
|
|
||||||
|
|
||||||
|
async def map_config_to_db(config: FullConfig, user: KhojUser):
|
||||||
|
if config.content_type:
|
||||||
|
if config.content_type.org:
|
||||||
|
await LocalOrgConfig.objects.filter(user=user).adelete()
|
||||||
|
await LocalOrgConfig.objects.acreate(
|
||||||
|
input_files=config.content_type.org.input_files,
|
||||||
|
input_filter=config.content_type.org.input_filter,
|
||||||
|
index_heading_entries=config.content_type.org.index_heading_entries,
|
||||||
|
user=user,
|
||||||
|
)
|
||||||
|
if config.content_type.markdown:
|
||||||
|
await LocalMarkdownConfig.objects.filter(user=user).adelete()
|
||||||
|
await LocalMarkdownConfig.objects.acreate(
|
||||||
|
input_files=config.content_type.markdown.input_files,
|
||||||
|
input_filter=config.content_type.markdown.input_filter,
|
||||||
|
index_heading_entries=config.content_type.markdown.index_heading_entries,
|
||||||
|
user=user,
|
||||||
|
)
|
||||||
|
if config.content_type.pdf:
|
||||||
|
await LocalPdfConfig.objects.filter(user=user).adelete()
|
||||||
|
await LocalPdfConfig.objects.acreate(
|
||||||
|
input_files=config.content_type.pdf.input_files,
|
||||||
|
input_filter=config.content_type.pdf.input_filter,
|
||||||
|
index_heading_entries=config.content_type.pdf.index_heading_entries,
|
||||||
|
user=user,
|
||||||
|
)
|
||||||
|
if config.content_type.plaintext:
|
||||||
|
await LocalPlaintextConfig.objects.filter(user=user).adelete()
|
||||||
|
await LocalPlaintextConfig.objects.acreate(
|
||||||
|
input_files=config.content_type.plaintext.input_files,
|
||||||
|
input_filter=config.content_type.plaintext.input_filter,
|
||||||
|
index_heading_entries=config.content_type.plaintext.index_heading_entries,
|
||||||
|
user=user,
|
||||||
|
)
|
||||||
|
if config.content_type.github:
|
||||||
|
await adapters.set_user_github_config(
|
||||||
|
user=user,
|
||||||
|
pat_token=config.content_type.github.pat_token,
|
||||||
|
repos=config.content_type.github.repos,
|
||||||
|
)
|
||||||
|
if config.content_type.notion:
|
||||||
|
await adapters.set_notion_config(
|
||||||
|
user=user,
|
||||||
|
token=config.content_type.notion.token,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _initialize_config():
|
||||||
|
if state.config is None:
|
||||||
|
state.config = FullConfig()
|
||||||
|
state.config.search_type = SearchConfig.model_validate(constants.default_config["search-type"])
|
||||||
|
|
|
@ -269,11 +269,11 @@ def test_get_api_config_types(client, sample_org_data, default_user: KhojUser):
|
||||||
text_search.setup(OrgToEntries, sample_org_data, regenerate=False, user=default_user)
|
text_search.setup(OrgToEntries, sample_org_data, regenerate=False, user=default_user)
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
response = client.get(f"/api/configure/types", headers=headers)
|
response = client.get(f"/api/content/types", headers=headers)
|
||||||
|
|
||||||
# Assert
|
# Assert
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
assert response.json() == ["all", "org", "plaintext"]
|
assert set(response.json()) == {"all", "org", "plaintext"}
|
||||||
|
|
||||||
|
|
||||||
# ----------------------------------------------------------------------------------------------------
|
# ----------------------------------------------------------------------------------------------------
|
||||||
|
@ -289,7 +289,7 @@ def test_get_configured_types_with_no_content_config(fastapi_app: FastAPI):
|
||||||
client = TestClient(fastapi_app)
|
client = TestClient(fastapi_app)
|
||||||
|
|
||||||
# Act
|
# Act
|
||||||
response = client.get(f"/api/configure/types")
|
response = client.get(f"/api/content/types")
|
||||||
|
|
||||||
# Assert
|
# Assert
|
||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
|
|
Loading…
Reference in a new issue