mirror of
https://github.com/khoj-ai/khoj.git
synced 2025-02-17 08:04:21 +00:00
Reuse get config data logic across config pages on web client
- Put logic to get config data, detailed or basic into router helpers module - Use the get config func across the config pages on web clients - Put configure content and get_notion_auth_url funcs in router helper module to avoid circular import
This commit is contained in:
Debanjum Singh Solanky
parent
1a5405e24c
commit
e8176b41ef
4 changed files with 347 additions and 448 deletions
|
|
@ -5,6 +5,7 @@ import io
|
|||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
|
@ -35,6 +36,7 @@ from PIL import Image
|
|||
from starlette.authentication import has_required_scope
|
||||
from starlette.requests import URL
|
||||
|
||||
from khoj.database import adapters
|
||||
from khoj.database.adapters import (
|
||||
AgentAdapters,
|
||||
AutomationAdapters,
|
||||
|
|
@ -42,18 +44,30 @@ from khoj.database.adapters import (
|
|||
EntryAdapters,
|
||||
create_khoj_token,
|
||||
get_khoj_tokens,
|
||||
get_user_name,
|
||||
get_user_subscription_state,
|
||||
run_with_process_lock,
|
||||
)
|
||||
from khoj.database.models import (
|
||||
ChatModelOptions,
|
||||
ClientApplication,
|
||||
Conversation,
|
||||
GithubConfig,
|
||||
KhojUser,
|
||||
NotionConfig,
|
||||
ProcessLock,
|
||||
Subscription,
|
||||
TextToImageModelConfig,
|
||||
UserRequests,
|
||||
)
|
||||
from khoj.processor.content.docx.docx_to_entries import DocxToEntries
|
||||
from khoj.processor.content.github.github_to_entries import GithubToEntries
|
||||
from khoj.processor.content.images.image_to_entries import ImageToEntries
|
||||
from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries
|
||||
from khoj.processor.content.notion.notion_to_entries import NotionToEntries
|
||||
from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
|
||||
from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries
|
||||
from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries
|
||||
from khoj.processor.conversation import prompts
|
||||
from khoj.processor.conversation.anthropic.anthropic_chat import (
|
||||
anthropic_send_message_to_model,
|
||||
|
|
@ -69,11 +83,15 @@ from khoj.processor.conversation.utils import (
|
|||
generate_chatml_messages_with_context,
|
||||
save_to_conversation_log,
|
||||
)
|
||||
from khoj.processor.speech.text_to_speech import is_eleven_labs_enabled
|
||||
from khoj.routers.email import is_resend_enabled, send_task_email
|
||||
from khoj.routers.storage import upload_image
|
||||
from khoj.routers.twilio import is_twilio_enabled
|
||||
from khoj.search_type import text_search
|
||||
from khoj.utils import state
|
||||
from khoj.utils.config import OfflineChatProcessorModel
|
||||
from khoj.utils.helpers import (
|
||||
LRU,
|
||||
ConversationCommand,
|
||||
ImageIntentType,
|
||||
is_none_or_empty,
|
||||
|
|
@ -1186,3 +1204,286 @@ def construct_automation_created_message(automation: Job, crontime: str, query_t
|
|||
|
||||
Manage your automations [here](/automations).
|
||||
""".strip()
|
||||
|
||||
|
||||
def get_user_config(user: KhojUser, request: Request, is_detailed: bool = False):
|
||||
user_picture = request.session.get("user", {}).get("picture")
|
||||
is_active = has_required_scope(request, ["premium"])
|
||||
has_documents = EntryAdapters.user_has_entries(user=user)
|
||||
|
||||
if not is_detailed:
|
||||
return {
|
||||
"request": request,
|
||||
"username": user.username if user else None,
|
||||
"user_photo": user_picture,
|
||||
"is_active": is_active,
|
||||
"has_documents": has_documents,
|
||||
"khoj_version": state.khoj_version,
|
||||
}
|
||||
|
||||
user_subscription_state = get_user_subscription_state(user.email)
|
||||
user_subscription = adapters.get_user_subscription(user.email)
|
||||
subscription_renewal_date = (
|
||||
user_subscription.renewal_date.strftime("%d %b %Y")
|
||||
if user_subscription and user_subscription.renewal_date
|
||||
else (user_subscription.created_at + timedelta(days=7)).strftime("%d %b %Y")
|
||||
)
|
||||
given_name = get_user_name(user)
|
||||
|
||||
enabled_content_source = set(EntryAdapters.get_unique_file_sources(user))
|
||||
successfully_configured = {
|
||||
"computer": ("computer" in enabled_content_source),
|
||||
"github": ("github" in enabled_content_source),
|
||||
"notion": ("notion" in enabled_content_source),
|
||||
}
|
||||
|
||||
selected_conversation_config = ConversationAdapters.get_conversation_config(user)
|
||||
conversation_options = ConversationAdapters.get_conversation_processor_options().all()
|
||||
all_conversation_options = list()
|
||||
for conversation_option in conversation_options:
|
||||
all_conversation_options.append({"chat_model": conversation_option.chat_model, "id": conversation_option.id})
|
||||
|
||||
search_model_options = adapters.get_or_create_search_models().all()
|
||||
all_search_model_options = list()
|
||||
for search_model_option in search_model_options:
|
||||
all_search_model_options.append({"name": search_model_option.name, "id": search_model_option.id})
|
||||
|
||||
current_search_model_option = adapters.get_user_search_model_or_default(user)
|
||||
|
||||
selected_paint_model_config = ConversationAdapters.get_user_text_to_image_model_config(user)
|
||||
paint_model_options = ConversationAdapters.get_text_to_image_model_options().all()
|
||||
all_paint_model_options = list()
|
||||
for paint_model in paint_model_options:
|
||||
all_paint_model_options.append({"model_name": paint_model.model_name, "id": paint_model.id})
|
||||
|
||||
notion_oauth_url = get_notion_auth_url(user)
|
||||
|
||||
eleven_labs_enabled = is_eleven_labs_enabled()
|
||||
|
||||
voice_models = ConversationAdapters.get_voice_model_options()
|
||||
voice_model_options = list()
|
||||
for voice_model in voice_models:
|
||||
voice_model_options.append({"name": voice_model.name, "id": voice_model.model_id})
|
||||
|
||||
if len(voice_model_options) == 0:
|
||||
eleven_labs_enabled = False
|
||||
|
||||
selected_voice_config = ConversationAdapters.get_voice_model_config(user)
|
||||
|
||||
return {
|
||||
"request": request,
|
||||
"username": user.username if user else None,
|
||||
"user_photo": user_picture,
|
||||
"is_active": is_active,
|
||||
"has_documents": has_documents,
|
||||
"khoj_version": state.khoj_version,
|
||||
"current_model_state": successfully_configured,
|
||||
"anonymous_mode": state.anonymous_mode,
|
||||
"given_name": given_name,
|
||||
"search_model_options": all_search_model_options,
|
||||
"selected_search_model_config": current_search_model_option.id,
|
||||
"conversation_options": all_conversation_options,
|
||||
"selected_conversation_config": selected_conversation_config.id if selected_conversation_config else None,
|
||||
"paint_model_options": all_paint_model_options,
|
||||
"selected_paint_model_config": selected_paint_model_config.id if selected_paint_model_config else None,
|
||||
"user_photo": user_picture,
|
||||
"billing_enabled": state.billing_enabled,
|
||||
"subscription_state": user_subscription_state,
|
||||
"subscription_renewal_date": subscription_renewal_date,
|
||||
"khoj_cloud_subscription_url": os.getenv("KHOJ_CLOUD_SUBSCRIPTION_URL"),
|
||||
"is_twilio_enabled": is_twilio_enabled(),
|
||||
"is_eleven_labs_enabled": eleven_labs_enabled,
|
||||
"voice_model_options": voice_model_options,
|
||||
"selected_voice_config": selected_voice_config.model_id if selected_voice_config else None,
|
||||
"phone_number": user.phone_number,
|
||||
"is_phone_number_verified": user.verified_phone_number,
|
||||
"notion_oauth_url": notion_oauth_url,
|
||||
}
|
||||
|
||||
|
||||
def configure_content(
|
||||
files: Optional[dict[str, dict[str, str]]],
|
||||
regenerate: bool = False,
|
||||
t: Optional[state.SearchType] = state.SearchType.All,
|
||||
full_corpus: bool = True,
|
||||
user: KhojUser = None,
|
||||
) -> bool:
|
||||
success = True
|
||||
if t == None:
|
||||
t = state.SearchType.All
|
||||
|
||||
if t is not None and t in [type.value for type in state.SearchType]:
|
||||
t = state.SearchType(t)
|
||||
|
||||
if t is not None and not t.value in [type.value for type in state.SearchType]:
|
||||
logger.warning(f"🚨 Invalid search type: {t}")
|
||||
return False
|
||||
|
||||
search_type = t.value if t else None
|
||||
|
||||
no_documents = all([not files.get(file_type) for file_type in files])
|
||||
|
||||
if files is None:
|
||||
logger.warning(f"🚨 No files to process for {search_type} search.")
|
||||
return True
|
||||
|
||||
try:
|
||||
# Initialize Org Notes Search
|
||||
if (search_type == state.SearchType.All.value or search_type == state.SearchType.Org.value) and files["org"]:
|
||||
logger.info("🦄 Setting up search for orgmode notes")
|
||||
# Extract Entries, Generate Notes Embeddings
|
||||
text_search.setup(
|
||||
OrgToEntries,
|
||||
files.get("org"),
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup org: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
try:
|
||||
# Initialize Markdown Search
|
||||
if (search_type == state.SearchType.All.value or search_type == state.SearchType.Markdown.value) and files[
|
||||
"markdown"
|
||||
]:
|
||||
logger.info("💎 Setting up search for markdown notes")
|
||||
# Extract Entries, Generate Markdown Embeddings
|
||||
text_search.setup(
|
||||
MarkdownToEntries,
|
||||
files.get("markdown"),
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup markdown: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
try:
|
||||
# Initialize PDF Search
|
||||
if (search_type == state.SearchType.All.value or search_type == state.SearchType.Pdf.value) and files["pdf"]:
|
||||
logger.info("🖨️ Setting up search for pdf")
|
||||
# Extract Entries, Generate PDF Embeddings
|
||||
text_search.setup(
|
||||
PdfToEntries,
|
||||
files.get("pdf"),
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup PDF: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
try:
|
||||
# Initialize Plaintext Search
|
||||
if (search_type == state.SearchType.All.value or search_type == state.SearchType.Plaintext.value) and files[
|
||||
"plaintext"
|
||||
]:
|
||||
logger.info("📄 Setting up search for plaintext")
|
||||
# Extract Entries, Generate Plaintext Embeddings
|
||||
text_search.setup(
|
||||
PlaintextToEntries,
|
||||
files.get("plaintext"),
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup plaintext: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
try:
|
||||
if no_documents:
|
||||
github_config = GithubConfig.objects.filter(user=user).prefetch_related("githubrepoconfig").first()
|
||||
if (
|
||||
search_type == state.SearchType.All.value or search_type == state.SearchType.Github.value
|
||||
) and github_config is not None:
|
||||
logger.info("🐙 Setting up search for github")
|
||||
# Extract Entries, Generate Github Embeddings
|
||||
text_search.setup(
|
||||
GithubToEntries,
|
||||
None,
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
config=github_config,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup GitHub: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
try:
|
||||
if no_documents:
|
||||
# Initialize Notion Search
|
||||
notion_config = NotionConfig.objects.filter(user=user).first()
|
||||
if (
|
||||
search_type == state.SearchType.All.value or search_type == state.SearchType.Notion.value
|
||||
) and notion_config:
|
||||
logger.info("🔌 Setting up search for notion")
|
||||
text_search.setup(
|
||||
NotionToEntries,
|
||||
None,
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
config=notion_config,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup Notion: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
try:
|
||||
# Initialize Image Search
|
||||
if (search_type == state.SearchType.All.value or search_type == state.SearchType.Image.value) and files[
|
||||
"image"
|
||||
]:
|
||||
logger.info("🖼️ Setting up search for images")
|
||||
# Extract Entries, Generate Image Embeddings
|
||||
text_search.setup(
|
||||
ImageToEntries,
|
||||
files.get("image"),
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup images: {e}", exc_info=True)
|
||||
success = False
|
||||
try:
|
||||
if (search_type == state.SearchType.All.value or search_type == state.SearchType.Docx.value) and files["docx"]:
|
||||
logger.info("📄 Setting up search for docx")
|
||||
text_search.setup(
|
||||
DocxToEntries,
|
||||
files.get("docx"),
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup docx: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
# Invalidate Query Cache
|
||||
if user:
|
||||
state.query_cache[user.uuid] = LRU()
|
||||
|
||||
return success
|
||||
|
||||
|
||||
NOTION_OAUTH_CLIENT_ID = os.getenv("NOTION_OAUTH_CLIENT_ID")
|
||||
NOTION_OAUTH_CLIENT_SECRET = os.getenv("NOTION_OAUTH_CLIENT_SECRET")
|
||||
NOTION_REDIRECT_URI = os.getenv("NOTION_REDIRECT_URI")
|
||||
|
||||
|
||||
def get_notion_auth_url(user: KhojUser):
|
||||
if not NOTION_OAUTH_CLIENT_ID or not NOTION_OAUTH_CLIENT_SECRET or not NOTION_REDIRECT_URI:
|
||||
return None
|
||||
return f"https://api.notion.com/v1/oauth/authorize?client_id={NOTION_OAUTH_CLIENT_ID}&redirect_uri={NOTION_REDIRECT_URI}&response_type=code&state={user.uuid}"
|
||||
|
|
|
|||
|
|
@ -6,20 +6,14 @@ from fastapi import APIRouter, Depends, Header, Request, Response, UploadFile
|
|||
from pydantic import BaseModel
|
||||
from starlette.authentication import requires
|
||||
|
||||
from khoj.database.models import GithubConfig, KhojUser, NotionConfig
|
||||
from khoj.processor.content.docx.docx_to_entries import DocxToEntries
|
||||
from khoj.processor.content.github.github_to_entries import GithubToEntries
|
||||
from khoj.processor.content.images.image_to_entries import ImageToEntries
|
||||
from khoj.processor.content.markdown.markdown_to_entries import MarkdownToEntries
|
||||
from khoj.processor.content.notion.notion_to_entries import NotionToEntries
|
||||
from khoj.processor.content.org_mode.org_to_entries import OrgToEntries
|
||||
from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries
|
||||
from khoj.processor.content.plaintext.plaintext_to_entries import PlaintextToEntries
|
||||
from khoj.routers.helpers import ApiIndexedDataLimiter, update_telemetry_state
|
||||
from khoj.search_type import text_search
|
||||
from khoj.routers.helpers import (
|
||||
ApiIndexedDataLimiter,
|
||||
configure_content,
|
||||
update_telemetry_state,
|
||||
)
|
||||
from khoj.utils import constants, state
|
||||
from khoj.utils.config import SearchModels
|
||||
from khoj.utils.helpers import LRU, get_file_type
|
||||
from khoj.utils.helpers import get_file_type
|
||||
from khoj.utils.rawconfig import ContentConfig, FullConfig, SearchConfig
|
||||
from khoj.utils.yaml import save_config_to_file_updated_state
|
||||
|
||||
|
|
@ -170,180 +164,3 @@ def configure_search(search_models: SearchModels, search_config: Optional[Search
|
|||
search_models = SearchModels()
|
||||
|
||||
return search_models
|
||||
|
||||
|
||||
def configure_content(
|
||||
files: Optional[dict[str, dict[str, str]]],
|
||||
regenerate: bool = False,
|
||||
t: Optional[state.SearchType] = state.SearchType.All,
|
||||
full_corpus: bool = True,
|
||||
user: KhojUser = None,
|
||||
) -> bool:
|
||||
success = True
|
||||
if t == None:
|
||||
t = state.SearchType.All
|
||||
|
||||
if t is not None and t in [type.value for type in state.SearchType]:
|
||||
t = state.SearchType(t)
|
||||
|
||||
if t is not None and not t.value in [type.value for type in state.SearchType]:
|
||||
logger.warning(f"🚨 Invalid search type: {t}")
|
||||
return False
|
||||
|
||||
search_type = t.value if t else None
|
||||
|
||||
no_documents = all([not files.get(file_type) for file_type in files])
|
||||
|
||||
if files is None:
|
||||
logger.warning(f"🚨 No files to process for {search_type} search.")
|
||||
return True
|
||||
|
||||
try:
|
||||
# Initialize Org Notes Search
|
||||
if (search_type == state.SearchType.All.value or search_type == state.SearchType.Org.value) and files["org"]:
|
||||
logger.info("🦄 Setting up search for orgmode notes")
|
||||
# Extract Entries, Generate Notes Embeddings
|
||||
text_search.setup(
|
||||
OrgToEntries,
|
||||
files.get("org"),
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup org: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
try:
|
||||
# Initialize Markdown Search
|
||||
if (search_type == state.SearchType.All.value or search_type == state.SearchType.Markdown.value) and files[
|
||||
"markdown"
|
||||
]:
|
||||
logger.info("💎 Setting up search for markdown notes")
|
||||
# Extract Entries, Generate Markdown Embeddings
|
||||
text_search.setup(
|
||||
MarkdownToEntries,
|
||||
files.get("markdown"),
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup markdown: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
try:
|
||||
# Initialize PDF Search
|
||||
if (search_type == state.SearchType.All.value or search_type == state.SearchType.Pdf.value) and files["pdf"]:
|
||||
logger.info("🖨️ Setting up search for pdf")
|
||||
# Extract Entries, Generate PDF Embeddings
|
||||
text_search.setup(
|
||||
PdfToEntries,
|
||||
files.get("pdf"),
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup PDF: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
try:
|
||||
# Initialize Plaintext Search
|
||||
if (search_type == state.SearchType.All.value or search_type == state.SearchType.Plaintext.value) and files[
|
||||
"plaintext"
|
||||
]:
|
||||
logger.info("📄 Setting up search for plaintext")
|
||||
# Extract Entries, Generate Plaintext Embeddings
|
||||
text_search.setup(
|
||||
PlaintextToEntries,
|
||||
files.get("plaintext"),
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup plaintext: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
try:
|
||||
if no_documents:
|
||||
github_config = GithubConfig.objects.filter(user=user).prefetch_related("githubrepoconfig").first()
|
||||
if (
|
||||
search_type == state.SearchType.All.value or search_type == state.SearchType.Github.value
|
||||
) and github_config is not None:
|
||||
logger.info("🐙 Setting up search for github")
|
||||
# Extract Entries, Generate Github Embeddings
|
||||
text_search.setup(
|
||||
GithubToEntries,
|
||||
None,
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
config=github_config,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup GitHub: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
try:
|
||||
if no_documents:
|
||||
# Initialize Notion Search
|
||||
notion_config = NotionConfig.objects.filter(user=user).first()
|
||||
if (
|
||||
search_type == state.SearchType.All.value or search_type == state.SearchType.Notion.value
|
||||
) and notion_config:
|
||||
logger.info("🔌 Setting up search for notion")
|
||||
text_search.setup(
|
||||
NotionToEntries,
|
||||
None,
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
config=notion_config,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup Notion: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
try:
|
||||
# Initialize Image Search
|
||||
if (search_type == state.SearchType.All.value or search_type == state.SearchType.Image.value) and files[
|
||||
"image"
|
||||
]:
|
||||
logger.info("🖼️ Setting up search for images")
|
||||
# Extract Entries, Generate Image Embeddings
|
||||
text_search.setup(
|
||||
ImageToEntries,
|
||||
files.get("image"),
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup images: {e}", exc_info=True)
|
||||
success = False
|
||||
try:
|
||||
if (search_type == state.SearchType.All.value or search_type == state.SearchType.Docx.value) and files["docx"]:
|
||||
logger.info("📄 Setting up search for docx")
|
||||
text_search.setup(
|
||||
DocxToEntries,
|
||||
files.get("docx"),
|
||||
regenerate=regenerate,
|
||||
full_corpus=full_corpus,
|
||||
user=user,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"🚨 Failed to setup docx: {e}", exc_info=True)
|
||||
success = False
|
||||
|
||||
# Invalidate Query Cache
|
||||
if user:
|
||||
state.query_cache[user.uuid] = LRU()
|
||||
|
||||
return success
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from starlette.responses import RedirectResponse
|
|||
|
||||
from khoj.database.adapters import aget_user_by_uuid
|
||||
from khoj.database.models import KhojUser, NotionConfig
|
||||
from khoj.routers.indexer import configure_content
|
||||
from khoj.routers.helpers import configure_content
|
||||
from khoj.utils.state import SearchType
|
||||
|
||||
NOTION_OAUTH_CLIENT_ID = os.getenv("NOTION_OAUTH_CLIENT_ID")
|
||||
|
|
@ -25,12 +25,6 @@ executor = ThreadPoolExecutor()
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_notion_auth_url(user: KhojUser):
|
||||
if not NOTION_OAUTH_CLIENT_ID or not NOTION_OAUTH_CLIENT_SECRET or not NOTION_REDIRECT_URI:
|
||||
return None
|
||||
return f"https://api.notion.com/v1/oauth/authorize?client_id={NOTION_OAUTH_CLIENT_ID}&redirect_uri={NOTION_REDIRECT_URI}&response_type=code&state={user.uuid}"
|
||||
|
||||
|
||||
async def run_in_executor(func, *args):
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(executor, func, *args)
|
||||
|
|
|
|||
|
|
@ -1,30 +1,21 @@
|
|||
# System Packages
|
||||
import json
|
||||
import os
|
||||
from datetime import timedelta
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from starlette.authentication import has_required_scope, requires
|
||||
from starlette.authentication import requires
|
||||
|
||||
from khoj.database import adapters
|
||||
from khoj.database.adapters import (
|
||||
AgentAdapters,
|
||||
ConversationAdapters,
|
||||
EntryAdapters,
|
||||
PublicConversationAdapters,
|
||||
get_user_github_config,
|
||||
get_user_name,
|
||||
get_user_notion_config,
|
||||
get_user_subscription_state,
|
||||
)
|
||||
from khoj.database.models import KhojUser
|
||||
from khoj.processor.speech.text_to_speech import is_eleven_labs_enabled
|
||||
from khoj.routers.helpers import get_next_url
|
||||
from khoj.routers.notion import get_notion_auth_url
|
||||
from khoj.routers.twilio import is_twilio_enabled
|
||||
from khoj.routers.helpers import get_next_url, get_user_config
|
||||
from khoj.utils import constants, state
|
||||
from khoj.utils.rawconfig import (
|
||||
GithubContentConfig,
|
||||
|
|
@ -42,80 +33,36 @@ templates = Jinja2Templates([constants.web_directory, constants.next_js_director
|
|||
@requires(["authenticated"], redirect="login_page")
|
||||
def index(request: Request):
|
||||
user = request.user.object
|
||||
user_picture = request.session.get("user", {}).get("picture")
|
||||
has_documents = EntryAdapters.user_has_entries(user=user)
|
||||
user_config = get_user_config(user, request)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"chat.html",
|
||||
context={
|
||||
"request": request,
|
||||
"username": user.username,
|
||||
"user_photo": user_picture,
|
||||
"is_active": has_required_scope(request, ["premium"]),
|
||||
"has_documents": has_documents,
|
||||
"khoj_version": state.khoj_version,
|
||||
},
|
||||
)
|
||||
return templates.TemplateResponse("chat.html", context=user_config)
|
||||
|
||||
|
||||
@web_client.post("/", response_class=FileResponse)
|
||||
@requires(["authenticated"], redirect="login_page")
|
||||
def index_post(request: Request):
|
||||
user = request.user.object
|
||||
user_picture = request.session.get("user", {}).get("picture")
|
||||
has_documents = EntryAdapters.user_has_entries(user=user)
|
||||
user_config = get_user_config(user, request)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"chat.html",
|
||||
context={
|
||||
"request": request,
|
||||
"username": user.username,
|
||||
"user_photo": user_picture,
|
||||
"is_active": has_required_scope(request, ["premium"]),
|
||||
"has_documents": has_documents,
|
||||
"khoj_version": state.khoj_version,
|
||||
},
|
||||
)
|
||||
return templates.TemplateResponse("chat.html", context=user_config)
|
||||
|
||||
|
||||
@web_client.get("/search", response_class=FileResponse)
|
||||
@requires(["authenticated"], redirect="login_page")
|
||||
def search_page(request: Request):
|
||||
user = request.user.object
|
||||
user_picture = request.session.get("user", {}).get("picture")
|
||||
has_documents = EntryAdapters.user_has_entries(user=user)
|
||||
user_config = get_user_config(user, request)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"search.html",
|
||||
context={
|
||||
"request": request,
|
||||
"username": user.username,
|
||||
"user_photo": user_picture,
|
||||
"is_active": has_required_scope(request, ["premium"]),
|
||||
"has_documents": has_documents,
|
||||
"khoj_version": state.khoj_version,
|
||||
},
|
||||
)
|
||||
return templates.TemplateResponse("search.html", context=user_config)
|
||||
|
||||
|
||||
@web_client.get("/chat", response_class=FileResponse)
|
||||
@requires(["authenticated"], redirect="login_page")
|
||||
def chat_page(request: Request):
|
||||
user = request.user.object
|
||||
user_picture = request.session.get("user", {}).get("picture")
|
||||
has_documents = EntryAdapters.user_has_entries(user=user)
|
||||
user_config = get_user_config(user, request)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"chat.html",
|
||||
context={
|
||||
"request": request,
|
||||
"username": user.username,
|
||||
"user_photo": user_picture,
|
||||
"is_active": has_required_scope(request, ["premium"]),
|
||||
"has_documents": has_documents,
|
||||
"khoj_version": state.khoj_version,
|
||||
},
|
||||
)
|
||||
return templates.TemplateResponse("chat.html", context=user_config)
|
||||
|
||||
|
||||
@web_client.get("/experimental", response_class=FileResponse)
|
||||
|
|
@ -169,25 +116,14 @@ def agents_page(request: Request):
|
|||
@web_client.get("/agent/{agent_slug}", response_class=HTMLResponse)
|
||||
def agent_page(request: Request, agent_slug: str):
|
||||
user: KhojUser = request.user.object if request.user.is_authenticated else None
|
||||
user_picture = request.session.get("user", {}).get("picture") if user else None
|
||||
|
||||
user_config = get_user_config(user, request)
|
||||
agent = AgentAdapters.get_agent_by_slug(agent_slug)
|
||||
has_documents = EntryAdapters.user_has_entries(user=user)
|
||||
|
||||
if agent == None:
|
||||
return templates.TemplateResponse(
|
||||
"404.html",
|
||||
context={
|
||||
"request": request,
|
||||
"khoj_version": state.khoj_version,
|
||||
"username": user.username if user else None,
|
||||
"has_documents": False,
|
||||
"is_active": has_required_scope(request, ["premium"]),
|
||||
"user_photo": user_picture,
|
||||
},
|
||||
)
|
||||
user_config["has_documents"] = False
|
||||
return templates.TemplateResponse("404.html", context=user_config)
|
||||
|
||||
agent_metadata = {
|
||||
user_config["agent"] = {
|
||||
"slug": agent.slug,
|
||||
"avatar": agent.avatar,
|
||||
"name": agent.name,
|
||||
|
|
@ -199,115 +135,23 @@ def agent_page(request: Request, agent_slug: str):
|
|||
"creator_not_self": agent.creator != user,
|
||||
}
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"agent.html",
|
||||
context={
|
||||
"request": request,
|
||||
"agent": agent_metadata,
|
||||
"khoj_version": state.khoj_version,
|
||||
"username": user.username if user else None,
|
||||
"has_documents": has_documents,
|
||||
"is_active": has_required_scope(request, ["premium"]),
|
||||
"user_photo": user_picture,
|
||||
},
|
||||
)
|
||||
return templates.TemplateResponse("agent.html", context=user_config)
|
||||
|
||||
|
||||
@web_client.get("/config", response_class=HTMLResponse)
|
||||
@requires(["authenticated"], redirect="login_page")
|
||||
def config_page(request: Request):
|
||||
user: KhojUser = request.user.object
|
||||
user_picture = request.session.get("user", {}).get("picture")
|
||||
has_documents = EntryAdapters.user_has_entries(user=user)
|
||||
user_config = get_user_config(user, request, is_detailed=True)
|
||||
|
||||
user_subscription_state = get_user_subscription_state(user.email)
|
||||
user_subscription = adapters.get_user_subscription(user.email)
|
||||
subscription_renewal_date = (
|
||||
user_subscription.renewal_date.strftime("%d %b %Y")
|
||||
if user_subscription and user_subscription.renewal_date
|
||||
else (user_subscription.created_at + timedelta(days=7)).strftime("%d %b %Y")
|
||||
)
|
||||
given_name = get_user_name(user)
|
||||
|
||||
enabled_content_source = set(EntryAdapters.get_unique_file_sources(user))
|
||||
successfully_configured = {
|
||||
"computer": ("computer" in enabled_content_source),
|
||||
"github": ("github" in enabled_content_source),
|
||||
"notion": ("notion" in enabled_content_source),
|
||||
}
|
||||
|
||||
selected_conversation_config = ConversationAdapters.get_conversation_config(user)
|
||||
conversation_options = ConversationAdapters.get_conversation_processor_options().all()
|
||||
all_conversation_options = list()
|
||||
for conversation_option in conversation_options:
|
||||
all_conversation_options.append({"chat_model": conversation_option.chat_model, "id": conversation_option.id})
|
||||
|
||||
search_model_options = adapters.get_or_create_search_models().all()
|
||||
all_search_model_options = list()
|
||||
for search_model_option in search_model_options:
|
||||
all_search_model_options.append({"name": search_model_option.name, "id": search_model_option.id})
|
||||
|
||||
current_search_model_option = adapters.get_user_search_model_or_default(user)
|
||||
|
||||
selected_paint_model_config = ConversationAdapters.get_user_text_to_image_model_config(user)
|
||||
paint_model_options = ConversationAdapters.get_text_to_image_model_options().all()
|
||||
all_paint_model_options = list()
|
||||
for paint_model in paint_model_options:
|
||||
all_paint_model_options.append({"model_name": paint_model.model_name, "id": paint_model.id})
|
||||
|
||||
notion_oauth_url = get_notion_auth_url(user)
|
||||
|
||||
eleven_labs_enabled = is_eleven_labs_enabled()
|
||||
|
||||
voice_models = ConversationAdapters.get_voice_model_options()
|
||||
voice_model_options = list()
|
||||
for voice_model in voice_models:
|
||||
voice_model_options.append({"name": voice_model.name, "id": voice_model.model_id})
|
||||
|
||||
if len(voice_model_options) == 0:
|
||||
eleven_labs_enabled = False
|
||||
|
||||
selected_voice_config = ConversationAdapters.get_voice_model_config(user)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"config.html",
|
||||
context={
|
||||
"request": request,
|
||||
"current_model_state": successfully_configured,
|
||||
"anonymous_mode": state.anonymous_mode,
|
||||
"username": user.username,
|
||||
"given_name": given_name,
|
||||
"search_model_options": all_search_model_options,
|
||||
"selected_search_model_config": current_search_model_option.id,
|
||||
"conversation_options": all_conversation_options,
|
||||
"selected_conversation_config": selected_conversation_config.id if selected_conversation_config else None,
|
||||
"paint_model_options": all_paint_model_options,
|
||||
"selected_paint_model_config": selected_paint_model_config.id if selected_paint_model_config else None,
|
||||
"user_photo": user_picture,
|
||||
"billing_enabled": state.billing_enabled,
|
||||
"subscription_state": user_subscription_state,
|
||||
"subscription_renewal_date": subscription_renewal_date,
|
||||
"khoj_cloud_subscription_url": os.getenv("KHOJ_CLOUD_SUBSCRIPTION_URL"),
|
||||
"is_active": has_required_scope(request, ["premium"]),
|
||||
"has_documents": has_documents,
|
||||
"is_twilio_enabled": is_twilio_enabled(),
|
||||
"is_eleven_labs_enabled": eleven_labs_enabled,
|
||||
"voice_model_options": voice_model_options,
|
||||
"selected_voice_config": selected_voice_config.model_id if selected_voice_config else None,
|
||||
"phone_number": user.phone_number,
|
||||
"is_phone_number_verified": user.verified_phone_number,
|
||||
"khoj_version": state.khoj_version,
|
||||
"notion_oauth_url": notion_oauth_url,
|
||||
},
|
||||
)
|
||||
return templates.TemplateResponse("config.html", context=user_config)
|
||||
|
||||
|
||||
@web_client.get("/config/content-source/github", response_class=HTMLResponse)
|
||||
@requires(["authenticated"], redirect="login_page")
|
||||
def github_config_page(request: Request):
|
||||
user = request.user.object
|
||||
user_picture = request.session.get("user", {}).get("picture")
|
||||
has_documents = EntryAdapters.user_has_entries(user=user)
|
||||
user_config = get_user_config(user, request)
|
||||
current_github_config = get_user_github_config(user)
|
||||
|
||||
if current_github_config:
|
||||
|
|
@ -329,66 +173,32 @@ def github_config_page(request: Request):
|
|||
else:
|
||||
current_config = {} # type: ignore
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"content_source_github_input.html",
|
||||
context={
|
||||
"request": request,
|
||||
"current_config": current_config,
|
||||
"username": user.username,
|
||||
"user_photo": user_picture,
|
||||
"is_active": has_required_scope(request, ["premium"]),
|
||||
"has_documents": has_documents,
|
||||
"khoj_version": state.khoj_version,
|
||||
},
|
||||
)
|
||||
user_config["current_config"] = current_config
|
||||
return templates.TemplateResponse("content_source_github_input.html", context=user_config)
|
||||
|
||||
|
||||
@web_client.get("/config/content-source/notion", response_class=HTMLResponse)
|
||||
@requires(["authenticated"], redirect="login_page")
|
||||
def notion_config_page(request: Request):
|
||||
user = request.user.object
|
||||
user_picture = request.session.get("user", {}).get("picture")
|
||||
has_documents = EntryAdapters.user_has_entries(user=user)
|
||||
user_config = get_user_config(user, request)
|
||||
|
||||
current_notion_config = get_user_notion_config(user)
|
||||
|
||||
current_config = NotionContentConfig(
|
||||
token=current_notion_config.token if current_notion_config else "",
|
||||
)
|
||||
|
||||
token = current_notion_config.token if current_notion_config else ""
|
||||
current_config = NotionContentConfig(token=token)
|
||||
current_config = json.loads(current_config.model_dump_json())
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"content_source_notion_input.html",
|
||||
context={
|
||||
"request": request,
|
||||
"current_config": current_config,
|
||||
"username": user.username,
|
||||
"user_photo": user_picture,
|
||||
"is_active": has_required_scope(request, ["premium"]),
|
||||
"has_documents": has_documents,
|
||||
"khoj_version": state.khoj_version,
|
||||
},
|
||||
)
|
||||
user_config["current_config"] = current_config
|
||||
return templates.TemplateResponse("content_source_notion_input.html", context=user_config)
|
||||
|
||||
|
||||
@web_client.get("/config/content-source/computer", response_class=HTMLResponse)
|
||||
@requires(["authenticated"], redirect="login_page")
|
||||
def computer_config_page(request: Request):
|
||||
user = request.user.object if request.user.is_authenticated else None
|
||||
user_picture = request.session.get("user", {}).get("picture") if user else None
|
||||
has_documents = EntryAdapters.user_has_entries(user=user) if user else False
|
||||
user_config = get_user_config(user, request)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"content_source_computer_input.html",
|
||||
context={
|
||||
"request": request,
|
||||
"username": user.username,
|
||||
"user_photo": user_picture,
|
||||
"is_active": has_required_scope(request, ["premium"]),
|
||||
"has_documents": has_documents,
|
||||
"khoj_version": state.khoj_version,
|
||||
},
|
||||
)
|
||||
return templates.TemplateResponse("content_source_computer_input.html", context=user_config)
|
||||
|
||||
|
||||
@web_client.get("/share/chat/{public_conversation_slug}", response_class=HTMLResponse)
|
||||
|
|
@ -404,8 +214,9 @@ def view_public_conversation(request: Request):
|
|||
},
|
||||
)
|
||||
user = request.user.object if request.user.is_authenticated else None
|
||||
user_picture = request.session.get("user", {}).get("picture") if user else None
|
||||
has_documents = EntryAdapters.user_has_entries(user=user) if user else False
|
||||
user_config = get_user_config(user, request)
|
||||
user_config["public_conversation_slug"] = public_conversation_slug
|
||||
user_config["google_client_id"] = os.environ.get("GOOGLE_CLIENT_ID")
|
||||
|
||||
all_agents = AgentAdapters.get_all_accessible_agents(request.user.object if request.user.is_authenticated else None)
|
||||
|
||||
|
|
@ -420,28 +231,15 @@ def view_public_conversation(request: Request):
|
|||
"name": agent.name,
|
||||
}
|
||||
)
|
||||
user_config["agents"] = agents_packet
|
||||
|
||||
google_client_id = os.environ.get("GOOGLE_CLIENT_ID")
|
||||
redirect_uri = str(request.app.url_path_for("auth"))
|
||||
next_url = str(
|
||||
request.app.url_path_for("view_public_conversation", public_conversation_slug=public_conversation_slug)
|
||||
)
|
||||
user_config["redirect_uri"] = f"{redirect_uri}?next={next_url}"
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"public_conversation.html",
|
||||
context={
|
||||
"request": request,
|
||||
"username": user.username if user else None,
|
||||
"user_photo": user_picture,
|
||||
"is_active": has_required_scope(request, ["premium"]),
|
||||
"has_documents": has_documents,
|
||||
"khoj_version": state.khoj_version,
|
||||
"public_conversation_slug": public_conversation_slug,
|
||||
"agents": agents_packet,
|
||||
"google_client_id": google_client_id,
|
||||
"redirect_uri": f"{redirect_uri}?next={next_url}",
|
||||
},
|
||||
)
|
||||
return templates.TemplateResponse("public_conversation.html", context=user_config)
|
||||
|
||||
|
||||
@web_client.get("/automations", response_class=HTMLResponse)
|
||||
|
|
@ -452,20 +250,9 @@ def automations_config_page(
|
|||
queryToRun: Optional[str] = None,
|
||||
):
|
||||
user = request.user.object if request.user.is_authenticated else None
|
||||
user_picture = request.session.get("user", {}).get("picture")
|
||||
has_documents = EntryAdapters.user_has_entries(user=user) if user else False
|
||||
user_config = get_user_config(user, request)
|
||||
user_config["subject"] = subject if subject else ""
|
||||
user_config["crontime"] = crontime if crontime else ""
|
||||
user_config["queryToRun"] = queryToRun if queryToRun else ""
|
||||
|
||||
return templates.TemplateResponse(
|
||||
"config_automation.html",
|
||||
context={
|
||||
"request": request,
|
||||
"username": user.username if user else None,
|
||||
"user_photo": user_picture,
|
||||
"is_active": has_required_scope(request, ["premium"]),
|
||||
"has_documents": has_documents,
|
||||
"khoj_version": state.khoj_version,
|
||||
"subject": subject if subject else "",
|
||||
"crontime": crontime if crontime else "",
|
||||
"queryToRun": queryToRun if queryToRun else "",
|
||||
},
|
||||
)
|
||||
return templates.TemplateResponse("config_automation.html", context=user_config)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue