From d1bfb245dff959580bed347c0ad563c2a6b5af71 Mon Sep 17 00:00:00 2001 From: Debanjum Date: Mon, 29 Jan 2024 18:03:43 +0530 Subject: [PATCH] Improve Khoj Chat and Settings UI (#630) * Fix license in pyproject.toml. Remove unused utils.state import * Use single debug mode check function. Disable telemetry in debug mode - Use single logic to check if khoj is running in debug mode. Previously there were 3 different variants of the check - Do not log telemetry if KHOJ_DEBUG is set to true. Previously didn't log telemetry even if KHOJ_DEBUG set to false * Respect line breaks in user, khoj chat messages to improve formatting * Disable Whatsapp config section on web client if Twilio not configured Simplify Whatsapp configuration status checking js by standardizing external input to lower case * Disable Phone API when Twilio not setup and rate limit calls to it - Move phone api to separate router and only enable it if Twilio enabled - Add rate-limiting to OTP and verification calls * Add slugs for phone rate limiting --------- Co-authored-by: sabaimran --- pyproject.toml | 2 +- src/interface/desktop/chat.html | 1 + src/khoj/app/settings.py | 4 +- src/khoj/configure.py | 17 ++++-- src/khoj/interface/web/chat.html | 1 + src/khoj/interface/web/config.html | 20 ++++--- src/khoj/main.py | 4 +- src/khoj/routers/api_config.py | 72 ------------------------- src/khoj/routers/api_phone.py | 86 ++++++++++++++++++++++++++++++ src/khoj/routers/helpers.py | 5 ++ src/khoj/utils/cli.py | 4 +- src/khoj/utils/helpers.py | 6 +++ src/khoj/utils/models.py | 2 - 13 files changed, 134 insertions(+), 90 deletions(-) create mode 100644 src/khoj/routers/api_phone.py diff --git a/pyproject.toml b/pyproject.toml index 9924dc0e..bfe8cc2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" name = "khoj-assistant" description = "An AI copilot for your Second Brain" readme = "README.md" -license = "GPL-3.0-or-later" +license = "AGPL-3.0-or-later" requires-python = ">=3.8" authors = [ { name = "Debanjum Singh Solanky, Saba Imran" }, diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html index 11ccc466..df6fadab 100644 --- a/src/interface/desktop/chat.html +++ b/src/interface/desktop/chat.html @@ -892,6 +892,7 @@ display: inline-block; max-width: 80%; text-align: left; + white-space: pre-line; } /* color chat bubble by khoj blue */ .chat-message-text.khoj { diff --git a/src/khoj/app/settings.py b/src/khoj/app/settings.py index c0d2d8a8..ce6b4eca 100644 --- a/src/khoj/app/settings.py +++ b/src/khoj/app/settings.py @@ -13,6 +13,8 @@ https://docs.djangoproject.com/en/4.2/ref/settings/ import os from pathlib import Path +from khoj.utils.helpers import in_debug_mode + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -24,7 +26,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent SECRET_KEY = os.getenv("KHOJ_DJANGO_SECRET_KEY", "!secret") # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = os.getenv("KHOJ_DEBUG") == "True" +DEBUG = in_debug_mode() # All Subdomains of KHOJ_DOMAIN are trusted KHOJ_DOMAIN = os.getenv("KHOJ_DOMAIN", "khoj.dev") diff --git a/src/khoj/configure.py b/src/khoj/configure.py index 5d82fa4d..620e83cc 100644 --- a/src/khoj/configure.py +++ b/src/khoj/configure.py @@ -34,6 +34,7 @@ from khoj.database.adapters import ( from khoj.database.models import ClientApplication, KhojUser, Subscription from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel from khoj.routers.indexer import configure_content, configure_search, load_content +from khoj.routers.twilio import is_twilio_enabled from khoj.utils import constants, state from khoj.utils.config import SearchType from khoj.utils.fs_syncer import collect_files @@ -258,18 +259,26 @@ def configure_routes(app): from khoj.routers.api_config import api_config from khoj.routers.auth import auth_router from khoj.routers.indexer import indexer - from khoj.routers.subscription import subscription_router from khoj.routers.web_client import web_client app.include_router(api, prefix="/api") app.include_router(api_config, prefix="/api/config") app.include_router(indexer, prefix="/api/v1/index") - if state.billing_enabled: - logger.info("💳 Enabled Billing") - app.include_router(subscription_router, prefix="/api/subscription") app.include_router(web_client) app.include_router(auth_router, prefix="/auth") + if state.billing_enabled: + from khoj.routers.subscription import subscription_router + + logger.info("💳 Enabled Billing") + app.include_router(subscription_router, prefix="/api/subscription") + + if is_twilio_enabled(): + logger.info("📞 Enabled Twilio") + from khoj.routers.api_phone import api_phone + + app.include_router(api_phone, prefix="/api/config/phone") + def configure_middleware(app): app.add_middleware(AuthenticationMiddleware, backend=UserAuthenticationBackend()) diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index 462ae05d..09659453 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -965,6 +965,7 @@ To get started, just start typing below. You can also type / to see a list of co display: inline-block; max-width: 80%; text-align: left; + white-space: pre-line; } /* color chat bubble by khoj blue */ .chat-message-text.khoj { diff --git a/src/khoj/interface/web/config.html b/src/khoj/interface/web/config.html index a045d58b..ab8de4f3 100644 --- a/src/khoj/interface/web/config.html +++ b/src/khoj/interface/web/config.html @@ -194,7 +194,9 @@ -
+
+
+
WhatsApp icon

WhatsApp

@@ -610,15 +612,21 @@ const phonenumberVerifiedText = document.getElementById("api-settings-card-description-verified"); const phonenumberUnverifiedText = document.getElementById("api-settings-card-description-unverified"); - let preExistingPhoneNumber = "{{ phone_number }}"; + const preExistingPhoneNumber = "{{ phone_number }}"; let isPhoneNumberVerified = "{{ is_phone_number_verified }}"; let isTwilioEnabled = "{{ is_twilio_enabled }}"; + isPhoneNumberVerified = isPhoneNumberVerified.toLowerCase(); + isTwilioEnabled = isTwilioEnabled.toLowerCase(); - if (preExistingPhoneNumber != "None" && (isPhoneNumberVerified == "True" || isPhoneNumberVerified == "true")) { + if (isTwilioEnabled !== "true" ) { + const phoneNumberVerificationCard = document.getElementById("phone-number-input-card"); + phoneNumberVerificationCard.style.display = "none"; + } + if (preExistingPhoneNumber != "None" && isPhoneNumberVerified === "true") { phonenumberVerifyButton.style.display = "none"; phonenumberRemoveButton.style.display = ""; - } else if (preExistingPhoneNumber != "None" && (isPhoneNumberVerified == "False" || isPhoneNumberVerified == "false")) { - if (isTwilioEnabled == "True" || isTwilioEnabled == "true") { + } else if (preExistingPhoneNumber != "None" && isPhoneNumberVerified === "false") { + if (isTwilioEnabled == "true") { phonenumberVerifyButton.style.display = ""; phonenumberRemoveButton.style.display = "none"; } else { @@ -759,7 +767,5 @@ }, 5000); }); }) - - {% endblock %} diff --git a/src/khoj/main.py b/src/khoj/main.py index b5421979..72036695 100644 --- a/src/khoj/main.py +++ b/src/khoj/main.py @@ -14,6 +14,8 @@ import threading import warnings from importlib.metadata import version +from khoj.utils.helpers import in_debug_mode + # Ignore non-actionable warnings warnings.filterwarnings("ignore", message=r"snapshot_download.py has been made private", category=FutureWarning) warnings.filterwarnings("ignore", message=r"legacy way to download files from the HF hub,", category=FutureWarning) @@ -45,7 +47,7 @@ with redirect_stdout(collectstatic_output): call_command("collectstatic", "--noinput") # Initialize the Application Server -if os.getenv("KHOJ_DEBUG", "false").lower() == "true": +if in_debug_mode(): app = FastAPI(debug=True) else: app = FastAPI(docs_url=None) # Disable Swagger UI in production diff --git a/src/khoj/routers/api_config.py b/src/khoj/routers/api_config.py index 12f53abb..26729f41 100644 --- a/src/khoj/routers/api_config.py +++ b/src/khoj/routers/api_config.py @@ -22,7 +22,6 @@ from khoj.database.models import ( NotionConfig, ) from khoj.routers.helpers import CommonQueryParams, update_telemetry_state -from khoj.routers.twilio import create_otp, is_twilio_enabled, verify_otp from khoj.utils import constants, state from khoj.utils.rawconfig import ( FullConfig, @@ -279,77 +278,6 @@ async def update_search_model( return {"status": "ok"} -@api_config.post("/phone", status_code=200) -@requires(["authenticated"]) -async def update_phone_number( - request: Request, - phone_number: str, - client: Optional[str] = None, -): - user = request.user.object - - await adapters.aset_user_phone_number(user, phone_number) - - if is_twilio_enabled(): - create_otp(user) - else: - logger.warning("Phone verification is not enabled") - - update_telemetry_state( - request=request, - telemetry_type="api", - api="set_phone_number", - client=client, - metadata={"phone_number": phone_number}, - ) - - return {"status": "ok"} - - -@api_config.delete("/phone", status_code=200) -@requires(["authenticated"]) -async def delete_phone_number( - request: Request, - client: Optional[str] = None, -): - user = request.user.object - - await adapters.aremove_phone_number(user) - - update_telemetry_state( - request=request, - telemetry_type="api", - api="delete_phone_number", - client=client, - ) - - return {"status": "ok"} - - -@api_config.post("/phone/verify", status_code=200) -@requires(["authenticated"]) -async def verify_mobile_otp( - request: Request, - code: str, - client: Optional[str] = None, -): - user: KhojUser = request.user.object - - update_telemetry_state( - request=request, - telemetry_type="api", - api="verify_phone_number", - client=client, - ) - - if is_twilio_enabled() and not verify_otp(user, code): - raise HTTPException(status_code=400, detail="Invalid OTP") - - user.verified_phone_number = True - await user.asave() - return {"status": "ok"} - - @api_config.get("/index/size", response_model=Dict[str, int]) @requires(["authenticated"]) async def get_indexed_data_size(request: Request, common: CommonQueryParams): diff --git a/src/khoj/routers/api_phone.py b/src/khoj/routers/api_phone.py new file mode 100644 index 00000000..84c9a5d2 --- /dev/null +++ b/src/khoj/routers/api_phone.py @@ -0,0 +1,86 @@ +import logging +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException, Request +from starlette.authentication import requires + +from khoj.database import adapters +from khoj.database.models import KhojUser +from khoj.routers.helpers import ApiUserRateLimiter, update_telemetry_state +from khoj.routers.twilio import create_otp, verify_otp + +api_phone = APIRouter() +logger = logging.getLogger(__name__) + + +@api_phone.post("", status_code=200) +@requires(["authenticated"]) +async def update_phone_number( + request: Request, + phone_number: str, + client: Optional[str] = None, + rate_limiter_per_day=Depends( + ApiUserRateLimiter(requests=5, subscribed_requests=5, window=60 * 60 * 24, slug="update_phone") + ), +): + user = request.user.object + + await adapters.aset_user_phone_number(user, phone_number) + create_otp(user) + + update_telemetry_state( + request=request, + telemetry_type="api", + api="set_phone_number", + client=client, + metadata={"phone_number": phone_number}, + ) + + return {"status": "ok"} + + +@api_phone.delete("", status_code=200) +@requires(["authenticated"]) +async def delete_phone_number( + request: Request, + client: Optional[str] = None, +): + user = request.user.object + + await adapters.aremove_phone_number(user) + + update_telemetry_state( + request=request, + telemetry_type="api", + api="delete_phone_number", + client=client, + ) + + return {"status": "ok"} + + +@api_phone.post("/verify", status_code=200) +@requires(["authenticated"]) +async def verify_mobile_otp( + request: Request, + code: str, + client: Optional[str] = None, + rate_limiter_per_day=Depends( + ApiUserRateLimiter(requests=5, subscribed_requests=5, window=60 * 60 * 24, slug="verify_phone") + ), +): + user: KhojUser = request.user.object + + update_telemetry_state( + request=request, + telemetry_type="api", + api="verify_phone_number", + client=client, + ) + + if not verify_otp(user, code): + raise HTTPException(status_code=400, detail="Invalid OTP") + + user.verified_phone_number = True + await user.asave() + return {"status": "ok"} diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py index b0d8c348..6bfdc9b9 100644 --- a/src/khoj/routers/helpers.py +++ b/src/khoj/routers/helpers.py @@ -360,6 +360,11 @@ class ApiUserRateLimiter: if subscribed and count_requests >= self.subscribed_requests: raise HTTPException(status_code=429, detail="Slow down! Too Many Requests") if not subscribed and count_requests >= self.requests: + if self.subscribed_requests == self.requests: + raise HTTPException( + status_code=429, + detail="Slow down! Too Many Requests", + ) raise HTTPException( status_code=429, detail="We're glad you're enjoying Khoj! You've exceeded your usage limit for today. Come back tomorrow or subscribe to increase your rate limit via [your settings](https://app.khoj.dev/config).", diff --git a/src/khoj/utils/cli.py b/src/khoj/utils/cli.py index 968232e6..efbb596e 100644 --- a/src/khoj/utils/cli.py +++ b/src/khoj/utils/cli.py @@ -16,7 +16,7 @@ from khoj.migrations.migrate_processor_config_openai import ( ) from khoj.migrations.migrate_server_pg import migrate_server_pg from khoj.migrations.migrate_version import migrate_config_to_version -from khoj.utils.helpers import resolve_absolute_path +from khoj.utils.helpers import in_debug_mode, resolve_absolute_path from khoj.utils.yaml import parse_config_from_file @@ -73,7 +73,7 @@ def cli(args=None): else: args = run_migrations(args) args.config = parse_config_from_file(args.config_file) - if os.environ.get("KHOJ_DEBUG"): + if in_debug_mode(): args.config.app.should_log_telemetry = False return args diff --git a/src/khoj/utils/helpers.py b/src/khoj/utils/helpers.py index df2a3cd0..503ede14 100644 --- a/src/khoj/utils/helpers.py +++ b/src/khoj/utils/helpers.py @@ -317,3 +317,9 @@ def batcher(iterable, max_n): if not chunk: return yield (x for x in chunk if x is not None) + + +def in_debug_mode(): + """Check if Khoj is running in debug mode. + Set KHOJ_DEBUG environment variable to true to enable debug mode.""" + return os.getenv("KHOJ_DEBUG", "false").lower() == "true" diff --git a/src/khoj/utils/models.py b/src/khoj/utils/models.py index b76d369a..f848d4e4 100644 --- a/src/khoj/utils/models.py +++ b/src/khoj/utils/models.py @@ -5,8 +5,6 @@ import openai import torch from tqdm import trange -from khoj.utils import state - class BaseEncoder(ABC): @abstractmethod