diff --git a/documentation/docs/clients/whatsapp.md b/documentation/docs/clients/whatsapp.md new file mode 100644 index 00000000..fc0d38e1 --- /dev/null +++ b/documentation/docs/clients/whatsapp.md @@ -0,0 +1,28 @@ +--- +sidebar_position: 5 +--- + +# WhatsApp + +> Query your Second Brain from WhatsApp + +Text [+1 (848) 800 4242](https://wa.me/18488004242) or scan [this QR code](https://khoj.dev/whatsapp) on your phone to chat with Khoj on WhatsApp. + +Without any desktop clients, you can start chatting with Khoj on WhatsApp. Bear in mind you do need one of the desktop clients in order to share and sync your data with Khoj. The WhatsApp AI bot will work right away for answering generic queries and using Khoj in default mode. + +In order to use Khoj on WhatsApp with your own data, you need to setup a Khoj Cloud account and connect your WhatsApp account to it. This is a one time setup and you can do it from the [Khoj Cloud config page](https://app.khoj.dev/config). + +If you hit usage limits for the WhatsApp bot, upgrade to [a paid plan](https://khoj.dev/pricing) on Khoj Cloud. + +## Features + +- **Slash Commands**: Use slash commands to quickly access Khoj features + - `/online`: Get responses from Khoj powered by online search. + - `/dream`: Generate an image in response to your prompt. + - `/notes`: Explicitly force Khoj to retrieve context from your notes. Note: You'll need to connect your WhatsApp account to a Khoj Cloud account for this to work. + +We have more commands under development, including `/share` to uploading documents directly to your Khoj account from WhatsApp, and `/speak` in order to get a speech response from Khoj. Feel free to [raise an issue](https://github.com/khoj-ai/flint/issues) if you have any suggestions for new commands. + +## Nerdy Details + +You can find all of the code for the WhatsApp bot in the the [flint repository](https://github.com/khoj-ai/flint). As all of our code, it is open source and you can contribute to it. diff --git a/pyproject.toml b/pyproject.toml index 7f969f3a..4b0c4140 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ dependencies = [ "openai-whisper >= 20231117", "django-phonenumber-field == 7.3.0", "phonenumbers == 8.13.27", + "twilio == 8.11" ] dynamic = ["version"] diff --git a/src/khoj/database/adapters/__init__.py b/src/khoj/database/adapters/__init__.py index 1b5e4d1b..8089ee40 100644 --- a/src/khoj/database/adapters/__init__.py +++ b/src/khoj/database/adapters/__init__.py @@ -95,6 +95,36 @@ async def aget_or_create_user_by_phone_number(phone_number: str) -> KhojUser: return user +async def aset_user_phone_number(user: KhojUser, phone_number: str) -> KhojUser: + if is_none_or_empty(phone_number): + return None + phone_number = phone_number.strip() + if not phone_number.startswith("+"): + phone_number = f"+{phone_number}" + existing_user_with_phone_number = await aget_user_by_phone_number(phone_number) + if existing_user_with_phone_number and existing_user_with_phone_number.id != user.id: + if is_none_or_empty(existing_user_with_phone_number.email): + # Transfer conversation history to the new user. If they don't have an associated email, they are effectively a new user + async for conversation in Conversation.objects.filter(user=existing_user_with_phone_number).aiterator(): + conversation.user = user + await conversation.asave() + + await existing_user_with_phone_number.adelete() + else: + raise HTTPException(status_code=400, detail="Phone number already exists") + + user.phone_number = phone_number + await user.asave() + return user + + +async def aremove_phone_number(user: KhojUser) -> KhojUser: + user.phone_number = None + user.verified_phone_number = False + await user.asave() + return user + + async def acreate_user_by_phone_number(phone_number: str) -> KhojUser: if is_none_or_empty(phone_number): return None @@ -213,7 +243,20 @@ async def get_user_by_token(token: dict) -> KhojUser: async def aget_user_by_phone_number(phone_number: str) -> KhojUser: if is_none_or_empty(phone_number): return None - return await KhojUser.objects.filter(phone_number=phone_number).prefetch_related("subscription").afirst() + matched_user = await KhojUser.objects.filter(phone_number=phone_number).prefetch_related("subscription").afirst() + + if not matched_user: + return None + + # If the user with this phone number does not have an email account with Khoj, return the user + if is_none_or_empty(matched_user.email): + return matched_user + + # If the user has an email account with Khoj and a verified number, return the user + if matched_user.verified_phone_number: + return matched_user + + return None async def retrieve_user(session_id: str) -> KhojUser: @@ -307,11 +350,11 @@ class ClientApplicationAdapters: class ConversationAdapters: @staticmethod - def get_conversation_by_user(user: KhojUser): - conversation = Conversation.objects.filter(user=user) + def get_conversation_by_user(user: KhojUser, client_application: ClientApplication = None): + conversation = Conversation.objects.filter(user=user, client=client_application) if conversation.exists(): return conversation.first() - return Conversation.objects.create(user=user) + return Conversation.objects.create(user=user, client=client_application) @staticmethod async def aget_conversation_by_user(user: KhojUser, client_application: ClientApplication = None): @@ -383,12 +426,12 @@ class ConversationAdapters: return await ChatModelOptions.objects.filter().afirst() @staticmethod - def save_conversation(user: KhojUser, conversation_log: dict): - conversation = Conversation.objects.filter(user=user) + def save_conversation(user: KhojUser, conversation_log: dict, client_application: ClientApplication = None): + conversation = Conversation.objects.filter(user=user, client=client_application) if conversation.exists(): conversation.update(conversation_log=conversation_log) else: - Conversation.objects.create(user=user, conversation_log=conversation_log) + Conversation.objects.create(user=user, conversation_log=conversation_log, client=client_application) @staticmethod def get_conversation_processor_options(): diff --git a/src/khoj/database/admin.py b/src/khoj/database/admin.py index 521e81de..9b07efde 100644 --- a/src/khoj/database/admin.py +++ b/src/khoj/database/admin.py @@ -58,6 +58,7 @@ class ConversationAdmin(admin.ModelAdmin): "user", "created_at", "updated_at", + "client", ) search_fields = ("conversation_id",) ordering = ("-created_at",) diff --git a/src/khoj/database/migrations/0028_khojuser_verified_phone_number.py b/src/khoj/database/migrations/0028_khojuser_verified_phone_number.py new file mode 100644 index 00000000..88b4b140 --- /dev/null +++ b/src/khoj/database/migrations/0028_khojuser_verified_phone_number.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.7 on 2024-01-19 13:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("database", "0027_merge_20240118_1324"), + ] + + operations = [ + migrations.AddField( + model_name="khojuser", + name="verified_phone_number", + field=models.BooleanField(default=False), + ), + ] diff --git a/src/khoj/database/models/__init__.py b/src/khoj/database/models/__init__.py index 93f4f2ac..688ca9af 100644 --- a/src/khoj/database/models/__init__.py +++ b/src/khoj/database/models/__init__.py @@ -26,6 +26,7 @@ class ClientApplication(BaseModel): class KhojUser(AbstractUser): uuid = models.UUIDField(models.UUIDField(default=uuid.uuid4, editable=False)) phone_number = PhoneNumberField(null=True, default=None, blank=True) + verified_phone_number = models.BooleanField(default=False) def save(self, *args, **kwargs): if not self.uuid: diff --git a/src/khoj/interface/web/assets/icons/whatsapp.svg b/src/khoj/interface/web/assets/icons/whatsapp.svg new file mode 100644 index 00000000..28b171c0 --- /dev/null +++ b/src/khoj/interface/web/assets/icons/whatsapp.svg @@ -0,0 +1,17 @@ + + + + + Whatsapp-color + Created with Sketch. + + + + + + + + + + + diff --git a/src/khoj/interface/web/base_config.html b/src/khoj/interface/web/base_config.html index b3a5a3a3..285936b6 100644 --- a/src/khoj/interface/web/base_config.html +++ b/src/khoj/interface/web/base_config.html @@ -7,6 +7,11 @@ Khoj - Settings + + @@ -330,6 +335,29 @@ text-decoration: none; } + div#phone-number-input-element { + display: flex; + align-items: center; + } + + p#phone-number-plus { + padding: 8px; + } + + div#clients { + grid-gap: 12px; + } + + input#country-code-phone-number-input { + max-width: 100px; + margin-right: 8px; + } + + input#country-code-phone-number-input { + max-width: 100px; + margin-right: 8px; + } + @media screen and (max-width: 700px) { .section-cards { grid-template-columns: 1fr; diff --git a/src/khoj/interface/web/config.html b/src/khoj/interface/web/config.html index 4d85e254..ae20c800 100644 --- a/src/khoj/interface/web/config.html +++ b/src/khoj/interface/web/config.html @@ -194,6 +194,37 @@ +
+
+ WhatsApp icon +

WhatsApp

+
+
+

Your number is connected. You can now chat with Khoj on WhatsApp at +1-848-800-4242. Learn more about the integration here.

+

Connect your number to chat with Khoj on WhatsApp. Learn more about the integration here.

+
+
+ {% if phone_number %} + + {% else %} + + {% endif %} +
+ +
+ + + + +
+
{% if billing_enabled %}
@@ -555,5 +586,181 @@ } }) } + + var phoneInputField = document.querySelector("#mobile_code"); + const iti = window.intlTelInput(phoneInputField, { + initialCountry: "auto", + geoIpLookup: callback => { + fetch("https://ipapi.co/json") + .then(res => res.json()) + .then(data => callback(data.country_code)) + .catch(() => callback("us")) + }, + separateDialCode: true, + utilsScript: "https://cdn.jsdelivr.net/npm/intl-tel-input@18.2.1/build/js/utils.js", + }); + + const errorMap = ["Invalid number", "Invalid country code", "Too short", "Too long", "Invalid number"]; + const phoneNumberUpdateCallback = document.getElementById("phone-number-update-callback"); + + const phonenumberVerifyButton = document.getElementById("whatsapp-verify"); + const phonenumberRemoveButton = document.getElementById("whatsapp-remove"); + const phonenumberVerifyOTPButton = document.getElementById("whatsapp-verify-otp"); + const phonenumberOTPInput = document.getElementById("whatsapp_otp"); + const phonenumberVerifiedText = document.getElementById("api-settings-card-description-verified"); + const phonenumberUnverifiedText = document.getElementById("api-settings-card-description-unverified"); + + let preExistingPhoneNumber = "{{ phone_number }}"; + let isPhoneNumberVerified = "{{ is_phone_number_verified }}"; + let isTwilioEnabled = "{{ is_twilio_enabled }}"; + + if (preExistingPhoneNumber != "None" && (isPhoneNumberVerified == "True" || isPhoneNumberVerified == "true")) { + phonenumberVerifyButton.style.display = "none"; + phonenumberRemoveButton.style.display = ""; + } else if (preExistingPhoneNumber != "None" && (isPhoneNumberVerified == "False" || isPhoneNumberVerified == "false")) { + if (isTwilioEnabled == "True" || isTwilioEnabled == "true") { + phonenumberVerifyButton.style.display = ""; + phonenumberRemoveButton.style.display = "none"; + } else { + phonenumberVerifyButton.style.display = "none"; + phonenumberRemoveButton.style.display = "none"; + } + } + else { + phonenumberVerifyButton.style.display = ""; + phonenumberRemoveButton.style.display = "none"; + } + + phoneInputField.addEventListener("keyup", () => { + if (iti.isValidNumber() == false) { + phonenumberVerifyButton.style.display = "none"; + phonenumberRemoveButton.style.display = "none"; + } else { + phonenumberVerifyButton.style.display = ""; + phonenumberRemoveButton.style.display = "none"; + } + }) + + phonenumberRemoveButton.addEventListener("click", () => { + fetch('/api/config/phone', { + method: 'DELETE', + headers: { + 'Content-Type': 'application/json', + } + }) + .then(response => response.json()) + .then(data => { + if (data.status == "ok") { + phonenumberVerifyButton.style.display = ""; + phonenumberRemoveButton.style.display = "none"; + phonenumberVerifiedText.style.display = "none"; + phonenumberUnverifiedText.style.display = "block"; + } + }) + }) + + phonenumberVerifyButton.addEventListener("click", () => { + console.log(iti.getValidationError()); + if (iti.isValidNumber() == false) { + phoneNumberUpdateCallback.innerHTML = "Invalid phone number: " + errorMap[iti.getValidationError()]; + phoneNumberUpdateCallback.style.display = "block"; + setTimeout(function() { + phoneNumberUpdateCallback.style.display = "none"; + }, 5000); + } else { + alert("Valid number"); + const mobileNumber = iti.getNumber(); + fetch('/api/config/phone?phone_number=' + mobileNumber, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + } + }) + .then(response => response.json()) + .then(data => { + if (data.status == "ok") { + if (isTwilioEnabled == "True" || isTwilioEnabled == "true") { + phoneNumberUpdateCallback.innerHTML = "OTP sent to your phone number"; + phonenumberVerifyOTPButton.style.display = "block"; + phonenumberOTPInput.style.display = "block"; + } else { + phonenumberVerifiedText.style.display = "block"; + phoneNumberUpdateCallback.innerHTML = "Phone number updated"; + phonenumberUnverifiedText.style.display = "none"; + } + phonenumberVerifyButton.style.display = "none"; + phoneNumberUpdateCallback.style.display = "block"; + setTimeout(function() { + phoneNumberUpdateCallback.style.display = "none"; + }, 5000); + } else { + phoneNumberUpdateCallback.innerHTML = "Error updating phone number"; + phoneNumberUpdateCallback.style.display = "block"; + setTimeout(function() { + phoneNumberUpdateCallback.style.display = "none"; + }, 5000); + } + }) + .catch((error) => { + console.error('Error:', error); + phoneNumberUpdateCallback.innerHTML = "Error updating phone number"; + phoneNumberUpdateCallback.style.display = "block"; + setTimeout(function() { + phoneNumberUpdateCallback.style.display = "none"; + }, 5000); + }); + } + }) + + phonenumberVerifyOTPButton.addEventListener("click", () => { + const otp = phonenumberOTPInput.value; + if (otp.length != 6) { + phoneNumberUpdateCallback.innerHTML = "Your OTP should be exactly 6 digits"; + phoneNumberUpdateCallback.style.display = "block"; + setTimeout(function() { + phoneNumberUpdateCallback.style.display = "none"; + }, 5000); + return; + } + + fetch('/api/config/phone/verify?code=' + otp, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + } + }) + .then(response => response.json()) + .then(data => { + if (data.status == "ok") { + phoneNumberUpdateCallback.innerHTML = "Phone number updated"; + phonenumberVerifiedText.style.display = "block"; + phonenumberUnverifiedText.style.display = "none"; + phoneNumberUpdateCallback.style.display = "block"; + phonenumberRemoveButton.style.display = ""; + phonenumberVerifyButton.style.display = "none"; + phonenumberVerifyOTPButton.style.display = "none"; + phonenumberOTPInput.style.display = "none"; + setTimeout(function() { + phoneNumberUpdateCallback.style.display = "none"; + }, 5000); + } else { + phoneNumberUpdateCallback.innerHTML = "Error updating phone number"; + phoneNumberUpdateCallback.style.display = "block"; + setTimeout(function() { + phoneNumberUpdateCallback.style.display = "none"; + }, 5000); + } + }) + .catch((error) => { + console.error('Error:', error); + phoneNumberUpdateCallback.innerHTML = "Error updating phone number"; + phoneNumberUpdateCallback.style.display = "block"; + setTimeout(function() { + phoneNumberUpdateCallback.style.display = "none"; + }, 5000); + }); + }) + + {% endblock %} diff --git a/src/khoj/processor/conversation/utils.py b/src/khoj/processor/conversation/utils.py index 32049918..a05605e4 100644 --- a/src/khoj/processor/conversation/utils.py +++ b/src/khoj/processor/conversation/utils.py @@ -10,7 +10,7 @@ from langchain.schema import ChatMessage from transformers import AutoTokenizer from khoj.database.adapters import ConversationAdapters -from khoj.database.models import KhojUser +from khoj.database.models import ClientApplication, KhojUser from khoj.utils.helpers import merge_dicts logger = logging.getLogger(__name__) @@ -98,6 +98,7 @@ def save_to_conversation_log( online_results: Dict[str, Any] = {}, inferred_queries: List[str] = [], intent_type: str = "remember", + client_application: ClientApplication = None, ): user_message_time = user_message_time or datetime.now().strftime("%Y-%m-%d %H:%M:%S") updated_conversation = message_to_log( @@ -111,7 +112,7 @@ def save_to_conversation_log( }, conversation_log=meta_log.get("chat", []), ) - ConversationAdapters.save_conversation(user, {"chat": updated_conversation}) + ConversationAdapters.save_conversation(user, {"chat": updated_conversation}, client_application=client_application) def generate_chatml_messages_with_context( diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index 3047087d..e0fcf454 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -241,7 +241,9 @@ def chat_history( validate_conversation_config() # Load Conversation History - meta_log = ConversationAdapters.get_conversation_by_user(user=user).conversation_log + meta_log = ConversationAdapters.get_conversation_by_user( + user=user, client_application=request.user.client_app + ).conversation_log update_telemetry_state( request=request, @@ -424,7 +426,13 @@ async def chat( } return Response(content=json.dumps(content_obj), media_type="application/json", status_code=status_code) await sync_to_async(save_to_conversation_log)( - q, image, user, meta_log, intent_type="text-to-image", inferred_queries=[improved_image_prompt] + q, + image, + user, + meta_log, + intent_type="text-to-image", + inferred_queries=[improved_image_prompt], + client_application=request.user.client_app, ) content_obj = {"image": image, "intentType": "text-to-image", "inferredQueries": [improved_image_prompt]} # type: ignore return Response(content=json.dumps(content_obj), media_type="application/json", status_code=status_code) @@ -438,6 +446,7 @@ async def chat( inferred_queries, conversation_command, user, + request.user.client_app, ) chat_metadata.update({"conversation_command": conversation_command.value}) diff --git a/src/khoj/routers/api_config.py b/src/khoj/routers/api_config.py index 6169eb4a..12f53abb 100644 --- a/src/khoj/routers/api_config.py +++ b/src/khoj/routers/api_config.py @@ -22,6 +22,7 @@ from khoj.database.models import ( NotionConfig, ) from khoj.routers.helpers import CommonQueryParams, update_telemetry_state +from khoj.routers.twilio import create_otp, is_twilio_enabled, verify_otp from khoj.utils import constants, state from khoj.utils.rawconfig import ( FullConfig, @@ -278,7 +279,77 @@ async def update_search_model( return {"status": "ok"} -# Create Routes +@api_config.post("/phone", status_code=200) +@requires(["authenticated"]) +async def update_phone_number( + request: Request, + phone_number: str, + client: Optional[str] = None, +): + user = request.user.object + + await adapters.aset_user_phone_number(user, phone_number) + + if is_twilio_enabled(): + create_otp(user) + else: + logger.warning("Phone verification is not enabled") + + update_telemetry_state( + request=request, + telemetry_type="api", + api="set_phone_number", + client=client, + metadata={"phone_number": phone_number}, + ) + + return {"status": "ok"} + + +@api_config.delete("/phone", status_code=200) +@requires(["authenticated"]) +async def delete_phone_number( + request: Request, + client: Optional[str] = None, +): + user = request.user.object + + await adapters.aremove_phone_number(user) + + update_telemetry_state( + request=request, + telemetry_type="api", + api="delete_phone_number", + client=client, + ) + + return {"status": "ok"} + + +@api_config.post("/phone/verify", status_code=200) +@requires(["authenticated"]) +async def verify_mobile_otp( + request: Request, + code: str, + client: Optional[str] = None, +): + user: KhojUser = request.user.object + + update_telemetry_state( + request=request, + telemetry_type="api", + api="verify_phone_number", + client=client, + ) + + if is_twilio_enabled() and not verify_otp(user, code): + raise HTTPException(status_code=400, detail="Invalid OTP") + + user.verified_phone_number = True + await user.asave() + return {"status": "ok"} + + @api_config.get("/index/size", response_model=Dict[str, int]) @requires(["authenticated"]) async def get_indexed_data_size(request: Request, common: CommonQueryParams): diff --git a/src/khoj/routers/helpers.py b/src/khoj/routers/helpers.py index 70c7254b..dced1b76 100644 --- a/src/khoj/routers/helpers.py +++ b/src/khoj/routers/helpers.py @@ -208,6 +208,7 @@ def generate_chat_response( inferred_queries: List[str] = [], conversation_command: ConversationCommand = ConversationCommand.Default, user: KhojUser = None, + client_application: ClientApplication = None, ) -> Tuple[Union[ThreadedGenerator, Iterator[str]], Dict[str, str]]: # Initialize Variables chat_response = None @@ -224,6 +225,7 @@ def generate_chat_response( compiled_references=compiled_references, online_results=online_results, inferred_queries=inferred_queries, + client_application=client_application, ) conversation_config = ConversationAdapters.get_valid_conversation_config(user) diff --git a/src/khoj/routers/twilio.py b/src/khoj/routers/twilio.py new file mode 100644 index 00000000..da0f2c50 --- /dev/null +++ b/src/khoj/routers/twilio.py @@ -0,0 +1,36 @@ +import logging +import os + +from twilio.rest import Client + +from khoj.database.models import KhojUser + +logger = logging.getLogger(__name__) + +account_sid = os.getenv("TWILIO_ACCOUNT_SID") +auth_token = os.getenv("TWILIO_AUTH_TOKEN") +verification_service_sid = os.getenv("TWILIO_VERIFICATION_SID") + +twilio_enabled = account_sid is not None and auth_token is not None and verification_service_sid is not None +if twilio_enabled: + client = Client(account_sid, auth_token) + + +def is_twilio_enabled(): + return twilio_enabled + + +def create_otp(user: KhojUser): + """Create a new OTP for the user""" + verification = client.verify.v2.services(verification_service_sid).verifications.create( + to=str(user.phone_number), channel="whatsapp" + ) + return verification.sid is not None + + +def verify_otp(user: KhojUser, code: str): + """Verify the OTP for the user""" + verification_check = client.verify.v2.services(verification_service_sid).verification_checks.create( + to=str(user.phone_number), code=code + ) + return verification_check.status == "approved" diff --git a/src/khoj/routers/web_client.py b/src/khoj/routers/web_client.py index 21d804ec..a51222c7 100644 --- a/src/khoj/routers/web_client.py +++ b/src/khoj/routers/web_client.py @@ -1,6 +1,5 @@ # System Packages import json -import math import os from datetime import timedelta @@ -18,6 +17,7 @@ from khoj.database.adapters import ( get_user_subscription_state, ) from khoj.database.models import KhojUser +from khoj.routers.twilio import is_twilio_enabled from khoj.utils import constants, state from khoj.utils.rawconfig import ( GithubContentConfig, @@ -174,6 +174,9 @@ def config_page(request: Request): "khoj_cloud_subscription_url": os.getenv("KHOJ_CLOUD_SUBSCRIPTION_URL"), "is_active": has_required_scope(request, ["premium"]), "has_documents": has_documents, + "phone_number": user.phone_number, + "is_twilio_enabled": is_twilio_enabled(), + "is_phone_number_verified": user.verified_phone_number, }, )