Add support for magic link email sign-in (#820)

* Add magic link email sign-in option

* Adding backend routes and model changes to keep state of email verification code and status

* Test and fix end to end email verification flow

* Add documentation for how to use the magic link sign-in when self-hosting Khoj

* Add magic link sign in to public conversation page

documentation/docs/miscellaneous/authentication.md

@@ -0,0 +1,30 @@
+# Authentication when Self-Hosting
+
+By default, most of the instructions for self-hosting Khoj assume a single user, and so the default configuration is to run in anonymous mode. However, if you want to enable authentication, you can do so either with [Google Auth](/miscellaneous/google_auth) or with magic links, as shown below. This can be helpful if you want to make sure your Khoj instance is only accessible to you and your team.
+
+:::tip[Note]
+Remove the `--anonymous-mode` flag in your start up command to enable authentication.
+:::
+
+The most secure way to do this is to integrate with [Resend](https://resend.com) by setting up an account and adding an environment variable for `RESEND_API_KEY`. You can get your API key [here](https://resend.com/api-keys). This will allow you to automatically send sign-in links to users who want to log in.
+
+It's still possible to use the magic links feature without Resend, but you'll need to manually send the magic links to users who want to log in.
+
+## Manually sending magic links
+
+1. The user will have to enter their email address in the login form.
+They'll click `Send Magic Link`. Without the Resend API key, this will just create an unverified account for them in the backend
+<img src="/img/magic_link.png" alt="Magic link login form" width="400"/>
+
+2. You can get their magic link using the admin panel
+Go to the [admin panel](http://localhost:42110/server/admin/database/khojuser/). You'll see a list of users. Search for the user you want to send a magic link to. Tick the checkbox next to their row, and use the action drop down at the top to 'Get email login URL'. This will generate a magic link that you can send to the user, which will appear at the top of the admin interface.
+
+| Get email login URL | Retrieved login URL |
+|---------------------|---------------------|
+| <img src="/img/admin_get_emali_login.png" alt="Get user magic sign in link" width="400" />| <img src="/img/admin_successful_login_url.png" alt="Successfully retrieved a login URL" width="400" />|
+
+3. Send the magic link to the user. They can click on it to log in.
+
+Once they click on the link, they'll automatically be logged in. They'll have to repeat this process for every new device they want to log in from, but they shouldn't have to repeat it on the same device.
+
+A given magic link can only be used once. If the user tries to use it again, they'll be redirected to the login page to get a new magic link.

src/khoj/database/adapters/__init__.py

@@ -160,10 +160,14 @@ async def acreate_user_by_phone_number(phone_number: str) -> KhojUser:
     return user
 
 
-async def get_or_create_user_by_email(email: str) -> KhojUser:
+async def aget_or_create_user_by_email(email: str) -> KhojUser:
     user, _ = await KhojUser.objects.filter(email=email).aupdate_or_create(defaults={"username": email, "email": email})
     await user.asave()
 
+    if user:
+        user.email_verification_code = secrets.token_urlsafe(18)
+        await user.asave()
+
     user_subscription = await Subscription.objects.filter(user=user).afirst()
     if not user_subscription:
         await Subscription.objects.acreate(user=user, type="trial")
@@ -171,10 +175,23 @@ async def get_or_create_user_by_email(email: str) -> KhojUser:
     return user
 
 
+async def aget_user_validated_by_email_verification_code(code: str) -> KhojUser:
+    user = await KhojUser.objects.filter(email_verification_code=code).afirst()
+    if not user:
+        return None
+
+    user.email_verification_code = None
+    user.verified_email = True
+    await user.asave()
+
+    return user
+
+
 async def create_user_by_google_token(token: dict) -> KhojUser:
     user, _ = await KhojUser.objects.filter(email=token.get("email")).aupdate_or_create(
         defaults={"username": token.get("email"), "email": token.get("email")}
     )
+    user.verified_email = True
     await user.asave()
 
     await GoogleUser.objects.acreate(
@@ -228,7 +245,7 @@ async def set_user_subscription(
     email: str, is_recurring=None, renewal_date=None, type="standard"
 ) -> Optional[Subscription]:
     # Get or create the user object and their subscription
-    user = await get_or_create_user_by_email(email)
+    user = await aget_or_create_user_by_email(email)
     user_subscription = await Subscription.objects.filter(user=user).afirst()
 
     # Update the user subscription state

src/khoj/database/admin.py

@@ -2,7 +2,7 @@ import csv
 import json
 
 from apscheduler.job import Job
-from django.contrib import admin
+from django.contrib import admin, messages
 from django.contrib.auth.admin import UserAdmin
 from django.http import HttpResponse
 from django_apscheduler.admin import DjangoJobAdmin
@@ -73,7 +73,19 @@ class KhojUserAdmin(UserAdmin):
     search_fields = ("email", "username", "phone_number", "uuid")
     filter_horizontal = ("groups", "user_permissions")
 
-    fieldsets = (("Personal info", {"fields": ("phone_number",)}),) + UserAdmin.fieldsets
+    fieldsets = (("Personal info", {"fields": ("phone_number", "email_verification_code")}),) + UserAdmin.fieldsets
+
+    actions = ["get_email_login_url"]
+
+    def get_email_login_url(self, request, queryset):
+        for user in queryset:
+            if user.email:
+                host = request.get_host()
+                unique_id = user.email_verification_code
+                login_url = f"{host}/auth/magic?code={unique_id}"
+                messages.info(request, f"Email login URL for {user.email}: {login_url}")
+
+    get_email_login_url.short_description = "Get email login URL"  # type: ignore
 
 
 admin.site.register(KhojUser, KhojUserAdmin)

src/khoj/database/migrations/0046_khojuser_email_verification_code_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.11 on 2024-06-17 08:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0045_fileobject"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="khojuser",
+            name="email_verification_code",
+            field=models.CharField(blank=True, default=None, max_length=200, null=True),
+        ),
+        migrations.AddField(
+            model_name="khojuser",
+            name="verified_email",
+            field=models.BooleanField(default=False),
+        ),
+    ]

src/khoj/database/models/__init__.py

@@ -32,6 +32,8 @@ class KhojUser(AbstractUser):
     uuid = models.UUIDField(models.UUIDField(default=uuid.uuid4, editable=False))
     phone_number = PhoneNumberField(null=True, default=None, blank=True)
     verified_phone_number = models.BooleanField(default=False)
+    verified_email = models.BooleanField(default=False)
+    email_verification_code = models.CharField(max_length=200, null=True, default=None, blank=True)
 
     def save(self, *args, **kwargs):
         if not self.uuid:

src/khoj/interface/email/magic_link.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>Welcome to Khoj</title>
+    </head>
+<body>
+<body style="font-family: 'Verdana', sans-serif; font-weight: 400; font-style: normal; padding: 0; text-align: left; width: 600px; margin: 20px auto;">
+<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
+<a class="logo" href="https://khoj.dev" target="_blank" style="text-decoration: none; text-decoration: underline dotted;">
+    <img src="https://khoj.dev/khoj-logo-sideways-500.png" alt="Khoj Logo" style="width: 100px;">
+</a>
+<p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">Hi! <a href="{{ link }}" target="_blank" style="text-decoration: none; text-decoration: underline dotted;">Click here to sign in on this browser.</a></p>
+
+<p style="color: #333; font-size: large; margin-top: 20px; padding: 0; line-height: 1.5;">- The Khoj Team</p>
+
+</body>
+</html>

src/khoj/interface/web/login.html

@@ -12,35 +12,74 @@
 
     <body>
         <div class="khoj-header"></div>
+            <!-- Login Modal -->
+            <div id="login-modal">
+                <img class="khoj-logo" src="/static/assets/icons/favicon-128x128.png" alt="Khoj"></img>
+                <div class="login-modal-title">Login to Khoj</div>
+                <!-- Sign in with Magic Link -->
+                <div class="khoj-magic-link">
+                    <input type="email" id="email" placeholder="Email" required>
+                    <button id="magic-link-button">Send Magic Link</button>
+                </div>
+                <!-- Divider -->
+                <div style="text-align: center; font-size: 16px; font-weight: 500; border-top: 1px solid black;">OR</div>
+                <!-- Sign Up/Login with Google OAuth -->
+                <div
+                    class="g_id_signin"
+                    data-shape="circle"
+                    data-text="continue_with"
+                    data-logo_alignment="center"
+                    data-size="large"
+                    data-type="standard">
+                </div>
+                <div id="g_id_onload"
+                    data-client_id="{{ google_client_id }}"
+                    data-ux_mode="popup"
+                    data-use_fedcm_for_prompt="true"
+                    data-login_uri="{{ redirect_uri }}"
+                    data-auto-select="true">
+                </div>
+            </div>
 
-        <!-- Login Modal -->
-        <div id="login-modal">
-            <img class="khoj-logo" src="/static/assets/icons/favicon-128x128.png" alt="Khoj"></img>
-            <div class="login-modal-title">Login to Khoj</div>
-            <!-- Sign Up/Login with Google OAuth -->
-            <div
-                class="g_id_signin"
-                data-shape="circle"
-                data-text="continue_with"
-                data-logo_alignment="center"
-                data-size="large"
-                data-type="standard">
-            </div>
-            <div id="g_id_onload"
-                data-client_id="{{ google_client_id }}"
-                data-ux_mode="popup"
-                data-use_fedcm_for_prompt="true"
-                data-login_uri="{{ redirect_uri }}"
-                data-auto-select="true">
-            </div>
+            <div class="khoj-footer"></div>
         </div>
-
-        <div class="khoj-footer"></div>
-   </div>
-
-
     </body>
 
+    <script>
+        const magicLinkButton = document.getElementById('magic-link-button');
+        const emailInput = document.getElementById('email');
+
+        magicLinkButton.addEventListener('click', async () => {
+            const email = emailInput.value;
+            if (!email) {
+                alert('Please enter a valid email address');
+                return;
+            }
+
+            if (!email.includes('@')) {
+                alert('Please enter a valid email address');
+                return;
+            }
+
+            magicLinkButton.disabled = true;
+            magicLinkButton.innerText = 'Check your email for a sign-in link!';
+
+            const response = await fetch('/auth/magic', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({ "email": email }),
+            })
+
+            if (response.status === 200) {
+                console.log('Magic link sent to your email');
+            } else {
+                alert('Failed to send magic link');
+            }
+        });
+    </script>
+
     <style>
         @media only screen and (max-width: 700px) {
             body {
@@ -56,7 +95,7 @@
         @media only screen and (min-width: 700px) {
             body {
                 display: grid;
-                grid-template-columns: 1fr min(70vw, 100%) 1fr;
+                grid-template-columns: 1fr min(50vw, 100%) 1fr;
                 grid-template-rows: 1fr auto 1fr;
             }
             body > * {
@@ -110,7 +149,6 @@
         div#login-modal {
             display: grid;
             grid-template-columns: 1fr;
-            grid-template-rows: 1fr auto auto 1fr;
             gap: 32px;
             min-height: 300px;
             margin-left: 25%;
@@ -118,6 +156,40 @@
             z-index: 1;
         }
 
+        .khoj-magic-link {
+            display: grid;
+            grid-template-columns: 1fr;
+            gap: 16px;
+            text-align: center;
+        }
+
+        #email {
+            padding: 10px;
+            font-size: 16px;
+            border: 1px solid var(--main-text-color);
+            border-radius: 5px;
+            width: 100%;
+        }
+
+        #email:focus {
+            box-shadow: 0 0 10px var(--main-text-color);
+        }
+
+        #magic-link-button {
+            padding: 10px;
+            font-size: 16px;
+            border: 1px solid var(--main-text-color);
+            border-radius: 5px;
+            width: 100%;
+            background: var(--main-text-color);
+            color: var(--frosted-background-color);
+            cursor: pointer;
+        }
+
+        #magic-link-button:hover {
+            box-shadow: 0 0 10px var(--main-text-color);
+        }
+
         div.g_id_signin {
             margin: 0 auto;
             display: block;

src/khoj/interface/web/public_conversation.html

@@ -656,7 +656,7 @@ Learn more [here](https://khoj.dev).
             event.preventDefault();
 
             if (!isLoggedIn) {
-                document.getElementById("login-modal").style.display = "block";
+                document.getElementById("login-modal").style.display = "flex";
                 return;
             }
 
@@ -851,6 +851,13 @@ Learn more [here](https://khoj.dev).
         <div id="login-modal" style="display: none;">
             <img class="khoj-logo" src="/static/assets/icons/favicon-128x128.png" alt="Khoj"></img>
             <div class="login-modal-title">Login to continue</div>
+            <!-- Sign in with Magic Link -->
+            <div class="khoj-magic-link">
+                <input type="email" id="email" placeholder="Email" required>
+                <button id="magic-link-button">Send Magic Link</button>
+            </div>
+            <!-- Divider -->
+            <div style="text-align: center; font-size: 16px; font-weight: 500; border-top: 1px solid black;">OR</div>
             <!-- Sign Up/Login with Google OAuth -->
             <div
                 class="g_id_signin"
@@ -964,6 +971,40 @@ Learn more [here](https://khoj.dev).
         if (chatNav) {
             chatNav.classList.add("khoj-nav-selected");
         }
+
+        const magicLinkButton = document.getElementById('magic-link-button');
+        const emailInput = document.getElementById('email');
+
+        magicLinkButton.addEventListener('click', async () => {
+            const email = emailInput.value;
+            if (!email) {
+                alert('Please enter a valid email address');
+                return;
+            }
+
+            if (!email.includes('@')) {
+                alert('Please enter a valid email address');
+                return;
+            }
+
+            magicLinkButton.disabled = true;
+            magicLinkButton.innerText = 'Check your email for a sign-in link!';
+
+            const response = await fetch('/auth/magic', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({ "email": email }),
+            })
+
+            if (response.status === 200) {
+                console.log('Magic link sent to your email');
+            } else {
+                alert('Failed to send magic link');
+            }
+        });
+
     </script>
     <style>
         html, body {
@@ -1405,10 +1446,6 @@ Learn more [here](https://khoj.dev).
         }
 
         div#login-modal {
-            display: grid;
-            grid-template-columns: 1fr;
-            grid-template-rows: 1fr auto auto 1fr;
-            gap: 32px;
             min-height: 300px;
             z-index: 1;
             position: absolute;
@@ -1419,6 +1456,45 @@ Learn more [here](https://khoj.dev).
             border-radius: 8px;
             padding: 40px;
             background: linear-gradient(145deg, var(--background-color), var(--primary-hover));
+            display: flex;
+            flex-direction: column;
+            align-content: center;
+            justify-content: center;
+            align-items: center;
+            gap: 24px;
+        }
+
+        .khoj-magic-link {
+            display: grid;
+            grid-template-columns: 1fr;
+            gap: 16px;
+            text-align: center;
+        }
+
+        #email {
+            padding: 10px;
+            font-size: 16px;
+            border: 1px solid var(--main-text-color);
+            border-radius: 5px;
+        }
+
+        #email:focus {
+            box-shadow: 0 0 10px var(--main-text-color);
+        }
+
+        #magic-link-button {
+            padding: 10px;
+            font-size: 16px;
+            border: 1px solid var(--main-text-color);
+            border-radius: 5px;
+            width: 100%;
+            background: var(--main-text-color);
+            color: var(--frosted-background-color);
+            cursor: pointer;
+        }
+
+        #magic-link-button:hover {
+            box-shadow: 0 0 10px var(--main-text-color);
         }
 
         div.g_id_signin {

src/khoj/routers/auth.py

@@ -5,6 +5,7 @@ import os
 from typing import Optional
 
 from fastapi import APIRouter
+from pydantic import BaseModel, EmailStr
 from starlette.authentication import requires
 from starlette.config import Config
 from starlette.requests import Request
@@ -13,11 +14,13 @@ from starlette.status import HTTP_302_FOUND
 
 from khoj.database.adapters import (
     acreate_khoj_token,
+    aget_or_create_user_by_email,
+    aget_user_validated_by_email_verification_code,
     delete_khoj_token,
     get_khoj_tokens,
     get_or_create_user,
 )
-from khoj.routers.email import send_welcome_email
+from khoj.routers.email import send_magic_link_email, send_welcome_email
 from khoj.routers.helpers import get_next_url, update_telemetry_state
 from khoj.utils import state
 
@@ -26,6 +29,10 @@ logger = logging.getLogger(__name__)
 auth_router = APIRouter()
 
 
+class MagicLinkForm(BaseModel):
+    email: EmailStr
+
+
 if not state.anonymous_mode:
     missing_requirements = []
     from authlib.integrations.starlette_client import OAuth, OAuthError
@@ -62,6 +69,35 @@ async def login(request: Request):
     return await oauth.google.authorize_redirect(request, redirect_uri)
 
 
+@auth_router.post("/magic")
+async def login_magic_link(request: Request, form: MagicLinkForm):
+    if request.user.is_authenticated:
+        # Clear the session if user is already authenticated
+        request.session.pop("user", None)
+
+    email = form.email
+    user = await aget_or_create_user_by_email(email)
+    unique_id = user.email_verification_code
+
+    if user:
+        await send_magic_link_email(email, unique_id, request.base_url)
+
+    return Response(status_code=200)
+
+
+@auth_router.get("/magic")
+async def sign_in_with_magic_link(request: Request, code: str):
+    user = await aget_user_validated_by_email_verification_code(code)
+    if user:
+        id_info = {
+            "email": user.email,
+        }
+
+        request.session["user"] = dict(id_info)
+        return RedirectResponse(url="/")
+    return RedirectResponse(request.app.url_path_for("login_page"))
+
+
 @auth_router.post("/token")
 @requires(["authenticated"], redirect="login_page")
 async def generate_token(request: Request, token_name: Optional[str] = None):

src/khoj/routers/email.py

@@ -32,6 +32,22 @@ def is_resend_enabled():
     return bool(RESEND_API_KEY)
 
 
+async def send_magic_link_email(email, unique_id, host):
+    sign_in_link = f"{host}auth/magic?code={unique_id}"
+
+    if not is_resend_enabled():
+        logger.debug(f"Email sending disabled. Share this sign-in link with the user: {sign_in_link}")
+        return
+
+    template = env.get_template("magic_link.html")
+
+    html_content = template.render(link=f"{host}auth/magic?code={unique_id}")
+
+    resend.Emails.send(
+        {"sender": "noreply@khoj.dev", "to": email, "subject": "Your Sign-In Link for Khoj 🚀", "html": html_content}
+    )
+
+
 async def send_welcome_email(name, email):
     if not is_resend_enabled():
         logger.debug("Email sending disabled")