Serve image assets from Khoj domain, not directly from S3 bucket (#734)

- Serve generated images from Khoj domain instead of directly from AWS S3 - Rename assets URL from Khoj S3 bucket to assets.khoj.dev
2024-11-27 09:25:06 +01:00 · 2024-06-02 17:24:35 +05:30 · 2024-06-02 17:24:35 +05:30 · cf8c9c2a3d
commit cf8c9c2a3d
parent 5bb3689562 7823ef09dc
8 changed files with 71 additions and 7 deletions
--- a/documentation/docs/data-sources/share_your_data.md
+++ b/documentation/docs/data-sources/share_your_data.md
@ -13,4 +13,4 @@ There are several ways you can get started with sharing your data with the Khoj
 - Setup the sync options for either [Obsidian](/clients/obsidian) or [Emacs](/clients/emacs) to automatically sync your documents with Khoj. This is best if you are already using these tools and want to leverage Khoj's AI capabilities.
 - Configure your [Notion](/data-sources/notion_integration) or [Github](/data-sources/github_integration) to sync with Khoj. By providing your credentials, you can keep the data synced in the background.

-![demo of dragging and dropping a file](https://khoj-web-bucket.s3.amazonaws.com/drag_drop_file.gif)
+![demo of dragging and dropping a file](https://assets.khoj.dev/drag_drop_file.gif)
--- a/documentation/docusaurus.config.js
+++ b/documentation/docusaurus.config.js
@ -80,7 +80,7 @@ const config = {
        {name: 'og:type', content: 'website'},
        {name: 'og:site_name', content: 'Khoj Documentation'},
        {name: 'og:description', content: 'Quickly get started with using or self-hosting Khoj'},
-        {name: 'og:image', content: 'https://khoj-web-bucket.s3.amazonaws.com/link_preview_docs.png'},
+        {name: 'og:image', content: 'https://assets.khoj.dev/link_preview_docs.png'},
        {name: 'og:url', content: 'https://docs.khoj.dev'},
        {name: 'keywords', content: 'khoj, khoj ai, chatgpt, open ai, open source, productivity'}
      ],
--- a/src/interface/desktop/search.html
+++ b/src/interface/desktop/search.html
@ -2,7 +2,7 @@
    <head>
        <meta charset="utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
-        <meta property="og:image" content="https://khoj-web-bucket.s3.amazonaws.com/khoj_hero.png">
+        <meta property="og:image" content="https://assets.khoj.dev/khoj_hero.png">
        <title>Khoj - Search</title>

        <link rel="icon" type="image/png" sizes="128x128" href="./assets/icons/favicon-128x128.png">
--- a/src/khoj/database/adapters/init.py
+++ b/src/khoj/database/adapters/init.py
@ -493,7 +493,7 @@ class ClientApplicationAdapters:

 class AgentAdapters:
    DEFAULT_AGENT_NAME = "Khoj"
-    DEFAULT_AGENT_AVATAR = "https://khoj-web-bucket.s3.amazonaws.com/lamp-128.png"
+    DEFAULT_AGENT_AVATAR = "https://assets.khoj.dev/lamp-128.png"
    DEFAULT_AGENT_SLUG = "khoj"

    @staticmethod
--- a/src/khoj/database/management/commands/change_generated_images_url.py
+++ b/src/khoj/database/management/commands/change_generated_images_url.py
@ -0,0 +1,61 @@
+from django.core.management.base import BaseCommand
+from tqdm import tqdm
+
+from khoj.database.models import Conversation
+from khoj.utils.helpers import ImageIntentType, is_none_or_empty
+
+
+class Command(BaseCommand):
+    help = "Serve Khoj generated images from a different URL."
+
+    def add_arguments(self, parser):
+        # Pass Source URL
+        parser.add_argument(
+            "--source",
+            action="store",
+            help="URL from which generated images are currently served.",
+        )
+        # Pass Destination URL
+        parser.add_argument("--destination", action="store", help="URL to serve generated image from going forward.")
+
+        # Add a new argument 'reverse' to the command
+        parser.add_argument(
+            "--reverse",
+            action="store_true",
+            help="Revert to serve generated images from source instead of destination URL.",
+        )
+
+    def handle(self, *args, **options):
+        updated_count = 0
+        if not options.get("source") or not options.get("destination"):
+            self.stdout.write(
+                self.style.ERROR(
+                    "Set --source, --destination args to migrate serving images from source to destination URL."
+                )
+            )
+            return
+
+        destination = options["source"] if options["reverse"] else options["destination"]
+        source = options["destination"] if options["reverse"] else options["source"]
+        for conversation in Conversation.objects.all():
+            conversation_updated = False
+            for chat in tqdm(conversation.conversation_log.get("chat", []), desc="Processing Conversations"):
+                if (
+                    chat.get("by", "") == "khoj"
+                    and not is_none_or_empty(chat.get("message"))
+                    and chat.get("message", "").startswith(source)
+                    and chat.get("intent", {}).get("type", "") == ImageIntentType.TEXT_TO_IMAGE2.value
+                    and chat.get("message", "").endswith(".webp")
+                ):
+                    # Convert source url to destination url
+                    chat["message"] = chat["message"].replace(source, destination)
+                    conversation_updated = True
+                    updated_count += 1
+
+            if conversation_updated:
+                print(f"Save the updated conversation {conversation.id} to the database.")
+                conversation.save()
+
+        if updated_count > 0:
+            success = f"Successfully converted {updated_count} image URLs from {source} to {destination}.".strip()
+            self.stdout.write(self.style.SUCCESS(success))
--- a/src/khoj/interface/web/chat.html
+++ b/src/khoj/interface/web/chat.html
@ -3,7 +3,7 @@
    <head>
        <meta charset="utf-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
-        <meta property="og:image" content="https://khoj-web-bucket.s3.amazonaws.com/khoj_hero.png">
+        <meta property="og:image" content="https://assets.khoj.dev/khoj_hero.png">
        <title>Khoj - Chat</title>

        <link rel="stylesheet" href="/static/assets/khoj.css?v={{ khoj_version }}">
--- a/src/khoj/interface/web/login.html
+++ b/src/khoj/interface/web/login.html
@ -7,7 +7,7 @@
        <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png">
        <link rel="manifest" href="/static/khoj.webmanifest">
        <link rel="stylesheet" href="/static/assets/khoj.css">
-        <meta property="og:image" content="https://khoj-web-bucket.s3.amazonaws.com/khoj_hero.png">
+        <meta property="og:image" content="https://assets.khoj.dev/khoj_hero.png">
    </head>

    <body>
--- a/src/khoj/routers/storage.py
+++ b/src/khoj/routers/storage.py
@ -6,6 +6,9 @@ logger = logging.getLogger(__name__)

 AWS_ACCESS_KEY = os.getenv("AWS_ACCESS_KEY")
 AWS_SECRET_KEY = os.getenv("AWS_SECRET_KEY")
+# S3 supports serving assets via your domain. Khoj expects this to be used in production. To enable it:
+# 1. Your bucket name for images should be of the form sub.domain.tld. For example, generated.khoj.dev
+# 2. Add CNAME entry to your domain's DNS records pointing to the S3 bucket. For example, CNAME generated.khoj.dev generated-khoj-dev.s3.amazonaws.com
 AWS_UPLOAD_IMAGE_BUCKET_NAME = os.getenv("AWS_IMAGE_UPLOAD_BUCKET")

 aws_enabled = AWS_ACCESS_KEY is not None and AWS_SECRET_KEY is not None and AWS_UPLOAD_IMAGE_BUCKET_NAME is not None
@ -25,7 +28,7 @@ def upload_image(image: bytes, user_id: uuid.UUID):
    image_key = f"{user_id}/{uuid.uuid4()}.webp"
    try:
        s3_client.put_object(Bucket=AWS_UPLOAD_IMAGE_BUCKET_NAME, Key=image_key, Body=image, ACL="public-read")
-        url = f"https://{AWS_UPLOAD_IMAGE_BUCKET_NAME}.s3.amazonaws.com/{image_key}"
+        url = f"https://{AWS_UPLOAD_IMAGE_BUCKET_NAME}/{image_key}"
        return url
    except Exception as e:
        logger.error(f"Failed to upload image to S3: {e}")