sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-11-23 23:48:56 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 16

Serve image assets from Khoj domain, not directly from S3 bucket (#734)

Browse source 
- Serve generated images from Khoj domain instead of directly from AWS S3
- Rename assets URL from Khoj S3 bucket to assets.khoj.dev
This commit is contained in:
Debanjum 2024-06-02 17:24:35 +05:30 committed by GitHub
commit cf8c9c2a3d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 71 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
documentation/docs/data-sources/share_your_data.md
View file

@ -13,4 +13,4 @@ There are several ways you can get started with sharing your data with the Khoj
- Setup the sync options for either [Obsidian](/clients/obsidian) or [Emacs](/clients/emacs) to automatically sync your documents with Khoj. This is best if you are already using these tools and want to leverage Khoj's AI capabilities. - Setup the sync options for either [Obsidian](/clients/obsidian) or [Emacs](/clients/emacs) to automatically sync your documents with Khoj. This is best if you are already using these tools and want to leverage Khoj's AI capabilities.
- Configure your [Notion](/data-sources/notion_integration) or [Github](/data-sources/github_integration) to sync with Khoj. By providing your credentials, you can keep the data synced in the background. - Configure your [Notion](/data-sources/notion_integration) or [Github](/data-sources/github_integration) to sync with Khoj. By providing your credentials, you can keep the data synced in the background.
![demo of dragging and dropping a file](https://khoj-web-bucket.s3.amazonaws.com/drag_drop_file.gif) ![demo of dragging and dropping a file](https://assets.khoj.dev/drag_drop_file.gif)

2
documentation/docusaurus.config.js
View file

@ -80,7 +80,7 @@ const config = {
{name: 'og:type', content: 'website'}, {name: 'og:type', content: 'website'},
{name: 'og:site_name', content: 'Khoj Documentation'}, {name: 'og:site_name', content: 'Khoj Documentation'},
{name: 'og:description', content: 'Quickly get started with using or self-hosting Khoj'}, {name: 'og:description', content: 'Quickly get started with using or self-hosting Khoj'},
{name: 'og:image', content: 'https://khoj-web-bucket.s3.amazonaws.com/link_preview_docs.png'}, {name: 'og:image', content: 'https://assets.khoj.dev/link_preview_docs.png'},
{name: 'og:url', content: 'https://docs.khoj.dev'}, {name: 'og:url', content: 'https://docs.khoj.dev'},
{name: 'keywords', content: 'khoj, khoj ai, chatgpt, open ai, open source, productivity'} {name: 'keywords', content: 'khoj, khoj ai, chatgpt, open ai, open source, productivity'}
], ],

2
src/interface/desktop/search.html
View file

@ -2,7 +2,7 @@
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
<meta property="og:image" content="https://khoj-web-bucket.s3.amazonaws.com/khoj_hero.png"> <meta property="og:image" content="https://assets.khoj.dev/khoj_hero.png">
<title>Khoj - Search</title> <title>Khoj - Search</title>
<link rel="icon" type="image/png" sizes="128x128" href="./assets/icons/favicon-128x128.png"> <link rel="icon" type="image/png" sizes="128x128" href="./assets/icons/favicon-128x128.png">

2
src/khoj/database/adapters/__init__.py
View file

@ -493,7 +493,7 @@ class ClientApplicationAdapters:
class AgentAdapters: class AgentAdapters:
DEFAULT_AGENT_NAME = "Khoj" DEFAULT_AGENT_NAME = "Khoj"
DEFAULT_AGENT_AVATAR = "https://khoj-web-bucket.s3.amazonaws.com/lamp-128.png" DEFAULT_AGENT_AVATAR = "https://assets.khoj.dev/lamp-128.png"
DEFAULT_AGENT_SLUG = "khoj" DEFAULT_AGENT_SLUG = "khoj"
@staticmethod @staticmethod

61
src/khoj/database/management/commands/change_generated_images_url.py Normal file
View file

@ -0,0 +1,61 @@
from django.core.management.base import BaseCommand
from tqdm import tqdm
from khoj.database.models import Conversation
from khoj.utils.helpers import ImageIntentType, is_none_or_empty
class Command(BaseCommand):
help = "Serve Khoj generated images from a different URL."
def add_arguments(self, parser):
# Pass Source URL
parser.add_argument(
"--source",
action="store",
help="URL from which generated images are currently served.",
)
# Pass Destination URL
parser.add_argument("--destination", action="store", help="URL to serve generated image from going forward.")
# Add a new argument 'reverse' to the command
parser.add_argument(
"--reverse",
action="store_true",
help="Revert to serve generated images from source instead of destination URL.",
)
def handle(self, *args, **options):
updated_count = 0
if not options.get("source") or not options.get("destination"):
self.stdout.write(
self.style.ERROR(
"Set --source, --destination args to migrate serving images from source to destination URL."
)
)
return
destination = options["source"] if options["reverse"] else options["destination"]
source = options["destination"] if options["reverse"] else options["source"]
for conversation in Conversation.objects.all():
conversation_updated = False
for chat in tqdm(conversation.conversation_log.get("chat", []), desc="Processing Conversations"):
if (
chat.get("by", "") == "khoj"
and not is_none_or_empty(chat.get("message"))
and chat.get("message", "").startswith(source)
and chat.get("intent", {}).get("type", "") == ImageIntentType.TEXT_TO_IMAGE2.value
and chat.get("message", "").endswith(".webp")
):
# Convert source url to destination url
chat["message"] = chat["message"].replace(source, destination)
conversation_updated = True
updated_count += 1
if conversation_updated:
print(f"Save the updated conversation {conversation.id} to the database.")
conversation.save()
if updated_count > 0:
success = f"Successfully converted {updated_count} image URLs from {source} to {destination}.".strip()
self.stdout.write(self.style.SUCCESS(success))

2
src/khoj/interface/web/chat.html
View file

@ -3,7 +3,7 @@
<head> <head>
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
<meta property="og:image" content="https://khoj-web-bucket.s3.amazonaws.com/khoj_hero.png"> <meta property="og:image" content="https://assets.khoj.dev/khoj_hero.png">
<title>Khoj - Chat</title> <title>Khoj - Chat</title>
<link rel="stylesheet" href="/static/assets/khoj.css?v={{ khoj_version }}"> <link rel="stylesheet" href="/static/assets/khoj.css?v={{ khoj_version }}">

2
src/khoj/interface/web/login.html
View file

@ -7,7 +7,7 @@
<link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png"> <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png">
<link rel="manifest" href="/static/khoj.webmanifest"> <link rel="manifest" href="/static/khoj.webmanifest">
<link rel="stylesheet" href="/static/assets/khoj.css"> <link rel="stylesheet" href="/static/assets/khoj.css">
<meta property="og:image" content="https://khoj-web-bucket.s3.amazonaws.com/khoj_hero.png"> <meta property="og:image" content="https://assets.khoj.dev/khoj_hero.png">
</head> </head>
<body> <body>

5
src/khoj/routers/storage.py
View file

@ -6,6 +6,9 @@ logger = logging.getLogger(__name__)
AWS_ACCESS_KEY = os.getenv("AWS_ACCESS_KEY") AWS_ACCESS_KEY = os.getenv("AWS_ACCESS_KEY")
AWS_SECRET_KEY = os.getenv("AWS_SECRET_KEY") AWS_SECRET_KEY = os.getenv("AWS_SECRET_KEY")
# S3 supports serving assets via your domain. Khoj expects this to be used in production. To enable it:
# 1. Your bucket name for images should be of the form sub.domain.tld. For example, generated.khoj.dev
# 2. Add CNAME entry to your domain's DNS records pointing to the S3 bucket. For example, CNAME generated.khoj.dev generated-khoj-dev.s3.amazonaws.com
AWS_UPLOAD_IMAGE_BUCKET_NAME = os.getenv("AWS_IMAGE_UPLOAD_BUCKET") AWS_UPLOAD_IMAGE_BUCKET_NAME = os.getenv("AWS_IMAGE_UPLOAD_BUCKET")
aws_enabled = AWS_ACCESS_KEY is not None and AWS_SECRET_KEY is not None and AWS_UPLOAD_IMAGE_BUCKET_NAME is not None aws_enabled = AWS_ACCESS_KEY is not None and AWS_SECRET_KEY is not None and AWS_UPLOAD_IMAGE_BUCKET_NAME is not None
@ -25,7 +28,7 @@ def upload_image(image: bytes, user_id: uuid.UUID):
image_key = f"{user_id}/{uuid.uuid4()}.webp" image_key = f"{user_id}/{uuid.uuid4()}.webp"
try: try:
s3_client.put_object(Bucket=AWS_UPLOAD_IMAGE_BUCKET_NAME, Key=image_key, Body=image, ACL="public-read") s3_client.put_object(Bucket=AWS_UPLOAD_IMAGE_BUCKET_NAME, Key=image_key, Body=image, ACL="public-read")
url = f"https://{AWS_UPLOAD_IMAGE_BUCKET_NAME}.s3.amazonaws.com/{image_key}" url = f"https://{AWS_UPLOAD_IMAGE_BUCKET_NAME}/{image_key}"
return url return url
except Exception as e: except Exception as e:
logger.error(f"Failed to upload image to S3: {e}") logger.error(f"Failed to upload image to S3: {e}")