From fdf78525b4bb6631e74fc0f8408f51e32049140f Mon Sep 17 00:00:00 2001 From: sabaimran <65192171+sabaimran@users.noreply.github.com> Date: Tue, 26 Mar 2024 05:43:24 -0700 Subject: [PATCH] Part 2: Add web UI updates for basic agent interactions (#675) * Initial pass at backend changes to support agents - Add a db model for Agents, attaching them to conversations - When an agent is added to a conversation, override the system prompt to tweak the instructions - Agents can be configured with prompt modification, model specification, a profile picture, and other things - Admin-configured models will not be editable by individual users - Add unit tests to verify agent behavior. Unit tests demonstrate imperfect adherence to prompt specifications * Customize default behaviors for conversations without agents or with default agents * Add a new web client route for viewing all agents * Use agent_id for getting correct agent * Add web UI views for agents - Add a page to view all agents - Add slugs to manage agents - Add a view to view single agent - Display active agent when in chat window - Fix post-login redirect issue * Fix agent view * Spruce up the 404 page and improve the overall layout for agents pages * Create chat actor for directly reading webpages based on user message - Add prompt for the read webpages chat actor to extract, infer webpage links - Make chat actor infer or extract webpage to read directly from user message - Rename previous read_webpage function to more narrow read_webpage_at_url function * Rename agents_page -> agent_page * Fix unit test for adding the filename to the compiled markdown entry * Fix layout of agent, agents pages * Merge migrations * Let the name, slug of the default agent be Khoj, khoj * Fix chat-related unit tests * Add webpage chat command for read web pages requested by user Update auto chat command inference prompt to show example of when to use webpage chat command (i.e when url is directly provided in link) * Support webpage command in chat API - Fallback to use webpage when SERPER not setup and online command was attempted - Do not stop responding if can't retrieve online results. Try to respond without the online context * Test select webpage as data source and extract web urls chat actors * Tweak prompts to extract information from webpages, online results - Show more of the truncated messages for debugging context - Update Khoj personality prompt to encourage it to remember it's capabilities * Rename extract_content online results field to webpages * Parallelize simple webpage read and extractor Similar to what is being done with search_online with olostep * Pass multiple webpages with their urls in online results context Previously even if MAX_WEBPAGES_TO_READ was > 1, only 1 extracted content would ever be passed. URL of the extracted webpage content wasn't passed to clients in online results context. This limited them from being rendered * Render webpage read in chat response references on Web, Desktop apps * Time chat actor responses & chat api request start for perf analysis * Increase the keep alive timeout in the main application for testing * Do not pipe access/error logs to separate files. Flow to stdout/stderr * [Temp] Reduce to 1 gunicorn worker * Change prod docker image to use jammy, rather than nvidia base image * Use Khoj icon when Khoj web is installed on iOS as a PWA * Make slug required for agents * Simplify calling logic and prevent agent access for unauthenticated users * Standardize to use personality over tuning in agent nomenclature * Make filtering logic more stringent for accessible agents and remove unused method: * Format chat message query --------- Co-authored-by: Debanjum Singh Solanky --- gunicorn-config.py | 6 +- prod.Dockerfile | 5 +- src/interface/desktop/chat.html | 11 +- src/khoj/configure.py | 2 + src/khoj/database/adapters/__init__.py | 50 ++- .../0031_agent_conversation_agent.py | 3 +- src/khoj/database/models/__init__.py | 20 +- src/khoj/interface/web/404.html | 42 ++- src/khoj/interface/web/agent.html | 286 ++++++++++++++ src/khoj/interface/web/agents.html | 201 ++++++++++ src/khoj/interface/web/assets/khoj.css | 2 +- src/khoj/interface/web/base_config.html | 2 +- src/khoj/interface/web/chat.html | 351 ++++++++++++++++-- src/khoj/interface/web/search.html | 1 + src/khoj/interface/web/utils.html | 36 +- src/khoj/main.py | 4 +- .../content/markdown/markdown_to_entries.py | 2 +- .../conversation/offline/chat_model.py | 4 +- src/khoj/processor/conversation/openai/gpt.py | 8 +- src/khoj/processor/conversation/prompts.py | 61 ++- src/khoj/processor/tools/online_search.py | 56 ++- src/khoj/routers/api_agents.py | 43 +++ src/khoj/routers/api_chat.py | 62 +++- src/khoj/routers/auth.py | 6 +- src/khoj/routers/helpers.py | 56 ++- src/khoj/routers/web_client.py | 83 ++++- src/khoj/utils/helpers.py | 15 +- tests/test_gpt4all_chat_director.py | 4 +- tests/test_helpers.py | 7 +- tests/test_markdown_to_entries.py | 4 +- tests/test_openai_chat_actors.py | 29 ++ 31 files changed, 1332 insertions(+), 130 deletions(-) create mode 100644 src/khoj/interface/web/agent.html create mode 100644 src/khoj/interface/web/agents.html create mode 100644 src/khoj/routers/api_agents.py diff --git a/gunicorn-config.py b/gunicorn-config.py index bfed49e7..ea382346 100644 --- a/gunicorn-config.py +++ b/gunicorn-config.py @@ -1,10 +1,10 @@ import multiprocessing bind = "0.0.0.0:42110" -workers = 8 +workers = 1 worker_class = "uvicorn.workers.UvicornWorker" timeout = 120 keep_alive = 60 -accesslog = "access.log" -errorlog = "error.log" +accesslog = "-" +errorlog = "-" loglevel = "debug" diff --git a/prod.Dockerfile b/prod.Dockerfile index 413835d0..0da5363a 100644 --- a/prod.Dockerfile +++ b/prod.Dockerfile @@ -1,12 +1,9 @@ -# Use Nvidia's latest Ubuntu 22.04 image as the base image -FROM nvidia/cuda:12.2.0-devel-ubuntu22.04 +FROM ubuntu:jammy LABEL org.opencontainers.image.source https://github.com/khoj-ai/khoj # Install System Dependencies RUN apt update -y && apt -y install python3-pip libsqlite3-0 ffmpeg libsm6 libxext6 -# Install Optional Dependencies -RUN apt install vim -y WORKDIR /app diff --git a/src/interface/desktop/chat.html b/src/interface/desktop/chat.html index 94cde782..f37ae562 100644 --- a/src/interface/desktop/chat.html +++ b/src/interface/desktop/chat.html @@ -87,7 +87,7 @@ function generateOnlineReference(reference, index) { // Generate HTML for Chat Reference - let title = reference.title; + let title = reference.title || reference.link; let link = reference.link; let snippet = reference.snippet; let question = reference.question; @@ -191,6 +191,15 @@ referenceSection.appendChild(polishedReference); } } + + if (onlineReference.webpages && onlineReference.webpages.length > 0) { + numOnlineReferences += onlineReference.webpages.length; + for (let index in onlineReference.webpages) { + let reference = onlineReference.webpages[index]; + let polishedReference = generateOnlineReference(reference, index); + referenceSection.appendChild(polishedReference); + } + } } return numOnlineReferences; diff --git a/src/khoj/configure.py b/src/khoj/configure.py index fb3a93ce..0adbe889 100644 --- a/src/khoj/configure.py +++ b/src/khoj/configure.py @@ -268,6 +268,7 @@ def initialize_content(regenerate: bool, search_type: Optional[SearchType] = Non def configure_routes(app): # Import APIs here to setup search types before while configuring server from khoj.routers.api import api + from khoj.routers.api_agents import api_agents from khoj.routers.api_chat import api_chat from khoj.routers.api_config import api_config from khoj.routers.indexer import indexer @@ -275,6 +276,7 @@ def configure_routes(app): app.include_router(api, prefix="/api") app.include_router(api_chat, prefix="/api/chat") + app.include_router(api_agents, prefix="/api/agents") app.include_router(api_config, prefix="/api/config") app.include_router(indexer, prefix="/api/v1/index") app.include_router(web_client) diff --git a/src/khoj/database/adapters/__init__.py b/src/khoj/database/adapters/__init__.py index cb317275..25f781fe 100644 --- a/src/khoj/database/adapters/__init__.py +++ b/src/khoj/database/adapters/__init__.py @@ -394,20 +394,32 @@ class ClientApplicationAdapters: class AgentAdapters: - DEFAULT_AGENT_NAME = "khoj" + DEFAULT_AGENT_NAME = "Khoj" DEFAULT_AGENT_AVATAR = "https://khoj-web-bucket.s3.amazonaws.com/lamp-128.png" + DEFAULT_AGENT_SLUG = "khoj" @staticmethod - async def aget_agent_by_id(agent_id: int, user: KhojUser): - agent = await Agent.objects.filter(id=agent_id).afirst() - # Check if it's accessible to the user - if agent and (agent.public or agent.creator == user): - return agent - return None + async def aget_agent_by_slug(agent_slug: str, user: KhojUser): + return await Agent.objects.filter( + (Q(slug__iexact=agent_slug.lower())) & (Q(public=True) | Q(creator=user)) + ).afirst() + + @staticmethod + def get_agent_by_slug(slug: str, user: KhojUser = None): + if user: + return Agent.objects.filter((Q(slug__iexact=slug.lower())) & (Q(public=True) | Q(creator=user))).first() + return Agent.objects.filter(slug__iexact=slug.lower(), public=True).first() @staticmethod def get_all_accessible_agents(user: KhojUser = None): - return Agent.objects.filter(Q(public=True) | Q(creator=user)).distinct() + if user: + return Agent.objects.filter(Q(public=True) | Q(creator=user)).distinct().order_by("created_at") + return Agent.objects.filter(public=True).order_by("created_at") + + @staticmethod + async def aget_all_accessible_agents(user: KhojUser = None) -> List[Agent]: + agents = await sync_to_async(AgentAdapters.get_all_accessible_agents)(user) + return await sync_to_async(list)(agents) @staticmethod def get_conversation_agent_by_id(agent_id: int): @@ -423,12 +435,19 @@ class AgentAdapters: @staticmethod def create_default_agent(): - # First delete the existing default - Agent.objects.filter(name=AgentAdapters.DEFAULT_AGENT_NAME).delete() - default_conversation_config = ConversationAdapters.get_default_conversation_config() default_personality = prompts.personality.format(current_date="placeholder") + agent = Agent.objects.filter(name=AgentAdapters.DEFAULT_AGENT_NAME).first() + + if agent: + agent.personality = default_personality + agent.chat_model = default_conversation_config + agent.slug = AgentAdapters.DEFAULT_AGENT_SLUG + agent.name = AgentAdapters.DEFAULT_AGENT_NAME + agent.save() + return agent + # The default agent is public and managed by the admin. It's handled a little differently than other agents. return Agent.objects.create( name=AgentAdapters.DEFAULT_AGENT_NAME, @@ -438,6 +457,7 @@ class AgentAdapters: personality=default_personality, tools=["*"], avatar=AgentAdapters.DEFAULT_AGENT_AVATAR, + slug=AgentAdapters.DEFAULT_AGENT_SLUG, ) @staticmethod @@ -486,10 +506,12 @@ class ConversationAdapters: @staticmethod async def acreate_conversation_session( - user: KhojUser, client_application: ClientApplication = None, agent_id: int = None + user: KhojUser, client_application: ClientApplication = None, agent_slug: str = None ): - if agent_id: - agent = await AgentAdapters.aget_agent_by_id(agent_id, user) + if agent_slug: + agent = await AgentAdapters.aget_agent_by_slug(agent_slug, user) + if agent is None: + raise HTTPException(status_code=400, detail="No such agent currently exists.") return await Conversation.objects.acreate(user=user, client=client_application, agent=agent) return await Conversation.objects.acreate(user=user, client=client_application) diff --git a/src/khoj/database/migrations/0031_agent_conversation_agent.py b/src/khoj/database/migrations/0031_agent_conversation_agent.py index 16586499..1d08a118 100644 --- a/src/khoj/database/migrations/0031_agent_conversation_agent.py +++ b/src/khoj/database/migrations/0031_agent_conversation_agent.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.10 on 2024-03-11 05:12 +# Generated by Django 4.2.10 on 2024-03-13 07:38 import django.db.models.deletion from django.conf import settings @@ -23,6 +23,7 @@ class Migration(migrations.Migration): ("tools", models.JSONField(default=list)), ("public", models.BooleanField(default=False)), ("managed_by_admin", models.BooleanField(default=False)), + ("slug", models.CharField(max_length=200)), ( "chat_model", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="database.chatmodeloptions"), diff --git a/src/khoj/database/models/__init__.py b/src/khoj/database/models/__init__.py index b8eeb8b1..364a6d1a 100644 --- a/src/khoj/database/models/__init__.py +++ b/src/khoj/database/models/__init__.py @@ -1,4 +1,5 @@ import uuid +from random import choice from django.contrib.auth.models import AbstractUser from django.core.exceptions import ValidationError @@ -94,13 +95,28 @@ class Agent(BaseModel): public = models.BooleanField(default=False) managed_by_admin = models.BooleanField(default=False) chat_model = models.ForeignKey(ChatModelOptions, on_delete=models.CASCADE) + slug = models.CharField(max_length=200) @receiver(pre_save, sender=Agent) -def check_public_name(sender, instance, **kwargs): - if instance.public: +def verify_agent(sender, instance, **kwargs): + # check if this is a new instance + if instance._state.adding: if Agent.objects.filter(name=instance.name, public=True).exists(): raise ValidationError(f"A public Agent with the name {instance.name} already exists.") + if Agent.objects.filter(name=instance.name, creator=instance.creator).exists(): + raise ValidationError(f"A private Agent with the name {instance.name} already exists.") + + slug = instance.name.lower().replace(" ", "-") + observed_random_numbers = set() + while Agent.objects.filter(slug=slug).exists(): + try: + random_number = choice([i for i in range(0, 1000) if i not in observed_random_numbers]) + except IndexError: + raise ValidationError("Unable to generate a unique slug for the Agent. Please try again later.") + observed_random_numbers.add(random_number) + slug = f"{slug}-{random_number}" + instance.slug = slug class NotionConfig(BaseModel): diff --git a/src/khoj/interface/web/404.html b/src/khoj/interface/web/404.html index 7041ff80..0762bde8 100644 --- a/src/khoj/interface/web/404.html +++ b/src/khoj/interface/web/404.html @@ -2,14 +2,19 @@ Khoj: An AI Personal Assistant for your digital brain - - + + + + + {% import 'utils.html' as utils %} + {{ utils.heading_pane(user_photo, username, is_active, has_documents) }} +
-

Oops, this is awkward. That page couldn't be found.

+

Oops, this is awkward. Looks like there's nothing here.

- Go Home + Go Home @@ -18,5 +23,34 @@ body.not-found { padding: 0 10% } + + body { + background-color: var(--background-color); + color: var(--main-text-color); + text-align: center; + font-family: var(--font-family); + font-size: medium; + font-weight: 300; + line-height: 1.5em; + height: 100vh; + margin: 0; + } + + body a.redirect-link { + font-size: 18px; + font-weight: bold; + background-color: var(--primary); + text-decoration: none; + border: 1px solid var(--main-text-color); + color: var(--main-text-color); + border-radius: 8px; + padding: 4px; + } + + body a.redirect-link:hover { + background-color: var(--main-text-color); + color: var(--primary); + } + diff --git a/src/khoj/interface/web/agent.html b/src/khoj/interface/web/agent.html new file mode 100644 index 00000000..6e6619cf --- /dev/null +++ b/src/khoj/interface/web/agent.html @@ -0,0 +1,286 @@ + + + + + Khoj - Agents + + + + + + + + + {% import 'utils.html' as utils %} + {{ utils.heading_pane(user_photo, username, is_active, has_documents) }} +
+
+
+
Agent Settings
+
+
+
+
+ Agent Avatar + +
+
Instructions
+
+

{{ agent.personality }}

+
+
+
+

Public

+ +
+ + + +
+
+
+ + + + + diff --git a/src/khoj/interface/web/agents.html b/src/khoj/interface/web/agents.html new file mode 100644 index 00000000..dc26606c --- /dev/null +++ b/src/khoj/interface/web/agents.html @@ -0,0 +1,201 @@ + + + + + Khoj - Agents + + + + + + + + + {% import 'utils.html' as utils %} + {{ utils.heading_pane(user_photo, username, is_active, has_documents) }} + + +
+
+
+

Agents

+ +
+ {% for agent in agents %} +
+ +
+ {{ agent.name }} +
+ +
+ +

{{ agent.name }}

+ +

{{ agent.personality }}

+
+
+ +
+
+ {% endfor %} +
+
+ + + + + diff --git a/src/khoj/interface/web/assets/khoj.css b/src/khoj/interface/web/assets/khoj.css index 7ba93c6a..3d7e7d4a 100644 --- a/src/khoj/interface/web/assets/khoj.css +++ b/src/khoj/interface/web/assets/khoj.css @@ -130,7 +130,7 @@ img.khoj-logo { background-color: var(--background-color); min-width: 160px; box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2); - right: 15vw; + right: 5vw; top: 64px; z-index: 1; opacity: 0; diff --git a/src/khoj/interface/web/base_config.html b/src/khoj/interface/web/base_config.html index 5c26e060..870f4eb9 100644 --- a/src/khoj/interface/web/base_config.html +++ b/src/khoj/interface/web/base_config.html @@ -162,7 +162,7 @@ height: 40px; } .card-title { - font-size: 20px; + font-size: medium; font-weight: normal; margin: 0; padding: 0; diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html index 35047c31..52750c12 100644 --- a/src/khoj/interface/web/chat.html +++ b/src/khoj/interface/web/chat.html @@ -5,6 +5,7 @@ Khoj - Chat + @@ -12,15 +13,16 @@ @@ -1205,13 +1328,27 @@ To get started, just start typing below. You can also type / to see a list of co +
-
+ + +