sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-11-27 17:35:07 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Format online, notes, code context with YAML to be legibile for LLM

Browse source
This commit is contained in:
Debanjum 2024-10-28 05:56:31 -07:00
parent 5bea0c705b
commit d75ce4a9e3
5 changed files with 28 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
src/khoj/processor/conversation/anthropic/anthropic_chat.py
View file

@ -19,6 +19,7 @@ from khoj.processor.conversation.utils import (
) )
from khoj.utils.helpers import ConversationCommand, is_none_or_empty from khoj.utils.helpers import ConversationCommand, is_none_or_empty
from khoj.utils.rawconfig import LocationData from khoj.utils.rawconfig import LocationData
from khoj.utils.yaml import yaml_dump
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -152,7 +153,6 @@ def converse_anthropic(
""" """
# Initialize Variables # Initialize Variables
current_date = datetime.now() current_date = datetime.now()
compiled_references = "\n\n".join({f"# File: {item['file']}\n## {item['compiled']}\n" for item in references})
if agent and agent.personality: if agent and agent.personality:
system_prompt = prompts.custom_personality.format( system_prompt = prompts.custom_personality.format(
@ -176,7 +176,7 @@ def converse_anthropic(
system_prompt = f"{system_prompt}\n{user_name_prompt}" system_prompt = f"{system_prompt}\n{user_name_prompt}"
# Get Conversation Primer appropriate to Conversation Type # Get Conversation Primer appropriate to Conversation Type
if conversation_commands == [ConversationCommand.Notes] and is_none_or_empty(compiled_references): if conversation_commands == [ConversationCommand.Notes] and is_none_or_empty(references):
completion_func(chat_response=prompts.no_notes_found.format()) completion_func(chat_response=prompts.no_notes_found.format())
return iter([prompts.no_notes_found.format()]) return iter([prompts.no_notes_found.format()])
elif conversation_commands == [ConversationCommand.Online] and is_none_or_empty(online_results): elif conversation_commands == [ConversationCommand.Online] and is_none_or_empty(online_results):
@ -184,12 +184,12 @@ def converse_anthropic(
return iter([prompts.no_online_results_found.format()]) return iter([prompts.no_online_results_found.format()])
context_message = "" context_message = ""
if not is_none_or_empty(compiled_references): if not is_none_or_empty(references):
context_message = f"{prompts.notes_conversation.format(query=user_query, references=compiled_references)}\n\n" context_message = f"{prompts.notes_conversation.format(query=user_query, references=yaml_dump(references))}\n\n"
if ConversationCommand.Online in conversation_commands or ConversationCommand.Webpage in conversation_commands: if ConversationCommand.Online in conversation_commands or ConversationCommand.Webpage in conversation_commands:
context_message += f"{prompts.online_search_conversation.format(online_results=str(online_results))}\n\n" context_message += f"{prompts.online_search_conversation.format(online_results=yaml_dump(online_results))}\n\n"
if ConversationCommand.Code in conversation_commands and not is_none_or_empty(code_results): if ConversationCommand.Code in conversation_commands and not is_none_or_empty(code_results):
context_message += f"{prompts.code_executed_context.format(code_results=str(code_results))}\n\n" context_message += f"{prompts.code_executed_context.format(code_results=yaml_dump(code_results))}\n\n"
context_message = context_message.strip() context_message = context_message.strip()
# Setup Prompt with Primer or Conversation History # Setup Prompt with Primer or Conversation History

12
src/khoj/processor/conversation/google/gemini_chat.py
View file

@ -19,6 +19,7 @@ from khoj.processor.conversation.utils import (
) )
from khoj.utils.helpers import ConversationCommand, is_none_or_empty from khoj.utils.helpers import ConversationCommand, is_none_or_empty
from khoj.utils.rawconfig import LocationData from khoj.utils.rawconfig import LocationData
from khoj.utils.yaml import yaml_dump
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -157,7 +158,6 @@ def converse_gemini(
""" """
# Initialize Variables # Initialize Variables
current_date = datetime.now() current_date = datetime.now()
compiled_references = "\n\n".join({f"# File: {item['file']}\n## {item['compiled']}\n" for item in references})
if agent and agent.personality: if agent and agent.personality:
system_prompt = prompts.custom_personality.format( system_prompt = prompts.custom_personality.format(
@ -182,7 +182,7 @@ def converse_gemini(
system_prompt = f"{system_prompt}\n{user_name_prompt}" system_prompt = f"{system_prompt}\n{user_name_prompt}"
# Get Conversation Primer appropriate to Conversation Type # Get Conversation Primer appropriate to Conversation Type
if conversation_commands == [ConversationCommand.Notes] and is_none_or_empty(compiled_references): if conversation_commands == [ConversationCommand.Notes] and is_none_or_empty(references):
completion_func(chat_response=prompts.no_notes_found.format()) completion_func(chat_response=prompts.no_notes_found.format())
return iter([prompts.no_notes_found.format()]) return iter([prompts.no_notes_found.format()])
elif conversation_commands == [ConversationCommand.Online] and is_none_or_empty(online_results): elif conversation_commands == [ConversationCommand.Online] and is_none_or_empty(online_results):
@ -190,12 +190,12 @@ def converse_gemini(
return iter([prompts.no_online_results_found.format()]) return iter([prompts.no_online_results_found.format()])
context_message = "" context_message = ""
if not is_none_or_empty(compiled_references): if not is_none_or_empty(references):
context_message = f"{prompts.notes_conversation.format(query=user_query, references=compiled_references)}\n\n" context_message = f"{prompts.notes_conversation.format(query=user_query, references=yaml_dump(references))}\n\n"
if ConversationCommand.Online in conversation_commands or ConversationCommand.Webpage in conversation_commands: if ConversationCommand.Online in conversation_commands or ConversationCommand.Webpage in conversation_commands:
context_message += f"{prompts.online_search_conversation.format(online_results=str(online_results))}\n\n" context_message += f"{prompts.online_search_conversation.format(online_results=yaml_dump(online_results))}\n\n"
if ConversationCommand.Code in conversation_commands and not is_none_or_empty(code_results): if ConversationCommand.Code in conversation_commands and not is_none_or_empty(code_results):
context_message += f"{prompts.code_executed_context.format(code_results=str(code_results))}\n\n" context_message += f"{prompts.code_executed_context.format(code_results=yaml_dump(code_results))}\n\n"
context_message = context_message.strip() context_message = context_message.strip()
# Setup Prompt with Primer or Conversation History # Setup Prompt with Primer or Conversation History

13
src/khoj/processor/conversation/offline/chat_model.py
View file

@ -19,6 +19,7 @@ from khoj.utils import state
from khoj.utils.constants import empty_escape_sequences from khoj.utils.constants import empty_escape_sequences
from khoj.utils.helpers import ConversationCommand, in_debug_mode, is_none_or_empty from khoj.utils.helpers import ConversationCommand, in_debug_mode, is_none_or_empty
from khoj.utils.rawconfig import LocationData from khoj.utils.rawconfig import LocationData
from khoj.utils.yaml import yaml_dump
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -185,26 +186,24 @@ def converse_offline(
system_prompt = f"{system_prompt}\n{user_name_prompt}" system_prompt = f"{system_prompt}\n{user_name_prompt}"
# Get Conversation Primer appropriate to Conversation Type # Get Conversation Primer appropriate to Conversation Type
if conversation_commands == [ConversationCommand.Notes] and is_none_or_empty(compiled_references): if conversation_commands == [ConversationCommand.Notes] and is_none_or_empty(references):
return iter([prompts.no_notes_found.format()]) return iter([prompts.no_notes_found.format()])
elif conversation_commands == [ConversationCommand.Online] and is_none_or_empty(online_results): elif conversation_commands == [ConversationCommand.Online] and is_none_or_empty(online_results):
completion_func(chat_response=prompts.no_online_results_found.format()) completion_func(chat_response=prompts.no_online_results_found.format())
return iter([prompts.no_online_results_found.format()]) return iter([prompts.no_online_results_found.format()])
context_message = "" context_message = ""
if not is_none_or_empty(compiled_references): if not is_none_or_empty(references):
context_message = f"{prompts.notes_conversation_offline.format(references=compiled_references)}\n\n" context_message = f"{prompts.notes_conversation_offline.format(references=yaml_dump(references))}\n\n"
if ConversationCommand.Online in conversation_commands or ConversationCommand.Webpage in conversation_commands: if ConversationCommand.Online in conversation_commands or ConversationCommand.Webpage in conversation_commands:
simplified_online_results = online_results.copy() simplified_online_results = online_results.copy()
for result in online_results: for result in online_results:
if online_results[result].get("webpages"): if online_results[result].get("webpages"):
simplified_online_results[result] = online_results[result]["webpages"] simplified_online_results[result] = online_results[result]["webpages"]
context_message += ( context_message += f"{prompts.online_search_conversation_offline.format(online_results=yaml_dump(simplified_online_results))}\n\n"
f"{prompts.online_search_conversation_offline.format(online_results=str(simplified_online_results))}\n\n"
)
if ConversationCommand.Code in conversation_commands and not is_none_or_empty(code_results): if ConversationCommand.Code in conversation_commands and not is_none_or_empty(code_results):
context_message += f"{prompts.code_executed_context.format(code_results=str(code_results))}\n\n" context_message += f"{prompts.code_executed_context.format(code_results=yaml_dump(code_results))}\n\n"
context_message = context_message.strip() context_message = context_message.strip()
# Setup Prompt with Primer or Conversation History # Setup Prompt with Primer or Conversation History

12
src/khoj/processor/conversation/openai/gpt.py
View file

@ -18,6 +18,7 @@ from khoj.processor.conversation.utils import (
) )
from khoj.utils.helpers import ConversationCommand, is_none_or_empty from khoj.utils.helpers import ConversationCommand, is_none_or_empty
from khoj.utils.rawconfig import LocationData from khoj.utils.rawconfig import LocationData
from khoj.utils.yaml import yaml_dump
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -155,7 +156,6 @@ def converse(
""" """
# Initialize Variables # Initialize Variables
current_date = datetime.now() current_date = datetime.now()
compiled_references = "\n\n".join({f"# File: {item['file']}\n## {item['compiled']}\n" for item in references})
if agent and agent.personality: if agent and agent.personality:
system_prompt = prompts.custom_personality.format( system_prompt = prompts.custom_personality.format(
@ -179,7 +179,7 @@ def converse(
system_prompt = f"{system_prompt}\n{user_name_prompt}" system_prompt = f"{system_prompt}\n{user_name_prompt}"
# Get Conversation Primer appropriate to Conversation Type # Get Conversation Primer appropriate to Conversation Type
if conversation_commands == [ConversationCommand.Notes] and is_none_or_empty(compiled_references): if conversation_commands == [ConversationCommand.Notes] and is_none_or_empty(references):
completion_func(chat_response=prompts.no_notes_found.format()) completion_func(chat_response=prompts.no_notes_found.format())
return iter([prompts.no_notes_found.format()]) return iter([prompts.no_notes_found.format()])
elif conversation_commands == [ConversationCommand.Online] and is_none_or_empty(online_results): elif conversation_commands == [ConversationCommand.Online] and is_none_or_empty(online_results):
@ -187,12 +187,12 @@ def converse(
return iter([prompts.no_online_results_found.format()]) return iter([prompts.no_online_results_found.format()])
context_message = "" context_message = ""
if not is_none_or_empty(compiled_references): if not is_none_or_empty(references):
context_message = f"{prompts.notes_conversation.format(references=compiled_references)}\n\n" context_message = f"{prompts.notes_conversation.format(references=yaml_dump(references))}\n\n"
if not is_none_or_empty(online_results): if not is_none_or_empty(online_results):
context_message += f"{prompts.online_search_conversation.format(online_results=str(online_results))}\n\n" context_message += f"{prompts.online_search_conversation.format(online_results=yaml_dump(online_results))}\n\n"
if not is_none_or_empty(code_results): if not is_none_or_empty(code_results):
context_message += f"{prompts.code_executed_context.format(code_results=str(code_results))}\n\n" context_message += f"{prompts.code_executed_context.format(code_results=yaml_dump(code_results))}\n\n"
context_message = context_message.strip() context_message = context_message.strip()
# Setup Prompt with Primer or Conversation History # Setup Prompt with Primer or Conversation History

4
src/khoj/utils/yaml.py
View file

@ -41,3 +41,7 @@ def parse_config_from_string(yaml_config: dict) -> FullConfig:
def parse_config_from_file(yaml_config_file): def parse_config_from_file(yaml_config_file):
"Parse and validate config in YML file" "Parse and validate config in YML file"
return parse_config_from_string(load_config_from_file(yaml_config_file)) return parse_config_from_string(load_config_from_file(yaml_config_file))
def yaml_dump(data):
return yaml.dump(data, allow_unicode=True, sort_keys=False, default_flow_style=False)