diff --git a/src/interface/obsidian/src/main.ts b/src/interface/obsidian/src/main.ts index 6d4af194..935945dd 100644 --- a/src/interface/obsidian/src/main.ts +++ b/src/interface/obsidian/src/main.ts @@ -38,9 +38,9 @@ export default class Khoj extends Plugin { id: 'chat', name: 'Chat', checkCallback: (checking) => { - if (!checking && this.settings.connectedToBackend && !!this.settings.openaiApiKey) + if (!checking && this.settings.connectedToBackend && (!!this.settings.openaiApiKey || this.settings.enableOfflineChat)) new KhojChatModal(this.app, this.settings).open(); - return !!this.settings.openaiApiKey; + return !!this.settings.openaiApiKey || this.settings.enableOfflineChat; } }); diff --git a/src/interface/obsidian/src/settings.ts b/src/interface/obsidian/src/settings.ts index bf4ed19f..c013f10c 100644 --- a/src/interface/obsidian/src/settings.ts +++ b/src/interface/obsidian/src/settings.ts @@ -2,6 +2,7 @@ import { App, Notice, PluginSettingTab, request, Setting } from 'obsidian'; import Khoj from 'src/main'; export interface KhojSetting { + enableOfflineChat: boolean; openaiApiKey: string; resultsCount: number; khojUrl: string; @@ -10,6 +11,7 @@ export interface KhojSetting { } export const DEFAULT_SETTINGS: KhojSetting = { + enableOfflineChat: false, resultsCount: 6, khojUrl: 'http://127.0.0.1:42110', connectedToBackend: false, @@ -35,7 +37,7 @@ export class KhojSettingTab extends PluginSettingTab { // Add khoj settings configurable from the plugin settings tab new Setting(containerEl) .setName('Khoj URL') - .setDesc('The URL of the Khoj backend') + .setDesc('The URL of the Khoj backend.') .addText(text => text .setValue(`${this.plugin.settings.khojUrl}`) .onChange(async (value) => { @@ -45,16 +47,25 @@ export class KhojSettingTab extends PluginSettingTab { })); new Setting(containerEl) .setName('OpenAI API Key') - .setDesc('Your OpenAI API Key for Khoj Chat') + .setDesc('Use OpenAI for Khoj Chat with your API key.') .addText(text => text .setValue(`${this.plugin.settings.openaiApiKey}`) .onChange(async (value) => { this.plugin.settings.openaiApiKey = value.trim(); await this.plugin.saveSettings(); })); + new Setting(containerEl) + .setName('Enable Offline Chat') + .setDesc('Chat privately without an internet connection. Enabling this will use offline chat even if OpenAI is configured.') + .addToggle(toggle => toggle + .setValue(this.plugin.settings.enableOfflineChat) + .onChange(async (value) => { + this.plugin.settings.enableOfflineChat = value; + await this.plugin.saveSettings(); + })); new Setting(containerEl) .setName('Results Count') - .setDesc('The number of results to show in search and use for chat') + .setDesc('The number of results to show in search and use for chat.') .addSlider(slider => slider .setLimits(1, 10, 1) .setValue(this.plugin.settings.resultsCount) @@ -65,7 +76,7 @@ export class KhojSettingTab extends PluginSettingTab { })); new Setting(containerEl) .setName('Auto Configure') - .setDesc('Automatically configure the Khoj backend') + .setDesc('Automatically configure the Khoj backend.') .addToggle(toggle => toggle .setValue(this.plugin.settings.autoConfigure) .onChange(async (value) => { @@ -75,7 +86,7 @@ export class KhojSettingTab extends PluginSettingTab { let indexVaultSetting = new Setting(containerEl); indexVaultSetting .setName('Index Vault') - .setDesc('Manually force Khoj to re-index your Obsidian Vault') + .setDesc('Manually force Khoj to re-index your Obsidian Vault.') .addButton(button => button .setButtonText('Update') .setCta() diff --git a/src/interface/obsidian/src/utils.ts b/src/interface/obsidian/src/utils.ts index 13d72de1..cb333310 100644 --- a/src/interface/obsidian/src/utils.ts +++ b/src/interface/obsidian/src/utils.ts @@ -9,6 +9,19 @@ export function getVaultAbsolutePath(vault: Vault): string { return ''; } +type OpenAIType = null | { + "chat-model": string; + "api-key": string; +}; + +interface ProcessorData { + conversation: { + "conversation-logfile": string; + openai: OpenAIType; + "enable-offline-chat": boolean; + }; +} + export async function configureKhojBackend(vault: Vault, setting: KhojSetting, notify: boolean = true) { let vaultPath = getVaultAbsolutePath(vault); let mdInVault = `${vaultPath}/**/*.md`; @@ -132,48 +145,36 @@ export async function configureKhojBackend(vault: Vault, setting: KhojSetting, n } } - // If OpenAI API key not set in Khoj plugin settings - if (!setting.openaiApiKey) { - // Disable khoj processors, as not required - delete data["processor"]; + let conversationLogFile = data?.["processor"]?.["conversation"]?.["conversation-logfile"] ?? `${khojDefaultChatDirectory}/conversation.json`; + + let processorData: ProcessorData = { + "conversation": { + "conversation-logfile": conversationLogFile, + "openai": null, + "enable-offline-chat": setting.enableOfflineChat, + } } - // Else if khoj backend not configured yet - else if (!khoj_already_configured || !data["processor"]) { - data["processor"] = { + + // If the Open AI API Key was configured in the plugin settings + if (!!setting.openaiApiKey) { + + let openAIChatModel = data?.["processor"]?.["conversation"]?.["openai"]?.["chat-model"] ?? khojDefaultChatModelName; + + processorData = { "conversation": { - "conversation-logfile": `${khojDefaultChatDirectory}/conversation.json`, + "conversation-logfile": conversationLogFile, "openai": { - "chat-model": khojDefaultChatModelName, + "chat-model": openAIChatModel, "api-key": setting.openaiApiKey, - } - }, - } - } - // Else if khoj config has no conversation processor config - else if (!data["processor"]["conversation"] || !data["processor"]["conversation"]["openai"]) { - data["processor"] = { - "conversation": { - "conversation-logfile": `${khojDefaultChatDirectory}/conversation.json`, - "openai": { - "chat-model": khojDefaultChatModelName, - "api-key": setting.openaiApiKey, - } - }, - } - } - // Else if khoj is not configured with OpenAI API key from khoj plugin settings - else if (data["processor"]["conversation"]["openai"]["api-key"] !== setting.openaiApiKey) { - data["processor"] = { - "conversation": { - "conversation-logfile": data["processor"]["conversation"]["conversation-logfile"], - "openai": { - "chat-model": data["processor"]["conversation"]["openai"]["chat-model"], - "api-key": setting.openaiApiKey, - } + }, + "enable-offline-chat": setting.enableOfflineChat, }, } } + // Set khoj processor config to conversation processor config + data["processor"] = processorData; + // Save updated config and refresh index on khoj backend updateKhojBackend(setting.khojUrl, data); if (!khoj_already_configured) diff --git a/src/khoj/configure.py b/src/khoj/configure.py index 5adde214..5e569f2a 100644 --- a/src/khoj/configure.py +++ b/src/khoj/configure.py @@ -169,7 +169,7 @@ def configure_content( try: # Initialize Org Notes Search - if (t == state.SearchType.Org or t == None) and content_config.org and search_models.text_search: + if (t == None or t.value == state.SearchType.Org.value) and content_config.org and search_models.text_search: logger.info("🦄 Setting up search for orgmode notes") # Extract Entries, Generate Notes Embeddings content_index.org = text_search.setup( @@ -181,7 +181,11 @@ def configure_content( ) # Initialize Markdown Search - if (t == state.SearchType.Markdown or t == None) and content_config.markdown and search_models.text_search: + if ( + (t == None or t.value == state.SearchType.Markdown.value) + and content_config.markdown + and search_models.text_search + ): logger.info("💎 Setting up search for markdown notes") # Extract Entries, Generate Markdown Embeddings content_index.markdown = text_search.setup( @@ -193,7 +197,7 @@ def configure_content( ) # Initialize PDF Search - if (t == state.SearchType.Pdf or t == None) and content_config.pdf and search_models.text_search: + if (t == None or t.value == state.SearchType.Pdf.value) and content_config.pdf and search_models.text_search: logger.info("🖨️ Setting up search for pdf") # Extract Entries, Generate PDF Embeddings content_index.pdf = text_search.setup( @@ -205,14 +209,22 @@ def configure_content( ) # Initialize Image Search - if (t == state.SearchType.Image or t == None) and content_config.image and search_models.image_search: + if ( + (t == None or t.value == state.SearchType.Image.value) + and content_config.image + and search_models.image_search + ): logger.info("🌄 Setting up search for images") # Extract Entries, Generate Image Embeddings content_index.image = image_search.setup( content_config.image, search_models.image_search.image_encoder, regenerate=regenerate ) - if (t == state.SearchType.Github or t == None) and content_config.github and search_models.text_search: + if ( + (t == None or t.value == state.SearchType.Github.value) + and content_config.github + and search_models.text_search + ): logger.info("🐙 Setting up search for github") # Extract Entries, Generate Github Embeddings content_index.github = text_search.setup( @@ -223,6 +235,21 @@ def configure_content( filters=[DateFilter(), WordFilter(), FileFilter()], ) + # Initialize Notion Search + if ( + (t == None or t.value in state.SearchType.Notion.value) + and content_config.notion + and search_models.text_search + ): + logger.info("🔌 Setting up search for notion") + content_index.notion = text_search.setup( + NotionToJsonl, + content_config.notion, + search_models.text_search.bi_encoder, + regenerate=regenerate, + filters=[DateFilter(), WordFilter(), FileFilter()], + ) + # Initialize External Plugin Search if (t == None or t in state.SearchType) and content_config.plugins and search_models.text_search: logger.info("🔌 Setting up search for plugins") @@ -236,17 +263,6 @@ def configure_content( filters=[DateFilter(), WordFilter(), FileFilter()], ) - # Initialize Notion Search - if (t == None or t in state.SearchType) and content_config.notion and search_models.text_search: - logger.info("🔌 Setting up search for notion") - content_index.notion = text_search.setup( - NotionToJsonl, - content_config.notion, - search_models.text_search.bi_encoder, - regenerate=regenerate, - filters=[DateFilter(), WordFilter(), FileFilter()], - ) - except Exception as e: logger.error(f"🚨 Failed to setup search: {e}", exc_info=True) raise e diff --git a/src/khoj/processor/github/github_to_jsonl.py b/src/khoj/processor/github/github_to_jsonl.py index ddfa6a67..58df09a9 100644 --- a/src/khoj/processor/github/github_to_jsonl.py +++ b/src/khoj/processor/github/github_to_jsonl.py @@ -38,6 +38,9 @@ class GithubToJsonl(TextToJsonl): return def process(self, previous_entries=[]): + if self.config.pat_token is None or self.config.pat_token == "": + logger.error(f"Github PAT token is not set. Skipping github content") + raise ValueError("Github PAT token is not set. Skipping github content") current_entries = [] for repo in self.config.repos: current_entries += self.process_repo(repo) diff --git a/tests/test_client.py b/tests/test_client.py index f1f4951b..c60b0ca2 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -53,13 +53,26 @@ def test_update_with_invalid_content_type(client): # ---------------------------------------------------------------------------------------------------- def test_update_with_valid_content_type(client): - for content_type in ["all", "org", "markdown", "image", "pdf", "github", "notion", "plugin1"]: + for content_type in ["all", "org", "markdown", "image", "pdf", "notion", "plugin1"]: # Act response = client.get(f"/api/update?t={content_type}") # Assert assert response.status_code == 200, f"Returned status: {response.status_code} for content type: {content_type}" +# ---------------------------------------------------------------------------------------------------- +def test_update_with_github_fails_without_pat(client): + # Act + response = client.get(f"/api/update?t=github") + + # Assert + assert response.status_code == 500, f"Returned status: {response.status_code} for content type: github" + assert ( + response.json()["detail"] + == "🚨 Failed to update server via API: Github PAT token is not set. Skipping github content" + ) + + # ---------------------------------------------------------------------------------------------------- def test_regenerate_with_invalid_content_type(client): # Act @@ -71,13 +84,26 @@ def test_regenerate_with_invalid_content_type(client): # ---------------------------------------------------------------------------------------------------- def test_regenerate_with_valid_content_type(client): - for content_type in ["all", "org", "markdown", "image", "pdf", "github", "notion", "plugin1"]: + for content_type in ["all", "org", "markdown", "image", "pdf", "notion", "plugin1"]: # Act response = client.get(f"/api/update?force=true&t={content_type}") # Assert assert response.status_code == 200, f"Returned status: {response.status_code} for content type: {content_type}" +# ---------------------------------------------------------------------------------------------------- +def test_regenerate_with_github_fails_without_pat(client): + # Act + response = client.get(f"/api/update?force=true&t=github") + + # Assert + assert response.status_code == 500, f"Returned status: {response.status_code} for content type: github" + assert ( + response.json()["detail"] + == "🚨 Failed to update server via API: Github PAT token is not set. Skipping github content" + ) + + # ---------------------------------------------------------------------------------------------------- def test_get_configured_types_via_api(client): # Act