diff --git a/docker-compose.yml b/docker-compose.yml index 1b79f784..3c5fe9f4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,11 +44,20 @@ services: - KHOJ_DEBUG=False - KHOJ_ADMIN_EMAIL=username@example.com - KHOJ_ADMIN_PASSWORD=password - # Uncomment the following lines to make your instance publicly accessible. - # Replace the domain with your domain. Proceed with caution, especially if you are using anonymous mode. + # Uncomment lines below to use chat models by each provider. + # Ensure you set your provider specific API keys. + # --- + # - OPENAI_API_KEY=your_openai_api_key + # - GEMINI_API_KEY=your_gemini_api_key + # - ANTHROPIC_API_KEY=your_anthropic_api_key + # Uncomment the necessary lines below to make your instance publicly accessible. + # Replace the KHOJ_DOMAIN with either your domain or IP address (no http/https prefix). + # Proceed with caution, especially if you are using anonymous mode. + # --- # - KHOJ_NO_HTTPS=True # - KHOJ_DOMAIN=192.168.0.104 - command: --host="0.0.0.0" --port=42110 -vv --anonymous-mode + # - KHOJ_DOMAIN=khoj.example.com + command: --host="0.0.0.0" --port=42110 -vv --anonymous-mode --non-interactive volumes: diff --git a/documentation/assets/img/example_search_model_admin_settings.png b/documentation/assets/img/example_search_model_admin_settings.png new file mode 100644 index 00000000..9efd0abb Binary files /dev/null and b/documentation/assets/img/example_search_model_admin_settings.png differ diff --git a/documentation/docs/advanced/admin.md b/documentation/docs/advanced/admin.md new file mode 100644 index 00000000..305741a8 --- /dev/null +++ b/documentation/docs/advanced/admin.md @@ -0,0 +1,73 @@ +# Admin Panel +> Describes the Khoj settings configurable via the admin panel + +## App Settings +### Agents +Add all the agents you want to use for your different use-cases like Writer, Researcher, Therapist etc. +- `Personality`: This is a prompt to tell the chat model how to tune the personality of the agent. +- `Chat model`: The chat model to use for the agent. +- `Name`: The name of the agent. This field helps give the agent a unique identity across the app. +- `Avatar`: Url to the agents profile picture. It help give the agent a unique visual identity across the app. +- `Style color`, `Style icon`: These fields help give the agent a unique, visually identifiable identity across the app. +- `Slug`: This is the agent name to use in urls. +- `Public`: Check this if the agent is expected to be visible to all users on this Khoj server. +- `Managed by admin`: Check this if the agent is managed by admin, not by any user. +- `Creator`: The user who created the agent. +- `Tools`: The list of tools available to this agent. Tools include notes, image, online. This field is not currently configurable and only supports all tools (i.e `["*"]`) + +### Chat Model Options +Add all the chat models you want to try, use and switch between for your different use-cases. For each chat model you add: +- `Chat model`: The name of an [OpenAI](https://platform.openai.com/docs/models), [Anthropic](https://docs.anthropic.com/en/docs/about-claude/models#model-names), [Gemini](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models) or [Offline](https://huggingface.co/models?pipeline_tag=text-generation&library=gguf) chat model. +- `Model type`: The chat model provider like `OpenAI`, `Offline`. +- `Vision enabled`: Set to `true` if your model supports vision. This is currently only supported for vision capable OpenAI models like `gpt-4o` +- `Max prompt size`, `Subscribed max prompt size`: These are optional fields. They are used to truncate the context to the maximum context size that can be passed to the model. This can help with accuracy and cost-saving.
+- `Tokenizer`: This is an optional field. It is used to accurately count tokens and truncate context passed to the chat model to stay within the models max prompt size. + ![example configuration for chat model options](/img/example_chatmodel_option.png) + +### Server Chat Settings +The server chat settings are used as: +1. The default chat models for subscribed (`Advanced` field) and unsubscribed (`Default` field) users. +2. The chat model for all intermediate steps like intent detection, web search etc. during chat response generation. + +If a server chat setting is not added the first ChatModelOption in your config is used as the default chat model. + +To add a server chat setting: +- Set your preferred default chat models in the `Default` fields of your [ServerChatSettings](http://localhost:42110/server/admin/database/serverchatsettings/) +- The `Advanced` field doesn't need to be set when self-hosting. When unset, the `Default` chat model is used for all users and the intermediate steps. + + +### OpenAI Processor Conversation Configs +These settings configure chat model providers to be accessed over API. +The name of this setting is kind of a misnomer, we know, it'll hopefully be changed at some point. +For each chat model provider you [add](http://localhost:42110/server/admin/database/openaiprocessorconversationconfig/add): +- `Api key`: Set to your [OpenAI](https://platform.openai.com/api-keys), [Anthropic](https://console.anthropic.com/account/keys) or [Gemini](https://aistudio.google.com/app/apikey) API keys. +- `Name`: Give the configuration any friendly name like `OpenAI`, `Gemini`, `Anthropic`. +- `Api base url`: Set the API base URL. This is only relevant to set if you're using another OpenAI-compatible proxy server like [Ollama](/advanced/ollama) or [LMStudio](/advanced/lmstudio). + ![example configuration for openai processor](/img/example_openai_processor_config.png) + +### Search Model Configs +Search models are used to generate vector embeddings of your documents for natural language search and chat. You can choose any [embeddings models on HuggingFace](https://huggingface.co/models?pipeline_tag=sentence-similarity) to try, use for your to create vector embeddings of your documents for natural language search and chat. + +Example Search Model Settings + +### Text to Image Model Options +Add text to image generation models with these settings. Khoj currently supports text to image models available via OpenAI, Stability or Replicate API +- `api-key`: Set to your OpenAI, Stability or Replicate API key +- `model`: Set the model name available over the selected model provider +- `model-type`: Set to the appropiate model provider +- `openai-config`: For image generation models available via OpenAI (compatible) API you can set the appropriate OpenAI Processor Conversation Settings instead of specifying the `api-key` field above + +### Speech to Text Model Options +Add speech to text models with these settings. Khoj currently only supports whisper speech to text model via OpenAI API or Offline + +### Voice Model Options +Add text to speech models with these settings. Khoj currently supports models from [ElevenLabs](https://elevenlabs.io/). + +## User Data +- Users, Entrys, Conversations, Subscriptions, Github configs, Notion configs, User search configs, User conversation configs, User voice configs + +## Miscellaneous Data +- Process Locks: Persistent Locks for Automations +- Client Applications: + +Client applications allow you to setup third party applications that can query your Khoj server using a client application ID + secret. The secret would go in a bearer token. diff --git a/documentation/docs/advanced/authentication.md b/documentation/docs/advanced/authentication.md index 456f0c0c..e5a002d4 100644 --- a/documentation/docs/advanced/authentication.md +++ b/documentation/docs/advanced/authentication.md @@ -7,7 +7,7 @@ This is only helpful for self-hosted users or teams. If you're using [Khoj Cloud By default, most of the instructions for self-hosting Khoj assume a single user, and so the default configuration is to run in anonymous mode. However, if you want to enable authentication, you can do so either with with [Magic Links](#using-magic-links) or [Google OAuth](#using-google-oauth) as shown below. This can be helpful to make Khoj securely accessible to you and your team. :::tip[Note] -Remove the `--anonymous-mode` flag in your start up command to enable authentication. +Remove the `--anonymous-mode` flag from your khoj start up command or docker-compose file to enable authentication. ::: ## Using Magic Links diff --git a/documentation/docs/advanced/remote.md b/documentation/docs/advanced/remote.md new file mode 100644 index 00000000..94aece1f --- /dev/null +++ b/documentation/docs/advanced/remote.md @@ -0,0 +1,20 @@ +# Remote Access + +By default self-hosted Khoj is only accessible on the machine it is running. To securely access it from a remote machine: +- Set the `KHOJ_DOMAIN` environment variable to your remotely accessible ip or domain via shell or docker-compose.yml. + Examples: `KHOJ_DOMAIN=my.khoj-domain.com`, `KHOJ_DOMAIN=192.168.0.4`. +- Ensure the Khoj Admin password and `KHOJ_DJANGO_SECRET_KEY` environment variable are securely set. +- Setup [Authentication](/advanced/authentication). +- Open access to the Khoj port (default: 42110) from your OS and Network firewall. + +:::warning[Use HTTPS certificate] +To expose Khoj on a custom domain over the public internet, use of an SSL certificate is strongly recommended. You can use [Let's Encrypt](https://letsencrypt.org/) to get a free SSL certificate for your domain. + +To disable HTTPS, set the `KHOJ_NO_HTTPS` environment variable to `True`. This can be useful if Khoj is only accessible behind a secure, private network. +::: + +:::info[Try Tailscale] +You can use [Tailscale](https://tailscale.com/) for easy, secure access to your self-hosted Khoj over the network. +1. Set `KHOJ_DOMAIN` to your machines [tailscale ip](https://tailscale.com/kb/1452/connect-to-devices#identify-your-devices) or [fqdn on tailnet](https://tailscale.com/kb/1081/magicdns#fully-qualified-domain-names-vs-machine-names). E.g `KHOJ_DOMAIN=100.4.2.0` or `KHOJ_DOMAIN=khoj.tailfe8c.ts.net` +2. Access Khoj by opening `http://tailscale-ip-of-server:42110` or `http://fqdn-of-server:42110` from any device on your tailscale network +::: diff --git a/documentation/docs/get-started/setup.mdx b/documentation/docs/get-started/setup.mdx index 40b892ae..4d479825 100644 --- a/documentation/docs/get-started/setup.mdx +++ b/documentation/docs/get-started/setup.mdx @@ -14,37 +14,63 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; ``` -## Setup +## Setup Khoj These are the general setup instructions for self-hosted Khoj. -You can install the Khoj server using either Docker or Pip. +You can install the Khoj server using either [Docker](?server=docker) or [Pip](?server=pip). :::info[Offline Model + GPU] If you want to use the offline chat model and you have a GPU, you should use Installation Option 2 - local setup via the Python package directly. Our Docker image doesn't currently support running the offline chat model on GPU, making inference times really slow. ::: -### 1A. Install Method 1: Docker + + + + +

Prerequisites

+ Install [Docker Desktop](https://docs.docker.com/desktop/install/mac-install/) + + +

Prerequisites

+ 1. Install [WSL2](https://learn.microsoft.com/en-us/windows/wsl/install) and restart your machine + ```shell + # Run in PowerShell + wsl --install + ``` + 2. Install [Docker Desktop](https://docs.docker.com/desktop/install/windows-install/) with **[WSL2 backend](https://docs.docker.com/desktop/wsl/#turn-on-docker-desktop-wsl-2)** (default) + + +

Prerequisites

+ Install [Docker Desktop](https://docs.docker.com/desktop/install/windows-install/). + You can also use your package manager to install Docker Engine & Docker Compose. + + -#### Prerequisites -1. Install Docker Engine. See [official instructions](https://docs.docker.com/engine/install/). -2. Ensure you have Docker Compose. See [official instructions](https://docs.docker.com/compose/install/). - -#### Setup - -1. Get the sample docker-compose file [from Github](https://github.com/khoj-ai/khoj/blob/master/docker-compose.yml). -2. Configure the environment variables in the docker-compose.yml to your choosing.
- Note: *Your admin account will automatically be created based on the admin credentials in that file, so pay attention to those.* -3. Now start the container by running the following command in the same directory as your docker-compose.yml file. This will automatically setup the database and run the Khoj server. +

Setup

+1. Download the Khoj docker-compose.yml file [from Github](https://github.com/khoj-ai/khoj/blob/master/docker-compose.yml) ```shell + # Windows users should use their WSL2 terminal to run these commands + mkdir ~/.khoj && cd ~/.khoj + wget https://raw.githubusercontent.com/khoj-ai/khoj/master/docker-compose.yml + ``` +2. Configure the environment variables in the docker-compose.yml + - Set `KHOJ_ADMIN_PASSWORD`, `KHOJ_DJANGO_SECRET_KEY` (and optionally the `KHOJ_ADMIN_EMAIL`) to something secure. This allows you to customize Khoj later via the admin panel. + - Set `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, or `GEMINI_API_KEY` to your API key if you want to use OpenAI, Anthropic or Gemini chat models respectively. +3. Start Khoj by running the following command in the same directory as your docker-compose.yml file. + ```shell + # Windows users should use their WSL2 terminal to run these commands + cd ~/.khoj docker-compose up ``` -Khoj should now be running at http://localhost:42110! You can see the web UI in your browser. +:::info[Remote Access] +By default Khoj is only accessible on the machine it is running. To access Khoj from a remote machine see [Remote Access Docs](/advanced/remote). +::: -### 1B. Install Method 2: Pip +Your setup is complete once you see `🌖 Khoj is ready to use` in the server logs on your terminal. + + -#### Prerequisites - -##### Install Postgres (with PgVector) +

1. Install Postgres (with PgVector)

Khoj uses Postgres DB for all server configuration and to scale to multi-user setups. It uses the pgvector package in Postgres to manage your document embeddings. Both Postgres and pgvector need to be installed for Khoj to work. @@ -66,32 +92,32 @@ Install [Postgres.app](https://postgresapp.com/). This comes pre-installed with -##### Create the Khoj database +

2. Create the Khoj database

-Make sure to update your environment variables to match your Postgres configuration if you're using a different name. The default values should work for most people. When prompted for a password, you can use the default password `postgres`, or configure it to your preference. Make sure to set the environment variable `POSTGRES_PASSWORD` to the same value as the password you set here. - - - + + ```shell -createdb khoj -U postgres --password + createdb khoj -U postgres --password ``` - - + + ```shell -createdb -U postgres khoj --password - ``` - - - ```shell -sudo -u postgres createdb khoj --password - ``` - - + createdb -U postgres khoj --password + ``` + + + ```shell + sudo -u postgres createdb khoj --password + ``` + + -#### Install Khoj server +:::info[Postgres Env Config] +Make sure to update the `POSTGRES_HOST`, `POSTGRES_PORT`, `POSTGRES_USER`, `POSTGRES_DB` or `POSTGRES_PASSWORD` environment variables to match any customizations in your Postgres configuration. +::: -##### Install Khoj Server -- *Make sure [python](https://realpython.com/installing-python/) and [pip](https://pip.pypa.io/en/stable/installation/) are installed on your machine* +

3. Install Khoj Server

+- Make sure [python](https://realpython.com/installing-python/) and [pip](https://pip.pypa.io/en/stable/installation/) are installed on your machine - Check [llama-cpp-python setup](https://python.langchain.com/docs/integrations/llms/llamacpp#installation) if you hit any llama-cpp issues with the installation Run the following command in your terminal to install the Khoj server. @@ -107,7 +133,7 @@ python -m pip install khoj ``` - In PowerShell on Windows + Run the following command in PowerShell on Windows ```shell # 1. (Optional) To use NVIDIA (CUDA) GPU $env:CMAKE_ARGS = "-DLLAMA_OPENBLAS=on" @@ -134,56 +160,47 @@ python -m pip install khoj -##### Start Khoj Server +

4. Start Khoj Server

-Before getting started, configure the following environment variables in your terminal for the first run - - - - ```shell - export KHOJ_ADMIN_EMAIL= - export KHOJ_ADMIN_PASSWORD= - ``` - - - If you're using PowerShell: - ```shell - $env:KHOJ_ADMIN_EMAIL="" - $env:KHOJ_ADMIN_PASSWORD="" - ``` - - - ```shell - export KHOJ_ADMIN_EMAIL= - export KHOJ_ADMIN_PASSWORD= - ``` - - - - -Run the following command from your terminal to start the Khoj backend and open Khoj in your browser. +Run the following command from your terminal to start the Khoj service. ```shell khoj --anonymous-mode ``` -`--anonymous-mode` allows you to run the server without setting up Google credentials for login. This allows you to use any of the clients without a login wall. If you want to use Google login, you can skip this flag, but you will have to add your Google developer credentials. +`--anonymous-mode` allows access to Khoj without requiring login. This is usually fine for local only, single user setups. If you need authentication follow the [authentication setup docs](/advanced/authentication). -On the first run, you will be prompted to input credentials for your admin account and do some basic configuration for your chat model settings. Once created, you can go to http://localhost:42110/server/admin and login with the credentials you just created. - -Khoj should now be running at http://localhost:42110. You can see the web UI in your browser. - -Note: To start Khoj automatically in the background use [Task scheduler](https://www.windowscentral.com/how-create-automated-task-using-task-scheduler-windows-10) on Windows or [Cron](https://en.wikipedia.org/wiki/Cron) on Mac, Linux (e.g. with `@reboot khoj`) +

First Run

+On the first run of the above command, you will be prompted to: +1. Create an admin account with a email and secure password +2. Customize the chat models to enable + - Keep your [OpenAI](https://platform.openai.com/api-keys), [Anthropic](https://console.anthropic.com/account/keys), [Gemini](https://aistudio.google.com/app/apikey) API keys and [OpenAI](https://platform.openai.com/docs/models), [Anthropic](https://docs.anthropic.com/en/docs/about-claude/models#model-names), [Gemini](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models), [Offline](https://huggingface.co/models?pipeline_tag=text-generation&library=gguf) chat model names handy to set any of them up during first run. +3. Your setup is complete once you see `🌖 Khoj is ready to use` in the server logs on your terminal! -### 2. Configure -#### Login to the Khoj Admin Panel +:::tip[Auto Start] +To start Khoj automatically in the background use [Task scheduler](https://www.windowscentral.com/how-create-automated-task-using-task-scheduler-windows-10) on Windows or [Cron](https://en.wikipedia.org/wiki/Cron) on Mac, Linux (e.g. with `@reboot khoj`) +::: + + + +## Use Khoj + +You can now open the web app at http://localhost:42110 and start interacting!
+Nothing else is necessary, but you can customize your setup further by following the steps below. + +:::info[First Message to Offline Chat Model] +The offline chat model gets downloaded when you first send a message to it. The download can take a few minutes! Subsequent messages should be faster. +::: + +### Add Chat Models +

Login to the Khoj Admin Panel

Go to http://localhost:42110/server/admin and login with the admin credentials you setup during installation. :::info[CSRF Error] Ensure you are using **localhost, not 127.0.0.1**, to access the admin panel to avoid the CSRF error. ::: -:::info[DISALLOWED HOST Error] +:::info[DISALLOWED HOST or Bad Request (400) Error] You may hit this if you try access Khoj exposed on a custom domain (e.g. 192.168.12.3 or example.com) or over HTTP. Set the environment variables KHOJ_DOMAIN=your-domain and KHOJ_NO_HTTPS=false if required to avoid this error. ::: @@ -192,93 +209,132 @@ Set the environment variables KHOJ_DOMAIN=your-domain and KHOJ_NO_HTTPS=false if Using Safari on Mac? You might not be able to login to the admin panel. Try using Chrome or Firefox instead. ::: -#### Configure Chat Model +

Configure Chat Model

Setup which chat model you'd want to use. Khoj supports local and online chat models. -:::tip[Multiple Chat Models] -Add a `ServerChatSettings` with `Default` and `Summarizer` fields set to your preferred chat model via [the admin panel](http://localhost:42110/server/admin/database/serverchatsettings/add/). Otherwise Khoj defaults to use the first chat model in your [ChatModelOptions](http://localhost:42110/server/admin/database/chatmodeloptions/) for all non chat response generation tasks. -::: - -##### Configure OpenAI Chat + + :::info[Ollama Integration] Using Ollama? See the [Ollama Integration](/advanced/ollama) section for more custom setup instructions. ::: -1. Go to the [OpenAI settings](http://localhost:42110/server/admin/database/openaiprocessorconversationconfig/) in the server admin settings to add an OpenAI processor conversation config. This is where you set your API key and server API base URL. The API base URL is optional - it's only relevant if you're using another OpenAI-compatible proxy server. - +1. Create a new [OpenAI processor conversation config](http://localhost:42110/server/admin/database/openaiprocessorconversationconfig/add) in the server admin settings. This is kind of a misnomer, we know. + - Add your [OpenAI API key](https://platform.openai.com/api-keys) + - Give the configuration a friendly name like `OpenAI` + - (Optional) Set the API base URL. It is only relevant if you're using another OpenAI-compatible proxy server like [Ollama](/advanced/ollama) or [LMStudio](/advanced/lmstudio). ![example configuration for openai processor](/img/example_openai_processor_config.png) - -2. Go over to configure your [chat model options](http://localhost:42110/server/admin/database/chatmodeloptions/). Set the `chat-model` field to a supported chat model[^1] of your choice. For example, you can specify `gpt-4o` if you're using OpenAI. +2. Create a new [chat model options](http://localhost:42110/server/admin/database/chatmodeloptions/add) + - Set the `chat-model` field to an [OpenAI chat model](https://platform.openai.com/docs/models). Example: `gpt-4o`. - Make sure to set the `model-type` field to `OpenAI`. - - The `tokenizer` and `max-prompt-size` fields are optional. Set them only if you're sure of the tokenizer or token limit for the model you're using. Contact us if you're unsure what to do here. - If your model supports vision, set the `vision enabled` field to `true`. This is currently only supported for OpenAI models with vision capabilities. - -![example configuration for chat model options](/img/example_chatmodel_option.png) - -##### Configure Anthropic Chat -1. Go to the [OpenAI settings](http://localhost:42110/server/admin/database/openaiprocessorconversationconfig/) in the server admin settings to add an OpenAI processor conversation config. This is kind of a misnomer, we know. Do not configure the API base url. Just add your API key and give the configuration a friendly name. -2. Go over to configure your [chat model options](http://localhost:42110/server/admin/database/chatmodeloptions/). Set the `chat-model` field to a supported chat model by Anthropic of your choice. For example, you can specify `claude-3-5-sonnet-20240620`. - - Make sure to set the `model-type` field to `Anthropic`. - The `tokenizer` and `max-prompt-size` fields are optional. Set them only if you're sure of the tokenizer or token limit for the model you're using. Contact us if you're unsure what to do here. +![example configuration for chat model options](/img/example_chatmodel_option.png) + + +1. Create a new [OpenAI processor conversation config](http://localhost:42110/server/admin/database/openaiprocessorconversationconfig/add) in the server admin settings. This is kind of a misnomer, we know. + - Add your [Anthropic API key](https://console.anthropic.com/account/keys) + - Give the configuration a friendly name like `Anthropic`. Do not configure the API base url. +2. Create a new [chat model options](http://localhost:42110/server/admin/database/chatmodeloptions/add) + - Set the `chat-model` field to an [Anthropic chat model](https://docs.anthropic.com/en/docs/about-claude/models#model-names). Example: `claude-3-5-sonnet-20240620`. + - Set the `model-type` field to `Anthropic`. + - Set the `Openai config` field to the OpenAI processor conversation config for Anthropic you created in step 1. + + +1. Create a new [OpenAI processor conversation config](http://localhost:42110/server/admin/database/openaiprocessorconversationconfig/add) in the server admin settings. This is kind of a misnomer, we know. + - Add your [Gemini API key](https://aistudio.google.com/app/apikey) + - Give the configuration a friendly name like `Gemini`. Do not configure the API base url. +2. Create a new [chat model options](http://localhost:42110/server/admin/database/chatmodeloptions/add) + - Set the `chat-model` field to a [Google Gemini chat model](https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models). Example: `gemini-1.5-flash`. + - Set the `model-type` field to `Gemini`. + - Set the `Openai config` field to the OpenAI processor conversation config for Gemini you created in step 1. -##### Configure Offline Chat + + +Offline chat stays completely private and can work without internet using any open-weights model. -Offline chat stays completely private and can work without internet using open-source models. - -**System Requirements**: +:::tip[System Requirements] - Minimum 8 GB RAM. Recommend **16Gb VRAM** - Minimum **5 GB of Disk** available -- A CPU supporting [AVX or AVX2 instructions](https://en.wikipedia.org/wiki/Advanced_Vector_Extensions) is required -- An Nvidia, AMD GPU or a Mac M1+ machine would significantly speed up chat response times +- A Nvidia, AMD GPU or a Mac M1+ machine would significantly speed up chat responses +::: -Any chat model on Huggingface in GGUF format can be used for local chat. Here's how you can set it up: +1. Get the name of your preferred chat model from [HuggingFace](https://huggingface.co/models?pipeline_tag=text-generation&library=gguf). *Most GGUF format chat models are supported*. +2. Open the [create chat model page](http://localhost:42110/server/admin/database/chatmodeloptions/add/) on the admin panel +3. Set the `chat-model` field to the name of your preferred chat model + - Make sure the `model-type` is set to `Offline` +4. Set the newly added chat model as your preferred model in your [User chat settings](http://localhost:42110/settings) and [Server chat settings](http://localhost:42110/server/admin/database/serverchatsettings/). +5. Restart the Khoj server and [start chatting](http://localhost:42110) with your new offline model! + + -1. No need to setup a conversation processor config! -2. Go over to configure your [chat model options](http://localhost:42110/server/admin/database/chatmodeloptions/). Set the `chat-model` field to a supported chat model[^1] of your choice. For example, we recommend `bartowski/Meta-Llama-3.1-8B-Instruct-GGUF`, but [any gguf model on huggingface](https://huggingface.co/models?library=gguf) should work. - - Make sure to set the `model-type` to `Offline`. Do not set `openai config`. - - The `tokenizer` and `max-prompt-size` fields are optional. You can set these for non-standard models (i.e not Mistral or Llama based models) or when you know the token limit of the model to improve context stuffing. +:::tip[Multiple Chat Models] +Set your preferred default chat model in the `Default`, `Advanced` fields of your [ServerChatSettings](http://localhost:42110/server/admin/database/serverchatsettings/). +Khoj uses these chat model for all intermediate steps like intent detection, web search etc. +::: -#### Share your data -You can sync your files and folders with Khoj using the [Desktop](/clients/desktop#setup), [Obsidian](/clients/obsidian#setup), or [Emacs](/clients/emacs#setup) clients or just drag and drop specific files on the [website](/clients/web#upload-documents). You can also directly sync your [Notion workspace](/data-sources/notion_integration). +:::info[Chat Model Fields] + - The `tokenizer` and `max-prompt-size` fields are optional. Set them only if you're sure of the tokenizer or token limit for the model you're using. This improves context stuffing. Contact us if you're unsure what to do here. + - Only tick the `vision enabled` field for OpenAI models with vision capabilities like gpt-4o. Vision capabilities in other chat models is not currently utilized. +::: -[^1]: Khoj, by default, can use [OpenAI GPT3.5+ chat models](https://platform.openai.com/docs/models/overview) or [GGUF chat models](https://huggingface.co/models?library=gguf). See [this section](/advanced/use-openai-proxy) on how to locally use OpenAI-format compatible proxy servers. +### Sync your Knowledge -### 3. Use Khoj 🚀 +- You can chat with your notes and documents using Khoj. +- Khoj can keep your files and folders synced using the Khoj [Desktop](/clients/desktop#setup), [Obsidian](/clients/obsidian#setup) or [Emacs](/clients/emacs#setup) clients. +- Your [Notion workspace](/data-sources/notion_integration) can be directly synced from the web app. +- You can also just drag and drop specific files you want to chat with on the [Web app](/clients/web#upload-documents). -Now open http://localhost:42110 to start interacting with Khoj! +### Setup Khoj Clients +The Khoj web app is available by default to chat, search and configure Khoj.
+You can also install a Khoj client to easily access it from Obsidian, Emacs, Whatsapp or your OS and keep your documents synced with Khoj. -### 4. Install Khoj Clients (Optional) -Khoj exposes a web interface to search, chat and configure by default.
-You can install a Khoj client to sync your documents or to easily access Khoj from within Obsidian, Emacs or your OS. - -- **Khoj Desktop**:
-[Install](/clients/desktop#setup) the Khoj Desktop app. - -- **Khoj Obsidian**:
-[Install](/clients/obsidian#setup) the Khoj Obsidian plugin. - -- **Khoj Emacs**:
-[Install](/clients/emacs#setup) khoj.el - -#### Setup host URL +:::info[Note] Set the host URL on your clients settings page to your Khoj server URL. By default, use `http://127.0.0.1:42110` or `http://localhost:42110`. Note that `localhost` may not work in all cases. +::: + + + + - Read the Khoj Desktop app [setup docs](/clients/desktop#setup). + + + - Read the Khoj Emacs package [setup docs](/clients/emacs#setup). + + + - Read the Khoj Obsidian plugin [setup docs](/clients/obsidian#setup). + + + - Read the Khoj Whatsapp app [setup docs](/clients/whatsapp). + + ## Upgrade - - +### Upgrade Server + + ```shell pip install --upgrade khoj ``` *Note: To upgrade to the latest pre-release version of the khoj server run below command* - From the same directory where you have your `docker-compose` file, this will fetch the latest build and upgrade your server. + Run the commands below from the same directory where you have your `docker-compose.yml` file. + This will fetch the latest build and upgrade your server. ```shell + # Windows users should use their WSL2 terminal to run these commands + cd ~/.khoj # assuming your khoj docker-compose.yml file is here docker-compose up --build ``` + + +### Upgrade Clients + + + - The Desktop app automatically updates to the latest released version on restart. + - You can manually download the latest version from the [Khoj Website](https://khoj.dev/downloads). + - Use your Emacs Package Manager to Upgrade - See [khoj.el package setup](/clients/emacs#setup) for details @@ -290,9 +346,9 @@ Set the host URL on your clients settings page to your Khoj server URL. By defau ## Uninstall - - - +### Uninstall Server + + ```shell # uninstall khoj server pip uninstall khoj @@ -302,19 +358,25 @@ Set the host URL on your clients settings page to your Khoj server URL. By defau ``` - From the same directory where you have your `docker-compose` file, run the command below to remove the server to delete its containers, networks, images and volumes. + Run the command below from the same directory where you have your `docker-compose` file. + This will remove the server containers, networks, images and volumes. ```shell docker-compose down --volumes ``` + + +### Uninstall Clients + + + Uninstall the Khoj Desktop client in the standard way from your OS. + - Uninstall the khoj Emacs, or desktop client in the standard way from Emacs or your OS respectively - You can also `rm -rf ~/.khoj` to remove the Khoj data directory if did a local install. + Uninstall the Khoj Emacs package in the standard way from Emacs. - Uninstall the khoj Obisidan, or desktop client in the standard way from Obsidian or your OS respectively - You can also `rm -rf ~/.khoj` to remove the Khoj data directory if did a local install. + Uninstall via the Community plugins tab on the settings pane in the Obsidian app @@ -331,7 +393,6 @@ Set the host URL on your clients settings page to your Khoj server URL. By defau ``` 3. Now start `khoj` using the standard steps described earlier - #### Install fails while building Tokenizer dependency - **Details**: `pip install khoj` fails while building the `tokenizers` dependency. Complains about Rust. - **Fix**: Install Rust to build the tokenizers package. For example on Mac run: @@ -342,18 +403,6 @@ Set the host URL on your clients settings page to your Khoj server URL. By defau ``` - **Refer**: [Issue with Fix](https://github.com/khoj-ai/khoj/issues/82#issuecomment-1241890946) for more details - #### Khoj in Docker errors out with \"Killed\" in error message - **Fix**: Increase RAM available to Docker Containers in Docker Settings - **Refer**: [StackOverflow Solution](https://stackoverflow.com/a/50770267), [Configure Resources on Docker for Mac](https://docs.docker.com/desktop/mac/#resources) - -## Advanced -### Self Host on Custom Domain - -You can self-host Khoj behind a custom domain as well. To do so, you need to set the `KHOJ_DOMAIN` environment variable to your domain (e.g., `export KHOJ_DOMAIN=my-khoj-domain.com` or add it to your `docker-compose.yml`). By default, the Khoj server you set up will not be accessible outside of `localhost` or `127.0.0.1`. - -:::warning[Without HTTPS certificate] -To expose Khoj on a custom domain over the public internet, use of an SSL certificate is strongly recommended. You can use [Let's Encrypt](https://letsencrypt.org/) to get a free SSL certificate for your domain. - -To disable HTTPS, set the `KHOJ_NO_HTTPS` environment variable to `True`. This can be useful if Khoj is only accessible behind a secure, private network. -::: diff --git a/documentation/yarn.lock b/documentation/yarn.lock index 48c26f36..677b8d7a 100644 --- a/documentation/yarn.lock +++ b/documentation/yarn.lock @@ -1794,9 +1794,9 @@ config-chain "^1.1.11" "@polka/url@^1.0.0-next.24": - version "1.0.0-next.25" - resolved "https://registry.yarnpkg.com/@polka/url/-/url-1.0.0-next.25.tgz#f077fdc0b5d0078d30893396ff4827a13f99e817" - integrity sha512-j7P6Rgr3mmtdkeDGTe0E/aYyWEWVtc5yFXtHCRHs28/jptDEWfaVOc5T7cblqy1XKPPfCxJc/8DwQ5YgLOZOVQ== + version "1.0.0-next.28" + resolved "https://registry.yarnpkg.com/@polka/url/-/url-1.0.0-next.28.tgz#d45e01c4a56f143ee69c54dd6b12eade9e270a73" + integrity sha512-8LduaNlMZGwdZ6qWrKlfa+2M4gahzFkprZiAt2TF8uS0qQgBizKXpXURqvTJ4WtmupWxaLqjRb2UCTe72mu+Aw== "@sideway/address@^4.1.5": version "4.1.5" @@ -2009,9 +2009,9 @@ "@types/estree" "*" "@types/estree@*", "@types/estree@^1.0.0", "@types/estree@^1.0.5": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" - integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== + version "1.0.6" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.6.tgz#628effeeae2064a1b4e79f78e81d87b7e5fc7b50" + integrity sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw== "@types/express-serve-static-core@*", "@types/express-serve-static-core@^4.17.33": version "4.19.5" @@ -2126,9 +2126,9 @@ "@types/node" "*" "@types/node@*": - version "22.5.4" - resolved "https://registry.yarnpkg.com/@types/node/-/node-22.5.4.tgz#83f7d1f65bc2ed223bdbf57c7884f1d5a4fa84e8" - integrity sha512-FDuKUJQm/ju9fT/SeX/6+gBzoPzlVCzfzmGkwKvRHQVxi4BntVbyIwf6a4Xn62mrvndLiml6z/UBXIdEVjQLXg== + version "22.5.5" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.5.5.tgz#52f939dd0f65fc552a4ad0b392f3c466cc5d7a44" + integrity sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA== dependencies: undici-types "~6.19.2" @@ -2148,14 +2148,14 @@ integrity sha512-rlAnzkW2sZOjbqZ743IHUhFcvzaGbqijwOu8QZnZCjfQzBqFE3s4lOTJEsxikImav9uzz/42I+O7YUs1mWgMlg== "@types/prop-types@*": - version "15.7.12" - resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.12.tgz#12bb1e2be27293c1406acb6af1c3f3a1481d98c6" - integrity sha512-5zvhXYtRNRluoE/jAp4GVsSduVUzNWKkOZrCDBWYtE7biZywwdC2AcEzg+cSMLFRfVgeAFqpfNabiPjxFddV1Q== + version "15.7.13" + resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.13.tgz#2af91918ee12d9d32914feb13f5326658461b451" + integrity sha512-hCZTSvwbzWGvhqxp/RqVqwU999pBf2vp7hzIjiYOsl8wqOmUxkQ6ddw1cV3l8811+kdUFus/q4d1Y3E3SyEifA== "@types/qs@*": - version "6.9.15" - resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.15.tgz#adde8a060ec9c305a82de1babc1056e73bd64dce" - integrity sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg== + version "6.9.16" + resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.16.tgz#52bba125a07c0482d26747d5d4947a64daf8f794" + integrity sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A== "@types/range-parser@*": version "1.2.7" @@ -2189,9 +2189,9 @@ "@types/react" "*" "@types/react@*": - version "18.3.5" - resolved "https://registry.yarnpkg.com/@types/react/-/react-18.3.5.tgz#5f524c2ad2089c0ff372bbdabc77ca2c4dbadf8f" - integrity sha512-WeqMfGJLGuLCqHGYRGHxnKrXcTitc6L/nBUWfWPcTarG3t9PsquqUMuVeXZeca+mglY4Vo5GZjCi0A3Or2lnxA== + version "18.3.8" + resolved "https://registry.yarnpkg.com/@types/react/-/react-18.3.8.tgz#1672ab19993f8aca7c7dc844c07d5d9e467d5a79" + integrity sha512-syBUrW3/XpnW4WJ41Pft+I+aPoDVbrBVQGEnbD7NijDGlVC+8gV/XKRY+7vMDlfPpbwYt0l1vd/Sj8bJGMbs9Q== dependencies: "@types/prop-types" "*" csstype "^3.0.2" @@ -2613,9 +2613,9 @@ autoprefixer@^10.4.14, autoprefixer@^10.4.19: postcss-value-parser "^4.2.0" babel-loader@^9.1.3: - version "9.1.3" - resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-9.1.3.tgz#3d0e01b4e69760cc694ee306fe16d358aa1c6f9a" - integrity sha512-xG3ST4DglodGf8qSwv0MdeWLhrDsw/32QMdTO5T1ZIp9gQur0HkCyFs7Awskr10JKXFXwpAhiCuYX5oGXnRGbw== + version "9.2.1" + resolved "https://registry.yarnpkg.com/babel-loader/-/babel-loader-9.2.1.tgz#04c7835db16c246dd19ba0914418f3937797587b" + integrity sha512-fqe8naHt46e0yIdkjUZYqddSXfej3AHajX+CSO5X7oy0EmPc6o5Xh+RClNoHjnieWz9AW4kZxW9yyFMhVB1QLA== dependencies: find-cache-dir "^4.0.0" schema-utils "^4.0.0" @@ -2838,9 +2838,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001646: - version "1.0.30001660" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001660.tgz#31218de3463fabb44d0b7607b652e56edf2e2355" - integrity sha512-GacvNTTuATm26qC74pt+ad1fW15mlQ/zuTzzY1ZoIzECTP8HURDfF43kNxPgf7H1jmelCBQTTbBNxdSXOA7Bqg== + version "1.0.30001662" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001662.tgz#3574b22dfec54a3f3b6787331da1040fe8e763ec" + integrity sha512-sgMUVwLmGseH8ZIrm1d51UbrhqMCH3jvS7gF/M6byuHOnKyLOBL7W8yz5V02OHwgLGA36o/AFhWzzh4uc5aqTA== ccount@^2.0.0: version "2.0.1" @@ -3625,9 +3625,9 @@ ee-first@1.1.1: integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== electron-to-chromium@^1.5.4: - version "1.5.19" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.19.tgz#aeaa0a076f3f0f0e8db2c57fd10158508f00725a" - integrity sha512-kpLJJi3zxTR1U828P+LIUDZ5ohixyo68/IcYOHLqnbTPr/wdgn4i1ECvmALN9E16JPA6cvCG5UG79gVwVdEK5w== + version "1.5.27" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.27.tgz#5203ce5d6054857d84ba84d3681cbe59132ade78" + integrity sha512-o37j1vZqCoEgBuWWXLHQgTN/KDKe7zwpiY5CPeq2RvUqOyJw9xnrULzZAEVQ5p4h+zjMk7hgtOoPdnLxr7m/jw== emoji-regex@^8.0.0: version "8.0.0" @@ -3868,9 +3868,9 @@ execa@^5.0.0: strip-final-newline "^2.0.0" express@^4.17.3: - version "4.20.0" - resolved "https://registry.yarnpkg.com/express/-/express-4.20.0.tgz#f1d08e591fcec770c07be4767af8eb9bcfd67c48" - integrity sha512-pLdae7I6QqShF5PnNTCVn4hI91Dx0Grkn2+IAsMTgMIKuQVte2dN9PeGSSAME2FR8anOhVA62QDIUaWVfEXVLw== + version "4.21.0" + resolved "https://registry.yarnpkg.com/express/-/express-4.21.0.tgz#d57cb706d49623d4ac27833f1cbc466b668eb915" + integrity sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng== dependencies: accepts "~1.3.8" array-flatten "1.1.1" @@ -3884,7 +3884,7 @@ express@^4.17.3: encodeurl "~2.0.0" escape-html "~1.0.3" etag "~1.8.1" - finalhandler "1.2.0" + finalhandler "1.3.1" fresh "0.5.2" http-errors "2.0.0" merge-descriptors "1.0.3" @@ -3893,11 +3893,11 @@ express@^4.17.3: parseurl "~1.3.3" path-to-regexp "0.1.10" proxy-addr "~2.0.7" - qs "6.11.0" + qs "6.13.0" range-parser "~1.2.1" safe-buffer "5.2.1" send "0.19.0" - serve-static "1.16.0" + serve-static "1.16.2" setprototypeof "1.2.0" statuses "2.0.1" type-is "~1.6.18" @@ -3997,13 +3997,13 @@ fill-range@^7.1.1: dependencies: to-regex-range "^5.0.1" -finalhandler@1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.2.0.tgz#7d23fe5731b207b4640e4fcd00aec1f9207a7b32" - integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== +finalhandler@1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.3.1.tgz#0c575f1d1d324ddd1da35ad7ece3df7d19088019" + integrity sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ== dependencies: debug "2.6.9" - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" on-finished "2.4.1" parseurl "~1.3.3" @@ -4726,10 +4726,10 @@ inline-style-parser@0.1.1: resolved "https://registry.yarnpkg.com/inline-style-parser/-/inline-style-parser-0.1.1.tgz#ec8a3b429274e9c0a1f1c4ffa9453a7fef72cea1" integrity sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q== -inline-style-parser@0.2.3: - version "0.2.3" - resolved "https://registry.yarnpkg.com/inline-style-parser/-/inline-style-parser-0.2.3.tgz#e35c5fb45f3a83ed7849fe487336eb7efa25971c" - integrity sha512-qlD8YNDqyTKTyuITrDOffsl6Tdhv+UC4hcdAVuQsK4IMQ99nSgd1MIA/Q+jQYoh9r3hVUXhYh7urSRmXPkW04g== +inline-style-parser@0.2.4: + version "0.2.4" + resolved "https://registry.yarnpkg.com/inline-style-parser/-/inline-style-parser-0.2.4.tgz#f4af5fe72e612839fcd453d989a586566d695f22" + integrity sha512-0aO8FkhNZlj/ZIbNi7Lxxr12obT7cL1moPfE4tg1LkX7LlLfC6DeX4l2ZEud1ukP9jNQyNnfzQVqwbwmAATY4Q== interpret@^1.0.0: version "1.4.0" @@ -5323,9 +5323,9 @@ mdast-util-gfm@^3.0.0: mdast-util-to-markdown "^2.0.0" mdast-util-mdx-expression@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.0.tgz#4968b73724d320a379110d853e943a501bfd9d87" - integrity sha512-fGCu8eWdKUKNu5mohVGkhBXCXGnOTLuFqOvGMvdikr+J1w7lDJgxThOKpwRWzzbyXAU2hhSwsmssOY4yTokluw== + version "2.0.1" + resolved "https://registry.yarnpkg.com/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz#43f0abac9adc756e2086f63822a38c8d3c3a5096" + integrity sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ== dependencies: "@types/estree-jsx" "^1.0.0" "@types/hast" "^3.0.0" @@ -5484,9 +5484,9 @@ micromark-core-commonmark@^2.0.0: micromark-util-types "^2.0.0" micromark-extension-directive@^3.0.0: - version "3.0.1" - resolved "https://registry.yarnpkg.com/micromark-extension-directive/-/micromark-extension-directive-3.0.1.tgz#67b3985bb991a69dbcae52664c57ee54b22f635a" - integrity sha512-VGV2uxUzhEZmaP7NSFo2vtq7M2nUD+WfmYQD+d8i/1nHbzE+rMy9uzTvUybBbNiVbrhOZibg3gbyoARGqgDWyg== + version "3.0.2" + resolved "https://registry.yarnpkg.com/micromark-extension-directive/-/micromark-extension-directive-3.0.2.tgz#2eb61985d1995a7c1ff7621676a4f32af29409e8" + integrity sha512-wjcXHgk+PPdmvR58Le9d7zQYWy+vKEU9Se44p2CrCDPiLr2FMyiT4Fyb5UFKFC66wGB3kPlgD7q3TnoqPS7SZA== dependencies: devlop "^1.0.0" micromark-factory-space "^2.0.0" @@ -6326,9 +6326,9 @@ path-to-regexp@2.2.1: integrity sha512-gu9bD6Ta5bwGrrU8muHzVOBFFREpp2iRkVfhBJahwJ6p6Xw20SjT0MxLnwkjOibQmGSYhiUnf2FLe7k+jcFmGQ== path-to-regexp@^1.7.0: - version "1.8.0" - resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.8.0.tgz#887b3ba9d84393e87a0a0b9f4cb756198b53548a" - integrity sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA== + version "1.9.0" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.9.0.tgz#5dc0753acbf8521ca2e0f137b4578b917b10cf24" + integrity sha512-xIp7/apCFJuUHdDLWe8O1HIkb0kQrOMb/0u6FXQjemHn/ii5LrIzU6bdECnsiTF/GjZkMEKg1xdiZwNqDYlZ6g== dependencies: isarray "0.0.1" @@ -6346,7 +6346,7 @@ periscopic@^3.0.0: estree-walker "^3.0.0" is-reference "^3.0.0" -picocolors@^1.0.0, picocolors@^1.0.1: +picocolors@^1.0.0, picocolors@^1.0.1, picocolors@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.0.tgz#5358b76a78cde483ba5cef6a9dc9671440b27d59" integrity sha512-TQ92mBOW0l3LeMeyLV6mzy/kWr8lkd/hp3mTg7wYK7zJhuBStmGMBG0BdeDZS/dZx1IukaX6Bk11zcln25o1Aw== @@ -6651,13 +6651,13 @@ postcss-zindex@^6.0.2: integrity sha512-5BxW9l1evPB/4ZIc+2GobEBoKC+h8gPGCMi+jxsYvd2x0mjq7wazk6DrP71pStqxE9Foxh5TVnonbWpFZzXaYg== postcss@^8.4.21, postcss@^8.4.24, postcss@^8.4.26, postcss@^8.4.33, postcss@^8.4.38: - version "8.4.45" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.45.tgz#538d13d89a16ef71edbf75d895284ae06b79e603" - integrity sha512-7KTLTdzdZZYscUc65XmjFiB73vBhBfbPztCYdUNvlaso9PrzjzcmjqBPR0lNGkcVlcO4BjiO5rK/qNz+XAen1Q== + version "8.4.47" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.47.tgz#5bf6c9a010f3e724c503bf03ef7947dcb0fea365" + integrity sha512-56rxCq7G/XfB4EkXq9Egn5GCqugWvDFjafDOThIdMBsI15iqPqR5r15TfSr1YPYeEI19YeaXMCbY6u88Y76GLQ== dependencies: nanoid "^3.3.7" - picocolors "^1.0.1" - source-map-js "^1.2.0" + picocolors "^1.1.0" + source-map-js "^1.2.1" pretty-error@^4.0.0: version "4.0.0" @@ -6742,13 +6742,6 @@ pupa@^3.1.0: dependencies: escape-goat "^4.0.0" -qs@6.11.0: - version "6.11.0" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" - integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== - dependencies: - side-channel "^1.0.4" - qs@6.13.0: version "6.13.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.13.0.tgz#6ca3bd58439f7e245655798997787b0d88a51906" @@ -6993,9 +6986,9 @@ recursive-readdir@^2.2.2: minimatch "^3.0.5" regenerate-unicode-properties@^10.1.0: - version "10.1.1" - resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-10.1.1.tgz#6b0e05489d9076b04c436f318d9b067bba459480" - integrity sha512-X007RyZLsCJVVrjgEFVpLUTZwyOZk3oiL75ZcuYjlIWd6rNJtOjkBwQc5AsRrpbKVkxN6sklw/k/9m2jJYOf8Q== + version "10.2.0" + resolved "https://registry.yarnpkg.com/regenerate-unicode-properties/-/regenerate-unicode-properties-10.2.0.tgz#626e39df8c372338ea9b8028d1f99dc3fd9c3db0" + integrity sha512-DqHn3DwbmmPVzeKj9woBadqmXxLvQoQIwu7nopMc72ztvxVmVk2SBhSnx67zuye5TP+lJsb/TBQsjLKhnDf3MA== dependencies: regenerate "^1.4.2" @@ -7125,9 +7118,9 @@ remark-parse@^11.0.0: unified "^11.0.0" remark-rehype@^11.0.0: - version "11.1.0" - resolved "https://registry.yarnpkg.com/remark-rehype/-/remark-rehype-11.1.0.tgz#d5f264f42bcbd4d300f030975609d01a1697ccdc" - integrity sha512-z3tJrAs2kIs1AqIIy6pzHmAHlF1hWQ+OdY4/hv+Wxe35EhyLKcajL33iUEn3ScxtFox9nUvRufR/Zre8Q08H/g== + version "11.1.1" + resolved "https://registry.yarnpkg.com/remark-rehype/-/remark-rehype-11.1.1.tgz#f864dd2947889a11997c0a2667cd6b38f685bca7" + integrity sha512-g/osARvjkBXb6Wo0XvAeXQohVta8i84ACbenPpoSsxTOQH/Ae0/RGP4WZgnMH5pMLpsj4FG7OHmcIcXxpza8eQ== dependencies: "@types/hast" "^3.0.0" "@types/mdast" "^4.0.0" @@ -7333,25 +7326,6 @@ semver@^7.3.2, semver@^7.3.5, semver@^7.3.7, semver@^7.5.4: resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== -send@0.18.0: - version "0.18.0" - resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" - integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== - dependencies: - debug "2.6.9" - depd "2.0.0" - destroy "1.2.0" - encodeurl "~1.0.2" - escape-html "~1.0.3" - etag "~1.8.1" - fresh "0.5.2" - http-errors "2.0.0" - mime "1.6.0" - ms "2.1.3" - on-finished "2.4.1" - range-parser "~1.2.1" - statuses "2.0.1" - send@0.19.0: version "0.19.0" resolved "https://registry.yarnpkg.com/send/-/send-0.19.0.tgz#bbc5a388c8ea6c048967049dbeac0e4a3f09d7f8" @@ -7405,15 +7379,15 @@ serve-index@^1.9.1: mime-types "~2.1.17" parseurl "~1.3.2" -serve-static@1.16.0: - version "1.16.0" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.16.0.tgz#2bf4ed49f8af311b519c46f272bf6ac3baf38a92" - integrity sha512-pDLK8zwl2eKaYrs8mrPZBJua4hMplRWJ1tIFksVC3FtBEBnl8dxgeHtsaMS8DhS9i4fLObaon6ABoc4/hQGdPA== +serve-static@1.16.2: + version "1.16.2" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.16.2.tgz#b6a5343da47f6bdd2673848bf45754941e803296" + integrity sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw== dependencies: - encodeurl "~1.0.2" + encodeurl "~2.0.0" escape-html "~1.0.3" parseurl "~1.3.3" - send "0.18.0" + send "0.19.0" set-function-length@^1.2.1: version "1.2.2" @@ -7475,7 +7449,7 @@ shelljs@^0.8.5: interpret "^1.0.0" rechoir "^0.6.2" -side-channel@^1.0.4, side-channel@^1.0.6: +side-channel@^1.0.6: version "1.0.6" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.6.tgz#abd25fb7cd24baf45466406b1096b7831c9215f2" integrity sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA== @@ -7553,7 +7527,7 @@ sort-css-media-queries@2.2.0: resolved "https://registry.yarnpkg.com/sort-css-media-queries/-/sort-css-media-queries-2.2.0.tgz#aa33cf4a08e0225059448b6c40eddbf9f1c8334c" integrity sha512-0xtkGhWCC9MGt/EzgnvbbbKhqWjl1+/rncmhTh5qCpbYguXh6S/qwePfv/JQ8jePXXmqingylxoC49pCkSPIbA== -source-map-js@^1.0.1, source-map-js@^1.2.0: +source-map-js@^1.0.1, source-map-js@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.2.1.tgz#1ce5650fddd87abc099eda37dcff024c2667ae46" integrity sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA== @@ -7720,11 +7694,11 @@ style-to-object@^0.4.0: inline-style-parser "0.1.1" style-to-object@^1.0.0: - version "1.0.7" - resolved "https://registry.yarnpkg.com/style-to-object/-/style-to-object-1.0.7.tgz#8604fb6018ac3db83e97207a4f85f579068f661c" - integrity sha512-uSjr59G5u6fbxUfKbb8GcqMGT3Xs9v5IbPkjb0S16GyOeBLAzSRK0CixBv5YrYvzO6TDLzIS6QCn78tkqWngPw== + version "1.0.8" + resolved "https://registry.yarnpkg.com/style-to-object/-/style-to-object-1.0.8.tgz#67a29bca47eaa587db18118d68f9d95955e81292" + integrity sha512-xT47I/Eo0rwJmaXC4oilDGDWLohVhR6o/xAQcPQN8q6QBuZVL8qMYL85kLmST5cPjAorwvqIA4qXTRQoYHaL6g== dependencies: - inline-style-parser "0.2.3" + inline-style-parser "0.2.4" stylehacks@^6.1.1: version "6.1.1" @@ -7800,9 +7774,9 @@ terser-webpack-plugin@^5.3.10, terser-webpack-plugin@^5.3.9: terser "^5.26.0" terser@^5.10.0, terser@^5.15.1, terser@^5.26.0: - version "5.32.0" - resolved "https://registry.yarnpkg.com/terser/-/terser-5.32.0.tgz#ee811c0d2d6b741c1cc34a2bc5bcbfc1b5b1f96c" - integrity sha512-v3Gtw3IzpBJ0ugkxEX8U0W6+TnPKRRCWGh1jC/iM/e3Ki5+qvO1L1EAZ56bZasc64aXHwRHNIQEzm6//i5cemQ== + version "5.33.0" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.33.0.tgz#8f9149538c7468ffcb1246cfec603c16720d2db1" + integrity sha512-JuPVaB7s1gdFKPKTelwUyRq5Sid2A3Gko2S0PncwdBq7kN9Ti9HPWDQ06MPsEDGsZeVESjKEnyGy68quBk1w6g== dependencies: "@jridgewell/source-map" "^0.3.3" acorn "^8.8.2" @@ -7897,9 +7871,9 @@ undici-types@~6.19.2: integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== unicode-canonical-property-names-ecmascript@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz#301acdc525631670d39f6146e0e77ff6bbdebddc" - integrity sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ== + version "2.0.1" + resolved "https://registry.yarnpkg.com/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.1.tgz#cb3173fe47ca743e228216e4a3ddc4c84d628cc2" + integrity sha512-dA8WbNeb2a6oQzAQ55YlT5vQAWGV9WXOsi3SskE3bcCdM0P4SDd+24zS/OCacdRq5BkdsRj9q3Pg6YyQoxIGqg== unicode-emoji-modifier-base@^1.0.0: version "1.0.0" @@ -7915,9 +7889,9 @@ unicode-match-property-ecmascript@^2.0.0: unicode-property-aliases-ecmascript "^2.0.0" unicode-match-property-value-ecmascript@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.1.0.tgz#cb5fffdcd16a05124f5a4b0bf7c3770208acbbe0" - integrity sha512-qxkjQt6qjg/mYscYMC0XKRn3Rh0wFPlfxB0xkt9CfyTvpX1Ra0+rAmdX2QyAobptSEvuy4RtpPRui6XkV+8wjA== + version "2.2.0" + resolved "https://registry.yarnpkg.com/unicode-match-property-value-ecmascript/-/unicode-match-property-value-ecmascript-2.2.0.tgz#a0401aee72714598f739b68b104e4fe3a0cb3c71" + integrity sha512-4IehN3V/+kkr5YeSSDDQG8QLqO26XpL2XP3GQtqwlT/QYSECAwFztxVHjlbh0+gjJ3XmNLS0zDsbgs9jWKExLg== unicode-property-aliases-ecmascript@^2.0.0: version "2.1.0" diff --git a/pyproject.toml b/pyproject.toml index b21fc1d5..727bbfe0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ dependencies = [ "psycopg2-binary == 2.9.9", "lxml == 4.9.3", "tzdata == 2023.3", - "rapidocr-onnxruntime == 1.3.22", + "rapidocr-onnxruntime == 1.3.24", "openai-whisper >= 20231117", "django-phonenumber-field == 7.3.0", "phonenumbers == 8.13.27", diff --git a/src/khoj/main.py b/src/khoj/main.py index 0a492e91..272c9bb8 100644 --- a/src/khoj/main.py +++ b/src/khoj/main.py @@ -131,7 +131,7 @@ def run(should_start_server=True): logger.info(f"📦 Initializing DB:\n{db_migrate_output.getvalue().strip()}") logger.debug(f"🌍 Initializing Web Client:\n{collectstatic_output.getvalue().strip()}") - initialization() + initialization(not args.non_interactive) # Create app directory, if it doesn't exist state.config_file.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/khoj/processor/content/images/image_to_entries.py b/src/khoj/processor/content/images/image_to_entries.py index d28518b7..6dcd728c 100644 --- a/src/khoj/processor/content/images/image_to_entries.py +++ b/src/khoj/processor/content/images/image_to_entries.py @@ -4,8 +4,6 @@ import os from datetime import datetime from typing import Dict, List, Tuple -from rapidocr_onnxruntime import RapidOCR - from khoj.database.models import Entry as DbEntry from khoj.database.models import KhojUser from khoj.processor.content.text_to_entries import TextToEntries @@ -58,7 +56,6 @@ class ImageToEntries(TextToEntries): entry_to_location_map: List[Tuple[str, str]] = [] for image_file in image_files: try: - loader = RapidOCR() bytes = image_files[image_file] # write the image to a temporary file timestamp_now = datetime.utcnow().timestamp() @@ -71,13 +68,18 @@ class ImageToEntries(TextToEntries): bytes = image_files[image_file] f.write(bytes) try: + from rapidocr_onnxruntime import RapidOCR + + loader = RapidOCR() image_entries_per_file = "" result, _ = loader(tmp_file) if result: expanded_entries = [text[1] for text in result] image_entries_per_file = " ".join(expanded_entries) except ImportError: - logger.warning(f"Unable to process file: {image_file}. This file will not be indexed.") + logger.warning( + f"Unable to process image or scanned file for text: {image_file}. This file will not be indexed." + ) continue entry_to_location_map.append((image_entries_per_file, image_file)) entries.extend([image_entries_per_file]) diff --git a/src/khoj/processor/conversation/utils.py b/src/khoj/processor/conversation/utils.py index b7794361..3f397798 100644 --- a/src/khoj/processor/conversation/utils.py +++ b/src/khoj/processor/conversation/utils.py @@ -18,13 +18,20 @@ from khoj.utils.helpers import is_none_or_empty, merge_dicts logger = logging.getLogger(__name__) model_to_prompt_size = { + # OpenAI Models "gpt-3.5-turbo": 12000, - "gpt-3.5-turbo-0125": 12000, - "gpt-4-0125-preview": 20000, "gpt-4-turbo-preview": 20000, + "gpt-4o": 20000, "gpt-4o-mini": 20000, "o1-preview": 20000, "o1-mini": 20000, + # Google Models + "gemini-1.5-flash": 20000, + "gemini-1.5-pro": 20000, + # Anthropic Models + "claude-3-5-sonnet-20240620": 20000, + "claude-3-opus-20240229": 20000, + # Offline Models "TheBloke/Mistral-7B-Instruct-v0.2-GGUF": 3500, "NousResearch/Hermes-2-Pro-Mistral-7B-GGUF": 3500, "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF": 20000, @@ -163,7 +170,7 @@ def generate_chatml_messages_with_context( if loaded_model: max_prompt_size = infer_max_tokens(loaded_model.n_ctx(), model_to_prompt_size.get(model_name, math.inf)) else: - max_prompt_size = model_to_prompt_size.get(model_name, 2000) + max_prompt_size = model_to_prompt_size.get(model_name, 10000) # Scale lookback turns proportional to max prompt size supported by model lookback_turns = max_prompt_size // 750 diff --git a/src/khoj/utils/cli.py b/src/khoj/utils/cli.py index dd3388d9..2e2bb77d 100644 --- a/src/khoj/utils/cli.py +++ b/src/khoj/utils/cli.py @@ -50,6 +50,12 @@ def cli(args=None): default=False, help="Run Khoj in anonymous mode. This does not require any login for connecting users.", ) + parser.add_argument( + "--non-interactive", + action="store_true", + default=False, + help="Start Khoj in non-interactive mode. Assumes interactive shell unavailable for config. E.g when run via Docker.", + ) args, remaining_args = parser.parse_known_args(args) diff --git a/src/khoj/utils/constants.py b/src/khoj/utils/constants.py index bbbbd000..d91cc84a 100644 --- a/src/khoj/utils/constants.py +++ b/src/khoj/utils/constants.py @@ -8,8 +8,15 @@ empty_escape_sequences = "\n|\r|\t| " app_env_filepath = "~/.khoj/env" telemetry_server = "https://khoj.beta.haletic.com/v1/telemetry" content_directory = "~/.khoj/content/" -default_offline_chat_model = "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF" -default_online_chat_model = "gpt-4o-mini" +default_offline_chat_models = [ + "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF", + "bartowski/gemma-2-9b-it-GGUF", + "bartowski/gemma-2-2b-it-GGUF", + "bartowski/Phi-3.5-mini-instruct-GGUF", +] +default_openai_chat_models = ["gpt-4o-mini", "gpt-4o"] +default_gemini_chat_models = ["gemini-1.5-flash", "gemini-1.5-pro"] +default_anthropic_chat_models = ["claude-3-5-sonnet-20240620", "claude-3-opus-20240229"] empty_config = { "search-type": { diff --git a/src/khoj/utils/initialization.py b/src/khoj/utils/initialization.py index 8ef0ef53..90bb9921 100644 --- a/src/khoj/utils/initialization.py +++ b/src/khoj/utils/initialization.py @@ -1,25 +1,37 @@ import logging import os +from typing import Tuple from khoj.database.adapters import ConversationAdapters from khoj.database.models import ( ChatModelOptions, KhojUser, OpenAIProcessorConversationConfig, + ServerChatSettings, SpeechToTextModelOptions, TextToImageModelConfig, ) from khoj.processor.conversation.utils import model_to_prompt_size, model_to_tokenizer -from khoj.utils.constants import default_offline_chat_model, default_online_chat_model +from khoj.utils.constants import ( + default_anthropic_chat_models, + default_gemini_chat_models, + default_offline_chat_models, + default_openai_chat_models, +) logger = logging.getLogger(__name__) -def initialization(): +def initialization(interactive: bool = True): def _create_admin_user(): logger.info( "👩‍✈️ Setting up admin user. These credentials will allow you to configure your server at /server/admin." ) + if not interactive and (not os.getenv("KHOJ_ADMIN_EMAIL") or not os.getenv("KHOJ_ADMIN_PASSWORD")): + logger.error( + "🚨 Admin user cannot be created. Please set the KHOJ_ADMIN_EMAIL, KHOJ_ADMIN_PASSWORD environment variables or start server in interactive mode." + ) + exit(1) email_addr = os.getenv("KHOJ_ADMIN_EMAIL") or input("Email: ") password = os.getenv("KHOJ_ADMIN_PASSWORD") or input("Password: ") admin_user = KhojUser.objects.create_superuser(email=email_addr, username=email_addr, password=password) @@ -27,87 +39,103 @@ def initialization(): def _create_chat_configuration(): logger.info( - "🗣️ Configure chat models available to your server. You can always update these at /server/admin using the credentials of your admin account" + "🗣️ Configure chat models available to your server. You can always update these at /server/admin using your admin account" ) - try: - use_offline_model = input("Use offline chat model? (y/n): ") - if use_offline_model == "y": - logger.info("🗣️ Setting up offline chat model") - - offline_chat_model = input( - f"Enter the offline chat model you want to use. See HuggingFace for available GGUF models (default: {default_offline_chat_model}): " - ) - if offline_chat_model == "": - ChatModelOptions.objects.create( - chat_model=default_offline_chat_model, model_type=ChatModelOptions.ModelType.OFFLINE - ) - else: - default_max_tokens = model_to_prompt_size.get(offline_chat_model, 2000) - max_tokens = input( - f"Enter the maximum number of tokens to use for the offline chat model (default {default_max_tokens}):" - ) - max_tokens = max_tokens or default_max_tokens - - default_tokenizer = model_to_tokenizer.get( - offline_chat_model, "hf-internal-testing/llama-tokenizer" - ) - tokenizer = input( - f"Enter the tokenizer to use for the offline chat model (default: {default_tokenizer}):" - ) - tokenizer = tokenizer or default_tokenizer - - ChatModelOptions.objects.create( - chat_model=offline_chat_model, - model_type=ChatModelOptions.ModelType.OFFLINE, - max_prompt_size=max_tokens, - tokenizer=tokenizer, - ) - except ModuleNotFoundError as e: - logger.warning("Offline models are not supported on this device.") - - use_openai_model = input("Use OpenAI models? (y/n): ") - if use_openai_model == "y": - logger.info("🗣️ Setting up your OpenAI configuration") - api_key = input("Enter your OpenAI API key: ") - OpenAIProcessorConversationConfig.objects.create(api_key=api_key) - - openai_chat_model = input( - f"Enter the OpenAI chat model you want to use (default: {default_online_chat_model}): " - ) - openai_chat_model = openai_chat_model or default_online_chat_model - - default_max_tokens = model_to_prompt_size.get(openai_chat_model, 2000) - max_tokens = input( - f"Enter the maximum number of tokens to use for the OpenAI chat model (default: {default_max_tokens}): " - ) - max_tokens = max_tokens or default_max_tokens - ChatModelOptions.objects.create( - chat_model=openai_chat_model, model_type=ChatModelOptions.ModelType.OPENAI, max_prompt_size=max_tokens - ) + # Set up OpenAI's online chat models + openai_configured, openai_provider = _setup_chat_model_provider( + ChatModelOptions.ModelType.OPENAI, + default_openai_chat_models, + default_api_key=os.getenv("OPENAI_API_KEY"), + vision_enabled=True, + is_offline=False, + interactive=interactive, + ) + # Setup OpenAI speech to text model + if openai_configured: default_speech2text_model = "whisper-1" - openai_speech2text_model = input( - f"Enter the OpenAI speech to text model you want to use (default: {default_speech2text_model}): " - ) - openai_speech2text_model = openai_speech2text_model or default_speech2text_model + if interactive: + openai_speech2text_model = input( + f"Enter the OpenAI speech to text model you want to use (default: {default_speech2text_model}): " + ) + openai_speech2text_model = openai_speech2text_model or default_speech2text_model + else: + openai_speech2text_model = default_speech2text_model SpeechToTextModelOptions.objects.create( model_name=openai_speech2text_model, model_type=SpeechToTextModelOptions.ModelType.OPENAI ) + # Setup OpenAI text to image model + if openai_configured: default_text_to_image_model = "dall-e-3" - openai_text_to_image_model = input( - f"Enter the OpenAI text to image model you want to use (default: {default_text_to_image_model}): " - ) - openai_speech2text_model = openai_text_to_image_model or default_text_to_image_model + if interactive: + openai_text_to_image_model = input( + f"Enter the OpenAI text to image model you want to use (default: {default_text_to_image_model}): " + ) + openai_text_to_image_model = openai_text_to_image_model or default_text_to_image_model + else: + openai_text_to_image_model = default_text_to_image_model TextToImageModelConfig.objects.create( - model_name=openai_text_to_image_model, model_type=TextToImageModelConfig.ModelType.OPENAI + model_name=openai_text_to_image_model, + model_type=TextToImageModelConfig.ModelType.OPENAI, + openai_config=openai_provider, ) - if use_offline_model == "y" or use_openai_model == "y": - logger.info("🗣️ Chat model configuration complete") + # Set up Google's Gemini online chat models + _setup_chat_model_provider( + ChatModelOptions.ModelType.GOOGLE, + default_gemini_chat_models, + default_api_key=os.getenv("GEMINI_API_KEY"), + vision_enabled=False, + is_offline=False, + interactive=interactive, + provider_name="Google Gemini", + ) - use_offline_speech2text_model = input("Use offline speech to text model? (y/n): ") + # Set up Anthropic's online chat models + _setup_chat_model_provider( + ChatModelOptions.ModelType.ANTHROPIC, + default_anthropic_chat_models, + default_api_key=os.getenv("ANTHROPIC_API_KEY"), + vision_enabled=False, + is_offline=False, + interactive=interactive, + ) + + # Set up offline chat models + _setup_chat_model_provider( + ChatModelOptions.ModelType.OFFLINE, + default_offline_chat_models, + default_api_key=None, + vision_enabled=False, + is_offline=True, + interactive=interactive, + ) + + # Explicitly set default chat model + chat_models_configured = ChatModelOptions.objects.count() + if chat_models_configured > 0: + default_chat_model_name = ChatModelOptions.objects.first().chat_model + # If there are multiple chat models, ask the user to choose the default chat model + if chat_models_configured > 1 and interactive: + user_chat_model_name = input( + f"Enter the default chat model to use (default: {default_chat_model_name}): " + ) + else: + user_chat_model_name = None + + # If the user's choice is valid, set it as the default chat model + if user_chat_model_name and ChatModelOptions.objects.filter(chat_model=user_chat_model_name).exists(): + default_chat_model_name = user_chat_model_name + + # Create a server chat settings object with the default chat model + default_chat_model = ChatModelOptions.objects.filter(chat_model=default_chat_model_name).first() + ServerChatSettings.objects.create(chat_default=default_chat_model) + logger.info("🗣️ Chat model configuration complete") + + # Set up offline speech to text model + use_offline_speech2text_model = "n" if not interactive else input("Use offline speech to text model? (y/n): ") if use_offline_speech2text_model == "y": logger.info("🗣️ Setting up offline speech to text model") # Delete any existing speech to text model options. There can only be one. @@ -124,6 +152,64 @@ def initialization(): logger.info(f"🗣️ Offline speech to text model configured to {offline_speech2text_model}") + def _setup_chat_model_provider( + model_type: ChatModelOptions.ModelType, + default_chat_models: list, + default_api_key: str, + interactive: bool, + vision_enabled: bool = False, + is_offline: bool = False, + provider_name: str = None, + ) -> Tuple[bool, OpenAIProcessorConversationConfig]: + supported_vision_models = ["gpt-4o-mini", "gpt-4o"] + provider_name = provider_name or model_type.name.capitalize() + default_use_model = {True: "y", False: "n"}[default_api_key is not None or is_offline] + use_model_provider = ( + default_use_model if not interactive else input(f"Add {provider_name} chat models? (y/n): ") + ) + + if use_model_provider != "y": + return False, None + + logger.info(f"️💬 Setting up your {provider_name} chat configuration") + + chat_model_provider = None + if not is_offline: + if interactive: + user_api_key = input(f"Enter your {provider_name} API key (default: {default_api_key}): ") + api_key = user_api_key if user_api_key != "" else default_api_key + else: + api_key = default_api_key + chat_model_provider = OpenAIProcessorConversationConfig.objects.create(api_key=api_key, name=provider_name) + + if interactive: + chat_model_names = input( + f"Enter the {provider_name} chat models you want to use (default: {','.join(default_chat_models)}): " + ) + chat_models = chat_model_names.split(",") if chat_model_names != "" else default_chat_models + chat_models = [model.strip() for model in chat_models] + else: + chat_models = default_chat_models + + for chat_model in chat_models: + default_max_tokens = model_to_prompt_size.get(chat_model) + default_tokenizer = model_to_tokenizer.get(chat_model) + vision_enabled = vision_enabled and chat_model in supported_vision_models + + chat_model_options = { + "chat_model": chat_model, + "model_type": model_type, + "max_prompt_size": default_max_tokens, + "vision_enabled": vision_enabled, + "tokenizer": default_tokenizer, + "openai_config": chat_model_provider, + } + + ChatModelOptions.objects.create(**chat_model_options) + + logger.info(f"🗣️ {provider_name} chat model configuration complete") + return True, chat_model_provider + admin_user = KhojUser.objects.filter(is_staff=True).first() if admin_user is None: while True: @@ -139,7 +225,8 @@ def initialization(): try: _create_chat_configuration() break - # Some environments don't support interactive input. We catch the exception and return if that's the case. The admin can still configure their settings from the admin page. + # Some environments don't support interactive input. We catch the exception and return if that's the case. + # The admin can still configure their settings from the admin page. except EOFError: return except Exception as e: diff --git a/tests/test_offline_chat_actors.py b/tests/test_offline_chat_actors.py index ed17cac9..2a0383f1 100644 --- a/tests/test_offline_chat_actors.py +++ b/tests/test_offline_chat_actors.py @@ -19,12 +19,12 @@ from khoj.processor.conversation.offline.chat_model import ( from khoj.processor.conversation.offline.utils import download_model from khoj.processor.conversation.utils import message_to_log from khoj.routers.helpers import aget_relevant_output_modes -from khoj.utils.constants import default_offline_chat_model +from khoj.utils.constants import default_offline_chat_models @pytest.fixture(scope="session") def loaded_model(): - return download_model(default_offline_chat_model, max_tokens=5000) + return download_model(default_offline_chat_models[0], max_tokens=5000) freezegun.configure(extend_ignore_list=["transformers"])