mirror of
https://github.com/khoj-ai/khoj.git
synced 2024-11-27 09:25:06 +01:00
Log clients calling API endpoints on Khoj server
- Make API endpoints on Khoj server accept `client` as request parameter - Khoj API endpoints: /chat, /search, /update - Make Khoj clients set `client` request param when calling the API endpoints on the Khoj server - Khoj clients: Emacs, Obsidian and Web - Also log khoj server_version running to telemetry server
This commit is contained in:
Debanjum
GitHub
commit
c68cde4803
10 changed files with 27 additions and 22 deletions
|
|
@ -634,7 +634,7 @@ CONFIG is json obtained from Khoj config API."
|
|||
(buffer-string)))
|
||||
;; Update index on khoj server after configuration update
|
||||
(let ((khoj--server-ready? nil))
|
||||
(url-retrieve (format "%s/api/update?t=org" khoj-server-url) #'identity)))
|
||||
(url-retrieve (format "%s/api/update?t=org&client=emacs" khoj-server-url) #'identity)))
|
||||
|
||||
(defun khoj--get-enabled-content-types ()
|
||||
"Get content types enabled for search from API."
|
||||
|
|
@ -651,7 +651,7 @@ CONFIG is json obtained from Khoj config API."
|
|||
Use QUERY, CONTENT-TYPE and (optional) RERANK as query params"
|
||||
(let ((rerank (or rerank "false"))
|
||||
(encoded-query (url-hexify-string query)))
|
||||
(format "%s/api/search?q=%s&t=%s&r=%s&n=%s" khoj-server-url encoded-query content-type rerank khoj-results-count)))
|
||||
(format "%s/api/search?q=%s&t=%s&r=%s&n=%s&client=emacs" khoj-server-url encoded-query content-type rerank khoj-results-count)))
|
||||
|
||||
(defun khoj--query-search-api-and-render-results (query-url content-type query buffer-name)
|
||||
"Query Khoj Search with QUERY-URL.
|
||||
|
|
@ -788,7 +788,7 @@ Render results in BUFFER-NAME using QUERY, CONTENT-TYPE."
|
|||
"Send QUERY to Khoj Chat API."
|
||||
(let* ((url-request-method "GET")
|
||||
(encoded-query (url-hexify-string query))
|
||||
(query-url (format "%s/api/chat?q=%s" khoj-server-url encoded-query)))
|
||||
(query-url (format "%s/api/chat?q=%s&client=emacs" khoj-server-url encoded-query)))
|
||||
(with-temp-buffer
|
||||
(condition-case ex
|
||||
(progn
|
||||
|
|
@ -1031,7 +1031,7 @@ Paragraph only starts at first text after blank line."
|
|||
(let* ((force-update (if (member "--force-update" args) "true" "false"))
|
||||
;; set content type to: specified > last used > based on current buffer > default type
|
||||
(content-type (or (transient-arg-value "--content-type=" args) (khoj--buffer-name-to-content-type (buffer-name))))
|
||||
(update-url (format "%s/api/update?t=%s&force=%s" khoj-server-url content-type force-update))
|
||||
(update-url (format "%s/api/update?t=%s&force=%s&client=emacs" khoj-server-url content-type force-update))
|
||||
(url-request-method "GET"))
|
||||
(progn
|
||||
(setq khoj--content-type content-type)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ export class KhojChatModal extends Modal {
|
|||
contentEl.createDiv({ attr: { id: "khoj-chat-body", class: "khoj-chat-body" } });
|
||||
|
||||
// Get conversation history from Khoj backend
|
||||
let chatUrl = `${this.setting.khojUrl}/api/chat?`;
|
||||
let chatUrl = `${this.setting.khojUrl}/api/chat?client=obsidian`;
|
||||
let response = await request(chatUrl);
|
||||
let chatLogs = JSON.parse(response).response;
|
||||
chatLogs.forEach((chatLog: any) => {
|
||||
|
|
@ -120,7 +120,7 @@ export class KhojChatModal extends Modal {
|
|||
|
||||
// Get chat response from Khoj backend
|
||||
let encodedQuery = encodeURIComponent(query);
|
||||
let chatUrl = `${this.setting.khojUrl}/api/chat?q=${encodedQuery}`;
|
||||
let chatUrl = `${this.setting.khojUrl}/api/chat?q=${encodedQuery}&client=obsidian`;
|
||||
let response = await request(chatUrl);
|
||||
let data = JSON.parse(response);
|
||||
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ export class KhojSearchModal extends SuggestModal<SearchResult> {
|
|||
async getSuggestions(query: string): Promise<SearchResult[]> {
|
||||
// Query Khoj backend for search results
|
||||
let encodedQuery = encodeURIComponent(query);
|
||||
let searchUrl = `${this.setting.khojUrl}/api/search?q=${encodedQuery}&n=${this.setting.resultsCount}&r=${this.rerank}`;
|
||||
let searchUrl = `${this.setting.khojUrl}/api/search?q=${encodedQuery}&n=${this.setting.resultsCount}&r=${this.rerank}&client=obsidian`;
|
||||
|
||||
// Get search results for markdown and pdf files
|
||||
let mdResponse = await request(`${searchUrl}&t=markdown`);
|
||||
|
|
|
|||
|
|
@ -107,8 +107,8 @@ export class KhojSettingTab extends PluginSettingTab {
|
|||
}, 300);
|
||||
this.plugin.registerInterval(progress_indicator);
|
||||
|
||||
await request(`${this.plugin.settings.khojUrl}/api/update?t=markdown&force=true`);
|
||||
await request(`${this.plugin.settings.khojUrl}/api/update?t=pdf&force=true`);
|
||||
await request(`${this.plugin.settings.khojUrl}/api/update?t=markdown&force=true&client=obsidian`);
|
||||
await request(`${this.plugin.settings.khojUrl}/api/update?t=pdf&force=true&client=obsidian`);
|
||||
new Notice('✅ Updated Khoj index.');
|
||||
|
||||
// Reset button once index is updated
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ regenerateButton.addEventListener("click", (event) => {
|
|||
event.preventDefault();
|
||||
regenerateButton.style.cursor = "progress";
|
||||
regenerateButton.disabled = true;
|
||||
fetch("/api/update?force=true")
|
||||
fetch("/api/update?force=true&client=web")
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
regenerateButton.style.cursor = "pointer";
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@
|
|||
document.getElementById("chat-input").value = "";
|
||||
|
||||
// Generate backend API URL to execute query
|
||||
let url = `/api/chat?q=${encodeURIComponent(query)}`;
|
||||
let url = `/api/chat?q=${encodeURIComponent(query)}&client=web`;
|
||||
|
||||
// Call specified Khoj API
|
||||
fetch(url)
|
||||
|
|
@ -82,7 +82,7 @@
|
|||
}
|
||||
|
||||
window.onload = function () {
|
||||
fetch('/api/chat')
|
||||
fetch('/api/chat?client=web')
|
||||
.then(response => response.json())
|
||||
.then(data => data.response)
|
||||
.then(chat_logs => {
|
||||
|
|
|
|||
|
|
@ -90,8 +90,8 @@
|
|||
|
||||
// Generate Backend API URL to execute Search
|
||||
url = type === "image"
|
||||
? `/api/search?q=${encodeURIComponent(query)}&t=${type}&n=${results_count}`
|
||||
: `/api/search?q=${encodeURIComponent(query)}&t=${type}&n=${results_count}&r=${rerank}`;
|
||||
? `/api/search?q=${encodeURIComponent(query)}&t=${type}&n=${results_count}&client=web`
|
||||
: `/api/search?q=${encodeURIComponent(query)}&t=${type}&n=${results_count}&r=${rerank}&client=web`;
|
||||
|
||||
// Execute Search and Render Results
|
||||
fetch(url)
|
||||
|
|
@ -107,7 +107,7 @@
|
|||
|
||||
function updateIndex() {
|
||||
type = document.getElementById("type").value;
|
||||
fetch(`/api/update?t=${type}`)
|
||||
fetch(`/api/update?t=${type}&client=web`)
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
console.log(data);
|
||||
|
|
|
|||
|
|
@ -70,6 +70,7 @@ def search(
|
|||
r: Optional[bool] = False,
|
||||
score_threshold: Optional[Union[float, None]] = None,
|
||||
dedupe: Optional[bool] = True,
|
||||
client: Optional[str] = None,
|
||||
):
|
||||
results: List[SearchResponse] = []
|
||||
if q is None or q == "":
|
||||
|
|
@ -181,14 +182,16 @@ def search(
|
|||
|
||||
# Only log telemetry if query is new and not a continuation of previous query
|
||||
if state.previous_query is None or state.previous_query not in user_query:
|
||||
state.telemetry += [log_telemetry(telemetry_type="api", api="search", app_config=state.config.app)]
|
||||
state.telemetry += [
|
||||
log_telemetry(telemetry_type="api", api="search", client=client, app_config=state.config.app)
|
||||
]
|
||||
state.previous_query = user_query
|
||||
|
||||
return results
|
||||
|
||||
|
||||
@api.get("/update")
|
||||
def update(t: Optional[SearchType] = None, force: Optional[bool] = False):
|
||||
def update(t: Optional[SearchType] = None, force: Optional[bool] = False, client: Optional[str] = None):
|
||||
try:
|
||||
state.search_index_lock.acquire()
|
||||
state.model = configure_search(state.model, state.config, regenerate=force, t=t)
|
||||
|
|
@ -207,13 +210,13 @@ def update(t: Optional[SearchType] = None, force: Optional[bool] = False):
|
|||
else:
|
||||
logger.info("📬 Processor reconfigured via API")
|
||||
|
||||
state.telemetry += [log_telemetry(telemetry_type="api", api="update", app_config=state.config.app)]
|
||||
state.telemetry += [log_telemetry(telemetry_type="api", api="update", client=client, app_config=state.config.app)]
|
||||
|
||||
return {"status": "ok", "message": "khoj reloaded"}
|
||||
|
||||
|
||||
@api.get("/chat")
|
||||
def chat(q: Optional[str] = None):
|
||||
def chat(q: Optional[str] = None, client: Optional[str] = None):
|
||||
if (
|
||||
state.processor_config is None
|
||||
or state.processor_config.conversation is None
|
||||
|
|
@ -277,6 +280,6 @@ def chat(q: Optional[str] = None):
|
|||
conversation_log=meta_log.get("chat", []),
|
||||
)
|
||||
|
||||
state.telemetry += [log_telemetry(telemetry_type="api", api="chat", app_config=state.config.app)]
|
||||
state.telemetry += [log_telemetry(telemetry_type="api", api="chat", client=client, app_config=state.config.app)]
|
||||
|
||||
return {"status": status, "response": gpt_response, "context": compiled_references}
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@ from __future__ import annotations # to avoid quoting type hints
|
|||
from collections import OrderedDict
|
||||
import datetime
|
||||
from importlib import import_module
|
||||
from importlib.metadata import version
|
||||
import logging
|
||||
from os import path
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import requests
|
||||
import sys
|
||||
from time import perf_counter
|
||||
import torch
|
||||
|
|
@ -184,6 +184,7 @@ def log_telemetry(telemetry_type: str, api: str = None, client: str = None, app_
|
|||
request_body = {
|
||||
"telemetry_type": telemetry_type,
|
||||
"server_id": get_server_id(),
|
||||
"server_version": version("khoj-assistant"),
|
||||
"os": platform.system(),
|
||||
"timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ def v1_telemetry(telemetry_data: List[Dict[str, str]]):
|
|||
# Log telemetry data
|
||||
for item in telemetry_data:
|
||||
cur.execute(
|
||||
"INSERT INTO usage (time, type, server_id, os, api, client) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
"INSERT INTO usage (time, type, server_id, os, api, client, server_version) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
item["timestamp"],
|
||||
item["telemetry_type"],
|
||||
|
|
@ -61,6 +61,7 @@ def v1_telemetry(telemetry_data: List[Dict[str, str]]):
|
|||
item["os"],
|
||||
item.get("api"),
|
||||
item.get("client"),
|
||||
item.get("server_version", None),
|
||||
),
|
||||
)
|
||||
# Commit the changes
|
||||
|
|
|
|||
Loading…
Reference in a new issue