mirror of
https://github.com/khoj-ai/khoj.git
synced 2024-12-19 10:57:45 +00:00
469a1cb6a2
Pull out /api/configure/content API endpoints into /api/content to allow for more logical organization of API path hierarchy This should make the url more succinct and API request intent more understandable by using existing HTTP method semantics along with the path. The /configure URL path segment was either - redundant (e.g POST /configure/notion) or - incorrect (e.g GET /configure/files) Some example of naming improvements: - GET /configure/types -> GET /content/types - GET /configure/files -> GET /content/files - DELETE /configure/files -> DELETE /content/files This should also align, merge better the the content indexing API triggered via PUT, PATCH /content Refactor Flow 1. Rename /api/configure/types -> /api/content/types 2. Rename /api/configure -> /api 3. Move /api/content to api_content from under api_config
458 lines
17 KiB
HTML
458 lines
17 KiB
HTML
<html>
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
|
|
<meta property="og:image" content="https://assets.khoj.dev/khoj_hero.png">
|
|
<title>Khoj - Search</title>
|
|
|
|
<link rel="icon" type="image/png" sizes="128x128" href="./assets/icons/favicon-128x128.png">
|
|
<link rel="manifest" href="./khoj.webmanifest">
|
|
<link rel="stylesheet" href="./assets/khoj.css">
|
|
</head>
|
|
<script type="text/javascript" src="./assets/org.min.js"></script>
|
|
<script type="text/javascript" src="./assets/markdown-it.min.js"></script>
|
|
<script src="./utils.js"></script>
|
|
|
|
<script>
|
|
function render_image(item) {
|
|
return `
|
|
<div class="results-image">
|
|
<a href="${item.entry}" class="image-link">
|
|
<img id=${item.score} src="${item.entry}?${Math.random()}"
|
|
title="Effective Score: ${item.score}, Meta: ${item.additional.metadata_score}, Image: ${item.additional.image_score}"
|
|
class="image">
|
|
</a>
|
|
</div>`;
|
|
}
|
|
|
|
function render_org(query, data, classPrefix="") {
|
|
return data.map(function (item) {
|
|
var orgParser = new Org.Parser();
|
|
var orgDocument = orgParser.parse(item.entry);
|
|
var orgHTMLDocument = orgDocument.convert(Org.ConverterHTML, { htmlClassPrefix: classPrefix, suppressNewLines: true });
|
|
return `<div class="results-org">` + orgHTMLDocument.toString() + `</div>`;
|
|
}).join("\n");
|
|
}
|
|
|
|
function render_markdown(query, data) {
|
|
var md = window.markdownit();
|
|
return data.map(function (item) {
|
|
let rendered = "";
|
|
if (item.additional.file.startsWith("http")) {
|
|
lines = item.entry.split("\n");
|
|
rendered = md.render(`${lines[0]}\t[*](${item.additional.file})\n${lines.slice(1).join("\n")}`);
|
|
}
|
|
else {
|
|
rendered = md.render(`${item.entry}`);
|
|
}
|
|
return `<div class="results-markdown">` + rendered + `</div>`;
|
|
}).join("\n");
|
|
}
|
|
|
|
function render_pdf(query, data) {
|
|
return data.map(function (item) {
|
|
let compiled_lines = item.additional.compiled.split("\n");
|
|
let filename = compiled_lines.shift();
|
|
let text_match = compiled_lines.join("\n")
|
|
return `<div class="results-pdf">` + `<h2>${filename}</h2>\n<p>${text_match}</p>` + `</div>`;
|
|
}).join("\n");
|
|
}
|
|
|
|
function render_html(query, data) {
|
|
return data.map(function (item) {
|
|
let document = new DOMParser().parseFromString(item.entry, "text/html");
|
|
// Scrub the HTML to remove any script tags and associated content
|
|
let script_tags = document.querySelectorAll("script");
|
|
for (let i = 0; i < script_tags.length; i++) {
|
|
script_tags[i].remove();
|
|
}
|
|
// Scrub the HTML to remove any style tags and associated content
|
|
let style_tags = document.querySelectorAll("style");
|
|
for (let i = 0; i < style_tags.length; i++) {
|
|
style_tags[i].remove();
|
|
}
|
|
// Scrub the HTML to remove any noscript tags and associated content
|
|
let noscript_tags = document.querySelectorAll("noscript");
|
|
for (let i = 0; i < noscript_tags.length; i++) {
|
|
noscript_tags[i].remove();
|
|
}
|
|
// Scrub the HTML to remove any iframe tags and associated content
|
|
let iframe_tags = document.querySelectorAll("iframe");
|
|
for (let i = 0; i < iframe_tags.length; i++) {
|
|
iframe_tags[i].remove();
|
|
}
|
|
// Scrub the HTML to remove any object tags and associated content
|
|
let object_tags = document.querySelectorAll("object");
|
|
for (let i = 0; i < object_tags.length; i++) {
|
|
object_tags[i].remove();
|
|
}
|
|
// Scrub the HTML to remove any embed tags and associated content
|
|
let embed_tags = document.querySelectorAll("embed");
|
|
for (let i = 0; i < embed_tags.length; i++) {
|
|
embed_tags[i].remove();
|
|
}
|
|
let scrubbedHTML = document.body.outerHTML;
|
|
return `<div class="results-html">` + scrubbedHTML + `</div>`;
|
|
}).join("\n");
|
|
}
|
|
|
|
function render_xml(query, data) {
|
|
return data.map(function (item) {
|
|
return `<div class="results-xml">` +
|
|
`<b><a href="${item.additional.file}">${item.additional.heading}</a></b>` +
|
|
`<xml>${item.entry}</xml>` +
|
|
`</div>`
|
|
}).join("\n");
|
|
}
|
|
|
|
function render_multiple(query, data, type) {
|
|
let html = "";
|
|
data.forEach(item => {
|
|
if (item.additional.file.endsWith(".org")) {
|
|
html += render_org(query, [item], "org-");
|
|
} else if (
|
|
item.additional.file.endsWith(".md") ||
|
|
item.additional.file.endsWith(".markdown") ||
|
|
(item.additional.file.includes("issues") && item.additional.source === "github") ||
|
|
(item.additional.file.includes("commit") && item.additional.source === "github")
|
|
)
|
|
{
|
|
html += render_markdown(query, [item]);
|
|
} else if (item.additional.file.endsWith(".pdf")) {
|
|
html += render_pdf(query, [item]);
|
|
} else if (item.additional.source == "notion") {
|
|
html += `<div class="results-notion">` + `<b><a href="${item.additional.file}">${item.additional.heading}</a></b>` + `<p>${item.entry}</p>` + `</div>`;
|
|
} else if (item.additional.file.endsWith(".html")) {
|
|
html += render_html(query, [item]);
|
|
} else if (item.additional.file.endsWith(".xml")) {
|
|
html += render_xml(query, [item])
|
|
} else {
|
|
html += `<div class="results-plugin">` + `<b><a href="${item.additional.file}">${item.additional.heading}</a></b>` + `<p>${item.entry}</p>` + `</div>`;
|
|
}
|
|
});
|
|
return html;
|
|
}
|
|
|
|
function render_results(data, query, type) {
|
|
let results = "";
|
|
if (type === "markdown") {
|
|
results = render_markdown(query, data);
|
|
} else if (type === "org") {
|
|
results = render_org(query, data, "org-");
|
|
} else if (type === "image") {
|
|
results = data.map(render_image).join('');
|
|
} else if (type === "pdf") {
|
|
results = render_pdf(query, data);
|
|
} else if (type === "github" || type === "all" || type === "notion") {
|
|
results = render_multiple(query, data, type);
|
|
} else {
|
|
results = data.map((item) => `<div class="results-plugin">` + `<p>${item.entry}</p>` + `</div>`).join("\n")
|
|
}
|
|
|
|
// Any POST rendering goes here.
|
|
|
|
let renderedResults = document.createElement("div");
|
|
renderedResults.id = `results-${type}`;
|
|
renderedResults.innerHTML = results;
|
|
|
|
// For all elements that are of type img in the results html and have a src with 'avatar' in the URL, add the class 'avatar'
|
|
// This is used to make the avatar images round
|
|
let images = renderedResults.querySelectorAll("img[src*='avatar']");
|
|
for (let i = 0; i < images.length; i++) {
|
|
images[i].classList.add("avatar");
|
|
}
|
|
|
|
return renderedResults.outerHTML;
|
|
}
|
|
|
|
async function search(rerank=false) {
|
|
// Extract required fields for search from form
|
|
query = document.getElementById("query").value.trim();
|
|
type = 'all';
|
|
results_count = localStorage.getItem("khojResultsCount") || 5;
|
|
console.log(`Query: ${query}, Type: ${type}, Results Count: ${results_count}`);
|
|
|
|
// Short circuit on empty query
|
|
if (query.length === 0) {
|
|
return;
|
|
}
|
|
|
|
// If set query field in url query param on rerank
|
|
if (rerank)
|
|
setQueryFieldInUrl(query);
|
|
|
|
// Execute Search and Render Results
|
|
url = await createRequestUrl(query, type, results_count || 5, rerank);
|
|
const khojToken = await window.tokenAPI.getToken();
|
|
const headers = { 'Authorization': `Bearer ${khojToken}` };
|
|
|
|
fetch(url, { headers })
|
|
.then(response => response.json())
|
|
.then(data => {
|
|
document.getElementById("results").innerHTML = render_results(data, query, type);
|
|
});
|
|
}
|
|
|
|
let debounceTimeout;
|
|
function incrementalSearch(event) {
|
|
// Run incremental search only after waitTime passed since the last key press
|
|
let waitTime = 300;
|
|
clearTimeout(debounceTimeout);
|
|
debounceTimeout = setTimeout(() => {
|
|
type = 'all';
|
|
// Search with reranking on 'Enter'
|
|
let should_rerank = event.key === 'Enter';
|
|
search(rerank=should_rerank);
|
|
}, waitTime);
|
|
}
|
|
|
|
async function populate_type_dropdown() {
|
|
const hostURL = await window.hostURLAPI.getURL();
|
|
const khojToken = await window.tokenAPI.getToken();
|
|
const headers = { 'Authorization': `Bearer ${khojToken}` };
|
|
|
|
// Populate type dropdown field with enabled content types only
|
|
fetch(`${hostURL}/api/content/types`, { headers })
|
|
.then(response => response.json())
|
|
.then(enabled_types => {
|
|
// Show warning if no content types are enabled
|
|
if (enabled_types.detail) {
|
|
document.getElementById("results").innerHTML = "<div id='results-error'>To use Khoj search, setup your content plugins on the Khoj <a class='inline-chat-link' href='/settings'>settings page</a>.</div>";
|
|
document.getElementById("query").setAttribute("disabled", "disabled");
|
|
document.getElementById("query").setAttribute("placeholder", "Configure Khoj to enable search");
|
|
return [];
|
|
}
|
|
|
|
return enabled_types;
|
|
});
|
|
}
|
|
|
|
async function createRequestUrl(query, type, results_count, rerank) {
|
|
// Generate Backend API URL to execute Search
|
|
const hostURL = await window.hostURLAPI.getURL();
|
|
|
|
let url = `${hostURL}/api/search?q=${encodeURIComponent(query)}&n=${results_count}&client=web`;
|
|
// If type is not 'all', append type to URL
|
|
if (type !== 'all')
|
|
url += `&t=${type}`;
|
|
// Rerank is only supported by text types
|
|
if (type !== "image")
|
|
url += `&r=${rerank}`;
|
|
return url;
|
|
}
|
|
|
|
function setQueryFieldInUrl(query) {
|
|
var url = new URL(window.location.href);
|
|
url.searchParams.set("q", query);
|
|
window.history.pushState({}, "", url.href);
|
|
}
|
|
|
|
window.addEventListener("DOMContentLoaded", async() => {
|
|
// Setup the header pane
|
|
document.getElementById("khoj-header").innerHTML = await populateHeaderPane();
|
|
// Setup the nav menu
|
|
document.getElementById("profile-picture").addEventListener("click", toggleNavMenu);
|
|
// Set the active nav pane
|
|
document.getElementById("search-nav")?.classList.add("khoj-nav-selected");
|
|
})
|
|
|
|
|
|
window.addEventListener("load", async function() {
|
|
// Dynamically populate type dropdown based on enabled content types and type passed as URL query parameter
|
|
await populate_type_dropdown();
|
|
|
|
// Fill query field with value passed in URL query parameters, if any.
|
|
var query_via_url = new URLSearchParams(window.location.search).get("q");
|
|
if (query_via_url)
|
|
document.getElementById("query").value = query_via_url;
|
|
});
|
|
</script>
|
|
|
|
<body>
|
|
<!--Add Header Logo and Nav Pane-->
|
|
<div id="khoj-header" class="khoj-header"></div>
|
|
|
|
<!--Add Text Box To Enter Query, Trigger Incremental Search OnChange -->
|
|
<input type="text" id="query" class="option" onkeyup=incrementalSearch(event) autofocus="autofocus" placeholder="Search your knowledge base using natural language">
|
|
|
|
<!-- Section to Render Results -->
|
|
<div id="results"></div>
|
|
</body>
|
|
|
|
<style>
|
|
@media only screen and (max-width: 600px) {
|
|
body {
|
|
display: grid;
|
|
grid-template-columns: 1fr;
|
|
grid-template-rows: 1fr auto auto auto minmax(80px, 100%);
|
|
font-size: small!important;
|
|
}
|
|
body > * {
|
|
grid-column: 1;
|
|
}
|
|
}
|
|
@media only screen and (min-width: 600px) {
|
|
body {
|
|
display: grid;
|
|
grid-template-columns: 1fr min(70vw, 100%) 1fr;
|
|
grid-template-rows: 1fr auto auto auto minmax(80px, 100%);
|
|
padding-top: 60vw;
|
|
}
|
|
body > * {
|
|
grid-column: 2;
|
|
}
|
|
}
|
|
body {
|
|
padding: 0px;
|
|
margin: 0px;
|
|
background: var(--background-color);
|
|
color: var(--main-text-color);
|
|
font-family: var(--font-family);
|
|
font-size: small;
|
|
font-weight: 300;
|
|
line-height: 1.5em;
|
|
}
|
|
body > * {
|
|
padding: 10px;
|
|
margin: 10px;
|
|
}
|
|
#options {
|
|
padding: 0;
|
|
display: grid;
|
|
grid-template-columns: 1fr;
|
|
}
|
|
#options > * {
|
|
padding: 15px;
|
|
border-radius: 5px;
|
|
border: 1px solid #475569;
|
|
background: #f9fafc
|
|
}
|
|
.option:hover {
|
|
box-shadow: 0 0 11px #aaa;
|
|
}
|
|
#options > button {
|
|
margin-right: 10px;
|
|
}
|
|
|
|
#query {
|
|
font-size: small;
|
|
}
|
|
#results {
|
|
font-size: small;
|
|
margin: 0px;
|
|
line-height: 20px;
|
|
}
|
|
.results-image {
|
|
display: grid;
|
|
grid-template-columns: repeat(3, 1fr);
|
|
}
|
|
.image-link {
|
|
place-self: center;
|
|
}
|
|
.image {
|
|
width: 20vw;
|
|
border-radius: 10px;
|
|
border: 1px solid #475569;
|
|
}
|
|
#json {
|
|
white-space: pre-wrap;
|
|
}
|
|
.results-pdf,
|
|
.results-notion,
|
|
.results-html,
|
|
.results-plugin {
|
|
text-align: left;
|
|
white-space: pre-line;
|
|
}
|
|
.results-markdown,
|
|
.results-github {
|
|
text-align: left;
|
|
}
|
|
.results-org {
|
|
text-align: left;
|
|
/* white-space: pre-line; */
|
|
}
|
|
.results-org h3 {
|
|
margin: 20px 0 0 0;
|
|
font-size: small;
|
|
}
|
|
span.org-task-status {
|
|
color: white;
|
|
padding: 3.5px 3.5px 0;
|
|
margin-right: 5px;
|
|
border-radius: 5px;
|
|
background-color: #eab308;
|
|
font-size: small;
|
|
}
|
|
span.org-task-status.todo {
|
|
background-color: #3b82f6
|
|
}
|
|
span.org-task-status.done {
|
|
background-color: #22c55e;
|
|
}
|
|
span.org-task-tag {
|
|
color: white;
|
|
padding: 3.5px 3.5px 0;
|
|
margin-right: 5px;
|
|
border-radius: 5px;
|
|
border: 1px solid #475569;
|
|
background-color: #ef4444;
|
|
font-size: small;
|
|
}
|
|
|
|
pre {
|
|
max-width: 100;
|
|
}
|
|
|
|
a {
|
|
color: #3b82f6;
|
|
text-decoration: none;
|
|
}
|
|
|
|
img.avatar {
|
|
width: 20px;
|
|
height: 20px;
|
|
border-radius: 50%;
|
|
}
|
|
|
|
div#results-error,
|
|
div.results-markdown,
|
|
div.results-notion,
|
|
div.results-org,
|
|
div.results-plugin,
|
|
div.results-html,
|
|
div.results-pdf {
|
|
text-align: left;
|
|
box-shadow: 2px 2px 2px var(--primary-hover);
|
|
border-radius: 5px;
|
|
padding: 10px;
|
|
margin: 10px 0;
|
|
border: 4px solid rgb(229, 229, 229);
|
|
}
|
|
|
|
div#results-error {
|
|
box-shadow: 2px 2px 2px #FF5722;
|
|
}
|
|
|
|
img {
|
|
max-width: 90%;
|
|
}
|
|
|
|
@keyframes gradient {
|
|
0% {
|
|
background-position: 0% 50%;
|
|
}
|
|
50% {
|
|
background-position: 100% 50%;
|
|
}
|
|
100% {
|
|
background-position: 0% 50%;
|
|
}
|
|
}
|
|
|
|
a.khoj-logo {
|
|
text-align: center;
|
|
}
|
|
|
|
</style>
|
|
</html>
|