diff --git a/collector/extensions/index.js b/collector/extensions/index.js
new file mode 100644
index 000000000..7b131b646
--- /dev/null
+++ b/collector/extensions/index.js
@@ -0,0 +1,52 @@
+const { reqBody } = require("../utils/http");
+
+function extensions(app) {
+ if (!app) return;
+
+ app.post("/ext/github-repo", async function (request, response) {
+ try {
+ const loadGithubRepo = require("../utils/extensions/GithubRepo");
+ const { success, reason, data } = await loadGithubRepo(reqBody(request));
+ response.status(200).json({
+ success,
+ reason,
+ data
+ });
+ } catch (e) {
+ console.error(e);
+ response.status(200).json({
+ success: false,
+ reason: e.message || "A processing error occurred.",
+ data: {},
+ });
+ }
+ return;
+ });
+
+ // gets all branches for a specific repo
+ app.post("/ext/github-repo/branches", async function (request, response) {
+ try {
+ const GithubRepoLoader = require("../utils/extensions/GithubRepo/RepoLoader");
+ const allBranches = await (new GithubRepoLoader(reqBody(request))).getRepoBranches()
+ response.status(200).json({
+ success: true,
+ reason: null,
+ data: {
+ branches: allBranches
+ }
+ });
+ } catch (e) {
+ console.error(e);
+ response.status(400).json({
+ success: false,
+ reason: e.message,
+ data: {
+ branches: []
+ }
+ });
+ }
+ return;
+ });
+}
+
+module.exports = extensions;
diff --git a/collector/index.js b/collector/index.js
index 4cbca6c21..5070ae72f 100644
--- a/collector/index.js
+++ b/collector/index.js
@@ -11,6 +11,7 @@ const { reqBody } = require("./utils/http");
const { processSingleFile } = require("./processSingleFile");
const { processLink } = require("./processLink");
const { wipeCollectorStorage } = require("./utils/files");
+const extensions = require("./extensions");
const app = express();
app.use(cors({ origin: true }));
@@ -57,6 +58,8 @@ app.post("/process-link", async function (request, response) {
return;
});
+extensions(app);
+
app.get("/accepts", function (_, response) {
response.status(200).json(ACCEPTED_MIMES);
});
diff --git a/collector/package.json b/collector/package.json
index 180a20888..fb9bed67a 100644
--- a/collector/package.json
+++ b/collector/package.json
@@ -24,6 +24,7 @@
"express": "^4.18.2",
"extract-zip": "^2.0.1",
"fluent-ffmpeg": "^2.1.2",
+ "ignore": "^5.3.0",
"js-tiktoken": "^1.0.8",
"langchain": "0.0.201",
"mammoth": "^1.6.0",
@@ -35,6 +36,7 @@
"pdf-parse": "^1.1.1",
"puppeteer": "^21.6.1",
"slugify": "^1.6.6",
+ "url-pattern": "^1.0.3",
"uuid": "^9.0.0",
"wavefile": "^11.0.0"
},
@@ -42,4 +44,4 @@
"nodemon": "^2.0.22",
"prettier": "^2.4.1"
}
-}
\ No newline at end of file
+}
diff --git a/collector/utils/extensions/GithubRepo/RepoLoader/index.js b/collector/utils/extensions/GithubRepo/RepoLoader/index.js
new file mode 100644
index 000000000..7f1c1c057
--- /dev/null
+++ b/collector/utils/extensions/GithubRepo/RepoLoader/index.js
@@ -0,0 +1,149 @@
+class RepoLoader {
+ constructor(args = {}) {
+ this.ready = false;
+ this.repo = args?.repo;
+ this.branch = args?.branch;
+ this.accessToken = args?.accessToken || null;
+ this.ignorePaths = args?.ignorePaths || [];
+
+ this.author = null;
+ this.project = null;
+ this.branches = [];
+ }
+
+ #validGithubUrl() {
+ const UrlPattern = require("url-pattern");
+ const pattern = new UrlPattern("https\\://github.com/(:author)/(:project)");
+ const match = pattern.match(this.repo);
+ if (!match) return false;
+
+ this.author = match.author;
+ this.project = match.project;
+ return true;
+ }
+
+ // Ensure the branch provided actually exists
+ // and if it does not or has not been set auto-assign to primary branch.
+ async #validBranch() {
+ await this.getRepoBranches();
+ if (!!this.branch && this.branches.includes(this.branch)) return;
+
+ console.log(
+ "[Github Loader]: Branch not set! Auto-assigning to a default branch."
+ );
+ this.branch = this.branches.includes("main") ? "main" : "master";
+ console.log(`[Github Loader]: Branch auto-assigned to ${this.branch}.`);
+ return;
+ }
+
+ async #validateAccessToken() {
+ if (!this.accessToken) return;
+ const valid = await fetch("https://api.github.com/octocat", {
+ method: "GET",
+ headers: {
+ Authorization: `Bearer ${this.accessToken}`,
+ "X-GitHub-Api-Version": "2022-11-28",
+ },
+ })
+ .then((res) => {
+ if (!res.ok) throw new Error(res.statusText);
+ return res.ok;
+ })
+ .catch((e) => {
+ console.error(
+ "Invalid Github Access Token provided! Access token will not be used",
+ e.message
+ );
+ return false;
+ });
+
+ if (!valid) this.accessToken = null;
+ return;
+ }
+
+ async init() {
+ if (!this.#validGithubUrl()) return;
+ await this.#validBranch();
+ await this.#validateAccessToken();
+ this.ready = true;
+ return this;
+ }
+
+ async recursiveLoader() {
+ if (!this.ready) throw new Error("[Github Loader]: not in ready state!");
+ const {
+ GithubRepoLoader: LCGithubLoader,
+ } = require("langchain/document_loaders/web/github");
+
+ if (this.accessToken)
+ console.log(
+ `[Github Loader]: Access token set! Recursive loading enabled!`
+ );
+
+ const loader = new LCGithubLoader(this.repo, {
+ accessToken: this.accessToken,
+ branch: this.branch,
+ recursive: !!this.accessToken, // Recursive will hit rate limits.
+ maxConcurrency: 5,
+ unknown: "ignore",
+ ignorePaths: this.ignorePaths,
+ });
+
+ const docs = [];
+ for await (const doc of loader.loadAsStream()) docs.push(doc);
+ return docs;
+ }
+
+ // Sort branches to always show either main or master at the top of the result.
+ #branchPrefSort(branches = []) {
+ const preferredSort = ["main", "master"];
+ return branches.reduce((acc, branch) => {
+ if (preferredSort.includes(branch)) return [branch, ...acc];
+ return [...acc, branch];
+ }, []);
+ }
+
+ // Get all branches for a given repo.
+ async getRepoBranches() {
+ if (!this.#validGithubUrl() || !this.author || !this.project) return [];
+ await this.#validateAccessToken(); // Ensure API access token is valid for pre-flight
+
+ let page = 0;
+ let polling = true;
+ const branches = [];
+
+ while (polling) {
+ console.log(`Fetching page ${page} of branches for ${this.project}`);
+ await fetch(
+ `https://api.github.com/repos/${this.author}/${this.project}/branches?per_page=100&page=${page}`,
+ {
+ method: "GET",
+ headers: {
+ ...(this.accessToken
+ ? { Authorization: `Bearer ${this.accessToken}` }
+ : {}),
+ "X-GitHub-Api-Version": "2022-11-28",
+ },
+ }
+ )
+ .then((res) => {
+ if (res.ok) return res.json();
+ throw new Error(`Invalid request to Github API: ${res.statusText}`);
+ })
+ .then((branchObjects) => {
+ polling = branchObjects.length > 0;
+ branches.push(branchObjects.map((branch) => branch.name));
+ page++;
+ })
+ .catch((err) => {
+ polling = false;
+ console.log(`RepoLoader.branches`, err);
+ });
+ }
+
+ this.branches = [...new Set(branches.flat())];
+ return this.#branchPrefSort(this.branches);
+ }
+}
+
+module.exports = RepoLoader;
diff --git a/collector/utils/extensions/GithubRepo/index.js b/collector/utils/extensions/GithubRepo/index.js
new file mode 100644
index 000000000..d459e6357
--- /dev/null
+++ b/collector/utils/extensions/GithubRepo/index.js
@@ -0,0 +1,78 @@
+const RepoLoader = require("./RepoLoader");
+const fs = require("fs");
+const path = require("path");
+const { default: slugify } = require("slugify");
+const { v4 } = require("uuid");
+const { writeToServerDocuments } = require("../../files");
+const { tokenizeString } = require("../../tokenizer");
+
+async function loadGithubRepo(args) {
+ const repo = new RepoLoader(args);
+ await repo.init();
+
+ if (!repo.ready)
+ return {
+ success: false,
+ reason: "Could not prepare Github repo for loading! Check URL",
+ };
+
+ console.log(
+ `-- Working Github ${repo.author}/${repo.project}:${repo.branch} --`
+ );
+ const docs = await repo.recursiveLoader();
+ if (!docs.length) {
+ return {
+ success: false,
+ reason: "No files were found for those settings.",
+ };
+ }
+
+ console.log(`[Github Loader]: Found ${docs.length} source files. Saving...`);
+ const outFolder = slugify(
+ `${repo.author}-${repo.project}-${repo.branch}-${v4().slice(0, 4)}`
+ ).toLowerCase();
+ const outFolderPath = path.resolve(
+ __dirname,
+ `../../../../server/storage/documents/${outFolder}`
+ );
+ fs.mkdirSync(outFolderPath);
+
+ for (const doc of docs) {
+ if (!doc.pageContent) continue;
+ const data = {
+ id: v4(),
+ url: "github://" + doc.metadata.source,
+ title: doc.metadata.source,
+ docAuthor: repo.author,
+ description: "No description found.",
+ docSource: repo.repo,
+ chunkSource: doc.metadata.source,
+ published: new Date().toLocaleString(),
+ wordCount: doc.pageContent.split(" ").length,
+ pageContent: doc.pageContent,
+ token_count_estimate: tokenizeString(doc.pageContent).length,
+ };
+ console.log(
+ `[Github Loader]: Saving ${doc.metadata.source} to ${outFolder}`
+ );
+ writeToServerDocuments(
+ data,
+ `${slugify(doc.metadata.source)}-${data.id}`,
+ outFolderPath
+ );
+ }
+
+ return {
+ success: true,
+ reason: null,
+ data: {
+ author: repo.author,
+ repo: repo.project,
+ branch: repo.branch,
+ files: docs.length,
+ destination: outFolder,
+ },
+ };
+}
+
+module.exports = loadGithubRepo;
diff --git a/collector/yarn.lock b/collector/yarn.lock
index c6f442a80..28c610926 100644
--- a/collector/yarn.lock
+++ b/collector/yarn.lock
@@ -1530,6 +1530,11 @@ ignore-by-default@^1.0.1:
resolved "https://registry.yarnpkg.com/ignore-by-default/-/ignore-by-default-1.0.1.tgz#48ca6d72f6c6a3af00a9ad4ae6876be3889e2b09"
integrity sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==
+ignore@^5.3.0:
+ version "5.3.0"
+ resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.0.tgz#67418ae40d34d6999c95ff56016759c718c82f78"
+ integrity sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==
+
immediate@~3.0.5:
version "3.0.6"
resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
@@ -3127,6 +3132,11 @@ unpipe@1.0.0, unpipe@~1.0.0:
resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==
+url-pattern@^1.0.3:
+ version "1.0.3"
+ resolved "https://registry.yarnpkg.com/url-pattern/-/url-pattern-1.0.3.tgz#0409292471b24f23c50d65a47931793d2b5acfc1"
+ integrity sha512-uQcEj/2puA4aq1R3A2+VNVBgaWYR24FdWjl7VNW83rnWftlhyzOZ/tBjezRiC2UkIzuxC8Top3IekN3vUf1WxA==
+
url-template@^2.0.8:
version "2.0.8"
resolved "https://registry.yarnpkg.com/url-template/-/url-template-2.0.8.tgz#fc565a3cccbff7730c775f5641f9555791439f21"
diff --git a/frontend/package.json b/frontend/package.json
index a1531b9cc..19fe1d0ab 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -27,6 +27,7 @@
"react-loading-icons": "^1.1.0",
"react-loading-skeleton": "^3.1.0",
"react-router-dom": "^6.3.0",
+ "react-tag-input-component": "^2.0.2",
"react-toastify": "^9.1.3",
"text-case": "^1.0.9",
"truncate": "^3.0.0",
diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx
index 2d1eeb7d1..7224f2e9c 100644
--- a/frontend/src/App.jsx
+++ b/frontend/src/App.jsx
@@ -36,6 +36,12 @@ const GeneralExportImport = lazy(() =>
import("@/pages/GeneralSettings/ExportImport")
);
const GeneralSecurity = lazy(() => import("@/pages/GeneralSettings/Security"));
+const DataConnectors = lazy(() =>
+ import("@/pages/GeneralSettings/DataConnectors")
+);
+const DataConnectorSetup = lazy(() =>
+ import("@/pages/GeneralSettings/DataConnectors/Connectors")
+);
const OnboardingFlow = lazy(() => import("@/pages/OnboardingFlow"));
export default function App() {
@@ -103,6 +109,15 @@ export default function App() {
path="/settings/workspaces"
element={<ManagerRoute Component={AdminWorkspaces} />}
/>
+ <Route
+ path="/settings/data-connectors"
+ element={<ManagerRoute Component={DataConnectors} />}
+ />
+ <Route
+ path="/settings/data-connectors/:connector"
+ element={<ManagerRoute Component={DataConnectorSetup} />}
+ />
+
{/* Onboarding Flow */}
<Route path="/onboarding" element={<OnboardingFlow />} />
</Routes>
diff --git a/frontend/src/components/DataConnectorOption/index.jsx b/frontend/src/components/DataConnectorOption/index.jsx
new file mode 100644
index 000000000..84af0ff1e
--- /dev/null
+++ b/frontend/src/components/DataConnectorOption/index.jsx
@@ -0,0 +1,39 @@
+import paths from "@/utils/paths";
+import ConnectorImages from "./media";
+
+export default function DataConnectorOption({ slug }) {
+ if (!DATA_CONNECTORS.hasOwnProperty(slug)) return null;
+ const { path, image, name, description, link } = DATA_CONNECTORS[slug];
+
+ return (
+ <a href={path}>
+ <label className="transition-all duration-300 inline-flex flex-col h-full w-60 cursor-pointer items-start justify-between rounded-2xl bg-preference-gradient border-2 border-transparent shadow-md px-5 py-4 text-white hover:bg-selected-preference-gradient hover:border-white/60 peer-checked:border-white peer-checked:border-opacity-90 peer-checked:bg-selected-preference-gradient">
+ <div className="flex items-center">
+ <img src={image} alt={name} className="h-10 w-10 rounded" />
+ <div className="ml-4 text-sm font-semibold">{name}</div>
+ </div>
+ <div className="mt-2 text-xs font-base text-white tracking-wide">
+ {description}
+ </div>
+ <a
+ href={link}
+ target="_blank"
+ className="mt-2 text-xs text-white font-medium underline"
+ >
+ {link}
+ </a>
+ </label>
+ </a>
+ );
+}
+
+export const DATA_CONNECTORS = {
+ github: {
+ name: "GitHub Repo",
+ path: paths.settings.dataConnectors.github(),
+ image: ConnectorImages.github,
+ description:
+ "Import an entire public or private Github repository in a single click.",
+ link: "https://github.com",
+ },
+};
diff --git a/frontend/src/components/DataConnectorOption/media/github.png b/frontend/src/components/DataConnectorOption/media/github.png
new file mode 100644
index 000000000..835221bab
Binary files /dev/null and b/frontend/src/components/DataConnectorOption/media/github.png differ
diff --git a/frontend/src/components/DataConnectorOption/media/index.js b/frontend/src/components/DataConnectorOption/media/index.js
new file mode 100644
index 000000000..a339328ef
--- /dev/null
+++ b/frontend/src/components/DataConnectorOption/media/index.js
@@ -0,0 +1,5 @@
+import Github from "./github.png";
+const ConnectorImages = {
+ github: Github,
+};
+export default ConnectorImages;
diff --git a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FileRow/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FileRow/index.jsx
index b83d695a0..f83a9e34c 100644
--- a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FileRow/index.jsx
+++ b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FileRow/index.jsx
@@ -33,7 +33,7 @@ export default function FileRow({
try {
setLoading(true);
setLoadingMessage("This may take a while for large documents");
- await System.deleteDocument(`${folderName}/${item.name}`, item);
+ await System.deleteDocument(`${folderName}/${item.name}`);
await fetchKeys(true);
} catch (error) {
console.error("Failed to delete the document:", error);
@@ -60,7 +60,7 @@ export default function FileRow({
selected ? "bg-sky-500/20" : ""
} ${expanded ? "bg-sky-500/10" : ""}`}`}
>
- <div className="col-span-4 flex gap-x-[4px] items-center">
+ <div className="pl-4 col-span-4 flex gap-x-[4px] items-center">
<div
className="w-3 h-3 rounded border-[1px] border-white flex justify-center items-center cursor-pointer"
role="checkbox"
diff --git a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FolderRow/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FolderRow/index.jsx
index f2d5d5c71..c93a45cd3 100644
--- a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FolderRow/index.jsx
+++ b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FolderRow/index.jsx
@@ -1,7 +1,8 @@
import { useState } from "react";
import FileRow from "../FileRow";
-import { CaretDown, FolderNotch } from "@phosphor-icons/react";
+import { CaretDown, FolderNotch, Trash } from "@phosphor-icons/react";
import { middleTruncate } from "@/utils/directories";
+import System from "@/models/system";
export default function FolderRow({
item,
@@ -12,8 +13,32 @@ export default function FolderRow({
fetchKeys,
setLoading,
setLoadingMessage,
+ autoExpanded = false,
}) {
- const [expanded, setExpanded] = useState(true);
+ const [expanded, setExpanded] = useState(autoExpanded);
+
+ const onTrashClick = async (event) => {
+ event.stopPropagation();
+ if (
+ !window.confirm(
+ "Are you sure you want to delete this folder?\nThis will require you to re-upload and re-embed it.\nAny documents in this folder will be removed from any workspace that is currently referencing it.\nThis action is not reversible."
+ )
+ ) {
+ return false;
+ }
+
+ try {
+ setLoading(true);
+ setLoadingMessage("This may take a while for large folders");
+ await System.deleteFolder(item.name);
+ await fetchKeys(true);
+ } catch (error) {
+ console.error("Failed to delete the document:", error);
+ }
+
+ if (selected) toggleSelection(item);
+ setLoading(false);
+ };
const handleExpandClick = (event) => {
event.stopPropagation();
@@ -30,7 +55,7 @@ export default function FolderRow({
>
<div className="col-span-4 flex gap-x-[4px] items-center">
<div
- className="w-3 h-3 rounded border-[1px] border-white flex justify-center items-center cursor-pointer"
+ className="shrink-0 w-3 h-3 rounded border-[1px] border-white flex justify-center items-center cursor-pointer"
role="checkbox"
aria-checked={selected}
tabIndex={0}
@@ -46,7 +71,7 @@ export default function FolderRow({
<CaretDown className="text-base font-bold w-4 h-4" />
</div>
<FolderNotch
- className="text-base font-bold w-4 h-4 mr-[3px]"
+ className="shrink-0 text-base font-bold w-4 h-4 mr-[3px]"
weight="fill"
/>
<p className="whitespace-nowrap overflow-show">
@@ -56,7 +81,14 @@ export default function FolderRow({
<p className="col-span-2 pl-3.5" />
<p className="col-span-2 pl-3" />
<p className="col-span-2 pl-2" />
- <div className="col-span-2 flex justify-end items-center" />
+ <div className="col-span-2 flex justify-end items-center">
+ {item.name !== "custom-documents" && (
+ <Trash
+ onClick={onTrashClick}
+ className="text-base font-bold w-4 h-4 ml-2 flex-shrink-0 cursor-pointer"
+ />
+ )}
+ </div>
</div>
{expanded && (
<div className="col-span-full">
diff --git a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx
index 072010336..dcf625c5e 100644
--- a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx
+++ b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx
@@ -106,6 +106,7 @@ export default function Directory({
isSelected={isSelected}
setLoading={setLoading}
setLoadingMessage={setLoadingMessage}
+ autoExpanded={index === 0}
/>
)
)
diff --git a/frontend/src/components/SettingsSidebar/index.jsx b/frontend/src/components/SettingsSidebar/index.jsx
index e599ee2b2..f4c9552e3 100644
--- a/frontend/src/components/SettingsSidebar/index.jsx
+++ b/frontend/src/components/SettingsSidebar/index.jsx
@@ -22,6 +22,7 @@ import {
X,
List,
FileCode,
+ Plugs,
} from "@phosphor-icons/react";
import useUser from "@/hooks/useUser";
import { USER_BACKGROUND_COLOR } from "@/utils/constants";
@@ -127,6 +128,11 @@ export default function SettingsSidebar() {
btnText="Vector Database"
icon={<Database className="h-5 w-5 flex-shrink-0" />}
/>
+ <Option
+ href={paths.settings.dataConnectors.list()}
+ btnText="Data Connectors"
+ icon={<Plugs className="h-5 w-5 flex-shrink-0" />}
+ />
</>
)}
diff --git a/frontend/src/components/Sidebar/IndexCount.jsx b/frontend/src/components/Sidebar/IndexCount.jsx
deleted file mode 100644
index 9e0e126c6..000000000
--- a/frontend/src/components/Sidebar/IndexCount.jsx
+++ /dev/null
@@ -1,34 +0,0 @@
-import pluralize from "pluralize";
-import React, { useEffect, useState } from "react";
-import System from "@/models/system";
-import { numberWithCommas } from "@/utils/numbers";
-
-export default function IndexCount() {
- const [indexes, setIndexes] = useState(null);
- useEffect(() => {
- async function indexCount() {
- setIndexes(await System.totalIndexes());
- }
- indexCount();
- }, []);
-
- if (indexes === null || indexes === 0) {
- return (
- <div className="flex w-full items-center justify-end gap-x-2">
- <div className="flex items-center gap-x-1 px-2 rounded-full">
- <p className="text-slate-400 leading-tight text-sm"></p>
- </div>
- </div>
- );
- }
-
- return (
- <div className="flex w-full items-center justify-end gap-x-2">
- <div className="flex items-center gap-x-1 px-2 rounded-full">
- <p className="text-slate-400 leading-tight text-sm">
- {numberWithCommas(indexes)} {pluralize("vector", indexes)}
- </p>
- </div>
- </div>
- );
-}
diff --git a/frontend/src/components/Sidebar/LLMStatus.jsx b/frontend/src/components/Sidebar/LLMStatus.jsx
deleted file mode 100644
index 733dcb1e7..000000000
--- a/frontend/src/components/Sidebar/LLMStatus.jsx
+++ /dev/null
@@ -1,49 +0,0 @@
-import React, { useEffect, useState } from "react";
-import { WarningCircle, Circle } from "@phosphor-icons/react";
-import System from "@/models/system";
-
-export default function LLMStatus() {
- const [status, setStatus] = useState(null);
- useEffect(() => {
- async function checkPing() {
- setStatus(await System.ping());
- }
- checkPing();
- }, []);
-
- if (status === null) {
- return (
- <div className="flex w-full items-center justify-start gap-x-2">
- <p className="text-slate-400 leading-loose text-sm">LLM</p>
- <div className="flex items-center gap-x-1 border border-slate-400 px-2 rounded-full">
- <p className="text-slate-400 leading-tight text-sm">unknown</p>
- <Circle className="h-3 w-3 stroke-slate-700 fill-slate-400 animate-pulse" />
- </div>
- </div>
- );
- }
-
- // TODO: add modal or toast on click to identify why this is broken
- // need to likely start server.
- if (status === false) {
- return (
- <div className="flex w-full items-center justify-end gap-x-2">
- <p className="text-slate-400 leading-loose text-sm">LLM</p>
- <div className="flex items-center gap-x-1 border border-red-400 px-2 bg-red-200 rounded-full">
- <p className="text-red-700 leading-tight text-sm">offline</p>
- <WarningCircle className="h-3 w-3 stroke-red-100 fill-red-400" />
- </div>
- </div>
- );
- }
-
- return (
- <div className="flex w-full items-center justify-end gap-x-2">
- <p className="text-slate-400 leading-loose text-sm">LLM</p>
- <div className="flex items-center gap-x-1 border border-slate-400 px-2 rounded-full">
- <p className="text-slate-400 leading-tight text-sm">online</p>
- <Circle className="h-3 w-3 stroke-green-100 fill-green-400 animate-pulse" />
- </div>
- </div>
- );
-}
diff --git a/frontend/src/components/Sidebar/index.jsx b/frontend/src/components/Sidebar/index.jsx
index ac92f4830..58af4048b 100644
--- a/frontend/src/components/Sidebar/index.jsx
+++ b/frontend/src/components/Sidebar/index.jsx
@@ -71,25 +71,6 @@ export default function Sidebar() {
<ActiveWorkspaces />
</div>
<div className="flex flex-col flex-grow justify-end mb-2">
- {/* <div className="flex flex-col gap-y-2">
- <div className="w-full flex items-center justify-between">
- <LLMStatus />
- <IndexCount />
- </div>
- <a
- href={paths.feedback()}
- target="_blank"
- className="flex flex-grow w-[100%] h-[36px] gap-x-2 py-[5px] px-4 border border-transparent rounded-lg text-slate-200 justify-center items-center bg-stone-800 hover:bg-stone-900"
- >
- <AtSign className="h-4 w-4" />
- <p className="text-slate-200 text-xs leading-loose font-semibold">
- Feedback form
- </p>
- </a>
- <ManagedHosting />
- <LogoutButton />
- </div> */}
-
{/* Footer */}
<div className="flex justify-center mt-2">
<div className="flex space-x-4">
diff --git a/frontend/src/index.css b/frontend/src/index.css
index f0f04bbcc..a7aef9a7e 100644
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -385,3 +385,7 @@ dialog::backdrop {
@apply border-blue-500 bg-blue-400/10 text-blue-800;
}
}
+
+.rti--container {
+ @apply !bg-zinc-900 !text-white !placeholder-white !placeholder-opacity-60 !text-sm !rounded-lg !p-2.5;
+}
diff --git a/frontend/src/models/dataConnector.js b/frontend/src/models/dataConnector.js
new file mode 100644
index 000000000..45d575024
--- /dev/null
+++ b/frontend/src/models/dataConnector.js
@@ -0,0 +1,47 @@
+import { API_BASE } from "@/utils/constants";
+import { baseHeaders } from "@/utils/request";
+import showToast from "@/utils/toast";
+
+const DataConnector = {
+ github: {
+ branches: async ({ repo, accessToken }) => {
+ return await fetch(`${API_BASE}/ext/github/branches`, {
+ method: "POST",
+ headers: baseHeaders(),
+ cache: "force-cache",
+ body: JSON.stringify({ repo, accessToken }),
+ })
+ .then((res) => res.json())
+ .then((res) => {
+ if (!res.success) throw new Error(res.reason);
+ return res.data;
+ })
+ .then((data) => {
+ return { branches: data?.branches || [], error: null };
+ })
+ .catch((e) => {
+ console.error(e);
+ showToast(e.message, "error");
+ return { branches: [], error: e.message };
+ });
+ },
+ collect: async function ({ repo, accessToken, branch, ignorePaths = [] }) {
+ return await fetch(`${API_BASE}/ext/github/repo`, {
+ method: "POST",
+ headers: baseHeaders(),
+ body: JSON.stringify({ repo, accessToken, branch, ignorePaths }),
+ })
+ .then((res) => res.json())
+ .then((res) => {
+ if (!res.success) throw new Error(res.reason);
+ return { data: res.data, error: null };
+ })
+ .catch((e) => {
+ console.error(e);
+ return { data: null, error: e.message };
+ });
+ },
+ },
+};
+
+export default DataConnector;
diff --git a/frontend/src/models/system.js b/frontend/src/models/system.js
index 79c203d94..9b71a6055 100644
--- a/frontend/src/models/system.js
+++ b/frontend/src/models/system.js
@@ -1,5 +1,6 @@
import { API_BASE, AUTH_TIMESTAMP } from "@/utils/constants";
import { baseHeaders } from "@/utils/request";
+import DataConnector from "./dataConnector";
const System = {
ping: async function () {
@@ -133,11 +134,23 @@ const System = {
return false;
});
},
- deleteDocument: async (name, meta) => {
+ deleteDocument: async (name) => {
return await fetch(`${API_BASE}/system/remove-document`, {
method: "DELETE",
headers: baseHeaders(),
- body: JSON.stringify({ name, meta }),
+ body: JSON.stringify({ name }),
+ })
+ .then((res) => res.ok)
+ .catch((e) => {
+ console.error(e);
+ return false;
+ });
+ },
+ deleteFolder: async (name) => {
+ return await fetch(`${API_BASE}/system/remove-folder`, {
+ method: "DELETE",
+ headers: baseHeaders(),
+ body: JSON.stringify({ name }),
})
.then((res) => res.ok)
.catch((e) => {
@@ -431,6 +444,7 @@ const System = {
return { success: false, error: e.message };
});
},
+ dataConnectors: DataConnector,
};
export default System;
diff --git a/frontend/src/pages/GeneralSettings/DataConnectors/Connectors/Github/index.jsx b/frontend/src/pages/GeneralSettings/DataConnectors/Connectors/Github/index.jsx
new file mode 100644
index 000000000..fdcc8cb57
--- /dev/null
+++ b/frontend/src/pages/GeneralSettings/DataConnectors/Connectors/Github/index.jsx
@@ -0,0 +1,294 @@
+import React, { useEffect, useState } from "react";
+import Sidebar, { SidebarMobileHeader } from "@/components/SettingsSidebar";
+import { isMobile } from "react-device-detect";
+import { DATA_CONNECTORS } from "@/components/DataConnectorOption";
+import System from "@/models/system";
+import { Info } from "@phosphor-icons/react/dist/ssr";
+import showToast from "@/utils/toast";
+import pluralize from "pluralize";
+import { TagsInput } from "react-tag-input-component";
+
+const DEFAULT_BRANCHES = ["main", "master"];
+export default function GithubConnectorSetup() {
+ const { image } = DATA_CONNECTORS.github;
+ const [loading, setLoading] = useState(false);
+ const [repo, setRepo] = useState(null);
+ const [accessToken, setAccessToken] = useState(null);
+ const [ignores, setIgnores] = useState([]);
+
+ const [settings, setSettings] = useState({
+ repo: null,
+ accessToken: null,
+ });
+
+ const handleSubmit = async (e) => {
+ e.preventDefault();
+ const form = new FormData(e.target);
+
+ try {
+ setLoading(true);
+ showToast(
+ "Fetching all files for repo - this may take a while.",
+ "info",
+ { clear: true, autoClose: false }
+ );
+ const { data, error } = await System.dataConnectors.github.collect({
+ repo: form.get("repo"),
+ accessToken: form.get("accessToken"),
+ branch: form.get("branch"),
+ ignorePaths: ignores,
+ });
+
+ if (!!error) {
+ showToast(error, "error", { clear: true });
+ setLoading(false);
+ return;
+ }
+
+ showToast(
+ `${data.files} ${pluralize("file", data.files)} collected from ${
+ data.author
+ }/${data.repo}:${data.branch}. Output folder is ${data.destination}.`,
+ "success",
+ { clear: true }
+ );
+ e.target.reset();
+ setLoading(false);
+ return;
+ } catch (e) {
+ console.error(e);
+ showToast(e.message, "error", { clear: true });
+ setLoading(false);
+ }
+ };
+
+ return (
+ <div className="w-screen h-screen overflow-hidden bg-sidebar flex">
+ {!isMobile && <Sidebar />}
+ <div
+ style={{ height: isMobile ? "100%" : "calc(100% - 32px)" }}
+ className="relative md:ml-[2px] md:mr-[16px] md:my-[16px] md:rounded-[26px] bg-main-gradient w-full h-full overflow-y-scroll border-4 border-accent"
+ >
+ {isMobile && <SidebarMobileHeader />}
+ <div className="flex w-full">
+ <div className="flex flex-col w-full px-1 md:px-20 md:py-12 py-16">
+ <div className="flex w-full gap-x-4 items-center pb-6 border-white border-b-2 border-opacity-10">
+ <img src={image} alt="Github" className="rounded-lg h-16 w-16" />
+ <div className="w-full flex flex-col gap-y-1">
+ <div className="items-center flex gap-x-4">
+ <p className="text-2xl font-semibold text-white">
+ Import GitHub Repository
+ </p>
+ </div>
+ <p className="text-sm font-base text-white text-opacity-60">
+ Import all files from a public or private Github repository
+ and have its files be available in your workspace.
+ </p>
+ </div>
+ </div>
+
+ <form className="w-full" onSubmit={handleSubmit}>
+ {!accessToken && (
+ <div className="flex flex-col gap-y-1 py-4 ">
+ <div className="flex flex-col w-fit gap-y-2 bg-blue-600/20 rounded-lg px-4 py-2">
+ <div className="flex items-center gap-x-2">
+ <Info size={20} className="shrink-0 text-blue-400" />
+ <p className="text-blue-400 text-sm">
+ Trying to collect a GitHub repo without a{" "}
+ <a
+ href="https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens"
+ rel="noreferrer"
+ target="_blank"
+ className="underline"
+ >
+ Personal Access Token
+ </a>{" "}
+ will fail to collect all files due to GitHub API limits.
+ </p>
+ </div>
+ <a
+ href="https://github.com/settings/personal-access-tokens/new"
+ rel="noreferrer"
+ target="_blank"
+ className="text-blue-400 hover:underline"
+ >
+ Create a temporary Access Token for this data connector
+ →
+ </a>
+ </div>
+ </div>
+ )}
+
+ <div className="w-full flex flex-col py-2">
+ <div className="w-full flex items-center gap-4">
+ <div className="flex flex-col w-60">
+ <div className="flex flex-col gap-y-1 mb-4">
+ <label className="text-white text-sm font-semibold block">
+ GitHub Repo URL
+ </label>
+ <p className="text-xs text-zinc-300">
+ Url of the GitHub repo you wish to collect.
+ </p>
+ </div>
+ <input
+ type="url"
+ name="repo"
+ className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
+ placeholder="https://github.com/Mintplex-Labs/anything-llm"
+ required={true}
+ autoComplete="off"
+ onChange={(e) => setRepo(e.target.value)}
+ onBlur={() => setSettings({ ...settings, repo })}
+ spellCheck={false}
+ />
+ </div>
+ <div className="flex flex-col w-60">
+ <div className="flex flex-col gap-y-1 mb-4">
+ <label className="text-white text-sm block flex gap-x-2 items-center">
+ <p className="font-semibold ">Github Access Token</p>{" "}
+ <p className="text-xs text-zinc-300 font-base!">
+ <i>optional</i>
+ </p>
+ </label>
+ <p className="text-xs text-zinc-300 flex gap-x-2">
+ Access Token to prevent rate limiting.
+ </p>
+ </div>
+ <input
+ type="text"
+ name="accessToken"
+ className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
+ placeholder="github_pat_1234_abcdefg"
+ required={false}
+ autoComplete="off"
+ spellCheck={false}
+ onChange={(e) => setAccessToken(e.target.value)}
+ onBlur={() => setSettings({ ...settings, accessToken })}
+ />
+ </div>
+ <GitHubBranchSelection
+ repo={settings.repo}
+ accessToken={settings.accessToken}
+ />
+ </div>
+
+ <div className="flex flex-col w-1/2 py-4">
+ <div className="flex flex-col gap-y-1 mb-4">
+ <label className="text-white text-sm block flex gap-x-2 items-center">
+ <p className="font-semibold ">File Ignores</p>
+ </label>
+ <p className="text-xs text-zinc-300 flex gap-x-2">
+ List in .gitignore format to ignore specific files during
+ collection. Press enter after each entry you want to save.
+ </p>
+ </div>
+ <TagsInput
+ value={ignores}
+ onChange={setIgnores}
+ name="ignores"
+ placeholder="!*.js, images/*, .DS_Store, bin/*"
+ classNames={{
+ tag: "bg-blue-300/10 text-zinc-800 m-1",
+ input:
+ "flex bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white p-2.5",
+ }}
+ />
+ </div>
+ </div>
+
+ <div className="flex flex-col gap-y-2 w-fit">
+ <button
+ type="submit"
+ disabled={loading}
+ className="mt-2 text-lg w-fit border border-slate-200 px-4 py-1 rounded-lg text-slate-200 items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 disabled:bg-slate-200 disabled:text-slate-800"
+ >
+ {loading
+ ? "Collecting files..."
+ : "Collect all files from GitHub repo"}
+ </button>
+ {loading && (
+ <p className="text-xs text-zinc-300">
+ Once complete, all files will be available for embedding
+ into workspaces in the document picker.
+ </p>
+ )}
+ </div>
+ </form>
+ </div>
+ </div>
+ </div>
+ </div>
+ );
+}
+
+function GitHubBranchSelection({ repo, accessToken }) {
+ const [allBranches, setAllBranches] = useState(DEFAULT_BRANCHES);
+ const [loading, setLoading] = useState(true);
+
+ useEffect(() => {
+ async function fetchAllBranches() {
+ if (!repo) {
+ setAllBranches(DEFAULT_BRANCHES);
+ setLoading(false);
+ return;
+ }
+
+ setLoading(true);
+ const { branches } = await System.dataConnectors.github.branches({
+ repo,
+ accessToken,
+ });
+ setAllBranches(branches.length > 0 ? branches : DEFAULT_BRANCHES);
+ setLoading(false);
+ }
+ fetchAllBranches();
+ }, [repo, accessToken]);
+
+ if (loading) {
+ return (
+ <div className="flex flex-col w-60">
+ <div className="flex flex-col gap-y-1 mb-4">
+ <label className="text-white text-sm font-semibold block">
+ Branch
+ </label>
+ <p className="text-xs text-zinc-300">
+ Branch you wish to collect files of
+ </p>
+ </div>
+ <select
+ name="branch"
+ required={true}
+ className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+ >
+ <option disabled={true} selected={true}>
+ -- loading available models --
+ </option>
+ </select>
+ </div>
+ );
+ }
+
+ return (
+ <div className="flex flex-col w-60">
+ <div className="flex flex-col gap-y-1 mb-4">
+ <label className="text-white text-sm font-semibold block">Branch</label>
+ <p className="text-xs text-zinc-300">
+ Branch you wish to collect files of
+ </p>
+ </div>
+ <select
+ name="branch"
+ required={true}
+ className="bg-zinc-900 border border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+ >
+ {allBranches.map((branch) => {
+ return (
+ <option key={branch} value={branch}>
+ {branch}
+ </option>
+ );
+ })}
+ </select>
+ </div>
+ );
+}
diff --git a/frontend/src/pages/GeneralSettings/DataConnectors/Connectors/index.jsx b/frontend/src/pages/GeneralSettings/DataConnectors/Connectors/index.jsx
new file mode 100644
index 000000000..cbd66f08a
--- /dev/null
+++ b/frontend/src/pages/GeneralSettings/DataConnectors/Connectors/index.jsx
@@ -0,0 +1,19 @@
+import paths from "@/utils/paths";
+import { lazy } from "react";
+import { useParams } from "react-router-dom";
+const Github = lazy(() => import("./Github"));
+
+const CONNECTORS = {
+ github: Github,
+};
+
+export default function DataConnectorSetup() {
+ const { connector } = useParams();
+ if (!connector || !CONNECTORS.hasOwnProperty(connector)) {
+ window.location = paths.home();
+ return;
+ }
+
+ const Page = CONNECTORS[connector];
+ return <Page />;
+}
diff --git a/frontend/src/pages/GeneralSettings/DataConnectors/index.jsx b/frontend/src/pages/GeneralSettings/DataConnectors/index.jsx
new file mode 100644
index 000000000..76dc13d0a
--- /dev/null
+++ b/frontend/src/pages/GeneralSettings/DataConnectors/index.jsx
@@ -0,0 +1,38 @@
+import React from "react";
+import Sidebar, { SidebarMobileHeader } from "@/components/SettingsSidebar";
+import { isMobile } from "react-device-detect";
+import DataConnectorOption from "@/components/DataConnectorOption";
+
+export default function DataConnectors() {
+ return (
+ <div className="w-screen h-screen overflow-hidden bg-sidebar flex">
+ {!isMobile && <Sidebar />}
+ <div
+ style={{ height: isMobile ? "100%" : "calc(100% - 32px)" }}
+ className="relative md:ml-[2px] md:mr-[16px] md:my-[16px] md:rounded-[26px] bg-main-gradient w-full h-full overflow-y-scroll border-4 border-accent"
+ >
+ {isMobile && <SidebarMobileHeader />}
+ <div className="flex w-full">
+ <div className="flex flex-col w-full px-1 md:px-20 md:py-12 py-16">
+ <div className="w-full flex flex-col gap-y-1 pb-6 border-white border-b-2 border-opacity-10">
+ <div className="items-center flex gap-x-4">
+ <p className="text-2xl font-semibold text-white">
+ Data Connectors
+ </p>
+ </div>
+ <p className="text-sm font-base text-white text-opacity-60">
+ Verified data connectors allow you to add more content to your
+ AnythingLLM workspaces with no custom code or complexity.
+ <br />
+ Guaranteed to work with your AnythingLLM instance.
+ </p>
+ </div>
+ <div className="py-4 w-full flex md:flex-wrap overflow-x-scroll gap-4 max-w-full">
+ <DataConnectorOption slug="github" />
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ );
+}
diff --git a/frontend/src/utils/paths.js b/frontend/src/utils/paths.js
index bfeb8b1bd..c21c1500b 100644
--- a/frontend/src/utils/paths.js
+++ b/frontend/src/utils/paths.js
@@ -76,5 +76,13 @@ export default {
apiKeys: () => {
return "/settings/api-keys";
},
+ dataConnectors: {
+ list: () => {
+ return "/settings/data-connectors";
+ },
+ github: () => {
+ return "/settings/data-connectors/github";
+ },
+ },
},
};
diff --git a/frontend/yarn.lock b/frontend/yarn.lock
index fdb7aae69..1d0639f44 100644
--- a/frontend/yarn.lock
+++ b/frontend/yarn.lock
@@ -2061,6 +2061,11 @@ react-router@6.12.1:
dependencies:
"@remix-run/router" "1.6.3"
+react-tag-input-component@^2.0.2:
+ version "2.0.2"
+ resolved "https://registry.yarnpkg.com/react-tag-input-component/-/react-tag-input-component-2.0.2.tgz#f62f013c6a535141dd1c6c3a88858223170150f1"
+ integrity sha512-dydI9luVwwv9vrjE5u1TTnkcOVkOVL6mhFti8r6hLi78V2F2EKWQOLptURz79UYbDHLSk6tnbvGl8FE+sMpADg==
+
react-toastify@^9.1.3:
version "9.1.3"
resolved "https://registry.yarnpkg.com/react-toastify/-/react-toastify-9.1.3.tgz#1e798d260d606f50e0fab5ee31daaae1d628c5ff"
diff --git a/server/endpoints/extensions/index.js b/server/endpoints/extensions/index.js
new file mode 100644
index 000000000..fc545ce3c
--- /dev/null
+++ b/server/endpoints/extensions/index.js
@@ -0,0 +1,53 @@
+const { Telemetry } = require("../../models/telemetry");
+const {
+ forwardExtensionRequest,
+} = require("../../utils/files/documentProcessor");
+const {
+ flexUserRoleValid,
+} = require("../../utils/middleware/multiUserProtected");
+const { validatedRequest } = require("../../utils/middleware/validatedRequest");
+
+function extensionEndpoints(app) {
+ if (!app) return;
+
+ app.post(
+ "/ext/github/branches",
+ [validatedRequest, flexUserRoleValid],
+ async (request, response) => {
+ try {
+ const responseFromProcessor = await forwardExtensionRequest({
+ endpoint: "/ext/github-repo/branches",
+ method: "POST",
+ body: request.body,
+ });
+ response.status(200).json(responseFromProcessor);
+ } catch (e) {
+ console.error(e);
+ response.sendStatus(500).end();
+ }
+ }
+ );
+
+ app.post(
+ "/ext/github/repo",
+ [validatedRequest, flexUserRoleValid],
+ async (request, response) => {
+ try {
+ const responseFromProcessor = await forwardExtensionRequest({
+ endpoint: "/ext/github-repo",
+ method: "POST",
+ body: request.body,
+ });
+ await Telemetry.sendTelemetry("extension_invoked", {
+ type: "github_repo",
+ });
+ response.status(200).json(responseFromProcessor);
+ } catch (e) {
+ console.error(e);
+ response.sendStatus(500).end();
+ }
+ }
+ );
+}
+
+module.exports = { extensionEndpoints };
diff --git a/server/endpoints/system.js b/server/endpoints/system.js
index 790305cff..fe54f6d2b 100644
--- a/server/endpoints/system.js
+++ b/server/endpoints/system.js
@@ -7,7 +7,7 @@ const {
checkProcessorAlive,
acceptedFileTypes,
} = require("../utils/files/documentProcessor");
-const { purgeDocument } = require("../utils/files/purgeDocument");
+const { purgeDocument, purgeFolder } = require("../utils/files/purgeDocument");
const { getVectorDbClass } = require("../utils/helpers");
const { updateENV, dumpENV } = require("../utils/helpers/updateENV");
const {
@@ -196,8 +196,23 @@ function systemEndpoints(app) {
[validatedRequest],
async (request, response) => {
try {
- const { name, meta } = reqBody(request);
- await purgeDocument(name, meta);
+ const { name } = reqBody(request);
+ await purgeDocument(name);
+ response.sendStatus(200).end();
+ } catch (e) {
+ console.log(e.message, e);
+ response.sendStatus(500).end();
+ }
+ }
+ );
+
+ app.delete(
+ "/system/remove-folder",
+ [validatedRequest],
+ async (request, response) => {
+ try {
+ const { name } = reqBody(request);
+ await purgeFolder(name);
response.sendStatus(200).end();
} catch (e) {
console.log(e.message, e);
diff --git a/server/index.js b/server/index.js
index d6999dd35..1a106053c 100644
--- a/server/index.js
+++ b/server/index.js
@@ -18,6 +18,7 @@ const { utilEndpoints } = require("./endpoints/utils");
const { Telemetry } = require("./models/telemetry");
const { developerEndpoints } = require("./endpoints/api");
const setupTelemetry = require("./utils/telemetry");
+const { extensionEndpoints } = require("./endpoints/extensions");
const app = express();
const apiRouter = express.Router();
const FILE_LIMIT = "3GB";
@@ -34,6 +35,7 @@ app.use(
app.use("/api", apiRouter);
systemEndpoints(apiRouter);
+extensionEndpoints(apiRouter);
workspaceEndpoints(apiRouter);
chatEndpoints(apiRouter);
adminEndpoints(apiRouter);
diff --git a/server/utils/files/documentProcessor.js b/server/utils/files/documentProcessor.js
index ea47a1582..5239a8708 100644
--- a/server/utils/files/documentProcessor.js
+++ b/server/utils/files/documentProcessor.js
@@ -59,9 +59,32 @@ async function processLink(link = "") {
});
}
+// We will not ever expose the document processor to the frontend API so instead we relay
+// all requests through the server. You can use this function to directly expose a specific endpoint
+// on the document processor.
+async function forwardExtensionRequest({ endpoint, method, body }) {
+ return await fetch(`${PROCESSOR_API}${endpoint}`, {
+ method,
+ body, // Stringified JSON!
+ headers: {
+ "Content-Type": "application/json",
+ },
+ })
+ .then((res) => {
+ if (!res.ok) throw new Error("Response could not be completed");
+ return res.json();
+ })
+ .then((res) => res)
+ .catch((e) => {
+ console.log(e.message);
+ return { success: false, data: {}, reason: e.message };
+ });
+}
+
module.exports = {
checkProcessorAlive,
processDocument,
processLink,
acceptedFileTypes,
+ forwardExtensionRequest,
};
diff --git a/server/utils/files/index.js b/server/utils/files/index.js
index 83505f8b4..c86221876 100644
--- a/server/utils/files/index.js
+++ b/server/utils/files/index.js
@@ -144,18 +144,14 @@ async function storeVectorResult(vectorData = [], filename = null) {
// Purges a file from the documents/ folder.
async function purgeSourceDocument(filename = null) {
if (!filename) return;
- console.log(`Purging document of ${filename}.`);
+ console.log(`Purging source document of ${filename}.`);
const filePath =
process.env.NODE_ENV === "development"
? path.resolve(__dirname, `../../storage/documents`, filename)
: path.resolve(process.env.STORAGE_DIR, `documents`, filename);
- if (!fs.existsSync(filePath)) {
- console.log(`Could not located cachefile for ${filename}`, filePath);
- return;
- }
-
+ if (!fs.existsSync(filePath)) return;
fs.rmSync(filePath);
return;
}
@@ -163,7 +159,7 @@ async function purgeSourceDocument(filename = null) {
// Purges a vector-cache file from the vector-cache/ folder.
async function purgeVectorCache(filename = null) {
if (!filename) return;
- console.log(`Purging cached vectorized results of ${filename}.`);
+ console.log(`Purging vector-cache of ${filename}.`);
const digest = uuidv5(filename, uuidv5.URL);
const filePath =
@@ -171,11 +167,7 @@ async function purgeVectorCache(filename = null) {
? path.resolve(__dirname, `../../storage/vector-cache`, `${digest}.json`)
: path.resolve(process.env.STORAGE_DIR, `vector-cache`, `${digest}.json`);
- if (!fs.existsSync(filePath)) {
- console.log(`Could not located cache file for ${filename}`, filePath);
- return;
- }
-
+ if (!fs.existsSync(filePath)) return;
fs.rmSync(filePath);
return;
}
diff --git a/server/utils/files/purgeDocument.js b/server/utils/files/purgeDocument.js
index a584a4261..27fe14710 100644
--- a/server/utils/files/purgeDocument.js
+++ b/server/utils/files/purgeDocument.js
@@ -1,8 +1,11 @@
+const fs = require("fs");
+const path = require("path");
+
const { purgeVectorCache, purgeSourceDocument } = require(".");
const { Document } = require("../../models/documents");
const { Workspace } = require("../../models/workspace");
-async function purgeDocument(filename, meta) {
+async function purgeDocument(filename) {
const workspaces = await Workspace.where();
for (const workspace of workspaces) {
await Document.removeDocuments(workspace, [filename]);
@@ -12,6 +15,45 @@ async function purgeDocument(filename, meta) {
return;
}
+async function purgeFolder(folderName) {
+ if (folderName === "custom-documents") return;
+ const documentsFolder =
+ process.env.NODE_ENV === "development"
+ ? path.resolve(__dirname, `../../storage/documents`)
+ : path.resolve(process.env.STORAGE_DIR, `documents`);
+
+ const folderPath = path.resolve(documentsFolder, folderName);
+ const filenames = fs
+ .readdirSync(folderPath)
+ .map((file) => path.join(folderName, file));
+ const workspaces = await Workspace.where();
+
+ const purgePromises = [];
+ // Remove associated Vector-cache files
+ for (const filename of filenames) {
+ const rmVectorCache = () =>
+ new Promise((resolve) =>
+ purgeVectorCache(filename).then(() => resolve(true))
+ );
+ purgePromises.push(rmVectorCache);
+ }
+
+ // Remove workspace document associations
+ for (const workspace of workspaces) {
+ const rmWorkspaceDoc = () =>
+ new Promise((resolve) =>
+ Document.removeDocuments(workspace, filenames).then(() => resolve(true))
+ );
+ purgePromises.push(rmWorkspaceDoc);
+ }
+
+ await Promise.all(purgePromises.flat().map((f) => f()));
+ fs.rmSync(folderPath, { recursive: true }); // Delete root document and source files.
+
+ return;
+}
+
module.exports = {
purgeDocument,
+ purgeFolder,
};