From 48cb8f2897b27913100347e19e241677375aeebf Mon Sep 17 00:00:00 2001
From: Timothy Carambat <rambat1010@gmail.com>
Date: Wed, 7 Feb 2024 15:17:32 -0800
Subject: [PATCH] Add support to upload rawText document via api (#692)
* Add support to upload rawText document via api
* update API doc endpoint with correct textContent key
* update response swagger doc
---
collector/index.js | 24 +++
collector/processRawText/index.js | 69 +++++++++
server/endpoints/api/document/index.js | 187 +++++++++++++++++++++++-
server/endpoints/api/workspace/index.js | 16 +-
server/swagger/openapi.json | 147 ++++++++++++++++++-
server/utils/files/documentProcessor.js | 20 +++
6 files changed, 453 insertions(+), 10 deletions(-)
create mode 100644 collector/processRawText/index.js
diff --git a/collector/index.js b/collector/index.js
index 062d78959..9ebe5f1ce 100644
--- a/collector/index.js
+++ b/collector/index.js
@@ -12,6 +12,7 @@ const { processSingleFile } = require("./processSingleFile");
const { processLink } = require("./processLink");
const { wipeCollectorStorage } = require("./utils/files");
const extensions = require("./extensions");
+const { processRawText } = require("./processRawText");
const app = express();
app.use(cors({ origin: true }));
@@ -66,6 +67,29 @@ app.post("/process-link", async function (request, response) {
return;
});
+app.post("/process-raw-text", async function (request, response) {
+ const { textContent, metadata } = reqBody(request);
+ try {
+ const {
+ success,
+ reason,
+ documents = [],
+ } = await processRawText(textContent, metadata);
+ response
+ .status(200)
+ .json({ filename: metadata.title, success, reason, documents });
+ } catch (e) {
+ console.error(e);
+ response.status(200).json({
+ filename: metadata?.title || "Unknown-doc.txt",
+ success: false,
+ reason: "A processing error occurred.",
+ documents: [],
+ });
+ }
+ return;
+});
+
extensions(app);
app.get("/accepts", function (_, response) {
diff --git a/collector/processRawText/index.js b/collector/processRawText/index.js
new file mode 100644
index 000000000..d435c9e7e
--- /dev/null
+++ b/collector/processRawText/index.js
@@ -0,0 +1,69 @@
+const { v4 } = require("uuid");
+const { writeToServerDocuments } = require("../utils/files");
+const { tokenizeString } = require("../utils/tokenizer");
+const { default: slugify } = require("slugify");
+
+// Will remove the last .extension from the input
+// and stringify the input + move to lowercase.
+function stripAndSlug(input) {
+ if (!input.includes('.')) return slugify(input, { lower: true });
+ return slugify(input.split('.').slice(0, -1).join('-'), { lower: true })
+}
+
+const METADATA_KEYS = {
+ possible: {
+ url: ({ url, title }) => {
+ let validUrl;
+ try {
+ const u = new URL(url);
+ validUrl = ["https:", "http:"].includes(u.protocol);
+ } catch { }
+
+ if (validUrl) return `web://${url.toLowerCase()}.website`;
+ return `file://${stripAndSlug(title)}.txt`;
+ },
+ title: ({ title }) => `${stripAndSlug(title)}.txt`,
+ docAuthor: ({ docAuthor }) => { return typeof docAuthor === 'string' ? docAuthor : 'no author specified' },
+ description: ({ description }) => { return typeof description === 'string' ? description : 'no description found' },
+ docSource: ({ docSource }) => { return typeof docSource === 'string' ? docSource : 'no source set' },
+ chunkSource: ({ chunkSource, title }) => { return typeof chunkSource === 'string' ? chunkSource : `${stripAndSlug(title)}.txt` },
+ published: ({ published }) => {
+ if (isNaN(Number(published))) return new Date().toLocaleString();
+ return new Date(Number(published)).toLocaleString()
+ },
+ }
+}
+
+async function processRawText(textContent, metadata) {
+ console.log(`-- Working Raw Text doc ${metadata.title} --`);
+ if (!textContent || textContent.length === 0) {
+ return {
+ success: false,
+ reason: "textContent was empty - nothing to process.",
+ documents: [],
+ };
+ }
+
+ const data = {
+ id: v4(),
+ url: METADATA_KEYS.possible.url(metadata),
+ title: METADATA_KEYS.possible.title(metadata),
+ docAuthor: METADATA_KEYS.possible.docAuthor(metadata),
+ description: METADATA_KEYS.possible.description(metadata),
+ docSource: METADATA_KEYS.possible.docSource(metadata),
+ chunkSource: METADATA_KEYS.possible.chunkSource(metadata),
+ published: METADATA_KEYS.possible.published(metadata),
+ wordCount: textContent.split(" ").length,
+ pageContent: textContent,
+ token_count_estimate: tokenizeString(textContent).length,
+ };
+
+ const document = writeToServerDocuments(
+ data,
+ `raw-${stripAndSlug(metadata.title)}-${data.id}`
+ );
+ console.log(`[SUCCESS]: Raw text and metadata saved & ready for embedding.\n`);
+ return { success: true, reason: null, documents: [document] };
+}
+
+module.exports = { processRawText }
\ No newline at end of file
diff --git a/server/endpoints/api/document/index.js b/server/endpoints/api/document/index.js
index b72debbdb..c210fff4a 100644
--- a/server/endpoints/api/document/index.js
+++ b/server/endpoints/api/document/index.js
@@ -6,6 +6,7 @@ const {
acceptedFileTypes,
processDocument,
processLink,
+ processRawText,
} = require("../../../utils/files/documentProcessor");
const {
viewLocalFiles,
@@ -90,6 +91,7 @@ function apiDocumentEndpoints(app) {
error: `Document processing API is not online. Document ${originalname} will not be processed automatically.`,
})
.end();
+ return;
}
const { success, reason, documents } =
@@ -127,7 +129,7 @@ function apiDocumentEndpoints(app) {
#swagger.requestBody = {
description: 'Link of web address to be scraped.',
required: true,
- type: 'file',
+ type: 'object',
content: {
"application/json": {
schema: {
@@ -186,6 +188,7 @@ function apiDocumentEndpoints(app) {
error: `Document processing API is not online. Link ${link} will not be processed automatically.`,
})
.end();
+ return;
}
const { success, reason, documents } = await processLink(link);
@@ -212,6 +215,138 @@ function apiDocumentEndpoints(app) {
}
);
+ app.post(
+ "/v1/document/raw-text",
+ [validApiKey],
+ async (request, response) => {
+ /*
+ #swagger.tags = ['Documents']
+ #swagger.description = 'Upload a file by specifying its raw text content and metadata values without having to upload a file.'
+ #swagger.requestBody = {
+ description: 'Text content and metadata of the file to be saved to the system. Use metadata-schema endpoint to get the possible metadata keys',
+ required: true,
+ type: 'object',
+ content: {
+ "application/json": {
+ schema: {
+ type: 'object',
+ example: {
+ "textContent": "This is the raw text that will be saved as a document in AnythingLLM.",
+ "metadata": {
+ keyOne: "valueOne",
+ keyTwo: "valueTwo",
+ etc: "etc"
+ }
+ }
+ }
+ }
+ }
+ }
+ #swagger.responses[200] = {
+ content: {
+ "application/json": {
+ schema: {
+ type: 'object',
+ example: {
+ success: true,
+ error: null,
+ documents: [
+ {
+ "id": "c530dbe6-bff1-4b9e-b87f-710d539d20bc",
+ "url": "file://my-document.txt",
+ "title": "hello-world.txt",
+ "docAuthor": "no author found",
+ "description": "No description found.",
+ "docSource": "My custom description set during upload",
+ "chunkSource": "no chunk source specified",
+ "published": "1/16/2024, 3:46:33 PM",
+ "wordCount": 252,
+ "pageContent": "AnythingLLM is the best....",
+ "token_count_estimate": 447,
+ "location": "custom-documents/raw-my-doc-text-c530dbe6-bff1-4b9e-b87f-710d539d20bc.json"
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ #swagger.responses[403] = {
+ schema: {
+ "$ref": "#/definitions/InvalidAPIKey"
+ }
+ }
+ */
+ try {
+ const requiredMetadata = ["title"];
+ const { textContent, metadata = {} } = reqBody(request);
+ const processingOnline = await checkProcessorAlive();
+
+ if (!processingOnline) {
+ response
+ .status(500)
+ .json({
+ success: false,
+ error: `Document processing API is not online. Request will not be processed.`,
+ })
+ .end();
+ return;
+ }
+
+ if (
+ !requiredMetadata.every(
+ (reqKey) =>
+ Object.keys(metadata).includes(reqKey) && !!metadata[reqKey]
+ )
+ ) {
+ response
+ .status(422)
+ .json({
+ success: false,
+ error: `You are missing required metadata key:value pairs in your request. Required metadata key:values are ${requiredMetadata
+ .map((v) => `'${v}'`)
+ .join(", ")}`,
+ })
+ .end();
+ return;
+ }
+
+ if (!textContent || textContent?.length === 0) {
+ response
+ .status(422)
+ .json({
+ success: false,
+ error: `The 'textContent' key cannot have an empty value.`,
+ })
+ .end();
+ return;
+ }
+
+ const { success, reason, documents } = await processRawText(
+ textContent,
+ metadata
+ );
+ if (!success) {
+ response
+ .status(500)
+ .json({ success: false, error: reason, documents })
+ .end();
+ return;
+ }
+
+ console.log(
+ `Document created successfully. It is now available in documents.`
+ );
+ await Telemetry.sendTelemetry("raw_document_uploaded");
+ await EventLogs.logEvent("api_raw_document_uploaded");
+ response.status(200).json({ success: true, error: null, documents });
+ } catch (e) {
+ console.log(e.message, e);
+ response.sendStatus(500).end();
+ }
+ }
+ );
+
app.get("/v1/documents", [validApiKey], async (_, response) => {
/*
#swagger.tags = ['Documents']
@@ -367,6 +502,56 @@ function apiDocumentEndpoints(app) {
}
}
);
+
+ app.get(
+ "/v1/document/metadata-schema",
+ [validApiKey],
+ async (_, response) => {
+ /*
+ #swagger.tags = ['Documents']
+ #swagger.description = 'Get the known available metadata schema for when doing a raw-text upload and the acceptable type of value for each key.'
+ #swagger.responses[200] = {
+ content: {
+ "application/json": {
+ schema: {
+ type: 'object',
+ example: {
+ "schema": {
+ "keyOne": "string | number | nullable",
+ "keyTwo": "string | number | nullable",
+ "specialKey": "number",
+ "title": "string",
+ }
+ }
+ }
+ }
+ }
+ }
+ #swagger.responses[403] = {
+ schema: {
+ "$ref": "#/definitions/InvalidAPIKey"
+ }
+ }
+ */
+ try {
+ response.status(200).json({
+ schema: {
+ // If you are updating this be sure to update the collector METADATA_KEYS constant in /processRawText.
+ url: "string | nullable",
+ title: "string",
+ docAuthor: "string | nullable",
+ description: "string | nullable",
+ docSource: "string | nullable",
+ chunkSource: "string | nullable",
+ published: "epoch timestamp in ms | nullable",
+ },
+ });
+ } catch (e) {
+ console.log(e.message, e);
+ response.sendStatus(500).end();
+ }
+ }
+ );
}
module.exports = { apiDocumentEndpoints };
diff --git a/server/endpoints/api/workspace/index.js b/server/endpoints/api/workspace/index.js
index 885d0f1ae..b846f3f8d 100644
--- a/server/endpoints/api/workspace/index.js
+++ b/server/endpoints/api/workspace/index.js
@@ -26,17 +26,17 @@ function apiWorkspaceEndpoints(app) {
#swagger.tags = ['Workspaces']
#swagger.description = 'Create a new workspace'
#swagger.requestBody = {
- description: 'JSON object containing new display name of workspace.',
- required: true,
- type: 'object',
- content: {
- "application/json": {
- example: {
- name: "My New Workspace",
- }
+ description: 'JSON object containing new display name of workspace.',
+ required: true,
+ type: 'object',
+ content: {
+ "application/json": {
+ example: {
+ name: "My New Workspace",
}
}
}
+ }
#swagger.responses[200] = {
content: {
"application/json": {
diff --git a/server/swagger/openapi.json b/server/swagger/openapi.json
index fd2ef8898..4554a7aad 100644
--- a/server/swagger/openapi.json
+++ b/server/swagger/openapi.json
@@ -973,7 +973,7 @@
"requestBody": {
"description": "Link of web address to be scraped.",
"required": true,
- "type": "file",
+ "type": "object",
"content": {
"application/json": {
"schema": {
@@ -987,6 +987,96 @@
}
}
},
+ "/v1/document/raw-text": {
+ "post": {
+ "tags": [
+ "Documents"
+ ],
+ "description": "Upload a file by specifying its raw text content and metadata values without having to upload a file.",
+ "parameters": [
+ {
+ "name": "Authorization",
+ "in": "header",
+ "schema": {
+ "type": "string"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "example": {
+ "success": true,
+ "error": null,
+ "documents": [
+ {
+ "id": "c530dbe6-bff1-4b9e-b87f-710d539d20bc",
+ "url": "file://my-document.txt",
+ "title": "hello-world.txt",
+ "docAuthor": "no author found",
+ "description": "No description found.",
+ "docSource": "My custom description set during upload",
+ "chunkSource": "no chunk source specified",
+ "published": "1/16/2024, 3:46:33 PM",
+ "wordCount": 252,
+ "pageContent": "AnythingLLM is the best....",
+ "token_count_estimate": 447,
+ "location": "custom-documents/raw-my-doc-text-c530dbe6-bff1-4b9e-b87f-710d539d20bc.json"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "403": {
+ "description": "Forbidden",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/InvalidAPIKey"
+ }
+ },
+ "application/xml": {
+ "schema": {
+ "$ref": "#/components/schemas/InvalidAPIKey"
+ }
+ }
+ }
+ },
+ "422": {
+ "description": "Unprocessable Entity"
+ },
+ "500": {
+ "description": "Internal Server Error"
+ }
+ },
+ "requestBody": {
+ "description": "Text content and metadata of the file to be saved to the system. Use metadata-schema endpoint to get the possible metadata keys",
+ "required": true,
+ "type": "object",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "example": {
+ "textContent": "This is the raw text that will be saved as a document in AnythingLLM.",
+ "metadata": {
+ "keyOne": "valueOne",
+ "keyTwo": "valueTwo",
+ "etc": "etc"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
"/v1/documents": {
"get": {
"tags": [
@@ -1195,6 +1285,61 @@
}
}
},
+ "/v1/document/metadata-schema": {
+ "get": {
+ "tags": [
+ "Documents"
+ ],
+ "description": "Get the known available metadata schema for when doing a raw-text upload and the acceptable type of value for each key.",
+ "parameters": [
+ {
+ "name": "Authorization",
+ "in": "header",
+ "schema": {
+ "type": "string"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "content": {
+ "application/json": {
+ "schema": {
+ "type": "object",
+ "example": {
+ "schema": {
+ "keyOne": "string | number | nullable",
+ "keyTwo": "string | number | nullable",
+ "specialKey": "number",
+ "title": "string"
+ }
+ }
+ }
+ }
+ }
+ },
+ "403": {
+ "description": "Forbidden",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/InvalidAPIKey"
+ }
+ },
+ "application/xml": {
+ "schema": {
+ "$ref": "#/components/schemas/InvalidAPIKey"
+ }
+ }
+ }
+ },
+ "500": {
+ "description": "Internal Server Error"
+ }
+ }
+ }
+ },
"/v1/workspace/new": {
"post": {
"tags": [
diff --git a/server/utils/files/documentProcessor.js b/server/utils/files/documentProcessor.js
index 27d0f5f2b..ef8eba17c 100644
--- a/server/utils/files/documentProcessor.js
+++ b/server/utils/files/documentProcessor.js
@@ -59,6 +59,25 @@ async function processLink(link = "") {
});
}
+async function processRawText(textContent = "", metadata = {}) {
+ return await fetch(`${PROCESSOR_API}/process-raw-text`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({ textContent, metadata }),
+ })
+ .then((res) => {
+ if (!res.ok) throw new Error("Response could not be completed");
+ return res.json();
+ })
+ .then((res) => res)
+ .catch((e) => {
+ console.log(e.message);
+ return { success: false, reason: e.message, documents: [] };
+ });
+}
+
// We will not ever expose the document processor to the frontend API so instead we relay
// all requests through the server. You can use this function to directly expose a specific endpoint
// on the document processor.
@@ -85,6 +104,7 @@ module.exports = {
checkProcessorAlive,
processDocument,
processLink,
+ processRawText,
acceptedFileTypes,
forwardExtensionRequest,
};