Export embedded chat history (#2329)

export embedded chat history

Co-authored-by: Timothy Carambat <rambat1010@gmail.com>

frontend/src/locales/de/common.js

@@ -421,6 +421,7 @@ const TRANSLATIONS = {
 
   "embed-chats": {
     title: "Eingebettete Chats",
+    export: "Exportieren",
     description:
       "Dies sind alle aufgezeichneten Chats und Nachrichten von jeder Einbettung, die Sie veröffentlicht haben.",
     table: {

frontend/src/locales/en/common.js

@@ -430,6 +430,7 @@ const TRANSLATIONS = {
 
   "embed-chats": {
     title: "Embed Chats",
+    export: "Export",
     description:
       "These are all the recorded chats and messages from any embed that you have published.",
     table: {

frontend/src/locales/es/common.js

@@ -424,6 +424,7 @@ const TRANSLATIONS = {
 
   "embed-chats": {
     title: "Incrustar chats",
+    export: "Exportar",
     description:
       "Estos son todos los chats y mensajes grabados de cualquier incrustación que hayas publicado.",
     table: {

frontend/src/locales/fr/common.js

@@ -438,6 +438,7 @@ const TRANSLATIONS = {
 
   "embed-chats": {
     title: "Chats intégrés",
+    export: "Exporter",
     description:
       "Voici tous les chats et messages enregistrés de tout widget intégré que vous avez publié.",
     table: {

frontend/src/locales/he/common.js

@@ -422,6 +422,7 @@ const TRANSLATIONS = {
 
   "embed-chats": {
     title: "הטמעת שיחות",
+    export: "ייצוא",
     description: "אלה כל השיחות וההודעות שנרשמו מכל הטמעה שפרסמת.",
     table: {
       embed: "הטמעה",

frontend/src/locales/it/common.js

@@ -435,6 +435,7 @@ const TRANSLATIONS = {
 
   "embed-chats": {
     title: "Chat incorporate",
+    export: "Esporta",
     description:
       "Queste sono tutte le chat e i messaggi registrati da qualsiasi embedding che hai pubblicato.",
     table: {

frontend/src/locales/ko/common.js

@@ -421,6 +421,7 @@ const TRANSLATIONS = {
 
   "embed-chats": {
     title: "임베드 채팅",
+    export: "내보내기",
     description: "게시한 임베드에서의 모든 채팅과 메시지의 기록입니다.",
     table: {
       embed: "임베드",

frontend/src/locales/pt_BR/common.js

@@ -433,6 +433,7 @@ const TRANSLATIONS = {
 
   "embed-chats": {
     title: "Incorporar Chats",
+    export: "Exportar",
     description:
       "Estes são todos os chats e mensagens registrados de qualquer incorporação que você publicou.",
     table: {

frontend/src/locales/ru/common.js

@@ -405,6 +405,7 @@ const TRANSLATIONS = {
   },
   "embed-chats": {
     title: "Встраивание чатов",
+    export: "Экспорт",
     description:
       "Это все записанные чаты и сообщения от любого встраивания, которое вы опубликовали.",
     table: {

frontend/src/locales/zh/common.js

@@ -406,6 +406,7 @@ const TRANSLATIONS = {
   // Embeddable Chat History
   "embed-chats": {
     title: "嵌入的聊天历史纪录",
+    export: "导出",
     description: "这些是您发布的任何嵌入的所有记录的聊天和消息。",
     table: {
       embed: "嵌入",

frontend/src/models/system.js

@@ -577,9 +577,10 @@ const System = {
         return { success: false, error: e.message };
       });
   },
-  exportChats: async (type = "csv") => {
+  exportChats: async (type = "csv", chatType = "workspace") => {
     const url = new URL(`${fullApiUrl()}/system/export-chats`);
     url.searchParams.append("type", encodeURIComponent(type));
+    url.searchParams.append("chatType", encodeURIComponent(chatType));
     return await fetch(url, {
       method: "GET",
       headers: baseHeaders(),

frontend/src/pages/GeneralSettings/Chats/index.jsx

@@ -59,7 +59,7 @@ export default function WorkspaceChats() {
   const { t } = useTranslation();
 
   const handleDumpChats = async (exportType) => {
-    const chats = await System.exportChats(exportType);
+    const chats = await System.exportChats(exportType, "workspace");
     if (!!chats) {
       const { name, mimeType, fileExtension, filenameFunc } =
         exportOptions[exportType];

frontend/src/pages/GeneralSettings/EmbedChats/index.jsx

@@ -1,4 +1,4 @@
-import { useEffect, useState } from "react";
+import { useEffect, useState, useRef } from "react";
 import Sidebar from "@/components/SettingsSidebar";
 import { isMobile } from "react-device-detect";
 import * as Skeleton from "react-loading-skeleton";
@@ -7,10 +7,86 @@ import useQuery from "@/hooks/useQuery";
 import ChatRow from "./ChatRow";
 import Embed from "@/models/embed";
 import { useTranslation } from "react-i18next";
+import { CaretDown, Download } from "@phosphor-icons/react";
+import showToast from "@/utils/toast";
+import { saveAs } from "file-saver";
+import System from "@/models/system";
+
+const exportOptions = {
+  csv: {
+    name: "CSV",
+    mimeType: "text/csv",
+    fileExtension: "csv",
+    filenameFunc: () => {
+      return `anythingllm-embed-chats-${new Date().toLocaleDateString()}`;
+    },
+  },
+  json: {
+    name: "JSON",
+    mimeType: "application/json",
+    fileExtension: "json",
+    filenameFunc: () => {
+      return `anythingllm-embed-chats-${new Date().toLocaleDateString()}`;
+    },
+  },
+  jsonl: {
+    name: "JSONL",
+    mimeType: "application/jsonl",
+    fileExtension: "jsonl",
+    filenameFunc: () => {
+      return `anythingllm-embed-chats-${new Date().toLocaleDateString()}-lines`;
+    },
+  },
+  jsonAlpaca: {
+    name: "JSON (Alpaca)",
+    mimeType: "application/json",
+    fileExtension: "json",
+    filenameFunc: () => {
+      return `anythingllm-embed-chats-${new Date().toLocaleDateString()}-alpaca`;
+    },
+  },
+};
 
 export default function EmbedChats() {
-  // TODO [FEAT]: Add export of embed chats
+  const [showMenu, setShowMenu] = useState(false);
+  const menuRef = useRef();
+  const openMenuButton = useRef();
   const { t } = useTranslation();
+
+  const handleDumpChats = async (exportType) => {
+    const chats = await System.exportChats(exportType, "embed");
+    if (!!chats) {
+      const { name, mimeType, fileExtension, filenameFunc } =
+        exportOptions[exportType];
+      const blob = new Blob([chats], { type: mimeType });
+      saveAs(blob, `${filenameFunc()}.${fileExtension}`);
+      showToast(`Embed chats exported successfully as ${name}.`, "success");
+    } else {
+      showToast("Failed to export embed chats.", "error");
+    }
+  };
+
+  const toggleMenu = () => {
+    setShowMenu(!showMenu);
+  };
+
+  useEffect(() => {
+    function handleClickOutside(event) {
+      if (
+        menuRef.current &&
+        !menuRef.current.contains(event.target) &&
+        !openMenuButton.current.contains(event.target)
+      ) {
+        setShowMenu(false);
+      }
+    }
+
+    document.addEventListener("mousedown", handleClickOutside);
+    return () => {
+      document.removeEventListener("mousedown", handleClickOutside);
+    };
+  }, []);
+
   return (
     <div className="w-screen h-screen overflow-hidden bg-sidebar flex">
       <Sidebar />
@@ -24,6 +100,38 @@ export default function EmbedChats() {
               <p className="text-lg leading-6 font-bold text-white">
                 {t("embed-chats.title")}
               </p>
+              <div className="relative">
+                <button
+                  ref={openMenuButton}
+                  onClick={toggleMenu}
+                  className="flex items-center gap-x-2 px-4 py-1 rounded-lg bg-primary-button hover:text-white text-xs font-semibold hover:bg-secondary shadow-[0_4px_14px_rgba(0,0,0,0.25)] h-[34px] w-fit"
+                >
+                  <Download size={18} weight="bold" />
+                  {t("embed-chats.export")}
+                  <CaretDown size={18} weight="bold" />
+                </button>
+                <div
+                  ref={menuRef}
+                  className={`${
+                    showMenu ? "slide-down" : "slide-up hidden"
+                  } z-20 w-fit rounded-lg absolute top-full right-0 bg-secondary mt-2 shadow-md`}
+                >
+                  <div className="py-2">
+                    {Object.entries(exportOptions).map(([key, data]) => (
+                      <button
+                        key={key}
+                        onClick={() => {
+                          handleDumpChats(key);
+                          setShowMenu(false);
+                        }}
+                        className="w-full text-left px-4 py-2 text-white text-sm hover:bg-[#3D4147]"
+                      >
+                        {data.name}
+                      </button>
+                    ))}
+                  </div>
+                </div>
+              </div>
             </div>
             <p className="text-xs leading-[18px] font-base text-white text-opacity-60">
               {t("embed-chats.description")}

server/endpoints/system.js

@@ -39,10 +39,7 @@ const {
   isMultiUserSetup,
 } = require("../utils/middleware/multiUserProtected");
 const { fetchPfp, determinePfpFilepath } = require("../utils/files/pfp");
-const {
-  prepareWorkspaceChatsForExport,
-  exportChatsAsType,
-} = require("../utils/helpers/chat/convertTo");
+const { exportChatsAsType } = require("../utils/helpers/chat/convertTo");
 const { EventLogs } = require("../models/eventLogs");
 const { CollectorApi } = require("../utils/collectorApi");
 const {
@@ -1009,13 +1006,13 @@ function systemEndpoints(app) {
     [validatedRequest, flexUserRoleValid([ROLES.manager, ROLES.admin])],
     async (request, response) => {
       try {
-        const { type = "jsonl" } = request.query;
-        const chats = await prepareWorkspaceChatsForExport(type);
-        const { contentType, data } = await exportChatsAsType(chats, type);
+        const { type = "jsonl", chatType = "workspace" } = request.query;
+        const { contentType, data } = await exportChatsAsType(type, chatType);
         await EventLogs.logEvent(
           "exported_chats",
           {
             type,
+            chatType,
           },
           response.locals.user?.id
         );

server/utils/helpers/chat/convertTo.js

@@ -1,22 +1,25 @@
 // Helpers that convert workspace chats to some supported format
 // for external use by the user.
 
-const { Workspace } = require("../../../models/workspace");
 const { WorkspaceChats } = require("../../../models/workspaceChats");
+const { EmbedChats } = require("../../../models/embedChats");
 const { safeJsonParse } = require("../../http");
 
 async function convertToCSV(preparedData) {
-  const rows = ["id,username,workspace,prompt,response,sent_at,rating"];
+  const headers = new Set(["id", "workspace", "prompt", "response", "sent_at"]);
+  preparedData.forEach((item) =>
+    Object.keys(item).forEach((key) => headers.add(key))
+  );
+
+  const rows = [Array.from(headers).join(",")];
+
   for (const item of preparedData) {
-    const record = [
-      item.id,
-      escapeCsv(item.username),
-      escapeCsv(item.workspace),
-      escapeCsv(item.prompt),
-      escapeCsv(item.response),
-      item.sent_at,
-      item.feedback,
-    ].join(",");
+    const record = Array.from(headers)
+      .map((header) => {
+        const value = item[header] ?? "";
+        return escapeCsv(String(value));
+      })
+      .join(",");
     rows.push(record);
   }
   return rows.join("\n");
@@ -37,29 +40,56 @@ async function convertToJSONL(workspaceChatsMap) {
     .join("\n");
 }
 
-async function prepareWorkspaceChatsForExport(format = "jsonl") {
+async function prepareChatsForExport(format = "jsonl", chatType = "workspace") {
   if (!exportMap.hasOwnProperty(format))
-    throw new Error("Invalid export type.");
+    throw new Error(`Invalid export type: ${format}`);
 
-  const chats = await WorkspaceChats.whereWithData({}, null, null, {
-    id: "asc",
-  });
+  let chats;
+  if (chatType === "workspace") {
+    chats = await WorkspaceChats.whereWithData({}, null, null, {
+      id: "asc",
+    });
+  } else if (chatType === "embed") {
+    chats = await EmbedChats.whereWithEmbedAndWorkspace(
+      {},
+      null,
+      {
+        id: "asc",
+      },
+      null
+    );
+  } else {
+    throw new Error(`Invalid chat type: ${chatType}`);
+  }
 
   if (format === "csv" || format === "json") {
     const preparedData = chats.map((chat) => {
       const responseJson = JSON.parse(chat.response);
-      return {
+      const baseData = {
         id: chat.id,
+        prompt: chat.prompt,
+        response: responseJson.text,
+        sent_at: chat.createdAt,
+      };
+
+      if (chatType === "embed") {
+        return {
+          ...baseData,
+          workspace: chat.embed_config
+            ? chat.embed_config.workspace.name
+            : "unknown workspace",
+        };
+      }
+
+      return {
+        ...baseData,
+        workspace: chat.workspace ? chat.workspace.name : "unknown workspace",
         username: chat.user
           ? chat.user.username
           : chat.api_session_id !== null
             ? "API"
             : "unknown user",
-        workspace: chat.workspace ? chat.workspace.name : "unknown workspace",
-        prompt: chat.prompt,
-        response: responseJson.text,
-        sent_at: chat.createdAt,
-        feedback:
+        rating:
           chat.feedbackScore === null
             ? "--"
             : chat.feedbackScore
@@ -71,22 +101,13 @@ async function prepareWorkspaceChatsForExport(format = "jsonl") {
     return preparedData;
   }
 
-  const workspaceIds = [...new Set(chats.map((chat) => chat.workspaceId))];
-  const workspacesWithPrompts = await Promise.all(
-    workspaceIds.map((id) => Workspace.get({ id: Number(id) }))
-  );
-  const workspacePromptsMap = workspacesWithPrompts.reduce((acc, workspace) => {
-    acc[workspace.id] = workspace.openAiPrompt;
-    return acc;
-  }, {});
-
   if (format === "jsonAlpaca") {
     const preparedData = chats.map((chat) => {
       const responseJson = JSON.parse(chat.response);
       return {
         instruction: buildSystemPrompt(
           chat,
-          workspacePromptsMap[chat.workspaceId]
+          chat.workspace ? chat.workspace.openAiPrompt : null
         ),
         input: chat.prompt,
         output: responseJson.text,
@@ -106,7 +127,7 @@ async function prepareWorkspaceChatsForExport(format = "jsonl") {
           {
             role: "system",
             content:
-              workspacePromptsMap[workspaceId] ||
+              chat.workspace?.openAiPrompt ||
               "Given the following conversation, relevant context, and a follow up question, reply with an answer to the current question the user is asking. Return only your response to the question given the above information following the users instructions as needed.",
           },
         ],
@@ -150,16 +171,18 @@ const exportMap = {
 };
 
 function escapeCsv(str) {
+  if (str === null || str === undefined) return '""';
   return `"${str.replace(/"/g, '""').replace(/\n/g, " ")}"`;
 }
 
-async function exportChatsAsType(workspaceChatsMap, format = "jsonl") {
+async function exportChatsAsType(format = "jsonl", chatType = "workspace") {
   const { contentType, func } = exportMap.hasOwnProperty(format)
     ? exportMap[format]
     : exportMap.jsonl;
+  const chats = await prepareChatsForExport(format, chatType);
   return {
     contentType,
-    data: await func(workspaceChatsMap),
+    data: await func(chats),
   };
 }
 
@@ -181,6 +204,6 @@ function buildSystemPrompt(chat, prompt = null) {
 }
 
 module.exports = {
-  prepareWorkspaceChatsForExport,
+  prepareChatsForExport,
   exportChatsAsType,
 };