From 2f3db0e63a203cf881ab24ea08a8f221c4542571 Mon Sep 17 00:00:00 2001 From: Sean Hatfield <seanhatfield5@gmail.com> Date: Mon, 22 Jan 2024 16:41:20 -0800 Subject: [PATCH] [FEAT] support pinecone serverless (#639) * migrate pinecone package to latest version and migrate pinecone vectordb provider class * remove pinecone environment name env variable and update docs to reflect removal & serverless support complete * migrate query for pinecone db * typo in log --------- Co-authored-by: timothycarambat <rambat1010@gmail.com> --- docker/.env.example | 1 - .../PineconeDBOptions/index.jsx | 17 ----- server/.env.example | 1 - server/endpoints/api/system/index.js | 1 - server/models/systemSettings.js | 1 - server/package.json | 4 +- server/swagger/openapi.json | 1 - server/utils/helpers/updateENV.js | 5 -- .../pinecone/PINECONE_SETUP.md | 1 - .../utils/vectorDbProviders/pinecone/index.js | 72 ++++++++----------- server/yarn.lock | 38 ++++++++-- 11 files changed, 64 insertions(+), 78 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index f3eba2418..0adabbdf7 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -76,7 +76,6 @@ GID='1000' # Enable all below if you are using vector database: Pinecone. # VECTOR_DB="pinecone" -# PINECONE_ENVIRONMENT= # PINECONE_API_KEY= # PINECONE_INDEX= diff --git a/frontend/src/components/VectorDBSelection/PineconeDBOptions/index.jsx b/frontend/src/components/VectorDBSelection/PineconeDBOptions/index.jsx index 5491f758c..bb3381fe1 100644 --- a/frontend/src/components/VectorDBSelection/PineconeDBOptions/index.jsx +++ b/frontend/src/components/VectorDBSelection/PineconeDBOptions/index.jsx @@ -17,23 +17,6 @@ export default function PineconeDBOptions({ settings }) { spellCheck={false} /> </div> - - <div className="flex flex-col w-60"> - <label className="text-white text-sm font-semibold block mb-4"> - Pinecone Index Environment - </label> - <input - type="text" - name="PineConeEnvironment" - className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5" - placeholder="us-gcp-west-1" - defaultValue={settings?.PineConeEnvironment} - required={true} - autoComplete="off" - spellCheck={false} - /> - </div> - <div className="flex flex-col w-60"> <label className="text-white text-sm font-semibold block mb-4"> Pinecone Index Name diff --git a/server/.env.example b/server/.env.example index 23e20bb13..e44748b41 100644 --- a/server/.env.example +++ b/server/.env.example @@ -73,7 +73,6 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea # Enable all below if you are using vector database: Pinecone. # VECTOR_DB="pinecone" -# PINECONE_ENVIRONMENT= # PINECONE_API_KEY= # PINECONE_INDEX= diff --git a/server/endpoints/api/system/index.js b/server/endpoints/api/system/index.js index b18019b14..fa1237135 100644 --- a/server/endpoints/api/system/index.js +++ b/server/endpoints/api/system/index.js @@ -40,7 +40,6 @@ function apiSystemEndpoints(app) { example: { "settings": { "VectorDB": "pinecone", - "PineConeEnvironment": "us-west4-gcp-free", "PineConeKey": true, "PineConeIndex": "my-pinecone-index", "LLMProvider": "azure", diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 90de463f0..b4e93bde6 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -32,7 +32,6 @@ const SystemSettings = { LocalAiApiKey: !!process.env.LOCAL_AI_API_KEY, ...(vectorDB === "pinecone" ? { - PineConeEnvironment: process.env.PINECONE_ENVIRONMENT, PineConeKey: !!process.env.PINECONE_API_KEY, PineConeIndex: process.env.PINECONE_INDEX, } diff --git a/server/package.json b/server/package.json index 58913fe35..c8a41b795 100644 --- a/server/package.json +++ b/server/package.json @@ -24,7 +24,7 @@ "@azure/openai": "1.0.0-beta.10", "@google/generative-ai": "^0.1.3", "@googleapis/youtube": "^9.0.0", - "@pinecone-database/pinecone": "^0.1.6", + "@pinecone-database/pinecone": "^2.0.1", "@prisma/client": "5.3.0", "@qdrant/js-client-rest": "^1.4.0", "@xenova/transformers": "^2.14.0", @@ -77,4 +77,4 @@ "nodemon": "^2.0.22", "prettier": "^3.0.3" } -} \ No newline at end of file +} diff --git a/server/swagger/openapi.json b/server/swagger/openapi.json index c7532059d..fd2ef8898 100644 --- a/server/swagger/openapi.json +++ b/server/swagger/openapi.json @@ -1941,7 +1941,6 @@ "example": { "settings": { "VectorDB": "pinecone", - "PineConeEnvironment": "us-west4-gcp-free", "PineConeKey": true, "PineConeIndex": "my-pinecone-index", "LLMProvider": "azure", diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 8114927b8..b061061e8 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -171,11 +171,6 @@ const KEY_MAPPING = { envKey: "QDRANT_API_KEY", checks: [], }, - - PineConeEnvironment: { - envKey: "PINECONE_ENVIRONMENT", - checks: [], - }, PineConeKey: { envKey: "PINECONE_API_KEY", checks: [], diff --git a/server/utils/vectorDbProviders/pinecone/PINECONE_SETUP.md b/server/utils/vectorDbProviders/pinecone/PINECONE_SETUP.md index 789f56f90..235f37b19 100644 --- a/server/utils/vectorDbProviders/pinecone/PINECONE_SETUP.md +++ b/server/utils/vectorDbProviders/pinecone/PINECONE_SETUP.md @@ -19,7 +19,6 @@ ``` VECTOR_DB="pinecone" -PINECONE_ENVIRONMENT=us-west4-gcp-free PINECONE_API_KEY=sklive-123xyz PINECONE_INDEX=my-primary-index # the value from the first instruction! ``` diff --git a/server/utils/vectorDbProviders/pinecone/index.js b/server/utils/vectorDbProviders/pinecone/index.js index 260c0a257..b8f288c06 100644 --- a/server/utils/vectorDbProviders/pinecone/index.js +++ b/server/utils/vectorDbProviders/pinecone/index.js @@ -1,4 +1,4 @@ -const { PineconeClient } = require("@pinecone-database/pinecone"); +const { Pinecone } = require("@pinecone-database/pinecone"); const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter"); const { storeVectorResult, cachedVectorInformation } = require("../../files"); const { v4: uuidv4 } = require("uuid"); @@ -8,37 +8,35 @@ const { getEmbeddingEngineSelection, } = require("../../helpers"); -const Pinecone = { +const PineconeDB = { name: "Pinecone", connect: async function () { if (process.env.VECTOR_DB !== "pinecone") throw new Error("Pinecone::Invalid ENV settings"); - const client = new PineconeClient(); - await client.init({ + const client = new Pinecone({ apiKey: process.env.PINECONE_API_KEY, - environment: process.env.PINECONE_ENVIRONMENT, - }); - const pineconeIndex = client.Index(process.env.PINECONE_INDEX); - const { status } = await client.describeIndex({ - indexName: process.env.PINECONE_INDEX, }); - if (!status.ready) throw new Error("Pinecode::Index not ready."); + const pineconeIndex = client.Index(process.env.PINECONE_INDEX); + const { status } = await client.describeIndex(process.env.PINECONE_INDEX); + + if (!status.ready) throw new Error("Pinecone::Index not ready."); return { client, pineconeIndex, indexName: process.env.PINECONE_INDEX }; }, totalVectors: async function () { const { pineconeIndex } = await this.connect(); - const { namespaces } = await pineconeIndex.describeIndexStats1(); + const { namespaces } = await pineconeIndex.describeIndexStats(); + return Object.values(namespaces).reduce( - (a, b) => a + (b?.vectorCount || 0), + (a, b) => a + (b?.recordCount || 0), 0 ); }, namespaceCount: async function (_namespace = null) { const { pineconeIndex } = await this.connect(); const namespace = await this.namespace(pineconeIndex, _namespace); - return namespace?.vectorCount || 0; + return namespace?.recordCount || 0; }, similarityResponse: async function ( index, @@ -52,13 +50,12 @@ const Pinecone = { sourceDocuments: [], scores: [], }; - const response = await index.query({ - queryRequest: { - namespace, - vector: queryVector, - topK: topN, - includeMetadata: true, - }, + + const pineconeNamespace = index.namespace(namespace); + const response = await pineconeNamespace.query({ + vector: queryVector, + topK: topN, + includeMetadata: true, }); response.matches.forEach((match) => { @@ -70,10 +67,9 @@ const Pinecone = { return result; }, - namespace: async function (index, namespace = null) { if (!namespace) throw new Error("No namespace value provided."); - const { namespaces } = await index.describeIndexStats1(); + const { namespaces } = await index.describeIndexStats(); return namespaces.hasOwnProperty(namespace) ? namespaces[namespace] : null; }, hasNamespace: async function (namespace = null) { @@ -83,11 +79,12 @@ const Pinecone = { }, namespaceExists: async function (index, namespace = null) { if (!namespace) throw new Error("No namespace value provided."); - const { namespaces } = await index.describeIndexStats1(); + const { namespaces } = await index.describeIndexStats(); return namespaces.hasOwnProperty(namespace); }, deleteVectorsInNamespace: async function (index, namespace = null) { - await index.delete1({ namespace, deleteAll: true }); + const pineconeNamespace = index.namespace(namespace); + await pineconeNamespace.deleteAll(); return true; }, addDocumentToNamespace: async function ( @@ -104,6 +101,7 @@ const Pinecone = { const cacheResult = await cachedVectorInformation(fullFilePath); if (cacheResult.exists) { const { pineconeIndex } = await this.connect(); + const pineconeNamespace = pineconeIndex.namespace(namespace); const { chunks } = cacheResult; const documentVectors = []; @@ -115,14 +113,7 @@ const Pinecone = { documentVectors.push({ docId, vectorId: id }); return { ...chunk, id }; }); - - // Push chunks with new ids to pinecone. - await pineconeIndex.upsert({ - upsertRequest: { - vectors: [...newChunks], - namespace, - }, - }); + await pineconeNamespace.upsert([...newChunks]); } await DocumentVectors.bulkInsert(documentVectors); @@ -170,15 +161,11 @@ const Pinecone = { if (vectors.length > 0) { const chunks = []; const { pineconeIndex } = await this.connect(); + const pineconeNamespace = pineconeIndex.namespace(namespace); console.log("Inserting vectorized chunks into Pinecone."); for (const chunk of toChunks(vectors, 100)) { chunks.push(chunk); - await pineconeIndex.upsert({ - upsertRequest: { - vectors: [...chunk], - namespace, - }, - }); + await pineconeNamespace.upsert([...chunk]); } await storeVectorResult(chunks, fullFilePath); } @@ -199,11 +186,10 @@ const Pinecone = { if (knownDocuments.length === 0) return; const vectorIds = knownDocuments.map((doc) => doc.vectorId); + + const pineconeNamespace = pineconeIndex.namespace(namespace); for (const batchOfVectorIds of toChunks(vectorIds, 1000)) { - await pineconeIndex.delete1({ - ids: batchOfVectorIds, - namespace, - }); + await pineconeNamespace.deleteMany(batchOfVectorIds); } const indexes = knownDocuments.map((doc) => doc.id); @@ -285,4 +271,4 @@ const Pinecone = { }, }; -module.exports.Pinecone = Pinecone; +module.exports.Pinecone = PineconeDB; diff --git a/server/yarn.lock b/server/yarn.lock index b25104dd1..67b4e11b6 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -625,12 +625,15 @@ "@octokit/webhooks-types" "7.1.0" aggregate-error "^3.1.0" -"@pinecone-database/pinecone@^0.1.6": - version "0.1.6" - resolved "https://registry.yarnpkg.com/@pinecone-database/pinecone/-/pinecone-0.1.6.tgz#13374ae9462c8eea0dc26683cafeddc4e7c0375f" - integrity sha512-tCnVc28udecthhgSBTdcMhYEW+xsR++AdZasp+ZE/AvUD1hOR2IR3edjk9m0sDxZyvXbno2KeqUbLIOZr7sCTw== +"@pinecone-database/pinecone@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@pinecone-database/pinecone/-/pinecone-2.0.1.tgz#1dce6e06e299dfe2c26490aacffe1103e316b8fc" + integrity sha512-a1ejzrqdSQ2yW+9QUi2TVlKwYUbrvGH+QH6POJhITyaOz9ANE+EhXqToC9af93Ctzq9n87+bOUvBvewLeW++Mw== dependencies: + "@sinclair/typebox" "^0.29.0" + ajv "^8.12.0" cross-fetch "^3.1.5" + encoding "^0.1.13" "@pkgr/core@^0.1.0": version "0.1.0" @@ -743,6 +746,11 @@ resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df" integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ== +"@sinclair/typebox@^0.29.0": + version "0.29.6" + resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.29.6.tgz#4cd8372f9247372edd5fc5af44f67e2032c46e2f" + integrity sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ== + "@tootallnate/once@1": version "1.1.2" resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" @@ -955,6 +963,16 @@ ajv@^6.12.4: json-schema-traverse "^0.4.1" uri-js "^4.2.2" +ajv@^8.12.0: + version "8.12.0" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1" + integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA== + dependencies: + fast-deep-equal "^3.1.1" + json-schema-traverse "^1.0.0" + require-from-string "^2.0.2" + uri-js "^4.2.2" + ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" @@ -1934,7 +1952,7 @@ encodeurl@~1.0.2: resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== -encoding@^0.1.12: +encoding@^0.1.12, encoding@^0.1.13: version "0.1.13" resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.13.tgz#56574afdd791f54a8e9b2785c0582a2d26210fa9" integrity sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A== @@ -3321,6 +3339,11 @@ json-schema-traverse@^0.4.1: resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== +json-schema-traverse@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2" + integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug== + json-stable-stringify-without-jsonify@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" @@ -4674,6 +4697,11 @@ require-directory@^2.1.1: resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== +require-from-string@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" + integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== + resolve-from@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"