From 2f3db0e63a203cf881ab24ea08a8f221c4542571 Mon Sep 17 00:00:00 2001
From: Sean Hatfield <seanhatfield5@gmail.com>
Date: Mon, 22 Jan 2024 16:41:20 -0800
Subject: [PATCH] [FEAT] support pinecone serverless (#639)
* migrate pinecone package to latest version and migrate pinecone vectordb provider class
* remove pinecone environment name env variable and update docs to reflect removal & serverless support complete
* migrate query for pinecone db
* typo in log
---------
Co-authored-by: timothycarambat <rambat1010@gmail.com>
---
docker/.env.example | 1 -
.../PineconeDBOptions/index.jsx | 17 -----
server/.env.example | 1 -
server/endpoints/api/system/index.js | 1 -
server/models/systemSettings.js | 1 -
server/package.json | 4 +-
server/swagger/openapi.json | 1 -
server/utils/helpers/updateENV.js | 5 --
.../pinecone/PINECONE_SETUP.md | 1 -
.../utils/vectorDbProviders/pinecone/index.js | 72 ++++++++-----------
server/yarn.lock | 38 ++++++++--
11 files changed, 64 insertions(+), 78 deletions(-)
diff --git a/docker/.env.example b/docker/.env.example
index f3eba2418..0adabbdf7 100644
--- a/docker/.env.example
+++ b/docker/.env.example
@@ -76,7 +76,6 @@ GID='1000'
# Enable all below if you are using vector database: Pinecone.
# VECTOR_DB="pinecone"
-# PINECONE_ENVIRONMENT=
# PINECONE_API_KEY=
# PINECONE_INDEX=
diff --git a/frontend/src/components/VectorDBSelection/PineconeDBOptions/index.jsx b/frontend/src/components/VectorDBSelection/PineconeDBOptions/index.jsx
index 5491f758c..bb3381fe1 100644
--- a/frontend/src/components/VectorDBSelection/PineconeDBOptions/index.jsx
+++ b/frontend/src/components/VectorDBSelection/PineconeDBOptions/index.jsx
@@ -17,23 +17,6 @@ export default function PineconeDBOptions({ settings }) {
spellCheck={false}
/>
</div>
-
- <div className="flex flex-col w-60">
- <label className="text-white text-sm font-semibold block mb-4">
- Pinecone Index Environment
- </label>
- <input
- type="text"
- name="PineConeEnvironment"
- className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
- placeholder="us-gcp-west-1"
- defaultValue={settings?.PineConeEnvironment}
- required={true}
- autoComplete="off"
- spellCheck={false}
- />
- </div>
-
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-4">
Pinecone Index Name
diff --git a/server/.env.example b/server/.env.example
index 23e20bb13..e44748b41 100644
--- a/server/.env.example
+++ b/server/.env.example
@@ -73,7 +73,6 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
# Enable all below if you are using vector database: Pinecone.
# VECTOR_DB="pinecone"
-# PINECONE_ENVIRONMENT=
# PINECONE_API_KEY=
# PINECONE_INDEX=
diff --git a/server/endpoints/api/system/index.js b/server/endpoints/api/system/index.js
index b18019b14..fa1237135 100644
--- a/server/endpoints/api/system/index.js
+++ b/server/endpoints/api/system/index.js
@@ -40,7 +40,6 @@ function apiSystemEndpoints(app) {
example: {
"settings": {
"VectorDB": "pinecone",
- "PineConeEnvironment": "us-west4-gcp-free",
"PineConeKey": true,
"PineConeIndex": "my-pinecone-index",
"LLMProvider": "azure",
diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
index 90de463f0..b4e93bde6 100644
--- a/server/models/systemSettings.js
+++ b/server/models/systemSettings.js
@@ -32,7 +32,6 @@ const SystemSettings = {
LocalAiApiKey: !!process.env.LOCAL_AI_API_KEY,
...(vectorDB === "pinecone"
? {
- PineConeEnvironment: process.env.PINECONE_ENVIRONMENT,
PineConeKey: !!process.env.PINECONE_API_KEY,
PineConeIndex: process.env.PINECONE_INDEX,
}
diff --git a/server/package.json b/server/package.json
index 58913fe35..c8a41b795 100644
--- a/server/package.json
+++ b/server/package.json
@@ -24,7 +24,7 @@
"@azure/openai": "1.0.0-beta.10",
"@google/generative-ai": "^0.1.3",
"@googleapis/youtube": "^9.0.0",
- "@pinecone-database/pinecone": "^0.1.6",
+ "@pinecone-database/pinecone": "^2.0.1",
"@prisma/client": "5.3.0",
"@qdrant/js-client-rest": "^1.4.0",
"@xenova/transformers": "^2.14.0",
@@ -77,4 +77,4 @@
"nodemon": "^2.0.22",
"prettier": "^3.0.3"
}
-}
\ No newline at end of file
+}
diff --git a/server/swagger/openapi.json b/server/swagger/openapi.json
index c7532059d..fd2ef8898 100644
--- a/server/swagger/openapi.json
+++ b/server/swagger/openapi.json
@@ -1941,7 +1941,6 @@
"example": {
"settings": {
"VectorDB": "pinecone",
- "PineConeEnvironment": "us-west4-gcp-free",
"PineConeKey": true,
"PineConeIndex": "my-pinecone-index",
"LLMProvider": "azure",
diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js
index 8114927b8..b061061e8 100644
--- a/server/utils/helpers/updateENV.js
+++ b/server/utils/helpers/updateENV.js
@@ -171,11 +171,6 @@ const KEY_MAPPING = {
envKey: "QDRANT_API_KEY",
checks: [],
},
-
- PineConeEnvironment: {
- envKey: "PINECONE_ENVIRONMENT",
- checks: [],
- },
PineConeKey: {
envKey: "PINECONE_API_KEY",
checks: [],
diff --git a/server/utils/vectorDbProviders/pinecone/PINECONE_SETUP.md b/server/utils/vectorDbProviders/pinecone/PINECONE_SETUP.md
index 789f56f90..235f37b19 100644
--- a/server/utils/vectorDbProviders/pinecone/PINECONE_SETUP.md
+++ b/server/utils/vectorDbProviders/pinecone/PINECONE_SETUP.md
@@ -19,7 +19,6 @@
```
VECTOR_DB="pinecone"
-PINECONE_ENVIRONMENT=us-west4-gcp-free
PINECONE_API_KEY=sklive-123xyz
PINECONE_INDEX=my-primary-index # the value from the first instruction!
```
diff --git a/server/utils/vectorDbProviders/pinecone/index.js b/server/utils/vectorDbProviders/pinecone/index.js
index 260c0a257..b8f288c06 100644
--- a/server/utils/vectorDbProviders/pinecone/index.js
+++ b/server/utils/vectorDbProviders/pinecone/index.js
@@ -1,4 +1,4 @@
-const { PineconeClient } = require("@pinecone-database/pinecone");
+const { Pinecone } = require("@pinecone-database/pinecone");
const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
const { storeVectorResult, cachedVectorInformation } = require("../../files");
const { v4: uuidv4 } = require("uuid");
@@ -8,37 +8,35 @@ const {
getEmbeddingEngineSelection,
} = require("../../helpers");
-const Pinecone = {
+const PineconeDB = {
name: "Pinecone",
connect: async function () {
if (process.env.VECTOR_DB !== "pinecone")
throw new Error("Pinecone::Invalid ENV settings");
- const client = new PineconeClient();
- await client.init({
+ const client = new Pinecone({
apiKey: process.env.PINECONE_API_KEY,
- environment: process.env.PINECONE_ENVIRONMENT,
- });
- const pineconeIndex = client.Index(process.env.PINECONE_INDEX);
- const { status } = await client.describeIndex({
- indexName: process.env.PINECONE_INDEX,
});
- if (!status.ready) throw new Error("Pinecode::Index not ready.");
+ const pineconeIndex = client.Index(process.env.PINECONE_INDEX);
+ const { status } = await client.describeIndex(process.env.PINECONE_INDEX);
+
+ if (!status.ready) throw new Error("Pinecone::Index not ready.");
return { client, pineconeIndex, indexName: process.env.PINECONE_INDEX };
},
totalVectors: async function () {
const { pineconeIndex } = await this.connect();
- const { namespaces } = await pineconeIndex.describeIndexStats1();
+ const { namespaces } = await pineconeIndex.describeIndexStats();
+
return Object.values(namespaces).reduce(
- (a, b) => a + (b?.vectorCount || 0),
+ (a, b) => a + (b?.recordCount || 0),
0
);
},
namespaceCount: async function (_namespace = null) {
const { pineconeIndex } = await this.connect();
const namespace = await this.namespace(pineconeIndex, _namespace);
- return namespace?.vectorCount || 0;
+ return namespace?.recordCount || 0;
},
similarityResponse: async function (
index,
@@ -52,13 +50,12 @@ const Pinecone = {
sourceDocuments: [],
scores: [],
};
- const response = await index.query({
- queryRequest: {
- namespace,
- vector: queryVector,
- topK: topN,
- includeMetadata: true,
- },
+
+ const pineconeNamespace = index.namespace(namespace);
+ const response = await pineconeNamespace.query({
+ vector: queryVector,
+ topK: topN,
+ includeMetadata: true,
});
response.matches.forEach((match) => {
@@ -70,10 +67,9 @@ const Pinecone = {
return result;
},
-
namespace: async function (index, namespace = null) {
if (!namespace) throw new Error("No namespace value provided.");
- const { namespaces } = await index.describeIndexStats1();
+ const { namespaces } = await index.describeIndexStats();
return namespaces.hasOwnProperty(namespace) ? namespaces[namespace] : null;
},
hasNamespace: async function (namespace = null) {
@@ -83,11 +79,12 @@ const Pinecone = {
},
namespaceExists: async function (index, namespace = null) {
if (!namespace) throw new Error("No namespace value provided.");
- const { namespaces } = await index.describeIndexStats1();
+ const { namespaces } = await index.describeIndexStats();
return namespaces.hasOwnProperty(namespace);
},
deleteVectorsInNamespace: async function (index, namespace = null) {
- await index.delete1({ namespace, deleteAll: true });
+ const pineconeNamespace = index.namespace(namespace);
+ await pineconeNamespace.deleteAll();
return true;
},
addDocumentToNamespace: async function (
@@ -104,6 +101,7 @@ const Pinecone = {
const cacheResult = await cachedVectorInformation(fullFilePath);
if (cacheResult.exists) {
const { pineconeIndex } = await this.connect();
+ const pineconeNamespace = pineconeIndex.namespace(namespace);
const { chunks } = cacheResult;
const documentVectors = [];
@@ -115,14 +113,7 @@ const Pinecone = {
documentVectors.push({ docId, vectorId: id });
return { ...chunk, id };
});
-
- // Push chunks with new ids to pinecone.
- await pineconeIndex.upsert({
- upsertRequest: {
- vectors: [...newChunks],
- namespace,
- },
- });
+ await pineconeNamespace.upsert([...newChunks]);
}
await DocumentVectors.bulkInsert(documentVectors);
@@ -170,15 +161,11 @@ const Pinecone = {
if (vectors.length > 0) {
const chunks = [];
const { pineconeIndex } = await this.connect();
+ const pineconeNamespace = pineconeIndex.namespace(namespace);
console.log("Inserting vectorized chunks into Pinecone.");
for (const chunk of toChunks(vectors, 100)) {
chunks.push(chunk);
- await pineconeIndex.upsert({
- upsertRequest: {
- vectors: [...chunk],
- namespace,
- },
- });
+ await pineconeNamespace.upsert([...chunk]);
}
await storeVectorResult(chunks, fullFilePath);
}
@@ -199,11 +186,10 @@ const Pinecone = {
if (knownDocuments.length === 0) return;
const vectorIds = knownDocuments.map((doc) => doc.vectorId);
+
+ const pineconeNamespace = pineconeIndex.namespace(namespace);
for (const batchOfVectorIds of toChunks(vectorIds, 1000)) {
- await pineconeIndex.delete1({
- ids: batchOfVectorIds,
- namespace,
- });
+ await pineconeNamespace.deleteMany(batchOfVectorIds);
}
const indexes = knownDocuments.map((doc) => doc.id);
@@ -285,4 +271,4 @@ const Pinecone = {
},
};
-module.exports.Pinecone = Pinecone;
+module.exports.Pinecone = PineconeDB;
diff --git a/server/yarn.lock b/server/yarn.lock
index b25104dd1..67b4e11b6 100644
--- a/server/yarn.lock
+++ b/server/yarn.lock
@@ -625,12 +625,15 @@
"@octokit/webhooks-types" "7.1.0"
aggregate-error "^3.1.0"
-"@pinecone-database/pinecone@^0.1.6":
- version "0.1.6"
- resolved "https://registry.yarnpkg.com/@pinecone-database/pinecone/-/pinecone-0.1.6.tgz#13374ae9462c8eea0dc26683cafeddc4e7c0375f"
- integrity sha512-tCnVc28udecthhgSBTdcMhYEW+xsR++AdZasp+ZE/AvUD1hOR2IR3edjk9m0sDxZyvXbno2KeqUbLIOZr7sCTw==
+"@pinecone-database/pinecone@^2.0.1":
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/@pinecone-database/pinecone/-/pinecone-2.0.1.tgz#1dce6e06e299dfe2c26490aacffe1103e316b8fc"
+ integrity sha512-a1ejzrqdSQ2yW+9QUi2TVlKwYUbrvGH+QH6POJhITyaOz9ANE+EhXqToC9af93Ctzq9n87+bOUvBvewLeW++Mw==
dependencies:
+ "@sinclair/typebox" "^0.29.0"
+ ajv "^8.12.0"
cross-fetch "^3.1.5"
+ encoding "^0.1.13"
"@pkgr/core@^0.1.0":
version "0.1.0"
@@ -743,6 +746,11 @@
resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df"
integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==
+"@sinclair/typebox@^0.29.0":
+ version "0.29.6"
+ resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.29.6.tgz#4cd8372f9247372edd5fc5af44f67e2032c46e2f"
+ integrity sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==
+
"@tootallnate/once@1":
version "1.1.2"
resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82"
@@ -955,6 +963,16 @@ ajv@^6.12.4:
json-schema-traverse "^0.4.1"
uri-js "^4.2.2"
+ajv@^8.12.0:
+ version "8.12.0"
+ resolved "https://registry.yarnpkg.com/ajv/-/ajv-8.12.0.tgz#d1a0527323e22f53562c567c00991577dfbe19d1"
+ integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==
+ dependencies:
+ fast-deep-equal "^3.1.1"
+ json-schema-traverse "^1.0.0"
+ require-from-string "^2.0.2"
+ uri-js "^4.2.2"
+
ansi-regex@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304"
@@ -1934,7 +1952,7 @@ encodeurl@~1.0.2:
resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==
-encoding@^0.1.12:
+encoding@^0.1.12, encoding@^0.1.13:
version "0.1.13"
resolved "https://registry.yarnpkg.com/encoding/-/encoding-0.1.13.tgz#56574afdd791f54a8e9b2785c0582a2d26210fa9"
integrity sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==
@@ -3321,6 +3339,11 @@ json-schema-traverse@^0.4.1:
resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660"
integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==
+json-schema-traverse@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz#ae7bcb3656ab77a73ba5c49bf654f38e6b6860e2"
+ integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==
+
json-stable-stringify-without-jsonify@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651"
@@ -4674,6 +4697,11 @@ require-directory@^2.1.1:
resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==
+require-from-string@^2.0.2:
+ version "2.0.2"
+ resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909"
+ integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==
+
resolve-from@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6"