From 21857530684f91b80df33041f19c8e2198bc3a59 Mon Sep 17 00:00:00 2001
From: timothycarambat <rambat1010@gmail.com>
Date: Mon, 22 Jul 2024 12:53:11 -0700
Subject: [PATCH] patch text.substring bug from compressor
---
server/utils/helpers/tiktoken.js | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/server/utils/helpers/tiktoken.js b/server/utils/helpers/tiktoken.js
index c5852892f..a3fa3b639 100644
--- a/server/utils/helpers/tiktoken.js
+++ b/server/utils/helpers/tiktoken.js
@@ -19,8 +19,13 @@ class TokenManager {
// https://github.com/openai/tiktoken/blob/9e79899bc248d5313c7dd73562b5e211d728723d/tiktoken/core.py#L91C20-L91C38
// Returns number[]
tokensFromString(input = "") {
- const tokens = this.encoder.encode(input, undefined, []);
- return tokens;
+ try {
+ const tokens = this.encoder.encode(String(input), undefined, []);
+ return tokens;
+ } catch (e) {
+ console.error(e);
+ return [];
+ }
}
bytesFromTokens(tokens = []) {