sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2025-02-20 06:55:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

Show debug log, not warning when use default tokenizer for context stuffing

Browse source
This commit is contained in:
Debanjum Singh Solanky 2024-09-12 20:20:14 -07:00
parent f173188dcf
commit 178b78f87b
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/khoj/processor/conversation/utils.py
View file

@ -238,7 +238,7 @@ def truncate_messages(
else: else:
encoder = AutoTokenizer.from_pretrained(default_tokenizer) encoder = AutoTokenizer.from_pretrained(default_tokenizer)
state.pretrained_tokenizers[default_tokenizer] = encoder state.pretrained_tokenizers[default_tokenizer] = encoder
logger.warning( logger.debug(
f"Fallback to default chat model tokenizer: {tokenizer_name}.\nConfigure tokenizer for unsupported model: {model_name} in Khoj settings to improve context stuffing." f"Fallback to default chat model tokenizer: {tokenizer_name}.\nConfigure tokenizer for unsupported model: {model_name} in Khoj settings to improve context stuffing."
) )