sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-11-27 17:35:07 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Split entries by max tokens while converting Markdown entries To JSONL

Browse source
This commit is contained in:
Debanjum Singh Solanky 2022-12-26 13:14:15 -03:00
parent 24676f95d8
commit f209e30a3b
1 changed files with 6 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/processor/markdown/markdown_to_jsonl.py
View file

@ -35,6 +35,12 @@ class MarkdownToJsonl(TextToJsonl):
end = time.time()
logger.debug(f"Parse entries from Markdown files into dictionaries: {end - start} seconds")
# Split entries by max tokens supported by model
start = time.time()
current_entries = self.split_entries_by_max_tokens(current_entries, max_tokens=256)
end = time.time()
logger.debug(f"Split entries by max token size supported by model: {end - start} seconds")
# Identify, mark and merge any new entries with previous entries
start = time.time()
if not previous_entries: