sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-11-27 17:35:07 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix process for generating embeddings for Notion entries (#648)

Browse source 
* Fix process for generating embeddings for Notion entries
* If no title field found, just log a warning and set the title to
This commit is contained in:
sabaimran 2024-02-20 13:46:56 -08:00 committed by GitHub
parent 43013c4fd4
commit 138f5223bd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/khoj/processor/content/notion/notion_to_entries.py
View file

@ -112,7 +112,9 @@ class NotionToEntries(TextToEntries):
page_entries = self.process_page(p_or_d) page_entries = self.process_page(p_or_d)
current_entries.extend(page_entries) current_entries.extend(page_entries)
return self.update_entries_with_ids(current_entries, user) current_entries = TextToEntries.split_entries_by_max_tokens(current_entries, max_tokens=256)
return self.update_entries_with_ids(current_entries, user=user)
def process_page(self, page): def process_page(self, page):
page_id = page["id"] page_id = page["id"]
@ -232,8 +234,9 @@ class NotionToEntries(TextToEntries):
elif "Event" in properties: elif "Event" in properties:
title_field = "Event" title_field = "Event"
elif title_field not in properties: elif title_field not in properties:
logger.error(f"Page {page_id} does not have a title field") logger.warning(f"Title field not found for page {page_id}. Setting title as None...")
return None, None title = None
return title, content
try: try:
title = page["properties"][title_field]["title"][0]["text"]["content"] title = page["properties"][title_field]["title"][0]["text"]["content"]
except Exception as e: except Exception as e: