From e4c445f805c6ad453f65aaed4fbf25acce011c42 Mon Sep 17 00:00:00 2001 From: sabaimran Date: Sun, 2 Jul 2023 13:35:02 -0700 Subject: [PATCH] Add try-except-finally blocks around configure calls in /update --- src/khoj/configure.py | 182 +++++++++++++++++++++------------------- src/khoj/routers/api.py | 9 +- 2 files changed, 102 insertions(+), 89 deletions(-) diff --git a/src/khoj/configure.py b/src/khoj/configure.py index de543349..19a07d44 100644 --- a/src/khoj/configure.py +++ b/src/khoj/configure.py @@ -93,98 +93,106 @@ def configure_search(model: SearchModels, config: FullConfig, regenerate: bool, logger.warning("🚨 No Content or Search type is configured.") return - # Initialize Org Notes Search - if (t == state.SearchType.Org or t == None) and config.content_type.org and config.search_type.asymmetric: - logger.info("🦄 Setting up search for orgmode notes") - # Extract Entries, Generate Notes Embeddings - model.org_search = text_search.setup( - OrgToJsonl, - config.content_type.org, - search_config=config.search_type.asymmetric, - regenerate=regenerate, - filters=[DateFilter(), WordFilter(), FileFilter()], - ) - - # Initialize Org Music Search - if (t == state.SearchType.Music or t == None) and config.content_type.music and config.search_type.asymmetric: - logger.info("🎺 Setting up search for org-music") - # Extract Entries, Generate Music Embeddings - model.music_search = text_search.setup( - OrgToJsonl, - config.content_type.music, - search_config=config.search_type.asymmetric, - regenerate=regenerate, - filters=[DateFilter(), WordFilter()], - ) - - # Initialize Markdown Search - if (t == state.SearchType.Markdown or t == None) and config.content_type.markdown and config.search_type.asymmetric: - logger.info("💎 Setting up search for markdown notes") - # Extract Entries, Generate Markdown Embeddings - model.markdown_search = text_search.setup( - MarkdownToJsonl, - config.content_type.markdown, - search_config=config.search_type.asymmetric, - regenerate=regenerate, - filters=[DateFilter(), WordFilter(), FileFilter()], - ) - - # Initialize Ledger Search - if (t == state.SearchType.Ledger or t == None) and config.content_type.ledger and config.search_type.symmetric: - logger.info("💸 Setting up search for ledger") - # Extract Entries, Generate Ledger Embeddings - model.ledger_search = text_search.setup( - BeancountToJsonl, - config.content_type.ledger, - search_config=config.search_type.symmetric, - regenerate=regenerate, - filters=[DateFilter(), WordFilter(), FileFilter()], - ) - - # Initialize PDF Search - if (t == state.SearchType.Pdf or t == None) and config.content_type.pdf and config.search_type.asymmetric: - logger.info("🖨️ Setting up search for pdf") - # Extract Entries, Generate PDF Embeddings - model.pdf_search = text_search.setup( - PdfToJsonl, - config.content_type.pdf, - search_config=config.search_type.asymmetric, - regenerate=regenerate, - filters=[DateFilter(), WordFilter(), FileFilter()], - ) - - # Initialize Image Search - if (t == state.SearchType.Image or t == None) and config.content_type.image and config.search_type.image: - logger.info("🌄 Setting up search for images") - # Extract Entries, Generate Image Embeddings - model.image_search = image_search.setup( - config.content_type.image, search_config=config.search_type.image, regenerate=regenerate - ) - - if (t == state.SearchType.Github or t == None) and config.content_type.github and config.search_type.asymmetric: - logger.info("🐙 Setting up search for github") - # Extract Entries, Generate Github Embeddings - model.github_search = text_search.setup( - GithubToJsonl, - config.content_type.github, - search_config=config.search_type.asymmetric, - regenerate=regenerate, - filters=[DateFilter(), WordFilter(), FileFilter()], - ) - - # Initialize External Plugin Search - if (t == None or t in state.SearchType) and config.content_type.plugins: - logger.info("🔌 Setting up search for plugins") - model.plugin_search = {} - for plugin_type, plugin_config in config.content_type.plugins.items(): - model.plugin_search[plugin_type] = text_search.setup( - JsonlToJsonl, - plugin_config, + try: + # Initialize Org Notes Search + if (t == state.SearchType.Org or t == None) and config.content_type.org and config.search_type.asymmetric: + logger.info("🦄 Setting up search for orgmode notes") + # Extract Entries, Generate Notes Embeddings + model.org_search = text_search.setup( + OrgToJsonl, + config.content_type.org, search_config=config.search_type.asymmetric, regenerate=regenerate, filters=[DateFilter(), WordFilter(), FileFilter()], ) + # Initialize Org Music Search + if (t == state.SearchType.Music or t == None) and config.content_type.music and config.search_type.asymmetric: + logger.info("🎺 Setting up search for org-music") + # Extract Entries, Generate Music Embeddings + model.music_search = text_search.setup( + OrgToJsonl, + config.content_type.music, + search_config=config.search_type.asymmetric, + regenerate=regenerate, + filters=[DateFilter(), WordFilter()], + ) + + # Initialize Markdown Search + if ( + (t == state.SearchType.Markdown or t == None) + and config.content_type.markdown + and config.search_type.asymmetric + ): + logger.info("💎 Setting up search for markdown notes") + # Extract Entries, Generate Markdown Embeddings + model.markdown_search = text_search.setup( + MarkdownToJsonl, + config.content_type.markdown, + search_config=config.search_type.asymmetric, + regenerate=regenerate, + filters=[DateFilter(), WordFilter(), FileFilter()], + ) + + # Initialize Ledger Search + if (t == state.SearchType.Ledger or t == None) and config.content_type.ledger and config.search_type.symmetric: + logger.info("💸 Setting up search for ledger") + # Extract Entries, Generate Ledger Embeddings + model.ledger_search = text_search.setup( + BeancountToJsonl, + config.content_type.ledger, + search_config=config.search_type.symmetric, + regenerate=regenerate, + filters=[DateFilter(), WordFilter(), FileFilter()], + ) + + # Initialize PDF Search + if (t == state.SearchType.Pdf or t == None) and config.content_type.pdf and config.search_type.asymmetric: + logger.info("🖨️ Setting up search for pdf") + # Extract Entries, Generate PDF Embeddings + model.pdf_search = text_search.setup( + PdfToJsonl, + config.content_type.pdf, + search_config=config.search_type.asymmetric, + regenerate=regenerate, + filters=[DateFilter(), WordFilter(), FileFilter()], + ) + + # Initialize Image Search + if (t == state.SearchType.Image or t == None) and config.content_type.image and config.search_type.image: + logger.info("🌄 Setting up search for images") + # Extract Entries, Generate Image Embeddings + model.image_search = image_search.setup( + config.content_type.image, search_config=config.search_type.image, regenerate=regenerate + ) + + if (t == state.SearchType.Github or t == None) and config.content_type.github and config.search_type.asymmetric: + logger.info("🐙 Setting up search for github") + # Extract Entries, Generate Github Embeddings + model.github_search = text_search.setup( + GithubToJsonl, + config.content_type.github, + search_config=config.search_type.asymmetric, + regenerate=regenerate, + filters=[DateFilter(), WordFilter(), FileFilter()], + ) + + # Initialize External Plugin Search + if (t == None or t in state.SearchType) and config.content_type.plugins: + logger.info("🔌 Setting up search for plugins") + model.plugin_search = {} + for plugin_type, plugin_config in config.content_type.plugins.items(): + model.plugin_search[plugin_type] = text_search.setup( + JsonlToJsonl, + plugin_config, + search_config=config.search_type.asymmetric, + regenerate=regenerate, + filters=[DateFilter(), WordFilter(), FileFilter()], + ) + except Exception as e: + logger.error("🚨 Failed to setup search") + raise e + # Invalidate Query Cache state.query_cache = LRU() diff --git a/src/khoj/routers/api.py b/src/khoj/routers/api.py index bec66ac8..91a67589 100644 --- a/src/khoj/routers/api.py +++ b/src/khoj/routers/api.py @@ -357,8 +357,13 @@ def update( ): try: state.search_index_lock.acquire() - state.model = configure_search(state.model, state.config, regenerate=force or False, t=t) - state.search_index_lock.release() + try: + state.model = configure_search(state.model, state.config, regenerate=force or False, t=t) + except Exception as e: + logger.error(e) + raise HTTPException(status_code=500, detail=str(e)) + finally: + state.search_index_lock.release() except ValueError as e: logger.error(e) raise HTTPException(status_code=500, detail=str(e))