From 52e3dd98356c12cdf90dc418e6cd4c25b1f40d88 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Sun, 11 Sep 2022 10:09:17 +0300 Subject: [PATCH] Pass the whole TextContentConfig as argument to text_to_jsonl methods - Let the specific text_to_jsonl method decide which of the TextContentConfig fields it needs to convert type to jsonl - This simplifies extending TextContentConfig for a specific type without modifying all text_to_jsonl methods - It keeps the number of args being passed to the `text_to_jsonl' methods in check --- src/processor/ledger/beancount_to_jsonl.py | 6 +++++- src/processor/markdown/markdown_to_jsonl.py | 6 +++++- src/processor/org_mode/org_to_jsonl.py | 6 +++++- src/search_type/text_search.py | 2 +- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/processor/ledger/beancount_to_jsonl.py b/src/processor/ledger/beancount_to_jsonl.py index 3af45c7a..d0d43f1a 100644 --- a/src/processor/ledger/beancount_to_jsonl.py +++ b/src/processor/ledger/beancount_to_jsonl.py @@ -13,13 +13,17 @@ import time from src.utils.helpers import get_absolute_path, is_none_or_empty, mark_entries_for_update from src.utils.constants import empty_escape_sequences from src.utils.jsonl import dump_jsonl, compress_jsonl_data +from src.utils.rawconfig import TextContentConfig logger = logging.getLogger(__name__) # Define Functions -def beancount_to_jsonl(beancount_files, beancount_file_filter, output_file, previous_entries=None): +def beancount_to_jsonl(config: TextContentConfig, previous_entries=None): + # Extract required fields from config + beancount_files, beancount_file_filter, output_file = config.input_files, config.input_filter, config.compressed_jsonl + # Input Validation if is_none_or_empty(beancount_files) and is_none_or_empty(beancount_file_filter): print("At least one of beancount-files or beancount-file-filter is required to be specified") diff --git a/src/processor/markdown/markdown_to_jsonl.py b/src/processor/markdown/markdown_to_jsonl.py index e7fc2779..9856c502 100644 --- a/src/processor/markdown/markdown_to_jsonl.py +++ b/src/processor/markdown/markdown_to_jsonl.py @@ -13,13 +13,17 @@ import time from src.utils.helpers import get_absolute_path, is_none_or_empty, mark_entries_for_update from src.utils.constants import empty_escape_sequences from src.utils.jsonl import dump_jsonl, compress_jsonl_data +from src.utils.rawconfig import TextContentConfig logger = logging.getLogger(__name__) # Define Functions -def markdown_to_jsonl(markdown_files, markdown_file_filter, output_file, previous_entries=None): +def markdown_to_jsonl(config: TextContentConfig, previous_entries=None): + # Extract required fields from config + markdown_files, markdown_file_filter, output_file = config.input_files, config.input_filter, config.compressed_jsonl + # Input Validation if is_none_or_empty(markdown_files) and is_none_or_empty(markdown_file_filter): print("At least one of markdown-files or markdown-file-filter is required to be specified") diff --git a/src/processor/org_mode/org_to_jsonl.py b/src/processor/org_mode/org_to_jsonl.py index f166810f..25ab1d9b 100644 --- a/src/processor/org_mode/org_to_jsonl.py +++ b/src/processor/org_mode/org_to_jsonl.py @@ -14,13 +14,17 @@ from src.processor.org_mode import orgnode from src.utils.helpers import get_absolute_path, is_none_or_empty, mark_entries_for_update from src.utils.jsonl import dump_jsonl, compress_jsonl_data from src.utils import state +from src.utils.rawconfig import TextContentConfig logger = logging.getLogger(__name__) # Define Functions -def org_to_jsonl(org_files, org_file_filter, output_file, previous_entries=None): +def org_to_jsonl(config: TextContentConfig, previous_entries=None): + # Extract required fields from config + org_files, org_file_filter, output_file = config.input_files, config.input_filter, config.compressed_jsonl + # Input Validation if is_none_or_empty(org_files) and is_none_or_empty(org_file_filter): print("At least one of org-files or org-file-filter is required to be specified") diff --git a/src/search_type/text_search.py b/src/search_type/text_search.py index 238c4736..e48b8803 100644 --- a/src/search_type/text_search.py +++ b/src/search_type/text_search.py @@ -183,7 +183,7 @@ def setup(text_to_jsonl, config: TextContentConfig, search_config: TextSearchCon # Map notes in text files to (compressed) JSONL formatted file config.compressed_jsonl = resolve_absolute_path(config.compressed_jsonl) previous_entries = extract_entries(config.compressed_jsonl) if config.compressed_jsonl.exists() else None - entries_with_indices = text_to_jsonl(config.input_files, config.input_filter, config.compressed_jsonl, previous_entries) + entries_with_indices = text_to_jsonl(config, previous_entries) # Extract Updated Entries entries = extract_entries(config.compressed_jsonl)