From 3aa0c30fee10014b0fdf237711bca778c3081243 Mon Sep 17 00:00:00 2001 From: Debanjum Singh Solanky Date: Mon, 16 Aug 2021 13:22:46 -0700 Subject: [PATCH] Use absolute file path to open files in org-to-jsonl.py, asymmetric.py Exit script if neither org_files, org_file_filter is present --- processor/org-mode/org-to-jsonl.py | 7 ++++--- search_types/asymmetric.py | 10 +++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/processor/org-mode/org-to-jsonl.py b/processor/org-mode/org-to-jsonl.py index d23bdfbd..20678a67 100644 --- a/processor/org-mode/org-to-jsonl.py +++ b/processor/org-mode/org-to-jsonl.py @@ -12,7 +12,7 @@ import gzip # Define Functions def dump_jsonl(jsonl_data, output_path, verbose=0): "Write List of JSON objects to JSON line file" - with open(output_path, 'w', encoding='utf-8') as f: + with open(get_absolute_path(output_path), 'w', encoding='utf-8') as f: f.write(jsonl_data) if verbose > 0: @@ -30,7 +30,7 @@ def compress_jsonl_data(jsonl_data, output_path, verbose=0): def load_jsonl(input_path, verbose=0): "Read List of JSON objects from JSON line file" data = [] - with open(input_path, 'r', encoding='utf-8') as f: + with open(get_absolute_path(input_path), 'r', encoding='utf-8') as f: for line in f: data.append(json.loads(line.rstrip('\n|\r'))) @@ -105,7 +105,7 @@ def get_absolute_path(filepath): if __name__ == '__main__': - # Setup argument parser + # Setup Argument Parser parser = argparse.ArgumentParser(description="Map Org-Mode notes into (compressed) JSONL format") parser.add_argument('--jsonl-file', '-o', type=pathlib.Path, required=True, help="Output file for JSONL formatted notes") parser.add_argument('--org-files', '-i', nargs='*', help="List of org-mode files to process") @@ -116,6 +116,7 @@ if __name__ == '__main__': if is_none_or_empty(args.org_files) and is_none_or_empty(args.org_file_filter): print("At least one of org-files or org-file-filter is required to be specified") + exit(1) # Get Org Files to Process org_files = get_org_files(args.org_files, args.org_file_filter) diff --git a/search_types/asymmetric.py b/search_types/asymmetric.py index ff191b11..9dc6123d 100644 --- a/search_types/asymmetric.py +++ b/search_types/asymmetric.py @@ -23,7 +23,7 @@ def initialize_model(): def extract_entries(notesfile, verbose=False): "Load entries from compressed jsonl" entries = [] - with gzip.open(str(notesfile.expanduser()), 'rt', encoding='utf8') as jsonl: + with gzip.open(get_absolute_path(notesfile), 'rt', encoding='utf8') as jsonl: for line in jsonl: note = json.loads(line.strip()) @@ -44,13 +44,13 @@ def compute_embeddings(entries, bi_encoder, embeddings_file, verbose=False): "Compute (and Save) Embeddings or Load Pre-Computed Embeddings" # Load pre-computed embeddings from file if exists if embeddings_file.exists(): - corpus_embeddings = torch.load(str(embeddings_file.expanduser())) + corpus_embeddings = torch.load(get_absolute_path(embeddings_file)) if verbose: print(f"Loaded embeddings from {embeddings_file}") else: # Else compute the corpus_embeddings from scratch, which can take a while corpus_embeddings = bi_encoder.encode(entries, convert_to_tensor=True, show_progress_bar=True) - torch.save(corpus_embeddings, str(embeddings_file.expanduser())) + torch.save(corpus_embeddings, get_absolute_path(embeddings_file)) if verbose: print(f"Computed embeddings and save them to {embeddings_file}") @@ -140,6 +140,10 @@ def collate_results(hits, entries, count=5, verbose=False): in hits[0:count]] +def get_absolute_path(filepath): + return str(pathlib.Path(filepath).expanduser().absolute()) + + if __name__ == '__main__': # Setup Argument Parser parser = argparse.ArgumentParser(description="Map Org-Mode notes into JSONL format")