sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-11-23 15:38:55 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Use absolute file path to open files in org-to-jsonl.py, asymmetric.py

Browse source 
Exit script if neither org_files, org_file_filter is present
This commit is contained in:
Debanjum Singh Solanky 2021-08-16 13:22:46 -07:00
parent e773611558
commit 3aa0c30fee
2 changed files with 11 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
processor/org-mode/org-to-jsonl.py
View file

@ -12,7 +12,7 @@ import gzip
# Define Functions
def dump_jsonl(jsonl_data, output_path, verbose=0):
"Write List of JSON objects to JSON line file"
with open(output_path, 'w', encoding='utf-8') as f:
with open(get_absolute_path(output_path), 'w', encoding='utf-8') as f:
f.write(jsonl_data)
if verbose > 0:
@ -30,7 +30,7 @@ def compress_jsonl_data(jsonl_data, output_path, verbose=0):
def load_jsonl(input_path, verbose=0):
"Read List of JSON objects from JSON line file"
data = []
with open(input_path, 'r', encoding='utf-8') as f:
with open(get_absolute_path(input_path), 'r', encoding='utf-8') as f:
for line in f:
data.append(json.loads(line.rstrip('\n|\r')))
@ -105,7 +105,7 @@ def get_absolute_path(filepath):
if __name__ == '__main__':
# Setup argument parser
# Setup Argument Parser
parser = argparse.ArgumentParser(description="Map Org-Mode notes into (compressed) JSONL format")
parser.add_argument('--jsonl-file', '-o', type=pathlib.Path, required=True, help="Output file for JSONL formatted notes")
parser.add_argument('--org-files', '-i', nargs='*', help="List of org-mode files to process")
@ -116,6 +116,7 @@ if __name__ == '__main__':
if is_none_or_empty(args.org_files) and is_none_or_empty(args.org_file_filter):
print("At least one of org-files or org-file-filter is required to be specified")
exit(1)
# Get Org Files to Process
org_files = get_org_files(args.org_files, args.org_file_filter)

10
search_types/asymmetric.py
View file

@ -23,7 +23,7 @@ def initialize_model():
def extract_entries(notesfile, verbose=False):
"Load entries from compressed jsonl"
entries = []
with gzip.open(str(notesfile.expanduser()), 'rt', encoding='utf8') as jsonl:
with gzip.open(get_absolute_path(notesfile), 'rt', encoding='utf8') as jsonl:
for line in jsonl:
note = json.loads(line.strip())
@ -44,13 +44,13 @@ def compute_embeddings(entries, bi_encoder, embeddings_file, verbose=False):
"Compute (and Save) Embeddings or Load Pre-Computed Embeddings"
# Load pre-computed embeddings from file if exists
if embeddings_file.exists():
corpus_embeddings = torch.load(str(embeddings_file.expanduser()))
corpus_embeddings = torch.load(get_absolute_path(embeddings_file))
if verbose:
print(f"Loaded embeddings from {embeddings_file}")
else: # Else compute the corpus_embeddings from scratch, which can take a while
corpus_embeddings = bi_encoder.encode(entries, convert_to_tensor=True, show_progress_bar=True)
torch.save(corpus_embeddings, str(embeddings_file.expanduser()))
torch.save(corpus_embeddings, get_absolute_path(embeddings_file))
if verbose:
print(f"Computed embeddings and save them to {embeddings_file}")
@ -140,6 +140,10 @@ def collate_results(hits, entries, count=5, verbose=False):
in hits[0:count]]
def get_absolute_path(filepath):
return str(pathlib.Path(filepath).expanduser().absolute())
if __name__ == '__main__':
# Setup Argument Parser
parser = argparse.ArgumentParser(description="Map Org-Mode notes into JSONL format")