sij/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2024-11-30 10:53:02 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve log messages in text_entries and memory leak unit test

Browse source
This commit is contained in:
Debanjum Singh Solanky 2023-11-06 19:26:54 -08:00
parent 38f24a037d
commit a08b152358
2 changed files with 4 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/khoj/processor/text_to_entries.py
View file

@ -93,7 +93,7 @@ class TextToEntries(ABC):
num_deleted_entries = 0 num_deleted_entries = 0
if regenerate: if regenerate:
with timer("Prepared dataset for regeneration in", logger): with timer("Cleared existing dataset for regeneration in", logger):
logger.debug(f"Deleting all entries for file type {file_type}") logger.debug(f"Deleting all entries for file type {file_type}")
num_deleted_entries = EntryAdapters.delete_all_entries(user, file_type) num_deleted_entries = EntryAdapters.delete_all_entries(user, file_type)

4
tests/test_helpers.py
View file

@ -64,6 +64,7 @@ def test_encode_docs_memory_leak():
batch_size = 20 batch_size = 20
embeddings_model = EmbeddingsModel() embeddings_model = EmbeddingsModel()
memory_usage_trend = [] memory_usage_trend = []
device = f"{helpers.get_device()}".upper()
# Act # Act
# Encode random strings repeatedly and record memory usage trend # Encode random strings repeatedly and record memory usage trend
@ -76,8 +77,9 @@ def test_encode_docs_memory_leak():
# Calculate slope of line fitting memory usage history # Calculate slope of line fitting memory usage history
memory_usage_trend = np.array(memory_usage_trend) memory_usage_trend = np.array(memory_usage_trend)
slope, _, _, _, _ = linregress(np.arange(len(memory_usage_trend)), memory_usage_trend) slope, _, _, _, _ = linregress(np.arange(len(memory_usage_trend)), memory_usage_trend)
print(f"Memory usage increased at ~{slope:.2f} MB per iteration on {device}")
# Assert # Assert
# If slope is positive memory utilization is increasing # If slope is positive memory utilization is increasing
# Positive threshold of 2, from observing memory usage trend on MPS vs CPU device # Positive threshold of 2, from observing memory usage trend on MPS vs CPU device
assert slope < 2, f"Memory usage increasing at ~{slope:.2f} MB per iteration" assert slope < 2, f"Memory leak suspected on {device}. Memory usage increased at ~{slope:.2f} MB per iteration"