From d7fc4a91d57d94b7518a8c4f1cfb5b98e7bdfce5 Mon Sep 17 00:00:00 2001 From: Debanjum Date: Thu, 7 Nov 2024 15:23:30 -0800 Subject: [PATCH] Only evaluate non-empty responses to reduce eval script latency, cost Empty responses by Khoj will always be an incorrect response, so no need to make call to an evaluator agent to check that --- tests/eval_frames.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/eval_frames.py b/tests/eval_frames.py index 8a1d849e..ae44f63e 100644 --- a/tests/eval_frames.py +++ b/tests/eval_frames.py @@ -113,7 +113,11 @@ def process_batch(batch, counter, results, dataset_length): agent_response = get_agent_response(prompt) # Evaluate response - evaluation = evaluate_response(prompt, agent_response, answer) + if agent_response is None or agent_response.strip() == "": + evaluation["decision"] = False + evaluation["explanation"] = "Agent response is empty. This maybe due to a service error." + else: + evaluation = evaluate_response(prompt, agent_response, answer) # Store results results.append(