diff --git a/src/khoj/processor/conversation/utils.py b/src/khoj/processor/conversation/utils.py
index 12fc736f..49c7fb2e 100644
--- a/src/khoj/processor/conversation/utils.py
+++ b/src/khoj/processor/conversation/utils.py
@@ -333,9 +333,10 @@ def construct_structured_message(
         ChatModelOptions.ModelType.GOOGLE,
         ChatModelOptions.ModelType.ANTHROPIC,
     ]:
-        constructed_messages: List[Any] = [
-            {"type": "text", "text": message},
-        ]
+        if not attached_file_context and not (vision_enabled and images):
+            return message
+
+        constructed_messages: List[Any] = [{"type": "text", "text": message}]
 
         if not is_none_or_empty(attached_file_context):
             constructed_messages.append({"type": "text", "text": attached_file_context})