Strategic Navigation or Stochastic Search? How Agents and Humans Reason Over Document Collections

# BM25 MLLM Agent core loop (Python pseudocode based on paper Algorithm 1)

from whoosh import index, qparser
from PIL import Image
import base64

SYSTEM_PROMPT = """
You are a document QA assistant with access to a search tool.
The answer is definitely in the documents.
If search returns no results, try different terms (synonyms, abbreviations, rephrasing).

Once relevant pages are found, provide:
1. answer: List of short answer values (exact document words preferred)
2. citations: List of {file, page} dicts
"""

def bm25_agent(question: str, search_index, vlm_client, max_steps=10, top_k=5):
    messages = [{"role": "system", "content": SYSTEM_PROMPT},
                {"role": "user", "content": question}]
    
    tools = [{
        "name": "search_documents",
        "description": "Search document collection. Supports boolean ops (AND/OR/NOT), phrases in quotes, wildcards (*). Example: '\"annual report\" AND revenue'",
        "parameters": {"query": {"type": "string"}}
    }]
    
    for step in range(max_steps):
        force_answer = (step == max_steps - 1)
        
        response = vlm_client.chat(
            messages=messages,
            tools=None if force_answer else tools,
            # force structured answer output on last step
        )
        
        if response.type == "answer":
            return response.answer, response.citations
        
        elif response.type == "tool_call" and response.tool == "search_documents":
            query = response.args["query"]
            
            # BM25 search then convert to page images
            results = search_index.search(query, limit=top_k)  # returns (file, page) tuples
            page_images = [render_page_as_image(file, page) for file, page in results]
            
            # Add images to messages (VLM analyzes visually)
            tool_result = {"role": "tool", "content": [
                {"type": "image", "data": img_to_base64(img)} for img in page_images
            ]}
            messages.append(tool_result)
    
    return fallback_answer(messages)

# Key point: boldly use different queries when search fails
# Claude Sonnet 4.5's average query drift of 0.38 is the key to performance
# Repeating similar queries like GPT-4.1 Nano (drift 0.10) leads to performance degradation