Structured Distillation for Personalized Agent Memory: 11x Token Reduction with Retrieval Preservation

# Distillation prompt (based on Appendix B, using Claude Haiku 4.5)
prompt = """
Distill this conversation exchange into JSON:
- "exchange_core": 1-2 sentences. What was accomplished or decided?
  Use the specific terms from the exchange. Do not invent details
  not present in the text.
- "specific_context": One concrete detail from the text: a number,
  error message, parameter name, or file path. Copy it exactly.
- "room_assignments": 1-3 rooms. Each room is a topic this exchange
  belongs to. {"room_type": "<file|concept|workflow>",
  "room_key": "<identifier>", "room_label": "<short label>",
  "relevance": <0.0-1.0>}

Project: {project_id}
Exchange (messages {ply_start}-{ply_end}):
{messages_text}

Respond with ONLY valid JSON.
"""

# files_touched is NOT LLM-generated — extracted via regex
import re
def extract_files_touched(exchange_text):
    pattern = r'[\w./\-]+\.(?:py|ts|js|go|rs|yaml|json|toml|md)'
    return list(set(re.findall(pattern, exchange_text)))

# Embedding + indexing
from sentence_transformers import SentenceTransformer
import faiss

model = SentenceTransformer('all-MiniLM-L6-v2')  # 22M params, CPU OK

def build_distill_index(palace_objects):
    texts = [f"{obj['exchange_core']}\n{obj['specific_context']}" 
             for obj in palace_objects]
    embeddings = model.encode(texts, show_progress_bar=True)
    
    index = faiss.IndexFlatL2(384)  # Exact search
    index.add(embeddings)
    return index, texts

# Cross-layer search: BM25 on verbatim + HNSW on distilled
from rank_bm25 import BM25Okapi

def cross_layer_search(query, verbatim_texts, distilled_texts, 
                        distill_index, top_k=10):
    # BM25 on verbatim
    tokenized = [t.split() for t in verbatim_texts]
    bm25 = BM25Okapi(tokenized)
    bm25_scores = bm25.get_scores(query.split())
    
    # Vector search on distilled
    q_emb = model.encode([query])
    _, vec_indices = distill_index.search(q_emb, top_k)
    
    # RRF fusion
    bm25_ranks = {i: r+1 for r, i in enumerate(bm25_scores.argsort()[::-1][:top_k])}
    vec_ranks = {i: r+1 for r, i in enumerate(vec_indices[0])}
    
    all_ids = set(bm25_ranks) | set(vec_ranks)
    rrf_scores = {i: 1/(60 + bm25_ranks.get(i, top_k+1)) + 
                     1/(60 + vec_ranks.get(i, top_k+1)) 
                  for i in all_ids}
    
    return sorted(rrf_scores, key=rrf_scores.get, reverse=True)[:top_k]