LLM Calibration 재정의: 단일 응답 정확도에서 모델 능력 추정으로

# Verbalized Confidence 프롬프트 (API-only 모델용)
prompt = """
Question: {question}

How likely are you to answer the question correctly?
You may refer to the following probabilities P:
- 0.0-0.1: "Almost no chance"
- 0.1-0.2: "Highly unlikely"
- 0.2-0.3: "Chances are slight"
- 0.3-0.4: "Unlikely"
- 0.4-0.5: "Less than even"
- 0.5-0.6: "Better than even"
- 0.6-0.7: "Likely"
- 0.7-0.8: "Very good chance"
- 0.8-0.9: "Highly likely"
- 0.9-1.0: "Almost certain"

Reason about your uncertainty and confidence, then provide a probability P between 0.0 and 1.0 in the format of \\boxed{P}.
"""

# Capability-calibrated confidence로 pass@k 시뮬레이션
def simulate_pass_at_k(confidence_scores: list[float], k: int) -> float:
    """capability-calibrated confidence p로 pass@k 추정"""
    pass_at_k_per_instance = [1 - (1 - p) ** k for p in confidence_scores]
    return sum(pass_at_k_per_instance) / len(pass_at_k_per_instance)

# 예시: 10개 질문, 각 confidence, k=5일 때 예상 성공률
confidences = [0.9, 0.3, 0.7, 0.5, 0.8, 0.2, 0.6, 0.4, 0.95, 0.1]
print(f"Estimated pass@5: {simulate_pass_at_k(confidences, k=5):.3f}")

# Inference Budget Greedy 분배 (Damani et al. 2024 방식)
def greedy_budget_allocation(confidences: list[float], total_budget: int) -> list[int]:
    """capability confidence 기반 greedy 예산 분배"""
    import heapq
    n = len(confidences)
    allocations = [1] * n  # 최소 1개씩
    remaining = total_budget - n
    
    # gain = p * (1-p)^k — 현재 할당에서 1개 더 줄 때의 기대 이득
    heap = []
    for i, p in enumerate(confidences):
        gain = p * (1 - p) ** 1  # k=1 기준 초기 gain
        heapq.heappush(heap, (-gain, i))
    
    for _ in range(remaining):
        neg_gain, i = heapq.heappop(heap)
        allocations[i] += 1
        k = allocations[i]
        p = confidences[i]
        new_gain = p * (1 - p) ** k
        heapq.heappush(heap, (-new_gain, i))
    
    return allocations