Clustered Self-Assessment: A Simple yet Effective Method for Uncertainty Quantification in Large Language Models

# Clustered Self-Assessment 핵심 흐름 (pseudo-code)
from transformers import pipeline

# 1. NLI 모델 로드
nli = pipeline("text-classification", model="cross-encoder/nli-deberta-v3-large")

def cluster_answers(answers):
    """NLI로 의미가 같은 답변끼리 묶기"""
    clusters = []
    for ans in answers:
        placed = False
        for cluster in clusters:
            rep = cluster[0]
            # 양방향 entailment 확인
            r1 = nli(f"{rep} [SEP] {ans}")[0]
            r2 = nli(f"{ans} [SEP] {rep}")[0]
            labels = {r1['label'], r2['label']}
            if 'CONTRADICTION' not in labels or 'ENTAILMENT' in labels:
                cluster.append(ans)
                placed = True
                break
        if not placed:
            clusters.append([ans])
    return clusters

def build_mcq(question, clusters):
    """클러스터 대표값으로 객관식 구성"""
    choices = [c[0] for c in clusters]
    labels = [chr(65 + i) for i in range(len(choices))]  # A, B, C...
    last_label = chr(65 + len(choices))
    
    prompt = f"""Task:
Select the one correct answer to the question from the choices provided.
If none of the provided choices is correct, select the final choice ({last_label}) None of the above.

Question:
{question}

Choices:
"""
    for label, choice in zip(labels, choices):
        prompt += f"({label}) {choice}\n"
    prompt += f"({last_label}) None of the above\n\nAnswer:\nThe answer is ("
    return prompt, labels

# 2. 사용 예시
question = "Where is the Eiffel Tower?"
# greedy 답변 + temperature 샘플링 2개
sampled_answers = ["Paris", "It's Paris", "Rome"]

clusters = cluster_answers(sampled_answers)
# clusters = [["Paris", "It's Paris"], ["Rome"]]

mcq_prompt, labels = build_mcq(question, clusters)
print(mcq_prompt)
# -> (A) Paris  (B) Rome  (C) None of the above

# 3. LLM에 MCQ 질의 후 첫 번째 선택지(A = greedy 답변 클러스터)의 토큰 확률을 confidence score로 사용
# confidence_score = logit_prob[token_A]  # 모델의 logit에서 직접 추출