From 9d56760d45c34bce892e9c5f534018eaac050cf8 Mon Sep 17 00:00:00 2001 From: Abhijeet Sharma Date: Fri, 14 Aug 2026 21:49:30 +0530 Subject: [PATCH] fix(longmemeval): judge abstention questions with the abstention rubric LongMemEval marks its 30 abstention questions with an `_abs` suffix on the question id and keeps the original `question_type`. Judge prompt selection only looked at the question type, so those questions were graded with the exact-answer rubric against a reference that is an explanation of why the question is unanswerable, which scores a correct abstention as incorrect. Carry an `isAbstention` flag from the benchmark into the judge and use it to pick the rubric, matching the abstention switch in LongMemEval's own evaluator. LoCoMo (`adversarial`) and ConvoMem (`abstention_evidence`) now set the flag as well and keep working through the existing question type check. --- src/benchmarks/convomem/index.ts | 1 + src/benchmarks/locomo/index.ts | 1 + src/benchmarks/longmemeval/index.ts | 12 +++++ src/judges/base.test.ts | 70 +++++++++++++++++++++++++++++ src/judges/base.ts | 20 ++++++++- src/orchestrator/phases/evaluate.ts | 1 + src/prompts/defaults.ts | 9 ++-- src/types/judge.ts | 5 +++ src/types/unified.ts | 7 +++ 9 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 src/judges/base.test.ts diff --git a/src/benchmarks/convomem/index.ts b/src/benchmarks/convomem/index.ts index 9c253fa..bfa9bbb 100644 --- a/src/benchmarks/convomem/index.ts +++ b/src/benchmarks/convomem/index.ts @@ -167,6 +167,7 @@ export class ConvoMemBenchmark implements Benchmark { questionType: category, groundTruth: item.answer, haystackSessionIds: sessionIds, + isAbstention: category === "abstention_evidence", metadata: { evidences: item.message_evidences, }, diff --git a/src/benchmarks/locomo/index.ts b/src/benchmarks/locomo/index.ts index 7b8c86c..2d462ca 100644 --- a/src/benchmarks/locomo/index.ts +++ b/src/benchmarks/locomo/index.ts @@ -132,6 +132,7 @@ export class LoCoMoBenchmark implements Benchmark { questionType, groundTruth: String(qa.answer), haystackSessionIds: sessionIds, + isAbstention: questionType === "adversarial", metadata: { sampleId: item.sample_id, evidence: qa.evidence, diff --git a/src/benchmarks/longmemeval/index.ts b/src/benchmarks/longmemeval/index.ts index 158d3f9..b6376e2 100644 --- a/src/benchmarks/longmemeval/index.ts +++ b/src/benchmarks/longmemeval/index.ts @@ -47,6 +47,17 @@ function parseLongMemEvalDate(dateStr: string): { iso: string; formatted: string } } +/** + * LongMemEval marks abstention questions with an `_abs` suffix on the question id and + * keeps the original `question_type` (e.g. `single-session-user`). For these entries the + * `answer` field is an explanation of why the question is unanswerable, not an answer to + * match against. The official evaluator switches rubric on the same signal, see + * `get_anscheck_prompt(..., abstention=...)` in LongMemEval's `src/evaluation/evaluate_qa.py`. + */ +export function isAbstentionQuestionId(questionId: string): boolean { + return questionId.endsWith("_abs") +} + /** * LongMemEval question types - native string types from the dataset. */ @@ -199,6 +210,7 @@ export class LongMemEvalBenchmark implements Benchmark { questionType: item.question_type, groundTruth: item.answer, haystackSessionIds: sessionIds, + isAbstention: isAbstentionQuestionId(item.question_id), metadata: { questionDate: item.question_date, }, diff --git a/src/judges/base.test.ts b/src/judges/base.test.ts new file mode 100644 index 0000000..dcfe2c9 --- /dev/null +++ b/src/judges/base.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, test } from "bun:test" +import { buildJudgePrompt } from "./base" +import { + ABSTENTION_JUDGE_PROMPT, + DEFAULT_JUDGE_PROMPT, + PREFERENCE_JUDGE_PROMPT, + TEMPORAL_JUDGE_PROMPT, +} from "../prompts/defaults" +import { isAbstentionQuestionId } from "../benchmarks/longmemeval" + +describe("isAbstentionQuestionId", () => { + test("detects the LongMemEval abstention suffix", () => { + expect(isAbstentionQuestionId("0862e8bf_abs")).toBe(true) + expect(isAbstentionQuestionId("gpt4_372c3eed_abs")).toBe(true) + }) + + test("leaves answerable question ids alone", () => { + expect(isAbstentionQuestionId("0862e8bf")).toBe(false) + expect(isAbstentionQuestionId("gpt4_372c3eed")).toBe(false) + }) +}) + +describe("buildJudgePrompt", () => { + const base = { + question: "How many chapters of my hamster care book did I read?", + groundTruth: "You did not mention this information. You mentioned your cat Luna.", + hypothesis: "I don't know.", + } + + test("uses the abstention rubric when the benchmark flags the question", () => { + const prompt = buildJudgePrompt({ + ...base, + questionType: "single-session-user", + isAbstention: true, + }) + + expect(prompt).toContain(ABSTENTION_JUDGE_PROMPT) + expect(prompt).toContain("Explanation: ") + expect(prompt).not.toContain("Ground Truth Answer: ") + }) + + test("keeps exact-answer rubrics for answerable questions of the same type", () => { + const prompt = buildJudgePrompt({ ...base, questionType: "single-session-user" }) + + expect(prompt).toContain(DEFAULT_JUDGE_PROMPT) + expect(prompt).toContain("Ground Truth Answer: ") + }) + + test("still routes abstention by question type for LoCoMo and ConvoMem", () => { + expect(buildJudgePrompt({ ...base, questionType: "adversarial" })).toContain( + ABSTENTION_JUDGE_PROMPT + ) + expect(buildJudgePrompt({ ...base, questionType: "abstention_evidence" })).toContain( + ABSTENTION_JUDGE_PROMPT + ) + }) + + test("does not disturb the other type-specific rubrics", () => { + expect(buildJudgePrompt({ ...base, questionType: "temporal-reasoning" })).toContain( + TEMPORAL_JUDGE_PROMPT + ) + + const preferencePrompt = buildJudgePrompt({ + ...base, + questionType: "single-session-preference", + }) + expect(preferencePrompt).toContain(PREFERENCE_JUDGE_PROMPT) + expect(preferencePrompt).toContain("Rubric: ") + }) +}) diff --git a/src/judges/base.ts b/src/judges/base.ts index bfdddae..fac20d5 100644 --- a/src/judges/base.ts +++ b/src/judges/base.ts @@ -6,6 +6,17 @@ export function getJudgePrompt(questionType: string, providerPrompts?: ProviderP return getJudgePromptForType(questionType) } +/** + * An abstention question is either flagged by the benchmark or identified by its type name. + * The flag is needed because LongMemEval keeps the original question type on its abstention + * entries and only marks them with an `_abs` question id suffix. + */ +function isAbstentionQuestion(input: JudgeInput): boolean { + if (input.isAbstention) return true + const type = input.questionType.toLowerCase() + return type.includes("abstention") || type.includes("adversarial") +} + export function buildJudgePrompt(input: JudgeInput): string { if (input.providerPrompts?.judgePrompt) { const prompts = input.providerPrompts.judgePrompt( @@ -16,9 +27,14 @@ export function buildJudgePrompt(input: JudgeInput): string { return prompts[input.questionType] ?? prompts.default } - const systemPrompt = getJudgePromptForType(input.questionType) + const isAbstention = isAbstentionQuestion(input) + const systemPrompt = getJudgePromptForType(input.questionType, isAbstention) const isPreference = input.questionType.toLowerCase().includes("preference") - const groundTruthLabel = isPreference ? "Rubric" : "Ground Truth Answer" + const groundTruthLabel = isAbstention + ? "Explanation" + : isPreference + ? "Rubric" + : "Ground Truth Answer" return `${systemPrompt} diff --git a/src/orchestrator/phases/evaluate.ts b/src/orchestrator/phases/evaluate.ts index a36205f..1d72dda 100644 --- a/src/orchestrator/phases/evaluate.ts +++ b/src/orchestrator/phases/evaluate.ts @@ -62,6 +62,7 @@ export async function runEvaluatePhase( questionType: question.questionType, groundTruth: question.groundTruth, hypothesis, + isAbstention: question.isAbstention, providerPrompts: provider?.prompts, }), calculateRetrievalMetrics( diff --git a/src/prompts/defaults.ts b/src/prompts/defaults.ts index 7fcc1c9..6c8e220 100644 --- a/src/prompts/defaults.ts +++ b/src/prompts/defaults.ts @@ -32,7 +32,7 @@ Respond with ONLY a JSON object: {"score": 1, "label": "correct", "explanation": "..."} if the response contains the correct answer {"score": 0, "label": "incorrect", "explanation": "..."} if the response does not contain the correct answer` -export const ABSTENTION_JUDGE_PROMPT = `You are evaluating an abstention question. The correct answer is that the information was NOT in the conversation, so the system should abstain or say it doesn't know. +export const ABSTENTION_JUDGE_PROMPT = `You are evaluating an abstention question. The correct answer is that the information was NOT in the conversation, so the system should abstain or say it doesn't know. You are given an explanation of why the question is unanswerable, not an answer to match against. The hypothesis is CORRECT if the system correctly abstains, says "I don't know", indicates uncertainty, or explicitly states the information is not available. It is INCORRECT if the system makes up an answer or hallucinates. @@ -58,10 +58,13 @@ Respond with ONLY a JSON object: {"score": 1, "label": "correct", "explanation": "..."} if the response satisfies the rubric {"score": 0, "label": "incorrect", "explanation": "..."} if the response does not satisfy the rubric` -export function getJudgePromptForType(questionType: string): string { +export function getJudgePromptForType(questionType: string, isAbstention?: boolean): string { const type = questionType.toLowerCase() - if (type.includes("abstention") || type.includes("adversarial")) { + // `isAbstention` comes from the benchmark and wins over the type name, because not every + // benchmark encodes abstention in the question type (LongMemEval keeps the original type + // and marks abstention with an `_abs` question id suffix). + if (isAbstention || type.includes("abstention") || type.includes("adversarial")) { return ABSTENTION_JUDGE_PROMPT } diff --git a/src/types/judge.ts b/src/types/judge.ts index cf8bcb3..f88fc6e 100644 --- a/src/types/judge.ts +++ b/src/types/judge.ts @@ -11,6 +11,11 @@ export interface JudgeInput { questionType: string groundTruth: string hypothesis: string + /** + * True when the question is unanswerable and the system is expected to abstain. + * Set by the benchmark, since not every benchmark encodes abstention in `questionType`. + */ + isAbstention?: boolean context?: string /** Optional provider-specific judge prompts */ providerPrompts?: ProviderPrompts diff --git a/src/types/unified.ts b/src/types/unified.ts index e4a0deb..0b300a8 100644 --- a/src/types/unified.ts +++ b/src/types/unified.ts @@ -25,6 +25,13 @@ export interface UnifiedQuestion { questionType: string groundTruth: string haystackSessionIds: string[] + /** + * True when the question is unanswerable from the conversation and the system is + * expected to abstain. Benchmarks that encode abstention outside of `questionType` + * (LongMemEval marks it with an `_abs` question id suffix) must set this so the + * judge uses the abstention rubric instead of exact-answer matching. + */ + isAbstention?: boolean metadata?: Record }