fix(longmemeval): judge abstention questions with the abstention rubric

src/benchmarks/convomem/index.ts

@@ -167,6 +167,7 @@ export class ConvoMemBenchmark implements Benchmark {
       questionType: category,
       groundTruth: item.answer,
       haystackSessionIds: sessionIds,
+      isAbstention: category === "abstention_evidence",
       metadata: {
         evidences: item.message_evidences,
       },

src/benchmarks/locomo/index.ts

@@ -132,6 +132,7 @@ export class LoCoMoBenchmark implements Benchmark {
           questionType,
           groundTruth: String(qa.answer),
           haystackSessionIds: sessionIds,
+          isAbstention: questionType === "adversarial",
           metadata: {
             sampleId: item.sample_id,
             evidence: qa.evidence,

src/benchmarks/longmemeval/index.ts

@@ -47,6 +47,17 @@ function parseLongMemEvalDate(dateStr: string): { iso: string; formatted: string
   }
 }
 
+/**
+ * LongMemEval marks abstention questions with an `_abs` suffix on the question id and
+ * keeps the original `question_type` (e.g. `single-session-user`). For these entries the
+ * `answer` field is an explanation of why the question is unanswerable, not an answer to
+ * match against. The official evaluator switches rubric on the same signal, see
+ * `get_anscheck_prompt(..., abstention=...)` in LongMemEval's `src/evaluation/evaluate_qa.py`.
+ */
+export function isAbstentionQuestionId(questionId: string): boolean {
+  return questionId.endsWith("_abs")
+}
+
 /**
  * LongMemEval question types - native string types from the dataset.
  */
@@ -199,6 +210,7 @@ export class LongMemEvalBenchmark implements Benchmark {
         questionType: item.question_type,
         groundTruth: item.answer,
         haystackSessionIds: sessionIds,
+        isAbstention: isAbstentionQuestionId(item.question_id),
         metadata: {
           questionDate: item.question_date,
         },

src/judges/base.test.ts

@@ -0,0 +1,70 @@
+import { describe, expect, test } from "bun:test"
+import { buildJudgePrompt } from "./base"
+import {
+  ABSTENTION_JUDGE_PROMPT,
+  DEFAULT_JUDGE_PROMPT,
+  PREFERENCE_JUDGE_PROMPT,
+  TEMPORAL_JUDGE_PROMPT,
+} from "../prompts/defaults"
+import { isAbstentionQuestionId } from "../benchmarks/longmemeval"
+
+describe("isAbstentionQuestionId", () => {
+  test("detects the LongMemEval abstention suffix", () => {
+    expect(isAbstentionQuestionId("0862e8bf_abs")).toBe(true)
+    expect(isAbstentionQuestionId("gpt4_372c3eed_abs")).toBe(true)
+  })
+
+  test("leaves answerable question ids alone", () => {
+    expect(isAbstentionQuestionId("0862e8bf")).toBe(false)
+    expect(isAbstentionQuestionId("gpt4_372c3eed")).toBe(false)
+  })
+})
+
+describe("buildJudgePrompt", () => {
+  const base = {
+    question: "How many chapters of my hamster care book did I read?",
+    groundTruth: "You did not mention this information. You mentioned your cat Luna.",
+    hypothesis: "I don't know.",
+  }
+
+  test("uses the abstention rubric when the benchmark flags the question", () => {
+    const prompt = buildJudgePrompt({
+      ...base,
+      questionType: "single-session-user",
+      isAbstention: true,
+    })
+
+    expect(prompt).toContain(ABSTENTION_JUDGE_PROMPT)
+    expect(prompt).toContain("Explanation: ")
+    expect(prompt).not.toContain("Ground Truth Answer: ")
+  })
+
+  test("keeps exact-answer rubrics for answerable questions of the same type", () => {
+    const prompt = buildJudgePrompt({ ...base, questionType: "single-session-user" })
+
+    expect(prompt).toContain(DEFAULT_JUDGE_PROMPT)
+    expect(prompt).toContain("Ground Truth Answer: ")
+  })
+
+  test("still routes abstention by question type for LoCoMo and ConvoMem", () => {
+    expect(buildJudgePrompt({ ...base, questionType: "adversarial" })).toContain(
+      ABSTENTION_JUDGE_PROMPT
+    )
+    expect(buildJudgePrompt({ ...base, questionType: "abstention_evidence" })).toContain(
+      ABSTENTION_JUDGE_PROMPT
+    )
+  })
+
+  test("does not disturb the other type-specific rubrics", () => {
+    expect(buildJudgePrompt({ ...base, questionType: "temporal-reasoning" })).toContain(
+      TEMPORAL_JUDGE_PROMPT
+    )
+
+    const preferencePrompt = buildJudgePrompt({
+      ...base,
+      questionType: "single-session-preference",
+    })
+    expect(preferencePrompt).toContain(PREFERENCE_JUDGE_PROMPT)
+    expect(preferencePrompt).toContain("Rubric: ")
+  })
+})

src/judges/base.ts

@@ -6,6 +6,17 @@ export function getJudgePrompt(questionType: string, providerPrompts?: ProviderP
   return getJudgePromptForType(questionType)
 }
 
+/**
+ * An abstention question is either flagged by the benchmark or identified by its type name.
+ * The flag is needed because LongMemEval keeps the original question type on its abstention
+ * entries and only marks them with an `_abs` question id suffix.
+ */
+function isAbstentionQuestion(input: JudgeInput): boolean {
+  if (input.isAbstention) return true
+  const type = input.questionType.toLowerCase()
+  return type.includes("abstention") || type.includes("adversarial")
+}
+
 export function buildJudgePrompt(input: JudgeInput): string {
   if (input.providerPrompts?.judgePrompt) {
     const prompts = input.providerPrompts.judgePrompt(
@@ -16,9 +27,14 @@ export function buildJudgePrompt(input: JudgeInput): string {
     return prompts[input.questionType] ?? prompts.default
   }
 
-  const systemPrompt = getJudgePromptForType(input.questionType)
+  const isAbstention = isAbstentionQuestion(input)
+  const systemPrompt = getJudgePromptForType(input.questionType, isAbstention)
   const isPreference = input.questionType.toLowerCase().includes("preference")
-  const groundTruthLabel = isPreference ? "Rubric" : "Ground Truth Answer"
+  const groundTruthLabel = isAbstention
+    ? "Explanation"
+    : isPreference
+      ? "Rubric"
+      : "Ground Truth Answer"
 
   return `${systemPrompt}
 

src/orchestrator/phases/evaluate.ts

@@ -62,6 +62,7 @@ export async function runEvaluatePhase(
             questionType: question.questionType,
             groundTruth: question.groundTruth,
             hypothesis,
+            isAbstention: question.isAbstention,
             providerPrompts: provider?.prompts,
           }),
           calculateRetrievalMetrics(

src/prompts/defaults.ts

@@ -32,7 +32,7 @@ Respond with ONLY a JSON object:
 {"score": 1, "label": "correct", "explanation": "..."} if the response contains the correct answer
 {"score": 0, "label": "incorrect", "explanation": "..."} if the response does not contain the correct answer`
 
-export const ABSTENTION_JUDGE_PROMPT = `You are evaluating an abstention question. The correct answer is that the information was NOT in the conversation, so the system should abstain or say it doesn't know.
+export const ABSTENTION_JUDGE_PROMPT = `You are evaluating an abstention question. The correct answer is that the information was NOT in the conversation, so the system should abstain or say it doesn't know. You are given an explanation of why the question is unanswerable, not an answer to match against.
 
 The hypothesis is CORRECT if the system correctly abstains, says "I don't know", indicates uncertainty, or explicitly states the information is not available. It is INCORRECT if the system makes up an answer or hallucinates.
 
@@ -58,10 +58,13 @@ Respond with ONLY a JSON object:
 {"score": 1, "label": "correct", "explanation": "..."} if the response satisfies the rubric
 {"score": 0, "label": "incorrect", "explanation": "..."} if the response does not satisfy the rubric`
 
-export function getJudgePromptForType(questionType: string): string {
+export function getJudgePromptForType(questionType: string, isAbstention?: boolean): string {
   const type = questionType.toLowerCase()
 
-  if (type.includes("abstention") || type.includes("adversarial")) {
+  // `isAbstention` comes from the benchmark and wins over the type name, because not every
+  // benchmark encodes abstention in the question type (LongMemEval keeps the original type
+  // and marks abstention with an `_abs` question id suffix).
+  if (isAbstention || type.includes("abstention") || type.includes("adversarial")) {
     return ABSTENTION_JUDGE_PROMPT
   }
 

src/types/judge.ts

@@ -11,6 +11,11 @@ export interface JudgeInput {
   questionType: string
   groundTruth: string
   hypothesis: string
+  /**
+   * True when the question is unanswerable and the system is expected to abstain.
+   * Set by the benchmark, since not every benchmark encodes abstention in `questionType`.
+   */
+  isAbstention?: boolean
   context?: string
   /** Optional provider-specific judge prompts */
   providerPrompts?: ProviderPrompts

src/types/unified.ts

@@ -25,6 +25,13 @@ export interface UnifiedQuestion {
   questionType: string
   groundTruth: string
   haystackSessionIds: string[]
+  /**
+   * True when the question is unanswerable from the conversation and the system is
+   * expected to abstain. Benchmarks that encode abstention outside of `questionType`
+   * (LongMemEval marks it with an `_abs` question id suffix) must set this so the
+   * judge uses the abstention rubric instead of exact-answer matching.
+   */
+  isAbstention?: boolean
   metadata?: Record<string, unknown>
 }