From 5b4fd35705cd4d698ebeaa3894fee96d15eef981 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Tue, 28 Jul 2026 17:55:39 +0900 Subject: [PATCH 01/14] chore: upgrade cloakbrowser within cooldown --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 9591f1d0..72f57319 100644 --- a/uv.lock +++ b/uv.lock @@ -438,16 +438,16 @@ wheels = [ [[package]] name = "cloakbrowser" -version = "0.4.7" +version = "0.4.12" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, { name = "httpx" }, { name = "playwright" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/36/8e/42edfe6b2ef5429c7ee4c86747ccd7a66c453179b78d41fa43778ef40112/cloakbrowser-0.4.7.tar.gz", hash = "sha256:b7efdb2199f1612d9561af15739c3e90198131e30b319d73cab55c819ef0cddb", size = 5569399, upload-time = "2026-07-02T22:24:51.447Z" } +sdist = { url = "https://files.pythonhosted.org/packages/32/0f/085ac9e2ac78b6800d55a59cefc077489a3ec9dd43e31362e9bc06734cfa/cloakbrowser-0.4.12.tar.gz", hash = "sha256:683d150a53cc768f6db6e3e758268d1b2e35144e0b4847336bd8bc71fdcb8baf", size = 5609654, upload-time = "2026-07-18T21:34:24.567Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1f/62/13cf0715f9f1bd1784d0527f6064d2deeeb91406af9187c0adafac2c7ae9/cloakbrowser-0.4.7-py3-none-any.whl", hash = "sha256:4e9019795133e3ec425602813b27140c164da7075be5f064a71c44bfd4eb32bd", size = 98055, upload-time = "2026-07-02T22:24:49.808Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e2/ec81d25a853d9fc0af91827b1ed5b97d0edcd34f3c7c2d1741915994dd53/cloakbrowser-0.4.12-py3-none-any.whl", hash = "sha256:0415acff4aa5f49c18bc9cbd6a65ae806591dfd71ddf5d862238c61cd8471142", size = 107156, upload-time = "2026-07-18T21:34:22.814Z" }, ] [[package]] From 68e252de35a8b5e3164e4e670457ba165a2b25ff Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Tue, 4 Aug 2026 16:24:21 +0900 Subject: [PATCH 02/14] ci: scope live agent evals to agent changes --- .github/workflows/test.yml | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 25030852..6cf44269 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -141,20 +141,27 @@ jobs: run: | if [[ -z "${BASE_SHA:-}" || "$BASE_SHA" =~ ^0+$ ]]; then echo "run=true" >> "$GITHUB_OUTPUT" + echo "run_live=true" >> "$GITHUB_OUTPUT" exit 0 fi changed_files="$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")" echo "$changed_files" + run=false + run_live=false while IFS= read -r file; do case "$file" in - .github/workflows/test.yml|scripts/agent_eval.py|packages/shared/src/five08/agent/*|packages/shared/src/five08/model_catalog.py|packages/shared/src/five08/data/model-profiles.json|apps/discord_bot/src/five08/discord_bot/cogs/agent.py|tests/evals/discord-agent/*|tests/unit/test_agent_cog.py|tests/unit/test_agent_evals.py|tests/unit/test_agent_gateway.py|pyproject.toml|uv.lock) - echo "run=true" >> "$GITHUB_OUTPUT" - exit 0 + .github/workflows/test.yml|scripts/agent_eval.py|tests/unit/test_agent_cog.py|tests/unit/test_agent_evals.py|tests/unit/test_agent_gateway.py|pyproject.toml|uv.lock) + run=true + ;; + packages/shared/src/five08/agent/*|packages/shared/src/five08/model_catalog.py|packages/shared/src/five08/data/model-profiles.json|apps/discord_bot/src/five08/discord_bot/cogs/agent.py|tests/evals/discord-agent/*) + run=true + run_live=true ;; esac done <<< "$changed_files" - echo "run=false" >> "$GITHUB_OUTPUT" + echo "run=$run" >> "$GITHUB_OUTPUT" + echo "run_live=$run_live" >> "$GITHUB_OUTPUT" - name: Skip Discord agent evals for unrelated changes if: steps.agent-eval-changes.outputs.run != 'true' @@ -182,7 +189,7 @@ jobs: uv run python scripts/agent_eval.py --suite canonical --model primary --no-env-file - name: Check live planner credentials - if: steps.agent-eval-changes.outputs.run == 'true' + if: steps.agent-eval-changes.outputs.run_live == 'true' id: live-planner-key env: LIVE_EVAL_DIRECT_KEY: ${{ secrets.OPENAI_API_KEY_DIRECT }} @@ -202,7 +209,7 @@ jobs: fi - name: Run canonical Discord agent live-planner evals - if: steps.agent-eval-changes.outputs.run == 'true' && steps.live-planner-key.outputs.available == 'true' + if: steps.agent-eval-changes.outputs.run_live == 'true' && steps.live-planner-key.outputs.available == 'true' env: LIVE_EVAL_DIRECT_KEY: ${{ secrets.OPENAI_API_KEY_DIRECT }} LIVE_EVAL_PROVIDER_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -216,10 +223,15 @@ jobs: fi - name: Skip live planner evals without provider credentials - if: steps.agent-eval-changes.outputs.run == 'true' && steps.live-planner-key.outputs.available != 'true' + if: steps.agent-eval-changes.outputs.run_live == 'true' && steps.live-planner-key.outputs.available != 'true' run: | echo "Skipping live planner evals because the matching OPENAI_API_KEY_DIRECT/OPENAI_API_KEY credential is not configured." + - name: Skip live planner evals for non-agent changes + if: steps.agent-eval-changes.outputs.run == 'true' && steps.agent-eval-changes.outputs.run_live != 'true' + run: | + echo "No agent behavior or fixture changes detected; skipping live planner evals." + - name: Upload Discord agent eval reports uses: actions/upload-artifact@v7 if: always() && steps.agent-eval-changes.outputs.run == 'true' From 69af3cecbc668c76d8bcd1b76c760d3bb4c10da1 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Tue, 4 Aug 2026 16:37:53 +0900 Subject: [PATCH 03/14] fix: run live evals for harness changes --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6cf44269..baf43aca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -151,12 +151,12 @@ jobs: run_live=false while IFS= read -r file; do case "$file" in - .github/workflows/test.yml|scripts/agent_eval.py|tests/unit/test_agent_cog.py|tests/unit/test_agent_evals.py|tests/unit/test_agent_gateway.py|pyproject.toml|uv.lock) + .github/workflows/test.yml|scripts/agent_eval.py|packages/shared/src/five08/agent/*|packages/shared/src/five08/model_catalog.py|packages/shared/src/five08/data/model-profiles.json|apps/discord_bot/src/five08/discord_bot/cogs/agent.py|tests/evals/discord-agent/*) run=true + run_live=true ;; - packages/shared/src/five08/agent/*|packages/shared/src/five08/model_catalog.py|packages/shared/src/five08/data/model-profiles.json|apps/discord_bot/src/five08/discord_bot/cogs/agent.py|tests/evals/discord-agent/*) + tests/unit/test_agent_cog.py|tests/unit/test_agent_evals.py|tests/unit/test_agent_gateway.py|pyproject.toml|uv.lock) run=true - run_live=true ;; esac done <<< "$changed_files" From 6ecb24d8e26102c4bf434fd68ba358bffa721f7d Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Tue, 4 Aug 2026 16:50:41 +0900 Subject: [PATCH 04/14] fix: prefer deterministic agent routes --- .../shared/src/five08/agent/orchestrator.py | 31 +++++++++++++++++-- tests/unit/test_agent_gateway.py | 26 ++++++++++++++-- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/packages/shared/src/five08/agent/orchestrator.py b/packages/shared/src/five08/agent/orchestrator.py index a6d4d771..8b263f86 100644 --- a/packages/shared/src/five08/agent/orchestrator.py +++ b/packages/shared/src/five08/agent/orchestrator.py @@ -128,13 +128,21 @@ def plan(self, message: str, context: AgentIdentityContext) -> AgentResponse: if resolved_member_agreement is not None: return resolved_member_agreement + planner: LiteralPlanner = "deterministic_regex" + planning_text = text + action = self._parse_action(text) + if action is not None: + return self._response_for_deterministic_action( + action=action, + context=context, + planning_text=planning_text, + planner=planner, + ) + planned_response = self._plan_with_model(text, context) if planned_response is not None: return planned_response - planner: LiteralPlanner = "deterministic_regex" - planning_text = text - action = self._parse_action(text) if action is None and not re.search( r"\bcreate\s+(?:a\s+)?task\b", text, re.IGNORECASE ): @@ -159,6 +167,23 @@ def plan(self, message: str, context: AgentIdentityContext) -> AgentResponse: "Try asking me to manage a task, GitHub issue, CRM contact, or member account." ), ) + return self._response_for_deterministic_action( + action=action, + context=context, + planning_text=planning_text, + planner=planner, + ) + + def _response_for_deterministic_action( + self, + *, + action: AgentToolAction, + context: AgentIdentityContext, + planning_text: str, + planner: LiteralPlanner, + ) -> AgentResponse: + """Plan a known workflow before asking a model to infer an intent.""" + if action.tool_name == "task_read.search_tasks" and not action.arguments.get( "project" ): diff --git a/tests/unit/test_agent_gateway.py b/tests/unit/test_agent_gateway.py index 0111cd7a..7000aa31 100644 --- a/tests/unit/test_agent_gateway.py +++ b/tests/unit/test_agent_gateway.py @@ -1779,10 +1779,30 @@ def test_task_creation_is_not_rerouted_to_member_agreement_submission() -> None: assert response.plan.actions[0].tool_name == "task_write.create_task" +def test_explicit_task_creation_does_not_depend_on_model_routing() -> None: + class FailingPlanner: + def plan(self, **_kwargs: object) -> AgentPlannerResult: + raise AssertionError("planner should not run for a known workflow") + + response = AgentOrchestrator(planner=FailingPlanner()).plan( + "Create a task to follow up on GitHub issue 123 in project Atlas", + _context(), + ) + + assert response.status == "requires_confirmation" + assert response.plan is not None + assert response.plan.planner == "deterministic_regex" + assert response.plan.actions[0].tool_name == "task_write.create_task" + assert response.plan.actions[0].arguments == { + "title": "follow up on GitHub issue 123", + "project": "Atlas", + } + + def test_agent_uses_structured_planner_for_multi_action_confirmation() -> None: class FakePlanner: def plan(self, **kwargs: object) -> AgentPlannerResult: - assert kwargs["model_tier"] == "strong" + assert kwargs["model_tier"] == "fast" return AgentPlannerResult( draft=PlannerDraft( status="planned", @@ -1812,7 +1832,9 @@ def plan(self, **kwargs: object) -> AgentPlannerResult: orchestrator = AgentOrchestrator(planner=FakePlanner()) - response = orchestrator.plan("Invite Sarah to Outline", _context(roles=["Admin"])) + response = orchestrator.plan( + "Help Sarah get started at 508", _context(roles=["Admin"]) + ) assert response.status == "requires_confirmation" assert response.plan is not None From 75f0307dab6b98544d1e9beb85e3cb406423e669 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Tue, 4 Aug 2026 16:56:26 +0900 Subject: [PATCH 05/14] fix: align live eval routing with production --- packages/shared/src/five08/agent/evals.py | 11 ++++ tests/evals/discord-agent/README.md | 3 + tests/unit/test_agent_evals.py | 70 +++++++++++++++++++++-- 3 files changed, 80 insertions(+), 4 deletions(-) diff --git a/packages/shared/src/five08/agent/evals.py b/packages/shared/src/five08/agent/evals.py index 47d65cb1..b07cf229 100644 --- a/packages/shared/src/five08/agent/evals.py +++ b/packages/shared/src/five08/agent/evals.py @@ -1008,6 +1008,17 @@ def _response_from_live_draft( ) if resolved_member_agreement is not None: return resolved_member_agreement + # The live call is retained as a planner-quality probe and its raw response + # is reported, but executable behavior must use the same deterministic + # routing boundary as production. + deterministic_action = orchestrator._parse_action(message) + if deterministic_action is not None: + return orchestrator._response_for_deterministic_action( + action=deterministic_action, + context=context, + planning_text=message, + planner="deterministic_regex", + ) if draft.status == "needs_clarification" or not draft.actions: fallback_action = _deterministic_live_planner_fallback( orchestrator=orchestrator, diff --git a/tests/evals/discord-agent/README.md b/tests/evals/discord-agent/README.md index b97dc4ca..58691bbc 100644 --- a/tests/evals/discord-agent/README.md +++ b/tests/evals/discord-agent/README.md @@ -62,6 +62,9 @@ With `--live-planner`, the configured provider returns a structured tool-call draft. The harness still does not let the model authorize users or perform side effects: deterministic policy checks scopes, write actions stop at confirmation, and read actions use fixture stubs when provided. +For a request that production can route deterministically, the harness evaluates +that same production route while retaining the provider draft in +`raw_model_output` for inspection. Without `--live-planner`, the runner uses the deterministic parser path. That is useful for local no-key debugging, but it is not the main PR gate. diff --git a/tests/unit/test_agent_evals.py b/tests/unit/test_agent_evals.py index de3b1abe..7887ecb8 100644 --- a/tests/unit/test_agent_evals.py +++ b/tests/unit/test_agent_evals.py @@ -12,6 +12,7 @@ AgentEvalObservedAction, evaluate_observed, load_env_file, + load_fixtures, list_eval_model_profiles, resolve_eval_model_profile, run_live_planner_eval_suite, @@ -254,6 +255,59 @@ def fake_post(*args: object, **kwargs: object) -> FakeResponse: } +def test_live_planner_eval_preserves_raw_misroute_but_uses_production_route( + monkeypatch, +) -> None: + raw_output = json.dumps( + { + "status": "planned", + "intent": "create_task", + "clarification_question": None, + "actions": [ + { + "tool_name": "github_issue.get_issue", + "arguments": { + "repository": "508-dev/todos", + "issue_number": 123, + }, + "summary": "Retrieve GitHub issue 123", + } + ], + } + ) + + class FakeResponse: + status_code = 200 + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, object]: + return {"choices": [{"message": {"content": raw_output}}]} + + monkeypatch.setenv("OPENAI_API_KEY_DIRECT", "direct-key") + monkeypatch.setattr( + "five08.agent.evals.requests.post", + lambda *_args, **_kwargs: FakeResponse(), + ) + + report = run_live_planner_eval_suite( + suite="canonical", + model="openai-direct", + ids=["task_create_mentions_github_issue_001"], + timeout_seconds=1, + ) + + assert report.summary["passed"] == 1 + scenario = report.scenarios[0] + assert scenario.observed.raw_model_output == raw_output + assert scenario.observed.actions[0].tool_name == "task_write.create_task" + assert scenario.observed.actions[0].arguments == { + "title": "follow up on GitHub issue 123", + "project": "Atlas", + } + + def test_live_planner_eval_retries_one_bad_plan(monkeypatch) -> None: class FakeResponse: status_code = 200 @@ -280,8 +334,8 @@ def json(self) -> dict[str, object]: '{"status":"planned","intent":"search_crm_contacts",' '"clarification_question":null,' '"actions":[{"tool_name":"crm_read.search_contacts",' - '"arguments":{"query":"Sarah","limit":5},' - '"summary":"Search CRM contacts matching Sarah"}]}', + '"arguments":{"query":"Caleb","limit":5},' + '"summary":"Search CRM contacts matching Caleb"}]}', ] calls: list[str] = [] @@ -291,11 +345,15 @@ def fake_post(*args: object, **kwargs: object) -> FakeResponse: monkeypatch.setenv("OPENAI_API_KEY_DIRECT", "direct-key") monkeypatch.setattr("five08.agent.evals.requests.post", fake_post) + fixture = load_fixtures(suite="canonical", ids=["crm_contact_info_lookup_001"])[ + 0 + ].model_copy(update={"known_failure": None}) + monkeypatch.setattr("five08.agent.evals.load_fixtures", lambda **_kwargs: [fixture]) report = run_live_planner_eval_suite( suite="canonical", model="openai-direct", - ids=["crm_contact_search_001"], + ids=["crm_contact_info_lookup_001"], timeout_seconds=1, ) @@ -336,11 +394,15 @@ def json(self) -> dict[str, object]: "five08.agent.evals.requests.post", lambda *_args, **_kwargs: FakeResponse(), ) + fixture = load_fixtures(suite="canonical", ids=["crm_contact_info_lookup_001"])[ + 0 + ].model_copy(update={"known_failure": None}) + monkeypatch.setattr("five08.agent.evals.load_fixtures", lambda **_kwargs: [fixture]) report = run_live_planner_eval_suite( suite="canonical", model="openai-direct", - ids=["crm_contact_update_confirmation_001"], + ids=["crm_contact_info_lookup_001"], timeout_seconds=1, ) From eb14e01dfd7fbf3e67f5dcc51c4acccf96e8120b Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Tue, 4 Aug 2026 17:13:27 +0900 Subject: [PATCH 06/14] fix: report live planner probe mismatches --- packages/shared/src/five08/agent/evals.py | 366 ++++++++++-------- .../shared/src/five08/agent/orchestrator.py | 62 ++- tests/evals/discord-agent/README.md | 7 +- tests/unit/test_agent_evals.py | 102 ++++- 4 files changed, 360 insertions(+), 177 deletions(-) diff --git a/packages/shared/src/five08/agent/evals.py b/packages/shared/src/five08/agent/evals.py index b07cf229..98d10f2b 100644 --- a/packages/shared/src/five08/agent/evals.py +++ b/packages/shared/src/five08/agent/evals.py @@ -233,6 +233,13 @@ class AgentEvalCheck(BaseModel): observed: Any = None +class AgentEvalProviderDraftProbe(BaseModel): + """Semantic quality of a live provider draft, separate from execution.""" + + status: Literal["passed", "failed", "parse_failed"] + checks: list[AgentEvalCheck] = Field(default_factory=list) + + class AgentEvalScenarioResult(BaseModel): """One fixture's observed output and check list.""" @@ -242,6 +249,7 @@ class AgentEvalScenarioResult(BaseModel): known_failure: AgentEvalKnownFailure | None = None checks: list[AgentEvalCheck] observed: AgentEvalObserved + provider_draft: AgentEvalProviderDraftProbe | None = None class AgentEvalReport(BaseModel): @@ -487,6 +495,18 @@ def run_live_planner_eval_suite( parse_successes = sum( 1 for result in scenario_results if result.observed.parse_success is True ) + provider_draft_failures = sum( + 1 + for result in scenario_results + if result.provider_draft is not None + and result.provider_draft.status != "passed" + ) + provider_draft_parse_failures = sum( + 1 + for result in scenario_results + if result.provider_draft is not None + and result.provider_draft.status == "parse_failed" + ) latencies = _scenario_latencies(scenario_results) return AgentEvalReport( mode="live_planner", @@ -505,8 +525,11 @@ def run_live_planner_eval_suite( "parse_successes": parse_successes, "parse_failures": len(scenario_results) - parse_successes, "parse_success_rate": _rate(parse_successes, len(scenario_results)), - "bad_plans": failed, - "bad_plan_rate": _rate(failed, len(scenario_results)), + "provider_draft_failures": provider_draft_failures, + "provider_draft_parse_failures": provider_draft_parse_failures, + "bad_plans": provider_draft_failures, + "bad_plan_rate": _rate(provider_draft_failures, len(scenario_results)), + "production_failures": failed, "avg_latency_ms": _average_int(latencies), "max_latency_ms": max(latencies) if latencies else None, "estimated_cost_usd": _sum_estimated_costs(scenario_results), @@ -611,20 +634,32 @@ def run_fixture_with_live_planner( model_config=profile.agent_model_config, ) message = fixture.request.current_message() + context = orchestrator._load_request_context(context) + deterministic_response = orchestrator._plan_deterministic_workflow( + message.strip(), context + ) + + # Probe the provider even for deterministic routes so its raw semantic + # quality remains observable, but always execute the production routing + # boundary before and independently of that call. call = _call_live_planner( profile=profile, fixture=fixture, message=message, timeout_seconds=timeout_seconds, ) - response = _response_from_live_draft( - orchestrator=orchestrator, - draft=call.draft, - message=message, - context=context, - profile=profile, - parse_error=call.error, - ) + provider_draft = _evaluate_provider_draft(fixture.expect, call) + if deterministic_response is not None: + response = deterministic_response + else: + response = _response_from_live_draft( + orchestrator=orchestrator, + draft=call.draft, + message=message, + context=context, + profile=profile, + parse_error=call.error, + ) observed = observe_response( fixture=fixture, message=message, @@ -637,7 +672,7 @@ def run_fixture_with_live_planner( estimated_cost_usd=call.estimated_cost_usd, ) checks = evaluate_observed(fixture.expect, observed, strict=False) - if not call.parse_success: + if deterministic_response is None and not call.parse_success: checks.append( AgentEvalCheck( name="live_planner.parse_success", @@ -661,6 +696,7 @@ def run_fixture_with_live_planner( known_failure=fixture.known_failure, checks=checks, observed=observed, + provider_draft=provider_draft, ) @@ -806,6 +842,7 @@ def _call_live_planner( timeout_seconds: float, ) -> LivePlannerCallResult: started = time.perf_counter() + raw_output: str | None = None try: if profile.live_provider == "anthropic": raw_output, token_usage, estimated_cost_usd = _call_anthropic_live_planner( @@ -836,7 +873,7 @@ def _call_live_planner( ) except Exception as exc: return LivePlannerCallResult( - raw_output=None, + raw_output=raw_output, latency_ms=_elapsed_ms(started), parse_success=False, error=str(exc), @@ -987,6 +1024,107 @@ def _parse_live_planner_json(raw_output: str) -> LivePlannerDraft: return LivePlannerDraft.model_validate(normalized) +def _evaluate_provider_draft( + expect: AgentEvalExpect, + call: LivePlannerCallResult, +) -> AgentEvalProviderDraftProbe: + """Evaluate raw provider semantics without conflating them with execution.""" + + checks = [ + _check( + "provider_draft.parse_success", + True, + call.parse_success, + ) + ] + if not call.parse_success or call.draft is None: + return AgentEvalProviderDraftProbe(status="parse_failed", checks=checks) + + draft = call.draft + expected_status = ( + "needs_clarification" if expect.status == "needs_clarification" else "planned" + ) + checks.append( + _check( + "provider_draft.status", + expected_status, + draft.status, + ) + ) + checks.append( + _check_optional( + "provider_draft.intent", + expect.intent, + draft.intent, + strict=False, + ) + ) + checks.append( + _check_optional( + "provider_draft.clarification_question", + expect.clarification_question, + draft.clarification_question, + strict=False, + ) + ) + if expect.actions is not None: + checks.append( + _check( + "provider_draft.action_count", + len(expect.actions), + len(draft.actions), + strict=False, + ) + ) + for index, expected_action in enumerate(expect.actions): + observed_action = ( + draft.actions[index] if index < len(draft.actions) else None + ) + prefix = f"provider_draft.actions[{index}]" + if observed_action is None: + checks.append( + AgentEvalCheck( + name=f"{prefix}.present", + passed=False, + expected=expected_action.model_dump(mode="json"), + observed=None, + ) + ) + continue + checks.append( + _check( + f"{prefix}.tool_name", + expected_action.tool_name, + observed_action.tool_name, + strict=False, + ) + ) + if expected_action.arguments is not None: + checks.append( + _check( + f"{prefix}.arguments", + expected_action.arguments, + observed_action.arguments, + strict=False, + ) + ) + if expected_action.arguments_contains is not None: + for key, value in expected_action.arguments_contains.items(): + checks.append( + _check( + f"{prefix}.arguments.{key}", + value, + observed_action.arguments.get(key), + strict=False, + ) + ) + checks = [check for check in checks if check.name] + return AgentEvalProviderDraftProbe( + status="passed" if all(check.passed for check in checks) else "failed", + checks=checks, + ) + + def _response_from_live_draft( *, orchestrator: AgentOrchestrator, @@ -1001,38 +1139,13 @@ def _response_from_live_draft( status="failed", message=f"Live planner failed: {parse_error or 'unknown error'}", ) - resolved_member_agreement = orchestrator._plan_member_agreement_from_crm( - message, - context, - planner="live_model", - ) - if resolved_member_agreement is not None: - return resolved_member_agreement - # The live call is retained as a planner-quality probe and its raw response - # is reported, but executable behavior must use the same deterministic - # routing boundary as production. - deterministic_action = orchestrator._parse_action(message) - if deterministic_action is not None: - return orchestrator._response_for_deterministic_action( - action=deterministic_action, - context=context, - planning_text=message, - planner="deterministic_regex", - ) if draft.status == "needs_clarification" or not draft.actions: - fallback_action = _deterministic_live_planner_fallback( - orchestrator=orchestrator, - message=message, + question = draft.clarification_question or "What should I do next?" + return AgentResponse( + status="needs_clarification", + message=question, + clarification_question=question, ) - if fallback_action is not None: - actions = [fallback_action] - else: - question = draft.clarification_question or "What should I do next?" - return AgentResponse( - status="needs_clarification", - message=question, - clarification_question=question, - ) else: actions = [ AgentToolAction( @@ -1139,114 +1252,6 @@ def _response_from_live_draft( ) -def _deterministic_live_planner_fallback( - *, - orchestrator: AgentOrchestrator, - message: str, -) -> AgentToolAction | None: - action = orchestrator._parse_action(message) - if action is None: - return None - if action.tool_name != "task_read.search_tasks": - return None - if orchestrator.registry.get(action.tool_name) is None: - return None - if _live_action_clarification(orchestrator, action) is not None: - return None - action.summary = action.summary or f"Call {action.tool_name}" - return action - - -def _live_action_clarification( - orchestrator: AgentOrchestrator, - action: AgentToolAction, -) -> str | None: - """Return a clarification question for malformed live-drafted actions.""" - args = action.arguments - tool_name = action.tool_name - if tool_name == "task_read.search_tasks": - if not _non_empty_arg(args, "project"): - return "Which project should I search?" - return None - if tool_name == "task_write.create_task": - if not _non_empty_arg(args, "title"): - return "What should the task be?" - return None - if tool_name == "task_write.update_task": - if not _non_empty_arg(args, "task_id"): - return "Which task should I update?" - if not any( - _non_empty_arg(args, key) - for key in ("title", "project", "assignee", "due_date", "status") - ): - return "What should I change on that task?" - return None - if tool_name == "github_issue.search_issues": - if not _non_empty_arg(args, "query"): - return "What GitHub issues should I search for?" - if not _non_empty_arg(args, "repository") and not _non_empty_text( - orchestrator.registry.runtime_config.github_default_repo - ): - return "Which GitHub repository should I search?" - return None - if tool_name == "github_issue.create_issue": - if not _non_empty_arg(args, "title"): - return "What should be the title of the GitHub issue?" - if not _non_empty_arg(args, "repository") and not _non_empty_text( - orchestrator.registry.runtime_config.github_default_repo - ): - return "Which GitHub repository should I create the issue in?" - return None - if tool_name == "crm_read.search_contacts": - if not _non_empty_arg(args, "query"): - return "Who should I look up?" - return None - if tool_name == "crm_write.update_contact": - if not _non_empty_arg(args, "contact_id"): - return "Which CRM contact should I update?" - updates = args.get("updates") - if not isinstance(updates, dict) or not updates: - return "What should I update on that CRM contact?" - return None - if tool_name == "docuseal_write.create_member_agreement_submission": - if not _non_empty_arg(args, "submitter_email"): - return "What email address should I use for the member agreement?" - return None - if tool_name == "mail_write.create_mailbox": - if not _non_empty_arg(args, "local_part"): - return "What mailbox should I create?" - if not _non_empty_arg(args, "backup_email"): - return "What backup email should I use?" - if not _non_empty_arg(args, "name"): - return "What display name should I use?" - if tool_name == "sso_write.create_user": - if not _has_contact_reference(args): - return "Which CRM contact should I create the SSO user for?" - return None - if tool_name == "outline_write.invite_user": - if not _non_empty_arg(args, "email") and not _has_contact_reference(args): - return "Who should I invite to Outline?" - return None - if tool_name == "account_write.create_user_accounts": - if not _has_contact_reference(args): - return "Which CRM contact should I create accounts for?" - if not _non_empty_arg(args, "mailbox_username"): - return "What 508 mailbox username should I create?" - return None - - -def _non_empty_arg(args: dict[str, Any], key: str) -> bool: - return _non_empty_text(args.get(key)) - - -def _has_contact_reference(args: dict[str, Any]) -> bool: - return _non_empty_arg(args, "contact_id") or _non_empty_arg(args, "contact_query") - - -def _non_empty_text(value: Any) -> bool: - return isinstance(value, str) and bool(value.strip()) - - def _live_model_selection( *, profile: AgentEvalModelProfile, @@ -1530,7 +1535,10 @@ def render_markdown_report(report: AgentEvalReport) -> str: "time_to_first_turn_ms", "parse_success_rate", "parse_failures", + "provider_draft_failures", + "provider_draft_parse_failures", "bad_plan_rate", + "production_failures", "avg_latency_ms", "max_latency_ms", "estimated_cost_usd", @@ -1543,19 +1551,27 @@ def render_markdown_report(report: AgentEvalReport) -> str: lines.extend( [ "", - "| Scenario | Status | Failed checks | Latency ms | Parse |", - "| --- | --- | --- | --- | --- |", + "| Scenario | Production | Provider draft | Production failed checks | Provider draft failures | Latency ms | Parse |", + "| --- | --- | --- | --- | --- | --- | --- |", ] ) for scenario in report.scenarios: failed_checks = [check.name for check in scenario.checks if not check.passed] + provider_draft = scenario.provider_draft + provider_failures = ( + [check.name for check in provider_draft.checks if not check.passed] + if provider_draft is not None + else [] + ) lines.append( "| " + " | ".join( [ scenario.fixture_id, scenario.status, + provider_draft.status if provider_draft is not None else "-", ", ".join(failed_checks) if failed_checks else "-", + ", ".join(provider_failures) if provider_failures else "-", str(scenario.observed.latency_ms or "-"), str(scenario.observed.parse_success) if scenario.observed.parse_success is not None @@ -1579,6 +1595,12 @@ def render_trace_report(report: AgentEvalReport) -> str: ] for scenario in report.scenarios: failed_checks = [check for check in scenario.checks if not check.passed] + provider_draft = scenario.provider_draft + provider_failures = ( + [check for check in provider_draft.checks if not check.passed] + if provider_draft is not None + else [] + ) lines.extend( [ f"## {scenario.fixture_id}", @@ -1607,6 +1629,36 @@ def render_trace_report(report: AgentEvalReport) -> str: ) else: lines.append("- none") + lines.extend( + [ + "", + "### Provider Draft Probe", + "", + ] + ) + if provider_draft is None: + lines.append("- not run") + else: + lines.append(f"- Status: `{provider_draft.status}`") + lines.append( + "- Raw output: " + + ( + "available in observed JSON" + if scenario.observed.raw_model_output + else "-" + ) + ) + if provider_failures: + for check in provider_failures: + lines.extend( + [ + f"- `{check.name}`", + f" - expected: `{_compact_json(check.expected)}`", + f" - observed: `{_compact_json(check.observed)}`", + ] + ) + else: + lines.append("- Semantic mismatches: none") lines.extend( [ "", @@ -1638,6 +1690,12 @@ def render_ctrf_report(report: AgentEvalReport) -> dict[str, Any]: tests: list[dict[str, Any]] = [] for scenario in report.scenarios: failed_checks = [check.name for check in scenario.checks if not check.passed] + provider_draft = scenario.provider_draft + provider_failures = ( + [check.name for check in provider_draft.checks if not check.passed] + if provider_draft is not None + else [] + ) status = ( "passed" if scenario.status in {"passed", "known_failure"} else "failed" ) @@ -1650,6 +1708,10 @@ def render_ctrf_report(report: AgentEvalReport) -> dict[str, Any]: "suite": ["discord-agent", report.suite, report.model], "tags": scenario.observed.tags, "rawStatus": scenario.status, + "providerDraftStatus": ( + provider_draft.status if provider_draft is not None else None + ), + "providerDraftFailures": provider_failures, "filePath": f"tests/evals/discord-agent/fixtures/v1/{scenario.fixture_id}.json", } ) diff --git a/packages/shared/src/five08/agent/orchestrator.py b/packages/shared/src/five08/agent/orchestrator.py index 8b263f86..8c3e2039 100644 --- a/packages/shared/src/five08/agent/orchestrator.py +++ b/packages/shared/src/five08/agent/orchestrator.py @@ -108,11 +108,7 @@ def policy(self) -> PolicyEngine: def plan(self, message: str, context: AgentIdentityContext) -> AgentResponse: text = message.strip() - loaded_context = self.context_loader.load( - context=context, - bounds=self.context_bounds, - ) - context = context.model_copy(update={"context_snippets": loaded_context}) + context = self._load_request_context(context) if not text: return AgentResponse( status="needs_clarification", @@ -120,25 +116,13 @@ def plan(self, message: str, context: AgentIdentityContext) -> AgentResponse: clarification_question="What task or project action should I take?", ) - resolved_member_agreement = self._plan_member_agreement_from_crm( - text, - context, - planner="deterministic_regex", - ) - if resolved_member_agreement is not None: - return resolved_member_agreement + deterministic_response = self._plan_deterministic_workflow(text, context) + if deterministic_response is not None: + return deterministic_response planner: LiteralPlanner = "deterministic_regex" planning_text = text - action = self._parse_action(text) - if action is not None: - return self._response_for_deterministic_action( - action=action, - context=context, - planning_text=planning_text, - planner=planner, - ) - + action: AgentToolAction | None = None planned_response = self._plan_with_model(text, context) if planned_response is not None: return planned_response @@ -174,6 +158,42 @@ def plan(self, message: str, context: AgentIdentityContext) -> AgentResponse: planner=planner, ) + def _load_request_context( + self, context: AgentIdentityContext + ) -> AgentIdentityContext: + """Load bounded request context before deterministic or model planning.""" + + loaded_context = self.context_loader.load( + context=context, + bounds=self.context_bounds, + ) + return context.model_copy(update={"context_snippets": loaded_context}) + + def _plan_deterministic_workflow( + self, + text: str, + context: AgentIdentityContext, + ) -> AgentResponse | None: + """Return the production response for an explicitly recognized workflow.""" + + resolved_member_agreement = self._plan_member_agreement_from_crm( + text, + context, + planner="deterministic_regex", + ) + if resolved_member_agreement is not None: + return resolved_member_agreement + + action = self._parse_action(text) + if action is not None: + return self._response_for_deterministic_action( + action=action, + context=context, + planning_text=text, + planner="deterministic_regex", + ) + return None + def _response_for_deterministic_action( self, *, diff --git a/tests/evals/discord-agent/README.md b/tests/evals/discord-agent/README.md index 58691bbc..f73675d2 100644 --- a/tests/evals/discord-agent/README.md +++ b/tests/evals/discord-agent/README.md @@ -63,8 +63,11 @@ draft. The harness still does not let the model authorize users or perform side effects: deterministic policy checks scopes, write actions stop at confirmation, and read actions use fixture stubs when provided. For a request that production can route deterministically, the harness evaluates -that same production route while retaining the provider draft in -`raw_model_output` for inspection. +that same production route before independently probing the provider draft. +Production checks control the scenario and CI result; the raw draft's parse and +semantic mismatches are retained in `raw_model_output`, `bad_plans` metrics, +and the Markdown/trace reports so deterministic routing does not hide provider +quality regressions. Without `--live-planner`, the runner uses the deterministic parser path. That is useful for local no-key debugging, but it is not the main PR gate. diff --git a/tests/unit/test_agent_evals.py b/tests/unit/test_agent_evals.py index 7887ecb8..9b8d3685 100644 --- a/tests/unit/test_agent_evals.py +++ b/tests/unit/test_agent_evals.py @@ -5,6 +5,8 @@ import json from pathlib import Path +import requests + from five08.agent.evals import ( AgentEvalExpect, AgentEvalExpectedAction, @@ -15,6 +17,8 @@ load_fixtures, list_eval_model_profiles, resolve_eval_model_profile, + render_markdown_report, + render_trace_report, run_live_planner_eval_suite, run_eval_suite, write_report, @@ -204,7 +208,9 @@ def fake_post(*args: object, **kwargs: object) -> FakeResponse: } -def test_live_planner_eval_falls_back_for_parseable_clarification(monkeypatch) -> None: +def test_live_planner_eval_uses_production_route_when_draft_clarifies( + monkeypatch, +) -> None: class FakeResponse: status_code = 200 @@ -247,15 +253,18 @@ def fake_post(*args: object, **kwargs: object) -> FakeResponse: assert report.summary["passed"] == 1 assert report.summary["failed"] == 0 + assert report.metrics["bad_plans"] == 1 scenario = report.scenarios[0] assert scenario.status == "passed" + assert scenario.provider_draft is not None + assert scenario.provider_draft.status == "failed" assert scenario.observed.actions[0].arguments == { "project": "Atlas", "query": "onboarding", } -def test_live_planner_eval_preserves_raw_misroute_but_uses_production_route( +def test_live_planner_eval_records_raw_misroute_but_uses_production_route( monkeypatch, ) -> None: raw_output = json.dumps( @@ -299,6 +308,8 @@ def json(self) -> dict[str, object]: ) assert report.summary["passed"] == 1 + assert report.metrics["bad_plans"] == 1 + assert report.metrics["provider_draft_failures"] == 1 scenario = report.scenarios[0] assert scenario.observed.raw_model_output == raw_output assert scenario.observed.actions[0].tool_name == "task_write.create_task" @@ -306,6 +317,93 @@ def json(self) -> dict[str, object]: "title": "follow up on GitHub issue 123", "project": "Atlas", } + assert scenario.provider_draft is not None + assert scenario.provider_draft.status == "failed" + assert { + check.name for check in scenario.provider_draft.checks if not check.passed + } >= { + "provider_draft.actions[0].tool_name", + "provider_draft.actions[0].arguments.title", + "provider_draft.actions[0].arguments.project", + } + assert "Provider draft failures" in render_markdown_report(report) + trace = render_trace_report(report) + assert "### Provider Draft Probe" in trace + assert "provider_draft.actions[0].tool_name" in trace + + +def test_live_planner_eval_records_provider_timeout_for_deterministic_route( + monkeypatch, +) -> None: + calls: list[str] = [] + + def fake_post(*_args: object, **_kwargs: object) -> object: + calls.append("call") + raise requests.Timeout("provider timeout") + + monkeypatch.setenv("OPENAI_API_KEY_DIRECT", "direct-key") + monkeypatch.setattr("five08.agent.evals.requests.post", fake_post) + + report = run_live_planner_eval_suite( + suite="canonical", + model="openai-direct", + ids=["task_create_mentions_github_issue_001"], + timeout_seconds=1, + ) + + assert calls == ["call"] + assert report.summary["passed"] == 1 + assert report.summary["failed"] == 0 + assert report.metrics["parse_failures"] == 1 + assert report.metrics["bad_plans"] == 1 + scenario = report.scenarios[0] + assert scenario.observed.parse_success is False + assert scenario.observed.actions[0].tool_name == "task_write.create_task" + assert scenario.provider_draft is not None + assert scenario.provider_draft.status == "parse_failed" + assert len(scenario.provider_draft.checks) == 1 + check = scenario.provider_draft.checks[0] + assert check.name == "provider_draft.parse_success" + assert check.passed is False + assert check.expected is True + assert check.observed is False + + +def test_live_planner_eval_records_invalid_draft_for_deterministic_route( + monkeypatch, +) -> None: + raw_output = "not valid planner JSON" + + class FakeResponse: + status_code = 200 + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, object]: + return {"choices": [{"message": {"content": raw_output}}]} + + monkeypatch.setenv("OPENAI_API_KEY_DIRECT", "direct-key") + monkeypatch.setattr( + "five08.agent.evals.requests.post", + lambda *_args, **_kwargs: FakeResponse(), + ) + + report = run_live_planner_eval_suite( + suite="canonical", + model="openai-direct", + ids=["task_create_mentions_github_issue_001"], + timeout_seconds=1, + ) + + assert report.summary["passed"] == 1 + assert report.metrics["bad_plans"] == 1 + scenario = report.scenarios[0] + assert scenario.observed.parse_success is False + assert scenario.observed.raw_model_output == raw_output + assert scenario.observed.actions[0].tool_name == "task_write.create_task" + assert scenario.provider_draft is not None + assert scenario.provider_draft.status == "parse_failed" def test_live_planner_eval_retries_one_bad_plan(monkeypatch) -> None: From 97e7a8be79c2d8f7fb4fcd3058be4b8a0072c1e7 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Tue, 4 Aug 2026 17:25:28 +0900 Subject: [PATCH 07/14] fix: preserve planner handling for compound commands --- .../shared/src/five08/agent/orchestrator.py | 18 ++++++ tests/unit/test_agent_gateway.py | 64 +++++++++++++++++-- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/packages/shared/src/five08/agent/orchestrator.py b/packages/shared/src/five08/agent/orchestrator.py index 8c3e2039..e797d56b 100644 --- a/packages/shared/src/five08/agent/orchestrator.py +++ b/packages/shared/src/five08/agent/orchestrator.py @@ -41,6 +41,10 @@ r"\s+(\d{1,2})(?:,\s*|\s+)(20\d{2})\b", re.IGNORECASE, ) +_WORKFLOW_CLAUSE_SEPARATOR_RE = re.compile( + r"\s*(?:;|\b(?:and\s+then|and\s+also|then|also|and)\b)\s*", + re.IGNORECASE, +) _WEEKDAYS = { "monday": 0, "tuesday": 1, @@ -176,6 +180,9 @@ def _plan_deterministic_workflow( ) -> AgentResponse | None: """Return the production response for an explicitly recognized workflow.""" + if self._has_multiple_deterministic_workflows(text): + return None + resolved_member_agreement = self._plan_member_agreement_from_crm( text, context, @@ -194,6 +201,17 @@ def _plan_deterministic_workflow( ) return None + def _has_multiple_deterministic_workflows(self, text: str) -> bool: + """Avoid collapsing separately recognized commands into one regex action.""" + + workflow_count = sum( + self._parse_action(clause) is not None + or self._extract_member_agreement_recipient(clause) is not None + for clause in _WORKFLOW_CLAUSE_SEPARATOR_RE.split(text) + if clause.strip() + ) + return workflow_count > 1 + def _response_for_deterministic_action( self, *, diff --git a/tests/unit/test_agent_gateway.py b/tests/unit/test_agent_gateway.py index 7000aa31..4e709706 100644 --- a/tests/unit/test_agent_gateway.py +++ b/tests/unit/test_agent_gateway.py @@ -1780,15 +1780,26 @@ def test_task_creation_is_not_rerouted_to_member_agreement_submission() -> None: def test_explicit_task_creation_does_not_depend_on_model_routing() -> None: - class FailingPlanner: - def plan(self, **_kwargs: object) -> AgentPlannerResult: - raise AssertionError("planner should not run for a known workflow") + planner_calls: list[dict[str, object]] = [] + + class RecordingPlanner: + def plan(self, **kwargs: object) -> AgentPlannerResult: + planner_calls.append(kwargs) + return AgentPlannerResult( + draft=PlannerDraft( + status="needs_clarification", + clarification_question="What should I do next?", + ), + model=AgentModelConfig().resolve("fast"), + latency_ms=1, + ) - response = AgentOrchestrator(planner=FailingPlanner()).plan( + response = AgentOrchestrator(planner=RecordingPlanner()).plan( "Create a task to follow up on GitHub issue 123 in project Atlas", _context(), ) + assert planner_calls == [] assert response.status == "requires_confirmation" assert response.plan is not None assert response.plan.planner == "deterministic_regex" @@ -1799,6 +1810,51 @@ def plan(self, **_kwargs: object) -> AgentPlannerResult: } +def test_compound_deterministic_workflows_use_the_model_planner() -> None: + planner_calls: list[dict[str, object]] = [] + + class RecordingPlanner: + def plan(self, **kwargs: object) -> AgentPlannerResult: + planner_calls.append(kwargs) + return AgentPlannerResult( + draft=PlannerDraft( + status="planned", + intent="invite_and_follow_up", + actions=[ + { + "tool_name": "outline_write.invite_user", + "arguments": {"email": "sarah@example.com"}, + "summary": "Invite Sarah to Outline", + }, + { + "tool_name": "task_write.create_task", + "arguments": {"title": "follow up"}, + "summary": "Create a follow-up task", + }, + ], + ), + model=AgentModelConfig().resolve("fast"), + latency_ms=1, + ) + + response = AgentOrchestrator(planner=RecordingPlanner()).plan( + "Invite sarah@example.com to Outline and create a task to follow up", + _context(roles=["Admin"]), + ) + + assert len(planner_calls) == 1 + assert planner_calls[0]["message"] == ( + "Invite sarah@example.com to Outline and create a task to follow up" + ) + assert response.status == "requires_confirmation" + assert response.plan is not None + assert response.plan.planner == "live_model" + assert [action.tool_name for action in response.plan.actions] == [ + "outline_write.invite_user", + "task_write.create_task", + ] + + def test_agent_uses_structured_planner_for_multi_action_confirmation() -> None: class FakePlanner: def plan(self, **kwargs: object) -> AgentPlannerResult: From aaaa13af2ce6416ed1c5522007679b6863fb1383 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Tue, 4 Aug 2026 18:15:23 +0900 Subject: [PATCH 08/14] fix: harden live agent evaluation routing --- packages/shared/src/five08/agent/evals.py | 27 ++++- .../shared/src/five08/agent/orchestrator.py | 20 +++- tests/evals/discord-agent/README.md | 8 +- tests/unit/test_agent_evals.py | 108 ++++++++++++++++++ tests/unit/test_agent_gateway.py | 42 +++++++ 5 files changed, 196 insertions(+), 9 deletions(-) diff --git a/packages/shared/src/five08/agent/evals.py b/packages/shared/src/five08/agent/evals.py index 98d10f2b..ed4f8cd4 100644 --- a/packages/shared/src/five08/agent/evals.py +++ b/packages/shared/src/five08/agent/evals.py @@ -648,7 +648,11 @@ def run_fixture_with_live_planner( message=message, timeout_seconds=timeout_seconds, ) - provider_draft = _evaluate_provider_draft(fixture.expect, call) + provider_draft = _evaluate_provider_draft( + fixture.expect, + call, + registry=orchestrator.registry, + ) if deterministic_response is not None: response = deterministic_response else: @@ -1027,6 +1031,8 @@ def _parse_live_planner_json(raw_output: str) -> LivePlannerDraft: def _evaluate_provider_draft( expect: AgentEvalExpect, call: LivePlannerCallResult, + *, + registry: ToolRegistry, ) -> AgentEvalProviderDraftProbe: """Evaluate raw provider semantics without conflating them with execution.""" @@ -1067,6 +1073,23 @@ def _evaluate_provider_draft( strict=False, ) ) + for index, observed_action in enumerate(draft.actions): + try: + registry.validate_planner_action( + observed_action.tool_name, + observed_action.arguments, + ) + except ValueError: + schema_valid = False + else: + schema_valid = True + checks.append( + _check( + f"provider_draft.actions[{index}].schema_valid", + True, + schema_valid, + ) + ) if expect.actions is not None: checks.append( _check( @@ -1096,7 +1119,7 @@ def _evaluate_provider_draft( f"{prefix}.tool_name", expected_action.tool_name, observed_action.tool_name, - strict=False, + strict=True, ) ) if expected_action.arguments is not None: diff --git a/packages/shared/src/five08/agent/orchestrator.py b/packages/shared/src/five08/agent/orchestrator.py index e797d56b..0916f3ba 100644 --- a/packages/shared/src/five08/agent/orchestrator.py +++ b/packages/shared/src/five08/agent/orchestrator.py @@ -45,6 +45,10 @@ r"\s*(?:;|\b(?:and\s+then|and\s+also|then|also|and)\b)\s*", re.IGNORECASE, ) +_ELLIPTICAL_TASK_CLAUSE_RE = re.compile( + r"^\s*(?:another|(?:a\s+)?(?:second|third|fourth)|one\s+more|an\s+additional)\s+task\b", + re.IGNORECASE, +) _WEEKDAYS = { "monday": 0, "tuesday": 1, @@ -202,15 +206,23 @@ def _plan_deterministic_workflow( return None def _has_multiple_deterministic_workflows(self, text: str) -> bool: - """Avoid collapsing separately recognized commands into one regex action.""" + """Avoid collapsing separate or elliptical commands into one regex action.""" + clauses = [ + clause + for clause in _WORKFLOW_CLAUSE_SEPARATOR_RE.split(text) + if clause.strip() + ] workflow_count = sum( self._parse_action(clause) is not None or self._extract_member_agreement_recipient(clause) is not None - for clause in _WORKFLOW_CLAUSE_SEPARATOR_RE.split(text) - if clause.strip() + for clause in clauses + ) + if workflow_count > 1: + return True + return workflow_count > 0 and any( + _ELLIPTICAL_TASK_CLAUSE_RE.match(clause) for clause in clauses ) - return workflow_count > 1 def _response_for_deterministic_action( self, diff --git a/tests/evals/discord-agent/README.md b/tests/evals/discord-agent/README.md index f73675d2..a547edbf 100644 --- a/tests/evals/discord-agent/README.md +++ b/tests/evals/discord-agent/README.md @@ -65,9 +65,11 @@ and read actions use fixture stubs when provided. For a request that production can route deterministically, the harness evaluates that same production route before independently probing the provider draft. Production checks control the scenario and CI result; the raw draft's parse and -semantic mismatches are retained in `raw_model_output`, `bad_plans` metrics, -and the Markdown/trace reports so deterministic routing does not hide provider -quality regressions. +semantic mismatches are retained in each scenario's `provider_draft.checks`, +in the `provider_draft_failures` and `provider_draft_parse_failures` metrics, +and in the Markdown/trace reports, so deterministic routing does not hide +provider quality regressions. The raw provider text stays in +`raw_model_output`; `bad_plans` remains an alias for provider-draft failures. Without `--live-planner`, the runner uses the deterministic parser path. That is useful for local no-key debugging, but it is not the main PR gate. diff --git a/tests/unit/test_agent_evals.py b/tests/unit/test_agent_evals.py index 9b8d3685..ce076ed1 100644 --- a/tests/unit/test_agent_evals.py +++ b/tests/unit/test_agent_evals.py @@ -332,6 +332,114 @@ def json(self) -> dict[str, object]: assert "provider_draft.actions[0].tool_name" in trace +def test_live_planner_eval_rejects_provider_tool_name_prefixes(monkeypatch) -> None: + raw_output = json.dumps( + { + "status": "planned", + "intent": "create_task", + "clarification_question": None, + "actions": [ + { + "tool_name": "task_write.create_task_v2", + "arguments": { + "title": "follow up on GitHub issue 123", + "project": "Atlas", + }, + "summary": "Create a follow-up task", + } + ], + } + ) + + class FakeResponse: + status_code = 200 + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, object]: + return {"choices": [{"message": {"content": raw_output}}]} + + monkeypatch.setenv("OPENAI_API_KEY_DIRECT", "direct-key") + monkeypatch.setattr( + "five08.agent.evals.requests.post", + lambda *_args, **_kwargs: FakeResponse(), + ) + + report = run_live_planner_eval_suite( + suite="canonical", + model="openai-direct", + ids=["task_create_mentions_github_issue_001"], + timeout_seconds=1, + ) + + assert report.summary["passed"] == 1 + assert report.metrics["bad_plans"] == 1 + scenario = report.scenarios[0] + assert scenario.provider_draft is not None + assert scenario.provider_draft.status == "failed" + assert { + check.name for check in scenario.provider_draft.checks if not check.passed + } >= { + "provider_draft.actions[0].tool_name", + "provider_draft.actions[0].schema_valid", + } + + +def test_live_planner_eval_rejects_unknown_provider_action_arguments( + monkeypatch, +) -> None: + raw_output = json.dumps( + { + "status": "planned", + "intent": "create_task", + "clarification_question": None, + "actions": [ + { + "tool_name": "task_write.create_task", + "arguments": { + "title": "follow up on GitHub issue 123", + "project": "Atlas", + "organization_id": "org-1", + }, + "summary": "Create a follow-up task", + } + ], + } + ) + + class FakeResponse: + status_code = 200 + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, object]: + return {"choices": [{"message": {"content": raw_output}}]} + + monkeypatch.setenv("OPENAI_API_KEY_DIRECT", "direct-key") + monkeypatch.setattr( + "five08.agent.evals.requests.post", + lambda *_args, **_kwargs: FakeResponse(), + ) + + report = run_live_planner_eval_suite( + suite="canonical", + model="openai-direct", + ids=["task_create_mentions_github_issue_001"], + timeout_seconds=1, + ) + + assert report.summary["passed"] == 1 + assert report.metrics["bad_plans"] == 1 + scenario = report.scenarios[0] + assert scenario.provider_draft is not None + assert scenario.provider_draft.status == "failed" + assert { + check.name for check in scenario.provider_draft.checks if not check.passed + } == {"provider_draft.actions[0].schema_valid"} + + def test_live_planner_eval_records_provider_timeout_for_deterministic_route( monkeypatch, ) -> None: diff --git a/tests/unit/test_agent_gateway.py b/tests/unit/test_agent_gateway.py index 4e709706..c879b126 100644 --- a/tests/unit/test_agent_gateway.py +++ b/tests/unit/test_agent_gateway.py @@ -1855,6 +1855,48 @@ def plan(self, **kwargs: object) -> AgentPlannerResult: ] +def test_elliptical_compound_task_request_uses_the_model_planner() -> None: + planner_calls: list[dict[str, object]] = [] + + class RecordingPlanner: + def plan(self, **kwargs: object) -> AgentPlannerResult: + planner_calls.append(kwargs) + return AgentPlannerResult( + draft=PlannerDraft( + status="planned", + intent="create_tasks", + actions=[ + { + "tool_name": "task_write.create_task", + "arguments": {"title": "draft the agenda"}, + "summary": "Draft the agenda", + }, + { + "tool_name": "task_write.create_task", + "arguments": {"title": "book a room"}, + "summary": "Book a room", + }, + ], + ), + model=AgentModelConfig().resolve("fast"), + latency_ms=1, + ) + + response = AgentOrchestrator(planner=RecordingPlanner()).plan( + "Create a task to draft the agenda and another task to book a room", + _context(), + ) + + assert len(planner_calls) == 1 + assert response.status == "requires_confirmation" + assert response.plan is not None + assert response.plan.planner == "live_model" + assert [action.arguments["title"] for action in response.plan.actions] == [ + "draft the agenda", + "book a room", + ] + + def test_agent_uses_structured_planner_for_multi_action_confirmation() -> None: class FakePlanner: def plan(self, **kwargs: object) -> AgentPlannerResult: From 6bcb7420c2acbf89a6b07bc562cff9d088e99f1e Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Tue, 4 Aug 2026 18:37:47 +0900 Subject: [PATCH 09/14] fix: separate raw provider eval expectations --- packages/shared/src/five08/agent/evals.py | 3 +- .../v1/member_agreement_crm_resolve_001.json | 4 ++ tests/unit/test_agent_evals.py | 48 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/packages/shared/src/five08/agent/evals.py b/packages/shared/src/five08/agent/evals.py index ed4f8cd4..30dd8802 100644 --- a/packages/shared/src/five08/agent/evals.py +++ b/packages/shared/src/five08/agent/evals.py @@ -168,6 +168,7 @@ class AgentEvalFixture(BaseModel): seed: AgentEvalSeed = Field(default_factory=AgentEvalSeed) request: AgentEvalRequest expect: AgentEvalExpect + provider_expect: AgentEvalExpect | None = None known_failure: AgentEvalKnownFailure | None = None @@ -649,7 +650,7 @@ def run_fixture_with_live_planner( timeout_seconds=timeout_seconds, ) provider_draft = _evaluate_provider_draft( - fixture.expect, + fixture.provider_expect or fixture.expect, call, registry=orchestrator.registry, ) diff --git a/tests/evals/discord-agent/fixtures/v1/member_agreement_crm_resolve_001.json b/tests/evals/discord-agent/fixtures/v1/member_agreement_crm_resolve_001.json index f0aeddf3..964cc15e 100644 --- a/tests/evals/discord-agent/fixtures/v1/member_agreement_crm_resolve_001.json +++ b/tests/evals/discord-agent/fixtures/v1/member_agreement_crm_resolve_001.json @@ -28,6 +28,10 @@ "request": { "message": "Send member agreement to Caleb" }, + "provider_expect": { + "status": "needs_clarification", + "actions": [] + }, "expect": { "status": "requires_confirmation", "intent": "send_member_agreement", diff --git a/tests/unit/test_agent_evals.py b/tests/unit/test_agent_evals.py index ce076ed1..81fca585 100644 --- a/tests/unit/test_agent_evals.py +++ b/tests/unit/test_agent_evals.py @@ -264,6 +264,54 @@ def fake_post(*args: object, **kwargs: object) -> FakeResponse: } +def test_live_planner_eval_uses_raw_provider_expectations_before_enrichment( + monkeypatch, +) -> None: + class FakeResponse: + status_code = 200 + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, object]: + return { + "choices": [ + { + "message": { + "content": ( + '{"status":"needs_clarification",' + '"intent":"send_member_agreement",' + '"clarification_question":"What is Caleb\'s email address?",' + '"actions":[]}' + ) + } + } + ] + } + + monkeypatch.setenv("OPENAI_API_KEY_DIRECT", "direct-key") + monkeypatch.setattr( + "five08.agent.evals.requests.post", + lambda *_args, **_kwargs: FakeResponse(), + ) + + report = run_live_planner_eval_suite( + suite="canonical", + model="openai-direct", + ids=["member_agreement_crm_resolve_001"], + timeout_seconds=1, + ) + + assert report.summary["passed"] == 1 + assert report.metrics["bad_plans"] == 0 + scenario = report.scenarios[0] + assert scenario.provider_draft is not None + assert scenario.provider_draft.status == "passed" + assert ( + scenario.observed.actions[0].arguments["submitter_email"] == "caleb@example.com" + ) + + def test_live_planner_eval_records_raw_misroute_but_uses_production_route( monkeypatch, ) -> None: From 58edea5b272af08d6dca46d7fc198929e71a6589 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Tue, 4 Aug 2026 18:52:22 +0900 Subject: [PATCH 10/14] fix: harden compound agent eval routing --- packages/shared/src/five08/agent/evals.py | 11 +++++ .../shared/src/five08/agent/orchestrator.py | 17 +++++++ .../schema/trajectory.v1.schema.json | 1 + tests/unit/test_agent_evals.py | 46 +++++++++++++++++++ tests/unit/test_agent_gateway.py | 24 ++++++++++ 5 files changed, 99 insertions(+) diff --git a/packages/shared/src/five08/agent/evals.py b/packages/shared/src/five08/agent/evals.py index 30dd8802..a3d84ba5 100644 --- a/packages/shared/src/five08/agent/evals.py +++ b/packages/shared/src/five08/agent/evals.py @@ -1074,6 +1074,17 @@ def _evaluate_provider_draft( strict=False, ) ) + if expected_status == "needs_clarification": + checks.append( + _check( + "provider_draft.clarification_question_present", + True, + bool( + draft.clarification_question + and draft.clarification_question.strip() + ), + ) + ) for index, observed_action in enumerate(draft.actions): try: registry.validate_planner_action( diff --git a/packages/shared/src/five08/agent/orchestrator.py b/packages/shared/src/five08/agent/orchestrator.py index 0916f3ba..2bdf99bc 100644 --- a/packages/shared/src/five08/agent/orchestrator.py +++ b/packages/shared/src/five08/agent/orchestrator.py @@ -49,6 +49,12 @@ r"^\s*(?:another|(?:a\s+)?(?:second|third|fourth)|one\s+more|an\s+additional)\s+task\b", re.IGNORECASE, ) +_SHARED_WORKFLOW_VERB_RE = re.compile( + r"^\s*(?:please\s+)?(?P" + r"search|find|list|show|create|open|update|edit|close|assign|add|invite|send|provision" + r")\b", + re.IGNORECASE, +) _WEEKDAYS = { "monday": 0, "tuesday": 1, @@ -220,6 +226,17 @@ def _has_multiple_deterministic_workflows(self, text: str) -> bool: ) if workflow_count > 1: return True + shared_verb_match = ( + _SHARED_WORKFLOW_VERB_RE.match(clauses[0]) if clauses else None + ) + if shared_verb_match is not None: + verb = shared_verb_match.group("verb") + if any( + self._parse_action(clause) is None + and self._parse_action(f"{verb} {clause}") is not None + for clause in clauses[1:] + ): + return True return workflow_count > 0 and any( _ELLIPTICAL_TASK_CLAUSE_RE.match(clause) for clause in clauses ) diff --git a/tests/evals/discord-agent/schema/trajectory.v1.schema.json b/tests/evals/discord-agent/schema/trajectory.v1.schema.json index c28fb5ee..9aac3385 100644 --- a/tests/evals/discord-agent/schema/trajectory.v1.schema.json +++ b/tests/evals/discord-agent/schema/trajectory.v1.schema.json @@ -47,6 +47,7 @@ "additionalProperties": false }, "expect": { "type": "object" }, + "provider_expect": { "type": "object" }, "known_failure": { "type": "object", "required": ["reason"], diff --git a/tests/unit/test_agent_evals.py b/tests/unit/test_agent_evals.py index 81fca585..d4d501df 100644 --- a/tests/unit/test_agent_evals.py +++ b/tests/unit/test_agent_evals.py @@ -312,6 +312,52 @@ def json(self) -> dict[str, object]: ) +def test_live_planner_eval_rejects_clarification_without_a_question( + monkeypatch, +) -> None: + class FakeResponse: + status_code = 200 + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, object]: + return { + "choices": [ + { + "message": { + "content": ( + '{"status":"needs_clarification",' + '"intent":"send_member_agreement","actions":[]}' + ) + } + } + ] + } + + monkeypatch.setenv("OPENAI_API_KEY_DIRECT", "direct-key") + monkeypatch.setattr( + "five08.agent.evals.requests.post", + lambda *_args, **_kwargs: FakeResponse(), + ) + + report = run_live_planner_eval_suite( + suite="canonical", + model="openai-direct", + ids=["member_agreement_crm_resolve_001"], + timeout_seconds=1, + ) + + assert report.summary["passed"] == 1 + assert report.metrics["bad_plans"] == 1 + scenario = report.scenarios[0] + assert scenario.provider_draft is not None + assert scenario.provider_draft.status == "failed" + assert { + check.name for check in scenario.provider_draft.checks if not check.passed + } == {"provider_draft.clarification_question_present"} + + def test_live_planner_eval_records_raw_misroute_but_uses_production_route( monkeypatch, ) -> None: diff --git a/tests/unit/test_agent_gateway.py b/tests/unit/test_agent_gateway.py index c879b126..fd26ef3a 100644 --- a/tests/unit/test_agent_gateway.py +++ b/tests/unit/test_agent_gateway.py @@ -1897,6 +1897,30 @@ def plan(self, **kwargs: object) -> AgentPlannerResult: ] +def test_shared_verb_compound_workflow_uses_the_model_planner() -> None: + planner_calls: list[dict[str, object]] = [] + + class RecordingPlanner: + def plan(self, **kwargs: object) -> AgentPlannerResult: + planner_calls.append(kwargs) + return AgentPlannerResult( + draft=PlannerDraft( + status="needs_clarification", + clarification_question="Which task and issue details should I use?", + ), + model=AgentModelConfig().resolve("fast"), + latency_ms=1, + ) + + request = "Search tasks in project Atlas and GitHub issues for onboarding" + response = AgentOrchestrator(planner=RecordingPlanner()).plan(request, _context()) + + assert len(planner_calls) == 1 + assert planner_calls[0]["message"] == request + assert response.status == "needs_clarification" + assert response.plan is None + + def test_agent_uses_structured_planner_for_multi_action_confirmation() -> None: class FakePlanner: def plan(self, **kwargs: object) -> AgentPlannerResult: From 10cfca96c19b3199b935ca0cf899802416eaecd9 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Wed, 5 Aug 2026 09:09:55 +0900 Subject: [PATCH 11/14] fix: retry failed provider eval probes --- packages/shared/src/five08/agent/evals.py | 14 +++++- tests/unit/test_agent_evals.py | 55 ++++++++++++++++++++++- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/packages/shared/src/five08/agent/evals.py b/packages/shared/src/five08/agent/evals.py index a3d84ba5..b977d456 100644 --- a/packages/shared/src/five08/agent/evals.py +++ b/packages/shared/src/five08/agent/evals.py @@ -478,14 +478,16 @@ def run_live_planner_eval_suite( ) if time_to_first_turn_ms is None: time_to_first_turn_ms = _elapsed_ms(suite_started) - if result.status == "failed": + if result.status == "failed" or _provider_draft_failed(result): retry_result = run_fixture_with_live_planner( fixture=fixture, profile=profile, timeout_seconds=timeout_seconds, ) retries += 1 - if retry_result.status != "failed": + if retry_result.status != "failed" and not _provider_draft_failed( + retry_result + ): result = retry_result scenario_results.append(result) passed = sum(1 for result in scenario_results if result.status == "passed") @@ -541,6 +543,14 @@ def run_live_planner_eval_suite( ) +def _provider_draft_failed(result: AgentEvalScenarioResult) -> bool: + """Return whether the provider probe failed independently of production.""" + + return ( + result.provider_draft is not None and result.provider_draft.status != "passed" + ) + + def run_fixture( *, fixture: AgentEvalFixture, model: str = "primary" ) -> AgentEvalScenarioResult: diff --git a/tests/unit/test_agent_evals.py b/tests/unit/test_agent_evals.py index d4d501df..2adf9f6f 100644 --- a/tests/unit/test_agent_evals.py +++ b/tests/unit/test_agent_evals.py @@ -534,7 +534,7 @@ def json(self) -> dict[str, object]: } == {"provider_draft.actions[0].schema_valid"} -def test_live_planner_eval_records_provider_timeout_for_deterministic_route( +def test_live_planner_eval_retries_provider_timeout_for_deterministic_route( monkeypatch, ) -> None: calls: list[str] = [] @@ -553,7 +553,7 @@ def fake_post(*_args: object, **_kwargs: object) -> object: timeout_seconds=1, ) - assert calls == ["call"] + assert calls == ["call", "call"] assert report.summary["passed"] == 1 assert report.summary["failed"] == 0 assert report.metrics["parse_failures"] == 1 @@ -563,6 +563,7 @@ def fake_post(*_args: object, **_kwargs: object) -> object: assert scenario.observed.actions[0].tool_name == "task_write.create_task" assert scenario.provider_draft is not None assert scenario.provider_draft.status == "parse_failed" + assert report.metrics["retries"] == 1 assert len(scenario.provider_draft.checks) == 1 check = scenario.provider_draft.checks[0] assert check.name == "provider_draft.parse_success" @@ -664,6 +665,56 @@ def fake_post(*args: object, **kwargs: object) -> FakeResponse: assert report.scenarios[0].status == "passed" +def test_live_planner_eval_retries_bad_provider_probe_for_deterministic_route( + monkeypatch, +) -> None: + class FakeResponse: + status_code = 200 + + def __init__(self, content: str) -> None: + self._content = content + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, object]: + return {"choices": [{"message": {"content": self._content}}]} + + responses = [ + '{"status":"needs_clarification","intent":null,' + '"clarification_question":"Which project?","actions":[]}', + '{"status":"planned","intent":"search_tasks",' + '"clarification_question":null,"actions":[' + '{"tool_name":"task_read.search_tasks",' + '"arguments":{"query":"onboarding","project":"Atlas"},' + '"summary":"Search Atlas tasks"}]}', + ] + calls: list[str] = [] + + def fake_post(*args: object, **kwargs: object) -> FakeResponse: + calls.append("call") + return FakeResponse(responses.pop(0)) + + monkeypatch.setenv("OPENAI_API_KEY_DIRECT", "direct-key") + monkeypatch.setattr("five08.agent.evals.requests.post", fake_post) + fixture = load_fixtures(suite="canonical", ids=["search_project_tasks_001"])[0] + monkeypatch.setattr("five08.agent.evals.load_fixtures", lambda **_kwargs: [fixture]) + + report = run_live_planner_eval_suite( + suite="canonical", + model="openai-direct", + ids=["search_project_tasks_001"], + timeout_seconds=1, + ) + + assert len(calls) == 2 + assert report.summary["passed"] == 1 + assert report.metrics["bad_plans"] == 0 + assert report.metrics["retries"] == 1 + assert report.scenarios[0].provider_draft is not None + assert report.scenarios[0].provider_draft.status == "passed" + + def test_live_planner_eval_applies_production_argument_gate(monkeypatch) -> None: class FakeResponse: status_code = 200 From a31f37f10f4642f3bda73ee8136bfcd5e9199ed7 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Thu, 6 Aug 2026 09:11:01 +0900 Subject: [PATCH 12/14] fix: preserve compound agent workflows --- packages/shared/src/five08/agent/evals.py | 9 ++-- .../shared/src/five08/agent/orchestrator.py | 12 ++++- tests/unit/test_agent_evals.py | 50 +++++++++++++++++++ tests/unit/test_agent_gateway.py | 48 ++++++++++++++++++ 4 files changed, 115 insertions(+), 4 deletions(-) diff --git a/packages/shared/src/five08/agent/evals.py b/packages/shared/src/five08/agent/evals.py index b977d456..0010fae6 100644 --- a/packages/shared/src/five08/agent/evals.py +++ b/packages/shared/src/five08/agent/evals.py @@ -1112,16 +1112,19 @@ def _evaluate_provider_draft( schema_valid, ) ) - if expect.actions is not None: + expected_actions = expect.actions + if expected_actions is None and expected_status == "needs_clarification": + expected_actions = [] + if expected_actions is not None: checks.append( _check( "provider_draft.action_count", - len(expect.actions), + len(expected_actions), len(draft.actions), strict=False, ) ) - for index, expected_action in enumerate(expect.actions): + for index, expected_action in enumerate(expected_actions): observed_action = ( draft.actions[index] if index < len(draft.actions) else None ) diff --git a/packages/shared/src/five08/agent/orchestrator.py b/packages/shared/src/five08/agent/orchestrator.py index 2bdf99bc..b0fdbf9b 100644 --- a/packages/shared/src/five08/agent/orchestrator.py +++ b/packages/shared/src/five08/agent/orchestrator.py @@ -46,9 +46,10 @@ re.IGNORECASE, ) _ELLIPTICAL_TASK_CLAUSE_RE = re.compile( - r"^\s*(?:another|(?:a\s+)?(?:second|third|fourth)|one\s+more|an\s+additional)\s+task\b", + r"^\s*(?:another(?:\s+task)?|(?:a\s+)?(?:second|third|fourth)|one\s+more|an\s+additional)\s+(?:task\b|to\b)", re.IGNORECASE, ) +_EMAIL_ADDRESS_RE = re.compile(r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", re.IGNORECASE) _SHARED_WORKFLOW_VERB_RE = re.compile( r"^\s*(?:please\s+)?(?P" r"search|find|list|show|create|open|update|edit|close|assign|add|invite|send|provision" @@ -237,6 +238,15 @@ def _has_multiple_deterministic_workflows(self, text: str) -> bool: for clause in clauses[1:] ): return True + if ( + verb.casefold() == "invite" + and "outline" in clauses[0].casefold() + and any( + _EMAIL_ADDRESS_RE.fullmatch(clause.strip()) is not None + for clause in clauses[1:] + ) + ): + return True return workflow_count > 0 and any( _ELLIPTICAL_TASK_CLAUSE_RE.match(clause) for clause in clauses ) diff --git a/tests/unit/test_agent_evals.py b/tests/unit/test_agent_evals.py index 2adf9f6f..8a65ff3c 100644 --- a/tests/unit/test_agent_evals.py +++ b/tests/unit/test_agent_evals.py @@ -358,6 +358,56 @@ def json(self) -> dict[str, object]: } == {"provider_draft.clarification_question_present"} +def test_live_planner_eval_rejects_clarification_draft_actions( + monkeypatch, +) -> None: + class FakeResponse: + status_code = 200 + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, object]: + return { + "choices": [ + { + "message": { + "content": ( + '{"status":"needs_clarification",' + '"intent":"search_tasks",' + '"clarification_question":"Which project?",' + '"actions":[{"tool_name":"task_read.search_tasks",' + '"arguments":{"query":"onboarding","project":"Atlas"},' + '"summary":"Search Atlas tasks"}]}' + ) + } + } + ] + } + + monkeypatch.setenv("OPENAI_API_KEY_DIRECT", "direct-key") + monkeypatch.setattr( + "five08.agent.evals.requests.post", + lambda *_args, **_kwargs: FakeResponse(), + ) + + report = run_live_planner_eval_suite( + suite="canonical", + model="openai-direct", + ids=["missing_project_clarification_001"], + timeout_seconds=1, + ) + + assert report.summary["passed"] == 1 + assert report.metrics["bad_plans"] == 1 + scenario = report.scenarios[0] + assert scenario.provider_draft is not None + assert scenario.provider_draft.status == "failed" + assert "provider_draft.action_count" in { + check.name for check in scenario.provider_draft.checks if not check.passed + } + + def test_live_planner_eval_records_raw_misroute_but_uses_production_route( monkeypatch, ) -> None: diff --git a/tests/unit/test_agent_gateway.py b/tests/unit/test_agent_gateway.py index fd26ef3a..d1bcd1b7 100644 --- a/tests/unit/test_agent_gateway.py +++ b/tests/unit/test_agent_gateway.py @@ -1897,6 +1897,54 @@ def plan(self, **kwargs: object) -> AgentPlannerResult: ] +def test_pronoun_only_compound_task_request_uses_the_model_planner() -> None: + planner_calls: list[dict[str, object]] = [] + + class RecordingPlanner: + def plan(self, **kwargs: object) -> AgentPlannerResult: + planner_calls.append(kwargs) + return AgentPlannerResult( + draft=PlannerDraft( + status="needs_clarification", + clarification_question="What are the two tasks?", + ), + model=AgentModelConfig().resolve("fast"), + latency_ms=1, + ) + + request = "Create a task to draft the agenda and another to book a room" + response = AgentOrchestrator(planner=RecordingPlanner()).plan(request, _context()) + + assert len(planner_calls) == 1 + assert planner_calls[0]["message"] == request + assert response.status == "needs_clarification" + assert response.plan is None + + +def test_shared_target_outline_invites_use_the_model_planner() -> None: + planner_calls: list[dict[str, object]] = [] + + class RecordingPlanner: + def plan(self, **kwargs: object) -> AgentPlannerResult: + planner_calls.append(kwargs) + return AgentPlannerResult( + draft=PlannerDraft( + status="needs_clarification", + clarification_question="Should I invite both people?", + ), + model=AgentModelConfig().resolve("fast"), + latency_ms=1, + ) + + request = "Invite alice@example.com to Outline and bob@example.com" + response = AgentOrchestrator(planner=RecordingPlanner()).plan(request, _context()) + + assert len(planner_calls) == 1 + assert planner_calls[0]["message"] == request + assert response.status == "needs_clarification" + assert response.plan is None + + def test_shared_verb_compound_workflow_uses_the_model_planner() -> None: planner_calls: list[dict[str, object]] = [] From 95c97590c2173ab0e03ceb6fe4c233f40b6d6435 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Thu, 6 Aug 2026 09:16:52 +0900 Subject: [PATCH 13/14] test: assert all compound actions require confirmation --- tests/unit/test_agent_gateway.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_agent_gateway.py b/tests/unit/test_agent_gateway.py index d1bcd1b7..79fe876c 100644 --- a/tests/unit/test_agent_gateway.py +++ b/tests/unit/test_agent_gateway.py @@ -1891,6 +1891,7 @@ def plan(self, **kwargs: object) -> AgentPlannerResult: assert response.status == "requires_confirmation" assert response.plan is not None assert response.plan.planner == "live_model" + assert all(action.requires_confirmation for action in response.plan.actions) assert [action.arguments["title"] for action in response.plan.actions] == [ "draft the agenda", "book a room", From 04c1175779d6914c784434202089603687b8f339 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Sat, 15 Aug 2026 17:14:09 +0900 Subject: [PATCH 14/14] chore: update cloakbrowser lockfile --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 72f57319..fb8df27b 100644 --- a/uv.lock +++ b/uv.lock @@ -438,16 +438,16 @@ wheels = [ [[package]] name = "cloakbrowser" -version = "0.4.12" +version = "0.5.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cryptography" }, { name = "httpx" }, { name = "playwright" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/32/0f/085ac9e2ac78b6800d55a59cefc077489a3ec9dd43e31362e9bc06734cfa/cloakbrowser-0.4.12.tar.gz", hash = "sha256:683d150a53cc768f6db6e3e758268d1b2e35144e0b4847336bd8bc71fdcb8baf", size = 5609654, upload-time = "2026-07-18T21:34:24.567Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/66/e7986afaa1309881e7174ddb8f579b96b82ebcc43eeb3a2705c1908db296/cloakbrowser-0.5.6.tar.gz", hash = "sha256:47f95a26d8d7bad608598d5af8373149368777a5f3006bc8b17a409c7dd728df", size = 5694461, upload-time = "2026-08-07T22:17:14.735Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2f/e2/ec81d25a853d9fc0af91827b1ed5b97d0edcd34f3c7c2d1741915994dd53/cloakbrowser-0.4.12-py3-none-any.whl", hash = "sha256:0415acff4aa5f49c18bc9cbd6a65ae806591dfd71ddf5d862238c61cd8471142", size = 107156, upload-time = "2026-07-18T21:34:22.814Z" }, + { url = "https://files.pythonhosted.org/packages/81/91/0972dbfb475111180c260667bff1220b49b0a513e2d605cf31acc740900d/cloakbrowser-0.5.6-py3-none-any.whl", hash = "sha256:6acd11f110f480d5ef7e01907f7fcf5f30aa09ccb0efbc13a1749b715454b964", size = 125088, upload-time = "2026-08-07T22:17:13.171Z" }, ] [[package]]