From 8cc1e502cdb869e594a9e8a7011030657df82c71 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 15:35:17 -0400 Subject: [PATCH 01/40] Carry canonical SPM choices and receipts through simulation workers --- .../budget_window_state.py | 1 + .../gateway_models.py | 34 +- .../macro_output.py | 4 + .../policyengine_simulation_contract/spm.py | 284 ++++++++ .../tests/test_gateway_models.py | 3 + .../tests/test_spm_generated_client.py | 58 ++ .../src/policyengine_simulation_entry/app.py | 30 +- .../tests/test_openapi.py | 48 +- .../fixtures/test_simulation_api_contracts.py | 4 + .../src/modal/budget_window_batch.py | 5 + .../src/modal/budget_window_results.py | 5 + .../src/modal/budget_window_scheduler.py | 20 + .../src/modal/segmented_national.py | 21 + .../modal/utils/update_version_registry.py | 11 +- .../artifact_keys.py | 31 +- .../baseline_artifacts.py | 26 +- .../compat_models.py | 2 + .../precompute.py | 12 +- .../simulation.py | 27 +- .../simulation_output_common.py | 6 +- .../simulation_runtime.py | 35 +- .../policyengine_simulation_executor/spm.py | 142 ++++ .../tests/test_canonical_spm.py | 626 ++++++++++++++++++ .../tests/test_canonical_spm_native.py | 153 +++++ .../endpoints.py | 66 +- .../responses.py | 21 +- .../tests/test_endpoints.py | 5 +- 27 files changed, 1640 insertions(+), 40 deletions(-) create mode 100644 libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py create mode 100644 projects/policyengine-apis-integ/tests/test_spm_generated_client.py create mode 100644 projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py create mode 100644 projects/policyengine-simulation-executor/tests/test_canonical_spm.py create mode 100644 projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/budget_window_state.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/budget_window_state.py index ae3b5515d..c451bc3d1 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/budget_window_state.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/budget_window_state.py @@ -254,6 +254,7 @@ def build_batch_status_response( child_jobs=state.child_jobs, result=state.result, error=state.error, + errors=state.errors, resolved_app_name=state.resolved_app_name, policyengine_bundle=state.policyengine_bundle, run_id=state.run_id, diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py index 9ba41c34a..83babe3bd 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py @@ -14,11 +14,16 @@ model_validator, ) +from policyengine_simulation_contract.spm import ( + SPMSelection, + SPMCapability, + SPMComparisonProvenance, + SPMErrorDetail, +) from policyengine_simulation_contract.json_types import JsonObject from policyengine_simulation_contract.macro_output import SingleYearMacroOutput from policyengine_simulation_observability.telemetry import TelemetryEnvelope - # Hard cap on request body size (bytes). SimulationOptions + telemetry + any # reform/baseline parameter tree should fit comfortably in ~256 KB. A hostile # client that tries to stream a multi-MB reform dict is rejected with 422 @@ -95,6 +100,7 @@ class GatewayRequestBase(BaseModel): """ country: str + spm: Optional[SPMSelection] = None version: Optional[str] = None policyengine_version: Optional[str] = None telemetry: TelemetryEnvelope | None = None @@ -156,6 +162,17 @@ class PolicyEngineBundle(BaseModel): policyengine_version: Optional[str] = None data_version: Optional[str] = None dataset: Optional[str] = None + spm: Optional[SPMCapability] = None + + +class SimulationErrorResponse(BaseModel): + """Existing route errors and public typed SPM input failures.""" + + status: Literal["failed"] | None = None + result: None = None + error: str | None = None + errors: list[SPMErrorDetail] | None = None + detail: str | None = None class JobSubmitResponse(BaseModel): @@ -176,6 +193,7 @@ class JobStatusResponse(BaseModel): status: Literal["running", "complete", "failed"] result: Optional[SingleYearMacroOutput] = None + errors: Optional[list[SPMErrorDetail]] = None error: Optional[str] = None resolved_app_name: Optional[str] = None policyengine_bundle: Optional[PolicyEngineBundle] = None @@ -223,6 +241,8 @@ def validate_end_year(self) -> "BudgetWindowBatchRequest": class BudgetWindowAnnualImpact(BaseModel): """Annual budget-window impact row.""" + spm_config: Optional[SPMSelection] = None + spm_provenance: Optional[SPMComparisonProvenance] = None year: str taxRevenueImpact: float federalTaxRevenueImpact: float @@ -265,6 +285,7 @@ class BatchChildJobStatus(BaseModel): "failed", "cancelled", ] + errors: Optional[list[SPMErrorDetail]] = None error: Optional[str] = None @@ -292,6 +313,7 @@ class BudgetWindowBatchStatusResponse(BaseModel): failed_years: list[str] = Field(default_factory=list) child_jobs: dict[str, BatchChildJobStatus] = Field(default_factory=dict) result: Optional[BudgetWindowResult] = None + errors: Optional[list[SPMErrorDetail]] = None error: Optional[str] = None resolved_app_name: Optional[str] = None policyengine_bundle: Optional[PolicyEngineBundle] = None @@ -323,6 +345,7 @@ class BudgetWindowBatchState(BaseModel): default_factory=dict ) result: Optional[BudgetWindowResult] = None + errors: Optional[list[SPMErrorDetail]] = None error: Optional[str] = None created_at: str updated_at: str @@ -348,6 +371,15 @@ class VersionMap(RootModel[dict[str, str]]): class VersionsResponse(BaseModel): """All supported simulation routing version maps.""" + spm_capabilities: dict[str, SPMCapability] = Field( + default_factory=dict, + description=( + "Canonical SPM capabilities keyed by exact PolicyEngine wrapper version. " + "Resolve a country-model route to its app in the routing maps, then " + "find that app's exact version in the policyengine map. App names, " + "country-model versions and latest aliases are not capability keys." + ), + ) policyengine: VersionMap us: VersionMap uk: VersionMap diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py index 229131bb8..7b6eae156 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, ConfigDict, RootModel +from policyengine_simulation_contract.spm import SPMSelection, SPMComparisonProvenance T = TypeVar("T") @@ -188,6 +189,9 @@ class CongressionalDistrictImpactOutput(MacroOutputModel): class SingleYearMacroOutput(MacroOutputModel): """Completed response returned by a single-year macro simulation.""" + spm_config: SPMSelection | None = None + spm_provenance: SPMComparisonProvenance | None = None + model_version: str data_version: str budget: BudgetaryImpact diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py new file mode 100644 index 000000000..8523b9131 --- /dev/null +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py @@ -0,0 +1,284 @@ +"""Dependency-light public models for US SPM selection and calculation receipts.""" + +import re +from datetime import date +from typing import Any, Literal, Optional + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + SerializerFunctionWrapHandler, + field_validator, + model_serializer, + model_validator, +) + + +def _selection_schema(schema): + # An omitted option inherits the selected bundle's default. Advertising + # Python attribute defaults here makes generated clients send an explicit + # county selection even when their caller omitted geography entirely. + for field in schema.get("properties", {}).values(): + field.pop("default", None) + + +class SPMSelection(BaseModel): + """Select from the bundle's pinned artifact; national geography is explicit. + + County mode reads the household's observed ``county_fips``. A state alone + does not identify an SPM area. These settings contain no provider or path. + """ + + model_config = ConfigDict( + frozen=True, extra="forbid", json_schema_extra=_selection_schema + ) + + forecast_content_sha256: Optional[str] = Field( + default=None, pattern=r"^[0-9a-f]{64}$" + ) + scenario: Optional[str] = Field(default=None, min_length=1, pattern=r"^\S+$") + geography_kind: Literal["county", "national", "metro"] = "county" + geography_id: Optional[str] = Field(default=None, min_length=1) + county_vintage: str = Field(default="2020", pattern=r"^[0-9]{4}$") + as_of: Optional[str] = None + + @model_serializer(mode="wrap") + def serialize_selection(self, handler: SerializerFunctionWrapHandler): + """Preserve inherited options through ordinary and nested JSON.""" + return { + name: value + for name, value in handler(self).items() + if name in self.model_fields_set + } + + @field_validator("as_of") + @classmethod + def validate_as_of(cls, value): + if value is not None: + if date.fromisoformat(value).isoformat() != value: + raise ValueError("as_of must be an ISO calendar date (YYYY-MM-DD)") + return value + + @model_validator(mode="after") + def validate_location(self): + if self.geography_kind == "metro": + if not self.geography_id or not self.geography_id.strip(): + raise ValueError("An SPM area selection requires geography_id") + elif self.geography_id is not None: + raise ValueError("Only an SPM area selection accepts geography_id") + return self + + +class SPMProvenance(BaseModel): + """Detached calculation receipt; data certification is a separate claim.""" + + model_config = ConfigDict(extra="forbid") + + forecast_id: str + forecast_sha256: str + scenario: str + geography_kind: str + runtime_versions: dict[str, Optional[str]] + years: dict[str, dict[str, Any]] + geographies: list[dict[str, Any]] + composition_method: str + storage_method: str + + +SPM_CONTRACT_VERSION = "canonical-spm-v1" +SPM_ERROR_CODES = frozenset( + { + "SPM_GEOGRAPHY_REQUIRED", + "SPM_GEOGRAPHY_UNAVAILABLE", + "SPM_COMPOSITION_REQUIRED", + "SPM_CONFIGURATION_UNAVAILABLE", + "SPM_SETTINGS_INVALID", + } +) + + +class SPMErrorDetail(BaseModel): + code: str + message: str + + +class SPMInputError(ValueError): + """Transportable error shared by the control plane and worker.""" + + def __init__(self, code: str, message: str): + self.code = code + self.message = message + super().__init__(code, message) + + def __str__(self): + return self.message + + def to_dict(self): + return {"code": self.code, "message": self.message} + + +def spm_error_detail(exc: BaseException) -> SPMErrorDetail | None: + """Recognize only public typed errors, including wrapped country errors.""" + seen = set() + current: BaseException | None = exc + while current is not None and id(current) not in seen: + seen.add(id(current)) + code = getattr(current, "code", None) + if isinstance(current, ValueError) and code in SPM_ERROR_CODES: + return SPMErrorDetail(code=code, message=str(current)) + current = current.__cause__ or current.__context__ + return None + + +class SPMCapability(BaseModel): + model_config = ConfigDict(extra="forbid") + contract_version: Literal["canonical-spm-v1"] = SPM_CONTRACT_VERSION + defaults: SPMSelection + + @model_validator(mode="after") + def pinned_defaults(self): + if not self.defaults.forecast_content_sha256 or not self.defaults.scenario: + raise ValueError( + "Certified SPM defaults must pin artifact hash and scenario" + ) + self.defaults = SPMSelection( + **{name: getattr(self.defaults, name) for name in SPMSelection.model_fields} + ) + return self + + +class SPMComparisonProvenance(BaseModel): + """One receipt per executed regional segment, separately for each policy.""" + + model_config = ConfigDict(extra="forbid") + baseline: list[SPMProvenance] = Field(min_length=1) + reform: list[SPMProvenance] = Field(min_length=1) + + +def resolve_spm_selection( + country, selection, *, capability, policyengine_version, model_version +): + """Resolve only certified metadata; unrecognized future bundles fail closed.""" + if country.lower() != "us": + if selection is not None: + raise SPMInputError( + "SPM_SETTINGS_INVALID", "SPM settings are supported only for the US" + ) + return None + if capability is None: + # Existing versioned worker bundles predate canonical SPM. This does + # not certify a future bundle or change the routing registry default. + version_parts = str(policyengine_version or "").split(".") + historical = ( + len(version_parts) == 3 + and all(re.fullmatch(r"[0-9]+", p) for p in version_parts) + and tuple(map(int, version_parts)) < (5, 2, 0) + ) + historical = historical or ( + policyengine_version in {"5.2.0", "5.3.0"} and model_version == "1.764.6" + ) + if selection is None and historical: + return None + raise SPMInputError( + "SPM_CONFIGURATION_UNAVAILABLE", + "This worker bundle has no certified canonical SPM capability", + ) + try: + cap = SPMCapability.model_validate(capability) + chosen = SPMSelection.model_validate({} if selection is None else selection) + defaults = cap.defaults + if chosen.forecast_content_sha256 not in ( + None, + defaults.forecast_content_sha256, + ): + raise ValueError( + "SPM selection does not match the certified bundle artifact hash" + ) + values = {name: getattr(defaults, name) for name in SPMSelection.model_fields} + values.update(chosen.model_dump(exclude_unset=True)) + if ( + "geography_kind" in chosen.model_fields_set + and chosen.geography_kind != defaults.geography_kind + ): + values["geography_id"] = chosen.geography_id + values["forecast_content_sha256"] = defaults.forecast_content_sha256 + values["scenario"] = chosen.scenario or defaults.scenario + return SPMSelection.model_validate(values).model_dump(mode="json") + except ValueError as exc: + raise SPMInputError("SPM_SETTINGS_INVALID", str(exc)) from exc + + +def validate_spm_result( + result: dict, selection: Any, *, expected_year: int | str | None = None +): + """Do not accept incomplete or mixed-method child/cached output.""" + if selection is None: + if ( + result.get("spm_config") is not None + or result.get("spm_provenance") is not None + ): + raise SPMInputError( + "SPM_CONFIGURATION_UNAVAILABLE", + "Unexpected SPM receipt for a historical result", + ) + return None + try: + chosen = SPMSelection.model_validate(selection) + result_selection = SPMSelection.model_validate(result.get("spm_config")) + required = { + "forecast_content_sha256", + "scenario", + "geography_kind", + "county_vintage", + } + if not required <= result_selection.model_fields_set: + raise ValueError("Result has no complete resolved SPM selection") + selection = {name: getattr(chosen, name) for name in SPMSelection.model_fields} + if { + name: getattr(result_selection, name) for name in SPMSelection.model_fields + } != selection: + raise ValueError("Result SPM selection differs from the request") + provenance = SPMComparisonProvenance.model_validate( + result.get("spm_provenance") + ) + for receipt in provenance.baseline + provenance.reform: + if ( + receipt.forecast_sha256 != selection["forecast_content_sha256"] + or receipt.scenario != selection["scenario"] + or receipt.geography_kind != selection["geography_kind"] + ): + raise ValueError("Result SPM provenance differs from the request") + if expected_year is not None and str(expected_year) not in receipt.years: + raise ValueError( + "Result SPM provenance does not cover the requested year" + ) + return provenance + except ValueError as exc: + raise SPMInputError("SPM_CONFIGURATION_UNAVAILABLE", str(exc)) from exc + + +def combine_spm_results( + results: list[dict], + selection: dict | None, + *, + expected_year: int | str | None = None, +) -> dict: + receipts = [ + validate_spm_result(result, selection, expected_year=expected_year) + for result in results + ] + if selection is None: + return {} + validated: list[SPMComparisonProvenance] = [] + for receipt in receipts: + if receipt is None: + raise SPMInputError( + "SPM_CONFIGURATION_UNAVAILABLE", "Missing canonical SPM receipt" + ) + validated.append(receipt) + combined = SPMComparisonProvenance( + baseline=[r for item in validated for r in item.baseline], + reform=[r for item in validated for r in item.reform], + ) + return {"spm_config": selection, "spm_provenance": combined.model_dump(mode="json")} diff --git a/libs/policyengine-simulation-contract/tests/test_gateway_models.py b/libs/policyengine-simulation-contract/tests/test_gateway_models.py index a6bd60dd3..75d90e918 100644 --- a/libs/policyengine-simulation-contract/tests/test_gateway_models.py +++ b/libs/policyengine-simulation-contract/tests/test_gateway_models.py @@ -416,6 +416,7 @@ def test_versions_response_has_named_maps(self): ) assert response.model_dump(mode="json") == { + "spm_capabilities": {}, "policyengine": {"latest": "4.10.0"}, "us": {"latest": "1.500.0"}, "uk": {"latest": "2.66.0"}, @@ -575,6 +576,7 @@ def test_budget_window_batch_submit_response_serializes_correctly(self): "model_version": "1.500.0", "policyengine_version": None, "data_version": None, + "spm": None, "dataset": "default", }, "run_id": "batch-run-123", @@ -628,5 +630,6 @@ def test_budget_window_batch_status_response_accepts_child_jobs_and_result(self) "job_id": "fc-2026", "status": "complete", "error": None, + "errors": None, } assert dumped["result"]["kind"] == "budgetWindow" diff --git a/projects/policyengine-apis-integ/tests/test_spm_generated_client.py b/projects/policyengine-apis-integ/tests/test_spm_generated_client.py new file mode 100644 index 000000000..31c9e9d49 --- /dev/null +++ b/projects/policyengine-apis-integ/tests/test_spm_generated_client.py @@ -0,0 +1,58 @@ +"""Hermetic wire checks against the actual generated public Python client.""" + +import json + +import pytest + +from policyengine_api_simulation_client.models import ( + SPMCapability, + SPMComparisonProvenance, + SPMSelection, + SimulationRequest, +) + + +def test_omitted_selection_options_stay_omitted_in_generated_requests(): + selection = SPMSelection(scenario="zero_real") + assert selection.to_dict() == {"scenario": "zero_real"} + request = SimulationRequest.from_dict({"country": "us", "spm": selection.to_dict()}) + assert json.loads(json.dumps(request.to_dict()))["spm"] == {"scenario": "zero_real"} + + +def test_explicit_null_date_survives_generated_request_roundtrip(): + selected = {"geography_kind": "national", "as_of": None} + request = SimulationRequest.from_dict({"country": "us", "spm": selected}) + assert request.to_dict()["spm"] == selected + + +def test_generated_capability_keeps_and_enforces_contract_version(): + capability = { + "contract_version": "canonical-spm-v1", + "defaults": { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + }, + } + assert SPMCapability.from_dict(capability).to_dict() == capability + with pytest.raises(ValueError, match="contract_version"): + SPMCapability.from_dict({**capability, "contract_version": "unknown-v2"}) + + +def test_generated_receipts_preserve_arbitrary_dated_metadata(): + receipt = { + "forecast_id": "test-only", + "forecast_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "runtime_versions": {"policyengine-us": "test-only", "optional": None}, + "years": {"2026": {"status": "forecast", "window": [2021, 2025]}}, + "geographies": [{"year": 2026, "county_assignment": None}], + "composition_method": "test-only", + "storage_method": "test-only", + } + provenance = {"baseline": [receipt], "reform": [receipt]} + assert SPMComparisonProvenance.from_dict(provenance).to_dict() == provenance diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index f3bbe511f..fefa907ba 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -24,6 +24,7 @@ PingResponse, ReadinessResponse, SimulationRequest, + SimulationErrorResponse, VersionMap, VersionsResponse, ) @@ -50,7 +51,6 @@ RequestIdentifiers, ) - logger = logging.getLogger(__name__) BACKEND_RESPONSE_HEADER = { "X-PolicyEngine-Simulation-Backend": "old_gateway", @@ -61,9 +61,13 @@ def _model_json(model: BaseModel) -> JsonObject: - return _json_object_adapter.validate_python( - model.model_dump(mode="json", by_alias=True, exclude_none=True) - ) + payload = model.model_dump(mode="json", by_alias=True, exclude_none=True) + selection = getattr(model, "spm", None) + if selection is not None: + # An explicit null information date clears a bundle cutoff. Preserve + # that distinction while leaving omitted request fields omitted. + payload["spm"] = selection.model_dump(mode="json", exclude_unset=True) + return _json_object_adapter.validate_python(payload) def _response(result: BackendResponse) -> Response: @@ -271,7 +275,10 @@ async def forward( response_model_exclude_none=True, responses={ 200: {"description": "Job submitted successfully"}, - 400: {"description": "Invalid request (unknown country/version)"}, + 400: { + "description": "Invalid request or SPM selection", + "model": SimulationErrorResponse, + }, }, dependencies=protected, ) @@ -299,7 +306,10 @@ async def submit_comparison( response_model_exclude_none=True, responses={ 200: {"description": "Budget-window batch submitted successfully"}, - 400: {"description": "Invalid request (unknown country/version/year)"}, + 400: { + "description": "Invalid request or SPM selection", + "model": SimulationErrorResponse, + }, }, dependencies=protected, ) @@ -332,6 +342,10 @@ async def submit_budget_window( responses={ 200: {"description": "Job complete", "model": JobStatusResponse}, 202: {"description": "Job still running"}, + 400: { + "description": "SPM input or configuration error", + "model": JobStatusResponse, + }, 404: {"description": "Job not found"}, 500: {"description": "Job failed"}, }, @@ -360,6 +374,10 @@ async def get_job(job_id: str, request: Request) -> Response: "model": BudgetWindowBatchStatusResponse, }, 202: {"description": "Batch submitted or running"}, + 400: { + "description": "SPM input or configuration error", + "model": BudgetWindowBatchStatusResponse, + }, 404: {"description": "Batch job not found"}, 500: {"description": "Batch failed"}, }, diff --git a/projects/policyengine-simulation-entry/tests/test_openapi.py b/projects/policyengine-simulation-entry/tests/test_openapi.py index 9b1c16afb..380bf0902 100644 --- a/projects/policyengine-simulation-entry/tests/test_openapi.py +++ b/projects/policyengine-simulation-entry/tests/test_openapi.py @@ -7,7 +7,6 @@ from policyengine_simulation_entry.app import create_app - type HttpMethod = Literal["get", "post", "put", "patch", "delete"] @@ -60,16 +59,56 @@ def operation_ids(spec: OpenAPIDocument) -> dict[tuple[str, str], str]: def normalized_compatibility_paths(spec: OpenAPIDocument) -> OpenAPIPathMap: - """Remove the two intentional Cloud Run-only OpenAPI additions.""" + """Remove intentional Cloud Run and additive SPM error documentation.""" paths = deepcopy(spec["paths"]) paths.pop("/ready", None) for operations in paths.values(): for method, operation in operations.items(): if method.lower() in {"get", "post", "put", "patch", "delete"}: operation.pop("security", None) + for path, description in { + "/simulate/economy/comparison": "Invalid request (unknown country/version)", + "/simulate/economy/budget-window": "Invalid request (unknown country/version/year)", + }.items(): + paths[path]["post"]["responses"]["400"] = {"description": description} + for path in ("/jobs/{job_id}", "/budget-window-jobs/{batch_job_id}"): + paths[path]["get"]["responses"].pop("400", None) return paths +def without_spm_extensions(schemas): + schemas = deepcopy(schemas) + for name in list(schemas): + if name.startswith("SPM") or name == "SimulationErrorResponse": + schemas.pop(name) + continue + for field in ( + "spm", + "spm_config", + "spm_provenance", + "errors", + "spm_capabilities", + ): + schemas[name].get("properties", {}).pop(field, None) + return schemas + + +def test_canonical_spm_extensions_are_public(): + spec = create_app().openapi() + schemas = spec["components"]["schemas"] + assert schemas["SPMSelection"]["additionalProperties"] is False + assert all( + "default" not in field + for field in schemas["SPMSelection"]["properties"].values() + ) + assert "spm" in schemas["SimulationRequest"]["properties"] + assert "spm" in schemas["BudgetWindowBatchRequest"]["properties"] + assert "spm_provenance" in schemas["SingleYearMacroOutput"]["properties"] + assert "spm_provenance" in schemas["BudgetWindowAnnualImpact"]["properties"] + for path in ("/jobs/{job_id}", "/budget-window-jobs/{batch_job_id}"): + assert "400" in spec["paths"][path]["get"]["responses"] + + def test_route_table_is_frozen(): assert methods(cast(OpenAPIDocument, create_app().openapi())) == EXPECTED_ROUTES @@ -110,7 +149,10 @@ def test_normalized_contract_matches_old_gateway(): ) == normalized_compatibility_paths(gateway_spec) cloud_run_schemas = deepcopy(cloud_run_spec["components"]["schemas"]) cloud_run_schemas.pop("ReadinessResponse") - assert cloud_run_schemas == gateway_spec["components"]["schemas"] + assert ( + without_spm_extensions(cloud_run_schemas) + == gateway_spec["components"]["schemas"] + ) gateway_operation_ids = operation_ids(gateway_spec) cloud_run_operation_ids = operation_ids(cloud_run_spec) assert { diff --git a/projects/policyengine-simulation-executor/fixtures/test_simulation_api_contracts.py b/projects/policyengine-simulation-executor/fixtures/test_simulation_api_contracts.py index 23804049d..f296f91dc 100644 --- a/projects/policyengine-simulation-executor/fixtures/test_simulation_api_contracts.py +++ b/projects/policyengine-simulation-executor/fixtures/test_simulation_api_contracts.py @@ -1,6 +1,8 @@ """Fixtures for simulation API contract tests.""" CURRENT_SINGLE_YEAR_MACRO_KEYS = { + "spm_config", + "spm_provenance", "model_version", "data_version", "budget", @@ -30,6 +32,8 @@ } CURRENT_SINGLE_YEAR_MACRO_RESULT = { + "spm_config": None, + "spm_provenance": None, "model_version": "1.715.2", "data_version": "1.115.5", "budget": { diff --git a/projects/policyengine-simulation-executor/src/modal/budget_window_batch.py b/projects/policyengine-simulation-executor/src/modal/budget_window_batch.py index a8f93c603..b1c04010c 100644 --- a/projects/policyengine-simulation-executor/src/modal/budget_window_batch.py +++ b/projects/policyengine-simulation-executor/src/modal/budget_window_batch.py @@ -13,6 +13,11 @@ def run_budget_window_batch_impl(params: dict[str, Any]) -> dict[str, Any]: + from policyengine_simulation_executor.spm import normalize_runtime_spm + + selection = normalize_runtime_spm(params) + if selection is not None: + params = {**params, "spm": selection} batch_job_id = modal.current_function_call_id() set_attribute("batch_job_id", batch_job_id) with segment(SegmentName.BUDGET_WINDOW_CONTEXT): diff --git a/projects/policyengine-simulation-executor/src/modal/budget_window_results.py b/projects/policyengine-simulation-executor/src/modal/budget_window_results.py index 2b9d7ca47..cedff79b8 100644 --- a/projects/policyengine-simulation-executor/src/modal/budget_window_results.py +++ b/projects/policyengine-simulation-executor/src/modal/budget_window_results.py @@ -3,6 +3,7 @@ from __future__ import annotations from decimal import Decimal +from policyengine_simulation_contract.spm import SPMSelection, validate_spm_result from typing import Any from policyengine_simulation_contract.gateway_models import ( @@ -36,7 +37,9 @@ def extract_annual_impact( *, simulation_year: str, child_result: dict[str, Any], + spm: Any = None, ) -> BudgetWindowAnnualImpact: + receipt = validate_spm_result(child_result, spm, expected_year=simulation_year) budget = child_result.get("budget", {}) if not isinstance(budget, dict): raise ValueError("Malformed budget-window child result: missing budget object") @@ -61,6 +64,8 @@ def extract_annual_impact( state_tax_revenue_impact = 0.0 return BudgetWindowAnnualImpact( + spm_config=SPMSelection.model_validate(spm) if spm is not None else None, + spm_provenance=receipt, year=simulation_year, taxRevenueImpact=tax_revenue_impact, federalTaxRevenueImpact=tax_revenue_impact - state_tax_revenue_impact, diff --git a/projects/policyengine-simulation-executor/src/modal/budget_window_scheduler.py b/projects/policyengine-simulation-executor/src/modal/budget_window_scheduler.py index 262c5a9b5..c54bee76c 100644 --- a/projects/policyengine-simulation-executor/src/modal/budget_window_scheduler.py +++ b/projects/policyengine-simulation-executor/src/modal/budget_window_scheduler.py @@ -30,6 +30,7 @@ put_batch_job_seed, put_batch_job_state, ) +from policyengine_simulation_contract.spm import spm_error_detail from policyengine_simulation_observability.errors import log_and_redact_exception from policyengine_simulation_observability.observability import SegmentName @@ -186,6 +187,15 @@ def poll_running_children_once(self) -> bool: except TimeoutError: continue except Exception as exc: + detail = spm_error_detail(exc) + if detail: + self.state.errors = [detail] + self.fail_batch_for_child_error( + simulation_year=simulation_year, error=detail.message + ) + self.state.child_jobs[simulation_year].errors = [detail] + put_batch_job_state(self.state) + return False redacted = log_and_redact_exception( exc, scope="budget_window_child_call", @@ -212,8 +222,18 @@ def poll_running_children_once(self) -> bool: annual_impact = extract_annual_impact( simulation_year=simulation_year, child_result=child_result, + spm=self.state.request_payload.get("spm"), ) except Exception as exc: + detail = spm_error_detail(exc) + if detail: + self.state.errors = [detail] + self.fail_batch_for_child_error( + simulation_year=simulation_year, error=detail.message + ) + self.state.child_jobs[simulation_year].errors = [detail] + put_batch_job_state(self.state) + return False redacted = log_and_redact_exception( exc, scope="budget_window_child_result_parsing", diff --git a/projects/policyengine-simulation-executor/src/modal/segmented_national.py b/projects/policyengine-simulation-executor/src/modal/segmented_national.py index 310c1e0da..e2c4cd583 100644 --- a/projects/policyengine-simulation-executor/src/modal/segmented_national.py +++ b/projects/policyengine-simulation-executor/src/modal/segmented_national.py @@ -33,6 +33,11 @@ from policyengine_simulation_executor.segmented_national_reduce import ( build_national_output, ) +from policyengine_simulation_contract.spm import ( + spm_error_detail, + SPMInputError, + combine_spm_results, +) from policyengine_simulation_observability.errors import log_and_redact_exception from policyengine_simulation_observability.observability import SegmentName from policyengine_simulation_observability.telemetry import split_internal_payload @@ -235,6 +240,10 @@ def _collect(self, handles: list[tuple[list[str], Any]]) -> list[dict]: poll_errors.pop(index, None) continue except Exception as exc: + detail = spm_error_detail(exc) + if detail: + self._cancel_all(handles) + raise SPMInputError(detail.code, detail.message) from exc # One transient poll-RPC blip must not kill the job; a # real child failure re-raises on the next probe too. attempts = poll_errors.get(index, 0) + 1 @@ -300,6 +309,13 @@ def _reduce(self, child_results: list[dict], *, country_module) -> dict[str, Any year=_parse_year(simulation_params), resolved_data_version=_requested_data_version(simulation_params), ) + output.update( + combine_spm_results( + child_results, + simulation_params.get("spm"), + expected_year=_parse_year(simulation_params), + ) + ) for key in ("model_version", "data_version"): if output.get(key): set_attribute(key, str(output[key])) @@ -315,6 +331,11 @@ def run_segmented_national_impl( def dispatch_run_simulation(params: dict[str, Any], *, app_name: str) -> dict[str, Any]: """The run_simulation entrypoint's routing: segmented national fan-out for eligible requests, the monolithic path for everything else.""" + from policyengine_simulation_executor.spm import normalize_runtime_spm + + selection = normalize_runtime_spm(params) + if selection is not None: + params = {**params, "spm": selection} if should_run_segmented_national(params): return run_segmented_national_impl(params, app_name=app_name) diff --git a/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py b/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py index 78027db9e..649a652a6 100644 --- a/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py +++ b/projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py @@ -5,7 +5,7 @@ import modal from packaging.version import InvalidVersion, Version -from typing import Iterable, TypedDict +from typing import Iterable, TypedDict, NotRequired POLICYENGINE_VERSION_DICT_NAME = "simulation-api-policyengine-versions" US_VERSION_DICT_NAME = "simulation-api-us-versions" @@ -38,6 +38,7 @@ class BundleManifestMetadata(TypedDict): policyengine_version: str us: CountryBundleMetadata uk: CountryBundleMetadata + spm: NotRequired[dict] class LatestVersions(TypedDict): @@ -104,12 +105,18 @@ def build_bundle_manifest_metadata( app_name: str, policyengine_version: str, ) -> BundleManifestMetadata: - return { + from policyengine_simulation_executor.spm import runtime_spm_capability + + capability = runtime_spm_capability() + metadata: BundleManifestMetadata = { "app_name": app_name, "policyengine_version": policyengine_version, "us": _country_bundle_metadata("us"), "uk": _country_bundle_metadata("uk"), } + if capability is not None: + metadata["spm"] = capability.model_dump(mode="json") + return metadata def _empty_routing_state() -> RoutingState: diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py index cb83156e5..5cdcffba9 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py @@ -74,6 +74,7 @@ def dataset_key( data_build_fingerprint: Optional[str], model_version: str, policyengine_version: str, + spm: Optional[dict] = None, ) -> str: """Digest identifying one single-year dataset artifact. @@ -86,7 +87,8 @@ def dataset_key( """ return canonical_digest( { - "schema": DATASET_KEY_SCHEMA, + "schema": DATASET_KEY_SCHEMA if spm is None else "ds2-spm", + **({"spm": spm} if spm is not None else {}), "country": country.lower(), "dataset": dataset, "year": int(year), @@ -108,6 +110,7 @@ def baseline_key( dataset_digest: str, model_version: str, policyengine_version: str, + spm: Optional[dict] = None, policy: str = CURRENT_LAW_POLICY, ) -> str: """Digest identifying one precomputed baseline simulation artifact. @@ -119,7 +122,8 @@ def baseline_key( """ return canonical_digest( { - "schema": BASELINE_KEY_SCHEMA, + "schema": BASELINE_KEY_SCHEMA if spm is None else "bl2-spm", + **({"spm": spm} if spm is not None else {}), "country": country.lower(), "region": region, "scope_key": scope_key, @@ -206,11 +210,13 @@ class DatasetArtifactIdentity: data_build_fingerprint: Optional[str] model_version: str policyengine_version: str + spm: Optional[dict] = None @property def digest(self) -> str: return dataset_key( country=self.country, + spm=self.spm, dataset=self.dataset, year=self.year, data_version=self.data_version, @@ -238,12 +244,14 @@ class BaselineArtifactIdentity: region: str scope_key: Optional[str] dataset: DatasetArtifactIdentity + spm: Optional[dict] = None @property def digest(self) -> str: return baseline_key( country=self.country, region=self.region, + spm=self.spm, scope_key=self.scope_key, dataset_digest=self.dataset.digest, model_version=self.dataset.model_version, @@ -254,9 +262,15 @@ def digest(self) -> str: def simulation_id(self) -> str: return baseline_simulation_id(self.digest) + @property + def storage_id(self) -> str: + if self.spm is None: + return self.simulation_id + return f"{self.simulation_id}-spm-{canonical_digest(self.spm)}" + @property def store_path(self) -> str: - return baseline_artifact_path(self.country, self.digest, self.simulation_id) + return baseline_artifact_path(self.country, self.digest, self.storage_id) def _receipt_source_sha256(country: str, data_version: str) -> Optional[str]: @@ -310,7 +324,11 @@ def collect_dataset_identity(country: str, year: int) -> DatasetArtifactIdentity stem = dataset_logical_name( resolve_dataset_reference(bundle.country, bundle.default_dataset) ) + from policyengine_simulation_executor.spm import normalize_runtime_spm + + selection = normalize_runtime_spm({"country": country, "time_period": year}) return DatasetArtifactIdentity( + spm=selection, country=bundle.country, dataset=bundle.default_dataset, stem=stem, @@ -330,8 +348,15 @@ def collect_baseline_identity( *, region: str, scope_key: Optional[str], + spm: Optional[dict] = None, ) -> BaselineArtifactIdentity: + from policyengine_simulation_executor.spm import normalize_runtime_spm + + selection = normalize_runtime_spm( + {"country": country, "time_period": year, "spm": spm} + ) return BaselineArtifactIdentity( + spm=selection, country=country.lower(), region=region, scope_key=scope_key, diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py index dadb9672e..00bd588f9 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py @@ -102,7 +102,11 @@ def qualifying_baseline_identity( return None return artifact_keys.collect_baseline_identity( - country, year, region=region, scope_key=scope_key + country, + year, + region=region, + scope_key=scope_key, + **({"spm": params["spm"]} if params.get("spm") is not None else {}), ) @@ -130,7 +134,11 @@ def deterministic_baseline_id( scoping_strategy=scoping_strategy, year=year, ) - except Exception: + except Exception as exc: + from policyengine_simulation_contract.spm import spm_error_detail + + if spm_error_detail(exc): + raise logger.warning( "Could not collect baseline artifact identity for %s; " "using a random simulation id", @@ -177,6 +185,13 @@ def ensure(self) -> None: self._record_outcome(OUTCOME_MISS) return + from policyengine_simulation_executor.spm import simulation_spm_result + + selection = getattr(self, "spm_config", None) + if selection is not None: + simulation_spm_result( + self, self, selection, expected_year=self.dataset.year + ) missing = self._missing_output_columns() if not missing: self._record_outcome(OUTCOME_HIT) @@ -197,8 +212,11 @@ def ensure(self) -> None: # existing key and exposes no remove, so evict directly first. from policyengine.core.simulation import _cache - _cache._cache.pop(self.id, None) - _cache.add(self.id, self) + _cache._cache.pop(getattr(self, "storage_id", self.id), None) + _cache.add( + getattr(self, "storage_id", self.id), + self.model_copy(deep=False) if selection is not None else self, + ) def _missing_output_columns(self) -> list[tuple[str, str]]: data = getattr(self.output_dataset, "data", None) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/compat_models.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/compat_models.py index 64e880015..12d545d98 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/compat_models.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/compat_models.py @@ -5,6 +5,7 @@ from typing import Any, Optional from pydantic import BaseModel, ConfigDict +from policyengine_simulation_contract.spm import SPMSelection from policyengine_simulation_executor.simulation_macro_output import ( SingleYearMacroOutput, @@ -15,6 +16,7 @@ class SimulationOptions(BaseModel): """Legacy request schema name kept for generated clients.""" country: str + spm: Optional[SPMSelection] = None scope: Optional[str] = None data: Optional[str] = None time_period: Optional[str | int] = None diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py index 1232ff54d..3370a7261 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py @@ -152,7 +152,7 @@ def build_manifest(plan: PrecomputePlan) -> ArtifactManifest: ManifestArtifact( type="baseline", path=entry.path, - filename=baseline_artifact_filename(entry.simulation_id), + filename=entry.path.rsplit("/", maxsplit=1)[-1], year=entry.year, digest=entry.digest, ) @@ -341,7 +341,9 @@ def compute_baseline_impl( baseline.ensure() compute_seconds = time.monotonic() - started - artifact_file = data_folder / baseline_artifact_filename(baseline.id) + artifact_file = data_folder / baseline_artifact_filename( + getattr(baseline, "storage_id", baseline.id) + ) if not artifact_file.exists(): raise RuntimeError(f"ensure() left no artifact at {artifact_file}") uploaded = store.upload_file(expected.path, artifact_file) @@ -373,7 +375,9 @@ def verify_determinism_impl( from policyengine_simulation_executor.baseline_artifacts import OUTCOME_HIT store, data_folder, baseline = _prepare_cohort_baseline(bucket, expected) - artifact_file = data_folder / baseline_artifact_filename(baseline.id) + artifact_file = data_folder / baseline_artifact_filename( + getattr(baseline, "storage_id", baseline.id) + ) if not artifact_file.exists(): store.download_file(expected.path, artifact_file) baseline.ensure() @@ -383,7 +387,9 @@ def verify_determinism_impl( f"(outcome={baseline.artifact_outcome})" ) + selection = getattr(baseline, "spm_config", None) fresh = Simulation( + **({"spm": selection} if selection is not None else {}), dataset=baseline.dataset, tax_benefit_model_version=baseline.tax_benefit_model_version, policy=None, diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation.py index a8b80bd3c..309f44315 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation.py @@ -1,6 +1,8 @@ import logging from fastapi import APIRouter +from fastapi.responses import JSONResponse +from policyengine_simulation_contract.spm import spm_error_detail from policyengine_simulation_executor.simulation_runtime import run_simulation_impl from policyengine_simulation_executor.compat_models import ( @@ -15,11 +17,28 @@ def create_router(): router = APIRouter() @router.post("/simulate/economy/comparison", response_model=EconomyComparison) - async def simulate(parameters: SimulationOptions) -> EconomyComparison: + async def simulate( + parameters: SimulationOptions, + ) -> EconomyComparison | JSONResponse: logger.info("Calculating comparison") - result = run_simulation_impl( - parameters.model_dump(mode="json", exclude_none=True) - ) + try: + payload = parameters.model_dump(mode="json", exclude_none=True) + if parameters.spm is not None: + payload["spm"] = parameters.spm.model_dump(mode="json") + result = run_simulation_impl(payload) + except ValueError as exc: + detail = spm_error_detail(exc) + if detail: + return JSONResponse( + status_code=400, + content={ + "status": "failed", + "result": None, + "error": detail.message, + "errors": [detail.model_dump()], + }, + ) + raise logger.info("Comparison complete") return EconomyComparison.model_validate(result) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py index fb804d9f2..c031b2ee7 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py @@ -98,6 +98,10 @@ def _poverty_module_function(name: str): def _try_compute_output(label: str, fn, *args, **kwargs): try: return fn(*args, **kwargs) - except Exception: + except Exception as exc: + from policyengine_simulation_contract.spm import spm_error_detail + + if spm_error_detail(exc) is not None: + raise logger.warning("Unable to calculate %s", label, exc_info=True) return None diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py index d75582f4a..5a90b5322 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py @@ -136,7 +136,20 @@ def run_simulation_impl(params: dict) -> dict: # Set up GCP credentials if needed. The credentials temp file is # cleaned up on exit so we never leave signed JSON material on disk. with setup_gcp_credentials(): - return _run_simulation_impl_core(params) + try: + return _run_simulation_impl_core(params) + except ValueError as exc: + # Gateway images intentionally do not install country packages. + # Transport the public error through the shared contract instead. + from policyengine_simulation_contract.spm import ( + SPMInputError, + spm_error_detail, + ) + + detail = spm_error_detail(exc) + if detail is not None: + raise SPMInputError(detail.code, detail.message) from None + raise def _parse_year(params: dict[str, Any]) -> int: @@ -490,6 +503,11 @@ def _build_simulation( deterministic_baseline_id, ) + from policyengine_simulation_executor.spm import normalize_runtime_spm + + selection = normalize_runtime_spm(params) + params = {**params, **({"spm": selection} if selection is not None else {})} + spm_kwargs = {"spm": selection} if selection is not None else {} country = params.get("country", "us") country_module = _country_module(country) simulation_id = deterministic_baseline_id( @@ -505,6 +523,7 @@ def _build_simulation( # when one is baked beside the dataset, and the subclass validates # the load (falling back to run()) — see baseline_artifacts. return ArtifactBaselineSimulation( + **spm_kwargs, id=simulation_id, dataset=dataset, tax_benefit_model_version=country_module.model, @@ -512,6 +531,7 @@ def _build_simulation( scoping_strategy=scoping_strategy, ) return Simulation( + **spm_kwargs, dataset=dataset, tax_benefit_model_version=country_module.model, policy=policy, @@ -523,6 +543,14 @@ def _run_simulation_impl_core(params: dict) -> dict: with segment(SegmentName.REQUEST_PARSE): simulation_params, telemetry, metadata = split_internal_payload(params) metadata = metadata or {} + from policyengine_simulation_executor.spm import ( + normalize_runtime_spm, + simulation_spm_result, + ) + + selection = normalize_runtime_spm(simulation_params) + if selection is not None: + simulation_params["spm"] = selection logger.info( "Starting simulation for country=%s run_id=%s process_id=%s", @@ -590,6 +618,11 @@ def _run_simulation_impl_core(params: dict) -> dict: resolved_region_code=region_resolution.code, ) output = builder.serialize() + output.update( + simulation_spm_result( + baseline, reform, selection, expected_year=_parse_year(simulation_params) + ) + ) # ensure() has run inside the builder by now, so the artifact outcome # (hit / incomplete / miss) is known for deterministic-id baselines. artifact_outcome = getattr(baseline, "artifact_outcome", None) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py new file mode 100644 index 000000000..b0193555a --- /dev/null +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py @@ -0,0 +1,142 @@ +"""Certified SPM runtime selection, independent of caller-supplied metadata.""" + +from functools import lru_cache + +from policyengine_simulation_contract.spm import ( + SPMCapability, + SPMInputError, + SPMProvenance, + combine_spm_results, + resolve_spm_selection, + spm_error_detail, +) + + +@lru_cache(maxsize=4) +def _forecast(expected_sha256): + from spm_calculator import load_forecast + + return load_forecast(expected_sha256=expected_sha256) + + +def runtime_spm_capability(): + from policyengine.bundle import get_current_bundle + from policyengine.core import Simulation + + try: + bundle = get_current_bundle() + configured = bundle.get("measurements", {}).get("spm") + except (ImportError, ValueError, TypeError, AttributeError) as exc: + raise SPMInputError( + "SPM_CONFIGURATION_UNAVAILABLE", + "The installed SPM bundle configuration is unavailable", + ) from exc + if configured is None: + return None + if "spm" not in Simulation.model_fields or not hasattr( + Simulation, "spm_provenance" + ): + raise SPMInputError( + "SPM_CONFIGURATION_UNAVAILABLE", + "The installed wrapper does not support canonical SPM", + ) + try: + from policyengine_us import Microsimulation + + if not ( + hasattr(Microsimulation, "spm_config") + and callable(getattr(Microsimulation, "spm_provenance", None)) + ): + raise ValueError("The installed US model does not support canonical SPM") + capability = SPMCapability(defaults=configured) + forecast = _forecast(capability.defaults.forecast_content_sha256) + forecast.entry( + forecast.years[0], + scenario=capability.defaults.scenario, + as_of=capability.defaults.as_of, + ) + return capability + except (ImportError, ValueError, TypeError, OSError) as exc: + raise SPMInputError("SPM_CONFIGURATION_UNAVAILABLE", str(exc)) from exc + + +def normalize_runtime_spm(params): + """Resolve before dataset loading, artifact lookup, or child submission.""" + from policyengine_simulation_executor.release_bundle import ( + get_country_release_bundle, + ) + + country = params.get("country", "us").lower() + bundle = get_country_release_bundle(country) + selection = resolve_spm_selection( + country, + params.get("spm"), + capability=runtime_spm_capability() if country == "us" else None, + policyengine_version=bundle.policyengine_version, + model_version=bundle.model_version, + ) + if selection is not None: + try: + forecast = _forecast(selection["forecast_content_sha256"]) + if selection["county_vintage"] != "2020": + raise ValueError("Unsupported county vintage: use 2020") + from policyengine_simulation_executor.simulation_runtime import _parse_year + + start = int(params.get("start_year") or _parse_year(params)) + for year in range(start, start + int(params.get("window_size", 1))): + forecast.entry( + year, scenario=selection["scenario"], as_of=selection["as_of"] + ) + if selection["geography_kind"] == "metro": + try: + forecast.geography_factor( + year, + "renter", + kind="metro", + geoid=selection["geography_id"], + scenario=selection["scenario"], + as_of=selection["as_of"], + ) + except ValueError as exc: + raise SPMInputError( + "SPM_GEOGRAPHY_UNAVAILABLE", str(exc) + ) from None + except ValueError as exc: + detail = spm_error_detail(exc) + if detail: + raise SPMInputError(detail.code, detail.message) from exc + raise SPMInputError("SPM_SETTINGS_INVALID", str(exc)) from exc + return selection + + +def simulation_spm_result(baseline, reform, selection, *, expected_year=None): + if selection is None: + return {} + receipts = [] + for simulation in (baseline, reform): + if simulation.spm_config != selection: + raise SPMInputError( + "SPM_CONFIGURATION_UNAVAILABLE", + "Simulation ignored the requested SPM selection", + ) + try: + receipts.append( + SPMProvenance.model_validate(simulation.spm_provenance()).model_dump( + mode="json" + ) + ) + except ValueError as exc: + raise SPMInputError( + "SPM_CONFIGURATION_UNAVAILABLE", + "Simulation has no valid SPM calculation receipt", + ) from exc + return combine_spm_results( + [ + { + "spm_config": selection, + "spm_provenance": {"baseline": [receipts[0]], "reform": [receipts[1]]}, + } + ], + selection, + expected_year=expected_year, + ) diff --git a/projects/policyengine-simulation-executor/tests/test_canonical_spm.py b/projects/policyengine-simulation-executor/tests/test_canonical_spm.py new file mode 100644 index 000000000..d9db1105d --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_canonical_spm.py @@ -0,0 +1,626 @@ +"""Bounded canonical worker contracts; no population simulation or Modal calls.""" + +import json +import pickle +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from policyengine_simulation_contract.gateway_models import ( + SimulationRequest, + BudgetWindowBatchRequest, +) +from policyengine_simulation_contract.spm import ( + SPMCapability, + SPMSelection, + SPMInputError, + resolve_spm_selection, + combine_spm_results, +) +from policyengine_simulation_executor import artifact_keys, simulation_runtime +from policyengine_simulation_executor.simulation import create_router +from src.modal.budget_window_results import ( + extract_annual_impact, + build_budget_window_result, +) +from src.modal.fanout import build_child_payload + +SELECTION = SPMSelection( + forecast_content_sha256="a" * 64, + scenario="ce_trend", + geography_kind="national", + geography_id=None, + county_vintage="2020", + as_of=None, +).model_dump() +CAPABILITY = SPMCapability(defaults=SELECTION).model_dump() + + +def result(selection=SELECTION, year="2026"): + receipt = dict( + forecast_id="test-only", + forecast_sha256=selection["forecast_content_sha256"], + scenario=selection["scenario"], + geography_kind=selection["geography_kind"], + runtime_versions={"policyengine-us": "test-only"}, + years={year: {"status": "forecast"}}, + geographies=[], + composition_method="classified-inputs", + storage_method="formula", + ) + return { + "spm_config": deepcopy(selection), + "spm_provenance": {"baseline": [receipt], "reform": [deepcopy(receipt)]}, + "budget": { + "tax_revenue_impact": 1, + "benefit_spending_impact": 2, + "budgetary_impact": -1, + }, + } + + +@pytest.mark.parametrize( + "model,extra", + [ + (SimulationRequest, {}), + ( + BudgetWindowBatchRequest, + {"region": "us", "start_year": "2026", "window_size": 2}, + ), + ], +) +def test_public_request_roundtrip(model, extra): + parsed = model(country="us", spm={"geography_kind": "national"}, **extra) + assert json.loads(parsed.model_dump_json())["spm"]["geography_kind"] == "national" + with pytest.raises(ValueError): + model(country="us", spm={"provider_path": "x"}, **extra) + with pytest.raises(ValueError): + model( + country="us", + spm={"geography_kind": "national", "geography_id": "12345"}, + **extra, + ) + + +def test_unknown_bundle_fails_closed_and_historical_uk_remain(): + for country, version, model in [ + ("us", "5.2.0", "1.764.6"), + ("us", "5.3.0", "1.764.6"), + ("uk", "future-test", "test"), + ]: + assert ( + resolve_spm_selection( + country, + None, + capability=None, + policyengine_version=version, + model_version=model, + ) + is None + ) + for selection in (None, {}, {"geography_kind": "national"}): + with pytest.raises(SPMInputError, match="no certified"): + resolve_spm_selection( + "us", + selection, + capability=None, + policyengine_version="future-test", + model_version="test", + ) + assert ( + resolve_spm_selection( + "us", + {}, + capability=CAPABILITY, + policyengine_version="test-only", + model_version="test", + ) + == SELECTION + ) + with pytest.raises(SPMInputError, match="hash"): + resolve_spm_selection( + "us", + {"forecast_content_sha256": "b" * 64}, + capability=CAPABILITY, + policyengine_version="test-only", + model_version="test", + ) + + +@pytest.mark.parametrize( + "kind,area", [("county", None), ("national", None), ("metro", "35620")] +) +def test_partial_selection_survives_ordinary_nested_json(kind, area): + capability = { + "contract_version": "canonical-spm-v1", + "defaults": { + **SELECTION, + "geography_kind": kind, + "geography_id": area, + "county_vintage": "2010", + "as_of": "2026-01-01", + }, + } + partial = {"scenario": "zero_real"} + request = SimulationRequest(country="us", spm=partial) + wire = json.loads(request.model_dump_json()) + assert wire["spm"] == partial + resolved = resolve_spm_selection( + "us", + wire["spm"], + capability=capability, + policyengine_version="test", + model_version="test", + ) + assert resolved == {**capability["defaults"], **partial} + assert SPMSelection.model_validate(resolved).model_dump() == resolved + explicit = {"geography_kind": "county", "county_vintage": "2020", "as_of": None} + assert resolve_spm_selection( + "us", + json.loads(SPMSelection(**explicit).model_dump_json()), + capability=capability, + policyengine_version="test", + model_version="test", + ) == {**capability["defaults"], **explicit, "geography_id": None} + + +def test_historical_allowance_does_not_certify_other_models(): + for version, model in [("5.3.0", "1.824.7"), ("5.3.1", "1.764.6")]: + with pytest.raises(SPMInputError, match="no certified"): + resolve_spm_selection( + "us", + None, + capability=None, + policyengine_version=version, + model_version=model, + ) + + +def test_annual_child_and_segment_receipts_must_cover_requested_year(): + wrong_year = result(year="2025") + with pytest.raises(SPMInputError, match="requested year"): + extract_annual_impact( + simulation_year="2026", child_result=wrong_year, spm=SELECTION + ) + with pytest.raises(SPMInputError, match="requested year"): + combine_spm_results([result(), wrong_year], SELECTION, expected_year=2026) + + +def test_result_transport_may_omit_nulls_but_not_resolved_options(): + transported = result() + transported["spm_config"] = { + key: value + for key, value in transported["spm_config"].items() + if value is not None + } + assert combine_spm_results([transported], SELECTION)["spm_config"] == SELECTION + for field in ( + "forecast_content_sha256", + "scenario", + "geography_kind", + "county_vintage", + ): + malformed = deepcopy(transported) + del malformed["spm_config"][field] + with pytest.raises(SPMInputError, match="complete resolved"): + combine_spm_results([malformed], SELECTION) + + +def test_sync_compatibility_endpoint_preserves_explicit_null(monkeypatch): + import policyengine_simulation_executor.simulation as simulation + + captured = [] + + def capture(params): + captured.append(params) + raise SPMInputError("SPM_GEOGRAPHY_REQUIRED", "intentional test stop") + + monkeypatch.setattr(simulation, "run_simulation_impl", capture) + app = FastAPI() + app.include_router(create_router()) + response = TestClient(app).post( + "/simulate/economy/comparison", + json={"country": "us", "spm": {"as_of": None}}, + ) + assert response.status_code == 400 + assert captured[0]["spm"] == {"as_of": None} + + +def test_year_alias_is_validated_before_dataset_loading(monkeypatch): + from policyengine_simulation_executor import release_bundle, spm + + monkeypatch.setattr( + release_bundle, + "get_country_release_bundle", + lambda country: SimpleNamespace( + policyengine_version="test", model_version="test" + ), + ) + monkeypatch.setattr(spm, "runtime_spm_capability", lambda: CAPABILITY) + entry = Mock(side_effect=ValueError("2040 is unavailable")) + monkeypatch.setattr(spm, "_forecast", lambda sha: SimpleNamespace(entry=entry)) + with pytest.raises(SPMInputError, match="2040 is unavailable"): + spm.normalize_runtime_spm({"country": "us", "year": "2040", "spm": SELECTION}) + assert entry.call_args.args == (2040,) + + +@pytest.mark.parametrize( + "code", + ["SPM_GEOGRAPHY_REQUIRED", "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED"], +) +def test_actual_http_formula_error_contract(monkeypatch, code): + error = SPMInputError(code, "Explicit input is required") + assert pickle.loads(pickle.dumps(error)).to_dict() == error.to_dict() + monkeypatch.setattr( + "policyengine_simulation_executor.simulation.run_simulation_impl", + Mock(side_effect=error), + ) + app = FastAPI() + app.include_router(create_router()) + response = TestClient(app).post( + "/simulate/economy/comparison", + json={"country": "us", "spm": {"geography_kind": "national"}}, + ) + assert response.status_code == 400 + assert response.json()["errors"] == [error.to_dict()] + assert response.json()["result"] is None + + +@pytest.mark.parametrize( + "key,value", + [ + ("forecast_content_sha256", "b" * 64), + ("scenario", "zero_real"), + ("geography_kind", "county"), + ("geography_id", "12345"), + ("county_vintage", "2010"), + ("as_of", "2025-01-01"), + ], +) +def test_each_setting_rotates_artifact_identity(key, value): + common = dict( + country="us", + region="national", + scope_key=None, + dataset_digest="d" * 64, + model_version="test", + policyengine_version="test", + ) + changed = {**SELECTION, key: value} + assert artifact_keys.baseline_key( + **common, spm=SELECTION + ) != artifact_keys.baseline_key(**common, spm=changed) + ds = dict( + country="us", + dataset="test", + year=2026, + data_version="test", + data_artifact_revision="test", + source_sha256=None, + data_build_fingerprint=None, + model_version="test", + policyengine_version="test", + ) + assert artifact_keys.dataset_key(**ds, spm=SELECTION) != artifact_keys.dataset_key( + **ds, spm=changed + ) + + +def test_fanout_and_budget_window_preserve_settings_and_receipts(): + child = build_child_payload( + {"country": "us", "spm": SELECTION, "window_size": 2}, + strip_fields={"window_size"}, + overrides={"time_period": "2027"}, + ) + assert child["spm"] == SELECTION + rows = [ + extract_annual_impact( + simulation_year=year, child_result=result(year=year), spm=SELECTION + ) + for year in ("2026", "2027") + ] + window = build_budget_window_result( + start_year="2026", window_size=2, annual_impacts=rows + ) + dumped = json.loads(window.model_dump_json()) + assert dumped["annualImpacts"][1]["spm_provenance"]["baseline"][0]["years"] == { + "2027": {"status": "forecast"} + } + merged = combine_spm_results([result(), result()], SELECTION) + assert len(merged["spm_provenance"]["baseline"]) == 2 + changed = result({**SELECTION, "scenario": "zero_real"}) + for outputs in ([result(), changed], [result(), {"budget": {}}]): + with pytest.raises(SPMInputError): + combine_spm_results(outputs, SELECTION) + + +def test_worker_build_passes_selection_to_baseline_and_reform(monkeypatch): + import policyengine.core + from policyengine_simulation_executor import baseline_artifacts + import policyengine_simulation_executor.spm as spm + + calls = [] + monkeypatch.setattr(spm, "normalize_runtime_spm", lambda params: SELECTION) + monkeypatch.setattr( + simulation_runtime, "_country_module", lambda _: SimpleNamespace(model="test") + ) + monkeypatch.setattr( + baseline_artifacts, + "deterministic_baseline_id", + lambda *a, **k: "baseline-id" if k["policy"] is None else None, + ) + monkeypatch.setattr( + baseline_artifacts, "ArtifactBaselineSimulation", lambda **k: calls.append(k) + ) + monkeypatch.setattr(policyengine.core, "Simulation", lambda **k: calls.append(k)) + for policy in (None, {"test": 123}): + simulation_runtime._build_simulation( + {"country": "us", "spm": SELECTION}, + dataset="tiny", + policy=policy, + region_code="us", + ) + assert len(calls) == 2 + assert all(call["spm"] == SELECTION for call in calls) + assert calls[0]["id"] == "baseline-id" + + +def test_identity_errors_do_not_degrade_to_an_unidentified_baseline(monkeypatch): + from policyengine_simulation_executor import baseline_artifacts + + monkeypatch.setattr( + baseline_artifacts, + "qualifying_baseline_identity", + Mock(side_effect=SPMInputError("SPM_CONFIGURATION_UNAVAILABLE", "Uncertified")), + ) + with pytest.raises(SPMInputError): + baseline_artifacts.deterministic_baseline_id( + {}, + country="us", + policy=None, + region_code="us", + scoping_strategy=None, + year=2026, + ) + + +@pytest.mark.parametrize( + "code", + ["SPM_GEOGRAPHY_REQUIRED", "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED"], +) +def test_gateway_poll_returns_structured_400(monkeypatch, code): + from policyengine_simulation_gateway import endpoints + from policyengine_simulation_gateway.testing import create_gateway_app + + monkeypatch.setattr(endpoints, "_job_metadata_store", lambda: {"job": {}}) + monkeypatch.setattr( + endpoints, + "modal", + SimpleNamespace( + FunctionCall=SimpleNamespace( + from_id=lambda _: SimpleNamespace( + get=Mock( + side_effect=SPMInputError(code, "Explicit input is required") + ) + ) + ) + ), + ) + response = TestClient(create_gateway_app()).get("/jobs/job") + assert response.status_code == 400 + assert response.json()["errors"] == [ + {"code": code, "message": "Explicit input is required"} + ] + assert response.json()["result"] is None + + +def test_gateway_submission_uses_registry_capability_before_spawn(monkeypatch): + from policyengine_simulation_gateway import endpoints + from policyengine_simulation_gateway.testing import create_gateway_app + from policyengine_simulation_contract.gateway_models import PolicyEngineBundle + + route = SimpleNamespace( + app_name="test-app", + response_version="test-only", + policyengine_version="test-only", + ) + monkeypatch.setattr(endpoints, "resolve_route", lambda *args: route) + bundle = PolicyEngineBundle( + model_version="test-only", policyengine_version="test-only", spm=CAPABILITY + ) + monkeypatch.setattr(endpoints, "_build_policyengine_bundle", lambda *args: bundle) + spawn = Mock(return_value=SimpleNamespace(object_id="job")) + monkeypatch.setattr( + endpoints, + "modal", + SimpleNamespace( + Function=SimpleNamespace( + from_name=lambda *args: SimpleNamespace(spawn=spawn) + ) + ), + ) + monkeypatch.setattr(endpoints, "_job_metadata_store", lambda: {}) + response = TestClient(create_gateway_app()).post( + "/simulate/economy/comparison", + json={"country": "us", "spm": {"geography_kind": "national"}}, + ) + assert response.status_code == 200 + assert spawn.call_args.args[0]["spm"] == SELECTION + assert ( + SPMCapability.model_validate( + response.json()["policyengine_bundle"]["spm"] + ).model_dump() + == CAPABILITY + ) + bundle.spm = None + spawn.reset_mock() + response = TestClient(create_gateway_app()).post( + "/simulate/economy/comparison", json={"country": "us"} + ) + assert response.status_code == 400 + spawn.assert_not_called() + + +def test_segmented_child_input_error_is_not_redacted_or_retried(monkeypatch): + from src.modal.segmented_national import SegmentedNationalRunner + + child = SimpleNamespace( + get=Mock( + side_effect=SPMInputError("SPM_GEOGRAPHY_REQUIRED", "County required") + ), + cancel=Mock(), + ) + runner = object.__new__(SegmentedNationalRunner) + runner.poll_interval_initial_seconds = 0.001 + runner.poll_interval_max_seconds = 0.001 + with pytest.raises(SPMInputError) as error: + runner._collect([(["state/ca"], child)]) + assert error.value.code == "SPM_GEOGRAPHY_REQUIRED" + child.get.assert_called_once() + child.cancel.assert_called_once() + + +def test_budget_window_state_keeps_typed_failure_on_replay(): + from policyengine_simulation_contract.gateway_models import BudgetWindowBatchState + from policyengine_simulation_contract.budget_window_state import ( + build_batch_status_response, + ) + from policyengine_simulation_gateway.responses import batch_status_response + + state = BudgetWindowBatchState( + batch_job_id="job", + status="failed", + country="us", + region="us", + version="test", + resolved_app_name="app", + policyengine_bundle={"model_version": "test"}, + start_year="2026", + window_size=2, + max_parallel=2, + created_at="test", + updated_at="test", + error="County required", + errors=[{"code": "SPM_GEOGRAPHY_REQUIRED", "message": "County required"}], + ) + replay = BudgetWindowBatchState.model_validate_json(state.model_dump_json()) + response = batch_status_response(build_batch_status_response(replay)) + assert response.status_code == 400 + assert json.loads(response.body)["errors"][0]["code"] == "SPM_GEOGRAPHY_REQUIRED" + + +@pytest.mark.parametrize( + "code", + ["SPM_GEOGRAPHY_REQUIRED", "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED"], +) +def test_country_error_is_transportable_without_country_package(monkeypatch, code): + from contextlib import nullcontext + + class CountryError(ValueError): + def __init__(self, code, message): + self.code = code + super().__init__(message) + + monkeypatch.setattr(simulation_runtime, "setup_gcp_credentials", nullcontext) + monkeypatch.setattr( + simulation_runtime, + "_run_simulation_impl_core", + Mock(side_effect=CountryError(code, "Observed input required")), + ) + with pytest.raises(SPMInputError) as caught: + simulation_runtime.run_simulation_impl({"country": "us"}) + transported = pickle.loads(pickle.dumps(caught.value)) + assert transported.to_dict() == {"code": code, "message": "Observed input required"} + + +@pytest.mark.parametrize( + "code", + ["SPM_GEOGRAPHY_REQUIRED", "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED"], +) +def test_optional_analysis_does_not_swallow_spm_input_errors(code): + from policyengine_simulation_executor.simulation_output_common import ( + _try_compute_output, + ) + + with pytest.raises(SPMInputError): + _try_compute_output( + "optional impact", + Mock(side_effect=SPMInputError(code, "Explicit input required")), + ) + assert ( + _try_compute_output( + "optional impact", Mock(side_effect=RuntimeError("legacy optional output")) + ) + is None + ) + + +def test_explicit_null_as_of_survives_entrypoint_and_budget_parent(): + from policyengine_simulation_entry.app import _model_json + from policyengine_simulation_gateway.endpoints import ( + _resolve_request_spm, + _build_budget_window_parent_payload, + ) + from policyengine_simulation_contract.gateway_models import PolicyEngineBundle + + request = BudgetWindowBatchRequest( + country="us", + region="us", + start_year="2026", + window_size=2, + spm={"geography_kind": "national", "as_of": None}, + ) + entry_payload = _model_json(request) + assert "as_of" in entry_payload["spm"] + assert "forecast_content_sha256" not in entry_payload["spm"] + request = BudgetWindowBatchRequest.model_validate(entry_payload) + bundle = PolicyEngineBundle( + model_version="test", + policyengine_version="test", + spm={ + "contract_version": "canonical-spm-v1", + "defaults": {**SELECTION, "as_of": "2026-01-01"}, + }, + ) + selection = _resolve_request_spm(request, bundle) + assert selection["as_of"] is None + parent = _build_budget_window_parent_payload( + request, resolved_version="test", resolved_app_name="test", bundle=bundle + ) + assert parent["spm"] == selection + + +def test_versions_http_retains_capability_contract_version(monkeypatch): + from policyengine_simulation_gateway import endpoints + from policyengine_simulation_gateway.testing import create_gateway_app + from src.modal.utils import update_version_registry as registry + + metadata = { + "app_name": "test-app", + "policyengine_version": "9.9.9", + "us": {"model_version": "test-us"}, + "uk": {"model_version": "test-uk"}, + "spm": CAPABILITY, + } + monkeypatch.setattr( + registry, "build_bundle_manifest_metadata", lambda **kwargs: metadata + ) + state = registry.build_next_routing_state( + current_state=None, + app_name="test-app", + policyengine_version="9.9.9", + us_version="test-us", + uk_version="test-uk", + ) + monkeypatch.setattr(endpoints, "_active_routing_state", lambda: state) + response = TestClient(create_gateway_app()).get("/versions") + assert response.status_code == 200 + payload = response.json() + assert payload["us"]["test-us"] == "test-app" + assert payload["policyengine"]["9.9.9"] == "test-app" + assert payload["spm_capabilities"] == {"9.9.9": CAPABILITY} + assert set(state["bundles"]) == {"9.9.9"} diff --git a/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py b/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py new file mode 100644 index 000000000..223486d04 --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py @@ -0,0 +1,153 @@ +"""One-household real formula smoke, enabled explicitly with a local native H5. + +This test never loads or simulates the full population. It selects one household +and its native member records using the producer's HDF table indexes. +""" + +import json +import os + +import pytest + +SOURCE = os.environ.get("SPM_NATIVE_SMOKE_SOURCE") +pytestmark = pytest.mark.skipif( + not SOURCE, + reason="Set SPM_NATIVE_SMOKE_SOURCE to the local native H5 for the bounded source integration smoke", +) + + +@pytest.fixture +def native_dataset(tmp_path): + import pandas as pd + from policyengine.tax_benefit_models.us import ensure_datasets + + tiny = tmp_path / "native.h5" + with pd.HDFStore(SOURCE, "r") as source, pd.HDFStore(tiny, "w") as dest: + household = source.select("household", start=0, stop=1) + household_id = int(household.household_id.iloc[0]) + people = source.select("person", where=f"person_household_id == {household_id}") + assert 0 < len(people) <= 20 + dest.put("household", household, format="table", data_columns=True) + dest.put("person", people, format="table", data_columns=True) + for entity in ("spm_unit", "tax_unit", "family", "marital_unit"): + ids = [int(value) for value in people[f"person_{entity}_id"].unique()] + dest.put( + entity, + source.select(entity, where=f"{entity}_id in {ids}"), + format="table", + data_columns=True, + ) + dest.put("_time_period", source.select("_time_period"), format="table") + # Unmanaged is an explicit test-only path; production keeps certified bundle + # loading. This exercises the real native-to-year materializer. + datasets = ensure_datasets( + datasets=[str(tiny)], + years=[2024], + data_folder=str(tmp_path / "year-data"), + allow_unmanaged=True, + ) + dataset = next(iter(datasets.values())) + dataset.load() + assert len(dataset.data.person) == len(people) + return dataset + + +def test_worker_baseline_reform_national_local_cache_and_receipts( + native_dataset, tmp_path +): + from policyengine.core.simulation import _cache + from policyengine_simulation_executor.simulation_runtime import _build_simulation + from policyengine_simulation_executor.spm import simulation_spm_result + + params = { + "country": "us", + "scope": "macro", + "time_period": "2024", + "data": str(tmp_path / "native.h5"), + "spm": {"geography_kind": "national"}, + } + baseline = _build_simulation( + params, dataset=native_dataset, policy=None, region_code="us" + ) + reform = _build_simulation( + params, + dataset=native_dataset, + policy={"gov.irs.credits.ctc.amount.base[0].amount": 3000}, + region_code="us", + ) + for simulation in (baseline, reform): + simulation.extra_variables = {"spm_unit": ["spm_unit_spm_threshold"]} + simulation.ensure() + assert ( + simulation.output_dataset.data.spm_unit["spm_unit_spm_threshold"] > 0 + ).all() + result = simulation_spm_result(baseline, reform, baseline.spm_config) + dumped = json.loads(json.dumps(result)) + assert dumped["spm_config"] == baseline.spm_config == reform.spm_config + assert dumped["spm_provenance"]["reform"][0]["years"]["2024"] + original = baseline.spm_provenance() + original["years"].clear() + assert baseline.spm_provenance()["years"] + # Disk replay uses the same identifier, selected config, and actual receipt. + _cache._cache.clear() + replay = _build_simulation( + params, dataset=native_dataset, policy=None, region_code="us" + ) + replay.id = baseline.id + replay.ensure() + assert replay.spm_provenance() == baseline.spm_provenance() + # A shared caller id cannot reuse national output for county selection. + local = _build_simulation( + {**params, "spm": {"geography_kind": "county"}}, + dataset=native_dataset, + policy=None, + region_code="us", + ) + local.id = baseline.id + assert local.storage_id != baseline.storage_id + local.ensure() + assert local.spm_provenance()["geography_kind"] == "county" + + +def test_native_state_only_worker_requires_geography_and_explicit_national_works( + native_dataset, +): + from policyengine_simulation_executor.simulation_runtime import _build_simulation + from policyengine_simulation_contract.spm import spm_error_detail + + native_dataset.data.household.drop(columns=["county_fips"], inplace=True) + params = { + "country": "us", + "scope": "macro", + "time_period": "2024", + "data": "test-only-custom-data", + } + simulation = _build_simulation( + params, dataset=native_dataset, policy=None, region_code="us" + ) + with pytest.raises(ValueError) as error: + simulation.run() + assert spm_error_detail(error.value).code == "SPM_GEOGRAPHY_REQUIRED" + national = _build_simulation( + {**params, "spm": {"geography_kind": "national"}}, + dataset=native_dataset, + policy=None, + region_code="us", + ) + national.run() + assert national.spm_provenance()["geography_kind"] == "national" + + +def test_actual_provider_unknown_metro_is_a_typed_input_error(): + from policyengine_simulation_executor.spm import normalize_runtime_spm + from policyengine_simulation_contract.spm import SPMInputError + + with pytest.raises(SPMInputError) as error: + normalize_runtime_spm( + { + "country": "us", + "time_period": "2024", + "spm": {"geography_kind": "metro", "geography_id": "00000"}, + } + ) + assert error.value.code == "SPM_GEOGRAPHY_UNAVAILABLE" diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py index 235eaf154..0228ab9d6 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py @@ -24,6 +24,11 @@ put_batch_job_state, ) from policyengine_simulation_gateway.auth import require_auth +from policyengine_simulation_contract.spm import ( + SPMCapability, + spm_error_detail, + resolve_spm_selection, +) from policyengine_simulation_observability.errors import log_and_redact_exception from policyengine_simulation_contract.gateway_models import ( BudgetWindowBatchRequest, @@ -575,9 +580,25 @@ def _build_policyengine_bundle( ), data_version=str(data_version) if isinstance(data_version, str) else None, dataset=resolved_dataset, + spm=app_bundle.get("spm") if country.lower() == "us" else None, ) +def _resolve_request_spm(request, bundle): + selection = resolve_spm_selection( + request.country, + request.spm, + capability=bundle.spm, + policyengine_version=bundle.policyengine_version, + model_version=bundle.model_version, + ) + if selection is not None: + from policyengine_simulation_contract.spm import SPMSelection + + request.spm = SPMSelection.model_validate(selection) + return selection + + def _serialize_job_metadata( resolved_app_name: str, bundle: PolicyEngineBundle, @@ -602,6 +623,8 @@ def _build_budget_window_parent_payload( mode="json", exclude_none=True, ) + if request.spm is not None: + payload["spm"] = request.spm.model_dump(mode="json") payload["version"] = resolved_version if request.telemetry is not None: payload["_telemetry"] = request.telemetry.model_dump(mode="json") @@ -694,10 +717,19 @@ async def submit_simulation(request: SimulationRequest): route, payload, ) + _resolve_request_spm(request, bundle) except (ValueError, HuggingFaceDatasetReferenceError) as exc: + detail = spm_error_detail(exc) + if detail: + return failed_job_response( + error=detail.message, errors=[detail.model_dump()] + ) record_error(exc, handled=True, status_code=400, include_stack=False) raise HTTPException(status_code=400, detail=str(exc)) from exc + if request.spm is not None: + payload["spm"] = request.spm.model_dump(mode="json") + logger.info( "Routing %s:%s to app %s (run_id=%s)", request.country, @@ -766,7 +798,13 @@ async def submit_budget_window_batch(request: BudgetWindowBatchRequest): route, request.model_dump(mode="json"), ) + _resolve_request_spm(request, bundle) except (ValueError, HuggingFaceDatasetReferenceError) as exc: + detail = spm_error_detail(exc) + if detail: + return failed_job_response( + error=detail.message, errors=[detail.model_dump()] + ) record_error(exc, handled=True, status_code=400, include_stack=False) raise HTTPException(status_code=400, detail=str(exc)) from exc with segment(SegmentName.REQUEST_PARSE): @@ -854,6 +892,13 @@ async def get_job_status(job_id: str): if _is_modal_job_not_found(exc): record_error(exc, handled=True, status_code=404, include_stack=False) raise HTTPException(status_code=404, detail=f"Job not found: {job_id}") + detail = spm_error_detail(exc) + if detail: + return failed_job_response( + error=detail.message, + errors=[detail.model_dump()], + job_metadata=job_metadata, + ) redacted = log_and_redact_exception( exc, scope="simulation_job_status", @@ -922,13 +967,19 @@ async def get_budget_window_job_status(batch_job_id: str): # "submitted" status from the seed store (#448). We deliberately # overwrite the main job store entry as well as the seed so either # lookup path observes the terminal failed state. - redacted = log_and_redact_exception( - exc, - scope="budget_window_parent_call", - context={"batch_job_id": batch_job_id}, + detail = spm_error_detail(exc) + message = ( + detail.message + if detail + else log_and_redact_exception( + exc, + scope="budget_window_parent_call", + context={"batch_job_id": batch_job_id}, + ) ) seed_state.status = "failed" - seed_state.error = redacted + seed_state.errors = [detail] if detail else None + seed_state.error = message with segment(SegmentName.BUDGET_WINDOW_STATE_WRITE): put_batch_job_state(seed_state) put_batch_job_seed(seed_state) @@ -948,6 +999,11 @@ async def list_versions() -> VersionsResponse: state = _active_routing_state() if state: return VersionsResponse( + spm_capabilities={ + name: SPMCapability.model_validate(bundle["spm"]) + for name, bundle in state.get("bundles", {}).items() + if isinstance(bundle, dict) and bundle.get("spm") is not None + }, policyengine=_version_map_from_state(state, "policyengine"), us=_version_map_from_state(state, "us"), uk=_version_map_from_state(state, "uk"), diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/responses.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/responses.py index 9aa2b2967..562d42b94 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/responses.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/responses.py @@ -25,6 +25,11 @@ def __init__(self, content: dict): def batch_status_payload(response: BudgetWindowBatchStatusResponse) -> dict: payload = response.model_dump(mode="json") + if payload.get("errors") is None: + payload.pop("errors", None) + for child in payload.get("child_jobs", {}).values(): + if child.get("errors") is None: + child.pop("errors", None) if response.policyengine_bundle is not None: payload["policyengine_bundle"] = response.policyengine_bundle.model_dump( mode="json", @@ -38,7 +43,9 @@ def batch_status_response(response: BudgetWindowBatchStatusResponse): if response.status in {"submitted", "running"}: return AcceptedResponse(payload) if response.status == "failed": - return ServerErrorResponse(payload) + return JSONResponse( + status_code=400 if response.errors else 500, content=payload + ) return response @@ -54,13 +61,15 @@ def running_job_response(job_metadata: dict | None = None) -> AcceptedResponse: def failed_job_response( - *, error: str, job_metadata: dict | None = None -) -> ServerErrorResponse: - return ServerErrorResponse( - { + *, error: str, job_metadata: dict | None = None, errors: list[dict] | None = None +) -> JSONResponse: + return JSONResponse( + status_code=400 if errors else 500, + content={ + **({"errors": errors} if errors else {}), "status": "failed", "result": None, "error": error, **(job_metadata or {}), - } + }, ) diff --git a/projects/policyengine-simulation-gateway/tests/test_endpoints.py b/projects/policyengine-simulation-gateway/tests/test_endpoints.py index b29169cc7..244cc5d8a 100644 --- a/projects/policyengine-simulation-gateway/tests/test_endpoints.py +++ b/projects/policyengine-simulation-gateway/tests/test_endpoints.py @@ -1019,6 +1019,7 @@ def test__given_no_active_state__then_versions_fall_back_to_old_dicts( assert response.status_code == 200 assert response.json() == { + "spm_capabilities": {}, "policyengine": { "latest": "4.9.0", "4.9.0": "old-py-app", @@ -1236,7 +1237,9 @@ def test__given_budget_window_submission__then_initial_poll_returns_seed_state( def test__given_batch_state__then_poll_returns_completed_response( self, mock_modal, client: TestClient ): - from policyengine_simulation_contract.budget_window_state import put_batch_job_state + from policyengine_simulation_contract.budget_window_state import ( + put_batch_job_state, + ) from policyengine_simulation_contract.gateway_models import ( BudgetWindowAnnualImpact, BudgetWindowBatchState, From dcfe4fd89d97145882cb147a9aa2ffae2cd10ebf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 15:44:45 -0400 Subject: [PATCH 02/40] Preserve unavailable SPM year errors across worker transport --- .../policyengine_simulation_contract/spm.py | 1 + .../tests/test_canonical_spm.py | 28 ++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py index 8523b9131..623fc9a39 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py @@ -92,6 +92,7 @@ class SPMProvenance(BaseModel): "SPM_GEOGRAPHY_REQUIRED", "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED", + "SPM_YEAR_UNAVAILABLE", "SPM_CONFIGURATION_UNAVAILABLE", "SPM_SETTINGS_INVALID", } diff --git a/projects/policyengine-simulation-executor/tests/test_canonical_spm.py b/projects/policyengine-simulation-executor/tests/test_canonical_spm.py index d9db1105d..6e3729139 100644 --- a/projects/policyengine-simulation-executor/tests/test_canonical_spm.py +++ b/projects/policyengine-simulation-executor/tests/test_canonical_spm.py @@ -250,7 +250,12 @@ def test_year_alias_is_validated_before_dataset_loading(monkeypatch): @pytest.mark.parametrize( "code", - ["SPM_GEOGRAPHY_REQUIRED", "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED"], + [ + "SPM_GEOGRAPHY_REQUIRED", + "SPM_GEOGRAPHY_UNAVAILABLE", + "SPM_COMPOSITION_REQUIRED", + "SPM_YEAR_UNAVAILABLE", + ], ) def test_actual_http_formula_error_contract(monkeypatch, code): error = SPMInputError(code, "Explicit input is required") @@ -390,7 +395,12 @@ def test_identity_errors_do_not_degrade_to_an_unidentified_baseline(monkeypatch) @pytest.mark.parametrize( "code", - ["SPM_GEOGRAPHY_REQUIRED", "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED"], + [ + "SPM_GEOGRAPHY_REQUIRED", + "SPM_GEOGRAPHY_UNAVAILABLE", + "SPM_COMPOSITION_REQUIRED", + "SPM_YEAR_UNAVAILABLE", + ], ) def test_gateway_poll_returns_structured_400(monkeypatch, code): from policyengine_simulation_gateway import endpoints @@ -515,7 +525,12 @@ def test_budget_window_state_keeps_typed_failure_on_replay(): @pytest.mark.parametrize( "code", - ["SPM_GEOGRAPHY_REQUIRED", "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED"], + [ + "SPM_GEOGRAPHY_REQUIRED", + "SPM_GEOGRAPHY_UNAVAILABLE", + "SPM_COMPOSITION_REQUIRED", + "SPM_YEAR_UNAVAILABLE", + ], ) def test_country_error_is_transportable_without_country_package(monkeypatch, code): from contextlib import nullcontext @@ -539,7 +554,12 @@ def __init__(self, code, message): @pytest.mark.parametrize( "code", - ["SPM_GEOGRAPHY_REQUIRED", "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED"], + [ + "SPM_GEOGRAPHY_REQUIRED", + "SPM_GEOGRAPHY_UNAVAILABLE", + "SPM_COMPOSITION_REQUIRED", + "SPM_YEAR_UNAVAILABLE", + ], ) def test_optional_analysis_does_not_swallow_spm_input_errors(code): from policyengine_simulation_executor.simulation_output_common import ( From c34f0e489219e930c4ea982bcab018a2d8c7bc69 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:20:30 -0400 Subject: [PATCH 03/40] Track PR677 Fable fixes and verification plan --- rollout/worker-fable-fixes/PROGRESS.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 rollout/worker-fable-fixes/PROGRESS.md diff --git a/rollout/worker-fable-fixes/PROGRESS.md b/rollout/worker-fable-fixes/PROGRESS.md new file mode 100644 index 000000000..b268110b4 --- /dev/null +++ b/rollout/worker-fable-fixes/PROGRESS.md @@ -0,0 +1,22 @@ +# PR677 Fable fixes + +## State + +- Authorized continuation from dcfe4fd89d97145882cb147a9aa2ffae2cd10ebf on max/spm-simulation-canonical-20260909. +- Exact Fable review read: gate 20260909-170551-pr-abdf4629, round 001-4542bf03eca1. Findings are valid; dispatch BrokenPipeError means no gate agreement exists. +- Existing untracked root PROGRESS.md and WORKER-CANONICAL-SPM-HANDOFF.md are preserved without edits. This committed progress file tracks this continuation. +- Initial git fetch origin and gh repo view failed because GitHub DNS/API connectivity is unavailable in the shell. Current upstream verification remains pending. +- No browser, population job, publication, deployment, merge, .err or .lane.log reads. + +## Done + +- Inspected status, remotes, starting HEAD, and attempted upstream fetch before edits. +- Read repository AGENTS.md and canonical testing/GitHub PR skills; read exact reviewer output. + +## Next + +- Add failing regressions and fix legacy route classification, typed runtime year/scenario errors, incomplete artifact recomputation, and precompute storage identity checks. +- Disposition related ambiguity, malformed capability, and partial-selection notes. +- Run focused and installed-environment qualifications, lint, type, and format checks; record source binding and exact evidence. +- Commit coherent steps; push verified commits to existing canonical draft PR677 and update its body if connectivity permits. +- Write FINAL-REPORT.md in this directory. From c3b47a4c34fbc9f8ccb1cc8db1bc857c4385a521 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:23:32 -0400 Subject: [PATCH 04/40] Add isolated installed-wheel qualification harness for Fable fixes --- rollout/worker-fable-fixes/.gitignore | 6 + .../run_installed_qualification.py | 212 ++++++++++++++++++ 2 files changed, 218 insertions(+) create mode 100644 rollout/worker-fable-fixes/.gitignore create mode 100644 rollout/worker-fable-fixes/run_installed_qualification.py diff --git a/rollout/worker-fable-fixes/.gitignore b/rollout/worker-fable-fixes/.gitignore new file mode 100644 index 000000000..0b6840c44 --- /dev/null +++ b/rollout/worker-fable-fixes/.gitignore @@ -0,0 +1,6 @@ +pytest-*/ +__pycache__/ +*.log +native-wheel-receipt.json +*-pyright.json +*-pyrightconfig.json diff --git a/rollout/worker-fable-fixes/run_installed_qualification.py b/rollout/worker-fable-fixes/run_installed_qualification.py new file mode 100644 index 000000000..d23ebe729 --- /dev/null +++ b/rollout/worker-fable-fixes/run_installed_qualification.py @@ -0,0 +1,212 @@ +"""Four-person worker and lazy-year smoke against new exact installed candidate wheels.""" + +import hashlib +import importlib +import importlib.metadata as metadata +import importlib.util +import json +import os +from pathlib import Path +import sys +import subprocess +import zipfile + +OUT = Path(__file__).resolve().parent +QUALIFICATION = Path( + "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification" +) +ROLLOUT = QUALIFICATION.parent +WORKTREES = ROLLOUT.parent / "worktrees" +WORKER = WORKTREES / "policyengine-sim-api-canonical" +SOURCE = ROLLOUT / "producer-final-candidate-20260909/artifacts/populace_us_2024.h5" +CONTENT = "3d86d5c4c0423480e6b69b75d222ffa4a7a2639e4094df5ba2504af01be17173" +WHEELS = { + "policyengine": ( + ROLLOUT / "postfix-qualification/wheels/policyengine-5.3.0-py3-none-any.whl", + "8c640d967575dddad70840bcbe938cea251c56958eded647c83eb9c5902735f1", + ), + "policyengine_us": ( + ROLLOUT + / "postfix-qualification/wheels/policyengine_us-1.824.7-py3-none-any.whl", + "7644819916a4f8ca2aa37a4a9d992fa834bfa66ab2d441326a9686baa5c1a688", + ), + "spm_calculator": ( + QUALIFICATION / "wheels/spm_calculator-1.0.0-py3-none-any.whl", + "c49c41da5fd482e563eaea956e205a3ba6841cadd4dd32bef4c0c3dbce17ffba", + ), + "policyengine_core": ( + ROLLOUT + / "postfix-qualification/wheels/policyengine_core-3.30.1-py3-none-any.whl", + "2dbcf5f590a0199a7b7c77fcbbda2ff6bc289f6c6169ca0af3beace35d8b3e63", + ), +} + + +def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +sys.dont_write_bytecode = True +os.environ["PYTHONDONTWRITEBYTECODE"] = "1" +os.environ["POLICYENGINE_SKIP_COUNTRY_IMPORTS"] = "1" +os.environ["SPM_NATIVE_SMOKE_SOURCE"] = str(SOURCE) +assert not os.environ.get("PYTHONPATH"), "No inherited source overlays allowed" +for relative in ( + "projects/policyengine-simulation-executor/src", + "projects/policyengine-simulation-entry/src", + "projects/policyengine-simulation-gateway/src", + "libs/policyengine-simulation-contract/src", + "libs/policyengine-simulation-observability/src", + "libs/policyengine-fastapi/src", +): + sys.path.insert(0, str(WORKER / relative)) + +receipt = { + "worker_head": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=WORKER, text=True + ).strip(), + "scope": "four_person_native_worker_installed_candidate_wheel_qualification", + "data_certification": "not_certified", + "external_package_publication": "not_attested", + "population_acceptance": "not_attested", + "python": sys.executable, + "wheels": {}, + "worker_source_overlay_only": True, +} +for module_name, (wheel, expected_hash) in WHEELS.items(): + assert sha256(wheel) == expected_hash, wheel + dist = metadata.distribution(module_name) + members = {} + with zipfile.ZipFile(wheel) as archive: + for member in archive.namelist(): + if not member.startswith(module_name + "/") or member.endswith("/"): + continue + installed = Path(dist.locate_file(member)).resolve() + assert installed.is_relative_to(QUALIFICATION / "venv"), installed + expected = archive.read(member) + assert installed.read_bytes() == expected, installed + members[member] = hashlib.sha256(expected).hexdigest() + assert members, module_name + spec = importlib.util.find_spec(module_name) + assert Path(spec.origin).resolve().is_relative_to(QUALIFICATION / "venv"), ( + spec.origin + ) + receipt["wheels"][module_name] = { + "path": str(wheel), + "sha256": expected_hash, + "version": dist.version, + "package_file_count": len(members), + "installed_package_files": members, + "import_origin_before": spec.origin, + } + +helper = WORKTREES / "policyengine-wrapper-production/tests/fixtures/spm_development.py" +spec = importlib.util.spec_from_file_location( + "worker_spm_development_bootstrap", helper +) +bootstrap = importlib.util.module_from_spec(spec) +spec.loader.exec_module(bootstrap) +manifest = QUALIFICATION / "development-manifest.json" +bootstrap.activate_spm_development_manifest(manifest) +receipt["development_fixture"] = { + "manifest_path": str(manifest), + "manifest_sha256": sha256(manifest), + "bootstrap_path": str(helper), + "bootstrap_sha256": sha256(helper), + "compatibility_basis": "unverified_development_fixture", +} +for name in WHEELS: + module = importlib.import_module(name) + origin = str(Path(module.__file__).resolve()) + assert Path(origin).is_relative_to(QUALIFICATION / "venv"), origin + receipt["wheels"][name]["import_origin_after"] = origin + +from spm_calculator import load_forecast +from policyengine_simulation_executor.spm import runtime_spm_capability + +assert load_forecast(expected_sha256=CONTENT).content_sha256 == CONTENT +capability = runtime_spm_capability() +assert capability.defaults.forecast_content_sha256 == CONTENT +receipt["runtime_capability"] = capability.model_dump(mode="json") +receipt["source_h5_sha256_before"] = sha256(SOURCE) +assert ( + receipt["source_h5_sha256_before"] + == "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" +) + +import pandas as pd + +with pd.HDFStore(SOURCE, "r") as source: + household = source.select("household", start=0, stop=1) + household_id = int(household.household_id.iloc[0]) + people = source.select("person", where=f"person_household_id == {household_id}") + receipt["selected_households"] = len(household) + receipt["selected_people"] = len(people) + assert len(household) == 1 and len(people) == 4 + +import pytest + + +class OutcomeRecorder: + def __init__(self): + self.passed = [] + self.failed = [] + self.skipped = [] + + def pytest_runtest_logreport(self, report): + if report.skipped: + self.skipped.append(report.nodeid) + elif report.failed: + self.failed.append(report.nodeid) + elif report.when == "call" and report.passed: + self.passed.append(report.nodeid) + + +outcomes = OutcomeRecorder() +exit_code = pytest.main( + [ + str( + WORKER + / "projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py" + ), + str(QUALIFICATION / "test_installed_year_boundary.py"), + str( + WORKER + / "projects/policyengine-simulation-executor/tests/test_baseline_artifacts_spm_native.py" + ), + str( + WORKER + / "projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py" + ), + "-q", + "--basetemp", + str(OUT / "pytest-native"), + "-p", + "no:cacheprovider", + ], + plugins=[outcomes], +) +receipt["pytest_outcomes"] = vars(outcomes) +assert not outcomes.skipped, outcomes.skipped +assert len(outcomes.passed) >= 15 and not outcomes.failed, vars(outcomes) +receipt["pytest_exit_code"] = int(exit_code) +source_files = subprocess.check_output( + ["git", "ls-files", "projects", "libs"], cwd=WORKER, text=True +).splitlines() +receipt["worker_source_files"] = { + name: sha256(WORKER / name) + for name in source_files + if name.endswith(".py") + and ("/src/" in name or "/tests/" in name or "/fixtures/" in name) +} +receipt["qualification_harness_sha256"] = sha256(Path(__file__)) +receipt["source_h5_sha256_after"] = sha256(SOURCE) +assert receipt["source_h5_sha256_after"] == receipt["source_h5_sha256_before"] +(OUT / "native-wheel-receipt.json").write_text( + json.dumps(receipt, indent=2, sort_keys=True) + "\n" +) +raise SystemExit(exit_code) From 1d6a6ed4b14b9cbd46c100bd18d69ddebeb727c5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:24:16 -0400 Subject: [PATCH 05/40] test: reproduce SPM legacy routing and partial selection regressions --- .../tests/test_spm_selection.py | 52 ++++++ .../tests/test_spm_routes.py | 157 ++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 libs/policyengine-simulation-contract/tests/test_spm_selection.py create mode 100644 projects/policyengine-simulation-gateway/tests/test_spm_routes.py diff --git a/libs/policyengine-simulation-contract/tests/test_spm_selection.py b/libs/policyengine-simulation-contract/tests/test_spm_selection.py new file mode 100644 index 000000000..f69f5dc58 --- /dev/null +++ b/libs/policyengine-simulation-contract/tests/test_spm_selection.py @@ -0,0 +1,52 @@ +"""Partial public selections inherit pinned bundle defaults before validation.""" + +import pytest + +from policyengine_simulation_contract.gateway_models import SimulationRequest +from policyengine_simulation_contract.spm import SPMInputError, resolve_spm_selection + + +def resolve(selection, *, kind="metro", area="35620"): + return resolve_spm_selection( + "us", + selection, + capability={ + "defaults": { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": kind, + "geography_id": area, + }, + }, + policyengine_version="5.3.0", + model_version="1.824.7", + ) + + +def test_partial_area_override_inherits_metro_default_through_request(): + request = SimulationRequest(country="us", spm={"geography_id": "31080"}) + payload = request.model_dump(mode="json")["spm"] + assert payload == {"geography_id": "31080"} + resolved = resolve(payload) + assert resolved["geography_kind"] == "metro" + assert resolved["geography_id"] == "31080" + + +@pytest.mark.parametrize("kind", ["county", "national"]) +def test_partial_area_override_rejects_incompatible_defaults(kind): + with pytest.raises(SPMInputError) as error: + resolve({"geography_id": "31080"}, kind=kind, area=None) + assert error.value.code == "SPM_SETTINGS_INVALID" + + +def test_explicit_metro_inherits_area_only_from_metro_defaults(): + assert resolve({"geography_kind": "metro"})["geography_id"] == "35620" + with pytest.raises(SPMInputError) as error: + resolve({"geography_kind": "metro"}, kind="county", area=None) + assert error.value.code == "SPM_SETTINGS_INVALID" + + +def test_explicit_national_discards_default_metro_area(): + resolved = resolve({"geography_kind": "national"}) + assert resolved["geography_kind"] == "national" + assert resolved["geography_id"] is None diff --git a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py new file mode 100644 index 000000000..b31afe469 --- /dev/null +++ b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py @@ -0,0 +1,157 @@ +"""SPM capability resolution on historical and ambiguous registry routes.""" + +from copy import deepcopy + +import pytest + +from fixtures.gateway_endpoints import TEST_ROUTING_STATE + + +CAPABILITY = { + "contract_version": "canonical-spm-v1", + "defaults": {"forecast_content_sha256": "a" * 64, "scenario": "ce_trend"}, +} +ENDPOINTS = [ + ("/simulate/economy/comparison", {"scope": "macro"}), + ( + "/simulate/economy/budget-window", + {"region": "us", "start_year": "2026", "window_size": 2}, + ), +] + + +def legacy_route(mock_modal, source, model_version): + app_name = "policyengine-simulation-us1-715-2-uk2-88-20" + if source == "legacy-country-dict": + del mock_modal["dicts"]["simulation-api-routing-state"] + mock_modal["dicts"]["simulation-api-us-versions"] = { + "latest": model_version, + model_version: app_name, + } + else: + mock_modal["dicts"]["simulation-api-routing-state"] = { + "active": { + "schema_version": 1, + "generation": source, + "latest": {"us": model_version}, + "routes": {"policyengine": {}, "us": {model_version: app_name}}, + "bundles": {}, + } + } + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +@pytest.mark.parametrize("source", ["legacy-country-dict", "legacy-seed"]) +@pytest.mark.parametrize("version", [None, "1.715.2"]) +def test_historical_no_wrapper_route_submits_without_spm( + mock_modal, client, endpoint, extra, source, version +): + legacy_route(mock_modal, source, "1.715.2") + response = client.post( + endpoint, json={"country": "us", "version": version, **extra} + ) + assert response.status_code == 200, response.text + assert "spm" not in mock_modal["func"].last_payload + assert response.json()["policyengine_bundle"]["model_version"] == "1.715.2" + assert "policyengine_version" not in response.json()["policyengine_bundle"] + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +@pytest.mark.parametrize("source", ["legacy-country-dict", "legacy-seed"]) +@pytest.mark.parametrize("selection", [{}, {"geography_kind": "national"}]) +def test_historical_route_rejects_any_explicit_spm( + mock_modal, client, endpoint, extra, source, selection +): + legacy_route(mock_modal, source, "1.715.2") + response = client.post( + endpoint, json={"country": "us", "spm": selection, **extra} + ) + assert response.status_code == 400 + assert response.json()["errors"][0]["code"] == "SPM_CONFIGURATION_UNAVAILABLE" + assert mock_modal["func"].calls == [] + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +@pytest.mark.parametrize( + "source,model_version", + [ + ("legacy-country-dict", "1.824.7"), + ("legacy-seed", "1.824.7"), + ("legacy-seed", "future-model"), + ("unknown-generation", "1.715.2"), + ], +) +def test_missing_wrapper_does_not_certify_unknown_routes( + mock_modal, client, endpoint, extra, source, model_version +): + legacy_route(mock_modal, source, model_version) + response = client.post(endpoint, json={"country": "us", **extra}) + assert response.status_code == 400 + assert response.json()["errors"][0]["code"] == "SPM_CONFIGURATION_UNAVAILABLE" + assert mock_modal["func"].calls == [] + + +def shared_app_state(mock_modal, *, sibling_model): + state = deepcopy(TEST_ROUTING_STATE) + original = state["bundles"]["4.10.0"] + app_name = original["app_name"] + state["routes"]["policyengine"]["5.3.0"] = app_name + state["bundles"]["5.3.0"] = { + **deepcopy(original), + "policyengine_version": "5.3.0", + "us": {**original["us"], "model_version": sibling_model}, + "spm": deepcopy(CAPABILITY), + } + state["routes"]["us"][sibling_model] = app_name + mock_modal["dicts"]["simulation-api-routing-state"] = {"active": state} + return state + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +def test_shared_app_resolves_unique_country_model_bundle( + mock_modal, client, endpoint, extra +): + shared_app_state(mock_modal, sibling_model="1.824.7") + response = client.post( + endpoint, json={"country": "us", "version": "1.824.7", **extra} + ) + assert response.status_code == 200, response.text + assert response.json()["policyengine_bundle"]["policyengine_version"] == "5.3.0" + assert mock_modal["func"].last_payload["spm"]["scenario"] == "ce_trend" + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +def test_shared_app_with_ambiguous_country_model_requires_explicit_bundle( + mock_modal, client, endpoint, extra +): + shared_app_state(mock_modal, sibling_model="1.500.0") + response = client.post( + endpoint, json={"country": "us", "version": "1.500.0", **extra} + ) + assert response.status_code == 400 + assert "policyengine_version" in response.json()["detail"] + assert mock_modal["func"].calls == [] + explicit = client.post( + endpoint, + json={"country": "us", "policyengine_version": "5.3.0", **extra}, + ) + assert explicit.status_code == 200 + assert mock_modal["func"].last_payload["spm"]["scenario"] == "ce_trend" + + +def test_versions_omits_malformed_capability_and_submission_rejects_it( + mock_modal, client +): + state = shared_app_state(mock_modal, sibling_model="1.824.7") + state["bundles"]["4.10.0"]["spm"] = { + "contract_version": "canonical-spm-v1", "defaults": {"scenario": "ce_trend"} + } + response = client.get("/versions") + assert response.status_code == 200 + assert set(response.json()["spm_capabilities"]) == {"5.3.0"} + rejected = client.post( + "/simulate/economy/comparison", + json={"country": "us", "policyengine_version": "4.10.0"}, + ) + assert rejected.status_code == 400 + assert mock_modal["func"].calls == [] From 8a7f6e8634728ba67ac7c3360782a28fe13c0855 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:24:57 -0400 Subject: [PATCH 06/40] Record upstream identity and reproduced Fable regressions --- rollout/worker-fable-fixes/PROGRESS.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rollout/worker-fable-fixes/PROGRESS.md b/rollout/worker-fable-fixes/PROGRESS.md index b268110b4..67ec25894 100644 --- a/rollout/worker-fable-fixes/PROGRESS.md +++ b/rollout/worker-fable-fixes/PROGRESS.md @@ -5,13 +5,17 @@ - Authorized continuation from dcfe4fd89d97145882cb147a9aa2ffae2cd10ebf on max/spm-simulation-canonical-20260909. - Exact Fable review read: gate 20260909-170551-pr-abdf4629, round 001-4542bf03eca1. Findings are valid; dispatch BrokenPipeError means no gate agreement exists. - Existing untracked root PROGRESS.md and WORKER-CANONICAL-SPM-HANDOFF.md are preserved without edits. This committed progress file tracks this continuation. -- Initial git fetch origin and gh repo view failed because GitHub DNS/API connectivity is unavailable in the shell. Current upstream verification remains pending. +- Initial git fetch origin and gh repo view failed because GitHub DNS/API connectivity is unavailable in the shell. The authenticated GitHub connector independently confirms live main is 414c631d622f5f587eedd187296a80182a923db2, identical to local origin/main, and PR677 is still a canonical draft at the requested starting head. Issue #676 is open and appropriate. Git transport remains pending. - No browser, population job, publication, deployment, merge, .err or .lane.log reads. ## Done - Inspected status, remotes, starting HEAD, and attempted upstream fetch before edits. - Read repository AGENTS.md and canonical testing/GitHub PR skills; read exact reviewer output. +- Read installed PolicyEngine analysis, standards and API skills, parent PolicyEngine guidance and wrapper repository guidance. Repository-specific instructions and the user's bounded qualification scope govern this work. +- Committed an isolated qualification harness that authenticates the existing c49c/76448/8c640/2dbc wheels and reuses the existing explicit development bootstrap without modifying it, the packages, or historical evidence. +- Routing regressions before fixes: 13 gateway failures (historical routes, sibling ambiguity, malformed capability) and two partial-selection failures reproduced. +- Archived starting source to /tmp/worker-fable-base for like-for-like installed-environment type diagnostics; no checkout or handoff edits. ## Next From f4cc5ea8abbc68406abd5c9d5b9a0f6c7da3d463 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:25:36 -0400 Subject: [PATCH 07/40] test: reproduce worker SPM year and scenario error boundaries --- .../tests/test_spm_runtime_native.py | 124 +++++++ .../worker-fable-fixes/runtime-evidence.md | 17 + .../runtime-regression-red.txt | 337 ++++++++++++++++++ 3 files changed, 478 insertions(+) create mode 100644 projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py create mode 100644 rollout/worker-fable-fixes/runtime-evidence.md create mode 100644 rollout/worker-fable-fixes/runtime-regression-red.txt diff --git a/projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py b/projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py new file mode 100644 index 000000000..14115940a --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py @@ -0,0 +1,124 @@ +"""Worker prevalidation against the installed canonical calculator contracts. + +Enabled by the authenticated native qualification harness. These cases stop +before dataset loading or child submission and never run a population job. +""" + +import os +from unittest.mock import Mock + +import pytest + +pytestmark = pytest.mark.skipif( + not os.environ.get("SPM_NATIVE_SMOKE_SOURCE"), + reason="Requires the installed canonical SPM qualification environment", +) + + +@pytest.fixture +def forecast(): + from policyengine_simulation_executor.spm import _forecast, runtime_spm_capability + + return _forecast(runtime_spm_capability().defaults.forecast_content_sha256) + + +def _worker_entrypoint(params, monkeypatch): + from policyengine_simulation_executor import simulation_runtime + from src.modal import budget_window_batch + + if "start_year" in params: + entrypoint = budget_window_batch.run_budget_window_batch_impl + boundary = Mock(side_effect=AssertionError("Must reject before child setup")) + monkeypatch.setattr(budget_window_batch, "build_batch_context", boundary) + else: + entrypoint = simulation_runtime._run_simulation_impl_core + boundary = Mock(side_effect=AssertionError("Must reject before dataset lookup")) + monkeypatch.setattr(simulation_runtime, "_resolve_dataset_reference", boundary) + return entrypoint, boundary + + +@pytest.mark.parametrize( + "period,unavailable_year", + [ + ({"time_period": "2021"}, 2021), + ({"time_period": "2036"}, 2036), + ({"year": "2040"}, 2040), + ({"start_year": "2021", "window_size": 1}, 2021), + ({"start_year": "2036", "window_size": 1}, 2036), + ({"start_year": "2035", "window_size": 2}, 2036), + ], +) +def test_worker_year_boundary_matches_real_provider( + forecast, period, unavailable_year, monkeypatch +): + from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider + from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail + + provider = PolicyEngineSPMProvider(forecast, geography_kind="national") + with pytest.raises(ValueError) as provider_error: + provider.year_metadata(unavailable_year) + expected = spm_error_detail(provider_error.value) + assert expected.code == "SPM_YEAR_UNAVAILABLE" + assert provider.provenance()["years"] == {} + params = {"country": "us", "spm": {"geography_kind": "national"}, **period} + entrypoint, boundary = _worker_entrypoint(params, monkeypatch) + with pytest.raises(SPMInputError) as error: + entrypoint(params) + assert error.value.to_dict() == expected.model_dump() + boundary.assert_not_called() + + +@pytest.mark.parametrize( + "period", + [ + {"time_period": "2024"}, + {"start_year": "2024", "window_size": 2}, + ], +) +def test_worker_scenario_boundary_matches_real_calculator_contract( + forecast, period, monkeypatch +): + from spm_calculator.axiom_adapter import export_build_spec + from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail + + scenario = "scenario-not-in-the-pinned-forecast" + with pytest.raises(ValueError) as adapter_error: + export_build_spec(forecast, years=[2024], scenario=scenario) + assert adapter_error.value.code == "SPM_SCENARIO_UNAVAILABLE" + expected = spm_error_detail(adapter_error.value) + assert expected is not None + params = { + "country": "us", + "spm": {"geography_kind": "national", "scenario": scenario}, + **period, + } + entrypoint, boundary = _worker_entrypoint(params, monkeypatch) + with pytest.raises(SPMInputError) as error: + entrypoint(params) + assert error.value.to_dict() == expected.model_dump() + boundary.assert_not_called() + + +def test_valid_worker_prevalidation_does_not_evaluate_amounts(forecast, monkeypatch): + from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider + from policyengine_simulation_executor.spm import normalize_runtime_spm + + calculate = Mock(side_effect=AssertionError("Prevalidation is not measurement")) + monkeypatch.setattr(PolicyEngineSPMProvider, "calculate_unit", calculate) + resolved = normalize_runtime_spm( + { + "country": "us", + "start_year": "2034", + "window_size": 2, + "spm": {"geography_kind": "national"}, + } + ) + assert resolved["geography_kind"] == "national" + provider = PolicyEngineSPMProvider( + forecast=forecast, + geography_kind=resolved["geography_kind"], + scenario=resolved["scenario"], + as_of=resolved["as_of"], + ) + assert provider.provenance()["years"] == {} + calculate.assert_not_called() diff --git a/rollout/worker-fable-fixes/runtime-evidence.md b/rollout/worker-fable-fixes/runtime-evidence.md new file mode 100644 index 000000000..3d69e2191 --- /dev/null +++ b/rollout/worker-fable-fixes/runtime-evidence.md @@ -0,0 +1,17 @@ +# Runtime year and scenario error evidence + +State: regression reproduced against the authenticated installed calculator and country/wrapper environment; implementation pending. + +The exact Fable reviewer output and repository testing skill were read. The installed `PolicyEngineSPMProvider.year_metadata` supplies the typed year contract. Its constructor still raises a plain unknown-scenario `ValueError`; the calculator's public Axiom `export_build_spec` translates that condition into `SPM_SCENARIO_UNAVAILABLE`. Worker prevalidation currently reports `SPM_SETTINGS_INVALID` instead and the shared error recognizer drops the scenario code entirely. + +Regression-first command: + +```sh +env -u PYTHONPATH UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache uv run --no-project --python /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/bin/python /private/tmp/run_worker_runtime_regressions.py > /private/tmp/worker-runtime-regression-red.txt 2>&1 +``` + +The temporary launcher executes only the original qualification script's authenticated wheel/bootstrap setup (before `import pytest`) and invokes the committed `tests/test_spm_runtime_native.py` with pytest `-q -p no:cacheprovider --basetemp /private/tmp/worker-runtime-regression-pytest`. It neither rewrites historical evidence nor modifies installed packages. Initial invocation with inherited PYTHONPATH stopped at the original bootstrap's explicit guard; rerun unsets it. + +Result before fixes: **8 failed, 1 passed, zero skipped, 1.21 seconds**. Six annual/year-alias/budget-window start/end cases showed the wrong year code; two real calculator scenario-contract cases showed the missing public scenario-code recognition. Valid-year prevalidation performs no amount calculation. Two plugin-rewrite warnings reflect explicit bootstrap before pytest, as in the prior qualification. + +Next: preserve typed adapter errors, test full public error transport, run the integrated installed qualification including the prior real-country lazy tax-only tests, and record final outcomes/source binding in the final report. diff --git a/rollout/worker-fable-fixes/runtime-regression-red.txt b/rollout/worker-fable-fixes/runtime-regression-red.txt new file mode 100644 index 000000000..034d0afee --- /dev/null +++ b/rollout/worker-fable-fixes/runtime-regression-red.txt @@ -0,0 +1,337 @@ +WARN `--no-project` was provided, but no project was found +warning: `--frozen` has no effect when used alongside `--no-project` +RUN_EXPERIMENT EVENTS: [] +wrote /tmp/v/results/invented_result.json +domain allocation: occupied_housing_unit=100 group_quarters_person=10 +whole-arm reference: occupied_housing_unit=1050/11 group_quarters_person=50/11 +refusal cases demonstrated: 12, unrefused: [] +FFFFFFFF. [100%] +=================================== FAILURES =================================== +________ test_worker_year_boundary_matches_real_provider[period0-2021] _________ + +forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) +period = {'time_period': '2021'}, unavailable_year = 2021 +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13862c2b0> + + @pytest.mark.parametrize( + "period,unavailable_year", + [ + ({"time_period": "2021"}, 2021), + ({"time_period": "2036"}, 2036), + ({"year": "2040"}, 2040), + ({"start_year": "2021", "window_size": 1}, 2021), + ({"start_year": "2036", "window_size": 1}, 2036), + ({"start_year": "2035", "window_size": 2}, 2036), + ], + ) + def test_worker_year_boundary_matches_real_provider( + forecast, period, unavailable_year, monkeypatch + ): + from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider + from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail + + provider = PolicyEngineSPMProvider(forecast, geography_kind="national") + with pytest.raises(ValueError) as provider_error: + provider.year_metadata(unavailable_year) + expected = spm_error_detail(provider_error.value) + assert expected.code == "SPM_YEAR_UNAVAILABLE" + assert provider.provenance()["years"] == {} + params = {"country": "us", "spm": {"geography_kind": "national"}, **period} + entrypoint, boundary = _worker_entrypoint(params, monkeypatch) + with pytest.raises(SPMInputError) as error: + entrypoint(params) +> assert error.value.to_dict() == expected.model_dump() +E AssertionError: assert {'code': 'SPM...try for 2021'} == {'code': 'SPM...try for 2021'} +E +E Omitting 1 identical items, use -vv to show +E Differing items: +E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_YEAR_UNAVAILABLE'} +E Use -v to get more diff + +projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:67: AssertionError +________ test_worker_year_boundary_matches_real_provider[period1-2036] _________ + +forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) +period = {'time_period': '2036'}, unavailable_year = 2036 +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13862c050> + + @pytest.mark.parametrize( + "period,unavailable_year", + [ + ({"time_period": "2021"}, 2021), + ({"time_period": "2036"}, 2036), + ({"year": "2040"}, 2040), + ({"start_year": "2021", "window_size": 1}, 2021), + ({"start_year": "2036", "window_size": 1}, 2036), + ({"start_year": "2035", "window_size": 2}, 2036), + ], + ) + def test_worker_year_boundary_matches_real_provider( + forecast, period, unavailable_year, monkeypatch + ): + from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider + from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail + + provider = PolicyEngineSPMProvider(forecast, geography_kind="national") + with pytest.raises(ValueError) as provider_error: + provider.year_metadata(unavailable_year) + expected = spm_error_detail(provider_error.value) + assert expected.code == "SPM_YEAR_UNAVAILABLE" + assert provider.provenance()["years"] == {} + params = {"country": "us", "spm": {"geography_kind": "national"}, **period} + entrypoint, boundary = _worker_entrypoint(params, monkeypatch) + with pytest.raises(SPMInputError) as error: + entrypoint(params) +> assert error.value.to_dict() == expected.model_dump() +E AssertionError: assert {'code': 'SPM...try for 2036'} == {'code': 'SPM...try for 2036'} +E +E Omitting 1 identical items, use -vv to show +E Differing items: +E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_YEAR_UNAVAILABLE'} +E Use -v to get more diff + +projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:67: AssertionError +________ test_worker_year_boundary_matches_real_provider[period2-2040] _________ + +forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) +period = {'year': '2040'}, unavailable_year = 2040 +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a0a28d0> + + @pytest.mark.parametrize( + "period,unavailable_year", + [ + ({"time_period": "2021"}, 2021), + ({"time_period": "2036"}, 2036), + ({"year": "2040"}, 2040), + ({"start_year": "2021", "window_size": 1}, 2021), + ({"start_year": "2036", "window_size": 1}, 2036), + ({"start_year": "2035", "window_size": 2}, 2036), + ], + ) + def test_worker_year_boundary_matches_real_provider( + forecast, period, unavailable_year, monkeypatch + ): + from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider + from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail + + provider = PolicyEngineSPMProvider(forecast, geography_kind="national") + with pytest.raises(ValueError) as provider_error: + provider.year_metadata(unavailable_year) + expected = spm_error_detail(provider_error.value) + assert expected.code == "SPM_YEAR_UNAVAILABLE" + assert provider.provenance()["years"] == {} + params = {"country": "us", "spm": {"geography_kind": "national"}, **period} + entrypoint, boundary = _worker_entrypoint(params, monkeypatch) + with pytest.raises(SPMInputError) as error: + entrypoint(params) +> assert error.value.to_dict() == expected.model_dump() +E AssertionError: assert {'code': 'SPM...try for 2040'} == {'code': 'SPM...try for 2040'} +E +E Omitting 1 identical items, use -vv to show +E Differing items: +E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_YEAR_UNAVAILABLE'} +E Use -v to get more diff + +projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:67: AssertionError +________ test_worker_year_boundary_matches_real_provider[period3-2021] _________ + +forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) +period = {'start_year': '2021', 'window_size': 1}, unavailable_year = 2021 +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a01b9b0> + + @pytest.mark.parametrize( + "period,unavailable_year", + [ + ({"time_period": "2021"}, 2021), + ({"time_period": "2036"}, 2036), + ({"year": "2040"}, 2040), + ({"start_year": "2021", "window_size": 1}, 2021), + ({"start_year": "2036", "window_size": 1}, 2036), + ({"start_year": "2035", "window_size": 2}, 2036), + ], + ) + def test_worker_year_boundary_matches_real_provider( + forecast, period, unavailable_year, monkeypatch + ): + from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider + from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail + + provider = PolicyEngineSPMProvider(forecast, geography_kind="national") + with pytest.raises(ValueError) as provider_error: + provider.year_metadata(unavailable_year) + expected = spm_error_detail(provider_error.value) + assert expected.code == "SPM_YEAR_UNAVAILABLE" + assert provider.provenance()["years"] == {} + params = {"country": "us", "spm": {"geography_kind": "national"}, **period} + entrypoint, boundary = _worker_entrypoint(params, monkeypatch) + with pytest.raises(SPMInputError) as error: + entrypoint(params) +> assert error.value.to_dict() == expected.model_dump() +E AssertionError: assert {'code': 'SPM...try for 2021'} == {'code': 'SPM...try for 2021'} +E +E Omitting 1 identical items, use -vv to show +E Differing items: +E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_YEAR_UNAVAILABLE'} +E Use -v to get more diff + +projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:67: AssertionError +________ test_worker_year_boundary_matches_real_provider[period4-2036] _________ + +forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) +period = {'start_year': '2036', 'window_size': 1}, unavailable_year = 2036 +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a0b09e0> + + @pytest.mark.parametrize( + "period,unavailable_year", + [ + ({"time_period": "2021"}, 2021), + ({"time_period": "2036"}, 2036), + ({"year": "2040"}, 2040), + ({"start_year": "2021", "window_size": 1}, 2021), + ({"start_year": "2036", "window_size": 1}, 2036), + ({"start_year": "2035", "window_size": 2}, 2036), + ], + ) + def test_worker_year_boundary_matches_real_provider( + forecast, period, unavailable_year, monkeypatch + ): + from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider + from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail + + provider = PolicyEngineSPMProvider(forecast, geography_kind="national") + with pytest.raises(ValueError) as provider_error: + provider.year_metadata(unavailable_year) + expected = spm_error_detail(provider_error.value) + assert expected.code == "SPM_YEAR_UNAVAILABLE" + assert provider.provenance()["years"] == {} + params = {"country": "us", "spm": {"geography_kind": "national"}, **period} + entrypoint, boundary = _worker_entrypoint(params, monkeypatch) + with pytest.raises(SPMInputError) as error: + entrypoint(params) +> assert error.value.to_dict() == expected.model_dump() +E AssertionError: assert {'code': 'SPM...try for 2036'} == {'code': 'SPM...try for 2036'} +E +E Omitting 1 identical items, use -vv to show +E Differing items: +E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_YEAR_UNAVAILABLE'} +E Use -v to get more diff + +projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:67: AssertionError +________ test_worker_year_boundary_matches_real_provider[period5-2036] _________ + +forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) +period = {'start_year': '2035', 'window_size': 2}, unavailable_year = 2036 +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a03f650> + + @pytest.mark.parametrize( + "period,unavailable_year", + [ + ({"time_period": "2021"}, 2021), + ({"time_period": "2036"}, 2036), + ({"year": "2040"}, 2040), + ({"start_year": "2021", "window_size": 1}, 2021), + ({"start_year": "2036", "window_size": 1}, 2036), + ({"start_year": "2035", "window_size": 2}, 2036), + ], + ) + def test_worker_year_boundary_matches_real_provider( + forecast, period, unavailable_year, monkeypatch + ): + from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider + from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail + + provider = PolicyEngineSPMProvider(forecast, geography_kind="national") + with pytest.raises(ValueError) as provider_error: + provider.year_metadata(unavailable_year) + expected = spm_error_detail(provider_error.value) + assert expected.code == "SPM_YEAR_UNAVAILABLE" + assert provider.provenance()["years"] == {} + params = {"country": "us", "spm": {"geography_kind": "national"}, **period} + entrypoint, boundary = _worker_entrypoint(params, monkeypatch) + with pytest.raises(SPMInputError) as error: + entrypoint(params) +> assert error.value.to_dict() == expected.model_dump() +E AssertionError: assert {'code': 'SPM...try for 2036'} == {'code': 'SPM...try for 2036'} +E +E Omitting 1 identical items, use -vv to show +E Differing items: +E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_YEAR_UNAVAILABLE'} +E Use -v to get more diff + +projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:67: AssertionError +___ test_worker_scenario_boundary_matches_real_calculator_contract[period0] ____ + +forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) +period = {'time_period': '2024'} +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a03ed50> + + @pytest.mark.parametrize( + "period", + [ + {"time_period": "2024"}, + {"start_year": "2024", "window_size": 2}, + ], + ) + def test_worker_scenario_boundary_matches_real_calculator_contract( + forecast, period, monkeypatch + ): + from spm_calculator.axiom_adapter import export_build_spec + from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail + + scenario = "scenario-not-in-the-pinned-forecast" + with pytest.raises(ValueError) as adapter_error: + export_build_spec(forecast, years=[2024], scenario=scenario) + assert adapter_error.value.code == "SPM_SCENARIO_UNAVAILABLE" + expected = spm_error_detail(adapter_error.value) +> assert expected is not None +E assert None is not None + +projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:89: AssertionError +___ test_worker_scenario_boundary_matches_real_calculator_contract[period1] ____ + +forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) +period = {'start_year': '2024', 'window_size': 2} +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a043c50> + + @pytest.mark.parametrize( + "period", + [ + {"time_period": "2024"}, + {"start_year": "2024", "window_size": 2}, + ], + ) + def test_worker_scenario_boundary_matches_real_calculator_contract( + forecast, period, monkeypatch + ): + from spm_calculator.axiom_adapter import export_build_spec + from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail + + scenario = "scenario-not-in-the-pinned-forecast" + with pytest.raises(ValueError) as adapter_error: + export_build_spec(forecast, years=[2024], scenario=scenario) + assert adapter_error.value.code == "SPM_SCENARIO_UNAVAILABLE" + expected = spm_error_detail(adapter_error.value) +> assert expected is not None +E assert None is not None + +projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:89: AssertionError +=============================== warnings summary =============================== +../../rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290 + /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290: PytestAssertRewriteWarning: Module already imported so cannot be rewritten; logfire + self._mark_plugins_for_rewrite(hook, disable_autoload) + +../../rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290 + /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290: PytestAssertRewriteWarning: Module already imported so cannot be rewritten; anyio + self._mark_plugins_for_rewrite(hook, disable_autoload) + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +=========================== short test summary info ============================ +FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period0-2021] +FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period1-2036] +FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period2-2040] +FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period3-2021] +FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period4-2036] +FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period5-2036] +FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_scenario_boundary_matches_real_calculator_contract[period0] +FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_scenario_boundary_matches_real_calculator_contract[period1] +8 failed, 1 passed, 2 warnings in 1.21s From 80c08f711b4c35b4233d43223b8e52553b282160 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:27:35 -0400 Subject: [PATCH 08/40] Document authenticated import order and handoff fingerprints --- rollout/worker-fable-fixes/handoff-preservation.json | 4 ++++ rollout/worker-fable-fixes/run_installed_qualification.py | 3 +++ 2 files changed, 7 insertions(+) create mode 100644 rollout/worker-fable-fixes/handoff-preservation.json diff --git a/rollout/worker-fable-fixes/handoff-preservation.json b/rollout/worker-fable-fixes/handoff-preservation.json new file mode 100644 index 000000000..77cf087d7 --- /dev/null +++ b/rollout/worker-fable-fixes/handoff-preservation.json @@ -0,0 +1,4 @@ +{ + "PROGRESS.md": "505f06cd28aa0624c21b1292bdb20f426dd71f15dee1f7db6ccf9ebc9ee98833", + "WORKER-CANONICAL-SPM-HANDOFF.md": "6d21bc5dd4b094d0e2b48d175fc021e13152584c7a964f471bfebb62401299ef" +} diff --git a/rollout/worker-fable-fixes/run_installed_qualification.py b/rollout/worker-fable-fixes/run_installed_qualification.py index d23ebe729..4ef7ed25b 100644 --- a/rollout/worker-fable-fixes/run_installed_qualification.py +++ b/rollout/worker-fable-fixes/run_installed_qualification.py @@ -1,5 +1,8 @@ """Four-person worker and lazy-year smoke against new exact installed candidate wheels.""" +# Imports below authentication must run after the explicit development bootstrap. +# ruff: noqa: E402 + import hashlib import importlib import importlib.metadata as metadata From 0bf61dfcd3de2b93e189ad14ef6888936f0c3eb0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:28:44 -0400 Subject: [PATCH 09/40] fix: preserve proven historical SPM routes and isolate bundle capabilities --- .../policyengine_simulation_contract/spm.py | 24 +++- .../tests/test_spm_selection.py | 34 ++++++ .../endpoints.py | 103 +++++++++++++----- .../tests/test_spm_routes.py | 45 ++++++-- 4 files changed, 172 insertions(+), 34 deletions(-) diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py index 623fc9a39..9b3f245ce 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py @@ -62,6 +62,10 @@ def validate_as_of(cls, value): @model_validator(mode="after") def validate_location(self): + # Request options inherit bundle defaults. Validate coupled options + # once both are explicit (including after defaults are resolved). + if not {"geography_kind", "geography_id"} <= self.model_fields_set: + return self if self.geography_kind == "metro": if not self.geography_id or not self.geography_id.strip(): raise ValueError("An SPM area selection requires geography_id") @@ -93,6 +97,7 @@ class SPMProvenance(BaseModel): "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED", "SPM_YEAR_UNAVAILABLE", + "SPM_SCENARIO_UNAVAILABLE", "SPM_CONFIGURATION_UNAVAILABLE", "SPM_SETTINGS_INVALID", } @@ -158,7 +163,13 @@ class SPMComparisonProvenance(BaseModel): def resolve_spm_selection( - country, selection, *, capability, policyengine_version, model_version + country, + selection, + *, + capability, + policyengine_version, + model_version, + route_provenance=None, ): """Resolve only certified metadata; unrecognized future bundles fail closed.""" if country.lower() != "us": @@ -179,6 +190,17 @@ def resolve_spm_selection( historical = historical or ( policyengine_version in {"5.2.0", "5.3.0"} and model_version == "1.764.6" ) + # Country-only routes seeded from the original registry may have no + # wrapper version. Require both their actual route provenance and a + # pre-canonical US model; absence of capability alone proves nothing. + model_parts = str(model_version or "").split(".") + historical = historical or ( + policyengine_version is None + and route_provenance in {"legacy-country-dict", "legacy-seed"} + and len(model_parts) == 3 + and all(re.fullmatch(r"[0-9]+", p) for p in model_parts) + and tuple(map(int, model_parts)) <= (1, 764, 6) + ) if selection is None and historical: return None raise SPMInputError( diff --git a/libs/policyengine-simulation-contract/tests/test_spm_selection.py b/libs/policyengine-simulation-contract/tests/test_spm_selection.py index f69f5dc58..452ab51c0 100644 --- a/libs/policyengine-simulation-contract/tests/test_spm_selection.py +++ b/libs/policyengine-simulation-contract/tests/test_spm_selection.py @@ -50,3 +50,37 @@ def test_explicit_national_discards_default_metro_area(): resolved = resolve({"geography_kind": "national"}) assert resolved["geography_kind"] == "national" assert resolved["geography_id"] is None + + +@pytest.mark.parametrize( + "provenance,wrapper,model,accepted", + [ + ("legacy-country-dict", None, "1.500.0", True), + ("legacy-seed", None, "1.715.2", True), + ("legacy-seed", None, "1.764.6", True), + (None, None, "1.715.2", False), + ("unknown", None, "1.715.2", False), + ("legacy-seed", None, "1.764.7", False), + ("legacy-seed", None, "unknown", False), + ("legacy-country-dict", "5.3.1", "1.715.2", False), + ], +) +def test_legacy_allowance_requires_route_provenance_and_historical_model( + provenance, wrapper, model, accepted +): + kwargs = dict( + capability=None, + policyengine_version=wrapper, + model_version=model, + route_provenance=provenance, + ) + if accepted: + assert resolve_spm_selection("us", None, **kwargs) is None + else: + with pytest.raises(SPMInputError) as error: + resolve_spm_selection("us", None, **kwargs) + assert error.value.code == "SPM_CONFIGURATION_UNAVAILABLE" + for selection in ({}, {"geography_kind": "national"}): + with pytest.raises(SPMInputError) as error: + resolve_spm_selection("us", selection, **kwargs) + assert error.value.code == "SPM_CONFIGURATION_UNAVAILABLE" diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py index 0228ab9d6..c53be3a59 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py @@ -26,6 +26,7 @@ from policyengine_simulation_gateway.auth import require_auth from policyengine_simulation_contract.spm import ( SPMCapability, + SPMInputError, spm_error_detail, resolve_spm_selection, ) @@ -81,6 +82,7 @@ class RouteResolution: response_version: str policyengine_version: str | None bundle_manifest: dict + route_provenance: str | None = None def _job_metadata_store(): @@ -343,22 +345,51 @@ def _bundle_manifest(state: dict, policyengine_version: str | None) -> dict: return manifest if isinstance(manifest, dict) else {} -def _policyengine_version_for_app(state: dict, app_name: str) -> str | None: - for policyengine_version, routed_app in _routing_state_routes( - state, "policyengine" - ).items(): - if routed_app == app_name: - return ( - policyengine_version if isinstance(policyengine_version, str) else None - ) - - for policyengine_version, manifest in _routing_state_bundles(state).items(): - if isinstance(manifest, dict) and manifest.get("app_name") == app_name: - return ( - policyengine_version if isinstance(policyengine_version, str) else None - ) - - return None +def _policyengine_version_for_app( + state: dict, app_name: str, *, country: str, model_version: str +) -> str | None: + candidates = { + version + for version, routed_app in _routing_state_routes(state, "policyengine").items() + if isinstance(version, str) and version != "latest" and routed_app == app_name + } | { + version + for version, manifest in _routing_state_bundles(state).items() + if isinstance(version, str) + and version != "latest" + and isinstance(manifest, dict) + and manifest.get("app_name") == app_name + } + matching = [] + unclassified = [] + for version in candidates: + manifest = _bundle_manifest(state, version) + country_bundle = manifest.get(country) + if ( + isinstance(country_bundle, dict) + and country_bundle.get("model_version") == model_version + ): + matching.append(version) + elif not isinstance(country_bundle, dict) or not isinstance( + country_bundle.get("model_version"), str + ): + unclassified.append(version) + if len(matching) == 1 and not unclassified: + return matching[0] + if len(candidates) > 1: + raise ValueError( + f"Ambiguous bundle for {country} version {model_version}; pass " + "policyengine_version to select a bundle explicitly" + ) + if not candidates: + return None + version = next(iter(candidates)) + _validate_legacy_version_matches_bundle( + country=country, + requested_version=model_version, + manifest=_bundle_manifest(state, version), + ) + return version def _policyengine_version_from_app_name(app_name: str) -> str | None: @@ -397,12 +428,17 @@ def _resolve_country_route( app_name = _routing_state_routes(state, country).get(version) if not isinstance(app_name, str): return None - policyengine_version = _policyengine_version_for_app(state, app_name) + policyengine_version = _policyengine_version_for_app( + state, app_name, country=country, model_version=version + ) return RouteResolution( app_name=app_name, response_version=version, policyengine_version=policyengine_version, bundle_manifest=_bundle_manifest(state, policyengine_version), + route_provenance=( + "legacy-seed" if state.get("generation") == "legacy-seed" else None + ), ) @@ -539,6 +575,7 @@ def _resolve_from_legacy_dicts( response_version=resolved_version, policyengine_version=_policyengine_version_from_app_name(app_name), bundle_manifest={}, + route_provenance="legacy-country-dict", ) @@ -573,6 +610,15 @@ def _build_policyengine_bundle( policyengine_version = app_bundle.get( "policyengine_version", resolution.policyengine_version ) + capability = app_bundle.get("spm") if country.lower() == "us" else None + if capability is not None: + try: + capability = SPMCapability.model_validate(capability) + except ValueError as exc: + raise SPMInputError( + "SPM_CONFIGURATION_UNAVAILABLE", + "This worker bundle has invalid canonical SPM capability metadata", + ) from exc return PolicyEngineBundle( model_version=str(model_version), policyengine_version=( @@ -580,17 +626,18 @@ def _build_policyengine_bundle( ), data_version=str(data_version) if isinstance(data_version, str) else None, dataset=resolved_dataset, - spm=app_bundle.get("spm") if country.lower() == "us" else None, + spm=capability, ) -def _resolve_request_spm(request, bundle): +def _resolve_request_spm(request, bundle, route): selection = resolve_spm_selection( request.country, request.spm, capability=bundle.spm, policyengine_version=bundle.policyengine_version, model_version=bundle.model_version, + route_provenance=route.route_provenance, ) if selection is not None: from policyengine_simulation_contract.spm import SPMSelection @@ -717,7 +764,7 @@ async def submit_simulation(request: SimulationRequest): route, payload, ) - _resolve_request_spm(request, bundle) + _resolve_request_spm(request, bundle, route) except (ValueError, HuggingFaceDatasetReferenceError) as exc: detail = spm_error_detail(exc) if detail: @@ -798,7 +845,7 @@ async def submit_budget_window_batch(request: BudgetWindowBatchRequest): route, request.model_dump(mode="json"), ) - _resolve_request_spm(request, bundle) + _resolve_request_spm(request, bundle, route) except (ValueError, HuggingFaceDatasetReferenceError) as exc: detail = spm_error_detail(exc) if detail: @@ -998,12 +1045,16 @@ async def list_versions() -> VersionsResponse: with segment(SegmentName.ROUTE_RESOLUTION): state = _active_routing_state() if state: + capabilities = {} + for name, bundle in _routing_state_bundles(state).items(): + if not isinstance(bundle, dict) or bundle.get("spm") is None: + continue + try: + capabilities[name] = SPMCapability.model_validate(bundle["spm"]) + except ValueError: + logger.warning("Omitting invalid SPM capability for bundle %s", name) return VersionsResponse( - spm_capabilities={ - name: SPMCapability.model_validate(bundle["spm"]) - for name, bundle in state.get("bundles", {}).items() - if isinstance(bundle, dict) and bundle.get("spm") is not None - }, + spm_capabilities=capabilities, policyengine=_version_map_from_state(state, "policyengine"), us=_version_map_from_state(state, "us"), uk=_version_map_from_state(state, "uk"), diff --git a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py index b31afe469..22953d8dd 100644 --- a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py +++ b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py @@ -23,6 +23,7 @@ def legacy_route(mock_modal, source, model_version): app_name = "policyengine-simulation-us1-715-2-uk2-88-20" if source == "legacy-country-dict": + app_name = "legacy-app" del mock_modal["dicts"]["simulation-api-routing-state"] mock_modal["dicts"]["simulation-api-us-versions"] = { "latest": model_version, @@ -63,9 +64,7 @@ def test_historical_route_rejects_any_explicit_spm( mock_modal, client, endpoint, extra, source, selection ): legacy_route(mock_modal, source, "1.715.2") - response = client.post( - endpoint, json={"country": "us", "spm": selection, **extra} - ) + response = client.post(endpoint, json={"country": "us", "spm": selection, **extra}) assert response.status_code == 400 assert response.json()["errors"][0]["code"] == "SPM_CONFIGURATION_UNAVAILABLE" assert mock_modal["func"].calls == [] @@ -120,6 +119,16 @@ def test_shared_app_resolves_unique_country_model_bundle( assert mock_modal["func"].last_payload["spm"]["scenario"] == "ce_trend" +def test_latest_route_alias_does_not_create_ambiguous_bundle(mock_modal, client): + state = shared_app_state(mock_modal, sibling_model="1.824.7") + state["routes"]["policyengine"]["latest"] = state["routes"]["policyengine"]["5.3.0"] + response = client.post( + "/simulate/economy/comparison", json={"country": "us", "version": "1.824.7"} + ) + assert response.status_code == 200 + assert response.json()["policyengine_bundle"]["policyengine_version"] == "5.3.0" + + @pytest.mark.parametrize("endpoint,extra", ENDPOINTS) def test_shared_app_with_ambiguous_country_model_requires_explicit_bundle( mock_modal, client, endpoint, extra @@ -139,13 +148,34 @@ def test_shared_app_with_ambiguous_country_model_requires_explicit_bundle( assert mock_modal["func"].last_payload["spm"]["scenario"] == "ce_trend" +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +def test_shared_app_with_missing_model_metadata_is_ambiguous( + mock_modal, client, endpoint, extra +): + state = shared_app_state(mock_modal, sibling_model="1.824.7") + del state["bundles"]["4.10.0"]["us"]["model_version"] + response = client.post( + endpoint, json={"country": "us", "version": "1.824.7", **extra} + ) + assert response.status_code == 400 + assert "policyengine_version" in response.json()["detail"] + assert mock_modal["func"].calls == [] + + +@pytest.mark.parametrize( + "invalid_capability", + [ + {"contract_version": "canonical-spm-v1", "defaults": {"scenario": "ce_trend"}}, + {**CAPABILITY, "contract_version": "unknown-contract"}, + {**CAPABILITY, "undeclared": True}, + {"defaults": {**CAPABILITY["defaults"], "geography_kind": "metro"}}, + ], +) def test_versions_omits_malformed_capability_and_submission_rejects_it( - mock_modal, client + mock_modal, client, invalid_capability ): state = shared_app_state(mock_modal, sibling_model="1.824.7") - state["bundles"]["4.10.0"]["spm"] = { - "contract_version": "canonical-spm-v1", "defaults": {"scenario": "ce_trend"} - } + state["bundles"]["4.10.0"]["spm"] = invalid_capability response = client.get("/versions") assert response.status_code == 200 assert set(response.json()["spm_capabilities"]) == {"5.3.0"} @@ -154,4 +184,5 @@ def test_versions_omits_malformed_capability_and_submission_rejects_it( json={"country": "us", "policyengine_version": "4.10.0"}, ) assert rejected.status_code == 400 + assert rejected.json()["errors"][0]["code"] == "SPM_CONFIGURATION_UNAVAILABLE" assert mock_modal["func"].calls == [] From 38020ce7891876aecc9a826037776463eef3a74f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:30:13 -0400 Subject: [PATCH 10/40] fix: preserve calculator SPM year and scenario errors in worker --- .../policyengine_simulation_executor/spm.py | 22 ++++++- .../tests/test_canonical_spm.py | 30 +++++++-- .../tests/test_spm_runtime_native.py | 19 ++++++ .../worker-fable-fixes/runtime-evidence.md | 30 +++++++-- .../runtime-regression-green.txt | 14 ++++ .../runtime-regression-red.txt | 65 +++++++++++++------ 6 files changed, 144 insertions(+), 36 deletions(-) create mode 100644 rollout/worker-fable-fixes/runtime-regression-green.txt diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py index b0193555a..6abb0ac2a 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py @@ -77,16 +77,27 @@ def normalize_runtime_spm(params): ) if selection is not None: try: + from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider + forecast = _forecast(selection["forecast_content_sha256"]) if selection["county_vintage"] != "2020": raise ValueError("Unsupported county vintage: use 2020") + provider = PolicyEngineSPMProvider( + forecast, + **{ + key: value + for key, value in selection.items() + if key != "forecast_content_sha256" + }, + ) from policyengine_simulation_executor.simulation_runtime import _parse_year start = int(params.get("start_year") or _parse_year(params)) for year in range(start, start + int(params.get("window_size", 1))): - forecast.entry( - year, scenario=selection["scenario"], as_of=selection["as_of"] - ) + # Use the country adapter's typed year contract. This temporary + # provider validates metadata without measuring any SPM amount + # or modifying the actual simulation's calculation receipts. + provider.year_metadata(year) if selection["geography_kind"] == "metro": try: forecast.geography_factor( @@ -105,6 +116,11 @@ def normalize_runtime_spm(params): detail = spm_error_detail(exc) if detail: raise SPMInputError(detail.code, detail.message) from exc + # The installed provider constructor still exposes the forecast's + # plain scenario error; use the same narrow translation as the + # calculator's Frame and Axiom adapters, leaving other errors alone. + if str(exc).startswith("Unknown forecast scenario:"): + raise SPMInputError("SPM_SCENARIO_UNAVAILABLE", str(exc)) from exc raise SPMInputError("SPM_SETTINGS_INVALID", str(exc)) from exc return selection diff --git a/projects/policyengine-simulation-executor/tests/test_canonical_spm.py b/projects/policyengine-simulation-executor/tests/test_canonical_spm.py index 6e3729139..966b08f4b 100644 --- a/projects/policyengine-simulation-executor/tests/test_canonical_spm.py +++ b/projects/policyengine-simulation-executor/tests/test_canonical_spm.py @@ -2,6 +2,7 @@ import json import pickle +import sys from copy import deepcopy from types import SimpleNamespace from unittest.mock import Mock @@ -241,11 +242,23 @@ def test_year_alias_is_validated_before_dataset_loading(monkeypatch): ), ) monkeypatch.setattr(spm, "runtime_spm_capability", lambda: CAPABILITY) - entry = Mock(side_effect=ValueError("2040 is unavailable")) - monkeypatch.setattr(spm, "_forecast", lambda sha: SimpleNamespace(entry=entry)) - with pytest.raises(SPMInputError, match="2040 is unavailable"): + year_metadata = Mock( + side_effect=SPMInputError("SPM_YEAR_UNAVAILABLE", "2040 is unavailable") + ) + monkeypatch.setattr(spm, "_forecast", lambda sha: object()) + monkeypatch.setitem( + sys.modules, + "spm_calculator.policyengine_adapter", + SimpleNamespace( + PolicyEngineSPMProvider=lambda *a, **k: SimpleNamespace( + year_metadata=year_metadata + ) + ), + ) + with pytest.raises(SPMInputError, match="2040 is unavailable") as error: spm.normalize_runtime_spm({"country": "us", "year": "2040", "spm": SELECTION}) - assert entry.call_args.args == (2040,) + assert error.value.code == "SPM_YEAR_UNAVAILABLE" + assert year_metadata.call_args.args == (2040,) @pytest.mark.parametrize( @@ -255,6 +268,7 @@ def test_year_alias_is_validated_before_dataset_loading(monkeypatch): "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED", "SPM_YEAR_UNAVAILABLE", + "SPM_SCENARIO_UNAVAILABLE", ], ) def test_actual_http_formula_error_contract(monkeypatch, code): @@ -400,6 +414,7 @@ def test_identity_errors_do_not_degrade_to_an_unidentified_baseline(monkeypatch) "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED", "SPM_YEAR_UNAVAILABLE", + "SPM_SCENARIO_UNAVAILABLE", ], ) def test_gateway_poll_returns_structured_400(monkeypatch, code): @@ -437,6 +452,7 @@ def test_gateway_submission_uses_registry_capability_before_spawn(monkeypatch): app_name="test-app", response_version="test-only", policyengine_version="test-only", + route_provenance=None, ) monkeypatch.setattr(endpoints, "resolve_route", lambda *args: route) bundle = PolicyEngineBundle( @@ -530,6 +546,7 @@ def test_budget_window_state_keeps_typed_failure_on_replay(): "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED", "SPM_YEAR_UNAVAILABLE", + "SPM_SCENARIO_UNAVAILABLE", ], ) def test_country_error_is_transportable_without_country_package(monkeypatch, code): @@ -559,6 +576,7 @@ def __init__(self, code, message): "SPM_GEOGRAPHY_UNAVAILABLE", "SPM_COMPOSITION_REQUIRED", "SPM_YEAR_UNAVAILABLE", + "SPM_SCENARIO_UNAVAILABLE", ], ) def test_optional_analysis_does_not_swallow_spm_input_errors(code): @@ -606,7 +624,9 @@ def test_explicit_null_as_of_survives_entrypoint_and_budget_parent(): "defaults": {**SELECTION, "as_of": "2026-01-01"}, }, ) - selection = _resolve_request_spm(request, bundle) + selection = _resolve_request_spm( + request, bundle, SimpleNamespace(route_provenance=None) + ) assert selection["as_of"] is None parent = _build_budget_window_parent_payload( request, resolved_version="test", resolved_app_name="test", bundle=bundle diff --git a/projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py b/projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py index 14115940a..a43aee2f9 100644 --- a/projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py +++ b/projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py @@ -122,3 +122,22 @@ def test_valid_worker_prevalidation_does_not_evaluate_amounts(forecast, monkeypa ) assert provider.provenance()["years"] == {} calculate.assert_not_called() + + +@pytest.mark.parametrize( + "selection", + [{"as_of": "2020-01-01"}, {"county_vintage": "2010"}], +) +def test_other_invalid_settings_keep_the_settings_code(selection): + from policyengine_simulation_contract.spm import SPMInputError + from policyengine_simulation_executor.spm import normalize_runtime_spm + + with pytest.raises(SPMInputError) as error: + normalize_runtime_spm( + { + "country": "us", + "time_period": "2024", + "spm": {"geography_kind": "national", **selection}, + } + ) + assert error.value.code == "SPM_SETTINGS_INVALID" diff --git a/rollout/worker-fable-fixes/runtime-evidence.md b/rollout/worker-fable-fixes/runtime-evidence.md index 3d69e2191..b11963b04 100644 --- a/rollout/worker-fable-fixes/runtime-evidence.md +++ b/rollout/worker-fable-fixes/runtime-evidence.md @@ -1,17 +1,33 @@ # Runtime year and scenario error evidence -State: regression reproduced against the authenticated installed calculator and country/wrapper environment; implementation pending. +State: Fable finding 2 fixed and verified against the authenticated installed calculator, country, core, and wrapper. Root-owned integrated qualification and final source binding remain separate. -The exact Fable reviewer output and repository testing skill were read. The installed `PolicyEngineSPMProvider.year_metadata` supplies the typed year contract. Its constructor still raises a plain unknown-scenario `ValueError`; the calculator's public Axiom `export_build_spec` translates that condition into `SPM_SCENARIO_UNAVAILABLE`. Worker prevalidation currently reports `SPM_SETTINGS_INVALID` instead and the shared error recognizer drops the scenario code entirely. +Worker prevalidation now uses the installed `PolicyEngineSPMProvider.year_metadata` typed year contract. Its constructor still emits a plain unknown-scenario `ValueError`; the worker uses the same narrow scenario translation as the calculator's Frame and Axiom adapters. Other invalid settings retain `SPM_SETTINGS_INVALID`. The shared contract recognizes `SPM_SCENARIO_UNAVAILABLE` so transport does not redact that code. Scientific packages and their source were not changed. -Regression-first command: +The new native tests compare actual provider/Axiom errors with annual and budget-window worker entrypoints before dataset/child setup. They cover 2021, 2036, the `year=2040` alias, unavailable window starts and a 2035–2036 window end, unknown scenarios, and invalid as-of/vintage settings. The valid-year case forbids amount evaluation and checks that a fresh real provider still has empty receipt years. Existing HTTP, polling, pickle, country-error, and optional-output tests now include the scenario code. The parent integrated run also reuses the prior 12 installed provider/country tests, including actual out-of-range tax-only calculations followed by typed threshold failure. + +## Clean regression and verification commands ```sh -env -u PYTHONPATH UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache uv run --no-project --python /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/bin/python /private/tmp/run_worker_runtime_regressions.py > /private/tmp/worker-runtime-regression-red.txt 2>&1 +env -u PYTHONPATH PYTHONSAFEPATH=1 UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache uv run --no-project --python /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/bin/python /private/tmp/run_worker_runtime_regressions.py --original-runtime --basetemp /private/tmp/worker-runtime-clean-red > /private/tmp/worker-runtime-clean-red.txt 2>&1 + +env -u PYTHONPATH PYTHONSAFEPATH=1 UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache uv run --no-project --python /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/bin/python /private/tmp/run_worker_runtime_regressions.py projects/policyengine-simulation-executor/tests/test_canonical_spm.py --basetemp /private/tmp/worker-runtime-clean-green > /private/tmp/worker-runtime-clean-green.txt 2>&1 + +UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache uv run --project projects/policyengine-simulation-executor --no-sync ruff format projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py projects/policyengine-simulation-executor/tests/test_canonical_spm.py projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py + +UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache uv run --project projects/policyengine-simulation-executor --no-sync ruff check projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py projects/policyengine-simulation-executor/tests/test_canonical_spm.py projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py ``` -The temporary launcher executes only the original qualification script's authenticated wheel/bootstrap setup (before `import pytest`) and invokes the committed `tests/test_spm_runtime_native.py` with pytest `-q -p no:cacheprovider --basetemp /private/tmp/worker-runtime-regression-pytest`. It neither rewrites historical evidence nor modifies installed packages. Initial invocation with inherited PYTHONPATH stopped at the original bootstrap's explicit guard; rerun unsets it. +The temporary launcher executes only the original qualification script's authenticated wheel/bootstrap setup (before `import pytest`) and invokes committed `tests/test_spm_runtime_native.py` with pytest `-q -p no:cacheprovider`. It asserts `/tmp` and `/private/tmp` are absent from `sys.path`. With `--original-runtime`, it overlays only the archived `dcfe4fd` executor `spm.py` from `/private/tmp/worker-fable-base`; it retains current shared transport recognition so the scenario regression reaches the original worker mapping itself. No checkout, installed-package edit, or fabricated metadata is involved. + +- Clean RED: **8 failed, 3 passed, zero skipped, 2.12 seconds**. All six year and both scenario boundary cases fail on the original worker with `SPM_SETTINGS_INVALID` in place of the calculator code. Full output: `runtime-regression-red.txt`. +- Clean GREEN: **56 passed, zero skipped, 0.28 seconds** across the 11 native cases and 45 canonical worker/transport cases. Full output: `runtime-regression-green.txt`. +- Ruff formatting and lint: pass. The two pytest plugin-rewrite warnings reflect explicit bootstrap before pytest, as in the prior qualification. + +## Superseded attempts and limits + +The first launcher invocation safely stopped at the inherited-PYTHONPATH guard. A service-local `uv run --no-sync pytest tests/test_canonical_spm.py -q` attempt selected an ambient Python 3.14 tool and could not collect the missing contract package; it is not validation evidence. No environment was changed. -Result before fixes: **8 failed, 1 passed, zero skipped, 1.21 seconds**. Six annual/year-alias/budget-window start/end cases showed the wrong year code; two real calculator scenario-contract cases showed the missing public scenario-code recognition. Valid-year prevalidation performs no amount calculation. Two plugin-rewrite warnings reflect explicit bootstrap before pytest, as in the prior qualification. +Early temporary-launcher RED/GREEN outputs lacked Python safe-path isolation: unrelated `/private/tmp/h2.py` shadowed optional HTTP/2 and imported an unrelated demonstration. Those outputs are superseded and are not qualification evidence. The unrelated files were not edited. The committed RED output was replaced by the clean isolated run above; both clean runs have no unrelated demo output. The final root-owned harness lives inside the repository and authenticates installed sources again. -Next: preserve typed adapter errors, test full public error transport, run the integrated installed qualification including the prior real-country lazy tax-only tests, and record final outcomes/source binding in the final report. +Remaining coupling: the installed PolicyEngine adapter lacks a public typed scenario-selection validator, so the narrow scenario translation follows the real calculator adapters' current contract. The tests execute the public Axiom adapter to detect drift. No amount formulas, installed receipt metadata, publication, population execution, deployment, or merge changed. diff --git a/rollout/worker-fable-fixes/runtime-regression-green.txt b/rollout/worker-fable-fixes/runtime-regression-green.txt new file mode 100644 index 000000000..87774a0f2 --- /dev/null +++ b/rollout/worker-fable-fixes/runtime-regression-green.txt @@ -0,0 +1,14 @@ +WARN `--no-project` was provided, but no project was found +warning: `--frozen` has no effect when used alongside `--no-project` +........................................................ [100%] +=============================== warnings summary =============================== +../../rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290 + /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290: PytestAssertRewriteWarning: Module already imported so cannot be rewritten; logfire + self._mark_plugins_for_rewrite(hook, disable_autoload) + +../../rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290 + /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290: PytestAssertRewriteWarning: Module already imported so cannot be rewritten; anyio + self._mark_plugins_for_rewrite(hook, disable_autoload) + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +56 passed, 2 warnings in 0.28s diff --git a/rollout/worker-fable-fixes/runtime-regression-red.txt b/rollout/worker-fable-fixes/runtime-regression-red.txt index 034d0afee..20ec7a72d 100644 --- a/rollout/worker-fable-fixes/runtime-regression-red.txt +++ b/rollout/worker-fable-fixes/runtime-regression-red.txt @@ -1,17 +1,12 @@ WARN `--no-project` was provided, but no project was found warning: `--frozen` has no effect when used alongside `--no-project` -RUN_EXPERIMENT EVENTS: [] -wrote /tmp/v/results/invented_result.json -domain allocation: occupied_housing_unit=100 group_quarters_person=10 -whole-arm reference: occupied_housing_unit=1050/11 group_quarters_person=50/11 -refusal cases demonstrated: 12, unrefused: [] -FFFFFFFF. [100%] +FFFFFFFF... [100%] =================================== FAILURES =================================== ________ test_worker_year_boundary_matches_real_provider[period0-2021] _________ forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) period = {'time_period': '2021'}, unavailable_year = 2021 -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13862c2b0> +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x138a20c30> @pytest.mark.parametrize( "period,unavailable_year", @@ -53,7 +48,7 @@ ________ test_worker_year_boundary_matches_real_provider[period1-2036] _________ forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) period = {'time_period': '2036'}, unavailable_year = 2036 -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13862c050> +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x138a20e90> @pytest.mark.parametrize( "period,unavailable_year", @@ -95,7 +90,7 @@ ________ test_worker_year_boundary_matches_real_provider[period2-2040] _________ forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) period = {'year': '2040'}, unavailable_year = 2040 -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a0a28d0> +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13bd58290> @pytest.mark.parametrize( "period,unavailable_year", @@ -137,7 +132,7 @@ ________ test_worker_year_boundary_matches_real_provider[period3-2021] _________ forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) period = {'start_year': '2021', 'window_size': 1}, unavailable_year = 2021 -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a01b9b0> +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13bdae7a0> @pytest.mark.parametrize( "period,unavailable_year", @@ -179,7 +174,7 @@ ________ test_worker_year_boundary_matches_real_provider[period4-2036] _________ forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) period = {'start_year': '2036', 'window_size': 1}, unavailable_year = 2036 -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a0b09e0> +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13bdaf9b0> @pytest.mark.parametrize( "period,unavailable_year", @@ -221,7 +216,7 @@ ________ test_worker_year_boundary_matches_real_provider[period5-2036] _________ forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) period = {'start_year': '2035', 'window_size': 2}, unavailable_year = 2036 -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a03f650> +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13bd96650> @pytest.mark.parametrize( "period,unavailable_year", @@ -263,7 +258,7 @@ ___ test_worker_scenario_boundary_matches_real_calculator_contract[period0] ____ forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) period = {'time_period': '2024'} -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a03ed50> +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13bd95d50> @pytest.mark.parametrize( "period", @@ -283,15 +278,29 @@ monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a03ed50> export_build_spec(forecast, years=[2024], scenario=scenario) assert adapter_error.value.code == "SPM_SCENARIO_UNAVAILABLE" expected = spm_error_detail(adapter_error.value) -> assert expected is not None -E assert None is not None + assert expected is not None + params = { + "country": "us", + "spm": {"geography_kind": "national", "scenario": scenario}, + **period, + } + entrypoint, boundary = _worker_entrypoint(params, monkeypatch) + with pytest.raises(SPMInputError) as error: + entrypoint(params) +> assert error.value.to_dict() == expected.model_dump() +E AssertionError: assert {'code': 'SPM...ned-forecast'} == {'code': 'SPM...ned-forecast'} +E +E Omitting 1 identical items, use -vv to show +E Differing items: +E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_SCENARIO_UNAVAILABLE'} +E Use -v to get more diff -projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:89: AssertionError +projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:98: AssertionError ___ test_worker_scenario_boundary_matches_real_calculator_contract[period1] ____ forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) period = {'start_year': '2024', 'window_size': 2} -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a043c50> +monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13bd834d0> @pytest.mark.parametrize( "period", @@ -311,10 +320,24 @@ monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13a043c50> export_build_spec(forecast, years=[2024], scenario=scenario) assert adapter_error.value.code == "SPM_SCENARIO_UNAVAILABLE" expected = spm_error_detail(adapter_error.value) -> assert expected is not None -E assert None is not None + assert expected is not None + params = { + "country": "us", + "spm": {"geography_kind": "national", "scenario": scenario}, + **period, + } + entrypoint, boundary = _worker_entrypoint(params, monkeypatch) + with pytest.raises(SPMInputError) as error: + entrypoint(params) +> assert error.value.to_dict() == expected.model_dump() +E AssertionError: assert {'code': 'SPM...ned-forecast'} == {'code': 'SPM...ned-forecast'} +E +E Omitting 1 identical items, use -vv to show +E Differing items: +E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_SCENARIO_UNAVAILABLE'} +E Use -v to get more diff -projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:89: AssertionError +projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:98: AssertionError =============================== warnings summary =============================== ../../rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290 /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290: PytestAssertRewriteWarning: Module already imported so cannot be rewritten; logfire @@ -334,4 +357,4 @@ FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.p FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period5-2036] FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_scenario_boundary_matches_real_calculator_contract[period0] FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_scenario_boundary_matches_real_calculator_contract[period1] -8 failed, 1 passed, 2 warnings in 1.21s +8 failed, 3 passed, 2 warnings in 2.12s From 203877850be3463b3fc2b93f6f183b7f664142c5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:30:39 -0400 Subject: [PATCH 11/40] test: reproduce incomplete SPM artifact and precompute key failures --- .../tests/native_spm_support.py | 36 ++++ .../test_baseline_artifacts_spm_native.py | 163 ++++++++++++++++++ .../tests/test_canonical_spm_native.py | 35 +--- .../tests/test_precompute.py | 9 + .../worker-fable-fixes/artifact-evidence.md | 45 +++++ .../worker-fable-fixes/run_artifact_native.py | 14 ++ 6 files changed, 270 insertions(+), 32 deletions(-) create mode 100644 projects/policyengine-simulation-executor/tests/native_spm_support.py create mode 100644 projects/policyengine-simulation-executor/tests/test_baseline_artifacts_spm_native.py create mode 100644 rollout/worker-fable-fixes/artifact-evidence.md create mode 100644 rollout/worker-fable-fixes/run_artifact_native.py diff --git a/projects/policyengine-simulation-executor/tests/native_spm_support.py b/projects/policyengine-simulation-executor/tests/native_spm_support.py new file mode 100644 index 000000000..e88abf6af --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/native_spm_support.py @@ -0,0 +1,36 @@ +"""One-household native dataset shared by installed SPM integration tests.""" + + +def materialize_native_household(source_path, tmp_path): + import pandas as pd + from policyengine.tax_benefit_models.us import ensure_datasets + + tiny = tmp_path / "native.h5" + with pd.HDFStore(source_path, "r") as source, pd.HDFStore(tiny, "w") as dest: + household = source.select("household", start=0, stop=1) + household_id = int(household.household_id.iloc[0]) + people = source.select("person", where=f"person_household_id == {household_id}") + assert 0 < len(people) <= 20 + dest.put("household", household, format="table", data_columns=True) + dest.put("person", people, format="table", data_columns=True) + for entity in ("spm_unit", "tax_unit", "family", "marital_unit"): + ids = [int(value) for value in people[f"person_{entity}_id"].unique()] + dest.put( + entity, + source.select(entity, where=f"{entity}_id in {ids}"), + format="table", + data_columns=True, + ) + dest.put("_time_period", source.select("_time_period"), format="table") + # Unmanaged is an explicit test-only path; production keeps certified bundle + # loading. This exercises the real native-to-year materializer. + datasets = ensure_datasets( + datasets=[str(tiny)], + years=[2024], + data_folder=str(tmp_path / "year-data"), + allow_unmanaged=True, + ) + dataset = next(iter(datasets.values())) + dataset.load() + assert len(dataset.data.person) == len(people) + return dataset diff --git a/projects/policyengine-simulation-executor/tests/test_baseline_artifacts_spm_native.py b/projects/policyengine-simulation-executor/tests/test_baseline_artifacts_spm_native.py new file mode 100644 index 000000000..45e9ed271 --- /dev/null +++ b/projects/policyengine-simulation-executor/tests/test_baseline_artifacts_spm_native.py @@ -0,0 +1,163 @@ +"""Real one-household SPM artifacts with deliberate local corruption. + +The producer always calculates authentic receipts with the installed country and +calculator. Corruption cases then damage only that temporary result artifact. +""" + +import json +import os +from pathlib import Path + +import pytest + +from native_spm_support import materialize_native_household + +pytestmark = pytest.mark.skipif( + not os.environ.get("SPM_NATIVE_SMOKE_SOURCE"), + reason="Requires the explicitly selected local native H5 and installed SPM bundle", +) + + +@pytest.fixture +def native_dataset(tmp_path): + return materialize_native_household(os.environ["SPM_NATIVE_SMOKE_SOURCE"], tmp_path) + + +def _baseline(dataset, *, geography="national", tax_only=False): + from policyengine_simulation_executor.baseline_artifacts import ( + ArtifactBaselineSimulation, + ) + from policyengine_simulation_executor.simulation_runtime import _build_simulation + + built = _build_simulation( + { + "country": "us", + "scope": "macro", + "time_period": "2024", + "data": "bounded-native-test", + "spm": {"geography_kind": geography}, + }, + dataset=dataset, + policy=None, + region_code="us", + ) + baseline = ArtifactBaselineSimulation.model_construct(**built.__dict__) + baseline.id = "bl1-native-receipt" + if tax_only: + # Produce a genuine lazy, tax-only output under the same selection. + model = baseline.tax_benefit_model_version + baseline.tax_benefit_model_version = model.model_copy( + update={ + "entity_variables": { + entity: ["income_tax"] if entity == "tax_unit" else [] + for entity in model.entity_variables + } + } + ) + return baseline + + +@pytest.mark.parametrize( + "damage, expected_outcome", + [ + ("tax_only_columns", "incomplete"), + ("empty_years", "incomplete"), + ("missing_receipt", "miss"), + ("malformed_receipt", "miss"), + ("different_selection", "miss"), + ("corrupt_hdf", "miss"), + ], +) +def test_incomplete_or_corrupt_spm_disk_artifact_recomputes( + native_dataset, monkeypatch, damage, expected_outcome +): + import h5py + from policyengine.core.simulation import _cache + from policyengine_simulation_executor.baseline_artifacts import ( + ArtifactBaselineSimulation, + ) + from policyengine_simulation_executor.spm import simulation_spm_result + + _cache._cache.clear() + producer = _baseline(native_dataset, tax_only=damage == "tax_only_columns") + producer.run() + producer.save() + path = Path(producer.output_dataset.filepath) + if damage == "tax_only_columns": + assert producer.spm_provenance()["years"] == {} + assert ( + "spm_unit_is_in_spm_poverty" + not in producer.output_dataset.data.spm_unit.columns + ) + elif damage == "corrupt_hdf": + path.write_bytes(b"deliberately corrupted test artifact") + else: + with h5py.File(path, "a") as stream: + recorded = json.loads(stream["policyengine_spm"].asstr()[()]) + del stream["policyengine_spm"] + if damage == "empty_years": + recorded["provenance"]["years"] = {} + elif damage == "malformed_receipt": + recorded["provenance"]["unexpected_test_field"] = True + elif damage == "different_selection": + recorded["config"]["geography_kind"] = "county" + if damage != "missing_receipt": + stream.create_dataset( + "policyengine_spm", + data=json.dumps(recorded), + dtype=h5py.string_dtype("utf-8"), + ) + + runs = [] + real_run = ArtifactBaselineSimulation.run + + def counted_run(simulation): + runs.append(simulation) + return real_run(simulation) + + monkeypatch.setattr(ArtifactBaselineSimulation, "run", counted_run) + consumer = _baseline(native_dataset) + requested = consumer.spm_config + consumer.ensure() + assert consumer.artifact_outcome == expected_outcome + assert runs == [consumer] + assert consumer.spm_config == requested + assert not consumer._missing_output_columns() + simulation_spm_result(consumer, consumer, requested, expected_year=2024) + # A second request uses the repaired cache; clearing it tests repaired disk. + cached = _baseline(native_dataset) + cached.ensure() + assert cached.artifact_outcome == "hit" + _cache._cache.clear() + reloaded = _baseline(native_dataset) + reloaded.ensure() + assert reloaded.artifact_outcome == "hit" + assert runs == [consumer] + assert reloaded.spm_provenance() == consumer.spm_provenance() + + +@pytest.mark.parametrize("damage", ["missing_receipt", "different_selection"]) +def test_invalid_spm_cache_entry_recomputes_requested_selection(native_dataset, damage): + from policyengine.core.simulation import _cache + from policyengine_simulation_executor.spm import simulation_spm_result + + _cache._cache.clear() + producer = _baseline( + native_dataset, + geography="county" if damage == "different_selection" else "national", + ) + producer.run() + if damage == "missing_receipt": + producer.spm_receipt = None + consumer = _baseline(native_dataset) + requested = consumer.spm_config + # Deliberate local cache corruption must never replace the request selection. + _cache.add(consumer.storage_id, producer) + consumer.ensure() + assert consumer.artifact_outcome == "incomplete" + assert consumer.spm_config == requested + simulation_spm_result(consumer, consumer, requested, expected_year=2024) + later = _baseline(native_dataset) + later.ensure() + assert later.artifact_outcome == "hit" + assert later.spm_provenance() == consumer.spm_provenance() diff --git a/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py b/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py index 223486d04..e6d4987da 100644 --- a/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py +++ b/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py @@ -9,6 +9,8 @@ import pytest +from native_spm_support import materialize_native_household + SOURCE = os.environ.get("SPM_NATIVE_SMOKE_SOURCE") pytestmark = pytest.mark.skipif( not SOURCE, @@ -18,38 +20,7 @@ @pytest.fixture def native_dataset(tmp_path): - import pandas as pd - from policyengine.tax_benefit_models.us import ensure_datasets - - tiny = tmp_path / "native.h5" - with pd.HDFStore(SOURCE, "r") as source, pd.HDFStore(tiny, "w") as dest: - household = source.select("household", start=0, stop=1) - household_id = int(household.household_id.iloc[0]) - people = source.select("person", where=f"person_household_id == {household_id}") - assert 0 < len(people) <= 20 - dest.put("household", household, format="table", data_columns=True) - dest.put("person", people, format="table", data_columns=True) - for entity in ("spm_unit", "tax_unit", "family", "marital_unit"): - ids = [int(value) for value in people[f"person_{entity}_id"].unique()] - dest.put( - entity, - source.select(entity, where=f"{entity}_id in {ids}"), - format="table", - data_columns=True, - ) - dest.put("_time_period", source.select("_time_period"), format="table") - # Unmanaged is an explicit test-only path; production keeps certified bundle - # loading. This exercises the real native-to-year materializer. - datasets = ensure_datasets( - datasets=[str(tiny)], - years=[2024], - data_folder=str(tmp_path / "year-data"), - allow_unmanaged=True, - ) - dataset = next(iter(datasets.values())) - dataset.load() - assert len(dataset.data.person) == len(people) - return dataset + return materialize_native_household(os.environ["SPM_NATIVE_SMOKE_SOURCE"], tmp_path) def test_worker_baseline_reform_national_local_cache_and_receipts( diff --git a/projects/policyengine-simulation-executor/tests/test_precompute.py b/projects/policyengine-simulation-executor/tests/test_precompute.py index 172a3a57d..361420cd4 100644 --- a/projects/policyengine-simulation-executor/tests/test_precompute.py +++ b/projects/policyengine-simulation-executor/tests/test_precompute.py @@ -768,6 +768,15 @@ def test_refuses_to_act_under_a_mismatched_id(self, cohort_stubs): precompute.compute_baseline_impl("bucket-x", self._entry("bl1-other")) assert cohort_stubs.uploads == [] + def test_refuses_a_mismatched_planned_storage_id(self, cohort_stubs): + entry = self._entry() + entry.path = "baselines/us/bl-d/bl1-cohort-spm-planned.h5" + with pytest.raises(RuntimeError, match="storage ids disagree"): + precompute.compute_baseline_impl("bucket-x", entry) + assert cohort_stubs.configured == [] + assert cohort_stubs.uploads == [] + assert not (cohort_stubs.folder / "bl1-cohort.h5").exists() + def test_refuses_a_plain_simulation(self, cohort_stubs): cohort_stubs.baseline = SimpleNamespace(id="bl1-cohort") with pytest.raises(RuntimeError, match="plain Simulation"): diff --git a/rollout/worker-fable-fixes/artifact-evidence.md b/rollout/worker-fable-fixes/artifact-evidence.md new file mode 100644 index 000000000..f8eb5cfa0 --- /dev/null +++ b/rollout/worker-fable-fixes/artifact-evidence.md @@ -0,0 +1,45 @@ +# Artifact findings 3 and 4 + +State: regression-first tests added; implementation follows in the next commit. +The exact Fable review from gate `20260909-170551-pr-abdf4629`, round +`001-4542bf03eca1`, was read along with the repository testing skill. No durable +agreement is inferred from that review. + +The installed regression harness authenticates the original c49c calculator, +76448 country, 8c640 wrapper and 2dbc core wheels with the existing qualification +script's complete verification/bootstrap prefix. It uses the original explicit +unverified development manifest; it does not synthesize receipt/version metadata. +Tests calculate authentic one-household receipts and deliberately damage only +local temporary artifacts or cache entries when exercising corruption handling. + +Clean regression command (worker repository root): + +```sh +env -u PYTHONPATH UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache \ + uv run --no-project \ + --python /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/bin/python \ + python -P rollout/worker-fable-fixes/run_artifact_native.py \ + projects/policyengine-simulation-executor/tests/test_baseline_artifacts_spm_native.py \ + projects/policyengine-simulation-executor/tests/test_precompute.py \ + -q -p no:cacheprovider --basetemp /private/tmp/worker-fable-artifact-red-clean \ + > rollout/worker-fable-fixes/artifact-native-red-clean.log 2>&1 +``` + +The clean run imports the unchanged artifact implementations from `dcfe4fd`. +Both tax-only missing SPM columns and a complete artifact with empty receipt years +reproduced failures before implementation edits. The remaining test outcomes will +be recorded after completion. The missing/malformed receipt, damaged HDF, and +recorded-selection isolation cases exercise the actual installed wrapper load, +save, cache, and model calculation paths. A focused precompute test expects +rejection when simulation ids agree but the planned storage filename differs. + +An earlier attempt used `/private/tmp/run_artifact_native.py`; the script directory +made unrelated `/private/tmp/h2.py` shadow an optional dependency, so that run was +interrupted and is superseded. The repository-owned launcher plus `python -P` +removes that import path; its clean output has none of the unrelated experiment +text. An initial bare `uv run --no-sync pytest` selected an inherited Python 3.14 +executable and failed collection; it provides no test evidence. All qualification +commands now select the authenticated Python 3.13 environment explicitly. + +Ruff check passes for the four new/updated test files after formatting. Root +handoff files were left untouched. diff --git a/rollout/worker-fable-fixes/run_artifact_native.py b/rollout/worker-fable-fixes/run_artifact_native.py new file mode 100644 index 000000000..754ca2add --- /dev/null +++ b/rollout/worker-fable-fixes/run_artifact_native.py @@ -0,0 +1,14 @@ +from pathlib import Path +import sys + +original = Path( + "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/run_installed_native_qualification.py" +) +namespace = {"__file__": str(original), "__name__": "artifact_regression"} +exec( + compile(original.read_text().split("import pytest\n", 1)[0], str(original), "exec"), + namespace, +) +import pytest + +raise SystemExit(pytest.main(sys.argv[1:])) From 56da1675ee15787202a476108a0d40e166fd155c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:30:52 -0400 Subject: [PATCH 12/40] docs: record Fable routing fixes and regression evidence --- .../worker-fable-fixes/routing-evidence.md | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 rollout/worker-fable-fixes/routing-evidence.md diff --git a/rollout/worker-fable-fixes/routing-evidence.md b/rollout/worker-fable-fixes/routing-evidence.md new file mode 100644 index 000000000..34065a7ff --- /dev/null +++ b/rollout/worker-fable-fixes/routing-evidence.md @@ -0,0 +1,56 @@ +# Gateway routing and selection evidence + +State: Fable finding 1 and the related shared-app, malformed-capability, and partial-selection notes are fixed. Regression tests were committed as `1d6a6ed`; production fixes and expanded rejection coverage were committed as `0bf61df`. The latter includes the runtime agent's narrow `SPM_SCENARIO_UNAVAILABLE` shared error-code addition. + +Done: + +- Read the exact reviewer output at `/Users/maxghenis/chief-of-staff/state/subfleet/gates/20260909-170551-pr-abdf4629/rounds/001-4542bf03eca1/peer-output.md`, repository testing/PR skills, and the actual `build_legacy_seed_routing_state` producer. +- Route resolution now carries internal provenance for the original country dictionary and `generation="legacy-seed"`. A missing wrapper version permits an ordinary no-SPM request only with this provenance and a numeric US model version at or before the already recognized historical `1.764.6` model. Missing provenance, unknown or later model metadata, future wrapper versions, and every explicit SPM selection fail closed. The classification is not a public request field. +- Annual and budget-window submission tests cover real supported registry shapes, explicit and latest country-version routing, missing wrapper versions, rejection without spawning, and unknown routes. Legacy-dictionary tests use `legacy-app`, for which wrapper-version inference is impossible. +- A shared app resolves a country-model request only when one bundle's country metadata matches and every sibling has enough country metadata to exclude it. Multiple matches or missing sibling model metadata require an explicit `policyengine_version`; latest aliases are excluded from exact-version candidates. +- Malformed stored capabilities are omitted from `/versions` with a warning. Submission against them returns typed `SPM_CONFIGURATION_UNAVAILABLE`, including otherwise historical bundles; no valid-looking capability or worker submission is produced. Tests cover missing pinned hash, unknown contract, extra fields, and incomplete metro defaults. +- Partial geography options are validated together after bundle defaults resolve. A geography-ID-only request can inherit metro defaults, an explicit metro can inherit its existing area, and incompatible county/national defaults still fail. Changing from metro to national clears the area. + +Regression-first commands (executed before production edits): + +```sh +# cwd: projects/policyengine-simulation-gateway +UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_PROJECT_ENVIRONMENT=../policyengine-simulation-executor/.venv PYTHONPATH=src:../../libs/policyengine-simulation-contract/src:../../libs/policyengine-simulation-observability/src:../../libs/policyengine-fastapi/src uv run --no-sync python -m pytest tests/test_spm_routes.py -q +# 13 failed, 16 passed: historical submissions, sibling capability selection, +# ambiguous shared-app selection, and malformed capability listing reproduced. + +# cwd: libs/policyengine-simulation-contract +UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_PROJECT_ENVIRONMENT=../../projects/policyengine-simulation-executor/.venv PYTHONPATH=src:../policyengine-simulation-observability/src uv run --no-sync python -m pytest tests/test_spm_selection.py -q +# 2 failed, 3 passed: geography-ID-only and metro-only inheritance reproduced. +``` + +Verification commands after fixes: + +```sh +# cwd: projects/policyengine-simulation-gateway +UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_PROJECT_ENVIRONMENT=../policyengine-simulation-executor/.venv PYTHONPATH=src:../../libs/policyengine-simulation-contract/src:../../libs/policyengine-simulation-observability/src:../../libs/policyengine-fastapi/src uv run --no-sync python -m pytest tests/test_endpoints.py tests/test_spm_routes.py -q +# 88 passed in 0.53 seconds. + +UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_PROJECT_ENVIRONMENT=../policyengine-simulation-executor/.venv PYTHONPATH=src:../../libs/policyengine-simulation-contract/src:../../libs/policyengine-simulation-observability/src:../../libs/policyengine-fastapi/src uv run --no-sync python -m pytest tests/ -q +# Initial full run: 138 passed, 1 deselected, 1 failed in 0.57 seconds. +# The checked-in OpenAPI golden omitted all pre-existing canonical SPM schemas. +# Root is updating that pre-existing stale golden and rerunning the full suite. + +# cwd: libs/policyengine-simulation-contract +UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_PROJECT_ENVIRONMENT=../../projects/policyengine-simulation-executor/.venv PYTHONPATH=src:../policyengine-simulation-observability/src uv run --no-sync python -m pytest tests/ -q +# 73 passed in 0.04 seconds. + +# cwd: repository root +ruff format --check projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py projects/policyengine-simulation-gateway/tests/test_spm_routes.py libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py libs/policyengine-simulation-contract/tests/test_spm_selection.py +ruff check projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py projects/policyengine-simulation-gateway/tests/test_spm_routes.py libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py libs/policyengine-simulation-contract/tests/test_spm_selection.py +git diff --check +# All pass. +``` + +The installed worker environment is Python 3.13.9; source overlays load the owned gateway/contract code. All gateway network/Modal seams are mocked. Native calculator verification belongs to the integrated qualification owned by root and the runtime agent. + +Schema equivalence was checked by loading the original contract source with `git show dcfe4fd89d97145882cb147a9aa2ffae2cd10ebf:libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py` into a separate Python module and comparing `model_json_schema()` for `SPMSelection`, `SPMCapability`, `SPMErrorDetail`, `SPMProvenance`, and `SPMComparisonProvenance`. All five are byte-structure equivalent; these routing/validation changes do not introduce a new public schema. + +Residual rollout constraint: routine unmeasured future bundles remain unavailable for default US requests. In particular, a newer model without certified `measurements.spm` does not inherit the historical allowance merely because its wrapper is 5.2.0 or 5.3.0. Incomplete route provenance likewise remains unavailable until its metadata is resolved explicitly. No live routes or deployment state were modified. + +Next: root completes integrated checks, authenticated native qualification, final source binding/report, and the existing canonical draft PR update. This subtask performed no push, publication, population job, deployment, merge, or changes to untracked root handoffs. From c97d3391c2184626f399bb685bfd29798e426905 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:31:01 -0400 Subject: [PATCH 13/40] Refresh gateway SPM schema and verify entrypoint contract parity --- .../tests/test_openapi.py | 23 +- .../tests/golden/openapi.json | 328 ++++++++++++++++++ 2 files changed, 330 insertions(+), 21 deletions(-) diff --git a/projects/policyengine-simulation-entry/tests/test_openapi.py b/projects/policyengine-simulation-entry/tests/test_openapi.py index 380bf0902..c7cc77054 100644 --- a/projects/policyengine-simulation-entry/tests/test_openapi.py +++ b/projects/policyengine-simulation-entry/tests/test_openapi.py @@ -76,23 +76,6 @@ def normalized_compatibility_paths(spec: OpenAPIDocument) -> OpenAPIPathMap: return paths -def without_spm_extensions(schemas): - schemas = deepcopy(schemas) - for name in list(schemas): - if name.startswith("SPM") or name == "SimulationErrorResponse": - schemas.pop(name) - continue - for field in ( - "spm", - "spm_config", - "spm_provenance", - "errors", - "spm_capabilities", - ): - schemas[name].get("properties", {}).pop(field, None) - return schemas - - def test_canonical_spm_extensions_are_public(): spec = create_app().openapi() schemas = spec["components"]["schemas"] @@ -149,10 +132,8 @@ def test_normalized_contract_matches_old_gateway(): ) == normalized_compatibility_paths(gateway_spec) cloud_run_schemas = deepcopy(cloud_run_spec["components"]["schemas"]) cloud_run_schemas.pop("ReadinessResponse") - assert ( - without_spm_extensions(cloud_run_schemas) - == gateway_spec["components"]["schemas"] - ) + cloud_run_schemas.pop("SimulationErrorResponse") + assert cloud_run_schemas == gateway_spec["components"]["schemas"] gateway_operation_ids = operation_ids(gateway_spec) cloud_run_operation_ids = operation_ids(cloud_run_spec) assert { diff --git a/projects/policyengine-simulation-gateway/tests/golden/openapi.json b/projects/policyengine-simulation-gateway/tests/golden/openapi.json index 606d51a3d..2282cbf4b 100644 --- a/projects/policyengine-simulation-gateway/tests/golden/openapi.json +++ b/projects/policyengine-simulation-gateway/tests/golden/openapi.json @@ -370,6 +370,20 @@ ], "title": "Status" }, + "errors": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/SPMErrorDetail" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Errors" + }, "error": { "anyOf": [ { @@ -392,6 +406,26 @@ }, "BudgetWindowAnnualImpact": { "properties": { + "spm_config": { + "anyOf": [ + { + "$ref": "#/components/schemas/SPMSelection" + }, + { + "type": "null" + } + ] + }, + "spm_provenance": { + "anyOf": [ + { + "$ref": "#/components/schemas/SPMComparisonProvenance" + }, + { + "type": "null" + } + ] + }, "year": { "type": "string", "title": "Year" @@ -435,6 +469,16 @@ "type": "string", "title": "Country" }, + "spm": { + "anyOf": [ + { + "$ref": "#/components/schemas/SPMSelection" + }, + { + "type": "null" + } + ] + }, "version": { "anyOf": [ { @@ -696,6 +740,20 @@ } ] }, + "errors": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/SPMErrorDetail" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Errors" + }, "error": { "anyOf": [ { @@ -1262,6 +1320,20 @@ } ] }, + "errors": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/SPMErrorDetail" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Errors" + }, "error": { "anyOf": [ { @@ -1635,6 +1707,16 @@ } ], "title": "Dataset" + }, + "spm": { + "anyOf": [ + { + "$ref": "#/components/schemas/SPMCapability" + }, + { + "type": "null" + } + ] } }, "type": "object", @@ -1721,12 +1803,230 @@ ], "title": "RacePovertyOutput" }, + "SPMCapability": { + "properties": { + "contract_version": { + "type": "string", + "const": "canonical-spm-v1", + "title": "Contract Version", + "default": "canonical-spm-v1" + }, + "defaults": { + "$ref": "#/components/schemas/SPMSelection" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "defaults" + ], + "title": "SPMCapability" + }, + "SPMComparisonProvenance": { + "properties": { + "baseline": { + "items": { + "$ref": "#/components/schemas/SPMProvenance" + }, + "type": "array", + "minItems": 1, + "title": "Baseline" + }, + "reform": { + "items": { + "$ref": "#/components/schemas/SPMProvenance" + }, + "type": "array", + "minItems": 1, + "title": "Reform" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "baseline", + "reform" + ], + "title": "SPMComparisonProvenance", + "description": "One receipt per executed regional segment, separately for each policy." + }, + "SPMErrorDetail": { + "properties": { + "code": { + "type": "string", + "title": "Code" + }, + "message": { + "type": "string", + "title": "Message" + } + }, + "type": "object", + "required": [ + "code", + "message" + ], + "title": "SPMErrorDetail" + }, + "SPMProvenance": { + "properties": { + "forecast_id": { + "type": "string", + "title": "Forecast Id" + }, + "forecast_sha256": { + "type": "string", + "title": "Forecast Sha256" + }, + "scenario": { + "type": "string", + "title": "Scenario" + }, + "geography_kind": { + "type": "string", + "title": "Geography Kind" + }, + "runtime_versions": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "type": "object", + "title": "Runtime Versions" + }, + "years": { + "additionalProperties": { + "additionalProperties": true, + "type": "object" + }, + "type": "object", + "title": "Years" + }, + "geographies": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array", + "title": "Geographies" + }, + "composition_method": { + "type": "string", + "title": "Composition Method" + }, + "storage_method": { + "type": "string", + "title": "Storage Method" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "forecast_id", + "forecast_sha256", + "scenario", + "geography_kind", + "runtime_versions", + "years", + "geographies", + "composition_method", + "storage_method" + ], + "title": "SPMProvenance", + "description": "Detached calculation receipt; data certification is a separate claim." + }, + "SPMSelection": { + "properties": { + "forecast_content_sha256": { + "anyOf": [ + { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + { + "type": "null" + } + ], + "title": "Forecast Content Sha256" + }, + "scenario": { + "anyOf": [ + { + "type": "string", + "minLength": 1, + "pattern": "^\\S+$" + }, + { + "type": "null" + } + ], + "title": "Scenario" + }, + "geography_kind": { + "type": "string", + "enum": [ + "county", + "national", + "metro" + ], + "title": "Geography Kind" + }, + "geography_id": { + "anyOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "null" + } + ], + "title": "Geography Id" + }, + "county_vintage": { + "type": "string", + "pattern": "^[0-9]{4}$", + "title": "County Vintage" + }, + "as_of": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "As Of" + } + }, + "additionalProperties": false, + "type": "object", + "title": "SPMSelection", + "description": "Select from the bundle's pinned artifact; national geography is explicit.\n\nCounty mode reads the household's observed ``county_fips``. A state alone\ndoes not identify an SPM area. These settings contain no provider or path." + }, "SimulationRequest": { "properties": { "country": { "type": "string", "title": "Country" }, + "spm": { + "anyOf": [ + { + "$ref": "#/components/schemas/SPMSelection" + }, + { + "type": "null" + } + ] + }, "version": { "anyOf": [ { @@ -1903,6 +2203,26 @@ }, "SingleYearMacroOutput": { "properties": { + "spm_config": { + "anyOf": [ + { + "$ref": "#/components/schemas/SPMSelection" + }, + { + "type": "null" + } + ] + }, + "spm_provenance": { + "anyOf": [ + { + "$ref": "#/components/schemas/SPMComparisonProvenance" + }, + { + "type": "null" + } + ] + }, "model_version": { "type": "string", "title": "Model Version" @@ -2195,6 +2515,14 @@ }, "VersionsResponse": { "properties": { + "spm_capabilities": { + "additionalProperties": { + "$ref": "#/components/schemas/SPMCapability" + }, + "type": "object", + "title": "Spm Capabilities", + "description": "Canonical SPM capabilities keyed by exact PolicyEngine wrapper version. Resolve a country-model route to its app in the routing maps, then find that app's exact version in the policyengine map. App names, country-model versions and latest aliases are not capability keys." + }, "policyengine": { "$ref": "#/components/schemas/VersionMap" }, From 9d16effa0a9b176c308b97c329e157f3ce57d85a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:33:04 -0400 Subject: [PATCH 14/40] test: reject malformed legacy seed and sibling metadata --- .../tests/test_spm_routes.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py index 22953d8dd..0ca79f1db 100644 --- a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py +++ b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py @@ -90,6 +90,32 @@ def test_missing_wrapper_does_not_certify_unknown_routes( assert mock_modal["func"].calls == [] +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +def test_legacy_seed_cannot_erase_future_wrapper_from_app_name( + mock_modal, client, endpoint, extra +): + legacy_route(mock_modal, "legacy-seed", "1.715.2") + state = mock_modal["dicts"]["simulation-api-routing-state"]["active"] + state["routes"]["us"]["1.715.2"] = "policyengine-simulation-py99-0-0" + response = client.post(endpoint, json={"country": "us", **extra}) + assert response.status_code == 400 + assert response.json()["errors"][0]["code"] == "SPM_CONFIGURATION_UNAVAILABLE" + assert mock_modal["func"].calls == [] + + +@pytest.mark.parametrize("schema_version", [None, 2]) +def test_legacy_seed_requires_supported_registry_schema( + mock_modal, client, schema_version +): + legacy_route(mock_modal, "legacy-seed", "1.715.2") + state = mock_modal["dicts"]["simulation-api-routing-state"]["active"] + state["schema_version"] = schema_version + response = client.post("/simulate/economy/comparison", json={"country": "us"}) + assert response.status_code == 400 + assert response.json()["errors"][0]["code"] == "SPM_CONFIGURATION_UNAVAILABLE" + assert mock_modal["func"].calls == [] + + def shared_app_state(mock_modal, *, sibling_model): state = deepcopy(TEST_ROUTING_STATE) original = state["bundles"]["4.10.0"] @@ -149,11 +175,12 @@ def test_shared_app_with_ambiguous_country_model_requires_explicit_bundle( @pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +@pytest.mark.parametrize("sibling_model", [None, "", " ", 42]) def test_shared_app_with_missing_model_metadata_is_ambiguous( - mock_modal, client, endpoint, extra + mock_modal, client, endpoint, extra, sibling_model ): state = shared_app_state(mock_modal, sibling_model="1.824.7") - del state["bundles"]["4.10.0"]["us"]["model_version"] + state["bundles"]["4.10.0"]["us"]["model_version"] = sibling_model response = client.post( endpoint, json={"country": "us", "version": "1.824.7", **extra} ) From 2ee006d9d38088ec325f9db3434a369955466463 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:33:39 -0400 Subject: [PATCH 15/40] fix: reject incomplete legacy route and sibling metadata --- .../endpoints.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py index c53be3a59..dafbfda4b 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py @@ -365,15 +365,15 @@ def _policyengine_version_for_app( for version in candidates: manifest = _bundle_manifest(state, version) country_bundle = manifest.get(country) - if ( - isinstance(country_bundle, dict) - and country_bundle.get("model_version") == model_version - ): - matching.append(version) - elif not isinstance(country_bundle, dict) or not isinstance( - country_bundle.get("model_version"), str - ): + candidate_model = ( + country_bundle.get("model_version") + if isinstance(country_bundle, dict) + else None + ) + if not isinstance(candidate_model, str) or not candidate_model.strip(): unclassified.append(version) + elif candidate_model == model_version: + matching.append(version) if len(matching) == 1 and not unclassified: return matching[0] if len(candidates) > 1: @@ -431,13 +431,20 @@ def _resolve_country_route( policyengine_version = _policyengine_version_for_app( state, app_name, country=country, model_version=version ) + if policyengine_version is None: + # The legacy seed producer infers these same wrapper routes. Missing + # registry metadata must not erase a known future app version. + policyengine_version = _policyengine_version_from_app_name(app_name) return RouteResolution( app_name=app_name, response_version=version, policyengine_version=policyengine_version, bundle_manifest=_bundle_manifest(state, policyengine_version), route_provenance=( - "legacy-seed" if state.get("generation") == "legacy-seed" else None + "legacy-seed" + if state.get("generation") == "legacy-seed" + and state.get("schema_version") == 1 + else None ), ) From 35f17d014a85fc67f6904b8ddf4b09f01b01f65b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:34:10 -0400 Subject: [PATCH 16/40] docs: record adversarial route provenance qualification --- rollout/worker-fable-fixes/routing-evidence.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rollout/worker-fable-fixes/routing-evidence.md b/rollout/worker-fable-fixes/routing-evidence.md index 34065a7ff..464d0a160 100644 --- a/rollout/worker-fable-fixes/routing-evidence.md +++ b/rollout/worker-fable-fixes/routing-evidence.md @@ -54,3 +54,7 @@ Schema equivalence was checked by loading the original contract source with `git Residual rollout constraint: routine unmeasured future bundles remain unavailable for default US requests. In particular, a newer model without certified `measurements.spm` does not inherit the historical allowance merely because its wrapper is 5.2.0 or 5.3.0. Incomplete route provenance likewise remains unavailable until its metadata is resolved explicitly. No live routes or deployment state were modified. Next: root completes integrated checks, authenticated native qualification, final source binding/report, and the existing canonical draft PR update. This subtask performed no push, publication, population job, deployment, merge, or changes to untracked root handoffs. + +Adversarial follow-up: regression commit `9d16eff` reproduced eight additional failures (37 passing cases) with `tests/test_spm_routes.py -q --tb=no` under the same gateway command environment. A seed could omit the wrapper route that the actual seed producer would infer from a future `policyengine-simulation-py99-0-0` app; an absent/unsupported seed schema was also accepted. Blank sibling model metadata incorrectly excluded that sibling. Fix `2ee006d` requires the supported seed schema, infers the app's wrapper version only when no exact registry candidate exists, and treats blank sibling models as unclassified. Exact registry versions still take precedence over app-name inference. + +After the follow-up, the gateway command with `tests/test_spm_routes.py tests/test_endpoints.py -q` passed **98 tests in 0.71 seconds**. Ruff format/check and `git diff --check` passed. Additional direct gateway-helper probes confirmed that a future bundle with each of `spm=None`, `False`, `0`, `[]`, a string, or `{}` raises `SPM_CONFIGURATION_UNAVAILABLE`; none produces an accepted submission. Native runtime changes were independently reviewed against the installed `PolicyEngineSPMProvider`: no actionable typed year/scenario or laziness defect was found, and prevalidation receipts remain private to the discarded provider. From 728f462a0342379a6b86022e21119b4b98381f09 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:36:09 -0400 Subject: [PATCH 17/40] Record passing worker regressions and installed type comparison --- rollout/worker-fable-fixes/PROGRESS.md | 6 +++ .../worker-fable-fixes/typing-comparison.json | 54 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 rollout/worker-fable-fixes/typing-comparison.json diff --git a/rollout/worker-fable-fixes/PROGRESS.md b/rollout/worker-fable-fixes/PROGRESS.md index 67ec25894..b6eeb7d35 100644 --- a/rollout/worker-fable-fixes/PROGRESS.md +++ b/rollout/worker-fable-fixes/PROGRESS.md @@ -16,6 +16,12 @@ - Committed an isolated qualification harness that authenticates the existing c49c/76448/8c640/2dbc wheels and reuses the existing explicit development bootstrap without modifying it, the packages, or historical evidence. - Routing regressions before fixes: 13 gateway failures (historical routes, sibling ambiguity, malformed capability) and two partial-selection failures reproduced. - Archived starting source to /tmp/worker-fable-base for like-for-like installed-environment type diagnostics; no checkout or handoff edits. +- Routing and runtime fixes committed, including adversarial follow-up for malformed seed schemas, future prefixed apps with absent routes, and unclassified sibling metadata. Latest full gateway suite: 149 passed; contract: 73 passed; entry: 66 passed; actual regenerated client: four passed. +- Full executor suite: 454 passed, 22 explicitly gated native tests skipped, two integration cases deselected, one environment-dependent version-extraction failure. That exact test passed with UV_NO_SYNC=1 against the existing environment (no download or package mutation), completing 455 unit checks. +- ./scripts/generate-clients.sh completed successfully; refreshed stale gateway OpenAPI golden and strengthened entry/gateway comparison to include SPM schemas. +- Clean native runtime regressions: eight failures before repair, then 56 canonical/runtime checks passed with zero skips. Earlier /tmp launchers were superseded after discovering unrelated pre-existing /tmp/h2.py import shadowing; clean launchers use safe-path mode and explicit installed Python 3.13. +- Clean artifact regressions reproduced receipt/column/cache-selection/storage identity failures. Artifact fixes and full native qualification are completing. +- Installed uv pip check: 180 compatible distributions. Current source Pyright: entry/contract zero errors; executor 145 and gateway 78, exactly matching unchanged starting source diagnostics. ## Next diff --git a/rollout/worker-fable-fixes/typing-comparison.json b/rollout/worker-fable-fixes/typing-comparison.json new file mode 100644 index 000000000..a2edf4073 --- /dev/null +++ b/rollout/worker-fable-fixes/typing-comparison.json @@ -0,0 +1,54 @@ +{ + "executor": { + "current": { + "filesAnalyzed": 43, + "errorCount": 145, + "warningCount": 0, + "informationCount": 0, + "timeInSec": 2.096 + }, + "base": { + "filesAnalyzed": 43, + "errorCount": 145, + "warningCount": 0, + "informationCount": 0, + "timeInSec": 1.352 + }, + "new_diagnostics": [] + }, + "gateway": { + "current": { + "filesAnalyzed": 8, + "errorCount": 78, + "warningCount": 0, + "informationCount": 0, + "timeInSec": 0.997 + }, + "base": { + "filesAnalyzed": 8, + "errorCount": 78, + "warningCount": 0, + "informationCount": 0, + "timeInSec": 0.684 + }, + "new_diagnostics": [] + }, + "contract": { + "current": { + "filesAnalyzed": 8, + "errorCount": 0, + "warningCount": 0, + "informationCount": 0, + "timeInSec": 0.741 + } + }, + "entry": { + "current": { + "filesAnalyzed": 7, + "errorCount": 0, + "warningCount": 0, + "informationCount": 0, + "timeInSec": 0.767 + } + } +} From 580673cf75a3da9df3e42b68c440f89343a8ef7e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:36:48 -0400 Subject: [PATCH 18/40] Reject temporary-directory import shadowing in qualification --- rollout/worker-fable-fixes/run_installed_qualification.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rollout/worker-fable-fixes/run_installed_qualification.py b/rollout/worker-fable-fixes/run_installed_qualification.py index 4ef7ed25b..80d7851f3 100644 --- a/rollout/worker-fable-fixes/run_installed_qualification.py +++ b/rollout/worker-fable-fixes/run_installed_qualification.py @@ -58,6 +58,9 @@ def sha256(path): os.environ["POLICYENGINE_SKIP_COUNTRY_IMPORTS"] = "1" os.environ["SPM_NATIVE_SMOKE_SOURCE"] = str(SOURCE) assert not os.environ.get("PYTHONPATH"), "No inherited source overlays allowed" +assert Path("/tmp").resolve() not in {Path(path).resolve() for path in sys.path}, ( + "Temporary-directory modules must not shadow installed dependencies" +) for relative in ( "projects/policyengine-simulation-executor/src", "projects/policyengine-simulation-entry/src", From c431d138bdbad9be71c12b6994851879cf5d05aa Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:37:25 -0400 Subject: [PATCH 19/40] fix: recompute incomplete SPM artifacts and verify storage keys --- .../baseline_artifacts.py | 32 ++++++-- .../precompute.py | 8 ++ .../tests/test_precompute.py | 4 +- .../worker-fable-fixes/artifact-evidence.md | 82 ++++++++++++++++++- .../worker-fable-fixes/run_artifact_native.py | 3 +- 5 files changed, 113 insertions(+), 16 deletions(-) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py index 00bd588f9..f8dfa290c 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py @@ -33,7 +33,8 @@ # Artifact outcomes reported via the `baseline_artifact` observability # attribute. "hit": loaded (disk or in-process cache) and complete; -# "incomplete": loaded but missing requested columns -> recomputed; +# "incomplete": loaded but missing requested columns or a valid SPM receipt +# -> recomputed; # "miss": no artifact -> computed (today's behavior). OUTCOME_HIT = "hit" OUTCOME_INCOMPLETE = "incomplete" @@ -179,30 +180,45 @@ def _record_outcome(self, outcome: str) -> None: self._artifact_outcome = outcome def ensure(self) -> None: + # The wrapper restores selection metadata on load/cache hits. Keep the + # request's selection so an invalid cached entry cannot replace it. + selection = getattr(self, "spm_config", None) + requested_spm = getattr(self, "spm", None) self._computed_this_process = False super().ensure() if self._computed_this_process: self._record_outcome(OUTCOME_MISS) return + from policyengine_simulation_contract.spm import SPMInputError + from policyengine_simulation_executor.spm import simulation_spm_result - selection = getattr(self, "spm_config", None) - if selection is not None: - simulation_spm_result( - self, self, selection, expected_year=self.dataset.year - ) + # A tax-only artifact can legitimately have no receipt years. Check its + # columns first, then treat unusable cached receipts as another gap. missing = self._missing_output_columns() - if not missing: + receipt_error = None + if not missing and selection is not None: + try: + simulation_spm_result( + self, self, selection, expected_year=self.dataset.year + ) + except SPMInputError as exc: + receipt_error = str(exc) + if not missing and receipt_error is None: self._record_outcome(OUTCOME_HIT) return logger.warning( - "Baseline artifact %s is missing output columns %s; recomputing", + "Baseline artifact %s is incomplete (missing columns=%s, " + "SPM receipt=%s); recomputing", self.id, missing, + receipt_error, ) self._record_outcome(OUTCOME_INCOMPLETE) + if selection is not None: + setattr(self, "spm", requested_spm) self.run() self.save() # Replace the in-process cache entry so this request's second diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py index 3370a7261..f26bc6640 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py @@ -310,6 +310,14 @@ def _prepare_cohort_baseline(bucket: str, expected: BaselinePlanEntry): f"({expected.simulation_id} != {baseline.id}); refusing to act " "under a mismatched key." ) + planned_storage_id = Path(expected.path).stem + storage_id = getattr(baseline, "storage_id", baseline.id) + if storage_id != planned_storage_id: + raise RuntimeError( + "Planned and in-container baseline storage ids disagree " + f"({planned_storage_id} != {storage_id}); refusing to act " + "under a mismatched key." + ) # The extras economic_impact_analysis applies unconditionally before # ensure(); the artifact must carry them or every request would fail diff --git a/projects/policyengine-simulation-executor/tests/test_precompute.py b/projects/policyengine-simulation-executor/tests/test_precompute.py index 361420cd4..7c051bde3 100644 --- a/projects/policyengine-simulation-executor/tests/test_precompute.py +++ b/projects/policyengine-simulation-executor/tests/test_precompute.py @@ -390,9 +390,7 @@ def test_precompute_identity_equals_runtime_id(self, identity_stubs, fake_region year=2026, ) assert writer_identity.simulation_id == reader_id - assert writer_identity.store_path.endswith( - f"/{writer_identity.simulation_id}.h5" - ) + assert writer_identity.store_path.endswith(f"/{writer_identity.storage_id}.h5") def test_dataset_filename_matches_runtime_stem_lookup(self, monkeypatch): """Real-manifest coverage: the artifact filename equals the exact diff --git a/rollout/worker-fable-fixes/artifact-evidence.md b/rollout/worker-fable-fixes/artifact-evidence.md index f8eb5cfa0..24016bc98 100644 --- a/rollout/worker-fable-fixes/artifact-evidence.md +++ b/rollout/worker-fable-fixes/artifact-evidence.md @@ -1,6 +1,6 @@ # Artifact findings 3 and 4 -State: regression-first tests added; implementation follows in the next commit. +State: findings 3 and 4 fixed; regression-first and installed qualification passed. The exact Fable review from gate `20260909-170551-pr-abdf4629`, round `001-4542bf03eca1`, was read along with the repository testing skill. No durable agreement is inferred from that review. @@ -26,9 +26,12 @@ env -u PYTHONPATH UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache \ ``` The clean run imports the unchanged artifact implementations from `dcfe4fd`. -Both tax-only missing SPM columns and a complete artifact with empty receipt years -reproduced failures before implementation edits. The remaining test outcomes will -be recorded after completion. The missing/malformed receipt, damaged HDF, and +The clean RED run completed with **6 failed, 40 passed in 173.47 seconds**. Five +failures reproduce the target defects: tax-only missing columns, complete columns +with empty receipt years, cached missing receipt, cached sibling selection, and +planned storage-id mismatch. A sixth, previously existing test assertion assumed +all planned filenames were bare simulation ids; it now asserts the planner +`storage_id`, which preserves the legacy contract and handles actual SPM bundles. The missing/malformed receipt, damaged HDF, and recorded-selection isolation cases exercise the actual installed wrapper load, save, cache, and model calculation paths. A focused precompute test expects rejection when simulation ids agree but the planned storage filename differs. @@ -43,3 +46,74 @@ commands now select the authenticated Python 3.13 environment explicitly. Ruff check passes for the four new/updated test files after formatting. Root handoff files were left untouched. + + +## Final implementation and checks + +The guard captures the original request selection before the installed wrapper +restores cached metadata. Missing columns are checked before receipt validation; +invalid cached receipts take `OUTCOME_INCOMPLETE` and the existing run/save/cache +replacement path. The request selection is restored before recompute. Real run or +save errors still propagate. Missing/malformed HDF receipts, differing recorded +selections, and corrupt HDF bytes retain the wrapper's existing miss/recompute +behavior. The repaired artifacts and cache entries then replay as hits with valid +actual calculation receipts and without another model run. + +Precompute compares the real wrapper's `storage_id` (falling back to id only for +historical wrappers) with `Path(expected.path).stem`, which is the planner's +serialized storage id. This occurs beside the simulation-id comparison, before +configuration, ensure, or upload. No precompute wire schema changed. + +Final GREEN used the clean command above with +`--basetemp /private/tmp/worker-fable-artifact-green` and +`-k 'not test_precompute_identity_equals_runtime_id'`, writing +`artifact-native-green.log`: **45 passed, 1 deselected, 2 warnings in 207.33 seconds**. +All eight native artifact cases passed. The deselected assertion had been +corrected separately after the GREEN process imported its test module; the +following exact installed check closes that case: **1 passed, 2 warnings in +0.09 seconds** (`artifact-installed-identity.log`). + +```sh +env -u PYTHONPATH UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache \ + uv run --no-project \ + --python /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/bin/python \ + python -P rollout/worker-fable-fixes/run_artifact_native.py \ + projects/policyengine-simulation-executor/tests/test_precompute.py::TestWriterReaderContract::test_precompute_identity_equals_runtime_id \ + -q -p no:cacheprovider --basetemp /private/tmp/worker-fable-artifact-identity \ + > rollout/worker-fable-fixes/artifact-installed-identity.log 2>&1 +``` + +From `projects/policyengine-simulation-executor`, the explicit historical Python +3.13 focused suite also passed **68 tests in 13.02 seconds**: + +```sh +env -u PYTHONPATH UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache \ + uv run --no-sync --python .venv/bin/python .venv/bin/python -m pytest \ + tests/test_baseline_artifacts.py tests/test_precompute.py -q +``` + +After the storage filename assertion correction, repeating just +`tests/test_precompute.py -q` with the same command prefix passed **38 tests in +0.88 seconds**. Ruff check and format-check pass for both source files, all four +artifact/native test/helper files, and the launcher (seven Python files total). +The executor test venv lacks the `pyright` module; the coordinator's established +full typecheck covers both changed source files and reports the same 145 +pre-existing executor diagnostics as `dcfe4fd`, with zero new diagnostics. + +The runtime-errors agent independently reviewed the artifact/precompute diff +against the installed wrapper's actual ensure/load/run/cache code and found no +additional actionable safety defect. This is a bounded peer review, not a durable +Fable agreement. No artifact upload, deployment, publication, or population +calculation was performed. Remote cached artifacts were not changed; these fixes +repair a loaded incomplete artifact locally by recomputing. + +## Source binding + +These final source hashes bind the implementation. GREEN executed identical +Python code before an outcome-comment-only edit. Root's final qualification and +source inventory bind the integrated branch head and all test files. + +| Source | SHA256 | +| --- | --- | +| `baseline_artifacts.py` | `2ec0d48bd97bad7a663deaf49b7a12c5b887d6b29f571e2cfe90d3c4247ad9c1` | +| `precompute.py` | `2c2a9bb6c824f4cce603f5728ea4cc58a693f8f354beff10b4cb370d41914e45` | diff --git a/rollout/worker-fable-fixes/run_artifact_native.py b/rollout/worker-fable-fixes/run_artifact_native.py index 754ca2add..46cb72bd7 100644 --- a/rollout/worker-fable-fixes/run_artifact_native.py +++ b/rollout/worker-fable-fixes/run_artifact_native.py @@ -1,4 +1,5 @@ from pathlib import Path +from importlib import import_module import sys original = Path( @@ -9,6 +10,6 @@ compile(original.read_text().split("import pytest\n", 1)[0], str(original), "exec"), namespace, ) -import pytest +pytest = import_module("pytest") raise SystemExit(pytest.main(sys.argv[1:])) From 2384726da01ceaaa09a10a6174c952c60de18d13 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:45:34 -0400 Subject: [PATCH 20/40] Record qualified Fable fixes, source binding and delivery blocker --- rollout/worker-fable-fixes/FINAL-REPORT.md | 134 ++++++++++++ rollout/worker-fable-fixes/PR-BODY.md | 13 ++ rollout/worker-fable-fixes/PROGRESS.md | 40 ++-- .../changed-python-files.txt | 16 ++ .../native-wheel-summary.json | 111 ++++++++++ .../worker-fable-fixes/source-binding.json | 198 ++++++++++++++++++ 6 files changed, 490 insertions(+), 22 deletions(-) create mode 100644 rollout/worker-fable-fixes/FINAL-REPORT.md create mode 100644 rollout/worker-fable-fixes/PR-BODY.md create mode 100644 rollout/worker-fable-fixes/changed-python-files.txt create mode 100644 rollout/worker-fable-fixes/native-wheel-summary.json create mode 100644 rollout/worker-fable-fixes/source-binding.json diff --git a/rollout/worker-fable-fixes/FINAL-REPORT.md b/rollout/worker-fable-fixes/FINAL-REPORT.md new file mode 100644 index 000000000..f808cafc8 --- /dev/null +++ b/rollout/worker-fable-fixes/FINAL-REPORT.md @@ -0,0 +1,134 @@ +# PR677 Fable fixes + +All four findings are fixed locally. The final combined qualification passed **34 installed native tests, zero skips or failures**, in 288.39 seconds. Service/client checks complete **747 passes** (including the explicitly retested version-extraction case). Remote delivery is blocked by this session's access restrictions, not by a code or test failure. + +## Source and review binding + +- Owned checkout: `/Users/maxghenis/spm-rebuild-20260908/worktrees/policyengine-sim-api-canonical`. +- Starting worker head: `dcfe4fd89d97145882cb147a9aa2ffae2cd10ebf`. +- Final implementation and tests: `c431d138bdbad9be71c12b6994851879cf5d05aa`; later commits contain qualification evidence and reports only. +- Live canonical `main`: `414c631d622f5f587eedd187296a80182a923db2`, independently fetched through the authenticated GitHub connector and equal to local `origin/main`. No base integration or checkout was needed. +- Existing issue: [#676](https://github.com/PolicyEngine/policyengine-sim-api/issues/676), verified open and appropriate. Existing [PR677](https://github.com/PolicyEngine/policyengine-sim-api/pull/677) is draft, unmerged, on canonical branch `max/spm-simulation-canonical-20260909`, still remotely at the starting head. +- Exact reviewer output read: `/Users/maxghenis/chief-of-staff/state/subfleet/gates/20260909-170551-pr-abdf4629/rounds/001-4542bf03eca1/peer-output.md`. Its complete findings remain valid despite dispatch's BrokenPipeError. **No durable Fable agreement exists or is claimed.** +- Repository testing/PR skills, installed PolicyEngine analysis/standards/API skills, parent PolicyEngine guidance, and wrapper repository guidance were read. GitNexus debugging guidance was read; no callable GitNexus graph was available, so source tracing used the actual repository and installed packages. +- Untracked root `PROGRESS.md` and `WORKER-CANONICAL-SPM-HANDOFF.md` were preserved. Their original fingerprints are committed in `handoff-preservation.json`. This continuation's committed state/done/next journal is the adjacent `PROGRESS.md`. + +## Finding disposition + +| Finding | Disposition and concrete proof | +| --- | --- | +| 1. Historical routes without wrapper versions incorrectly return 400 | Fixed. Internal route provenance distinguishes the actual legacy country dictionary and schema-v1 `legacy-seed` paths. Unknown-wrapper acceptance additionally requires a numeric historical US model through 1.764.6. A future wrapper inferable from the app name cannot be erased by missing seed metadata. Annual and budget-window submissions accept supported historical shapes; explicit SPM selections, future/unknown models, unproven seeds, ambiguous siblings and malformed capabilities reject. | +| 2. Year/scenario prevalidation loses typed errors | Fixed. Uses the installed `PolicyEngineSPMProvider.year_metadata` contract, preserving `SPM_YEAR_UNAVAILABLE` at annual/year-alias and budget-window start/end boundaries. Narrow scenario translation matches the calculator's public adapter contract and preserves `SPM_SCENARIO_UNAVAILABLE` through HTTP/poll/pickle/optional-analysis transport. Other invalid settings retain `SPM_SETTINGS_INVALID`. Prevalidation does not measure amounts; actual country tax-only calculations remain lazy outside forecast years. | +| 3. Incomplete cached receipt becomes fatal | Fixed. Check required columns first; an unusable cached receipt follows incomplete → run/save/cache replacement. Capture the requested selection before wrapper cache restoration and restore it before recomputation. Eight real one-household HDF/cache cases cover tax-only missing columns, empty years, missing/malformed receipt, changed recorded selection, corrupt HDF and cache selection isolation; repaired disk and memory results replay as valid hits. Real run/save errors still propagate. | +| 4. Precompute omits storage identity check | Fixed. Compare actual wrapper `storage_id` with `Path(expected.path).stem`, the planner's serialized storage identity, beside the existing simulation-id guard and before configuring, running or uploading. Historical wrappers without the property retain their bare-id contract. A mismatch rejects even when simulation ids agree; the installed-wrapper identity test passes. | + +Related notes are also resolved: shared-app routing uses exact country metadata only when every sibling is positively classified, excludes `latest` aliases, and rejects missing/blank metadata or unresolved ambiguity. Invalid stored capabilities are omitted with a warning from `/versions` and rejected with a typed configuration error on submission. Coupled geography validation happens after defaults resolve, so partial metro area selections obey the declared inheritance contract while invalid resolved combinations fail. Public SPM schemas remain unchanged by those validation fixes. The stale gateway golden from the original PR was regenerated, and entry/gateway schema comparison now covers their shared SPM schema instead of stripping it. + +The runtime-capability performance note was intentionally left alone. No performance refactor or unrelated model/style change was made. See `routing-evidence.md`, `runtime-evidence.md`, and `artifact-evidence.md` for detailed regression-first evidence and bounded independent cross-reviews. + +## Checks and exact commands + +Commands below were run in this checkout, with service commands run from the stated directory. `WORKER`, `QUAL`, `PY`, and `OUT` expand as follows: + +```sh +WORKER=/Users/maxghenis/spm-rebuild-20260908/worktrees/policyengine-sim-api-canonical +QUAL=/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification +PY="$WORKER/projects/policyengine-simulation-executor/.venv/bin/python" +OUT="$WORKER/rollout/worker-fable-fixes" +``` + +| Check | Result | +| --- | --- | +| Gateway full unit suite | 149 passed, 1 integration case deselected; 0.77 seconds. | +| Entry full unit suite | 66 passed; 1.20 seconds. | +| Shared contract full suite | 73 passed; 0.05 seconds. | +| Executor full suite | 454 passed, 22 opt-in native cases skipped, 2 integration cases deselected; one dependency-download failure, 23.62 seconds. That exact version-extraction test passed with syncing disabled, 1.21 seconds, completing 455 unit checks. | +| Actual regenerated Python client | 4 passed; 0.13 seconds. | +| Installed native/runtime regression lane | Clean RED: 8 failures / 3 passes. GREEN: 56 canonical/runtime checks passed, zero skips. | +| Installed artifact/precompute lane | Clean RED: 6 failures / 40 passes (five target defects plus an existing bare-id filename assertion). GREEN: 45 passed plus the separately corrected identity assertion passed; all 8 actual native artifact cases passed. | +| Combined authenticated native qualification | **34 passed, zero skipped/failed**, 2 expected plugin-rewrite warnings; 288.39 seconds. Covers original native bridge, prior lazy tax-only/provider behavior, all 8 artifact cases and 11 new runtime boundary cases. | +| Installed dependency check | All 180 distributions compatible. | +| Ruff format/lint and diff whitespace | All 16 changed Python files pass; `git diff --check` passes. | +| Pyright against exact installed environment | Entry/contract: zero diagnostics. Executor: 145; gateway: 78. Counts and diagnostic identities exactly match the archived unchanged starting source, with **zero new diagnostics**. | + +Full service commands, from the corresponding gateway, entry, executor, or shared-contract project directory: + +```sh +env -u PYTHONPATH UV_CACHE_DIR=/tmp/worker-fable-uv-cache \ + uv run --offline --no-project --python "$PY" python -m pytest tests/ -q +``` + +The executor's pre-existing script test calls `uv run` internally, which attempted an unavailable NumPy download. It was rerun against the existing installed environment without synchronization: + +```sh +# cwd: projects/policyengine-simulation-executor +env -u PYTHONPATH UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_NO_SYNC=1 \ + uv run --offline --no-project --python "$PY" python -m pytest \ + tests/test_modal_scripts.py::TestModalExtractVersions::test_extracts_versions_from_policyengine_bundle -q +``` + +Client generation and actual generated-client checks: + +```sh +# cwd: checkout root +UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_OFFLINE=1 ./scripts/generate-clients.sh +# cwd: projects/policyengine-apis-integ +PYTHONPATH="$WORKER/projects/policyengine-simulation-entry/artifacts/clients/python" \ + UV_CACHE_DIR=/tmp/worker-fable-uv-cache \ + uv run --offline --no-project --python "$PY" python -m pytest \ + tests/test_spm_generated_client.py -q +``` + +The gateway golden was regenerated with the actual module, then copied from its generated output as documented by the golden test: + +```sh +# cwd: projects/policyengine-simulation-gateway +env -u PYTHONPATH UV_CACHE_DIR=/tmp/worker-fable-uv-cache \ + uv run --offline --no-project --python "$PY" python -m policyengine_simulation_gateway.generate_openapi +cp artifacts/openapi.json tests/golden/openapi.json +``` + +Combined installed qualification, dependency and lint commands, from the checkout root: + +```sh +env -u PYTHONPATH PYTHONSAFEPATH=1 UV_CACHE_DIR=/tmp/worker-fable-uv-cache \ + uv run --offline --no-project --python "$QUAL/venv/bin/python" \ + python -P rollout/worker-fable-fixes/run_installed_qualification.py +UV_CACHE_DIR=/tmp/worker-fable-uv-cache uv pip check --python "$QUAL/venv/bin/python" +git diff --name-only dcfe4fd HEAD -- '*.py' > "$OUT/changed-python-files.txt" +UV_CACHE_DIR=/tmp/worker-fable-uv-cache uv run --offline --no-project ruff format --check $(cat "$OUT/changed-python-files.txt") +UV_CACHE_DIR=/tmp/worker-fable-uv-cache uv run --offline --no-project ruff check $(cat "$OUT/changed-python-files.txt") +git diff --check +``` + +Pyright used the installed cached 1.1.411 distribution, with identical Python 3.13 settings, source overlays and authenticated qualification venv on both sides. The base was created with `git archive dcfe4fd89d97145882cb147a9aa2ffae2cd10ebf | tar -x -C /tmp/worker-fable-base`, without a checkout. Each of the executor/gateway `base` and `current`, plus current entry/contract checks used: + +```sh +node /Users/maxghenis/.cache/uv/archive-v0/ELraG5tqAzXmiwNiWpYsc/pyright/dist/index.js \ + --project "$OUT/current-executor-pyrightconfig.json" --outputjson +``` + +Replace the config basename with `base-executor`, `base-gateway`, `current-gateway`, `current-entry` or `current-contract` as appropriate. The exact configurations and raw diagnostic JSON remain locally in this evidence directory; the diagnostic comparison is committed as `typing-comparison.json`. Bulk service/native logs are retained locally but ignored by git; selected runtime regression outputs are committed. Regression-first commands and outcomes are in the three committed lane evidence files. + +## Authenticated scientific inputs and qualification limits + +| Package | Version | Wheel SHA256 | +| --- | --- | --- | +| policyengine | 5.3.0 | `8c640d967575dddad70840bcbe938cea251c56958eded647c83eb9c5902735f1` | +| policyengine-us | 1.824.7 | `7644819916a4f8ca2aa37a4a9d992fa834bfa66ab2d441326a9686baa5c1a688` | +| policyengine-core | 3.30.1 | `2dbcf5f590a0199a7b7c77fcbbda2ff6bc289f6c6169ca0af3beace35d8b3e63` | +| spm-calculator | 1.0.0 | `c49c41da5fd482e563eaea956e205a3ba6841cadd4dd32bef4c0c3dbce17ffba` | + +The harness checks all 17,613 installed scientific package files byte for byte against these wheels and checks real import origins. Only worker/shared-library source is overlaid. It reuses the existing reviewed development manifest and explicit bootstrap; it does not fabricate receipt/version metadata or certify the inherited data. Actual artifact receipts are produced by the installed model/calculator, then deliberately damaged only within temporary regression fixtures. The model source itself is unchanged. `native-wheel-summary.json` records all 34 passing cases, import origins and the raw authenticated receipt digest; `source-binding.json` binds all 190 tracked worker/source test files byte for byte to the implementation commit. Post-run checks confirm those files and both untracked handoffs are unchanged. + +Forecast content identity: `3d86d5c4c0423480e6b69b75d222ffa4a7a2639e4094df5ba2504af01be17173`. Native H5 identity: `6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84`; only its indexed first household and four people are simulated, and its full hash is checked before and after. The existing explicit development manifest is SHA256 `c78faac072f85109ac50d36a9e538a4dae56af4cdc98d2281603a3f4849a591f`; the bootstrap is SHA256 `f762892a374c3950af54834f0fb234708addc59ad4af90aab99866eb37e27a5e`. + +Earlier temporary launchers exposed an unrelated pre-existing `/private/tmp/h2.py` through Python's script-directory import path; that file printed another experiment's output. Those runs were discarded/interrupted and superseded by clean safe-path runs. The unrelated file was not modified. The qualification command clears inherited `PYTHONPATH` and uses explicit installed Python 3.13; the final harness requires the variable to be absent and rejects `/tmp` on `sys.path`. + +Residual risks: no durable Fable agreement; existing executor/gateway type debt remains; final published model/data bundle, required CI, Docker/service integration and live rollout verification remain outside this bounded local qualification. Unknown future legacy bundles still need explicit measured capability or a reviewed routing/model contract; absence of capability never broadly implies legacy. Scenario error translation remains coupled to the calculator adapters' narrow, tested message prefix. No browser, population job, package/data publication, deployment, merge, external messaging, `.err` read or `.lane.log` read occurred. + +## Delivery status + +Local coherent steps are committed throughout. `git fetch origin`, `gh repo view`, and `git push origin HEAD:refs/heads/max/spm-simulation-canonical-20260909` failed on GitHub DNS/API connectivity. Authenticated connector reads confirmed the canonical repository, base, issue and draft PR. The fallback GitHub write tool returned: **“MCP tool call requires approval, but approval policy is never.”** No remote Git object, branch or PR-body mutation occurred; the API transfer was stopped at that boundary. + +The reviewed, update-ready body is `PR-BODY.md`, beginning `Fixes #676`. It is prepared locally and has not been posted. Per `docs/engineering/skills/github-prs.md`, “If you cannot push to the canonical repository, stop and ask for access. Do not create a fork PR as a fallback.” Finishing delivery requires Git network access or an approved GitHub write channel; no fork fallback or merge is authorized or attempted. diff --git a/rollout/worker-fable-fixes/PR-BODY.md b/rollout/worker-fable-fixes/PR-BODY.md new file mode 100644 index 000000000..82362a9f1 --- /dev/null +++ b/rollout/worker-fable-fixes/PR-BODY.md @@ -0,0 +1,13 @@ +Fixes #676 + +Carry canonical SPM choices through annual comparisons, budget windows, segmented execution and baseline reuse. Exact bundle capabilities, request/cache identities and baseline/reform receipts retain forecast hash, scenario, geography and covered years; typed errors survive worker and HTTP transport. + +Preserve ordinary no-SPM submissions on proven historical country-dictionary and schema-v1 legacy-seed routes, using model metadata and route provenance. Unknown/future bundles, incomplete seed metadata, ambiguous sibling bundles and explicit unsupported canonical selections fail closed. Invalid registry capabilities are omitted from discovery and rejected on submission; partial selections resolve against declared defaults before coupled geography validation. + +Worker prevalidation uses the installed calculator adapter's typed year contract and retains `SPM_SCENARIO_UNAVAILABLE`. Incomplete cached SPM output or receipts recompute with the original requested selection, while corruption and selection-isolation safeguards remain intact. Precompute verifies the wrapper's actual storage identity against the planned filename before computation/upload. Gateway OpenAPI golden and entrypoint schema parity now include the original PR's SPM extensions; clients were regenerated. + +Validation: 149 gateway, 66 entrypoint, 73 shared-contract and four actual generated-client tests pass. Executor checks complete 455 passes: the full run had 454 passes and one dependency-download failure, then that exact test passed against the existing environment with syncing disabled; 22 opt-in native cases were skipped and two integration cases deselected in the ordinary unit run. Regression-first installed tests reproduced the defects. Combined authenticated installed qualification passed 34 native tests with zero skips/failures in 288.39 seconds; exact source binding and commands are recorded in `rollout/worker-fable-fixes/FINAL-REPORT.md`. Ruff format/lint and whitespace checks pass. Entry/contract type checks have zero diagnostics; executor/gateway have zero new diagnostics against unchanged starting source (145/78 existing). + +Native qualification uses only one household/four people and the authenticated calculator c49c41da…, country 76448199…, wrapper 8c640d96… and Core 2dbcf5f5… wheels. The 17,613 installed scientific package files are verified against their wheels. The existing development manifest/bootstrap is explicit qualification evidence; no receipt/version metadata is fabricated and no model package source changes. + +This remains a draft worker PR. The complete Fable review's dispatch failed with BrokenPipeError, so there is no durable gate agreement; the four findings are dispositioned with concrete tests and independent bounded cross-review. Final published model/data bundle qualification, required CI, Docker/service integration, live rollout verification and fresh durable review remain pending. No browser, population job, package/data publication, deployment or merge was performed. diff --git a/rollout/worker-fable-fixes/PROGRESS.md b/rollout/worker-fable-fixes/PROGRESS.md index b6eeb7d35..bf0e7c90e 100644 --- a/rollout/worker-fable-fixes/PROGRESS.md +++ b/rollout/worker-fable-fixes/PROGRESS.md @@ -2,31 +2,27 @@ ## State -- Authorized continuation from dcfe4fd89d97145882cb147a9aa2ffae2cd10ebf on max/spm-simulation-canonical-20260909. -- Exact Fable review read: gate 20260909-170551-pr-abdf4629, round 001-4542bf03eca1. Findings are valid; dispatch BrokenPipeError means no gate agreement exists. -- Existing untracked root PROGRESS.md and WORKER-CANONICAL-SPM-HANDOFF.md are preserved without edits. This committed progress file tracks this continuation. -- Initial git fetch origin and gh repo view failed because GitHub DNS/API connectivity is unavailable in the shell. The authenticated GitHub connector independently confirms live main is 414c631d622f5f587eedd187296a80182a923db2, identical to local origin/main, and PR677 is still a canonical draft at the requested starting head. Issue #676 is open and appropriate. Git transport remains pending. -- No browser, population job, publication, deployment, merge, .err or .lane.log reads. +- Local implementation, regressions, installed qualification and reporting are complete. Implementation/test head: c431d138bdbad9be71c12b6994851879cf5d05aa; later commits contain evidence/reports only. +- All four Fable findings and related capability/selection notes are dispositioned. Complete review read from gate 20260909-170551-pr-abdf4629, round 001-4542bf03eca1; dispatch BrokenPipeError means no durable gate agreement exists. +- Remote delivery is blocked: Git/gh cannot reach GitHub from the shell; authenticated connector reads work, but the write tool requires approval while this session's policy is never. No remote mutation occurred. Existing canonical draft PR677 remains at dcfe4fd89d97145882cb147a9aa2ffae2cd10ebf. +- Live main verified through authenticated GitHub reads at 414c631d622f5f587eedd187296a80182a923db2, identical to local origin/main; issue #676 verified appropriate. No destructive checkout or base integration. +- Existing untracked root PROGRESS.md and WORKER-CANONICAL-SPM-HANDOFF.md preserved byte for byte. This committed progress file tracks only this continuation. ## Done -- Inspected status, remotes, starting HEAD, and attempted upstream fetch before edits. -- Read repository AGENTS.md and canonical testing/GitHub PR skills; read exact reviewer output. -- Read installed PolicyEngine analysis, standards and API skills, parent PolicyEngine guidance and wrapper repository guidance. Repository-specific instructions and the user's bounded qualification scope govern this work. -- Committed an isolated qualification harness that authenticates the existing c49c/76448/8c640/2dbc wheels and reuses the existing explicit development bootstrap without modifying it, the packages, or historical evidence. -- Routing regressions before fixes: 13 gateway failures (historical routes, sibling ambiguity, malformed capability) and two partial-selection failures reproduced. -- Archived starting source to /tmp/worker-fable-base for like-for-like installed-environment type diagnostics; no checkout or handoff edits. -- Routing and runtime fixes committed, including adversarial follow-up for malformed seed schemas, future prefixed apps with absent routes, and unclassified sibling metadata. Latest full gateway suite: 149 passed; contract: 73 passed; entry: 66 passed; actual regenerated client: four passed. -- Full executor suite: 454 passed, 22 explicitly gated native tests skipped, two integration cases deselected, one environment-dependent version-extraction failure. That exact test passed with UV_NO_SYNC=1 against the existing environment (no download or package mutation), completing 455 unit checks. -- ./scripts/generate-clients.sh completed successfully; refreshed stale gateway OpenAPI golden and strengthened entry/gateway comparison to include SPM schemas. -- Clean native runtime regressions: eight failures before repair, then 56 canonical/runtime checks passed with zero skips. Earlier /tmp launchers were superseded after discovering unrelated pre-existing /tmp/h2.py import shadowing; clean launchers use safe-path mode and explicit installed Python 3.13. -- Clean artifact regressions reproduced receipt/column/cache-selection/storage identity failures. Artifact fixes and full native qualification are completing. -- Installed uv pip check: 180 compatible distributions. Current source Pyright: entry/contract zero errors; executor 145 and gateway 78, exactly matching unchanged starting source diagnostics. +- Inspected status/HEAD/remotes and attempted fetch before edits; read exact reviewer output, prior rollout evidence, repository testing/PR guidance and relevant installed PolicyEngine skills. +- Committed failing regression steps, implementation steps, further adversarial route fixes and evidence throughout. +- Fixed proven historical routing with fail-closed future/ambiguous capabilities, typed calculator year/scenario errors at annual/window boundaries, incomplete artifact recomputation with selection isolation, and precompute storage identity rejection. +- Resolved malformed discovery/submission capabilities and partial geography defaults. Refreshed gateway OpenAPI golden, strengthened schema parity and regenerated actual clients. +- Service/client checks: 747 passes total (gateway149, entry66, contract73, executor454 plus its one environment-dependent script test successfully retested without syncing, generated client4). Ordinary executor run deliberately skipped22 opt-in native cases and deselected2 integration cases. +- Final combined authenticated native qualification: 34 passed, zero skips/failures, 288.39 seconds, two expected plugin-rewrite warnings. Authenticated all17,613 installed scientific package files against c49c/76448/8c640/2dbc wheels; model source unchanged. Indexed execution only: one household/four people. Source H5 unchanged. +- Final190 tracked worker/test source files match implementation commit byte for byte. Summary and full source binding committed; raw3MB receipt and bulk logs retained locally. +- Installed uv pip check: all180 distributions compatible. Ruff format/lint passes all16 changed Python files; diff whitespace passes. Pyright entry/contract zero diagnostics; executor145/gateway78 identical to unchanged starting source, zero new diagnostics. +- Independent bounded cross-reviews found no additional actionable defects. Earlier temporary launcher results were discarded after unrelated pre-existing /tmp/h2.py import shadowing; clean safe-path launchers reproduced and passed the regressions. Unrelated file untouched. +- Wrote FINAL-REPORT.md with exact commands/results/source binding/risks and PR-BODY.md beginning Fixes #676. No browser, population job, package/data publication, deployment, merge, external messaging, .err or .lane.log reads. ## Next -- Add failing regressions and fix legacy route classification, typed runtime year/scenario errors, incomplete artifact recomputation, and precompute storage identity checks. -- Disposition related ambiguity, malformed capability, and partial-selection notes. -- Run focused and installed-environment qualifications, lint, type, and format checks; record source binding and exact evidence. -- Commit coherent steps; push verified commits to existing canonical draft PR677 and update its body if connectivity permits. -- Write FINAL-REPORT.md in this directory. +- Provide Git network access or an approved GitHub write channel to deliver the already committed changes. Repository PR skill explicitly requires asking for access when canonical push is unavailable; do not create a fork fallback. +- Once access is available, recheck live base and existing PR head, push this branch to PolicyEngine/policyengine-sim-api, apply PR-BODY.md and verify canonical draft/head/message. Do not merge. +- Fresh durable Fable agreement, required CI and release/live rollout gates remain coordinator-owned; local test results are not deployment or publication approval. diff --git a/rollout/worker-fable-fixes/changed-python-files.txt b/rollout/worker-fable-fixes/changed-python-files.txt new file mode 100644 index 000000000..60011faeb --- /dev/null +++ b/rollout/worker-fable-fixes/changed-python-files.txt @@ -0,0 +1,16 @@ +libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py +libs/policyengine-simulation-contract/tests/test_spm_selection.py +projects/policyengine-simulation-entry/tests/test_openapi.py +projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py +projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py +projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py +projects/policyengine-simulation-executor/tests/native_spm_support.py +projects/policyengine-simulation-executor/tests/test_baseline_artifacts_spm_native.py +projects/policyengine-simulation-executor/tests/test_canonical_spm.py +projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py +projects/policyengine-simulation-executor/tests/test_precompute.py +projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py +projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py +projects/policyengine-simulation-gateway/tests/test_spm_routes.py +rollout/worker-fable-fixes/run_artifact_native.py +rollout/worker-fable-fixes/run_installed_qualification.py diff --git a/rollout/worker-fable-fixes/native-wheel-summary.json b/rollout/worker-fable-fixes/native-wheel-summary.json new file mode 100644 index 000000000..9d572e4f9 --- /dev/null +++ b/rollout/worker-fable-fixes/native-wheel-summary.json @@ -0,0 +1,111 @@ +{ + "data_certification": "not_certified", + "development_fixture": { + "bootstrap_path": "/Users/maxghenis/spm-rebuild-20260908/worktrees/policyengine-wrapper-production/tests/fixtures/spm_development.py", + "bootstrap_sha256": "f762892a374c3950af54834f0fb234708addc59ad4af90aab99866eb37e27a5e", + "compatibility_basis": "unverified_development_fixture", + "manifest_path": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/development-manifest.json", + "manifest_sha256": "c78faac072f85109ac50d36a9e538a4dae56af4cdc98d2281603a3f4849a591f" + }, + "external_package_publication": "not_attested", + "population_acceptance": "not_attested", + "pytest_exit_code": 0, + "pytest_outcomes": { + "failed": [], + "passed": [ + "tests/test_canonical_spm_native.py::test_worker_baseline_reform_national_local_cache_and_receipts", + "tests/test_canonical_spm_native.py::test_native_state_only_worker_requires_geography_and_explicit_national_works", + "tests/test_canonical_spm_native.py::test_actual_provider_unknown_metro_is_a_typed_input_error", + "::test_installed_provider_missing_year[national-2021]", + "::test_installed_provider_missing_year[national-2036]", + "::test_installed_provider_missing_year[metro-2021]", + "::test_installed_provider_missing_year[metro-2036]", + "::test_installed_provider_missing_year[county-2021]", + "::test_installed_provider_missing_year[county-2036]", + "::test_installed_provider_missing_year_metadata[2021]", + "::test_installed_provider_missing_year_metadata[2036]", + "::test_installed_country_tax_only_then_measurement[national-2021]", + "::test_installed_country_tax_only_then_measurement[national-2036]", + "::test_installed_country_tax_only_then_measurement[metro-2021]", + "::test_installed_country_tax_only_then_measurement[metro-2036]", + "tests/test_baseline_artifacts_spm_native.py::test_incomplete_or_corrupt_spm_disk_artifact_recomputes[tax_only_columns-incomplete]", + "tests/test_baseline_artifacts_spm_native.py::test_incomplete_or_corrupt_spm_disk_artifact_recomputes[empty_years-incomplete]", + "tests/test_baseline_artifacts_spm_native.py::test_incomplete_or_corrupt_spm_disk_artifact_recomputes[missing_receipt-miss]", + "tests/test_baseline_artifacts_spm_native.py::test_incomplete_or_corrupt_spm_disk_artifact_recomputes[malformed_receipt-miss]", + "tests/test_baseline_artifacts_spm_native.py::test_incomplete_or_corrupt_spm_disk_artifact_recomputes[different_selection-miss]", + "tests/test_baseline_artifacts_spm_native.py::test_incomplete_or_corrupt_spm_disk_artifact_recomputes[corrupt_hdf-miss]", + "tests/test_baseline_artifacts_spm_native.py::test_invalid_spm_cache_entry_recomputes_requested_selection[missing_receipt]", + "tests/test_baseline_artifacts_spm_native.py::test_invalid_spm_cache_entry_recomputes_requested_selection[different_selection]", + "tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period0-2021]", + "tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period1-2036]", + "tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period2-2040]", + "tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period3-2021]", + "tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period4-2036]", + "tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period5-2036]", + "tests/test_spm_runtime_native.py::test_worker_scenario_boundary_matches_real_calculator_contract[period0]", + "tests/test_spm_runtime_native.py::test_worker_scenario_boundary_matches_real_calculator_contract[period1]", + "tests/test_spm_runtime_native.py::test_valid_worker_prevalidation_does_not_evaluate_amounts", + "tests/test_spm_runtime_native.py::test_other_invalid_settings_keep_the_settings_code[selection0]", + "tests/test_spm_runtime_native.py::test_other_invalid_settings_keep_the_settings_code[selection1]" + ], + "skipped": [] + }, + "python": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/bin/python", + "qualification_harness_sha256": "aae743b84f0223c82ff15a115d766a49ceddea85605fe8bc96fd94903d5eb21f", + "raw_receipt_sha256": "7fd3c88b6b8b371f5b163e8ca731d8d0de72681cf97d5c6197742fe8078fa6e1", + "runtime_capability": { + "contract_version": "canonical-spm-v1", + "defaults": { + "as_of": null, + "county_vintage": "2020", + "forecast_content_sha256": "3d86d5c4c0423480e6b69b75d222ffa4a7a2639e4094df5ba2504af01be17173", + "geography_id": null, + "geography_kind": "county", + "scenario": "ce_trend" + } + }, + "scientific_package_files_verified": 17613, + "scope": "four_person_native_worker_installed_candidate_wheel_qualification", + "selected_households": 1, + "selected_people": 4, + "source_h5_sha256_after": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", + "source_h5_sha256_before": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", + "test_elapsed_seconds": 288.39, + "wheels": { + "policyengine": { + "import_origin_after": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/policyengine/__init__.py", + "import_origin_before": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/policyengine/__init__.py", + "package_file_count": 104, + "path": "/Users/maxghenis/spm-rebuild-20260908/rollout/postfix-qualification/wheels/policyengine-5.3.0-py3-none-any.whl", + "sha256": "8c640d967575dddad70840bcbe938cea251c56958eded647c83eb9c5902735f1", + "version": "5.3.0" + }, + "policyengine_core": { + "import_origin_after": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/policyengine_core/__init__.py", + "import_origin_before": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/policyengine_core/__init__.py", + "package_file_count": 193, + "path": "/Users/maxghenis/spm-rebuild-20260908/rollout/postfix-qualification/wheels/policyengine_core-3.30.1-py3-none-any.whl", + "sha256": "2dbcf5f590a0199a7b7c77fcbbda2ff6bc289f6c6169ca0af3beace35d8b3e63", + "version": "3.30.1" + }, + "policyengine_us": { + "import_origin_after": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/policyengine_us/__init__.py", + "import_origin_before": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/policyengine_us/__init__.py", + "package_file_count": 17254, + "path": "/Users/maxghenis/spm-rebuild-20260908/rollout/postfix-qualification/wheels/policyengine_us-1.824.7-py3-none-any.whl", + "sha256": "7644819916a4f8ca2aa37a4a9d992fa834bfa66ab2d441326a9686baa5c1a688", + "version": "1.824.7" + }, + "spm_calculator": { + "import_origin_after": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/spm_calculator/__init__.py", + "import_origin_before": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/spm_calculator/__init__.py", + "package_file_count": 62, + "path": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/wheels/spm_calculator-1.0.0-py3-none-any.whl", + "sha256": "c49c41da5fd482e563eaea956e205a3ba6841cadd4dd32bef4c0c3dbce17ffba", + "version": "1.0.0" + } + }, + "worker_head": "c431d138bdbad9be71c12b6994851879cf5d05aa", + "worker_source_files_verified": 190, + "worker_source_overlay_only": true +} diff --git a/rollout/worker-fable-fixes/source-binding.json b/rollout/worker-fable-fixes/source-binding.json new file mode 100644 index 000000000..151568528 --- /dev/null +++ b/rollout/worker-fable-fixes/source-binding.json @@ -0,0 +1,198 @@ +{ + "harness_sha256": "aae743b84f0223c82ff15a115d766a49ceddea85605fe8bc96fd94903d5eb21f", + "implementation_head": "c431d138bdbad9be71c12b6994851879cf5d05aa", + "raw_receipt_sha256": "7fd3c88b6b8b371f5b163e8ca731d8d0de72681cf97d5c6197742fe8078fa6e1", + "tree": "d90dc393446c0ac42ce2f620ee6efb66f6b6d0a8", + "worker_source_files": { + "libs/policyengine-fastapi/fixtures/__init__.py": "8824a809592f0bcc83da4922b4c248dcc31728b9d7e2edb88c5a60c8546b9ff0", + "libs/policyengine-fastapi/fixtures/ping/__init__.py": "5d3f2b1aeb257772823091a01bbd0204abac815bcbb032f6ba2985b9c0359274", + "libs/policyengine-fastapi/fixtures/ping/shared.py": "dd0ef1399255a84d74380ede657f5bb40d0478e526709663ae730830c3a73cea", + "libs/policyengine-fastapi/src/policyengine_fastapi/__init__.py": "bd236453db17527bdae849e1594021d4774ca8d679eea8f433207225aa54c21f", + "libs/policyengine-fastapi/src/policyengine_fastapi/auth/__init__.py": "9471634f7819cc9d1ad75d9187b655a46fda57f1f258f5dcd40c5bce6c86e972", + "libs/policyengine-fastapi/src/policyengine_fastapi/auth/jwt_decoder.py": "fe00648b65e7407b919244c4feedc6b72698dc8b99afc9d74fdeac41e48ba0db", + "libs/policyengine-fastapi/src/policyengine_fastapi/database.py": "64d8d809c9df27d708db953afbd1c9e5646a6fade26db5a35345cbaeedd1d04e", + "libs/policyengine-fastapi/src/policyengine_fastapi/exit.py": "01cbbc6053ab2e1ce203a0151282c737b0b22b4a8eb79fc8e5ec88358021fef7", + "libs/policyengine-fastapi/src/policyengine_fastapi/health/__init__.py": "3f8edf1a242e95344b646f8fc387b451300c3dcac6817fa31fff4368677242b9", + "libs/policyengine-fastapi/src/policyengine_fastapi/observability/__init__.py": "0fd247358c47b47823ed39783f6af36747d2e2a9d9984954b7e291a6fb088edd", + "libs/policyengine-fastapi/src/policyengine_fastapi/observability/config.py": "fd166b2a8baf8b33eeed6df508c9e66901ab19742dc3fcc15997d534bd7da0ae", + "libs/policyengine-fastapi/src/policyengine_fastapi/observability/contracts.py": "a5038b0f09464405cdfea26d53aaee8f765bb49712a6feb118e2a0c87888039a", + "libs/policyengine-fastapi/src/policyengine_fastapi/observability/correlation.py": "b31882080c06236b64ad6be4b7ca7677f6486a0cffe961b7e119b4b6b69e2c8d", + "libs/policyengine-fastapi/src/policyengine_fastapi/observability/emitters.py": "bf9fe7b9ceba7d651a9069f2bce2b72c43393b8fbd33c0092b21ffa590968679", + "libs/policyengine-fastapi/src/policyengine_fastapi/observability/provider.py": "1cf85f524b1ed8aeb2bbfa1dfffdbb625b4e9591a4febe4b9600647e42ce2c02", + "libs/policyengine-fastapi/src/policyengine_fastapi/observability/stages.py": "7674cd880b9ee474323ac392b222d0e8e15e7b9305794f0ae302f281d26b609e", + "libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/__init__.py": "1ecb4aeaca6794c1c2ba756d940c36f4345a6eedf9dddd9b420e82ba706b55da", + "libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/console.py": "73abaa0d751f45ed34a691da59c7d053533dcec75c735beea3bc32a4e93f2438", + "libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py": "ec20841d1c9ddf027a319397ab78c17a96e848d22b120e148b8c74f039e56ea8", + "libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/instrumentor.py": "bd63b2175997cf5bff4223a7afd85bedea92a1279903f547b59e539a1642d14b", + "libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/middleware.py": "35d833171d9fb0b86cd5fd04f854e4a1fafc9d267e85b2d12d804bdf7af7a0a8", + "libs/policyengine-fastapi/src/policyengine_fastapi/ping/__init__.py": "b03e435d8c5c6c9d251669441c06e720f62dbdc9e75bd2f7c5fc44674e806644", + "libs/policyengine-fastapi/tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "libs/policyengine-fastapi/tests/conftest.py": "4bc41f67c99e97d38a2577988d2d6df7daf86466be8f9308204985681f24452a", + "libs/policyengine-fastapi/tests/ping/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "libs/policyengine-fastapi/tests/ping/test_alive.py": "21ac2c13efa5eac90c78771f9bc0c01c62e00729bd8f62b1622f4935b1583112", + "libs/policyengine-fastapi/tests/ping/test_ping.py": "a4a7ea1a770cff99c828b9252475665b804f0a77c7759ef2b9c411eac97ca56d", + "libs/policyengine-fastapi/tests/ping/test_started.py": "0a9c2d4c7b120895da51ca2aaa66472161b64bc9a4745721cd0c799b46bdc4a9", + "libs/policyengine-fastapi/tests/test_exit.py": "4e493b7470457d5a5d247890846384d4a70d72476cfe5d94cf94002d750db3ee", + "libs/policyengine-fastapi/tests/test_observability.py": "aab8025c15fa6304cb9a4fc8262c497793b44f79942128387b65e8ba3ffb86ed", + "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/__init__.py": "052d987a106e9a83adf5e3728ab44beaef3849e965dc470b66876afe40f379ae", + "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/budget_window_state.py": "d4b1976cf1bd9ca693e468882ab78764a88b7c7aad7c35c588e37f1e692fe8ae", + "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/dataset_uri.py": "855f4ca52c36e2626f0dbeb09d0de4b1ab5473eeadcc5964cdfd33b5b9afe302", + "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py": "ec4669c02b53bad9e602c7571e4533d9e39ed923d39c769c7454cc0634040a51", + "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/hf_dataset.py": "728b8eb8fe01dc4194466e49446efb94d65740ca3dff4daf9f8db8cc18522078", + "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/json_types.py": "65b54f228c8c127201b71b2ef3ca9a779d6fbdfaeeba684c2ad7e00f1351111c", + "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py": "c860ede3f7ab397ff1f24ee6b9eedf689eeabfc994434ff52e1b2e55fe39e8c6", + "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py": "9a751b3ded7a637b8446d7ef2eeef34cbcc77835b2d112f66a525f4e7d9cacc2", + "libs/policyengine-simulation-contract/tests/conftest.py": "3c78401c6ddd675b1666b7a0f481133a0b6214d788ce13c0a2147ce2ad6ba080", + "libs/policyengine-simulation-contract/tests/test_budget_window_state.py": "0b9f6c2f9007628a67695879d8d6d9f7eaca9e832486285498ebfa5d88de2705", + "libs/policyengine-simulation-contract/tests/test_dataset_uri.py": "d0d99e6c30bc8bcc3a085d43560ceb52565b5cd4b0ca8f2b22bc648c7ea749ff", + "libs/policyengine-simulation-contract/tests/test_gateway_models.py": "b16632e54af9101cdb9f8839bef431957503022fc3389ace7d2ed7c1fcc87e4d", + "libs/policyengine-simulation-contract/tests/test_hf_dataset.py": "a88adca95e0fbab294940ddbdb5d29fe5153b986242a7f58525dffe0c9ce681b", + "libs/policyengine-simulation-contract/tests/test_spm_selection.py": "cc7c92ae1d2b329e6e945d9d8b15111621983f600f7ea94c06c1a0b58562a696", + "libs/policyengine-simulation-observability/src/policyengine_simulation_observability/__init__.py": "e798650c4dac75dff7dc14bc65cafa1eee2cf95030eb59f435fe7f86846445b9", + "libs/policyengine-simulation-observability/src/policyengine_simulation_observability/errors.py": "72d5d712a0faeb33b709ae924cb03df1f62908c389e5c8bebdf9d9863c262b11", + "libs/policyengine-simulation-observability/src/policyengine_simulation_observability/logfire_legacy.py": "8f399e6a7d5c8179569e79d309677d4894cb1f35d9b87a8e7790e5ff02dc847a", + "libs/policyengine-simulation-observability/src/policyengine_simulation_observability/observability.py": "af0f07dd96dfc286ccf615d059d6e97278533162a8403bc76934d5799a62dd65", + "libs/policyengine-simulation-observability/src/policyengine_simulation_observability/telemetry.py": "8c04ba096e9f81f4324b5d97206c93a17cfd343ec5ef3fdfa7287f898ec859f7", + "libs/policyengine-simulation-observability/tests/test_errors.py": "7cf9c8512a639f4735dd705e0ad746241842bbc1fa9ef63537a15e7728c23382", + "libs/policyengine-simulation-observability/tests/test_logfire_legacy.py": "9371b2e2121efe99479522618cec268d1529b22f4e87bd10845d7344527225eb", + "libs/policyengine-simulation-observability/tests/test_observability.py": "2d8ce6cc1363880a559624bf9e74b9e85e0d3f5ad345d80c0db63b0630d258bb", + "libs/policyengine-simulation-observability/tests/test_telemetry.py": "c8e77b6b928fc9b106d529f8947130c1e7a4f42e139c26856e556405806d01f5", + "projects/policyengine-apis-integ/tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "projects/policyengine-apis-integ/tests/simulation/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "projects/policyengine-apis-integ/tests/simulation/conftest.py": "5901dec759cb22a66b09b36e41b970166c4f0e3b2776cb6c4e3a9dc0231c92dd", + "projects/policyengine-apis-integ/tests/simulation/test_auth_smoke.py": "2fedca4601f4615dd52dcf7ca362493c5b16d08635d1ddc678e8b5fcccb19503", + "projects/policyengine-apis-integ/tests/simulation/test_budget_window.py": "1da04eea75c6e5f010f1bc7eada8585ca3de37efc31a3e49a9b13c1cb7ff9431", + "projects/policyengine-apis-integ/tests/simulation/test_calculate.py": "63f2994a5beca6fd8d2d39250d7a7f1ef97f5afe7190c6cedcd3f14e3bec68ae", + "projects/policyengine-apis-integ/tests/simulation/test_ping.py": "c02a38408ad8fa26e39353704383d4625fa5f1ae19ffe066478b0b3a6772d6f0", + "projects/policyengine-apis-integ/tests/test_spm_generated_client.py": "cf213da3d895f558bf39e2d025998694989ebb338964aac9a19c1b457fbeaad9", + "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/__init__.py": "5a781097ce32e248443e2585e8118602bd4932f11cd0e2412e9d0e5647c8a989", + "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py": "1423d7485251dcb89a69aa61bfe02d9269b8228e6434602e28b8d240bef4c60d", + "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/auth.py": "6b8077c854e5273b004eb3a279d036717510d2aeeb29435e2609a0ac5de28449", + "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/backend.py": "62ecce3d42f9ed2b76ac42ac630401a10ac0cb6f450b56e381c2e576b4ada19b", + "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py": "a382920bc4600f5ff9702cdc4a8df9cbf14f5ae556387374122bf265ab1ab981", + "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/generate_openapi.py": "52e1f056053c39f4f36a949eae75259226094780d5e33e97255ba2a2f6e69c2b", + "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py": "a253c53dba6dd1fb732f912d04b4ff8a9b4525cd602cc3d86c9ec515a02f2c9c", + "projects/policyengine-simulation-entry/tests/conftest.py": "2a70c6344369b43d88b65e5f7096f5df32fb7f6ca384e5eaa98463c06c298b3b", + "projects/policyengine-simulation-entry/tests/test_app.py": "58871ea57ffaca310699e10bafbd12bb5bd9cbfd45bb69adf37b744de59bfe99", + "projects/policyengine-simulation-entry/tests/test_auth.py": "ca9f1060432c9f609073da0e2575bf27779331aed13909eec499a1dd59177357", + "projects/policyengine-simulation-entry/tests/test_backend.py": "8829701bda722198614a3525c33a344bed1b700113cae17c2a7671ae982eaa67", + "projects/policyengine-simulation-entry/tests/test_config.py": "8cad43b56f1258b7595b53114f9619bb87e348d96ea6c9af4aa4c791873a09f4", + "projects/policyengine-simulation-entry/tests/test_deployment_assets.py": "f9f33b62d2141f1f27339aee8fa910262c9ab26863651a99faec036a6c3ba588", + "projects/policyengine-simulation-entry/tests/test_openapi.py": "f0a00748a78bc748cd87419fb5fbb7c3854d6aa06f5b76be6a8b0455fb76cb8b", + "projects/policyengine-simulation-executor/fixtures/__init__.py": "2382b67dc056a47f519de1cddc56b7dc54da71ffbe747e439933108a8ff4a9e5", + "projects/policyengine-simulation-executor/fixtures/fake_modal.py": "033eec5c860a24f635823e2f1253069e49d42b6651ef6deb46768af56b80cab1", + "projects/policyengine-simulation-executor/fixtures/identity_stubs.py": "c97d961868804d94be7007f55cbc2e1ea67ef6f6d5ac0adc3dd3b8a7f0d91409", + "projects/policyengine-simulation-executor/fixtures/test_modal_scripts.py": "2346c68ad3b90b1c74be645c7f319ee7b89fc5566a27a2ab5e3a136a88a70128", + "projects/policyengine-simulation-executor/fixtures/test_policyengine_package_update_scripts.py": "81783c06aa2d1623752a090351f3ec791d6618ec5020877ef7b258ab09b9d485", + "projects/policyengine-simulation-executor/fixtures/test_simulation_api_contracts.py": "4698e89d3b75f9a72995ac330e37968cb669fabebce0e82302b002358e65ddd0", + "projects/policyengine-simulation-executor/fixtures/test_simulation_output_builder.py": "5a12e638b3a4486e5f12f0def071cb4e67e2bdc4e3ed6b29dce1661c5f87e4b3", + "projects/policyengine-simulation-executor/fixtures/test_support.py": "c4760869aa6a35cdc8b7a53ccc45a007094f50f8ca693e08b6e9aa6f6e4b8475", + "projects/policyengine-simulation-executor/src/modal/__init__.py": "d3cbe2a2075ca1f6557cc6d89a277db60bb068fd26a503497cc0faa907e89aa0", + "projects/policyengine-simulation-executor/src/modal/_image_setup.py": "2fc8e800e188ada9eebd010516b36768ccbb4473bb92b6da43348d1d385c7f5f", + "projects/policyengine-simulation-executor/src/modal/app.py": "42fe7bcb2787593aced114c70c69fbe846bb9f90ba1cdfcf81a1df11d64e90e6", + "projects/policyengine-simulation-executor/src/modal/budget_window_batch.py": "3eaa98933b1634579e3516e73e33eb2e9bebd8bd1ef68859ddcaae6193362d52", + "projects/policyengine-simulation-executor/src/modal/budget_window_context.py": "e45eb44c85fec41b41aa65b004ec127858a09dd0f1b5fa153a03d9ae4364edab", + "projects/policyengine-simulation-executor/src/modal/budget_window_results.py": "76fe205ea9f0a6a4e2a7d6ce795214978273c0ffccfab8776bd1dbff3bac2fab", + "projects/policyengine-simulation-executor/src/modal/budget_window_scheduler.py": "153d169999ecb8a10887325e29ea647ddb5ebbac7ded8f0c80d5cea8880d42ee", + "projects/policyengine-simulation-executor/src/modal/dependency_pins.py": "fda1e8c64df0d0361d4940ba26b9a9f294c2b82b083f4bc2ca3437ad3a4d05b8", + "projects/policyengine-simulation-executor/src/modal/fanout.py": "23adcd39d819b5c0ca3e8565616045afc09df0b6a0d96352bb71f1c625d09da5", + "projects/policyengine-simulation-executor/src/modal/logging_redaction.py": "3991cd1e575b81d038d9e3b7e19a59e1eebceb41f24eb7aa7a69d90e95dd092e", + "projects/policyengine-simulation-executor/src/modal/precompute_app.py": "8b6e92e0bcfefd3ddf8368baf1c45e63dff79c757ee1c213da89ce236de21de2", + "projects/policyengine-simulation-executor/src/modal/segmented_national.py": "e15246852fabe616b68e15bc3afa5857c14a96213dcc77115c46da73730abd9d", + "projects/policyengine-simulation-executor/src/modal/smoke_app.py": "cebc2ee8c202fb46dc66b606c92d5ce8c44e849fcef26cc514d91c199cac1181", + "projects/policyengine-simulation-executor/src/modal/utils/extract_bundle_versions.py": "612b6a8b7fcc536567269085f9369d2c1718ecc1e51929c7b9012893db34c183", + "projects/policyengine-simulation-executor/src/modal/utils/record_deployment.py": "2757e092db4975f0d63cd28777ddac12d81695644ccb8a1adf1fd786fa805ac2", + "projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py": "1e840d2ebbd39aacef2d5aaf9a2113da4ffd7575f15c8d5e5eeae1f719ccefe8", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/__init__.py": "716b0000bc200acb3f5680b5d90057b3bf46f05f4abd0644de1870644a96e624", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py": "aa144894e2015752666c40e230e33f3aaee6c86f34eebfced2fcee774c1af87a", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_store.py": "35796f1c645ce0fb1a3cb2f28ed0034947a4c148b52cecccf8ce93a443af6fe9", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py": "2ec0d48bd97bad7a663deaf49b7a12c5b887d6b29f571e2cfe90d3c4247ad9c1", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/compat_models.py": "5895c1cbaaa13092e604f9dfd0f1048c0a233c1cf3d8b2fa09ecd009b7e2df1d", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/generate_openapi.py": "730125691928ad5c159b5d16f5902b3366949466cda7ebcc2cdcc03f44dd2fea", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/main.py": "85ff24d4ce177c6580ee25515a5667f7891c9f829905efc3670c256d04c59674", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/national_partition.py": "0bf9b334cea28176c0d6c8348f1df68e0af1666bcfbf010a11a09836735b259f", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py": "2c2a9bb6c824f4cce603f5728ea4cc58a693f8f354beff10b4cb370d41914e45", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute_models.py": "825fb81870bb8439e89a7415f9c1b9c52e2676e5435314a7eb9db6f7ade1f565", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/release_bundle.py": "71941fba56a4b755ba875ecb1d8b2dc48a9eb817132c8bde13f2e8e4ef063548", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/segmented_national_reduce.py": "72cfc7bdcf150cd747533ecb533b8b07a2e743b3fc9fec0ae5f536c9144121fb", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/settings.py": "652a575875df40ac79a3bf460de49a045967cfe2241125c77c68c8ea9dd0401c", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation.py": "a8f9c2a8deddb39972e0973c6c4307372b779dd19739c42c83f9469a57b0d84c", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_macro_output.py": "43bfae627139dc747ebf905d9abc3ef7c58856a8f86f8e638ed62cd87bb25740", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_microdata.py": "9e3a985e061e8ec1a4646738c63e91039b5de780e68475a883eb65faabf1812f", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_budget.py": "034ebaf9ed6d22bb2bcae4f5ba87f9b6abdddfdc0da459bcede5dddd6603ad74", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py": "d15a15b6c9b0d3444434568e034f90cd618655344387a1b2b5661a21cdfdb412", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_cliff.py": "5bd5aac3b005e19e64710e127de782714b2dc7f320131ea7deb438bb397574a6", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py": "e76e9b37f0494c211f0f6945a294c370470c52426c24ebb28a38bf555fb72b22", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_distribution.py": "9fb21e82df52f90bb68f67882fb3cff3fcf9bf56302a10b70300d0a47b236caf", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py": "0467b98ffd514c93bb932ebf24ee6a9eaef449dbbb2f182a5c725e5b9fc90d14", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_inequality.py": "caa2528af2bf1ddd6e82532755ca55f096787b0c36fc8bcdf256ba3a43c946f9", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_labor.py": "1cc2822aeeae7e87155ddf56c6b81996fcf9e2fb3e22f9480e7402755692567b", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_poverty.py": "1ab65438d874bb979da886ea8ee8e08be8216b14d915d6afe6c4d007c8688875", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py": "898326620cb91285d9c3a1f481150ff1e31c91730825bd0770272a228d02d684", + "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py": "82e71283aff85507216e7eefdb471b26f20173351702a998087bb3ebe35e9a48", + "projects/policyengine-simulation-executor/tests/conftest.py": "eb16bfd02b3270fe5ba393286c0acdceb9b7297069f2c0e846d81e93fcc2e855", + "projects/policyengine-simulation-executor/tests/integration/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "projects/policyengine-simulation-executor/tests/integration/test_budget_window_ephemeral_modal.py": "a84d5dd179ff572b922c91c7ad38ff98e0ccd3387c9ac492a31309af8626e87d", + "projects/policyengine-simulation-executor/tests/integration/test_image_smoke_modal.py": "d981b1ed2eb4a5278a755ebc3ebbae539f8ab0892c6341a6ccb27623fb1ba473", + "projects/policyengine-simulation-executor/tests/native_spm_support.py": "c8d4b66b4d3991fccc31c2f2b82684cde920f984388bf461222e4c6194ccdae3", + "projects/policyengine-simulation-executor/tests/test_app_redaction.py": "2965a253b47fedb7166f507d14d7293e497ef7d674e69582b4c16b353f4ccb4b", + "projects/policyengine-simulation-executor/tests/test_artifact_keys.py": "12245baef89b40864aec1e5f0a3aca20b943af378bba44a1d71a26ca3332d606", + "projects/policyengine-simulation-executor/tests/test_artifact_store.py": "5c2742563f1a01442c20a236d3e9340ee302ecb432daac7a62030bfe3c6b801d", + "projects/policyengine-simulation-executor/tests/test_baseline_artifacts.py": "7459eeaaa96076647b1d08ce8e224cc051ccde714ba859ccc69b6bc93466b55c", + "projects/policyengine-simulation-executor/tests/test_baseline_artifacts_spm_native.py": "13cae4f802c28ad605f0bbe0898e58d4029c9e6dee050dbe7c36a46e60c85533", + "projects/policyengine-simulation-executor/tests/test_budget_window_batch.py": "22c4e350c2835a3b832136f08080f932024478ae04ce8b52b5224bf9c20c0e52", + "projects/policyengine-simulation-executor/tests/test_budget_window_context.py": "9e29c42db83337e2e2333409048ee508d90da385e91049ed26a252b0715b4749", + "projects/policyengine-simulation-executor/tests/test_budget_window_results.py": "5ac9b558ed20498a151d50eda026b244eac9e03dfa5276ee3efff854328ece66", + "projects/policyengine-simulation-executor/tests/test_budget_window_scheduler.py": "a29eb56005648e7cd8321347d58a3e036347bec0c213b66a0e93744e652594a8", + "projects/policyengine-simulation-executor/tests/test_bundle_version_export.py": "23ccf3dfad99af01fe03468ab7e1857bd63880bf7ffea53bdea3feed3f7b5038", + "projects/policyengine-simulation-executor/tests/test_canonical_spm.py": "74e32f6ab77c8aa025181c7005c88a178c0ac8b694dc19910b22efb06fecc122", + "projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py": "a2d7877785af38af76b03ea37ed3b1f934cb8ebd0faa25402c63bca96e130860", + "projects/policyengine-simulation-executor/tests/test_gcp_credentials.py": "3b8b8743f38eabef7fab48bd3d756c7c7742cfeff7d8b71527e52f2e24ec54cb", + "projects/policyengine-simulation-executor/tests/test_image_setup_fetch.py": "1275099acd7d95b632132811709e539fb70b7e67a139fc856b58c4c09018441e", + "projects/policyengine-simulation-executor/tests/test_modal_bundle_image.py": "df05f3b496deaa4b929970e4dea7ab91d06b6192ac8fed3c5158b6df61f99512", + "projects/policyengine-simulation-executor/tests/test_modal_scripts.py": "2fd24ac6d2c35212d74f46c7cee4e437c9352859b5f7ce66b28802ce65fb18e1", + "projects/policyengine-simulation-executor/tests/test_national_partition.py": "d3111ad8dda7150257c595d1dde9211eb34abc5e7e2716851c0e3ac730fc79ef", + "projects/policyengine-simulation-executor/tests/test_pandas3_compatibility.py": "f230a82892f3076952361fa5522095269a8634c99bf6c66be2cf4daaccb637cc", + "projects/policyengine-simulation-executor/tests/test_placeholder.py": "f835fd2d1ece403312811e9fdba7f2c911ad0b7dc53de4892d29b2e9c69da68a", + "projects/policyengine-simulation-executor/tests/test_policyengine_dependency_source.py": "d3b5fab0d7be9ad1251717104bbe3e71186be88efd5024c15fc8eb2396d669c9", + "projects/policyengine-simulation-executor/tests/test_policyengine_package_update_scripts.py": "965c7e8c79e3f9a8d604d4c74e7fba3fcc28bcb506cd6e8d5688644e73799a38", + "projects/policyengine-simulation-executor/tests/test_precompute.py": "512762d5dff607c66682aacb64d05e9a6f6c6a29bb1c118f9e2bb192a7a76ea0", + "projects/policyengine-simulation-executor/tests/test_precompute_app.py": "892276cd8e229b98010aba072f041f9283d46f11a9bc99e4467bf1edefdb2b73", + "projects/policyengine-simulation-executor/tests/test_record_deployment.py": "0b8dd6310c496ac6f736b6c6812671ae54768e52e7c3c5f8d2d069e06c271f81", + "projects/policyengine-simulation-executor/tests/test_region_group_resolution.py": "cb4b65d6301d4515f466f5e4f933e44396cfc04f99e62a8fe3f10a1091fa2c44", + "projects/policyengine-simulation-executor/tests/test_release_bundle.py": "8477397d5d24d8b4db257b35a5cfbd03d858dbd265634c5ec85d06353b388a25", + "projects/policyengine-simulation-executor/tests/test_segmented_national.py": "f236d257071d33803a37ff27b771cb6a18b973a70023025eb77b291c31ac0dfd", + "projects/policyengine-simulation-executor/tests/test_segmented_national_reduce.py": "4024783eb555f28e10ef83ad3ecddd30316ed0966d39b5a60c6447a64a834efa", + "projects/policyengine-simulation-executor/tests/test_settings.py": "c32a6c23d93e33accb0deb044a73bebc9faaecb5bfa34e33bf1d30c6eca939e8", + "projects/policyengine-simulation-executor/tests/test_simulation_api_contracts.py": "00d72ce7622b2999af10757afed8fc853cf3f987e508e31db9a7b4ed8ddcf546", + "projects/policyengine-simulation-executor/tests/test_simulation_microdata.py": "926f3d4a3b09ee9c27aef5cd9bd5a7532cc4098cfccccb90578e0db1eb887000", + "projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py": "f2d3b27b0968418c5b5dd117c90c3cab09ec5fb9bd71539bdcf97628e3974180", + "projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py": "af7dafb2639620e0644d2df75e57809070a8252b8ea62ae8c2280ad83ffc8555", + "projects/policyengine-simulation-executor/tests/test_standalone_simulation_contract.py": "a09935edd48c11b91090a9bd1a5652f0393959c1202630e0a83dd6695e3dfbae", + "projects/policyengine-simulation-executor/tests/test_update_version_registry.py": "92f0e2dfb8fa24c6edd24c018f6a89f27c94f508a095376bb2004eb056890f6f", + "projects/policyengine-simulation-gateway/fixtures/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "projects/policyengine-simulation-gateway/fixtures/gateway_endpoints.py": "1402ad55117a2ae76e97e666a321eae361ff59797be269ce1edc8a2e6b503fe7", + "projects/policyengine-simulation-gateway/fixtures/package_imports.py": "1864c22dbe8438c342be9a14fe100ac91b3d8927f8532cf45b7963325b6e29cc", + "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/__init__.py": "374a88d7888c27fddada550766f489ebd83121610be4f56c3f392816b52b04e6", + "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/app.py": "fef0ca6f10bfb12248d0107cc29bc68194c7cca863bdded00dbf3624d550c708", + "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/auth.py": "fc512f7dc9dfd9eb8b056f166dacb7f24f6541205a9c1d4b2f5752f07cec227b", + "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py": "adaae4cb7c0a21dd728ca4c8c0b85db934af964068a302af17f1a9852d20d8f3", + "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/generate_openapi.py": "7b2cc5be0a20fe2cdd75a05fc852cee701a119c8cf9622f7d8b28b9e800e047c", + "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/responses.py": "7e48b9d01ca6990df78f46df9f281620a9e16e37f33be237358112c90b9c4d93", + "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/smoke_app.py": "7fa96ae81dbc27b90e6dc77300e55275e9b2ad11f50e8c40bfac4b9d22d06c84", + "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/testing.py": "25f95c79db79f9ec2d3cc693b75d74e2c2ba9683b0a1a1ff377bd35bada52ab3", + "projects/policyengine-simulation-gateway/tests/conftest.py": "873d596435f011f3fc23a274c9ecd7ea39abb0b7a25fc5d5c39f7a5a9ada1ac2", + "projects/policyengine-simulation-gateway/tests/integration/test_image_smoke_modal.py": "141f363280f9f7ca2f84f70fc24cc31608db4e460a83e3ca29e7273a979f81c6", + "projects/policyengine-simulation-gateway/tests/test_auth.py": "edce21cced7146d90fbc2ed6c0fd226f8f6aa4f893527197bb99075ea70e5fed", + "projects/policyengine-simulation-gateway/tests/test_endpoints.py": "284a4cc6636c9f929f76012d91f817d5f20e49140351aa6dbee22be2721ff778", + "projects/policyengine-simulation-gateway/tests/test_health.py": "429a017d164e8a4324f2c1aa102e3caf91bcad3176cc510f5496bd642192fd41", + "projects/policyengine-simulation-gateway/tests/test_import_coverage.py": "9437892c4461e15b6695c44fd67dda55b249fe6c423456a6fe6a80b656616f2f", + "projects/policyengine-simulation-gateway/tests/test_modal_gateway_image.py": "e4ca9a47c9a5b17c2986db300f48f537b40590f05179394e63918468a77867e5", + "projects/policyengine-simulation-gateway/tests/test_observability.py": "c9492dcee6dfb09dac6f0408d025e395433befa8875bbcb6aac7e2ca92123b5d", + "projects/policyengine-simulation-gateway/tests/test_openapi_golden.py": "ab02b0316789b7e3f94843af34f84d0fa5f37aa642313b452af50438577a3290", + "projects/policyengine-simulation-gateway/tests/test_package_imports.py": "2e05408df800e4f6855ac8dc56cdbe6d28135a8c563c5bb572132aaef3b85102", + "projects/policyengine-simulation-gateway/tests/test_ping.py": "4d33e6ba7573e83d0e96292dfc66822b417135e8e35422597157a5fee5731d05", + "projects/policyengine-simulation-gateway/tests/test_route_table.py": "28778a2b3da017cac8fad9de5dcf899f8d96b0e913527ddd7a341f9ff159c34a", + "projects/policyengine-simulation-gateway/tests/test_spm_routes.py": "a97d31eb784504d01f64e80f1ec2e84f72ada13770edfd37f981681612fc08cf" + } +} From 7e886763c4853776780cde15a18e70e8cfed23ef Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 17:56:25 -0400 Subject: [PATCH 21/40] Keep review evidence outside the product source --- rollout/worker-fable-fixes/.gitignore | 6 - rollout/worker-fable-fixes/FINAL-REPORT.md | 134 ------- rollout/worker-fable-fixes/PR-BODY.md | 13 - rollout/worker-fable-fixes/PROGRESS.md | 28 -- .../worker-fable-fixes/artifact-evidence.md | 119 ------ .../changed-python-files.txt | 16 - .../handoff-preservation.json | 4 - .../native-wheel-summary.json | 111 ------ .../worker-fable-fixes/routing-evidence.md | 60 --- .../worker-fable-fixes/run_artifact_native.py | 15 - .../run_installed_qualification.py | 218 ----------- .../worker-fable-fixes/runtime-evidence.md | 33 -- .../runtime-regression-green.txt | 14 - .../runtime-regression-red.txt | 360 ------------------ .../worker-fable-fixes/source-binding.json | 198 ---------- .../worker-fable-fixes/typing-comparison.json | 54 --- 16 files changed, 1383 deletions(-) delete mode 100644 rollout/worker-fable-fixes/.gitignore delete mode 100644 rollout/worker-fable-fixes/FINAL-REPORT.md delete mode 100644 rollout/worker-fable-fixes/PR-BODY.md delete mode 100644 rollout/worker-fable-fixes/PROGRESS.md delete mode 100644 rollout/worker-fable-fixes/artifact-evidence.md delete mode 100644 rollout/worker-fable-fixes/changed-python-files.txt delete mode 100644 rollout/worker-fable-fixes/handoff-preservation.json delete mode 100644 rollout/worker-fable-fixes/native-wheel-summary.json delete mode 100644 rollout/worker-fable-fixes/routing-evidence.md delete mode 100644 rollout/worker-fable-fixes/run_artifact_native.py delete mode 100644 rollout/worker-fable-fixes/run_installed_qualification.py delete mode 100644 rollout/worker-fable-fixes/runtime-evidence.md delete mode 100644 rollout/worker-fable-fixes/runtime-regression-green.txt delete mode 100644 rollout/worker-fable-fixes/runtime-regression-red.txt delete mode 100644 rollout/worker-fable-fixes/source-binding.json delete mode 100644 rollout/worker-fable-fixes/typing-comparison.json diff --git a/rollout/worker-fable-fixes/.gitignore b/rollout/worker-fable-fixes/.gitignore deleted file mode 100644 index 0b6840c44..000000000 --- a/rollout/worker-fable-fixes/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -pytest-*/ -__pycache__/ -*.log -native-wheel-receipt.json -*-pyright.json -*-pyrightconfig.json diff --git a/rollout/worker-fable-fixes/FINAL-REPORT.md b/rollout/worker-fable-fixes/FINAL-REPORT.md deleted file mode 100644 index f808cafc8..000000000 --- a/rollout/worker-fable-fixes/FINAL-REPORT.md +++ /dev/null @@ -1,134 +0,0 @@ -# PR677 Fable fixes - -All four findings are fixed locally. The final combined qualification passed **34 installed native tests, zero skips or failures**, in 288.39 seconds. Service/client checks complete **747 passes** (including the explicitly retested version-extraction case). Remote delivery is blocked by this session's access restrictions, not by a code or test failure. - -## Source and review binding - -- Owned checkout: `/Users/maxghenis/spm-rebuild-20260908/worktrees/policyengine-sim-api-canonical`. -- Starting worker head: `dcfe4fd89d97145882cb147a9aa2ffae2cd10ebf`. -- Final implementation and tests: `c431d138bdbad9be71c12b6994851879cf5d05aa`; later commits contain qualification evidence and reports only. -- Live canonical `main`: `414c631d622f5f587eedd187296a80182a923db2`, independently fetched through the authenticated GitHub connector and equal to local `origin/main`. No base integration or checkout was needed. -- Existing issue: [#676](https://github.com/PolicyEngine/policyengine-sim-api/issues/676), verified open and appropriate. Existing [PR677](https://github.com/PolicyEngine/policyengine-sim-api/pull/677) is draft, unmerged, on canonical branch `max/spm-simulation-canonical-20260909`, still remotely at the starting head. -- Exact reviewer output read: `/Users/maxghenis/chief-of-staff/state/subfleet/gates/20260909-170551-pr-abdf4629/rounds/001-4542bf03eca1/peer-output.md`. Its complete findings remain valid despite dispatch's BrokenPipeError. **No durable Fable agreement exists or is claimed.** -- Repository testing/PR skills, installed PolicyEngine analysis/standards/API skills, parent PolicyEngine guidance, and wrapper repository guidance were read. GitNexus debugging guidance was read; no callable GitNexus graph was available, so source tracing used the actual repository and installed packages. -- Untracked root `PROGRESS.md` and `WORKER-CANONICAL-SPM-HANDOFF.md` were preserved. Their original fingerprints are committed in `handoff-preservation.json`. This continuation's committed state/done/next journal is the adjacent `PROGRESS.md`. - -## Finding disposition - -| Finding | Disposition and concrete proof | -| --- | --- | -| 1. Historical routes without wrapper versions incorrectly return 400 | Fixed. Internal route provenance distinguishes the actual legacy country dictionary and schema-v1 `legacy-seed` paths. Unknown-wrapper acceptance additionally requires a numeric historical US model through 1.764.6. A future wrapper inferable from the app name cannot be erased by missing seed metadata. Annual and budget-window submissions accept supported historical shapes; explicit SPM selections, future/unknown models, unproven seeds, ambiguous siblings and malformed capabilities reject. | -| 2. Year/scenario prevalidation loses typed errors | Fixed. Uses the installed `PolicyEngineSPMProvider.year_metadata` contract, preserving `SPM_YEAR_UNAVAILABLE` at annual/year-alias and budget-window start/end boundaries. Narrow scenario translation matches the calculator's public adapter contract and preserves `SPM_SCENARIO_UNAVAILABLE` through HTTP/poll/pickle/optional-analysis transport. Other invalid settings retain `SPM_SETTINGS_INVALID`. Prevalidation does not measure amounts; actual country tax-only calculations remain lazy outside forecast years. | -| 3. Incomplete cached receipt becomes fatal | Fixed. Check required columns first; an unusable cached receipt follows incomplete → run/save/cache replacement. Capture the requested selection before wrapper cache restoration and restore it before recomputation. Eight real one-household HDF/cache cases cover tax-only missing columns, empty years, missing/malformed receipt, changed recorded selection, corrupt HDF and cache selection isolation; repaired disk and memory results replay as valid hits. Real run/save errors still propagate. | -| 4. Precompute omits storage identity check | Fixed. Compare actual wrapper `storage_id` with `Path(expected.path).stem`, the planner's serialized storage identity, beside the existing simulation-id guard and before configuring, running or uploading. Historical wrappers without the property retain their bare-id contract. A mismatch rejects even when simulation ids agree; the installed-wrapper identity test passes. | - -Related notes are also resolved: shared-app routing uses exact country metadata only when every sibling is positively classified, excludes `latest` aliases, and rejects missing/blank metadata or unresolved ambiguity. Invalid stored capabilities are omitted with a warning from `/versions` and rejected with a typed configuration error on submission. Coupled geography validation happens after defaults resolve, so partial metro area selections obey the declared inheritance contract while invalid resolved combinations fail. Public SPM schemas remain unchanged by those validation fixes. The stale gateway golden from the original PR was regenerated, and entry/gateway schema comparison now covers their shared SPM schema instead of stripping it. - -The runtime-capability performance note was intentionally left alone. No performance refactor or unrelated model/style change was made. See `routing-evidence.md`, `runtime-evidence.md`, and `artifact-evidence.md` for detailed regression-first evidence and bounded independent cross-reviews. - -## Checks and exact commands - -Commands below were run in this checkout, with service commands run from the stated directory. `WORKER`, `QUAL`, `PY`, and `OUT` expand as follows: - -```sh -WORKER=/Users/maxghenis/spm-rebuild-20260908/worktrees/policyengine-sim-api-canonical -QUAL=/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification -PY="$WORKER/projects/policyengine-simulation-executor/.venv/bin/python" -OUT="$WORKER/rollout/worker-fable-fixes" -``` - -| Check | Result | -| --- | --- | -| Gateway full unit suite | 149 passed, 1 integration case deselected; 0.77 seconds. | -| Entry full unit suite | 66 passed; 1.20 seconds. | -| Shared contract full suite | 73 passed; 0.05 seconds. | -| Executor full suite | 454 passed, 22 opt-in native cases skipped, 2 integration cases deselected; one dependency-download failure, 23.62 seconds. That exact version-extraction test passed with syncing disabled, 1.21 seconds, completing 455 unit checks. | -| Actual regenerated Python client | 4 passed; 0.13 seconds. | -| Installed native/runtime regression lane | Clean RED: 8 failures / 3 passes. GREEN: 56 canonical/runtime checks passed, zero skips. | -| Installed artifact/precompute lane | Clean RED: 6 failures / 40 passes (five target defects plus an existing bare-id filename assertion). GREEN: 45 passed plus the separately corrected identity assertion passed; all 8 actual native artifact cases passed. | -| Combined authenticated native qualification | **34 passed, zero skipped/failed**, 2 expected plugin-rewrite warnings; 288.39 seconds. Covers original native bridge, prior lazy tax-only/provider behavior, all 8 artifact cases and 11 new runtime boundary cases. | -| Installed dependency check | All 180 distributions compatible. | -| Ruff format/lint and diff whitespace | All 16 changed Python files pass; `git diff --check` passes. | -| Pyright against exact installed environment | Entry/contract: zero diagnostics. Executor: 145; gateway: 78. Counts and diagnostic identities exactly match the archived unchanged starting source, with **zero new diagnostics**. | - -Full service commands, from the corresponding gateway, entry, executor, or shared-contract project directory: - -```sh -env -u PYTHONPATH UV_CACHE_DIR=/tmp/worker-fable-uv-cache \ - uv run --offline --no-project --python "$PY" python -m pytest tests/ -q -``` - -The executor's pre-existing script test calls `uv run` internally, which attempted an unavailable NumPy download. It was rerun against the existing installed environment without synchronization: - -```sh -# cwd: projects/policyengine-simulation-executor -env -u PYTHONPATH UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_NO_SYNC=1 \ - uv run --offline --no-project --python "$PY" python -m pytest \ - tests/test_modal_scripts.py::TestModalExtractVersions::test_extracts_versions_from_policyengine_bundle -q -``` - -Client generation and actual generated-client checks: - -```sh -# cwd: checkout root -UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_OFFLINE=1 ./scripts/generate-clients.sh -# cwd: projects/policyengine-apis-integ -PYTHONPATH="$WORKER/projects/policyengine-simulation-entry/artifacts/clients/python" \ - UV_CACHE_DIR=/tmp/worker-fable-uv-cache \ - uv run --offline --no-project --python "$PY" python -m pytest \ - tests/test_spm_generated_client.py -q -``` - -The gateway golden was regenerated with the actual module, then copied from its generated output as documented by the golden test: - -```sh -# cwd: projects/policyengine-simulation-gateway -env -u PYTHONPATH UV_CACHE_DIR=/tmp/worker-fable-uv-cache \ - uv run --offline --no-project --python "$PY" python -m policyengine_simulation_gateway.generate_openapi -cp artifacts/openapi.json tests/golden/openapi.json -``` - -Combined installed qualification, dependency and lint commands, from the checkout root: - -```sh -env -u PYTHONPATH PYTHONSAFEPATH=1 UV_CACHE_DIR=/tmp/worker-fable-uv-cache \ - uv run --offline --no-project --python "$QUAL/venv/bin/python" \ - python -P rollout/worker-fable-fixes/run_installed_qualification.py -UV_CACHE_DIR=/tmp/worker-fable-uv-cache uv pip check --python "$QUAL/venv/bin/python" -git diff --name-only dcfe4fd HEAD -- '*.py' > "$OUT/changed-python-files.txt" -UV_CACHE_DIR=/tmp/worker-fable-uv-cache uv run --offline --no-project ruff format --check $(cat "$OUT/changed-python-files.txt") -UV_CACHE_DIR=/tmp/worker-fable-uv-cache uv run --offline --no-project ruff check $(cat "$OUT/changed-python-files.txt") -git diff --check -``` - -Pyright used the installed cached 1.1.411 distribution, with identical Python 3.13 settings, source overlays and authenticated qualification venv on both sides. The base was created with `git archive dcfe4fd89d97145882cb147a9aa2ffae2cd10ebf | tar -x -C /tmp/worker-fable-base`, without a checkout. Each of the executor/gateway `base` and `current`, plus current entry/contract checks used: - -```sh -node /Users/maxghenis/.cache/uv/archive-v0/ELraG5tqAzXmiwNiWpYsc/pyright/dist/index.js \ - --project "$OUT/current-executor-pyrightconfig.json" --outputjson -``` - -Replace the config basename with `base-executor`, `base-gateway`, `current-gateway`, `current-entry` or `current-contract` as appropriate. The exact configurations and raw diagnostic JSON remain locally in this evidence directory; the diagnostic comparison is committed as `typing-comparison.json`. Bulk service/native logs are retained locally but ignored by git; selected runtime regression outputs are committed. Regression-first commands and outcomes are in the three committed lane evidence files. - -## Authenticated scientific inputs and qualification limits - -| Package | Version | Wheel SHA256 | -| --- | --- | --- | -| policyengine | 5.3.0 | `8c640d967575dddad70840bcbe938cea251c56958eded647c83eb9c5902735f1` | -| policyengine-us | 1.824.7 | `7644819916a4f8ca2aa37a4a9d992fa834bfa66ab2d441326a9686baa5c1a688` | -| policyengine-core | 3.30.1 | `2dbcf5f590a0199a7b7c77fcbbda2ff6bc289f6c6169ca0af3beace35d8b3e63` | -| spm-calculator | 1.0.0 | `c49c41da5fd482e563eaea956e205a3ba6841cadd4dd32bef4c0c3dbce17ffba` | - -The harness checks all 17,613 installed scientific package files byte for byte against these wheels and checks real import origins. Only worker/shared-library source is overlaid. It reuses the existing reviewed development manifest and explicit bootstrap; it does not fabricate receipt/version metadata or certify the inherited data. Actual artifact receipts are produced by the installed model/calculator, then deliberately damaged only within temporary regression fixtures. The model source itself is unchanged. `native-wheel-summary.json` records all 34 passing cases, import origins and the raw authenticated receipt digest; `source-binding.json` binds all 190 tracked worker/source test files byte for byte to the implementation commit. Post-run checks confirm those files and both untracked handoffs are unchanged. - -Forecast content identity: `3d86d5c4c0423480e6b69b75d222ffa4a7a2639e4094df5ba2504af01be17173`. Native H5 identity: `6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84`; only its indexed first household and four people are simulated, and its full hash is checked before and after. The existing explicit development manifest is SHA256 `c78faac072f85109ac50d36a9e538a4dae56af4cdc98d2281603a3f4849a591f`; the bootstrap is SHA256 `f762892a374c3950af54834f0fb234708addc59ad4af90aab99866eb37e27a5e`. - -Earlier temporary launchers exposed an unrelated pre-existing `/private/tmp/h2.py` through Python's script-directory import path; that file printed another experiment's output. Those runs were discarded/interrupted and superseded by clean safe-path runs. The unrelated file was not modified. The qualification command clears inherited `PYTHONPATH` and uses explicit installed Python 3.13; the final harness requires the variable to be absent and rejects `/tmp` on `sys.path`. - -Residual risks: no durable Fable agreement; existing executor/gateway type debt remains; final published model/data bundle, required CI, Docker/service integration and live rollout verification remain outside this bounded local qualification. Unknown future legacy bundles still need explicit measured capability or a reviewed routing/model contract; absence of capability never broadly implies legacy. Scenario error translation remains coupled to the calculator adapters' narrow, tested message prefix. No browser, population job, package/data publication, deployment, merge, external messaging, `.err` read or `.lane.log` read occurred. - -## Delivery status - -Local coherent steps are committed throughout. `git fetch origin`, `gh repo view`, and `git push origin HEAD:refs/heads/max/spm-simulation-canonical-20260909` failed on GitHub DNS/API connectivity. Authenticated connector reads confirmed the canonical repository, base, issue and draft PR. The fallback GitHub write tool returned: **“MCP tool call requires approval, but approval policy is never.”** No remote Git object, branch or PR-body mutation occurred; the API transfer was stopped at that boundary. - -The reviewed, update-ready body is `PR-BODY.md`, beginning `Fixes #676`. It is prepared locally and has not been posted. Per `docs/engineering/skills/github-prs.md`, “If you cannot push to the canonical repository, stop and ask for access. Do not create a fork PR as a fallback.” Finishing delivery requires Git network access or an approved GitHub write channel; no fork fallback or merge is authorized or attempted. diff --git a/rollout/worker-fable-fixes/PR-BODY.md b/rollout/worker-fable-fixes/PR-BODY.md deleted file mode 100644 index 82362a9f1..000000000 --- a/rollout/worker-fable-fixes/PR-BODY.md +++ /dev/null @@ -1,13 +0,0 @@ -Fixes #676 - -Carry canonical SPM choices through annual comparisons, budget windows, segmented execution and baseline reuse. Exact bundle capabilities, request/cache identities and baseline/reform receipts retain forecast hash, scenario, geography and covered years; typed errors survive worker and HTTP transport. - -Preserve ordinary no-SPM submissions on proven historical country-dictionary and schema-v1 legacy-seed routes, using model metadata and route provenance. Unknown/future bundles, incomplete seed metadata, ambiguous sibling bundles and explicit unsupported canonical selections fail closed. Invalid registry capabilities are omitted from discovery and rejected on submission; partial selections resolve against declared defaults before coupled geography validation. - -Worker prevalidation uses the installed calculator adapter's typed year contract and retains `SPM_SCENARIO_UNAVAILABLE`. Incomplete cached SPM output or receipts recompute with the original requested selection, while corruption and selection-isolation safeguards remain intact. Precompute verifies the wrapper's actual storage identity against the planned filename before computation/upload. Gateway OpenAPI golden and entrypoint schema parity now include the original PR's SPM extensions; clients were regenerated. - -Validation: 149 gateway, 66 entrypoint, 73 shared-contract and four actual generated-client tests pass. Executor checks complete 455 passes: the full run had 454 passes and one dependency-download failure, then that exact test passed against the existing environment with syncing disabled; 22 opt-in native cases were skipped and two integration cases deselected in the ordinary unit run. Regression-first installed tests reproduced the defects. Combined authenticated installed qualification passed 34 native tests with zero skips/failures in 288.39 seconds; exact source binding and commands are recorded in `rollout/worker-fable-fixes/FINAL-REPORT.md`. Ruff format/lint and whitespace checks pass. Entry/contract type checks have zero diagnostics; executor/gateway have zero new diagnostics against unchanged starting source (145/78 existing). - -Native qualification uses only one household/four people and the authenticated calculator c49c41da…, country 76448199…, wrapper 8c640d96… and Core 2dbcf5f5… wheels. The 17,613 installed scientific package files are verified against their wheels. The existing development manifest/bootstrap is explicit qualification evidence; no receipt/version metadata is fabricated and no model package source changes. - -This remains a draft worker PR. The complete Fable review's dispatch failed with BrokenPipeError, so there is no durable gate agreement; the four findings are dispositioned with concrete tests and independent bounded cross-review. Final published model/data bundle qualification, required CI, Docker/service integration, live rollout verification and fresh durable review remain pending. No browser, population job, package/data publication, deployment or merge was performed. diff --git a/rollout/worker-fable-fixes/PROGRESS.md b/rollout/worker-fable-fixes/PROGRESS.md deleted file mode 100644 index bf0e7c90e..000000000 --- a/rollout/worker-fable-fixes/PROGRESS.md +++ /dev/null @@ -1,28 +0,0 @@ -# PR677 Fable fixes - -## State - -- Local implementation, regressions, installed qualification and reporting are complete. Implementation/test head: c431d138bdbad9be71c12b6994851879cf5d05aa; later commits contain evidence/reports only. -- All four Fable findings and related capability/selection notes are dispositioned. Complete review read from gate 20260909-170551-pr-abdf4629, round 001-4542bf03eca1; dispatch BrokenPipeError means no durable gate agreement exists. -- Remote delivery is blocked: Git/gh cannot reach GitHub from the shell; authenticated connector reads work, but the write tool requires approval while this session's policy is never. No remote mutation occurred. Existing canonical draft PR677 remains at dcfe4fd89d97145882cb147a9aa2ffae2cd10ebf. -- Live main verified through authenticated GitHub reads at 414c631d622f5f587eedd187296a80182a923db2, identical to local origin/main; issue #676 verified appropriate. No destructive checkout or base integration. -- Existing untracked root PROGRESS.md and WORKER-CANONICAL-SPM-HANDOFF.md preserved byte for byte. This committed progress file tracks only this continuation. - -## Done - -- Inspected status/HEAD/remotes and attempted fetch before edits; read exact reviewer output, prior rollout evidence, repository testing/PR guidance and relevant installed PolicyEngine skills. -- Committed failing regression steps, implementation steps, further adversarial route fixes and evidence throughout. -- Fixed proven historical routing with fail-closed future/ambiguous capabilities, typed calculator year/scenario errors at annual/window boundaries, incomplete artifact recomputation with selection isolation, and precompute storage identity rejection. -- Resolved malformed discovery/submission capabilities and partial geography defaults. Refreshed gateway OpenAPI golden, strengthened schema parity and regenerated actual clients. -- Service/client checks: 747 passes total (gateway149, entry66, contract73, executor454 plus its one environment-dependent script test successfully retested without syncing, generated client4). Ordinary executor run deliberately skipped22 opt-in native cases and deselected2 integration cases. -- Final combined authenticated native qualification: 34 passed, zero skips/failures, 288.39 seconds, two expected plugin-rewrite warnings. Authenticated all17,613 installed scientific package files against c49c/76448/8c640/2dbc wheels; model source unchanged. Indexed execution only: one household/four people. Source H5 unchanged. -- Final190 tracked worker/test source files match implementation commit byte for byte. Summary and full source binding committed; raw3MB receipt and bulk logs retained locally. -- Installed uv pip check: all180 distributions compatible. Ruff format/lint passes all16 changed Python files; diff whitespace passes. Pyright entry/contract zero diagnostics; executor145/gateway78 identical to unchanged starting source, zero new diagnostics. -- Independent bounded cross-reviews found no additional actionable defects. Earlier temporary launcher results were discarded after unrelated pre-existing /tmp/h2.py import shadowing; clean safe-path launchers reproduced and passed the regressions. Unrelated file untouched. -- Wrote FINAL-REPORT.md with exact commands/results/source binding/risks and PR-BODY.md beginning Fixes #676. No browser, population job, package/data publication, deployment, merge, external messaging, .err or .lane.log reads. - -## Next - -- Provide Git network access or an approved GitHub write channel to deliver the already committed changes. Repository PR skill explicitly requires asking for access when canonical push is unavailable; do not create a fork fallback. -- Once access is available, recheck live base and existing PR head, push this branch to PolicyEngine/policyengine-sim-api, apply PR-BODY.md and verify canonical draft/head/message. Do not merge. -- Fresh durable Fable agreement, required CI and release/live rollout gates remain coordinator-owned; local test results are not deployment or publication approval. diff --git a/rollout/worker-fable-fixes/artifact-evidence.md b/rollout/worker-fable-fixes/artifact-evidence.md deleted file mode 100644 index 24016bc98..000000000 --- a/rollout/worker-fable-fixes/artifact-evidence.md +++ /dev/null @@ -1,119 +0,0 @@ -# Artifact findings 3 and 4 - -State: findings 3 and 4 fixed; regression-first and installed qualification passed. -The exact Fable review from gate `20260909-170551-pr-abdf4629`, round -`001-4542bf03eca1`, was read along with the repository testing skill. No durable -agreement is inferred from that review. - -The installed regression harness authenticates the original c49c calculator, -76448 country, 8c640 wrapper and 2dbc core wheels with the existing qualification -script's complete verification/bootstrap prefix. It uses the original explicit -unverified development manifest; it does not synthesize receipt/version metadata. -Tests calculate authentic one-household receipts and deliberately damage only -local temporary artifacts or cache entries when exercising corruption handling. - -Clean regression command (worker repository root): - -```sh -env -u PYTHONPATH UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache \ - uv run --no-project \ - --python /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/bin/python \ - python -P rollout/worker-fable-fixes/run_artifact_native.py \ - projects/policyengine-simulation-executor/tests/test_baseline_artifacts_spm_native.py \ - projects/policyengine-simulation-executor/tests/test_precompute.py \ - -q -p no:cacheprovider --basetemp /private/tmp/worker-fable-artifact-red-clean \ - > rollout/worker-fable-fixes/artifact-native-red-clean.log 2>&1 -``` - -The clean run imports the unchanged artifact implementations from `dcfe4fd`. -The clean RED run completed with **6 failed, 40 passed in 173.47 seconds**. Five -failures reproduce the target defects: tax-only missing columns, complete columns -with empty receipt years, cached missing receipt, cached sibling selection, and -planned storage-id mismatch. A sixth, previously existing test assertion assumed -all planned filenames were bare simulation ids; it now asserts the planner -`storage_id`, which preserves the legacy contract and handles actual SPM bundles. The missing/malformed receipt, damaged HDF, and -recorded-selection isolation cases exercise the actual installed wrapper load, -save, cache, and model calculation paths. A focused precompute test expects -rejection when simulation ids agree but the planned storage filename differs. - -An earlier attempt used `/private/tmp/run_artifact_native.py`; the script directory -made unrelated `/private/tmp/h2.py` shadow an optional dependency, so that run was -interrupted and is superseded. The repository-owned launcher plus `python -P` -removes that import path; its clean output has none of the unrelated experiment -text. An initial bare `uv run --no-sync pytest` selected an inherited Python 3.14 -executable and failed collection; it provides no test evidence. All qualification -commands now select the authenticated Python 3.13 environment explicitly. - -Ruff check passes for the four new/updated test files after formatting. Root -handoff files were left untouched. - - -## Final implementation and checks - -The guard captures the original request selection before the installed wrapper -restores cached metadata. Missing columns are checked before receipt validation; -invalid cached receipts take `OUTCOME_INCOMPLETE` and the existing run/save/cache -replacement path. The request selection is restored before recompute. Real run or -save errors still propagate. Missing/malformed HDF receipts, differing recorded -selections, and corrupt HDF bytes retain the wrapper's existing miss/recompute -behavior. The repaired artifacts and cache entries then replay as hits with valid -actual calculation receipts and without another model run. - -Precompute compares the real wrapper's `storage_id` (falling back to id only for -historical wrappers) with `Path(expected.path).stem`, which is the planner's -serialized storage id. This occurs beside the simulation-id comparison, before -configuration, ensure, or upload. No precompute wire schema changed. - -Final GREEN used the clean command above with -`--basetemp /private/tmp/worker-fable-artifact-green` and -`-k 'not test_precompute_identity_equals_runtime_id'`, writing -`artifact-native-green.log`: **45 passed, 1 deselected, 2 warnings in 207.33 seconds**. -All eight native artifact cases passed. The deselected assertion had been -corrected separately after the GREEN process imported its test module; the -following exact installed check closes that case: **1 passed, 2 warnings in -0.09 seconds** (`artifact-installed-identity.log`). - -```sh -env -u PYTHONPATH UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache \ - uv run --no-project \ - --python /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/bin/python \ - python -P rollout/worker-fable-fixes/run_artifact_native.py \ - projects/policyengine-simulation-executor/tests/test_precompute.py::TestWriterReaderContract::test_precompute_identity_equals_runtime_id \ - -q -p no:cacheprovider --basetemp /private/tmp/worker-fable-artifact-identity \ - > rollout/worker-fable-fixes/artifact-installed-identity.log 2>&1 -``` - -From `projects/policyengine-simulation-executor`, the explicit historical Python -3.13 focused suite also passed **68 tests in 13.02 seconds**: - -```sh -env -u PYTHONPATH UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache \ - uv run --no-sync --python .venv/bin/python .venv/bin/python -m pytest \ - tests/test_baseline_artifacts.py tests/test_precompute.py -q -``` - -After the storage filename assertion correction, repeating just -`tests/test_precompute.py -q` with the same command prefix passed **38 tests in -0.88 seconds**. Ruff check and format-check pass for both source files, all four -artifact/native test/helper files, and the launcher (seven Python files total). -The executor test venv lacks the `pyright` module; the coordinator's established -full typecheck covers both changed source files and reports the same 145 -pre-existing executor diagnostics as `dcfe4fd`, with zero new diagnostics. - -The runtime-errors agent independently reviewed the artifact/precompute diff -against the installed wrapper's actual ensure/load/run/cache code and found no -additional actionable safety defect. This is a bounded peer review, not a durable -Fable agreement. No artifact upload, deployment, publication, or population -calculation was performed. Remote cached artifacts were not changed; these fixes -repair a loaded incomplete artifact locally by recomputing. - -## Source binding - -These final source hashes bind the implementation. GREEN executed identical -Python code before an outcome-comment-only edit. Root's final qualification and -source inventory bind the integrated branch head and all test files. - -| Source | SHA256 | -| --- | --- | -| `baseline_artifacts.py` | `2ec0d48bd97bad7a663deaf49b7a12c5b887d6b29f571e2cfe90d3c4247ad9c1` | -| `precompute.py` | `2c2a9bb6c824f4cce603f5728ea4cc58a693f8f354beff10b4cb370d41914e45` | diff --git a/rollout/worker-fable-fixes/changed-python-files.txt b/rollout/worker-fable-fixes/changed-python-files.txt deleted file mode 100644 index 60011faeb..000000000 --- a/rollout/worker-fable-fixes/changed-python-files.txt +++ /dev/null @@ -1,16 +0,0 @@ -libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py -libs/policyengine-simulation-contract/tests/test_spm_selection.py -projects/policyengine-simulation-entry/tests/test_openapi.py -projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py -projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py -projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py -projects/policyengine-simulation-executor/tests/native_spm_support.py -projects/policyengine-simulation-executor/tests/test_baseline_artifacts_spm_native.py -projects/policyengine-simulation-executor/tests/test_canonical_spm.py -projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py -projects/policyengine-simulation-executor/tests/test_precompute.py -projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py -projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py -projects/policyengine-simulation-gateway/tests/test_spm_routes.py -rollout/worker-fable-fixes/run_artifact_native.py -rollout/worker-fable-fixes/run_installed_qualification.py diff --git a/rollout/worker-fable-fixes/handoff-preservation.json b/rollout/worker-fable-fixes/handoff-preservation.json deleted file mode 100644 index 77cf087d7..000000000 --- a/rollout/worker-fable-fixes/handoff-preservation.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "PROGRESS.md": "505f06cd28aa0624c21b1292bdb20f426dd71f15dee1f7db6ccf9ebc9ee98833", - "WORKER-CANONICAL-SPM-HANDOFF.md": "6d21bc5dd4b094d0e2b48d175fc021e13152584c7a964f471bfebb62401299ef" -} diff --git a/rollout/worker-fable-fixes/native-wheel-summary.json b/rollout/worker-fable-fixes/native-wheel-summary.json deleted file mode 100644 index 9d572e4f9..000000000 --- a/rollout/worker-fable-fixes/native-wheel-summary.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "data_certification": "not_certified", - "development_fixture": { - "bootstrap_path": "/Users/maxghenis/spm-rebuild-20260908/worktrees/policyengine-wrapper-production/tests/fixtures/spm_development.py", - "bootstrap_sha256": "f762892a374c3950af54834f0fb234708addc59ad4af90aab99866eb37e27a5e", - "compatibility_basis": "unverified_development_fixture", - "manifest_path": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/development-manifest.json", - "manifest_sha256": "c78faac072f85109ac50d36a9e538a4dae56af4cdc98d2281603a3f4849a591f" - }, - "external_package_publication": "not_attested", - "population_acceptance": "not_attested", - "pytest_exit_code": 0, - "pytest_outcomes": { - "failed": [], - "passed": [ - "tests/test_canonical_spm_native.py::test_worker_baseline_reform_national_local_cache_and_receipts", - "tests/test_canonical_spm_native.py::test_native_state_only_worker_requires_geography_and_explicit_national_works", - "tests/test_canonical_spm_native.py::test_actual_provider_unknown_metro_is_a_typed_input_error", - "::test_installed_provider_missing_year[national-2021]", - "::test_installed_provider_missing_year[national-2036]", - "::test_installed_provider_missing_year[metro-2021]", - "::test_installed_provider_missing_year[metro-2036]", - "::test_installed_provider_missing_year[county-2021]", - "::test_installed_provider_missing_year[county-2036]", - "::test_installed_provider_missing_year_metadata[2021]", - "::test_installed_provider_missing_year_metadata[2036]", - "::test_installed_country_tax_only_then_measurement[national-2021]", - "::test_installed_country_tax_only_then_measurement[national-2036]", - "::test_installed_country_tax_only_then_measurement[metro-2021]", - "::test_installed_country_tax_only_then_measurement[metro-2036]", - "tests/test_baseline_artifacts_spm_native.py::test_incomplete_or_corrupt_spm_disk_artifact_recomputes[tax_only_columns-incomplete]", - "tests/test_baseline_artifacts_spm_native.py::test_incomplete_or_corrupt_spm_disk_artifact_recomputes[empty_years-incomplete]", - "tests/test_baseline_artifacts_spm_native.py::test_incomplete_or_corrupt_spm_disk_artifact_recomputes[missing_receipt-miss]", - "tests/test_baseline_artifacts_spm_native.py::test_incomplete_or_corrupt_spm_disk_artifact_recomputes[malformed_receipt-miss]", - "tests/test_baseline_artifacts_spm_native.py::test_incomplete_or_corrupt_spm_disk_artifact_recomputes[different_selection-miss]", - "tests/test_baseline_artifacts_spm_native.py::test_incomplete_or_corrupt_spm_disk_artifact_recomputes[corrupt_hdf-miss]", - "tests/test_baseline_artifacts_spm_native.py::test_invalid_spm_cache_entry_recomputes_requested_selection[missing_receipt]", - "tests/test_baseline_artifacts_spm_native.py::test_invalid_spm_cache_entry_recomputes_requested_selection[different_selection]", - "tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period0-2021]", - "tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period1-2036]", - "tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period2-2040]", - "tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period3-2021]", - "tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period4-2036]", - "tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period5-2036]", - "tests/test_spm_runtime_native.py::test_worker_scenario_boundary_matches_real_calculator_contract[period0]", - "tests/test_spm_runtime_native.py::test_worker_scenario_boundary_matches_real_calculator_contract[period1]", - "tests/test_spm_runtime_native.py::test_valid_worker_prevalidation_does_not_evaluate_amounts", - "tests/test_spm_runtime_native.py::test_other_invalid_settings_keep_the_settings_code[selection0]", - "tests/test_spm_runtime_native.py::test_other_invalid_settings_keep_the_settings_code[selection1]" - ], - "skipped": [] - }, - "python": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/bin/python", - "qualification_harness_sha256": "aae743b84f0223c82ff15a115d766a49ceddea85605fe8bc96fd94903d5eb21f", - "raw_receipt_sha256": "7fd3c88b6b8b371f5b163e8ca731d8d0de72681cf97d5c6197742fe8078fa6e1", - "runtime_capability": { - "contract_version": "canonical-spm-v1", - "defaults": { - "as_of": null, - "county_vintage": "2020", - "forecast_content_sha256": "3d86d5c4c0423480e6b69b75d222ffa4a7a2639e4094df5ba2504af01be17173", - "geography_id": null, - "geography_kind": "county", - "scenario": "ce_trend" - } - }, - "scientific_package_files_verified": 17613, - "scope": "four_person_native_worker_installed_candidate_wheel_qualification", - "selected_households": 1, - "selected_people": 4, - "source_h5_sha256_after": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", - "source_h5_sha256_before": "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84", - "test_elapsed_seconds": 288.39, - "wheels": { - "policyengine": { - "import_origin_after": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/policyengine/__init__.py", - "import_origin_before": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/policyengine/__init__.py", - "package_file_count": 104, - "path": "/Users/maxghenis/spm-rebuild-20260908/rollout/postfix-qualification/wheels/policyengine-5.3.0-py3-none-any.whl", - "sha256": "8c640d967575dddad70840bcbe938cea251c56958eded647c83eb9c5902735f1", - "version": "5.3.0" - }, - "policyengine_core": { - "import_origin_after": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/policyengine_core/__init__.py", - "import_origin_before": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/policyengine_core/__init__.py", - "package_file_count": 193, - "path": "/Users/maxghenis/spm-rebuild-20260908/rollout/postfix-qualification/wheels/policyengine_core-3.30.1-py3-none-any.whl", - "sha256": "2dbcf5f590a0199a7b7c77fcbbda2ff6bc289f6c6169ca0af3beace35d8b3e63", - "version": "3.30.1" - }, - "policyengine_us": { - "import_origin_after": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/policyengine_us/__init__.py", - "import_origin_before": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/policyengine_us/__init__.py", - "package_file_count": 17254, - "path": "/Users/maxghenis/spm-rebuild-20260908/rollout/postfix-qualification/wheels/policyengine_us-1.824.7-py3-none-any.whl", - "sha256": "7644819916a4f8ca2aa37a4a9d992fa834bfa66ab2d441326a9686baa5c1a688", - "version": "1.824.7" - }, - "spm_calculator": { - "import_origin_after": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/spm_calculator/__init__.py", - "import_origin_before": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/spm_calculator/__init__.py", - "package_file_count": 62, - "path": "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/wheels/spm_calculator-1.0.0-py3-none-any.whl", - "sha256": "c49c41da5fd482e563eaea956e205a3ba6841cadd4dd32bef4c0c3dbce17ffba", - "version": "1.0.0" - } - }, - "worker_head": "c431d138bdbad9be71c12b6994851879cf5d05aa", - "worker_source_files_verified": 190, - "worker_source_overlay_only": true -} diff --git a/rollout/worker-fable-fixes/routing-evidence.md b/rollout/worker-fable-fixes/routing-evidence.md deleted file mode 100644 index 464d0a160..000000000 --- a/rollout/worker-fable-fixes/routing-evidence.md +++ /dev/null @@ -1,60 +0,0 @@ -# Gateway routing and selection evidence - -State: Fable finding 1 and the related shared-app, malformed-capability, and partial-selection notes are fixed. Regression tests were committed as `1d6a6ed`; production fixes and expanded rejection coverage were committed as `0bf61df`. The latter includes the runtime agent's narrow `SPM_SCENARIO_UNAVAILABLE` shared error-code addition. - -Done: - -- Read the exact reviewer output at `/Users/maxghenis/chief-of-staff/state/subfleet/gates/20260909-170551-pr-abdf4629/rounds/001-4542bf03eca1/peer-output.md`, repository testing/PR skills, and the actual `build_legacy_seed_routing_state` producer. -- Route resolution now carries internal provenance for the original country dictionary and `generation="legacy-seed"`. A missing wrapper version permits an ordinary no-SPM request only with this provenance and a numeric US model version at or before the already recognized historical `1.764.6` model. Missing provenance, unknown or later model metadata, future wrapper versions, and every explicit SPM selection fail closed. The classification is not a public request field. -- Annual and budget-window submission tests cover real supported registry shapes, explicit and latest country-version routing, missing wrapper versions, rejection without spawning, and unknown routes. Legacy-dictionary tests use `legacy-app`, for which wrapper-version inference is impossible. -- A shared app resolves a country-model request only when one bundle's country metadata matches and every sibling has enough country metadata to exclude it. Multiple matches or missing sibling model metadata require an explicit `policyengine_version`; latest aliases are excluded from exact-version candidates. -- Malformed stored capabilities are omitted from `/versions` with a warning. Submission against them returns typed `SPM_CONFIGURATION_UNAVAILABLE`, including otherwise historical bundles; no valid-looking capability or worker submission is produced. Tests cover missing pinned hash, unknown contract, extra fields, and incomplete metro defaults. -- Partial geography options are validated together after bundle defaults resolve. A geography-ID-only request can inherit metro defaults, an explicit metro can inherit its existing area, and incompatible county/national defaults still fail. Changing from metro to national clears the area. - -Regression-first commands (executed before production edits): - -```sh -# cwd: projects/policyengine-simulation-gateway -UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_PROJECT_ENVIRONMENT=../policyengine-simulation-executor/.venv PYTHONPATH=src:../../libs/policyengine-simulation-contract/src:../../libs/policyengine-simulation-observability/src:../../libs/policyengine-fastapi/src uv run --no-sync python -m pytest tests/test_spm_routes.py -q -# 13 failed, 16 passed: historical submissions, sibling capability selection, -# ambiguous shared-app selection, and malformed capability listing reproduced. - -# cwd: libs/policyengine-simulation-contract -UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_PROJECT_ENVIRONMENT=../../projects/policyengine-simulation-executor/.venv PYTHONPATH=src:../policyengine-simulation-observability/src uv run --no-sync python -m pytest tests/test_spm_selection.py -q -# 2 failed, 3 passed: geography-ID-only and metro-only inheritance reproduced. -``` - -Verification commands after fixes: - -```sh -# cwd: projects/policyengine-simulation-gateway -UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_PROJECT_ENVIRONMENT=../policyengine-simulation-executor/.venv PYTHONPATH=src:../../libs/policyengine-simulation-contract/src:../../libs/policyengine-simulation-observability/src:../../libs/policyengine-fastapi/src uv run --no-sync python -m pytest tests/test_endpoints.py tests/test_spm_routes.py -q -# 88 passed in 0.53 seconds. - -UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_PROJECT_ENVIRONMENT=../policyengine-simulation-executor/.venv PYTHONPATH=src:../../libs/policyengine-simulation-contract/src:../../libs/policyengine-simulation-observability/src:../../libs/policyengine-fastapi/src uv run --no-sync python -m pytest tests/ -q -# Initial full run: 138 passed, 1 deselected, 1 failed in 0.57 seconds. -# The checked-in OpenAPI golden omitted all pre-existing canonical SPM schemas. -# Root is updating that pre-existing stale golden and rerunning the full suite. - -# cwd: libs/policyengine-simulation-contract -UV_CACHE_DIR=/tmp/worker-fable-uv-cache UV_PROJECT_ENVIRONMENT=../../projects/policyengine-simulation-executor/.venv PYTHONPATH=src:../policyengine-simulation-observability/src uv run --no-sync python -m pytest tests/ -q -# 73 passed in 0.04 seconds. - -# cwd: repository root -ruff format --check projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py projects/policyengine-simulation-gateway/tests/test_spm_routes.py libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py libs/policyengine-simulation-contract/tests/test_spm_selection.py -ruff check projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py projects/policyengine-simulation-gateway/tests/test_spm_routes.py libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py libs/policyengine-simulation-contract/tests/test_spm_selection.py -git diff --check -# All pass. -``` - -The installed worker environment is Python 3.13.9; source overlays load the owned gateway/contract code. All gateway network/Modal seams are mocked. Native calculator verification belongs to the integrated qualification owned by root and the runtime agent. - -Schema equivalence was checked by loading the original contract source with `git show dcfe4fd89d97145882cb147a9aa2ffae2cd10ebf:libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py` into a separate Python module and comparing `model_json_schema()` for `SPMSelection`, `SPMCapability`, `SPMErrorDetail`, `SPMProvenance`, and `SPMComparisonProvenance`. All five are byte-structure equivalent; these routing/validation changes do not introduce a new public schema. - -Residual rollout constraint: routine unmeasured future bundles remain unavailable for default US requests. In particular, a newer model without certified `measurements.spm` does not inherit the historical allowance merely because its wrapper is 5.2.0 or 5.3.0. Incomplete route provenance likewise remains unavailable until its metadata is resolved explicitly. No live routes or deployment state were modified. - -Next: root completes integrated checks, authenticated native qualification, final source binding/report, and the existing canonical draft PR update. This subtask performed no push, publication, population job, deployment, merge, or changes to untracked root handoffs. - -Adversarial follow-up: regression commit `9d16eff` reproduced eight additional failures (37 passing cases) with `tests/test_spm_routes.py -q --tb=no` under the same gateway command environment. A seed could omit the wrapper route that the actual seed producer would infer from a future `policyengine-simulation-py99-0-0` app; an absent/unsupported seed schema was also accepted. Blank sibling model metadata incorrectly excluded that sibling. Fix `2ee006d` requires the supported seed schema, infers the app's wrapper version only when no exact registry candidate exists, and treats blank sibling models as unclassified. Exact registry versions still take precedence over app-name inference. - -After the follow-up, the gateway command with `tests/test_spm_routes.py tests/test_endpoints.py -q` passed **98 tests in 0.71 seconds**. Ruff format/check and `git diff --check` passed. Additional direct gateway-helper probes confirmed that a future bundle with each of `spm=None`, `False`, `0`, `[]`, a string, or `{}` raises `SPM_CONFIGURATION_UNAVAILABLE`; none produces an accepted submission. Native runtime changes were independently reviewed against the installed `PolicyEngineSPMProvider`: no actionable typed year/scenario or laziness defect was found, and prevalidation receipts remain private to the discarded provider. diff --git a/rollout/worker-fable-fixes/run_artifact_native.py b/rollout/worker-fable-fixes/run_artifact_native.py deleted file mode 100644 index 46cb72bd7..000000000 --- a/rollout/worker-fable-fixes/run_artifact_native.py +++ /dev/null @@ -1,15 +0,0 @@ -from pathlib import Path -from importlib import import_module -import sys - -original = Path( - "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/run_installed_native_qualification.py" -) -namespace = {"__file__": str(original), "__name__": "artifact_regression"} -exec( - compile(original.read_text().split("import pytest\n", 1)[0], str(original), "exec"), - namespace, -) -pytest = import_module("pytest") - -raise SystemExit(pytest.main(sys.argv[1:])) diff --git a/rollout/worker-fable-fixes/run_installed_qualification.py b/rollout/worker-fable-fixes/run_installed_qualification.py deleted file mode 100644 index 80d7851f3..000000000 --- a/rollout/worker-fable-fixes/run_installed_qualification.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Four-person worker and lazy-year smoke against new exact installed candidate wheels.""" - -# Imports below authentication must run after the explicit development bootstrap. -# ruff: noqa: E402 - -import hashlib -import importlib -import importlib.metadata as metadata -import importlib.util -import json -import os -from pathlib import Path -import sys -import subprocess -import zipfile - -OUT = Path(__file__).resolve().parent -QUALIFICATION = Path( - "/Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification" -) -ROLLOUT = QUALIFICATION.parent -WORKTREES = ROLLOUT.parent / "worktrees" -WORKER = WORKTREES / "policyengine-sim-api-canonical" -SOURCE = ROLLOUT / "producer-final-candidate-20260909/artifacts/populace_us_2024.h5" -CONTENT = "3d86d5c4c0423480e6b69b75d222ffa4a7a2639e4094df5ba2504af01be17173" -WHEELS = { - "policyengine": ( - ROLLOUT / "postfix-qualification/wheels/policyengine-5.3.0-py3-none-any.whl", - "8c640d967575dddad70840bcbe938cea251c56958eded647c83eb9c5902735f1", - ), - "policyengine_us": ( - ROLLOUT - / "postfix-qualification/wheels/policyengine_us-1.824.7-py3-none-any.whl", - "7644819916a4f8ca2aa37a4a9d992fa834bfa66ab2d441326a9686baa5c1a688", - ), - "spm_calculator": ( - QUALIFICATION / "wheels/spm_calculator-1.0.0-py3-none-any.whl", - "c49c41da5fd482e563eaea956e205a3ba6841cadd4dd32bef4c0c3dbce17ffba", - ), - "policyengine_core": ( - ROLLOUT - / "postfix-qualification/wheels/policyengine_core-3.30.1-py3-none-any.whl", - "2dbcf5f590a0199a7b7c77fcbbda2ff6bc289f6c6169ca0af3beace35d8b3e63", - ), -} - - -def sha256(path): - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -sys.dont_write_bytecode = True -os.environ["PYTHONDONTWRITEBYTECODE"] = "1" -os.environ["POLICYENGINE_SKIP_COUNTRY_IMPORTS"] = "1" -os.environ["SPM_NATIVE_SMOKE_SOURCE"] = str(SOURCE) -assert not os.environ.get("PYTHONPATH"), "No inherited source overlays allowed" -assert Path("/tmp").resolve() not in {Path(path).resolve() for path in sys.path}, ( - "Temporary-directory modules must not shadow installed dependencies" -) -for relative in ( - "projects/policyengine-simulation-executor/src", - "projects/policyengine-simulation-entry/src", - "projects/policyengine-simulation-gateway/src", - "libs/policyengine-simulation-contract/src", - "libs/policyengine-simulation-observability/src", - "libs/policyengine-fastapi/src", -): - sys.path.insert(0, str(WORKER / relative)) - -receipt = { - "worker_head": subprocess.check_output( - ["git", "rev-parse", "HEAD"], cwd=WORKER, text=True - ).strip(), - "scope": "four_person_native_worker_installed_candidate_wheel_qualification", - "data_certification": "not_certified", - "external_package_publication": "not_attested", - "population_acceptance": "not_attested", - "python": sys.executable, - "wheels": {}, - "worker_source_overlay_only": True, -} -for module_name, (wheel, expected_hash) in WHEELS.items(): - assert sha256(wheel) == expected_hash, wheel - dist = metadata.distribution(module_name) - members = {} - with zipfile.ZipFile(wheel) as archive: - for member in archive.namelist(): - if not member.startswith(module_name + "/") or member.endswith("/"): - continue - installed = Path(dist.locate_file(member)).resolve() - assert installed.is_relative_to(QUALIFICATION / "venv"), installed - expected = archive.read(member) - assert installed.read_bytes() == expected, installed - members[member] = hashlib.sha256(expected).hexdigest() - assert members, module_name - spec = importlib.util.find_spec(module_name) - assert Path(spec.origin).resolve().is_relative_to(QUALIFICATION / "venv"), ( - spec.origin - ) - receipt["wheels"][module_name] = { - "path": str(wheel), - "sha256": expected_hash, - "version": dist.version, - "package_file_count": len(members), - "installed_package_files": members, - "import_origin_before": spec.origin, - } - -helper = WORKTREES / "policyengine-wrapper-production/tests/fixtures/spm_development.py" -spec = importlib.util.spec_from_file_location( - "worker_spm_development_bootstrap", helper -) -bootstrap = importlib.util.module_from_spec(spec) -spec.loader.exec_module(bootstrap) -manifest = QUALIFICATION / "development-manifest.json" -bootstrap.activate_spm_development_manifest(manifest) -receipt["development_fixture"] = { - "manifest_path": str(manifest), - "manifest_sha256": sha256(manifest), - "bootstrap_path": str(helper), - "bootstrap_sha256": sha256(helper), - "compatibility_basis": "unverified_development_fixture", -} -for name in WHEELS: - module = importlib.import_module(name) - origin = str(Path(module.__file__).resolve()) - assert Path(origin).is_relative_to(QUALIFICATION / "venv"), origin - receipt["wheels"][name]["import_origin_after"] = origin - -from spm_calculator import load_forecast -from policyengine_simulation_executor.spm import runtime_spm_capability - -assert load_forecast(expected_sha256=CONTENT).content_sha256 == CONTENT -capability = runtime_spm_capability() -assert capability.defaults.forecast_content_sha256 == CONTENT -receipt["runtime_capability"] = capability.model_dump(mode="json") -receipt["source_h5_sha256_before"] = sha256(SOURCE) -assert ( - receipt["source_h5_sha256_before"] - == "6496cc4393d4d3c6574f76eca231de5898c803b9067645591fd5c4d3e65aee84" -) - -import pandas as pd - -with pd.HDFStore(SOURCE, "r") as source: - household = source.select("household", start=0, stop=1) - household_id = int(household.household_id.iloc[0]) - people = source.select("person", where=f"person_household_id == {household_id}") - receipt["selected_households"] = len(household) - receipt["selected_people"] = len(people) - assert len(household) == 1 and len(people) == 4 - -import pytest - - -class OutcomeRecorder: - def __init__(self): - self.passed = [] - self.failed = [] - self.skipped = [] - - def pytest_runtest_logreport(self, report): - if report.skipped: - self.skipped.append(report.nodeid) - elif report.failed: - self.failed.append(report.nodeid) - elif report.when == "call" and report.passed: - self.passed.append(report.nodeid) - - -outcomes = OutcomeRecorder() -exit_code = pytest.main( - [ - str( - WORKER - / "projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py" - ), - str(QUALIFICATION / "test_installed_year_boundary.py"), - str( - WORKER - / "projects/policyengine-simulation-executor/tests/test_baseline_artifacts_spm_native.py" - ), - str( - WORKER - / "projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py" - ), - "-q", - "--basetemp", - str(OUT / "pytest-native"), - "-p", - "no:cacheprovider", - ], - plugins=[outcomes], -) -receipt["pytest_outcomes"] = vars(outcomes) -assert not outcomes.skipped, outcomes.skipped -assert len(outcomes.passed) >= 15 and not outcomes.failed, vars(outcomes) -receipt["pytest_exit_code"] = int(exit_code) -source_files = subprocess.check_output( - ["git", "ls-files", "projects", "libs"], cwd=WORKER, text=True -).splitlines() -receipt["worker_source_files"] = { - name: sha256(WORKER / name) - for name in source_files - if name.endswith(".py") - and ("/src/" in name or "/tests/" in name or "/fixtures/" in name) -} -receipt["qualification_harness_sha256"] = sha256(Path(__file__)) -receipt["source_h5_sha256_after"] = sha256(SOURCE) -assert receipt["source_h5_sha256_after"] == receipt["source_h5_sha256_before"] -(OUT / "native-wheel-receipt.json").write_text( - json.dumps(receipt, indent=2, sort_keys=True) + "\n" -) -raise SystemExit(exit_code) diff --git a/rollout/worker-fable-fixes/runtime-evidence.md b/rollout/worker-fable-fixes/runtime-evidence.md deleted file mode 100644 index b11963b04..000000000 --- a/rollout/worker-fable-fixes/runtime-evidence.md +++ /dev/null @@ -1,33 +0,0 @@ -# Runtime year and scenario error evidence - -State: Fable finding 2 fixed and verified against the authenticated installed calculator, country, core, and wrapper. Root-owned integrated qualification and final source binding remain separate. - -Worker prevalidation now uses the installed `PolicyEngineSPMProvider.year_metadata` typed year contract. Its constructor still emits a plain unknown-scenario `ValueError`; the worker uses the same narrow scenario translation as the calculator's Frame and Axiom adapters. Other invalid settings retain `SPM_SETTINGS_INVALID`. The shared contract recognizes `SPM_SCENARIO_UNAVAILABLE` so transport does not redact that code. Scientific packages and their source were not changed. - -The new native tests compare actual provider/Axiom errors with annual and budget-window worker entrypoints before dataset/child setup. They cover 2021, 2036, the `year=2040` alias, unavailable window starts and a 2035–2036 window end, unknown scenarios, and invalid as-of/vintage settings. The valid-year case forbids amount evaluation and checks that a fresh real provider still has empty receipt years. Existing HTTP, polling, pickle, country-error, and optional-output tests now include the scenario code. The parent integrated run also reuses the prior 12 installed provider/country tests, including actual out-of-range tax-only calculations followed by typed threshold failure. - -## Clean regression and verification commands - -```sh -env -u PYTHONPATH PYTHONSAFEPATH=1 UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache uv run --no-project --python /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/bin/python /private/tmp/run_worker_runtime_regressions.py --original-runtime --basetemp /private/tmp/worker-runtime-clean-red > /private/tmp/worker-runtime-clean-red.txt 2>&1 - -env -u PYTHONPATH PYTHONSAFEPATH=1 UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache uv run --no-project --python /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/bin/python /private/tmp/run_worker_runtime_regressions.py projects/policyengine-simulation-executor/tests/test_canonical_spm.py --basetemp /private/tmp/worker-runtime-clean-green > /private/tmp/worker-runtime-clean-green.txt 2>&1 - -UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache uv run --project projects/policyengine-simulation-executor --no-sync ruff format projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py projects/policyengine-simulation-executor/tests/test_canonical_spm.py projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py - -UV_CACHE_DIR=/private/tmp/worker-fable-uv-cache uv run --project projects/policyengine-simulation-executor --no-sync ruff check projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py projects/policyengine-simulation-executor/tests/test_canonical_spm.py projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py -``` - -The temporary launcher executes only the original qualification script's authenticated wheel/bootstrap setup (before `import pytest`) and invokes committed `tests/test_spm_runtime_native.py` with pytest `-q -p no:cacheprovider`. It asserts `/tmp` and `/private/tmp` are absent from `sys.path`. With `--original-runtime`, it overlays only the archived `dcfe4fd` executor `spm.py` from `/private/tmp/worker-fable-base`; it retains current shared transport recognition so the scenario regression reaches the original worker mapping itself. No checkout, installed-package edit, or fabricated metadata is involved. - -- Clean RED: **8 failed, 3 passed, zero skipped, 2.12 seconds**. All six year and both scenario boundary cases fail on the original worker with `SPM_SETTINGS_INVALID` in place of the calculator code. Full output: `runtime-regression-red.txt`. -- Clean GREEN: **56 passed, zero skipped, 0.28 seconds** across the 11 native cases and 45 canonical worker/transport cases. Full output: `runtime-regression-green.txt`. -- Ruff formatting and lint: pass. The two pytest plugin-rewrite warnings reflect explicit bootstrap before pytest, as in the prior qualification. - -## Superseded attempts and limits - -The first launcher invocation safely stopped at the inherited-PYTHONPATH guard. A service-local `uv run --no-sync pytest tests/test_canonical_spm.py -q` attempt selected an ambient Python 3.14 tool and could not collect the missing contract package; it is not validation evidence. No environment was changed. - -Early temporary-launcher RED/GREEN outputs lacked Python safe-path isolation: unrelated `/private/tmp/h2.py` shadowed optional HTTP/2 and imported an unrelated demonstration. Those outputs are superseded and are not qualification evidence. The unrelated files were not edited. The committed RED output was replaced by the clean isolated run above; both clean runs have no unrelated demo output. The final root-owned harness lives inside the repository and authenticates installed sources again. - -Remaining coupling: the installed PolicyEngine adapter lacks a public typed scenario-selection validator, so the narrow scenario translation follows the real calculator adapters' current contract. The tests execute the public Axiom adapter to detect drift. No amount formulas, installed receipt metadata, publication, population execution, deployment, or merge changed. diff --git a/rollout/worker-fable-fixes/runtime-regression-green.txt b/rollout/worker-fable-fixes/runtime-regression-green.txt deleted file mode 100644 index 87774a0f2..000000000 --- a/rollout/worker-fable-fixes/runtime-regression-green.txt +++ /dev/null @@ -1,14 +0,0 @@ -WARN `--no-project` was provided, but no project was found -warning: `--frozen` has no effect when used alongside `--no-project` -........................................................ [100%] -=============================== warnings summary =============================== -../../rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290 - /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290: PytestAssertRewriteWarning: Module already imported so cannot be rewritten; logfire - self._mark_plugins_for_rewrite(hook, disable_autoload) - -../../rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290 - /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290: PytestAssertRewriteWarning: Module already imported so cannot be rewritten; anyio - self._mark_plugins_for_rewrite(hook, disable_autoload) - --- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html -56 passed, 2 warnings in 0.28s diff --git a/rollout/worker-fable-fixes/runtime-regression-red.txt b/rollout/worker-fable-fixes/runtime-regression-red.txt deleted file mode 100644 index 20ec7a72d..000000000 --- a/rollout/worker-fable-fixes/runtime-regression-red.txt +++ /dev/null @@ -1,360 +0,0 @@ -WARN `--no-project` was provided, but no project was found -warning: `--frozen` has no effect when used alongside `--no-project` -FFFFFFFF... [100%] -=================================== FAILURES =================================== -________ test_worker_year_boundary_matches_real_provider[period0-2021] _________ - -forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) -period = {'time_period': '2021'}, unavailable_year = 2021 -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x138a20c30> - - @pytest.mark.parametrize( - "period,unavailable_year", - [ - ({"time_period": "2021"}, 2021), - ({"time_period": "2036"}, 2036), - ({"year": "2040"}, 2040), - ({"start_year": "2021", "window_size": 1}, 2021), - ({"start_year": "2036", "window_size": 1}, 2036), - ({"start_year": "2035", "window_size": 2}, 2036), - ], - ) - def test_worker_year_boundary_matches_real_provider( - forecast, period, unavailable_year, monkeypatch - ): - from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider - from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail - - provider = PolicyEngineSPMProvider(forecast, geography_kind="national") - with pytest.raises(ValueError) as provider_error: - provider.year_metadata(unavailable_year) - expected = spm_error_detail(provider_error.value) - assert expected.code == "SPM_YEAR_UNAVAILABLE" - assert provider.provenance()["years"] == {} - params = {"country": "us", "spm": {"geography_kind": "national"}, **period} - entrypoint, boundary = _worker_entrypoint(params, monkeypatch) - with pytest.raises(SPMInputError) as error: - entrypoint(params) -> assert error.value.to_dict() == expected.model_dump() -E AssertionError: assert {'code': 'SPM...try for 2021'} == {'code': 'SPM...try for 2021'} -E -E Omitting 1 identical items, use -vv to show -E Differing items: -E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_YEAR_UNAVAILABLE'} -E Use -v to get more diff - -projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:67: AssertionError -________ test_worker_year_boundary_matches_real_provider[period1-2036] _________ - -forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) -period = {'time_period': '2036'}, unavailable_year = 2036 -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x138a20e90> - - @pytest.mark.parametrize( - "period,unavailable_year", - [ - ({"time_period": "2021"}, 2021), - ({"time_period": "2036"}, 2036), - ({"year": "2040"}, 2040), - ({"start_year": "2021", "window_size": 1}, 2021), - ({"start_year": "2036", "window_size": 1}, 2036), - ({"start_year": "2035", "window_size": 2}, 2036), - ], - ) - def test_worker_year_boundary_matches_real_provider( - forecast, period, unavailable_year, monkeypatch - ): - from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider - from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail - - provider = PolicyEngineSPMProvider(forecast, geography_kind="national") - with pytest.raises(ValueError) as provider_error: - provider.year_metadata(unavailable_year) - expected = spm_error_detail(provider_error.value) - assert expected.code == "SPM_YEAR_UNAVAILABLE" - assert provider.provenance()["years"] == {} - params = {"country": "us", "spm": {"geography_kind": "national"}, **period} - entrypoint, boundary = _worker_entrypoint(params, monkeypatch) - with pytest.raises(SPMInputError) as error: - entrypoint(params) -> assert error.value.to_dict() == expected.model_dump() -E AssertionError: assert {'code': 'SPM...try for 2036'} == {'code': 'SPM...try for 2036'} -E -E Omitting 1 identical items, use -vv to show -E Differing items: -E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_YEAR_UNAVAILABLE'} -E Use -v to get more diff - -projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:67: AssertionError -________ test_worker_year_boundary_matches_real_provider[period2-2040] _________ - -forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) -period = {'year': '2040'}, unavailable_year = 2040 -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13bd58290> - - @pytest.mark.parametrize( - "period,unavailable_year", - [ - ({"time_period": "2021"}, 2021), - ({"time_period": "2036"}, 2036), - ({"year": "2040"}, 2040), - ({"start_year": "2021", "window_size": 1}, 2021), - ({"start_year": "2036", "window_size": 1}, 2036), - ({"start_year": "2035", "window_size": 2}, 2036), - ], - ) - def test_worker_year_boundary_matches_real_provider( - forecast, period, unavailable_year, monkeypatch - ): - from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider - from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail - - provider = PolicyEngineSPMProvider(forecast, geography_kind="national") - with pytest.raises(ValueError) as provider_error: - provider.year_metadata(unavailable_year) - expected = spm_error_detail(provider_error.value) - assert expected.code == "SPM_YEAR_UNAVAILABLE" - assert provider.provenance()["years"] == {} - params = {"country": "us", "spm": {"geography_kind": "national"}, **period} - entrypoint, boundary = _worker_entrypoint(params, monkeypatch) - with pytest.raises(SPMInputError) as error: - entrypoint(params) -> assert error.value.to_dict() == expected.model_dump() -E AssertionError: assert {'code': 'SPM...try for 2040'} == {'code': 'SPM...try for 2040'} -E -E Omitting 1 identical items, use -vv to show -E Differing items: -E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_YEAR_UNAVAILABLE'} -E Use -v to get more diff - -projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:67: AssertionError -________ test_worker_year_boundary_matches_real_provider[period3-2021] _________ - -forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) -period = {'start_year': '2021', 'window_size': 1}, unavailable_year = 2021 -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13bdae7a0> - - @pytest.mark.parametrize( - "period,unavailable_year", - [ - ({"time_period": "2021"}, 2021), - ({"time_period": "2036"}, 2036), - ({"year": "2040"}, 2040), - ({"start_year": "2021", "window_size": 1}, 2021), - ({"start_year": "2036", "window_size": 1}, 2036), - ({"start_year": "2035", "window_size": 2}, 2036), - ], - ) - def test_worker_year_boundary_matches_real_provider( - forecast, period, unavailable_year, monkeypatch - ): - from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider - from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail - - provider = PolicyEngineSPMProvider(forecast, geography_kind="national") - with pytest.raises(ValueError) as provider_error: - provider.year_metadata(unavailable_year) - expected = spm_error_detail(provider_error.value) - assert expected.code == "SPM_YEAR_UNAVAILABLE" - assert provider.provenance()["years"] == {} - params = {"country": "us", "spm": {"geography_kind": "national"}, **period} - entrypoint, boundary = _worker_entrypoint(params, monkeypatch) - with pytest.raises(SPMInputError) as error: - entrypoint(params) -> assert error.value.to_dict() == expected.model_dump() -E AssertionError: assert {'code': 'SPM...try for 2021'} == {'code': 'SPM...try for 2021'} -E -E Omitting 1 identical items, use -vv to show -E Differing items: -E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_YEAR_UNAVAILABLE'} -E Use -v to get more diff - -projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:67: AssertionError -________ test_worker_year_boundary_matches_real_provider[period4-2036] _________ - -forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) -period = {'start_year': '2036', 'window_size': 1}, unavailable_year = 2036 -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13bdaf9b0> - - @pytest.mark.parametrize( - "period,unavailable_year", - [ - ({"time_period": "2021"}, 2021), - ({"time_period": "2036"}, 2036), - ({"year": "2040"}, 2040), - ({"start_year": "2021", "window_size": 1}, 2021), - ({"start_year": "2036", "window_size": 1}, 2036), - ({"start_year": "2035", "window_size": 2}, 2036), - ], - ) - def test_worker_year_boundary_matches_real_provider( - forecast, period, unavailable_year, monkeypatch - ): - from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider - from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail - - provider = PolicyEngineSPMProvider(forecast, geography_kind="national") - with pytest.raises(ValueError) as provider_error: - provider.year_metadata(unavailable_year) - expected = spm_error_detail(provider_error.value) - assert expected.code == "SPM_YEAR_UNAVAILABLE" - assert provider.provenance()["years"] == {} - params = {"country": "us", "spm": {"geography_kind": "national"}, **period} - entrypoint, boundary = _worker_entrypoint(params, monkeypatch) - with pytest.raises(SPMInputError) as error: - entrypoint(params) -> assert error.value.to_dict() == expected.model_dump() -E AssertionError: assert {'code': 'SPM...try for 2036'} == {'code': 'SPM...try for 2036'} -E -E Omitting 1 identical items, use -vv to show -E Differing items: -E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_YEAR_UNAVAILABLE'} -E Use -v to get more diff - -projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:67: AssertionError -________ test_worker_year_boundary_matches_real_provider[period5-2036] _________ - -forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) -period = {'start_year': '2035', 'window_size': 2}, unavailable_year = 2036 -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13bd96650> - - @pytest.mark.parametrize( - "period,unavailable_year", - [ - ({"time_period": "2021"}, 2021), - ({"time_period": "2036"}, 2036), - ({"year": "2040"}, 2040), - ({"start_year": "2021", "window_size": 1}, 2021), - ({"start_year": "2036", "window_size": 1}, 2036), - ({"start_year": "2035", "window_size": 2}, 2036), - ], - ) - def test_worker_year_boundary_matches_real_provider( - forecast, period, unavailable_year, monkeypatch - ): - from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider - from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail - - provider = PolicyEngineSPMProvider(forecast, geography_kind="national") - with pytest.raises(ValueError) as provider_error: - provider.year_metadata(unavailable_year) - expected = spm_error_detail(provider_error.value) - assert expected.code == "SPM_YEAR_UNAVAILABLE" - assert provider.provenance()["years"] == {} - params = {"country": "us", "spm": {"geography_kind": "national"}, **period} - entrypoint, boundary = _worker_entrypoint(params, monkeypatch) - with pytest.raises(SPMInputError) as error: - entrypoint(params) -> assert error.value.to_dict() == expected.model_dump() -E AssertionError: assert {'code': 'SPM...try for 2036'} == {'code': 'SPM...try for 2036'} -E -E Omitting 1 identical items, use -vv to show -E Differing items: -E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_YEAR_UNAVAILABLE'} -E Use -v to get more diff - -projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:67: AssertionError -___ test_worker_scenario_boundary_matches_real_calculator_contract[period0] ____ - -forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) -period = {'time_period': '2024'} -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13bd95d50> - - @pytest.mark.parametrize( - "period", - [ - {"time_period": "2024"}, - {"start_year": "2024", "window_size": 2}, - ], - ) - def test_worker_scenario_boundary_matches_real_calculator_contract( - forecast, period, monkeypatch - ): - from spm_calculator.axiom_adapter import export_build_spec - from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail - - scenario = "scenario-not-in-the-pinned-forecast" - with pytest.raises(ValueError) as adapter_error: - export_build_spec(forecast, years=[2024], scenario=scenario) - assert adapter_error.value.code == "SPM_SCENARIO_UNAVAILABLE" - expected = spm_error_detail(adapter_error.value) - assert expected is not None - params = { - "country": "us", - "spm": {"geography_kind": "national", "scenario": scenario}, - **period, - } - entrypoint, boundary = _worker_entrypoint(params, monkeypatch) - with pytest.raises(SPMInputError) as error: - entrypoint(params) -> assert error.value.to_dict() == expected.model_dump() -E AssertionError: assert {'code': 'SPM...ned-forecast'} == {'code': 'SPM...ned-forecast'} -E -E Omitting 1 identical items, use -vv to show -E Differing items: -E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_SCENARIO_UNAVAILABLE'} -E Use -v to get more diff - -projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:98: AssertionError -___ test_worker_scenario_boundary_matches_real_calculator_contract[period1] ____ - -forecast = SPMForecast(_json=b'{"areas":{"1002":{"area_type":"state_nonmetro","name":"Alabama Nonmetro"},"10180":{"area_type":"ms...002', '56035': '56002', '56037': '56002', '56039': '56002', '56041': '56002', '56043': '56002', '56045': '56002'})})})) -period = {'start_year': '2024', 'window_size': 2} -monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x13bd834d0> - - @pytest.mark.parametrize( - "period", - [ - {"time_period": "2024"}, - {"start_year": "2024", "window_size": 2}, - ], - ) - def test_worker_scenario_boundary_matches_real_calculator_contract( - forecast, period, monkeypatch - ): - from spm_calculator.axiom_adapter import export_build_spec - from policyengine_simulation_contract.spm import SPMInputError, spm_error_detail - - scenario = "scenario-not-in-the-pinned-forecast" - with pytest.raises(ValueError) as adapter_error: - export_build_spec(forecast, years=[2024], scenario=scenario) - assert adapter_error.value.code == "SPM_SCENARIO_UNAVAILABLE" - expected = spm_error_detail(adapter_error.value) - assert expected is not None - params = { - "country": "us", - "spm": {"geography_kind": "national", "scenario": scenario}, - **period, - } - entrypoint, boundary = _worker_entrypoint(params, monkeypatch) - with pytest.raises(SPMInputError) as error: - entrypoint(params) -> assert error.value.to_dict() == expected.model_dump() -E AssertionError: assert {'code': 'SPM...ned-forecast'} == {'code': 'SPM...ned-forecast'} -E -E Omitting 1 identical items, use -vv to show -E Differing items: -E {'code': 'SPM_SETTINGS_INVALID'} != {'code': 'SPM_SCENARIO_UNAVAILABLE'} -E Use -v to get more diff - -projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py:98: AssertionError -=============================== warnings summary =============================== -../../rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290 - /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290: PytestAssertRewriteWarning: Module already imported so cannot be rewritten; logfire - self._mark_plugins_for_rewrite(hook, disable_autoload) - -../../rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290 - /Users/maxghenis/spm-rebuild-20260908/rollout/year-boundary-qualification/venv/lib/python3.13/site-packages/_pytest/config/__init__.py:1290: PytestAssertRewriteWarning: Module already imported so cannot be rewritten; anyio - self._mark_plugins_for_rewrite(hook, disable_autoload) - --- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html -=========================== short test summary info ============================ -FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period0-2021] -FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period1-2036] -FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period2-2040] -FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period3-2021] -FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period4-2036] -FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_year_boundary_matches_real_provider[period5-2036] -FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_scenario_boundary_matches_real_calculator_contract[period0] -FAILED projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py::test_worker_scenario_boundary_matches_real_calculator_contract[period1] -8 failed, 3 passed, 2 warnings in 2.12s diff --git a/rollout/worker-fable-fixes/source-binding.json b/rollout/worker-fable-fixes/source-binding.json deleted file mode 100644 index 151568528..000000000 --- a/rollout/worker-fable-fixes/source-binding.json +++ /dev/null @@ -1,198 +0,0 @@ -{ - "harness_sha256": "aae743b84f0223c82ff15a115d766a49ceddea85605fe8bc96fd94903d5eb21f", - "implementation_head": "c431d138bdbad9be71c12b6994851879cf5d05aa", - "raw_receipt_sha256": "7fd3c88b6b8b371f5b163e8ca731d8d0de72681cf97d5c6197742fe8078fa6e1", - "tree": "d90dc393446c0ac42ce2f620ee6efb66f6b6d0a8", - "worker_source_files": { - "libs/policyengine-fastapi/fixtures/__init__.py": "8824a809592f0bcc83da4922b4c248dcc31728b9d7e2edb88c5a60c8546b9ff0", - "libs/policyengine-fastapi/fixtures/ping/__init__.py": "5d3f2b1aeb257772823091a01bbd0204abac815bcbb032f6ba2985b9c0359274", - "libs/policyengine-fastapi/fixtures/ping/shared.py": "dd0ef1399255a84d74380ede657f5bb40d0478e526709663ae730830c3a73cea", - "libs/policyengine-fastapi/src/policyengine_fastapi/__init__.py": "bd236453db17527bdae849e1594021d4774ca8d679eea8f433207225aa54c21f", - "libs/policyengine-fastapi/src/policyengine_fastapi/auth/__init__.py": "9471634f7819cc9d1ad75d9187b655a46fda57f1f258f5dcd40c5bce6c86e972", - "libs/policyengine-fastapi/src/policyengine_fastapi/auth/jwt_decoder.py": "fe00648b65e7407b919244c4feedc6b72698dc8b99afc9d74fdeac41e48ba0db", - "libs/policyengine-fastapi/src/policyengine_fastapi/database.py": "64d8d809c9df27d708db953afbd1c9e5646a6fade26db5a35345cbaeedd1d04e", - "libs/policyengine-fastapi/src/policyengine_fastapi/exit.py": "01cbbc6053ab2e1ce203a0151282c737b0b22b4a8eb79fc8e5ec88358021fef7", - "libs/policyengine-fastapi/src/policyengine_fastapi/health/__init__.py": "3f8edf1a242e95344b646f8fc387b451300c3dcac6817fa31fff4368677242b9", - "libs/policyengine-fastapi/src/policyengine_fastapi/observability/__init__.py": "0fd247358c47b47823ed39783f6af36747d2e2a9d9984954b7e291a6fb088edd", - "libs/policyengine-fastapi/src/policyengine_fastapi/observability/config.py": "fd166b2a8baf8b33eeed6df508c9e66901ab19742dc3fcc15997d534bd7da0ae", - "libs/policyengine-fastapi/src/policyengine_fastapi/observability/contracts.py": "a5038b0f09464405cdfea26d53aaee8f765bb49712a6feb118e2a0c87888039a", - "libs/policyengine-fastapi/src/policyengine_fastapi/observability/correlation.py": "b31882080c06236b64ad6be4b7ca7677f6486a0cffe961b7e119b4b6b69e2c8d", - "libs/policyengine-fastapi/src/policyengine_fastapi/observability/emitters.py": "bf9fe7b9ceba7d651a9069f2bce2b72c43393b8fbd33c0092b21ffa590968679", - "libs/policyengine-fastapi/src/policyengine_fastapi/observability/provider.py": "1cf85f524b1ed8aeb2bbfa1dfffdbb625b4e9591a4febe4b9600647e42ce2c02", - "libs/policyengine-fastapi/src/policyengine_fastapi/observability/stages.py": "7674cd880b9ee474323ac392b222d0e8e15e7b9305794f0ae302f281d26b609e", - "libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/__init__.py": "1ecb4aeaca6794c1c2ba756d940c36f4345a6eedf9dddd9b420e82ba706b55da", - "libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/console.py": "73abaa0d751f45ed34a691da59c7d053533dcec75c735beea3bc32a4e93f2438", - "libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/gcp.py": "ec20841d1c9ddf027a319397ab78c17a96e848d22b120e148b8c74f039e56ea8", - "libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/instrumentor.py": "bd63b2175997cf5bff4223a7afd85bedea92a1279903f547b59e539a1642d14b", - "libs/policyengine-fastapi/src/policyengine_fastapi/opentelemetry/middleware.py": "35d833171d9fb0b86cd5fd04f854e4a1fafc9d267e85b2d12d804bdf7af7a0a8", - "libs/policyengine-fastapi/src/policyengine_fastapi/ping/__init__.py": "b03e435d8c5c6c9d251669441c06e720f62dbdc9e75bd2f7c5fc44674e806644", - "libs/policyengine-fastapi/tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "libs/policyengine-fastapi/tests/conftest.py": "4bc41f67c99e97d38a2577988d2d6df7daf86466be8f9308204985681f24452a", - "libs/policyengine-fastapi/tests/ping/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "libs/policyengine-fastapi/tests/ping/test_alive.py": "21ac2c13efa5eac90c78771f9bc0c01c62e00729bd8f62b1622f4935b1583112", - "libs/policyengine-fastapi/tests/ping/test_ping.py": "a4a7ea1a770cff99c828b9252475665b804f0a77c7759ef2b9c411eac97ca56d", - "libs/policyengine-fastapi/tests/ping/test_started.py": "0a9c2d4c7b120895da51ca2aaa66472161b64bc9a4745721cd0c799b46bdc4a9", - "libs/policyengine-fastapi/tests/test_exit.py": "4e493b7470457d5a5d247890846384d4a70d72476cfe5d94cf94002d750db3ee", - "libs/policyengine-fastapi/tests/test_observability.py": "aab8025c15fa6304cb9a4fc8262c497793b44f79942128387b65e8ba3ffb86ed", - "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/__init__.py": "052d987a106e9a83adf5e3728ab44beaef3849e965dc470b66876afe40f379ae", - "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/budget_window_state.py": "d4b1976cf1bd9ca693e468882ab78764a88b7c7aad7c35c588e37f1e692fe8ae", - "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/dataset_uri.py": "855f4ca52c36e2626f0dbeb09d0de4b1ab5473eeadcc5964cdfd33b5b9afe302", - "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/gateway_models.py": "ec4669c02b53bad9e602c7571e4533d9e39ed923d39c769c7454cc0634040a51", - "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/hf_dataset.py": "728b8eb8fe01dc4194466e49446efb94d65740ca3dff4daf9f8db8cc18522078", - "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/json_types.py": "65b54f228c8c127201b71b2ef3ca9a779d6fbdfaeeba684c2ad7e00f1351111c", - "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/macro_output.py": "c860ede3f7ab397ff1f24ee6b9eedf689eeabfc994434ff52e1b2e55fe39e8c6", - "libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py": "9a751b3ded7a637b8446d7ef2eeef34cbcc77835b2d112f66a525f4e7d9cacc2", - "libs/policyengine-simulation-contract/tests/conftest.py": "3c78401c6ddd675b1666b7a0f481133a0b6214d788ce13c0a2147ce2ad6ba080", - "libs/policyengine-simulation-contract/tests/test_budget_window_state.py": "0b9f6c2f9007628a67695879d8d6d9f7eaca9e832486285498ebfa5d88de2705", - "libs/policyengine-simulation-contract/tests/test_dataset_uri.py": "d0d99e6c30bc8bcc3a085d43560ceb52565b5cd4b0ca8f2b22bc648c7ea749ff", - "libs/policyengine-simulation-contract/tests/test_gateway_models.py": "b16632e54af9101cdb9f8839bef431957503022fc3389ace7d2ed7c1fcc87e4d", - "libs/policyengine-simulation-contract/tests/test_hf_dataset.py": "a88adca95e0fbab294940ddbdb5d29fe5153b986242a7f58525dffe0c9ce681b", - "libs/policyengine-simulation-contract/tests/test_spm_selection.py": "cc7c92ae1d2b329e6e945d9d8b15111621983f600f7ea94c06c1a0b58562a696", - "libs/policyengine-simulation-observability/src/policyengine_simulation_observability/__init__.py": "e798650c4dac75dff7dc14bc65cafa1eee2cf95030eb59f435fe7f86846445b9", - "libs/policyengine-simulation-observability/src/policyengine_simulation_observability/errors.py": "72d5d712a0faeb33b709ae924cb03df1f62908c389e5c8bebdf9d9863c262b11", - "libs/policyengine-simulation-observability/src/policyengine_simulation_observability/logfire_legacy.py": "8f399e6a7d5c8179569e79d309677d4894cb1f35d9b87a8e7790e5ff02dc847a", - "libs/policyengine-simulation-observability/src/policyengine_simulation_observability/observability.py": "af0f07dd96dfc286ccf615d059d6e97278533162a8403bc76934d5799a62dd65", - "libs/policyengine-simulation-observability/src/policyengine_simulation_observability/telemetry.py": "8c04ba096e9f81f4324b5d97206c93a17cfd343ec5ef3fdfa7287f898ec859f7", - "libs/policyengine-simulation-observability/tests/test_errors.py": "7cf9c8512a639f4735dd705e0ad746241842bbc1fa9ef63537a15e7728c23382", - "libs/policyengine-simulation-observability/tests/test_logfire_legacy.py": "9371b2e2121efe99479522618cec268d1529b22f4e87bd10845d7344527225eb", - "libs/policyengine-simulation-observability/tests/test_observability.py": "2d8ce6cc1363880a559624bf9e74b9e85e0d3f5ad345d80c0db63b0630d258bb", - "libs/policyengine-simulation-observability/tests/test_telemetry.py": "c8e77b6b928fc9b106d529f8947130c1e7a4f42e139c26856e556405806d01f5", - "projects/policyengine-apis-integ/tests/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "projects/policyengine-apis-integ/tests/simulation/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "projects/policyengine-apis-integ/tests/simulation/conftest.py": "5901dec759cb22a66b09b36e41b970166c4f0e3b2776cb6c4e3a9dc0231c92dd", - "projects/policyengine-apis-integ/tests/simulation/test_auth_smoke.py": "2fedca4601f4615dd52dcf7ca362493c5b16d08635d1ddc678e8b5fcccb19503", - "projects/policyengine-apis-integ/tests/simulation/test_budget_window.py": "1da04eea75c6e5f010f1bc7eada8585ca3de37efc31a3e49a9b13c1cb7ff9431", - "projects/policyengine-apis-integ/tests/simulation/test_calculate.py": "63f2994a5beca6fd8d2d39250d7a7f1ef97f5afe7190c6cedcd3f14e3bec68ae", - "projects/policyengine-apis-integ/tests/simulation/test_ping.py": "c02a38408ad8fa26e39353704383d4625fa5f1ae19ffe066478b0b3a6772d6f0", - "projects/policyengine-apis-integ/tests/test_spm_generated_client.py": "cf213da3d895f558bf39e2d025998694989ebb338964aac9a19c1b457fbeaad9", - "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/__init__.py": "5a781097ce32e248443e2585e8118602bd4932f11cd0e2412e9d0e5647c8a989", - "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py": "1423d7485251dcb89a69aa61bfe02d9269b8228e6434602e28b8d240bef4c60d", - "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/auth.py": "6b8077c854e5273b004eb3a279d036717510d2aeeb29435e2609a0ac5de28449", - "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/backend.py": "62ecce3d42f9ed2b76ac42ac630401a10ac0cb6f450b56e381c2e576b4ada19b", - "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py": "a382920bc4600f5ff9702cdc4a8df9cbf14f5ae556387374122bf265ab1ab981", - "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/generate_openapi.py": "52e1f056053c39f4f36a949eae75259226094780d5e33e97255ba2a2f6e69c2b", - "projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py": "a253c53dba6dd1fb732f912d04b4ff8a9b4525cd602cc3d86c9ec515a02f2c9c", - "projects/policyengine-simulation-entry/tests/conftest.py": "2a70c6344369b43d88b65e5f7096f5df32fb7f6ca384e5eaa98463c06c298b3b", - "projects/policyengine-simulation-entry/tests/test_app.py": "58871ea57ffaca310699e10bafbd12bb5bd9cbfd45bb69adf37b744de59bfe99", - "projects/policyengine-simulation-entry/tests/test_auth.py": "ca9f1060432c9f609073da0e2575bf27779331aed13909eec499a1dd59177357", - "projects/policyengine-simulation-entry/tests/test_backend.py": "8829701bda722198614a3525c33a344bed1b700113cae17c2a7671ae982eaa67", - "projects/policyengine-simulation-entry/tests/test_config.py": "8cad43b56f1258b7595b53114f9619bb87e348d96ea6c9af4aa4c791873a09f4", - "projects/policyengine-simulation-entry/tests/test_deployment_assets.py": "f9f33b62d2141f1f27339aee8fa910262c9ab26863651a99faec036a6c3ba588", - "projects/policyengine-simulation-entry/tests/test_openapi.py": "f0a00748a78bc748cd87419fb5fbb7c3854d6aa06f5b76be6a8b0455fb76cb8b", - "projects/policyengine-simulation-executor/fixtures/__init__.py": "2382b67dc056a47f519de1cddc56b7dc54da71ffbe747e439933108a8ff4a9e5", - "projects/policyengine-simulation-executor/fixtures/fake_modal.py": "033eec5c860a24f635823e2f1253069e49d42b6651ef6deb46768af56b80cab1", - "projects/policyengine-simulation-executor/fixtures/identity_stubs.py": "c97d961868804d94be7007f55cbc2e1ea67ef6f6d5ac0adc3dd3b8a7f0d91409", - "projects/policyengine-simulation-executor/fixtures/test_modal_scripts.py": "2346c68ad3b90b1c74be645c7f319ee7b89fc5566a27a2ab5e3a136a88a70128", - "projects/policyengine-simulation-executor/fixtures/test_policyengine_package_update_scripts.py": "81783c06aa2d1623752a090351f3ec791d6618ec5020877ef7b258ab09b9d485", - "projects/policyengine-simulation-executor/fixtures/test_simulation_api_contracts.py": "4698e89d3b75f9a72995ac330e37968cb669fabebce0e82302b002358e65ddd0", - "projects/policyengine-simulation-executor/fixtures/test_simulation_output_builder.py": "5a12e638b3a4486e5f12f0def071cb4e67e2bdc4e3ed6b29dce1661c5f87e4b3", - "projects/policyengine-simulation-executor/fixtures/test_support.py": "c4760869aa6a35cdc8b7a53ccc45a007094f50f8ca693e08b6e9aa6f6e4b8475", - "projects/policyengine-simulation-executor/src/modal/__init__.py": "d3cbe2a2075ca1f6557cc6d89a277db60bb068fd26a503497cc0faa907e89aa0", - "projects/policyengine-simulation-executor/src/modal/_image_setup.py": "2fc8e800e188ada9eebd010516b36768ccbb4473bb92b6da43348d1d385c7f5f", - "projects/policyengine-simulation-executor/src/modal/app.py": "42fe7bcb2787593aced114c70c69fbe846bb9f90ba1cdfcf81a1df11d64e90e6", - "projects/policyengine-simulation-executor/src/modal/budget_window_batch.py": "3eaa98933b1634579e3516e73e33eb2e9bebd8bd1ef68859ddcaae6193362d52", - "projects/policyengine-simulation-executor/src/modal/budget_window_context.py": "e45eb44c85fec41b41aa65b004ec127858a09dd0f1b5fa153a03d9ae4364edab", - "projects/policyengine-simulation-executor/src/modal/budget_window_results.py": "76fe205ea9f0a6a4e2a7d6ce795214978273c0ffccfab8776bd1dbff3bac2fab", - "projects/policyengine-simulation-executor/src/modal/budget_window_scheduler.py": "153d169999ecb8a10887325e29ea647ddb5ebbac7ded8f0c80d5cea8880d42ee", - "projects/policyengine-simulation-executor/src/modal/dependency_pins.py": "fda1e8c64df0d0361d4940ba26b9a9f294c2b82b083f4bc2ca3437ad3a4d05b8", - "projects/policyengine-simulation-executor/src/modal/fanout.py": "23adcd39d819b5c0ca3e8565616045afc09df0b6a0d96352bb71f1c625d09da5", - "projects/policyengine-simulation-executor/src/modal/logging_redaction.py": "3991cd1e575b81d038d9e3b7e19a59e1eebceb41f24eb7aa7a69d90e95dd092e", - "projects/policyengine-simulation-executor/src/modal/precompute_app.py": "8b6e92e0bcfefd3ddf8368baf1c45e63dff79c757ee1c213da89ce236de21de2", - "projects/policyengine-simulation-executor/src/modal/segmented_national.py": "e15246852fabe616b68e15bc3afa5857c14a96213dcc77115c46da73730abd9d", - "projects/policyengine-simulation-executor/src/modal/smoke_app.py": "cebc2ee8c202fb46dc66b606c92d5ce8c44e849fcef26cc514d91c199cac1181", - "projects/policyengine-simulation-executor/src/modal/utils/extract_bundle_versions.py": "612b6a8b7fcc536567269085f9369d2c1718ecc1e51929c7b9012893db34c183", - "projects/policyengine-simulation-executor/src/modal/utils/record_deployment.py": "2757e092db4975f0d63cd28777ddac12d81695644ccb8a1adf1fd786fa805ac2", - "projects/policyengine-simulation-executor/src/modal/utils/update_version_registry.py": "1e840d2ebbd39aacef2d5aaf9a2113da4ffd7575f15c8d5e5eeae1f719ccefe8", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/__init__.py": "716b0000bc200acb3f5680b5d90057b3bf46f05f4abd0644de1870644a96e624", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_keys.py": "aa144894e2015752666c40e230e33f3aaee6c86f34eebfced2fcee774c1af87a", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/artifact_store.py": "35796f1c645ce0fb1a3cb2f28ed0034947a4c148b52cecccf8ce93a443af6fe9", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/baseline_artifacts.py": "2ec0d48bd97bad7a663deaf49b7a12c5b887d6b29f571e2cfe90d3c4247ad9c1", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/compat_models.py": "5895c1cbaaa13092e604f9dfd0f1048c0a233c1cf3d8b2fa09ecd009b7e2df1d", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/generate_openapi.py": "730125691928ad5c159b5d16f5902b3366949466cda7ebcc2cdcc03f44dd2fea", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/main.py": "85ff24d4ce177c6580ee25515a5667f7891c9f829905efc3670c256d04c59674", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/national_partition.py": "0bf9b334cea28176c0d6c8348f1df68e0af1666bcfbf010a11a09836735b259f", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute.py": "2c2a9bb6c824f4cce603f5728ea4cc58a693f8f354beff10b4cb370d41914e45", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/precompute_models.py": "825fb81870bb8439e89a7415f9c1b9c52e2676e5435314a7eb9db6f7ade1f565", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/release_bundle.py": "71941fba56a4b755ba875ecb1d8b2dc48a9eb817132c8bde13f2e8e4ef063548", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/segmented_national_reduce.py": "72cfc7bdcf150cd747533ecb533b8b07a2e743b3fc9fec0ae5f536c9144121fb", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/settings.py": "652a575875df40ac79a3bf460de49a045967cfe2241125c77c68c8ea9dd0401c", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation.py": "a8f9c2a8deddb39972e0973c6c4307372b779dd19739c42c83f9469a57b0d84c", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_macro_output.py": "43bfae627139dc747ebf905d9abc3ef7c58856a8f86f8e638ed62cd87bb25740", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_microdata.py": "9e3a985e061e8ec1a4646738c63e91039b5de780e68475a883eb65faabf1812f", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_budget.py": "034ebaf9ed6d22bb2bcae4f5ba87f9b6abdddfdc0da459bcede5dddd6603ad74", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_builder.py": "d15a15b6c9b0d3444434568e034f90cd618655344387a1b2b5661a21cdfdb412", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_cliff.py": "5bd5aac3b005e19e64710e127de782714b2dc7f320131ea7deb438bb397574a6", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_common.py": "e76e9b37f0494c211f0f6945a294c370470c52426c24ebb28a38bf555fb72b22", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_distribution.py": "9fb21e82df52f90bb68f67882fb3cff3fcf9bf56302a10b70300d0a47b236caf", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_geographic.py": "0467b98ffd514c93bb932ebf24ee6a9eaef449dbbb2f182a5c725e5b9fc90d14", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_inequality.py": "caa2528af2bf1ddd6e82532755ca55f096787b0c36fc8bcdf256ba3a43c946f9", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_labor.py": "1cc2822aeeae7e87155ddf56c6b81996fcf9e2fb3e22f9480e7402755692567b", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_output_poverty.py": "1ab65438d874bb979da886ea8ee8e08be8216b14d915d6afe6c4d007c8688875", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/simulation_runtime.py": "898326620cb91285d9c3a1f481150ff1e31c91730825bd0770272a228d02d684", - "projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py": "82e71283aff85507216e7eefdb471b26f20173351702a998087bb3ebe35e9a48", - "projects/policyengine-simulation-executor/tests/conftest.py": "eb16bfd02b3270fe5ba393286c0acdceb9b7297069f2c0e846d81e93fcc2e855", - "projects/policyengine-simulation-executor/tests/integration/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "projects/policyengine-simulation-executor/tests/integration/test_budget_window_ephemeral_modal.py": "a84d5dd179ff572b922c91c7ad38ff98e0ccd3387c9ac492a31309af8626e87d", - "projects/policyengine-simulation-executor/tests/integration/test_image_smoke_modal.py": "d981b1ed2eb4a5278a755ebc3ebbae539f8ab0892c6341a6ccb27623fb1ba473", - "projects/policyengine-simulation-executor/tests/native_spm_support.py": "c8d4b66b4d3991fccc31c2f2b82684cde920f984388bf461222e4c6194ccdae3", - "projects/policyengine-simulation-executor/tests/test_app_redaction.py": "2965a253b47fedb7166f507d14d7293e497ef7d674e69582b4c16b353f4ccb4b", - "projects/policyengine-simulation-executor/tests/test_artifact_keys.py": "12245baef89b40864aec1e5f0a3aca20b943af378bba44a1d71a26ca3332d606", - "projects/policyengine-simulation-executor/tests/test_artifact_store.py": "5c2742563f1a01442c20a236d3e9340ee302ecb432daac7a62030bfe3c6b801d", - "projects/policyengine-simulation-executor/tests/test_baseline_artifacts.py": "7459eeaaa96076647b1d08ce8e224cc051ccde714ba859ccc69b6bc93466b55c", - "projects/policyengine-simulation-executor/tests/test_baseline_artifacts_spm_native.py": "13cae4f802c28ad605f0bbe0898e58d4029c9e6dee050dbe7c36a46e60c85533", - "projects/policyengine-simulation-executor/tests/test_budget_window_batch.py": "22c4e350c2835a3b832136f08080f932024478ae04ce8b52b5224bf9c20c0e52", - "projects/policyengine-simulation-executor/tests/test_budget_window_context.py": "9e29c42db83337e2e2333409048ee508d90da385e91049ed26a252b0715b4749", - "projects/policyengine-simulation-executor/tests/test_budget_window_results.py": "5ac9b558ed20498a151d50eda026b244eac9e03dfa5276ee3efff854328ece66", - "projects/policyengine-simulation-executor/tests/test_budget_window_scheduler.py": "a29eb56005648e7cd8321347d58a3e036347bec0c213b66a0e93744e652594a8", - "projects/policyengine-simulation-executor/tests/test_bundle_version_export.py": "23ccf3dfad99af01fe03468ab7e1857bd63880bf7ffea53bdea3feed3f7b5038", - "projects/policyengine-simulation-executor/tests/test_canonical_spm.py": "74e32f6ab77c8aa025181c7005c88a178c0ac8b694dc19910b22efb06fecc122", - "projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py": "a2d7877785af38af76b03ea37ed3b1f934cb8ebd0faa25402c63bca96e130860", - "projects/policyengine-simulation-executor/tests/test_gcp_credentials.py": "3b8b8743f38eabef7fab48bd3d756c7c7742cfeff7d8b71527e52f2e24ec54cb", - "projects/policyengine-simulation-executor/tests/test_image_setup_fetch.py": "1275099acd7d95b632132811709e539fb70b7e67a139fc856b58c4c09018441e", - "projects/policyengine-simulation-executor/tests/test_modal_bundle_image.py": "df05f3b496deaa4b929970e4dea7ab91d06b6192ac8fed3c5158b6df61f99512", - "projects/policyengine-simulation-executor/tests/test_modal_scripts.py": "2fd24ac6d2c35212d74f46c7cee4e437c9352859b5f7ce66b28802ce65fb18e1", - "projects/policyengine-simulation-executor/tests/test_national_partition.py": "d3111ad8dda7150257c595d1dde9211eb34abc5e7e2716851c0e3ac730fc79ef", - "projects/policyengine-simulation-executor/tests/test_pandas3_compatibility.py": "f230a82892f3076952361fa5522095269a8634c99bf6c66be2cf4daaccb637cc", - "projects/policyengine-simulation-executor/tests/test_placeholder.py": "f835fd2d1ece403312811e9fdba7f2c911ad0b7dc53de4892d29b2e9c69da68a", - "projects/policyengine-simulation-executor/tests/test_policyengine_dependency_source.py": "d3b5fab0d7be9ad1251717104bbe3e71186be88efd5024c15fc8eb2396d669c9", - "projects/policyengine-simulation-executor/tests/test_policyengine_package_update_scripts.py": "965c7e8c79e3f9a8d604d4c74e7fba3fcc28bcb506cd6e8d5688644e73799a38", - "projects/policyengine-simulation-executor/tests/test_precompute.py": "512762d5dff607c66682aacb64d05e9a6f6c6a29bb1c118f9e2bb192a7a76ea0", - "projects/policyengine-simulation-executor/tests/test_precompute_app.py": "892276cd8e229b98010aba072f041f9283d46f11a9bc99e4467bf1edefdb2b73", - "projects/policyengine-simulation-executor/tests/test_record_deployment.py": "0b8dd6310c496ac6f736b6c6812671ae54768e52e7c3c5f8d2d069e06c271f81", - "projects/policyengine-simulation-executor/tests/test_region_group_resolution.py": "cb4b65d6301d4515f466f5e4f933e44396cfc04f99e62a8fe3f10a1091fa2c44", - "projects/policyengine-simulation-executor/tests/test_release_bundle.py": "8477397d5d24d8b4db257b35a5cfbd03d858dbd265634c5ec85d06353b388a25", - "projects/policyengine-simulation-executor/tests/test_segmented_national.py": "f236d257071d33803a37ff27b771cb6a18b973a70023025eb77b291c31ac0dfd", - "projects/policyengine-simulation-executor/tests/test_segmented_national_reduce.py": "4024783eb555f28e10ef83ad3ecddd30316ed0966d39b5a60c6447a64a834efa", - "projects/policyengine-simulation-executor/tests/test_settings.py": "c32a6c23d93e33accb0deb044a73bebc9faaecb5bfa34e33bf1d30c6eca939e8", - "projects/policyengine-simulation-executor/tests/test_simulation_api_contracts.py": "00d72ce7622b2999af10757afed8fc853cf3f987e508e31db9a7b4ed8ddcf546", - "projects/policyengine-simulation-executor/tests/test_simulation_microdata.py": "926f3d4a3b09ee9c27aef5cd9bd5a7532cc4098cfccccb90578e0db1eb887000", - "projects/policyengine-simulation-executor/tests/test_simulation_output_builder.py": "f2d3b27b0968418c5b5dd117c90c3cab09ec5fb9bd71539bdcf97628e3974180", - "projects/policyengine-simulation-executor/tests/test_spm_runtime_native.py": "af7dafb2639620e0644d2df75e57809070a8252b8ea62ae8c2280ad83ffc8555", - "projects/policyengine-simulation-executor/tests/test_standalone_simulation_contract.py": "a09935edd48c11b91090a9bd1a5652f0393959c1202630e0a83dd6695e3dfbae", - "projects/policyengine-simulation-executor/tests/test_update_version_registry.py": "92f0e2dfb8fa24c6edd24c018f6a89f27c94f508a095376bb2004eb056890f6f", - "projects/policyengine-simulation-gateway/fixtures/__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "projects/policyengine-simulation-gateway/fixtures/gateway_endpoints.py": "1402ad55117a2ae76e97e666a321eae361ff59797be269ce1edc8a2e6b503fe7", - "projects/policyengine-simulation-gateway/fixtures/package_imports.py": "1864c22dbe8438c342be9a14fe100ac91b3d8927f8532cf45b7963325b6e29cc", - "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/__init__.py": "374a88d7888c27fddada550766f489ebd83121610be4f56c3f392816b52b04e6", - "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/app.py": "fef0ca6f10bfb12248d0107cc29bc68194c7cca863bdded00dbf3624d550c708", - "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/auth.py": "fc512f7dc9dfd9eb8b056f166dacb7f24f6541205a9c1d4b2f5752f07cec227b", - "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py": "adaae4cb7c0a21dd728ca4c8c0b85db934af964068a302af17f1a9852d20d8f3", - "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/generate_openapi.py": "7b2cc5be0a20fe2cdd75a05fc852cee701a119c8cf9622f7d8b28b9e800e047c", - "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/responses.py": "7e48b9d01ca6990df78f46df9f281620a9e16e37f33be237358112c90b9c4d93", - "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/smoke_app.py": "7fa96ae81dbc27b90e6dc77300e55275e9b2ad11f50e8c40bfac4b9d22d06c84", - "projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/testing.py": "25f95c79db79f9ec2d3cc693b75d74e2c2ba9683b0a1a1ff377bd35bada52ab3", - "projects/policyengine-simulation-gateway/tests/conftest.py": "873d596435f011f3fc23a274c9ecd7ea39abb0b7a25fc5d5c39f7a5a9ada1ac2", - "projects/policyengine-simulation-gateway/tests/integration/test_image_smoke_modal.py": "141f363280f9f7ca2f84f70fc24cc31608db4e460a83e3ca29e7273a979f81c6", - "projects/policyengine-simulation-gateway/tests/test_auth.py": "edce21cced7146d90fbc2ed6c0fd226f8f6aa4f893527197bb99075ea70e5fed", - "projects/policyengine-simulation-gateway/tests/test_endpoints.py": "284a4cc6636c9f929f76012d91f817d5f20e49140351aa6dbee22be2721ff778", - "projects/policyengine-simulation-gateway/tests/test_health.py": "429a017d164e8a4324f2c1aa102e3caf91bcad3176cc510f5496bd642192fd41", - "projects/policyengine-simulation-gateway/tests/test_import_coverage.py": "9437892c4461e15b6695c44fd67dda55b249fe6c423456a6fe6a80b656616f2f", - "projects/policyengine-simulation-gateway/tests/test_modal_gateway_image.py": "e4ca9a47c9a5b17c2986db300f48f537b40590f05179394e63918468a77867e5", - "projects/policyengine-simulation-gateway/tests/test_observability.py": "c9492dcee6dfb09dac6f0408d025e395433befa8875bbcb6aac7e2ca92123b5d", - "projects/policyengine-simulation-gateway/tests/test_openapi_golden.py": "ab02b0316789b7e3f94843af34f84d0fa5f37aa642313b452af50438577a3290", - "projects/policyengine-simulation-gateway/tests/test_package_imports.py": "2e05408df800e4f6855ac8dc56cdbe6d28135a8c563c5bb572132aaef3b85102", - "projects/policyengine-simulation-gateway/tests/test_ping.py": "4d33e6ba7573e83d0e96292dfc66822b417135e8e35422597157a5fee5731d05", - "projects/policyengine-simulation-gateway/tests/test_route_table.py": "28778a2b3da017cac8fad9de5dcf899f8d96b0e913527ddd7a341f9ff159c34a", - "projects/policyengine-simulation-gateway/tests/test_spm_routes.py": "a97d31eb784504d01f64e80f1ec2e84f72ada13770edfd37f981681612fc08cf" - } -} diff --git a/rollout/worker-fable-fixes/typing-comparison.json b/rollout/worker-fable-fixes/typing-comparison.json deleted file mode 100644 index a2edf4073..000000000 --- a/rollout/worker-fable-fixes/typing-comparison.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "executor": { - "current": { - "filesAnalyzed": 43, - "errorCount": 145, - "warningCount": 0, - "informationCount": 0, - "timeInSec": 2.096 - }, - "base": { - "filesAnalyzed": 43, - "errorCount": 145, - "warningCount": 0, - "informationCount": 0, - "timeInSec": 1.352 - }, - "new_diagnostics": [] - }, - "gateway": { - "current": { - "filesAnalyzed": 8, - "errorCount": 78, - "warningCount": 0, - "informationCount": 0, - "timeInSec": 0.997 - }, - "base": { - "filesAnalyzed": 8, - "errorCount": 78, - "warningCount": 0, - "informationCount": 0, - "timeInSec": 0.684 - }, - "new_diagnostics": [] - }, - "contract": { - "current": { - "filesAnalyzed": 8, - "errorCount": 0, - "warningCount": 0, - "informationCount": 0, - "timeInSec": 0.741 - } - }, - "entry": { - "current": { - "filesAnalyzed": 7, - "errorCount": 0, - "warningCount": 0, - "informationCount": 0, - "timeInSec": 0.767 - } - } -} From b0447ca8bf29d45b8ea9163f7be2c1dc12d3ee63 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 9 Sep 2026 20:12:22 -0400 Subject: [PATCH 22/40] Fix isolated service coverage for explicit SPM dates --- .../tests/test_app.py | 27 +++++++++++++++++++ .../tests/test_canonical_spm.py | 15 +++++------ 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/projects/policyengine-simulation-entry/tests/test_app.py b/projects/policyengine-simulation-entry/tests/test_app.py index bbfa7bdea..c43bd6fcb 100644 --- a/projects/policyengine-simulation-entry/tests/test_app.py +++ b/projects/policyengine-simulation-entry/tests/test_app.py @@ -358,3 +358,30 @@ def test_upstream_error_status_and_body_are_preserved(client, backend, status_co assert result.json() == {"detail": "upstream response"} assert result.headers[REQUEST_ID_HEADER] == "request-upstream-error" assert "x-request-id" not in result.headers + + +@pytest.mark.parametrize( + "path,extra", + [ + ("/simulate/economy/comparison", {}), + ( + "/simulate/economy/budget-window", + {"region": "us", "start_year": "2026", "window_size": 2}, + ), + ], +) +@pytest.mark.parametrize("date_fields", [{"as_of": None}, {}]) +def test_submission_preserves_spm_field_presence( + client, backend, path, extra, date_fields +): + selection = {"geography_kind": "national", **date_fields} + + result = client.post(path, json={"country": "us", "spm": selection, **extra}) + + assert result.status_code == 200 + forwarded = backend.requests[-1] + assert forwarded.method == "POST" + assert forwarded.path == path + # Explicit null clears the downstream bundle cutoff; an omitted date inherits it. + # Other omitted fields must remain absent so bundle defaults can fill them. + assert forwarded.json_body["spm"] == selection diff --git a/projects/policyengine-simulation-executor/tests/test_canonical_spm.py b/projects/policyengine-simulation-executor/tests/test_canonical_spm.py index 966b08f4b..6637d5371 100644 --- a/projects/policyengine-simulation-executor/tests/test_canonical_spm.py +++ b/projects/policyengine-simulation-executor/tests/test_canonical_spm.py @@ -597,8 +597,11 @@ def test_optional_analysis_does_not_swallow_spm_input_errors(code): ) -def test_explicit_null_as_of_survives_entrypoint_and_budget_parent(): - from policyengine_simulation_entry.app import _model_json +@pytest.mark.parametrize( + "date_fields,expected_as_of", + [({"as_of": None}, None), ({}, "2026-01-01")], +) +def test_as_of_presence_survives_budget_parent(date_fields, expected_as_of): from policyengine_simulation_gateway.endpoints import ( _resolve_request_spm, _build_budget_window_parent_payload, @@ -610,12 +613,8 @@ def test_explicit_null_as_of_survives_entrypoint_and_budget_parent(): region="us", start_year="2026", window_size=2, - spm={"geography_kind": "national", "as_of": None}, + spm={"geography_kind": "national", **date_fields}, ) - entry_payload = _model_json(request) - assert "as_of" in entry_payload["spm"] - assert "forecast_content_sha256" not in entry_payload["spm"] - request = BudgetWindowBatchRequest.model_validate(entry_payload) bundle = PolicyEngineBundle( model_version="test", policyengine_version="test", @@ -627,7 +626,7 @@ def test_explicit_null_as_of_survives_entrypoint_and_budget_parent(): selection = _resolve_request_spm( request, bundle, SimpleNamespace(route_provenance=None) ) - assert selection["as_of"] is None + assert selection["as_of"] == expected_as_of parent = _build_budget_window_parent_payload( request, resolved_version="test", resolved_app_name="test", bundle=bundle ) From a11c4aeb60125d01b43699a4652fb2709625a275 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 11:51:08 -0400 Subject: [PATCH 23/40] Add progress tracker for review fixes Co-Authored-By: Claude Fable 5.1 --- PROGRESS.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 PROGRESS.md diff --git a/PROGRESS.md b/PROGRESS.md new file mode 100644 index 000000000..e7894e041 --- /dev/null +++ b/PROGRESS.md @@ -0,0 +1,29 @@ +# PR #677 Fable-review fixes — progress + +Branch: `max/spm-simulation-canonical-fixes-20260911` (from PR head `b0447ca`) +Target PR branch: `max/spm-simulation-canonical-20260909` +Base: `main` = `414c631d622f5f587eedd187296a80182a923db2` + +## State + +Worktree created at PR head. Investigating the nine review findings. + +## Findings checklist + +- [ ] Medium 1 — completed results drop explicit nulls from `spm_config` on the wire +- [ ] Medium 2 — legacy no-SPM provenance `legacy-seed` expires on next publish +- [ ] Medium 3 — precompute storage identity never shown to match the wrapper +- [ ] Medium 4 — budget-window scheduler typed-error branches untested +- [ ] Low 1 — legacy no-SPM poll bodies not byte-identical to base +- [ ] Low 2 — stale country routes now fail; undocumented, untested +- [ ] Low 3 — segmented reduce with SPM children untested end to end +- [ ] Low 4 — hermetic CI never exercises receipt validation in `ensure()` +- [ ] Low 5 — repeated prevalidation per request (perf) + +## Done + +- Created worktree, verified head/base SHAs. + +## Next + +- Deep-read each finding site; write fixes with regression tests. From b074b955885b82c54d833b3ea61c265eae7bcafd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 12:03:15 -0400 Subject: [PATCH 24/40] Keep resolved SPM nulls on the wire and legacy bodies unchanged Co-Authored-By: Claude Fable 5.1 --- PROGRESS.md | 21 +- .../policyengine_simulation_contract/spm.py | 29 ++- .../endpoints.py | 17 +- .../tests/test_spm_routes.py | 189 ++++++++++++++++++ 4 files changed, 248 insertions(+), 8 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index e7894e041..2ff25972c 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -10,11 +10,11 @@ Worktree created at PR head. Investigating the nine review findings. ## Findings checklist -- [ ] Medium 1 — completed results drop explicit nulls from `spm_config` on the wire +- [x] Medium 1 — completed results drop explicit nulls from `spm_config` on the wire - [ ] Medium 2 — legacy no-SPM provenance `legacy-seed` expires on next publish - [ ] Medium 3 — precompute storage identity never shown to match the wrapper - [ ] Medium 4 — budget-window scheduler typed-error branches untested -- [ ] Low 1 — legacy no-SPM poll bodies not byte-identical to base +- [x] Low 1 — legacy no-SPM poll bodies not byte-identical to base - [ ] Low 2 — stale country routes now fail; undocumented, untested - [ ] Low 3 — segmented reduce with SPM children untested end to end - [ ] Low 4 — hermetic CI never exercises receipt validation in `ensure()` @@ -23,7 +23,22 @@ Worktree created at PR head. Investigating the nine review findings. ## Done - Created worktree, verified head/base SHAs. +- Baseline suites green at PR head: contract 73, gateway 149, entry 70, + executor 456 passed / 22 skipped. +- **Medium 1**: `SPMSelection.serialize_selection` now restores options that + were explicitly selected as null when the caller serializes with + `exclude_none` (the poll routes do, via `response_model_exclude_none`). + Tests: `test_completed_result_body_keeps_resolved_nulls`, + `test_completed_budget_window_rows_keep_resolved_nulls` — both verified + failing against the pre-fix serializer. +- **Low 1**: new `_bundle_payload` helper drops the `spm` key from raw 202/500 + job bodies and the worker `_metadata` when a route has no capability, so a + legacy no-SPM body is byte-identical to base. Tests: + `test_legacy_poll_bodies_carry_no_spm_key`, + `test_legacy_budget_window_metadata_carries_no_spm_key`, + `test_canonical_poll_bodies_still_carry_the_capability` — the first two + verified failing against the pre-fix endpoints. ## Next -- Deep-read each finding site; write fixes with regression tests. +- Medium 2 (durable legacy-route provenance), Medium 3, Medium 4, Low 2-5. diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py index 9b3f245ce..2ae4e1789 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py @@ -8,6 +8,7 @@ BaseModel, ConfigDict, Field, + SerializationInfo, SerializerFunctionWrapHandler, field_validator, model_serializer, @@ -44,11 +45,33 @@ class SPMSelection(BaseModel): as_of: Optional[str] = None @model_serializer(mode="wrap") - def serialize_selection(self, handler: SerializerFunctionWrapHandler): - """Preserve inherited options through ordinary and nested JSON.""" + def serialize_selection( + self, handler: SerializerFunctionWrapHandler, info: SerializationInfo + ): + """Preserve inherited options through ordinary and nested JSON. + + Presence is the contract: an omitted option inherits the bundle + default, so an option explicitly selected as null has to stay on the + wire. ``exclude_none`` would otherwise turn a completed result's + resolved selection back into a partial request, and the poll routes + apply it to every body via ``response_model_exclude_none``. + """ + dumped = handler(self) + if info.exclude_none: + excluded = info.exclude or frozenset() + dumped = { + name: dumped.get(name) + for name in type(self).model_fields + if name in dumped + or ( + name in self.model_fields_set + and name not in excluded + and getattr(self, name) is None + ) + } return { name: value - for name, value in handler(self).items() + for name, value in dumped.items() if name in self.model_fields_set } diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py index dafbfda4b..0e79a64d9 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py @@ -653,6 +653,19 @@ def _resolve_request_spm(request, bundle, route): return selection +def _bundle_payload(bundle: PolicyEngineBundle, **dump_kwargs) -> dict: + """Dump a bundle, omitting ``spm`` when the route has no capability. + + The 202 and 500 job bodies splat this dict in directly, bypassing the + routes' ``response_model_exclude_none``. Every other optional bundle + field was already emitted as an explicit null before canonical SPM, so + only the new key is dropped: a legacy no-SPM body stays byte-identical. + """ + return bundle.model_dump( + exclude=None if bundle.spm is not None else {"spm"}, **dump_kwargs + ) + + def _serialize_job_metadata( resolved_app_name: str, bundle: PolicyEngineBundle, @@ -660,7 +673,7 @@ def _serialize_job_metadata( ) -> dict: return { "resolved_app_name": resolved_app_name, - "policyengine_bundle": bundle.model_dump(), + "policyengine_bundle": _bundle_payload(bundle), "run_id": run_id, } @@ -685,7 +698,7 @@ def _build_budget_window_parent_payload( payload["_metadata"] = { "resolved_version": resolved_version, "resolved_app_name": resolved_app_name, - "policyengine_bundle": bundle.model_dump(mode="json"), + "policyengine_bundle": _bundle_payload(bundle, mode="json"), } return payload diff --git a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py index 0ca79f1db..d94fdbff5 100644 --- a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py +++ b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py @@ -213,3 +213,192 @@ def test_versions_omits_malformed_capability_and_submission_rejects_it( assert rejected.status_code == 400 assert rejected.json()["errors"][0]["code"] == "SPM_CONFIGURATION_UNAVAILABLE" assert mock_modal["func"].calls == [] + + +RESOLVED_SELECTION = { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} + + +def spm_receipt(year): + return { + "forecast_id": "ce-forecast", + "forecast_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "runtime_versions": {"spm_calculator": "0.3.1"}, + "years": {str(year): {"entry": f"{year}-01-01"}}, + "geographies": [{"kind": "national"}], + "composition_method": "national", + "storage_method": "artifact", + } + + +def spm_provenance(year): + return {"baseline": [spm_receipt(year)], "reform": [spm_receipt(year)]} + + +def test_completed_result_body_keeps_resolved_nulls(mock_modal, client): + """A client must be able to replay the selection it was handed back. + + ``response_model_exclude_none`` applies to every poll body, and an + omitted option inherits the bundle default. Dropping a resolved null + would silently re-resolve ``as_of`` and ``geography_id`` on the next + request, changing the baseline key and the receipts. + """ + shared_app_state(mock_modal, sibling_model="1.824.7") + submitted = client.post( + "/simulate/economy/comparison", + json={ + "country": "us", + "version": "1.824.7", + "spm": {"geography_kind": "national", "as_of": None}, + }, + ) + assert submitted.status_code == 200, submitted.text + assert mock_modal["func"].last_payload["spm"] == RESOLVED_SELECTION + + job_id = submitted.json()["job_id"] + call = mock_modal["function_call"].registry[job_id] + call.result = { + **call.result, + "spm_config": RESOLVED_SELECTION, + "spm_provenance": spm_provenance(2026), + } + + polled = client.get(f"/jobs/{job_id}") + assert polled.status_code == 200, polled.text + assert polled.json()["result"]["spm_config"] == RESOLVED_SELECTION + + +def test_completed_budget_window_rows_keep_resolved_nulls(mock_modal, client): + """Each ``annualImpacts`` row carries the same replayable selection.""" + from policyengine_simulation_contract.budget_window_state import ( + put_batch_job_state, + ) + from policyengine_simulation_contract.gateway_models import ( + BudgetWindowAnnualImpact, + BudgetWindowBatchState, + BudgetWindowResult, + BudgetWindowTotals, + PolicyEngineBundle, + ) + + shared_app_state(mock_modal, sibling_model="1.824.7") + impact = BudgetWindowAnnualImpact( + spm_config=RESOLVED_SELECTION, + spm_provenance=spm_provenance(2026), + year="2026", + taxRevenueImpact=10, + federalTaxRevenueImpact=7, + stateTaxRevenueImpact=3, + benefitSpendingImpact=5, + budgetaryImpact=15, + ) + put_batch_job_state( + BudgetWindowBatchState( + batch_job_id="mock-batch-job-id-123", + status="complete", + country="us", + region="us", + version="1.824.7", + target="general", + resolved_app_name="policyengine-simulation-py4-10-0", + policyengine_bundle=PolicyEngineBundle(model_version="1.824.7"), + start_year="2026", + window_size=1, + max_parallel=1, + request_payload={"country": "us", "spm": RESOLVED_SELECTION}, + years=["2026"], + queued_years=[], + running_years=[], + completed_years=["2026"], + failed_years=[], + child_jobs={}, + partial_annual_impacts={}, + result=BudgetWindowResult( + startYear="2026", + endYear="2026", + windowSize=1, + annualImpacts=[impact], + totals=BudgetWindowTotals( + taxRevenueImpact=10, + federalTaxRevenueImpact=7, + stateTaxRevenueImpact=3, + benefitSpendingImpact=5, + budgetaryImpact=15, + ), + ), + error=None, + created_at="2026-01-01T00:00:00+00:00", + updated_at="2026-01-01T00:00:01+00:00", + run_id="batch-run-123", + ) + ) + + polled = client.get("/budget-window-jobs/mock-batch-job-id-123") + assert polled.status_code == 200, polled.text + row = polled.json()["result"]["annualImpacts"][0] + assert row["spm_config"] == RESOLVED_SELECTION + + +# Every optional field the pre-SPM gateway emitted in a raw 202/500 body. +LEGACY_BUNDLE_KEYS = {"model_version", "policyengine_version", "data_version", "dataset"} + + +@pytest.mark.parametrize("source", ["legacy-country-dict", "legacy-seed"]) +def test_legacy_poll_bodies_carry_no_spm_key(mock_modal, client, source): + """The 202 and 500 bodies splat job metadata in raw, bypassing the + routes' ``response_model_exclude_none``. A legacy no-SPM job must stay + byte-identical to the pre-SPM gateway.""" + legacy_route(mock_modal, source, "1.715.2") + submitted = client.post("/simulate/economy/comparison", json={"country": "us"}) + assert submitted.status_code == 200, submitted.text + job_id = submitted.json()["job_id"] + call = mock_modal["function_call"].registry[job_id] + + call.running = True + running = client.get(f"/jobs/{job_id}") + assert running.status_code == 202 + assert set(running.json()["policyengine_bundle"]) == LEGACY_BUNDLE_KEYS + + call.running = False + call.error = RuntimeError("worker exploded") + failed = client.get(f"/jobs/{job_id}") + assert failed.status_code == 500 + assert set(failed.json()["policyengine_bundle"]) == LEGACY_BUNDLE_KEYS + + +@pytest.mark.parametrize("source", ["legacy-country-dict", "legacy-seed"]) +def test_legacy_budget_window_metadata_carries_no_spm_key(mock_modal, client, source): + """The same bundle rides to the worker in ``_metadata``.""" + legacy_route(mock_modal, source, "1.715.2") + submitted = client.post( + "/simulate/economy/budget-window", + json={"country": "us", "region": "us", "start_year": "2026", "window_size": 2}, + ) + assert submitted.status_code == 200, submitted.text + metadata = mock_modal["func"].last_payload["_metadata"] + assert set(metadata["policyengine_bundle"]) == LEGACY_BUNDLE_KEYS + + +def test_canonical_poll_bodies_still_carry_the_capability(mock_modal, client): + """Omission is scoped to routes with no capability, not to all routes.""" + shared_app_state(mock_modal, sibling_model="1.824.7") + submitted = client.post( + "/simulate/economy/comparison", json={"country": "us", "version": "1.824.7"} + ) + assert submitted.status_code == 200, submitted.text + job_id = submitted.json()["job_id"] + mock_modal["function_call"].registry[job_id].running = True + + running = client.get(f"/jobs/{job_id}") + assert running.status_code == 202 + bundle = running.json()["policyengine_bundle"] + assert set(bundle) == LEGACY_BUNDLE_KEYS | {"spm"} + assert bundle["spm"]["defaults"]["scenario"] == "ce_trend" From 0c3f9c609fb6b8fa23ad8d7dc4ca589b62d6e9ab Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 12:05:34 -0400 Subject: [PATCH 25/40] Cover segmented national SPM reduce end to end Co-Authored-By: Claude Fable 5.1 --- .../tests/test_segmented_national.py | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) diff --git a/projects/policyengine-simulation-executor/tests/test_segmented_national.py b/projects/policyengine-simulation-executor/tests/test_segmented_national.py index 0ac6ea7ce..e5280a4c9 100644 --- a/projects/policyengine-simulation-executor/tests/test_segmented_national.py +++ b/projects/policyengine-simulation-executor/tests/test_segmented_national.py @@ -5,7 +5,9 @@ import pytest from src.modal import segmented_national as sn +from policyengine_simulation_contract.spm import SPMInputError from policyengine_simulation_executor import simulation_runtime as sr +from policyengine_simulation_executor import spm as executor_spm from policyengine_simulation_executor.national_partition import ( US_NATIONAL_REGION_GROUPS, ) @@ -363,6 +365,9 @@ def fake_segmented(params, *, app_name): return {"segmented": True} monkeypatch.setattr(sn, "run_segmented_national_impl", fake_segmented) + monkeypatch.setattr( + executor_spm, "normalize_runtime_spm", lambda params: None + ) monkeypatch.setattr( sr, "run_simulation_impl", @@ -388,9 +393,175 @@ def test__everything_else_takes_the_monolithic_path( "run_segmented_national_impl", lambda *a, **k: pytest.fail("segmented path must not run"), ) + monkeypatch.setattr( + executor_spm, "normalize_runtime_spm", lambda params: None + ) monkeypatch.setattr( sr, "run_simulation_impl", lambda params: {"monolithic": True} ) assert sn.dispatch_run_simulation(params, app_name="app-x") == { "monolithic": True } + + +SPM_SELECTION = { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} + + +def _spm_receipt(label, *, selection=SPM_SELECTION, year="2026"): + return { + "forecast_id": label, + "forecast_sha256": selection["forecast_content_sha256"], + "scenario": selection["scenario"], + "geography_kind": selection["geography_kind"], + "runtime_versions": {"policyengine-us": "test-only"}, + "years": {year: {"status": "forecast"}}, + "geographies": [], + "composition_method": "classified-inputs", + "storage_method": "formula", + } + + +def _spm_child(index, *, selection=SPM_SELECTION, year="2026"): + """One child's result carrying that segment's own SPM receipts.""" + return { + "child": index, + "spm_config": dict(selection), + "spm_provenance": { + "baseline": [ + _spm_receipt(f"baseline-{index}", selection=selection, year=year) + ], + "reform": [_spm_receipt(f"reform-{index}", selection=selection, year=year)], + }, + } + + +def _spm_child_missing(index, field): + """A child whose transported selection lost a resolved option.""" + child = _spm_child(index) + del child["spm_config"][field] + return child + + +@pytest.fixture +def stub_reduce(monkeypatch): + """The microdata reduce itself is covered by + test_segmented_national_reduce; these tests pin the SPM merge.""" + monkeypatch.setattr( + sn, "build_national_output", lambda children, **kwargs: {"budget": {}} + ) + + +class TestSegmentedNationalSPM: + def test__child_receipts_combine_into_one_national_receipt( + self, bare_country, stub_reduce + ): + fake = FakeModal([FakeCall(_spm_child(i)) for i in range(20)]) + runner = _runner(fake, params={**NATIONAL, "spm": SPM_SELECTION}) + + output = runner.run() + + # The resolved selection rides to every child unchanged and comes + # back on the parent as the one national selection. + assert all(p["spm"] == SPM_SELECTION for p in fake.spawned_payloads) + assert output["spm_config"] == SPM_SELECTION + # Every executed segment's receipts survive, baseline and reform + # concatenated in group order: a national SPM result must account + # for all 20 segments, not just the first. + provenance = output["spm_provenance"] + assert [r["forecast_id"] for r in provenance["baseline"]] == [ + f"baseline-{i}" for i in range(20) + ] + assert [r["forecast_id"] for r in provenance["reform"]] == [ + f"reform-{i}" for i in range(20) + ] + assert output["budget"] == {} + + @pytest.mark.parametrize( + "rogue,reason", + [ + ( + _spm_child(7, selection={**SPM_SELECTION, "scenario": "zero_real"}), + "selection differs from the request", + ), + (_spm_child(7, year="2025"), "does not cover the requested year"), + (_spm_child_missing(7, "county_vintage"), "complete resolved"), + ], + ) + def test__child_receipt_disagreeing_with_the_request_is_typed( + self, bare_country, stub_reduce, rogue, reason + ): + # One segment computed something other than what was requested: + # fail with the public typed code, never publish a mixed national + # result whose receipt does not describe every segment. + calls = [FakeCall(_spm_child(i)) for i in range(20)] + calls[7] = FakeCall(rogue) + runner = _runner(FakeModal(calls), params={**NATIONAL, "spm": SPM_SELECTION}) + + with pytest.raises(SPMInputError, match=reason) as error: + runner.run() + assert error.value.code == "SPM_CONFIGURATION_UNAVAILABLE" + + def test__unrequested_child_receipt_is_typed(self, bare_country, stub_reduce): + # A historical request must not inherit a child's stray receipt. + calls = [FakeCall({"child": i}) for i in range(20)] + calls[3] = FakeCall(_spm_child(3)) + runner = _runner(FakeModal(calls)) + + with pytest.raises(SPMInputError, match="Unexpected SPM receipt"): + runner.run() + + def test__a_run_without_spm_adds_no_receipt_keys(self, bare_country, stub_reduce): + runner = _runner(FakeModal([FakeCall({"child": i}) for i in range(20)])) + assert runner.run() == {"budget": {}} + + +class TestDispatchResolvesSPM: + def test__selection_is_resolved_before_the_fan_out(self, monkeypatch): + # Children and the parent's receipt check must both see the + # worker-resolved selection, not the caller's partial one. + monkeypatch.setattr( + executor_spm, "normalize_runtime_spm", lambda params: SPM_SELECTION + ) + seen = {} + + def fake_segmented(params, *, app_name): + seen["params"] = params + return {"segmented": True} + + monkeypatch.setattr(sn, "run_segmented_national_impl", fake_segmented) + + result = sn.dispatch_run_simulation( + {**NATIONAL, "spm": {"geography_kind": "national"}}, app_name="app-x" + ) + + assert result == {"segmented": True} + assert seen["params"]["spm"] == SPM_SELECTION + + def test__typed_spm_error_spawns_nothing(self, monkeypatch): + # Resolution failures are cheap only if they happen before 20 + # children are spawned. + def unavailable(params): + raise SPMInputError("SPM_GEOGRAPHY_REQUIRED", "County required") + + monkeypatch.setattr(executor_spm, "normalize_runtime_spm", unavailable) + monkeypatch.setattr( + sn, + "run_segmented_national_impl", + lambda *a, **k: pytest.fail("segmented path must not run"), + ) + monkeypatch.setattr( + sr, + "run_simulation_impl", + lambda params: pytest.fail("monolithic path must not run"), + ) + + with pytest.raises(SPMInputError) as error: + sn.dispatch_run_simulation(dict(NATIONAL), app_name="app-x") + assert error.value.code == "SPM_GEOGRAPHY_REQUIRED" From 4992a19d18b3f146ccdde3ea3a5376fcc4434f65 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 12:10:41 -0400 Subject: [PATCH 26/40] Derive legacy route provenance from route shape, not registry generation Co-Authored-By: Claude Fable 5.1 --- PROGRESS.md | 2 +- .../policyengine_simulation_contract/spm.py | 17 ++- .../tests/test_spm_selection.py | 55 ++++++++- .../tests/test_canonical_spm.py | 8 +- .../endpoints.py | 36 +++++- .../tests/test_spm_routes.py | 116 +++++++++++++++++- 6 files changed, 214 insertions(+), 20 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 2ff25972c..b06b8d894 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -11,7 +11,7 @@ Worktree created at PR head. Investigating the nine review findings. ## Findings checklist - [x] Medium 1 — completed results drop explicit nulls from `spm_config` on the wire -- [ ] Medium 2 — legacy no-SPM provenance `legacy-seed` expires on next publish +- [x] Medium 2 — legacy no-SPM provenance `legacy-seed` expires on next publish - [ ] Medium 3 — precompute storage identity never shown to match the wrapper - [ ] Medium 4 — budget-window scheduler typed-error branches untested - [x] Low 1 — legacy no-SPM poll bodies not byte-identical to base diff --git a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py index 2ae4e1789..bbde945da 100644 --- a/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py +++ b/libs/policyengine-simulation-contract/src/policyengine_simulation_contract/spm.py @@ -210,16 +210,23 @@ def resolve_spm_selection( and all(re.fullmatch(r"[0-9]+", p) for p in version_parts) and tuple(map(int, version_parts)) < (5, 2, 0) ) + # The last two pre-canonical wrappers pin US 1.764.6. A route the + # registry carries no manifest for states no model version at all, + # and states nothing that contradicts the wrapper pin; a route that + # states a different model version does. historical = historical or ( - policyengine_version in {"5.2.0", "5.3.0"} and model_version == "1.764.6" + policyengine_version in {"5.2.0", "5.3.0"} + and model_version in (None, "1.764.6") ) - # Country-only routes seeded from the original registry may have no - # wrapper version. Require both their actual route provenance and a - # pre-canonical US model; absence of capability alone proves nothing. + # Country-only routes predating the bundle manifests have no wrapper + # version. Require both their route provenance -- derived from route + # shape, not from the registry's rewritable generation marker -- and + # a pre-canonical US model; absence of capability alone proves + # nothing. model_parts = str(model_version or "").split(".") historical = historical or ( policyengine_version is None - and route_provenance in {"legacy-country-dict", "legacy-seed"} + and route_provenance in {"legacy-country-dict", "legacy-country-route"} and len(model_parts) == 3 and all(re.fullmatch(r"[0-9]+", p) for p in model_parts) and tuple(map(int, model_parts)) <= (1, 764, 6) diff --git a/libs/policyengine-simulation-contract/tests/test_spm_selection.py b/libs/policyengine-simulation-contract/tests/test_spm_selection.py index 452ab51c0..a2b12ae84 100644 --- a/libs/policyengine-simulation-contract/tests/test_spm_selection.py +++ b/libs/policyengine-simulation-contract/tests/test_spm_selection.py @@ -56,12 +56,13 @@ def test_explicit_national_discards_default_metro_area(): "provenance,wrapper,model,accepted", [ ("legacy-country-dict", None, "1.500.0", True), - ("legacy-seed", None, "1.715.2", True), - ("legacy-seed", None, "1.764.6", True), + ("legacy-country-route", None, "1.715.2", True), + ("legacy-country-route", None, "1.764.6", True), (None, None, "1.715.2", False), ("unknown", None, "1.715.2", False), - ("legacy-seed", None, "1.764.7", False), - ("legacy-seed", None, "unknown", False), + ("legacy-seed", None, "1.715.2", False), + ("legacy-country-route", None, "1.764.7", False), + ("legacy-country-route", None, "unknown", False), ("legacy-country-dict", "5.3.1", "1.715.2", False), ], ) @@ -84,3 +85,49 @@ def test_legacy_allowance_requires_route_provenance_and_historical_model( with pytest.raises(SPMInputError) as error: resolve_spm_selection("us", selection, **kwargs) assert error.value.code == "SPM_CONFIGURATION_UNAVAILABLE" + + +@pytest.mark.parametrize("wrapper", ["5.2.0", "5.3.0"]) +@pytest.mark.parametrize("model", [None, "1.764.6"]) +def test_pinned_pre_canonical_wrapper_is_historical_without_a_stated_model( + wrapper, model +): + """A routing entry with no bundle manifest states no model version. + + Requiring the pin's model version there rejected live 5.2.0/5.3.0 + routes, because the response's model version falls back to the wrapper + version when the registry carries no manifest. + """ + assert ( + resolve_spm_selection( + "us", + None, + capability=None, + policyengine_version=wrapper, + model_version=model, + ) + is None + ) + + +@pytest.mark.parametrize( + "wrapper,model", + [ + ("5.2.0", "1.824.7"), + ("5.3.0", "1.824.7"), + ("5.4.0", None), + ("5.4.0", "1.764.6"), + ], +) +def test_unpinned_or_contradicted_wrapper_is_not_historical(wrapper, model): + """A stated model version that contradicts the pin, and any wrapper + outside the pinned pre-canonical set, still fail closed.""" + with pytest.raises(SPMInputError) as error: + resolve_spm_selection( + "us", + None, + capability=None, + policyengine_version=wrapper, + model_version=model, + ) + assert error.value.code == "SPM_CONFIGURATION_UNAVAILABLE" diff --git a/projects/policyengine-simulation-executor/tests/test_canonical_spm.py b/projects/policyengine-simulation-executor/tests/test_canonical_spm.py index 6637d5371..a990b5445 100644 --- a/projects/policyengine-simulation-executor/tests/test_canonical_spm.py +++ b/projects/policyengine-simulation-executor/tests/test_canonical_spm.py @@ -452,6 +452,8 @@ def test_gateway_submission_uses_registry_capability_before_spawn(monkeypatch): app_name="test-app", response_version="test-only", policyengine_version="test-only", + bundle_manifest={}, + country_model_version="test-only", route_provenance=None, ) monkeypatch.setattr(endpoints, "resolve_route", lambda *args: route) @@ -624,7 +626,11 @@ def test_as_of_presence_survives_budget_parent(date_fields, expected_as_of): }, ) selection = _resolve_request_spm( - request, bundle, SimpleNamespace(route_provenance=None) + request, + bundle, + SimpleNamespace( + bundle_manifest={}, country_model_version=None, route_provenance=None + ), ) assert selection["as_of"] == expected_as_of parent = _build_budget_window_parent_payload( diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py index 0e79a64d9..a4f70eae2 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py @@ -83,6 +83,10 @@ class RouteResolution: policyengine_version: str | None bundle_manifest: dict route_provenance: str | None = None + # The country model version this route names, when it names one. A + # country route's key is that version; a policyengine-keyed route's is + # not, and echoing a wrapper version there reads as a model version. + country_model_version: str | None = None def _job_metadata_store(): @@ -440,10 +444,16 @@ def _resolve_country_route( response_version=version, policyengine_version=policyengine_version, bundle_manifest=_bundle_manifest(state, policyengine_version), + country_model_version=version, + # A country route the registry ties to no wrapper bundle at all is + # pre-canonical by construction: every publish writes the wrapper + # route and the bundle manifest for the app it deploys, and the + # legacy seed only infers wrapper routes for prefixed app names. The + # registry's ``generation`` marker cannot carry this — the next + # publish rewrites it (update_version_registry). route_provenance=( - "legacy-seed" - if state.get("generation") == "legacy-seed" - and state.get("schema_version") == 1 + "legacy-country-route" + if policyengine_version is None and state.get("schema_version") == 1 else None ), ) @@ -582,6 +592,7 @@ def _resolve_from_legacy_dicts( response_version=resolved_version, policyengine_version=_policyengine_version_from_app_name(app_name), bundle_manifest={}, + country_model_version=resolved_version, route_provenance="legacy-country-dict", ) @@ -637,13 +648,30 @@ def _build_policyengine_bundle( ) +def _certified_model_version(country: str, route: RouteResolution) -> str | None: + """The country model version the registry states for this route. + + ``PolicyEngineBundle.model_version`` falls back to the routing response + version so the response always carries one. On a policyengine-keyed + route with no manifest that fallback is a wrapper version, and reading + it as a model version is how a legacy 5.2.0/5.3.0 route stopped + resolving. An unstated model version contradicts nothing. + """ + country_bundle = route.bundle_manifest.get(country.lower()) + if isinstance(country_bundle, dict): + stated = country_bundle.get("model_version") + if isinstance(stated, str) and stated.strip(): + return stated + return route.country_model_version + + def _resolve_request_spm(request, bundle, route): selection = resolve_spm_selection( request.country, request.spm, capability=bundle.spm, policyengine_version=bundle.policyengine_version, - model_version=bundle.model_version, + model_version=_certified_model_version(request.country, route), route_provenance=route.route_provenance, ) if selection is not None: diff --git a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py index d94fdbff5..c55aff1bb 100644 --- a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py +++ b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py @@ -20,7 +20,17 @@ ] +# ``generation`` is one global marker that every publish rewrites, so a +# seeded route has to resolve the same before and after a canonical deploy. +PUBLISHED_GENERATION = "5.4.0:policyengine-simulation-py5-4-0" +LEGACY_SOURCES = ["legacy-country-dict", "legacy-seed", PUBLISHED_GENERATION] + + def legacy_route(mock_modal, source, model_version): + """Two shapes of pre-canonical route: the original per-country Modal + dicts, and a seeded routing-state country route the registry ties to no + wrapper bundle. For the latter, ``source`` is the state's ``generation`` + marker, which must not change how the route resolves.""" app_name = "policyengine-simulation-us1-715-2-uk2-88-20" if source == "legacy-country-dict": app_name = "legacy-app" @@ -42,7 +52,7 @@ def legacy_route(mock_modal, source, model_version): @pytest.mark.parametrize("endpoint,extra", ENDPOINTS) -@pytest.mark.parametrize("source", ["legacy-country-dict", "legacy-seed"]) +@pytest.mark.parametrize("source", LEGACY_SOURCES) @pytest.mark.parametrize("version", [None, "1.715.2"]) def test_historical_no_wrapper_route_submits_without_spm( mock_modal, client, endpoint, extra, source, version @@ -58,7 +68,7 @@ def test_historical_no_wrapper_route_submits_without_spm( @pytest.mark.parametrize("endpoint,extra", ENDPOINTS) -@pytest.mark.parametrize("source", ["legacy-country-dict", "legacy-seed"]) +@pytest.mark.parametrize("source", LEGACY_SOURCES) @pytest.mark.parametrize("selection", [{}, {"geography_kind": "national"}]) def test_historical_route_rejects_any_explicit_spm( mock_modal, client, endpoint, extra, source, selection @@ -76,8 +86,9 @@ def test_historical_route_rejects_any_explicit_spm( [ ("legacy-country-dict", "1.824.7"), ("legacy-seed", "1.824.7"), + (PUBLISHED_GENERATION, "1.824.7"), ("legacy-seed", "future-model"), - ("unknown-generation", "1.715.2"), + (PUBLISHED_GENERATION, "future-model"), ], ) def test_missing_wrapper_does_not_certify_unknown_routes( @@ -351,7 +362,7 @@ def test_completed_budget_window_rows_keep_resolved_nulls(mock_modal, client): LEGACY_BUNDLE_KEYS = {"model_version", "policyengine_version", "data_version", "dataset"} -@pytest.mark.parametrize("source", ["legacy-country-dict", "legacy-seed"]) +@pytest.mark.parametrize("source", LEGACY_SOURCES) def test_legacy_poll_bodies_carry_no_spm_key(mock_modal, client, source): """The 202 and 500 bodies splat job metadata in raw, bypassing the routes' ``response_model_exclude_none``. A legacy no-SPM job must stay @@ -374,7 +385,7 @@ def test_legacy_poll_bodies_carry_no_spm_key(mock_modal, client, source): assert set(failed.json()["policyengine_bundle"]) == LEGACY_BUNDLE_KEYS -@pytest.mark.parametrize("source", ["legacy-country-dict", "legacy-seed"]) +@pytest.mark.parametrize("source", LEGACY_SOURCES) def test_legacy_budget_window_metadata_carries_no_spm_key(mock_modal, client, source): """The same bundle rides to the worker in ``_metadata``.""" legacy_route(mock_modal, source, "1.715.2") @@ -402,3 +413,98 @@ def test_canonical_poll_bodies_still_carry_the_capability(mock_modal, client): bundle = running.json()["policyengine_bundle"] assert set(bundle) == LEGACY_BUNDLE_KEYS | {"spm"} assert bundle["spm"]["defaults"]["scenario"] == "ce_trend" + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +def test_publishing_does_not_revoke_a_seeded_country_route( + mock_modal, client, endpoint, extra +): + """The route is unchanged across a publish; only the global marker moved. + + Keying the allowance on ``generation`` meant the first canonical deploy + turned every seeded country route's bare submission into a 400. + """ + for generation in ("legacy-seed", PUBLISHED_GENERATION): + legacy_route(mock_modal, generation, "1.715.2") + response = client.post(endpoint, json={"country": "us", **extra}) + assert response.status_code == 200, f"{generation}: {response.text}" + assert "spm" not in mock_modal["func"].last_payload + + +def wrapper_route_without_manifest(mock_modal, wrapper): + """A seeded wrapper route the app-release snapshot had no bundle for.""" + state = deepcopy(TEST_ROUTING_STATE) + app_name = f"policyengine-simulation-py{wrapper.replace('.', '-')}" + state["routes"]["policyengine"][wrapper] = app_name + mock_modal["dicts"]["simulation-api-routing-state"] = {"active": state} + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +@pytest.mark.parametrize("wrapper", ["5.2.0", "5.3.0"]) +def test_pinned_wrapper_route_without_manifest_submits_without_spm( + mock_modal, client, endpoint, extra, wrapper +): + """With no manifest the response's ``model_version`` echoes the wrapper + version. Reading that back as a country model version rejected the two + pinned pre-canonical bundles.""" + wrapper_route_without_manifest(mock_modal, wrapper) + response = client.post( + endpoint, json={"country": "us", "policyengine_version": wrapper, **extra} + ) + assert response.status_code == 200, response.text + assert "spm" not in mock_modal["func"].last_payload + assert response.json()["policyengine_bundle"]["model_version"] == wrapper + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +@pytest.mark.parametrize("wrapper", ["5.2.0", "5.3.0"]) +def test_pinned_wrapper_through_legacy_dicts_submits_without_spm( + mock_modal, client, endpoint, extra, wrapper +): + """Same route shape through the pre-registry Modal dicts.""" + del mock_modal["dicts"]["simulation-api-routing-state"] + app_name = f"policyengine-simulation-py{wrapper.replace('.', '-')}" + mock_modal["dicts"]["simulation-api-policyengine-versions"] = {wrapper: app_name} + response = client.post( + endpoint, json={"country": "us", "policyengine_version": wrapper, **extra} + ) + assert response.status_code == 200, response.text + assert "spm" not in mock_modal["func"].last_payload + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +def test_unpinned_wrapper_route_without_manifest_still_fails_closed( + mock_modal, client, endpoint, extra +): + """Only the two pinned pre-canonical wrappers are vouched for without a + manifest; an unrecognized bundle proves nothing.""" + wrapper_route_without_manifest(mock_modal, "5.4.0") + response = client.post( + endpoint, json={"country": "us", "policyengine_version": "5.4.0", **extra} + ) + assert response.status_code == 400 + assert response.json()["errors"][0]["code"] == "SPM_CONFIGURATION_UNAVAILABLE" + assert mock_modal["func"].calls == [] + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +def test_stated_model_version_still_contradicts_the_wrapper_pin( + mock_modal, client, endpoint, extra +): + """A manifest that states a post-canonical model under a pinned wrapper + is a republish we cannot vouch for.""" + state = deepcopy(TEST_ROUTING_STATE) + original = state["bundles"]["4.10.0"] + state["routes"]["policyengine"]["5.3.0"] = original["app_name"] + state["bundles"]["5.3.0"] = { + **deepcopy(original), + "policyengine_version": "5.3.0", + "us": {**original["us"], "model_version": "1.824.7"}, + } + mock_modal["dicts"]["simulation-api-routing-state"] = {"active": state} + response = client.post( + endpoint, json={"country": "us", "policyengine_version": "5.3.0", **extra} + ) + assert response.status_code == 400 + assert response.json()["errors"][0]["code"] == "SPM_CONFIGURATION_UNAVAILABLE" + assert mock_modal["func"].calls == [] From 94e980ef1fd1d8904798c78fe3516561ea47b746 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 12:18:52 -0400 Subject: [PATCH 27/40] Cover budget-window scheduler typed and redacted child failures The scheduler's four child-failure branches were unexercised: a typed SPMInputError from the child call, a typed receipt-validation failure during result parsing, and the redaction fallback on each. The semi integration runtime now takes per-year injection seams so a child can raise or return an arbitrary result, and the new cases assert that a typed failure reaches the poll body as a 400 with the code re-attached to the replaced child entry, that the batch returns early with the remaining years still queued, and that an untyped failure stays a 500 carrying no code and no internal detail. Co-Authored-By: Claude Fable 5.1 --- .../tests/test_budget_window_scheduler.py | 175 +++++++++++++++++- 1 file changed, 173 insertions(+), 2 deletions(-) diff --git a/projects/policyengine-simulation-executor/tests/test_budget_window_scheduler.py b/projects/policyengine-simulation-executor/tests/test_budget_window_scheduler.py index 475c80049..64bd0318f 100644 --- a/projects/policyengine-simulation-executor/tests/test_budget_window_scheduler.py +++ b/projects/policyengine-simulation-executor/tests/test_budget_window_scheduler.py @@ -17,9 +17,42 @@ import src.modal.budget_window_batch as batch_module import src.modal.budget_window_scheduler as scheduler_module import policyengine_simulation_contract.budget_window_state as state_module +from policyengine_simulation_contract.budget_window_state import ( + BUDGET_WINDOW_JOB_SEED_DICT_NAME, +) +from policyengine_simulation_contract.spm import SPMInputError, SPMSelection from policyengine_simulation_gateway.testing import create_gateway_app from policyengine_simulation_gateway import endpoints +SPM_SELECTION = SPMSelection( + forecast_content_sha256="a" * 64, + scenario="ce_trend", + geography_kind="national", + geography_id=None, + county_vintage="2020", + as_of=None, +).model_dump(mode="json") + + +def spm_child_result(runtime, simulation_year, *, receipt_year): + """A child result carrying a canonical SPM receipt for ``receipt_year``.""" + receipt = { + "forecast_id": "test-only", + "forecast_sha256": SPM_SELECTION["forecast_content_sha256"], + "scenario": SPM_SELECTION["scenario"], + "geography_kind": SPM_SELECTION["geography_kind"], + "runtime_versions": {"policyengine-us": "test-only"}, + "years": {receipt_year: {"status": "forecast"}}, + "geographies": [], + "composition_method": "classified-inputs", + "storage_method": "formula", + } + return { + "spm_config": dict(SPM_SELECTION), + "spm_provenance": {"baseline": [receipt], "reform": [dict(receipt)]}, + **runtime.child_result_for_year(simulation_year), + } + @dataclass class SemiIntegrationRuntime: @@ -30,6 +63,15 @@ class SemiIntegrationRuntime: next_parent_call_id: str = "parent-batch-123" active_child_calls: set[str] = field(default_factory=set) max_active_child_calls: int = 0 + # Per-year injection seams: a child call raises ``child_errors[year]`` + # instead of returning, or returns ``child_results[year]`` verbatim. + child_errors: dict[str, BaseException] = field(default_factory=dict) + child_results: dict[str, dict] = field(default_factory=dict) + + def child_outcome_for_year(self, simulation_year: str) -> dict: + return self.child_results.get( + simulation_year, self.child_result_for_year(simulation_year) + ) def child_result_for_year(self, simulation_year: str) -> dict: offset = int(simulation_year) - 2025 @@ -69,15 +111,23 @@ def get(self, key: str, default=None): class MockChildCall: def __init__( - self, runtime: SemiIntegrationRuntime, *, object_id: str, result: dict + self, + runtime: SemiIntegrationRuntime, + *, + object_id: str, + result: dict, + error: BaseException | None = None, ): self.runtime = runtime self.object_id = object_id self.result = result + self.error = error self.runtime.child_started(object_id) def get(self, timeout: int = 0): self.runtime.child_finished(self.object_id) + if self.error is not None: + raise self.error return self.result @@ -124,7 +174,8 @@ def spawn(self, payload: dict): call = MockChildCall( self.runtime, object_id=f"child-{simulation_year}", - result=self.runtime.child_result_for_year(simulation_year), + result=self.runtime.child_outcome_for_year(simulation_year), + error=self.runtime.child_errors.get(simulation_year), ) self.runtime.calls[call.object_id] = call return call @@ -246,3 +297,123 @@ def test_budget_window_submit_and_poll_exercise_gateway_worker_seams( assert all("window_size" not in payload for payload in runtime.child_payloads) assert all("max_parallel" not in payload for payload in runtime.child_payloads) assert all("_metadata" not in payload for payload in runtime.child_payloads) + + +def submit_budget_window(client, *, window_size=3, max_parallel=1): + response = client.post( + "/simulate/economy/budget-window", + json={ + "country": "us", + "region": "us", + "scope": "macro", + "reform": {}, + "start_year": "2026", + "window_size": window_size, + "max_parallel": max_parallel, + }, + ) + assert response.status_code == 200, response.text + return response.json()["batch_job_id"] + + +def test_typed_child_call_failure_persists_errors_and_stops_the_batch( + budget_window_semi_integration_client, +): + """A child that raises ``SPMInputError`` is transported, not redacted.""" + client, runtime = budget_window_semi_integration_client + runtime.child_errors["2026"] = SPMInputError( + "SPM_GEOGRAPHY_REQUIRED", "County required" + ) + + batch_job_id = submit_budget_window(client) + assert client.get(f"/budget-window-jobs/{batch_job_id}").status_code == 202 + + failure = client.get(f"/budget-window-jobs/{batch_job_id}") + assert failure.status_code == 400 + body = failure.json() + expected = [{"code": "SPM_GEOGRAPHY_REQUIRED", "message": "County required"}] + assert body["status"] == "failed" + assert body["errors"] == expected + assert body["error"] == "County required" + # ``mark_child_failed`` replaces the child entry, so the scheduler has to + # re-attach the typed payload after it. + assert body["child_jobs"]["2026"]["errors"] == expected + assert body["child_jobs"]["2026"]["status"] == "failed" + assert body["failed_years"] == ["2026"] + # The scheduler returns early: the remaining years are never spawned. + assert body["queued_years"] == ["2027", "2028"] + assert [payload["time_period"] for payload in runtime.child_payloads] == ["2026"] + + +def test_typed_result_validation_failure_persists_errors_and_stops_the_batch( + budget_window_semi_integration_client, +): + """A mismatched SPM receipt fails the batch with the typed code.""" + client, runtime = budget_window_semi_integration_client + batch_job_id = submit_budget_window(client) + + # The gateway records the resolved selection on the seed whenever the + # route advertises a canonical SPM capability; the parent reads it back + # from the seed to validate each child receipt. Seeding it here keeps the + # test on the scheduler seam instead of the registry's. + seed = runtime.dicts[BUDGET_WINDOW_JOB_SEED_DICT_NAME][batch_job_id] + seed["request_payload"]["spm"] = dict(SPM_SELECTION) + runtime.child_results["2026"] = spm_child_result( + runtime, "2026", receipt_year="1999" + ) + + assert client.get(f"/budget-window-jobs/{batch_job_id}").status_code == 202 + + failure = client.get(f"/budget-window-jobs/{batch_job_id}") + assert failure.status_code == 400 + body = failure.json() + expected = [ + { + "code": "SPM_CONFIGURATION_UNAVAILABLE", + "message": "Result SPM provenance does not cover the requested year", + } + ] + assert body["status"] == "failed" + assert body["errors"] == expected + assert body["child_jobs"]["2026"]["errors"] == expected + assert body["failed_years"] == ["2026"] + assert body["queued_years"] == ["2027", "2028"] + assert [payload["time_period"] for payload in runtime.child_payloads] == ["2026"] + + +def test_untyped_child_failure_is_redacted_and_carries_no_typed_errors( + budget_window_semi_integration_client, +): + """Only public typed errors escape redaction; others stay 500 with no code.""" + client, runtime = budget_window_semi_integration_client + runtime.child_errors["2026"] = RuntimeError("internal detail") + + batch_job_id = submit_budget_window(client) + assert client.get(f"/budget-window-jobs/{batch_job_id}").status_code == 202 + + failure = client.get(f"/budget-window-jobs/{batch_job_id}") + assert failure.status_code == 500 + body = failure.json() + assert "errors" not in body + assert "internal detail" not in body["error"] + assert "errors" not in body["child_jobs"]["2026"] + + +def test_untyped_result_parsing_failure_is_redacted_and_stops_the_batch( + budget_window_semi_integration_client, +): + """A malformed child result fails the batch without leaking the reason.""" + client, runtime = budget_window_semi_integration_client + runtime.child_results["2026"] = {"budget": "not-an-object"} + + batch_job_id = submit_budget_window(client) + assert client.get(f"/budget-window-jobs/{batch_job_id}").status_code == 202 + + failure = client.get(f"/budget-window-jobs/{batch_job_id}") + assert failure.status_code == 500 + body = failure.json() + assert "errors" not in body + assert "missing budget object" not in body["error"] + assert body["failed_years"] == ["2026"] + assert body["queued_years"] == ["2027", "2028"] + assert [payload["time_period"] for payload in runtime.child_payloads] == ["2026"] From bf215fbdc8d9992681ed1fec2c1d691d0fdf0e51 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 12:19:01 -0400 Subject: [PATCH 28/40] Record medium 2-4 and low 1-3 progress Co-Authored-By: Claude Fable 5.1 --- PROGRESS.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index b06b8d894..5ba311c0e 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -13,10 +13,10 @@ Worktree created at PR head. Investigating the nine review findings. - [x] Medium 1 — completed results drop explicit nulls from `spm_config` on the wire - [x] Medium 2 — legacy no-SPM provenance `legacy-seed` expires on next publish - [ ] Medium 3 — precompute storage identity never shown to match the wrapper -- [ ] Medium 4 — budget-window scheduler typed-error branches untested +- [x] Medium 4 — budget-window scheduler typed-error branches untested - [x] Low 1 — legacy no-SPM poll bodies not byte-identical to base - [ ] Low 2 — stale country routes now fail; undocumented, untested -- [ ] Low 3 — segmented reduce with SPM children untested end to end +- [x] Low 3 — segmented reduce with SPM children untested end to end - [ ] Low 4 — hermetic CI never exercises receipt validation in `ensure()` - [ ] Low 5 — repeated prevalidation per request (perf) @@ -39,6 +39,20 @@ Worktree created at PR head. Investigating the nine review findings. `test_canonical_poll_bodies_still_carry_the_capability` — the first two verified failing against the pre-fix endpoints. +- **Low 3**: `tests/test_segmented_national.py` now drives a segmented + national reduce with SPM children end to end. +- **Medium 2**: route provenance is now derived from route shape + (`policyengine_version is None and schema_version == 1` -> + `legacy-country-route`) instead of the registry's rewritable `generation` + marker, and `_certified_model_version` stops reading a wrapper version as + a country model version, so a pinned 5.2.0/5.3.0 route with no manifest + stays historical. Tests in `test_spm_selection.py` and `test_spm_routes.py`. +- **Medium 4**: `test_budget_window_scheduler.py` gained per-year child + injection seams and four cases covering the typed child-call failure, the + typed receipt-validation failure, and the redaction fallback on each. + Verified by mutation: deleting the child-entry re-attach fails both typed + cases. + ## Next -- Medium 2 (durable legacy-route provenance), Medium 3, Medium 4, Low 2-5. +- Medium 3, Low 2, Low 4, Low 5. From bf6764ceb5f119680a2073d0bce92feebdebe9e1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 12:23:29 -0400 Subject: [PATCH 29/40] Pin the null-preserving serializer at the contract layer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The explicit-null fix was covered only by the gateway's poll-body tests: reverting the serializer left the contract suite green. These cases hold the serializer itself, including the nested-model path the response models take and the two behaviours the fix must not disturb — an omitted option still disappears under exclude_none, and ordinary serialization is untouched. Co-Authored-By: Claude Fable 5.1 --- .../tests/test_spm_selection.py | 71 ++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/libs/policyengine-simulation-contract/tests/test_spm_selection.py b/libs/policyengine-simulation-contract/tests/test_spm_selection.py index a2b12ae84..0993b617f 100644 --- a/libs/policyengine-simulation-contract/tests/test_spm_selection.py +++ b/libs/policyengine-simulation-contract/tests/test_spm_selection.py @@ -3,7 +3,11 @@ import pytest from policyengine_simulation_contract.gateway_models import SimulationRequest -from policyengine_simulation_contract.spm import SPMInputError, resolve_spm_selection +from policyengine_simulation_contract.spm import ( + SPMInputError, + SPMSelection, + resolve_spm_selection, +) def resolve(selection, *, kind="metro", area="35620"): @@ -131,3 +135,68 @@ def test_unpinned_or_contradicted_wrapper_is_not_historical(wrapper, model): model_version=model, ) assert error.value.code == "SPM_CONFIGURATION_UNAVAILABLE" + + +class TestSerializerPreservesExplicitNulls: + """``exclude_none`` must not turn a resolved selection back into a partial. + + Presence is the contract: an omitted option inherits the bundle default, + so an option that was explicitly selected as null is a different request + from one that was omitted. The poll routes serialise every body with + ``response_model_exclude_none``, so the serializer is the only thing + standing between a completed result and a body a client cannot replay. + """ + + resolved = SPMSelection( + forecast_content_sha256="a" * 64, + scenario="ce_trend", + geography_kind="national", + geography_id=None, + county_vintage="2020", + as_of=None, + ) + + def test_exclude_none_keeps_explicitly_selected_nulls(self): + assert self.resolved.model_dump(mode="json", exclude_none=True) == { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + } + + def test_exclude_none_still_drops_options_the_caller_omitted(self): + partial = SPMSelection(geography_kind="national") + assert partial.model_dump(mode="json", exclude_none=True) == { + "geography_kind": "national" + } + + def test_exclude_none_survives_nesting_in_a_response_model(self): + nested = SimulationRequest(country="us", spm=self.resolved) + assert nested.model_dump(mode="json", exclude_none=True)["spm"] == { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + } + + def test_explicit_exclude_still_removes_a_null_option(self): + assert "as_of" not in self.resolved.model_dump( + mode="json", exclude_none=True, exclude={"as_of"} + ) + + def test_ordinary_serialization_is_unchanged(self): + assert self.resolved.model_dump(mode="json") == { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + } + assert SPMSelection(geography_kind="national").model_dump(mode="json") == { + "geography_kind": "national" + } From 638b7c6304cccfb0fb39b7d978192841c08a5e89 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 12:33:11 -0400 Subject: [PATCH 30/40] Exercise the SPM receipt guard in hermetic CI The receipt validation in ensure() was reachable only from the opt-in native tests, because the pinned pre-canonical policyengine has neither an spm field nor spm_provenance, so the guard read no selection and returned early. A small test double supplies the canonical wrapper's surface and its restore timing -- a load or cache hit overwrites the requested selection with the artifact's -- letting fixture-driven cases fix every decision the guard makes: a matching receipt loads as a hit, and a receipt from another selection, another forecast or another year, a missing or malformed one, and a stale cached selection each recompute under the request's own settings. Columns are still decided first, and a request with no selection never consults a receipt at all. The double stands in for the wrapper's shape, not for the wrapper: that the deployed bundle really emits receipts of this shape stays the native tests' claim. Co-Authored-By: Claude Fable 5.1 --- .../tests/test_baseline_artifacts.py | 266 ++++++++++++++++++ 1 file changed, 266 insertions(+) diff --git a/projects/policyengine-simulation-executor/tests/test_baseline_artifacts.py b/projects/policyengine-simulation-executor/tests/test_baseline_artifacts.py index 4520ea18f..7e2af9e1b 100644 --- a/projects/policyengine-simulation-executor/tests/test_baseline_artifacts.py +++ b/projects/policyengine-simulation-executor/tests/test_baseline_artifacts.py @@ -7,11 +7,15 @@ simulation cache is swapped per test. """ +from copy import deepcopy from types import SimpleNamespace import pandas as pd import pytest +from policyengine.core import Simulation +from pydantic import PrivateAttr + from policyengine_simulation_executor import artifact_keys as ak from policyengine_simulation_executor import baseline_artifacts as ba @@ -584,3 +588,265 @@ def test_incomplete_disk_artifact_recomputes_and_overwrites( year=2026, ) assert "employment_income" in reloaded.data.person.columns + + +SPM_SELECTION = { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} + + +def _spm_receipt(selection=None, *, year="2026"): + """One calculation receipt, the shape ``spm_provenance()`` returns. + + ``simulation_spm_result`` reads one receipt per simulation and pairs the + baseline's with the reform's itself, so the wrapper hands back a flat + receipt rather than a baseline/reform comparison. + """ + selection = selection or SPM_SELECTION + return { + "forecast_id": "test-only", + "forecast_sha256": selection["forecast_content_sha256"], + "scenario": selection["scenario"], + "geography_kind": selection["geography_kind"], + "runtime_versions": {"policyengine-us": "test-only"}, + "years": {year: {"status": "forecast"}}, + "geographies": [], + "composition_method": "classified-inputs", + "storage_method": "formula", + } + + +class SPMWrapperSimulation(Simulation): + """Test double for the canonical wrapper's SPM surface. + + The wrapper the canonical bundle installs adds an ``spm`` field, an + ``spm_config`` holding the selection an artifact was built under, and an + ``spm_provenance()`` calculation receipt; it restores that metadata on a + load or a cache hit. The ``policyengine`` this project pins has none of + it, which is why ``ensure()``'s receipt validation was reachable only + from the SPM_NATIVE_SMOKE_SOURCE-gated tests. + + This double reproduces that surface's *shape and restore timing*, which + is all the guard depends on. It does not prove the deployed wrapper + emits receipts of this shape — only the native tests do. + """ + + spm: dict | None = None + spm_config: dict | None = None + spm_receipt: dict | None = None + _provenance_reads: list = PrivateAttr(default_factory=list) + + def spm_provenance(self): + self._provenance_reads.append(deepcopy(self.spm_config)) + return self.spm_receipt + + def ensure(self): + from policyengine.core.simulation import _cache + + cached = _cache.get(self.id) + if cached is not None: + self.output_dataset = cached.output_dataset + # The restore the guard exists to survive: a cached entry's + # selection lands on this request's simulation, replacing the + # one the request asked for. + self.spm = deepcopy(getattr(cached, "spm", None)) + self.spm_config = deepcopy(getattr(cached, "spm_config", None)) + self.spm_receipt = deepcopy(getattr(cached, "spm_receipt", None)) + return + try: + self.tax_benefit_model_version.load(self) + except Exception: + self.run() + self.save() + _cache.add(self.id, self) + + +class CanonicalSPMSimulation(ba.ArtifactBaselineSimulation, SPMWrapperSimulation): + """``ArtifactBaselineSimulation`` over a wrapper that supports SPM.""" + + +class SPMModelVersion(FakeModelVersion): + """Restores an artifact's stored SPM metadata on load, as the wrapper does.""" + + def __init__( + self, *, load_result="complete", stored_config=None, stored_receipt=None + ): + super().__init__(load_result=load_result) + self.stored_config = stored_config + self.stored_receipt = stored_receipt + self.spm_at_run = [] + + def load(self, simulation): + super().load(simulation) + # An artifact carries the selection it was built under, and the + # wrapper restores it over the requested one. + simulation.spm = deepcopy(self.stored_config) + simulation.spm_config = deepcopy(self.stored_config) + simulation.spm_receipt = deepcopy(self.stored_receipt) + + def run(self, simulation): + super().run(simulation) + # A real recompute measures SPM under whatever ``spm`` now holds. + self.spm_at_run.append(deepcopy(simulation.spm)) + simulation.spm_config = deepcopy(simulation.spm) + simulation.spm_receipt = _spm_receipt(simulation.spm) + + +def _make_spm_sim(model_version, *, spm=None, sim_id="bl1-spm", year=2026): + return CanonicalSPMSimulation.model_construct( + id=sim_id, + dataset=SimpleNamespace(year=year), + tax_benefit_model_version=model_version, + policy=None, + dynamic=None, + scoping_strategy=None, + extra_variables={}, + output_dataset=None, + spm=deepcopy(SPM_SELECTION) if spm is None else deepcopy(spm), + # The wrapper carries the selection it was configured with from + # construction; a load or cache hit then overwrites it with whatever + # the artifact was built under, which is exactly what the guard in + # ``ensure()`` compares against the pre-load value. + spm_config=deepcopy(SPM_SELECTION) if spm is None else deepcopy(spm), + spm_receipt=None, + ) + + +class TestEnsureValidatesSPMReceipts: + """``ensure()``'s receipt validation, driven by fixtures rather than a model. + + Only the opt-in native tests reached this block before, so hermetic CI + could not tell a working guard from a dead one. These cases fix the + guard's *decisions*; the native tests remain the only evidence that real + wrapper receipts have the shape the decisions are made on. + """ + + def test_matching_receipt_loads_as_a_hit(self, fresh_cache): + model = SPMModelVersion( + stored_config=deepcopy(SPM_SELECTION), stored_receipt=_spm_receipt() + ) + sim = _make_spm_sim(model) + sim.ensure() + assert sim.artifact_outcome == ba.OUTCOME_HIT + assert model.calls == ["load"] + + def test_artifact_built_under_another_selection_recomputes(self, fresh_cache): + other = {**SPM_SELECTION, "scenario": "zero_real"} + model = SPMModelVersion(stored_config=other, stored_receipt=_spm_receipt(other)) + sim = _make_spm_sim(model) + sim.ensure() + assert sim.artifact_outcome == ba.OUTCOME_INCOMPLETE + assert model.calls == ["load", "run", "save"] + # The requested selection is restored before the recompute, so the + # new artifact is measured under the request, not the artifact. + assert model.spm_at_run == [SPM_SELECTION] + assert sim.spm_config == SPM_SELECTION + + def test_artifact_without_a_receipt_recomputes(self, fresh_cache): + model = SPMModelVersion( + stored_config=deepcopy(SPM_SELECTION), stored_receipt=None + ) + sim = _make_spm_sim(model) + sim.ensure() + assert sim.artifact_outcome == ba.OUTCOME_INCOMPLETE + assert model.calls == ["load", "run", "save"] + + def test_malformed_receipt_recomputes(self, fresh_cache): + malformed = _spm_receipt() + del malformed["composition_method"] + model = SPMModelVersion( + stored_config=deepcopy(SPM_SELECTION), stored_receipt=malformed + ) + sim = _make_spm_sim(model) + sim.ensure() + assert sim.artifact_outcome == ba.OUTCOME_INCOMPLETE + assert model.calls == ["load", "run", "save"] + + def test_receipt_from_another_forecast_recomputes(self, fresh_cache): + stale = _spm_receipt() + stale["forecast_sha256"] = "b" * 64 + model = SPMModelVersion( + stored_config=deepcopy(SPM_SELECTION), stored_receipt=stale + ) + sim = _make_spm_sim(model) + sim.ensure() + assert sim.artifact_outcome == ba.OUTCOME_INCOMPLETE + assert model.calls == ["load", "run", "save"] + + def test_receipt_for_another_year_recomputes(self, fresh_cache): + model = SPMModelVersion( + stored_config=deepcopy(SPM_SELECTION), + stored_receipt=_spm_receipt(year="2024"), + ) + sim = _make_spm_sim(model) + sim.ensure() + assert sim.artifact_outcome == ba.OUTCOME_INCOMPLETE + assert model.calls == ["load", "run", "save"] + + def test_stale_cached_selection_cannot_replace_the_request(self, fresh_cache): + """A cache hit restores the cached entry's selection onto this + request. The guard validates against the pre-load selection, so the + stale entry can only cost a recompute — never silently answer the + request under the wrong SPM settings.""" + other = {**SPM_SELECTION, "scenario": "zero_real"} + stale = _make_spm_sim(SPMModelVersion(), spm=other) + stale.output_dataset = _output(_complete_frames()) + stale.spm_config = other + stale.spm_receipt = _spm_receipt(other) + fresh_cache.add("bl1-spm", stale) + + model = SPMModelVersion() + sim = _make_spm_sim(model) + sim.ensure() + assert sim.artifact_outcome == ba.OUTCOME_INCOMPLETE + assert model.calls == ["run", "save"] + assert model.spm_at_run == [SPM_SELECTION] + assert sim.spm_config == SPM_SELECTION + + def test_recompute_replaces_the_cache_with_this_request_s_selection( + self, fresh_cache + ): + other = {**SPM_SELECTION, "scenario": "zero_real"} + model = SPMModelVersion(stored_config=other, stored_receipt=_spm_receipt(other)) + sim = _make_spm_sim(model) + sim.ensure() + assert fresh_cache.get("bl1-spm").spm_config == SPM_SELECTION + + # The next request in this container hits the replaced entry and + # validates clean: one recompute for the container, not one each. + later_model = SPMModelVersion(stored_config=other) + later = _make_spm_sim(later_model) + later.ensure() + assert later.artifact_outcome == ba.OUTCOME_HIT + assert later_model.calls == [] + + def test_missing_columns_are_decided_before_the_receipt(self, fresh_cache): + """A tax-only artifact can legitimately carry no usable receipt. + Columns are checked first so the outcome is attributed to the gap + that actually exists, and the receipt is never consulted.""" + model = SPMModelVersion( + load_result="incomplete", + stored_config=deepcopy(SPM_SELECTION), + stored_receipt=None, + ) + sim = _make_spm_sim(model) + sim.ensure() + assert sim.artifact_outcome == ba.OUTCOME_INCOMPLETE + assert sim._provenance_reads == [] + + def test_a_request_without_a_selection_never_validates_receipts(self, fresh_cache): + """The legacy path: no selection, so a complete artifact is a hit + even though the wrapper surface exists and holds no receipt.""" + model = SPMModelVersion(stored_config=None, stored_receipt=None) + sim = _make_spm_sim(model, sim_id="bl1-legacy") + sim.spm = None + sim.spm_config = None + sim.ensure() + assert sim.artifact_outcome == ba.OUTCOME_HIT + assert model.calls == ["load"] + assert sim._provenance_reads == [] From ae9489e6766139547216013c9ac006eaa3222a6a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 12:37:49 -0400 Subject: [PATCH 31/40] Hold the storage id against the wrapper that names the file Precompute plans a store path from BaselineArtifactIdentity.storage_id and the worker aborts when the wrapper's own Simulation.storage_id disagrees, so the two derivations have to be one identifier reached down two paths -- but nothing showed they agreed: the identity was compared to itself, the native case asserted only that two selections differ, and the remaining case covered the abort. The pinned policyengine is pre-canonical and exposes no storage_id, so hermetic CI cannot import an SPM-capable wrapper. It now asserts what it can: the no-selection arm against the real installed Simulation through the exact expression precompute uses, and the SPM arm against the canonical wrapper's expression transcribed from the wheel the native lane installs -- including a non-ASCII scenario, the one input where our explicit ensure_ascii could diverge from the wrapper's json defaults. A golden freezes the string, every selection field is shown to rotate it, and a skipped case starts asserting real equality by itself as soon as an SPM-capable wrapper is pinned. The native smoke now makes the same claim against the real wrapper rather than only asserting inequality. Reformatting the digest, truncating it, or flipping ensure_ascii each fail these tests; before them all three passed. Co-Authored-By: Claude Fable 5.1 --- .../tests/test_artifact_keys.py | 170 ++++++++++++++++++ .../tests/test_canonical_spm_native.py | 18 +- 2 files changed, 187 insertions(+), 1 deletion(-) diff --git a/projects/policyengine-simulation-executor/tests/test_artifact_keys.py b/projects/policyengine-simulation-executor/tests/test_artifact_keys.py index 955b91413..ea8318791 100644 --- a/projects/policyengine-simulation-executor/tests/test_artifact_keys.py +++ b/projects/policyengine-simulation-executor/tests/test_artifact_keys.py @@ -175,3 +175,173 @@ def test_baseline_identity_composes(self, stub_identity_sources): assert identity.store_path == ( f"baselines/us/{_BASELINE_GOLDEN}/bl1-f9cac05d94509895.h5" ) + + +_SPM_SELECTION = { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, +} +_SPM_STORAGE_GOLDEN = ( + "bl1-21f52b30719e20bb-spm-" + "7396bf5f4876c42bb6cba0f9533478098edc0861cc3657bd0d60f88dbb26ac39" +) + + +def _installed_wrapper_has_storage_id() -> bool: + from policyengine.core import Simulation + + return hasattr(Simulation, "storage_id") + + +_INSTALLED_WRAPPER_HAS_STORAGE_ID = _installed_wrapper_has_storage_id() + + +def _wrapper_storage_id(simulation_id: str, spm_config: dict | None) -> str: + """The canonical wrapper's own ``storage_id``, transcribed verbatim. + + ``policyengine/core/simulation.py`` in the SPM-capable wrapper:: + + @property + def storage_id(self) -> str: + config = self.spm_config + if config is None: + return self.id + encoded = json.dumps( + config, sort_keys=True, separators=(",", ":") + ).encode() + return f"{self.id}-spm-{hashlib.sha256(encoded).hexdigest()}" + + Read from ``policyengine-5.3.0-py3-none-any.whl`` sha256 + ``8c640d96…2735f1``, the build the native qualification lane installs. + """ + import hashlib + import json + + if spm_config is None: + return simulation_id + encoded = json.dumps(spm_config, sort_keys=True, separators=(",", ":")).encode() + return f"{simulation_id}-spm-{hashlib.sha256(encoded).hexdigest()}" + + +class TestWrapperStorageIdAgreement: + """The planner's storage id against the wrapper that names the file. + + Precompute plans a store path from ``BaselineArtifactIdentity.storage_id`` + and the in-container worker refuses to publish when the wrapper's own + ``Simulation.storage_id`` disagrees. The wrapper's value also names the + saved ``.h5``, so the two derivations are not merely compared — they are + the same identifier reached down two independent code paths, and a + disagreement blocks every canonical publish. + + The ``policyengine`` this project pins is pre-canonical: it has neither + an ``spm`` field nor a ``storage_id``, so an SPM-capable wrapper cannot + be imported in hermetic CI and these tests cannot prove agreement with + one. What they do prove: + + * the no-selection arm agrees with the **installed** wrapper, through the + exact expression ``precompute`` uses — that is the configuration + deployed today, asserted against the real object; + * the SPM arm agrees with the canonical wrapper's expression as read from + the wheel above, so our side cannot drift from the contract without a + reviewable diff, and the digest cannot be quietly reformatted; + * ``test_spm_arm_matches_the_installed_wrapper`` stops skipping and + starts asserting real equality on its own the moment an SPM-capable + wrapper is pinned. + + Agreement with the real 5.3.0 wrapper was checked out of band on + 2026-09-11 by resolving seven selection shapes through both that wheel's + ``resolve_spm_selection`` and this repo's, and comparing both the + resolved configs and the resulting storage ids: all seven agreed. That + is a recorded observation, not coverage — only the native lane re-runs + anything like it. + """ + + @pytest.fixture + def identity(self, stub_identity_sources): + """Build the identity directly from an already-resolved selection. + + ``collect_baseline_identity`` resolves the selection through the + installed bundle first; that resolution is the contract suite's + subject. What is under test here is only what the identity then + does with the resolved dict. + """ + + def _identity(spm=None): + return ak.BaselineArtifactIdentity( + spm=spm, + country="us", + region="national", + scope_key=_BASELINE_KWARGS["scope_key"], + dataset=ak.collect_dataset_identity("us", 2026), + ) + + return _identity + + @pytest.mark.parametrize( + "selection", + [ + None, + _SPM_SELECTION, + {**_SPM_SELECTION, "geography_kind": "county"}, + {**_SPM_SELECTION, "geography_kind": "metro", "geography_id": "35620"}, + {**_SPM_SELECTION, "as_of": "2025-01-01"}, + # Non-ASCII is the one input where our explicit ensure_ascii=True + # could diverge from the wrapper's json.dumps defaults. + {**_SPM_SELECTION, "scenario": "ce_trend_ü"}, + ], + ) + def test_storage_id_matches_the_wrapper_expression(self, identity, selection): + built = identity(selection) + assert built.storage_id == _wrapper_storage_id(built.simulation_id, selection) + + def test_spm_storage_id_golden(self, identity): + """Freeze the string. Changing it rotates every canonical artifact.""" + built = identity(_SPM_SELECTION) + assert built.storage_id == _SPM_STORAGE_GOLDEN + assert built.store_path.endswith(f"/{_SPM_STORAGE_GOLDEN}.h5") + + @pytest.mark.parametrize("field", sorted(_SPM_SELECTION)) + def test_every_selection_field_rotates_the_storage_id(self, identity, field): + perturbed = {**_SPM_SELECTION, field: "other"} + assert identity(perturbed).storage_id != _SPM_STORAGE_GOLDEN + + def test_storage_id_ignores_selection_key_order(self, identity): + reversed_selection = dict(reversed(list(_SPM_SELECTION.items()))) + assert identity(reversed_selection).storage_id == _SPM_STORAGE_GOLDEN + + def test_no_selection_keeps_the_plain_simulation_id(self, identity): + built = identity(None) + assert built.storage_id == built.simulation_id + assert built.store_path.endswith(f"/{built.simulation_id}.h5") + + def test_legacy_arm_matches_the_installed_wrapper(self, identity): + """The deployed-today arm, against the real ``Simulation`` object. + + ``precompute`` reads ``getattr(baseline, "storage_id", baseline.id)``; + with no selection that has to be the planned id on any wrapper, + canonical or not. + """ + from policyengine.core import Simulation + + built = identity(None) + wrapper = Simulation.model_construct(id=built.simulation_id) + assert getattr(wrapper, "storage_id", wrapper.id) == built.storage_id + + @pytest.mark.skipif( + not _INSTALLED_WRAPPER_HAS_STORAGE_ID, + reason=( + "The pinned policyengine is pre-canonical and has no storage_id; " + "this asserts real equality as soon as an SPM-capable wrapper is " + "pinned, replacing the transcribed expression above." + ), + ) + def test_spm_arm_matches_the_installed_wrapper(self, identity): + from policyengine.core import Simulation + + built = identity(_SPM_SELECTION) + wrapper = Simulation.model_construct(id=built.simulation_id, spm=_SPM_SELECTION) + assert wrapper.storage_id == built.storage_id diff --git a/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py b/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py index e6d4987da..2d03c1d9d 100644 --- a/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py +++ b/projects/policyengine-simulation-executor/tests/test_canonical_spm_native.py @@ -68,8 +68,9 @@ def test_worker_baseline_reform_national_local_cache_and_receipts( replay.ensure() assert replay.spm_provenance() == baseline.spm_provenance() # A shared caller id cannot reuse national output for county selection. + local_params = {**params, "spm": {"geography_kind": "county"}} local = _build_simulation( - {**params, "spm": {"geography_kind": "county"}}, + local_params, dataset=native_dataset, policy=None, region_code="us", @@ -79,6 +80,21 @@ def test_worker_baseline_reform_national_local_cache_and_receipts( local.ensure() assert local.spm_provenance()["geography_kind"] == "county" + # The precompute guard compares the planner's storage id against this + # wrapper property and refuses to publish on a mismatch, so agreement is + # what makes any canonical artifact publishable. Hermetic CI can only + # check our side against the wrapper's expression transcribed into + # test_artifact_keys; this is the same claim against the real wrapper. + from policyengine_simulation_executor.artifact_keys import canonical_digest + from policyengine_simulation_executor.spm import normalize_runtime_spm + + for simulation, request in ((baseline, params), (local, local_params)): + resolved = normalize_runtime_spm(request) + assert simulation.spm_config == resolved + assert simulation.storage_id == ( + f"{simulation.id}-spm-{canonical_digest(resolved)}" + ) + def test_native_state_only_worker_requires_geography_and_explicit_national_works( native_dataset, From 215d6facf2bbb3e7fd4b40b61c499e4b2cba784a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 12:40:37 -0400 Subject: [PATCH 32/40] Make an unstated bundle model version mean one thing The single-candidate country route validates the requested version against the serving bundle's manifest, which is right -- publishing never prunes country routes, so a re-published wrapper leaves the old country version pointing at an app that no longer serves it, and base returned a 202 whose version and model_version contradicted each other. But the check derived "states a model version" twice and differently: the ambiguity classifier treats a blank or whitespace value as unstated, while the shared validator reads any string as stated, so a blank manifest entry produced a 400 with a message that named no version at all -- and the same value is already treated as missing metadata by the ambiguity test. The rejection now reuses the classification already computed above. The behaviour change was also undocumented and untested. The resolver carries why it refuses, the gateway README tells callers pinning an old version what to do, and the new cases pin the refusal, the accepted route it is distinguished from, and all four ways a manifest can state nothing. Co-Authored-By: Claude Fable 5.1 --- .../policyengine-simulation-gateway/README.md | 24 ++++++ .../endpoints.py | 35 +++++++-- .../tests/test_spm_routes.py | 77 +++++++++++++++++++ 3 files changed, 131 insertions(+), 5 deletions(-) diff --git a/projects/policyengine-simulation-gateway/README.md b/projects/policyengine-simulation-gateway/README.md index 8372da0dc..619f6c6c4 100644 --- a/projects/policyengine-simulation-gateway/README.md +++ b/projects/policyengine-simulation-gateway/README.md @@ -4,6 +4,30 @@ The stable Modal gateway for the simulation service: routes simulation requests to versioned executor apps (`policyengine-simulation-py{X}`) via the routing state in `modal.Dict`, and serves the public API contract. +## Country-version routing + +A request that names a country model `version` resolves to the app the +routing state lists for it. When the registry ties that app to exactly one +wrapper bundle and the bundle's manifest states a *different* model version +for the country, the gateway refuses the request with a 400 instead of +routing it. + +That is a deliberate change from the pre-canonical gateway, which served +the route and returned a self-contradictory body — `version` from the stale +route, `policyengine_bundle.model_version` from the live manifest. It can +happen without anyone doing anything wrong: publishing only ever *adds* +country routes and overwrites the bundle manifest for the wrapper it +deploys, and nothing prunes the old country route, so re-publishing one +wrapper with an upgraded country model leaves the previous country version +pointing at an app that no longer serves it. Callers pinning an old +`version` (the `policyengine-apis-integ` suite pins one) should move to a +version the current bundle states, or pass `policyengine_version` +explicitly. + +A manifest that states *no* model version for the country — entry absent, +or the value absent, non-string, empty or whitespace — contradicts nothing +and still routes. + ## Image dependencies The Modal image installs with `uv_sync(frozen=True)` from this project's diff --git a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py index a4f70eae2..19c1566d3 100644 --- a/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py +++ b/projects/policyengine-simulation-gateway/src/policyengine_simulation_gateway/endpoints.py @@ -352,6 +352,26 @@ def _bundle_manifest(state: dict, policyengine_version: str | None) -> dict: def _policyengine_version_for_app( state: dict, app_name: str, *, country: str, model_version: str ) -> str | None: + """The wrapper version serving this country route, or None. + + Behaviour change from the pre-canonical gateway, deliberate: a country + route whose one candidate bundle states a *different* model version is + now rejected with a 400 instead of resolving. Publishing only ever adds + country routes and overwrites the bundle manifest for the wrapper + version it deploys (``update_version_registry``), so re-publishing one + wrapper with an upgraded country model leaves the old country route + pointing at an app whose manifest now states the new model. Before, that + route resolved and the 202 body contradicted itself -- ``version`` from + the stale route, ``policyengine_bundle.model_version`` from the live + manifest -- and with canonical SPM the caller would also have been + reading a capability the requested model never had. Nothing prunes the + stale route, so the gateway refuses it instead. + + "States a different model version" and "states none" are one + classification, computed once below: an absent country entry, an absent, + non-string, empty or whitespace ``model_version`` all contradict + nothing and still resolve. + """ candidates = { version for version, routed_app in _routing_state_routes(state, "policyengine").items() @@ -388,11 +408,16 @@ def _policyengine_version_for_app( if not candidates: return None version = next(iter(candidates)) - _validate_legacy_version_matches_bundle( - country=country, - requested_version=model_version, - manifest=_bundle_manifest(state, version), - ) + if not unclassified: + # Reuse the classification above rather than re-deriving it: the + # shared validator reads any string as a stated model version, so + # calling it unconditionally rejected a blank one here while the + # ambiguity check above treated blank as unstated. + _validate_legacy_version_matches_bundle( + country=country, + requested_version=model_version, + manifest=_bundle_manifest(state, version), + ) return version diff --git a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py index c55aff1bb..8bea53d2c 100644 --- a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py +++ b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py @@ -508,3 +508,80 @@ def test_stated_model_version_still_contradicts_the_wrapper_pin( assert response.status_code == 400 assert response.json()["errors"][0]["code"] == "SPM_CONFIGURATION_UNAVAILABLE" assert mock_modal["func"].calls == [] + + +def republished_wrapper_state(mock_modal, *, model_version): + """The registry a re-publish leaves behind. + + ``update_version_registry`` only ever adds country routes and it + overwrites the bundle manifest for the wrapper version it deploys, so + re-publishing one wrapper with an upgraded country model leaves + ``routes["us"]["1.459.0"]`` pointing at an app whose manifest now states + something else. Nothing prunes it. + """ + state = deepcopy(TEST_ROUTING_STATE) + state["bundles"]["3.9.0"]["us"] = { + **state["bundles"]["3.9.0"]["us"], + "model_version": model_version, + } + mock_modal["dicts"]["simulation-api-routing-state"] = {"active": state} + return state + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +def test_stale_country_route_is_refused_rather_than_served( + mock_modal, client, endpoint, extra +): + """The behaviour change. Before, this route resolved and the 202 body + contradicted itself: ``version`` from the stale route, + ``policyengine_bundle.model_version`` from the live manifest.""" + republished_wrapper_state(mock_modal, model_version="1.470.0") + response = client.post( + endpoint, json={"country": "us", "version": "1.459.0", **extra} + ) + assert response.status_code == 400 + detail = response.json()["detail"] + assert "1.459.0" in detail and "1.470.0" in detail + # A routing refusal, not an SPM one. + assert "errors" not in response.json() + assert mock_modal["func"].calls == [] + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +def test_a_country_route_its_bundle_still_states_is_served( + mock_modal, client, endpoint, extra +): + """The other half: a route the manifest agrees with keeps working, so + the refusal above is about disagreement and not about single-candidate + country routes in general.""" + republished_wrapper_state(mock_modal, model_version="1.459.0") + response = client.post( + endpoint, json={"country": "us", "version": "1.459.0", **extra} + ) + assert response.status_code == 200, response.text + bundle = response.json()["policyengine_bundle"] + assert bundle["policyengine_version"] == "3.9.0" + assert bundle["model_version"] == "1.459.0" + + +@pytest.mark.parametrize("endpoint,extra", ENDPOINTS) +@pytest.mark.parametrize("model_version", [None, "", " ", 42]) +def test_a_bundle_that_states_no_model_version_contradicts_nothing( + mock_modal, client, endpoint, extra, model_version +): + """Unstated is unstated, whichever way it is unstated. + + The ambiguity check classifies a blank or whitespace model version as + unstated; the shared validator reads any string as stated. Deriving the + same fact twice made a blank manifest entry a 400 here while + ``test_shared_app_with_missing_model_metadata_is_ambiguous`` treats the + identical value as missing metadata. + """ + state = republished_wrapper_state(mock_modal, model_version=model_version) + if model_version is None: + del state["bundles"]["3.9.0"]["us"]["model_version"] + response = client.post( + endpoint, json={"country": "us", "version": "1.459.0", **extra} + ) + assert response.status_code == 200, response.text + assert response.json()["policyengine_bundle"]["policyengine_version"] == "3.9.0" From 3b28b9a0df99e74b4095c1a1192b2a0d9cd79576 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 12:46:27 -0400 Subject: [PATCH 33/40] Key the SPM artifact cache tests on the storage id The receipt-guard double had no storage id, so the artifact class's cache key and its snapshot replacement fell back to the bare simulation id and were not under test: filing under self.id, or caching the live object instead of a copy, both left the suite green. The double now carries a storage id and files a snapshot, as the wrapper does, and three cases pin what that buys -- two selections under one simulation id keep separate entries and neither serves the other, the replacement is a copy that a later re-point cannot rewrite, and the container converges on the recomputed entry. The stale-cache case split in two. A cached entry whose selection disagrees with the key it is filed under is something a correct wrapper never writes, so it is now labelled as the defense-in-depth case it is, and the realistic one -- a usable-looking entry whose receipt does not cover the requested year -- is covered separately. Co-Authored-By: Claude Fable 5.1 --- .../tests/test_baseline_artifacts.py | 142 ++++++++++++++++-- 1 file changed, 129 insertions(+), 13 deletions(-) diff --git a/projects/policyengine-simulation-executor/tests/test_baseline_artifacts.py b/projects/policyengine-simulation-executor/tests/test_baseline_artifacts.py index 7e2af9e1b..d4562bdcc 100644 --- a/projects/policyengine-simulation-executor/tests/test_baseline_artifacts.py +++ b/projects/policyengine-simulation-executor/tests/test_baseline_artifacts.py @@ -621,6 +621,13 @@ def _spm_receipt(selection=None, *, year="2026"): } +def _installed_wrapper_supports_spm() -> bool: + return "spm" in Simulation.model_fields and hasattr(Simulation, "spm_provenance") + + +_INSTALLED_WRAPPER_SUPPORTS_SPM = _installed_wrapper_supports_spm() + + class SPMWrapperSimulation(Simulation): """Test double for the canonical wrapper's SPM surface. @@ -632,13 +639,28 @@ class SPMWrapperSimulation(Simulation): from the SPM_NATIVE_SMOKE_SOURCE-gated tests. This double reproduces that surface's *shape and restore timing*, which - is all the guard depends on. It does not prove the deployed wrapper - emits receipts of this shape — only the native tests do. + is all the guard depends on. It is deliberately more permissive than the + real wrapper, whose own load rejects an artifact built under a different + selection: the guard is written not to trust the wrapper, so it is + exercised here as the independent check it is meant to be. + + It is evidence about ``ensure()``'s decisions, never about the wrapper. + It cannot show that a real ``spm_provenance()`` returns a mapping + ``SPMProvenance`` accepts, that a real load or cache hit restores the + artifact's selection at all, or that a real ``storage_id`` separates two + selections. Those three are the native suite's claims; hermetic green + here is not wrapper conformance. + + ``storage_id`` is a plain field rather than a property derived from the + selection, so it stays put while a load rewrites ``spm`` — the double + makes no claim about how the wrapper computes it, only that the artifact + class keys the process cache on it. """ spm: dict | None = None spm_config: dict | None = None spm_receipt: dict | None = None + storage_id: str = "" _provenance_reads: list = PrivateAttr(default_factory=list) def spm_provenance(self): @@ -648,7 +670,8 @@ def spm_provenance(self): def ensure(self): from policyengine.core.simulation import _cache - cached = _cache.get(self.id) + cache_key = self.storage_id or self.id + cached = _cache.get(cache_key) if cached is not None: self.output_dataset = cached.output_dataset # The restore the guard exists to survive: a cached entry's @@ -663,7 +686,10 @@ def ensure(self): except Exception: self.run() self.save() - _cache.add(self.id, self) + # The wrapper files a snapshot, not the live object, so re-pointing + # this simulation at another selection cannot rewrite the entry an + # earlier key names. + _cache.add(cache_key, self.model_copy(deep=False)) class CanonicalSPMSimulation(ba.ArtifactBaselineSimulation, SPMWrapperSimulation): @@ -697,9 +723,13 @@ def run(self, simulation): simulation.spm_receipt = _spm_receipt(simulation.spm) -def _make_spm_sim(model_version, *, spm=None, sim_id="bl1-spm", year=2026): +def _make_spm_sim( + model_version, *, spm=None, sim_id="bl1-spm", storage_id=None, year=2026 +): + selection = SPM_SELECTION if spm is None else spm return CanonicalSPMSimulation.model_construct( id=sim_id, + storage_id=storage_id or f"{sim_id}-{selection['scenario']}", dataset=SimpleNamespace(year=year), tax_benefit_model_version=model_version, policy=None, @@ -788,17 +818,35 @@ def test_receipt_for_another_year_recomputes(self, fresh_cache): assert sim.artifact_outcome == ba.OUTCOME_INCOMPLETE assert model.calls == ["load", "run", "save"] - def test_stale_cached_selection_cannot_replace_the_request(self, fresh_cache): - """A cache hit restores the cached entry's selection onto this - request. The guard validates against the pre-load selection, so the - stale entry can only cost a recompute — never silently answer the - request under the wrong SPM settings.""" + def test_cached_entry_with_an_unusable_receipt_is_revalidated(self, fresh_cache): + """A cache hit skips the load entirely, so the entry's receipt is + the only thing standing between the request and someone else's + output. A tax-only run leaves a receipt with no years under the same + selection; the guard has to notice and recompute.""" + stale = _make_spm_sim(SPMModelVersion()) + stale.output_dataset = _output(_complete_frames()) + stale.spm_receipt = _spm_receipt(year="2024") + fresh_cache.add(stale.storage_id, stale) + + model = SPMModelVersion() + sim = _make_spm_sim(model) + sim.ensure() + assert sim.artifact_outcome == ba.OUTCOME_INCOMPLETE + # Cache hit short-circuits the load; the guard still forces the run. + assert model.calls == ["run", "save"] + + def test_cached_selection_cannot_replace_the_request(self, fresh_cache): + """Defense in depth: an entry whose selection disagrees with the key + it is filed under. A correct wrapper never writes one — the key is + derived from the selection — but a cache hit restores the entry's + selection onto this request without revalidating, so the guard + keeps the request's own selection and recomputes under it.""" other = {**SPM_SELECTION, "scenario": "zero_real"} stale = _make_spm_sim(SPMModelVersion(), spm=other) stale.output_dataset = _output(_complete_frames()) - stale.spm_config = other stale.spm_receipt = _spm_receipt(other) - fresh_cache.add("bl1-spm", stale) + request = _make_spm_sim(SPMModelVersion()) + fresh_cache.add(request.storage_id, stale) model = SPMModelVersion() sim = _make_spm_sim(model) @@ -815,7 +863,7 @@ def test_recompute_replaces_the_cache_with_this_request_s_selection( model = SPMModelVersion(stored_config=other, stored_receipt=_spm_receipt(other)) sim = _make_spm_sim(model) sim.ensure() - assert fresh_cache.get("bl1-spm").spm_config == SPM_SELECTION + assert fresh_cache.get(sim.storage_id).spm_config == SPM_SELECTION # The next request in this container hits the replaced entry and # validates clean: one recompute for the container, not one each. @@ -850,3 +898,71 @@ def test_a_request_without_a_selection_never_validates_receipts(self, fresh_cach assert sim.artifact_outcome == ba.OUTCOME_HIT assert model.calls == ["load"] assert sim._provenance_reads == [] + + def test_the_cache_is_keyed_by_storage_id_not_simulation_id(self, fresh_cache): + """One caller id must not serve two selections. + + The deterministic simulation id is shared by every selection; only + the storage id separates them, and the artifact class evicts and + replaces under it. Keyed on the id instead, a national recompute + would answer the next county request from cache. + """ + other = {**SPM_SELECTION, "scenario": "zero_real"} + first_model = SPMModelVersion( + stored_config=other, stored_receipt=_spm_receipt(other) + ) + first = _make_spm_sim(first_model) + first.ensure() + assert first.artifact_outcome == ba.OUTCOME_INCOMPLETE + + second_model = SPMModelVersion( + stored_config=other, stored_receipt=_spm_receipt(other) + ) + second = _make_spm_sim(second_model, spm=other) + assert second.id == first.id + assert second.storage_id != first.storage_id + second.ensure() + # It read its own artifact rather than the entry the first request + # left behind, and both entries survive side by side. + assert second_model.calls == ["load"] + assert second.artifact_outcome == ba.OUTCOME_HIT + # The artifact class filed the recompute under the storage id, and + # nothing was ever filed under the bare simulation id. + assert fresh_cache.get(first.storage_id).spm_config == SPM_SELECTION + assert fresh_cache.get(second.storage_id) is not None + assert fresh_cache.get(first.id) is None + + def test_the_replaced_cache_entry_is_a_snapshot(self, fresh_cache): + """The replacement is a copy so that re-pointing this simulation at + another selection and running it again cannot overwrite the output + the earlier key names.""" + other = {**SPM_SELECTION, "scenario": "zero_real"} + model = SPMModelVersion(stored_config=other, stored_receipt=_spm_receipt(other)) + sim = _make_spm_sim(model) + sim.ensure() + cached = fresh_cache.get(sim.storage_id) + assert cached is not sim + + # A later request in the same process re-points the live object. + model.load(sim) + assert sim.spm_config == other + assert cached.spm_config == SPM_SELECTION + + @pytest.mark.skipif( + not _INSTALLED_WRAPPER_SUPPORTS_SPM, + reason=( + "The pinned policyengine is pre-canonical; this checks the double " + "above against the real surface as soon as an SPM-capable wrapper " + "is pinned, at which point the double should be retired for it." + ), + ) + def test_the_double_matches_the_installed_wrapper_surface(self): + from policyengine.core import Simulation + + assert "spm" in Simulation.model_fields + assert callable(getattr(Simulation, "spm_provenance", None)) + assert isinstance( + getattr(type(Simulation), "storage_id", None) + or Simulation.__dict__.get("storage_id"), + property, + ) From 2b52202e16fb28f0efedfb3dd8b3a54177695120 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 12:52:11 -0400 Subject: [PATCH 34/40] Show the precompute write path publishing under a shared storage id The storage-id agreement fixture moved out of the key-discipline suite so the precompute writer can state the same claim. Nothing in hermetic CI ran compute_baseline_impl with a selection: the identity was compared to itself, and the one case that reached the guard covered the abort. A stub baseline that names its artifact the wrapper's way -- transcribed from the wheel the native lane installs, not from our planner -- now runs the canonical write end to end, so the guard passing is the claim rather than an assumption, and the abort is shown under a real selection too. Co-Authored-By: Claude Fable 5.1 --- .../fixtures/wrapper_spm.py | 46 ++++++++++ .../tests/test_artifact_keys.py | 61 ++++---------- .../tests/test_precompute.py | 84 +++++++++++++++++++ 3 files changed, 147 insertions(+), 44 deletions(-) create mode 100644 projects/policyengine-simulation-executor/fixtures/wrapper_spm.py diff --git a/projects/policyengine-simulation-executor/fixtures/wrapper_spm.py b/projects/policyengine-simulation-executor/fixtures/wrapper_spm.py new file mode 100644 index 000000000..41c639629 --- /dev/null +++ b/projects/policyengine-simulation-executor/fixtures/wrapper_spm.py @@ -0,0 +1,46 @@ +"""The canonical wrapper's ``storage_id``, transcribed for hermetic tests. + +The executor pins a pre-canonical ``policyengine``: its ``Simulation`` has +no ``spm`` field and no ``storage_id``, so hermetic CI cannot import the +property that names every canonical baseline artifact. Precompute plans a +store path from ``BaselineArtifactIdentity.storage_id`` and the +in-container worker aborts when the wrapper's value disagrees, so the two +derivations have to be one identifier reached down two paths. + +This is that second path, copied out of the SPM-capable wrapper so the +key-discipline tests and the precompute writer==reader tests state it once. +It proves our side has not drifted from the contract we read; it is not the +wrapper, and only the SPM_NATIVE_SMOKE_SOURCE-gated suites run against one. + +``policyengine/core/simulation.py``:: + + @property + def storage_id(self) -> str: + \"\"\"Include resolved SPM settings in cache and saved-result identity.\"\"\" + config = self.spm_config + if config is None: + return self.id + encoded = json.dumps(config, sort_keys=True, separators=(",", ":")).encode() + return f"{self.id}-spm-{hashlib.sha256(encoded).hexdigest()}" + +Read from ``policyengine-5.3.0-py3-none-any.whl`` sha256 +``8c640d967575dddad70840bcbe938cea251c56958eded647c83eb9c5902735f1``, the +build the native qualification lane installs. +""" + +import hashlib +import json + + +def wrapper_storage_id(simulation_id: str, spm_config: dict | None) -> str: + if spm_config is None: + return simulation_id + encoded = json.dumps(spm_config, sort_keys=True, separators=(",", ":")).encode() + return f"{simulation_id}-spm-{hashlib.sha256(encoded).hexdigest()}" + + +def installed_wrapper_has_storage_id() -> bool: + """True once an SPM-capable wrapper is pinned and this file can retire.""" + from policyengine.core import Simulation + + return hasattr(Simulation, "storage_id") diff --git a/projects/policyengine-simulation-executor/tests/test_artifact_keys.py b/projects/policyengine-simulation-executor/tests/test_artifact_keys.py index ea8318791..1512b8330 100644 --- a/projects/policyengine-simulation-executor/tests/test_artifact_keys.py +++ b/projects/policyengine-simulation-executor/tests/test_artifact_keys.py @@ -12,6 +12,10 @@ import pytest from fixtures.identity_stubs import install_identity_stubs +from fixtures.wrapper_spm import ( + installed_wrapper_has_storage_id, + wrapper_storage_id as _wrapper_storage_id, +) from policyengine_simulation_executor import artifact_keys as ak @@ -185,48 +189,13 @@ def test_baseline_identity_composes(self, stub_identity_sources): "county_vintage": "2020", "as_of": None, } +_INSTALLED_WRAPPER_HAS_STORAGE_ID = installed_wrapper_has_storage_id() _SPM_STORAGE_GOLDEN = ( "bl1-21f52b30719e20bb-spm-" "7396bf5f4876c42bb6cba0f9533478098edc0861cc3657bd0d60f88dbb26ac39" ) -def _installed_wrapper_has_storage_id() -> bool: - from policyengine.core import Simulation - - return hasattr(Simulation, "storage_id") - - -_INSTALLED_WRAPPER_HAS_STORAGE_ID = _installed_wrapper_has_storage_id() - - -def _wrapper_storage_id(simulation_id: str, spm_config: dict | None) -> str: - """The canonical wrapper's own ``storage_id``, transcribed verbatim. - - ``policyengine/core/simulation.py`` in the SPM-capable wrapper:: - - @property - def storage_id(self) -> str: - config = self.spm_config - if config is None: - return self.id - encoded = json.dumps( - config, sort_keys=True, separators=(",", ":") - ).encode() - return f"{self.id}-spm-{hashlib.sha256(encoded).hexdigest()}" - - Read from ``policyengine-5.3.0-py3-none-any.whl`` sha256 - ``8c640d96…2735f1``, the build the native qualification lane installs. - """ - import hashlib - import json - - if spm_config is None: - return simulation_id - encoded = json.dumps(spm_config, sort_keys=True, separators=(",", ":")).encode() - return f"{simulation_id}-spm-{hashlib.sha256(encoded).hexdigest()}" - - class TestWrapperStorageIdAgreement: """The planner's storage id against the wrapper that names the file. @@ -242,9 +211,9 @@ class TestWrapperStorageIdAgreement: be imported in hermetic CI and these tests cannot prove agreement with one. What they do prove: - * the no-selection arm agrees with the **installed** wrapper, through the - exact expression ``precompute`` uses — that is the configuration - deployed today, asserted against the real object; + * the no-selection arm agrees through the exact accessor ``precompute`` + uses, against the real installed object — the same answer on either + wrapper, which is the point: that arm must not move; * the SPM arm agrees with the canonical wrapper's expression as read from the wheel above, so our side cannot drift from the contract without a reviewable diff, and the digest cannot be quietly reformatted; @@ -318,12 +287,16 @@ def test_no_selection_keeps_the_plain_simulation_id(self, identity): assert built.storage_id == built.simulation_id assert built.store_path.endswith(f"/{built.simulation_id}.h5") - def test_legacy_arm_matches_the_installed_wrapper(self, identity): - """The deployed-today arm, against the real ``Simulation`` object. + def test_legacy_arm_matches_the_wrapper_accessor(self, identity): + """The no-selection arm, through the accessor precompute uses. - ``precompute`` reads ``getattr(baseline, "storage_id", baseline.id)``; - with no selection that has to be the planned id on any wrapper, - canonical or not. + ``precompute`` reads ``getattr(baseline, "storage_id", baseline.id)``. + With no selection that has to be the planned id on *any* wrapper: + pre-canonical, because the attribute is absent and the fallback is + the id; canonical, because its property short-circuits to the id + when there is no config. The assertion is the same either way, so + this pins the accessor's contract rather than telling the two + wrappers apart. """ from policyengine.core import Simulation diff --git a/projects/policyengine-simulation-executor/tests/test_precompute.py b/projects/policyengine-simulation-executor/tests/test_precompute.py index 7c051bde3..1212cc031 100644 --- a/projects/policyengine-simulation-executor/tests/test_precompute.py +++ b/projects/policyengine-simulation-executor/tests/test_precompute.py @@ -16,6 +16,7 @@ from pydantic import ValidationError from fixtures.identity_stubs import install_identity_stubs +from fixtures.wrapper_spm import wrapper_storage_id from policyengine_simulation_executor import precompute from policyengine_simulation_executor.artifact_keys import canonical_digest from policyengine_simulation_executor.precompute_models import ( @@ -667,8 +668,32 @@ class SilentBaseline(ArtifactBaselineSimulation): def ensure(self): self._artifact_outcome = "miss" + class SPMStubBaseline(ArtifactBaselineSimulation): + """A baseline that names its artifact the way the wrapper does. + + The canonical wrapper derives ``storage_id`` from the resolved + selection and saves under it; ``compute_baseline_impl`` plans a + path from the identity's own derivation and refuses to act when + the two disagree. Only a stub that computes its side + independently can show the guard passing rather than aborting. + """ + + @property + def spm_config(self): + return state.spm + + @property + def storage_id(self): + return wrapper_storage_id(self.id, self.spm_config) + + def ensure(self): + self._artifact_outcome = "miss" + (tmp_path / f"{self.storage_id}.h5").write_bytes(b"artifact-bytes") + + state.spm = None state.baseline = StubBaseline.model_construct(id="bl1-cohort") state.make_silent = lambda: SilentBaseline.model_construct(id="bl1-cohort") + state.make_spm = lambda: SPMStubBaseline.model_construct(id="bl1-cohort") class FakeStore: def __init__(self, bucket): @@ -775,6 +800,65 @@ def test_refuses_a_mismatched_planned_storage_id(self, cohort_stubs): assert cohort_stubs.uploads == [] assert not (cohort_stubs.folder / "bl1-cohort.h5").exists() + def test_publishes_a_selection_scoped_artifact_under_the_planned_path( + self, cohort_stubs + ): + """The canonical write path, end to end through the guard. + + Nothing else in hermetic CI runs ``compute_baseline_impl`` with a + selection: the identity's storage id was only ever compared to + itself, and the one case that reached the guard covered the abort. + Here the plan is keyed the identity's way and the container names + its artifact the wrapper's way, so the guard passing at all is the + claim -- if the two derivations differed, no canonical artifact + could be published and the deploy would be blocked. + """ + selection = { + "forecast_content_sha256": "a" * 64, + "scenario": "ce_trend", + "geography_kind": "national", + "geography_id": None, + "county_vintage": "2020", + "as_of": None, + } + cohort_stubs.spm = selection + cohort_stubs.baseline = cohort_stubs.make_spm() + + # The planner's side: BaselineArtifactIdentity.storage_id's own + # expression, applied to this cohort's simulation id. + planned_storage_id = f"bl1-cohort-spm-{canonical_digest(selection)}" + entry = self._entry() + entry.path = f"baselines/us/bl-d/{planned_storage_id}.h5" + + result = precompute.compute_baseline_impl("bucket-x", entry) + + assert cohort_stubs.uploads == [ + (entry.path, str(cohort_stubs.folder / f"{planned_storage_id}.h5")) + ] + assert result.simulation_id == "bl1-cohort" + assert result.uploaded is True + assert result.size_bytes == len(b"artifact-bytes") + # A selection-free plan for the same cohort is a different artifact. + assert planned_storage_id != "bl1-cohort" + + def test_refuses_a_selection_scoped_plan_the_container_does_not_share( + self, cohort_stubs + ): + """The abort the review worried about, under a real selection: a + container that resolved a different selection names a different + artifact, and nothing is published.""" + cohort_stubs.spm = {"scenario": "zero_real"} + cohort_stubs.baseline = cohort_stubs.make_spm() + entry = self._entry() + entry.path = ( + f"baselines/us/bl-d/bl1-cohort-spm-" + f"{canonical_digest({'scenario': 'ce_trend'})}.h5" + ) + with pytest.raises(RuntimeError, match="storage ids disagree"): + precompute.compute_baseline_impl("bucket-x", entry) + assert cohort_stubs.configured == [] + assert cohort_stubs.uploads == [] + def test_refuses_a_plain_simulation(self, cohort_stubs): cohort_stubs.baseline = SimpleNamespace(id="bl1-cohort") with pytest.raises(RuntimeError, match="plain Simulation"): From 5809d5bceb0e9143cb62ec024330646b61752f67 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 13:06:50 -0400 Subject: [PATCH 35/40] Format the gateway SPM route tests This file was ruff-clean at the PR head and the new cases left it not. Tests are outside the formatted paths (the Makefile and the CI lint job walk src only), so nothing catches it; reformatting now keeps the drift this branch introduced out of the next diff. Co-Authored-By: Claude Opus 5 --- .../tests/test_spm_routes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py index 8bea53d2c..ad7122b27 100644 --- a/projects/policyengine-simulation-gateway/tests/test_spm_routes.py +++ b/projects/policyengine-simulation-gateway/tests/test_spm_routes.py @@ -359,7 +359,12 @@ def test_completed_budget_window_rows_keep_resolved_nulls(mock_modal, client): # Every optional field the pre-SPM gateway emitted in a raw 202/500 body. -LEGACY_BUNDLE_KEYS = {"model_version", "policyengine_version", "data_version", "dataset"} +LEGACY_BUNDLE_KEYS = { + "model_version", + "policyengine_version", + "data_version", + "dataset", +} @pytest.mark.parametrize("source", LEGACY_SOURCES) From 19e27f83208b470113bce781ba942cc3d9015c90 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 13:06:59 -0400 Subject: [PATCH 36/40] Resolve an SPM selection once per container, not once per call site A national request normalized its selection about six times -- the worker entrypoint, `_build_simulation`, and the baseline and dataset identity collectors -- and each one re-read the installed capability and built a throwaway `PolicyEngineSPMProvider` to prevalidate the years. Neither input can move under a running container: the capability is a property of the image, and the prevalidation reads nothing but the resolved selection and the year range. Both are memoized on those keys. `lru_cache` does not keep exceptions, so every fail-closed path still re-derives and re-raises; the cache can only skip work that already succeeded. Keying on the range means the range has to be parsed before the selection is validated, where it used to be parsed between the provider's construction and its first use -- an unusable year would then have masked an unknown scenario, so that one request revalidates the selection over an empty range first and keeps the old order. Tests count the two pieces of repeated work: through `normalize_runtime_spm` directly and through `collect_baseline_identity`, which is where the repeat the review counted actually happens. Each cache is also shown not to weaken what it caches -- a rejected selection and an unavailable capability are re-derived every time -- and the executor suite clears both around every test so a stub is never served another test's answer. Co-Authored-By: Claude Opus 5 --- .../policyengine_simulation_executor/spm.py | 134 +++++++++---- .../tests/conftest.py | 18 ++ .../tests/test_canonical_spm.py | 186 ++++++++++++++++++ 3 files changed, 305 insertions(+), 33 deletions(-) diff --git a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py index 6abb0ac2a..bfb1c311a 100644 --- a/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py +++ b/projects/policyengine-simulation-executor/src/policyengine_simulation_executor/spm.py @@ -1,5 +1,6 @@ """Certified SPM runtime selection, independent of caller-supplied metadata.""" +import json from functools import lru_cache from policyengine_simulation_contract.spm import ( @@ -19,7 +20,23 @@ def _forecast(expected_sha256): return load_forecast(expected_sha256=expected_sha256) +@lru_cache(maxsize=1) def runtime_spm_capability(): + """The installed image's certified SPM capability, or None. + + Memoized for the life of the process because every input is a property + of the installed image -- the bundle configuration, the wrapper's model + fields, the country model's methods and the pinned forecast -- none of + which a running container can change. One national request resolves the + selection about six times (the worker entrypoint, ``_build_simulation``, + and the dataset and baseline identity collectors), and each resolution + called this. + + ``lru_cache`` does not memoize exceptions, so every fail-closed path + still re-derives and re-raises: the cache can only ever skip work that + already succeeded. Tests that stub the installed environment must call + :func:`reset_spm_runtime_caches` (the executor suite does, automatically). + """ from policyengine.bundle import get_current_bundle from policyengine.core import Simulation @@ -60,6 +77,78 @@ def runtime_spm_capability(): raise SPMInputError("SPM_CONFIGURATION_UNAVAILABLE", str(exc)) from exc +@lru_cache(maxsize=64) +def _prevalidate_selection(selection_json: str, start_year: int, window_size: int): + """Prove a resolved selection can be measured, before anything expensive. + + Keyed on the resolved selection and the year range, which is everything + it reads: the provider is constructed from the selection alone and asked + only for per-year metadata. The same request resolves the same selection + several times over (see :func:`runtime_spm_capability`), and the review + of this change counted about six provider constructions per national + request; memoizing collapses them to one per distinct selection and + range. + + Only successful validations are memoized -- ``lru_cache`` re-runs after + an exception -- so a rejected selection is rejected again, with the same + typed error, every time it is asked for. + """ + from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider + + selection = json.loads(selection_json) + forecast = _forecast(selection["forecast_content_sha256"]) + if selection["county_vintage"] != "2020": + raise ValueError("Unsupported county vintage: use 2020") + provider = PolicyEngineSPMProvider( + forecast, + **{ + key: value + for key, value in selection.items() + if key != "forecast_content_sha256" + }, + ) + for year in range(start_year, start_year + window_size): + # Use the country adapter's typed year contract. This temporary + # provider validates metadata without measuring any SPM amount + # or modifying the actual simulation's calculation receipts. + provider.year_metadata(year) + if selection["geography_kind"] == "metro": + try: + forecast.geography_factor( + year, + "renter", + kind="metro", + geoid=selection["geography_id"], + scenario=selection["scenario"], + as_of=selection["as_of"], + ) + except ValueError as exc: + raise SPMInputError("SPM_GEOGRAPHY_UNAVAILABLE", str(exc)) from None + + +# Bound at import, so clearing still works while a test has replaced one of +# these module attributes with a stub. +_MEMO_CLEARERS = ( + runtime_spm_capability.cache_clear, + _prevalidate_selection.cache_clear, +) + + +def reset_spm_runtime_caches(): + """Forget what was memoized from the installed environment. + + Production never needs this: a container's image is fixed for the life + of the process. Tests that stub the installed bundle, wrapper or + provider do, and the executor suite calls it around every test. + + ``_forecast`` is deliberately not cleared. It is keyed by the content + hash of what it loads, so it cannot go stale, and a test that stubs it + replaces the module attribute rather than filling the cache. + """ + for clear in _MEMO_CLEARERS: + clear() + + def normalize_runtime_spm(params): """Resolve before dataset loading, artifact lookup, or child submission.""" from policyengine_simulation_executor.release_bundle import ( @@ -76,42 +165,21 @@ def normalize_runtime_spm(params): model_version=bundle.model_version, ) if selection is not None: + canonical = json.dumps(selection, sort_keys=True, separators=(",", ":")) try: - from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider - - forecast = _forecast(selection["forecast_content_sha256"]) - if selection["county_vintage"] != "2020": - raise ValueError("Unsupported county vintage: use 2020") - provider = PolicyEngineSPMProvider( - forecast, - **{ - key: value - for key, value in selection.items() - if key != "forecast_content_sha256" - }, - ) from policyengine_simulation_executor.simulation_runtime import _parse_year - start = int(params.get("start_year") or _parse_year(params)) - for year in range(start, start + int(params.get("window_size", 1))): - # Use the country adapter's typed year contract. This temporary - # provider validates metadata without measuring any SPM amount - # or modifying the actual simulation's calculation receipts. - provider.year_metadata(year) - if selection["geography_kind"] == "metro": - try: - forecast.geography_factor( - year, - "renter", - kind="metro", - geoid=selection["geography_id"], - scenario=selection["scenario"], - as_of=selection["as_of"], - ) - except ValueError as exc: - raise SPMInputError( - "SPM_GEOGRAPHY_UNAVAILABLE", str(exc) - ) from None + try: + start = int(params.get("start_year") or _parse_year(params)) + window = int(params.get("window_size", 1)) + except ValueError: + # Keep the order the range's old parse position gave it: the + # selection was checked before the range, so an unusable year + # never masked an unknown scenario. An empty range asks for + # no year and reports the selection's own defect first. + _prevalidate_selection(canonical, 0, 0) + raise + _prevalidate_selection(canonical, start, window) except ValueError as exc: detail = spm_error_detail(exc) if detail: diff --git a/projects/policyengine-simulation-executor/tests/conftest.py b/projects/policyengine-simulation-executor/tests/conftest.py index c94efcc3b..466293e88 100644 --- a/projects/policyengine-simulation-executor/tests/conftest.py +++ b/projects/policyengine-simulation-executor/tests/conftest.py @@ -4,6 +4,8 @@ import sys from pathlib import Path +import pytest + pytest_plugins = () project_root = Path(__file__).parent.parent @@ -15,3 +17,19 @@ ).ensure_project_root_on_path ensure_project_root_on_path() + + +@pytest.fixture(autouse=True) +def _forget_spm_runtime_caches(): + """Keep the SPM runtime memoization out of every other test's way. + + ``runtime_spm_capability`` and the selection prevalidation are memoized + on the installed environment, which a container cannot change but a test + can. Clearing on both sides of each test means a stub is never served a + previous test's answer, and never leaves one behind. + """ + from policyengine_simulation_executor.spm import reset_spm_runtime_caches + + reset_spm_runtime_caches() + yield + reset_spm_runtime_caches() diff --git a/projects/policyengine-simulation-executor/tests/test_canonical_spm.py b/projects/policyengine-simulation-executor/tests/test_canonical_spm.py index a990b5445..cd990d087 100644 --- a/projects/policyengine-simulation-executor/tests/test_canonical_spm.py +++ b/projects/policyengine-simulation-executor/tests/test_canonical_spm.py @@ -261,6 +261,192 @@ def test_year_alias_is_validated_before_dataset_loading(monkeypatch): assert year_metadata.call_args.args == (2040,) +class TestResolutionIsNotRepeatedWork: + """One request resolves its selection about six times; pay once. + + ``_run_simulation_impl_core`` normalizes, ``_build_simulation`` + normalizes again, and the baseline and dataset identity collectors + normalize once each. Every one of those re-read the installed + capability and built a throwaway ``PolicyEngineSPMProvider`` to + prevalidate the years. The installed image cannot change under a + running container, and the prevalidation reads nothing but the + selection and the year range, so both are now memoized -- these cases + fix what that must and must not change. + """ + + @staticmethod + def _stub_bundle(monkeypatch): + from policyengine_simulation_executor import release_bundle + + monkeypatch.setattr( + release_bundle, + "get_country_release_bundle", + lambda country: SimpleNamespace( + policyengine_version="test", model_version="test" + ), + ) + + @staticmethod + def _stub_provider(monkeypatch, capability=CAPABILITY): + """Count the two pieces of work a resolution repeats.""" + import policyengine_simulation_executor.spm as spm + + year_metadata = Mock(return_value={}) + constructions = Mock( + side_effect=lambda *a, **k: SimpleNamespace(year_metadata=year_metadata) + ) + monkeypatch.setattr(spm, "runtime_spm_capability", lambda: capability) + monkeypatch.setattr(spm, "_forecast", lambda sha: object()) + monkeypatch.setitem( + sys.modules, + "spm_calculator.policyengine_adapter", + SimpleNamespace(PolicyEngineSPMProvider=constructions), + ) + return SimpleNamespace( + constructions=constructions, year_metadata=year_metadata, spm=spm + ) + + def test_one_selection_and_year_range_prevalidates_once(self, monkeypatch): + self._stub_bundle(monkeypatch) + stubs = self._stub_provider(monkeypatch) + request = {"country": "us", "time_period": "2026", "spm": SELECTION} + + resolutions = [stubs.spm.normalize_runtime_spm(request) for _ in range(6)] + + assert resolutions == [SELECTION] * 6 + assert stubs.constructions.call_count == 1 + assert stubs.year_metadata.call_count == 1 + + # The key is the selection and the range, so neither can be served + # the other's answer. + stubs.spm.normalize_runtime_spm({**request, "time_period": "2027"}) + assert stubs.constructions.call_count == 2 + stubs.spm.normalize_runtime_spm( + {**request, "spm": {**SELECTION, "scenario": "zero_real"}} + ) + assert stubs.constructions.call_count == 3 + stubs.spm.normalize_runtime_spm({**request, "window_size": 2}) + assert stubs.constructions.call_count == 4 + # One year each for the three single-year ranges, two for the window. + assert stubs.year_metadata.call_count == 5 + + stubs.spm.reset_spm_runtime_caches() + stubs.spm.normalize_runtime_spm(request) + assert stubs.constructions.call_count == 5 + + def test_a_rejected_selection_is_rejected_every_time(self, monkeypatch): + """Fail-closed cannot be weakened by a cache: exceptions are not kept.""" + self._stub_bundle(monkeypatch) + stubs = self._stub_provider(monkeypatch) + stubs.year_metadata.side_effect = SPMInputError( + "SPM_YEAR_UNAVAILABLE", "2040 is unavailable" + ) + request = {"country": "us", "time_period": "2040", "spm": SELECTION} + + for _ in range(3): + with pytest.raises(SPMInputError) as error: + stubs.spm.normalize_runtime_spm(request) + assert error.value.code == "SPM_YEAR_UNAVAILABLE" + assert stubs.constructions.call_count == 3 + + def test_an_unusable_year_does_not_mask_the_selection(self, monkeypatch): + """The order the memo had to preserve. + + The year range used to be parsed between the provider's construction + and its first use, so a selection the provider itself rejects was + reported as such even when the year was unusable. The memo is keyed + on the range, so the range has to be parsed first; an empty range + keeps the old order for the one request where it differs. + """ + self._stub_bundle(monkeypatch) + stubs = self._stub_provider(monkeypatch) + stubs.constructions.side_effect = ValueError( + "Unknown forecast scenario: made-up" + ) + unusable = {"country": "us", "time_period": "2026-01", "spm": SELECTION} + + with pytest.raises(SPMInputError) as error: + stubs.spm.normalize_runtime_spm(unusable) + assert error.value.code == "SPM_SCENARIO_UNAVAILABLE" + assert stubs.year_metadata.call_count == 0 + + with pytest.raises(SPMInputError) as error: + stubs.spm.normalize_runtime_spm({**unusable, "time_period": "2026"}) + assert error.value.code == "SPM_SCENARIO_UNAVAILABLE" + + def test_an_unusable_year_alone_is_a_settings_error(self, monkeypatch): + self._stub_bundle(monkeypatch) + stubs = self._stub_provider(monkeypatch) + + with pytest.raises(SPMInputError) as error: + stubs.spm.normalize_runtime_spm( + {"country": "us", "time_period": "2026-01", "spm": SELECTION} + ) + assert error.value.code == "SPM_SETTINGS_INVALID" + assert stubs.constructions.call_count == 1 + assert stubs.year_metadata.call_count == 0 + + def test_collecting_a_baseline_identity_prevalidates_once(self, monkeypatch): + """The repeat the review counted, through the real collectors. + + ``collect_baseline_identity`` normalizes the request's selection and + then ``collect_dataset_identity`` normalizes the bundle's default for + the same year. When both resolve to the same selection that is one + prevalidation for the pair, and repeating the collection -- as a + request does -- adds none. + """ + from fixtures.identity_stubs import install_identity_stubs + + install_identity_stubs(monkeypatch) + stubs = self._stub_provider(monkeypatch) + + first = artifact_keys.collect_baseline_identity( + "us", 2026, region="us", scope_key=None, spm={"geography_kind": "national"} + ) + assert first.spm == SELECTION + assert stubs.constructions.call_count == 1 + + second = artifact_keys.collect_baseline_identity( + "us", 2026, region="us", scope_key=None, spm={"geography_kind": "national"} + ) + assert second.storage_id == first.storage_id + assert stubs.constructions.call_count == 1 + + # A request that differs from the bundle default still resolves both. + county = artifact_keys.collect_baseline_identity( + "us", 2026, region="us", scope_key=None, spm={"geography_kind": "county"} + ) + assert county.storage_id != first.storage_id + assert stubs.constructions.call_count == 2 + + def test_the_installed_capability_is_read_once(self, monkeypatch): + import policyengine.bundle + import policyengine_simulation_executor.spm as spm + + reads = Mock(return_value={"measurements": {}}) + monkeypatch.setattr(policyengine.bundle, "get_current_bundle", reads) + + assert [spm.runtime_spm_capability() for _ in range(4)] == [None] * 4 + assert reads.call_count == 1 + + spm.reset_spm_runtime_caches() + assert spm.runtime_spm_capability() is None + assert reads.call_count == 2 + + def test_an_unavailable_capability_is_rederived_every_time(self, monkeypatch): + import policyengine.bundle + import policyengine_simulation_executor.spm as spm + + reads = Mock(side_effect=ValueError("bundle configuration unavailable")) + monkeypatch.setattr(policyengine.bundle, "get_current_bundle", reads) + + for _ in range(3): + with pytest.raises(SPMInputError) as error: + spm.runtime_spm_capability() + assert error.value.code == "SPM_CONFIGURATION_UNAVAILABLE" + assert reads.call_count == 3 + + @pytest.mark.parametrize( "code", [ From 3edb78b81221521cc3f87ccee7c2171a6429b9b0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 13:08:27 -0400 Subject: [PATCH 37/40] Record every finding closed Co-Authored-By: Claude Opus 5 --- PROGRESS.md | 89 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 34 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 5ba311c0e..2dbf8954a 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1,4 +1,4 @@ -# PR #677 Fable-review fixes — progress +# PR #677 review fixes — progress Branch: `max/spm-simulation-canonical-fixes-20260911` (from PR head `b0447ca`) Target PR branch: `max/spm-simulation-canonical-20260909` @@ -6,53 +6,74 @@ Base: `main` = `414c631d622f5f587eedd187296a80182a923db2` ## State -Worktree created at PR head. Investigating the nine review findings. +All nine review findings are addressed. Suites green locally; pushing to +the PR branch next. ## Findings checklist - [x] Medium 1 — completed results drop explicit nulls from `spm_config` on the wire - [x] Medium 2 — legacy no-SPM provenance `legacy-seed` expires on next publish -- [ ] Medium 3 — precompute storage identity never shown to match the wrapper +- [x] Medium 3 — precompute storage identity never shown to match the wrapper - [x] Medium 4 — budget-window scheduler typed-error branches untested - [x] Low 1 — legacy no-SPM poll bodies not byte-identical to base -- [ ] Low 2 — stale country routes now fail; undocumented, untested +- [x] Low 2 — stale country routes now fail; undocumented, untested - [x] Low 3 — segmented reduce with SPM children untested end to end -- [ ] Low 4 — hermetic CI never exercises receipt validation in `ensure()` -- [ ] Low 5 — repeated prevalidation per request (perf) +- [x] Low 4 — hermetic CI never exercises receipt validation in `ensure()` +- [x] Low 5 — repeated prevalidation per request (perf) ## Done -- Created worktree, verified head/base SHAs. -- Baseline suites green at PR head: contract 73, gateway 149, entry 70, - executor 456 passed / 22 skipped. -- **Medium 1**: `SPMSelection.serialize_selection` now restores options that - were explicitly selected as null when the caller serializes with - `exclude_none` (the poll routes do, via `response_model_exclude_none`). - Tests: `test_completed_result_body_keeps_resolved_nulls`, - `test_completed_budget_window_rows_keep_resolved_nulls` — both verified - failing against the pre-fix serializer. -- **Low 1**: new `_bundle_payload` helper drops the `spm` key from raw 202/500 - job bodies and the worker `_metadata` when a route has no capability, so a - legacy no-SPM body is byte-identical to base. Tests: - `test_legacy_poll_bodies_carry_no_spm_key`, - `test_legacy_budget_window_metadata_carries_no_spm_key`, - `test_canonical_poll_bodies_still_carry_the_capability` — the first two - verified failing against the pre-fix endpoints. - -- **Low 3**: `tests/test_segmented_national.py` now drives a segmented - national reduce with SPM children end to end. -- **Medium 2**: route provenance is now derived from route shape +- **Medium 1** (`b074b95`, `bf6764c`): `SPMSelection.serialize_selection` + keeps options that were explicitly selected as null when the caller + serializes with `exclude_none` (the poll routes do, via + `response_model_exclude_none`). Covered at the gateway + (`test_completed_result_body_keeps_resolved_nulls`, + `test_completed_budget_window_rows_keep_resolved_nulls`) and at the + contract layer, where reverting the serializer had left the suite green. +- **Medium 2** (`4992a19`): route provenance is derived from route shape (`policyengine_version is None and schema_version == 1` -> `legacy-country-route`) instead of the registry's rewritable `generation` marker, and `_certified_model_version` stops reading a wrapper version as a country model version, so a pinned 5.2.0/5.3.0 route with no manifest - stays historical. Tests in `test_spm_selection.py` and `test_spm_routes.py`. -- **Medium 4**: `test_budget_window_scheduler.py` gained per-year child - injection seams and four cases covering the typed child-call failure, the - typed receipt-validation failure, and the redaction fallback on each. - Verified by mutation: deleting the child-entry re-attach fails both typed - cases. + stays historical. +- **Medium 3** (`ae9489e`, `3b28b9a`, `2b52202`): the planner's storage id + is held against the wrapper's own derivation — transcribed from the + 5.3.0 wheel the native lane installs, in `fixtures/wrapper_spm.py` — with + a golden, per-field rotation, a non-ASCII scenario, and a skipped case + that starts asserting real equality the moment an SPM-capable wrapper is + pinned. Hermetic CI now runs `compute_baseline_impl` under a real + selection with a container that names its artifact the wrapper's way, so + the guard *passing* is the claim, not only the abort. The native smoke + makes the same claim against the real wrapper. +- **Medium 4** (`94e980e`): per-year child injection seams and four cases + covering the typed child-call failure, the typed receipt-validation + failure, and the redaction fallback on each. +- **Low 1** (`b074b95`): `_bundle_payload` drops the `spm` key from raw + 202/500 job bodies and the worker `_metadata` when a route has no + capability, so a legacy no-SPM body is byte-identical to base; a + canonical route still carries the capability. +- **Low 2** (`215d6fa`): the resolver carries why it refuses, the gateway + README tells callers pinning an old version what to do, and cases pin the + refusal, the accepted route it is distinguished from, and all four ways a + manifest can state nothing. The same commit made "states nothing" mean + one thing: a blank manifest entry had produced a 400 naming no version. +- **Low 3** (`0c3f9c6`): `tests/test_segmented_national.py` drives a + segmented national reduce with SPM children end to end. +- **Low 4** (`638b7c6`, `3b28b9a`): a test double supplies the canonical + wrapper's SPM surface and restore timing, so every decision `ensure()` + makes about a receipt is fixed without the native source; the cache key + and snapshot replacement are under test too. +- **Low 5** (`19e27f8`): `runtime_spm_capability` and the selection + prevalidation are memoized on the installed image and on the resolved + selection plus year range. Neither cache keeps exceptions, so fail-closed + is unchanged, and the executor suite clears both around every test. -## Next +## Not changed, deliberately -- Medium 3, Low 2, Low 4, Low 5. +- `changelog_entry.yaml`: no workflow, script or doc consumes it, there is + no `CHANGELOG.md`, and `AGENTS.md` states there is no repository-wide + changelog fragment requirement. Low 2's "changelog/doc note" is the + gateway README section. +- `libs/policyengine-fastapi/src` carries nine pre-existing `ruff check` + F401s, present at the PR head and on `main`. CI's lint job runs + `ruff format --check` only, which passes. From fd869e7c351cec91bc6ff3ce97dd9848cfd780c4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 13:26:38 -0400 Subject: [PATCH 38/40] Close the gaps an adversarial audit found in the storage-id and scheduler claims Three claims held up only because nothing tested them. The skipped `test_spm_arm_matches_the_installed_wrapper` promised to start asserting real equality the moment an SPM-capable wrapper was pinned. It would not have: that wrapper's `spm_config` refuses a model version whose country is not "us", so the bare `model_construct(id=..., spm=...)` raises "SPM selection is only supported by the US model" instead of comparing anything. It now supplies the US model version and the bundle the property re-resolves through, and the class says which preconditions those are. The precompute write path said it keyed the plan the identity's way and then restated the identity's expression by hand, so truncating the real `storage_id`'s digest, changing its separator, or upper-casing its hex all left the precompute suite green. It now builds a real `BaselineArtifactIdentity` and takes its `store_path`, `simulation_id` and `storage_id`; each of those three mutations now fails it. The scheduler's typed branches were driven only at `max_parallel=1`, where the early return has nothing to return early from: replacing it with `continue` changed nothing observable. A two-parallel case now pins that the sibling already running is cancelled rather than harvested into a partial window. Both branches also persist the re-attached errors before returning, which nothing re-read -- the poll body is serialized from the in-memory state -- so dropping the write was invisible; a case now reads the batch-state dict back. Agreement with the real wrapper is no longer only transcribed. Running the executor environment with the qualification lane's 5.3.0 wheel shadowing the pinned pre-canonical one, eight selection shapes resolve to the same config and the same storage id through both that wheel and this repo. The wheel's sha256 is doing real work in that record: a second local build carries the same filename and version and has no `storage_id` at all. Co-Authored-By: Claude Opus 5 --- .../fixtures/wrapper_spm.py | 12 ++- .../tests/test_artifact_keys.py | 73 ++++++++++++++----- .../tests/test_budget_window_scheduler.py | 61 ++++++++++++++++ .../tests/test_precompute.py | 72 ++++++++++++++---- 4 files changed, 186 insertions(+), 32 deletions(-) diff --git a/projects/policyengine-simulation-executor/fixtures/wrapper_spm.py b/projects/policyengine-simulation-executor/fixtures/wrapper_spm.py index 41c639629..46442315e 100644 --- a/projects/policyengine-simulation-executor/fixtures/wrapper_spm.py +++ b/projects/policyengine-simulation-executor/fixtures/wrapper_spm.py @@ -25,7 +25,17 @@ def storage_id(self) -> str: Read from ``policyengine-5.3.0-py3-none-any.whl`` sha256 ``8c640d967575dddad70840bcbe938cea251c56958eded647c83eb9c5902735f1``, the -build the native qualification lane installs. +unpublished development wheel the native qualification lane installs. The +hash is the identifier, not the version: another local build carries the +same filename and version and has no ``storage_id`` at all. + +``spm_config`` above is not a stored value. On the canonical wrapper it +refuses a non-US ``tax_benefit_model_version`` and otherwise re-resolves +``spm`` through the installed bundle, so an unset selection becomes the +bundle's defaults rather than None -- which is why this function takes a +config that has already been resolved, and why the agreement claim is +about the digest, not about the resolution. Both are checked against that +wheel in ``TestWrapperStorageIdAgreement``'s recorded out-of-band run. """ import hashlib diff --git a/projects/policyengine-simulation-executor/tests/test_artifact_keys.py b/projects/policyengine-simulation-executor/tests/test_artifact_keys.py index 1512b8330..9ece5ae4e 100644 --- a/projects/policyengine-simulation-executor/tests/test_artifact_keys.py +++ b/projects/policyengine-simulation-executor/tests/test_artifact_keys.py @@ -9,6 +9,8 @@ churn (or, worse, a writer/reader mismatch). """ +from types import SimpleNamespace + import pytest from fixtures.identity_stubs import install_identity_stubs @@ -218,15 +220,25 @@ class TestWrapperStorageIdAgreement: the wheel above, so our side cannot drift from the contract without a reviewable diff, and the digest cannot be quietly reformatted; * ``test_spm_arm_matches_the_installed_wrapper`` stops skipping and - starts asserting real equality on its own the moment an SPM-capable - wrapper is pinned. - - Agreement with the real 5.3.0 wrapper was checked out of band on - 2026-09-11 by resolving seven selection shapes through both that wheel's - ``resolve_spm_selection`` and this repo's, and comparing both the - resolved configs and the resulting storage ids: all seven agreed. That - is a recorded observation, not coverage — only the native lane re-runs - anything like it. + asserts real equality the moment an SPM-capable wrapper is pinned. It + supplies the two things that wrapper's ``spm_config`` needs and this + project cannot assume — a US model version, and a bundle pinning this + selection — because ``spm_config`` refuses a non-US model and + re-resolves the selection through ``get_current_bundle`` rather than + reading it off the object. + + Agreement with the real wrapper was checked out of band on 2026-09-11 by + running the executor environment with that wheel's ``policyengine`` + shadowing the pinned one, then resolving seven selection shapes (unset, + empty, national, county, metro, a moved ``as_of``, and a non-ASCII + scenario) through both the wheel's ``resolve_spm_selection`` and this + repo's, and comparing the resolved configs and the storage ids. All + seven agreed on both. That is a recorded observation, not coverage — + only the native lane re-runs anything like it. + + The wheel's sha256 is load-bearing, not decoration: two builds both + named ``policyengine-5.3.0-py3-none-any.whl`` are available locally, and + the other one (``962882ea…``) has no ``storage_id`` at all. """ @pytest.fixture @@ -291,12 +303,19 @@ def test_legacy_arm_matches_the_wrapper_accessor(self, identity): """The no-selection arm, through the accessor precompute uses. ``precompute`` reads ``getattr(baseline, "storage_id", baseline.id)``. - With no selection that has to be the planned id on *any* wrapper: - pre-canonical, because the attribute is absent and the fallback is - the id; canonical, because its property short-circuits to the id - when there is no config. The assertion is the same either way, so - this pins the accessor's contract rather than telling the two - wrappers apart. + With no resolved selection that has to be the planned id on *any* + wrapper: pre-canonical, because the attribute is absent and the + fallback is the id; canonical, because the property short-circuits + to the id when ``spm_config`` is None. The assertion is the same + either way, so this pins the accessor's contract rather than telling + the two wrappers apart. + + "No resolved selection" is the condition, not "the caller sent no + selection". On a canonical bundle the wrapper resolves an unset + ``spm`` into the bundle's defaults and returns a suffixed id — and + so does this project, through the same rule in + ``resolve_spm_selection``, which is why they still agree. A plain id + is what a bundle with no SPM measurement produces. """ from policyengine.core import Simulation @@ -312,9 +331,29 @@ def test_legacy_arm_matches_the_wrapper_accessor(self, identity): "pinned, replacing the transcribed expression above." ), ) - def test_spm_arm_matches_the_installed_wrapper(self, identity): + def test_spm_arm_matches_the_installed_wrapper(self, identity, monkeypatch): + """Real equality against the wrapper, once one can be imported. + + The canonical ``spm_config`` reads the model version's country and + re-resolves the selection through the installed bundle, so a bare + ``model_construct(id=..., spm=...)`` raises "SPM selection is only + supported by the US model" instead of comparing anything. Both are + supplied here so this activates on a pin rather than erroring. + """ + import policyengine.bundle from policyengine.core import Simulation built = identity(_SPM_SELECTION) - wrapper = Simulation.model_construct(id=built.simulation_id, spm=_SPM_SELECTION) + # Only after the planner has read the real installed bundle: this + # stub is for the wrapper's re-resolution, not for ours. + monkeypatch.setattr( + policyengine.bundle, + "get_current_bundle", + lambda: {"measurements": {"spm": _SPM_SELECTION}}, + ) + wrapper = Simulation.model_construct( + id=built.simulation_id, + spm=_SPM_SELECTION, + tax_benefit_model_version=SimpleNamespace(country_code="us"), + ) assert wrapper.storage_id == built.storage_id diff --git a/projects/policyengine-simulation-executor/tests/test_budget_window_scheduler.py b/projects/policyengine-simulation-executor/tests/test_budget_window_scheduler.py index 64bd0318f..4364f86d8 100644 --- a/projects/policyengine-simulation-executor/tests/test_budget_window_scheduler.py +++ b/projects/policyengine-simulation-executor/tests/test_budget_window_scheduler.py @@ -18,6 +18,7 @@ import src.modal.budget_window_scheduler as scheduler_module import policyengine_simulation_contract.budget_window_state as state_module from policyengine_simulation_contract.budget_window_state import ( + BUDGET_WINDOW_JOB_DICT_NAME, BUDGET_WINDOW_JOB_SEED_DICT_NAME, ) from policyengine_simulation_contract.spm import SPMInputError, SPMSelection @@ -381,6 +382,66 @@ def test_typed_result_validation_failure_persists_errors_and_stops_the_batch( assert [payload["time_period"] for payload in runtime.child_payloads] == ["2026"] +def test_a_typed_child_failure_cancels_the_siblings_already_running( + budget_window_semi_integration_client, +): + """The early return, where a single-parallel batch cannot show it. + + ``poll_running_children_once`` returns False on a typed failure instead + of continuing round the loop. With ``max_parallel=1`` there is never a + second running child, so replacing that return with ``continue`` changed + nothing observable. At two, the sibling that was already running must + be cancelled rather than harvested: the batch has failed, and a + completed 2027 alongside a failed 2026 would be a partial window nobody + asked for. + """ + client, runtime = budget_window_semi_integration_client + runtime.child_errors["2026"] = SPMInputError( + "SPM_GEOGRAPHY_REQUIRED", "County required" + ) + + batch_job_id = submit_budget_window(client, max_parallel=2) + assert client.get(f"/budget-window-jobs/{batch_job_id}").status_code == 202 + + body = client.get(f"/budget-window-jobs/{batch_job_id}").json() + + assert [payload["time_period"] for payload in runtime.child_payloads] == [ + "2026", + "2027", + ] + assert body["completed_years"] == [] + assert body["child_jobs"]["2027"]["status"] == "cancelled" + assert body["failed_years"] == ["2026"] + assert body["queued_years"] == ["2028"] + + +def test_a_typed_child_failure_is_persisted_not_only_returned( + budget_window_semi_integration_client, +): + """The poll body is serialized from the state run() returns in memory. + + Both typed branches write the re-attached errors back through + ``put_batch_job_state`` before returning, and nothing else re-reads the + batch-state dict, so deleting those writes left the suite green. A + later poll -- a different container, or the same one after a restart -- + reads the dict, so what is in it is what the client eventually sees. + """ + client, runtime = budget_window_semi_integration_client + runtime.child_errors["2026"] = SPMInputError( + "SPM_GEOGRAPHY_REQUIRED", "County required" + ) + + batch_job_id = submit_budget_window(client) + assert client.get(f"/budget-window-jobs/{batch_job_id}").status_code == 202 + assert client.get(f"/budget-window-jobs/{batch_job_id}").status_code == 400 + + persisted = runtime.dicts[BUDGET_WINDOW_JOB_DICT_NAME][batch_job_id] + expected = [{"code": "SPM_GEOGRAPHY_REQUIRED", "message": "County required"}] + assert persisted["status"] == "failed" + assert persisted["errors"] == expected + assert persisted["child_jobs"]["2026"]["errors"] == expected + + def test_untyped_child_failure_is_redacted_and_carries_no_typed_errors( budget_window_semi_integration_client, ): diff --git a/projects/policyengine-simulation-executor/tests/test_precompute.py b/projects/policyengine-simulation-executor/tests/test_precompute.py index 1212cc031..b96ed3c84 100644 --- a/projects/policyengine-simulation-executor/tests/test_precompute.py +++ b/projects/policyengine-simulation-executor/tests/test_precompute.py @@ -693,7 +693,9 @@ def ensure(self): state.spm = None state.baseline = StubBaseline.model_construct(id="bl1-cohort") state.make_silent = lambda: SilentBaseline.model_construct(id="bl1-cohort") - state.make_spm = lambda: SPMStubBaseline.model_construct(id="bl1-cohort") + state.make_spm = lambda sim_id="bl1-cohort": SPMStubBaseline.model_construct( + id=sim_id + ) class FakeStore: def __init__(self, bucket): @@ -800,6 +802,41 @@ def test_refuses_a_mismatched_planned_storage_id(self, cohort_stubs): assert cohort_stubs.uploads == [] assert not (cohort_stubs.folder / "bl1-cohort.h5").exists() + @staticmethod + def _planned_identity(selection): + """A real ``BaselineArtifactIdentity``, not a restated expression. + + The plan the deploy publishes comes from this object's properties, + so keying the test off them is what makes the write path a + writer==reader check: reformat or truncate ``storage_id``'s digest + and the container -- which derives its own the wrapper's way -- no + longer names the planned file. + """ + from policyengine_simulation_executor.artifact_keys import ( + BaselineArtifactIdentity, + DatasetArtifactIdentity, + ) + + return BaselineArtifactIdentity( + spm=selection, + country="us", + region="state/ca+state/wv", + scope_key="scoping", + dataset=DatasetArtifactIdentity( + spm=selection, + country="us", + dataset="populace_cps", + stem="populace", + year=2026, + data_version="1.2.3", + data_artifact_revision="rev-abc", + source_sha256=None, + data_build_fingerprint=None, + model_version="9.9.9", + policyengine_version="4.22.0", + ), + ) + def test_publishes_a_selection_scoped_artifact_under_the_planned_path( self, cohort_stubs ): @@ -808,10 +845,11 @@ def test_publishes_a_selection_scoped_artifact_under_the_planned_path( Nothing else in hermetic CI runs ``compute_baseline_impl`` with a selection: the identity's storage id was only ever compared to itself, and the one case that reached the guard covered the abort. - Here the plan is keyed the identity's way and the container names - its artifact the wrapper's way, so the guard passing at all is the - claim -- if the two derivations differed, no canonical artifact - could be published and the deploy would be blocked. + Here the plan is the planner's own ``BaselineArtifactIdentity`` and + the container names its artifact the wrapper's way, so the guard + passing at all is the claim -- if the two derivations differed, no + canonical artifact could be published and the deploy would be + blocked. """ selection = { "forecast_content_sha256": "a" * 64, @@ -821,25 +859,31 @@ def test_publishes_a_selection_scoped_artifact_under_the_planned_path( "county_vintage": "2020", "as_of": None, } + planned = self._planned_identity(selection) cohort_stubs.spm = selection - cohort_stubs.baseline = cohort_stubs.make_spm() + cohort_stubs.baseline = cohort_stubs.make_spm(planned.simulation_id) - # The planner's side: BaselineArtifactIdentity.storage_id's own - # expression, applied to this cohort's simulation id. - planned_storage_id = f"bl1-cohort-spm-{canonical_digest(selection)}" - entry = self._entry() - entry.path = f"baselines/us/bl-d/{planned_storage_id}.h5" + entry = self._entry(planned.simulation_id) + entry.digest = planned.digest + entry.path = planned.store_path result = precompute.compute_baseline_impl("bucket-x", entry) assert cohort_stubs.uploads == [ - (entry.path, str(cohort_stubs.folder / f"{planned_storage_id}.h5")) + ( + planned.store_path, + str(cohort_stubs.folder / f"{planned.storage_id}.h5"), + ) ] - assert result.simulation_id == "bl1-cohort" + assert result.simulation_id == planned.simulation_id assert result.uploaded is True assert result.size_bytes == len(b"artifact-bytes") + # The digest the container computed is the one the planner planned. + assert planned.storage_id == wrapper_storage_id( + planned.simulation_id, selection + ) # A selection-free plan for the same cohort is a different artifact. - assert planned_storage_id != "bl1-cohort" + assert planned.storage_id != planned.simulation_id def test_refuses_a_selection_scoped_plan_the_container_does_not_share( self, cohort_stubs From a1999a4f4e1079e40b8479aab19715c259ef2d62 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 13:26:55 -0400 Subject: [PATCH 39/40] Record the re-audit pass Co-Authored-By: Claude Opus 5 --- PROGRESS.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 2dbf8954a..1ef2edb58 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -6,8 +6,9 @@ Base: `main` = `414c631d622f5f587eedd187296a80182a923db2` ## State -All nine review findings are addressed. Suites green locally; pushing to -the PR branch next. +All nine review findings are addressed, and an adversarial re-audit of the +eight already-closed ones found three claims that only held because nothing +tested them; those are closed too (`fd869e7`). Suites green locally. ## Findings checklist @@ -68,6 +69,22 @@ the PR branch next. selection plus year range. Neither cache keeps exceptions, so fail-closed is unchanged, and the executor suite clears both around every test. +## Re-audit pass (`fd869e7`) + +- The skipped `test_spm_arm_matches_the_installed_wrapper` claimed it would + self-activate on a canonical pin. It would have raised instead: that + wrapper's `spm_config` refuses a non-US model version. It now supplies + the model version and the bundle the property re-resolves through. +- The precompute write-path test restated the identity's expression by + hand, so three mutations of the real `storage_id` left the precompute + suite green. It now builds a real `BaselineArtifactIdentity`; all three + fail it. +- The scheduler's typed branches ran only at `max_parallel=1`, so the early + return and the state write were both unpinned. Two cases now cover them. +- Agreement with the real wrapper is recorded, not just transcribed: eight + selection shapes resolve to the same config and storage id through the + qualification lane's 5.3.0 wheel and through this repo. + ## Not changed, deliberately - `changelog_entry.yaml`: no workflow, script or doc consumes it, there is From 054662a43306ae0b5a71b6ebe64f6310a08d967b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 11 Sep 2026 13:49:00 -0400 Subject: [PATCH 40/40] Drop the review-lane progress tracker from the repository root Co-Authored-By: Claude Fable 5.1 --- PROGRESS.md | 96 ----------------------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 PROGRESS.md diff --git a/PROGRESS.md b/PROGRESS.md deleted file mode 100644 index 1ef2edb58..000000000 --- a/PROGRESS.md +++ /dev/null @@ -1,96 +0,0 @@ -# PR #677 review fixes — progress - -Branch: `max/spm-simulation-canonical-fixes-20260911` (from PR head `b0447ca`) -Target PR branch: `max/spm-simulation-canonical-20260909` -Base: `main` = `414c631d622f5f587eedd187296a80182a923db2` - -## State - -All nine review findings are addressed, and an adversarial re-audit of the -eight already-closed ones found three claims that only held because nothing -tested them; those are closed too (`fd869e7`). Suites green locally. - -## Findings checklist - -- [x] Medium 1 — completed results drop explicit nulls from `spm_config` on the wire -- [x] Medium 2 — legacy no-SPM provenance `legacy-seed` expires on next publish -- [x] Medium 3 — precompute storage identity never shown to match the wrapper -- [x] Medium 4 — budget-window scheduler typed-error branches untested -- [x] Low 1 — legacy no-SPM poll bodies not byte-identical to base -- [x] Low 2 — stale country routes now fail; undocumented, untested -- [x] Low 3 — segmented reduce with SPM children untested end to end -- [x] Low 4 — hermetic CI never exercises receipt validation in `ensure()` -- [x] Low 5 — repeated prevalidation per request (perf) - -## Done - -- **Medium 1** (`b074b95`, `bf6764c`): `SPMSelection.serialize_selection` - keeps options that were explicitly selected as null when the caller - serializes with `exclude_none` (the poll routes do, via - `response_model_exclude_none`). Covered at the gateway - (`test_completed_result_body_keeps_resolved_nulls`, - `test_completed_budget_window_rows_keep_resolved_nulls`) and at the - contract layer, where reverting the serializer had left the suite green. -- **Medium 2** (`4992a19`): route provenance is derived from route shape - (`policyengine_version is None and schema_version == 1` -> - `legacy-country-route`) instead of the registry's rewritable `generation` - marker, and `_certified_model_version` stops reading a wrapper version as - a country model version, so a pinned 5.2.0/5.3.0 route with no manifest - stays historical. -- **Medium 3** (`ae9489e`, `3b28b9a`, `2b52202`): the planner's storage id - is held against the wrapper's own derivation — transcribed from the - 5.3.0 wheel the native lane installs, in `fixtures/wrapper_spm.py` — with - a golden, per-field rotation, a non-ASCII scenario, and a skipped case - that starts asserting real equality the moment an SPM-capable wrapper is - pinned. Hermetic CI now runs `compute_baseline_impl` under a real - selection with a container that names its artifact the wrapper's way, so - the guard *passing* is the claim, not only the abort. The native smoke - makes the same claim against the real wrapper. -- **Medium 4** (`94e980e`): per-year child injection seams and four cases - covering the typed child-call failure, the typed receipt-validation - failure, and the redaction fallback on each. -- **Low 1** (`b074b95`): `_bundle_payload` drops the `spm` key from raw - 202/500 job bodies and the worker `_metadata` when a route has no - capability, so a legacy no-SPM body is byte-identical to base; a - canonical route still carries the capability. -- **Low 2** (`215d6fa`): the resolver carries why it refuses, the gateway - README tells callers pinning an old version what to do, and cases pin the - refusal, the accepted route it is distinguished from, and all four ways a - manifest can state nothing. The same commit made "states nothing" mean - one thing: a blank manifest entry had produced a 400 naming no version. -- **Low 3** (`0c3f9c6`): `tests/test_segmented_national.py` drives a - segmented national reduce with SPM children end to end. -- **Low 4** (`638b7c6`, `3b28b9a`): a test double supplies the canonical - wrapper's SPM surface and restore timing, so every decision `ensure()` - makes about a receipt is fixed without the native source; the cache key - and snapshot replacement are under test too. -- **Low 5** (`19e27f8`): `runtime_spm_capability` and the selection - prevalidation are memoized on the installed image and on the resolved - selection plus year range. Neither cache keeps exceptions, so fail-closed - is unchanged, and the executor suite clears both around every test. - -## Re-audit pass (`fd869e7`) - -- The skipped `test_spm_arm_matches_the_installed_wrapper` claimed it would - self-activate on a canonical pin. It would have raised instead: that - wrapper's `spm_config` refuses a non-US model version. It now supplies - the model version and the bundle the property re-resolves through. -- The precompute write-path test restated the identity's expression by - hand, so three mutations of the real `storage_id` left the precompute - suite green. It now builds a real `BaselineArtifactIdentity`; all three - fail it. -- The scheduler's typed branches ran only at `max_parallel=1`, so the early - return and the state write were both unpinned. Two cases now cover them. -- Agreement with the real wrapper is recorded, not just transcribed: eight - selection shapes resolve to the same config and storage id through the - qualification lane's 5.3.0 wheel and through this repo. - -## Not changed, deliberately - -- `changelog_entry.yaml`: no workflow, script or doc consumes it, there is - no `CHANGELOG.md`, and `AGENTS.md` states there is no repository-wide - changelog fragment requirement. Low 2's "changelog/doc note" is the - gateway README section. -- `libs/policyengine-fastapi/src` carries nine pre-existing `ruff check` - F401s, present at the PR head and on `main`. CI's lint job runs - `ruff format --check` only, which passes.