Add parallel live simulation coverage

.github/scripts/run_live_staging_tests.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+python -m pytest \
+  -n 2 \
+  --dist load \
+  --maxschedchunk=1 \
+  tests/integration/test_live_budget_window_cache.py \
+  tests/integration/test_live_calculate.py \
+  tests/integration/test_live_economy.py \
+  -v

.github/workflows/push.yml

@@ -298,19 +298,45 @@ jobs:
           python-version: "3.12"
       - name: Install staging test dependencies
         run: pip install pytest httpx sqlalchemy psycopg[binary]
-      - name: Run staging smoke test
-        run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_v2_policies.py tests/integration/test_live_calculate.py tests/integration/test_live_economy.py tests/integration/test_live_budget_window_cache.py -v
+      - name: Run staging integration tests
+        run: python -m pytest tests/integration/test_cloud_run_candidate.py tests/integration/test_live_v2_metadata.py tests/integration/test_live_v2_policies.py -v
         env:
           API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}
           STAGING_API_TEST_PROBE_ID: cloud-run-${{ needs.deploy-cloud-run-staging.outputs.tag }}
           V2_MIGRATION_DATABASE_URL: ${{ secrets.V2_MIGRATION_DATABASE_URL }}
 
+  parallel-live-simulation-tests-staging-cloud-run:
+    name: Run live simulations against the staging candidate
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    needs:
+      - deploy-cloud-run-staging
+      - integration-tests-staging-cloud-run
+    if: |
+      (github.repository == 'PolicyEngine/policyengine-api')
+      && (github.event.head_commit.message == 'Update PolicyEngine API')
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install live test dependencies
+        run: pip install pytest pytest-xdist httpx
+      - name: Run live staging tests in two pytest workers
+        run: bash .github/scripts/run_live_staging_tests.sh
+        env:
+          API_BASE_URL: ${{ needs.deploy-cloud-run-staging.outputs.url }}
+          STAGING_API_TEST_PROBE_ID: cloud-run-${{ needs.deploy-cloud-run-staging.outputs.tag }}-${{ github.run_id }}-${{ github.run_attempt }}
+
   promote-cloud-run-staging:
     name: Promote staging Cloud Run traffic
     runs-on: ubuntu-latest
     needs:
       - deploy-cloud-run-staging
       - integration-tests-staging-cloud-run
+      - parallel-live-simulation-tests-staging-cloud-run
     if: |
       (github.repository == 'PolicyEngine/policyengine-api')
       && (github.event.head_commit.message == 'Update PolicyEngine API')

changelog.d/2400.added.md

@@ -0,0 +1 @@
+Add parallel live staging coverage for US federal and California tax credits and UK Universal Credit calculations.

tests/data/california_eitc_reform.json

@@ -0,0 +1,7 @@
+{
+  "data": {
+    "gov.states.ca.tax.income.credits.earned_income.phase_in.rate[1].amount": {
+      "2025-01-01.2100-12-31": 0.35
+    }
+  }
+}

tests/data/live_uk_universal_credit_household.json

@@ -0,0 +1,48 @@
+{
+  "household": {
+    "benunits": {
+      "benefit unit": {
+        "members": [
+          "parent",
+          "child"
+        ],
+        "universal_credit": {
+          "2025": null
+        },
+        "would_claim_uc": {
+          "2025": true
+        }
+      }
+    },
+    "households": {
+      "household": {
+        "members": [
+          "parent",
+          "child"
+        ],
+        "region": {
+          "2025": "SCOTLAND"
+        }
+      }
+    },
+    "people": {
+      "parent": {
+        "age": {
+          "2025": 35
+        },
+        "employment_income": {
+          "2025": 12000
+        }
+      },
+      "child": {
+        "age": {
+          "2025": 6
+        },
+        "employment_income": {
+          "2025": 0
+        }
+      }
+    }
+  },
+  "policy": {}
+}

tests/data/live_us_credits_household.json

@@ -0,0 +1,85 @@
+{
+  "household": {
+    "families": {
+      "family": {
+        "members": [
+          "parent",
+          "child"
+        ]
+      }
+    },
+    "households": {
+      "household": {
+        "members": [
+          "parent",
+          "child"
+        ],
+        "state_name": {
+          "2025": "CA"
+        }
+      }
+    },
+    "marital_units": {
+      "parent's marital unit": {
+        "members": [
+          "parent"
+        ]
+      },
+      "child's marital unit": {
+        "marital_unit_id": {
+          "2025": 1
+        },
+        "members": [
+          "child"
+        ]
+      }
+    },
+    "people": {
+      "parent": {
+        "age": {
+          "2025": 35
+        },
+        "employment_income": {
+          "2025": 20000
+        }
+      },
+      "child": {
+        "age": {
+          "2025": 6
+        },
+        "employment_income": {
+          "2025": 0
+        },
+        "is_tax_unit_dependent": {
+          "2025": true
+        }
+      }
+    },
+    "spm_units": {
+      "spm unit": {
+        "members": [
+          "parent",
+          "child"
+        ]
+      }
+    },
+    "tax_units": {
+      "tax unit": {
+        "members": [
+          "parent",
+          "child"
+        ],
+        "eitc": {
+          "2025": null
+        },
+        "ctc": {
+          "2025": null
+        },
+        "ca_eitc": {
+          "2025": null
+        }
+      }
+    }
+  },
+  "policy": {}
+}

tests/data/uk_universal_credit_reform.json

@@ -0,0 +1,7 @@
+{
+  "data": {
+    "gov.dwp.universal_credit.standard_allowance.amount.SINGLE_OLD": {
+      "2025-01-01.2100-12-31": 500
+    }
+  }
+}

tests/integration/conftest.py

@@ -85,6 +85,16 @@ def _response_summary(response: httpx.Response) -> str:
     return f"HTTP {response.status_code}: {response.text[:500]}"
 
 
+def _structured_error_payload(response: httpx.Response) -> dict | None:
+    try:
+        payload = response.json()
+    except ValueError:
+        return None
+    if isinstance(payload, dict) and payload.get("status") == "error":
+        return payload
+    return None
+
+
 def _poll_live_endpoint(
     api_client: httpx.Client,
     path: str,
@@ -102,6 +112,9 @@ def _poll_live_endpoint(
             last_response = f"{type(error).__name__}: {error}"
         else:
             if response.status_code in TRANSIENT_POLL_STATUS_CODES:
+                error_payload = _structured_error_payload(response)
+                if error_payload is not None:
+                    return error_payload
                 last_response = _response_summary(response)
             else:
                 response.raise_for_status()

tests/integration/test_live_calculate.py

@@ -1,12 +1,19 @@
+import json
 from pathlib import Path
 
+import pytest
+
 
 def _load_payload(filename: str) -> str:
     return (Path(__file__).resolve().parents[1] / "data" / filename).read_text(
         encoding="utf-8"
     )
 
 
+def _load_json_payload(filename: str) -> dict:
+    return json.loads(_load_payload(filename))
+
+
 def test_live_calculate_us_1(api_client):
     response = api_client.post(
         "/us/calculate",
@@ -61,3 +68,40 @@ def test_live_calculate_drops_deprecated_medical_input(
         "medical_out_of_pocket_expenses" in warning and "deprecated" in warning.lower()
         for warning in payload["warnings"]
     )
+
+
+def test_live_calculate_us_federal_and_california_credits(
+    api_client,
+    integration_probe_id,
+):
+    request_payload = _load_json_payload("live_us_credits_household.json")
+    request_payload["integration_probe_id"] = f"{integration_probe_id}-us-credits"
+
+    response = api_client.post("/us/calculate", json=request_payload)
+
+    assert response.status_code == 200, response.text
+    payload = response.json()
+    assert payload["status"] == "ok", payload
+    tax_unit = payload["result"]["tax_units"]["tax unit"]
+    assert tax_unit["eitc"]["2025"] == pytest.approx(4328.0, abs=0.01)
+    assert tax_unit["ctc"]["2025"] == pytest.approx(2200.0, abs=0.01)
+    assert tax_unit["ca_eitc"]["2025"] == pytest.approx(395.86, abs=0.01)
+
+
+def test_live_calculate_uk_universal_credit_in_scotland(
+    api_client,
+    integration_probe_id,
+):
+    request_payload = _load_json_payload("live_uk_universal_credit_household.json")
+    request_payload["integration_probe_id"] = f"{integration_probe_id}-uk-uc"
+
+    response = api_client.post("/uk/calculate", json=request_payload)
+
+    assert response.status_code == 200, response.text
+    payload = response.json()
+    assert payload["status"] == "ok", payload
+    benefit_unit = payload["result"]["benunits"]["benefit unit"]
+    assert benefit_unit["universal_credit"]["2025"] == pytest.approx(
+        6229.80,
+        abs=0.01,
+    )

tests/integration/test_live_economy.py

@@ -74,3 +74,75 @@ def test_live_utah_macro_reform(api_client, integration_probe_id, poll_live_endp
     assert isinstance(lose_less_than_5, int | float), result
     assert math.isfinite(lose_less_than_5), result
     assert 0 <= lose_less_than_5 <= 1, result
+
+
+def _assert_live_macro_reform(
+    api_client,
+    integration_probe_id,
+    poll_live_endpoint,
+    *,
+    country_id: str,
+    reform_filename: str,
+    region: str,
+    probe_suffix: str,
+) -> None:
+    metadata_response = api_client.get(f"/{country_id}/metadata")
+    metadata_response.raise_for_status()
+    metadata = metadata_response.json()["result"]
+
+    policy_response = api_client.post(
+        f"/{country_id}/policy",
+        json=_load_reform_payload(reform_filename),
+    )
+    assert policy_response.status_code in (200, 201), policy_response.text
+    policy_id = policy_response.json()["result"]["policy_id"]
+
+    payload = poll_live_endpoint(
+        api_client,
+        f"/{country_id}/economy/{policy_id}/over/{metadata['current_law_id']}",
+        {
+            "region": region,
+            "time_period": _pick_time_period(metadata),
+            "staging_probe": f"{integration_probe_id}-{probe_suffix}",
+        },
+        route_name=f"{country_id}-{probe_suffix}-economy",
+    )
+
+    assert payload["status"] == "ok", payload
+    result = payload["result"]
+    assert result is not None, payload
+    budgetary_impact = result["budget"]["budgetary_impact"]
+    assert isinstance(budgetary_impact, int | float), result
+    assert math.isfinite(budgetary_impact), result
+
+
+def test_live_california_eitc_macro_reform(
+    api_client,
+    integration_probe_id,
+    poll_live_endpoint,
+):
+    _assert_live_macro_reform(
+        api_client,
+        integration_probe_id,
+        poll_live_endpoint,
+        country_id="us",
+        reform_filename="california_eitc_reform.json",
+        region="state/ca",
+        probe_suffix="california-eitc",
+    )
+
+
+def test_live_uk_universal_credit_macro_reform_in_scotland(
+    api_client,
+    integration_probe_id,
+    poll_live_endpoint,
+):
+    _assert_live_macro_reform(
+        api_client,
+        integration_probe_id,
+        poll_live_endpoint,
+        country_id="uk",
+        reform_filename="uk_universal_credit_reform.json",
+        region="country/scotland",
+        probe_suffix="uk-scotland-uc",
+    )