Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 19 additions & 7 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -141,20 +141,27 @@ jobs:
run: |
if [[ -z "${BASE_SHA:-}" || "$BASE_SHA" =~ ^0+$ ]]; then
echo "run=true" >> "$GITHUB_OUTPUT"
echo "run_live=true" >> "$GITHUB_OUTPUT"
exit 0
fi

changed_files="$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")"
echo "$changed_files"
run=false
run_live=false
while IFS= read -r file; do
case "$file" in
.github/workflows/test.yml|scripts/agent_eval.py|packages/shared/src/five08/agent/*|packages/shared/src/five08/model_catalog.py|packages/shared/src/five08/data/model-profiles.json|apps/discord_bot/src/five08/discord_bot/cogs/agent.py|tests/evals/discord-agent/*|tests/unit/test_agent_cog.py|tests/unit/test_agent_evals.py|tests/unit/test_agent_gateway.py|pyproject.toml|uv.lock)
echo "run=true" >> "$GITHUB_OUTPUT"
exit 0
.github/workflows/test.yml|scripts/agent_eval.py|packages/shared/src/five08/agent/*|packages/shared/src/five08/model_catalog.py|packages/shared/src/five08/data/model-profiles.json|apps/discord_bot/src/five08/discord_bot/cogs/agent.py|tests/evals/discord-agent/*)
run=true
run_live=true
;;
tests/unit/test_agent_cog.py|tests/unit/test_agent_evals.py|tests/unit/test_agent_gateway.py|pyproject.toml|uv.lock)
run=true
;;
esac
done <<< "$changed_files"
echo "run=false" >> "$GITHUB_OUTPUT"
echo "run=$run" >> "$GITHUB_OUTPUT"
echo "run_live=$run_live" >> "$GITHUB_OUTPUT"

- name: Skip Discord agent evals for unrelated changes
if: steps.agent-eval-changes.outputs.run != 'true'
Expand Down Expand Up @@ -182,7 +189,7 @@ jobs:
uv run python scripts/agent_eval.py --suite canonical --model primary --no-env-file

- name: Check live planner credentials
if: steps.agent-eval-changes.outputs.run == 'true'
if: steps.agent-eval-changes.outputs.run_live == 'true'
id: live-planner-key
env:
LIVE_EVAL_DIRECT_KEY: ${{ secrets.OPENAI_API_KEY_DIRECT }}
Expand All @@ -202,7 +209,7 @@ jobs:
fi

- name: Run canonical Discord agent live-planner evals
if: steps.agent-eval-changes.outputs.run == 'true' && steps.live-planner-key.outputs.available == 'true'
if: steps.agent-eval-changes.outputs.run_live == 'true' && steps.live-planner-key.outputs.available == 'true'
env:
LIVE_EVAL_DIRECT_KEY: ${{ secrets.OPENAI_API_KEY_DIRECT }}
LIVE_EVAL_PROVIDER_KEY: ${{ secrets.OPENAI_API_KEY }}
Expand All @@ -216,10 +223,15 @@ jobs:
fi

- name: Skip live planner evals without provider credentials
if: steps.agent-eval-changes.outputs.run == 'true' && steps.live-planner-key.outputs.available != 'true'
if: steps.agent-eval-changes.outputs.run_live == 'true' && steps.live-planner-key.outputs.available != 'true'
run: |
echo "Skipping live planner evals because the matching OPENAI_API_KEY_DIRECT/OPENAI_API_KEY credential is not configured."

- name: Skip live planner evals for non-agent changes
if: steps.agent-eval-changes.outputs.run == 'true' && steps.agent-eval-changes.outputs.run_live != 'true'
run: |
echo "No agent behavior or fixture changes detected; skipping live planner evals."

- name: Upload Discord agent eval reports
uses: actions/upload-artifact@v7
if: always() && steps.agent-eval-changes.outputs.run == 'true'
Expand Down
Loading
Loading