diff --git a/src/google/adk/cli/cli_eval.py b/src/google/adk/cli/cli_eval.py index 15a311dff28..43e5321bcce 100644 --- a/src/google/adk/cli/cli_eval.py +++ b/src/google/adk/cli/cli_eval.py @@ -101,15 +101,27 @@ def parse_and_get_evals_to_run( eval_set_to_evals: dict[str, list[str]] = {} for input_eval_set in evals_to_run_info: evals = [] - if ":" not in input_eval_set: + drive_letter = input_eval_set[:1] + has_windows_drive_prefix = ( + len(input_eval_set) >= 3 + and drive_letter.isascii() + and drive_letter.isalpha() + and input_eval_set[1] == ":" + and input_eval_set[2] in ("\\", "/") + ) + selector_separator_index = input_eval_set.find( + ":", 3 if has_windows_drive_prefix else 0 + ) + if selector_separator_index == -1: # We don't have any eval cases specified. This would be the case where the # the user wants to run all eval cases in the eval set. eval_set = input_eval_set else: # There are eval cases that we need to parse. The user wants to run # specific eval cases from the eval set. - eval_set = input_eval_set.split(":")[0] - evals = input_eval_set.split(":")[1].split(",") + eval_set = input_eval_set[:selector_separator_index] + selector_list = input_eval_set[selector_separator_index + 1 :] + evals = selector_list.split(":")[0].split(",") evals = [s for s in evals if s.strip()] if eval_set not in eval_set_to_evals: diff --git a/tests/unittests/cli/utils/test_cli_eval.py b/tests/unittests/cli/utils/test_cli_eval.py index c6d21fa7078..5171a8efed3 100644 --- a/tests/unittests/cli/utils/test_cli_eval.py +++ b/tests/unittests/cli/utils/test_cli_eval.py @@ -19,6 +19,67 @@ from types import SimpleNamespace from unittest import mock +import pytest + + +@pytest.mark.parametrize( + ("input_eval_set", "expected"), + [ + pytest.param( + r"C:\tmp\agent\eval.evalset.json", + {r"C:\tmp\agent\eval.evalset.json": []}, + id="windows-backslash-path-without-selectors", + ), + pytest.param( + r"C:\tmp\agent\eval.evalset.json:case1", + {r"C:\tmp\agent\eval.evalset.json": ["case1"]}, + id="windows-backslash-path-with-one-selector", + ), + pytest.param( + r"C:\tmp\agent\eval.evalset.json:case1,case2", + {r"C:\tmp\agent\eval.evalset.json": ["case1", "case2"]}, + id="windows-backslash-path-with-multiple-selectors", + ), + pytest.param( + "C:/tmp/agent/eval.evalset.json:case1", + {"C:/tmp/agent/eval.evalset.json": ["case1"]}, + id="windows-forward-slash-path", + ), + pytest.param( + r"d:\tmp\agent\eval.evalset.json:case1", + {r"d:\tmp\agent\eval.evalset.json": ["case1"]}, + id="lowercase-windows-drive", + ), + pytest.param( + "/tmp/agent/eval.evalset.json:case1,case2", + {"/tmp/agent/eval.evalset.json": ["case1", "case2"]}, + id="posix-path-with-selectors", + ), + pytest.param( + "my_eval_set:case1,case2", + {"my_eval_set": ["case1", "case2"]}, + id="eval-set-id-with-selectors", + ), + pytest.param( + "my_eval_set", + {"my_eval_set": []}, + id="eval-set-id-without-selectors", + ), + pytest.param( + "/tmp/agent/eval.evalset.json", + {"/tmp/agent/eval.evalset.json": []}, + id="posix-path-without-selectors", + ), + ], +) +def test_parse_and_get_evals_to_run_parses_eval_set_and_selectors( + input_eval_set: str, expected: dict[str, list[str]] +): + """Eval-set paths and IDs retain their optional case selectors.""" + from google.adk.cli.cli_eval import parse_and_get_evals_to_run + + assert parse_and_get_evals_to_run([input_eval_set]) == expected + def test_get_eval_sets_manager_local(monkeypatch): mock_local_manager = mock.MagicMock()