diff --git a/contributing/samples/gepa/experiment.py b/contributing/samples/gepa/experiment.py index 2f5d03a772..f68b349d9c 100644 --- a/contributing/samples/gepa/experiment.py +++ b/contributing/samples/gepa/experiment.py @@ -43,7 +43,6 @@ from tau_bench.types import EnvRunResult from tau_bench.types import RunConfig import tau_bench_agent as tau_bench_agent_lib - import utils diff --git a/contributing/samples/gepa/run_experiment.py b/contributing/samples/gepa/run_experiment.py index cfd850b3a3..1bc4ee58c8 100644 --- a/contributing/samples/gepa/run_experiment.py +++ b/contributing/samples/gepa/run_experiment.py @@ -25,7 +25,6 @@ from absl import flags import experiment from google.genai import types - import utils _OUTPUT_DIR = flags.DEFINE_string( diff --git a/src/google/adk/cli/cli_create.py b/src/google/adk/cli/cli_create.py index 9085586e18..6fee8d855c 100644 --- a/src/google/adk/cli/cli_create.py +++ b/src/google/adk/cli/cli_create.py @@ -21,6 +21,8 @@ import click +from ..apps.app import validate_app_name + _INIT_PY_TEMPLATE = """\ from . import agent """ @@ -294,6 +296,12 @@ def run_cmd( VertexAI as backend. type: Optional[str], Whether to define agent with config file or code. """ + # Validate agent name is a valid Python identifier + try: + validate_app_name(agent_name) + except ValueError as e: + raise click.UsageError(str(e)) from e + agent_folder = os.path.join(os.getcwd(), agent_name) # check folder doesn't exist or it's empty. Otherwise, throw if os.path.exists(agent_folder) and os.listdir(agent_folder): diff --git a/src/google/adk/evaluation/eval_metrics.py b/src/google/adk/evaluation/eval_metrics.py index b7c544ccad..95ac8ba768 100644 --- a/src/google/adk/evaluation/eval_metrics.py +++ b/src/google/adk/evaluation/eval_metrics.py @@ -23,6 +23,7 @@ from pydantic import BaseModel from pydantic import ConfigDict from pydantic import Field +from pydantic import field_validator from pydantic.json_schema import SkipJsonSchema from typing_extensions import TypeAlias @@ -224,6 +225,20 @@ class MatchType(Enum): ), ) + @field_validator("match_type", mode="before") + @classmethod + def _validate_match_type(cls, v): + """Convert string enum names to enum values for backward compatibility.""" + if isinstance(v, str): + try: + return cls.MatchType[v] + except KeyError: + raise ValueError( + f"Invalid match_type: '{v}'. Must be one of: " + f"{', '.join(cls.MatchType.__members__)}" + ) + return v + class LlmBackedUserSimulatorCriterion(LlmAsAJudgeCriterion): """Criterion for LLM-backed User Simulator Evaluators.""" diff --git a/tests/unittests/cli/utils/test_cli_create.py b/tests/unittests/cli/utils/test_cli_create.py index 33b3f877a8..d41ac66279 100644 --- a/tests/unittests/cli/utils/test_cli_create.py +++ b/tests/unittests/cli/utils/test_cli_create.py @@ -301,3 +301,51 @@ def test_get_gcp_region_from_gcloud_fail( ), ) assert cli_create._get_gcp_region_from_gcloud() == "" + + +# run_cmd validation +@pytest.mark.parametrize( + "invalid_name, error_substring", + [ + ("my-agent", "must be a valid identifier"), + ("my agent", "must be a valid identifier"), + ("user", "reserved for end-user input"), + ], +) +def test_run_cmd_rejects_invalid_agent_names( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + invalid_name: str, + error_substring: str, +) -> None: + """run_cmd should reject invalid agent names.""" + monkeypatch.setattr(os, "getcwd", lambda: str(tmp_path)) + with pytest.raises(click.UsageError) as exc_info: + cli_create.run_cmd( + invalid_name, + model="gemini-2.5-flash", + google_api_key="test-key", + google_cloud_project=None, + google_cloud_region=None, + type="code", + ) + assert error_substring in str(exc_info.value) + + +@pytest.mark.parametrize("valid_name", ["my_agent", "agent123"]) +def test_run_cmd_accepts_valid_agent_names( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, valid_name: str +) -> None: + """run_cmd should accept valid Python identifiers.""" + monkeypatch.setattr(os, "getcwd", lambda: str(tmp_path)) + monkeypatch.setattr(click, "prompt", lambda *a, **k: "1") # Choose type + + cli_create.run_cmd( + valid_name, + model="gemini-2.5-flash", + google_api_key="test-key", + google_cloud_project=None, + google_cloud_region=None, + type="code", + ) + assert (tmp_path / valid_name).exists() diff --git a/tests/unittests/evaluation/test_trajectory_evaluator.py b/tests/unittests/evaluation/test_trajectory_evaluator.py index 0795739768..b8467f3d83 100644 --- a/tests/unittests/evaluation/test_trajectory_evaluator.py +++ b/tests/unittests/evaluation/test_trajectory_evaluator.py @@ -407,3 +407,39 @@ def test_evaluate_invocations_no_invocations(evaluator: TrajectoryEvaluator): assert result.overall_score is None assert result.overall_eval_status == EvalStatus.NOT_EVALUATED assert not result.per_invocation_results + + +@pytest.mark.parametrize( + "match_type_input, expected_enum", + [ + # String values + ("EXACT", ToolTrajectoryCriterion.MatchType.EXACT), + ("IN_ORDER", ToolTrajectoryCriterion.MatchType.IN_ORDER), + ("ANY_ORDER", ToolTrajectoryCriterion.MatchType.ANY_ORDER), + # Enum values + ( + ToolTrajectoryCriterion.MatchType.IN_ORDER, + ToolTrajectoryCriterion.MatchType.IN_ORDER, + ), + # Integer values + (0, ToolTrajectoryCriterion.MatchType.EXACT), + (1, ToolTrajectoryCriterion.MatchType.IN_ORDER), + (2, ToolTrajectoryCriterion.MatchType.ANY_ORDER), + ], +) +def test_tool_trajectory_criterion_accepts_valid_match_types( + match_type_input, expected_enum +): + """Tests that ToolTrajectoryCriterion accepts string, enum, and int values.""" + criterion = ToolTrajectoryCriterion( + threshold=0.5, match_type=match_type_input + ) + assert criterion.match_type == expected_enum + + +def test_tool_trajectory_criterion_rejects_invalid_string(): + """Tests that ToolTrajectoryCriterion rejects invalid string values.""" + with pytest.raises(ValueError) as exc_info: + ToolTrajectoryCriterion(threshold=0.5, match_type="INVALID") + assert "Invalid match_type" in str(exc_info.value) + assert "EXACT, IN_ORDER, ANY_ORDER" in str(exc_info.value)