diff --git a/src/eval/tasks/bfcl/task_context/bfcl_evaluation_code.py b/src/eval/tasks/bfcl/task_context/bfcl_evaluation_code.py index 5aedc8aa..a4698ca7 100644 --- a/src/eval/tasks/bfcl/task_context/bfcl_evaluation_code.py +++ b/src/eval/tasks/bfcl/task_context/bfcl_evaluation_code.py @@ -78,8 +78,16 @@ async def score(state: TaskState, target: Target) -> Score: target_obj = state.metadata["target_obj"] - args_identical = tool_calls[0].arguments == target_obj["arguments"] function_identical = tool_calls[0].function == target_obj["function"] + args_identical = function_identical and canonicalize( + tool_calls[0].function, + tool_calls[0].arguments, + state.metadata["tools"], + ) == canonicalize( + target_obj["function"], + target_obj["arguments"], + state.metadata["tools"], + ) logger.info( f"args: {tool_calls[0].arguments} == {target_obj['arguments']}\nfunction: {tool_calls[0].function} == {target_obj['function']}" ) @@ -94,6 +102,25 @@ async def score(state: TaskState, target: Target) -> Score: return score +def canonicalize( + function_name: str, + arguments: dict[str, Any], + tools: list[dict[str, Any]], +) -> dict[str, Any]: + """Fill optional argument defaults before comparing tool calls.""" + tool = next(tool for tool in tools if tool["name"] == function_name) + parameters = tool["parameters"] + required = set(parameters.get("required") or []) + defaults = { + name: schema["default"] + for name, schema in (parameters.get("properties") or {}).items() + if name not in required + and isinstance(schema, dict) + and "default" in schema + } + return defaults | arguments + + def record_to_sample(record: dict[str, Any]) -> Sample: assert len(record["question"]) == 1 assert len(record["ground_truth"]) == 1