| { |
| "examples": 192, |
| "seconds": 6.547292470932007, |
| "decoding": "raw greedy, no repair, no teacher fallback", |
| "checkpoint_sha256": "9489b5cde69c11f64b3e7031182e8248aa83790f1ad4aeefbf99a4e5e152e692", |
| "sql_metric": "Compile against the supplied schema, then compare results on two generated SQLite fixtures after dialect adaptation; not native MySQL/PostgreSQL execution.", |
| "metrics": { |
| "json_valid": { |
| "correct": 192, |
| "total": 192, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 192, |
| "total": 192, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 192, |
| "total": 192, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 192, |
| "total": 192, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 188, |
| "total": 192, |
| "rate": 0.9791666666666666 |
| }, |
| "sql_equivalent": { |
| "correct": 125, |
| "total": 128, |
| "rate": 0.9765625 |
| }, |
| "success": { |
| "correct": 188, |
| "total": 192, |
| "rate": 0.9791666666666666 |
| } |
| }, |
| "groups": { |
| "language:noisy_en": { |
| "json_valid": { |
| "correct": 48, |
| "total": 48, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 48, |
| "total": 48, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 48, |
| "total": 48, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 48, |
| "total": 48, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 48, |
| "total": 48, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 31, |
| "total": 31, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 48, |
| "total": 48, |
| "rate": 1.0 |
| } |
| }, |
| "backend:supabase": { |
| "json_valid": { |
| "correct": 92, |
| "total": 92, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 92, |
| "total": 92, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 92, |
| "total": 92, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 92, |
| "total": 92, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 90, |
| "total": 92, |
| "rate": 0.9782608695652174 |
| }, |
| "sql_equivalent": { |
| "correct": 64, |
| "total": 66, |
| "rate": 0.9696969696969697 |
| }, |
| "success": { |
| "correct": 90, |
| "total": 92, |
| "rate": 0.9782608695652174 |
| } |
| }, |
| "operation:missing_schema": { |
| "json_valid": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| } |
| }, |
| "operation:project": { |
| "json_valid": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 6, |
| "total": 7, |
| "rate": 0.8571428571428571 |
| }, |
| "sql_equivalent": { |
| "correct": 6, |
| "total": 7, |
| "rate": 0.8571428571428571 |
| }, |
| "success": { |
| "correct": 6, |
| "total": 7, |
| "rate": 0.8571428571428571 |
| } |
| }, |
| "language:hinglish": { |
| "json_valid": { |
| "correct": 52, |
| "total": 52, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 52, |
| "total": 52, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 52, |
| "total": 52, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 52, |
| "total": 52, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 51, |
| "total": 52, |
| "rate": 0.9807692307692307 |
| }, |
| "sql_equivalent": { |
| "correct": 36, |
| "total": 37, |
| "rate": 0.972972972972973 |
| }, |
| "success": { |
| "correct": 51, |
| "total": 52, |
| "rate": 0.9807692307692307 |
| } |
| }, |
| "operation:count_eq": { |
| "json_valid": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| } |
| }, |
| "language:en": { |
| "json_valid": { |
| "correct": 44, |
| "total": 44, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 44, |
| "total": 44, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 44, |
| "total": 44, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 44, |
| "total": 44, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 42, |
| "total": 44, |
| "rate": 0.9545454545454546 |
| }, |
| "sql_equivalent": { |
| "correct": 30, |
| "total": 31, |
| "rate": 0.967741935483871 |
| }, |
| "success": { |
| "correct": 42, |
| "total": 44, |
| "rate": 0.9545454545454546 |
| } |
| }, |
| "operation:group_count": { |
| "json_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| } |
| }, |
| "operation:distinct": { |
| "json_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| } |
| }, |
| "operation:year": { |
| "json_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| } |
| }, |
| "operation:max": { |
| "json_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| } |
| }, |
| "operation:bottom": { |
| "json_valid": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| } |
| }, |
| "operation:lt": { |
| "json_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| } |
| }, |
| "operation:avg": { |
| "json_valid": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| } |
| }, |
| "operation:sum": { |
| "json_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 3, |
| "total": 4, |
| "rate": 0.75 |
| }, |
| "sql_equivalent": { |
| "correct": 3, |
| "total": 4, |
| "rate": 0.75 |
| }, |
| "success": { |
| "correct": 3, |
| "total": 4, |
| "rate": 0.75 |
| } |
| }, |
| "language:hi": { |
| "json_valid": { |
| "correct": 48, |
| "total": 48, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 48, |
| "total": 48, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 48, |
| "total": 48, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 48, |
| "total": 48, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 47, |
| "total": 48, |
| "rate": 0.9791666666666666 |
| }, |
| "sql_equivalent": { |
| "correct": 28, |
| "total": 29, |
| "rate": 0.9655172413793104 |
| }, |
| "success": { |
| "correct": 47, |
| "total": 48, |
| "rate": 0.9791666666666666 |
| } |
| }, |
| "operation:project_eq": { |
| "json_valid": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| } |
| }, |
| "operation:not_null": { |
| "json_valid": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| } |
| }, |
| "operation:or": { |
| "json_valid": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| } |
| }, |
| "operation:date_after": { |
| "json_valid": { |
| "correct": 10, |
| "total": 10, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 10, |
| "total": 10, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 10, |
| "total": 10, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 10, |
| "total": 10, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 10, |
| "total": 10, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 10, |
| "total": 10, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 10, |
| "total": 10, |
| "rate": 1.0 |
| } |
| }, |
| "operation:top": { |
| "json_valid": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| } |
| }, |
| "operation:having": { |
| "json_valid": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| } |
| }, |
| "operation:min": { |
| "json_valid": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 5, |
| "total": 5, |
| "rate": 1.0 |
| } |
| }, |
| "operation:gt": { |
| "json_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| } |
| }, |
| "backend:mysql": { |
| "json_valid": { |
| "correct": 100, |
| "total": 100, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 100, |
| "total": 100, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 100, |
| "total": 100, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 100, |
| "total": 100, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 98, |
| "total": 100, |
| "rate": 0.98 |
| }, |
| "sql_equivalent": { |
| "correct": 61, |
| "total": 62, |
| "rate": 0.9838709677419355 |
| }, |
| "success": { |
| "correct": 98, |
| "total": 100, |
| "rate": 0.98 |
| } |
| }, |
| "operation:count": { |
| "json_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| } |
| }, |
| "operation:group_sum": { |
| "json_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| } |
| }, |
| "operation:between": { |
| "json_valid": { |
| "correct": 6, |
| "total": 6, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 6, |
| "total": 6, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 6, |
| "total": 6, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 6, |
| "total": 6, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 6, |
| "total": 6, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 6, |
| "total": 6, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 6, |
| "total": 6, |
| "rate": 1.0 |
| } |
| }, |
| "operation:sort_asc": { |
| "json_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| } |
| }, |
| "operation:sort_desc": { |
| "json_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| } |
| }, |
| "operation:sql_error": { |
| "json_valid": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 8, |
| "total": 9, |
| "rate": 0.8888888888888888 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 8, |
| "total": 9, |
| "rate": 0.8888888888888888 |
| } |
| }, |
| "operation:all": { |
| "json_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 3, |
| "total": 4, |
| "rate": 0.75 |
| }, |
| "sql_equivalent": { |
| "correct": 3, |
| "total": 4, |
| "rate": 0.75 |
| }, |
| "success": { |
| "correct": 3, |
| "total": 4, |
| "rate": 0.75 |
| } |
| }, |
| "operation:and": { |
| "json_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| } |
| }, |
| "operation:month": { |
| "json_valid": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| } |
| }, |
| "operation:join_filter": { |
| "json_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| } |
| }, |
| "operation:eq": { |
| "json_valid": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| } |
| }, |
| "operation:list_tables": { |
| "json_valid": { |
| "correct": 12, |
| "total": 12, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 12, |
| "total": 12, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 12, |
| "total": 12, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 12, |
| "total": 12, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 12, |
| "total": 12, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 12, |
| "total": 12, |
| "rate": 1.0 |
| } |
| }, |
| "operation:gte": { |
| "json_valid": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| } |
| }, |
| "operation:lte": { |
| "json_valid": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| } |
| }, |
| "operation:describe": { |
| "json_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 4, |
| "total": 4, |
| "rate": 1.0 |
| } |
| }, |
| "operation:join": { |
| "json_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 3, |
| "total": 3, |
| "rate": 1.0 |
| } |
| }, |
| "operation:null": { |
| "json_valid": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 2, |
| "total": 2, |
| "rate": 1.0 |
| } |
| }, |
| "operation:contains": { |
| "json_valid": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| }, |
| "success": { |
| "correct": 1, |
| "total": 1, |
| "rate": 1.0 |
| } |
| }, |
| "operation:search": { |
| "json_valid": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| } |
| }, |
| "operation:ticket": { |
| "json_valid": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 9, |
| "total": 9, |
| "rate": 1.0 |
| } |
| }, |
| "operation:weather": { |
| "json_valid": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| } |
| }, |
| "operation:read_file": { |
| "json_valid": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "schema_valid": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "action_correct": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "tool_correct": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "arguments_exact": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| }, |
| "sql_equivalent": { |
| "correct": 0, |
| "total": 0, |
| "rate": 0.0 |
| }, |
| "success": { |
| "correct": 7, |
| "total": 7, |
| "rate": 1.0 |
| } |
| } |
| }, |
| "rescoring": "Same saved raw outputs, with explicit supplied-schema compilation; no generation or output repair." |
| } |
