| """Audit every reference action and execute each distinct SQL case on two fixtures.""" |
| import argparse |
| import hashlib |
| import json |
| import sqlite3 |
| from pathlib import Path |
| import time |
| from jsonschema import Draft202012Validator |
| from tinyquery.data import SQL_OPS,compact,serialize,validate_sql,sqlite_sql |
| from tinyquery.evaluate import check_action |
|
|
|
|
| def main(): |
| p=argparse.ArgumentParser();p.add_argument('--data',required=True);p.add_argument('--out',required=True) |
| args=p.parse_args();start=time.time();ids=set();prompts=set();queries=set();context_queries=set();validators={};errors=[];count=0;sql_rows=0 |
| for line in Path(args.data).open(): |
| row=json.loads(line);count+=1 |
| try: |
| assert row['id'] not in ids,'Duplicate ID' |
| ids.add(row['id']);fingerprint=hashlib.sha256(row['prompt'].encode()).hexdigest() |
| assert fingerprint not in prompts,'Duplicate prompt' |
| prompts.add(fingerprint) |
| assert row['prompt']==serialize(row['context'],row['question']),'Prompt serialization differs' |
| assert row['response']==compact(row['target']),'Response serialization differs' |
| action=row['target'] |
| if action['action']=='call': |
| assert set(action)=={'action','name','arguments'} |
| tool=next(t for t in row['context']['tools'] if t['name']==action['name']) |
| schema=compact(tool['inputSchema']) |
| if schema not in validators:validators[schema]=Draft202012Validator(tool['inputSchema']) |
| validators[schema].validate(action['arguments']) |
| else:check_action(action,row['context']) |
| if row['operation'] in SQL_OPS: |
| sql_rows+=1;a=action['arguments'];sql=a.get('sql',a.get('query')) |
| context_key=hashlib.sha256(compact([row['backend'],row['context']['schema'],sql]).encode()).hexdigest() |
| if context_key not in context_queries: |
| import sqlglot |
| db=sqlite3.connect(':memory:');db.create_function('YEAR',1,lambda x:0);db.create_function('MONTH',1,lambda x:0) |
| try: |
| for ddl in row['context']['schema']:db.execute(ddl) |
| db.execute(sqlite_sql(sqlglot.parse_one(sql,read='postgres' if row['backend']=='supabase' else 'mysql'))) |
| finally:db.close() |
| context_queries.add(context_key) |
| key=hashlib.sha256(compact([row['backend'],row['slots'],sql]).encode()).hexdigest() |
| if key not in queries: |
| validate_sql(sql,row['backend'],row['slots']);queries.add(key) |
| except Exception as exc: |
| errors.append({'id':row['id'],'error':type(exc).__name__+': '+str(exc)}) |
| if len(errors)>=20:break |
| if count%10000==0:print(json.dumps({'rows':count,'unique_sql':len(queries),'seconds':time.time()-start}),flush=True) |
| report={'rows':count,'unique_ids':len(ids),'unique_prompts':len(prompts),'sql_rows':sql_rows,'unique_sql_cases':len(queries), |
| 'distinct_context_sql_compilations':len(context_queries), |
| 'fixtures_per_sql_case':2,'sql_engine':'SQLite after dialect parsing/adaptation; native coverage reported separately', |
| 'seconds':time.time()-start,'errors':errors,'passed':not errors} |
| Path(args.out).write_text(json.dumps(report,indent=2));print(json.dumps(report),flush=True) |
| if errors:raise SystemExit(1) |
|
|
|
|
| if __name__=='__main__':main() |
|
|
