no subprocess

app.py

@@ -1,157 +1,21 @@
 import gradio as gr
-import os
-import sys
-from pathlib import Path
-from datetime import datetime
-import json
+from evaluation_logic import run_evaluation, AVAILABLE_PROMPT_FORMATS
 
-# Add the duckdb-nsql directory to the Python path
-current_dir = Path(__file__).resolve().parent
-duckdb_nsql_dir = current_dir / 'duckdb-nsql'
-eval_dir = duckdb_nsql_dir / 'eval'
-sys.path.extend([str(current_dir), str(duckdb_nsql_dir), str(eval_dir)])
-
-# Import necessary functions and classes from predict.py and evaluate.py
-from eval.predict import predict, console, get_manifest, DefaultLoader
-from eval.constants import PROMPT_FORMATTERS
-from eval.evaluate import evaluate, compute_metrics, get_to_print
-from eval.evaluate import test_suite_evaluation, read_tables_json
-
-
-def run_evaluation(model_name):
-    results = []
-
-    if "OPENROUTER_API_KEY" not in os.environ:
-        return "Error: OPENROUTER_API_KEY not found in environment variables."
-
-    try:
-        # Set up the arguments similar to the CLI in predict.py
-        dataset_path = "duckdb-nsql/eval/data/dev.json"
-        table_meta_path = "duckdb-nsql/eval/data/tables.json"
-        output_dir = "duckdb-nsql/output/"
-        prompt_format = "duckdbinstgraniteshort"
-        stop_tokens = [';']
-        max_tokens = 30000
-        temperature = 0.1
-        num_beams = -1
-        manifest_client = "openrouter"
-        manifest_engine = model_name
-        manifest_connection = "http://localhost:5000"
-        overwrite_manifest = True
-        parallel = False
-
-        # Initialize necessary components
-        data_formatter = DefaultLoader()
-        prompt_formatter = PROMPT_FORMATTERS[prompt_format]()
-
-        # Load manifest
-        manifest = get_manifest(
-            manifest_client=manifest_client,
-            manifest_connection=manifest_connection,
-            manifest_engine=manifest_engine,
-        )
-
-        results.append(f"Using model: {manifest_engine}")
-
-        # Load data and metadata
-        results.append("Loading metadata and data...")
-        db_to_tables = data_formatter.load_table_metadata(table_meta_path)
-        data = data_formatter.load_data(dataset_path)
-
-        # Generate output filename
-        date_today = datetime.now().strftime("%y-%m-%d")
-        pred_filename = f"{prompt_format}_0docs_{manifest_engine.split('/')[-1]}_{Path(dataset_path).stem}_{date_today}.json"
-        pred_path = Path(output_dir) / pred_filename
-        results.append(f"Prediction will be saved to: {pred_path}")
-
-        # Debug: Print predict function signature
-        yield f"Predict function signature: {inspect.signature(predict)}"
-
-        # Run prediction
-        yield "Starting prediction..."
-        try:
-            predict(
-                dataset_path=dataset_path,
-                table_meta_path=table_meta_path,
-                output_dir=output_dir,
-                prompt_format=prompt_format,
-                stop_tokens=stop_tokens,
-                max_tokens=max_tokens,
-                temperature=temperature,
-                num_beams=num_beams,
-                manifest_client=manifest_client,
-                manifest_engine=manifest_engine,
-                manifest_connection=manifest_connection,
-                overwrite_manifest=overwrite_manifest,
-                parallel=parallel
-            )
-        except TypeError as e:
-            yield f"TypeError in predict function: {str(e)}"
-            yield "Attempting to call predict with only expected arguments..."
-            # Try calling predict with only the arguments it expects
-            predict_args = inspect.getfullargspec(predict).args
-            filtered_args = {k: v for k, v in locals().items() if k in predict_args}
-            predict(**filtered_args)
-            
-        results.append("Prediction completed.")
-
-        # Run evaluation
-        results.append("Starting evaluation...")
-
-        # Set up evaluation arguments
-        gold_path = Path(dataset_path)
-        db_dir = "duckdb-nsql/eval/data/databases/"
-        tables_path = Path(table_meta_path)
-
-        kmaps = test_suite_evaluation.build_foreign_key_map_from_json(str(tables_path))
-        db_schemas = read_tables_json(str(tables_path))
-
-        gold_sqls_dict = json.load(gold_path.open("r", encoding="utf-8"))
-        pred_sqls_dict = [json.loads(l) for l in pred_path.open("r").readlines()]
-
-        gold_sqls = [p.get("query", p.get("sql", "")) for p in gold_sqls_dict]
-        setup_sqls = [p["setup_sql"] for p in gold_sqls_dict]
-        validate_sqls = [p["validation_sql"] for p in gold_sqls_dict]
-        gold_dbs = [p.get("db_id", p.get("db", "")) for p in gold_sqls_dict]
-        pred_sqls = [p["pred"] for p in pred_sqls_dict]
-        categories = [p.get("category", "") for p in gold_sqls_dict]
-
-        metrics = compute_metrics(
-            gold_sqls=gold_sqls,
-            pred_sqls=pred_sqls,
-            gold_dbs=gold_dbs,
-            setup_sqls=setup_sqls,
-            validate_sqls=validate_sqls,
-            kmaps=kmaps,
-            db_schemas=db_schemas,
-            database_dir=db_dir,
-            lowercase_schema_match=False,
-            model_name=model_name,
-            categories=categories,
-        )
-
-        results.append("Evaluation completed.")
-
-        # Format and add the evaluation metrics to the results
-        if metrics:
-            to_print = get_to_print({"all": metrics}, "all", model_name, len(gold_sqls))
-            formatted_metrics = "\n".join([f"{k}: {v}" for k, v in to_print.items() if k not in ["slice", "model"]])
-            results.append(f"Evaluation metrics:\n{formatted_metrics}")
-        else:
-            results.append("No evaluation metrics returned.")
-
-    except Exception as e:
-        results.append(f"An unexpected error occurred: {str(e)}")
-
-    return "\n\n".join(results)
+def gradio_run_evaluation(model_name, prompt_format):
+    return run_evaluation(model_name, prompt_format)
 
 with gr.Blocks() as demo:
     gr.Markdown("# DuckDB SQL Evaluation App")
 
     model_name = gr.Textbox(label="Model Name (e.g., qwen/qwen-2.5-72b-instruct)")
+    prompt_format = gr.Dropdown(
+        label="Prompt Format", 
+        choices=AVAILABLE_PROMPT_FORMATS, 
+        value="duckdbinstgraniteshort"
+    )
     start_btn = gr.Button("Start Evaluation")
     output = gr.Textbox(label="Output", lines=20)
 
-    start_btn.click(fn=run_evaluation, inputs=[model_name], outputs=output)
+    start_btn.click(fn=gradio_run_evaluation, inputs=[model_name, prompt_format], outputs=output)
 
-demo.launch()
+demo.queue().launch()

evaluation_logic.py

@@ -0,0 +1,193 @@
+import os
+import sys
+from pathlib import Path
+from datetime import datetime
+import json
+import traceback
+
+# Add the necessary directories to the Python path
+current_dir = Path(__file__).resolve().parent
+duckdb_nsql_dir = current_dir / 'duckdb-nsql'
+eval_dir = duckdb_nsql_dir / 'eval'
+sys.path.extend([str(current_dir), str(duckdb_nsql_dir), str(eval_dir)])
+
+# Import necessary functions and classes
+from eval.predict import get_manifest, DefaultLoader, PROMPT_FORMATTERS, generate_sql
+from eval.evaluate import evaluate, compute_metrics, get_to_print
+from eval.evaluate import test_suite_evaluation, read_tables_json
+from eval.schema import TextToSQLParams, Table
+
+AVAILABLE_PROMPT_FORMATS = list(PROMPT_FORMATTERS.keys())
+
+def run_prediction(model_name, prompt_format, output_file):
+    dataset_path = str(eval_dir / "data/dev.json")
+    table_meta_path = str(eval_dir / "data/tables.json")
+    stop_tokens = [';']
+    max_tokens = 30000
+    temperature = 0.1
+    num_beams = -1
+    manifest_client = "openrouter"
+    manifest_engine = model_name
+    manifest_connection = "http://localhost:5000"
+    overwrite_manifest = True
+    parallel = False
+
+    yield "Starting prediction..."
+
+    try:
+        # Initialize necessary components
+        data_formatter = DefaultLoader()
+        prompt_formatter = PROMPT_FORMATTERS[prompt_format]()
+
+        # Load manifest
+        manifest = get_manifest(
+            manifest_client=manifest_client,
+            manifest_connection=manifest_connection,
+            manifest_engine=manifest_engine,
+        )
+
+        # Load data
+        data = data_formatter.load_data(dataset_path)
+        db_to_tables = data_formatter.load_table_metadata(table_meta_path)
+
+        # Prepare input for generate_sql
+        text_to_sql_inputs = []
+        for input_question in data:
+            question = input_question["question"]
+            db_id = input_question.get("db_id", "none")
+            if db_id != "none":
+                table_params = list(db_to_tables.get(db_id, {}).values())
+            else:
+                table_params = []
+            
+            if len(table_params) == 0:
+                yield f"[red] WARNING: No tables found for {db_id} [/red]"
+            
+            text_to_sql_inputs.append(TextToSQLParams(
+                instruction=question,
+                database=db_id,
+                tables=table_params,
+            ))
+
+        # Generate SQL
+        generated_sqls = generate_sql(
+            manifest=manifest,
+            text_to_sql_in=text_to_sql_inputs,
+            retrieved_docs=[[] for _ in text_to_sql_inputs],  # Assuming no retrieved docs
+            prompt_formatter=prompt_formatter,
+            stop_tokens=stop_tokens,
+            overwrite_manifest=overwrite_manifest,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            num_beams=num_beams,
+            parallel=parallel
+        )
+
+        # Save results
+        with output_file.open('w') as f:
+            for original_data, (sql, _) in zip(data, generated_sqls):
+                output = {**original_data, "pred": sql}
+                json.dump(output, f)
+                f.write('\n')
+
+        yield f"Prediction completed. Results saved to {output_file}"
+    except Exception as e:
+        yield f"Prediction failed with error: {str(e)}"
+        yield f"Error traceback: {traceback.format_exc()}"
+
+def run_evaluation(model_name, prompt_format="duckdbinstgraniteshort"):
+    if "OPENROUTER_API_KEY" not in os.environ:
+        yield "Error: OPENROUTER_API_KEY not found in environment variables."
+        return
+
+    try:
+        # Set up the arguments
+        dataset_path = str(eval_dir / "data/dev.json")
+        table_meta_path = str(eval_dir / "data/tables.json")
+        output_dir = eval_dir / "output"
+
+        yield f"Using model: {model_name}"
+        yield f"Using prompt format: {prompt_format}"
+
+        output_file = output_dir / f"{prompt_format}_0docs_{model_name.trim().replace('/', '_')}_dev_{datetime.now().strftime('%y-%m-%d')}.json"
+
+        # Ensure the output directory exists
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        if output_file.exists():
+            yield f"Prediction file already exists: {output_file}"
+            yield "Skipping prediction step and proceeding to evaluation."
+        else:
+            # Run prediction
+            for output in run_prediction(model_name, prompt_format, output_file):
+                yield output
+
+        # Run evaluation
+        yield "Starting evaluation..."
+
+        # Set up evaluation arguments
+        gold_path = Path(dataset_path)
+        db_dir = str(eval_dir / "data/databases/")
+        tables_path = Path(table_meta_path)
+
+        kmaps = test_suite_evaluation.build_foreign_key_map_from_json(str(tables_path))
+        db_schemas = read_tables_json(str(tables_path))
+
+        gold_sqls_dict = json.load(gold_path.open("r", encoding="utf-8"))
+        pred_sqls_dict = [json.loads(l) for l in output_file.open("r").readlines()]
+
+        gold_sqls = [p.get("query", p.get("sql", "")) for p in gold_sqls_dict]
+        setup_sqls = [p["setup_sql"] for p in gold_sqls_dict]
+        validate_sqls = [p["validation_sql"] for p in gold_sqls_dict]
+        gold_dbs = [p.get("db_id", p.get("db", "")) for p in gold_sqls_dict]
+        pred_sqls = [p["pred"] for p in pred_sqls_dict]
+        categories = [p.get("category", "") for p in gold_sqls_dict]
+
+        yield "Computing metrics..."
+        metrics = compute_metrics(
+            gold_sqls=gold_sqls,
+            pred_sqls=pred_sqls,
+            gold_dbs=gold_dbs,
+            setup_sqls=setup_sqls,
+            validate_sqls=validate_sqls,
+            kmaps=kmaps,
+            db_schemas=db_schemas,
+            database_dir=db_dir,
+            lowercase_schema_match=False,
+            model_name=model_name,
+            categories=categories,
+        )
+
+        yield "Evaluation completed."
+
+        if metrics:
+            yield "Overall Results:"
+            overall_metrics = metrics['exec']['all']
+            yield f"Count: {overall_metrics['count']}"
+            yield f"Execution Accuracy: {overall_metrics['exec']:.3f}"
+            yield f"Exact Match Accuracy: {overall_metrics['exact']:.3f}"
+            yield f"Equality: {metrics['equality']['equality']:.3f}"
+            yield f"Edit Distance: {metrics['edit_distance']['edit_distance']:.3f}"
+
+            yield "\nResults by Category:"
+            categories = ['easy', 'medium', 'hard', 'duckdb', 'ddl', 'all']
+            
+            for category in categories:
+                if category in metrics['exec']:
+                    yield f"\n{category}:"
+                    category_metrics = metrics['exec'][category]
+                    yield f"Count: {category_metrics['count']}"
+                    yield f"Execution Accuracy: {category_metrics['exec']:.3f}"
+                else:
+                    yield f"\n{category}: No data available"
+        else:
+            yield "No evaluation metrics returned."
+    except Exception as e:
+        yield f"An unexpected error occurred: {str(e)}"
+        yield f"Error traceback: {traceback.format_exc()}"
+
+if __name__ == "__main__":
+    model_name = input("Enter the model name: ")
+    prompt_format = input("Enter the prompt format (default is duckdbinstgraniteshort): ") or "duckdbinstgraniteshort"
+    for result in run_evaluation(model_name, prompt_format):
+        print(result, flush=True)

requirements.txt

@@ -20,6 +20,7 @@ peft==0.6.0
 packaging==23.2
 ninja==1.11.1.1
 langchain
+gradio
 pydantic
 packaging
 #./duckdb-nsql/manifest
@@ -28,3 +29,4 @@ flask
 diffusers
 deepspeed
 sentence_transformers
+tqdm