p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] manual_eval/clean_eval.py (python)
import json
import sys
import os

def clean_eval_file(input_path):
    """
    Loads a JSON/JSONL evaluation file and resets evaluation results.
    Sets 'equivalent' to null and 'reason' to an empty string.
    """
    if not os.path.exists(input_path):
        print(f"Error: File '{input_path}' not found.")
        return

    print(f"Processing '{input_path}'...")
    
    try:
        with open(input_path, 'r') as f:
            data = json.load(f)
    except json.JSONDecodeError:
        # If it fails, check if it's actual JSONL (one object per line)
        print("Detected non-standard JSON, trying to read as JSONL...")
        try:
            with open(input_path, 'r') as f:
                lines = f.readlines()
                if len(lines) == 1:
                    data = json.loads(lines[0])
                else:
                    # If it's true JSONL, we handle it differently if needed,
                    # but the requested file looks like a single object.
                    raise ValueError("File contains multiple lines and is not a single JSON object.")
        except Exception as e:
            print(f"Error decoding JSON: {e}")
            return

    # Process the data structure
    if 'generation_results' in data:
        for task in data['generation_results']:
            if 'sample_evaluations' in task:
                for sample in task['sample_evaluations']:
                    # Reset evaluation_results object if present
                    if 'evaluation_results' in sample:
                        sample['evaluation_results']['equivalent'] = None
                        sample['evaluation_results']['reason'] = ""
                    
                    # Also reset high-level status keys if they exist
                    if 'equivalent' in sample:
                        sample['equivalent'] = None
                    if 'exact_match' in sample:
                        sample['exact_match'] = None

    # Reset aggregate scores if they exist
    if 'evaluation_scores' in data:
        for key in data['evaluation_scores']:
            if any(term in key for term in ['avg', 'pass', 'score', 'match']):
                data['evaluation_scores'][key] = 0

    # Save the cleaned file
    output_path = input_path.replace('.jsonl', '_cleaned.jsonl')
    if output_path == input_path:
        output_path = input_path + ".cleaned"

    with open(output_path, 'w') as f:
        json.dump(data, f, indent=2)
    
    print(f"Successfully cleaned evaluation results.")
    print(f"Output saved to: {output_path}")

if __name__ == "__main__":
    if len(sys.argv) > 1:
        for arg in sys.argv[1:]:
            clean_eval_file(arg)
    else:
        # Use the specific file mentioned if no arguments are provided
        target = "/p/home/jusers/mahajan1/juwels/projects/envcomp/miles/manual_eval/qwen1.7B_lora_bs_16_13032800_1000_evaluations.jsonl"
        if os.path.exists(target):
            clean_eval_file(target)
        else:
            print("Usage: python clean_eval.py <path_to_jsonl_file>")
[terminal_command] TERMINAL ()
python /p/home/jusers/mahajan1/juwels/projects/envcomp/miles/manual_eval/clean_eval.py /p/home/jusers/mahajan1/juwels/projects/envcomp/miles/manual_eval/qwen1.7B_lora_bs_16_13032800_1000_evaluations.jsonl