p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] check_problem_difficulty.py (python)
#!/usr/bin/env python3
"""
Script to process JSONL files and print file names and entries.
"""

import json
import sys
import argparse
from google import genai
import os
gemini = genai.Client(api_key=os.getenv("GEMINI_API_KEY"))

def get_problem_difficulty(problem):
    """ask gemini to determine the difficulty of the problem"""
    prompt = f"""
    You are a helpful assistant that determines the difficulty of a math problem.
    The problem is: {problem}
    Return the difficulty of the problem on a scale of 1 to 10, where 1 is the easiest and 10 is the hardest.
    """
    response = gemini.models.generate_content(model="gemini-2.5-pro", contents=[prompt])
    return response.text

def process_jsonl_file(file_path):
    """Process a single JSONL file and print its entries."""
    print(f"
{'='*60}")
    print(f"Processing file: {file_path}")
    print(f"{'='*60}
")
    
    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            for line_num, line in enumerate(f, start=1):
                line = line.strip()
                if not line:  # Skip empty lines
                    continue
                
                try:
                    entry = json.loads(line)
                    print(f"Entry {line_num}:")
                    print(f"Problem:
 {entry['input'][0]['content']}")
                    difficulty = get_problem_difficulty(entry['input'][0]['content'])
                    print(f"Gemini Difficulty verdict: {difficulty}")
                    print("-" * 60)
                except json.JSONDecodeError as e:
                    print(f"Error parsing entry on line {line_num}: {e}")
                    print(f"Line content: {line}")
                    print("-" * 60)
    except FileNotFoundError:
        print(f"Error: File '{file_path}' not found.")
    except Exception as e:
        print(f"Error processing file '{file_path}': {e}")


def main():
    parser = argparse.ArgumentParser(
        description='Process JSONL files and print file names and entries.'
    )
    parser.add_argument(
        'files',
        nargs='+',
        help='One or more JSONL files to process'
    )
    
    args = parser.parse_args()
    
    for file_path in args.files:
        process_jsonl_file(file_path)


if __name__ == '__main__':
    main()
[tab] gemini_verdicts_easy_w_outputs.log (log)

============================================================
Processing file: easy_problems.jsonl
============================================================

Entry 1:
Problem:
 Solve the following math problem. Make sure to put the answer (and only answer) inside \boxed{}.

The probability of snow on 1st January is 1/20. What is the probability that snow will fall on the next 1st January?
Error processing file 'easy_problems.jsonl': 503 UNAVAILABLE. {'error': {'code': 503, 'message': 'The model is overloaded. Please try again later.', 'status': 'UNAVAILABLE'}}