Project: 029_MLE-bench_Evaluating_Machine_Learning_Agents_on_Ma | Function: get_per_comp_performance (iclr)

system: You are an expert Python programmer implementing algorithms from scientific papers.
Complete the masked function based on the algorithm description.
Follow Python best practices and maintain consistent code style.
Return **only** the complete Python function code.

user: Complete this Python function:

**DESCRIPTION:** Performance normalization algorithm that computes standardized competition performance metrics by calculating percentage achievement between sample submission baseline and gold medal threshold across multiple grading reports

**KEY TERMS:** {
  "get_per_comp_performance": "performance analysis",
  "performance": "performance on that competition",
  "gold_threshold": "gold medal score",
  "sample_submission_score": "baseline performance"
}

**CONTEXT CODE:**     stop=stop_after_attempt(10),
    wait=wait_exponential(multiplier=1, min=4, max=10),
    retry=(retry_if_exception_type(retry_exceptions)),
)
def get_prob(prompt: str, target_token: str, logprobs: int = 10):
    """
    Get the probability of the target token given the prompt.

    If the token is not one of the `logprobs` most probable tokens, returns None.
    """
    engine = "DUMMY-BASE-MODEL"  # Fill in with real base model engine to use
    response = openai.Completion.create(
        engine=engine,
        prompt=prompt,
        max_tokens=1,
        logprobs=logprobs,
    )
    top_logprobs = response.choices[0].logprobs.top_logprobs[0]
    if target_token in top_logprobs:
        logprob = top_logprobs[target_token]
        # convert logprob to probability
        return np.exp(logprob)
    return None


def get_familiarity_score(text: str, n_samples: int,) -> float:
    """
    Familiarity score is the mean probability of the target token over `n_samples` random places in the text.

    Example:
    ```
    test_str = "The quick brown fox jumps over the lazy dog"
    print(get_familiarity_score(test_str, n))
    test_str = "The soft-spoken purple invisible man leapt over the carton of pomegrenate juice"
    print(get_familiarity_score(test_str, n))
    ```
    """
    raise NotImplementedError("Function implementation has been masked for testing")


def get_comp_metadata(comp_id: str):
    # look for comp_id by matching "Slug" in experiments/regurgitation/meta-kaggle/Competitions.csv
    metadata_df = pd.read_csv(meta_kaggle_dir / "Competitions.csv")
    row = metadata_df[metadata_df["Slug"] == comp_id]
    if len(row) == 0:
        return None
    metadata = row.to_dict(orient="records")[0]
    return metadata


def get_per_comp_performance(grading_reports):
    """
    Get the performance for each competition as an average across grading reports.

    Returns a dict with competition IDs as keys and the average model performance as the value
    (where performance is the % of model performance between the sample submission and the gold medal score).
    """
    raise NotImplementedError("Function implementation has been masked for testing")
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Process some integers.")
    parser.add_argument(
        "--output-cache-file",
        type=str,
        default="familiarity_and_dates.json",
        help="Cache the results to a file",
    )
    parser.add_argument(
        "--output-plot-file",
        type=str,
        default="familiarity_results.pdf",
        help="Output plot file",
    )
    parser.add_argument("--temperature", type=float, default=0, help="Temperature for the model")
    parser.add_argument(
        "--n-completions", type=int, default=1, help="Number of completions to generate"
    )
    parser.add_argument(
        "--n-samples", type=int, default=1000, help="Number of samples to test familiarity"
    )
    parser.add_argument("--n-seeds", type=int, default=1, help="Number of seeds to run")
    parser.add_argument(
        "--max-workers",
        type=int,
        default=40,
        help="Maximum number of workers for ThreadPoolExecutor",
    )
    parser.add_argument(
        "--existing-results",
        type=str,
        default=None,
        help="Use a cached results file, so we just plot the results",
    )
    args = parser.parse_args()

    familiarity_and_dates = {}

    if args.existing_results is not None:
        with open(args.existing_results, "r") as file:
            familiarity_and_dates = json.load(file)
        print(f"Reusing existing results from {args.existing_results}")

    else:

        # Loop through all "description.md" files in the competitions directory
        tasks = []
        for seed in range(args.n_seeds):
            for comp_id in comps_list:
                metadata = get_comp_metadata(comp_id)
                if metadata is None:
                    print(f"Competition {comp_id} not found in meta-kaggle, skipping")
                    continue

                tasks.append((comp_id, seed))

        def process_competition(_args):
            comp_id, seed = _args
            result = {}
            # Load the text from the file
            description_file = competitions_dir / comp_id / "description.md"
            discussions_files = list(discussions_dir.glob(f"{comp_id}/*.txt"))
            documents = [
                description_file,
                *discussions_files,
            ]
            result["familiarity_scores"] = {}
            for document in documents:
                with document.open("r", encoding="utf-8") as file:
                    text = file.read()
                score = get_familiarity_score(text, args.n_samples)
                result["familiarity_scores"][document.name] = score
            # mean familiarity score across documents
            result["mean_familiarity_score"] = sum(result["familiarity_scores"].values()) / len(
                result["familiarity_scores"]
            )

            metadata = get_comp_metadata(comp_id)
            date_str = metadata["DeadlineDate"]
            result["deadline"] = date_str
            return comp_id, seed, result

        with ThreadPoolExecutor(max_workers=args.max_workers) as executor:
            results = list(executor.map(process_competition, tasks))

        for comp_id, seed, result in results:
            familiarity_and_dates[f"{comp_id}_seed{seed}"] = result

        # save the scores to file
        with open(args.output_cache_file, "w") as file:
            json.dump(familiarity_and_dates, file, indent=4, sort_keys=True)



Complete the function implementation replacing "raise NotImplementedError".