Project: 029_MLE-bench_Evaluating_Machine_Learning_Agents_on_Ma | Function: get_familiarity_score (iclr)

system: You are an expert Python programmer implementing algorithms from scientific papers.
Complete the masked function based on the algorithm description.
Follow Python best practices and maintain consistent code style.
Return **only** the complete Python function code.

user: Complete this Python function:

**DESCRIPTION:** Contamination detection algorithm via token probability analysis that computes familiarity scores to measure model exposure to competition data through mean token prediction probability sampling

**KEY TERMS:** {
  "get_familiarity_score": "model's familiarity",
  "n_samples": "random sampling",
  "get_prob": "probability a model assigns to each token",
  "mean_familiarity_score": "mean familiarity"
}

**CONTEXT CODE:** comps_list = [p.parent.name for p in sorted(competitions_dir.glob("**/description.md"))]
discussions_dir = Path("./discussions/")
meta_kaggle_dir = Path("./meta-kaggle/")
grading_reports_dir = Path("../../runs/")
test_eval_script = Path("../../tests/integration/expected_scores.py")

# import ALL_COMPS_AND_EXPECTED_SCORES from test_eval_script
try:
    from importlib.util import module_from_spec, spec_from_file_location
    spec = spec_from_file_location("expected_scores", test_eval_script)
    expected_scores = module_from_spec(spec)
    spec.loader.exec_module(expected_scores)
    sample_submissions_scores = expected_scores.ALL_COMPS_AND_EXPECTED_SCORES
except (FileNotFoundError, ImportError, AttributeError):
    # For testing purposes when file doesn't exist
    sample_submissions_scores = [("comp1", 0.5), ("comp2", 0.3), ("comp3", 0.8)]


try:
    from openai import error as openai_error
    retry_exceptions = (openai_error.APIError, IndexError)
except ImportError:
    retry_exceptions = (IndexError,)

@retry(
    stop=stop_after_attempt(10),
    wait=wait_exponential(multiplier=1, min=4, max=10),
    retry=(retry_if_exception_type(retry_exceptions)),
)
def get_prob(prompt: str, target_token: str, logprobs: int = 10):
    """
    Get the probability of the target token given the prompt.

    If the token is not one of the `logprobs` most probable tokens, returns None.
    """
    engine = "DUMMY-BASE-MODEL"  # Fill in with real base model engine to use
    response = openai.Completion.create(
        engine=engine,
        prompt=prompt,
        max_tokens=1,
        logprobs=logprobs,
    )
    top_logprobs = response.choices[0].logprobs.top_logprobs[0]
    if target_token in top_logprobs:
        logprob = top_logprobs[target_token]
        # convert logprob to probability
        return np.exp(logprob)
    return None


def get_familiarity_score(text: str, n_samples: int,) -> float:
    """
    Familiarity score is the mean probability of the target token over `n_samples` random places in the text.

    Example:
    ```
    test_str = "The quick brown fox jumps over the lazy dog"
    print(get_familiarity_score(test_str, n))
    test_str = "The soft-spoken purple invisible man leapt over the carton of pomegrenate juice"
    print(get_familiarity_score(test_str, n))
    ```
    """
    raise NotImplementedError("Function implementation has been masked for testing")


def get_comp_metadata(comp_id: str):
    # look for comp_id by matching "Slug" in experiments/regurgitation/meta-kaggle/Competitions.csv
    metadata_df = pd.read_csv(meta_kaggle_dir / "Competitions.csv")
    row = metadata_df[metadata_df["Slug"] == comp_id]
    if len(row) == 0:
        return None
    metadata = row.to_dict(orient="records")[0]
    return metadata


def get_per_comp_performance(grading_reports):
    """
    Get the performance for each competition as an average across grading reports.

    Returns a dict with competition IDs as keys and the average model performance as the value
    (where performance is the % of model performance between the sample submission and the gold medal score).
    """
    raise NotImplementedError("Function implementation has been masked for testing")
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Process some integers.")
    parser.add_argument(
        "--output-cache-file",
        type=str,
        default="familiarity_and_dates.json",
        help="Cache the results to a file",
    )
    parser.add_argument(
        "--output-plot-file",
        type=str,
        default="familiarity_results.pdf",
        help="Output plot file",
    )
    parser.add_argument("--temperature", type=float, default=0, help="Temperature for the model")
    parser.add_argument(
        "--n-completions", type=int, default=1, help="Number of completions to generate"
    )
    parser.add_argument(
        "--n-samples", type=int, default=1000, help="Number of samples to test familiarity"
    )
    parser.add_argument("--n-seeds", type=int, default=1, help="Number of seeds to run")
    parser.add_argument(
        "--max-workers",
        type=int,
        default=40,
        help="Maximum number of workers for ThreadPoolExecutor",
    )
    parser.add_argument(
        "--existing-results",
        type=str,
        default=None,
        help="Use a cached results file, so we just plot the results",
    )
    args = parser.parse_args()

    familiarity_and_dates = {}

    if args.existing_results is not None:
        with open(args.existing_results, "r") as file:
            familiarity_and_dates = json.load(file)
        print(f"Reusing existing results from {args.existing_results}")

    else:

        # Loop through all "description.md" files in the competitions directory
        tasks = []
        for seed in range(args.n_seeds):
            for comp_id in comps_list:
                metadata = get_comp_metadata(comp_id)
                if metadata is None:
                    print(f"Competition {comp_id} not found in meta-kaggle, skipping")
                    continue

                tasks.append((comp_id, seed))

        def process_competition(_args):
            comp_id, seed = _args
            result = {}
            # Load the text from the file
            description_file = competitions_dir / comp_id / "description.md"
            discussions_files = list(discussions_dir.glob(f"{comp_id}/*.txt"))
            documents = [
                description_file,
                *discussions_files,
            ]
            result["familiarity_scores"] = {}


Complete the function implementation replacing "raise NotImplementedError".