loubnabnl HF staff commited on
Commit
8635c9e
1 Parent(s): f01eda0

add tests file

Browse files
Files changed (1) hide show
  1. tests.py +14 -0
tests.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+ from evaluate import load
3
+
4
+ solution_sample1 = json.load(open("test_examples/solutions_problem_1.json", "r"))
5
+ solution_sample2 = json.load(open("test_examples/solutions_problem_2.json", "r"))
6
+ single_solutions = [solution_sample1[:1], solution_sample2[:1]]
7
+ multiple_solutions = [solution_sample1[:3], solution_sample2[:3]]
8
+
9
+ metric = load("loubnabnl/apps_metric")
10
+ result_1 = metric.compute(predictions=single_solutions, level="all")
11
+ result_2 = metric.compute(predictions=multiple_solutions, level="all", k_list=[1, 2, 3])
12
+
13
+ assert result_1 == {'avg_accuracy': 1.0, 'strict_accuracy': 1.0, 'pass_at_k': None}
14
+ assert result_2 == {'avg_accuracy': None, 'strict_accuracy': None, 'pass_at_k': {'pass@1': 1.0, 'pass@2': 1.0, 'pass@3': 1.0}}