CultriX commited on
Commit
3955083
1 Parent(s): 4e86d45

calculate.py

Browse files
Files changed (1) hide show
  1. calculate.py +57 -0
calculate.py ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Read the CSV data into a DataFrame
2
+ data = pd.read_csv('models.csv')
3
+
4
+ # Define the score columns
5
+ score_columns = ['Average', 'AGIEval', 'GPT4All', 'TruthfulQA', 'Bigbench']
6
+
7
+ # Function to calculate the highest combined score for a given column
8
+ def calculate_highest_combined_score(column):
9
+ start_time = time.time()
10
+ scores = data[column].tolist()
11
+ models = data['Model'].tolist()
12
+ top_combinations = {2: [], 3: [], 4: [], 5: [], 6: []}
13
+ calculations = {2: 0, 3: 0, 4: 0, 5: 0, 6: 0}
14
+
15
+ # Generate all unique combinations of two, three, four, and five models
16
+ for r in range(2, 7): # r is the combination size (2, 3, 4, 5 or 6)
17
+ for combination in combinations(zip(scores, models), r):
18
+ combined_score = sum(score for score, _ in combination)
19
+ # Add the combination to the list along with its score
20
+ top_combinations[r].append((combined_score, tuple(model for _, model in combination)))
21
+ calculations[r] += 1
22
+ # Sort the list in descending order by score and keep only the top three
23
+ top_combinations[r] = sorted(top_combinations[r], key=lambda x: x[0], reverse=True)[:3]
24
+
25
+ elapsed_time = time.time() - start_time
26
+ return column, top_combinations, calculations, elapsed_time
27
+
28
+ # Function to be executed in parallel
29
+ def worker(column):
30
+ return calculate_highest_combined_score(column)
31
+
32
+ if __name__ == '__main__':
33
+ with Pool() as pool:
34
+ results = pool.map(worker, score_columns)
35
+ # Sort results by max_score in descending order
36
+ sorted_results = sorted(results, key=lambda x: max(x[1][5])[0] if 5 in x[1] else 0, reverse=True)
37
+ # Print the sorted results
38
+ for column, top_combinations, calculations, elapsed_time in sorted_results:
39
+ for r in range(2, 7):
40
+ print(f"Column: {column}, Number of Models: {r}")
41
+ for score, combination in top_combinations[r]:
42
+ print(f"Combination: {combination}, Score: {score}")
43
+ print(f"Calculations required: {calculations[r]}")
44
+ print(f"Time taken: {elapsed_time:.4f} seconds")
45
+ print() # Add an empty line for better readability
46
+
47
+ # Count how many times each model is mentioned
48
+ model_mentions = Counter()
49
+ for _, top_combinations, _, _ in sorted_results:
50
+ for r in range(2, 7):
51
+ for _, combination in top_combinations[r]:
52
+ model_mentions.update(combination)
53
+
54
+ # Print the top 5 most mentioned models
55
+ print("Top 5 most mentioned models:")
56
+ for model, count in model_mentions.most_common(5):
57
+ print(f"{model}: {count} times")