Update chess-gpt-eval/main.py

chess-gpt-eval/main.py

@@ -179,6 +179,7 @@ def record_results(
     player_two: Player,
     game_state: str,
     player_one_illegal_moves: int,
+    player_one_illegal_attempts: int,
     player_two_illegal_moves: int,
     player_one_legal_moves: int,
     player_two_legal_moves: int,
@@ -189,6 +190,7 @@ def record_results(
     player_two_failed_to_find_legal_move: bool,
     total_moves: int,
     illegal_moves: int,
+    opening_moves: int
 ):
     unique_game_id = generate_unique_game_id()
 
@@ -222,6 +224,7 @@ def record_results(
             player_one_score = -1e10
             player_two_score = -1e10
 
+    played_moves = player_one_illegal_moves + player_one_legal_moves
     info_dict = {
         "game_id": unique_game_id,
         "transcript": game_state,
@@ -242,6 +245,10 @@ def record_results(
         "player_two_failed_to_find_legal_move": player_two_failed_to_find_legal_move,
         "game_title": f"{player_one_title} vs. {player_two_title}",
         "number_of_moves": board.fullmove_number,
+        "p1_illegal_attempts": player_one_illegal_attempts,
+        "p1_avg_attempts_per_illegal": 0 if player_one_illegal_moves == 0 else player_one_illegal_attempts / float(player_one_illegal_moves),
+        "p1_illegal_attemtps_pct": 1.0 if played_moves == 0 else player_one_illegal_attempts / float(player_one_illegal_attempts + player_one_legal_moves),
+        "p1_illegal_moves_pct": 1.0 if played_moves == 0 else player_one_illegal_moves / float(played_moves),
         "time_taken": total_time,
         "total_moves": total_moves,
         "illegal_moves": illegal_moves,
@@ -428,9 +435,11 @@ def play_game(
         board = chess.Board()
 
         if random_opening_seed:
-            game_state, board = initialize_game_with_opening(game_state, board)
-            #print(f"play_gamea after init: |{game_state}|")
+            game_state, board, opening_moves = initialize_game_with_opening(game_state, board)
+        else:
+            opening_moves = 0
         player_one_illegal_moves = 0
+        player_one_illegal_attempts = 0
         player_two_illegal_moves = 0
         player_one_legal_moves = 0
         player_two_legal_moves = 0
@@ -469,7 +478,8 @@ def play_game(
                 player_one_failed_to_find_legal_move,
                 illegal_moves_one,
             ) = play_turn(player_one, board, game_state, player_one=True)
-            player_one_illegal_moves += illegal_moves_one
+            player_one_illegal_moves  += 1 if illegal_moves_one > 0 else 0
+            player_one_illegal_attempts += illegal_moves_one
             if illegal_moves_one != 0:
                 player_one_legal_moves -= 1
             if (
@@ -485,7 +495,7 @@ def play_game(
                 player_two_failed_to_find_legal_move,
                 illegal_moves_two,
             ) = play_turn(player_two, board, game_state, player_one=False)
-            player_two_illegal_moves += illegal_moves_two
+            player_two_illegal_moves  += 1 if illegal_moves_two > 0 else 0
             if illegal_moves_two != 0:
                 player_two_legal_moves -= 1
             if (
@@ -512,6 +522,7 @@ def play_game(
             player_two,
             game_state,
             player_one_illegal_moves,
+            player_one_illegal_attempts,
             player_two_illegal_moves,
             player_one_legal_moves,
             player_two_legal_moves,
@@ -522,6 +533,7 @@ def play_game(
             player_two_failed_to_find_legal_move,
             total_moves,
             illegal_moves,
+            opening_moves
         )
     if isinstance(player_one, StockfishPlayer):
         player_one.close()
@@ -542,7 +554,7 @@ move_num_in_gamestate = False
 if __name__ == "__main__":
     for nanogpt_player in player_ones:
         for i in range(1): #range(11):
-            num_games = 300 #265 #265 instead of 250 for duplicates (for lc0, stockfish doesn't need it)
+            num_games = 312 #265 #265 instead of 250 for duplicates (for lc0, stockfish doesn't need it)
             # player_one = GPTPlayer(model="gpt-3.5-turbo-instruct")
             # player_one = LocalLlamaPlayer(model_name="meta-llama/Llama-2-7b-hf")
             # player_one = LocalLoraLlamaPlayer("meta-llama/Llama-2-7b-hf", "/workspace/axolotl/lora2-out")