Commit
•
5c0b2f9
1
Parent(s):
3bfb3cc
Upload 2 files
Browse files- chess_eval_main.py +561 -0
- meta.pkl +3 -0
chess_eval_main.py
ADDED
@@ -0,0 +1,561 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import openai
|
2 |
+
import chess
|
3 |
+
import chess.engine
|
4 |
+
import os
|
5 |
+
import csv
|
6 |
+
import random
|
7 |
+
import time
|
8 |
+
import platform
|
9 |
+
|
10 |
+
# NOTE: LLAMA AND NANOGPT ARE EXPERIMENTAL PLAYERS, if not using them, comment them out
|
11 |
+
# from llama_module import BaseLlamaPlayer, LocalLlamaPlayer, LocalLoraLlamaPlayer
|
12 |
+
from nanogpt.nanogpt_module import NanoGptPlayer
|
13 |
+
from mamba_module import MambaPlayer
|
14 |
+
import gpt_query
|
15 |
+
from lczero.backends import Weights, Backend, GameState
|
16 |
+
import numpy as np
|
17 |
+
|
18 |
+
from typing import Optional, Tuple
|
19 |
+
from dataclasses import dataclass
|
20 |
+
|
21 |
+
|
22 |
+
@dataclass
|
23 |
+
class LegalMoveResponse:
|
24 |
+
move_san: Optional[str] = None
|
25 |
+
move_uci: Optional[chess.Move] = None
|
26 |
+
attempts: int = 0
|
27 |
+
is_resignation: bool = False
|
28 |
+
is_illegal_move: bool = False
|
29 |
+
|
30 |
+
|
31 |
+
# Define base Player class
|
32 |
+
class Player:
|
33 |
+
def get_move(self, board: chess.Board, game_state: str, temperature: float) -> str:
|
34 |
+
raise NotImplementedError
|
35 |
+
|
36 |
+
def get_config(self) -> dict:
|
37 |
+
raise NotImplementedError
|
38 |
+
|
39 |
+
|
40 |
+
class GPTPlayer(Player):
|
41 |
+
def __init__(self, model: str):
|
42 |
+
with open("gpt_inputs/api_key.txt", "r") as f:
|
43 |
+
openai.api_key = f.read().strip()
|
44 |
+
self.model = model
|
45 |
+
|
46 |
+
def get_move(
|
47 |
+
self, board: chess.Board, game_state: str, temperature: float
|
48 |
+
) -> Optional[str]:
|
49 |
+
response = get_gpt_response(game_state, self.model, temperature)
|
50 |
+
return get_move_from_gpt_response(response)
|
51 |
+
|
52 |
+
def get_config(self) -> dict:
|
53 |
+
return {"model": self.model}
|
54 |
+
|
55 |
+
|
56 |
+
class LC0PLayer(Player):
|
57 |
+
# "11258-32x4-se.pb.gz" = stockfish level 0- = skill 0
|
58 |
+
# "11258-48x5-se.pb.gz" = stockfish level 0+ = skill 1
|
59 |
+
# "11258-80x7-se.pb.gz" = stockfish level 1 = skill 2
|
60 |
+
# "11258-104x9-se.pb.gz" = stockfish level 2 = skill 3
|
61 |
+
# "TK-6430 aka 128x10-BPR-64M-6430000.pb.gz" = stockfish level 3 = skill 4
|
62 |
+
# "00af53b081e80147172e6f281c01daf5ca19ada173321438914c730370aa4267" = stockfish level 4 = skill 5
|
63 |
+
# "b2ec465d0fb5b5eb39d2e1e3f74041a5d2fc92d413b71aa7ea0b6fb082ccba9c" = stockfish level 5+ = skill 6
|
64 |
+
def __init__(self, skill):
|
65 |
+
self.skill = skill
|
66 |
+
network_paths = ["./lc0/build/release/11258-32x4-se.pb.gz", "./lc0/build/release/11258-48x5-se.pb.gz", "./lc0/build/release/11258-80x7-se.pb.gz", "./lc0/build/release/11258-104x9-se.pb.gz", "./lc0/build/release/TK-6430 aka 128x10-BPR-64M-6430000.pb.gz", "./lc0/build/release/00af53b081e80147172e6f281c01daf5ca19ada173321438914c730370aa4267", "./lc0/build/release/b2ec465d0fb5b5eb39d2e1e3f74041a5d2fc92d413b71aa7ea0b6fb082ccba9c"]
|
67 |
+
print(f"\n\nLoading lc0 network: {network_paths[skill]}\n\n")
|
68 |
+
self.weights = Weights(network_paths[skill])
|
69 |
+
self.backend = Backend(weights=self.weights)
|
70 |
+
self.gamestate = GameState()
|
71 |
+
|
72 |
+
def get_move(self, board: chess.Board, game_state: str, temperature: float):
|
73 |
+
self.gamestate = GameState(fen=board.fen())
|
74 |
+
input_planes = self.gamestate.as_input(self.backend)
|
75 |
+
result = self.backend.evaluate(input_planes)[0]
|
76 |
+
moves = self.gamestate.moves()
|
77 |
+
policy_indices = self.gamestate.policy_indices()
|
78 |
+
move_probs = np.array(result.p_softmax(*policy_indices))
|
79 |
+
best_move_idx = move_probs.argmax()
|
80 |
+
best_move = moves[best_move_idx]
|
81 |
+
return board.san(chess.Move.from_uci(best_move))
|
82 |
+
|
83 |
+
def get_config(self) -> dict:
|
84 |
+
return {"network": self.weights, "skill_level": self.skill, "play_time": 0}
|
85 |
+
|
86 |
+
|
87 |
+
class StockfishPlayer(Player):
|
88 |
+
|
89 |
+
@staticmethod
|
90 |
+
def get_stockfish_path() -> str:
|
91 |
+
"""
|
92 |
+
Determines the operating system and returns the appropriate path for Stockfish.
|
93 |
+
|
94 |
+
Returns:
|
95 |
+
str: Path to the Stockfish executable based on the operating system.
|
96 |
+
"""
|
97 |
+
if platform.system() == 'Linux':
|
98 |
+
return "/usr/games/stockfish"
|
99 |
+
elif platform.system() == 'Darwin': # Darwin is the system name for macOS
|
100 |
+
return "stockfish"
|
101 |
+
elif platform.system() == 'Windows':
|
102 |
+
return r"C:\Users\Haile\Downloads\stockfish\stockfish-windows-x86-64-avx2.exe"
|
103 |
+
else:
|
104 |
+
raise OSError("Unsupported operating system")
|
105 |
+
|
106 |
+
def __init__(self, skill_level: int, play_time: float):
|
107 |
+
self._skill_level = skill_level
|
108 |
+
self._play_time = play_time
|
109 |
+
# If getting started, you need to run brew install stockfish
|
110 |
+
stockfish_path = StockfishPlayer.get_stockfish_path()
|
111 |
+
self._engine = chess.engine.SimpleEngine.popen_uci(stockfish_path)
|
112 |
+
|
113 |
+
def get_move(
|
114 |
+
self, board: chess.Board, game_state: str, temperature: float
|
115 |
+
) -> Optional[str]:
|
116 |
+
if self._skill_level == -2:
|
117 |
+
legal_moves = list(board.legal_moves)
|
118 |
+
random_move = random.choice(legal_moves)
|
119 |
+
return board.san(random_move)
|
120 |
+
elif self._skill_level < 0:
|
121 |
+
self._engine.configure({"Skill Level": 0})
|
122 |
+
result = self._engine.play(
|
123 |
+
board, chess.engine.Limit(time=1e-8, depth=1, nodes=1)
|
124 |
+
)
|
125 |
+
|
126 |
+
else:
|
127 |
+
self._engine.configure({"Skill Level": self._skill_level})
|
128 |
+
result = self._engine.play(board, chess.engine.Limit(time=self._play_time))
|
129 |
+
if result.move is None:
|
130 |
+
return None
|
131 |
+
return board.san(result.move)
|
132 |
+
|
133 |
+
def get_config(self) -> dict:
|
134 |
+
return {"skill_level": self._skill_level, "play_time": self._play_time}
|
135 |
+
|
136 |
+
def close(self):
|
137 |
+
self._engine.quit()
|
138 |
+
|
139 |
+
|
140 |
+
class HumanPlayer(Player):
|
141 |
+
def get_move(self, board: chess.Board, game_state: str, temperature: float) -> str:
|
142 |
+
# Print board for human player
|
143 |
+
print(board)
|
144 |
+
while True:
|
145 |
+
move = input("Enter your move (SAN format): ")
|
146 |
+
try:
|
147 |
+
move_uci = board.parse_san(move)
|
148 |
+
if move_uci in board.legal_moves:
|
149 |
+
return move
|
150 |
+
except:
|
151 |
+
print("Illegal move, try again.")
|
152 |
+
|
153 |
+
def get_config(self) -> dict:
|
154 |
+
return {"player": "human"}
|
155 |
+
|
156 |
+
|
157 |
+
def get_gpt_response(game_state: str, model: str, temperature: float) -> Optional[str]:
|
158 |
+
# trying to prevent what I believe to be rate limit issues
|
159 |
+
if model == "gpt-4":
|
160 |
+
time.sleep(0.4)
|
161 |
+
response = gpt_query.get_gpt_response(game_state, model, temperature)
|
162 |
+
return response
|
163 |
+
|
164 |
+
|
165 |
+
def get_move_from_gpt_response(response: Optional[str]) -> Optional[str]:
|
166 |
+
if response is None:
|
167 |
+
return None
|
168 |
+
|
169 |
+
# Parse the response to get only the first move
|
170 |
+
moves = response.split()
|
171 |
+
first_move = moves[0] if moves else None
|
172 |
+
|
173 |
+
return first_move
|
174 |
+
|
175 |
+
|
176 |
+
def record_results(
|
177 |
+
board: chess.Board,
|
178 |
+
player_one: Player,
|
179 |
+
player_two: Player,
|
180 |
+
game_state: str,
|
181 |
+
player_one_illegal_moves: int,
|
182 |
+
player_two_illegal_moves: int,
|
183 |
+
player_one_legal_moves: int,
|
184 |
+
player_two_legal_moves: int,
|
185 |
+
total_time: float,
|
186 |
+
player_one_resignation: bool,
|
187 |
+
player_two_resignation: bool,
|
188 |
+
player_one_failed_to_find_legal_move: bool,
|
189 |
+
player_two_failed_to_find_legal_move: bool,
|
190 |
+
total_moves: int,
|
191 |
+
illegal_moves: int,
|
192 |
+
):
|
193 |
+
unique_game_id = generate_unique_game_id()
|
194 |
+
|
195 |
+
(
|
196 |
+
player_one_title,
|
197 |
+
player_two_title,
|
198 |
+
player_one_time,
|
199 |
+
player_two_time,
|
200 |
+
) = get_player_titles_and_time(player_one, player_two)
|
201 |
+
|
202 |
+
if player_one_resignation or player_one_failed_to_find_legal_move:
|
203 |
+
result = "0-1"
|
204 |
+
player_one_score = 0
|
205 |
+
player_two_score = 1
|
206 |
+
elif player_two_resignation or player_two_failed_to_find_legal_move:
|
207 |
+
result = "1-0"
|
208 |
+
player_one_score = 1
|
209 |
+
player_two_score = 0
|
210 |
+
else:
|
211 |
+
result = board.result()
|
212 |
+
# Hmmm.... debating this one. Annoying if I leave it running and it fails here for some reason, probably involving some
|
213 |
+
# resignation / failed move situation I didn't think of
|
214 |
+
# -1e10 at least ensures it doesn't fail silently
|
215 |
+
if "-" in result:
|
216 |
+
player_one_score = result.split("-")[0]
|
217 |
+
player_two_score = result.split("-")[1]
|
218 |
+
elif result == "*": # Draw due to hitting max moves
|
219 |
+
player_one_score = 0#1/2
|
220 |
+
player_two_score = 1#1/2
|
221 |
+
else:
|
222 |
+
player_one_score = -1e10
|
223 |
+
player_two_score = -1e10
|
224 |
+
|
225 |
+
info_dict = {
|
226 |
+
"game_id": unique_game_id,
|
227 |
+
"transcript": game_state,
|
228 |
+
"result": result,
|
229 |
+
"player_one": player_one_title,
|
230 |
+
"player_two": player_two_title,
|
231 |
+
"player_one_time": player_one_time,
|
232 |
+
"player_two_time": player_two_time,
|
233 |
+
"player_one_score": player_one_score,
|
234 |
+
"player_two_score": player_two_score,
|
235 |
+
"player_one_illegal_moves": player_one_illegal_moves,
|
236 |
+
"player_two_illegal_moves": player_two_illegal_moves,
|
237 |
+
"player_one_legal_moves": player_one_legal_moves,
|
238 |
+
"player_two_legal_moves": player_two_legal_moves,
|
239 |
+
"player_one_resignation": player_one_resignation,
|
240 |
+
"player_two_resignation": player_two_resignation,
|
241 |
+
"player_one_failed_to_find_legal_move": player_one_failed_to_find_legal_move,
|
242 |
+
"player_two_failed_to_find_legal_move": player_two_failed_to_find_legal_move,
|
243 |
+
"game_title": f"{player_one_title} vs. {player_two_title}",
|
244 |
+
"number_of_moves": board.fullmove_number,
|
245 |
+
"time_taken": total_time,
|
246 |
+
"total_moves": total_moves,
|
247 |
+
"illegal_moves": illegal_moves,
|
248 |
+
}
|
249 |
+
|
250 |
+
if RUN_FOR_ANALYSIS:
|
251 |
+
csv_file_path = f"logs/{player_one_recording_name}_vs_{player_two_recording_name}"
|
252 |
+
csv_file_path = csv_file_path.replace(".", "_") # Because I'm using ckpt filenames for nanogpt models
|
253 |
+
csv_file_path += ".csv"
|
254 |
+
else:
|
255 |
+
csv_file_path = recording_file
|
256 |
+
|
257 |
+
|
258 |
+
|
259 |
+
# Determine if we need to write headers (in case the file doesn't exist yet)
|
260 |
+
write_headers = not os.path.exists(csv_file_path)
|
261 |
+
|
262 |
+
# Append the results to the CSV file
|
263 |
+
with open(csv_file_path, "a", newline="") as csv_file: # THIS WAS APPEND
|
264 |
+
writer = csv.DictWriter(csv_file, fieldnames=info_dict.keys())
|
265 |
+
if write_headers:
|
266 |
+
writer.writeheader()
|
267 |
+
writer.writerow(info_dict)
|
268 |
+
|
269 |
+
with open("game.txt", "w") as f:
|
270 |
+
f.write(game_state)
|
271 |
+
|
272 |
+
|
273 |
+
def generate_unique_game_id() -> str:
|
274 |
+
timestamp = int(time.time())
|
275 |
+
random_num = random.randint(1000, 9999) # 4-digit random number
|
276 |
+
return f"{timestamp}-{random_num}"
|
277 |
+
|
278 |
+
|
279 |
+
def get_player_titles_and_time(
|
280 |
+
player_one: Player, player_two: Player
|
281 |
+
) -> Tuple[str, str, Optional[float], Optional[float]]:
|
282 |
+
player_one_config = player_one.get_config()
|
283 |
+
player_two_config = player_two.get_config()
|
284 |
+
|
285 |
+
# For player one
|
286 |
+
if "model" in player_one_config:
|
287 |
+
player_one_title = player_one_config["model"]
|
288 |
+
player_one_time = None
|
289 |
+
else:
|
290 |
+
player_one_title = f"Stockfish {player_one_config['skill_level']}"
|
291 |
+
player_one_time = player_one_config["play_time"]
|
292 |
+
|
293 |
+
# For player two
|
294 |
+
if "model" in player_two_config:
|
295 |
+
player_two_title = player_two_config["model"]
|
296 |
+
player_two_time = None
|
297 |
+
else:
|
298 |
+
player_two_title = f"Stockfish {player_two_config['skill_level']}"
|
299 |
+
player_two_time = player_two_config["play_time"]
|
300 |
+
|
301 |
+
return (player_one_title, player_two_title, player_one_time, player_two_time)
|
302 |
+
|
303 |
+
|
304 |
+
used_openings = []
|
305 |
+
def initialize_game_with_opening(
|
306 |
+
game_state: str, board: chess.Board
|
307 |
+
) -> Tuple[str, chess.Board]:
|
308 |
+
global used_openings
|
309 |
+
with open("openings.csv", "r") as file:
|
310 |
+
lines = file.readlines()[1:] # Skip header
|
311 |
+
moves_string = random.choice(lines)
|
312 |
+
while moves_string in used_openings:
|
313 |
+
moves_string = random.choice(lines)
|
314 |
+
used_openings.append(moves_string)
|
315 |
+
if move_num_in_gamestate:
|
316 |
+
game_state = moves_string.rstrip() + " "
|
317 |
+
else:
|
318 |
+
game_state = ' '.join(['.' + m.split(".")[-1] if "." in m else m for m in moves_string.split()])
|
319 |
+
# Splitting the moves string on spaces
|
320 |
+
tokens = moves_string.split()
|
321 |
+
|
322 |
+
for token in tokens:
|
323 |
+
# If the token contains a period, it's a move number + move combination
|
324 |
+
if "." in token:
|
325 |
+
move = token.split(".")[-1] # Take the move part after the period
|
326 |
+
else:
|
327 |
+
move = token
|
328 |
+
|
329 |
+
board.push_san(move)
|
330 |
+
return game_state.rstrip(), board
|
331 |
+
|
332 |
+
|
333 |
+
# Return is (move_san, move_uci, attempts, is_resignation, is_illegal_move)
|
334 |
+
def get_legal_move(
|
335 |
+
player: Player,
|
336 |
+
board: chess.Board,
|
337 |
+
game_state: str,
|
338 |
+
player_one: bool,
|
339 |
+
max_attempts: int = 5,
|
340 |
+
) -> LegalMoveResponse:
|
341 |
+
"""Request a move from the player and ensure it's legal."""
|
342 |
+
move_san = None
|
343 |
+
move_uci = None
|
344 |
+
|
345 |
+
for attempt in range(max_attempts):
|
346 |
+
move_san = player.get_move(
|
347 |
+
board, game_state, min(((attempt / max_attempts) * 1) + 0.001, 0.75)
|
348 |
+
)
|
349 |
+
|
350 |
+
# Sometimes when GPT thinks it's the end of the game, it will just output the result
|
351 |
+
# Like "1-0". If so, this really isn't an illegal move, so we'll add a check for that.
|
352 |
+
if move_san is not None:
|
353 |
+
if move_san == "1-0" or move_san == "0-1" or move_san == "1/2-1/2":
|
354 |
+
print(f"{move_san}, player has resigned")
|
355 |
+
return LegalMoveResponse(
|
356 |
+
move_san=None,
|
357 |
+
move_uci=None,
|
358 |
+
attempts=attempt,
|
359 |
+
is_resignation=True,
|
360 |
+
)
|
361 |
+
|
362 |
+
try:
|
363 |
+
move_uci = board.parse_san(move_san)
|
364 |
+
except Exception as e:
|
365 |
+
print(f"Error parsing move {move_san}: {e}")
|
366 |
+
# check if player is gpt-3.5-turbo-instruct
|
367 |
+
# only recording errors for gpt-3.5-turbo-instruct because it's errors are so rare
|
368 |
+
if player.get_config()["model"] == "gpt-3.5-turbo-instruct":
|
369 |
+
with open("gpt-3.5-turbo-instruct-illegal-moves.txt", "a") as f:
|
370 |
+
f.write(f"{game_state}\n{move_san}\n")
|
371 |
+
continue
|
372 |
+
|
373 |
+
if move_uci in board.legal_moves:
|
374 |
+
if player_one == False:
|
375 |
+
if not move_san.startswith(" "):
|
376 |
+
move_san = " " + move_san
|
377 |
+
else:
|
378 |
+
if move_san.startswith(" "):
|
379 |
+
move_san = move_san[1:]
|
380 |
+
return LegalMoveResponse(move_san, move_uci, attempt)
|
381 |
+
print(f"Illegal move: {move_san}")
|
382 |
+
|
383 |
+
# If we reach here, the player has made illegal moves for all attempts.
|
384 |
+
print(f"{player} provided illegal moves for {max_attempts} attempts.")
|
385 |
+
return LegalMoveResponse(
|
386 |
+
move_san=None, move_uci=None, attempts=max_attempts, is_illegal_move=True
|
387 |
+
)
|
388 |
+
|
389 |
+
|
390 |
+
def play_turn(
|
391 |
+
player: Player, board: chess.Board, game_state: str, player_one: bool
|
392 |
+
) -> Tuple[str, bool, bool, int]:
|
393 |
+
result = get_legal_move(player, board, game_state, player_one, 5)
|
394 |
+
illegal_moves = result.attempts
|
395 |
+
move_san = result.move_san
|
396 |
+
move_uci = result.move_uci
|
397 |
+
resignation = result.is_resignation
|
398 |
+
failed_to_find_legal_move = result.is_illegal_move
|
399 |
+
|
400 |
+
if resignation:
|
401 |
+
print(f"{player} resigned with result: {board.result()}")
|
402 |
+
elif failed_to_find_legal_move:
|
403 |
+
print(f"Game over: 5 consecutive illegal moves from {player}")
|
404 |
+
elif move_san is None or move_uci is None:
|
405 |
+
print(f"Game over: {player} failed to find a legal move")
|
406 |
+
else:
|
407 |
+
board.push(move_uci)
|
408 |
+
game_state += move_san
|
409 |
+
print(move_san, end=" ")
|
410 |
+
|
411 |
+
return game_state, resignation, failed_to_find_legal_move, illegal_moves
|
412 |
+
|
413 |
+
|
414 |
+
def play_game(
|
415 |
+
player_one: Player,
|
416 |
+
player_two: Player,
|
417 |
+
max_games: int = 10,
|
418 |
+
random_opening_seed: bool = False,
|
419 |
+
):
|
420 |
+
for z in range(max_games):
|
421 |
+
print(f"\nGame {z} of {max_games}\n")
|
422 |
+
|
423 |
+
with open("gpt_inputs/prompt.txt", "r") as f:
|
424 |
+
game_state = f.read()
|
425 |
+
board = chess.Board()
|
426 |
+
|
427 |
+
if random_opening_seed:
|
428 |
+
game_state, board = initialize_game_with_opening(game_state, board)
|
429 |
+
player_one_illegal_moves = 0
|
430 |
+
player_two_illegal_moves = 0
|
431 |
+
player_one_legal_moves = 0
|
432 |
+
player_two_legal_moves = 0
|
433 |
+
player_one_resignation = False
|
434 |
+
player_two_resignation = False
|
435 |
+
player_one_failed_to_find_legal_move = False
|
436 |
+
player_two_failed_to_find_legal_move = False
|
437 |
+
start_time = time.time()
|
438 |
+
|
439 |
+
total_moves = 0
|
440 |
+
illegal_moves = 0
|
441 |
+
print_for_human = isinstance(player_one, HumanPlayer) or isinstance(player_two, HumanPlayer)
|
442 |
+
|
443 |
+
while not board.is_game_over():
|
444 |
+
if print_for_human:
|
445 |
+
print(board)
|
446 |
+
|
447 |
+
with open("game.txt", "w") as f:
|
448 |
+
f.write(game_state)
|
449 |
+
current_move_num = f"{board.fullmove_number if move_num_in_gamestate else ''}."
|
450 |
+
total_moves += 1
|
451 |
+
# I increment legal moves here so player_two isn't penalized for the game ending before its turn
|
452 |
+
player_one_legal_moves += 1
|
453 |
+
player_two_legal_moves += 1
|
454 |
+
|
455 |
+
# this if statement may be overkill, just trying to get format to exactly match PGN notation
|
456 |
+
if board.fullmove_number != 1:
|
457 |
+
game_state += " "
|
458 |
+
game_state += current_move_num
|
459 |
+
print(f"{current_move_num}", end=" ")
|
460 |
+
|
461 |
+
(
|
462 |
+
game_state,
|
463 |
+
player_one_resignation,
|
464 |
+
player_one_failed_to_find_legal_move,
|
465 |
+
illegal_moves_one,
|
466 |
+
) = play_turn(player_one, board, game_state, player_one=True)
|
467 |
+
player_one_illegal_moves += illegal_moves_one
|
468 |
+
if illegal_moves_one != 0:
|
469 |
+
player_one_legal_moves -= 1
|
470 |
+
if (
|
471 |
+
board.is_game_over()
|
472 |
+
or player_one_resignation
|
473 |
+
or player_one_failed_to_find_legal_move
|
474 |
+
):
|
475 |
+
break
|
476 |
+
|
477 |
+
(
|
478 |
+
game_state,
|
479 |
+
player_two_resignation,
|
480 |
+
player_two_failed_to_find_legal_move,
|
481 |
+
illegal_moves_two,
|
482 |
+
) = play_turn(player_two, board, game_state, player_one=False)
|
483 |
+
player_two_illegal_moves += illegal_moves_two
|
484 |
+
if illegal_moves_two != 0:
|
485 |
+
player_two_legal_moves -= 1
|
486 |
+
if (
|
487 |
+
board.is_game_over()
|
488 |
+
or player_two_resignation
|
489 |
+
or player_two_failed_to_find_legal_move
|
490 |
+
):
|
491 |
+
break
|
492 |
+
|
493 |
+
print("\n", end="")
|
494 |
+
|
495 |
+
if total_moves > MAX_MOVES:
|
496 |
+
break
|
497 |
+
|
498 |
+
end_time = time.time()
|
499 |
+
total_time = end_time - start_time
|
500 |
+
print(f"\nGame over. Total time: {total_time} seconds")
|
501 |
+
print(f"Result: {board.result()}")
|
502 |
+
print(board)
|
503 |
+
print()
|
504 |
+
record_results(
|
505 |
+
board,
|
506 |
+
player_one,
|
507 |
+
player_two,
|
508 |
+
game_state,
|
509 |
+
player_one_illegal_moves,
|
510 |
+
player_two_illegal_moves,
|
511 |
+
player_one_legal_moves,
|
512 |
+
player_two_legal_moves,
|
513 |
+
total_time,
|
514 |
+
player_one_resignation,
|
515 |
+
player_two_resignation,
|
516 |
+
player_one_failed_to_find_legal_move,
|
517 |
+
player_two_failed_to_find_legal_move,
|
518 |
+
total_moves,
|
519 |
+
illegal_moves,
|
520 |
+
)
|
521 |
+
if isinstance(player_one, StockfishPlayer):
|
522 |
+
player_one.close()
|
523 |
+
if isinstance(player_two, StockfishPlayer):
|
524 |
+
player_two.close()
|
525 |
+
|
526 |
+
# print(game_state)
|
527 |
+
|
528 |
+
|
529 |
+
RUN_FOR_ANALYSIS = True
|
530 |
+
MAX_MOVES = 999 # Due to nanogpt max input length of 1024
|
531 |
+
recording_file = "logs/determine.csv" # default recording file. Because we are using list [player_ones], recording_file is overwritten
|
532 |
+
# player_one_recording_name = "ckpt_8.pt"
|
533 |
+
#player_ones = ["ckpt_iter_20000.pt","ckpt_iter_40000.pt","ckpt_iter_60000.pt","ckpt_iter_80000.pt"] #["ckpt.pt"]
|
534 |
+
player_ones = ["anneal/anneal_complete.pt"]
|
535 |
+
player_two_recording_name = "stockfish_sweep" #"stockfish_sweep"
|
536 |
+
move_num_in_gamestate = True
|
537 |
+
if __name__ == "__main__":
|
538 |
+
for nanogpt_player in player_ones:
|
539 |
+
player_one_recording_name = nanogpt_player
|
540 |
+
for i in range(2): #range(11):
|
541 |
+
num_games = 235 #265 instead of 250 for duplicates (for lc0, stockfish doesn't need it)
|
542 |
+
# player_one = GPTPlayer(model="gpt-3.5-turbo-instruct")
|
543 |
+
# player_one = LocalLlamaPlayer(model_name="meta-llama/Llama-2-7b-hf")
|
544 |
+
# player_one = LocalLoraLlamaPlayer("meta-llama/Llama-2-7b-hf", "/workspace/axolotl/lora2-out")
|
545 |
+
# player_one = GPTPlayer(model="gpt-4")
|
546 |
+
# player_one = StockfishPlayer(skill_level=-1, play_time=0.1)
|
547 |
+
# player_one = NanoGptPlayer(model_name=player_one_recording_name)
|
548 |
+
|
549 |
+
player_one = MambaPlayer(model_name=player_one_recording_name)
|
550 |
+
player_two = StockfishPlayer(skill_level=i, play_time=0.1)
|
551 |
+
#player_two = LC0PLayer(skill=i)
|
552 |
+
|
553 |
+
# player_two = GPTPlayer(model="gpt-4")
|
554 |
+
# player_two = GPTPlayer(model="gpt-3.5-turbo-instruct")
|
555 |
+
|
556 |
+
print(f"\n\nSTARTING GAMES AGAINST STOCKFISH LEVEL {i}\n\n")
|
557 |
+
#print(f"\n\nSTARTING GAMES AGAINST LC0 LEVEL {i}\n\n")
|
558 |
+
|
559 |
+
play_game(player_one, player_two, num_games, random_opening_seed=True)
|
560 |
+
|
561 |
+
print("\n\n\n********\nDONE!\n********\n\n\n")
|
meta.pkl
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:f1121191e401988851de5744fe27fe463c3a086fc8c9a5538ef7fc12162bfb09
|
3 |
+
size 373
|