Repository: google-deepmind/open_spiel (language: python)

Issue:
Libtorch AlphaZero: Resume training from most recent checkpoint
This feature was talked about before with @selfsim and @SwiftGodDev.

I have been working on it a little bit and should hopefully have something soon, but I just thought I would make an issue to ensure that it is not forgotten about.


Discussion / hints:
Awesome @christianjans, thanks for continual contributions!

Fix patch:
diff --git a/open_spiel/CMakeLists.txt b/open_spiel/CMakeLists.txt
index 0120760e4c..986de783a4 100644
--- a/open_spiel/CMakeLists.txt
+++ b/open_spiel/CMakeLists.txt
@@ -106,6 +106,8 @@ openspiel_optional_dependency(OPEN_SPIEL_BUILD_WITH_HANABI         OFF
   "Build against the Hanabi game.")
 openspiel_optional_dependency(OPEN_SPIEL_BUILD_WITH_JULIA          OFF
   "Build binary for Julia.")
+openspiel_optional_dependency(OPEN_SPIEL_BUILD_WITH_LIBNOP         OFF
+  "Build with support for libnop.")
 openspiel_optional_dependency(OPEN_SPIEL_BUILD_WITH_LIBTORCH       OFF
   "Build with support for libtorch.")
 openspiel_optional_dependency(OPEN_SPIEL_BUILD_WITH_TENSORFLOW_CC  OFF
@@ -219,6 +221,10 @@ endif()
 if (OPEN_SPIEL_BUILD_WITH_ROSHAMBO)
   set(OPEN_SPIEL_OBJECTS ${OPEN_SPIEL_OBJECTS} $<TARGET_OBJECTS:roshambo>)
 endif()
+if (OPEN_SPIEL_BUILD_WITH_LIBNOP)
+  include_directories(libnop/libnop/include)
+  add_subdirectory(libnop)
+endif()
 if (OPEN_SPIEL_BUILD_WITH_LIBTORCH)
   list(APPEND CMAKE_PREFIX_PATH "${CMAKE_CURRENT_SOURCE_DIR}/libtorch/libtorch")
   find_package(Torch REQUIRED)
diff --git a/open_spiel/algorithms/alpha_zero_torch/README.md b/open_spiel/algorithms/alpha_zero_torch/README.md
index 791bbeaf5c..82611137ff 100644
--- a/open_spiel/algorithms/alpha_zero_torch/README.md
+++ b/open_spiel/algorithms/alpha_zero_torch/README.md
@@ -3,9 +3,9 @@
 This is a C++ implementation of the AlphaZero algorithm based on LibTorch,
 similar to the C++ TF-based AlphaZero.
 
-To build and use this implementation, you must set the optional global variable
-`OPEN_SPIEL_BUILD_WITH_LIBTORCH` to `ON` when installing dependencies and
-building OpenSpiel.
+To build and use this implementation, you must set the optional global variables
+`OPEN_SPIEL_BUILD_WITH_LIBTORCH` and `OPEN_SPIEL_BUILD_WITH_LIBNOP` to `ON` when
+installing dependencies and building OpenSpiel.
 
 Then, to get started, see `examples/alpha_zero_torch_example.cc`.
 
@@ -21,3 +21,52 @@ several of the same components. If you have any questions, feel free to ask the
 original author Christian Jans directly by following up on the PR linked above.
 The PR also includes some results of experiments run using this implementation
 that may be useful.
+
+## Setting up LibTorch AlphaZero
+
+1.  In [global_variables.sh](../../scripts/global_variables.sh), find the
+    `OPEN_SPIEL_BUILD_WITH_LIBNOP` variable and set its value to `"ON"`.
+2.  In [global_variables.sh](../../scripts/global_variables.sh), find the
+    `OPEN_SPIEL_BUILD_WITH_LIBTORCH` variable and set its value to `"ON"`.
+3.  In [global_variables.sh](../../scripts/global_variables.sh), find the
+    `OPEN_SPIEL_BUILD_WITH_LIBTORCH_DOWNLOAD_URL` variable and set its value to
+    the LibTorch version URL compatible with your OS and hardware (see the
+    comments in global_variables.sh for the URLs):
+4.  Download libnop and the specified version of LibTorch by running:
+    ```bash
+    $ ./install.sh
+    ```
+5.  Build OpenSpiel to compile LibTorch-dependent and libnop-dependent code
+    (such as LibTorch AlphaZero).
+    ```bash
+    $ ./open_spiel/scripts/build_and_run_tests.sh
+    ```
+
+## Starting LibTorch AlphaZero Training
+
+Starting training from scratch can be done by running
+`alpha_zero_torch_example`:
+```sh
+$ ./build/examples/alpha_zero_torch_example --game=tic_tac_toe --path=/home/me/az_example/
+```
+Run with the `--help` flag to see a complete list of flags and a brief
+description of each.
+
+## Resuming LibTorch AlphaZero Training
+
+Training can be resumed from the most recent checkpoint by providing the path to
+the `config.json` (which is created during the initial training run) as a
+positional argument:
+```sh
+$ ./build/examples/alpha_zero_torch_example /home/me/az_example/config.json
+```
+
+## Playing a Trained LibTorch AlphaZero
+
+A trained LibTorch AlphaZero can be played by running
+`alpha_zero_torch_game_example`:
+```sh
+$ ./build/examples/alpha_zero_torch_game_example --game=tic_tac_toe --player1=az --player2=mcts --az_path=/home/me/az_example/ --az_checkpoint=-1
+```
+Run with the `--help` flag to see a complete list of flags and a brief
+description of each.
diff --git a/open_spiel/algorithms/alpha_zero_torch/alpha_zero.cc b/open_spiel/algorithms/alpha_zero_torch/alpha_zero.cc
index 734e775530..c96edcf931 100644
--- a/open_spiel/algorithms/alpha_zero_torch/alpha_zero.cc
+++ b/open_spiel/algorithms/alpha_zero_torch/alpha_zero.cc
@@ -14,11 +14,13 @@
 
 #include "open_spiel/algorithms/alpha_zero_torch/alpha_zero.h"
 
+#include <algorithm>
 #include <cstdlib>
 #include <iostream>
 #include <memory>
 #include <random>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "open_spiel/abseil-cpp/absl/algorithm/container.h"
@@ -41,6 +43,7 @@
 #include "open_spiel/utils/json.h"
 #include "open_spiel/utils/logger.h"
 #include "open_spiel/utils/lru_cache.h"
+#include "open_spiel/utils/serializable_circular_buffer.h"
 #include "open_spiel/utils/stats.h"
 #include "open_spiel/utils/thread.h"
 #include "open_spiel/utils/threaded_queue.h"
@@ -49,6 +52,41 @@ namespace open_spiel {
 namespace algorithms {
 namespace torch_az {
 
+struct StartInfo {
+  absl::Time start_time;
+  int start_step;
+  int model_checkpoint_step;
+  int64_t total_trajectories;
+};
+
+StartInfo StartInfoFromLearnerJson(const std::string& path) {
+  StartInfo start_info;
+  file::File learner_file(path + "/learner.jsonl", "r");
+  std::vector<std::string> learner_lines = absl::StrSplit(
+      learner_file.ReadContents(), "\n");
+  std::string last_learner_line;
+
+  // Get the last non-empty line in learner.jsonl.
+  for (int i = learner_lines.size() - 1; i >= 0; i--) {
+    if (!learner_lines[i].empty()) {
+      last_learner_line = learner_lines[i];
+      break;
+    }
+  }
+
+  json::Object last_learner_json = json::FromString(
+      last_learner_line).value().GetObject();
+
+  start_info.start_time = absl::Now() - absl::Seconds(
+      last_learner_json["time_rel"].GetDouble());
+  start_info.start_step = last_learner_json["step"].GetInt() + 1;
+  start_info.model_checkpoint_step = VPNetModel::kMostRecentCheckpointStep;
+  start_info.total_trajectories =
+      last_learner_json["total_trajectories"].GetInt();
+
+  return start_info;
+}
+
 struct Trajectory {
   struct State {
     std::vector<float> observation;
@@ -250,19 +288,24 @@ void learner(const open_spiel::Game& game, const AlphaZeroConfig& config,
              DeviceManager* device_manager,
              std::shared_ptr<VPNetEvaluator> eval,
              ThreadedQueue<Trajectory>* trajectory_queue,
-             EvalResults* eval_results, StopToken* stop) {
-  FileLogger logger(config.path, "learner");
-  DataLoggerJsonLines data_logger(config.path, "learner", true);
+             EvalResults* eval_results, StopToken* stop,
+             const StartInfo& start_info) {
+  FileLogger logger(config.path, "learner", "a");
+  DataLoggerJsonLines data_logger(
+      config.path, "learner", true, "a", start_info.start_time);
   std::mt19937 rng;
 
   int device_id = 0;  // Do not change, the first device is the learner.
   logger.Print("Running the learner on device %d: %s", device_id,
                device_manager->Get(0, device_id)->Device());
 
-  CircularBuffer<VPNetModel::TrainInputs> replay_buffer(
+  SerializableCircularBuffer<VPNetModel::TrainInputs> replay_buffer(
       config.replay_buffer_size);
+  if (start_info.start_step > 1) {
+    replay_buffer.LoadBuffer(config.path + "/replay_buffer.data");
+  }
   int learn_rate = config.replay_buffer_size / config.replay_buffer_reuse;
-  int64_t total_trajectories = 0;
+  int64_t total_trajectories = start_info.total_trajectories;
 
   const int stage_count = 7;
   std::vector<open_spiel::BasicStats> value_accuracies(stage_count);
@@ -274,8 +317,9 @@ void learner(const open_spiel::Game& game, const AlphaZeroConfig& config,
   // Actor threads have likely been contributing for a while, so put `last` in
   // the past to avoid a giant spike on the first step.
   absl::Time last = absl::Now() - absl::Seconds(60);
-  for (int step = 1; !stop->StopRequested() &&
-                     (config.max_steps == 0 || step <= config.max_steps);
+  for (int step = start_info.start_step;
+       !stop->StopRequested() &&
+           (config.max_steps == 0 || step <= config.max_steps);
        ++step) {
     outcomes.Reset();
     game_lengths.Reset();
@@ -339,6 +383,8 @@ void learner(const open_spiel::Game& game, const AlphaZeroConfig& config,
 
     last = now;
 
+    replay_buffer.SaveBuffer(config.path + "/replay_buffer.data");
+
     VPNetModel::LossInfo losses;
     {  // Extra scope to return the device for use for inference asap.
       DeviceManager::DeviceLoan learn_model =
@@ -362,9 +408,11 @@ void learner(const open_spiel::Game& game, const AlphaZeroConfig& config,
 
     // Always save a checkpoint, either for keeping or for loading the weights
     // to the other sessions. It only allows numbers, so use -1 as "latest".
-    std::string checkpoint_path =
-        device_manager->Get(0, device_id)
-            ->SaveCheckpoint(step % config.checkpoint_freq == 0 ? step : -1);
+    std::string checkpoint_path = device_manager->Get(0, device_id)
+        ->SaveCheckpoint(VPNetModel::kMostRecentCheckpointStep);
+    if (step % config.checkpoint_freq == 0) {
+      device_manager->Get(0, device_id)->SaveCheckpoint(step);
+    }
     if (device_manager->Count() > 0) {
       for (int i = 0; i < device_manager->Count(); ++i) {
         if (i != device_id) {
@@ -376,6 +424,7 @@ void learner(const open_spiel::Game& game, const AlphaZeroConfig& config,
 
     DataLogger::Record record = {
         {"step", step},
+        // XXX: "total_states" is currently incorrect if resuming.
         {"total_states", replay_buffer.TotalAdded()},
         {"states_per_s", num_states / seconds},
         {"states_per_s_actor", num_states / (config.actors * seconds)},
@@ -434,7 +483,7 @@ void learner(const open_spiel::Game& game, const AlphaZeroConfig& config,
   }
 }
 
-bool AlphaZero(AlphaZeroConfig config, StopToken* stop) {
+bool AlphaZero(AlphaZeroConfig config, StopToken* stop, bool resuming) {
   std::shared_ptr<const open_spiel::Game> game =
       open_spiel::LoadGame(config.game);
 
@@ -491,6 +540,14 @@ bool AlphaZero(AlphaZeroConfig config, StopToken* stop) {
     fd.Write(json::ToString(config.ToJson(), true) + "\n");
   }
 
+  StartInfo start_info = {/*start_time=*/absl::Now(),
+                          /*start_step=*/1,
+                          /*model_checkpoint_step*/0,
+                          /*total_trajectories*/0};
+  if (resuming) {
+    start_info = StartInfoFromLearnerJson(config.path);
+  }
+
   DeviceManager device_manager;
   for (const absl::string_view& device : absl::StrSplit(config.devices, ',')) {
     device_manager.AddDevice(
@@ -511,10 +568,15 @@ bool AlphaZero(AlphaZeroConfig config, StopToken* stop) {
     return false;
   }
 
+  std::cerr << "Loading model from step " << start_info.model_checkpoint_step
+            << std::endl;
   {  // Make sure they're all in sync.
-    std::string first_checkpoint = device_manager.Get(0)->SaveCheckpoint(0);
-    for (int i = 1; i < device_manager.Count(); ++i) {
-      device_manager.Get(0, i)->LoadCheckpoint(first_checkpoint);
+    if (!resuming) {
+      device_manager.Get(0)->SaveCheckpoint(start_info.model_checkpoint_step);
+    }
+    for (int i = 0; i < device_manager.Count(); ++i) {
+      device_manager.Get(0, i)->LoadCheckpoint(
+          start_info.model_checkpoint_step);
     }
   }
 
@@ -540,7 +602,7 @@ bool AlphaZero(AlphaZeroConfig config, StopToken* stop) {
         [&, i]() { evaluator(*game, config, i, &eval_results, eval, stop); });
   }
   learner(*game, config, &device_manager, eval, &trajectory_queue,
-          &eval_results, stop);
+          &eval_results, stop, start_info);
 
   if (!stop->StopRequested()) {
     stop->Stop();
diff --git a/open_spiel/algorithms/alpha_zero_torch/alpha_zero.h b/open_spiel/algorithms/alpha_zero_torch/alpha_zero.h
index 3ee5e07d77..3f802d2f6d 100644
--- a/open_spiel/algorithms/alpha_zero_torch/alpha_zero.h
+++ b/open_spiel/algorithms/alpha_zero_torch/alpha_zero.h
@@ -15,6 +15,11 @@
 #ifndef OPEN_SPIEL_ALGORITHMS_ALPHA_ZERO_TORCH_ALPHA_ZERO_H_
 #define OPEN_SPIEL_ALGORITHMS_ALPHA_ZERO_TORCH_ALPHA_ZERO_H_
 
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "open_spiel/utils/file.h"
 #include "open_spiel/utils/json.h"
 #include "open_spiel/utils/thread.h"
 
@@ -91,9 +96,42 @@ struct AlphaZeroConfig {
         {"max_steps", max_steps},
     });
   }
+
+  void FromJson(const json::Object& config_json) {
+    game = config_json.at("game").GetString();
+    path = config_json.at("path").GetString();
+    graph_def = config_json.at("graph_def").GetString();
+    nn_model = config_json.at("nn_model").GetString();
+    nn_width = config_json.at("nn_width").GetInt();
+    nn_depth = config_json.at("nn_depth").GetInt();
+    devices = config_json.at("devices").GetString();
+    explicit_learning = config_json.at("explicit_learning").GetBool();
+    learning_rate = config_json.at("learning_rate").GetDouble();
+    weight_decay = config_json.at("weight_decay").GetDouble();
+    train_batch_size = config_json.at("train_batch_size").GetInt();
+    inference_batch_size = config_json.at("inference_batch_size").GetInt();
+    inference_threads = config_json.at("inference_threads").GetInt();
+    inference_cache = config_json.at("inference_cache").GetInt();
+    replay_buffer_size = config_json.at("replay_buffer_size").GetInt();
+    replay_buffer_reuse = config_json.at("replay_buffer_reuse").GetInt();
+    checkpoint_freq = config_json.at("checkpoint_freq").GetInt();
+    evaluation_window = config_json.at("evaluation_window").GetInt();
+    uct_c = config_json.at("uct_c").GetDouble();
+    max_simulations = config_json.at("max_simulations").GetInt();
+    policy_alpha = config_json.at("policy_alpha").GetDouble();
+    policy_epsilon = config_json.at("policy_epsilon").GetDouble();
+    temperature = config_json.at("temperature").GetDouble();
+    temperature_drop = config_json.at("temperature_drop").GetDouble();
+    cutoff_probability = config_json.at("cutoff_probability").GetDouble();
+    cutoff_value = config_json.at("cutoff_value").GetDouble();
+    actors = config_json.at("actors").GetInt();
+    evaluators = config_json.at("evaluators").GetInt();
+    eval_levels = config_json.at("eval_levels").GetInt();
+    max_steps = config_json.at("max_steps").GetInt();
+  }
 };
 
-bool AlphaZero(AlphaZeroConfig config, StopToken* stop);
+bool AlphaZero(AlphaZeroConfig config, StopToken* stop, bool resuming);
 
 }  // namespace torch_az
 }  // namespace algorithms
diff --git a/open_spiel/algorithms/alpha_zero_torch/vpnet.cc b/open_spiel/algorithms/alpha_zero_torch/vpnet.cc
index bed7f40d54..2957f9059d 100644
--- a/open_spiel/algorithms/alpha_zero_torch/vpnet.cc
+++ b/open_spiel/algorithms/alpha_zero_torch/vpnet.cc
@@ -108,8 +108,7 @@ bool CreateGraphDef(const Game& game, double learning_rate, double weight_decay,
 }
 
 VPNetModel::VPNetModel(const Game &game, const std::string &path,
-                       const std::string &file_name, const std::string &device,
-                       int checkpoint_step)
+                       const std::string &file_name, const std::string &device)
     : device_(device), path_(path),
       flat_input_size_(game.ObservationTensorSize()),
       num_actions_(game.NumDistinctActions()),
@@ -124,11 +123,6 @@ VPNetModel::VPNetModel(const Game &game, const std::string &path,
   SPIEL_CHECK_EQ(game.NumPlayers(), 2);
   SPIEL_CHECK_EQ(game.GetType().utility, GameType::Utility::kZeroSum);
 
-  // Load from specified checkpoint if a checkpoint step is given.
-  if (checkpoint_step != kInvalidCheckpointStep) {
-    LoadCheckpoint(absl::StrCat(path, "/checkpoint-", checkpoint_step));
-  }
-
   // Put this model on the specified device.
   model_->to(torch_device_);
 }
@@ -142,6 +136,11 @@ std::string VPNetModel::SaveCheckpoint(int step) {
   return full_path;
 }
 
+void VPNetModel::LoadCheckpoint(int step) {
+  // Load checkpoint from the path given at its initialization.
+  LoadCheckpoint(absl::StrCat(path_, "/checkpoint-", step));
+}
+
 void VPNetModel::LoadCheckpoint(const std::string& path) {
   torch::load(model_, absl::StrCat(path, ".pt"), torch_device_);
   torch::load(model_optimizer_, absl::StrCat(path, "-optimizer.pt"),
diff --git a/open_spiel/algorithms/alpha_zero_torch/vpnet.h b/open_spiel/algorithms/alpha_zero_torch/vpnet.h
index c05dbd1aca..32b756e59e 100644
--- a/open_spiel/algorithms/alpha_zero_torch/vpnet.h
+++ b/open_spiel/algorithms/alpha_zero_torch/vpnet.h
@@ -17,6 +17,8 @@
 
 #include <torch/torch.h>
 
+#include <nop/structure.h>
+
 #include "open_spiel/algorithms/alpha_zero_torch/model.h"
 #include "open_spiel/spiel.h"
 
@@ -92,6 +94,8 @@ class VPNetModel {
     std::vector<float> observations;
     ActionsAndProbs policy;
     double value;
+
+    NOP_STRUCTURE(TrainInputs, legal_actions, observations, policy, value);
   };
 
   enum CheckpointStep {
@@ -100,8 +104,8 @@ class VPNetModel {
   };
 
   VPNetModel(const Game &game, const std::string &path,
-             const std::string &file_name, const std::string &device = "/cpu:0",
-             int checkpoint_step = kInvalidCheckpointStep);
+             const std::string &file_name,
+             const std::string &device = "/cpu:0");
 
   // Move only, not copyable.
   VPNetModel(VPNetModel&& other) = default;
@@ -117,6 +121,7 @@ class VPNetModel {
   LossInfo Learn(const std::vector<TrainInputs>& inputs);
 
   std::string SaveCheckpoint(int step);
+  void LoadCheckpoint(int step);
   void LoadCheckpoint(const std::string& path);
 
   const std::string Device() const { return device_; }
diff --git a/open_spiel/examples/alpha_zero_torch_example.cc b/open_spiel/examples/alpha_zero_torch_example.cc
index 548083c987..63f41f65a1 100644
--- a/open_spiel/examples/alpha_zero_torch_example.cc
+++ b/open_spiel/examples/alpha_zero_torch_example.cc
@@ -17,6 +17,8 @@
 #include "open_spiel/abseil-cpp/absl/flags/flag.h"
 #include "open_spiel/abseil-cpp/absl/flags/parse.h"
 #include "open_spiel/algorithms/alpha_zero_torch/alpha_zero.h"
+#include "open_spiel/utils/file.h"
+#include "open_spiel/utils/json.h"
 #include "open_spiel/utils/thread.h"
 
 ABSL_FLAG(std::string, game, "tic_tac_toe", "The name of the game to play.");
@@ -95,41 +97,63 @@ void signal_installer() {
 }
 
 int main(int argc, char** argv) {
-  absl::ParseCommandLine(argc, argv);
+  std::vector<char*> positional_args = absl::ParseCommandLine(argc, argv);
   signal_installer();
 
+  bool resuming;
   open_spiel::algorithms::torch_az::AlphaZeroConfig config;
-  config.game = absl::GetFlag(FLAGS_game);
-  config.path = absl::GetFlag(FLAGS_path);
-  config.graph_def = absl::GetFlag(FLAGS_graph_def);
-  config.nn_model = absl::GetFlag(FLAGS_nn_model);
-  config.nn_width = absl::GetFlag(FLAGS_nn_width);
-  config.nn_depth = absl::GetFlag(FLAGS_nn_depth);
-  config.devices = absl::GetFlag(FLAGS_devices);
-  config.explicit_learning = absl::GetFlag(FLAGS_explicit_learning);
-  config.learning_rate = absl::GetFlag(FLAGS_learning_rate);
-  config.weight_decay = absl::GetFlag(FLAGS_weight_decay);
-  config.train_batch_size = absl::GetFlag(FLAGS_train_batch_size);
-  config.replay_buffer_size = absl::GetFlag(FLAGS_replay_buffer_size);
-  config.replay_buffer_reuse = absl::GetFlag(FLAGS_replay_buffer_reuse);
-  config.checkpoint_freq = absl::GetFlag(FLAGS_checkpoint_freq);
-  config.evaluation_window = absl::GetFlag(FLAGS_evaluation_window);
-  config.uct_c = absl::GetFlag(FLAGS_uct_c);
-  config.max_simulations = absl::GetFlag(FLAGS_max_simulations);
-  config.train_batch_size = absl::GetFlag(FLAGS_train_batch_size);
-  config.inference_batch_size = absl::GetFlag(FLAGS_inference_batch_size);
-  config.inference_threads = absl::GetFlag(FLAGS_inference_threads);
-  config.inference_cache = absl::GetFlag(FLAGS_inference_cache);
-  config.policy_alpha = absl::GetFlag(FLAGS_policy_alpha);
-  config.policy_epsilon = absl::GetFlag(FLAGS_policy_epsilon);
-  config.temperature = absl::GetFlag(FLAGS_temperature);
-  config.temperature_drop = absl::GetFlag(FLAGS_temperature_drop);
-  config.cutoff_probability = absl::GetFlag(FLAGS_cutoff_probability);
-  config.cutoff_value = absl::GetFlag(FLAGS_cutoff_value);
-  config.actors = absl::GetFlag(FLAGS_actors);
-  config.evaluators = absl::GetFlag(FLAGS_evaluators);
-  config.eval_levels = absl::GetFlag(FLAGS_eval_levels);
-  config.max_steps = absl::GetFlag(FLAGS_max_steps);
 
-  return !AlphaZero(config, &stop_token);
+  if (positional_args.size() > 1) {
+    // Resume training from a checkpoint.
+    resuming = true;
+
+    if (argc > 2) {
+      open_spiel::SpielFatalError(
+          "Specify only a path to a config.json to resume training.");
+    }
+
+    open_spiel::file::File config_file(positional_args[1], "r");
+    std::string config_string = config_file.ReadContents();
+    open_spiel::json::Object config_json = open_spiel::json::FromString(
+        config_string).value().GetObject();
+
+    config.FromJson(config_json);
+  } else {
+    // Start training from scratch.
+    resuming = false;
+
+    config.game = absl::GetFlag(FLAGS_game);
+    config.path = absl::GetFlag(FLAGS_path);
+    config.graph_def = absl::GetFlag(FLAGS_graph_def);
+    config.nn_model = absl::GetFlag(FLAGS_nn_model);
+    config.nn_width = absl::GetFlag(FLAGS_nn_width);
+    config.nn_depth = absl::GetFlag(FLAGS_nn_depth);
+    config.devices = absl::GetFlag(FLAGS_devices);
+    config.explicit_learning = absl::GetFlag(FLAGS_explicit_learning);
+    config.learning_rate = absl::GetFlag(FLAGS_learning_rate);
+    config.weight_decay = absl::GetFlag(FLAGS_weight_decay);
+    config.train_batch_size = absl::GetFlag(FLAGS_train_batch_size);
+    config.replay_buffer_size = absl::GetFlag(FLAGS_replay_buffer_size);
+    config.replay_buffer_reuse = absl::GetFlag(FLAGS_replay_buffer_reuse);
+    config.checkpoint_freq = absl::GetFlag(FLAGS_checkpoint_freq);
+    config.evaluation_window = absl::GetFlag(FLAGS_evaluation_window);
+    config.uct_c = absl::GetFlag(FLAGS_uct_c);
+    config.max_simulations = absl::GetFlag(FLAGS_max_simulations);
+    config.train_batch_size = absl::GetFlag(FLAGS_train_batch_size);
+    config.inference_batch_size = absl::GetFlag(FLAGS_inference_batch_size);
+    config.inference_threads = absl::GetFlag(FLAGS_inference_threads);
+    config.inference_cache = absl::GetFlag(FLAGS_inference_cache);
+    config.policy_alpha = absl::GetFlag(FLAGS_policy_alpha);
+    config.policy_epsilon = absl::GetFlag(FLAGS_policy_epsilon);
+    config.temperature = absl::GetFlag(FLAGS_temperature);
+    config.temperature_drop = absl::GetFlag(FLAGS_temperature_drop);
+    config.cutoff_probability = absl::GetFlag(FLAGS_cutoff_probability);
+    config.cutoff_value = absl::GetFlag(FLAGS_cutoff_value);
+    config.actors = absl::GetFlag(FLAGS_actors);
+    config.evaluators = absl::GetFlag(FLAGS_evaluators);
+    config.eval_levels = absl::GetFlag(FLAGS_eval_levels);
+    config.max_steps = absl::GetFlag(FLAGS_max_steps);
+  }
+
+  return !AlphaZero(config, &stop_token, resuming);
 }
diff --git a/open_spiel/examples/alpha_zero_torch_game_example.cc b/open_spiel/examples/alpha_zero_torch_game_example.cc
index cd899d1744..6ad6c4d47e 100644
--- a/open_spiel/examples/alpha_zero_torch_game_example.cc
+++ b/open_spiel/examples/alpha_zero_torch_game_example.cc
@@ -187,7 +187,8 @@ int main(int argc, char **argv) {
   open_spiel::algorithms::torch_az::DeviceManager device_manager;
   device_manager.AddDevice(open_spiel::algorithms::torch_az::VPNetModel(
       *game, absl::GetFlag(FLAGS_az_path), absl::GetFlag(FLAGS_az_graph_def),
-      "/cpu:0", absl::GetFlag(FLAGS_az_checkpoint)));
+      "/cpu:0"));
+  device_manager.Get(0, 0)->LoadCheckpoint(absl::GetFlag(FLAGS_az_checkpoint));
   auto az_evaluator =
       std::make_shared<open_spiel::algorithms::torch_az::VPNetEvaluator>(
           /*device_manager=*/&device_manager,
diff --git a/open_spiel/libnop/.gitignore b/open_spiel/libnop/.gitignore
new file mode 100644
index 0000000000..8d249ab4c6
--- /dev/null
+++ b/open_spiel/libnop/.gitignore
@@ -0,0 +1,1 @@
+libnop/
diff --git a/open_spiel/libnop/CMakeLists.txt b/open_spiel/libnop/CMakeLists.txt
new file mode 100644
index 0000000000..a9b24bfde0
--- /dev/null
+++ b/open_spiel/libnop/CMakeLists.txt
@@ -0,0 +1,4 @@
+# An integration test to make sure that we can link with libnop.
+add_executable(libnop_integration_test
+  libnop_integration_test.cc ${OPEN_SPIEL_OBJECTS})
+add_test(libnop_integration_test libnop_integration_test)
diff --git a/open_spiel/scripts/global_variables.sh b/open_spiel/scripts/global_variables.sh
index c093eb3a7e..bf795e6443 100644
--- a/open_spiel/scripts/global_variables.sh
+++ b/open_spiel/scripts/global_variables.sh
@@ -41,6 +41,10 @@ export OPEN_SPIEL_BUILD_WITH_XINXIN=${OPEN_SPIEL_BUILD_WITH_XINXIN:-$DEFAULT_OPT
 export OPEN_SPIEL_BUILD_WITH_ROSHAMBO=${OPEN_SPIEL_BUILD_WITH_ROSHAMBO:-$DEFAULT_OPTIONAL_DEPENDENCY}
 export OPEN_SPIEL_BUILD_WITH_GO=${OPEN_SPIEL_BUILD_WITH_GO:-$DEFAULT_OPTIONAL_DEPENDENCY}
 
+# Download the header-only library, libnop (https://github.com/google/libnop),
+# to support the serialization and deserialization of C++ data types.
+export OPEN_SPIEL_BUILD_WITH_LIBNOP="${OPEN_SPIEL_BUILD_WITH_LIBNOP:-"OFF"}"
+
 # Download precompiled binaries for libtorch (PyTorch C++ API).
 # See https://pytorch.org/cppdocs/ for C++ documentation.
 # This dependency is currently not supported by Travis CI test.
diff --git a/open_spiel/scripts/install.sh b/open_spiel/scripts/install.sh
index 54b580f071..6386a0f15c 100755
--- a/open_spiel/scripts/install.sh
+++ b/open_spiel/scripts/install.sh
@@ -127,6 +127,12 @@ if [[ ${OPEN_SPIEL_BUILD_WITH_ROSHAMBO:-"ON"} == "ON" ]] && [[ ! -d ${DIR} ]]; t
   git clone -b 'open_spiel' --single-branch --depth 1  https://github.com/jhtschultz/roshambo.git ${DIR}
 fi
 
+# This GitHub repository allows for serialization of custom C++ objects.
+DIR="open_spiel/libnop/libnop"
+if [[ ${OPEN_SPIEL_BUILD_WITH_LIBNOP:-"ON"} == "ON" ]] && [[ ! -d ${DIR} ]]; then
+  git clone -b 'master' --single-branch --depth 1  https://github.com/google/libnop.git ${DIR}
+fi
+
 # Add libtorch (PyTorch C++ API).
 # This downloads the precompiled binaries available from the pytorch website.
 DIR="open_spiel/libtorch/libtorch"
diff --git a/open_spiel/utils/CMakeLists.txt b/open_spiel/utils/CMakeLists.txt
index cc3c605819..f06826d9e1 100644
--- a/open_spiel/utils/CMakeLists.txt
+++ b/open_spiel/utils/CMakeLists.txt
@@ -17,6 +17,7 @@ add_library (utils OBJECT
   random.cc
   run_python.h
   run_python.cc
+  serializable_circular_buffer.h
   serialization.h
   stats.h
   tensor_view.h
@@ -69,6 +70,13 @@ if (OPEN_SPIEL_BUILD_WITH_PYTHON)
            --python_command=${Python3_EXECUTABLE})
 endif()
 
+if (OPEN_SPIEL_BUILD_WITH_LIBNOP)
+  add_executable(serializable_circular_buffer_test
+                 serializable_circular_buffer_test.cc ${OPEN_SPIEL_OBJECTS}
+                 $<TARGET_OBJECTS:tests>)
+  add_test(serializable_circular_buffer_test serializable_circular_buffer_test)
+endif()
+
 add_executable(stats_test stats_test.cc ${OPEN_SPIEL_OBJECTS}
                $<TARGET_OBJECTS:tests>)
 add_test(stats_test stats_test)
diff --git a/open_spiel/utils/circular_buffer.h b/open_spiel/utils/circular_buffer.h
index 1b265d12a0..8c9e9a5a79 100644
--- a/open_spiel/utils/circular_buffer.h
+++ b/open_spiel/utils/circular_buffer.h
@@ -61,7 +61,7 @@ class CircularBuffer {
   // How many elements have ever been added to the buffer.
   int64_t TotalAdded() const { return total_added_; }
 
- private:
+ protected:
   const int max_size_;
   int64_t total_added_;
   std::vector<T> data_;
diff --git a/open_spiel/utils/data_logger.cc b/open_spiel/utils/data_logger.cc
index b60ad7ad4a..c2b00c933d 100644
--- a/open_spiel/utils/data_logger.cc
+++ b/open_spiel/utils/data_logger.cc
@@ -24,10 +24,12 @@
 namespace open_spiel {
 
 DataLoggerJsonLines::DataLoggerJsonLines(const std::string& path,
-                                         const std::string& name, bool flush)
-    : fd_(absl::StrFormat("%s/%s.jsonl", path, name), "w"),
+                                         const std::string& name, bool flush,
+                                         const std::string& mode,
+                                         absl::Time start_time)
+    : fd_(absl::StrFormat("%s/%s.jsonl", path, name), mode),
       flush_(flush),
-      start_time_(absl::Now()) {}
+      start_time_(start_time) {}
 
 void DataLoggerJsonLines::Write(DataLogger::Record record) {
   static absl::TimeZone utc = absl::UTCTimeZone();
diff --git a/open_spiel/utils/data_logger.h b/open_spiel/utils/data_logger.h
index 143cfa494f..be3b53778e 100644
--- a/open_spiel/utils/data_logger.h
+++ b/open_spiel/utils/data_logger.h
@@ -38,7 +38,9 @@ class DataLogger {
 class DataLoggerJsonLines : public DataLogger {
  public:
   explicit DataLoggerJsonLines(const std::string& path, const std::string& name,
-                               bool flush = false);
+                               bool flush = false,
+                               const std::string& mode = "w",
+                               absl::Time start_time = absl::Now());
   ~DataLoggerJsonLines() override;
 
   // The json lines logger is move only.
diff --git a/open_spiel/utils/logger.h b/open_spiel/utils/logger.h
index fb76bfc683..3b9eb7ded6 100644
--- a/open_spiel/utils/logger.h
+++ b/open_spiel/utils/logger.h
@@ -42,8 +42,9 @@ class Logger {
 // A logger to print stuff to a file.
 class FileLogger : public Logger {
  public:
-  FileLogger(const std::string& path, const std::string& name)
-      : fd_(absl::StrFormat("%s/log-%s.txt", path, name), "w"),
+  FileLogger(const std::string& path, const std::string& name,
+             const std::string& mode = "w")
+      : fd_(absl::StrFormat("%s/log-%s.txt", path, name), mode),
         tz_(absl::LocalTimeZone()) {
     Print("%s started", name);
   }
diff --git a/open_spiel/utils/serializable_circular_buffer.h b/open_spiel/utils/serializable_circular_buffer.h
new file mode 100644
index 0000000000..54f3db781d
--- /dev/null
+++ b/open_spiel/utils/serializable_circular_buffer.h
@@ -0,0 +1,70 @@
+// Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef OPEN_SPIEL_UTILS_SERIALIZABLE_CIRCULAR_BUFFER_H_
+#define OPEN_SPIEL_UTILS_SERIALIZABLE_CIRCULAR_BUFFER_H_
+
+#include <nop/serializer.h>
+#include <nop/utility/stream_reader.h>
+#include <nop/utility/stream_writer.h>
+
+#include <algorithm>
+#include <fstream>
+#include <random>
+#include <string>
+#include <vector>
+
+#include "open_spiel/abseil-cpp/absl/strings/str_format.h"
+#include "open_spiel/spiel_utils.h"
+#include "open_spiel/utils/circular_buffer.h"
+
+namespace open_spiel {
+
+// A serializable circular buffer of fixed size.
+template <class T>
+class SerializableCircularBuffer : public CircularBuffer<T> {
+ public:
+  explicit SerializableCircularBuffer(int max_size)
+      : CircularBuffer<T>(max_size) {}
+
+  // Serialize the data of the buffer to a file.
+  void SaveBuffer(const std::string& path) const {
+    nop::Serializer<nop::StreamWriter<std::ofstream>> serializer{path};
+    serializer.Write(this->max_size_);
+    serializer.Write(this->total_added_);
+    serializer.Write(this->data_);
+  }
+
+  // Populate the buffer with data from a saved buffer's file.
+  void LoadBuffer(const std::string& path) {
+    nop::Deserializer<nop::StreamReader<std::ifstream>> deserializer{path};
+
+    // Ensure this buffer's max size equals the max size of the saved buffer.
+    int max_size;
+    deserializer.Read(&max_size);
+    if (max_size != this->max_size_) {
+      SpielFatalError(absl::StrFormat("Cannot load data from a buffer with max"
+                                      "size %d into a buffer with max size %d.",
+                                      max_size,
+                                      this->max_size_));
+    }
+
+    deserializer.Read(&(this->total_added_));
+    deserializer.Read(&(this->data_));
+  }
+};
+
+}  // namespace open_spiel
+
+#endif  // OPEN_SPIEL_UTILS_SERIALIZABLE_CIRCULAR_BUFFER_H_


Test patch:
diff --git a/open_spiel/libnop/libnop_integration_test.cc b/open_spiel/libnop/libnop_integration_test.cc
new file mode 100644
index 0000000000..6e2b000438
--- /dev/null
+++ b/open_spiel/libnop/libnop_integration_test.cc
@@ -0,0 +1,65 @@
+// Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <array>
+#include <iostream>
+#include <sstream>
+#include <string>
+
+#include <nop/serializer.h>
+#include <nop/structure.h>
+#include <nop/utility/stream_writer.h>
+
+namespace {
+
+// Libnop example taken from
+// https://github.com/google/libnop/blob/master/README.md
+
+// Contrived template type with private members.
+template <typename T>
+struct UserDefined {
+ public:
+  UserDefined() = default;
+  UserDefined(std::string label, std::vector<T> vector)
+      : label_{std::move(label)}, vector_{std::move(vector)} {}
+
+  const std::string label() const { return label_; }
+  const std::vector<T>& vector() const { return vector_; }
+
+ private:
+  std::string label_;
+  std::vector<T> vector_;
+
+  NOP_STRUCTURE(UserDefined, label_, vector_);
+};
+
+void TestSerialization() {
+  using Writer = nop::StreamWriter<std::stringstream>;
+  nop::Serializer<Writer> serializer;
+
+  serializer.Write(UserDefined<int>{"ABC", {1, 2, 3, 4, 5}});
+
+  using ArrayType = std::array<UserDefined<float>, 2>;
+  serializer.Write(
+      ArrayType{{{"ABC", {1, 2, 3, 4, 5}}, {"XYZ", {3.14, 2.72, 23.14}}}});
+
+  const std::string data = serializer.writer().stream().str();
+  std::cout << "Wrote " << data.size() << " bytes." << std::endl;
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  TestSerialization();
+}
diff --git a/open_spiel/utils/serializable_circular_buffer_test.cc b/open_spiel/utils/serializable_circular_buffer_test.cc
new file mode 100644
index 0000000000..01f291b2ef
--- /dev/null
+++ b/open_spiel/utils/serializable_circular_buffer_test.cc
@@ -0,0 +1,157 @@
+// Copyright 2019 DeepMind Technologies Ltd. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "open_spiel/utils/serializable_circular_buffer.h"
+
+#include <nop/structure.h>
+
+#include <random>
+#include <string>
+#include <vector>
+#include <utility>
+
+#include "open_spiel/spiel_utils.h"
+#include "open_spiel/utils/file.h"
+
+namespace open_spiel {
+namespace {
+
+const char* kSimpleSerializationFilename = "simple_buffer_data.nop";
+const char* kComplexSerializationFilename = "complex_buffer_data.nop";
+
+struct TestStruct {
+  std::vector<Action> action_vector;
+  std::vector<float> float_vector;
+  std::vector<std::pair<Action, double>> actions_and_probs;
+  double double_value;
+
+  bool operator==(const TestStruct& other_test_struct) const {
+    return action_vector == other_test_struct.action_vector &&
+           float_vector == other_test_struct.float_vector &&
+           actions_and_probs == other_test_struct.actions_and_probs &&
+           double_value == other_test_struct.double_value;
+  }
+
+  NOP_STRUCTURE(TestStruct,
+                action_vector,
+                float_vector,
+                actions_and_probs,
+                double_value);
+};
+
+void TestSerializableCircularBuffer() {
+  SerializableCircularBuffer<int> buffer(4);
+  std::mt19937 rng;
+  std::vector<int> sample;
+
+  SPIEL_CHECK_TRUE(buffer.Empty());
+  SPIEL_CHECK_EQ(buffer.Size(), 0);
+
+  buffer.Add(13);
+  SPIEL_CHECK_FALSE(buffer.Empty());
+  SPIEL_CHECK_EQ(buffer.Size(), 1);
+  SPIEL_CHECK_EQ(buffer.TotalAdded(), 1);
+  SPIEL_CHECK_EQ(buffer[0], 13);
+
+  sample = buffer.Sample(&rng, 1);
+  SPIEL_CHECK_EQ(sample.size(), 1);
+  SPIEL_CHECK_EQ(sample[0], 13);
+
+  buffer.Add(14);
+  buffer.Add(15);
+  buffer.Add(16);
+
+  SPIEL_CHECK_EQ(buffer.Size(), 4);
+  SPIEL_CHECK_EQ(buffer.TotalAdded(), 4);
+
+  sample = buffer.Sample(&rng, 2);
+  SPIEL_CHECK_EQ(sample.size(), 2);
+  SPIEL_CHECK_GE(sample[0], 13);
+  SPIEL_CHECK_LE(sample[0], 16);
+  SPIEL_CHECK_GE(sample[1], 13);
+  SPIEL_CHECK_LE(sample[1], 16);
+
+  buffer.Add(17);
+  buffer.Add(18);
+
+  SPIEL_CHECK_EQ(buffer.Size(), 4);
+  SPIEL_CHECK_EQ(buffer.TotalAdded(), 6);
+
+  sample = buffer.Sample(&rng, 1);
+  SPIEL_CHECK_EQ(sample.size(), 1);
+  SPIEL_CHECK_GE(sample[0], 15);
+  SPIEL_CHECK_LE(sample[0], 18);
+}
+
+void TestSimpleSerializableCircularBufferSerialization() {
+  SerializableCircularBuffer<int> original_buffer(6);
+  original_buffer.Add(1);
+  original_buffer.Add(2);
+  original_buffer.Add(3);
+  original_buffer.Add(4);
+  original_buffer.Add(5);
+  original_buffer.Add(6);
+  original_buffer.SaveBuffer(kSimpleSerializationFilename);
+
+  SerializableCircularBuffer<int> new_buffer(6);
+  new_buffer.LoadBuffer(kSimpleSerializationFilename);
+
+  SPIEL_CHECK_EQ(original_buffer.Size(), new_buffer.Size());
+  SPIEL_CHECK_EQ(original_buffer.TotalAdded(), new_buffer.TotalAdded());
+  SPIEL_CHECK_TRUE(original_buffer.Data() == new_buffer.Data());
+}
+
+void TestComplexSerializableCircularBufferSerialization() {
+  TestStruct struct1 = {.action_vector = {1, 2, 3},
+                        .float_vector = {1.0f, 2.0f, 3.0f},
+                        .actions_and_probs = {{1, 1.0}, {2, 2.0}, {3, 3.0}},
+                        .double_value = 1.23};
+  TestStruct struct2 = {.action_vector = {4, 5, 6},
+                        .float_vector = {4.0f, 5.0f, 6.0f},
+                        .actions_and_probs = {{4, 4.0}, {5, 5.0}, {6, 6.0}},
+                        .double_value = 4.56};
+  TestStruct struct3 = {.action_vector = {7, 8, 9},
+                        .float_vector = {7.0f, 8.0f, 9.0f},
+                        .actions_and_probs = {{7, 7.0}, {8, 8.0}, {9, 9.0}},
+                        .double_value = 7.89};
+
+  SerializableCircularBuffer<TestStruct> original_buffer(3);
+  original_buffer.Add(struct1);
+  original_buffer.Add(struct2);
+  original_buffer.Add(struct3);
+  original_buffer.SaveBuffer(kComplexSerializationFilename);
+
+  SerializableCircularBuffer<TestStruct> new_buffer(3);
+  new_buffer.LoadBuffer(kComplexSerializationFilename);
+
+  SPIEL_CHECK_EQ(original_buffer.Size(), new_buffer.Size());
+  SPIEL_CHECK_EQ(original_buffer.TotalAdded(), new_buffer.TotalAdded());
+  SPIEL_CHECK_TRUE(original_buffer.Data() == new_buffer.Data());
+}
+
+void EndCircularBufferTest() {
+  // Remove the files created in the serialization tests.
+  file::Remove(kSimpleSerializationFilename);
+  file::Remove(kComplexSerializationFilename);
+}
+
+}  // namespace
+}  // namespace open_spiel
+
+int main(int argc, char** argv) {
+  open_spiel::TestSerializableCircularBuffer();
+  open_spiel::TestSimpleSerializableCircularBufferSerialization();
+  open_spiel::TestComplexSerializableCircularBufferSerialization();
+  open_spiel::EndCircularBufferTest();
+}