# Buggy code
def extract_weights(self, state_dict, iteration):
    for key in state_dict.keys():
        vec = state_dict[key]
        weights_to_watch = min(
            self.number_of_weights, reduce(lambda x, y: x * y, list(vec.size()))
        )

        if key not in self.weights_dict:
            self._init_weights_index(key, state_dict, weights_to_watch)

        for i in range(weights_to_watch):
            vec = state_dict[key]
            for index in self.weights_dict[key][i]:
                vec = vec[index]

            value = vec.item()

            with open(self.weights_file, "a") as f:
                f.write("{}\t{}\t{}\t{}\n".format(iteration, key, i, float(value)))


# Traceback (TypeError)
----------------------------------------------------------------------------------------------------
2020-06-12 18:44:49,152 Model training base path: "dialogpt"
2020-06-12 18:44:49,152 ----------------------------------------------------------------------------------------------------

2020-06-12 18:44:49,153 Device: cuda:0
2020-06-12 18:44:49,154 ----------------------------------------------------------------------------------------------------
2020-06-12 18:44:49,155 Embeddings storage mode: cpu
2020-06-12 18:44:49,230 ----------------------------------------------------------------------------------------------------
2020-06-12 18:49:00,669 epoch 1 - iter 76/766 - loss 1.11887558 - samples/sec: 9.67

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-7-e12f92b71d0b> in <module>
7         mini_batch_chunk_size=2, # set this if you get OOM errors
8         max_epochs=1, # very few epochs of fine-tuning
----> 9         embeddings_storage_mode='cpu',
10         #checkpoint=True
11     )

~/venv36/lib64/python3.6/site-packages/flair/trainers/trainer.py in train(self, base_path, learning_rate, mini_batch_size, mini_batch_chunk_size, max_epochs, scheduler, anneal_factor, patience, initial_extra_patience, min_learning_rate, train_with_dev, monitor_train, monitor_test, embeddings_storage_mode, checkpoint, save_final_model, anneal_with_restarts, anneal_with_prestarts, batch_growth_annealing, shuffle, param_selection_mode, num_workers, sampler, use_amp, amp_opt_level, eval_on_train_fraction, eval_on_train_shuffle, **kwargs)
376                         if not param_selection_mode:
377                             weight_extractor.extract_weights(
--> 378                                 self.model.state_dict(), iteration
379                             )
380

~/venv36/lib64/python3.6/site-packages/flair/training_utils.py in extract_weights(self, state_dict, iteration)
262             vec = state_dict[key]
263             weights_to_watch = min(
--> 264                 self.number_of_weights, reduce(lambda x, y: x * y, list(vec.size()))
265             )
266

TypeError: reduce() of empty sequence with no initial value

# Fixed code
def extract_weights(self, state_dict, iteration):
    for key in state_dict.keys():
        vec = state_dict[key]
        # print(vec)
        try:
            weights_to_watch = min(
                self.number_of_weights, reduce(lambda x, y: x * y, list(vec.size()))
            )
        except:
            continue

        if key not in self.weights_dict:
            self._init_weights_index(key, state_dict, weights_to_watch)

        for i in range(weights_to_watch):
            vec = state_dict[key]
            for index in self.weights_dict[key][i]:
                vec = vec[index]

            value = vec.item()

            with open(self.weights_file, "a") as f:
                f.write("{}\t{}\t{}\t{}\n".format(iteration, key, i, float(value)))