# Buggy code
def _calculate_statistics(self):
    """Calculate and log simple summary statistics of the datasets"""
    logger.info("")
    logger.info("DATASETS SUMMARY")
    logger.info("================")

    self.counts = {}

    if self.data["train"]:
        self.counts["train"] = len(self.data["train"])
    else:
        self.counts["train"] = 0

    if self.data["dev"]:
        self.counts["dev"] = len(self.data["dev"])
    else:
        self.counts["dev"] = 0

    if self.data["test"]:
        self.counts["test"] = len(self.data["test"])
    else:
        self.counts["test"] = 0

    seq_lens = []
    if self.data["train"]:
        for dataset in self.data["train"].datasets:
            train_input_numpy = dataset[:][0].numpy()
            seq_lens.extend(
                np.sum(
                    train_input_numpy != self.processor.tokenizer.pad_token_id, axis=1
                )
            )
        max_seq_len = dataset[:][0].shape[1]

    self.clipped = np.mean(np.array(seq_lens) == max_seq_len) if seq_lens else 0
    self.ave_len = np.mean(seq_lens) if seq_lens else 0

    logger.info("Examples in train: {}".format(self.counts["train"]))
    logger.info("Examples in dev  : {}".format(self.counts["dev"]))
    logger.info("Examples in test : {}".format(self.counts["test"]))
    logger.info("")
    if self.data["train"]:
        logger.info(
            "Longest sequence length observed after clipping:     {}".format(
                max(seq_lens)
            )
        )
        logger.info("Average sequence length after clipping: {}".format(self.ave_len))
        logger.info("Proportion clipped:      {}".format(self.clipped))
        if self.clipped > 0.5:
            logger.info(
                "[Farmer's Tip] {}% of your samples got cut down to {} tokens. "
                "Consider increasing max_seq_len. "
                "This will lead to higher memory consumption but is likely to "
                "improve your model performance".format(
                    round(self.clipped * 100, 1), max_seq_len
                )
            )

    MlLogger.log_params(
        {
            "n_samples_train": self.counts["train"],
            "n_samples_dev": self.counts["dev"],
            "n_samples_test": self.counts["test"],
            "batch_size": self.batch_size,
            "ave_seq_len": self.ave_len,
            "clipped": self.clipped,
        }
    )


# Traceback (AttributeError)
Traceback (most recent call last):
File "/home/ubuntu/pycharm/FARM/examples/dpr_encoder.py", line 155, in <module>
dense_passage_retrieval()
File "/home/ubuntu/pycharm/FARM/examples/dpr_encoder.py", line 91, in dense_passage_retrieval
data_silo = DataSilo(processor=processor, batch_size=batch_size, distributed=distributed, max_processes=128)
File "/home/ubuntu/pycharm/FARM/farm/data_handler/data_silo.py", line 113, in __init__
self._load_data()
File "/home/ubuntu/pycharm/FARM/farm/data_handler/data_silo.py", line 272, in _load_data
self._calculate_statistics()
File "/home/ubuntu/pycharm/FARM/farm/data_handler/data_silo.py", line 454, in _calculate_statistics
seq_lens.extend(np.sum(train_input_numpy != self.processor.tokenizer.pad_token_id, axis=1))
AttributeError: 'NoneType' object has no attribute 'pad_token_id'

# Fixed code
def _calculate_statistics(self):
    """Calculate and log simple summary statistics of the datasets"""
    logger.info("")
    logger.info("DATASETS SUMMARY")
    logger.info("================")

    self.counts = {}

    if self.data["train"]:
        self.counts["train"] = len(self.data["train"])
        if "input_ids" in self.tensor_names:
            clipped, ave_len, seq_lens, max_seq_len = (
                self._calc_length_stats_single_encoder()
            )
        elif (
            "query_input_ids" in self.tensor_names
            and "passage_input_ids" in self.tensor_names
        ):
            clipped, ave_len, seq_lens, max_seq_len = (
                self._calc_length_stats_biencoder()
            )
        else:
            logger.warning(
                f"Could not compute length statistics because 'input_ids' or 'query_input_ids' and 'passage_input_ids' are missing."
            )
            clipped = -1
            ave_len = -1
    else:
        self.counts["train"] = 0

    if self.data["dev"]:
        self.counts["dev"] = len(self.data["dev"])
    else:
        self.counts["dev"] = 0

    if self.data["test"]:
        self.counts["test"] = len(self.data["test"])
    else:
        self.counts["test"] = 0

    logger.info("Examples in train: {}".format(self.counts["train"]))
    logger.info("Examples in dev  : {}".format(self.counts["dev"]))
    logger.info("Examples in test : {}".format(self.counts["test"]))
    logger.info("")
    if self.data["train"]:
        if "input_ids" in self.tensor_names:
            logger.info(
                "Longest sequence length observed after clipping:     {}".format(
                    max(seq_lens)
                )
            )
            logger.info("Average sequence length after clipping: {}".format(ave_len))
            logger.info("Proportion clipped:      {}".format(clipped))
            if clipped > 0.5:
                logger.info(
                    "[Farmer's Tip] {}% of your samples got cut down to {} tokens. "
                    "Consider increasing max_seq_len. "
                    "This will lead to higher memory consumption but is likely to "
                    "improve your model performance".format(
                        round(clipped * 100, 1), max_seq_len
                    )
                )
        elif (
            "query_input_ids" in self.tensor_names
            and "passage_input_ids" in self.tensor_names
        ):
            logger.info(
                "Longest query length observed after clipping: {}   - for max_query_len: {}".format(
                    max(seq_lens[0]), max_seq_len[0]
                )
            )
            logger.info(
                "Average query length after clipping:          {}".format(ave_len[0])
            )
            logger.info(
                "Proportion queries clipped:                   {}".format(clipped[0])
            )
            logger.info("")
            logger.info(
                "Longest passage length observed after clipping: {}   - for max_passage_len: {}".format(
                    max(seq_lens[1]), max_seq_len[1]
                )
            )
            logger.info(
                "Average passage length after clipping:          {}".format(ave_len[1])
            )
            logger.info(
                "Proportion passages clipped:                    {}".format(clipped[1])
            )

    MlLogger.log_params(
        {
            "n_samples_train": self.counts["train"],
            "n_samples_dev": self.counts["dev"],
            "n_samples_test": self.counts["test"],
            "batch_size": self.batch_size,
            "ave_seq_len": ave_len,
            "clipped": clipped,
        }
    )