Repository: data61/blocklib

Issue:
Convert printing to logging
As a library `blocklib` shouldn't `print`

Discussion:
codecov[bot]: # [Codecov](https://codecov.io/gh/data61/blocklib/pull/138?src=pr&el=h1) Report
> Merging [#138](https://codecov.io/gh/data61/blocklib/pull/138?src=pr&el=desc) (f79645e) into [master](https://codecov.io/gh/data61/blocklib/commit/2d10854d6ce7de9f3a9ae6f13b43badc9f16b67d?el=desc) (2d10854) will **decrease** coverage by `3.13%`.
> The diff coverage is `69.09%`.


```diff
@@            Coverage Diff             @@
##           master     #138      +/-   ##
==========================================
- Coverage   98.15%   95.02%   -3.14%     
==========================================
  Files          14       14              
  Lines         543      543              
==========================================
- Hits          533      516      -17     
- Misses         10       27      +17     
```



Patch:
diff --git a/blocklib/candidate_blocks_generator.py b/blocklib/candidate_blocks_generator.py
index 9957bb1..f15b151 100644
--- a/blocklib/candidate_blocks_generator.py
+++ b/blocklib/candidate_blocks_generator.py
@@ -1,7 +1,9 @@
 """Class that implement candidate block generations."""
-from typing import Dict, Sequence, Tuple, Type, List, Optional
+import sys
+
+from typing import Dict, Sequence, Tuple, Type, List, Optional, TextIO
 from .configuration import get_config
-from .pprlindex import PPRLIndex
+from .pprlindex import PPRLIndex, ReversedIndexResult
 from .pprlpsig import PPRLIndexPSignature
 from .pprllambdafold import PPRLIndexLambdaFold
 from .validation import validate_signature_config
@@ -13,22 +15,48 @@ PPRLSTATES = {
 }  # type: Dict[str, Type[PPRLIndex]]
 
 
-
 class CandidateBlockingResult:
-    """Object for holding candidate blocking results."""
+    """Object for holding candidate blocking results.
 
-    def __init__(self, blocks: Dict, state: PPRLIndex):
+    :ivar blocks: a dictionary that contains a mapping from the block ID to the record IDs in that block.
+    :ivar state: A PPRLIndex state that contains the configuration of blocking
+    :ivar stats: a dictionary containing the summary statistics of the generated blocks"""
+
+    def __init__(self, blocking_result: ReversedIndexResult, state: PPRLIndex):
         """
         Initialise a blocking result object.
-        :param blocks: A dictionary where key is set of 1 bits in bloom filter and value is a list of record IDs
+        :param blocking_result: A ReversedIndexResult object, containing the blocks and corresponding statistics
         :param state: A PPRLIndex state that contains configuration of blocking
         """
-        self.blocks = blocks
+        self.blocks = blocking_result.reversed_index
         self.state = state
+        self.stats = blocking_result.stats
 
-
-def generate_candidate_blocks(data: Sequence[Tuple[str, ...]], signature_config: Dict, header: Optional[List[str]] = None,
-                              verbose: bool = False):
+    def print_summary_statistics(self, output: TextIO = sys.stdout):
+        """
+        Print the summary statistics of this candidate blocking result to 'output'.
+        :param output: a file like object to write to. Defaults to sys.stdout
+        """
+        def print_stats(stats: Dict, out: TextIO):
+            out.write('\tNumber of Blocks:   {}\n'.format(stats['num_of_blocks']))
+            out.write('\tMinimum Block Size: {}\n'.format(stats['min_size']))
+            out.write('\tMaximum Block Size: {}\n'.format(stats['max_size']))
+            out.write('\tAverage Block Size: {}\n'.format(stats['avg_size']))
+            out.write('\tMedian Block Size:  {}\n'.format(stats['med_size']))
+            out.write('\tStandard Deviation of Block Size:  {}\n'.format(stats['std_size']))
+            if 'coverage' in stats:
+                out.write('\tCoverage:           {}%\n'.format(round(stats['coverage'] * 100, 2)))
+
+        output.write('Statistics for the generated blocks:\n')
+        print_stats(self.stats, output)
+        if 'statistics_per_strategy' in self.stats:
+            output.write('Individual statistics for each strategy:\n')
+            for stat in self.stats['statistics_per_strategy']:
+                output.write('Strategy: {}\n'.format(stat['strategy_idx']))
+                print_stats(stat, output)
+
+
+def generate_candidate_blocks(data: Sequence[Tuple[str, ...]], signature_config: Dict, header: Optional[List[str]] = None):
     """
     :param data: list of tuples E.g. ('0', 'Kenneth Bain', '1964/06/17', 'M')
     :param signature_config:
@@ -37,7 +65,6 @@ def generate_candidate_blocks(data: Sequence[Tuple[str, ...]], signature_config:
         ``docs/schema/signature-config-schema.json``
     :param header: column names (optional)
         Program should throw exception if block features are string but header is None
-    :param verbose: print additional information to std out.
 
     :return: A 2-tuple containing
         A list of "signatures" per record in data.
@@ -63,11 +90,8 @@ def generate_candidate_blocks(data: Sequence[Tuple[str, ...]], signature_config:
 
     if algorithm in PPRLSTATES:
         state = PPRLSTATES[algorithm](config)
-        reversed_index = state.build_reversed_index(data, verbose, header)
-        state.summarize_reversed_index(reversed_index)
-
-        # make candidate blocking result object
-        candidate_block_obj = CandidateBlockingResult(reversed_index, state)
+        reversed_index_result = state.build_reversed_index(data, header)
+        candidate_block_obj = CandidateBlockingResult(reversed_index_result, state)
 
     else:
         raise NotImplementedError('The algorithm {} is not supported yet'.format(algorithm))
diff --git a/blocklib/evaluation.py b/blocklib/evaluation.py
index e71a7bb..1663637 100644
--- a/blocklib/evaluation.py
+++ b/blocklib/evaluation.py
@@ -1,5 +1,6 @@
 """Module to evaluate blocking when ground truth is available."""
 from tqdm import tqdm
+import logging
 
 
 def assess_blocks_2party(filtered_reverse_indices, data):
@@ -48,7 +49,7 @@ def assess_blocks_2party(filtered_reverse_indices, data):
     # pair completeness is the "recall" before matching stage
     rr = 1.0 - float(num_cand_rec_pairs) / total_rec
     if num_all_true_matches == 0:
-        print("Pair completeness is zero, because there are no true matches in the provided data.")
+        logging.warning("Pair completeness is zero, because there are no true matches in the provided data.")
         pc = 0
     else:
         pc = float(num_block_true_matches) / num_all_true_matches
diff --git a/blocklib/pprlindex.py b/blocklib/pprlindex.py
index 052fc03..b3982d0 100644
--- a/blocklib/pprlindex.py
+++ b/blocklib/pprlindex.py
@@ -1,7 +1,6 @@
 import random
-from typing import Any, Dict, List, Sequence, Optional
+from typing import Dict, List, Sequence, Optional
 from blocklib.configuration import get_config
-from blocklib.stats import reversed_index_stats
 from blocklib.utils import check_header
 
 
@@ -13,12 +12,13 @@ class PPRLIndex:
         self.rec_dict = None
         self.ent_id_col = None
         self.rec_id_col = None
-        self.stats = {}  # type: Dict[str, Any]
 
     def get_feature_to_index_map(self, data: Sequence[Sequence], header: Optional[List[str]] = None):
         """Return feature name to feature index mapping if there is a header and feature is of type string."""
         feature_type = type(self.blocking_features[0])  # type: ignore
         feature_to_index = None
+        if len(data) == 0:
+            return feature_to_index
         tuple_type = type(data[0])  # if data is CLKs, then tuple_type will be str, otherwise a tuple
 
         if header and feature_type == str and tuple_type != str:
@@ -39,43 +39,16 @@ class PPRLIndex:
         else:
             self.blocking_features_index = blocking_features
 
-    def build_reversed_index(self, data: Sequence[Sequence], verbose: bool, header: Optional[List[str]]  = None):
+    def build_reversed_index(self, data: Sequence[Sequence],  header: Optional[List[str]] = None):
         """Method which builds the index for all database.
 
            :param data: list of tuples, PII dataset
-           :param verbose: print additional information to std out.
            :param header: file header, optional
-
+           :rtype: ReversedIndexResult
            See derived classes for actual implementations.
         """
         raise NotImplementedError("Derived class needs to implement")
 
-    def summarize_reversed_index(self, reversed_index: Dict):
-        """Summarize statistics of reverted index / blocks."""
-        assert len(reversed_index) > 0
-        # statistics of block
-        self.stats = reversed_index_stats(reversed_index)
-        # find how many blocks each entity / record is a member of
-        rec_to_block = {}  # type: Dict[Any, List[Any]]
-        for block_id, block in reversed_index.items():
-            for rec in block:
-                if rec in rec_to_block:
-                    rec_to_block[rec].append(block_id)
-                else:
-                    rec_to_block[rec] = [block_id]
-        num_of_blocks_per_rec = [len(x) for x in rec_to_block.values()]
-        self.stats['num_of_blocks_per_rec'] = num_of_blocks_per_rec
-
-        print('Statistics for the generated blocks:')
-        print('\tNumber of Blocks:   {}'.format(self.stats['num_of_blocks']))
-        print('\tMinimum Block Size: {}'.format(self.stats['min_size']))
-        print('\tMaximum Block Size: {}'.format(self.stats['max_size']))
-        print('\tAverage Block Size: {}'.format(self.stats['avg_size']))
-        print('\tMedian Block Size:  {}'.format(self.stats['med_size']))
-        print('\tStandard Deviation of Block Size:  {}'.format(self.stats['std_size']))
-
-        return self.stats
-
     def select_reference_value(self, reference_data: Sequence[Sequence], ref_data_config: Dict):
         """Load reference data for methods need reference."""
         # read configurations
@@ -92,3 +65,13 @@ class PPRLIndex:
 
         print('  Selected %d random reference values' % (len(ref_val_list)))
         return ref_val_list
+
+
+class ReversedIndexResult(object):
+
+    def __init__(self, reversed_index: Dict, stats: Dict):
+        self.reversed_index = reversed_index
+        self.stats = stats
+
+    def __eq__(self, other):
+        return self.reversed_index == other.reversed_index and self.stats == other.stats
diff --git a/blocklib/pprllambdafold.py b/blocklib/pprllambdafold.py
index 0b49fe0..0aaa1e4 100644
--- a/blocklib/pprllambdafold.py
+++ b/blocklib/pprllambdafold.py
@@ -1,11 +1,12 @@
 import random
-import numpy as np
+
 from collections import defaultdict
 from typing import Dict, Sequence, Any, List, Optional
 from blocklib.configuration import get_config
-from .pprlindex import PPRLIndex
+from .pprlindex import PPRLIndex, ReversedIndexResult
 from .encoding import generate_bloom_filter
-from .utils import deserialize_filters, check_header
+from .utils import deserialize_filters
+from .stats import reversed_index_stats
 
 
 class PPRLIndexLambdaFold(PPRLIndex):
@@ -47,12 +48,11 @@ class PPRLIndexLambdaFold(PPRLIndex):
         bloom_filter = generate_bloom_filter(grams, self.bf_len, self.num_hash_function)
         return bloom_filter
 
-    def build_reversed_index(self, data: Sequence[Any], verbose: bool = False, header: Optional[List[str]] = None):
+    def build_reversed_index(self, data: Sequence[Any], header: Optional[List[str]] = None):
         """Build inverted index for PPRL Lambda-fold blocking method.
-
         :param data: list of lists
-        :param verbose: ignored
-        :return:
+        :param header: file header, optional
+        :return: reversed index as ReversedIndexResult
         """
         feature_to_index = self.get_feature_to_index_map(data, header)
         self.set_blocking_features_index(self.blocking_features, feature_to_index)
@@ -82,5 +82,5 @@ class PPRLIndexLambdaFold(PPRLIndex):
                 lambda_table['{}{}'.format(i, block_key)].append(rec_id)
             invert_index.update(lambda_table)
 
-        return invert_index
+        return ReversedIndexResult(invert_index, reversed_index_stats(invert_index))
 
diff --git a/blocklib/pprlpsig.py b/blocklib/pprlpsig.py
index 0e39e57..f0b7449 100644
--- a/blocklib/pprlpsig.py
+++ b/blocklib/pprlpsig.py
@@ -1,4 +1,4 @@
-import statistics
+import logging
 from collections import defaultdict
 from typing import Dict, List, Sequence, Any, Optional
 
@@ -6,9 +6,9 @@ import numpy as np
 
 from .configuration import get_config
 from .encoding import flip_bloom_filter
-from .pprlindex import PPRLIndex
+from .pprlindex import PPRLIndex, ReversedIndexResult
 from .signature_generator import generate_signatures
-from .stats import reversed_index_per_strategy_stats
+from .stats import reversed_index_per_strategy_stats, reversed_index_stats
 
 
 class PPRLIndexPSignature(PPRLIndex):
@@ -35,7 +35,7 @@ class PPRLIndexPSignature(PPRLIndex):
         self.signature_strategies = get_config(config, 'signatureSpecs')
         self.rec_id_col = config.get("record-id-col", None)
 
-    def build_reversed_index(self, data: Sequence[Sequence], verbose: bool = False, header: Optional[List[str]] = None):
+    def build_reversed_index(self, data: Sequence[Sequence], header: Optional[List[str]] = None):
         """Build inverted index given P-Sig method."""
         feature_to_index = self.get_feature_to_index_map(data, header)
         self.set_blocking_features_index(self.blocking_features, feature_to_index)
@@ -57,21 +57,11 @@ class PPRLIndexPSignature(PPRLIndex):
             for i, signature in enumerate(signatures):
                 reversed_index_per_strategy[i][signature].append(rec_id)
 
-        n = len(data)
         reversed_index_per_strategy = [self.filter_reversed_index(data, reversed_index) for reversed_index in
                                        reversed_index_per_strategy]
-        if verbose:
-            strat_stats = reversed_index_per_strategy_stats(reversed_index_per_strategy, n)
-            print("Statistics for the individual strategies:")
-            for strat_stat in strat_stats:
-                print('Strategy {}:'.format(strat_stat["strategy_idx"]))
-                print('\tblock size {} min, {} max, {:.2f} avg, {} median, {:.2f} std'
-                      .format(strat_stat["min_size"], strat_stat["max_size"], strat_stat["avg_size"],
-                              strat_stat["med_size"], strat_stat["std_size"]))
-                print('\t {} blocks, {} filtered elements, {:.2f}% coverage'
-                      .format(strat_stat["num_of_blocks"], strat_stat["num_filtered_elements"],
-                              (strat_stat["coverage"] * 100)))
-
+        # somehow the reversed_index of the first strategy gets overwritten in the next step. Thus, we generate the
+        # statistics of the different strategies first.
+        strategy_stats = reversed_index_per_strategy_stats(reversed_index_per_strategy, len(data))
         # combine the reversed indices into one
         filtered_reversed_index = reversed_index_per_strategy[0]
         for rev_idx in reversed_index_per_strategy[1:]:
@@ -86,12 +76,13 @@ class PPRLIndexPSignature(PPRLIndex):
         for recids in filtered_reversed_index.values():
             for rid in recids:
                 entities.add(rid)
-        coverage = round(len(entities) / len(record_ids) * 100, 2)
-        if coverage == 100:
-            print('P-Sig: {}% records are covered in blocks'.format(coverage))
-        else:
-            print('P-Sig: Warning! only {}% records are covered in blocks. Please consider to improve signatures'
-                  .format(coverage))
+        coverage = len(entities) / len(record_ids)
+        if coverage < 1:
+            logging.warning(
+                f'The P-Sig configuration leads to incomplete coverage ({round(coverage * 100, 2)}%)!\n'
+                f'This means that not all records are part of at least one block. You can increase coverage by '
+                f'adjusting the filter to be less aggressive or by finding signatures that produce smaller block sizes.'
+            )
 
         # map signatures in reversed_index into bloom filter
         num_hash_func = int(self.blocking_config.get("number-hash-functions", None))
@@ -106,7 +97,11 @@ class PPRLIndexPSignature(PPRLIndex):
             else:
                 reversed_index[bf_set] = rec_ids
 
-        return reversed_index
+        # create some statistics around the blocking results
+        stats = reversed_index_stats(reversed_index)
+        stats['statistics_per_strategy'] = strategy_stats
+        stats['coverage'] = coverage
+        return ReversedIndexResult(reversed_index, stats)
 
     def filter_reversed_index(self, data: Sequence[Sequence], reversed_index: Dict):
         # Filter inverted index based on ratio
diff --git a/blocklib/signature_generator.py b/blocklib/signature_generator.py
index 66f37e0..04657f6 100644
--- a/blocklib/signature_generator.py
+++ b/blocklib/signature_generator.py
@@ -119,7 +119,6 @@ def generate_signatures(signature_strategies: List[List],
 
             if func is None:
                 strategy_type = spec['type']
-                print(spec)
                 raise NotImplementedError('Strategy {} is not implemented yet!'.format(strategy_type))
             else:
                 config.update(args)
diff --git a/docs/tutorial/tutorial_blocking.ipynb b/docs/tutorial/tutorial_blocking.ipynb
index 8603621..157053d 100644
--- a/docs/tutorial/tutorial_blocking.ipynb
+++ b/docs/tutorial/tutorial_blocking.ipynb
@@ -32,7 +32,11 @@
   {
    "cell_type": "code",
    "execution_count": 1,
-   "metadata": {},
+   "metadata": {
+    "pycharm": {
+     "is_executing": true
+    }
+   },
    "outputs": [
     {
      "data": {
@@ -64,7 +68,7 @@
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
-       "      <td>0</td>\n",
+       "      <th>0</th>\n",
        "      <td>761859</td>\n",
        "      <td>kate</td>\n",
        "      <td>chapman</td>\n",
@@ -72,7 +76,7 @@
        "      <td>4017</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <td>1</td>\n",
+       "      <th>1</th>\n",
        "      <td>1384455</td>\n",
        "      <td>lian</td>\n",
        "      <td>hurse</td>\n",
@@ -80,7 +84,7 @@
        "      <td>3464</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <td>2</td>\n",
+       "      <th>2</th>\n",
        "      <td>1933333</td>\n",
        "      <td>matthew</td>\n",
        "      <td>russo</td>\n",
@@ -88,7 +92,7 @@
        "      <td>4065</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <td>3</td>\n",
+       "      <th>3</th>\n",
        "      <td>1564695</td>\n",
        "      <td>lorraine</td>\n",
        "      <td>zammit</td>\n",
@@ -96,7 +100,7 @@
        "      <td>2770</td>\n",
        "    </tr>\n",
        "    <tr>\n",
-       "      <td>4</td>\n",
+       "      <th>4</th>\n",
        "      <td>5971993</td>\n",
        "      <td>ingo</td>\n",
        "      <td>richardson</td>\n",
@@ -290,7 +294,6 @@
     }
    ],
    "source": [
-    "# NBVAL_IGNORE_OUTPUT\n",
     "data_alice = df_alice.to_dict(orient='split')['data']\n",
     "print(\"Example PII\", data_alice[0])"
    ]
@@ -304,29 +307,46 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "P-Sig: 100.0% records are covered in blocks\n",
       "Statistics for the generated blocks:\n",
       "\tNumber of Blocks:   5029\n",
       "\tMinimum Block Size: 1\n",
       "\tMaximum Block Size: 61\n",
       "\tAverage Block Size: 1.8337641678266057\n",
       "\tMedian Block Size:  1\n",
-      "\tStandard Deviation of Block Size:  3.8368431973204213\n"
+      "\tStandard Deviation of Block Size:  3.8368431973204213\n",
+      "\tCoverage:           100.0%\n",
+      "Individual statistics for each strategy:\n",
+      "Strategy: 0\n",
+      "\tNumber of Blocks:   503\n",
+      "\tMinimum Block Size: 1\n",
+      "\tMaximum Block Size: 61\n",
+      "\tAverage Block Size: 9.16699801192843\n",
+      "\tMedian Block Size:  6\n",
+      "\tStandard Deviation of Block Size:  9.342740344535663\n",
+      "\tCoverage:           100.0%\n",
+      "Strategy: 1\n",
+      "\tNumber of Blocks:   4534\n",
+      "\tMinimum Block Size: 1\n",
+      "\tMaximum Block Size: 7\n",
+      "\tAverage Block Size: 1.0169827966475518\n",
+      "\tMedian Block Size:  1\n",
+      "\tStandard Deviation of Block Size:  0.1583699259848952\n",
+      "\tCoverage:           100.0%\n"
      ]
     }
    ],
    "source": [
-    "# NBVAL_IGNORE_OUTPUT\n",
     "from blocklib import generate_candidate_blocks\n",
     "\n",
-    "block_obj_alice = generate_candidate_blocks(data_alice, blocking_config)"
+    "block_obj_alice = generate_candidate_blocks(data_alice, blocking_config)\n",
+    "block_obj_alice.print_summary_statistics()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The statistics of blocks are printed for you to inspect the block distribution and decide if this is a good blocking result. Here both average and median block sizes are 1 which is resilient to frequency attack. \n",
+    "You can print the statistics of the blocks in order to inspect the block distribution and decide if this is a good blocking result. Here both average and median block sizes are 1 which is resilient to frequency attack. \n",
     "\n",
     "You can get the blocking instance and blocks/reversed indice in the `block_obj_alice`. Let's look at the first block in the reversed indcies:"
    ]
@@ -336,13 +356,6 @@
    "execution_count": 5,
    "metadata": {},
    "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "<blocklib.pprlpsig.PPRLIndexPSignature object at 0x115973510>\n"
-     ]
-    },
     {
      "data": {
       "text/plain": [
@@ -355,8 +368,6 @@
     }
    ],
    "source": [
-    "# NBVAL_IGNORE_OUTPUT\n",
-    "print(block_obj_alice.state)\n",
     "list(block_obj_alice.blocks.keys())[0]"
    ]
   },
@@ -378,7 +389,6 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "P-Sig: 100.0% records are covered in blocks\n",
       "Statistics for the generated blocks:\n",
       "\tNumber of Blocks:   5018\n",
       "\tMinimum Block Size: 1\n",
@@ -386,9 +396,24 @@
       "\tAverage Block Size: 1.8377839776803508\n",
       "\tMedian Block Size:  1\n",
       "\tStandard Deviation of Block Size:  3.838423809405143\n",
-      "<blocklib.pprlpsig.PPRLIndexPSignature object at 0x106318d10>\n",
-      "(1098, 707, 316, 1973)\n",
-      "[1, 25, 765, 1078, 1166, 1203, 1273, 1531, 1621, 1625, 1755, 1965, 2027, 2824, 3106, 3125, 3414, 3501, 3610, 4033, 4139, 4472, 4579]\n"
+      "\tCoverage:           100.0%\n",
+      "Individual statistics for each strategy:\n",
+      "Strategy: 0\n",
+      "\tNumber of Blocks:   500\n",
+      "\tMinimum Block Size: 1\n",
+      "\tMaximum Block Size: 59\n",
+      "\tAverage Block Size: 9.222\n",
+      "\tMedian Block Size:  6\n",
+      "\tStandard Deviation of Block Size:  9.337402753462053\n",
+      "\tCoverage:           100.0%\n",
+      "Strategy: 1\n",
+      "\tNumber of Blocks:   4529\n",
+      "\tMinimum Block Size: 1\n",
+      "\tMaximum Block Size: 4\n",
+      "\tAverage Block Size: 1.0181055420622653\n",
+      "\tMedian Block Size:  1\n",
+      "\tStandard Deviation of Block Size:  0.14447674684130893\n",
+      "\tCoverage:           100.0%\n"
      ]
     }
    ],
@@ -397,9 +422,7 @@
     "df_bob = pd.read_csv('data/bob.csv')\n",
     "data_bob = df_bob.to_dict(orient='split')['data']\n",
     "block_obj_bob = generate_candidate_blocks(data_bob, blocking_config)\n",
-    "print(block_obj_bob.state)\n",
-    "print(list(block_obj_bob.blocks.keys())[0])\n",
-    "print(list(block_obj_bob.blocks.values())[1])"
+    "block_obj_bob.print_summary_statistics()\n"
    ]
   },
   {
@@ -426,7 +449,6 @@
     }
    ],
    "source": [
-    "# NBVAL_IGNORE_OUTPUT\n",
     "from blocklib import generate_blocks\n",
     "\n",
     "filtered_blocks_alice, filtered_blocks_bob = generate_blocks([block_obj_alice, block_obj_bob], K=2)\n",
@@ -440,7 +462,7 @@
    "source": [
     "### Assess Blocking\n",
     "\n",
-    "We can assess the blocking result when we have ground truth. There are two main metrics to assess blocking result as we mentioned in the beginning of this tutorial. Here is a recap:\n",
+    "We can assess the blocking result when we know the ground truth. There are two main metrics to assess blocking result as we mentioned in the beginning of this tutorial. Here is a recap:\n",
     "\n",
     "* reduction ratio: relative reduction in the number of record pairs to be compared.\n",
     "* pair completeness: the percentage of true matches after blocking\n"
@@ -455,7 +477,22 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "assessing blocks: 100%|██████████| 2793/2793 [00:00<00:00, 97204.45key/s]\n"
+      "assessing blocks: 100%|██████████| 2793/2793 [00:00<00:00, 36770.31key/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "reduction ratio:   0.996\n",
+      "pair completeness: 1.0\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
      ]
     }
    ],
@@ -468,7 +505,9 @@
     "subdata2 = [x[0] for x in data_bob]\n",
     "\n",
     "rr, pc = assess_blocks_2party([filtered_blocks_alice, filtered_blocks_bob],\n",
-    "                              [subdata1, subdata2])"
+    "                              [subdata1, subdata2])\n",
+    "print(f'reduction ratio:   {round(rr, 3)}')\n",
+    "print(f'pair completeness: {pc}')"
    ]
   },
   {
@@ -482,7 +521,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -524,21 +563,38 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "P-Sig: 100.0% records are covered in blocks\n",
       "Statistics for the generated blocks:\n",
       "\tNumber of Blocks:   5029\n",
       "\tMinimum Block Size: 1\n",
       "\tMaximum Block Size: 61\n",
       "\tAverage Block Size: 1.8337641678266057\n",
       "\tMedian Block Size:  1\n",
-      "\tStandard Deviation of Block Size:  3.8368431973204213\n"
+      "\tStandard Deviation of Block Size:  3.8368431973204213\n",
+      "\tCoverage:           100.0%\n",
+      "Individual statistics for each strategy:\n",
+      "Strategy: 0\n",
+      "\tNumber of Blocks:   503\n",
+      "\tMinimum Block Size: 1\n",
+      "\tMaximum Block Size: 61\n",
+      "\tAverage Block Size: 9.16699801192843\n",
+      "\tMedian Block Size:  6\n",
+      "\tStandard Deviation of Block Size:  9.342740344535663\n",
+      "\tCoverage:           100.0%\n",
+      "Strategy: 1\n",
+      "\tNumber of Blocks:   4534\n",
+      "\tMinimum Block Size: 1\n",
+      "\tMaximum Block Size: 7\n",
+      "\tAverage Block Size: 1.0169827966475518\n",
+      "\tMedian Block Size:  1\n",
+      "\tStandard Deviation of Block Size:  0.1583699259848952\n",
+      "\tCoverage:           100.0%\n"
      ]
     }
    ],
@@ -546,7 +602,8 @@
     "data_alice = df_alice.to_dict(orient='split')['data']\n",
     "header = list(df_alice.columns)\n",
     "\n",
-    "block_obj_alice = generate_candidate_blocks(data_alice, blocking_config, header=header)"
+    "block_obj_alice = generate_candidate_blocks(data_alice, blocking_config, header=header)\n",
+    "block_obj_alice.print_summary_statistics()"
    ]
   },
   {
@@ -562,7 +619,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -601,7 +658,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
@@ -629,17 +686,18 @@
     }
    ],
    "source": [
-    "# NBVAL_IGNORE_OUTPUT\n",
     "print('Generating candidate blocks for Alice:')\n",
     "block_obj_alice = generate_candidate_blocks(data_alice, blocking_config)\n",
+    "block_obj_alice.print_summary_statistics()\n",
     "print()\n",
     "print('Generating candidate blocks for Bob: ')\n",
-    "block_obj_bob = generate_candidate_blocks(data_bob, blocking_config)"
+    "block_obj_bob = generate_candidate_blocks(data_bob, blocking_config)\n",
+    "block_obj_bob.print_summary_statistics()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -652,7 +710,6 @@
     }
    ],
    "source": [
-    "# NBVAL_IGNORE_OUTPUT\n",
     "filtered_blocks_alice, filtered_blocks_bob = generate_blocks([block_obj_alice, block_obj_bob], K=2)\n",
     "print('Alice: {} out of {} blocks are in common'.format(len(filtered_blocks_alice), len(block_obj_alice.blocks)))\n",
     "print('Bob:   {} out of {} blocks are in common'.format(len(filtered_blocks_bob), len(block_obj_bob.blocks)))\n"
@@ -660,29 +717,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "assessing blocks: 100%|██████████| 4167/4167 [00:00<00:00, 7690.70key/s] "
+      "assessing blocks: 100%|██████████| 4167/4167 [00:00<00:00, 7677.36key/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "RR=0.8823915973988634\n",
-      "PC=1.0\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\n"
+      "reduction ratio:   0.882\n",
+      "pair completeness: 1.0\n"
      ]
     }
    ],
@@ -690,9 +740,16 @@
     "# NBVAL_IGNORE_OUTPUT\n",
     "rr, pc = assess_blocks_2party([filtered_blocks_alice, filtered_blocks_bob],\n",
     "                              [subdata1, subdata2])\n",
-    "print('RR={}'.format(rr))\n",
-    "print('PC={}'.format(pc))"
+    "print(f'reduction ratio:   {round(rr, 3)}')\n",
+    "print(f'pair completeness: {pc}')"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -711,16 +768,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.4"
-  },
-  "pycharm": {
-   "stem_cell": {
-    "cell_type": "raw",
-    "metadata": {
-     "collapsed": false
-    },
-    "source": []
-   }
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,


Test patch:
diff --git a/tests/test_candidate_block_generator.py b/tests/test_candidate_block_generator.py
index f3f4fb1..7572124 100644
--- a/tests/test_candidate_block_generator.py
+++ b/tests/test_candidate_block_generator.py
@@ -1,5 +1,5 @@
 import pytest
-
+import io
 from blocklib import generate_candidate_blocks
 from blocklib import PPRLIndexPSignature
 from blocklib import flip_bloom_filter
@@ -74,3 +74,11 @@ class TestCandidateBlockGenerator:
         bf_set_fred = str(tuple(flip_bloom_filter('0_Fred', bf_len, num_hash_funcs)))
         bf_set_lindsay = str(tuple(flip_bloom_filter('0_Lindsay', bf_len, num_hash_funcs)))
         assert candidate_block_obj.blocks == {bf_set_fred: ['id4', 'id5'], bf_set_lindsay: ['id6']}
+        stringbuf = io.StringIO()
+        candidate_block_obj.print_summary_statistics(output=stringbuf)
+        stats = stringbuf.getvalue()
+        assert stats.startswith('Statistics for the generated blocks:')
+        assert 'Number of Blocks:   2' in stats
+        assert 'Coverage:           50.0%' in stats
+        assert 'Individual statistics for each strategy:' in stats
+
diff --git a/tests/test_pprlindex.py b/tests/test_pprlindex.py
index cedb475..ea9ad2b 100644
--- a/tests/test_pprlindex.py
+++ b/tests/test_pprlindex.py
@@ -1,4 +1,5 @@
 from blocklib import PPRLIndex
+from blocklib.stats import reversed_index_stats
 import random
 
 
@@ -9,12 +10,10 @@ def test_init():
     assert pprl.rec_dict is None
     assert pprl.ent_id_col is None
     assert pprl.rec_id_col is None
-    assert pprl.stats == {}
 
 
 def test_summarize_reversed_index():
     """Test summarize_reversed_index for base class PPRLIndex."""
-    pprl = PPRLIndex()
 
     reversed_index = {
         'Jo': ['id1', 'id2', 'id3'],
@@ -22,18 +21,13 @@ def test_summarize_reversed_index():
         'Li': ['id5']
     }
 
-    stats = pprl.summarize_reversed_index(reversed_index)
+    stats = reversed_index_stats(reversed_index)
     assert stats['num_of_blocks'] == 3
     assert stats['min_size'] == 1
     assert stats['max_size'] == 3
     assert stats['avg_size'] == 2
     assert stats['med_size'] == 2
 
-    # Same comment as the block length.
-    num_of_blocks_per_rec = stats['num_of_blocks_per_rec']
-    num_of_blocks_per_rec.sort()
-    assert num_of_blocks_per_rec == [1, 1, 1, 1, 2]
-
 
 def test_select_reference_value():
     """Test selection of reference value."""
@@ -56,3 +50,8 @@ def test_select_reference_value():
     expected = random.sample(combined, 3)
     assert ref_val_list == expected
 
+
+def test_get_feature_to_index_map():
+    pprl = PPRLIndex()
+    pprl.blocking_features = ['boo']
+    assert pprl.get_feature_to_index_map([]) is None
diff --git a/tests/test_pprllambdafold.py b/tests/test_pprllambdafold.py
index 8532ac6..025acb3 100644
--- a/tests/test_pprllambdafold.py
+++ b/tests/test_pprllambdafold.py
@@ -49,18 +49,23 @@ class TestLambdaFold(unittest.TestCase):
         lambdafold = PPRLIndexLambdaFold(config_index)
         data = [[1, 'Xu', 'Li'],
                 [2, 'Fred', 'Yu']]
-        reversed_index = lambdafold.build_reversed_index(data)
-        assert len(reversed_index) == 5 * 2
-        assert all([len(k) == 31 for k in reversed_index])
-        assert all([len(v) == 1 for v in reversed_index.values()])
+        reversed_index_result = lambdafold.build_reversed_index(data)
+        assert len(reversed_index_result.reversed_index) == 5 * 2
+        assert all([len(k) == 31 for k in reversed_index_result.reversed_index])
+        assert all([len(v) == 1 for v in reversed_index_result.reversed_index.values()])
+        stats = reversed_index_result.stats
+        assert stats['num_of_blocks'] == 10
+        assert stats['min_size'] == 1
+        assert stats['max_size'] == 1
+        assert len(stats) >= 7
 
         # build with row index
         del config_index['record-id-col']
         lambdafold = PPRLIndexLambdaFold(config_index)
-        reversed_index = lambdafold.build_reversed_index(data)
-        assert len(reversed_index) == 5 * 2
-        assert all([len(k) == 31 for k in reversed_index])
-        assert all([len(v) == 1 for v in reversed_index.values()])
+        reversed_index_result = lambdafold.build_reversed_index(data)
+        assert len(reversed_index_result.reversed_index) == 5 * 2
+        assert all([len(k) == 31 for k in reversed_index_result.reversed_index])
+        assert all([len(v) == 1 for v in reversed_index_result.reversed_index.values()])
 
         # build given headers
         config_name = {
@@ -74,11 +79,11 @@ class TestLambdaFold(unittest.TestCase):
         }
         header = ['ID', 'firstname', 'lastname']
         lambdafold_use_colname = PPRLIndexLambdaFold(config_name)
-        reversed_index_use_colname = lambdafold_use_colname.build_reversed_index(data, header=header)
-        assert len(reversed_index_use_colname) == 5 * 2
-        assert all([len(k) == 31 for k in reversed_index_use_colname])
-        assert all([len(v) == 1 for v in reversed_index_use_colname.values()])
-        assert reversed_index == reversed_index_use_colname
+        reversed_index_result_use_colname = lambdafold_use_colname.build_reversed_index(data, header=header)
+        assert len(reversed_index_result_use_colname.reversed_index) == 5 * 2
+        assert all([len(k) == 31 for k in reversed_index_result_use_colname.reversed_index])
+        assert all([len(v) == 1 for v in reversed_index_result_use_colname.reversed_index.values()])
+        assert reversed_index_result == reversed_index_result_use_colname
 
     def test_build_reversed_index_clks(self):
         """Test building the inverted index with CLKs input."""
@@ -97,9 +102,9 @@ class TestLambdaFold(unittest.TestCase):
         with clk_filepath.open() as f:
             data = json.load(f)['clks']\
 
-        reversed_index = lambdafold.build_reversed_index(data)
-        assert len(reversed_index) == 5 * 4
-        assert all([len(k) == 31 for k in reversed_index])
+        reversed_index_result = lambdafold.build_reversed_index(data)
+        assert len(reversed_index_result.reversed_index) == 5 * 4
+        assert all([len(k) == 31 for k in reversed_index_result.reversed_index])
 
     def test_header_with_feature_type(self):
         """Test different combination of header and feature column type."""
@@ -133,12 +138,12 @@ class TestLambdaFold(unittest.TestCase):
         lambda_fold_index = PPRLIndexLambdaFold(config_index)
         lambda_fold_name = PPRLIndexLambdaFold(config_name)
         # case1 header is given and feature type is column names
-        reversed_index1 = lambda_fold_name.build_reversed_index(data, verbose=True, header=header)
+        reversed_index1 = lambda_fold_name.build_reversed_index(data, header=header)
         # case2 header is given and feature type is index
-        reversed_index2 = lambda_fold_index.build_reversed_index(data, verbose=True, header=header)
+        reversed_index2 = lambda_fold_index.build_reversed_index(data, header=header)
         # case3 header is not given and feature type is index
-        reversed_index3 = lambda_fold_index.build_reversed_index(data, verbose=True, header=None)
+        reversed_index3 = lambda_fold_index.build_reversed_index(data, header=None)
 
         # above 3 cases should give exactly same results
         assert reversed_index1 == reversed_index2
-        assert reversed_index2 == reversed_index3
\ No newline at end of file
+        assert reversed_index2 == reversed_index3
diff --git a/tests/test_pprlpsig.py b/tests/test_pprlpsig.py
index 13ecc6f..322afcb 100644
--- a/tests/test_pprlpsig.py
+++ b/tests/test_pprlpsig.py
@@ -55,9 +55,9 @@ class TestPSig(unittest.TestCase):
 
         }
         psig = PPRLIndexPSignature(config)
-        reversed_index = psig.build_reversed_index(data)
-        assert len(reversed_index) == 1
-        assert set(next(iter(reversed_index.values()))) == {'id1', 'id2'}
+        reversed_index_result = psig.build_reversed_index(data)
+        assert len(reversed_index_result.reversed_index) == 1
+        assert set(next(iter(reversed_index_result.reversed_index.values()))) == {'id1', 'id2'}
 
     def test_build_reversed_index(self):
         """Test build revert index."""
@@ -82,10 +82,16 @@ class TestPSig(unittest.TestCase):
             ]
         }
         psig = PPRLIndexPSignature(config)
-        reversed_index = psig.build_reversed_index(data, verbose=True)
+        reversed_index_result = psig.build_reversed_index(data)
         bf_set = tuple(flip_bloom_filter("0_Fred", config['blocking-filter']['bf-len'],
                                          config['blocking-filter']['number-hash-functions']))
-        assert reversed_index == {str(bf_set): ['id4', 'id5']}
+        assert reversed_index_result.reversed_index == {str(bf_set): ['id4', 'id5']}
+        stats = reversed_index_result.stats
+        assert len(stats) >= 9
+        assert stats['num_of_blocks'] == 1
+        assert stats['min_size'] == 2
+        assert 'statistics_per_strategy' in stats
+        assert 'coverage' in stats
 
     def test_build_reversed_index_feature_name(self):
         """Test build revert index."""
@@ -113,10 +119,10 @@ class TestPSig(unittest.TestCase):
         }
 
         psig = PPRLIndexPSignature(config)
-        reversed_index = psig.build_reversed_index(data, verbose=True, header=header)
+        reversed_index_result = psig.build_reversed_index(data, header=header)
         bf_set = tuple(flip_bloom_filter("0_Fred", config['blocking-filter']['bf-len'],
                                          config['blocking-filter']['number-hash-functions']))
-        assert reversed_index == {str(bf_set): ['id4', 'id5']}
+        assert reversed_index_result.reversed_index == {str(bf_set): ['id4', 'id5']}
 
         # test if results with column name and column index are the same
         config_index = {
@@ -139,8 +145,8 @@ class TestPSig(unittest.TestCase):
             ]
         }
         psig_col_index = PPRLIndexPSignature(config_index)
-        reversed_index_col_index = psig_col_index.build_reversed_index(data, verbose=True, header=header)
-        assert reversed_index == reversed_index_col_index
+        reversed_index_result_col_index = psig_col_index.build_reversed_index(data, header=header)
+        assert reversed_index_result.reversed_index == reversed_index_result_col_index.reversed_index
 
     def test_inconsistent_header(self):
         """Test when header dimension is not consistent with data dimension."""
@@ -167,7 +173,7 @@ class TestPSig(unittest.TestCase):
         }
         psig = PPRLIndexPSignature(config)
         with self.assertRaises(AssertionError):
-            psig.build_reversed_index(data, verbose=True, header=header)
+            psig.build_reversed_index(data, header=header)
 
     def test_header_with_feature_type(self):
         """Test different combination of header and feature column type."""
@@ -214,13 +220,13 @@ class TestPSig(unittest.TestCase):
         psig_index = PPRLIndexPSignature(config_index)
         psig_name = PPRLIndexPSignature(config_name)
         # case1 header is given and feature type is column names
-        reversed_index1 = psig_name.build_reversed_index(data, verbose=True, header=header)
+        reversed_index1_result = psig_name.build_reversed_index(data, header=header)
         # case2 header is given and feature type is index
-        reversed_index2 = psig_index.build_reversed_index(data, verbose=True, header=header)
+        reversed_index2_result = psig_index.build_reversed_index(data, header=header)
         # case3 header is not given and feature type is index
-        reversed_index3 = psig_index.build_reversed_index(data, verbose=True, header=None)
+        reversed_index3_result = psig_index.build_reversed_index(data, header=None)
 
         # above 3 cases should give exactly same results
-        assert reversed_index1 == reversed_index2
-        assert reversed_index2 == reversed_index3
+        assert reversed_index1_result == reversed_index2_result
+        assert reversed_index2_result == reversed_index3_result