Repository: data61/blocklib

Issue:
Convert block key into string
Currently P-sig generates block keys of type `set` while Lambda-fold generates block key of type `string`. For entity service deployment, we prefer string block keys. This issue is to convert block key into string for P-Sig

Comments:

Patch:
diff --git a/blocklib/blocks_generator.py b/blocklib/blocks_generator.py
index a62e044..21a91cd 100644
--- a/blocklib/blocks_generator.py
+++ b/blocklib/blocks_generator.py
@@ -1,7 +1,7 @@
 """Module that implement final block generations."""
 from collections import defaultdict
 from typing import Any, Dict, Sequence, Set, List, cast
-
+import ast
 import numpy as np
 
 from blocklib import PPRLIndex
@@ -89,6 +89,7 @@ def generate_blocks_psig(reversed_indices: Sequence[Dict], block_states: Sequenc
     for reversed_index, state in zip(reversed_indices, block_states):
         cbf = set()  # type: Set[int]
         for bf_set in reversed_index:
+            bf_set = ast.literal_eval(bf_set)
             cbf = cbf.union(bf_set)
 
         bf_len = int(block_states[0].blocking_config.get("bf-len", None))
@@ -102,10 +103,11 @@ def generate_blocks_psig(reversed_indices: Sequence[Dict], block_states: Sequenc
 
     # filter reversed_indices with block filter
     for reversed_index in reversed_indices:
-        has_matches = {bf_set: all(block_filter[i] for i in bf_set) for bf_set in reversed_index}
+        has_matches = {bf_set: all(block_filter[i] for i in ast.literal_eval(bf_set)) for bf_set in reversed_index}
         for bf_set in has_matches:
             if not has_matches[bf_set]:
                 del reversed_index[bf_set]
+
     # because of collisions in counting bloom filter, there are blocks only unique to one filtered index
     # only keep blocks that exist in at least threshold many reversed indices
     keys = defaultdict(int)  # type: Dict[Set, int]
@@ -116,4 +118,5 @@ def generate_blocks_psig(reversed_indices: Sequence[Dict], block_states: Sequenc
     clean_reversed_indices = []  # type: List[Dict[Set, List]]
     for reversed_index in reversed_indices:
         clean_reversed_indices.append(dict((k, reversed_index[k]) for k in common_keys if k in reversed_index))
+
     return clean_reversed_indices
diff --git a/blocklib/pprlpsig.py b/blocklib/pprlpsig.py
index 6ba617d..d0eabb3 100644
--- a/blocklib/pprlpsig.py
+++ b/blocklib/pprlpsig.py
@@ -94,10 +94,10 @@ class PPRLIndexPSignature(PPRLIndex):
         num_hash_func = int(self.blocking_config.get("number-hash-functions", None))
         bf_len = int(self.blocking_config.get("bf-len", None))
 
-        reversed_index = {}  # type: Dict[Any, List[Any]]
+        reversed_index = {}  # type: Dict[str, List[Any]]
 
         for signature, rec_ids in filtered_reversed_index.items():
-            bf_set = tuple(flip_bloom_filter(signature, bf_len, num_hash_func))
+            bf_set = str(tuple(flip_bloom_filter(signature, bf_len, num_hash_func)))
             if bf_set in reversed_index:
                 reversed_index[bf_set].extend(rec_ids)
             else:


Test patch:
diff --git a/tests/test_blocks_generators.py b/tests/test_blocks_generators.py
index 3021379..931556d 100644
--- a/tests/test_blocks_generators.py
+++ b/tests/test_blocks_generators.py
@@ -107,7 +107,7 @@ class TestBlocksGenerator:
         for string in ['1_Fr', '0_Fred', '1_Li']:
             bf_set = flip_bloom_filter(string, config['blocking-filter']['bf-len'],
                                        config['blocking-filter']['number-hash-functions'])
-            expected_bf_sets[tuple(bf_set)] = True
+            expected_bf_sets[str(tuple(bf_set))] = True
 
         assert all(key in expected_bf_sets for key in filtered_alice)
         assert filtered_alice.keys() == filtered_bob.keys()
@@ -177,7 +177,7 @@ class TestBlocksGenerator:
         for string in ['1_Fr', '1_Jo']:
             bf_set = flip_bloom_filter(string, config['blocking-filter']['bf-len'],
                                        config['blocking-filter']['number-hash-functions'])
-            expected_bf_sets[string] = tuple(bf_set)
+            expected_bf_sets[string] = str(tuple(bf_set))
 
         expected_m1 = {expected_bf_sets['1_Fr']: ['m1-2'], expected_bf_sets['1_Jo']: ['m1-1']}
         expected_m2 = {expected_bf_sets['1_Fr']: ['m2-1'], expected_bf_sets['1_Jo']: ['m2-2']}
diff --git a/tests/test_candidate_block_generator.py b/tests/test_candidate_block_generator.py
index 9b65e82..e7755cb 100644
--- a/tests/test_candidate_block_generator.py
+++ b/tests/test_candidate_block_generator.py
@@ -61,6 +61,6 @@ class TestCandidateBlockGenerator:
                         'version': 1,
                         'config': config}
         candidate_block_obj = generate_candidate_blocks(data, block_config)
-        bf_set_fred = tuple(flip_bloom_filter('0_Fred', bf_len, num_hash_funcs))
-        bf_set_lindsay = tuple(flip_bloom_filter('0_Lindsay', bf_len, num_hash_funcs))
+        bf_set_fred = str(tuple(flip_bloom_filter('0_Fred', bf_len, num_hash_funcs)))
+        bf_set_lindsay = str(tuple(flip_bloom_filter('0_Lindsay', bf_len, num_hash_funcs)))
         assert candidate_block_obj.blocks == {bf_set_fred: ['id4', 'id5'], bf_set_lindsay: ['id6']}
diff --git a/tests/test_pprlpsig.py b/tests/test_pprlpsig.py
index c285172..258450a 100644
--- a/tests/test_pprlpsig.py
+++ b/tests/test_pprlpsig.py
@@ -84,4 +84,4 @@ class TestPSig(unittest.TestCase):
         reversed_index = psig.build_reversed_index(data, verbose=True)
         bf_set = tuple(flip_bloom_filter("0_Fred", config['blocking-filter']['bf-len'],
                                          config['blocking-filter']['number-hash-functions']))
-        assert reversed_index == {bf_set: ['id4', 'id5']}
+        assert reversed_index == {str(bf_set): ['id4', 'id5']}