Repository: castorini/pyserini

Issue:
Expose methods to convert between internal and external docid
This is a summary of the issue presented in #32 

Consider this fragment:

```
>>> from pyserini.index import pyutils
>>> 
>>> index_utils = pyutils.IndexReaderUtils('index-robust04-20191213/')
>>> postings_list = index_utils.get_postings_list('black')
>>> 
>>> for i in range(0, 10):
...     print('{}'.format(postings_list[i]))
... 
(6, 2) [555,606]
(29, 1) [410]
(32, 2) [65,462]
(35, 2) [288,475]
(56, 1) [662]
(60, 1) [69]
(61, 1) [110]
(63, 1) [195]
(74, 2) [230,518]
(96, 1) [107]
```

The docids (e.g., 6 in the first posting), refers to _internal_ Lucene docids, which are different from external docids (i.e., those in the collection).

Use this hidden method `convertLuceneDocidToDocid` to convert, as in:

```
>>> for i in range(0, 10):
...     print('{} {}'.format(index_utils.object.convertLuceneDocidToDocid(index_utils.reader, postings_list[i].docid), postings_list[i]))
... 
LA111289-0011 (6, 2) [555,606]
LA092890-0052 (29, 1) [410]
LA022489-0041 (32, 2) [65,462]
LA051990-0051 (35, 2) [288,475]
LA092890-0077 (56, 1) [662]
LA022489-0061 (60, 1) [69]
LA021889-0073 (61, 1) [110]
LA110689-0057 (63, 1) [195]
LA080789-0088 (74, 2) [230,518]
LA021889-0117 (96, 1) [107]
```

The TODO is to explicitly expose `convertLuceneDocidToDocid`.

Similarly, we can use `convertDocidToLuceneDocid` to convert an external collection docid into an internal docid:

```
>>> from jnius import autoclass
>>> JString = autoclass('java.lang.String')
>>> index_utils.object.convertDocidToLuceneDocid(index_utils.reader, JString("LA052189-0089"))
200443
```

We can verify as follows:

```
>>> for i in range(len(postings_list)):
...     if postings_list[i].docid == 200443:
...         print('{} {}'.format(index_utils.object.convertLuceneDocidToDocid(index_utils.reader, postings_list[i].docid), postings_list[i]))
... 
LA052189-0089 (200443, 64) [18,133,175,212,225,244,262,273,307,320,344,372,388,431,438,454,464,541,576,583,616,640,772,778,801,831,838,885,891,912,937,952,970,1123,1151,1165,1180,1210,1215,1231,1270,1307,1346,1431,1436,1507,1514,1542,1546,1550,1663,1676,1726,1750,1764,1769,1781,1784,1838,1847,1873,1880,1922,1971]
```
Which matches exactly what we get from `get_document_vector`:

```
>>> index_utils = pyutils.IndexReaderUtils('index-robust04-20191213/')
>>> doc_vector = index_utils.get_document_vector("LA052189-0089")
>>> doc_vector['black']
64
```


Discussion:

Resolving patch:
diff --git a/docs/source/conf.py b/docs/source/conf.py
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -14,7 +14,7 @@
 import sys
 sys.path.insert(0, os.path.abspath('.'))
 sys.path.insert(0, os.path.abspath('..'))
-
+sys.path.insert(0, os.path.abspath('../..'))
 
 # -- Project information -----------------------------------------------------
 
diff --git a/pyserini/index/pyutils.py b/pyserini/index/pyutils.py
--- a/pyserini/index/pyutils.py
+++ b/pyserini/index/pyutils.py
@@ -14,9 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-'''
+"""
 Module for providing python interface to Anserini index reader utils
-'''
+"""
+
 from ..pyclass import JIndexReaderUtils, JDocumentVectorWeight, JString
 
 import logging
@@ -25,14 +26,14 @@
 
 
 class IndexReaderUtils:
-    '''
+    """
     Wrapper class for Anserini's IndexReaderUtils.
 
     Parameters
     ----------
     index_dir : str
         Path to Lucene index directory
-    '''
+    """
 
     def __init__(self, index_dir):
         self.object = JIndexReaderUtils()
@@ -42,32 +43,17 @@ class DocumentVectorWeight:
         NONE = JDocumentVectorWeight.NONE
         TF_IDF = JDocumentVectorWeight.TF_IDF
 
-    class Posting:
-        '''
-        Basic Posting class for Postings List
-        '''
-        def __init__(self, docid, term_freq, positions):
-            self.docid = docid
-            self.term_freq = term_freq
-            self.positions = positions
-
-        def __repr__(self):
-            repr = '(' + str(self.docid) + ', ' + str(self.term_freq) + ')'
-            if self.positions:
-                repr += ' [' + ','.join([str(p) for p in self.positions]) + ']'
-            return repr
-
     class IndexTerm:
-        '''
+        """
         Basic IndexTerm class to represent each term in index
-        '''
+        """
         def __init__(self, term, doc_freq, total_term_freq):
             self.term = term
             self.doc_freq = doc_freq
             self.total_term_freq = total_term_freq
 
     def analyze(self, text):
-        '''
+        """
         Parameters
         ----------
         term : str
@@ -75,7 +61,7 @@ def analyze(self, text):
         -------
         result : str
             List of stemmed tokens
-        '''
+        """
         stemmed = self.object.analyze(JString(text))
         token_list = []
         for token in stemmed.toArray():
@@ -83,16 +69,16 @@ def analyze(self, text):
         return token_list
 
     def terms(self):
-        '''
+        """
         :return: generator over terms
-        '''
+        """
         term_iterator = self.object.getTerms(self.reader)
         while term_iterator.hasNext():
             cur_term = term_iterator.next()
             yield self.IndexTerm(cur_term.getTerm(), cur_term.getDF(), cur_term.getTotalTF())
 
     def get_term_counts(self, term):
-        '''
+        """
         Parameters
         ----------
         term : str
@@ -101,12 +87,12 @@ def get_term_counts(self, term):
         -------
         result : long, long
             Collection frequency and document frequency of term
-        '''
+        """
         term_map = self.object.getTermCounts(self.reader, JString(term))
         return term_map.get(JString('collectionFreq')), term_map.get(JString('docFreq'))
 
     def get_postings_list(self, term):
-        '''
+        """
         Parameters
         ----------
         term : str
@@ -114,15 +100,15 @@ def get_postings_list(self, term):
         -------
         result : list<Posting>
             Postings list for term
-        '''
+        """
         postings_list = self.object.getPostingsList(self.reader, JString(term))
         result = []
         for posting in postings_list.toArray():
-            result.append(self.Posting(posting.getDocid(), posting.getTF(), posting.getPositions()))
+            result.append(Posting(posting.getDocid(), posting.getTF(), posting.getPositions()))
         return result
 
     def get_document_vector(self, docid):
-        '''
+        """
         Parameters
         ----------
         docid : str
@@ -131,7 +117,7 @@ def get_document_vector(self, docid):
         -------
         result : dict
             Terms and their respective frequencies in document
-        '''
+        """
         doc_vector_map = self.object.getDocumentVector(self.reader, JString(docid))
         doc_vector_dict = {}
         for term in doc_vector_map.keySet().toArray():
@@ -139,7 +125,7 @@ def get_document_vector(self, docid):
         return doc_vector_dict
 
     def get_raw_document(self, docid):
-        '''
+        """
         Parameters
         ----------
         docid : str
@@ -147,11 +133,11 @@ def get_raw_document(self, docid):
         Returns
         -------
         result : raw document given its collection docid
-        '''
+        """
         return self.object.getRawDocument(self.reader, JString(docid))
 
     def get_bm25_term_weight(self, docid, term):
-        '''
+        """
         Parameters
         ----------
         docid : str
@@ -161,16 +147,70 @@ def get_bm25_term_weight(self, docid, term):
         -------
         result : float
             BM25 score (NaN if no documents match)
-        '''
+        """
         return self.object.getBM25TermWeight(self.reader, JString(docid), JString(term))
 
     def dump_document_vectors(self, reqDocidsPath, weight):
-        '''
+        """
         Parameters
         ----------
         reqDocidsPath : str
             dumps the document vector for all documents in reqDocidsPath
         weight : DocumentVectorWeight
             the weight for dumped document vector(s)
-        '''
-        self.object.dumpDocumentVectors(self.reader, reqDocidsPath, weight)
\ No newline at end of file
+        """
+        self.object.dumpDocumentVectors(self.reader, reqDocidsPath, weight)
+
+    def convert_internal_docid_to_collection_docid(self, docid: int) -> str:
+        """Converts Lucene's internal ``docid`` to its external collection ``docid``.
+
+        Parameters
+        ----------
+        docid : int
+            A Lucene internal ``docid``.
+
+        Returns
+        -------
+        str
+            The external collection ``docid`` corresponding to Lucene's internal ``docid``.
+        """
+        return self.object.convertLuceneDocidToDocid(self.reader, docid)
+
+    def convert_collection_docid_to_internal_docid(self, docid: str) -> int:
+        """Converts an external collection ``docid`` to its Lucene's internal ``docid``.
+
+        Parameters
+        ----------
+        docid : str
+            An external collection ``docid``.
+
+        Returns
+        -------
+        str
+            The Lucene internal ``docid`` corresponding to the external collection ``docid``.
+        """
+        return self.object.convertDocidToLuceneDocid(self.reader, docid)
+
+
+class Posting:
+    """Class representing a posting in a postings list.
+
+    Parameters
+    ----------
+    docid : int
+        The ``docid`` associated with this posting.
+    tf : int
+        The term frequency associated with this posting.
+    positions : List[int]
+        The list of positions associated with this posting.
+    """
+    def __init__(self, docid, tf, positions):
+        self.docid = docid
+        self.tf = tf
+        self.positions = positions
+
+    def __repr__(self):
+        repr = '(' + str(self.docid) + ', ' + str(self.tf) + ')'
+        if self.positions:
+            repr += ' [' + ','.join([str(p) for p in self.positions]) + ']'
+        return repr
\ No newline at end of file


Test patch:
diff --git a/tests/test_indexutils.py b/tests/test_indexutils.py
--- a/tests/test_indexutils.py
+++ b/tests/test_indexutils.py
@@ -42,10 +42,14 @@ def test_postings(self):
         self.assertEqual(len(postings_list), 138)
 
         self.assertEqual(postings_list[0].docid, 238)
-        self.assertEqual(postings_list[0].term_freq, 1)
+        self.assertEqual(self.index_utils.convert_internal_docid_to_collection_docid(postings_list[0].docid), 'CACM-0239')
+        self.assertEqual(postings_list[0].tf, 1)
+        self.assertEqual(len(postings_list[0].positions), 1)
 
         self.assertEqual(postings_list[-1].docid, 3168)
-        self.assertEqual(postings_list[-1].term_freq, 1)
+        self.assertEqual(self.index_utils.convert_internal_docid_to_collection_docid(postings_list[-1].docid), 'CACM-3169')
+        self.assertEqual(postings_list[-1].tf, 1)
+        self.assertEqual(len(postings_list[-1].positions), 1)
 
     def test_doc_vector(self):
         doc_vector = self.index_utils.get_document_vector('CACM-3134')
@@ -53,6 +57,20 @@ def test_doc_vector(self):
         self.assertEqual(doc_vector['inform'], 8)
         self.assertEqual(doc_vector['retriev'], 7)
 
+    def test_doc_vector_matches_index(self):
+        # From the document vector, look up the term frequency of "information".
+        doc_vector = self.index_utils.get_document_vector('CACM-3134')
+        self.assertEqual(doc_vector['inform'], 8)
+
+        # Now look up the postings list for "information".
+        term = 'information'
+        postings_list = list(self.index_utils.get_postings_list(term))
+
+        for i in range(len(postings_list)):
+            if self.index_utils.convert_internal_docid_to_collection_docid(postings_list[i].docid) == 'CACM-3134':
+                # The tf values should match.
+                self.assertEqual(postings_list[i].tf, 8)
+
     def test_raw_doc(self):
         lines = self.index_utils.get_raw_document('CACM-3134').splitlines()
         self.assertEqual(len(lines), 55)
@@ -63,6 +81,12 @@ def test_bm25_weight(self):
         self.assertAlmostEqual(self.index_utils.get_bm25_term_weight('CACM-3134', 'inform'), 1.925014, places=5)
         self.assertAlmostEqual(self.index_utils.get_bm25_term_weight('CACM-3134', 'retriev'), 2.496352, places=5)
 
+    def test_docid_converstion(self):
+        self.assertEqual(self.index_utils.convert_internal_docid_to_collection_docid(1), 'CACM-0002')
+        self.assertEqual(self.index_utils.convert_collection_docid_to_internal_docid('CACM-0002'), 1)
+        self.assertEqual(self.index_utils.convert_internal_docid_to_collection_docid(1000), 'CACM-1001')
+        self.assertEqual(self.index_utils.convert_collection_docid_to_internal_docid('CACM-1001'), 1000)
+
     def tearDown(self):
         os.remove(self.tarball_name)
         shutil.rmtree(self.index_dir)