Repository: castorini/pyserini

Issue:
get_document_vector() and get_postings_list() Stemming ?
Hi @lintool !
I have a new issue : 
I created a new index with the dataset "DUC-2001" by mean of this function : 

```
 sh anserini/target/appassembler/bin/IndexCollection \
            -collection TrecCollection \
            -generator JsoupGenerator \
            -threads 2 \
            -input ${EXP}/ \
            -index indexes/lucene-index.XXX \
            -storePositions -storeDocvectors -storeRawDocs
```

I also installed Luke Toolbox project to understand how the index working.

When i run this code : 

```
for id_ in docid:
    doc_vector = index_utils.get_document_vector(id_)
    bm25_score_one_doc = {}
    for term_ in doc_vector:
        postings_list = index_utils.get_postings_list(term_)

```
it works for some terms but not for all...

```
Traceback (most recent call last):
  File "doc2index_2.py", line 50, in <module>
    postings_list = index_utils.get_postings_list(term_)
  File "/home/poulain/.local/lib/python3.6/site-packages/pyserini/index/pyutils.py", line 118, in get_postings_list
    postings_list = self.object.getPostingsList(self.reader, JString(term))
  File "jnius/jnius_export_class.pxi", line 768, in jnius.JavaMethod.__call__
  File "jnius/jnius_export_class.pxi", line 934, in jnius.JavaMethod.call_staticmethod
  File "jnius/jnius_utils.pxi", line 91, in jnius.check_exception
jnius.JavaException: JVM exception occurred: java.lang.NullPointerException
```

I think there are two different indexes, the first one applies a stemming ( the word "Cherokee" become "cheroke") and the second keeps the word without stemming. 

So, how can i stemming the posting index ? 

Best regards 



Discussion:
hi @Oulaolay - welcome!

To be clear, you'd want a variant of `get_postings_list` that takes an already analyzed term, right?

There's actually already an outstanding issue:
https://github.com/castorini/anserini/issues/990

I'm not sure when we'll get to it... but you're welcome to send a pull request...

Resolving patch:
diff --git a/pyserini/index/pyutils.py b/pyserini/index/pyutils.py
--- a/pyserini/index/pyutils.py
+++ b/pyserini/index/pyutils.py
@@ -23,7 +23,7 @@
 import logging
 from typing import Dict, Iterator, List, Tuple
 
-from ..pyclass import JIndexReaderUtils, JString
+from ..pyclass import JIndexReaderUtils, JString, JAnalyzerUtils
 
 logger = logging.getLogger(__name__)
 
@@ -87,24 +87,28 @@ def __init__(self, index_dir):
         self.object = JIndexReaderUtils()
         self.reader = self.object.getReader(JString(index_dir))
 
-    def analyze(self, text: str) -> List[str]:
-        """Applies Anserini's default Lucene ``Analyzer`` to process a piece of text.
+    def analyze(self, text: str, analyzer=None) -> List[str]:
+        """Analyzes a piece of text. Applies Anserini's default Lucene analyzer if analyzer not specified.
 
         Parameters
         ----------
         text : str
             The piece of text to analyze.
-
+        analyzer : analyzer
+            The analyzer to apply.
         Returns
         -------
         List[str]
-            List of tokens corresponding to the output of the ``Analyzer``.
+            List of tokens corresponding to the output of the analyzer.
         """
-        stemmed = self.object.analyze(JString(text))
-        token_list = []
-        for token in stemmed.toArray():
-            token_list.append(token)
-        return token_list
+        if analyzer is None:
+            results = JAnalyzerUtils.analyze(JString(text.encode('utf-8')))
+        else:
+            results = JAnalyzerUtils.analyze(analyzer, JString(text.encode('utf-8')))
+        tokens = []
+        for token in results.toArray():
+            tokens.append(token)
+        return tokens
 
     def terms(self) -> Iterator[IndexTerm]:
         """Returns an iterator over (analyzed) terms in the index.
@@ -135,20 +139,28 @@ def get_term_counts(self, term: str) -> Tuple[int, int]:
         term_map = self.object.getTermCounts(self.reader, JString(term.encode('utf-8')))
         return term_map.get(JString('docFreq')), term_map.get(JString('collectionFreq'))
 
-    def get_postings_list(self, term: str) -> List[Posting]:
+    def get_postings_list(self, term: str, analyze=True) -> List[Posting]:
         """Returns the postings list for a term.
 
         Parameters
         ----------
         term : str
-            The raw (unanalyzed) term.
+            The raw term.
+        analyze : Bool
+            Whether or not analyze the term.
 
         Returns
         -------
         List[Posting]
             List of :class:`Posting` objects corresponding to the postings list for the term.
         """
-        postings_list = self.object.getPostingsList(self.reader, JString(term.encode('utf-8')))
+        if analyze:
+            postings_list = self.object.getPostingsListForUnanalyzedTerm(self.reader, JString(term.encode('utf-8')))
+        else:
+            postings_list = self.object.getPostingsListForAnalyzedTerm(self.reader, JString(term.encode('utf-8')))
+        if postings_list is None:
+            return None
+
         result = []
         for posting in postings_list.toArray():
             result.append(Posting(posting.getDocid(), posting.getTF(), posting.getPositions()))
diff --git a/pyserini/pyclass.py b/pyserini/pyclass.py
--- a/pyserini/pyclass.py
+++ b/pyserini/pyclass.py
@@ -66,6 +66,8 @@
 JIndexReaderUtils = autoclass('io.anserini.index.IndexReaderUtils')
 JDocumentVectorWeight = autoclass('io.anserini.index.IndexReaderUtils$DocumentVectorWeight')
 
+JAnalyzerUtils = autoclass('io.anserini.analysis.AnalyzerUtils')
+
 ### Generator
 
 class JIndexHelpers:


Test patch:
diff --git a/tests/test_analysis.py b/tests/test_analysis.py
--- a/tests/test_analysis.py
+++ b/tests/test_analysis.py
@@ -7,7 +7,7 @@
 from pyserini.analysis import pyanalysis
 from pyserini.search import pysearch
 from pyserini.index import pyutils
-from pyserini.pyclass import JString
+from pyserini.pyclass import JString, JAnalyzerUtils
 
 
 class TestAnalyzers(unittest.TestCase):
@@ -37,7 +37,7 @@ def test_different_analyzers_are_different(self):
     def test_analyze_with_analyzer(self):
         tokenizer = pyanalysis.get_analyzer('tokenize')
         query = JString('information retrieval')
-        only_tokenization = self.index_utils.object.analyzeWithAnalyzer(query, tokenizer)
+        only_tokenization = JAnalyzerUtils.analyze(tokenizer, query)
         token_list = []
         for token in only_tokenization.toArray():
             token_list.append(token)
diff --git a/tests/test_indexutils.py b/tests/test_indexutils.py
--- a/tests/test_indexutils.py
+++ b/tests/test_indexutils.py
@@ -21,6 +21,7 @@
 from random import randint
 from urllib.request import urlretrieve
 
+from pyserini.analysis import pyanalysis
 from pyserini.index import pyutils
 from pyserini.pyclass import JString
 
@@ -62,13 +63,20 @@ def test_analyze(self):
         self.assertEqual(' '.join(self.index_utils.analyze('retrieval')), 'retriev')
         self.assertEqual(' '.join(self.index_utils.analyze('rapid retrieval, space economy')),
                          'rapid retriev space economi')
+        tokenizer = pyanalysis.get_analyzer('tokenize')
+        self.assertEqual(' '.join(self.index_utils.analyze('retrieval', analyzer=tokenizer)), 'retrieval')
+        self.assertEqual(' '.join(self.index_utils.analyze('rapid retrieval, space economy', analyzer=tokenizer)),
+                         'rapid retrieval space economy')
+        # Test utf encoding:
+        self.assertEqual(self.index_utils.analyze('zoölogy')[0], 'zoölog')
+        self.assertEqual(self.index_utils.analyze('zoölogy', analyzer=tokenizer)[0], 'zoölogy')
 
     def test_term_stats(self):
         df, cf = self.index_utils.get_term_counts('retrieval')
         self.assertEqual(df, 138)
         self.assertEqual(cf, 275)
 
-    def test_postings(self):
+    def test_postings1(self):
         term = 'retrieval'
         postings = list(self.index_utils.get_postings_list(term))
         self.assertEqual(len(postings), 138)
@@ -83,6 +91,26 @@ def test_postings(self):
         self.assertEqual(postings[-1].tf, 1)
         self.assertEqual(len(postings[-1].positions), 1)
 
+    def test_postings2(self):
+        self.assertIsNone(self.index_utils.get_postings_list('asdf'))
+
+        postings = list(self.index_utils.get_postings_list('retrieval', analyze=True))
+        self.assertEqual(len(postings), 138)
+
+        # If we don't analyze, then we can't find the postings list:
+        self.assertIsNone(self.index_utils.get_postings_list('retrieval', analyze=False))
+
+        # Supply the analyzed form directly, and we're good:
+        postings = list(self.index_utils.get_postings_list('retriev', analyze=False))
+        self.assertEqual(len(postings), 138)
+        postings = list(self.index_utils.get_postings_list(self.index_utils.analyze('retrieval')[0], analyze=False))
+        self.assertEqual(len(postings), 138)
+
+        # Test utf encoding:
+        self.assertEqual(self.index_utils.get_postings_list('zoölogy'), None)
+        self.assertEqual(self.index_utils.get_postings_list('zoölogy', analyze=False), None)
+        self.assertEqual(self.index_utils.get_postings_list('zoölogy', analyze=True), None)
+
     def test_doc_vector(self):
         doc_vector = self.index_utils.get_document_vector('CACM-3134')
         self.assertEqual(len(doc_vector), 94)
@@ -123,11 +151,10 @@ def test_docid_converstion(self):
         self.assertEqual(self.index_utils.convert_collection_docid_to_internal_docid('CACM-1001'), 1000)
 
     def test_jstring_term(self):
-        self.assertEqual(self.index_utils.get_term_counts("zoölogy"), (0, 0))
+        self.assertEqual(self.index_utils.get_term_counts('zoölogy'), (0, 0))
         with self.assertRaises(ValueError):
             # Should fail when pyjnius has solved this internally.
             JString('zoölogy')
-        
 
     def tearDown(self):
         os.remove(self.tarball_name)