Repository: bst-mug/acres

Issue:
Baseline: most frequent n-gram that passes filtering
Should also support multi-word acronyms.

Comments:

Patch:
diff --git a/acres/fastngram/fastngram.py b/acres/fastngram/fastngram.py
new file mode 100644
index 0000000..276b87d
--- /dev/null
+++ b/acres/fastngram/fastngram.py
@@ -0,0 +1,109 @@
+"""
+A faster version of n-gram matching that uses dictionaries for speed-up.
+"""
+
+from collections import OrderedDict
+from typing import List, Dict, Set, Tuple, Iterator
+
+from acres.preprocess import resource_factory
+
+
+class ContextMap:
+    """
+    A map of contexts to center words.
+    """
+
+    def __init__(self) -> None:
+        self.map = {}  # type: Dict[Tuple[str, str], Set[str]]
+
+    def add_context(self, center: str, left_context: str, right_context: str) -> None:
+        """
+        Add a center n-gram with a context.
+
+        :param center:
+        :param left_context:
+        :param right_context:
+        :return:
+        """
+        context = (left_context, right_context)
+        self.map.setdefault(context, set())
+        self.map[context].add(center)
+
+    def centers(self, left_context: str, right_context: str) -> Set[str]:
+        """
+        Find center n-grams that happen on a given context.
+
+        :param left_context:
+        :param right_context:
+        :return:
+        """
+        context = (left_context, right_context)
+        if context not in self.map:
+            return set()
+        return self.map[context]
+
+
+def expandn(acronym: str, left_context: str = "", right_context: str = "") -> Iterator[str]:
+    """
+    Find an unlimited set of expansion candidates for an acronym given its left and right context. \
+    Note that no filtering is done here.
+
+    :param acronym: Not used.
+    :param left_context:
+    :param right_context:
+    :return:
+    """
+    model = resource_factory.get_fastngram()
+
+    # TODO support for n-grams (n > 1). May need an OrderedDict.
+    count_map = model[1]
+    for freq, context_map in count_map.items():
+        # TODO require a min_freq?
+        center_ngrams = context_map.centers(left_context, right_context)
+        for ngram in center_ngrams:
+            yield ngram
+
+
+def expand(acronym: str, left_context: str = "", right_context: str = "") -> List[str]:
+    """
+    Find a limited set of expansion candidates for an acronym given its left and right context.
+
+    :param acronym:
+    :param left_context:
+    :param right_context:
+    :return:
+    """
+    # Limit expansions while we don't use generators downstream
+    # TODO 1k may not be enough if we're not doing ANY filtering here (e.g. initial).
+    # https://github.com/bst-mug/acres/issues/28
+    limit = 1000
+    i = 0
+    ret = []  # type: List[str]
+    for ngram in expandn(acronym, left_context, right_context):
+        ret.append(ngram)
+        i += 1
+        if i > limit:
+            break
+    return ret
+
+
+def optimizer(ngrams: Dict[str, int]) -> 'Dict[int, OrderedDict[int, ContextMap]]':
+    """
+    Create a search-optimized represenation of an ngram-list.
+
+    :param ngrams:
+    :return:
+    """
+    model = {}  # type: Dict[int, OrderedDict[int, ContextMap]]
+
+    # Ensure ngrams are ordered by decreasing frequency.
+    sorted_ngrams = sorted(ngrams.items(), key=lambda x: x[1], reverse=True)
+
+    for ngram, freq in sorted_ngrams:
+        # size = len(ngram.split(" "))
+        context = ContextMap()
+        context.add_context(ngram, "", "")
+        model.setdefault(1, OrderedDict())
+        model[1][freq] = context
+
+    return model
diff --git a/acres/preprocess/resource_factory.py b/acres/preprocess/resource_factory.py
index 9ca2824..6f1d3fe 100644
--- a/acres/preprocess/resource_factory.py
+++ b/acres/preprocess/resource_factory.py
@@ -6,10 +6,12 @@ This module provides methods for lazily loading resources.
 import logging
 import os.path
 import pickle
+from collections import OrderedDict
 from typing import Dict, Set, List, Tuple, Any
 
 from gensim.models import Word2Vec
 
+from acres.fastngram import fastngram
 from acres.nn import train
 from acres.preprocess import dumps
 from acres.stats import dictionary
@@ -37,6 +39,7 @@ NGRAMSTAT = {}  # type: Dict[int, Tuple[int,str]]
 CHARACTER_NGRAMS = {}  # type: Dict[str, int]
 WORD_NGRAMS = {}  # type: Dict[str, int]
 DICTIONARY = {}  # type: Dict[str, List[str]]
+FAST_NGRAM = {}  # type: Dict[int, OrderedDict[int, Dict[str, Set[str]]]]
 
 
 def get_log_corpus_filename() -> str:
@@ -303,6 +306,22 @@ def get_dictionary() -> Dict[str, List[str]]:
     return DICTIONARY
 
 
+def get_fastngram() -> 'Dict[int, OrderedDict[int, fastngram.ContextMap]]':
+    """
+    Lazy load the fast n-gram model.
+
+    :return:
+    """
+    global FAST_NGRAM
+
+    if not FAST_NGRAM:
+        word_ngrams = get_word_ngrams()
+        logger.info("Optimizing ngrams...")
+        FAST_NGRAM = fastngram.optimizer(word_ngrams)
+
+    return FAST_NGRAM
+
+
 def reset() -> None:
     """
     Resets global variables to force model recreation.
diff --git a/acres/resolution/resolver.py b/acres/resolution/resolver.py
index e9c7470..162cafc 100644
--- a/acres/resolution/resolver.py
+++ b/acres/resolution/resolver.py
@@ -1,6 +1,7 @@
 from enum import Enum
 from typing import List, Dict, Tuple
 
+from acres.fastngram import fastngram
 from acres.ngram import finder
 from acres.nn import test
 from acres.rater import rater
@@ -15,11 +16,13 @@ class Strategy(Enum):
     NGRAM = 1
     WORD2VEC = 2
     DICTIONARY = 3
+    FASTNGRAM = 4
 
 
 NGRAM_CACHE = {}  # type: Dict[Tuple, List[str]]
 WORD2VEC_CACHE = {}  # type: Dict[Tuple, List[str]]
 DICTIONARY_CACHE = {}  # type: Dict[Tuple, List[str]]
+FASTNGRAM_CACHE = {}  # type: Dict[Tuple, List[str]]
 
 
 def cached_resolve(acronym: str, left_context: str, right_context: str,
@@ -39,7 +42,8 @@ def cached_resolve(acronym: str, left_context: str, right_context: str,
     switcher = {
         Strategy.NGRAM: NGRAM_CACHE,
         Strategy.WORD2VEC: WORD2VEC_CACHE,
-        Strategy.DICTIONARY: DICTIONARY_CACHE
+        Strategy.DICTIONARY: DICTIONARY_CACHE,
+        Strategy.FASTNGRAM: FASTNGRAM_CACHE
     }
 
     cache = switcher.get(strategy)
@@ -90,7 +94,8 @@ def resolve(acronym: str, left_context: str, right_context: str, strategy: Strat
     switcher = {
         Strategy.NGRAM: finder.robust_find_embeddings,
         Strategy.WORD2VEC: test.find_candidates,
-        Strategy.DICTIONARY: dictionary.expand
+        Strategy.DICTIONARY: dictionary.expand,
+        Strategy.FASTNGRAM: fastngram.expand,
     }
 
     func = switcher.get(strategy)


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 9d700a5..7cce7bd 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,4 +1,5 @@
-import pytest, os
+import os
+import pytest
 
 from acres.preprocess import resource_factory
 
@@ -49,6 +50,23 @@ def ngramstat():
     resource_factory.NGRAMSTAT = old
 
 
+@pytest.fixture(scope="module")
+def word_ngrams():
+    # Setup: save current one and assign a fake one
+    old = resource_factory.WORD_NGRAMS
+    resource_factory.WORD_NGRAMS = {"EKG": 500,
+                                    "Elektrokardiogramm": 200,
+                                    "performed EKG yesterday": 50,
+                                    "performed Elektrokardiogramm yesterday": 20,
+                                    "performed Elektro kardiogramm yesterday": 10,  # sic
+                                    "performed Effusion yesterday": 5
+                                    }
+    yield resource_factory.WORD_NGRAMS
+
+    # Teardown: revert back to old
+    resource_factory.WORD_NGRAMS = old
+
+
 @pytest.fixture(scope="module")
 def index():
     # Setup: save current one and assign a fake one
diff --git a/tests/fastngram/test_fastngram.py b/tests/fastngram/test_fastngram.py
new file mode 100644
index 0000000..36d7033
--- /dev/null
+++ b/tests/fastngram/test_fastngram.py
@@ -0,0 +1,6 @@
+from acres.fastngram.fastngram import expand
+
+
+def test_expand(word_ngrams):
+    expansions = expand("EKG")
+    assert "Elektrokardiogramm" in expansions