Repository: barrust/pyspellchecker

Issue:
Deprecate `word_probability`
`word_probability` is really a misnomer and should be deprecated. Instead, something like `word_usage_frequency` or something similar should be used. It really is the ratio of the word provided against all words in the dictionary. I believe issue #68 is partly due to this being named what it is named. 

Comments:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index eefac8f..8208ff8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # pyspellchecker
 
+## Version 0.6.1 (Future)
+* Deprecated `spell.word_probability` since the name makes it seem that it is building a true probability; use `spell.word_usage_frequency` instead
+
 ## Version 0.6.0
 * Remove **python 2.7** support
 * Updated automated `scripts/build_dictionary.py` script to support adding missing words
diff --git a/spellchecker/spellchecker.py b/spellchecker/spellchecker.py
index 336e405..85df817 100644
--- a/spellchecker/spellchecker.py
+++ b/spellchecker/spellchecker.py
@@ -2,12 +2,11 @@
     Peter Norvig. See: https://norvig.com/spell-correct.html """
 import gzip
 import json
-import os
 import pkgutil
 import string
 from collections import Counter
 
-from .utils import _parse_into_words, ensure_unicode, load_file, write_file
+from .utils import _parse_into_words, ensure_unicode, load_file, write_file, deprecated
 
 
 class SpellChecker(object):
@@ -52,14 +51,14 @@ class SpellChecker(object):
         elif language:
             filename = "resources/{}.json.gz".format(language.lower())
             try:
-                json_open = pkgutil.get_data('spellchecker', filename)
+                json_open = pkgutil.get_data("spellchecker", filename)
             except FileNotFoundError:
                 msg = (
                     "The provided dictionary language ({}) does not " "exist!"
                 ).format(language.lower())
                 raise ValueError(msg)
 
-            lang_dict = json.loads(gzip.decompress(json_open).decode('utf-8'))
+            lang_dict = json.loads(gzip.decompress(json_open).decode("utf-8"))
             self._word_frequency.load_json(lang_dict)
 
     def __contains__(self, key):
@@ -122,9 +121,9 @@ class SpellChecker(object):
         data = json.dumps(self.word_frequency.dictionary, sort_keys=True)
         write_file(filepath, encoding, gzipped, data)
 
-    def word_probability(self, word, total_words=None):
-        """ Calculate the probability of the `word` being the desired, correct
-            word
+    def word_usage_frequency(self, word, total_words=None):
+        """ Calculate the frequency to the `word` provided as seen across the
+            entire dictionary
 
             Args:
                 word (str): The word for which the word probability is \
@@ -134,11 +133,32 @@ class SpellChecker(object):
                 frequency
             Returns:
                 float: The probability that the word is the correct word """
-        if total_words is None:
+        if not total_words:
             total_words = self._word_frequency.total_words
         word = ensure_unicode(word)
         return self._word_frequency.dictionary[word] / total_words
 
+    @deprecated("Deprecated as of version 0.6.1; use word_usage_frequency instead")
+    def word_probability(self, word, total_words=None):
+        """ Calculate the frequency to the `word` provided as seen across the
+            entire dictionary; function was a misnomar and is therefore
+            deprecated!
+
+            Args:
+                word (str): The word for which the word probability is \
+                calculated
+                total_words (int): The total number of words to use in the \
+                calculation; use the default for using the whole word \
+                frequency
+            Returns:
+                float: The probability that the word is the correct word
+            Note:
+                Deprecated as of version 0.6.1; use `word_usage_frequency` \
+                instead
+            Note:
+                Will be removed in version 0.6.3 """
+        return self.word_usage_frequency(word, total_words)
+
     def correction(self, word):
         """ The most probable correct spelling for the word
 
@@ -148,7 +168,7 @@ class SpellChecker(object):
                 str: The most likely candidate """
         word = ensure_unicode(word)
         candidates = list(self.candidates(word))
-        return max(sorted(candidates), key=self.word_probability)
+        return max(sorted(candidates), key=self.__getitem__)
 
     def candidates(self, word):
         """ Generate possible spelling corrections for the provided word up to
@@ -191,8 +211,7 @@ class SpellChecker(object):
         return set(
             w
             for w in tmp
-            if w in self._word_frequency.dictionary
-            and self._check_if_should_check(w)
+            if w in self._word_frequency.dictionary and self._check_if_should_check(w)
         )
 
     def unknown(self, words):
@@ -221,7 +240,11 @@ class SpellChecker(object):
             Returns:
                 set: The set of strings that are edit distance one from the \
                 provided word """
-        word = ensure_unicode(word).lower() if not self._case_sensitive else ensure_unicode(word)
+        word = (
+            ensure_unicode(word).lower()
+            if not self._case_sensitive
+            else ensure_unicode(word)
+        )
         if self._check_if_should_check(word) is False:
             return {word}
         letters = self._word_frequency.letters
@@ -241,7 +264,11 @@ class SpellChecker(object):
             Returns:
                 set: The set of strings that are edit distance two from the \
                 provided word """
-        word = ensure_unicode(word).lower() if not self._case_sensitive else ensure_unicode(word)
+        word = (
+            ensure_unicode(word).lower()
+            if not self._case_sensitive
+            else ensure_unicode(word)
+        )
         return [
             e2 for e1 in self.edit_distance_1(word) for e2 in self.edit_distance_1(e1)
         ]
@@ -266,7 +293,9 @@ class SpellChecker(object):
     def _check_if_should_check(self, word):
         if len(word) == 1 and word in string.punctuation:
             return False
-        if len(word) > self._word_frequency.longest_word_length + 3:  # magic number to allow removal of up to 2 letters.
+        if (
+            len(word) > self._word_frequency.longest_word_length + 3
+        ):  # magic number to allow removal of up to 2 letters.
             return False
         try:  # check if it is a number (int, float, etc)
             float(word)
@@ -288,7 +317,7 @@ class WordFrequency(object):
         "_letters",
         "_tokenizer",
         "_case_sensitive",
-        "_longest_word_length"
+        "_longest_word_length",
     ]
 
     def __init__(self, tokenizer=None, case_sensitive=False):
diff --git a/spellchecker/utils.py b/spellchecker/utils.py
index 93a3560..437a7aa 100644
--- a/spellchecker/utils.py
+++ b/spellchecker/utils.py
@@ -2,25 +2,80 @@
 import contextlib
 import gzip
 import re
+import functools
+import warnings
 
+from .info import __version__
 
-def ensure_unicode(s, encoding='utf-8'):
+
+def fail_after(version):
+    """ Decorator to add to tests to ensure that they fail if a deprecated
+        feature is not removed before the specified version
+
+        Args:
+            version (str): The version to check against """
+
+    def decorator_wrapper(func):
+        @functools.wraps(func)
+        def test_inner(*args, **kwargs):
+            if [int(x) for x in version.split(".")] <= [
+                int(x) for x in __version__.split(".")
+            ]:
+                msg = "The function {} must be fully removed as it is depricated and must be removed by version {}".format(
+                    func.__name__, version
+                )
+                raise AssertionError(msg)
+            return func(*args, **kwargs)
+
+        return test_inner
+
+    return decorator_wrapper
+
+
+def deprecated(message=""):
+    """ A simplistic decorator to mark functions as deprecated. The function
+        will pass a message to the user on the first use of the function
+
+        Args:
+            message (str): The message to display if the function is deprecated
+    """
+
+    def decorator_wrapper(func):
+        @functools.wraps(func)
+        def function_wrapper(*args, **kwargs):
+            func_name = func.__name__
+            if func_name not in function_wrapper.deprecated_items:
+                msg = "Function {} is now deprecated! {}".format(func.__name__, message)
+                warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+                function_wrapper.deprecated_items.add(func_name)
+
+            return func(*args, **kwargs)
+
+        # set this up the first time the decorator is called
+        function_wrapper.deprecated_items = set()
+
+        return function_wrapper
+
+    return decorator_wrapper
+
+
+def ensure_unicode(_str, encoding="utf-8"):
     """ Simplify checking if passed in data are bytes or a string and decode
         bytes into unicode.
 
         Args:
-            s (str): The input string (possibly bytes)
+            _str (str): The input string (possibly bytes)
             encoding (str): The encoding to use if input is bytes
         Returns:
             str: The encoded string
     """
-    if isinstance(s, bytes):
-        return s.decode(encoding)
-    return s
+    if isinstance(_str, bytes):
+        return _str.decode(encoding)
+    return _str
 
 
 @contextlib.contextmanager
-def __gzip_read(filename, mode='rb', encoding='UTF-8'):
+def __gzip_read(filename, mode="rb", encoding="UTF-8"):
     """ Context manager to correctly handle the decoding of the output of \
         the gzip file
 
@@ -47,7 +102,7 @@ def load_file(filename, encoding):
             str: The string data from the file read
     """
     if filename[-3:].lower() == ".gz":
-        with __gzip_read(filename, mode='rt', encoding=encoding) as data:
+        with __gzip_read(filename, mode="rt", encoding=encoding) as data:
             yield data
     else:
         with open(filename, mode="r", encoding=encoding) as fobj:
@@ -65,7 +120,7 @@ def write_file(filepath, encoding, gzipped, data):
             data (str): The data to be written out
     """
     if gzipped:
-        with gzip.open(filepath, 'wt') as fobj:
+        with gzip.open(filepath, "wt") as fobj:
             fobj.write(data)
     else:
         with open(filepath, "w", encoding=encoding) as fobj:


Test patch:
diff --git a/tests/spellchecker_test.py b/tests/spellchecker_test.py
index b403117..ba0254a 100644
--- a/tests/spellchecker_test.py
+++ b/tests/spellchecker_test.py
@@ -5,6 +5,7 @@ import unittest
 import os
 
 from spellchecker import SpellChecker
+from spellchecker.utils import fail_after
 
 class TestSpellChecker(unittest.TestCase):
     ''' test the spell checker class '''
@@ -50,7 +51,17 @@ class TestSpellChecker(unittest.TestCase):
         # if the default load changes so will this...
         self.assertEqual(spell.word_frequency['the'], 76138318)
 
-    def test_word_probability(self):
+    def test_word_usage_frequency(self):
+        ''' test the word usage frequency calculation '''
+        spell = SpellChecker()
+        # if the default load changes so will this...
+        num = spell.word_frequency['the']
+        denom = spell.word_frequency.total_words
+        self.assertEqual(spell.word_usage_frequency('the'), num / denom)
+
+    # deprecated!
+    @fail_after("0.6.3")
+    def test_word_probability_calc(self):
         ''' test the word probability calculation '''
         spell = SpellChecker()
         # if the default load changes so will this...