Repository: mindsdb/lightwood

Issue:
One-hot encoder mode to work without unknown categories
## Task

Categorical data is fully supported by lightwood. One of the available encoders for this data type is the [one-hot encoder](https://github.com/mindsdb/lightwood/blob/staging/lightwood/encoder/categorical/onehot.py).

The one-hot encoder (OHE) currently always outputs a vector with size `n_categories+1`, where the extra index is reserved for `unknown` or `missing` categories.

This task consists in adding a mode to the OHE (i.e. a boolean flag in the constructor) so that it doesn't create an index for `None/Unknown`, so that it is forced to always predict a previously seen category.

## Steps :male_detective: :female_detective: 

- Fork the Lightwood repository, checkout the `staging` branch and from it create a new one.
- Implement the change. Please, feel free to ask questions in the comments below.
- Make the PR and address any comments that reviewers might make.

## Additional rewards :1st_place_medal: 

Each encoder PR brings 5️⃣  points for entry into the draw for a :computer:  Deep Learning Laptop powered by the NVIDIA RTX 3080 Max-Q GPU or other swag :shirt: :bear: .  For more info check out https://mindsdb.com/hacktoberfest/


Discussion:
What should the behaviour for this new feature be?
When encountering an unknown value, do we return a vector full of zeros (like we currently do for `None`s), or crash like in scikit-learn's one hot encoder?

The current behavior is:
**encode**: Unknown symbols become `[1, 0, 0, ...]`, `None` becomes `[0, 0, ...]`
**decode**: Both unknowns symbols and `None` become `UNCOMMON_WORD`

To clarify, we assume the input all has valid categories mapped and we just want a constructor that returns n-length vector instead of (n+1)-length vector? 

Resolving patch:
diff --git a/lightwood/encoder/categorical/onehot.py b/lightwood/encoder/categorical/onehot.py
--- a/lightwood/encoder/categorical/onehot.py
+++ b/lightwood/encoder/categorical/onehot.py
@@ -2,6 +2,7 @@
 import numpy as np
 from scipy.special import softmax
 from lightwood.encoder.text.helpers.rnn_helpers import Lang
+from lightwood.helpers.log import log
 from lightwood.encoder.base import BaseEncoder
 
 UNCOMMON_WORD = '__mdb_unknown_cat'
@@ -10,10 +11,16 @@
 
 class OneHotEncoder(BaseEncoder):
 
-    def __init__(self, is_target=False, target_class_distribution=None):
+    def __init__(self, is_target=False, target_class_distribution=None, handle_unknown='unknown_token'):
         super().__init__(is_target)
         self._lang = None
         self.rev_map = {}
+
+        if handle_unknown not in {"unknown_token", "return_zeros"}:
+            raise ValueError(f"handle_unknown should be either 'unknown_token' or 'return_zeros', got {handle_unknown}")
+        else:
+            self.handle_unknown = handle_unknown
+
         if self.is_target:
             self.target_class_distribution = target_class_distribution
             self.index_weights = None
@@ -22,18 +29,28 @@ def prepare(self, priming_data, max_dimensions=20000):
         if self._prepared:
             raise Exception('You can only call "prepare" once for a given encoder.')
 
-        priming_data = [x if x is not None else UNCOMMON_WORD for x in priming_data]
         self._lang = Lang('default')
-        self._lang.index2word = {UNCOMMON_TOKEN: UNCOMMON_WORD}
-        self._lang.word2index = {UNCOMMON_WORD: UNCOMMON_TOKEN}
-        self._lang.word2count[UNCOMMON_WORD] = 0
-        self._lang.n_words = 1
+        if self.handle_unknown == "return_zeros":
+            priming_data = [x for x in priming_data if x is not None]
+            self._lang.index2word = {}
+            self._lang.word2index = {}
+            self._lang.n_words = 0
+        else:  # self.handle_unknown == "unknown_token"
+            priming_data = [x if x is not None else UNCOMMON_WORD for x in priming_data]
+            self._lang.index2word = {UNCOMMON_TOKEN: UNCOMMON_WORD}
+            self._lang.word2index = {UNCOMMON_WORD: UNCOMMON_TOKEN}
+            self._lang.word2count[UNCOMMON_WORD] = 0
+            self._lang.n_words = 1
+
         for category in priming_data:
             if category is not None:
                 self._lang.addWord(str(category))
 
         while self._lang.n_words > max_dimensions:
-            necessary_words = UNCOMMON_WORD
+            if self.handle_unknown == "return_zeros":
+                necessary_words = []
+            else:  # self.handle_unknown == "unknown_token"
+                necessary_words = [UNCOMMON_WORD]
             least_occuring_words = self._lang.getLeastOccurring(n=len(necessary_words) + 1)
 
             word_to_remove = None
@@ -64,6 +81,7 @@ def prepare(self, priming_data, max_dimensions=20000):
     def encode(self, column_data):
         if not self._prepared:
             raise Exception('You need to call "prepare" before calling "encode" or "decode".')
+
         ret = []
         v_len = self._lang.n_words
 
@@ -71,8 +89,16 @@ def encode(self, column_data):
             encoded_word = [0] * v_len
             if word is not None:
                 word = str(word)
-                index = self._lang.word2index[word] if word in self._lang.word2index else UNCOMMON_TOKEN
-                encoded_word[index] = 1
+                if self.handle_unknown == "return_zeros":
+                    if word in self._lang.word2index:
+                        index = self._lang.word2index[word]
+                        encoded_word[index] = 1
+                    else:
+                        # Encoding an unknown value will result in a vector of zeros
+                        log.warning('Trying to encode a value never seen before, returning vector of zeros')
+                else:  # self.handle_unknown == "unknown_token"
+                    index = self._lang.word2index[word] if word in self._lang.word2index else UNCOMMON_TOKEN
+                    encoded_word[index] = 1
 
             ret.append(encoded_word)
 
@@ -88,8 +114,13 @@ def decode(self, encoded_data, return_raw=False):
             # But this explicitly operates on logits; it will take care of
             # the one hot (so you can pass something in the softmax logit space)
             # But will not affect something that is already OHE.
-            ohe_index = np.argmax(vector)
-            ret.append(self._lang.index2word[ohe_index])
+
+            all_zeros = not np.any(vector)
+            if self.handle_unknown == "return_zeros" and all_zeros:
+                ret.append(UNCOMMON_WORD)
+            else:  # self.handle_unknown == "unknown_token"
+                ohe_index = np.argmax(vector)
+                ret.append(self._lang.index2word[ohe_index])
 
             if return_raw:
                 probs.append(softmax(vector).tolist())


Test patch:
diff --git a/tests/unit_tests/encoder/categorical/test_onehot.py b/tests/unit_tests/encoder/categorical/test_onehot.py
--- a/tests/unit_tests/encoder/categorical/test_onehot.py
+++ b/tests/unit_tests/encoder/categorical/test_onehot.py
@@ -1,4 +1,5 @@
 import unittest
+from torch import Tensor
 from lightwood.encoder.categorical.onehot import (
     OneHotEncoder,
     UNCOMMON_TOKEN,
@@ -7,13 +8,14 @@
 
 
 class TestOnehot(unittest.TestCase):
-    def test_encode_and_decode(self):
+    def test_encode_and_decode_with_unknown_token(self):
         data = ['category 1', 'category 3', 'category 4', None]
 
-        enc = OneHotEncoder()
+        enc = OneHotEncoder(handle_unknown='unknown_token')
 
         enc.prepare(data)
         encoded_data = enc.encode(data)
+
         decoded_data = enc.decode(enc.encode(['category 2', 'category 1', 'category 3', None]))
 
         self.assertTrue(len(encoded_data) == 4)
@@ -43,3 +45,45 @@ def test_encode_and_decode(self):
             self.assertTrue(decoded_data[4] == UNCOMMON_WORD)
 
             self.assertTrue(decoded_data[3] == 'category 4')
+
+    def test_encode_and_decode_with_return_zeros(self):
+        enc = OneHotEncoder(handle_unknown="return_zeros")
+
+        data = ['category 1', 'category 3', 'category 4', None]
+        enc.prepare(data)
+        encoded_data = enc.encode(data)
+        self.assertTrue((encoded_data == Tensor([
+            [1., 0., 0.],  # category 1
+            [0., 1., 0.],  # category 3
+            [0., 0., 1.],  # category 4
+            [0., 0., 0.],  # None
+        ])).all())
+
+        decoded_data = enc.decode(enc.encode(['category 2', 'category 1', 'category 3', None]))
+        self.assertEqual(decoded_data, [
+            UNCOMMON_WORD,  # category 2 is not seen, thus encoded to zero, thus decoded to uncommon word
+            'category 1',  # seen before
+            'category 3',  # seen before
+            UNCOMMON_WORD  # None is encoded to zero, thus decoded to uncommon word
+        ])
+
+        # Test max_dimensions
+        for max_dimensions in [2, 3]:
+            data = ['category 1', 'category 1', 'category 3', 'category 4', 'category 4', 'category 4', None]
+
+            enc = OneHotEncoder()
+
+            enc.prepare(data, max_dimensions=max_dimensions)
+            encoded_data = enc.encode(data)
+            decoded_data = enc.decode(enc.encode(['category 1', 'category 2', 'category 3', 'category 4', None]))
+
+            if max_dimensions == 3:
+                self.assertTrue(decoded_data[0] == 'category 1')
+            else:
+                self.assertTrue(decoded_data[0] == UNCOMMON_WORD)
+
+            self.assertTrue(decoded_data[1] == UNCOMMON_WORD)
+            self.assertTrue(decoded_data[2] == UNCOMMON_WORD)
+            self.assertTrue(decoded_data[4] == UNCOMMON_WORD)
+
+            self.assertTrue(decoded_data[3] == 'category 4')