Repository: axa-group/nlp.js (JavaScript)

Issue:
wrong implementation of BertWordpieceTokenizer
This implementation of BertWordpieceTokenizer selects subtokens in the wrong order. For example, if you use the vocabulary of a `bert-base-uncased` and try to tokenize the word "vegan," the resulting tokens from your implementation will be v ##egan and not vega ##n.

**To Reproduce**
Steps to reproduce the behavior:
1. Load the workpiece tokenizer with the `bert-base-uncased` vocabulary file. I used the one from hugging face.
2. Tokenize 'vegan', the expected result is vega ##n, but you get v ##egan
3. Try other words like 'cumin' or 'ajinomoto.'

**Expected behavior**
The specification of wordpiece says that you have to find the longest prefixes and not the longest affixes! So your recursive implementation is doing precisely the opposite of what it should be doing, picking from back to front the longest chunks, where it should be looking from front to back.

Please see my working version here: https://gist.github.com/juancavallotti/25d619f81dabb9c0476dcec87acc3a0a


Discussion:
Copuld you please provide a PR with your changes? can also be done very easiely using GitHub Web be browing to the file an klicking the pencil button and the doing the needed chnages there and then do "propose file change" and then "create Pull request"
Thanks for pointing the problem and the solution @juancavallotti . As @Apollon77 suggested, would be nice if you can add this change using a PR. If this is not possible by whatever reason, we'll apply this fix and you will be mentioned in the commit.



Patch:
diff --git a/packages/bert-tokenizer/src/bert-word-piece-tokenizer.js b/packages/bert-tokenizer/src/bert-word-piece-tokenizer.js
index 4b246f65b..5d2bcbd8d 100644
--- a/packages/bert-tokenizer/src/bert-word-piece-tokenizer.js
+++ b/packages/bert-tokenizer/src/bert-word-piece-tokenizer.js
@@ -118,10 +118,25 @@ class BertWordPieceTokenizer extends Clonable {
     return result;
   }
 
-  getBestAffix(word) {
+  getBestPrefix(word) {
     const maxLength = Math.min(word.length - 1, this.affixMaxLength);
+
+    // we try searching from shortest to longest.
     for (let i = maxLength; i > 0; i -= 1) {
-      const current = word.slice(-i);
+      const current = word.substring(0, i);
+      if (this.words[current]) {
+        return current;
+      }
+    }
+    return undefined;
+  }
+
+  getBestAffix(word) {
+    const maxLength = Math.min(word.length, this.affixMaxLength);
+
+    // we try searching from shortest to longest.
+    for (let i = maxLength; i > 0; i -= 1) {
+      const current = word.substring(0, i);
       if (this.affixes[current]) {
         return current;
       }
@@ -129,20 +144,30 @@ class BertWordPieceTokenizer extends Clonable {
     return undefined;
   }
 
-  tokenizeWord(srcWord, useExtra = false) {
+  tokenizeWord(srcWord, useExtra = false, isInside = false) {
     const word = this.lowercase ? srcWord.toLowerCase() : srcWord;
     const result = {
       tokens: [],
       ids: [],
     };
-    const wordIndex = this.words[word];
+
+    if (srcWord.length === 0) {
+      return result;
+    }
+
+    const wordIndex = isInside ? this.affixes[word] : this.words[word];
     if (wordIndex !== undefined) {
-      result.tokens.push(word);
+      result.tokens.push((isInside ? '##' : '') + word);
       result.ids.push(wordIndex);
       return result;
     }
-    const bestAffix = this.getBestAffix(word);
-    if (!bestAffix) {
+
+    // this might be in the prefixes part
+    const bestPart = isInside
+      ? this.getBestAffix(word)
+      : this.getBestPrefix(word);
+
+    if (!bestPart) {
       if (useExtra) {
         const index = this.numWords + this.numExtra;
         this.extra[word] = index;
@@ -155,15 +180,17 @@ class BertWordPieceTokenizer extends Clonable {
       }
       return result;
     }
-    const newWord = word.slice(0, -bestAffix.length);
-    const newWordTokens = this.tokenizeWord(newWord, useExtra);
+    const newWord = word.substring(bestPart.length);
+    const newWordTokens = this.tokenizeWord(newWord, useExtra, true);
+
+    const text = bestPart;
+    result.tokens.push((isInside ? '##' : '') + text);
+    result.ids.push(isInside ? this.affixes[text] : this.words[text]);
+
     for (let i = 0; i < newWordTokens.tokens.length; i += 1) {
       result.tokens.push(newWordTokens.tokens[i]);
       result.ids.push(newWordTokens.ids[i]);
     }
-    const text = `##${bestAffix}`;
-    result.tokens.push(text);
-    result.ids.push(this.words[text]);
     return result;
   }
 


Test patch:
diff --git a/packages/bert-tokenizer/test/bert-word-piece-tokenizer.test.js b/packages/bert-tokenizer/test/bert-word-piece-tokenizer.test.js
index 3c66e4392..02fb59fa4 100644
--- a/packages/bert-tokenizer/test/bert-word-piece-tokenizer.test.js
+++ b/packages/bert-tokenizer/test/bert-word-piece-tokenizer.test.js
@@ -56,10 +56,20 @@ describe('BertWordPieceTokenizer', () => {
     });
   });
 
+  describe('Get Best Prefix', () => {
+    test('Should calculate the best prefix for a word', () => {
+      const input = 'Supervised';
+      const expected = 'Super';
+      const tokenizer = new BertWordPieceTokenizer({ vocabContent: vocabEn });
+      const actual = tokenizer.getBestPrefix(input);
+      expect(actual).toEqual(expected);
+    });
+  });
+
   describe('Get Best Affix', () => {
     test('Should calculate the best affix for a word', () => {
       const input = 'Supervised';
-      const expected = 'ised';
+      const expected = 'S';
       const tokenizer = new BertWordPieceTokenizer({ vocabContent: vocabEn });
       const actual = tokenizer.getBestAffix(input);
       expect(actual).toEqual(expected);
@@ -67,11 +77,31 @@ describe('BertWordPieceTokenizer', () => {
   });
 
   describe('Tokenize Word', () => {
+    test('Should return several tokens if word does not match - with isInside', () => {
+      const input = 'Supervised';
+      const expected = {
+        tokens: ['##S', '##upe', '##r', '##vise', '##d'],
+        ids: [1708, 26939, 1197, 16641, 1181],
+      };
+      const tokenizer = new BertWordPieceTokenizer({ vocabContent: vocabEn });
+      const actual = tokenizer.tokenizeWord(input, false, true);
+      expect(actual).toEqual(expected);
+    });
     test('Should return several tokens if word does not match and has affixes', () => {
       const input = 'Supervised';
       const expected = {
-        tokens: ['Super', '##v', '##ised'],
-        ids: [3198, 1964, 3673],
+        tokens: ['Super', '##vise', '##d'],
+        ids: [3198, 16641, 1181],
+      };
+      const tokenizer = new BertWordPieceTokenizer({ vocabContent: vocabEn });
+      const actual = tokenizer.tokenizeWord(input);
+      expect(actual).toEqual(expected);
+    });
+    test('Should return several tokens if word does not match and has affixes #2', () => {
+      const input = 'vegan';
+      const expected = {
+        tokens: ['ve', '##gan'],
+        ids: [1396, 3820],
       };
       const tokenizer = new BertWordPieceTokenizer({ vocabContent: vocabEn });
       const actual = tokenizer.tokenizeWord(input);