Repository: nltk/nltk

Issue:
emojis are not tokenized very well
Maybe add something like this as a pre or post processing step?

It might make sense to download the emoji list and store it as part of the build, so people do not need to load the emojis module, ...

```
import emoji
from emoji import unicode_codes
import re


EMOJI_UNICODE = unicode_codes.EMOJI_UNICODE['en']
emojis = sorted(EMOJI_UNICODE.values(), key=len, reverse=True)
print (emojis, sep='\n')
emoji_regexp = f"({'|'.join(re.escape(u) for u in emojis)})"
EMOJI_XL = re.compile(rf"\B({emoji_regexp})", flags=re.UNICODE)
EMOJI_XR = re.compile(rf"({emoji_regexp})\B", flags=re.UNICODE)
EMOJI_WL = re.compile(rf"(\w)({emoji_regexp})", flags=re.UNICODE)
EMOJI_WR = re.compile(rf"({emoji_regexp})(\w)", flags=re.UNICODE)
EMOJI_REGEX = re.compile(rf"({emoji_regexp})", flags=re.UNICODE)

def split_emoji(text):
    text = EMOJI_REGEX.sub(r' \1 ', text)
    text = text.replace('  ', ' ')
    return text

test =  "🤔 🙈 me así, se😌 ds 💕👭👙 hello 👩🏾‍🎓 emoji hello 👨‍👩‍👦‍👦 how are 😊 you today🙅🏽🙅🏽"
#test = "They are going to start a direct flight soon😠"

print(test)

print(split_emoji(test))
```


Discussion:
I'm in favor of this, as long as it doesn't make the tokenization considerably slower. I don't think it would have to. 
Perhaps a good place for it would be to include it in the TweetTokenizer? The regexps for those are defined [here](https://github.com/nltk/nltk/blob/a31de6c687165afe64a490053c73c8e0b20ff6db/nltk/tokenize/casual.py#L146).

I would also prefer not having additional dependencies for this, but perhaps the dependency allows us to automatically update to newer emoji lists. After all, they're up to v14.0 now, and will continue updating.

Resolving patch:
diff --git a/nltk/tokenize/casual.py b/nltk/tokenize/casual.py
--- a/nltk/tokenize/casual.py
+++ b/nltk/tokenize/casual.py
@@ -157,6 +157,12 @@
     r"""(?:\#+[\w_]+[\w\'_\-]*[\w_]+)""",
     # email addresses
     r"""[\w.+-]+@[\w-]+\.(?:[\w-]\.?)+[\w-]""",
+    # Zero-Width-Joiner and Skin tone modifier emojis
+    """.(?:
+        [\U0001F3FB-\U0001F3FF]?(?:\u200d.[\U0001F3FB-\U0001F3FF]?)+
+        |
+        [\U0001F3FB-\U0001F3FF]
+    )""",
     # Remaining word types:
     r"""
     (?:[^\W\d_](?:[^\W\d_]|['\-_])+[^\W\d_]) # Words with apostrophes or dashes.
@@ -194,6 +200,7 @@
     r"(([A-Za-z0-9_]){15}(?!@)|([A-Za-z0-9_]){1,14}(?![A-Za-z0-9_]*@))"
 )
 
+
 ######################################################################
 # Functions for converting html entities
 ######################################################################


Test patch:
diff --git a/nltk/test/unit/test_tokenize.py b/nltk/test/unit/test_tokenize.py
--- a/nltk/test/unit/test_tokenize.py
+++ b/nltk/test/unit/test_tokenize.py
@@ -334,6 +334,54 @@ def test_phone_tokenizer(self):
         result = tokenizer.tokenize(test2)
         assert result == expected
 
+    def test_emoji_tokenizer(self):
+        """
+        Test a string that contains Emoji ZWJ Sequences and skin tone modifier
+        """
+        tokenizer = TweetTokenizer()
+
+        # A Emoji ZWJ Sequences, they together build as a single emoji, should not be split.
+        test1 = "👨‍👩‍👧‍👧"
+        expected = ["👨‍👩‍👧‍👧"]
+        result = tokenizer.tokenize(test1)
+        assert result == expected
+
+        # A Emoji with skin tone modifier, the two characters build a single emoji, should not be split.
+        test2 = "👨🏿"
+        expected = ["👨🏿"]
+        result = tokenizer.tokenize(test2)
+        assert result == expected
+
+        # A string containing both skin tone modifier and ZWJ Sequences
+        test3 = "🤔 🙈 me así, se😌 ds 💕👭👙 hello 👩🏾‍🎓 emoji hello 👨‍👩‍👦‍👦 how are 😊 you today🙅🏽🙅🏽"
+        expected = [
+            "🤔",
+            "🙈",
+            "me",
+            "así",
+            ",",
+            "se",
+            "😌",
+            "ds",
+            "💕",
+            "👭",
+            "👙",
+            "hello",
+            "👩🏾\u200d🎓",
+            "emoji",
+            "hello",
+            "👨\u200d👩\u200d👦\u200d👦",
+            "how",
+            "are",
+            "😊",
+            "you",
+            "today",
+            "🙅🏽",
+            "🙅🏽",
+        ]
+        result = tokenizer.tokenize(test3)
+        assert result == expected
+
     def test_pad_asterisk(self):
         """
         Test padding of asterisk for word tokenization.