Clean data for emotional datasets

### Emotion dataset
emotion datset があるのは[ここ](https://github.com/abishekarun/Text-Emotion-Classification)と[ここ](https://dl.fbaipublicfiles.com/parlai/empatheticdialogues/empatheticdialogues.tar.gz)。

### Clean data
それぞれのデータセットについて行なう。
記号は「.」「,」「?」「!」だけ残しておいて他は除去する。


### Script

```tweets_clean.py
import csv
import string

import requests
import urllib
from bs4 import BeautifulSoup
import re
from tqdm import tqdm
import itertools
import numpy as np
import pandas as pd

def get_files():
    utterances  = []
    sentiments = []
    file = 'tweets_clean.txt'
    with open(file, 'r', encoding='utf-8') as f:
        data = f.readlines()
    content, labels = list(), list()
    for d in data:
        id, c, l = d.split('\t')
        content.append(c)
        labels.append(l.replace(':', ''))
        
    for text, label in  zip(content, labels):
        res_text = cleaning(text)
        if res_text != None:
            utterances.append(res_text)
            sentiments.append(label)
    return utterances, sentiments

def cleaning(text):
    txt = str(text)
    txt = re.sub(r"http\S+", "", txt)
    if len(txt) == 0:
        return None
    else:
        txt = txt.split()
        index = 0
        for j in tqdm(range(len(txt))):
            if txt[j][0] == '@':
                index = j
        txt = txt[index+1:]
        if len(txt) == 0:
            return None
        else:
            txt = ' '.join(txt)
            if len(txt) == 0:
                return None
            else:
                txt = txt.replace('"','').replace('&lt;', '<').replace('&gt;', '>').replace('&amp;', '&').replace('&quot;', "'")
                for s in string.punctuation:
                    if s not in '.?,!':
                        txt = txt.replace(s, '')
                return txt

def main():
    utters, sentiments = get_files()
    emotions = list(set(sentiments))
    with open('tweets_clean_utterances.txt', 'w', encoding='utf-8') as f:
        f.write('\n'.join(utters))
    with open('tweets_clean_sentiments.txt', 'w', encoding='utf-8') as f:
        f.write('\n'.join(sentiments))
    with open('tweets_clean_emotions.txt', 'w', encoding='utf-8') as f:
        f.write('\n'.join(emotions))
        
if __name__=='__main__':
    main()
```

```text_emotion.py
import csv
import string

import requests
import urllib
from bs4 import BeautifulSoup
import re
from tqdm import tqdm
import itertools
import numpy as np
import pandas as pd

def get_files():
    utterances  = []
    sentiments = []
    file = 'text_emotion.csv'
    df = pd.read_csv(file)
    content = df['content']
    labels = df['sentiment']
    for text, label in  zip(content, labels):
        res_text = cleaning(text)
        if res_text != None:
            utterances.append(res_text)
            sentiments.append(label)
    return utterances, sentiments

def cleaning(text):
    txt = str(text)
    txt = re.sub(r"http\S+", "", txt)
    if len(txt) == 0:
        return None
    else:
        txt = txt.split()
        index = 0
        for j in tqdm(range(len(txt))):
            if txt[j][0] == '@':
                index = j
        txt = txt[index+1:]
        if len(txt) == 0:
            return None
        else:
            txt = ' '.join(txt)
            if len(txt) == 0:
                return None
            else:
                txt = txt.replace('"','').replace('&lt;', '<').replace('&gt;', '>').replace('&amp;', '&').replace('&quot;', "'")
                for s in string.punctuation:
                    if s not in '.?,!':
                        txt = txt.replace(s, '')
                return txt

def main():
    utters, sentiments = get_files()
    emotions = list(set(sentiments))
    with open('text_emotion_utterances.txt', 'w', encoding='utf-8') as f:
        f.write('\n'.join(utters))
    with open('text_emotion_sentiments.txt', 'w', encoding='utf-8') as f:
        f.write('\n'.join(sentiments))
    with open('text_emotion_emotions.txt', 'w', encoding='utf-8') as f:
        f.write('\n'.join(emotions))

if __name__=='__main__':
    main()
```

```empatheticdialogues.py
import csv
import requests
import urllib
from bs4 import BeautifulSoup
import string
from tqdm import tqdm

def get_files():
    utterances  = []
    labels = []
    files = ['train.csv', 'test.csv', 'valid.csv']
    for file in files:
        with open(file, 'r', encoding='utf-8') as f:
            reader = list(csv.reader(f))
            for i, raw in tqdm(enumerate(reader[1:], start=1)):
                line = raw[5].replace('_comma_', ' ,')
                emotion = raw[2]
                line = line.replace('"','').replace('&lt;', '<').replace('&gt;', '>').replace('&amp;', '&').replace('&quot;', "'")
                for s in string.punctuation:
                    if s not in '.?,!':
                        line = line.replace(s, '')
                if line.strip() != "":
                    utterances.append(line)
                    labels.append(emotion)
                else:
                    pass
    return utterances, labels

def main():
    utters, emotions = get_files()
    with open('empatheticdialogues_utterances.txt', 'w', encoding='utf-8') as f:
        f.write('\n'.join(utters))
    with open('empatheticdialogues_sentiments.txt', 'w', encoding='utf-8') as f:
        f.write('\n'.join(emotions))
    with open('empatheticdialogues_emotions.txt', 'w', encoding='utf-8') as f:
        f.write('\n'.join(list(set(emotions))))

if __name__=='__main__':
    main()
 ```