features.py

import numpy as np
import pandas as pd
import language_tool_python
import readability
import enchant
from enchant.checker import SpellChecker
from collections import OrderedDict
import nltk
nltk.download('punkt')
from nltk.tokenize import word_tokenize


class FeatureGenerator:
    def __init__(self):
        self.spell_checker = SpellChecker('en_US')
        self.enchant_dict = enchant.Dict("en_US")

        common_words_1k_filename = './1-1000.txt'
        with open(common_words_1k_filename) as f:
            self.common_words_1k = set(x.strip() for x in f.readlines())

        common_words_filename_10k = './google-10000-english-no-swears.txt'
        with open(common_words_filename_10k) as f:
            self.common_words_10k = set(x.strip() for x in f.readlines())

        # make sure common_words are a subset as well
        self.common_words_10k.update(self.common_words_1k)

        profanity_filename = './profanity.txt'
        with open(profanity_filename) as f:
            self.profanity_set = set(x.strip() for x in f.readlines())

        words_freq_filename = "./count_1w.txt"
        self.words_freq = pd.read_csv(words_freq_filename,
                             names=['word', 'freq'],
                             sep='\t',
                             header=None,
                             dtype={'word': str, 'freq': int},
                             keep_default_na=False,
                             na_values=[''])

        self.words_freq = self.words_freq.set_index('word')

        self.language_tool = language_tool_python.LanguageTool('en-US')


    def text_preprocess(self, text: str):
        '''
        Transform text to be processed by readability

        :param text: input text

        :return: str preprocessed text
        '''
        text = text.strip()

        # new paragraph starts with \n\n
        # readability also requires each sentence to end with \n
        paragraphs = [p.strip()\
                        .replace('. ', '.\n')\
                        .replace('? ', '?\n')\
                        .replace('! ', '!\n') for p in text.split('\n\n')]

        return "\n\n".join(paragraphs)


    def misspelled_count(self, text: str):
        '''
        Get count out misspelled words by enchant SpellChecker

        :param text: input text

        :return: count of misspelled words in text
        '''
        self.spell_checker.set_text(text)
        return len(list(self.spell_checker))

    def flatten_readability(self, r: OrderedDict):
        '''
        Flatten readability output by adding prefixes

        :param r: OrderedDict of readability output

        :return: dict
        '''
        out = {}
        for k, group in r.items():
            prefix = {'readability grades': '',
                      'sentence info': '',
                      'word usage': 'wu_',
                      'sentence beginnings': 'sb_'}[k]
            for var_name, value in group.items():
                out[prefix + var_name] = value
        return out

    def get_noncommon_words_count(self, text, common_words_dict):
    # first, tokenize the text
    # second, iterate over tokens and see whether
    # a. it is a word
    # b. not in common_words
    # c. correctly spelled
    # d. does not have underscore '_'

        def is_noncommon_word(w):
            return  len(w) > 2 and\
                    w not in common_words_dict and\
                    '_' not in w and\
                    self.enchant_dict.check(w)

        return sum(is_noncommon_word(x) for x in word_tokenize(text.lower()))


    def get_noncommon_words_count_1k(self, text):
        return self.get_noncommon_words_count(text, self.common_words_1k)

    def get_noncommon_words_count_10k(self, text):
        return self.get_noncommon_words_count(text, self.common_words_10k)



    def get_profanity_count(self, text):
        return sum(x in self.profanity_set for x in word_tokenize(text.lower()))

    def get_uncommon_words_counts(self, text):
        word_freq_thresholds = np.array([1e8, 1e7, 1e6, 1e5], dtype=int)
        counts = np.zeros(len(word_freq_thresholds), dtype=int)
        for w in word_tokenize(text):
            if len(w) <  3 or '_' in w or not self.enchant_dict.check(w) or w not in self.words_freq.index:
                continue
            w_freq = self.words_freq.loc[w].values[0]
            counts += (word_freq_thresholds > w_freq)
        return counts



    # I have commented out categories giving 0 columns in train dataset
    # SEMANTICS is mostly 0, we drop it as well
    LT_categories = ['CASING',
                     #'COLLOQUIALISMS',
                     'COMPOUNDING',
                     'CONFUSED_WORDS',
                     #'FALSE_FRIENDS',
                     #'GENDER_NEUTRALITY',
                     'GRAMMAR',
                     'MISC',
                     'PUNCTUATION',
                     'REDUNDANCY',
                     #'REGIONALISMS',
                     #'REPETITIONS',
                     #'REPETITIONS_STYLE',
                     #'SEMANTICS',
                     'STYLE',
                     'TYPOGRAPHY',
                     'TYPOS',
                     'TOTAL',
                    ]
    def get_LT_features(self, text):
        '''
        Generates LanguateTool features: each category count and a total number
        '''
        matches = self.language_tool.check(text)
        cat_counts = [sum(m.category == cat for m in matches) for cat in self.LT_categories[:-1]]
        return cat_counts + [len(matches)]


    def generate_features(self, df: pd.DataFrame):
        '''
        Generate features from a dataframe with `full_text` column containing english text

        :param df: input dataframe

        :return: pd.DataFrame with features and possibly updated `full_text` column
        '''

        res_df = df.copy()
        #res_df['full_text'] = res_df['full_text'].apply(self.text_preprocess)



        features_df = res_df[['full_text']].apply(lambda row:
                                                     self.flatten_readability(
                                                         readability.getmeasures(
                                                             self.text_preprocess(row[0]), lang='en')),
                                                 axis='columns',
                                                 result_type='expand')

        features_df['text_len'] = res_df['full_text'].apply(lambda x: len(x))

        features_df['misspelled'] = res_df['full_text'].apply(self.misspelled_count)
        features_df['noncommon_words_1k'] = res_df['full_text'].apply(self.get_noncommon_words_count_1k)
        features_df['noncommon_words_10k'] = res_df['full_text'].apply(self.get_noncommon_words_count_10k)
        features_df['profanity_count'] = res_df['full_text'].apply(self.get_profanity_count)
        features_df[['uwc1e8', 'uwc1e7', 'uwc1e6', 'uwc1e5']] =\
            res_df[['full_text']].apply(lambda x: self.get_uncommon_words_counts(x[0]),
                                        axis='columns',
                                        result_type='expand')

        # Generate ratio features
        words_ratio_features = ['wordtypes',
                                'long_words',
                                'complex_words',
                                'complex_words_dc',
                                'wu_tobeverb',
                                'wu_auxverb',
                                'wu_conjunction',
                                'wu_pronoun',
                                'wu_preposition',
                                'wu_nominalization',
                                'misspelled',
                                'noncommon_words_1k',
                                'noncommon_words_10k',
                                'uwc1e8',
                                'uwc1e7',
                                'uwc1e6',
                                'uwc1e5',
                               ]
        features_df[[x + "_ratio" for x in words_ratio_features]] = features_df[words_ratio_features]\
                                                                        .div(features_df['words'], axis=0)
        sentences_ratio_features = ['sb_pronoun',
                                    'sb_interrogative',
                                    'sb_article',
                                    'sb_subordination',
                                    'sb_conjunction',
                                    'sb_preposition',
                                   ]
        features_df[[x + "_ratio" for x in sentences_ratio_features]] = features_df[sentences_ratio_features]\
                                                                            .div(features_df['sentences'], axis=0)


        features_df[['LT_' + x for x in self.LT_categories]] = res_df[['full_text']]\
                                                            .apply(lambda x: self.get_LT_features(x[0]),
                                                                   axis=1,
                                                                   result_type='expand')

        features_df[['LT_' + x + '_ratio' for x in self.LT_categories]] = features_df[['LT_' + x for x in self.LT_categories]]\
                                                                        .div(features_df['words'], axis=0)

        features_df = features_df.sort_index(axis=1)
        return pd.concat([res_df, features_df], axis='columns')