Kaggle competition: Hackathon Sentimento
<SOS> metric: categorizationaccuracy Kaggle data source: hackathon-sentimento<define_variables>
warnings.filterwarnings('ignore')
 sns.set()
%config InlineBackend.figure_format = 'svg'
print(os.listdir('.. /input/'))
i = 0
for k in list(vocab):
print(k, vocab[k])
 i += 1
if i > 5:
break<categorify>
df = pd.read_csv('.. /input/train.csv')
 df.head()
def load_embed(file):
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(file, encoding='latin'))
return embeddings_index<load_pretrained>
test_df = pd.read_csv('.. /input/test.csv' )
glove = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
print("Extracting GloVe embedding")
 embed_glove = load_embed(glove)
 print("Embedding extraction done" )<statistical_test>
df['char_count'] = df['review'].apply(len)
 df['word_count'] = df['review'].apply(lambda x: len(x.split()))
df['word_density'] = df['char_count'] /(df['word_count']+1)
 df['punctuation_count'] = df['review'].apply(lambda x: len("".join(_ for _ in x if _ in string.punctuation)) )
def load_glove_matrix(word_index, embeddings_index):
all_embs = np.stack(embeddings_index.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
EMBED_SIZE = all_embs.shape[1]
nb_words = min(MAX_FEATURES, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, EMBED_SIZE))
for word, i in word_index.items() :
if i >= MAX_FEATURES:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix<categorify>
X_train, X_val, y_train, y_val = train_test_split(df['review'], df['sentiment'], test_size=0.3,
random_state=17 )
def check_coverage(vocab, embeddings_index):
known_words = {}
unknown_words = {}
nb_known_words = 0
nb_unknown_words = 0
for word in vocab.keys() :
try:
known_words[word] = embeddings_index[word]
nb_known_words += vocab[word]
except:
unknown_words[word] = vocab[word]
nb_unknown_words += vocab[word]
pass
print('Found embeddings for {:.3%} of vocabulary'.format(len(known_words)/len(vocab)))
 print('Found embeddings for {:.3%} of all text'.format(nb_known_words/(nb_known_words + nb_unknown_words)))
 unknown_words = sorted(unknown_words.items() , key=operator.itemgetter(1)) [::-1]
return unknown_words<compute_test_metric>
tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\w{1,}', max_features=5000)
 tfidf_vect.fit(df['review'])
 xtrain_tfidf = tfidf_vect.transform(X_train)
 xvalid_tfidf = tfidf_vect.transform(X_val)
 tfidf_vect_ngram = TfidfVectorizer(analyzer='word', token_pattern=r'\w{1,}', ngram_range=(2,3), max_features=5000)
 tfidf_vect_ngram.fit(df['review'])
 xtrain_tfidf_ngram = tfidf_vect_ngram.transform(X_train)
 xvalid_tfidf_ngram = tfidf_vect_ngram.transform(X_val)
 tfidf_vect_ngram_chars = TfidfVectorizer(analyzer='char', token_pattern=r'\w{1,}', ngram_range=(2,3), max_features=5000)
 tfidf_vect_ngram_chars.fit(df['review'])
 xtrain_tfidf_ngram_chars = tfidf_vect_ngram_chars.transform(X_train)
 xvalid_tfidf_ngram_chars = tfidf_vect_ngram_chars.transform(X_val)
 xtest_tfidf = tfidf_vect.transform(test_df['review'])
 xtest_tfidf_ngram = tfidf_vect_ngram.transform(test_df['review'])
 xtest_tfidf_ngram_chars = tfidf_vect_ngram_chars.transform(test_df['review'] )
print("Glove: ")
 oov_glove = check_coverage(vocab, embed_glove )<categorify>
count_vect = CountVectorizer(analyzer='word', token_pattern=r'\w{1,}')
 count_vect.fit(df['review'])
 xtrain_count = count_vect.transform(X_train)
 xvalid_count = count_vect.transform(X_val)
 xtest_count = count_vect.transform(test_df['review'] )
def add_lower(embedding, vocab):
count = 0
for word in vocab:
if word in embedding and word.lower() not in embedding:
embedding[word.lower() ] = embedding[word]
count += 1
print(f"Added {count} words to embedding" )<data_type_conversions>
model1 = linear_model.LogisticRegression()
model1.fit(xtrain_count, y_train)
 accuracy=model1.score(xvalid_count, y_val)
 print('Accuracy Count LR:', accuracy)
 test_pred1=model1.predict(xtest_count)
 model2 = linear_model.LogisticRegression()
model2.fit(xtrain_tfidf, y_train)
 accuracy=model2.score(xvalid_tfidf, y_val)
 print('Accuracy TFIDF LR:', accuracy)
 test_pred2=model2.predict(xtest_tfidf)
 model3 = linear_model.LogisticRegression()
model3.fit(xtrain_tfidf_ngram, y_train)
 accuracy = model3.score(xvalid_tfidf_ngram, y_val)
 print('Accuracy TFIDF NGRAM LR:', accuracy)
 test_pred3 = model3.predict(xtest_tfidf_ngram )
train['question_text'] = train['question_text'].apply(lambda x: x.lower())
 test['question_text'] = test['question_text'].apply(lambda x: x.lower() )<compute_test_metric>
final_pred = np.array([])
 for i in range(0,len(test_df['review'])) :
final_pred = np.append(final_pred, np.argmax(np.bincount([test_pred1[i], test_pred2[i], test_pred3[i]])) )
print("Glove: ")
 oov_glove = check_coverage(vocab, embed_glove)
 add_lower(embed_glove, vocab)
 oov_glove = check_coverage(vocab, embed_glove )<define_variables>
sub_df = pd.DataFrame()
sub_df['Id'] = test_df['Id']
sub_df['sentiment'] = [int(i)for i in final_pred]
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}
len(contraction_mapping )<define_variables>
sub_df.to_csv('my_submission.csv', index=False )