Kaggle competition: Sentiment Analysis on Movie Reviews
training_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv' )<count_missing_values>
from fastai.text import *
training_data.isnull().sum()<count_missing_values>
train = pd.read_csv(".. /input/train.tsv", sep='\t')
 test = pd.read_csv(".. /input/test.tsv", sep='\t')
 train['Sentiment'] = train['Sentiment'].apply(str)
test_data.isnull().sum()<count_unique_values>
test_id = test['PhraseId']
training_data.PassengerId.nunique()
passengerId = test_data['PassengerId']<drop_column>
learn = language_model_learner(data,AWD_LSTM, drop_mult=0.3)
training_data.drop(labels='PassengerId', axis=1, inplace=True)
 test_data.drop(labels='PassengerId', axis=1, inplace=True )<groupby>
learn.lr_find()
print(training_data[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False ).mean() )<define_variables>
learn.fit_one_cycle(5, 1e-2, moms=(0.8,0.7))
<define_variables>
learn.unfreeze()
learn.fit_one_cycle(5, 1e-3, moms=(0.8,0.7))
<count_unique_values>
learn.save_encoder('fine_tuned_enc' )
training_data.Name.nunique()<feature_engineering>
test_datalist = TextList.from_df(test, cols='Phrase', vocab=data.vocab)
 data_clas =(TextList.from_df(train, cols='Phrase', vocab=data.vocab)
.split_by_rand_pct(0.2)
.label_from_df(cols= 'Sentiment')
.add_test(test_datalist)
.databunch(bs=32))
data_clas.show_batch()
training_data['Title'] = training_data['Name'].apply(lambda x: x.split(',')[1] ).apply(lambda x: x.split() [0])
 test_data['Title'] = test_data['Name'].apply(lambda x: x.split(',')[1] ).apply(lambda x: x.split() [0])
 training_data['Name_Len'] = training_data['Name'].apply(lambda x: len(x))
test_data['Name_Len'] = test_data['Name'].apply(lambda x: len(x))
training_data.drop(labels='Name', axis=1, inplace=True)
 test_data.drop(labels='Name', axis=1, inplace=True )<data_type_conversions>
learn_classifier = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5)
 learn_classifier.load_encoder('fine_tuned_enc')
 learn_classifier.freeze()
test_data.Name_Len =(test_data.Name_Len/10 ).astype(np.int64)+1
training_data.Name_Len =(training_data.Name_Len/10 ).astype(np.int64)+1<groupby>
learn_classifier.lr_find()
print(training_data[['Title', 'Survived']].groupby(['Title'], as_index=False ).mean() )<groupby>
learn_classifier.fit_one_cycle(5, 2e-2, moms=(0.8,0.7))
print(training_data[['Name_Len', 'Survived']].groupby(['Name_Len'], as_index=False ).mean() )<groupby>
learn_classifier.freeze_to(-2)
 learn_classifier.fit_one_cycle(5, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))
print(training_data[['Sex', 'Survived']].groupby(['Sex'], as_index = False ).mean() )<define_variables>
learn_classifier.freeze_to(-3)
 learn_classifier.fit_one_cycle(5, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))
<count_missing_values>
learn_classifier.show_results()
training_data.Age.isnull().sum()
training_age_n = training_data.Age.dropna(axis=0 )<concatenate>
preds, target = learn_classifier.get_preds(DatasetType.Test, ordered=True)
 labels = np.argmax(preds, axis =1)
full_data = pd.concat([training_data, test_data] )<feature_engineering>
submission = pd.DataFrame({'PhraseId': test_id, 'Sentiment': labels})
 submission.to_csv('submission.csv', index=False)
 submission.head()