Kaggle competition: Sentiment Analysis on Movie Reviews
data_train = data_dum.iloc[:891]
data_test = data_dum.iloc[891:]
X = data_train.values
test = data_test.values
y = survived_train.values
dep = np.arange(1,9)
 param_grid = {'max_depth' : dep}
clf = tree.DecisionTreeClassifier()
clf_cv = GridSearchCV(clf, param_grid=param_grid, cv=5)
 clf_cv.fit(X, y)
 print("Tuned Decision Tree Parameters: {}".format(clf_cv.best_params_))
print("Best score is {}".format(clf_cv.best_score_))<save_to_csv>
logging.basicConfig(level=logging.INFO )
Y_pred = clf_cv.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('dec_tree_feat_eng.csv', index=False )<save_to_csv>
!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py
logreg = LogisticRegression()
logreg.fit(X,y)
 Y_pred = logreg.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('log_reg_feat_eng.csv', index=False)
<save_to_csv>
%%time
module_url = "https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"
bert_layer = hub.KerasLayer(module_url, trainable=True )
c_space = np.logspace(-5, 8, 15)
 param_grid = {'C': c_space, 'penalty': ['l1', 'l2']}
logreg_cv = GridSearchCV(logreg,param_grid,cv=5)
 logreg_cv.fit(X,y)
 print("Tuned Logistic Regression Parameter: {}".format(logreg_cv.best_params_))
print("Tuned Logistic Regression Accuracy: {}".format(logreg_cv.best_score_))
Y_pred = logreg_cv.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('log_reg__feat_eng.csv', index=False )<save_to_csv>
train = pd.read_csv('train.tsv', sep='\t')
 test = pd.read_csv('test.tsv', sep='\t' )
rf_clf = RandomForestClassifier()
rf_clf.fit(X,y)
 Y_pred = rf_clf.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('random_forest_feat_eng.csv', index=False )<save_to_csv>
%%time
module_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'
bert_layer = hub.KerasLayer(module_url, trainable=True )
n_estimators = np.arange(10,50)
 params_grid = {'n_estimators':n_estimators}
rf_clf = RandomForestClassifier()
rf_clf_cv = GridSearchCV(rf_clf,params_grid,cv=5)
 rf_clf_cv.fit(X,y)
 print("Tuned Random Forest Classifier Parameter: {}".format(rf_clf_cv.best_params_))
print("Tuned Random Forest Classifier Accuracy: {}".format(rf_clf_cv.best_score_))
Y_pred = rf_clf_cv.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('random_forest_cv_feat_eng.csv', index=False )<save_to_csv>
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
 def bert_encode(texts, tokenizer, max_len=512):
all_tokens = []
all_masks = []
all_segments = []
for text in texts:
text = tokenizer.tokenize(text)
 text = text[:max_len-2]
input_sequence = ["[CLS]"] + text + ["[SEP]"]
pad_len = max_len - len(input_sequence)
 tokens = tokenizer.convert_tokens_to_ids(input_sequence)+ [0] * pad_len
pad_masks = [1] * len(input_sequence)+ [0] * pad_len
segment_ids = [0] * max_len
all_tokens.append(tokens)
 all_masks.append(pad_masks)
 all_segments.append(segment_ids)
 return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
 def build_model(bert_layer, max_len=512):
input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
 clf_output = sequence_output[:, 0, :]
net = tf.keras.layers.Dense(64, activation='relu' )(clf_output)
 net = tf.keras.layers.Dropout(0.2 )(net)
 net = tf.keras.layers.Dense(32, activation='relu' )(net)
 net = tf.keras.layers.Dropout(0.2 )(net)
 out = tf.keras.layers.Dense(5, activation='softmax' )(net)
 model = tf.keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
 model.compile(tf.keras.optimizers.Adam(lr=1e-5), loss='categorical_crossentropy', metrics=['accuracy'])
 return model
knn = KNeighborsClassifier()
knn.fit(X,y)
 Y_pred = knn.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('knn_feat_eng.csv', index=False )<save_to_csv>
max_len = 150
train_input = bert_encode(train.Phrase.values, tokenizer, max_len=max_len)
 test_input = bert_encode(test.Phrase.values, tokenizer, max_len=max_len)
 train_labels = tf.keras.utils.to_categorical(train.Sentiment.values, num_classes=5 )
n_neighbors = np.arange(1,20)
 params_grid = {'n_neighbors':n_neighbors}
knn = KNeighborsClassifier()
knn_cv = GridSearchCV(knn,params_grid,cv=5)
 knn_cv.fit(X,y)
 print("Tuned KNN Classifier Parameter: {}".format(knn_cv.best_params_))
print("Tuned KNN Classifier Accuracy: {}".format(knn_cv.best_score_))
Y_pred = knn_cv.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('knn_cv_feat_eng.csv', index=False )<set_options>
%%time
checkpoint = tf.keras.callbacks.ModelCheckpoint('model.h5', monitor='val_accuracy', save_best_only=True, verbose=1)
 earlystopping = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=5, verbose=1)
 train_history = model.fit(
 train_input, train_labels,
validation_split=0.2,
epochs=3,
callbacks=[checkpoint, earlystopping],
batch_size=32,
verbose=1
)
%matplotlib inline
warnings.filterwarnings('ignore')
 sns.set_palette('cool' )<load_from_csv>
%%time
model.load_weights('model.h5')
 test_pred = model.predict(test_input)
 sub = pd.read_csv('.. /input/sentiment-analysis-on-movie-reviews/sampleSubmission.csv')
 sub['Sentiment'] = np.argmax(test_pred, axis=-1)
 sub.to_csv('submission.csv', index=False)