Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<drop_column>
SimpleRNN, LSTM,
Bidirectional, Lambda, Embedding, Dropout, Conv1D, MaxPooling1D)
train_data = train_data_orig.drop(['id'], axis = 1)
 test_data = test_data_orig.drop(['id'], axis = 1 )<categorify>
df_train = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 df_test = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 df = pd.concat([df_train, df_test], keys=['train', 'test'])
 df.text = df.text.astype("string" )
color_le = preprocessing.LabelEncoder()
color_le.fit(train_data['color'])
 train_data['color_int'] = color_le.transform(train_data['color'])
 _ = sns.pairplot(train_data.drop('color', axis = 1), hue = 'type', palette = 'muted', diag_kind='kde')
 train_data.drop('color_int', axis = 1, inplace = True )<find_best_model_class>
vocab_size = 30000
sequence_length = 30
df = pd.get_dummies(train_data.drop('type', axis = 1))
X_train, X_test, y_train, y_test = train_test_split(df, train_data['type'], test_size = 0.25, random_state = 0)
 dt_clf = DecisionTreeClassifier(random_state = 0)
 dt_clf.fit(X_train, y_train)
 y_pred = dt_clf.predict(X_test)
 print(metrics.classification_report(y_test, y_pred))
print("
Accuracy Score is: " + str(metrics.accuracy_score(y_test, y_pred)) )<compute_test_metric>
regex_nonalpha = re.compile('[^a-zA-Z0-9\s]')
 regex_http = re.compile("http\S*")
 regex_apostrophe = re.compile("'")
 regex_space = re.compile("\s\s+")
 def alphanumeric(text):
text = regex_http.sub("", text)
 text = regex_apostrophe.sub("", text)
 text = regex_nonalpha.sub(" ", text)
 text = regex_space.sub(" ", text)
 return text.lower().strip()
accuracy_scorer = metrics.make_scorer(metrics.accuracy_score )<prepare_x_and_y>
def alphanumeric_tf(text_tensor):
text_tensor = tf.strings.regex_replace(text_tensor, "http\S*", "")
 text_tensor = tf.strings.regex_replace(text_tensor, "'", "")
 text_tensor = tf.strings.regex_replace(text_tensor, "[^a-zA-Z0-9\s]", " ")
 text_tensor = tf.strings.regex_replace(text_tensor, "\s\s+", " ")
 return tf.strings.strip(tf.strings.lower(text_tensor))
X_train = pd.get_dummies(train_data.drop('type', axis = 1))
y_train = train_data['type']
X_test = pd.get_dummies(test_data )<train_on_grid>
text_vectorizer = TextVectorization(max_tokens=vocab_size,
standardize=alphanumeric_tf,
output_sequence_length=sequence_length, 
)
 text_vectorizer.adapt(df.text.to_numpy())
 vocab = text_vectorizer.get_vocabulary()
params = {'n_estimators':[10, 20, 50, 100], 'criterion':['gini', 'entropy'], 'max_depth':[None, 5, 10, 25, 50]}
rf = RandomForestClassifier(random_state = 0)
 clf = GridSearchCV(rf, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(X_train, y_train)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
rf_best = RandomForestClassifier(n_estimators = 10, random_state = 0 )<train_on_grid>
text_vectorizer(df.iloc[:5].text.to_numpy() )
params = {'n_estimators':[10, 25, 50, 100], 'max_samples':[1, 3, 5, 10]}
bag = BaggingClassifier(random_state = 0)
 clf = GridSearchCV(bag, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(X_train, y_train)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
bag_best = BaggingClassifier(max_samples = 5, n_estimators = 25, random_state = 0 )<train_on_grid>
X_train = text_vectorizer(df_train.text.to_numpy())
 y = df_train.target.to_numpy()
X_test = text_vectorizer(df_test.text.to_numpy())
 ds_train = tf.data.Dataset.from_tensor_slices(( X_train, y))
ds_test = tf.data.Dataset.from_tensor_slices(( X_test))
params = {'learning_rate':[0.05, 0.1, 0.5], 'n_estimators':[100, 200, 500], 'max_depth':[2, 3, 5, 10]}
gbc = GradientBoostingClassifier(random_state = 0)
 clf = GridSearchCV(gbc, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(X_train, y_train)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
gbc_best = GradientBoostingClassifier(learning_rate = 0.1, max_depth = 5, n_estimators = 100, random_state = 0 )<train_on_grid>
ds = ds_train.shuffle(buffer_size=10000 ).cache().enumerate()
ds_train = ds.filter(lambda i, data: i % 10 <= 7 ).map(lambda _, data: data ).cache()
ds_dev = ds.filter(lambda i, data: i % 10 > 7 ).map(lambda _, data: data ).cache()
params = {'n_neighbors':[3, 5, 10, 20], 'leaf_size':[20, 30, 50], 'p':[1, 2, 5], 'weights':['uniform', 'distance']}
knc = KNeighborsClassifier()
clf = GridSearchCV(knc, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(X_train, y_train)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
knc_best = KNeighborsClassifier(n_neighbors = 10 )<train_on_grid>
BATCH_SIZE = 1024
ds_train = ds_train.batch(BATCH_SIZE)
 ds_dev = ds_dev.batch(BATCH_SIZE)
 ds_test = ds_test.batch(BATCH_SIZE )
params = {'penalty':['l1', 'l2'], 'C':[1, 2, 3, 5, 10]}
lr = LogisticRegression(random_state = 0)
 clf = GridSearchCV(lr, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(X_train, y_train)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
lr_best = LogisticRegression(penalty = 'l1', C = 1, random_state = 0 )<train_on_grid>
embedding_dim = 64
embedding_layer = tf.keras.layers.Embedding(vocab_size,
embedding_dim,
input_length=sequence_length,
mask_zero=True,
)
params = {'kernel':['linear', 'rbf'], 'C':[1, 3, 5, 10], 'degree':[3, 5, 10]}
svc = SVC(probability = True, random_state = 0)
 clf = GridSearchCV(svc, param_grid = params, scoring = accuracy_scorer, cv = 5, n_jobs = -1)
 clf.fit(X_train, y_train)
 print('Best score: {}'.format(clf.best_score_))
print('Best parameters: {}'.format(clf.best_params_))
svc_best = SVC(C = 10, degree = 3, kernel = 'linear', probability = True, random_state = 0 )<predict_on_test>
!wget http://nlp.stanford.edu/data/glove.6B.zip
!unzip -q glove.6B.zip
voting_clf = VotingClassifier(estimators=[('rf', rf_best),('bag', bag_best),('gbc', gbc_best),('lr', lr_best),('svc', svc_best)]
, voting='hard')
 voting_clf.fit(X_train, y_train)
 y_pred = voting_clf.predict(X_test)
 print("
Accuracy Score for VotingClassifier is: " + str(voting_clf.score(X_train, y_train)) )<save_to_csv>
word_index = dict(zip(vocab, range(len(vocab))))
path_to_glove_file = "./glove.6B.100d.txt"
embeddings_index = {}
with open(path_to_glove_file)as f:
for line in f:
word, coefs = line.split(maxsplit=1)
 coefs = np.fromstring(coefs, "f", sep=" ")
 embeddings_index[word] = coefs
print("Found %s word vectors." % len(embeddings_index))
submission = pd.DataFrame({'id':test_data_orig['id'], 'type':y_pred})
 submission.to_csv('.. /working/submission.csv', index=False )<import_modules>
num_tokens = len(vocab)+ 2
embedding_dim = 100
hits = 0
misses = 0
embedding_matrix = np.zeros(( num_tokens, embedding_dim))
for word, i in word_index.items() :
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
hits += 1
else:
misses += 1
print("Converted %d words(%d misses)" %(hits, misses))
import pandas as pd
import numpy as np
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
from catboost import CatBoostClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier
from sklearn.metrics import roc_auc_score
from rgf.sklearn import RGFClassifier<load_from_csv>
embedding_layer = Embedding(
 num_tokens,
embedding_dim,
embeddings_initializer=tf.keras.initializers.Constant(embedding_matrix),
trainable=False,
)
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<load_from_csv>
model = Sequential([
embedding_layer,
Conv1D(128, 2, activation="relu"),
MaxPooling1D(2),
Conv1D(128, 5, activation="relu"),
GlobalMaxPooling1D() ,
Dense(128, activation="relu"),
Dropout(0.5),
Dense(1)
 ])
 model.summary()
model.compile(optimizer='Adam',
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'], 
)
 history = model.fit(ds_train, epochs=100, validation_data=ds_dev, verbose=0,
callbacks=[TqdmCallback(verbose=1)]
).history
plt.plot(history['accuracy'])
 plt.plot(history['val_accuracy'] )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<categorify>
model.fit(ds_train.concatenate(ds_dev), epochs=100, verbose=0, callbacks=[TqdmCallback(verbose=1)])
 predictions = model.predict(ds_test )
id_test = test['id'].values
target_train = train['target'].values
train = train.drop(['target','id'], axis = 1)
 test = test.drop(['id'], axis = 1)
 col_to_drop = train.columns[train.columns.str.startswith('ps_calc_')]
train = train.drop(col_to_drop, axis=1)
 test = test.drop(col_to_drop, axis=1)
 train = train.replace(-1, np.nan)
 test = test.replace(-1, np.nan)
 cat_features = [a for a in train.columns if a.endswith('cat')]
train['intrain'] = True
test['intrain'] = False
both = pd.concat([train,test],axis=0)
 for column in cat_features:
	temp = pd.get_dummies(pd.Series(both[column]))
	both = pd.concat([both,temp],axis=1)
 	both = both.drop([column],axis=1)
 train = both[both.intrain==True].drop(['intrain'],axis=1)
 test = both[both.intrain==False].drop(['intrain'],axis=1)
 print(train.values.shape, test.values.shape )<predict_on_test>
predictions =(tf.sigmoid(model.predict(ds_test)).numpy() > 0.5 ).astype(int ).reshape(( -1,))
class Ensemble(object):
def __init__(self, n_splits, stacker, base_models):
self.n_splits = n_splits
self.stacker = stacker
self.base_models = base_models
def fit_predict(self, X, y, T):
X = np.array(X)
 y = np.array(y)
 T = np.array(T)
 folds = list(StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=2016 ).split(X, y))
S_train = np.zeros(( X.shape[0], len(self.base_models)))
 S_test = np.zeros(( T.shape[0], len(self.base_models)))
 for i, clf in enumerate(self.base_models):
S_test_i = np.zeros(( T.shape[0], self.n_splits))
for j,(train_idx, test_idx)in enumerate(folds):
X_train = X[train_idx]
y_train = y[train_idx]
X_holdout = X[test_idx]
print("Fit %s fold %d" %(str(clf ).split('(')[0], j+1))
clf.fit(X_train, y_train)
 y_pred = clf.predict_proba(X_holdout)[:,1]
S_train[test_idx, i] = y_pred
S_test_i[:, j] = clf.predict_proba(T)[:,1]
S_test[:, i] = S_test_i.mean(axis=1)
 print(" Model score: %.5f
" % roc_auc_score(y, S_train[:,i]))
self.base_preds = S_test
almost_zero = 1e-12
almost_one = 1 - almost_zero
S_train[S_train>almost_one] = almost_one
S_train[S_train<almost_zero] = almost_zero
S_train = np.log(S_train/(1-S_train))
S_test[S_test>almost_one] = almost_one
S_test[S_test<almost_zero] = almost_zero
S_test = np.log(S_test/(1-S_test))
results = cross_val_score(self.stacker, S_train, y, cv=self.n_splits, scoring='roc_auc')
 print("Stacker score: %.5f" %(results.mean()))
self.stacker.fit(S_train, y)
 print('Coefficients:', self.stacker.coef_)
 res = self.stacker.predict_proba(S_test)[:,1]
return res<init_hyperparams>
submission = pd.DataFrame({
'id': df_test.id.to_numpy() ,
'target': predictions,
} )
lgb_params = {}
lgb_params['learning_rate'] = 0.02
lgb_params['n_estimators'] = 650
lgb_params['max_bin'] = 10
lgb_params['subsample'] = 0.8
lgb_params['subsample_freq'] = 10
lgb_params['colsample_bytree'] = 0.8
lgb_params['min_child_samples'] = 500
lgb_params['random_state'] = 99
lgb_params2 = {}
lgb_params2['n_estimators'] = 1090
lgb_params2['learning_rate'] = 0.02
lgb_params2['colsample_bytree'] = 0.3
lgb_params2['subsample'] = 0.7
lgb_params2['subsample_freq'] = 2
lgb_params2['num_leaves'] = 16
lgb_params2['random_state'] = 99
lgb_params3 = {}
lgb_params3['n_estimators'] = 1100
lgb_params3['max_depth'] = 4
lgb_params3['learning_rate'] = 0.02
lgb_params3['random_state'] = 99 
<choose_model_class>
submission.to_csv('submission.csv', index=False )
<predict_on_test><EOS> 
submission.to_csv('submission.csv', index=False )