Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<train_model>
%matplotlib inline
warnings.filterwarnings("ignore" )
print("Base Accuracy - Predicting all labels as non toxic ")
(1-train_comb[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].mean())*100.0<compute_train_metric>
train_df=pd.read_csv("/kaggle/input/titanic/train.csv")
 test_df=pd.read_csv("/kaggle/input/titanic/test.csv")
 train_df.head()
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
self.max_score = 0
self.not_better_count = 0
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=1)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: %d - score: %.6f
" %(epoch+1, score))
if(score > self.max_score):
print("*** New High Score(previous: %.6f)
 " % self.max_score)
 model.save_weights("best_weights.h5")
 self.max_score=score
self.not_better_count = 0
else:
self.not_better_count += 1
if self.not_better_count > 3:
print("Epoch %05d: early stopping, high score = %.6f" %(epoch,self.max_score))
self.model.stop_training = True<choose_model_class>
y=train_df["Survived"].values
train_df.drop(["Survived","PassengerId"],inplace=True,axis=1)
 test_df.drop(["PassengerId"],inplace=True,axis=1)
 train_df.head()
def get_model(features,clipvalue=1.,num_filters=40,dropout=0.5,embed_size=200):
features_input = Input(shape=(features.shape[1],))
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_vectors], trainable=False,name='EmbeddingLayer' )(inp)
 x, x_h, x_c = Bidirectional(GRU(num_filters, return_sequences=True, return_state = True),name='BidirectionalGRU' )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, x_h, max_pool,features_input])
 outp = Dense(6, activation="sigmoid" )(x)
 model = Model(inputs=[inp,features_input], outputs=outp)
 adam = optimizers.adam(clipvalue=clipvalue)
 model.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=['accuracy'])
 return model<split>
train_df["train"]=1
test_df["train"]=0
combined_df=pd.concat([train_df,test_df] )
model = get_model(features)
 batch_size = 32
epochs = 5
gc.collect()
K.clear_session()
num_folds = 5
predict = np.zeros(( test.shape[0],6))
scores = []
oof_predict = np.zeros(( train.shape[0],6))
kf = KFold(n_splits=num_folds, shuffle=True, random_state=239)
 for train_index, test_index in kf.split(x_train):
kfold_y_train,kfold_y_test = y_train[train_index], y_train[test_index]
kfold_X_train = x_train[train_index]
kfold_X_features = features[train_index]
kfold_X_valid = x_train[test_index]
kfold_X_valid_features = features[test_index]
gc.collect()
K.clear_session()
model = get_model(features)
 ra_val = RocAucEvaluation(validation_data=([kfold_X_valid,kfold_X_valid_features], kfold_y_test), interval = 1)
 model.fit([kfold_X_train,kfold_X_features], kfold_y_train, batch_size=batch_size, epochs=epochs, verbose=1,
callbacks = [ra_val])
 gc.collect()
model.load_weights("best_weights.h5")
 predict += model.predict([x_test,test_features], batch_size=batch_size,verbose=1)/ num_folds
gc.collect()
oof_predict[test_index] = model.predict([kfold_X_valid, kfold_X_valid_features],batch_size=batch_size, verbose=1)
 cv_score = roc_auc_score(kfold_y_test, oof_predict[test_index])
 scores.append(cv_score)
 print('Cross Validation Score: ',cv_score)
 print("Model Completion for Keras DL")
 print('Total CV score is {}'.format(np.mean(scores)))
 print("Saving Predictions for offline upload")
 sample_submission = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv")
 class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
sample_submission[class_names] = predict
sample_submission.to_csv('UdactiyAssignment_v1.csv',index=False)
<train_model>
Image("/kaggle/input/missing-values-mechanism/Missingtheory.png" )
print("Code Run Completed" )<load_from_csv>
null_df,del_rows=null_info(combined_df,"Null values on train Data" )
train = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/train.csv")
 test = pd.read_csv(".. /input/jigsaw-toxic-comment-classification-challenge/test.csv")
 train.columns 
<feature_engineering>
model_df= combined_df[null_df]
train['length'] = train['comment_text'].apply(len )<categorify>
model_df.isna().sum()
tokenizer = Tokenizer()
tokenizer.fit_on_texts(train['comment_text'])
 train_x = tokenizer.texts_to_sequences(train['comment_text'])
 train_x = pad_sequences(train_x, maxlen=300)
<count_values>
Image("/kaggle/input/missing-values-mechanism/Missingtheory.png" )
print("Word count:",len(tokenizer.word_counts)) 
<count_values>
def imputation(data):
data_numeric=data.select_dtypes(include=np.number)
 data_categorical=data.select_dtypes(exclude=np.number)
 display(Markdown("
display(data[data_numeric.isna().values].head(2))
display(data[data_categorical.isna().values].head(2))
data.fillna(data_numeric.median() ,inplace=True)
 for i in data_categorical.columns:
data[i].fillna(data[i].value_counts().index[0], inplace=True)
 display(Markdown("
display(data[data_numeric.isna().values].head(2))
display(data[data_categorical.isna().values].head(2))
print("Document count:", tokenizer.document_count)
<categorify>
def remove_exists(data1,data2,del_rows):
data1.drop(data2.columns,axis=1,inplace=True)
 data1.drop(del_rows,axis=1,inplace=True)
 display(data1.head())
 remove_exists(combined_df,model_df,del_rows )
embeddings_index = {}
file = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
with open(file, encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
print('Loaded %s word vectors.' % len(embeddings_index))<define_variables>
df_cate.drop("Ticket",axis=1,inplace=True )
num_words = min(len(embeddings_index), len(tokenizer.word_index))
print(num_words )<feature_engineering>
model_df[df_cate.columns]=df_cate.copy()
model_df[df_num.columns]=df_num.copy()
embedding_matrix = np.zeros(( num_words, 300))
total_invalid_record_count = 0
for item in tokenizer.word_index.items() :
try:
word, index = item[0], item[1]
embedding_matrix[index] = embeddings_index[word]
except Exception as e:
print("Exception occured for record:", word)
 total_invalid_record_count += 1
print("total_invalid_record_count:",total_invalid_record_count)
<define_variables>
model_df["Sex"]=model_df.Sex.map({"male":0,"female":1} )
print("total_invalid_record_count:",total_invalid_record_count)
 print("Word Index:",len(tokenizer.word_index))
correct_records = len(tokenizer.word_index)- total_invalid_record_count
print("Total correct records:",correct_records )<feature_engineering>
model_df["Embarked"]=model_df.Embarked.map({"S":0,"C":1,"Q":2} )
embedding_matrix = np.zeros(( correct_records, 300))
total_invalid_record_count = 0
index_val = 0
for item in tokenizer.word_index.items() :
try:
word, index = item[0], item[1]
embedding_matrix[index_val] = embeddings_index[word]
index_val += 1
except Exception as e:
total_invalid_record_count += 1
print("total_invalid_record_count:",total_invalid_record_count )<import_modules>
model_df["family"]=model_df["SibSp"] + model_df["Parch"] + 1
from keras.layers import Dense, Input, LSTM, Bidirectional, Conv1D
from keras.layers import Dropout, Embedding
from keras.preprocessing import text, sequence
from keras.layers import GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate, SpatialDropout1D
from keras.models import Model
from keras.models import Sequential<choose_model_class>
titles = {"Mr": 1, "Miss": 2, "Mrs": 3, "Master": 4, "Rare": 5}
model_df['Title'] = model_df.Name.str.extract('([A-Za-z]+)\.', expand= False)
 model_df['Title'] = model_df['Title'].replace(['Lady', 'Countess','Capt', 'Col','Don', 'Dr','Major', 'Rev', 'Sir', 'Jonkheer', 'Dona'], 'Rare')
 model_df['Title'] = model_df['Title'].replace('Mlle', 'Miss')
 model_df['Title'] = model_df['Title'].replace('Ms', 'Miss')
 model_df['Title'] = model_df['Title'].replace('Mme', 'Mrs')
 model_df['Title'] = model_df['Title'].map(titles)
 model_df['Title'] = model_df['Title'].fillna(0 )
model = Sequential()
model.add(Embedding(correct_records, 300, weights=[embedding_matrix], input_length=300))
model.add(LSTM(128))
model.add(Dense(6, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
 print(model.summary() )<split>
model_df['Age'] = model_df['Age'].astype(int)
 model_df.loc[ model_df['Age'] <= 11, 'Age'] = 0
model_df.loc[(model_df['Age'] > 11)&(model_df['Age'] <= 18), 'Age'] = 1
model_df.loc[(model_df['Age'] > 18)&(model_df['Age'] <= 22), 'Age'] = 2
model_df.loc[(model_df['Age'] > 22)&(model_df['Age'] <= 27), 'Age'] = 3
model_df.loc[(model_df['Age'] > 27)&(model_df['Age'] <= 33), 'Age'] = 4
model_df.loc[(model_df['Age'] > 33)&(model_df['Age'] <= 40), 'Age'] = 5
model_df.loc[(model_df['Age'] > 40)&(model_df['Age'] <= 66), 'Age'] = 6
model_df.loc[ model_df['Age'] > 66, 'Age'] = 6
train_y = train[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']]
X_train, X_test, y_train, y_test = train_test_split(train_x, train_y, test_size=0.33, random_state=42 )<train_model>
model_df['Fare'] = model_df['Fare'].astype(int)
 model_df.loc[ model_df['Fare'] <= 7.91, 'Fare'] = 0
model_df.loc[(model_df['Fare'] > 7.91)&(model_df['Fare'] <= 14.454), 'Fare'] = 1
model_df.loc[(model_df['Fare'] > 14.454)&(model_df['Fare'] <= 31), 'Fare'] = 2
model_df.loc[(model_df['Fare'] > 31)&(model_df['Fare'] <= 99), 'Fare'] = 3
model_df.loc[(model_df['Fare'] > 99)&(model_df['Fare'] <= 250), 'Fare'] = 4
model_df.loc[ model_df['Fare'] > 250, 'Fare'] = 5
model.fit(X_train, y_train, epochs=1, batch_size=64 )<save_to_csv>
model_df.drop(["SibSp","Parch","Name"],axis=1,inplace=True)
 model_df.head()
tokenizer.fit_on_texts(test['comment_text'])
 test_x = tokenizer.texts_to_sequences(test['comment_text'])
 test_x = pad_sequences(test_x, maxlen=300)
 predictions = model.predict(test_x, batch_size=64, verbose=1)
 submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']] = predictions
submission.to_csv('submission.csv', index=False )<train_model>
X_train=model_df[model_df.train==1]
X_test=model_df[model_df.train==0]
Y_train=y
X_train.drop("train",axis=1,inplace=True)
 X_test.drop("train",axis=1,inplace=True )
<import_modules>
sgd = linear_model.SGDClassifier(max_iter=5, tol=None)
 sgd.fit(X_train, Y_train)
 acc_sgd = round(sgd.score(X_train, Y_train)* 100, 2 )
import pandas as pd<import_modules>
random_forest = RandomForestClassifier(n_estimators=100)
 random_forest.fit(X_train, Y_train)
 acc_random_forest = round(random_forest.score(X_train, Y_train)* 100, 2 )
import pandas as pd<load_from_csv>
GBC = GradientBoostingClassifier()
GBC.fit(X_train, Y_train)
 acc_GBC = round(GBC.score(X_train, Y_train)* 100, 2 )
glove = pd.read_csv(".. /input/nb-svm-strong-linear-baseline/submission.csv")
 subb = pd.read_csv('.. /input/fasttext-like-baseline-with-keras-lb-0-053/submission_bn_fasttext.csv')
 ave = pd.read_csv('.. /input/toxic-avenger/submission.csv')
 lstm = pd.read_csv('.. /input/toxicfiles/baselinelstm0069.csv')
 svm = pd.read_csv(".. /input/toxicfiles/lstmglove0072ge.csv" )<drop_column>
logreg = LogisticRegression()
logreg.fit(X_train, Y_train)
 acc_log = round(logreg.score(X_train, Y_train)* 100, 2 )
col = col.tolist()
col.remove('id' )<feature_engineering>
knn = KNeighborsClassifier(n_neighbors = 3)
 knn.fit(X_train, Y_train)
 acc_knn = round(knn.score(X_train, Y_train)* 100, 2 )
for i in col:
ble[i] =(2*subb[i] + 3*lstm[i] + 4*glove[i] + 5*svm[i] + ave[i])/ 15<save_to_csv>
gaussian = GaussianNB()
gaussian.fit(X_train, Y_train)
 acc_gaussian = round(gaussian.score(X_train, Y_train)* 100, 2 )
ble.to_csv('submission20.csv', index = False )<set_options>
perceptron = Perceptron(max_iter=5)
 perceptron.fit(X_train, Y_train)
 acc_perceptron = round(perceptron.score(X_train, Y_train)* 100, 2 )
np.random.seed(42)
 warnings.filterwarnings('ignore')
 os.environ['OMP_NUM_THREADS'] = '4'<load_from_csv>
linear_svc = LinearSVC()
linear_svc.fit(X_train, Y_train)
 acc_linear_svc = round(linear_svc.score(X_train, Y_train)* 100, 2 )
EMBEDDING_FILE = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv')
 submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 X_train = train["comment_text"].fillna("fillna" ).values
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test["comment_text"].fillna("fillna" ).values<string_transform>
decision_tree = DecisionTreeClassifier()
decision_tree.fit(X_train, Y_train)
 acc_decision_tree = round(decision_tree.score(X_train, Y_train)* 100, 2 )
max_features = 30000
maxlen = 100
embed_size = 300
tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(X_train)+ list(X_test))
X_train = tokenizer.texts_to_sequences(X_train)
 X_test = tokenizer.texts_to_sequences(X_test)
 x_train = sequence.pad_sequences(X_train, maxlen=maxlen)
 x_test = sequence.pad_sequences(X_test, maxlen=maxlen )<feature_engineering>
results = pd.DataFrame({
'Model': ['Support Vector Machines','GBC', 'KNN', 'Logistic Regression',
'Random Forest', 'Naive Bayes', 'Perceptron',
'Stochastic Gradient Decent',
'Decision Tree'],
'Score': [acc_linear_svc,acc_GBC, acc_knn, acc_log,
acc_random_forest, acc_gaussian, acc_perceptron,
acc_sgd, acc_decision_tree]})
 result_df = results.sort_values(by='Score', ascending=False)
 result_df = result_df.set_index('Score')
 result_df.head(10 )
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
rf = RandomForestClassifier(n_estimators=100)
 scores = cross_val_score(rf, X_train, Y_train, cv=10, scoring = "accuracy")
 print("Scores:", scores)
 print("Mean:", scores.mean())
 print("Standard Deviation:", scores.std() )
def get_model() :
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(units, return_sequences = True))(x)
 x = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 conc = concatenate([avg_pool, max_pool])
 outp = Dense(6, activation="sigmoid" )(conc)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
 return model
model = get_model()
model.summary()<train_model>
random_forest = RandomForestClassifier(criterion = "gini",
min_samples_leaf = 1,
min_samples_split = 10,
n_estimators=100,
max_features='auto',
oob_score= True,
random_state=1,
n_jobs=-1)
 random_forest.fit(X_train, Y_train)
 print("oob score:", round(random_forest.score(X_train,Y_train), 4)*100, "%" )
BATCH_SIZE = 32
EPOCHS = 2
VALIDATION_SPLIT = 0.1
file_path="weights_base.best.hdf5"
list_classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y = train[list_classes].values
checkpoint = ModelCheckpoint(file_path, monitor='val_loss', verbose=1, save_best_only=True,
mode='min')
 early = EarlyStopping(monitor="val_loss", mode="min", patience=20)
 callbacks_list = [checkpoint, early]
model.fit(x_train, y, batch_size=BATCH_SIZE, epochs=EPOCHS, verbose=1,
validation_split=0.0,shuffle=False, callbacks=callbacks_list )<save_to_csv>
predictions = cross_val_predict(random_forest, X_train, Y_train, cv=10)
 confusion_matrix(Y_train, predictions )
y_pred = model.predict(x_test, batch_size=1024)
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False)
 print(submission.values[0] )<define_variables>
print("Precision:", precision_score(Y_train, predictions))
print("Recall:",recall_score(Y_train, predictions))
<set_options>
y_scores = random_forest.predict_proba(X_train)
 y_scores = y_scores[:,1]
start_time = time.time()
np.random.seed(42)
 warnings.filterwarnings('ignore')
 os.environ['OMP_NUM_THREADS'] = '4'
cores = 4
train1 = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 train = train1
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
max_features = 50000
maxlen = 100
embed_size = 300<choose_model_class>
r_a_score = roc_auc_score(Y_train, y_scores)
 print("ROC-AUC-Score:", r_a_score )
lemmatizer = WordNetLemmatizer()
def penn_to_wn(tag):

 if tag.startswith('J'):
return wn.ADJ
elif tag.startswith('N'):
return wn.NOUN
elif tag.startswith('R'):
return wn.ADV
elif tag.startswith('V'):
return wn.VERB
return None
def clean_text(text):
text = text.replace("<br />", " ")
 return text
def swn_polarity(text):

 tokens_count = 0
sentiment = 0.0
text = clean_text(text)
 tagged_sentence = pos_tag(word_tokenize(text))
for word, tag in tagged_sentence:
wn_tag = penn_to_wn(tag)
 if wn_tag not in(wn.NOUN, wn.ADJ, wn.ADV):
continue
lemma = lemmatizer.lemmatize(word, pos=wn_tag)
 if not lemma:
continue
synsets = wn.synsets(lemma, pos=wn_tag)
 if not synsets:
continue
synset = synsets[0]
swn_synset = swn.senti_synset(synset.name())
 if swn_synset.obj_score() == 1:
sentiment += swn_synset.pos_score() - swn_synset.neg_score()
elif swn_synset.obj_score() != 1:
sentiment += swn_synset.pos_score() - swn_synset.neg_score() + swn_synset.obj_score()
tokens_count += 1
return sentiment
def sentiment(df):
return df.apply(lambda x: swn_polarity(x))
cores = 4
def parallelize_dataframe(df, func):
df_split = np.array_split(df, cores)
 with closing(Pool(cores)) as pool:
df = pd.concat(pool.map(func, df_split))
return df
def sentiment_score(df):
df['senti_score'] = parallelize_dataframe(df['comment_text'], sentiment)
 return df<compute_test_metric>
predictionss=random_forest.predict(X_test )
t1 = time.time()
train = sentiment_score(train)
 t2 = time.time()
print("Time taken is "+str(t2-t1))
print(train.shape )<normalization>
test_df=pd.read_csv("/kaggle/input/titanic/test.csv")
 submission =pd.DataFrame({'PassengerId':test_df.PassengerId,'Survived':predictionss})
 submission.head()
<data_type_conversions><EOS> 
submission.to_csv("submission.csv",index=False )