Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<create_dataframe>
%matplotlib inline
final_output_per_model = []
test_dataset = TweetDataset(df_test)
 data_loader = torch.utils.data.DataLoader(
 test_dataset,
batch_size=config.VALID_BATCH_SIZE,
shuffle=False,
num_workers=1 
)
 for model_path in config.get_model_paths() :
model = build_model(model_path, train=False, device=device)
 final_output = []
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 output_start, output_end = model(ids=ids, mask=mask, token_type_ids=token_type_ids)
 output_start = torch.softmax(output_start, dim=1 ).cpu().detach().numpy()
output_end = torch.softmax(output_end, dim=1 ).cpu().detach().numpy()
final_output.append((
output_start,
output_end, 
))
final_output_per_model.append(final_output )<categorify>
train_data = pd.read_csv('.. /input/titanic/train.csv')
 train_data.head(10 )
final_output = []
with torch.no_grad() :
tk0 = tqdm(data_loader, total=len(data_loader))
for bi, d in enumerate(tk0):
ids = d["ids"]
token_type_ids = d["token_type_ids"]
mask = d["mask"]
sentiment = d["sentiment"]
tweet = d["tweet"]
offsets = d["offsets"].numpy()
ids = ids.to(device, dtype=torch.long)
 token_type_ids = token_type_ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 outputs_start = [x[bi][0] for x in final_output_per_model]
outputs_end = [x[bi][1] for x in final_output_per_model]
output_start = sum(outputs_start)/ len(outputs_start)
 output_end = sum(outputs_end)/ len(outputs_end)
 for i in range(len(ids)) :
start_pred = np.argmax(output_start[i])
 end_pred = np.argmax(output_end[i])
 if start_pred > end_pred or sentiment[i] == 'neural' or len(tweet[i].split())< 3:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 final_output.append(pred)
<load_from_csv>
test_data = pd.read_csv('.. /input/titanic/test.csv')
 test_data.head(10 )
sample = pd.read_csv(config.SAMPLE_FILE)
 sample.loc[:, 'selected_text'] = final_output<save_to_csv>
train_data.isnull().sum()
if config.TRAIN:
subm_file = f'{config.VERSION}_submission.csv'
else:
subm_file = 'submission.csv'
sample.to_csv(subm_file, index=False)
 print("saved to", subm_file )<set_options>
test_data.isnull().sum()
warnings.filterwarnings('ignore' )<set_options>
train_data['Age'] = train_data['Age'].fillna(train_data['Age'].mean())
 test_data['Age'] = test_data['Age'].fillna(test_data['Age'].mean() )
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 os.environ['PYTHONHASHSEED'] = str(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed = 42
seed_everything(seed )<feature_engineering>
train_data.isnull().sum()
class TweetDataset(torch.utils.data.Dataset):
def __init__(self, df, max_len=96):
self.df = df
self.max_len = max_len
self.labeled = 'selected_text' in df
self.tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file='.. /input/roberta-base/vocab.json',
merges_file='.. /input/roberta-base/merges.txt',
lowercase=True,
add_prefix_space=True)
 def __getitem__(self, index):
data = {}
row = self.df.iloc[index]
ids, masks, tweet, offsets = self.get_input_data(row)
 data['ids'] = ids
data['masks'] = masks
data['tweet'] = tweet
data['offsets'] = offsets
if self.labeled:
start_idx, end_idx = self.get_target_idx(row, tweet, offsets)
 data['start_idx'] = start_idx
data['end_idx'] = end_idx
return data
def __len__(self):
return len(self.df)
 def get_input_data(self, row):
tweet = " " + " ".join(row.text.lower().split())
 encoding = self.tokenizer.encode(tweet)
 sentiment_id = self.tokenizer.encode(row.sentiment ).ids
ids = [0] + sentiment_id + [2, 2] + encoding.ids + [2]
offsets = [(0, 0)] * 4 + encoding.offsets + [(0, 0)]
pad_len = self.max_len - len(ids)
 if pad_len > 0:
ids += [1] * pad_len
offsets += [(0, 0)] * pad_len
ids = torch.tensor(ids)
 masks = torch.where(ids != 1, torch.tensor(1), torch.tensor(0))
offsets = torch.tensor(offsets)
 return ids, masks, tweet, offsets
def get_target_idx(self, row, tweet, offsets):
selected_text = " " + " ".join(row.selected_text.lower().split())
 len_st = len(selected_text)- 1
idx0 = None
idx1 = None
for ind in(i for i, e in enumerate(tweet)if e == selected_text[1]):
if " " + tweet[ind: ind+len_st] == selected_text:
idx0 = ind
idx1 = ind + len_st - 1
break
char_targets = [0] * len(tweet)
 if idx0 != None and idx1 != None:
for ct in range(idx0, idx1 + 1):
char_targets[ct] = 1
target_idx = []
for j,(offset1, offset2)in enumerate(offsets):
if sum(char_targets[offset1: offset2])> 0:
target_idx.append(j)
 start_idx = target_idx[0]
end_idx = target_idx[-1]
return start_idx, end_idx
def get_train_val_loaders(df, train_idx, val_idx, batch_size=8):
train_df = df.iloc[train_idx]
val_df = df.iloc[val_idx]
train_loader = torch.utils.data.DataLoader(
 TweetDataset(train_df),
batch_size=batch_size,
shuffle=True,
num_workers=2,
drop_last=True)
 val_loader = torch.utils.data.DataLoader(
 TweetDataset(val_df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 dataloaders_dict = {"train": train_loader, "val": val_loader}
return dataloaders_dict
def get_test_loader(df, batch_size=32):
loader = torch.utils.data.DataLoader(
 TweetDataset(df),
batch_size=batch_size,
shuffle=False,
num_workers=2)
 return loader<load_pretrained>
test_data.isnull().sum()
class TweetModel(nn.Module):
def __init__(self):
super(TweetModel, self ).__init__()
config = RobertaConfig.from_pretrained(
 '.. /input/roberta-base/config.json', output_hidden_states=True)
 self.roberta = RobertaModel.from_pretrained(
 '.. /input/roberta-base/pytorch_model.bin', config=config)
 self.dropout = nn.Dropout(0.5)
 self.fc = nn.Linear(config.hidden_size, 2)
 nn.init.normal_(self.fc.weight, std=0.02)
 nn.init.normal_(self.fc.bias, 0)
 def forward(self, input_ids, attention_mask):
_, _, hs = self.roberta(input_ids, attention_mask)
 x = torch.stack([hs[-1], hs[-2], hs[-3], hs[-4]])
 x = torch.mean(x, 0)
 x = self.dropout(x)
 x = self.fc(x)
 start_logits, end_logits = x.split(1, dim=-1)
 start_logits = start_logits.squeeze(-1)
 end_logits = end_logits.squeeze(-1)
 return start_logits, end_logits<compute_test_metric>
train_data['Embarked'] = train_data['Embarked'].fillna("S" )
def loss_fn(start_logits, end_logits, start_positions, end_positions):
ce_loss = nn.CrossEntropyLoss()
start_loss = ce_loss(start_logits, start_positions)
 end_loss = ce_loss(end_logits, end_positions)
 total_loss = start_loss + end_loss
return total_loss<statistical_test>
train_data.isnull().sum()
def get_selected_text(text, start_idx, end_idx, offsets):
selected_text = ""
for ix in range(start_idx, end_idx + 1):
selected_text += text[offsets[ix][0]: offsets[ix][1]]
if(ix + 1)< len(offsets)and offsets[ix][1] < offsets[ix + 1][0]:
selected_text += " "
return selected_text
def jaccard(str1, str2):
a = set(str1.lower().split())
 b = set(str2.lower().split())
 c = a.intersection(b)
 return float(len(c)) /(len(a)+ len(b)- len(c))
def compute_jaccard_score(text, start_idx, end_idx, start_logits, end_logits, offsets):
start_pred = np.argmax(start_logits)
 end_pred = np.argmax(end_logits)
 if start_pred > end_pred:
pred = text
else:
pred = get_selected_text(text, start_pred, end_pred, offsets)
 true = get_selected_text(text, start_idx, end_idx, offsets)
 return jaccard(true, pred )<train_model>
train_data['Cabin'] = train_data['Cabin'].fillna("Missing")
 test_data['Cabin'] = test_data['Cabin'].fillna("Missing" )
def train_model(model, dataloaders_dict, criterion, optimizer, num_epochs, filename):
model.cuda()
for epoch in range(num_epochs):
for phase in ['train', 'val']:
if phase == 'train':
model.train()
else:
model.eval()
epoch_loss = 0.0
epoch_jaccard = 0.0
for data in(dataloaders_dict[phase]):
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_idx = data['start_idx'].cuda()
end_idx = data['end_idx'].cuda()
optimizer.zero_grad()
with torch.set_grad_enabled(phase == 'train'):
start_logits, end_logits = model(ids, masks)
 loss = criterion(start_logits, end_logits, start_idx, end_idx)
 if phase == 'train':
loss.backward()
optimizer.step()
epoch_loss += loss.item() * len(ids)
 start_idx = start_idx.cpu().detach().numpy()
end_idx = end_idx.cpu().detach().numpy()
start_logits = torch.softmax(start_logits, dim=1 ).cpu().detach().numpy()
end_logits = torch.softmax(end_logits, dim=1 ).cpu().detach().numpy()
for i in range(len(ids)) :
jaccard_score = compute_jaccard_score(
 tweet[i],
start_idx[i],
end_idx[i],
start_logits[i],
end_logits[i],
offsets[i])
 epoch_jaccard += jaccard_score
epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
 epoch_jaccard = epoch_jaccard / len(dataloaders_dict[phase].dataset)
 print('Epoch {}/{} | {:^5} | Loss: {:.4f} | Jaccard: {:.4f}'.format(
 epoch + 1, num_epochs, phase, epoch_loss, epoch_jaccard))
torch.save(model.state_dict() , filename )<choose_model_class>
train_data.isnull().sum()
num_epochs = 3
batch_size = 32
skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=seed )<load_from_csv>
test_data.isnull().sum()
%%time
train_df = pd.read_csv('.. /input/tweet-sentiment-extraction/train.csv')
 train_df['text'] = train_df['text'].astype(str)
 train_df['selected_text'] = train_df['selected_text'].astype(str)
 for fold,(train_idx, val_idx)in enumerate(skf.split(train_df, train_df.sentiment), start=1):
print(f'Fold: {fold}')
 model = TweetModel()
optimizer = optim.AdamW(model.parameters() , lr=3e-5, betas=(0.9, 0.999))
criterion = loss_fn
dataloaders_dict = get_train_val_loaders(train_df, train_idx, val_idx, batch_size)
 train_model(
 model,
dataloaders_dict,
criterion,
optimizer,
num_epochs,
f'roberta_fold{fold}.pth' )<load_from_csv>
test_data['Fare'] = test_data['Fare'].median()
%%time
test_df = pd.read_csv('.. /input/tweet-sentiment-extraction/test.csv')
 test_df['text'] = test_df['text'].astype(str)
 test_loader = get_test_loader(test_df)
 predictions = []
models = []
for fold in range(skf.n_splits):
model = TweetModel()
model.cuda()
model.load_state_dict(torch.load(f'roberta_fold{fold+1}.pth'))
model.eval()
models.append(model)
 for data in test_loader:
ids = data['ids'].cuda()
masks = data['masks'].cuda()
tweet = data['tweet']
offsets = data['offsets'].numpy()
start_logits = []
end_logits = []
for model in models:
with torch.no_grad() :
output = model(ids, masks)
 start_logits.append(torch.softmax(output[0], dim=1 ).cpu().detach().numpy())
 end_logits.append(torch.softmax(output[1], dim=1 ).cpu().detach().numpy())
 start_logits = np.mean(start_logits, axis=0)
 end_logits = np.mean(end_logits, axis=0)
 for i in range(len(ids)) :
start_pred = np.argmax(start_logits[i])
 end_pred = np.argmax(end_logits[i])
 if start_pred > end_pred:
pred = tweet[i]
else:
pred = get_selected_text(tweet[i], start_pred, end_pred, offsets[i])
 predictions.append(pred )<save_to_csv>
train_data.isnull().sum()
sub_df = pd.read_csv('.. /input/tweet-sentiment-extraction/sample_submission.csv')
 sub_df['selected_text'] = predictions
sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('!!!!', '!')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('.. ', '.')if len(x.split())==1 else x)
 sub_df['selected_text'] = sub_df['selected_text'].apply(lambda x: x.replace('...', '.')if len(x.split())==1 else x)
 sub_df.to_csv('submission.csv', index=False)
 sub_df.head()<import_modules>
test_data.isnull().sum()
import numpy as np
import pandas as pd<load_from_csv>
train_data = pd.get_dummies(train_data, columns=["Sex"], drop_first=True)
 train_data = pd.get_dummies(train_data, columns=["Embarked"],drop_first=True)
 train_data['Fare'] = train_data['Fare'].astype(int)
 train_data.loc[train_data.Fare<=7.91,'Fare']=0
train_data.loc[(train_data.Fare>7.91)&(train_data.Fare<=14.454),'Fare']=1
train_data.loc[(train_data.Fare>14.454)&(train_data.Fare<=31),'Fare']=2
train_data.loc[(train_data.Fare>31),'Fare']=3
train_data['Age']=train_data['Age'].astype(int)
 train_data.loc[ train_data['Age'] <= 16, 'Age']= 0
train_data.loc[(train_data['Age'] > 16)&(train_data['Age'] <= 32), 'Age'] = 1
train_data.loc[(train_data['Age'] > 32)&(train_data['Age'] <= 48), 'Age'] = 2
train_data.loc[(train_data['Age'] > 48)&(train_data['Age'] <= 64), 'Age'] = 3
train_data.loc[train_data['Age'] > 64, 'Age'] = 4
ftgru = pd.read_csv('.. /input/leakyfasttext/leaky_submission.csv')
 ftlstm = pd.read_csv('.. /input/fasttextglove/fin.csv')
 ggru = pd.read_csv('.. /input/fasttextglove/submission_GloveGRU.csv')
 glstm = pd.read_csv('.. /input/fasttextglove/submission_GloveLSTM.csv')
<save_to_csv>
test_data = pd.get_dummies(test_data, columns=["Sex"], drop_first=True)
 test_data = pd.get_dummies(test_data, columns=["Embarked"],drop_first=True)
 test_data['Fare'] = test_data['Fare'].astype(int)
 test_data.loc[test_data.Fare<=7.91,'Fare']=0
test_data.loc[(test_data.Fare>7.91)&(test_data.Fare<=14.454),'Fare']=1
test_data.loc[(test_data.Fare>14.454)&(test_data.Fare<=31),'Fare']=2
test_data.loc[(test_data.Fare>31),'Fare']=3
test_data['Age']=test_data['Age'].astype(int)
 test_data.loc[ test_data['Age'] <= 16, 'Age']= 0
test_data.loc[(test_data['Age'] > 16)&(test_data['Age'] <= 32), 'Age'] = 1
test_data.loc[(test_data['Age'] > 32)&(test_data['Age'] <= 48), 'Age'] = 2
test_data.loc[(test_data['Age'] > 48)&(test_data['Age'] <= 64), 'Age'] = 3
test_data.loc[test_data['Age'] > 64, 'Age'] = 4
submission = pd.DataFrame()
submission['id'] = ggru['id']
for label in labels:
submission[label] = ftgru[label].rank(pct=True)* 0.25 + ftlstm[label].rank(pct=True)* 0.25 + ggru[label].rank(pct=True)* 0.25 + glstm[label].rank(pct=True)* 0.25
submission.to_csv('submission.csv', index=False )<import_modules>
train_data.drop(['Ticket','Cabin','Name'],axis=1,inplace=True)
 test_data.drop(['Ticket','Cabin','Name'],axis=1,inplace=True )
import numpy as np
import pandas as pd
from keras.layers import Dense,Input,Bidirectional,Conv1D,GRU
from keras.layers import Embedding,GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate, SpatialDropout1D
from keras.preprocessing import text, sequence
from keras import initializers, regularizers, constraints, optimizers, layers, callbacks
from keras.callbacks import EarlyStopping,ModelCheckpoint
from keras.models import Model
from keras.optimizers import Adam, SGD
from sklearn.model_selection import train_test_split
import spacy<load_from_csv>
train_data.Survived.value_counts() /len(train_data)*100
glove_file = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
train_file = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv.zip')
 test_file = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv.zip')
 X_train = train_file["comment_text"].str.lower()
X_test = test_file["comment_text"].str.lower()
y_train = train_file[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values<string_transform>
train_data.groupby("Survived" ).mean()
max_features=150000
maxlen=200
embed_size=300
tok=text.Tokenizer(num_words=max_features,lower=True)
 tok.fit_on_texts(list(X_train)+list(X_test))
X_train=tok.texts_to_sequences(X_train)
 X_test=tok.texts_to_sequences(X_test)
 x_train=sequence.pad_sequences(X_train,maxlen=maxlen)
 x_test=sequence.pad_sequences(X_test,maxlen=maxlen )<categorify>
train_data.groupby("Sex_male" ).mean()
embeddings_index = {}
counter = 0
with open(glove_file,encoding='utf8')as f:
for line in f:
counter += 1
if(counter%1000000)==0:
print(counter)
 values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs<init_hyperparams>
from sklearn.metrics import accuracy_score, log_loss
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
from sklearn import metrics
nlp = spacy.load('en');
word_index = tok.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.random.randn(num_words, embed_size)/4
kk = 0
moo = 0
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
kk += 1
else:
for x in nlp(word,disable=['parser', 'ner']):
embedding_vector = embeddings_index.get(x.lemma_)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
kk += 1
break 
<choose_model_class>
X = train_data.drop(['Survived'], axis=1)
 y = train_data["Survived"]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.22, random_state = 5 )
model_input = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False )(model_input)
 x = SpatialDropout1D(0.1 )(x)
 x = Bidirectional(GRU(200, return_sequences=True,dropout=0.25,recurrent_dropout=0.25,implementation=1))(x)
 x = Conv1D(128, kernel_size = 3 )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 preds = Dense(6, activation="sigmoid" )(x)
 model = Model(model_input, preds)
 model.summary()
model.compile(loss='binary_crossentropy',optimizer=Adam(lr=2e-4),metrics=['accuracy'])
<train_model>
logReg = LogisticRegression()
logReg.fit(X_train,y_train )
model.fit(x_train, y_train, batch_size=128, epochs=10, verbose=1 )<save_to_csv>
logReg_predict = logReg.predict(X_test)
 logReg_score = logReg.score(X_test,y_test)
 print("Logistic Regression Score :",logReg_score )
y_pred = model.predict(x_test,batch_size=1024,verbose=1)
 submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv.zip')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False )<import_modules>
print("Accuracy Score of Logistic Regression Model:")
 print(metrics.accuracy_score(y_test,logReg_predict))
print("
","Classification Report:")
 print(metrics.classification_report(y_test,logReg_predict),'
' )
print(os.listdir(".. /input")) 
<load_from_csv>
SVC_model = SVC(probability=True)
 SVC_model.fit(X_train,y_train )
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
train = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/train.csv')
 test = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/test.csv' )<prepare_x_and_y>
SVC_predict = SVC_model.predict(X_test)
 SVC_score = SVC_model.score(X_test,y_test)
 print("Support Vector Classifier Score :",SVC_score )
train["comment_text"].fillna("fillna")
 test["comment_text"].fillna("fillna")
 X_train = train["comment_text"].str.lower()
y_train = train[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]].values
X_test = test["comment_text"].str.lower()<define_variables>
print("Accuracy Score of Support Vector Classifier SVC Model:")
 print(metrics.accuracy_score(y_test,SVC_predict))
print("
","Classification Report:")
 print(metrics.classification_report(y_test,SVC_predict),'
' )
max_features=100000
maxlen=150
embed_size=300<compute_train_metric>
decisionTreeModel = DecisionTreeClassifier(max_leaf_nodes=17, random_state=0)
 decisionTreeModel.fit(X_train, y_train )
class RocAucEvaluation(Callback):
def __init__(self, validation_data=() , interval=1):
super(Callback, self ).__init__()
self.interval = interval
self.X_val, self.y_val = validation_data
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict(self.X_val, verbose=0)
 score = roc_auc_score(self.y_val, y_pred)
 print("
ROC-AUC - epoch: {:d} - score: {:.6f}".format(epoch+1, score))<categorify>
decisionTree_predict = decisionTreeModel.predict(X_test)
 decisionTree_score = decisionTreeModel.score(X_test,y_test)
 print("Decision Tree Classifier Score :",decisionTree_score )
tok=text.Tokenizer(num_words=max_features,lower=True)
 tok.fit_on_texts(list(X_train)+list(X_test))
X_train=tok.texts_to_sequences(X_train)
 X_test=tok.texts_to_sequences(X_test)
 x_train=sequence.pad_sequences(X_train,maxlen=maxlen)
 x_test=sequence.pad_sequences(X_test,maxlen=maxlen )<categorify>
print("Accuracy Score of Decision Tree Classifier Model:")
 print(metrics.accuracy_score(y_test,decisionTree_predict))
print("
","Classification Report:")
 print(metrics.classification_report(y_test,decisionTree_predict),'
' )
embeddings_index = {}
with open(EMBEDDING_FILE,encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs 
<feature_engineering>
Random_forest = RandomForestClassifier(n_estimators=17)
 Random_forest.fit(X_train,y_train )
word_index = tok.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector<choose_model_class>
randomForest_predict = Random_forest.predict(X_test)
 randomForest_score = Random_forest.score(X_test,y_test)
 print("Random Forest Score :",randomForest_score )
sequence_input = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix],trainable = False )(sequence_input)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)
 x = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform" )(x)
 avg_pool = GlobalAveragePooling1D()(x)
 max_pool = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool, max_pool])
 preds = Dense(6, activation="sigmoid" )(x)
 model = Model(sequence_input, preds)
 model.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-3),metrics=['accuracy'] )<split>
print("Accuracy Score of Random Forest Classifier Model:")
 print(metrics.accuracy_score(y_test,randomForest_predict))
print("
","Classification Report:")
 print(metrics.classification_report(y_test,randomForest_predict),'
' )
batch_size = 128
epochs = 4
X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.9, random_state=233 )<train_model>
KNN_model = KNeighborsClassifier(n_neighbors=37)
 KNN_model.fit(X_train, y_train )
model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),callbacks = callbacks_list,verbose=1)
 model.load_weights(filepath)
 print('Predicting.... ')
 y_pred = model.predict(x_test,batch_size=1024,verbose=1 )<save_to_csv>
KNN_predict = KNN_model.predict(X_test)
 KNN_score = KNN_model.score(X_test,y_test)
 print("KNN Classifier Score :",KNN_score )
submission = pd.read_csv('.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv')
 submission[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]] = y_pred
submission.to_csv('submission.csv', index=False )<load_from_csv>
print("Accuracy Score of KNN Model:")
 print(metrics.accuracy_score(y_test,KNN_predict))
print("
","Classification Report:")
 print(metrics.classification_report(y_test,KNN_predict),'
' )
f_attn_bi = '.. /input/module-12-attn-bi/10fold_attn_post_am(1 ).csv'
f_dpcnn = '.. /input/module-10-dpcnn/10fold_dpcnn_test(1 ).csv'
f_nbsvm = '.. /input/module-1-nbsvm/Module_1_submission.csv'
f_bilstm_f_pl = '.. /input/module-2-bilstm-f-pl/Module_2_submission.csv'
f_capsule_gru = '.. /input/module-7-capsule-gru/Module_7_submission.csv'
f_mish = '.. /input/module-6-mish/Mish_submission.csv'
f_bilstm_g_tta = '.. /input/module-3-bilstm-g-tta/Module_3_submission.csv'
f_capsule = '.. /input/module-11-capsule/10fold_capsule_am.csv'
f_dmcnn = '.. /input/module-9-dmcnn/10fold_dmcnn_am.csv'
f_bilstm_f = '.. /input/module-13-bilstm-f/Module_13_submission.csv'
f_dehyp = '.. /input/module-4-dehyph/Module_4_submission.csv'
f_bilstm_duembd_pl = '.. /input/module-5-bi-lstm-duembd-pl/Module_5_submission.csv'
f_bilstm_pp = '.. /input/module-8-bi-lstm-pp/10fold_lstmpp_am(1 ).csv'<load_from_csv>
gbk = GradientBoostingClassifier(random_state=101, n_estimators=150,min_samples_split=100, max_depth=6)
 gbk.fit(X_train, y_train )
p_attn_bi = pd.read_csv(f_attn_bi)
 p_dpcnn = pd.read_csv(f_dpcnn)
 p_nbsvm = pd.read_csv(f_nbsvm)
 p_bilstm_f_pl = pd.read_csv(f_bilstm_f_pl)
 p_capsule_gru = pd.read_csv(f_capsule_gru)
 p_mish = pd.read_csv(f_mish)
 p_bilstm_g_tta = pd.read_csv(f_bilstm_g_tta)
 p_capsule = pd.read_csv(f_capsule)
 p_bilstm_f = pd.read_csv(f_bilstm_f)
 p_dmcnn = pd.read_csv(f_dmcnn)
 p_dehyp = pd.read_csv(f_dehyp)
 p_bilstm_duembd_pl = pd.read_csv(f_bilstm_duembd_pl)
 p_bilstm_pp = pd.read_csv(f_bilstm_pp )<define_variables>
gbk_predict = gbk.predict(X_test)
 gbk_score = gbk.score(X_test,y_test)
 print("Gradient Boosting Score :",gbk_score )
label_cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
p_res = p_attn_bi.copy()
p_res[label_cols] =(p_attn_bi[label_cols] + p_dpcnn[label_cols] + p_nbsvm[label_cols] + p_bilstm_f_pl[label_cols] + p_capsule_gru[label_cols] + p_mish[label_cols] + p_bilstm_g_tta[label_cols] +(p_capsule[label_cols] * 3)+ p_bilstm_f[label_cols] + p_dmcnn[label_cols] + p_dehyp[label_cols] + p_bilstm_duembd_pl[label_cols] + p_bilstm_pp[label_cols])/ 15<save_to_csv>
print("Accuracy Score of Gradient Boosting Model:")
 print(metrics.accuracy_score(y_test,gbk_predict))
p_res.to_csv('submission.csv', index=False )<install_modules>
from sklearn import ensemble
from sklearn.model_selection import GridSearchCV
! pip install transformers<import_modules>
GridList =[ {'n_estimators' : [10, 15, 20, 25, 30, 35, 40], 'max_depth' : [5,10,15, 20]},]
randomForest_ensemble = ensemble.RandomForestClassifier(random_state=31, max_features= 3)
 gridSearchCV = GridSearchCV(randomForest_ensemble,GridList, cv = 5 )
import os
from typing import Tuple, List
from functools import partial
import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader, RandomSampler
from torch.nn.utils.rnn import pad_sequence
from transformers import BertTokenizer, BertModel, AdamW, get_linear_schedule_with_warmup, BertPreTrainedModel
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from tqdm import tqdm<load_from_csv>
gridSearchCV.fit(X_train,y_train )
! unzip.. /input/jigsaw-toxic-comment-classification-challenge/train.csv.zip;
! unzip.. /input/jigsaw-toxic-comment-classification-challenge/sample_submission.csv.zip;
! unzip.. /input/jigsaw-toxic-comment-classification-challenge/test.csv.zip;
! unzip.. /input/jigsaw-toxic-comment-classification-challenge/test_labels.csv.zip;<load_from_csv>
gridSearchCV_predict = gridSearchCV.predict(X_test)
 gridSearchCV_score = gridSearchCV.score(X_test,y_test)
 print("Grid SearchCV Score :",gridSearchCV_score )
path = "./"
bert_model_name = 'bert-base-cased'
device = torch.device('cpu')
 if torch.cuda.is_available() :
device = torch.device('cuda:0')
 tokenizer = BertTokenizer.from_pretrained(bert_model_name)
 assert tokenizer.pad_token_id == 0, "Padding value used in masks is set to zero, please change it everywhere"
train_df = pd.read_csv(os.path.join(path, 'train.csv'))
train_df, val_df = train_test_split(train_df, test_size=0.05 )<prepare_x_and_y>
from tabulate import tabulate
class ToxicDataset(Dataset):
def __init__(self, tokenizer: BertTokenizer, dataframe: pd.DataFrame, lazy: bool = False):
self.tokenizer = tokenizer
self.pad_idx = tokenizer.pad_token_id
self.lazy = lazy
if not self.lazy:
self.X = []
self.Y = []
for i,(row)in tqdm(dataframe.iterrows()):
x, y = self.row_to_tensor(self.tokenizer, row)
 self.X.append(x)
 self.Y.append(y)
 else:
self.df = dataframe
@staticmethod
def row_to_tensor(tokenizer: BertTokenizer, row: pd.Series)-> Tuple[torch.LongTensor, torch.LongTensor]:
tokens = tokenizer.encode(row["comment_text"], add_special_tokens=True)
 if len(tokens)> 120:
tokens = tokens[:119] + [tokens[-1]]
x = torch.LongTensor(tokens)
 y = torch.FloatTensor(row[["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]])
 return x, y
def __len__(self):
if self.lazy:
return len(self.df)
 else:
return len(self.X)
 def __getitem__(self, index: int)-> Tuple[torch.LongTensor, torch.LongTensor]:
if not self.lazy:
return self.X[index], self.Y[index]
else:
return self.row_to_tensor(self.tokenizer, self.df.iloc[index])
 def collate_fn(batch: List[Tuple[torch.LongTensor, torch.LongTensor]], device: torch.device)\
-> Tuple[torch.LongTensor, torch.LongTensor]:
x, y = list(zip(*batch))
x = pad_sequence(x, batch_first=True, padding_value=0)
 y = torch.stack(y)
 return x.to(device), y.to(device)
 train_dataset = ToxicDataset(tokenizer, train_df, lazy=True)
 dev_dataset = ToxicDataset(tokenizer, val_df, lazy=True)
 collate_fn = partial(collate_fn, device=device)
 BATCH_SIZE = 32
train_sampler = RandomSampler(train_dataset)
 dev_sampler = RandomSampler(dev_dataset)
 train_iterator = DataLoader(train_dataset, batch_size=BATCH_SIZE, sampler=train_sampler, collate_fn=collate_fn)
 dev_iterator = DataLoader(dev_dataset, batch_size=BATCH_SIZE, sampler=dev_sampler, collate_fn=collate_fn )<choose_model_class>
ids = test_data['PassengerId']
print(len(ids))
predictions = gridSearchCV.predict(test_data )
class BertClassifier(nn.Module):
def __init__(self, bert: BertModel, num_classes: int):
super().__init__()
self.bert = bert
self.classifier = nn.Linear(bert.config.hidden_size, num_classes)
 def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,
labels=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask)
 cls_output = outputs[1]
cls_output = self.classifier(cls_output)
 cls_output = torch.sigmoid(cls_output)
 criterion = nn.BCELoss()
loss = 0
if labels is not None:
loss = criterion(cls_output, labels)
 return loss, cls_output
model = BertClassifier(BertModel.from_pretrained(bert_model_name), 6 ).to(device )<train_model>
output = pd.DataFrame({ 'PassengerId' : ids, 'Survived': predictions } )
def train(model, iterator, optimizer, scheduler):
model.train()
total_loss = 0
for x, y in tqdm(iterator):
optimizer.zero_grad()
mask =(x != 0 ).float()
loss, outputs = model(x, attention_mask=mask, labels=y)
 total_loss += loss.item()
loss.backward()
optimizer.step()
scheduler.step()
print(f"Train loss {total_loss / len(iterator)}")
 def evaluate(model, iterator):
model.eval()
pred = []
true = []
with torch.no_grad() :
total_loss = 0
for x, y in tqdm(iterator):
mask =(x != 0 ).float()
loss, outputs = model(x, attention_mask=mask, labels=y)
 total_loss += loss
true += y.cpu().numpy().tolist()
pred += outputs.cpu().numpy().tolist()
true = np.array(true)
 pred = np.array(pred)
 for i, name in enumerate(['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']):
print(f"{name} roc_auc {roc_auc_score(true[:, i], pred[:, i])}")
 print(f"Evaluate loss {total_loss / len(iterator)}" )<choose_model_class>
output.to_csv('submission.csv', index=False )