Kaggle competition: Sentiment Analysis on Movie Reviews
<SOS> metric: CategorizationAccuracy Kaggle data source: sentiment-analysis-on-movie-reviews<count_values>
%%bash
pip install -q transformers
bins = [0, 16, 32, 48, 81]
names = [0, 1, 2, 3]
for df in combine:
df['AgeBin'] = pd.cut(df['Age'], bins, labels = names)
 df['AgeBin'] = pd.to_numeric(df['AgeBin'])
 print(train_df.AgeBin.value_counts())
 print("_"*40)
 print(test_df.AgeBin.value_counts() )<count_values>
import numpy as np
import pandas as pd
from pathlib import Path
import os
import torch
import torch.optim as optim
import random
from fastai import *
from fastai.text import *
from fastai.callbacks import *
from transformers import PreTrainedModel, PreTrainedTokenizer, PretrainedConfig
from transformers import BertForSequenceClassification, BertTokenizer, BertConfig
from transformers import RobertaForSequenceClassification, RobertaTokenizer, RobertaConfig
from transformers import XLNetForSequenceClassification, XLNetTokenizer, XLNetConfig
from transformers import XLMForSequenceClassification, XLMTokenizer, XLMConfig
from transformers import DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertConfig
for df in combine:
df['FareCat'] = pd.qcut(df.FarePP, 4)
 print(train_df.FareCat.value_counts())
 print("_"*40)
 print(test_df.FareCat.value_counts() )<count_values>
print('fastai version :', fastai.__version__)
 print('transformers version :', transformers.__version__ )
labels = [0, 1, 2, 3]
for df in combine:
df['FareCat'] = pd.qcut(df.FarePP, 4, labels=labels)
 df['FareCat'] = pd.to_numeric(df['FareCat'])
 print(train_df.FareCat.value_counts())
 print("_"*40)
 print(test_df.FareCat.value_counts() )<sort_values>
DATA_ROOT = Path(".. ")/ "/kaggle/input/sentiment-analysis-on-movie-reviews"
train = pd.read_csv(DATA_ROOT / 'train.tsv.zip', sep="\t")
 test = pd.read_csv(DATA_ROOT / 'test.tsv.zip', sep="\t")
 print(train.shape,test.shape)
 train.head()
train_df[['FareCat', 'Survived']].groupby(['FareCat'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<feature_engineering>
MODEL_CLASSES = {
'bert':(BertForSequenceClassification, BertTokenizer, BertConfig),
'xlnet':(XLNetForSequenceClassification, XLNetTokenizer, XLNetConfig),
'xlm':(XLMForSequenceClassification, XLMTokenizer, XLMConfig),
'roberta':(RobertaForSequenceClassification, RobertaTokenizer, RobertaConfig),
'distilbert':(DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertConfig)
 }
for dataset in combine:
df = dataset.groupby(['Pclass', 'FareCat'] ).size().unstack(0 ).fillna(0)
 df['first_perc'] =(df[df.columns[0]]/(df[df.columns[0]] + df[df.columns[1]] + df[df.columns[2]]))
df['second_perc'] =(df[df.columns[1]]/(df[df.columns[0]] + df[df.columns[1]] + df[df.columns[2]]))
df['third_perc'] =(df[df.columns[2]]/(df[df.columns[0]] + df[df.columns[1]] + df[df.columns[2]]))
print(df)
 print("_"*40 )<count_values>
seed = 42
use_fp16 = False
bs = 16
model_type = 'roberta'
pretrained_model_name = 'roberta-base'
for df in combine:
df['FamSize'] = 0
df.loc[(df.NumFam > 1), 'FamSize'] = 1
df.loc[(df.NumFam > 3), 'FamSize'] = 2
df.loc[(df.NumFam > 5), 'FamSize'] = 3
print(train_df.FamSize.value_counts())
 print("_"*40)
 print(test_df.FamSize.value_counts() )<sort_values>
model_class, tokenizer_class, config_class = MODEL_CLASSES[model_type]
train_df[['FamSize', 'Survived']].groupby(['FamSize'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<groupby>
def seed_all(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
train_df[["Pclass", "FamSize", "Survived"]].groupby(['FamSize', 'Pclass'],
as_index=True ).mean()<groupby>
seed_all(seed )
train_df[["Sex", "FamSize", "Survived"]].groupby(['FamSize', 'Sex'],
as_index=True ).mean()<feature_engineering>
class TransformersBaseTokenizer(BaseTokenizer):

 def __init__(self, pretrained_tokenizer: PreTrainedTokenizer, model_type = 'bert', **kwargs):
self._pretrained_tokenizer = pretrained_tokenizer
self.max_seq_len = pretrained_tokenizer.max_len
self.model_type = model_type
def __call__(self, *args, **kwargs):
return self
def tokenizer(self, t:str)-> List[str]:

 CLS = self._pretrained_tokenizer.cls_token
SEP = self._pretrained_tokenizer.sep_token
if self.model_type in ['roberta']:
tokens = self._pretrained_tokenizer.tokenize(t, add_prefix_space=True)[:self.max_seq_len - 2]
tokens = [CLS] + tokens + [SEP]
else:
tokens = self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2]
if self.model_type in ['xlnet']:
tokens = tokens + [SEP] + [CLS]
else:
tokens = [CLS] + tokens + [SEP]
return tokens
for df in combine:
df['Se_Cl'] = 0
df.loc[(( df.Sex == 1)&(df.Pclass == 1)) , 'Se_Cl'] = 1
df.loc[(( df.Sex == 1)&(df.Pclass == 2)) , 'Se_Cl'] = 2
df.loc[(( df.Sex == 1)&(df.Pclass == 3)) , 'Se_Cl'] = 3
df.loc[(( df.Sex == 2)&(df.Pclass == 1)) , 'Se_Cl'] = 4
df.loc[(( df.Sex == 2)&(df.Pclass == 2)) , 'Se_Cl'] = 5
df.loc[(( df.Sex == 2)&(df.Pclass == 3)) , 'Se_Cl'] = 6
print(train_df.Se_Cl.value_counts())
 print("_"*40)
 print(test_df.Se_Cl.value_counts())
 print("_"*40)
 print("_"*40)
 train_df[['Se_Cl', 'Survived']].groupby(['Se_Cl'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<feature_engineering>
transformer_tokenizer = tokenizer_class.from_pretrained(pretrained_model_name)
 transformer_base_tokenizer = TransformersBaseTokenizer(pretrained_tokenizer = transformer_tokenizer, model_type = model_type)
 fastai_tokenizer = Tokenizer(tok_func = transformer_base_tokenizer, pre_rules=[], post_rules=[] )
for df in combine:
df['Cl_IA'] = 0
df.loc[(( df.IsAlone == 1)&(df.Pclass == 1)) , 'Cl_IA'] = 1
df.loc[(( df.IsAlone == 1)&(df.Pclass == 2)) , 'Cl_IA'] = 2
df.loc[(( df.IsAlone == 1)&(df.Pclass == 3)) , 'Cl_IA'] = 3
df.loc[(( df.IsAlone == 0)&(df.Pclass == 1)) , 'Cl_IA'] = 4
df.loc[(( df.IsAlone == 0)&(df.Pclass == 2)) , 'Cl_IA'] = 5
df.loc[(( df.IsAlone == 0)&(df.Pclass == 3)) , 'Cl_IA'] = 6
print(train_df.Cl_IA.value_counts())
 print("_"*40)
 print(test_df.Cl_IA.value_counts())
 print("_"*40)
 print("_"*40)
 train_df[['Cl_IA', 'Survived']].groupby(['Cl_IA'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<feature_engineering>
class TransformersVocab(Vocab):
def __init__(self, tokenizer: PreTrainedTokenizer):
super(TransformersVocab, self ).__init__(itos = [])
 self.tokenizer = tokenizer
def numericalize(self, t:Collection[str])-> List[int]:
"Convert a list of tokens `t` to their ids."
return self.tokenizer.convert_tokens_to_ids(t)
 def textify(self, nums:Collection[int], sep=' ')-> List[str]:
"Convert a list of `nums` to their tokens."
nums = np.array(nums ).tolist()
return sep.join(self.tokenizer.convert_ids_to_tokens(nums)) if sep is not None else self.tokenizer.convert_ids_to_tokens(nums)
 def __getstate__(self):
return {'itos':self.itos, 'tokenizer':self.tokenizer}
def __setstate__(self, state:dict):
self.itos = state['itos']
self.tokenizer = state['tokenizer']
self.stoi = collections.defaultdict(int,{v:k for k,v in enumerate(self.itos)} )
for df in combine:
df['Ca_Cl'] = 0
df.loc[(( df.Cabin == 0)&(df.Pclass == 1)) , 'Ca_Cl'] = 1
df.loc[(( df.Cabin == 0)&(df.Pclass == 2)) , 'Ca_Cl'] = 2
df.loc[(( df.Cabin == 0)&(df.Pclass == 3)) , 'Ca_Cl'] = 3
df.loc[(( df.Cabin == 1)&(df.Pclass == 1)) , 'Ca_Cl'] = 4
df.loc[(( df.Cabin == 1)&(df.Pclass == 2)) , 'Ca_Cl'] = 5
df.loc[(( df.Cabin == 1)&(df.Pclass == 3)) , 'Ca_Cl'] = 6
print(train_df.Ca_Cl.value_counts())
 print("_"*40)
 print(test_df.Ca_Cl.value_counts())
 print("_"*40)
 print("_"*40)
 train_df[['Ca_Cl', 'Survived']].groupby(['Ca_Cl'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<feature_engineering>
transformer_vocab = TransformersVocab(tokenizer = transformer_tokenizer)
 numericalize_processor = NumericalizeProcessor(vocab=transformer_vocab)
 tokenize_processor = TokenizeProcessor(tokenizer=fastai_tokenizer, include_bos=False, include_eos=False)
 transformer_processor = [tokenize_processor, numericalize_processor]
for df in combine:
df['MA_Cl'] = 0
df.loc[(( df.MisAge == 0)&(df.Pclass == 1)) , 'MA_Cl'] = 1
df.loc[(( df.MisAge == 0)&(df.Pclass == 2)) , 'MA_Cl'] = 2
df.loc[(( df.MisAge == 0)&(df.Pclass == 3)) , 'MA_Cl'] = 3
df.loc[(( df.MisAge == 1)&(df.Pclass == 1)) , 'MA_Cl'] = 4
df.loc[(( df.MisAge == 1)&(df.Pclass == 2)) , 'MA_Cl'] = 5
df.loc[(( df.MisAge == 1)&(df.Pclass == 3)) , 'MA_Cl'] = 6
print(train_df.MA_Cl.value_counts())
 print("_"*40)
 print(test_df.MA_Cl.value_counts())
 print("_"*40)
 print("_"*40)
 train_df[['MA_Cl', 'Survived']].groupby(['MA_Cl'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<feature_engineering>
pad_first = bool(model_type in ['xlnet'])
 pad_idx = transformer_tokenizer.pad_token_id
for df in combine:
df['IK_Cl'] = 0
df.loc[(( df.IsKid == 0)&(df.Pclass == 1)) , 'IK_Cl'] = 1
df.loc[(( df.IsKid == 0)&(df.Pclass == 2)) , 'IK_Cl'] = 2
df.loc[(( df.IsKid == 0)&(df.Pclass == 3)) , 'IK_Cl'] = 3
df.loc[(( df.IsKid == 1)&(df.Pclass == 1)) , 'IK_Cl'] = 4
df.loc[(( df.IsKid == 1)&(df.Pclass == 2)) , 'IK_Cl'] = 5
df.loc[(( df.IsKid == 1)&(df.Pclass == 3)) , 'IK_Cl'] = 6
print(train_df.IK_Cl.value_counts())
 print("_"*40)
 print(test_df.IK_Cl.value_counts())
 print("_"*40)
 print("_"*40)
 train_df[['IK_Cl', 'Survived']].groupby(['IK_Cl'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<count_values>
tokens = transformer_tokenizer.tokenize('Salut c est moi, Hello it s me')
 print(tokens)
 ids = transformer_tokenizer.convert_tokens_to_ids(tokens)
 print(ids)
 transformer_tokenizer.convert_ids_to_tokens(ids)
for df in combine:
df["Em_Cl"] = df["Embarked"] * df["Pclass"]
print(train_df.Em_Cl.value_counts())
 print("_"*40)
 print(test_df.Em_Cl.value_counts())
 print("_"*40)
 print("_"*40)
 train_df[['Em_Cl', 'Survived']].groupby(['Em_Cl'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<feature_engineering>
databunch =(TextList.from_df(train, cols='Phrase', processor=transformer_processor)
.split_by_rand_pct(0.1,seed=seed)
.label_from_df(cols= 'Sentiment')
.add_test(test)
.databunch(bs=bs, pad_first=pad_first, pad_idx=pad_idx))
for df in combine:
df['Se_Ca'] = 0
df.loc[(( df.Sex == 1)&(df.Cabin == 0)) , 'Se_Ca'] = 1
df.loc[(( df.Sex == 1)&(df.Cabin == 1)) , 'Se_Ca'] = 2
df.loc[(( df.Sex == 2)&(df.Cabin == 0)) , 'Se_Ca'] = 3
df.loc[(( df.Sex == 2)&(df.Cabin == 1)) , 'Se_Ca'] = 4
print(train_df.Se_Ca.value_counts())
 print("_"*40)
 print(test_df.Se_Ca.value_counts())
 print("_"*40)
 print("_"*40)
 train_df[['Se_Ca', 'Survived']].groupby(['Se_Ca'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<feature_engineering>
print('[CLS] token :', transformer_tokenizer.cls_token)
 print('[SEP] token :', transformer_tokenizer.sep_token)
 print('[PAD] token :', transformer_tokenizer.pad_token)
 databunch.show_batch()
for df in combine:
df['MA_Ca'] = 0
df.loc[(( df.MisAge == 0)&(df.Cabin == 0)) , 'MA_Ca'] = 1
df.loc[(( df.MisAge == 0)&(df.Cabin == 1)) , 'MA_Ca'] = 2
df.loc[(( df.MisAge == 1)&(df.Cabin == 0)) , 'MA_Ca'] = 3
df.loc[(( df.MisAge == 1)&(df.Cabin == 1)) , 'MA_Ca'] = 4
print(train_df.MA_Ca.value_counts())
 print("_"*40)
 print(test_df.MA_Ca.value_counts())
 print("_"*40)
 print("_"*40)
 train_df[['MA_Ca', 'Survived']].groupby(['MA_Ca'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<feature_engineering>
print('[CLS] id :', transformer_tokenizer.cls_token_id)
 print('[SEP] id :', transformer_tokenizer.sep_token_id)
 print('[PAD] id :', pad_idx)
 test_one_batch = databunch.one_batch() [0]
print('Batch shape : ',test_one_batch.shape)
 print(test_one_batch )
for df in combine:
df['Se_IA'] = 0
df.loc[(( df.Sex == 1)&(df.IsAlone == 0)) , 'Se_IA'] = 1
df.loc[(( df.Sex == 1)&(df.IsAlone == 1)) , 'Se_IA'] = 2
df.loc[(( df.Sex == 2)&(df.IsAlone == 0)) , 'Se_IA'] = 3
df.loc[(( df.Sex == 2)&(df.IsAlone == 1)) , 'Se_IA'] = 4
print(train_df.Se_IA.value_counts())
 print("_"*40)
 print(test_df.Se_IA.value_counts())
 print("_"*40)
 print("_"*40)
 train_df[['Se_IA', 'Survived']].groupby(['Se_IA'],
as_index=True ).mean().sort_values(by='Survived',
ascending=False )<set_options>
class CustomTransformerModel(nn.Module):
def __init__(self, transformer_model: PreTrainedModel):
super(CustomTransformerModel,self ).__init__()
self.transformer = transformer_model
def forward(self, input_ids, attention_mask=None):
attention_mask =(input_ids!=pad_idx ).type(input_ids.type())
 logits = self.transformer(input_ids,
attention_mask = attention_mask)[0]
return logits
warnings.filterwarnings("ignore" )<prepare_x_and_y>
config = config_class.from_pretrained(pretrained_model_name)
 config.num_labels = 5
config.use_bfloat16 = use_fp16
print(config )
features = ['Pclass', 'Sex', 'Cabin', 'Embarked', 'Title', 'AgeBin', 'MisAge', 'IsKid',
'FamSize', 'Se_Cl', 'Cl_IA', 'Se_Ca', 'MA_Ca', 'Se_IA']
y = train_df['Survived'].copy()
X = train_df[features].copy()
test = test_df[features].copy()
X.head()<define_variables>
transformer_model = model_class.from_pretrained(pretrained_model_name, config = config)
 custom_transformer_model = CustomTransformerModel(transformer_model = transformer_model )
clf_list = [DecisionTreeClassifier() ,
RandomForestClassifier() ,
AdaBoostClassifier() ,
GradientBoostingClassifier() ,
XGBClassifier() ,
Perceptron() ,
LogisticRegression() ,
SVC() ,
LinearSVC() ,
KNeighborsClassifier() ,
GaussianNB() ,
SGDClassifier()
]<compute_train_metric>
CustomAdamW = partial(AdamW, correct_bias=False)
 learner = Learner(databunch,
custom_transformer_model,
opt_func = CustomAdamW,
metrics=[accuracy, error_rate])
 learner.callbacks.append(ShowGraph(learner))
if use_fp16: learner = learner.to_fp16()
mdl = []
bias_acc = []
var_acc = []
bias_f1 = []
var_f1 = []
bias_auc = []
var_auc = []
acc_scorer = make_scorer(f1_score)
 for clf in clf_list:
model = clf.__class__.__name__
res = cross_val_score(clf, X, y, scoring='accuracy', cv = 5)
 score = round(res.mean() * 100, 3)
 var = round(res.std() , 3)
 bias_acc.append(score)
 var_acc.append(var)
 res = cross_val_score(clf, X, y, scoring=acc_scorer, cv = 5)
 score = round(res.mean() * 100, 3)
 var = round(res.std() , 3)
 bias_f1.append(score)
 var_f1.append(var)
 res = cross_val_score(clf, X, y, scoring='roc_auc', cv = 5)
 score = round(res.mean() * 100, 3)
 var = round(res.std() , 3)
 bias_auc.append(score)
 var_auc.append(var)
 mdl.append(model)
 print(model)
 robcon = pd.DataFrame({'Model': mdl, 'Bias_acc':bias_acc,'Variance_acc':var_acc,
'Bias_f1':bias_f1,'Variance_f1':var_f1, 'Bias_auc':bias_auc,'Variance_auc':var_auc,
})
 robcon = robcon[['Model','Bias_acc','Variance_acc', 'Bias_f1','Variance_f1','Bias_auc','Variance_auc' ]]
robcon<sort_values>
print(learner.model )
print("Best for accuracy")
 print(robcon[['Model','Bias_acc']].sort_values(by= 'Bias_acc', ascending=False ).head(6))
print("_"*40)
 print("Best for f1")
 print(robcon[['Model','Bias_f1']].sort_values(by= 'Bias_f1', ascending=False ).head(6))
print("_"*40)
 print("Best for roc_auc")
 print(robcon[['Model','Bias_auc']].sort_values(by= 'Bias_auc', ascending=False ).head(6))
print("_"*40)
 print("Least variance for accuracy")
 print(robcon[['Model','Variance_acc']].sort_values(by= 'Variance_acc' ).head(6))
print("_"*40)
 print("Least variance for f1")
 print(robcon[['Model','Variance_f1']].sort_values(by= 'Variance_f1' ).head(6))
print("_"*40)
 print("Least variance for roc_auc")
 print(robcon[['Model','Variance_auc']].sort_values(by= 'Variance_auc' ).head(6))<split>
list_layers = [learner.model.transformer.roberta.embeddings,
learner.model.transformer.roberta.encoder.layer[0],
learner.model.transformer.roberta.encoder.layer[1],
learner.model.transformer.roberta.encoder.layer[2],
learner.model.transformer.roberta.encoder.layer[3],
learner.model.transformer.roberta.encoder.layer[4],
learner.model.transformer.roberta.encoder.layer[5],
learner.model.transformer.roberta.encoder.layer[6],
learner.model.transformer.roberta.encoder.layer[7],
learner.model.transformer.roberta.encoder.layer[8],
learner.model.transformer.roberta.encoder.layer[9],
learner.model.transformer.roberta.encoder.layer[10],
learner.model.transformer.roberta.encoder.layer[11],
learner.model.transformer.roberta.pooler]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=895 )<import_modules>
learner.split(list_layers)
 num_groups = len(learner.layer_groups)
 print('Learner split in',num_groups,'groups')
 print(learner.layer_groups)
from sklearn.feature_selection import RFECV<train_on_grid>
learner.save('untrain' )
FeatSel_log = RFECV(LogisticRegression() , step = 1, scoring = 'roc_auc', cv = 10)
 FeatSel_log.fit(X_train, y_train)
 BestFeat_log = X_train.columns.values[FeatSel_log.get_support() ]
BestFeat_log<train_on_grid>
seed_all(seed)
 learner.load('untrain');
param_grid = {'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000],
'tol': [0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1, 1],
'random_state' : [42]}
grid_log = GridSearchCV(LogisticRegression() , param_grid, cv = 10, scoring= 'roc_auc')
 %time grid_log.fit(X_train[BestFeat_log], y_train)
 best_log = grid_log.best_estimator_
print(best_log)
 print("_"*40)
 print(np.abs(grid_log.best_score_))
print("_"*40)
 predictions = best_log.predict(X_test[BestFeat_log])
 accuracy_score(y_true = y_test, y_pred = predictions )<find_best_params>
learner.freeze_to(-1 )
param_grid = {'C': np.arange(1,10),
'tol': [0.0001, 0.001, 0.01, 0.1, 1],
'kernel': ['linear', 'rbf', 'poly', 'sigmoid'],
'random_state': [42]}
grid_SVC = GridSearchCV(SVC() , param_grid, cv = 10, scoring= 'roc_auc')
 %time grid_SVC.fit(X_train, y_train)
 best_SVC = grid_SVC.best_estimator_
print(best_SVC)
 print("_"*40)
 print(np.abs(grid_SVC.best_score_))
predictions = best_SVC.predict(X_test)
 accuracy_score(y_true = y_test, y_pred = predictions )<train_on_grid>
learner.lr_find()
FeatSel_ada = RFECV(AdaBoostClassifier() , step = 1, scoring = 'roc_auc', cv = 10)
 FeatSel_ada.fit(X_train, y_train)
 BestFeat_ada = X_train.columns.values[FeatSel_ada.get_support() ]
BestFeat_ada<train_on_grid>
learner.fit_one_cycle(1,max_lr=2e-03,moms=(0.8,0.7))
param_grid = {'n_estimators': np.arange(50, 500, 50),
'learning_rate': [0.0001, 0.001, 0.01, 0.1, 1, 2],
'algorithm': ['SAMME', 'SAMME.R'],
'random_state': [42]}
grid_ada = GridSearchCV(AdaBoostClassifier() , param_grid, cv = 10, scoring= 'roc_auc')
 %time grid_ada.fit(X_train[BestFeat_ada], y_train)
 best_ada = grid_ada.best_estimator_
print(best_ada)
 print("_"*40)
 print(np.abs(grid_ada.best_score_))
predictions = best_ada.predict(X_test[BestFeat_ada])
 accuracy_score(y_true = y_test, y_pred = predictions )<train_on_grid>
learner.save('first_cycle' )
FeatSel_for = RFECV(RandomForestClassifier() , step = 1, scoring = 'roc_auc', cv = 10)
 FeatSel_for.fit(X_train, y_train)
 BestFeat_for = X_train.columns.values[FeatSel_for.get_support() ]
BestFeat_for<train_on_grid>
seed_all(seed)
 learner.load('first_cycle');
param_grid = {'n_estimators': np.arange(10, 100, 10),
'max_depth': np.arange(2,20),
'max_features' : ['auto', 'log2', None],
'criterion' : ['gini', 'entropy'],
'random_state' : [42]}
grid_forest = GridSearchCV(RandomForestClassifier() , param_grid, cv = 10, scoring= 'roc_auc')
 %time grid_forest.fit(X_train[BestFeat_for], y_train)
 best_forest = grid_forest.best_estimator_
print(best_forest)
 print("_"*40)
 print(np.abs(grid_forest.best_score_))
predictions = best_forest.predict(X_test[BestFeat_for])
 accuracy_score(y_true = y_test, y_pred = predictions )<train_on_grid>
learner.freeze_to(-2 )
FeatSel_XGB = RFECV(XGBClassifier() , step = 1, scoring = 'roc_auc', cv = 10)
 FeatSel_XGB.fit(X_train, y_train)
 BestFeat_XGB = X_train.columns.values[FeatSel_XGB.get_support() ]
BestFeat_XGB<train_on_grid>
lr = 1e-5
param_grid = {'learning_rate': [0.0001, 0.001, 0.01, 0.1, 1, 2],
'max_depth': np.arange(2,10),
'n_estimators': np.arange(50, 500, 50),
'random_state': [42]}
grid_XGB = GridSearchCV(XGBClassifier() , param_grid, cv = 10, scoring= 'roc_auc')
 %time grid_XGB.fit(X_train[BestFeat_XGB], y_train)
 best_XGB = grid_XGB.best_estimator_
print(best_XGB)
 print("_"*40)
 print(np.abs(grid_XGB.best_score_))
predictions = best_XGB.predict(X_test[BestFeat_XGB])
 accuracy_score(y_true = y_test, y_pred = predictions )<save_to_csv>
learner.fit_one_cycle(1, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))
submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": best_XGB.predict(test[BestFeat_XGB])
 })
 submission.to_csv('submission.csv', index=False )<set_options>
learner.save('second_cycle' )
%matplotlib inline
sns.set()<load_from_csv>
seed_all(seed)
 learner.load('second_cycle');
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 df_train.head()<count_values>
learner.freeze_to(-3 )
print(df_train[df_train.Sex == 'female'].Survived.sum() /df_train[df_train.Sex == 'female'].Survived.count())
 print(df_train[df_train.Sex == 'male'].Survived.sum() /df_train[df_train.Sex == 'male'].Survived.count() )<feature_engineering>
learner.fit_one_cycle(1, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))
data['Age'] = data.Age.fillna(data.Age.median())
 data['Fare'] = data.Fare.fillna(data.Fare.median())
 data.info()<categorify>
learner.save('third_cycle' )
data = pd.get_dummies(data, columns=['Sex'], drop_first=True)
 data.head()<prepare_x_and_y>
seed_all(seed)
 learner.load('third_cycle');
data_train = data.iloc[:891]
data_test = data.iloc[891:]
X = data_train.values
test = data_test.values
y = survived_train.values<train_model>
learner.unfreeze()
clf = tree.DecisionTreeClassifier(max_depth=3)
 clf.fit(X, y )<save_to_csv>
learner.fit_one_cycle(2, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))
Y_pred = clf.predict(test)
 df_test['Survived'] = Y_pred
df_test[['PassengerId', 'Survived']].to_csv('1st_dec_tree.csv', index=False )<load_from_csv>
learner.predict('This is the best movie of 2020' )
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 survived_train = df_train.Survived
data = pd.concat([df_train.drop(['Survived'], axis=1), df_test])
 data['Title'] = data.Name.apply(lambda x: re.search('([A-Z][a-z]+)\.', x ).group(1))
sns.countplot(x='Title', data=data);
plt.xticks(rotation=45);<feature_engineering>
learner.predict('This is the worst movie of 2020' )
data['Has_Cabin'] = ~data.Cabin.isnull()
data.drop(['Cabin', 'Name', 'PassengerId', 'Ticket'], axis=1, inplace=True)
 data.head()<data_type_conversions>
path = '/kaggle/working'
export_learner = load_learner(path, file = 'transformer.pkl' )
data['Age'] = data.Age.fillna(data.Age.median())
 data['Fare'] = data.Fare.fillna(data.Fare.median())
 data['Embarked'] = data['Embarked'].fillna('S')
 data.info()<feature_engineering>
export_learner.predict('This is the worst movie of 2020' )
data['CatAge'] = pd.qcut(data.Age, q=4, labels=False)
 data['CatFare']= pd.qcut(data.Fare, q=4, labels=False)
 data.head()<drop_column>
def get_preds_as_nparray(ds_type)-> np.ndarray:

 preds = learner.get_preds(ds_type)[0].detach().cpu().numpy()
sampler = [i for i in databunch.dl(ds_type ).sampler]
reverse_sampler = np.argsort(sampler)
 return preds[reverse_sampler, :]
test_preds = get_preds_as_nparray(DatasetType.Test )
data = data.drop(['Age', 'Fare','SibSp','Parch'], axis=1)
 data.head()<categorify>
sample_submission = pd.read_csv(DATA_ROOT / 'sampleSubmission.csv')
 sample_submission['Sentiment'] = np.argmax(test_preds,axis=1)
 sample_submission.to_csv("predictions.csv", index=False )
data_dum = pd.get_dummies(data, drop_first=True)
 data_dum.head()<train_on_grid>
def create_download_link(title = "Download CSV file", filename = "data.csv"):
html = '<a href={filename}>{title}</a>'
html = html.format(title=title,filename=filename)
 return HTML(html)
 create_download_link(filename='predictions.csv' )